glazed 0.1.0 → 0.2.0

data/lib/glazed/state_backends/sqlite.rb

@@ -34,7 +34,7 @@ module Glazed
       class SchemaMismatch < Glazed::Error; end
 
       # Version of the database layout.
-      SCHEMA_VERSION = 1 #: Integer
+      SCHEMA_VERSION = 2 #: Integer
       # Attempts at a change that keeps conflicting with other processes.
       MAX_ATTEMPTS = 5 #: Integer
       # Milliseconds to wait for another process holding the database lock.
@@ -113,6 +113,32 @@ module Glazed
         end
       end
 
+      # @yieldparam record [Proto::V1::Wakeup, nil] the current record, or nil when there is none
+      # @yieldreturn [Proto::V1::Wakeup, nil] the new record, or nil to delete it
+      # @return [Proto::V1::Wakeup, nil] the record held afterwards
+      # @raise [InvalidMessage] when the block returns an invalid record
+      # @raise [Storages::Base::Conflict] when other processes keep changing the database
+      #: () { (Proto::V1::Wakeup? record) -> Proto::V1::Wakeup? } -> Proto::V1::Wakeup?
+      def update_wakeup(&block)
+        write do |db|
+          stored = db.get_first_value("select record from wakeups where name = 'default'") #: String?
+          updated = block.call(stored && Proto::V1::Wakeup.decode(stored))
+          if updated
+            Validation.validate!(updated)
+            encoded = Proto::V1::Wakeup.encode(updated)
+            unless encoded == stored
+              db.execute(<<~SQL, [SQLite3::Blob.new(encoded)])
+                insert into wakeups (name, record) values ('default', ?)
+                on conflict (name) do update set record = excluded.record
+              SQL
+            end
+          elsif stored
+            db.execute("delete from wakeups where name = 'default'")
+          end
+          updated
+        end
+      end
+
       # @param workflow [String] workflow name
       # @param now [Time] the time; a lease ending at or before it has expired
       # @param limit [Integer, nil] records returned at most; all when nil
@@ -288,12 +314,23 @@ module Glazed
               ) strict
             SQL
             db.execute('create index schedules_due on schedules (workflow, due_at_ms)')
+            create_wakeups(db)
             db.execute('insert into glazed_schema (version) values (?)', [SCHEMA_VERSION])
+          when 1
+            create_wakeups(db)
+            db.execute('update glazed_schema set version = ?', [SCHEMA_VERSION])
           else
             raise SchemaMismatch, "#{@path} has state schema version #{version}, while this glazed uses #{SCHEMA_VERSION}"
           end
         end
       end
+
+      # @param db [SQLite3::Database] the database, in a transaction
+      # @return [void]
+      #: (SQLite3::Database db) -> void
+      private def create_wakeups(db)
+        db.execute('create table wakeups (name text primary key, record blob not null) strict')
+      end
     end
   end
 end

data/lib/glazed/step_context.rb

@@ -6,34 +6,52 @@
 require 'google/protobuf'
 require 'glazed'
 require 'glazed/proto'
+require 'glazed/secrets_resolver'
+require 'glazed/storage_context'
 require 'glazed/storages'
 
 module Glazed
-  # Named storages shared by the steps of a workflow, and where steps report progress.
+  # Named storages shared by the steps of a workflow, the resolver of the
+  # secrets they refer to, and where steps report progress.
   class StepContext
     # Raised when a step names a storage that is not configured.
     class UnknownStorage < Glazed::Error; end
 
+    # @return [Proto::V1::Redaction, nil] redaction settings shared by redact steps
+    #: Proto::V1::Redaction?
+    attr_reader :redaction
+
     # @param storages [Hash{String => Storages::Base}] storages by name
+    # @param redaction [Proto::V1::Redaction, nil] redaction settings shared by redact steps
     # @param listener [Proc] receives progress events from steps, possibly from several threads at once
-    #: (storages: Hash[String, Storages::Base], ?listener: Runner::listener) -> void
-    def initialize(storages:, listener: ->(_event) {})
+    # @param secrets [SecretsResolver] reads secrets of nodes and settings
+    #: (storages: Hash[String, Storages::Base], ?redaction: Proto::V1::Redaction?, ?listener: Runner::listener, ?secrets: SecretsResolver) -> void
+    def initialize(storages:, redaction: nil, listener: ->(_event) {}, secrets: SecretsResolver.new)
       @storages = storages #: Hash[String, Storages::Base]
+      @redaction = redaction #: Proto::V1::Redaction?
       @listener = listener #: Runner::listener
+      @secrets = secrets #: SecretsResolver
     end
 
-    # Builds every configured storage.
+    # @return [SecretsResolver] reads secrets of nodes and settings
+    #: SecretsResolver
+    attr_reader :secrets
+
+    # Builds every configured storage, with one secrets resolver for the storages and the steps.
     #
     # @param storages [Google::Protobuf::Map] storage configs by name, from {Proto::V1::Config#storages}
+    # @param redaction [Proto::V1::Redaction, nil] redaction settings shared by redact steps, from {Proto::V1::Config#redaction}
     # @param listener [Proc] receives progress events from steps, possibly from several threads at once
     # @return [StepContext] the context
     # @raise [Storages::MissingBackend] when a storage config sets no backend
-    #: (Google::Protobuf::Map[String, Proto::V1::StorageConfig] storages, ?listener: Runner::listener) -> StepContext
-    def self.from_proto(storages, listener: ->(_event) {})
+    #: (Google::Protobuf::Map[String, Proto::V1::StorageConfig] storages, ?redaction: Proto::V1::Redaction?, ?listener: Runner::listener) -> StepContext
+    def self.from_proto(storages, redaction: nil, listener: ->(_event) {})
+      secrets = SecretsResolver.new
+      storage_context = StorageContext.new(secrets:)
       # Map#to_h would turn the messages into hashes.
       built = {} #: Hash[String, Storages::Base]
-      storages.each { |name, config| built[name] = Storages.build(config) }
-      new(storages: built, listener:)
+      storages.each { |name, config| built[name] = Storages.build(config, context: storage_context) }
+      new(storages: built, redaction:, listener:, secrets:)
     end
 
     # Reports progress of a step.

data/lib/glazed/steps/{batch_shell.rb → batch_command.rb}

@@ -11,41 +11,41 @@ require 'glazed/steps'
 
 module Glazed
   module Steps
-    # Batch: runs a command with /bin/sh -c once, with the
-    # {Proto::V1::BatchState} of every node that got this far as ProtoJSON on
-    # stdin, for hooks such as notifications.
+    # Batch: runs a command line with /bin/sh -c, or a program with its
+    # arguments, once with the {Proto::V1::BatchState} of every node that got
+    # this far as ProtoJSON on stdin, for hooks such as notifications.
     #
     # @example Step definition
-    #   {"name": "notify", "batch_shell": {"run": "bin/notify-changes"}}
-    class BatchShell < BatchStep
+    #   {"name": "notify", "batch_command": {"run": "bin/notify-changes"}}
+    class BatchCommand < BatchStep
       # Seconds a command may run when the definition sets none.
       DEFAULT_TIMEOUT = 300 #: Integer
 
       # @param name [String] step name
-      # @param run [String] command line
+      # @param run [String, Array<String>] command line for /bin/sh -c, or program and its arguments
       # @param timeout [Numeric] seconds before the command is killed
-      #: (name: String, run: String, ?timeout: Numeric) -> void
+      #: (name: String, run: String | Array[String], ?timeout: Numeric) -> void
       def initialize(name:, run:, timeout: DEFAULT_TIMEOUT)
         super(name:)
-        @run = run #: String
+        @run = run #: String | Array[String]
         @timeout = timeout #: Numeric
       end
 
-      # Builds the step from {Proto::V1::BatchShellStep}.
+      # Builds the step from {Proto::V1::BatchCommandStep}.
       #
       # @param name [String] step name
-      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::BatchShellStep}
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::BatchCommandStep}
       # @param context [StepContext] named storages, unused
-      # @return [BatchShell] the step
-      #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> BatchShell
+      # @return [BatchCommand] the step
+      #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> BatchCommand
       def self.from_proto(name, settings, context:)
-        shell = Must.be_a(settings, Proto::V1::BatchShellStep)
-        new(name:, run: shell.run, timeout: shell.timeout_seconds.zero? ? DEFAULT_TIMEOUT : shell.timeout_seconds)
+        command = Must.be_a(settings, Proto::V1::BatchCommandStep)
+        new(name:, run: ShellCommand.run_of(command.run, command.exec), timeout: command.timeout_seconds.zero? ? DEFAULT_TIMEOUT : command.timeout_seconds)
       end
 
       # @param batch [Proto::V1::BatchState] node states
       # @return [Proto::V1::BatchState] the same states
-      # @raise [ShellCommand::Failed] when the command exits non-zero
+      # @raise [ShellCommand::Failed] when the command cannot start or exits non-zero
       # @raise [TimeoutError] when the command does not finish in time
       #: (Proto::V1::BatchState batch) -> Proto::V1::BatchState
       def call(batch)

data/lib/glazed/steps/{shell.rb → command.rb}

@@ -12,28 +12,31 @@ require 'glazed/storages'
 
 module Glazed
   module Steps
-    # Per node: runs a command with /bin/sh -c. The input artifact's content is
-    # on stdin and stdout becomes the output artifact; both are optional.
+    # Per node: runs a command line with /bin/sh -c, or a program with its
+    # arguments. The input artifact's content is on stdin and stdout becomes
+    # the output artifact; both are optional.
     #
     # @example Step definition
-    #   {"name": "normalize", "shell": {"run": "sed -e '/^ntp clock-period/d'",
+    #   {"name": "normalize", "command": {"run": "sed -e '/^ntp clock-period/d'",
     #    "input": "raw", "output": "normalized", "storage": "staging"}}
-    class Shell < NodeStep
+    # @example Step definition without a shell
+    #   {"name": "lint", "command": {"exec": ["bin/lint-config", "--strict"], "input": "raw"}}
+    class Command < NodeStep
       # Seconds a command may run when the definition sets none.
       DEFAULT_TIMEOUT = 300 #: Integer
 
       # @param name [String] step name
-      # @param run [String] command line
+      # @param run [String, Array<String>] command line for /bin/sh -c, or program and its arguments
       # @param input [String, nil] artifact written to stdin
       # @param output [String, nil] artifact produced from stdout
       # @param storage_name [String, nil] storage for the output, by name
       # @param context [StepContext] named storages
       # @param timeout [Numeric] seconds before the command is killed
       # @raise [StepContext::UnknownStorage] when the storage is not configured
-      #: (name: String, run: String, input: String?, output: String?, storage_name: String?, context: StepContext, ?timeout: Numeric) -> void
+      #: (name: String, run: String | Array[String], input: String?, output: String?, storage_name: String?, context: StepContext, ?timeout: Numeric) -> void
       def initialize(name:, run:, input:, output:, storage_name:, context:, timeout: DEFAULT_TIMEOUT)
         super(name:)
-        @run = run #: String
+        @run = run #: String | Array[String]
         @input = input #: String?
         @output = output #: String?
         @storage_name = storage_name #: String?
@@ -42,21 +45,21 @@ module Glazed
         @timeout = timeout #: Numeric
       end
 
-      # Builds the step from {Proto::V1::ShellStep}.
+      # Builds the step from {Proto::V1::CommandStep}.
       #
       # @param name [String] step name
-      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::ShellStep}
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::CommandStep}
       # @param context [StepContext] named storages
-      # @return [Shell] the step
-      #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Shell
+      # @return [Command] the step
+      #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Command
       def self.from_proto(name, settings, context:)
-        shell = Must.be_a(settings, Proto::V1::ShellStep)
+        command = Must.be_a(settings, Proto::V1::CommandStep)
         new(
-          name:, run: shell.run, context:,
-          input: shell.input.empty? ? nil : shell.input,
-          output: shell.output.empty? ? nil : shell.output,
-          storage_name: shell.storage.empty? ? nil : shell.storage,
-          timeout: shell.timeout_seconds.zero? ? DEFAULT_TIMEOUT : shell.timeout_seconds,
+          name:, run: ShellCommand.run_of(command.run, command.exec), context:,
+          input: command.input.empty? ? nil : command.input,
+          output: command.output.empty? ? nil : command.output,
+          storage_name: command.storage.empty? ? nil : command.storage,
+          timeout: command.timeout_seconds.zero? ? DEFAULT_TIMEOUT : command.timeout_seconds,
         )
       end
 
@@ -70,7 +73,7 @@ module Glazed
 
       # @param state [Proto::V1::NodeState] the node and its artifacts
       # @return [Proto::V1::NodeState] the state, with the output artifact when configured
-      # @raise [ShellCommand::Failed] when the command exits non-zero
+      # @raise [ShellCommand::Failed] when the command cannot start or exits non-zero
       # @raise [TimeoutError] when the command does not finish in time
       # @raise [MissingArtifact] when the input artifact is absent
       #: (Proto::V1::NodeState state) -> Proto::V1::NodeState

data/lib/glazed/steps/fetch.rb

@@ -58,7 +58,7 @@ module Glazed
       def call(state)
         node = state.node or raise ArgumentError, "#{name}: state has no node"
         progress = ->(message) { @context.notify(level: 'info', step: name, node: node.name, message:) } #: ^(String) -> void
-        config = Collector.new(node:, timeout: @timeout, progress:).call
+        config = Collector.new(node:, timeout: @timeout, progress:, secrets: @context.secrets).call
         result = @storage.write(key: Storages.key_for(node), content: config, message: "#{node.name}: #{name}")
         with_artifacts(state, @output => stored(@storage_name, result))
       end

data/lib/glazed/steps/redact.rb

@@ -67,7 +67,9 @@ module Glazed
       #: (Proto::V1::NodeState state) -> Proto::V1::NodeState
       def call(state)
         node = state.node or raise ArgumentError, "#{name}: state has no node"
-        redactor = Redactor.from_proto(@redaction, patterns: Dialects.find(node.dialect).sensitive_patterns)
+        redactor = Redactor.from_proto(
+          @redaction, patterns: Dialects.find(node.dialect).sensitive_patterns, global: @context.redaction, secrets: @context.secrets,
+        )
         content = redactor.call(@context.read(artifact(state, @input)))
         result = @storage.write(key: Storages.key_for(node), content:, message: "#{node.name}: #{name}")
         with_artifacts(state, @output => stored(@storage_name, result))

data/lib/glazed/storage_context.rb

@@ -0,0 +1,22 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed'
+require 'glazed/secrets_resolver'
+
+module Glazed
+  # What storages are built with, besides their own settings.
+  class StorageContext
+    # @param secrets [SecretsResolver] reads secrets the storages refer to
+    #: (?secrets: SecretsResolver) -> void
+    def initialize(secrets: SecretsResolver.new)
+      @secrets = secrets #: SecretsResolver
+    end
+
+    # @return [SecretsResolver] reads secrets the storages refer to
+    #: SecretsResolver
+    attr_reader :secrets
+  end
+end

data/lib/glazed/storages/base.rb

@@ -64,9 +64,10 @@ module Glazed
       # Builds a storage from its settings message.
       #
       # @param settings [Google::Protobuf::MessageExts] backend settings from {Proto::V1::StorageConfig}
+      # @param context [StorageContext] secrets the settings refer to
       # @return [Base] the storage
-      #: (Google::Protobuf::MessageExts settings) -> Base
-      def self.from_proto(settings) = raise NotImplementedError
+      #: (Google::Protobuf::MessageExts settings, context: StorageContext) -> Base
+      def self.from_proto(settings, context:) = raise NotImplementedError
 
       # Writes one file.
       #

data/lib/glazed/storages/filesystem.rb

@@ -33,10 +33,11 @@ module Glazed
       # Builds a storage from {Proto::V1::FilesystemStorage}.
       #
       # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::FilesystemStorage}
+      # @param context [StorageContext] secrets the settings refer to
       # @return [Filesystem] the storage
       # @raise [Must::Violation] when settings is another message
-      #: (Google::Protobuf::MessageExts settings) -> Filesystem
-      def self.from_proto(settings)
+      #: (Google::Protobuf::MessageExts settings, context: StorageContext) -> Filesystem
+      def self.from_proto(settings, context:)
         filesystem = Must.be_a(settings, Proto::V1::FilesystemStorage)
         new(path: filesystem.path)
       end

data/lib/glazed/storages/git.rb

@@ -34,24 +34,29 @@ module Glazed
       # @param remote [String, nil] remote name or URL to pull from and push to
       # @param author_name [String] commit author and committer name
       # @param author_email [String] commit author and committer email
-      #: (path: String, ?branch: String, ?remote: String?, ?author_name: String, ?author_email: String) -> void
-      def initialize(path:, branch: DEFAULT_BRANCH, remote: nil, author_name: DEFAULT_AUTHOR_NAME, author_email: DEFAULT_AUTHOR_EMAIL)
+      # @param subdirectory [String, nil] directory within the repository holding the files; the root when nil
+      # @raise [InvalidKey] for an unsafe subdirectory
+      #: (path: String, ?branch: String, ?remote: String?, ?author_name: String, ?author_email: String, ?subdirectory: String?) -> void
+      def initialize(path:, branch: DEFAULT_BRANCH, remote: nil, author_name: DEFAULT_AUTHOR_NAME, author_email: DEFAULT_AUTHOR_EMAIL, subdirectory: nil)
         super()
         @path = path #: String
         @branch = branch #: String
         @remote = remote #: String?
         @author_name = author_name #: String
         @author_email = author_email #: String
+        @subdirectory = subdirectory&.then { validate_key(it.chomp('/')) } #: String?
         @lock = Mutex.new #: Thread::Mutex
       end
 
       # Builds a storage from {Proto::V1::GitStorage}.
       #
       # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::GitStorage}
+      # @param context [StorageContext] secrets the settings refer to
       # @return [Git] the storage
       # @raise [Must::Violation] when settings is another message
-      #: (Google::Protobuf::MessageExts settings) -> Git
-      def self.from_proto(settings)
+      # @raise [InvalidKey] for an unsafe subdirectory
+      #: (Google::Protobuf::MessageExts settings, context: StorageContext) -> Git
+      def self.from_proto(settings, context:)
         git = Must.be_a(settings, Proto::V1::GitStorage)
         new(
           path: git.path,
@@ -59,6 +64,7 @@ module Glazed
           remote: git.remote.empty? ? nil : git.remote,
           author_name: git.author_name.empty? ? DEFAULT_AUTHOR_NAME : git.author_name,
           author_email: git.author_email.empty? ? DEFAULT_AUTHOR_EMAIL : git.author_email,
+          subdirectory: git.subdirectory.empty? ? nil : git.subdirectory,
         )
       end
 
@@ -71,6 +77,9 @@ module Glazed
       # @return [String, nil] remote name or URL
       #: String?
       attr_reader :remote
+      # @return [String, nil] directory within the repository holding the files
+      #: String?
+      attr_reader :subdirectory
 
       # Writes files in one commit. Nothing is committed when no content
       # changed, and the pointers then name the existing HEAD.
@@ -83,22 +92,25 @@ module Glazed
       #: (Array[Base::file] files, message: Base::message_builder) -> Array[WriteResult]
       def write_all(files, message:)
         keys = files.map { |key, _| validate_key(key) }
+        paths = keys.map { repository_path(it) }
         @lock.synchronize do
           prepare
           files.each do |key, content|
-            file = File.join(path, key)
+            file = File.join(path, repository_path(key))
             FileUtils.mkdir_p(File.dirname(file))
             File.write(file, content)
           end
-          git(['add', '--', *keys])
-          changed = git(['diff', '--cached', '--name-only', '-z', '--', *keys]).split("\0")
+          git(['add', '--', *paths])
+          changed_paths = git(['diff', '--cached', '--name-only', '-z', '--', *paths]).split("\0")
+          changed = keys.select { changed_paths.include?(repository_path(it)) }
           unless changed.empty?
             git(['commit', '--quiet', '--no-verify', '-m', message.call(changed)])
             git(['push', '--quiet', Must.expect(remote), "HEAD:refs/heads/#{branch}"]) if remote
           end
           commit = git(%w[rev-parse HEAD]).strip
           keys.map do |key|
-            WriteResult.new(pointer: pointer_to(Proto::V1::GitLocation.new(path: key, commit:)), changed: changed.include?(key))
+            location = Proto::V1::GitLocation.new(path: repository_path(key), commit:)
+            WriteResult.new(pointer: pointer_to(location), changed: changed.include?(key))
           end
         end
       end
@@ -115,6 +127,11 @@ module Glazed
         git(['show', "#{location.commit}:#{validate_key(location.path)}"])
       end
 
+      # @param key [String] validated file key
+      # @return [String] path of the file within the repository
+      #: (String key) -> String
+      private def repository_path(key) = subdirectory ? "#{subdirectory}/#{key}" : key
+
       # Initialises the repository when missing, switches to the branch and
       # pulls it from the remote when it exists there.
       #

data/lib/glazed/storages/github.rb

@@ -51,8 +51,10 @@ module Glazed
       # @param api_url [String] API endpoint
       # @param author_name [String, nil] commit author name; the token owner when nil
       # @param author_email [String, nil] commit author email; the token owner when nil
-      #: (repository: String, branch: String, token: Secret, ?base_branch: String?, ?api_url: String, ?author_name: String?, ?author_email: String?) -> void
-      def initialize(repository:, branch:, token:, base_branch: nil, api_url: DEFAULT_API_URL, author_name: nil, author_email: nil)
+      # @param subdirectory [String, nil] directory within the repository holding the files; the root when nil
+      # @raise [InvalidKey] for an unsafe subdirectory
+      #: (repository: String, branch: String, token: Secret, ?base_branch: String?, ?api_url: String, ?author_name: String?, ?author_email: String?, ?subdirectory: String?) -> void
+      def initialize(repository:, branch:, token:, base_branch: nil, api_url: DEFAULT_API_URL, author_name: nil, author_email: nil, subdirectory: nil)
         super()
         @repository = repository #: String
         @branch = branch #: String
@@ -60,26 +62,30 @@ module Glazed
         @base_branch = base_branch #: String?
         @api_url = api_url.chomp('/') #: String
         @author = (author_name && author_email ? {name: author_name, email: author_email} : nil) #: Hash[Symbol, String]?
+        @subdirectory = subdirectory&.then { validate_key(it.chomp('/')) } #: String?
         @http = nil #: HTTPX::Session?
       end
 
       # Builds a storage from {Proto::V1::GithubStorage}.
       #
       # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::GithubStorage}
+      # @param context [StorageContext] secrets the settings refer to
       # @return [Github] the storage
       # @raise [Must::Violation] when settings is another message
       # @raise [ArgumentError] when the token is missing
-      #: (Google::Protobuf::MessageExts settings) -> Github
-      def self.from_proto(settings)
+      # @raise [InvalidKey] for an unsafe subdirectory
+      #: (Google::Protobuf::MessageExts settings, context: StorageContext) -> Github
+      def self.from_proto(settings, context:)
         github = Must.be_a(settings, Proto::V1::GithubStorage)
         new(
           repository: github.repository,
           branch: github.branch,
-          token: Secret.maybe(github.token) || raise(ArgumentError, 'github storage needs a token'),
+          token: context.secrets.maybe(github.token) || raise(ArgumentError, 'github storage needs a token'),
           base_branch: github.base_branch.empty? ? nil : github.base_branch,
           api_url: github.api_url.empty? ? DEFAULT_API_URL : github.api_url,
           author_name: github.author_name.empty? ? nil : github.author_name,
           author_email: github.author_email.empty? ? nil : github.author_email,
+          subdirectory: github.subdirectory.empty? ? nil : github.subdirectory,
         )
       end
 
@@ -89,6 +95,9 @@ module Glazed
       # @return [String] branch to commit to
       #: String
       attr_reader :branch
+      # @return [String, nil] directory within the repository holding the files
+      #: String?
+      attr_reader :subdirectory
 
       # Writes the changed files in one commit. Nothing is committed when no
       # content changed, and the pointers then name the existing head.
@@ -103,7 +112,7 @@ module Glazed
         keys = files.map { |key, _| validate_key(key) }
         commit, changed = with_retries { commit_files(files, message) }
         keys.map do |key|
-          pointer = pointer_to(Proto::V1::GithubLocation.new(repository:, path: key, commit:))
+          pointer = pointer_to(Proto::V1::GithubLocation.new(repository:, path: repository_path(key), commit:))
           WriteResult.new(pointer:, changed: changed.include?(key))
         end
       end
@@ -147,12 +156,12 @@ module Glazed
         parent = head || ref_sha(base_branch) || raise(ApiError.new(status: 404, message: "#{repository}: base branch #{base_branch} not found"))
         base_tree = json(get("/repos/#{repository}/git/commits/#{parent}")).dig('tree', 'sha')
         existing = blob_shas(base_tree)
-        candidates = existing ? files.reject { |key, content| existing[key] == blob_sha(content) } : files
+        candidates = existing ? files.reject { |key, content| existing[repository_path(key)] == blob_sha(content) } : files
 
         commit = parent
         changed = [] #: Array[String]
         unless candidates.empty?
-          entries = candidates.map { |key, content| {path: key, mode: '100644', type: 'blob', content:} }
+          entries = candidates.map { |key, content| {path: repository_path(key), mode: '100644', type: 'blob', content:} }
           tree = json(post("/repos/#{repository}/git/trees", {base_tree:, tree: entries})).fetch('sha')
           unless tree == base_tree
             changed = candidates.map(&:first)
@@ -181,6 +190,11 @@ module Glazed
         entries.select { it['type'] == 'blob' }.to_h { [it.fetch('path'), it.fetch('sha')] }
       end
 
+      # @param key [String] validated file key
+      # @return [String] path of the file within the repository
+      #: (String key) -> String
+      private def repository_path(key) = subdirectory ? "#{subdirectory}/#{key}" : key
+
       # @param content [String] file content
       # @return [String] git blob SHA-1 of the content
       #: (String content) -> String

data/lib/glazed/storages/memory.rb

@@ -32,10 +32,11 @@ module Glazed
       # Builds a storage from {Proto::V1::MemoryStorage}.
       #
       # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::MemoryStorage}
+      # @param context [StorageContext] secrets the settings refer to
       # @return [Memory] the storage
       # @raise [Must::Violation] when settings is another message
-      #: (Google::Protobuf::MessageExts settings) -> Memory
-      def self.from_proto(settings)
+      #: (Google::Protobuf::MessageExts settings, context: StorageContext) -> Memory
+      def self.from_proto(settings, context:)
         Must.be_a(settings, Proto::V1::MemoryStorage)
         new
       end

data/lib/glazed/storages.rb

@@ -8,6 +8,7 @@ require 'glazed/finder'
 require 'glazed/must'
 require 'glazed/plugin_registry'
 require 'glazed/proto'
+require 'glazed/storage_context'
 require 'glazed/storages/base'
 require 'glazed/validation'
 
@@ -49,21 +50,22 @@ module Glazed
     # Builds the storage a config selects.
     #
     # @param config [Proto::V1::StorageConfig] storage settings
+    # @param context [StorageContext] secrets the settings refer to
     # @return [Base] the storage
     # @raise [InvalidMessage] when the config breaks the rules of its schema
     # @raise [Finder::NotFound] when the backend is not available
     # @raise [PluginRegistry::UnknownPlugin] when no plugin is registered for the settings type
-    #: (Proto::V1::StorageConfig config) -> Base
-    def self.build(config)
+    #: (Proto::V1::StorageConfig config, ?context: StorageContext) -> Base
+    def self.build(config, context: StorageContext.new)
       Validation.validate!(config)
       backend = config.backend or raise MissingBackend, 'storage config sets no backend'
       if (packed = config.plugin)
         implementation, settings = PLUGINS.resolve(packed)
-        return storage_class(implementation).from_proto(settings)
+        return storage_class(implementation).from_proto(settings, context:)
       end
 
       settings = Must.be_a(config[backend.to_s], Google::Protobuf::MessageExts)
-      find(backend).from_proto(settings)
+      find(backend).from_proto(settings, context:)
     end
 
     # @param implementation [Module] registered implementation

data/lib/glazed/transports/base.rb

@@ -4,7 +4,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 require 'glazed'
+require 'glazed/proto'
 require 'glazed/secret'
+require 'glazed/secrets_resolver'
 
 module Glazed
   module Transports
@@ -18,13 +20,17 @@ module Glazed
       # @param username [String] login user name
       # @param password [Secret, nil] login password
       # @param connect_timeout [Numeric] seconds to wait for a connection
-      #: (host: String, port: Integer?, username: String, password: Secret?, ?connect_timeout: Numeric) -> void
-      def initialize(host:, port:, username:, password:, connect_timeout: DEFAULT_CONNECT_TIMEOUT)
+      # @param transport_options [Proto::V1::TransportOptions, nil] per-transport connection options
+      # @param secrets [SecretsResolver] reads secrets in the transport options
+      #: (host: String, port: Integer?, username: String, password: Secret?, ?connect_timeout: Numeric, ?transport_options: Proto::V1::TransportOptions?, ?secrets: SecretsResolver) -> void
+      def initialize(host:, port:, username:, password:, connect_timeout: DEFAULT_CONNECT_TIMEOUT, transport_options: nil, secrets: SecretsResolver.new)
         @host = host #: String
         @port = port #: Integer?
         @username = username #: String
         @password = password #: Secret?
         @connect_timeout = connect_timeout #: Numeric
+        @transport_options = transport_options #: Proto::V1::TransportOptions?
+        @secrets = secrets #: SecretsResolver
         @pty_options = {} #: Hash[Symbol, untyped]
       end
 
@@ -43,6 +49,12 @@ module Glazed
       # @return [Numeric] seconds to wait for a connection
       #: Numeric
       attr_reader :connect_timeout
+      # @return [Proto::V1::TransportOptions, nil] per-transport connection options
+      #: Proto::V1::TransportOptions?
+      attr_reader :transport_options
+      # @return [SecretsResolver] reads secrets in the transport options
+      #: SecretsResolver
+      attr_reader :secrets
       # @return [Hash{Symbol => Object}] terminal settings overriding the transport's defaults; ignored by transports without a PTY
       #: Hash[Symbol, untyped]
       attr_accessor :pty_options