glazed-aws 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: abc0ee55acf9c806cea84fd269b250283e24bcdf32f03a41009e3ff75d579b66
-  data.tar.gz: 9d468db2b2b4e4264f9c9c0af4a16617513a87f5c3ace4c8714a47bc13d5236b
+  metadata.gz: da36b22c067ea1b077af0937adac343cfe582c5e096d59ab92ccc24d9137c8f3
+  data.tar.gz: de668481d4de70c3f1757d0c510a6c41d0917c70bbd4fb94a8c6652c9c418755
 SHA512:
-  metadata.gz: ae2aa7b00c8ab8be71f2a01b9ad64b05b2c905f8736391b8c291926909cab07e0ed239d48b953a6d568d4c45974509fe1ebf9c957c316a82b14e4d0b5e1b83e9
-  data.tar.gz: c4e701032a6baab3fac4a4672be3a4c2d26da14e1d4a1d0153c205184905eb2650b9b7cfb9c7d818ddbeadd99f81039ee6b63947e9300a8eadc2a29e2f422e1e
+  metadata.gz: 4505bc0c45940d21ead467725e6f8ed54c9fd1790b3a00f41d20c25b22d275eb583f1b95e172ba5e0cae5e95bbd8e3106c1077b7aa048715402e565d1b0078cb
+  data.tar.gz: ce0a8e0c3b9f29919dc9689fae1629fad2252a4c16a36e1870d8b1ae8616a3b203162fd62d6d39446262d2043624797dbfe8847207d4a0a62551a9777f52e380

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## [0.2.0] - 2026-10-15
+
+### Enhancements
+
+- The Lambda function returns the state machine definition for the event `{"state_machine": {}}`, so Terraform can generate it with `aws_lambda_invocation`
+- `ssm` secret source: `{"ssm": {"name": ...}}` reads an AWS Systems Manager Parameter Store parameter, decrypted, and reuses it for `cache_seconds`
+- The Lambda function reads its config from S3 with `GLAZED_CONFIG_S3_BUCKET`, `GLAZED_CONFIG_S3_KEY` and `GLAZED_CONFIG_S3_DIGEST`, checking the object's SHA-256
+- Run the orchestrated workflows on AWS Step Functions: a Lambda function performing each task (`Glazed::Aws::Lambda`), and `glazed-aws state-machine` printing the state machine definition. Signals come from EventBridge rules, Pipes and Scheduler, and debounce waits in the state machine ([docs/aws-step-functions.md](../docs/aws-step-functions.md))
+
 ## [0.1.0] - 2026-10-14
 
 Initial release.

data/README.md

@@ -1,3 +1,5 @@
 # glazed-aws
 
-AWS integrations for [glazed](https://github.com/sorah/glazed): the `s3` storage (`Glazed::Storages::S3`). Installing this gem makes `{"s3": {...}}` storage configs available.
+AWS integrations for [glazed](https://github.com/sorah/glazed): the `s3` storage (`Glazed::Storages::S3`) and the `ssm` secret source (`Glazed::Secrets::Ssm`). Installing this gem makes `{"s3": {...}}` storage configs and `{"ssm": {...}}` secret references available.
+
+It also runs the orchestrated workflows of a config on AWS Step Functions with a Lambda function (`Glazed::Aws::Lambda`), and the `glazed-aws state-machine` command prints the state machine definition. See [docs/aws-step-functions.md](../docs/aws-step-functions.md).

data/exe/glazed-aws

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed/aws/command'
+
+exit Glazed::Aws::Command.start(ARGV)

data/lib/glazed/aws/command/state_machine.rb

@@ -0,0 +1,57 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'json'
+require 'glazed/aws'
+require 'glazed/aws/state_machine'
+require 'glazed/command/base'
+
+module Glazed
+  module Aws
+    module Command
+      # `glazed-aws state-machine`: prints the Amazon States Language
+      # definition running the orchestrated workflows of a config with the
+      # glazed-aws Lambda function. Storages that need care under Step
+      # Functions are warned about on standard error.
+      class StateMachine < Glazed::Command::Base
+        desc 'Print the Step Functions state machine definition for the orchestrated workflows'
+        option :config, aliases: ['-c'], desc: 'Config file, ProtoJSON of glazed.v1.Config; $GLAZED_CONFIG_FILE when omitted'
+        option :function_arn, desc: 'ARN of the Lambda function with the glazed/aws/lambda handler'
+        option :max_concurrency, type: :integer, desc: 'Nodes processed in parallel; jobs in the config by default'
+        option :grace_seconds, type: :integer, desc: 'Seconds after waking until another execution may take over; the longest lease by default'
+        option :max_cycles, type: :integer, desc: 'Cycles after which an execution ends, leaving later ones to the next; 20 by default'
+
+        # Prints the definition.
+        #
+        # @param function_arn [String, nil] ARN of the Lambda function; required
+        # @param config [String, nil] path of the config file; {Glazed::Command::Base::CONFIG_FILE_ENV} when nil
+        # @param max_concurrency [String, nil] nodes processed in parallel
+        # @param grace_seconds [String, nil] seconds after waking until another execution may take over
+        # @param max_cycles [String, nil] cycles after which an execution ends
+        # @param _options [Hash] other options dry-cli passes, unused
+        # @return [void]
+        # @raise [Glazed::Error] when the config is invalid, or a workflow cannot run under Step Functions
+        #: (?function_arn: String?, ?config: String?, ?max_concurrency: String?, ?grace_seconds: String?, ?max_cycles: String?, **untyped _options) -> void
+        def call(function_arn: nil, config: nil, max_concurrency: nil, grace_seconds: nil, max_cycles: nil, **_options)
+          path = config_path(config, 'state-machine')
+          unless function_arn
+            err.puts 'glazed-aws state-machine: --function-arn is required'
+            exit 1
+          end
+
+          settings = load_config(path)
+          orchestrator_settings(settings, path)
+          # dry-cli passes integer options as strings.
+          definition = Aws::StateMachine.new(
+            config: settings, function_arn:,
+            max_concurrency: max_concurrency&.then { Integer(it) }, grace_seconds: grace_seconds&.then { Integer(it) }, max_cycles: max_cycles&.then { Integer(it) },
+          )
+          out.puts JSON.pretty_generate(definition.to_h)
+          definition.warnings.each { err.puts "glazed-aws state-machine: #{it}" }
+        end
+      end
+    end
+  end
+end

data/lib/glazed/aws/command.rb

@@ -0,0 +1,33 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'dry/cli'
+require 'glazed/aws'
+require 'glazed/aws/command/state_machine'
+
+module Glazed
+  module Aws
+    # Commands of the glazed-aws executable.
+    module Command
+      extend Dry::CLI::Registry
+
+      register 'state-machine', StateMachine
+
+      # Runs a command line.
+      #
+      # @param argv [Array<String>] command line arguments
+      # @param out [IO] standard output
+      # @param err [IO] standard error
+      # @return [Integer] exit status
+      #: (Array[String] argv, ?out: IO, ?err: IO) -> Integer
+      def self.start(argv, out: $stdout, err: $stderr)
+        Dry::CLI.new(self).call(arguments: argv, out:, err:)
+        0
+      rescue SystemExit => e
+        e.status
+      end
+    end
+  end
+end

data/lib/glazed/aws/lambda.rb

@@ -0,0 +1,196 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'digest'
+require 'json'
+require 'aws-sdk-s3'
+require 'glazed'
+require 'glazed/aws'
+require 'glazed/aws/signal_events'
+require 'glazed/aws/state_machine'
+require 'glazed/config'
+require 'glazed/coordinator'
+require 'glazed/coordinator/task_runner'
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/proto_json'
+require 'glazed/runner'
+require 'glazed/sources'
+require 'glazed/state_backends'
+require 'glazed/step_context'
+require 'glazed/workflow'
+
+module Glazed
+  module Aws
+    # AWS Lambda function performing the tasks of the state machine
+    # {StateMachine} generates. Set the handler to
+    # `glazed/aws/lambda.Glazed::Aws::Lambda.handler`. The config is read from
+    # the S3 object GLAZED_CONFIG_S3_BUCKET and GLAZED_CONFIG_S3_KEY name,
+    # which must have the SHA-256 in GLAZED_CONFIG_S3_DIGEST, or else from the
+    # file GLAZED_CONFIG_FILE names. Events are `{"task":
+    # <glazed.v1.CoordinatorTask>, "execution_input": <execution input>}`;
+    # signals in the execution input are added to a check task. The result is
+    # the task's result as ProtoJSON. The event `{"state_machine": {}}` returns
+    # the definition of the state machine invoking this function instead.
+    module Lambda
+      # Environment variable naming the config file.
+      CONFIG_FILE_ENV = 'GLAZED_CONFIG_FILE' #: String
+      # Environment variable naming the bucket of the config object.
+      CONFIG_S3_BUCKET_ENV = 'GLAZED_CONFIG_S3_BUCKET' #: String
+      # Environment variable naming the key of the config object.
+      CONFIG_S3_KEY_ENV = 'GLAZED_CONFIG_S3_KEY' #: String
+      # Environment variable holding the SHA-256 of the config object, in base64.
+      CONFIG_S3_DIGEST_ENV = 'GLAZED_CONFIG_S3_DIGEST' #: String
+      # Environment variable naming the region of the config bucket; the SDK default when unset.
+      CONFIG_S3_REGION_ENV = 'GLAZED_CONFIG_S3_REGION' #: String
+
+      # What a warm function keeps between invocations.
+      class Function
+        # Builds the task runner of a config.
+        #
+        # @param config [Proto::V1::Config] the config
+        # @param source [String] where the config comes from
+        # @param listener [Proc] receives events
+        # @return [Function] the function
+        # @raise [Glazed::Error] when the config lacks orchestrator settings
+        #: (Proto::V1::Config config, source: String, listener: Runner::listener) -> Function
+        def self.build(config, source:, listener:)
+          settings = config.orchestrator or raise Glazed::Error, "#{source}: orchestrator is not configured"
+          context = StepContext.from_proto(config.storages, listener:)
+          workflows = settings.workflows.keys.to_h do |name|
+            definition = config.workflows[name] or raise Glazed::Error, "#{source}: no workflow named #{name.inspect}"
+            [name, Workflow.from_proto(definition, context:)]
+          end
+          backend = StateBackends.build(Must.because(settings.state) { 'validation requires orchestrator.state' }, context:)
+          coordinator = Coordinator.new(settings:, backend:, listener:)
+          runner = Coordinator::TaskRunner.new(coordinator:, workflows:, inventory: -> { Sources.nodes(config.inventory.to_a) }, listener:)
+          new(config:, source:, runner:)
+        end
+
+        # @param config [Proto::V1::Config] the config
+        # @param source [String] where the config comes from
+        # @param runner [Coordinator::TaskRunner] performs the tasks
+        #: (config: Proto::V1::Config, source: String, runner: Coordinator::TaskRunner) -> void
+        def initialize(config:, source:, runner:)
+          @config = config #: Proto::V1::Config
+          @source = source #: String
+          @runner = runner #: Coordinator::TaskRunner
+        end
+
+        # @return [Proto::V1::Config] the config
+        #: Proto::V1::Config
+        attr_reader :config
+        # @return [String] where the config comes from
+        #: String
+        attr_reader :source
+        # @return [Coordinator::TaskRunner] performs the tasks
+        #: Coordinator::TaskRunner
+        attr_reader :runner
+      end
+
+      @function = nil #: Function?
+
+      # Entry point of the Lambda runtime.
+      #
+      # @param event [Hash] the event
+      # @param context [Object] the Lambda context
+      # @return [Hash] the task's result as ProtoJSON, or the state machine definition
+      #: (event: Hash[String, untyped], context: untyped) -> Hash[String, untyped]
+      def self.handler(event:, context:)
+        handle(event, environment: ENV.to_h, invoked_function_arn: context.invoked_function_arn)
+      end
+
+      # Performs the task of an event. The config is loaded once per source,
+      # so a changed object in S3 is read only when its digest changes.
+      #
+      # @param event [Hash] the event
+      # @param environment [Hash{String => String}] environment variables
+      # @param invoked_function_arn [String, nil] ARN the function was invoked with, including any qualifier
+      # @param listener [Proc] receives events; JSON lines on standard error by default
+      # @param s3 [Aws::S3::Client, nil] client reading the config object; built from the environment when nil
+      # @return [Hash] the task's result as ProtoJSON, or the state machine definition
+      # @raise [Glazed::Error] when the config is missing, invalid or not of the expected digest, or the task fails
+      #: (Hash[String, untyped] event, environment: Hash[String, String], ?invoked_function_arn: String?, ?listener: Runner::listener, ?s3: ::Aws::S3::Client?) -> Hash[String, untyped]
+      def self.handle(event, environment:, invoked_function_arn: nil, listener: ->(event) { $stderr.puts(JSON.generate(event)) }, s3: nil)
+        source = config_source(environment)
+        function = @function
+        if function.nil? || function.source != source
+          function = @function = Function.build(load_config(environment, s3:), source:, listener:)
+        end
+        return state_machine(function.config, event.fetch('state_machine'), invoked_function_arn:) if event.key?('state_machine')
+
+        task = ProtoJson.load(Proto::V1::CoordinatorTask, JSON.generate(event.fetch('task')))
+        if (check = task.check) && event.key?('execution_input')
+          SignalEvents.extract(event['execution_input'], listener:).each { check.signals << it }
+        end
+        JSON.parse(ProtoJson.dump(function.runner.perform(task)))
+      end
+
+      # Generates the state machine definition, as `glazed-aws state-machine` does.
+      #
+      # @param config [Proto::V1::Config] the config
+      # @param request [Hash, nil] options: `function_arn`, the invoked ARN by default, and
+      #   `max_concurrency`, `grace_seconds` and `max_cycles`
+      # @param invoked_function_arn [String, nil] ARN the function was invoked with
+      # @return [Hash] `definition`, the definition as a JSON string, and `warnings`
+      # @raise [Glazed::Error] when no function ARN is known, or a workflow cannot run under Step Functions
+      #: (Proto::V1::Config config, Hash[String, untyped]? request, invoked_function_arn: String?) -> Hash[String, untyped]
+      private_class_method def self.state_machine(config, request, invoked_function_arn:)
+        request ||= {}
+        function_arn = request.fetch('function_arn', invoked_function_arn) or raise Glazed::Error, 'state_machine.function_arn is required'
+        definition = StateMachine.new(
+          config:, function_arn:,
+          max_concurrency: request['max_concurrency']&.then { Integer(it) },
+          grace_seconds: request['grace_seconds']&.then { Integer(it) },
+          max_cycles: request['max_cycles']&.then { Integer(it) },
+        )
+        {'definition' => JSON.generate(definition.to_h), 'warnings' => definition.warnings}
+      end
+
+      # @param environment [Hash{String => String}] environment variables
+      # @return [String] where the config comes from: the S3 object and its digest, or the file
+      # @raise [Glazed::Error] when neither is set, or the S3 object is set partly
+      #: (Hash[String, String] environment) -> String
+      private_class_method def self.config_source(environment)
+        bucket = environment[CONFIG_S3_BUCKET_ENV]
+        return environment.fetch(CONFIG_FILE_ENV) { raise Glazed::Error, "#{CONFIG_S3_BUCKET_ENV} or #{CONFIG_FILE_ENV} is required" } unless bucket
+
+        key = environment.fetch(CONFIG_S3_KEY_ENV) { raise Glazed::Error, "#{CONFIG_S3_KEY_ENV} is required with #{CONFIG_S3_BUCKET_ENV}" }
+        digest = environment.fetch(CONFIG_S3_DIGEST_ENV) { raise Glazed::Error, "#{CONFIG_S3_DIGEST_ENV} is required with #{CONFIG_S3_BUCKET_ENV}" }
+        "s3://#{bucket}/#{key}##{digest}"
+      end
+
+      # @param environment [Hash{String => String}] environment variables
+      # @param s3 [Aws::S3::Client, nil] client reading the config object; built from the environment when nil
+      # @return [Proto::V1::Config] the config
+      # @raise [Glazed::Error] when the config is invalid, or the S3 object is not of the expected digest
+      #: (Hash[String, String] environment, s3: ::Aws::S3::Client?) -> Proto::V1::Config
+      private_class_method def self.load_config(environment, s3:)
+        bucket = environment[CONFIG_S3_BUCKET_ENV]
+        return Glazed::Config.load(environment.fetch(CONFIG_FILE_ENV)) unless bucket
+
+        key = environment.fetch(CONFIG_S3_KEY_ENV)
+        origin = "s3://#{bucket}/#{key}"
+        region = environment[CONFIG_S3_REGION_ENV]
+        s3 ||= region ? ::Aws::S3::Client.new(region:) : ::Aws::S3::Client.new
+        json = s3.get_object(bucket:, key:).body.read.force_encoding(Encoding::UTF_8)
+        # The digest pins the object the function was deployed with.
+        actual = [Digest::SHA256.digest(json)].pack('m0')
+        expected = environment.fetch(CONFIG_S3_DIGEST_ENV)
+        raise Glazed::Error, "#{origin} has SHA-256 #{actual}, not #{expected} in #{CONFIG_S3_DIGEST_ENV}" unless actual == expected
+
+        Glazed::Config.parse(json, origin:)
+      end
+
+      # Forgets the loaded config, so the next event loads it again.
+      #
+      # @return [void]
+      #: () -> void
+      def self.reset
+        @function = nil
+      end
+    end
+  end
+end

data/lib/glazed/aws/signal_events.rb

@@ -0,0 +1,106 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'json'
+require 'time'
+require 'google/protobuf'
+require 'glazed'
+require 'glazed/proto'
+require 'glazed/proto_json'
+require 'glazed/runner'
+require 'glazed/validation'
+
+module Glazed
+  module Aws
+    # Reads {Proto::V1::Signal}s from the input of a Step Functions execution,
+    # in the shapes its triggers deliver:
+    #
+    # - an EventBridge event, from a rule targeting the state machine, whose `detail` is a Signal
+    # - an array of SQS messages, from an EventBridge Pipe, whose bodies are Signals or EventBridge events
+    # - `{"signals": [...]}`, from EventBridge Scheduler or a manual StartExecution
+    #
+    # A Signal lacking `source` or `received_at` takes the event's source and
+    # time. Entries that do not decode or validate are reported and skipped,
+    # so one bad message does not hold back the others.
+    module SignalEvents
+      # @param input [Object] the execution input, decoded from JSON
+      # @param listener [Proc] receives an event for each skipped entry
+      # @return [Array<Proto::V1::Signal>] the signals
+      #: (untyped input, ?listener: Runner::listener) -> Array[Proto::V1::Signal]
+      def self.extract(input, listener: ->(_event) {})
+        case input
+        when Array then input.flat_map { from_sqs_message(it, listener:) }
+        when Hash
+          if input.key?('detail-type') then from_event(input, listener:)
+          elsif input.key?('signals') then Array(input['signals']).filter_map { decode(it, listener:) }
+          else []
+          end
+        else []
+        end
+      end
+
+      # @param message [Object] an SQS message as an EventBridge Pipe delivers it
+      # @param listener [Proc] receives an event when the message is skipped
+      # @return [Array<Proto::V1::Signal>] the signals in its body
+      #: (untyped message, listener: Runner::listener) -> Array[Proto::V1::Signal]
+      private_class_method def self.from_sqs_message(message, listener:)
+        body = message.is_a?(Hash) ? message['body'] : nil
+        unless body.is_a?(String)
+          skip(listener, 'SQS message without a body')
+          return []
+        end
+
+        begin
+          document = JSON.parse(body) #: untyped
+        rescue JSON::ParserError => e
+          skip(listener, "SQS message #{message['messageId']}: #{e.message}")
+          return []
+        end
+        return from_event(document, listener:) if document.is_a?(Hash) && document.key?('detail-type')
+
+        sent = message.dig('attributes', 'SentTimestamp')
+        received_at = (Time.at(Rational(sent.to_i, 1000)).utc.iso8601(3) if sent) #: String?
+        [decode(document, listener:, received_at:)].compact
+      end
+
+      # @param event [Hash] an EventBridge event
+      # @param listener [Proc] receives an event when the event is skipped
+      # @return [Array<Proto::V1::Signal>] the signal in its detail
+      #: (Hash[String, untyped] event, listener: Runner::listener) -> Array[Proto::V1::Signal]
+      private_class_method def self.from_event(event, listener:)
+        [decode(event['detail'], listener:, source: event['source'], received_at: event['time'])].compact
+      end
+
+      # @param document [Object] a Signal as ProtoJSON, decoded from JSON
+      # @param listener [Proc] receives an event when the document is skipped
+      # @param source [String, nil] source when the document has none
+      # @param received_at [String, nil] RFC 3339 reception time when the document has none
+      # @return [Proto::V1::Signal, nil] the signal; nil when it is invalid
+      #: (untyped document, listener: Runner::listener, ?source: String?, ?received_at: String?) -> Proto::V1::Signal?
+      private_class_method def self.decode(document, listener:, source: nil, received_at: nil)
+        unless document.is_a?(Hash)
+          skip(listener, "signal is not an object: #{JSON.generate(document)}")
+          return
+        end
+
+        defaults = {'source' => source, 'received_at' => received_at}.compact
+        signal = ProtoJson.load(Proto::V1::Signal, JSON.generate(defaults.merge(document)))
+        Validation.validate!(signal)
+        signal
+      rescue Google::Protobuf::ParseError, InvalidMessage => e
+        skip(listener, "invalid signal #{JSON.generate(document)}: #{e.message}")
+        nil
+      end
+
+      # @param listener [Proc] receives the event
+      # @param message [String] why an entry is skipped
+      # @return [void]
+      #: (Runner::listener listener, String message) -> void
+      private_class_method def self.skip(listener, message)
+        listener.call({level: 'error', message: "signal skipped: #{message}"})
+      end
+    end
+  end
+end

data/lib/glazed/aws/state_machine.rb

@@ -0,0 +1,261 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'json'
+require 'google/protobuf'
+require 'glazed'
+require 'glazed/aws'
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/steps'
+
+module Glazed
+  module Aws
+    # Amazon States Language definition of a Standard state machine running
+    # the orchestrated workflows of a config with the {Lambda} function. Each
+    # execution checks when nodes are due, waits, runs every workflow's
+    # batch of due nodes, and checks again, until another execution waits
+    # for the next due node instead. Each step is one state: consecutive
+    # per-node steps run in a Map over the batch, batch steps run once.
+    #
+    # @example
+    #   definition = Glazed::Aws::StateMachine.new(config:, function_arn: 'arn:aws:lambda:...:function:glazed').to_h
+    class StateMachine
+      # Raised when a workflow cannot run under Step Functions.
+      class Unsupported < Glazed::Error; end
+
+      # Nodes processed in parallel when neither the caller nor the config sets it.
+      DEFAULT_MAX_CONCURRENCY = 4 #: Integer
+      # Errors of the Lambda service rather than of the function, worth retrying.
+      LAMBDA_ERRORS = [
+        'Lambda.ServiceException', 'Lambda.AWSLambdaException', 'Lambda.SdkClientException', 'Lambda.TooManyRequestsException',
+      ].freeze #: Array[String]
+      # Longest state name Step Functions accepts.
+      MAX_STATE_NAME = 80 #: Integer
+
+      #: type state = Hash[String, untyped]
+
+      # @param config [Proto::V1::Config] the config, with orchestrator settings
+      # @param function_arn [String] ARN of the {Lambda} function
+      # @param max_concurrency [Integer, nil] nodes processed in parallel; the config's jobs when nil
+      # @param grace_seconds [Integer, nil] seconds after waking until another execution may take over;
+      #   the longest lease when nil
+      # @param max_cycles [Integer, nil] cycles after which an execution ends; 20 when nil
+      # @raise [Glazed::Error] when the config has no orchestrator settings
+      #: (config: Proto::V1::Config, function_arn: String, ?max_concurrency: Integer?, ?grace_seconds: Integer?, ?max_cycles: Integer?) -> void
+      def initialize(config:, function_arn:, max_concurrency: nil, grace_seconds: nil, max_cycles: nil)
+        @config = config #: Proto::V1::Config
+        settings = config.orchestrator or raise Glazed::Error, 'orchestrator is not configured'
+        @settings = settings #: Proto::V1::Orchestrator
+        @function_arn = function_arn #: String
+        jobs = config.jobs.zero? ? DEFAULT_MAX_CONCURRENCY : config.jobs
+        @max_concurrency = max_concurrency || jobs #: Integer
+        @grace_seconds = grace_seconds #: Integer?
+        @max_cycles = max_cycles #: Integer?
+      end
+
+      # Storages that work under Step Functions only with care, with why.
+      #
+      # @return [Array<String>] the warnings
+      #: () -> Array[String]
+      def warnings
+        storages_written.filter_map do |name|
+          case Must.because(@config.storages[name]) { 'steps are checked against the storages' }.backend
+          when :filesystem then "storage #{name} is a local directory; every Lambda instance has its own"
+          when :git
+            git = Must.because(@config.storages[name]&.git) { 'backend is git' }
+            "storage #{name} is a local git repository without remote; every Lambda instance has its own" if git.remote.empty?
+          end
+        end
+      end
+
+      # @return [Hash] the definition
+      # @raise [Unsupported] when a workflow writes to memory storage, which does not outlive one task
+      # @raise [Finder::NotFound, PluginRegistry::UnknownPlugin] when a step's implementation is not available
+      #: () -> Hash[String, untyped]
+      def to_h
+        memory = storages_written.select { @config.storages[it]&.backend == :memory }
+        raise Unsupported, "memory storages do not outlive a Lambda invocation: #{memory.join(', ')}" unless memory.empty?
+
+        {
+          'Comment' => 'glazed coordinator, generated by glazed-aws state-machine',
+          'QueryLanguage' => 'JSONata',
+          'StartAt' => 'Start',
+          'States' => {
+            'Start' => {'Type' => 'Pass', 'Assign' => {'cycle' => 0}, 'Next' => 'Check'},
+            'Check' => invoke(
+              {'check' => {
+                'token' => '{% $states.context.Execution.Name %}', 'cycle' => '{% $cycle %}',
+                'grace_seconds' => @grace_seconds || 0, 'max_cycles' => @max_cycles || 0,
+              }},
+              execution_input: '{% $cycle = 0 ? $states.context.Execution.Input : null %}',
+            ).merge('Retry' => [retry_on(LAMBDA_ERRORS + ['States.TaskFailed'])], 'Next' => 'Armed?'),
+            'Armed?' => {
+              'Type' => 'Choice',
+              'Choices' => [{'Condition' => '{% $exists($states.input.wait_until) %}', 'Next' => 'Sleep'}],
+              'Default' => 'Done',
+            },
+            'Sleep' => {'Type' => 'Wait', 'Timestamp' => '{% $states.input.wait_until %}', 'Next' => 'Prepare'},
+            'Prepare' => invoke({'prepare' => {}}).merge('Retry' => [retry_on(LAMBDA_ERRORS + ['States.TaskFailed'])], 'Next' => 'Workflows'),
+            'Workflows' => {
+              'Type' => 'Parallel',
+              'Branches' => @settings.workflows.keys.sort.map { branch(it) },
+              'Catch' => [{'ErrorEquals' => ['States.ALL'], 'Next' => 'Next cycle'}],
+              'Next' => 'Next cycle',
+            },
+            'Next cycle' => {'Type' => 'Pass', 'Assign' => {'cycle' => '{% $cycle + 1 %}'}, 'Next' => 'Check'},
+            'Done' => {'Type' => 'Succeed'},
+          },
+        }
+      end
+
+      # @param workflow [String] workflow name
+      # @return [Hash] the Parallel branch running the workflow's batch
+      #: (String workflow) -> StateMachine::state
+      private def branch(workflow)
+        steps = Must.because(@config.workflows[workflow]) { 'validation checks orchestrated workflows exist' }.steps.to_a
+        chunks = steps.chunk_while { |a, b| node_step?(a) && node_step?(b) }.to_a
+        complete = name(workflow, 'complete')
+        states = {} #: Hash[String, StateMachine::state]
+        maps = 0
+        names = chunks.map { |chunk| node_step?(chunk.fetch(0)) ? name(workflow, "each node ##{maps += 1}") : name(workflow, chunk.fetch(0).name) }
+        names << complete
+
+        start = name(workflow, 'batch')
+        states[start] = {'Type' => 'Pass', 'Output' => "{% $lookup($states.input.batches, #{JSON.generate(workflow)}) %}", 'Next' => name(workflow, 'any due?')}
+        states[name(workflow, 'any due?')] = {
+          'Type' => 'Choice',
+          'Choices' => [{'Condition' => '{% $count($states.input.nodes) > 0 %}', 'Next' => names.fetch(0)}],
+          'Default' => name(workflow, 'none due'),
+        }
+        states[name(workflow, 'none due')] = {'Type' => 'Succeed'}
+        chunks.each_with_index do |chunk, index|
+          following = names.fetch(index + 1)
+          if node_step?(chunk.fetch(0))
+            states[names.fetch(index)] = map(workflow, chunk, names.fetch(index), following)
+          else
+            step = chunk.fetch(0)
+            failed = "{% $merge([$states.input, {\"error\": #{JSON.generate("#{step.name}: ")} & $states.errorOutput.Error & \": \" & $states.errorOutput.Cause}]) %}"
+            # A failed batch skips the remaining steps.
+            after = following == complete ? complete : name(workflow, "#{step.name} failed?")
+            states[names.fetch(index)] = invoke({'batch_step' => {'step' => step.name, 'batch' => '{% $states.input %}'}}).merge(
+              'Retry' => [retry_on(LAMBDA_ERRORS)],
+              'Catch' => [{'ErrorEquals' => ['States.ALL'], 'Output' => failed, 'Next' => complete}],
+              'Next' => after,
+            )
+            if after != complete
+              states[after] = {
+                'Type' => 'Choice',
+                'Choices' => [{'Condition' => '{% $exists($states.input.error) %}', 'Next' => complete}],
+                'Default' => following,
+              }
+            end
+          end
+        end
+        states[complete] = invoke({'complete' => {'batch' => '{% $states.input %}'}}).merge(
+          'Retry' => [retry_on(LAMBDA_ERRORS + ['States.TaskFailed'])],
+          'Output' => '{% {"workflow": $states.result.Payload.workflow, "nodes": $count($states.result.Payload.nodes)} %}',
+          'End' => true,
+        )
+        {'StartAt' => start, 'States' => states}
+      end
+
+      # @param workflow [String] workflow name
+      # @param chunk [Array<Proto::V1::Step>] consecutive per-node steps
+      # @param map_name [String] name of the Map state
+      # @param following [String] state after the Map
+      # @return [Hash] the Map state running the steps for each node of the batch
+      #: (String workflow, Array[Proto::V1::Step] chunk, String map_name, String following) -> StateMachine::state
+      private def map(workflow, chunk, map_name, following)
+        done = name(workflow, "#{map_name.delete_prefix("#{workflow}: ")} done")
+        states = {} #: Hash[String, StateMachine::state]
+        chunk.each_with_index do |step, index|
+          after = chunk[index + 1]&.then { name(workflow, it.name) } || done
+          failure = '{% {"state": $states.input.state, "failure": ' \
+                    "{\"step\": #{JSON.generate(step.name)}, \"error\": $states.errorOutput.Error & \": \" & $states.errorOutput.Cause}} %}"
+          states[name(workflow, step.name)] = invoke(
+            {'node_step' => {'workflow' => workflow, 'step' => step.name, 'outcome' => '{% $states.input %}'}},
+          ).merge(
+            'Retry' => [retry_on(LAMBDA_ERRORS)],
+            'Catch' => [{'ErrorEquals' => ['States.ALL'], 'Output' => failure, 'Next' => after}],
+            'Next' => after,
+          )
+        end
+        states[done] = {'Type' => 'Pass', 'End' => true}
+        {
+          'Type' => 'Map',
+          'Items' => '{% $states.input.nodes %}',
+          'MaxConcurrency' => @max_concurrency,
+          'ItemProcessor' => {'ProcessorConfig' => {'Mode' => 'INLINE'}, 'StartAt' => name(workflow, chunk.fetch(0).name), 'States' => states},
+          'Output' => '{% $merge([$states.input, {"nodes": $states.result}]) %}',
+          'Next' => following,
+        }
+      end
+
+      # @param task [Hash] the glazed.v1.CoordinatorTask, with JSONata expressions
+      # @param execution_input [String, nil] JSONata expression of the execution input to pass
+      # @return [Hash] a Task state invoking the function
+      #: (Hash[String, untyped] task, ?execution_input: String?) -> StateMachine::state
+      private def invoke(task, execution_input: nil)
+        payload = {'task' => task} #: Hash[String, untyped]
+        payload['execution_input'] = execution_input if execution_input
+        {
+          'Type' => 'Task',
+          'Resource' => 'arn:aws:states:::lambda:invoke',
+          'Arguments' => {'FunctionName' => @function_arn, 'Payload' => payload},
+          'Output' => '{% $states.result.Payload %}',
+        }
+      end
+
+      # @param errors [Array<String>] errors to retry
+      # @return [Hash] a retrier with backoff
+      #: (Array[String] errors) -> Hash[String, untyped]
+      private def retry_on(errors)
+        {'ErrorEquals' => errors, 'IntervalSeconds' => 2, 'MaxAttempts' => 3, 'BackoffRate' => 2, 'JitterStrategy' => 'FULL'}
+      end
+
+      # @param workflow [String] workflow name
+      # @param label [String] what the state does
+      # @return [String] a state name unique across the state machine
+      # @raise [Unsupported] when the name exceeds what Step Functions accepts
+      #: (String workflow, String label) -> String
+      private def name(workflow, label)
+        name = "#{workflow}: #{label}"
+        raise Unsupported, "state name #{name.inspect} is longer than #{MAX_STATE_NAME} characters" if name.length > MAX_STATE_NAME
+
+        name
+      end
+
+      # @param step [Proto::V1::Step] step definition
+      # @return [Boolean] whether the step runs for each node
+      #: (Proto::V1::Step step) -> bool
+      private def node_step?(step)
+        implementation = if (packed = step.plugin)
+          Steps::PLUGINS.resolve(packed).first
+        else
+          Steps.find(Must.because(step.action) { 'validation requires an action' })
+        end
+        implementation.is_a?(Class) && implementation.ancestors.include?(Steps::NodeStep)
+      end
+
+      # @return [Array<String>] names of the storages steps of the orchestrated workflows write to
+      #: () -> Array[String]
+      private def storages_written
+        @settings.workflows.keys.flat_map do |workflow|
+          (@config.workflows[workflow]&.steps || []).filter_map do |step|
+            action = step.action
+            next unless action && action != :plugin
+
+            field = Proto::V1::Step.descriptor.lookup(action.to_s)&.subtype&.lookup('storage')
+            next unless field
+
+            storage = field.get(step[action.to_s]) #: untyped
+            storage.is_a?(String) && !storage.empty? ? storage : nil
+          end
+        end.uniq
+      end
+    end
+  end
+end

data/lib/glazed/aws/version.rb

@@ -6,6 +6,6 @@
 module Glazed
   module Aws
     # Gem version.
-    VERSION = '0.1.0' #: String
+    VERSION = '0.2.0' #: String
   end
 end

data/lib/glazed/secrets/ssm.rb

@@ -0,0 +1,97 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'aws-sdk-ssm'
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/secret'
+require 'glazed/secrets'
+
+module Glazed
+  # Namespace of secret sources; glazed-aws adds {Ssm}.
+  module Secrets
+    # AWS Systems Manager Parameter Store parameter, decrypted when it is a
+    # SecureString. A fetched value is reused for cache_seconds, so nodes of a
+    # batch sharing a password fetch it once per resolver.
+    #
+    # @example
+    #   Glazed::Secrets::Ssm.new(name: '/glazed/backup/password').value
+    class Ssm < Base
+      # Seconds a fetched value is reused when the settings set none.
+      DEFAULT_CACHE_SECONDS = 300 #: Integer
+
+      # @param name [String] name or ARN of the parameter
+      # @param region [String, nil] region of the parameter; the SDK default when nil
+      # @param cache_seconds [Integer] seconds a fetched value is reused
+      # @param client [Aws::SSM::Client, nil] client to use instead of building one
+      # @param clock [Proc] returns monotonic seconds
+      #: (name: String, ?region: String?, ?cache_seconds: Integer, ?client: ::Aws::SSM::Client?, ?clock: ^() -> Float) -> void
+      def initialize(name:, region: nil, cache_seconds: DEFAULT_CACHE_SECONDS, client: nil, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC).to_f })
+        super()
+        @name = name #: String
+        @region = region #: String?
+        @cache_seconds = cache_seconds #: Integer
+        @client = client #: ::Aws::SSM::Client?
+        @clock = clock #: ^() -> Float
+        @value = nil #: String?
+        @fetched_at = nil #: Float?
+        @lock = Mutex.new #: Thread::Mutex
+      end
+
+      # Builds a source from {Proto::V1::SsmParameter}.
+      #
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::SsmParameter}
+      # @return [Ssm] the source
+      # @raise [Must::Violation] when settings is another message
+      #: (Google::Protobuf::MessageExts settings) -> Ssm
+      def self.from_proto(settings)
+        ssm = Must.be_a(settings, Proto::V1::SsmParameter)
+        new(
+          name: ssm.name,
+          region: ssm.region.empty? ? nil : ssm.region,
+          cache_seconds: ssm.cache_seconds.zero? ? DEFAULT_CACHE_SECONDS : ssm.cache_seconds,
+        )
+      end
+
+      # @return [String] name or ARN of the parameter
+      #: String
+      attr_reader :name
+
+      # @return [String] the parameter value, fetched again once cache_seconds have passed
+      # @raise [Secret::Unavailable] when the parameter or its version does not exist
+      # @raise [Aws::SSM::Errors::ServiceError] when the request fails otherwise, such as for lack of permission
+      #: () -> String
+      def value
+        @lock.synchronize do
+          now = @clock.call
+          cached = @value
+          fetched_at = @fetched_at
+          return cached if cached && fetched_at && now - fetched_at < @cache_seconds
+
+          fetch.tap do |fetched|
+            @value = fetched
+            @fetched_at = now
+          end
+        end
+      end
+
+      # @return [String] the current value
+      # @raise [Secret::Unavailable] when the parameter or its version does not exist
+      #: () -> String
+      private def fetch
+        parameter = client.get_parameter(name:, with_decryption: true).parameter
+        parameter&.value or raise Secret::Unavailable, "SSM parameter #{name} has no value"
+      rescue ::Aws::SSM::Errors::ParameterNotFound, ::Aws::SSM::Errors::ParameterVersionNotFound => e
+        raise Secret::Unavailable, "SSM parameter #{name}: #{e.message}"
+      end
+
+      # @return [Aws::SSM::Client] the client
+      #: () -> ::Aws::SSM::Client
+      private def client
+        @client ||= @region ? ::Aws::SSM::Client.new(region: @region) : ::Aws::SSM::Client.new
+      end
+    end
+  end
+end

data/lib/glazed/storages/s3.rb

@@ -41,10 +41,11 @@ module Glazed
       # Builds a storage from {Proto::V1::S3Storage}.
       #
       # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::S3Storage}
+      # @param context [StorageContext] secrets the settings refer to
       # @return [S3] the storage
       # @raise [Must::Violation] when settings is another message
-      #: (Google::Protobuf::MessageExts settings) -> S3
-      def self.from_proto(settings)
+      #: (Google::Protobuf::MessageExts settings, context: StorageContext) -> S3
+      def self.from_proto(settings, context:)
         s3 = Must.be_a(settings, Proto::V1::S3Storage)
         new(bucket: s3.bucket, prefix: s3.prefix, region: s3.region.empty? ? nil : s3.region)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: glazed-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sorah Fukumori
@@ -11,6 +11,20 @@ date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: glazed
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -24,7 +38,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-s3
+  name: aws-sdk-ssm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dry-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +67,8 @@ dependencies:
         version: '0'
 email:
 - her@sorah.jp
-executables: []
+executables:
+- glazed-aws
 extensions: []
 extra_rdoc_files: []
 files:
@@ -49,8 +78,15 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/glazed-aws
 - lib/glazed/aws.rb
+- lib/glazed/aws/command.rb
+- lib/glazed/aws/command/state_machine.rb
+- lib/glazed/aws/lambda.rb
+- lib/glazed/aws/signal_events.rb
+- lib/glazed/aws/state_machine.rb
 - lib/glazed/aws/version.rb
+- lib/glazed/secrets/ssm.rb
 - lib/glazed/storages/s3.rb
 homepage: https://github.com/sorah/glazed
 licenses: