smith-agents 0.11.0 → 0.12.1

data/lib/smith/artifacts/active_record_store.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+require_relative "store_rules"
+
+module Smith
+  module Artifacts
+    # An artifact store in a database table, through the host's Active
+    # Record model (a class or its name):
+    #
+    #   create_table :smith_artifacts do |t|
+    #     t.string :ref, null: false, index: { unique: true }
+    #     t.binary :data, null: false, limit: 64.megabytes   # MySQL's blob holds 64 KiB
+    #     t.string :content_type, null: false
+    #     t.string :execution_namespace, index: true
+    #     t.timestamps
+    #   end
+    #
+    # Each artifact is one row, its ref a random UUID (after the store's
+    # tenant `namespace:` when one is set), with the run that stored it, so
+    # the store can say which run stored a ref.
+    class ActiveRecordStore
+      include StoreRules
+
+      INNER_REF = StoreRules::UUID
+
+      def initialize(model:, namespace: nil)
+        @model_source = model.is_a?(String) ? model.dup.freeze : model
+        @namespace = check_namespace!(namespace)
+      end
+
+      def store(data, content_type: "application/octet-stream", execution_namespace: nil)
+        enforce_tenant_isolation!
+        check_data!(data)
+        ref = generate_ref(SecureRandom.uuid)
+        model_class.create!(ref:, data:, content_type:, execution_namespace:)
+        ref
+      end
+
+      def fetch(ref)
+        enforce_tenant_isolation!
+        model_class.find_by(ref:)&.data if owns_ref?(ref)
+      end
+
+      # Whether a run (its execution namespace) stored this ref.
+      def stored_by?(ref, execution_namespace)
+        !execution_namespace.nil? && owns_ref?(ref) && model_class.where(ref:, execution_namespace:).exists?
+      end
+
+      def model = model_class
+
+      def expired(retention: nil, execution_namespace: nil)
+        retention = effective_retention(retention)
+        return [] unless retention
+
+        scope = model_class.where(model_class.arel_table[:created_at].lt(Time.now.utc - retention))
+        scope = scope.where(execution_namespace:) if execution_namespace
+        scope.pluck(:ref).select { |ref| owns_ref?(ref) }
+      end
+
+      # Removes the artifact (when a run is named, only if that run stored
+      # it).
+      def delete(ref, execution_namespace: nil)
+        enforce_tenant_isolation!
+        return unless owns_ref?(ref)
+
+        scope = model_class.where(ref:)
+        scope = scope.where(execution_namespace:) if execution_namespace
+        scope.delete_all
+        nil
+      end
+
+      private
+
+      def model_class = @model_source.is_a?(String) ? Object.const_get(@model_source) : @model_source
+    end
+  end
+end

data/lib/smith/artifacts/file.rb

@@ -1,38 +1,71 @@
 # frozen_string_literal: true
 
-require "securerandom"
+require "fileutils"
 require "json"
+require "securerandom"
+
+require_relative "store_rules"
 
 module Smith
   module Artifacts
+    # An artifact store in a directory (`dir:`), shared only when the
+    # directory is. Each artifact is a file named by a random UUID (after
+    # the store's tenant `namespace:` when one is set) beside a `.meta`
+    # file with its type and the run that stored it; no ref names any other
+    # file.
     class File
+      include StoreRules
+
+      INNER_REF = StoreRules::UUID
+
       def initialize(dir:, namespace: nil)
         @dir = dir
-        @namespace = namespace
+        @namespace = check_namespace!(namespace)
       end
 
       def store(data, content_type: "application/octet-stream", execution_namespace: nil)
-        ref = generate_ref
+        enforce_tenant_isolation!
+        check_data!(data)
+        ref = generate_ref(SecureRandom.uuid)
         meta = { content_type: content_type }
         meta[:execution_namespace] = execution_namespace if execution_namespace
-        ::File.write(::File.join(@dir, ref), data)
+        ::File.binwrite(::File.join(@dir, ref), data)
         ::File.write(::File.join(@dir, "#{ref}.meta"), JSON.generate(meta))
         ref
       end
 
       def fetch(ref)
+        enforce_tenant_isolation!
+        return unless owns_ref?(ref)
+
         path = ::File.join(@dir, ref)
-        ::File.exist?(path) ? ::File.read(path) : nil
+        ::File.binread(path) if ::File.file?(path)
+      end
+
+      # Removes the artifact (when a run is named, only if that run stored
+      # it).
+      def delete(ref, execution_namespace: nil)
+        enforce_tenant_isolation!
+        return unless owns_ref?(ref) && (execution_namespace.nil? || stored_by?(ref, execution_namespace))
+
+        [ref, "#{ref}.meta"].each { |name| ::FileUtils.rm_f(::File.join(@dir, name)) }
+        nil
+      end
+
+      # Whether a run (its execution namespace) stored this ref.
+      def stored_by?(ref, execution_namespace)
+        !execution_namespace.nil? && owns_ref?(ref) && stored_namespace(ref) == execution_namespace
       end
 
       def expired(retention: nil, execution_namespace: nil)
+        retention = effective_retention(retention)
         return [] unless retention
 
         cutoff = Time.now.utc - retention
-        Dir.glob(::File.join(@dir, "*")).reject { |f| f.end_with?(".meta") }.filter_map do |path|
+        Dir.glob(::File.join(@dir, "*")).filter_map do |path|
           ref = ::File.basename(path)
-          next unless ::File.mtime(path).utc < cutoff
-          next if execution_namespace && !matches_execution_namespace?(ref, execution_namespace)
+          next unless owns_ref?(ref) && ::File.mtime(path).utc < cutoff
+          next if execution_namespace && stored_namespace(ref) != execution_namespace
 
           ref
         end
@@ -40,19 +73,13 @@ module Smith
 
       private
 
-      def generate_ref
-        raw = SecureRandom.uuid
-        @namespace ? "#{@namespace}:#{raw}" : raw
-      end
-
-      def matches_execution_namespace?(ref, execution_namespace)
+      def stored_namespace(ref)
         meta_path = ::File.join(@dir, "#{ref}.meta")
-        return false unless ::File.exist?(meta_path)
+        return unless ::File.exist?(meta_path)
 
-        meta = JSON.parse(::File.read(meta_path), symbolize_names: true)
-        meta[:execution_namespace] == execution_namespace
+        JSON.parse(::File.read(meta_path))["execution_namespace"]
       rescue JSON::ParserError
-        false
+        nil
       end
     end
   end

data/lib/smith/artifacts/memory.rb

@@ -2,73 +2,84 @@
 
 require "digest"
 
+require_relative "store_rules"
+
 module Smith
   module Artifacts
+    # An artifact store in the process: for one process and for tests.
+    # Refs are content hashes, so runs that store the same bytes share one;
+    # each run's claim on it (and when it made the claim) is kept apart, so
+    # a run's delete drops only its own claim, and the bytes go with the
+    # last claim.
     class Memory
+      include StoreRules
+
+      INNER_REF = /\A\h{64}\z/
+      HOST = :host
+      private_constant :HOST
+
       attr_reader :namespace
 
       def initialize(namespace: nil)
-        @namespace = namespace
+        @namespace = check_namespace!(namespace)
         @store = {}
         @metadata = {}
       end
 
       def store(data, content_type: "application/octet-stream", execution_namespace: nil)
         enforce_tenant_isolation!
-        ref = generate_ref(data)
+        check_data!(data)
+        ref = generate_ref(Digest::SHA256.hexdigest(data))
         @store[ref] = data
-        @metadata[ref] ||= { content_type: content_type, stored_at: Time.now.utc, execution_namespaces: [] }
-        tag_execution_namespace(ref, execution_namespace)
+        meta = (@metadata[ref] ||= { content_type:, claims: {} })
+        meta[:claims][execution_namespace || HOST] = Time.now.utc
         ref
       end
 
       def fetch(ref)
         enforce_tenant_isolation!
-        return nil unless owns_ref?(ref)
-
-        @store[ref]
-      end
-
-      def expired(retention: nil, execution_namespace: nil)
-        effective_retention = retention || Smith.config.artifact_retention
-        return [] unless effective_retention
-
-        cutoff = Time.now.utc - effective_retention
-        @metadata.select { |ref, meta| expired_match?(ref, meta, cutoff, execution_namespace) }.keys
+        @store[ref] if owns_ref?(ref)
       end
 
-      private
+      # Drops a run's claim (or, with no run named, every claim); the bytes
+      # go when no claim is left.
+      def delete(ref, execution_namespace: nil)
+        enforce_tenant_isolation!
+        meta = @metadata[ref] if owns_ref?(ref)
+        return unless meta
 
-      def tag_execution_namespace(ref, execution_namespace)
-        return unless execution_namespace
+        execution_namespace ? meta[:claims].delete(execution_namespace) : meta[:claims].clear
+        return unless meta[:claims].empty?
 
-        namespaces = @metadata[ref][:execution_namespaces]
-        namespaces << execution_namespace unless namespaces.include?(execution_namespace)
+        @store.delete(ref)
+        @metadata.delete(ref)
+        nil
       end
 
-      def expired_match?(ref, meta, cutoff, execution_namespace)
-        owns_ref?(ref) &&
-          meta[:stored_at] < cutoff &&
-          (execution_namespace.nil? || meta[:execution_namespaces]&.include?(execution_namespace))
-      end
+      # Whether a run (its execution namespace) stored this ref.
+      def stored_by?(ref, execution_namespace)
+        return false if execution_namespace.nil? || !owns_ref?(ref)
 
-      def generate_ref(data)
-        content_hash = Digest::SHA256.hexdigest(data.to_s)
-        @namespace ? "#{@namespace}:#{content_hash}" : content_hash
+        @metadata.dig(ref, :claims)&.key?(execution_namespace) || false
       end
 
-      def owns_ref?(ref)
-        if @namespace
-          ref.start_with?("#{@namespace}:")
-        else
-          !ref.include?(":")
+      # Refs whose claims (or the run's claim, when one is named) are all
+      # older than the retention.
+      def expired(retention: nil, execution_namespace: nil)
+        retention = effective_retention(retention)
+        return [] unless retention
+
+        cutoff = Time.now.utc - retention
+        @metadata.filter_map do |ref, meta|
+          ref if owns_ref?(ref) && claims_before?(meta[:claims], execution_namespace, cutoff)
         end
       end
 
-      def enforce_tenant_isolation!
-        return unless Smith.config.artifact_tenant_isolation
+      private
 
-        raise Smith::Error, "artifact_tenant_isolation requires a namespace" unless @namespace
+      def claims_before?(claims, execution_namespace, cutoff)
+        claims = claims.slice(execution_namespace) if execution_namespace
+        !claims.empty? && claims.values.all? { |stored_at| stored_at < cutoff }
       end
     end
   end

data/lib/smith/artifacts/s3.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+require_relative "store_rules"
+
+module Smith
+  module Artifacts
+    # An artifact store in an S3-compatible bucket: AWS S3, Cloudflare R2
+    # (`endpoint: "https://<account>.r2.cloudflarestorage.com"`,
+    # `region: "auto"`), MinIO and the like. Each artifact is one object
+    # under `prefix`, its ref a random UUID (after the store's tenant
+    # `namespace:` when one is set), and the run that stored it is kept in
+    # the object's metadata, so the store can say which run stored a ref.
+    #
+    # The client is the host's `Aws::S3::Client` (`client:`), or one built
+    # from the remaining options (`region:`, `endpoint:`,
+    # `access_key_id:`, `secret_access_key:`, ...), which needs the
+    # `aws-sdk-s3` gem in the host's bundle; Smith does not depend on it.
+    # Only keys of the store's ref shape under the prefix are artifacts, so
+    # other objects in the bucket are neither read, listed nor deleted.
+    class S3
+      include StoreRules
+
+      INNER_REF = StoreRules::UUID
+      NAMESPACE_KEY = "smith-execution-namespace"
+
+      def initialize(bucket:, prefix: "smith/artifacts", namespace: nil, client: nil, **client_options)
+        if client && client_options.any?
+          raise ArgumentError, "give Smith::Artifacts::S3 a client: or the options to build one, not both"
+        end
+
+        @bucket = bucket
+        @prefix = prefix.to_s.delete_suffix("/")
+        @namespace = check_namespace!(namespace)
+        @client = client || self.class.build_client(**client_options)
+      end
+
+      def self.build_client(**)
+        require "aws-sdk-s3"
+        ::Aws::S3::Client.new(**)
+      rescue LoadError
+        raise Smith::Error, "Smith::Artifacts::S3 needs the aws-sdk-s3 gem in the host's bundle, or a client:"
+      end
+
+      def store(data, content_type: "application/octet-stream", execution_namespace: nil)
+        enforce_tenant_isolation!
+        check_data!(data)
+        ref = generate_ref(SecureRandom.uuid)
+        metadata = execution_namespace ? { NAMESPACE_KEY => execution_namespace } : {}
+        @client.put_object(bucket: @bucket, key: object_key(ref), body: data, content_type:, metadata:)
+        ref
+      end
+
+      def fetch(ref)
+        enforce_tenant_isolation!
+        return unless owns_ref?(ref)
+
+        @client.get_object(bucket: @bucket, key: object_key(ref)).body.read
+      rescue StandardError => e
+        raise unless missing?(e)
+      end
+
+      # Whether a run (its execution namespace) stored this ref.
+      def stored_by?(ref, execution_namespace)
+        return false if execution_namespace.nil? || !owns_ref?(ref)
+
+        @client.head_object(bucket: @bucket, key: object_key(ref)).metadata[NAMESPACE_KEY] == execution_namespace
+      rescue StandardError => e
+        raise unless missing?(e)
+
+        false
+      end
+
+      # Listing the prefix, then (for one run's refs) a HEAD per expired
+      # object.
+      def expired(retention: nil, execution_namespace: nil)
+        retention = effective_retention(retention)
+        return [] unless retention
+
+        cutoff = Time.now.utc - retention
+        objects.filter_map do |object|
+          ref = object.key.delete_prefix(key_prefix)
+          next unless object.last_modified.utc < cutoff && owns_ref?(ref)
+          next if execution_namespace && !stored_by?(ref, execution_namespace)
+
+          ref
+        end
+      end
+
+      # Removes the artifact (when a run is named, only if that run stored
+      # it).
+      def delete(ref, execution_namespace: nil)
+        enforce_tenant_isolation!
+        return unless owns_ref?(ref) && (execution_namespace.nil? || stored_by?(ref, execution_namespace))
+
+        @client.delete_object(bucket: @bucket, key: object_key(ref))
+        nil
+      end
+
+      private
+
+      def key_prefix = @prefix.empty? ? "" : "#{@prefix}/"
+
+      def object_key(ref) = "#{key_prefix}#{ref}"
+
+      # Every object under the prefix, page by page.
+      def objects
+        Enumerator.new do |yielder|
+          token = nil
+          loop do
+            page = @client.list_objects_v2(bucket: @bucket, prefix: key_prefix, continuation_token: token)
+            page.contents.each { |object| yielder << object }
+            break unless page.is_truncated
+
+            token = page.next_continuation_token
+          end
+        end
+      end
+
+      # The SDK's missing-object errors (NoSuchKey on a read, NotFound on a
+      # head), by name, so Smith loads no SDK class.
+      def missing?(error) = error.class.name.to_s.end_with?("::NoSuchKey", "::NotFound")
+    end
+  end
+end

data/lib/smith/artifacts/scoped_store.rb

@@ -2,27 +2,75 @@
 
 module Smith
   module Artifacts
+    # A run's view of the artifact store: it stores under its own
+    # namespace, and reads its own refs and those of the namespaces it may
+    # read (a child run's parent and that parent's ancestors). A store that
+    # records who stored a ref (`stored_by?`, as every built-in store does)
+    # is asked, so a ref names an artifact one of those runs stored and no
+    # other; with any other store the namespace prefix alone decides a read,
+    # and a delete is refused.
+    # Without a `backend:`, the configured store is resolved when a step
+    # first stores or reads, so a run that keeps no artifacts needs none.
     class ScopedStore
-      def initialize(backend:, namespace:)
+      def initialize(namespace:, backend: nil, readable: [])
         @backend = backend
         @namespace = namespace
+        @readable = [namespace, *readable].freeze
       end
 
       def store(data, content_type: "application/octet-stream")
-        inner_ref = @backend.store(data, content_type: content_type, execution_namespace: @namespace)
+        inner_ref = backend.store(data, content_type: content_type, execution_namespace: @namespace)
         "#{@namespace}:#{inner_ref}"
       end
 
       def fetch(ref)
-        return nil unless ref.start_with?("#{@namespace}:")
+        return unless ref.is_a?(String)
 
-        inner_ref = ref.delete_prefix("#{@namespace}:")
-        @backend.fetch(inner_ref)
+        namespace = @readable.find { |candidate| ref.start_with?("#{candidate}:") }
+        return nil unless namespace
+
+        inner = ref.delete_prefix("#{namespace}:")
+        return nil if vouches? && !backend.stored_by?(inner, namespace)
+
+        backend.fetch(inner)
+      end
+
+      # Removes one of this run's own refs (never a readable ancestor's or
+      # another run's): only through a store that can tell which run stored
+      # a ref. A store whose delete takes the run (every built-in one) is
+      # asked to remove only what this run stored, and Memory drops only
+      # this run's claim on bytes other runs share; any other store deletes
+      # a ref this run stored.
+      def delete(ref)
+        unless vouches? && backend.respond_to?(:delete)
+          raise Smith::Error, "deleting a run's artifact needs a store with stored_by? and delete"
+        end
+        return unless ref.is_a?(String)
+
+        inner = ref.delete_prefix("#{@namespace}:")
+        if per_run_delete? then backend.delete(inner, execution_namespace: @namespace)
+        elsif backend.stored_by?(inner, @namespace) then backend.delete(inner)
+        end
+        nil
       end
 
+      # Whether the store can tell which run stored a ref.
+      def vouches? = backend.respond_to?(:stored_by?)
+
       def expired(retention: nil)
-        @backend.expired(retention: retention, execution_namespace: @namespace)
-                .map { |ref| "#{@namespace}:#{ref}" }
+        raise Smith::Error, "the artifact store does not list expired refs" unless backend.respond_to?(:expired)
+
+        backend.expired(retention: retention, execution_namespace: @namespace).map { |ref| "#{@namespace}:#{ref}" }
+      end
+
+      private
+
+      def backend = @backend ||= Smith.artifact_backend
+
+      def per_run_delete?
+        backend.method(:delete).parameters.any? do |kind, name|
+          %i[key keyreq].include?(kind) && name == :execution_namespace
+        end
       end
     end
   end

data/lib/smith/artifacts/store_rules.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Smith
+  module Artifacts
+    # What every built-in store holds to: a tenant `namespace:` of letters,
+    # digits, `-` and `_` (so its refs stay readable), the
+    # `artifact_tenant_isolation` setting (refuse to work without a
+    # namespace), refs that are the store's own (its tenant's prefix, then
+    # the store's ref shape, nothing else), the `artifact_retention`
+    # setting when no retention is given, and no run behind a ref whose run
+    # is not named.
+    module StoreRules
+      NAMESPACE = /\A[A-Za-z0-9_-]+\z/
+      UUID = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/
+
+      private
+
+      def check_namespace!(namespace)
+        return namespace if namespace.nil? || (namespace.is_a?(String) && NAMESPACE.match?(namespace))
+
+        raise ArgumentError, "an artifact store's namespace: must be letters, digits, - and _"
+      end
+
+      def enforce_tenant_isolation!
+        return unless Smith.config.artifact_tenant_isolation
+
+        raise Smith::Error, "artifact_tenant_isolation requires a namespace" unless @namespace
+      end
+
+      # The store's tenant prefix, then a ref of the store's shape.
+      def owns_ref?(ref)
+        return false unless ref.is_a?(String)
+
+        inner = @namespace ? ref.delete_prefix("#{@namespace}:") : ref
+        (!@namespace || inner != ref) && self.class::INNER_REF.match?(inner)
+      end
+
+      def effective_retention(retention) = retention || Smith.config.artifact_retention
+
+      # An artifact is bytes: other values would be stored as whatever a
+      # store makes of them (Ruby's inspect, or a refusal).
+      def check_data!(data)
+        raise TypeError, "an artifact is bytes (a String); serialize other values first" unless data.is_a?(String)
+      end
+
+      def generate_ref(raw) = @namespace ? "#{@namespace}:#{raw}" : raw
+    end
+  end
+end

data/lib/smith/artifacts.rb

@@ -1,5 +1,102 @@
 # frozen_string_literal: true
 
 module Smith
-  module Artifacts; end
+  # Where steps keep payloads too large to keep inline, and the documents
+  # and images content parts name. Like workflow state, artifacts must
+  # outlive the process that stored them whenever a run waits, retries or
+  # resumes elsewhere, so the store is configured, never assumed:
+  #
+  #   Smith.configure do |config|
+  #     config.artifact_store = :s3            # or :active_record, :file, :memory, or a store
+  #     config.artifact_options = { bucket: "smith-artifacts",
+  #                                 endpoint: "https://<account>.r2.cloudflarestorage.com" }
+  #   end
+  #
+  # With nothing configured, `Smith.artifacts` raises
+  # Smith::ArtifactStoreMissing, except in `test_mode`, where it is an
+  # in-process Memory store.
+  module Artifacts
+    REQUIRED_METHODS = %i[store fetch].freeze
+    NAMED = { memory: :Memory, file: :File, s3: :S3, active_record: :ActiveRecordStore }.freeze
+
+    # The store a configuration names: a store itself (an object, or a
+    # class or module whose own methods store and fetch), a store class
+    # (built with the options), or one of :memory, :file, :s3 and
+    # :active_record. A store needs `store` and `fetch`; `stored_by?` lets
+    # runs read only their own refs, and `expired` and `delete` serve
+    # cleanup.
+    def self.resolve(store, **)
+      return if store.nil?
+      return store if REQUIRED_METHODS.all? { |method_name| store.respond_to?(method_name) }
+      return validate!(store.new(**)) if store.is_a?(Class)
+      return validate!(store) unless store.is_a?(Symbol) || store.is_a?(String)
+
+      named(store, **)
+    end
+
+    def self.named(store, **)
+      name = NAMED.fetch(store.to_sym) do
+        raise ArgumentError, "unknown artifact store #{store.inspect}; use :s3, :active_record, :file or :memory"
+      end
+      const_get(name).new(**)
+    end
+
+    def self.validate!(store)
+      missing = REQUIRED_METHODS.reject { |method_name| store.respond_to?(method_name) }
+      return store if missing.empty?
+
+      raise ArgumentError, "an artifact store must implement #{missing.join(", ")}"
+    end
+  end
+
+  # Inside a step, the run's scoped view of the artifact store; elsewhere
+  # the store itself.
+  def self.artifacts
+    scoped_artifacts || artifact_backend
+  end
+
+  # The configured store itself, never a run's scoped view of it.
+  # Built once per configuration, under a lock, so threads that resolve it
+  # together share one store.
+  def self.artifact_backend
+    signature = artifact_signature
+    current = @_artifact_current
+    return current.last if current&.first == signature
+
+    (@_artifact_lock ||= Mutex.new).synchronize do
+      current = @_artifact_current
+      return current.last if current&.first == signature
+
+      @_artifact_current = [signature, build_artifact_backend].freeze
+      @_artifact_current.last
+    end
+  end
+
+  def self.build_artifact_backend
+    resolved = Artifacts.resolve(config.artifact_store, **(config.artifact_options || {}))
+    resolved ||= Artifacts::Memory.new if config.test_mode
+    resolved || raise(ArtifactStoreMissing)
+  end
+  private_class_method :build_artifact_backend
+
+  # A configured store object is compared by identity; a name or class by
+  # value, with its options; test_mode matters only when nothing is set.
+  def self.artifact_signature
+    store = config.artifact_store
+    name = store.is_a?(Symbol) || store.is_a?(String) || store.is_a?(Module) ? snapshot_value(store) : store.object_id
+    [name, snapshot_value(config.artifact_options || {}), store.nil? && config.test_mode]
+  end
+  private_class_method :artifact_signature
+
+  def self.artifacts=(store)
+    config.artifact_store = store
+  end
+
+  def self.scoped_artifacts
+    Thread.current[:smith_scoped_artifacts]
+  end
+
+  def self.scoped_artifacts=(store)
+    Thread.current[:smith_scoped_artifacts] = store
+  end
 end