engram 0.4.0 → 0.5.0

data/lib/engram/embedding_metadata.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Engram
+  # Helpers for storing and validating embedding provenance under Engram's
+  # reserved metadata namespace.
+  module EmbeddingMetadata
+    RESERVED_KEY = "_engram"
+    EMBEDDING_KEY = "embedding"
+
+    module_function
+
+    def build(adapter:, dimensions:, model: nil, provider: nil)
+      data = {
+        "adapter" => adapter.to_s,
+        "dimensions" => dimensions
+      }
+      data["provider"] = provider.to_s if provider
+      data["model"] = model.to_s if model
+      data["fingerprint"] = fingerprint(data)
+      data
+    end
+
+    def for_embedder(embedder, embedding: nil)
+      return nil unless embedder.respond_to?(:embedding_metadata)
+
+      metadata = stringify_keys(embedder.embedding_metadata || {})
+      return nil if metadata.empty?
+
+      dimensions = embedding.respond_to?(:length) ? embedding.length : metadata["dimensions"]
+      metadata = metadata.merge("dimensions" => dimensions) if dimensions
+      metadata["fingerprint"] = fingerprint(metadata)
+      metadata
+    end
+
+    def attach(record, embedder: nil, embedding_metadata: nil)
+      metadata = stringify_keys(embedding_metadata || for_embedder(embedder, embedding: record.embedding)) || {}
+      return record if metadata.empty?
+
+      record.with(metadata: merge(record.metadata, metadata))
+    end
+
+    def extract(metadata)
+      metadata = stringify_keys(metadata || {})
+      reserved = metadata[RESERVED_KEY]
+      return {} unless reserved.is_a?(Hash)
+
+      stringify_keys(reserved[EMBEDDING_KEY] || {})
+    end
+
+    def merge(metadata, embedding_metadata)
+      Engram::ReservedMetadata.attach(
+        metadata, EMBEDDING_KEY, embedding_metadata || {}, namespace_description: "embedding metadata"
+      )
+    end
+
+    def search(store, embedding:, embedding_metadata:, scope:, limit:, kinds: nil)
+      store.search(
+        embedding: embedding,
+        embedding_metadata: embedding_metadata,
+        scope: scope,
+        limit: limit,
+        kinds: kinds
+      )
+    rescue ArgumentError => error
+      raise unless unknown_embedding_metadata_keyword?(error)
+
+      store.search(embedding: embedding, scope: scope, limit: limit, kinds: kinds)
+    end
+
+    def validate_query!(embedding, embedding_metadata)
+      metadata = stringify_keys(embedding_metadata || {})
+      return if metadata.empty?
+
+      expected = metadata["dimensions"]
+      return unless expected && embedding.respond_to?(:length) && embedding.length != expected.to_i
+
+      raise Engram::Error,
+        "embedding dimension mismatch: query vector has #{embedding.length} dimensions, metadata declares #{expected}"
+    end
+
+    def validate_record!(record, query_embedding, query_metadata)
+      if query_embedding.respond_to?(:length) && record.embedding.respond_to?(:length) &&
+          query_embedding.length != record.embedding.length
+        raise Engram::Error,
+          "embedding dimension mismatch: query vector has #{query_embedding.length} dimensions, " \
+          "record #{record.id.inspect} has #{record.embedding.length}"
+      end
+
+      stored = extract(record.metadata)
+      return if stored.empty?
+
+      stored_dimensions = stored["dimensions"]
+      if stored_dimensions && record.embedding.respond_to?(:length) && record.embedding.length != stored_dimensions.to_i
+        raise Engram::Error,
+          "embedding metadata mismatch: record #{record.id.inspect} has #{record.embedding.length} dimensions, " \
+          "metadata declares #{stored_dimensions}"
+      end
+
+      query_metadata = stringify_keys(query_metadata || {})
+      return if query_metadata.empty?
+
+      conflicting_key = %w[adapter provider model dimensions fingerprint].find do |key|
+        stored.key?(key) && query_metadata.key?(key) && stored[key].to_s != query_metadata[key].to_s
+      end
+      return unless conflicting_key
+
+      raise Engram::Error,
+        "embedding metadata mismatch for record #{record.id.inspect}: #{conflicting_key} " \
+        "#{stored[conflicting_key].inspect} does not match query #{query_metadata[conflicting_key].inspect}"
+    end
+
+    def fingerprint(metadata)
+      relevant = stringify_keys(metadata).values_at("adapter", "provider", "model", "dimensions")
+      Digest::SHA256.hexdigest(relevant.map { |value| value.nil? ? "" : value.to_s }.join("\0"))
+    end
+
+    def stringify_keys(value)
+      case value
+      when Hash
+        value.each_with_object({}) do |(key, nested), out|
+          out[key.to_s] = stringify_keys(nested)
+        end
+      else
+        value
+      end
+    end
+
+    def unknown_embedding_metadata_keyword?(error)
+      error.message.include?("unknown keyword: :embedding_metadata") ||
+        error.message.include?("unknown keyword: \"embedding_metadata\"")
+    end
+  end
+end

data/lib/engram/extractors/llm_extractor.rb

@@ -58,14 +58,15 @@ module Engram
           next if content.empty?
           next if (fact["confidence"] || 1.0).to_f < @min_confidence
 
-          Engram::Record.new(
+          embedding = @embedder.embed(content)
+          Engram::EmbeddingMetadata.attach(Engram::Record.new(
             content: content,
             scope: scope,
             kind: fact["kind"] || "fact",
             importance: (fact["importance"] || 1.0).to_f,
             metadata: {confidence: (fact["confidence"] || 1.0).to_f},
-            embedding: @embedder.embed(content)
-          )
+            embedding: embedding
+          ), embedder: @embedder)
         end
       end
 

data/lib/engram/memory.rb

@@ -16,10 +16,11 @@ module Engram
     # persistence policy rejects the record.
     def add(content, kind: :fact, importance: 1.0, metadata: {})
       Engram::Instrumentation.instrument("add", Engram::Instrumentation.payload(scope: scope, store: @store, kind: kind)) do
+        embedding = @embedder.embed(content)
         record = Record.new(
           content: content,
           scope: scope,
-          embedding: @embedder.embed(content),
+          embedding: embedding,
           kind: kind,
           importance: importance,
           metadata: metadata
@@ -85,6 +86,17 @@ module Engram
       @store.all(scope: scope)
     end
 
+    # Recompute embeddings (and embedding metadata) for memories in the scope.
+    # When `stale_only` is true, only records whose current metadata does not match the
+    # active embedder are rebuilt.
+    def rebuild_embeddings(stale_only: true, batch_size: 100)
+      UseCases::RebuildEmbeddings.new(store: @store, embedder: @embedder).call(
+        scope: scope,
+        stale_only: stale_only,
+        batch_size: batch_size
+      )
+    end
+
     # Prune stale memories. `older_than` is a duration in seconds; `min_importance` keeps
     # memories at or above that importance even when old. Returns the forgotten records.
     def forget_stale(older_than:, min_importance: Float::INFINITY)

data/lib/engram/persistence.rb

@@ -11,14 +11,17 @@ module Engram
       @persistence_policy = persistence_policy
     end
 
-    def add(record)
+    def add(record, scope: record.scope)
       record = prepare(record)
+      if record && record.scope != scope
+        raise Engram::Error, "cannot move memory across scopes"
+      end
       @store.add(record) if record
     end
 
-    def update(id:, record:)
+    def update(scope:, id:, record:)
       record = prepare(record)
-      @store.update(id: id, record: record) if record
+      @store.update(scope: scope, id: id, record: record) if record
     end
 
     private
@@ -27,8 +30,10 @@ module Engram
       original_content = record.content
       record = @before_persist.call(record) if @before_persist
       record = @persistence_policy.call(record) if record && @persistence_policy
-      record = record.with(embedding: @embedder.embed(record.content)) if record && record.content != original_content
-      record
+      if record && record.content != original_content
+        record = record.with(embedding: @embedder.embed(record.content))
+      end
+      record ? Engram::EmbeddingMetadata.attach(record, embedder: @embedder) : nil
     end
   end
 end

data/lib/engram/ports/memory_store.rb

@@ -13,28 +13,35 @@ module Engram
       # Return up to `limit` Records in `scope` nearest to `embedding`,
       # ordered most-relevant first. When `kinds` is provided, only records with
       # those canonical memory kinds are eligible.
-      def search(embedding:, scope:, limit:, kinds: nil)
+      def search(embedding:, scope:, limit:, kinds: nil, embedding_metadata: nil)
         raise NotImplementedError, "#{self.class} must implement #search"
       end
 
       # All Records for a scope (mostly for inspection/tests).
-      def all(scope:)
+      # Supports optional `limit` and `offset` for batching large sweeps.
+      # Returned records are sorted in stable `id` order when batching is used.
+      # Use `after_id` for keyset pagination.
+      def all(scope:, limit: nil, offset: 0, after_id: nil)
         raise NotImplementedError, "#{self.class} must implement #all"
       end
 
       # Replace the content/embedding of an existing memory. Used by consolidation
-      # (UPDATE). Returns the updated Record.
-      def update(id:, record:)
+      # (UPDATE). Returns the updated Record. Raises Engram::Error when the scoped
+      # record does not exist or the replacement would move it to another scope.
+      def update(scope:, id:, record:)
         raise NotImplementedError, "#{self.class} must implement #update"
       end
 
-      # Remove a memory by id. Used by consolidation (FORGET).
-      def delete(id:)
+      # Remove a memory by id. Used by consolidation (FORGET). Returns the number
+      # of affected rows: 1 when deleted, 0 when the scoped record does not exist.
+      def delete(scope:, id:)
         raise NotImplementedError, "#{self.class} must implement #delete"
       end
 
       # Update the last-accessed timestamp of a memory. Used by recency-aware recall.
-      def touch(id:, at: Time.now)
+      # Returns the number of affected rows: 1 when touched, 0 when the scoped record
+      # does not exist.
+      def touch(scope:, id:, at: Time.now)
         raise NotImplementedError, "#{self.class} must implement #touch"
       end
     end

data/lib/engram/ports/processed_turns.rb

@@ -2,18 +2,26 @@
 
 module Engram
   module Ports
-    # Contract for remembering which turns have already been observed, so observation is
-    # idempotent across retries and accidental double-calls.
+    # Scoped lifecycle for observation idempotency. Implementations suppress concurrent claims
+    # while a lease is live. Lease expiry may permit overlap; this contract does not provide
+    # token ownership, fencing, or exactly-once execution.
+    # #claim returns a truthy opaque token when the caller may proceed, or nil when suppressed.
     # Implementations: Adapters::InMemoryProcessedTurns, Rails::CacheProcessedTurns.
     module ProcessedTurns
-      # Has this idempotency key already been processed?
-      def seen?(key)
-        raise NotImplementedError, "#{self.class} must implement #seen?"
+      def claim(scope:, key:)
+        raise NotImplementedError, "#{self.class} must implement #claim"
       end
 
-      # Mark this idempotency key as processed.
-      def record(key)
-        raise NotImplementedError, "#{self.class} must implement #record"
+      def complete(scope:, key:, claim:)
+        raise NotImplementedError, "#{self.class} must implement #complete"
+      end
+
+      def release(scope:, key:, claim:)
+        raise NotImplementedError, "#{self.class} must implement #release"
+      end
+
+      def completed?(scope:, key:)
+        raise NotImplementedError, "#{self.class} must implement #completed?"
       end
     end
   end

data/lib/engram/provenance.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+module Engram
+  # Provider-neutral source provenance stored in Record metadata.
+  #
+  # Offsets are zero-based Unicode codepoint indexes into one host-owned source
+  # message. The end offset is exclusive. Engram stores references and spans,
+  # never the source text itself.
+  class Provenance
+    RESERVED_KEY = "_engram"
+    METADATA_KEY = "provenance"
+    SCHEMA_VERSION = 1
+    ALIGNMENTS = %i[exact normalized inferred ungrounded].freeze
+
+    class Span
+      OFFSET_UNIT = "unicode_codepoint"
+
+      attr_reader :start_offset, :end_offset, :offset_unit
+
+      def initialize(start_offset:, end_offset:, offset_unit: OFFSET_UNIT)
+        unless start_offset.is_a?(Integer) && end_offset.is_a?(Integer) &&
+            start_offset >= 0 && end_offset > start_offset
+          raise ArgumentError, "span offsets must be non-negative integers with end_offset > start_offset"
+        end
+        raise ArgumentError, "unsupported offset unit" unless offset_unit.to_s == OFFSET_UNIT
+
+        @start_offset = start_offset
+        @end_offset = end_offset
+        @offset_unit = OFFSET_UNIT
+        freeze
+      end
+
+      def end_exclusive? = true
+
+      def ==(other)
+        other.is_a?(self.class) && to_h == other.to_h
+      end
+      alias_method :eql?, :==
+
+      def hash = to_h.hash
+
+      def to_h
+        {"start_offset" => start_offset, "end_offset" => end_offset, "offset_unit" => offset_unit}
+      end
+    end
+
+    class Source
+      attr_reader :source_id, :source_type, :message_index, :role, :spans, :alignment
+
+      def initialize(source_id:, source_type:, message_index:, role:, spans:, alignment:)
+        raise ArgumentError, "source_id is required" if source_id.to_s.strip.empty?
+        raise ArgumentError, "source_type is required" if source_type.to_s.strip.empty?
+        unless message_index.is_a?(Integer) && !message_index.negative?
+          raise ArgumentError, "message_index must be a non-negative integer"
+        end
+        raise ArgumentError, "role is required" if role.to_s.strip.empty?
+
+        unless alignment.is_a?(String) || alignment.is_a?(Symbol)
+          raise ArgumentError, "unknown alignment #{alignment.inspect}"
+        end
+        normalized_alignment = alignment.to_sym
+        raise ArgumentError, "unknown alignment #{alignment.inspect}" unless ALIGNMENTS.include?(normalized_alignment)
+
+        @source_id = source_id.to_s.dup.freeze
+        @source_type = source_type.to_s.dup.freeze
+        @message_index = message_index
+        @role = role.to_s.dup.freeze
+        raise ArgumentError, "spans must be an array" unless spans.is_a?(Array)
+        raise ArgumentError, "spans must contain at least one span" if spans.empty?
+        @spans = spans.map do |span|
+          raise ArgumentError, "spans must contain Provenance::Span values" unless span.is_a?(Span)
+          span
+        end.freeze
+        @alignment = normalized_alignment
+        freeze
+      end
+
+      def ==(other)
+        other.is_a?(self.class) && to_h == other.to_h
+      end
+      alias_method :eql?, :==
+
+      def hash = to_h.hash
+
+      def to_h
+        data = {
+          "source_id" => source_id,
+          "source_type" => source_type,
+          "spans" => spans.map(&:to_h),
+          "alignment" => alignment.to_s
+        }
+        data["message_index"] = message_index
+        data["role"] = role
+        data
+      end
+    end
+
+    class Extractor
+      attr_reader :name, :provider, :model
+
+      def initialize(name:, model:, provider: nil)
+        raise ArgumentError, "extractor name is required" if name.to_s.strip.empty?
+        raise ArgumentError, "extractor model is required" if model.to_s.strip.empty?
+
+        @name = name.to_s.dup.freeze
+        @provider = provider&.to_s&.dup&.freeze
+        @model = model.to_s.dup.freeze
+        freeze
+      end
+
+      def ==(other)
+        other.is_a?(self.class) && to_h == other.to_h
+      end
+      alias_method :eql?, :==
+
+      def hash = to_h.hash
+
+      def to_h
+        {"name" => name, "model" => model}.tap do |data|
+          data["provider"] = provider if provider
+        end
+      end
+    end
+
+    attr_reader :sources, :extractor, :confidence
+
+    def initialize(sources:, extractor:, confidence:)
+      raise ArgumentError, "sources must be an array" unless sources.is_a?(Array)
+      raise ArgumentError, "sources must contain at least one source" if sources.empty?
+      @sources = sources.map do |source|
+        raise ArgumentError, "sources must contain Provenance::Source values" unless source.is_a?(Source)
+        source
+      end.freeze
+      raise ArgumentError, "extractor must be a Provenance::Extractor" unless extractor.is_a?(Extractor)
+      valid_confidence = (confidence.is_a?(Integer) || confidence.is_a?(Float)) && confidence.between?(0, 1)
+      unless valid_confidence
+        raise ArgumentError, "confidence must be an Integer or Float between 0 and 1"
+      end
+
+      @extractor = extractor
+      @confidence = confidence
+      freeze
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && to_h == other.to_h
+    end
+    alias_method :eql?, :==
+
+    def hash = to_h.hash
+
+    def to_h
+      {
+        "version" => SCHEMA_VERSION,
+        "sources" => sources.map(&:to_h),
+        "extractor" => extractor.to_h,
+        "confidence" => confidence
+      }
+    end
+
+    class << self
+      def attach(metadata, provenance)
+        raise ArgumentError, "provenance must be a Provenance value" unless provenance.is_a?(self)
+
+        Engram::ReservedMetadata.attach(metadata, METADATA_KEY, provenance.to_h)
+      end
+
+      def extract(metadata)
+        metadata = deep_stringify(metadata || {})
+        return nil unless metadata.is_a?(Hash)
+
+        reserved = metadata[RESERVED_KEY]
+        return nil unless reserved.is_a?(Hash)
+
+        data = reserved[METADATA_KEY]
+        return nil unless data.is_a?(Hash) && data["version"] == SCHEMA_VERSION
+
+        from_h(data)
+      end
+
+      private
+
+      def from_h(data)
+        sources = provenance_array(data["sources"], "_engram.provenance.sources")
+        extractor_data = provenance_hash(data["extractor"], "_engram.provenance.extractor")
+
+        parsed_sources = sources.each_with_index.map do |source, source_index|
+          source_path = "_engram.provenance.sources[#{source_index}]"
+          source = provenance_hash(source, source_path)
+          spans = provenance_array(source["spans"], "#{source_path}.spans")
+          parsed_spans = spans.each_with_index.map do |span, span_index|
+            span_path = "#{source_path}.spans[#{span_index}]"
+            span = provenance_hash(span, span_path)
+            build_provenance_value(span_path) do
+              Span.new(
+                start_offset: span["start_offset"],
+                end_offset: span["end_offset"],
+                offset_unit: span["offset_unit"]
+              )
+            end
+          end
+
+          build_provenance_value(source_path) do
+            Source.new(
+              source_id: source["source_id"],
+              source_type: source["source_type"],
+              message_index: source["message_index"],
+              role: source["role"],
+              spans: parsed_spans,
+              alignment: source["alignment"]
+            )
+          end
+        end
+
+        extractor = build_provenance_value("_engram.provenance.extractor") do
+          Extractor.new(**symbolize_extractor(extractor_data))
+        end
+        build_provenance_value("_engram.provenance") do
+          new(sources: parsed_sources, extractor: extractor, confidence: data["confidence"])
+        end
+      end
+
+      def provenance_hash(value, path)
+        return value if value.is_a?(Hash)
+
+        raise Engram::Error, "malformed provenance at #{path}: expected an object"
+      end
+
+      def provenance_array(value, path)
+        return value if value.is_a?(Array)
+
+        raise Engram::Error, "malformed provenance at #{path}: expected an array"
+      end
+
+      def build_provenance_value(path)
+        yield
+      rescue ArgumentError, TypeError, KeyError, NoMethodError => error
+        raise Engram::Error, "malformed provenance at #{path}: #{error.message}"
+      end
+
+      def symbolize_extractor(data)
+        {name: data["name"], provider: data["provider"], model: data["model"]}
+      end
+
+      def deep_stringify(value)
+        case value
+        when Hash
+          value.each_with_object({}) { |(key, nested), out| out[key.to_s] = deep_stringify(nested) }
+        when Array
+          value.map { |nested| deep_stringify(nested) }
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/engram/rails/cache_processed_turns.rb

@@ -1,30 +1,71 @@
 # frozen_string_literal: true
 
+require "digest"
+require "securerandom"
+
 module Engram
   module Rails
-    # ProcessedTurns backed by Rails.cache. Idempotency survives across processes and job
-    # retries when a shared cache (e.g. Solid Cache) is configured.
+    # Claims use atomic unless-exist writes. Generic ActiveSupport caches do not expose
+    # compare-and-delete, so release leaves a claim in place until lease expiry and completion
+    # never deletes it.
     class CacheProcessedTurns
       include Engram::Ports::ProcessedTurns
 
-      def initialize(namespace: "engram:processed_turns", ttl: 86_400)
+      class NonAtomicCacheError < Engram::Error; end
+      class CacheWriteError < Engram::Error; end
+
+      def initialize(namespace: "engram:processed_turns", ttl: 86_400, lease_ttl: 300, cache: ::Rails.cache)
         @namespace = namespace
         @ttl = ttl
+        @lease_ttl = lease_ttl
+        @cache = cache
+      end
+
+      def claim(scope:, key:)
+        return if completed?(scope: scope, key: key)
+        token = SecureRandom.uuid
+        result = @cache.write(claim_key(scope, key), token, expires_in: @lease_ttl, unless_exist: true)
+        unless result == true || result == false
+          raise NonAtomicCacheError, "cache backend must return true/false for atomic write(unless_exist: true)"
+        end
+        return unless result
+        return token unless completed?(scope: scope, key: key)
+
+        release(scope: scope, key: key, claim: token)
+        nil
       end
 
-      def seen?(key)
-        ::Rails.cache.exist?(cache_key(key))
+      def complete(scope:, key:, claim:)
+        written = @cache.write(completed_key(scope, key), true, expires_in: @ttl)
+        raise CacheWriteError, "cache backend failed to write completed observation" unless written
+        true
       end
 
-      def record(key)
-        ::Rails.cache.write(cache_key(key), true, expires_in: @ttl)
-        key
+      def release(scope:, key:, claim:)
+        # A read followed by delete could delete a successor lease.
+        nil
+      end
+
+      def completed?(scope:, key:)
+        @cache.exist?(completed_key(scope, key))
       end
 
       private
 
-      def cache_key(key)
-        "#{@namespace}:#{key}"
+      def digest(scope, key)
+        encoded = [scope, key].each_with_object(+"") do |component, buffer|
+          bytes = component.to_s.encode(Encoding::UTF_8)
+          buffer << [bytes.bytesize].pack("N") << bytes
+        end
+        Digest::SHA256.hexdigest(encoded)
+      end
+
+      def claim_key(scope, key)
+        "#{@namespace}:claim:#{digest(scope, key)}"
+      end
+
+      def completed_key(scope, key)
+        "#{@namespace}:completed:#{digest(scope, key)}"
       end
     end
   end

data/lib/engram/rails/observe_job.rb

@@ -4,6 +4,11 @@ module Engram
   # Background observation: runs extract → consolidate off the request path.
   # Defined only when ActiveJob is available (loaded via the Railtie).
   class ObserveJob < ActiveJob::Base
+    # Later retries must outlast the default claim lease.
+    retry_on Engram::ObservationInProgressError,
+      wait: ->(executions) { (executions**4) + 2 },
+      attempts: 8
+
     def perform(scope, messages)
       Engram::Memory.new(scope: scope).observe(messages)
     end

data/lib/engram/rails/tasks.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Loaded by the Railtie in Rails apps (where :environment boots the app and its
+# initializers) and by the gem's own Rakefile (which defines a no-op :environment).
+namespace :engram do
+  desc "Rebuild embeddings in a memory scope. Usage: bundle exec rake 'engram:rebuild_embeddings[user:1]'"
+  task :rebuild_embeddings, [:scope] => :environment do |_, args|
+    scope = args[:scope]
+    raise ArgumentError, "rebuild_embeddings requires a scope argument" if scope.to_s.empty?
+
+    stale_only = !%w[false 0 no].include?(ENV.fetch("STALE_ONLY", "true").downcase)
+    batch_size = Integer(ENV.fetch("BATCH_SIZE", "100"))
+    raise ArgumentError, "BATCH_SIZE must be greater than 0" unless batch_size.positive?
+
+    result = Engram::UseCases::RebuildEmbeddings.new(
+      store: Engram.config.store,
+      embedder: Engram.config.embedder
+    ).call(scope: scope, stale_only: stale_only, batch_size: batch_size)
+
+    puts "scope=#{result[:scope]} processed=#{result[:processed]} updated=#{result[:updated]} skipped=#{result[:skipped]} failed=#{result[:failed]}"
+
+    if result[:failed] > 0
+      abort "failed_ids=#{result[:failed_ids].join(",")}"
+    end
+  end
+end