engram 0.4.0 → 0.5.0

data/lib/engram/railtie.rb

@@ -19,5 +19,9 @@ module Engram
         require "engram/rails/observe_job"
       end
     end
+
+    rake_tasks do
+      load File.expand_path("rails/tasks.rake", __dir__)
+    end
   end
 end

data/lib/engram/reserved_metadata.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Engram
+  # Collision-preserving operations for schemas sharing Engram's metadata namespace.
+  module ReservedMetadata
+    KEY = "_engram"
+
+    module_function
+
+    def attach(metadata, schema_key, value, namespace_description: "metadata")
+      metadata = (metadata || {}).dup
+      reserved_values = [metadata.delete(KEY), metadata.delete(:_engram)].compact
+      unless reserved_values.all? { |reserved| reserved.is_a?(Hash) }
+        raise Engram::Error, "metadata key #{KEY.inspect} is reserved for Engram #{namespace_description}"
+      end
+
+      reserved = reserved_values.reduce({}) do |merged, reserved_value|
+        siblings = reserved_value.reject { |key, _nested| key.to_s == schema_key }
+        merge(merged, normalize(siblings))
+      end
+      metadata.merge(KEY => reserved.merge(schema_key => normalize(value)))
+    end
+
+    def normalize(value, path = [])
+      return value.map { |nested| normalize(nested, path) } if value.is_a?(Array)
+      return value unless value.is_a?(Hash)
+
+      value.each_with_object({}) do |(key, nested), normalized|
+        string_key = key.to_s
+        normalized_value = normalize(nested, path + [string_key])
+        normalized[string_key] = if normalized.key?(string_key)
+          merge_value(normalized[string_key], normalized_value, path + [string_key])
+        else
+          normalized_value
+        end
+      end
+    end
+
+    def merge(left, right, path = [])
+      right.each_with_object(left.dup) do |(key, value), merged|
+        merged[key] = merged.key?(key) ? merge_value(merged[key], value, path + [key]) : value
+      end
+    end
+
+    def merge_value(left, right, path)
+      return merge(left, right, path) if left.is_a?(Hash) && right.is_a?(Hash)
+      return left if left == right
+
+      raise Engram::Error, "conflicting reserved metadata at #{path.join(".")}"
+    end
+  end
+end

data/lib/engram/use_cases/forget.rb

@@ -20,8 +20,12 @@ module Engram
           timestamp && timestamp < cutoff && record.importance.to_f < min_importance
         end
 
-        stale.each { |record| @store.delete(id: record.id) if record.id }
-        stale
+        stale.select do |record|
+          next false unless record.id
+
+          deleted = @store.delete(scope: scope, id: record.id)
+          deleted && deleted != 0
+        end
       end
     end
   end

data/lib/engram/use_cases/observe.rb

@@ -26,38 +26,54 @@ module Engram
           idempotency_key_present: !idempotency_key.nil?
         )
         Engram::Instrumentation.instrument("observe", payload) do
-          if already_processed?(idempotency_key)
+          claim = acquire_claim(scope, idempotency_key)
+          if idempotency_key && @processed_turns && !claim
+            unless @processed_turns.completed?(scope: scope, key: idempotency_key)
+              # A live, incomplete lease is retryable, not a completed duplicate.
+              raise Engram::ObservationInProgressError,
+                "observation for this turn is claimed but not completed; retry after the claim lease expires"
+            end
+
             payload[:skipped] = true
             payload[:candidate_count] = 0
             payload[:decision_count] = 0
             next []
           end
 
-          candidates = extract(messages: messages, scope: scope)
-          payload[:candidate_count] = candidates.size
-          if candidates.empty?
-            mark_processed(idempotency_key)
-            payload[:decision_count] = 0
-            next []
-          end
+          begin
+            candidates = extract(messages: messages, scope: scope)
+            payload[:candidate_count] = candidates.size
+            if candidates.empty?
+              complete_claim(scope, idempotency_key, claim)
+              payload[:decision_count] = 0
+              next []
+            end
 
-          decisions = consolidate(candidates: candidates, scope: scope)
-          applied_decisions = decisions.filter_map { |decision| apply(decision) }
-          payload[:decision_count] = applied_decisions.size
-          payload[:decision_actions] = applied_decisions.map { |decision| decision.action.to_s }
-          mark_processed(idempotency_key)
-          applied_decisions
+            decisions = consolidate(candidates: candidates, scope: scope)
+            applied_decisions = decisions.filter_map { |decision| apply(decision, scope) }
+            payload[:decision_count] = applied_decisions.size
+            payload[:decision_actions] = applied_decisions.map { |decision| decision.action.to_s }
+            complete_claim(scope, idempotency_key, claim)
+            applied_decisions
+          rescue
+            release_claim(scope, idempotency_key, claim)
+            raise
+          end
         end
       end
 
       private
 
-      def already_processed?(key)
-        !!(key && @processed_turns&.seen?(key))
+      def acquire_claim(scope, key)
+        @processed_turns.claim(scope: scope, key: key) if key && @processed_turns
+      end
+
+      def complete_claim(scope, key, claim)
+        @processed_turns.complete(scope: scope, key: key, claim: claim) if claim
       end
 
-      def mark_processed(key)
-        @processed_turns.record(key) if key && @processed_turns
+      def release_claim(scope, key, claim)
+        @processed_turns.release(scope: scope, key: key, claim: claim) if claim
       end
 
       def extract(messages:, scope:)
@@ -79,16 +95,21 @@ module Engram
         end
       end
 
-      def apply(decision)
+      def apply(decision, scope)
         case decision.action
         when :add
-          decision if persistence.add(decision.candidate)
+          raise Engram::Error, "cannot move memory across scopes" unless decision.candidate.scope == scope
+
+          decision if persistence.add(decision.candidate, scope: scope)
         when :update
-          if decision.target_id && persistence.update(id: decision.target_id, record: decision.candidate)
+          if decision.target_id && persistence.update(scope: scope, id: decision.target_id, record: decision.candidate)
             decision
           end
         when :forget
-          decision if decision.target_id && @store.delete(id: decision.target_id)
+          if decision.target_id
+            deleted = @store.delete(scope: scope, id: decision.target_id)
+            decision if deleted && deleted != 0
+          end
         when :noop
           nil
         end

data/lib/engram/use_cases/rebuild_embeddings.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Engram
+  module UseCases
+    # Rebuild stored embeddings when the active embedder configuration changes.
+    # Returns counts useful for logging/observability:
+    #   - :processed: total rows examined in scope
+    #   - :updated: rows re-embedded and persisted
+    #   - :skipped: rows skipped because they were unchanged or incomplete
+    #   - :failed: count of rows that failed during rebuild
+    #   - :failed_ids: record IDs that failed during rebuild
+    #   - :failed_errors: error class/message keyed by failed record ID
+    class RebuildEmbeddings
+      def initialize(store:, embedder:)
+        @store = store
+        @embedder = embedder
+      end
+
+      def call(scope:, stale_only: true, batch_size: 100)
+        batch_size = Integer(batch_size)
+        raise ArgumentError, "batch_size must be greater than 0" unless batch_size.positive?
+
+        counts = {
+          processed: 0,
+          updated: 0,
+          skipped: 0,
+          failed: 0,
+          failed_ids: [],
+          failed_errors: {}
+        }
+
+        after_id = nil
+        legacy_index = 0
+        legacy_records = nil
+        processed_ids = Set.new
+        processed_nil_records = Set.new
+        loop do
+          batch_result = fetch_batch(scope:, batch_size:, after_id:, legacy_records:, legacy_index:)
+          legacy_records = batch_result[:legacy_records] if batch_result[:legacy_records]
+          batch = batch_result[:records]
+          if batch.empty?
+            break if legacy_records
+
+            # Reconcile with one stable snapshot: accepting keyset keywords does
+            # not guarantee that an adapter returns complete, ordered pages.
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          cursor_record = batch.reverse.find { |candidate| !candidate.id.nil? } unless legacy_records
+
+          if !legacy_records && fallback_before_processing?(batch:, after_id:)
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            batch = legacy_records.slice(legacy_index, batch_size) || []
+            break if batch.empty?
+          end
+
+          batch.each do |record|
+            identity = record.id.nil? ? record.object_id : record.id
+            seen = record.id.nil? ? processed_nil_records : processed_ids
+            next unless seen.add?(identity)
+
+            counts[:processed] += 1
+
+            if record.id.nil?
+              counts[:skipped] += 1
+              next
+            end
+
+            if stale_only && !stale?(record)
+              counts[:skipped] += 1
+              next
+            end
+
+            begin
+              rebuilt = record.with(embedding: @embedder.embed(record.content))
+              rebuilt = EmbeddingMetadata.attach(rebuilt, embedder: @embedder)
+              updated = @store.update(scope: scope, id: record.id, record: rebuilt)
+              raise Engram::Error, "memory update was not applied" unless updated && updated != 0
+
+              counts[:updated] += 1
+            rescue => error
+              counts[:failed] += 1
+              record_id = record.id
+              counts[:failed_ids] << record_id
+              counts[:failed_errors][record_id] = {
+                class: error.class.name,
+                message: error.message
+              }
+              warn "Rebuild failed for record ##{record_id}: #{error.class}: #{error.message}"
+            end
+          end
+
+          if legacy_records
+            legacy_index += batch.length
+            next
+          end
+
+          if cursor_record.nil?
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          if fallback_to_legacy_batching?(batch:, cursor_record:)
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          after_id = cursor_record.id
+        end
+
+        {scope: scope, **counts}
+      end
+
+      private
+
+      def fetch_batch(scope:, batch_size:, after_id:, legacy_records:, legacy_index:)
+        if legacy_records
+          return {records: legacy_records.slice(legacy_index, batch_size) || [], legacy_records:}
+        end
+
+        records = modern_batch(scope:, batch_size:, after_id:)
+        return {records:} if records
+
+        legacy_records = Array(@store.all(scope: scope))
+        {records: legacy_records.slice(legacy_index, batch_size) || [], legacy_records:}
+      end
+
+      def modern_batch(scope:, batch_size:, after_id:)
+        return unless batched_all_supported?
+
+        @store.all(scope: scope, limit: batch_size, after_id: after_id)
+      end
+
+      def batched_all_supported?
+        parameters = @store.method(:all).parameters
+        keyword_names = parameters.filter_map do |kind, name|
+          name if [:keyreq, :key].include?(kind)
+        end
+
+        keyword_names.include?(:limit) && keyword_names.include?(:after_id)
+      end
+
+      def fallback_to_legacy_batching?(batch:, cursor_record:)
+        cursor_record != batch.last
+      end
+
+      def fallback_before_processing?(batch:, after_id:)
+        ids = batch.filter_map(&:id)
+        return true if ids.uniq.length != ids.length
+        return false if after_id.nil?
+        return true if batch.any? { |record| record.id.nil? }
+
+        batch.any? { |record| record.id && record.id <= after_id }
+      end
+
+      def stale?(record)
+        stored = EmbeddingMetadata.extract(record.metadata)
+        return true if stored.empty?
+
+        expected = expected_metadata(record)
+        return true unless expected
+
+        required = %w[adapter provider model dimensions fingerprint]
+        return true if required.any? { |key| stored[key] != expected[key] }
+
+        return true if stored["dimensions"] &&
+          record.embedding.respond_to?(:length) &&
+          record.embedding.length != stored["dimensions"].to_i
+
+        false
+      end
+
+      # What the active embedder would produce for a rebuilt row. Declared dimensions take
+      # precedence so dimension-only configuration drift marks rows stale; fall back to the
+      # record's current vector length for embedders that do not declare dimensions.
+      def expected_metadata(record)
+        declared = EmbeddingMetadata.for_embedder(@embedder)
+        return declared if declared.nil? || declared["dimensions"]
+
+        EmbeddingMetadata.for_embedder(@embedder, embedding: record.embedding)
+      end
+    end
+  end
+end

data/lib/engram/use_cases/recall.rb

@@ -36,11 +36,19 @@ module Engram
         )
         Engram::Instrumentation.instrument("recall", payload) do
           embedding = @embedder.embed(query)
+          embedding_metadata = Engram::EmbeddingMetadata.for_embedder(@embedder, embedding: embedding)
           pool_limit = reranking? ? limit * @pool_factor : limit
-          pool = @store.search(embedding: embedding, scope: scope, limit: pool_limit, kinds: kinds)
+          pool = Engram::EmbeddingMetadata.search(
+            @store,
+            embedding: embedding,
+            embedding_metadata: embedding_metadata,
+            scope: scope,
+            limit: pool_limit,
+            kinds: kinds
+          )
 
           results = (reranking? ? rerank(pool, embedding) : pool).first(limit)
-          touch(results) if @touch
+          touch(results, scope) if @touch
           payload[:result_count] = results.size
           payload[:candidate_count] = pool.size
           results
@@ -72,8 +80,8 @@ module Engram
         0.5**(age / @recency_halflife)
       end
 
-      def touch(records)
-        records.each { |record| @store.touch(id: record.id, at: Time.now) if record.id }
+      def touch(records, scope)
+        records.each { |record| @store.touch(scope: scope, id: record.id, at: Time.now) if record.id }
       end
     end
   end

data/lib/engram/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Engram
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/engram.rb

@@ -3,6 +3,9 @@
 require_relative "engram/version"
 require_relative "engram/configuration"
 require_relative "engram/math"
+require_relative "engram/reserved_metadata"
+require_relative "engram/embedding_metadata"
+require_relative "engram/provenance"
 require_relative "engram/memory_kind"
 require_relative "engram/record"
 require_relative "engram/decision"
@@ -24,6 +27,7 @@ require_relative "engram/use_cases/recall"
 require_relative "engram/use_cases/inject"
 require_relative "engram/use_cases/observe"
 require_relative "engram/use_cases/forget"
+require_relative "engram/use_cases/rebuild_embeddings"
 
 # Built-in adapters (pure Ruby, no external deps)
 require_relative "engram/adapters/in_memory_store"
@@ -51,6 +55,9 @@ require_relative "engram/integrations/ruby_llm"
 module Engram
   class Error < StandardError; end
 
+  # Raised when a live claim suppresses an observation that has not completed.
+  class ObservationInProgressError < Error; end
+
   class << self
     def config
       @config ||= Configuration.new

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: engram
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Alexandr Kholodniak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-06 00:00:00.000000000 Z
+date: 2026-07-16 00:00:00.000000000 Z
 dependencies: []
 description: |
   Engram gives AI agents durable, long-term memory. It recalls relevant facts about a
@@ -36,6 +36,7 @@ files:
 - lib/engram/consolidators/heuristic_consolidator.rb
 - lib/engram/consolidators/llm_consolidator.rb
 - lib/engram/decision.rb
+- lib/engram/embedding_metadata.rb
 - lib/engram/extractors/llm_extractor.rb
 - lib/engram/instrumentation.rb
 - lib/engram/integrations/ruby_llm.rb
@@ -50,15 +51,19 @@ files:
 - lib/engram/ports/extractor.rb
 - lib/engram/ports/memory_store.rb
 - lib/engram/ports/processed_turns.rb
+- lib/engram/provenance.rb
 - lib/engram/rails/cache_processed_turns.rb
 - lib/engram/rails/has_memory.rb
 - lib/engram/rails/observe_job.rb
+- lib/engram/rails/tasks.rake
 - lib/engram/railtie.rb
 - lib/engram/record.rb
+- lib/engram/reserved_metadata.rb
 - lib/engram/turn_digest.rb
 - lib/engram/use_cases/forget.rb
 - lib/engram/use_cases/inject.rb
 - lib/engram/use_cases/observe.rb
+- lib/engram/use_cases/rebuild_embeddings.rb
 - lib/engram/use_cases/recall.rb
 - lib/engram/version.rb
 - lib/generators/engram/install_generator.rb
@@ -89,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Long-term memory for AI agents in Ruby — stored in your own Postgres.