engram 0.4.0 → 0.6.0

data/lib/engram/use_cases/rebuild_embeddings.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Engram
+  module UseCases
+    # Rebuild stored embeddings when the active embedder configuration changes.
+    # Returns counts useful for logging/observability:
+    #   - :processed: total rows examined in scope
+    #   - :updated: rows re-embedded and persisted
+    #   - :skipped: rows skipped because they were unchanged or incomplete
+    #   - :failed: count of rows that failed during rebuild
+    #   - :failed_ids: record IDs that failed during rebuild
+    #   - :failed_errors: error class/message keyed by failed record ID
+    class RebuildEmbeddings
+      def initialize(store:, embedder:)
+        @store = store
+        @embedder = embedder
+      end
+
+      def call(scope:, stale_only: true, batch_size: 100)
+        batch_size = Integer(batch_size)
+        raise ArgumentError, "batch_size must be greater than 0" unless batch_size.positive?
+
+        counts = {
+          processed: 0,
+          updated: 0,
+          skipped: 0,
+          failed: 0,
+          failed_ids: [],
+          failed_errors: {}
+        }
+
+        after_id = nil
+        legacy_index = 0
+        legacy_records = nil
+        processed_ids = Set.new
+        processed_nil_records = Set.new
+        loop do
+          batch_result = fetch_batch(scope:, batch_size:, after_id:, legacy_records:, legacy_index:)
+          legacy_records = batch_result[:legacy_records] if batch_result[:legacy_records]
+          batch = batch_result[:records]
+          if batch.empty?
+            break if legacy_records
+
+            # Reconcile with one stable snapshot: accepting keyset keywords does
+            # not guarantee that an adapter returns complete, ordered pages.
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          cursor_record = batch.reverse.find { |candidate| !candidate.id.nil? } unless legacy_records
+
+          if !legacy_records && fallback_before_processing?(batch:, after_id:)
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            batch = legacy_records.slice(legacy_index, batch_size) || []
+            break if batch.empty?
+          end
+
+          batch.each do |record|
+            identity = record.id.nil? ? record.object_id : record.id
+            seen = record.id.nil? ? processed_nil_records : processed_ids
+            next unless seen.add?(identity)
+
+            counts[:processed] += 1
+
+            if record.id.nil?
+              counts[:skipped] += 1
+              next
+            end
+
+            if stale_only && !stale?(record)
+              counts[:skipped] += 1
+              next
+            end
+
+            begin
+              rebuilt = record.with(embedding: @embedder.embed(record.content))
+              rebuilt = EmbeddingMetadata.attach(rebuilt, embedder: @embedder)
+              updated = @store.update(scope: scope, id: record.id, record: rebuilt)
+              raise Engram::Error, "memory update was not applied" unless updated && updated != 0
+
+              counts[:updated] += 1
+            rescue => error
+              counts[:failed] += 1
+              record_id = record.id
+              counts[:failed_ids] << record_id
+              counts[:failed_errors][record_id] = {
+                class: error.class.name,
+                message: error.message
+              }
+              warn "Rebuild failed for record ##{record_id}: #{error.class}: #{error.message}"
+            end
+          end
+
+          if legacy_records
+            legacy_index += batch.length
+            next
+          end
+
+          if cursor_record.nil?
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          if fallback_to_legacy_batching?(batch:, cursor_record:)
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          after_id = cursor_record.id
+        end
+
+        {scope: scope, **counts}
+      end
+
+      private
+
+      def fetch_batch(scope:, batch_size:, after_id:, legacy_records:, legacy_index:)
+        if legacy_records
+          return {records: legacy_records.slice(legacy_index, batch_size) || [], legacy_records:}
+        end
+
+        records = modern_batch(scope:, batch_size:, after_id:)
+        return {records:} if records
+
+        legacy_records = Array(@store.all(scope: scope))
+        {records: legacy_records.slice(legacy_index, batch_size) || [], legacy_records:}
+      end
+
+      def modern_batch(scope:, batch_size:, after_id:)
+        return unless batched_all_supported?
+
+        @store.all(scope: scope, limit: batch_size, after_id: after_id)
+      end
+
+      def batched_all_supported?
+        parameters = @store.method(:all).parameters
+        keyword_names = parameters.filter_map do |kind, name|
+          name if [:keyreq, :key].include?(kind)
+        end
+
+        keyword_names.include?(:limit) && keyword_names.include?(:after_id)
+      end
+
+      def fallback_to_legacy_batching?(batch:, cursor_record:)
+        cursor_record != batch.last
+      end
+
+      def fallback_before_processing?(batch:, after_id:)
+        ids = batch.filter_map(&:id)
+        return true if ids.uniq.length != ids.length
+        return false if after_id.nil?
+        return true if batch.any? { |record| record.id.nil? }
+
+        batch.any? { |record| record.id && record.id <= after_id }
+      end
+
+      def stale?(record)
+        stored = EmbeddingMetadata.extract(record.metadata)
+        return true if stored.empty?
+
+        expected = expected_metadata(record)
+        return true unless expected
+
+        required = %w[adapter provider model dimensions fingerprint]
+        return true if required.any? { |key| stored[key] != expected[key] }
+
+        return true if stored["dimensions"] &&
+          record.embedding.respond_to?(:length) &&
+          record.embedding.length != stored["dimensions"].to_i
+
+        false
+      end
+
+      # What the active embedder would produce for a rebuilt row. Declared dimensions take
+      # precedence so dimension-only configuration drift marks rows stale; fall back to the
+      # record's current vector length for embedders that do not declare dimensions.
+      def expected_metadata(record)
+        declared = EmbeddingMetadata.for_embedder(@embedder)
+        return declared if declared.nil? || declared["dimensions"]
+
+        EmbeddingMetadata.for_embedder(@embedder, embedding: record.embedding)
+      end
+    end
+  end
+end

data/lib/engram/use_cases/recall.rb

@@ -36,11 +36,19 @@ module Engram
         )
         Engram::Instrumentation.instrument("recall", payload) do
           embedding = @embedder.embed(query)
+          embedding_metadata = Engram::EmbeddingMetadata.for_embedder(@embedder, embedding: embedding)
           pool_limit = reranking? ? limit * @pool_factor : limit
-          pool = @store.search(embedding: embedding, scope: scope, limit: pool_limit, kinds: kinds)
+          pool = Engram::EmbeddingMetadata.search(
+            @store,
+            embedding: embedding,
+            embedding_metadata: embedding_metadata,
+            scope: scope,
+            limit: pool_limit,
+            kinds: kinds
+          )
 
           results = (reranking? ? rerank(pool, embedding) : pool).first(limit)
-          touch(results) if @touch
+          touch(results, scope) if @touch
           payload[:result_count] = results.size
           payload[:candidate_count] = pool.size
           results
@@ -72,8 +80,8 @@ module Engram
         0.5**(age / @recency_halflife)
       end
 
-      def touch(records)
-        records.each { |record| @store.touch(id: record.id, at: Time.now) if record.id }
+      def touch(records, scope)
+        records.each { |record| @store.touch(scope: scope, id: record.id, at: Time.now) if record.id }
       end
     end
   end

data/lib/engram/use_cases/source_impact.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Engram
+  module UseCases
+    # Look up which memories in a scope reference a given host source.
+    #
+    # Returns the records whose understood provenance lists a source matching both
+    # `source_id` and `source_type` exactly, with no trimming or normalization.
+    # Reads stay tolerant: legacy, malformed, and future-schema
+    # provenance simply do not match. Source IDs are references, not authorization
+    # boundaries, so callers still bound the lookup to a `scope`.
+    class SourceImpact
+      def initialize(store:)
+        @store = store
+      end
+
+      # Returns the Array<Record> in `scope` referencing the exact source. Result
+      # order follows the store's #all enumeration and is not otherwise guaranteed.
+      def call(scope:, source_id:, source_type:)
+        source_id = require_string!("source_id", source_id)
+        source_type = require_string!("source_type", source_type)
+
+        @store.all(scope: scope).select do |record|
+          provenance = record.provenance
+          provenance&.sources&.any? do |source|
+            source.source_id == source_id && source.source_type == source_type
+          end
+        end
+      end
+
+      private
+
+      def require_string!(name, value)
+        unless value.is_a?(String) && !value.strip.empty?
+          raise ArgumentError, "#{name} must be a non-empty String"
+        end
+
+        value
+      end
+    end
+  end
+end

data/lib/engram/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Engram
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/engram.rb

@@ -3,8 +3,15 @@
 require_relative "engram/version"
 require_relative "engram/configuration"
 require_relative "engram/math"
+require_relative "engram/internal/core_hash"
+require_relative "engram/reserved_metadata"
+require_relative "engram/embedding_metadata"
+require_relative "engram/provenance"
 require_relative "engram/memory_kind"
 require_relative "engram/record"
+require_relative "engram/internal/scope"
+require_relative "engram/internal/candidate_integrity"
+require_relative "engram/extraction"
 require_relative "engram/decision"
 require_relative "engram/turn_digest"
 require_relative "engram/persistence_policy"
@@ -24,6 +31,9 @@ require_relative "engram/use_cases/recall"
 require_relative "engram/use_cases/inject"
 require_relative "engram/use_cases/observe"
 require_relative "engram/use_cases/forget"
+require_relative "engram/use_cases/rebuild_embeddings"
+require_relative "engram/use_cases/source_impact"
+require_relative "engram/use_cases/grounding_report"
 
 # Built-in adapters (pure Ruby, no external deps)
 require_relative "engram/adapters/in_memory_store"
@@ -51,6 +61,9 @@ require_relative "engram/integrations/ruby_llm"
 module Engram
   class Error < StandardError; end
 
+  # Raised when a live claim suppresses an observation that has not completed.
+  class ObservationInProgressError < Error; end
+
   class << self
     def config
       @config ||= Configuration.new

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: engram
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Alexandr Kholodniak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-06 00:00:00.000000000 Z
+date: 2026-07-28 00:00:00.000000000 Z
 dependencies: []
 description: |
   Engram gives AI agents durable, long-term memory. It recalls relevant facts about a
@@ -36,9 +36,14 @@ files:
 - lib/engram/consolidators/heuristic_consolidator.rb
 - lib/engram/consolidators/llm_consolidator.rb
 - lib/engram/decision.rb
+- lib/engram/embedding_metadata.rb
+- lib/engram/extraction.rb
 - lib/engram/extractors/llm_extractor.rb
 - lib/engram/instrumentation.rb
 - lib/engram/integrations/ruby_llm.rb
+- lib/engram/internal/candidate_integrity.rb
+- lib/engram/internal/core_hash.rb
+- lib/engram/internal/scope.rb
 - lib/engram/math.rb
 - lib/engram/memory.rb
 - lib/engram/memory_kind.rb
@@ -50,16 +55,22 @@ files:
 - lib/engram/ports/extractor.rb
 - lib/engram/ports/memory_store.rb
 - lib/engram/ports/processed_turns.rb
+- lib/engram/provenance.rb
 - lib/engram/rails/cache_processed_turns.rb
 - lib/engram/rails/has_memory.rb
 - lib/engram/rails/observe_job.rb
+- lib/engram/rails/tasks.rake
 - lib/engram/railtie.rb
 - lib/engram/record.rb
+- lib/engram/reserved_metadata.rb
 - lib/engram/turn_digest.rb
 - lib/engram/use_cases/forget.rb
+- lib/engram/use_cases/grounding_report.rb
 - lib/engram/use_cases/inject.rb
 - lib/engram/use_cases/observe.rb
+- lib/engram/use_cases/rebuild_embeddings.rb
 - lib/engram/use_cases/recall.rb
+- lib/engram/use_cases/source_impact.rb
 - lib/engram/version.rb
 - lib/generators/engram/install_generator.rb
 - lib/generators/engram/templates/create_engram_memories.rb.tt
@@ -89,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Long-term memory for AI agents in Ruby — stored in your own Postgres.