engram 0.3.0 → 0.5.0

data/lib/engram/rails/cache_processed_turns.rb

@@ -1,30 +1,71 @@
 # frozen_string_literal: true
 
+require "digest"
+require "securerandom"
+
 module Engram
   module Rails
-    # ProcessedTurns backed by Rails.cache. Idempotency survives across processes and job
-    # retries when a shared cache (e.g. Solid Cache) is configured.
+    # Claims use atomic unless-exist writes. Generic ActiveSupport caches do not expose
+    # compare-and-delete, so release leaves a claim in place until lease expiry and completion
+    # never deletes it.
     class CacheProcessedTurns
       include Engram::Ports::ProcessedTurns
 
-      def initialize(namespace: "engram:processed_turns", ttl: 86_400)
+      class NonAtomicCacheError < Engram::Error; end
+      class CacheWriteError < Engram::Error; end
+
+      def initialize(namespace: "engram:processed_turns", ttl: 86_400, lease_ttl: 300, cache: ::Rails.cache)
         @namespace = namespace
         @ttl = ttl
+        @lease_ttl = lease_ttl
+        @cache = cache
+      end
+
+      def claim(scope:, key:)
+        return if completed?(scope: scope, key: key)
+        token = SecureRandom.uuid
+        result = @cache.write(claim_key(scope, key), token, expires_in: @lease_ttl, unless_exist: true)
+        unless result == true || result == false
+          raise NonAtomicCacheError, "cache backend must return true/false for atomic write(unless_exist: true)"
+        end
+        return unless result
+        return token unless completed?(scope: scope, key: key)
+
+        release(scope: scope, key: key, claim: token)
+        nil
       end
 
-      def seen?(key)
-        ::Rails.cache.exist?(cache_key(key))
+      def complete(scope:, key:, claim:)
+        written = @cache.write(completed_key(scope, key), true, expires_in: @ttl)
+        raise CacheWriteError, "cache backend failed to write completed observation" unless written
+        true
       end
 
-      def record(key)
-        ::Rails.cache.write(cache_key(key), true, expires_in: @ttl)
-        key
+      def release(scope:, key:, claim:)
+        # A read followed by delete could delete a successor lease.
+        nil
+      end
+
+      def completed?(scope:, key:)
+        @cache.exist?(completed_key(scope, key))
       end
 
       private
 
-      def cache_key(key)
-        "#{@namespace}:#{key}"
+      def digest(scope, key)
+        encoded = [scope, key].each_with_object(+"") do |component, buffer|
+          bytes = component.to_s.encode(Encoding::UTF_8)
+          buffer << [bytes.bytesize].pack("N") << bytes
+        end
+        Digest::SHA256.hexdigest(encoded)
+      end
+
+      def claim_key(scope, key)
+        "#{@namespace}:claim:#{digest(scope, key)}"
+      end
+
+      def completed_key(scope, key)
+        "#{@namespace}:completed:#{digest(scope, key)}"
       end
     end
   end

data/lib/engram/rails/observe_job.rb

@@ -4,6 +4,11 @@ module Engram
   # Background observation: runs extract → consolidate off the request path.
   # Defined only when ActiveJob is available (loaded via the Railtie).
   class ObserveJob < ActiveJob::Base
+    # Later retries must outlast the default claim lease.
+    retry_on Engram::ObservationInProgressError,
+      wait: ->(executions) { (executions**4) + 2 },
+      attempts: 8
+
     def perform(scope, messages)
       Engram::Memory.new(scope: scope).observe(messages)
     end

data/lib/engram/rails/tasks.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Loaded by the Railtie in Rails apps (where :environment boots the app and its
+# initializers) and by the gem's own Rakefile (which defines a no-op :environment).
+namespace :engram do
+  desc "Rebuild embeddings in a memory scope. Usage: bundle exec rake 'engram:rebuild_embeddings[user:1]'"
+  task :rebuild_embeddings, [:scope] => :environment do |_, args|
+    scope = args[:scope]
+    raise ArgumentError, "rebuild_embeddings requires a scope argument" if scope.to_s.empty?
+
+    stale_only = !%w[false 0 no].include?(ENV.fetch("STALE_ONLY", "true").downcase)
+    batch_size = Integer(ENV.fetch("BATCH_SIZE", "100"))
+    raise ArgumentError, "BATCH_SIZE must be greater than 0" unless batch_size.positive?
+
+    result = Engram::UseCases::RebuildEmbeddings.new(
+      store: Engram.config.store,
+      embedder: Engram.config.embedder
+    ).call(scope: scope, stale_only: stale_only, batch_size: batch_size)
+
+    puts "scope=#{result[:scope]} processed=#{result[:processed]} updated=#{result[:updated]} skipped=#{result[:skipped]} failed=#{result[:failed]}"
+
+    if result[:failed] > 0
+      abort "failed_ids=#{result[:failed_ids].join(",")}"
+    end
+  end
+end

data/lib/engram/railtie.rb

@@ -19,5 +19,9 @@ module Engram
         require "engram/rails/observe_job"
       end
     end
+
+    rake_tasks do
+      load File.expand_path("rails/tasks.rake", __dir__)
+    end
   end
 end

data/lib/engram/record.rb

@@ -5,25 +5,30 @@ module Engram
   #
   # `id` is assigned by the store on persistence (nil until then); consolidation uses it
   # to target UPDATE/FORGET. `scope` namespaces memories to an owner (e.g. "user:42").
-  # `kind` is a memory type (semantic / episodic / preference).
+  # `kind` is a memory type (fact / preference / instruction / episodic). The legacy
+  # `semantic` kind is normalized to `fact` for compatibility with pre-1.0 records.
   class Record
     attr_accessor :id, :last_accessed_at
     attr_reader :content, :embedding, :scope, :kind, :importance, :metadata,
       :created_at
 
-    def initialize(content:, scope:, id: nil, embedding: nil, kind: :semantic,
+    def initialize(content:, scope:, id: nil, embedding: nil, kind: :fact,
       importance: 1.0, metadata: {}, created_at: nil, last_accessed_at: nil)
       @id = id
       @content = content
       @scope = scope
       @embedding = embedding
-      @kind = kind
+      @kind = MemoryKind.normalize(kind)
       @importance = importance
       @metadata = metadata
       @created_at = created_at || Time.now
       @last_accessed_at = last_accessed_at
     end
 
+    def with(**attributes)
+      self.class.new(**to_h.merge(attributes))
+    end
+
     def to_h
       {
         id: id, content: content, scope: scope, embedding: embedding, kind: kind,

data/lib/engram/reserved_metadata.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Engram
+  # Collision-preserving operations for schemas sharing Engram's metadata namespace.
+  module ReservedMetadata
+    KEY = "_engram"
+
+    module_function
+
+    def attach(metadata, schema_key, value, namespace_description: "metadata")
+      metadata = (metadata || {}).dup
+      reserved_values = [metadata.delete(KEY), metadata.delete(:_engram)].compact
+      unless reserved_values.all? { |reserved| reserved.is_a?(Hash) }
+        raise Engram::Error, "metadata key #{KEY.inspect} is reserved for Engram #{namespace_description}"
+      end
+
+      reserved = reserved_values.reduce({}) do |merged, reserved_value|
+        siblings = reserved_value.reject { |key, _nested| key.to_s == schema_key }
+        merge(merged, normalize(siblings))
+      end
+      metadata.merge(KEY => reserved.merge(schema_key => normalize(value)))
+    end
+
+    def normalize(value, path = [])
+      return value.map { |nested| normalize(nested, path) } if value.is_a?(Array)
+      return value unless value.is_a?(Hash)
+
+      value.each_with_object({}) do |(key, nested), normalized|
+        string_key = key.to_s
+        normalized_value = normalize(nested, path + [string_key])
+        normalized[string_key] = if normalized.key?(string_key)
+          merge_value(normalized[string_key], normalized_value, path + [string_key])
+        else
+          normalized_value
+        end
+      end
+    end
+
+    def merge(left, right, path = [])
+      right.each_with_object(left.dup) do |(key, value), merged|
+        merged[key] = merged.key?(key) ? merge_value(merged[key], value, path + [key]) : value
+      end
+    end
+
+    def merge_value(left, right, path)
+      return merge(left, right, path) if left.is_a?(Hash) && right.is_a?(Hash)
+      return left if left == right
+
+      raise Engram::Error, "conflicting reserved metadata at #{path.join(".")}"
+    end
+  end
+end

data/lib/engram/use_cases/forget.rb

@@ -20,8 +20,12 @@ module Engram
           timestamp && timestamp < cutoff && record.importance.to_f < min_importance
         end
 
-        stale.each { |record| @store.delete(id: record.id) if record.id }
-        stale
+        stale.select do |record|
+          next false unless record.id
+
+          deleted = @store.delete(scope: scope, id: record.id)
+          deleted && deleted != 0
+        end
       end
     end
   end

data/lib/engram/use_cases/inject.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "cgi"
+
 module Engram
   module UseCases
     # Render recalled memories into a prompt as a clearly delimited block.
@@ -12,10 +14,22 @@ module Engram
 
       # Returns a new prompt string. If there are no memories, the prompt is unchanged.
       def call(prompt:, memories:)
-        return prompt if memories.nil? || memories.empty?
+        payload = {memory_count: memories&.size.to_i}
+        Engram::Instrumentation.instrument("inject", payload) do
+          next prompt if memories.nil? || memories.empty?
+
+          block = memories.map { |memory| render_memory(memory) }.join("\n")
+          "#{prompt}\n\n#{@header}:\n<engram-memories>\n#{block}\n</engram-memories>"
+        end
+      end
+
+      private
+
+      def render_memory(memory)
+        kind = CGI.escapeHTML((memory.kind || :fact).to_s)
+        content = CGI.escapeHTML(memory.content.to_s)
 
-        block = memories.map { |m| "- #{m.content}" }.join("\n")
-        "#{prompt}\n\n#{@header}:\n#{block}"
+        %(<engram-memory kind="#{kind}">#{content}</engram-memory>)
       end
     end
   end

data/lib/engram/use_cases/observe.rb

@@ -9,51 +9,115 @@ module Engram
     # When a ProcessedTurns store and an idempotency_key are provided, a turn that was
     # already processed is skipped (no extraction, no duplicate memories).
     class Observe
-      def initialize(store:, extractor:, consolidator:, processed_turns: nil)
+      def initialize(store:, extractor:, consolidator:, processed_turns: nil, embedder: Engram.config.embedder)
         @store = store
         @extractor = extractor
         @consolidator = consolidator
         @processed_turns = processed_turns
+        @embedder = embedder
       end
 
       # Returns the Array<Decision> that were applied (empty if skipped or nothing found).
       def call(messages:, scope:, idempotency_key: nil)
-        return [] if already_processed?(idempotency_key)
+        payload = Engram::Instrumentation.payload(
+          scope: scope,
+          store: @store,
+          message_count: messages.size,
+          idempotency_key_present: !idempotency_key.nil?
+        )
+        Engram::Instrumentation.instrument("observe", payload) do
+          claim = acquire_claim(scope, idempotency_key)
+          if idempotency_key && @processed_turns && !claim
+            unless @processed_turns.completed?(scope: scope, key: idempotency_key)
+              # A live, incomplete lease is retryable, not a completed duplicate.
+              raise Engram::ObservationInProgressError,
+                "observation for this turn is claimed but not completed; retry after the claim lease expires"
+            end
 
-        candidates = @extractor.extract(messages: messages, scope: scope)
-        if candidates.empty?
-          mark_processed(idempotency_key)
-          return []
-        end
+            payload[:skipped] = true
+            payload[:candidate_count] = 0
+            payload[:decision_count] = 0
+            next []
+          end
+
+          begin
+            candidates = extract(messages: messages, scope: scope)
+            payload[:candidate_count] = candidates.size
+            if candidates.empty?
+              complete_claim(scope, idempotency_key, claim)
+              payload[:decision_count] = 0
+              next []
+            end
 
-        decisions = @consolidator.reconcile_all(candidates: candidates, scope: scope)
-        decisions.each { |decision| apply(decision) }
-        mark_processed(idempotency_key)
-        decisions
+            decisions = consolidate(candidates: candidates, scope: scope)
+            applied_decisions = decisions.filter_map { |decision| apply(decision, scope) }
+            payload[:decision_count] = applied_decisions.size
+            payload[:decision_actions] = applied_decisions.map { |decision| decision.action.to_s }
+            complete_claim(scope, idempotency_key, claim)
+            applied_decisions
+          rescue
+            release_claim(scope, idempotency_key, claim)
+            raise
+          end
+        end
       end
 
       private
 
-      def already_processed?(key)
-        !!(key && @processed_turns&.seen?(key))
+      def acquire_claim(scope, key)
+        @processed_turns.claim(scope: scope, key: key) if key && @processed_turns
+      end
+
+      def complete_claim(scope, key, claim)
+        @processed_turns.complete(scope: scope, key: key, claim: claim) if claim
+      end
+
+      def release_claim(scope, key, claim)
+        @processed_turns.release(scope: scope, key: key, claim: claim) if claim
       end
 
-      def mark_processed(key)
-        @processed_turns.record(key) if key && @processed_turns
+      def extract(messages:, scope:)
+        payload = Engram::Instrumentation.payload(scope: scope, store: @store, message_count: messages.size)
+        Engram::Instrumentation.instrument("extract", payload) do
+          candidates = @extractor.extract(messages: messages, scope: scope)
+          payload[:candidate_count] = candidates.size
+          candidates
+        end
+      end
+
+      def consolidate(candidates:, scope:)
+        payload = Engram::Instrumentation.payload(scope: scope, store: @store, candidate_count: candidates.size)
+        Engram::Instrumentation.instrument("consolidate", payload) do
+          decisions = @consolidator.reconcile_all(candidates: candidates, scope: scope)
+          payload[:decision_count] = decisions.size
+          payload[:decision_actions] = decisions.map { |decision| decision.action.to_s }
+          decisions
+        end
       end
 
-      def apply(decision)
+      def apply(decision, scope)
         case decision.action
         when :add
-          @store.add(decision.candidate)
+          raise Engram::Error, "cannot move memory across scopes" unless decision.candidate.scope == scope
+
+          decision if persistence.add(decision.candidate, scope: scope)
         when :update
-          @store.update(id: decision.target_id, record: decision.candidate) if decision.target_id
+          if decision.target_id && persistence.update(scope: scope, id: decision.target_id, record: decision.candidate)
+            decision
+          end
         when :forget
-          @store.delete(id: decision.target_id) if decision.target_id
+          if decision.target_id
+            deleted = @store.delete(scope: scope, id: decision.target_id)
+            decision if deleted && deleted != 0
+          end
         when :noop
           nil
         end
       end
+
+      def persistence
+        @persistence ||= Persistence.new(store: @store, embedder: @embedder)
+      end
     end
   end
 end

data/lib/engram/use_cases/rebuild_embeddings.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Engram
+  module UseCases
+    # Rebuild stored embeddings when the active embedder configuration changes.
+    # Returns counts useful for logging/observability:
+    #   - :processed: total rows examined in scope
+    #   - :updated: rows re-embedded and persisted
+    #   - :skipped: rows skipped because they were unchanged or incomplete
+    #   - :failed: count of rows that failed during rebuild
+    #   - :failed_ids: record IDs that failed during rebuild
+    #   - :failed_errors: error class/message keyed by failed record ID
+    class RebuildEmbeddings
+      def initialize(store:, embedder:)
+        @store = store
+        @embedder = embedder
+      end
+
+      def call(scope:, stale_only: true, batch_size: 100)
+        batch_size = Integer(batch_size)
+        raise ArgumentError, "batch_size must be greater than 0" unless batch_size.positive?
+
+        counts = {
+          processed: 0,
+          updated: 0,
+          skipped: 0,
+          failed: 0,
+          failed_ids: [],
+          failed_errors: {}
+        }
+
+        after_id = nil
+        legacy_index = 0
+        legacy_records = nil
+        processed_ids = Set.new
+        processed_nil_records = Set.new
+        loop do
+          batch_result = fetch_batch(scope:, batch_size:, after_id:, legacy_records:, legacy_index:)
+          legacy_records = batch_result[:legacy_records] if batch_result[:legacy_records]
+          batch = batch_result[:records]
+          if batch.empty?
+            break if legacy_records
+
+            # Reconcile with one stable snapshot: accepting keyset keywords does
+            # not guarantee that an adapter returns complete, ordered pages.
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          cursor_record = batch.reverse.find { |candidate| !candidate.id.nil? } unless legacy_records
+
+          if !legacy_records && fallback_before_processing?(batch:, after_id:)
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            batch = legacy_records.slice(legacy_index, batch_size) || []
+            break if batch.empty?
+          end
+
+          batch.each do |record|
+            identity = record.id.nil? ? record.object_id : record.id
+            seen = record.id.nil? ? processed_nil_records : processed_ids
+            next unless seen.add?(identity)
+
+            counts[:processed] += 1
+
+            if record.id.nil?
+              counts[:skipped] += 1
+              next
+            end
+
+            if stale_only && !stale?(record)
+              counts[:skipped] += 1
+              next
+            end
+
+            begin
+              rebuilt = record.with(embedding: @embedder.embed(record.content))
+              rebuilt = EmbeddingMetadata.attach(rebuilt, embedder: @embedder)
+              updated = @store.update(scope: scope, id: record.id, record: rebuilt)
+              raise Engram::Error, "memory update was not applied" unless updated && updated != 0
+
+              counts[:updated] += 1
+            rescue => error
+              counts[:failed] += 1
+              record_id = record.id
+              counts[:failed_ids] << record_id
+              counts[:failed_errors][record_id] = {
+                class: error.class.name,
+                message: error.message
+              }
+              warn "Rebuild failed for record ##{record_id}: #{error.class}: #{error.message}"
+            end
+          end
+
+          if legacy_records
+            legacy_index += batch.length
+            next
+          end
+
+          if cursor_record.nil?
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          if fallback_to_legacy_batching?(batch:, cursor_record:)
+            legacy_records = Array(@store.all(scope: scope))
+            legacy_index = 0
+            next
+          end
+
+          after_id = cursor_record.id
+        end
+
+        {scope: scope, **counts}
+      end
+
+      private
+
+      def fetch_batch(scope:, batch_size:, after_id:, legacy_records:, legacy_index:)
+        if legacy_records
+          return {records: legacy_records.slice(legacy_index, batch_size) || [], legacy_records:}
+        end
+
+        records = modern_batch(scope:, batch_size:, after_id:)
+        return {records:} if records
+
+        legacy_records = Array(@store.all(scope: scope))
+        {records: legacy_records.slice(legacy_index, batch_size) || [], legacy_records:}
+      end
+
+      def modern_batch(scope:, batch_size:, after_id:)
+        return unless batched_all_supported?
+
+        @store.all(scope: scope, limit: batch_size, after_id: after_id)
+      end
+
+      def batched_all_supported?
+        parameters = @store.method(:all).parameters
+        keyword_names = parameters.filter_map do |kind, name|
+          name if [:keyreq, :key].include?(kind)
+        end
+
+        keyword_names.include?(:limit) && keyword_names.include?(:after_id)
+      end
+
+      def fallback_to_legacy_batching?(batch:, cursor_record:)
+        cursor_record != batch.last
+      end
+
+      def fallback_before_processing?(batch:, after_id:)
+        ids = batch.filter_map(&:id)
+        return true if ids.uniq.length != ids.length
+        return false if after_id.nil?
+        return true if batch.any? { |record| record.id.nil? }
+
+        batch.any? { |record| record.id && record.id <= after_id }
+      end
+
+      def stale?(record)
+        stored = EmbeddingMetadata.extract(record.metadata)
+        return true if stored.empty?
+
+        expected = expected_metadata(record)
+        return true unless expected
+
+        required = %w[adapter provider model dimensions fingerprint]
+        return true if required.any? { |key| stored[key] != expected[key] }
+
+        return true if stored["dimensions"] &&
+          record.embedding.respond_to?(:length) &&
+          record.embedding.length != stored["dimensions"].to_i
+
+        false
+      end
+
+      # What the active embedder would produce for a rebuilt row. Declared dimensions take
+      # precedence so dimension-only configuration drift marks rows stale; fall back to the
+      # record's current vector length for embedders that do not declare dimensions.
+      def expected_metadata(record)
+        declared = EmbeddingMetadata.for_embedder(@embedder)
+        return declared if declared.nil? || declared["dimensions"]
+
+        EmbeddingMetadata.for_embedder(@embedder, embedding: record.embedding)
+      end
+    end
+  end
+end

data/lib/engram/use_cases/recall.rb

@@ -24,16 +24,35 @@ module Engram
       end
 
       # Returns Array<Record>, most relevant first.
-      def call(query, scope:, limit: Engram.config.default_limit)
+      def call(query, scope:, limit: Engram.config.default_limit, kinds: nil)
         raise ArgumentError, "query must be a non-empty string" if query.to_s.strip.empty?
 
-        embedding = @embedder.embed(query)
-        pool_limit = reranking? ? limit * @pool_factor : limit
-        pool = @store.search(embedding: embedding, scope: scope, limit: pool_limit)
+        payload = Engram::Instrumentation.payload(
+          scope: scope,
+          store: @store,
+          limit: limit,
+          kinds: Array(kinds).map(&:to_s),
+          reranking: reranking?
+        )
+        Engram::Instrumentation.instrument("recall", payload) do
+          embedding = @embedder.embed(query)
+          embedding_metadata = Engram::EmbeddingMetadata.for_embedder(@embedder, embedding: embedding)
+          pool_limit = reranking? ? limit * @pool_factor : limit
+          pool = Engram::EmbeddingMetadata.search(
+            @store,
+            embedding: embedding,
+            embedding_metadata: embedding_metadata,
+            scope: scope,
+            limit: pool_limit,
+            kinds: kinds
+          )
 
-        results = (reranking? ? rerank(pool, embedding) : pool).first(limit)
-        touch(results) if @touch
-        results
+          results = (reranking? ? rerank(pool, embedding) : pool).first(limit)
+          touch(results, scope) if @touch
+          payload[:result_count] = results.size
+          payload[:candidate_count] = pool.size
+          results
+        end
       end
 
       private
@@ -61,8 +80,8 @@ module Engram
         0.5**(age / @recency_halflife)
       end
 
-      def touch(records)
-        records.each { |record| @store.touch(id: record.id, at: Time.now) if record.id }
+      def touch(records, scope)
+        records.each { |record| @store.touch(scope: scope, id: record.id, at: Time.now) if record.id }
       end
     end
   end

data/lib/engram/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Engram
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end