engram 0.4.0 → 0.6.0

data/lib/engram/rails/tasks.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Loaded by the Railtie in Rails apps (where :environment boots the app and its
+# initializers) and by the gem's own Rakefile (which defines a no-op :environment).
+namespace :engram do
+  desc "Rebuild embeddings in a memory scope. Usage: bundle exec rake 'engram:rebuild_embeddings[user:1]'"
+  task :rebuild_embeddings, [:scope] => :environment do |_, args|
+    scope = args[:scope]
+    raise ArgumentError, "rebuild_embeddings requires a scope argument" if scope.to_s.empty?
+
+    stale_only = !%w[false 0 no].include?(ENV.fetch("STALE_ONLY", "true").downcase)
+    batch_size = Integer(ENV.fetch("BATCH_SIZE", "100"))
+    raise ArgumentError, "BATCH_SIZE must be greater than 0" unless batch_size.positive?
+
+    result = Engram::UseCases::RebuildEmbeddings.new(
+      store: Engram.config.store,
+      embedder: Engram.config.embedder
+    ).call(scope: scope, stale_only: stale_only, batch_size: batch_size)
+
+    puts "scope=#{result[:scope]} processed=#{result[:processed]} updated=#{result[:updated]} skipped=#{result[:skipped]} failed=#{result[:failed]}"
+
+    if result[:failed] > 0
+      abort "failed_ids=#{result[:failed_ids].join(",")}"
+    end
+  end
+end

data/lib/engram/railtie.rb

@@ -19,5 +19,9 @@ module Engram
         require "engram/rails/observe_job"
       end
     end
+
+    rake_tasks do
+      load File.expand_path("rails/tasks.rake", __dir__)
+    end
   end
 end

data/lib/engram/record.rb

@@ -8,6 +8,10 @@ module Engram
   # `kind` is a memory type (fact / preference / instruction / episodic). The legacy
   # `semantic` kind is normalized to `fact` for compatibility with pre-1.0 records.
   class Record
+    STATE_READERS = %i[
+      id content scope embedding kind importance metadata created_at last_accessed_at
+    ].freeze
+
     attr_accessor :id, :last_accessed_at
     attr_reader :content, :embedding, :scope, :kind, :importance, :metadata,
       :created_at
@@ -29,12 +33,15 @@ module Engram
       self.class.new(**to_h.merge(attributes))
     end
 
+    # Return structured supporting-source metadata when this record carries a
+    # provenance schema understood by the installed Engram version.
+    # Legacy, malformed, and future-schema records remain readable and return nil.
+    def provenance
+      Engram::Provenance.extract(metadata)
+    end
+
     def to_h
-      {
-        id: id, content: content, scope: scope, embedding: embedding, kind: kind,
-        importance: importance, metadata: metadata,
-        created_at: created_at, last_accessed_at: last_accessed_at
-      }
+      STATE_READERS.to_h { |reader| [reader, public_send(reader)] }
     end
   end
 end

data/lib/engram/reserved_metadata.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Engram
+  # Collision-preserving operations for schemas sharing Engram's metadata namespace.
+  module ReservedMetadata
+    KEY = "_engram"
+
+    module_function
+
+    def attach(metadata, schema_key, value, namespace_description: "metadata")
+      metadata = (metadata || {}).dup
+      reserved_values = [metadata.delete(KEY), metadata.delete(:_engram)].compact
+      unless reserved_values.all? { |reserved| reserved.is_a?(Hash) }
+        raise Engram::Error, "metadata key #{KEY.inspect} is reserved for Engram #{namespace_description}"
+      end
+
+      reserved = reserved_values.reduce({}) do |merged, reserved_value|
+        siblings = reserved_value.reject { |key, _nested| key.to_s == schema_key }
+        merge(merged, normalize(siblings))
+      end
+      metadata.merge(KEY => reserved.merge(schema_key => normalize(value)))
+    end
+
+    def normalize(value, path = [])
+      return value.map { |nested| normalize(nested, path) } if value.is_a?(Array)
+      return value unless value.is_a?(Hash)
+
+      value.each_with_object({}) do |(key, nested), normalized|
+        string_key = key.to_s
+        normalized_value = normalize(nested, path + [string_key])
+        normalized[string_key] = if normalized.key?(string_key)
+          merge_value(normalized[string_key], normalized_value, path + [string_key])
+        else
+          normalized_value
+        end
+      end
+    end
+
+    def merge(left, right, path = [])
+      right.each_with_object(left.dup) do |(key, value), merged|
+        merged[key] = merged.key?(key) ? merge_value(merged[key], value, path + [key]) : value
+      end
+    end
+
+    def merge_value(left, right, path)
+      return merge(left, right, path) if left.is_a?(Hash) && right.is_a?(Hash)
+      return left if left == right
+
+      raise Engram::Error, "conflicting reserved metadata at #{path.join(".")}"
+    end
+  end
+end

data/lib/engram/use_cases/forget.rb

@@ -20,8 +20,12 @@ module Engram
           timestamp && timestamp < cutoff && record.importance.to_f < min_importance
         end
 
-        stale.each { |record| @store.delete(id: record.id) if record.id }
-        stale
+        stale.select do |record|
+          next false unless record.id
+
+          deleted = @store.delete(scope: scope, id: record.id)
+          deleted && deleted != 0
+        end
       end
     end
   end

data/lib/engram/use_cases/grounding_report.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Engram
+  module UseCases
+    # Count records in a scope by their weakest understood source alignment.
+    class GroundingReport
+      ALIGNMENT_RANK = Engram::Provenance::ALIGNMENTS.each_with_index.to_h.freeze
+      private_constant :ALIGNMENT_RANK
+
+      def initialize(store:)
+        @store = store
+      end
+
+      def call(scope:)
+        counts = {
+          exact: 0,
+          normalized: 0,
+          inferred: 0,
+          ungrounded: 0,
+          unattributed: 0,
+          total: 0
+        }
+
+        @store.all(scope: scope).each do |record|
+          provenance = record.provenance
+          alignment = provenance && weakest_alignment(provenance)
+          counts[alignment || :unattributed] += 1
+          counts[:total] += 1
+        end
+
+        counts.freeze
+      end
+
+      private
+
+      def weakest_alignment(provenance)
+        alignments = provenance.sources.map(&:alignment)
+        return unless alignments.all? { |alignment| ALIGNMENT_RANK.key?(alignment) }
+
+        alignments.max_by { |alignment| ALIGNMENT_RANK.fetch(alignment) }
+      end
+    end
+  end
+end

data/lib/engram/use_cases/observe.rb

@@ -26,69 +26,343 @@ module Engram
           idempotency_key_present: !idempotency_key.nil?
         )
         Engram::Instrumentation.instrument("observe", payload) do
-          if already_processed?(idempotency_key)
+          claim = acquire_claim(scope, idempotency_key)
+          if idempotency_key && @processed_turns && !claim
+            unless @processed_turns.completed?(scope: scope, key: idempotency_key)
+              # A live, incomplete lease is retryable, not a completed duplicate.
+              raise Engram::ObservationInProgressError,
+                "observation for this turn is claimed but not completed; retry after the claim lease expires"
+            end
+
             payload[:skipped] = true
             payload[:candidate_count] = 0
             payload[:decision_count] = 0
             next []
           end
 
-          candidates = extract(messages: messages, scope: scope)
-          payload[:candidate_count] = candidates.size
-          if candidates.empty?
-            mark_processed(idempotency_key)
-            payload[:decision_count] = 0
-            next []
-          end
+          begin
+            candidates = extract(messages: messages, scope: scope)
+            payload[:candidate_count] = candidates.size
+            if candidates.empty?
+              complete_claim(scope, idempotency_key, claim)
+              payload[:decision_count] = 0
+              next []
+            end
 
-          decisions = consolidate(candidates: candidates, scope: scope)
-          applied_decisions = decisions.filter_map { |decision| apply(decision) }
-          payload[:decision_count] = applied_decisions.size
-          payload[:decision_actions] = applied_decisions.map { |decision| decision.action.to_s }
-          mark_processed(idempotency_key)
-          applied_decisions
+            decisions, candidate_budget, detached_by_candidate = consolidate(candidates: candidates, scope: scope)
+            operations = preflight(decisions, candidate_budget, detached_by_candidate, scope)
+            applied_decisions = operations.filter_map { |operation| apply(operation, scope) }
+            payload[:decision_count] = applied_decisions.size
+            payload[:decision_actions] = applied_decisions.map { |decision| decision.action.to_s }
+            complete_claim(scope, idempotency_key, claim)
+            applied_decisions
+          rescue
+            release_claim(scope, idempotency_key, claim)
+            raise
+          end
         end
       end
 
       private
 
-      def already_processed?(key)
-        !!(key && @processed_turns&.seen?(key))
+      def acquire_claim(scope, key)
+        @processed_turns.claim(scope: scope, key: key) if key && @processed_turns
       end
 
-      def mark_processed(key)
-        @processed_turns.record(key) if key && @processed_turns
+      def complete_claim(scope, key, claim)
+        @processed_turns.complete(scope: scope, key: key, claim: claim) if claim
+      end
+
+      def release_claim(scope, key, claim)
+        @processed_turns.release(scope: scope, key: key, claim: claim) if claim
       end
 
       def extract(messages:, scope:)
         payload = Engram::Instrumentation.payload(scope: scope, store: @store, message_count: messages.size)
         Engram::Instrumentation.instrument("extract", payload) do
-          candidates = @extractor.extract(messages: messages, scope: scope)
+          candidates = normalize_extractions(@extractor.extract(messages: messages, scope: scope))
           payload[:candidate_count] = candidates.size
           candidates
         end
       end
 
+      def normalize_extractions(results)
+        unless core_array?(results)
+          raise Engram::Error, "extractor must return an Array containing only Engram::Record or Engram::Extraction values"
+        end
+
+        core_array_map(results) do |result|
+          candidate = case result
+          when Engram::Record then normalize_record(result)
+          when Engram::Extraction then normalize_record(result.to_record)
+          else
+            raise Engram::Error, "extractor must return an Array containing only Engram::Record or Engram::Extraction values"
+          end
+
+          raise Engram::Error, "observation candidates must not have an id" unless candidate.id.nil?
+
+          candidate
+        end
+      rescue Engram::Internal::CandidateIntegrity::Error => error
+        raise Engram::Error, error.message
+      end
+
+      def normalize_record(record)
+        if Object.instance_method(:instance_of?).bind_call(record, Engram::Record)
+          candidate_integrity.validate!(record)
+          return record
+        end
+
+        candidate_integrity.detach(record)
+      rescue Engram::Internal::CandidateIntegrity::Error => error
+        raise Engram::Error, error.message
+      end
+
       def consolidate(candidates:, scope:)
         payload = Engram::Instrumentation.payload(scope: scope, store: @store, candidate_count: candidates.size)
         Engram::Instrumentation.instrument("consolidate", payload) do
-          decisions = @consolidator.reconcile_all(candidates: candidates, scope: scope)
+          candidate_snapshot = snapshot_candidates(candidates)
+          detached_candidates = detach_candidates(candidates)
+          candidate_budget, detached_by_candidate = candidate_context(candidates, detached_candidates)
+          raw_decisions = @consolidator.reconcile_all(candidates: candidates, scope: scope)
+          decisions = canonicalize_decisions(raw_decisions)
+          verify_candidates!(candidates, candidate_snapshot)
+
           payload[:decision_count] = decisions.size
           payload[:decision_actions] = decisions.map { |decision| decision.action.to_s }
-          decisions
+          [decisions, candidate_budget, detached_by_candidate]
+        end
+      end
+
+      def snapshot_candidates(candidates)
+        candidate_integrity.snapshot(candidates)
+      rescue Engram::Internal::CandidateIntegrity::Error => error
+        raise Engram::Error, error.message
+      end
+
+      def verify_candidates!(candidates, snapshot)
+        candidate_integrity.verify!(candidates, snapshot)
+      rescue Engram::Internal::CandidateIntegrity::Error => error
+        raise Engram::Error, error.message
+      end
+
+      def detach_candidates(candidates)
+        candidates.map { |candidate| candidate_integrity.detach(candidate) }
+      rescue Engram::Internal::CandidateIntegrity::Error => error
+        raise Engram::Error, error.message
+      end
+
+      def candidate_context(candidates, detached_candidates)
+        candidate_budget = Hash.new(0).compare_by_identity
+        detached_by_candidate = {}.compare_by_identity
+        candidates.zip(detached_candidates) do |candidate, detached|
+          candidate_budget[candidate] += 1
+          (detached_by_candidate[candidate] ||= []) << detached
+        end
+        [candidate_budget, detached_by_candidate]
+      end
+
+      def candidate_integrity
+        @candidate_integrity ||= Engram::Internal::CandidateIntegrity.new
+      end
+
+      def canonicalize_decisions(raw_decisions)
+        unless core_array?(raw_decisions)
+          raise Engram::Error, "consolidator must return an Array of Engram::Decision values"
+        end
+
+        core_array_map(raw_decisions) do |raw_decision|
+          unless Object.instance_method(:is_a?).bind_call(raw_decision, Engram::Decision)
+            raise Engram::Error, "consolidator must return an Array of Engram::Decision values"
+          end
+
+          action = raw_decision.action
+          candidate = raw_decision.candidate
+          target_id = canonical_target_id(raw_decision.target_id)
+          reason = raw_decision.reason
+          unless Engram::Decision::ACTIONS.include?(action)
+            raise Engram::Error, "unsupported decision action #{action.inspect}"
+          end
+
+          Engram::Decision.new(action: action, candidate: candidate, target_id: target_id, reason: reason)
+        end
+      end
+
+      # Array subclasses are supported, but checks and traversal at an untrusted
+      # return boundary must not dispatch through singleton or subclass behavior.
+      def core_array?(value)
+        Object.instance_method(:is_a?).bind_call(value, Array)
+      rescue TypeError
+        false
+      end
+
+      def core_array_map(values)
+        mapped = []
+        Array.instance_method(:each).bind_call(values) { |value| mapped << yield(value) }
+        mapped
+      end
+
+      def canonical_target_id(target_id)
+        return if BasicObject.instance_method(:equal?).bind_call(target_id, nil)
+
+        target_class = Object.instance_method(:class).bind_call(target_id)
+        valid_class = target_class.equal?(Integer) || target_class.equal?(String)
+        valid_state = Object.instance_method(:instance_variables).bind_call(target_id).empty?
+        unless valid_class && valid_state
+          raise Engram::Error, "decision target_id must be a plain String or Integer"
         end
+
+        return target_id if target_class.equal?(Integer)
+
+        if custom_behavior?(target_id, target_class)
+          raise Engram::Error, "decision target_id must be a plain String or Integer"
+        end
+
+        String.instance_method(:dup).bind_call(target_id)
+      rescue TypeError
+        raise Engram::Error, "decision target_id must be a plain String or Integer"
       end
 
-      def apply(decision)
+      def custom_behavior?(value, value_class)
+        singleton_class = Object.instance_method(:singleton_class).bind_call(value)
+        return false if singleton_class.equal?(value_class)
+
+        method_visibilities = %i[
+          public_instance_methods protected_instance_methods private_instance_methods
+        ]
+        return true if method_visibilities.any? do |visibility|
+          Module.instance_method(visibility).bind_call(singleton_class, false).any?
+        end
+
+        singleton_ancestors = Module.instance_method(:ancestors).bind_call(singleton_class)
+        class_ancestors = Module.instance_method(:ancestors).bind_call(value_class)
+        return true unless singleton_ancestors.length == class_ancestors.length + 1 &&
+          class_ancestors.each_with_index.all? { |ancestor, index| singleton_ancestors[index + 1].equal?(ancestor) }
+
+        method_visibilities.any? do |visibility|
+          singleton_methods = Module.instance_method(visibility).bind_call(singleton_class, true)
+          class_methods = Module.instance_method(visibility).bind_call(value_class, true)
+          singleton_methods.length != class_methods.length ||
+            singleton_methods.any? { |method_name| !class_methods.include?(method_name) }
+        end
+      end
+
+      def preflight(decisions, candidate_budget, detached_by_candidate, scope)
+        decision_counts = Hash.new(0).compare_by_identity
+        safe_decisions = []
+
+        decisions.each do |decision|
+          unless decision.is_a?(Engram::Decision)
+            raise Engram::Error, "consolidator must return an Array of Engram::Decision values"
+          end
+          unless Engram::Decision::ACTIONS.include?(decision.action)
+            raise Engram::Error, "unsupported decision action #{decision.action.inspect}"
+          end
+          unless Object.instance_method(:is_a?).bind_call(decision.candidate, Engram::Record)
+            message = if decision.action == :forget
+              "forget decision candidate must be an Engram::Record"
+            else
+              "decision candidate must be an Engram::Record"
+            end
+            raise Engram::Error, message
+          end
+          unless candidate_budget.key?(decision.candidate)
+            raise Engram::Error, "decision must reference the actual candidate supplied to the consolidator"
+          end
+          decision_counts[decision.candidate] += 1
+          if decision_counts[decision.candidate] > candidate_budget.fetch(decision.candidate)
+            raise Engram::Error,
+              "consolidator must return no more than one decision per candidate occurrence; " \
+              "multiple decisions reference the same candidate"
+          end
+          detached_candidate = detached_by_candidate.fetch(decision.candidate).fetch(
+            decision_counts.fetch(decision.candidate) - 1
+          )
+          unless Engram::Internal::Scope.record_matches?(detached_candidate, scope)
+            raise Engram::Error, "cannot move memory across scopes"
+          end
+
+          if %i[update forget].include?(decision.action) && !decision.target_id
+            raise Engram::Error, "#{decision.action} decision requires a target_id"
+          end
+
+          if %i[add update forget].include?(decision.action)
+            Engram::Provenance.extract_for_persistence(detached_candidate.metadata)
+          end
+          safe_decisions << Engram::Decision.new(action: decision.action, candidate: detached_candidate,
+            target_id: decision.target_id, reason: decision.reason)
+        end
+
+        preflight_destructive_targets!(safe_decisions, scope)
+        decisions.zip(safe_decisions).map do |result_decision, safe_decision|
+          prepare_operation(safe_decision, scope, result_decision)
+        end
+      end
+
+      def preflight_destructive_targets!(decisions, scope)
+        target_ids = decisions.filter_map do |decision|
+          decision.target_id if %i[update forget].include?(decision.action)
+        end
+        return if target_ids.empty?
+
+        duplicate_id = target_ids.tally.find { |_, count| count > 1 }&.first
+        if duplicate_id
+          raise Engram::Error, "multiple decisions target memory #{duplicate_id.inspect}"
+        end
+
+        existing_ids = scoped_existing_ids(scope, target_ids)
+        existing_id_lookup = existing_ids.each_with_object({}) { |id, lookup| lookup[id] = true }
+        missing_id = target_ids.find { |target_id| !existing_id_lookup[target_id] }
+        if missing_id
+          raise Engram::Error, "no memory with id #{missing_id.inspect} in scope #{scope.inspect}"
+        end
+      end
+
+      def scoped_existing_ids(scope, target_ids)
+        if @store.respond_to?(:existing_ids)
+          begin
+            return Array(@store.existing_ids(scope: scope, ids: target_ids))
+          rescue NotImplementedError
+            # Optional capability: legacy adapters may inherit the default stub.
+          end
+        end
+
+        @store.all(scope: scope).map(&:id)
+      end
+
+      def prepare_operation(decision, scope, result_decision)
+        prepared = case decision.action
+        when :add
+          persistence.prepare(decision.candidate)
+        when :update
+          persistence.prepare(decision.candidate) if decision.target_id
+        when :forget
+          persistence.allowed?(decision.candidate) if decision.target_id
+        end
+
+        if %i[add update].include?(decision.action) && prepared &&
+            !Engram::Internal::Scope.record_matches?(prepared, scope)
+          raise Engram::Error, "cannot move memory across scopes"
+        end
+
+        [result_decision, decision, prepared]
+      end
+
+      def apply(operation, scope)
+        result_decision, decision, prepared = operation
         case decision.action
         when :add
-          decision if persistence.add(decision.candidate)
+          result_decision if prepared && persistence.add_prepared(prepared, scope: scope)
         when :update
-          if decision.target_id && persistence.update(id: decision.target_id, record: decision.candidate)
-            decision
+          if decision.target_id && prepared &&
+              persistence.update_prepared(scope: scope, id: decision.target_id, record: prepared)
+            result_decision
           end
         when :forget
-          decision if decision.target_id && @store.delete(id: decision.target_id)
+          if decision.target_id && prepared
+            deleted = @store.delete(scope: scope, id: decision.target_id)
+            result_decision if deleted && deleted != 0
+          end
         when :noop
           nil
         end