engram 0.4.0 → 0.6.0

data/lib/engram/persistence.rb

@@ -3,6 +3,8 @@
 module Engram
   # Applies persistence hooks and policy consistently before writing records.
   class Persistence
+    RECORD_METADATA_READER = Engram::Record.instance_method(:metadata)
+
     def initialize(store:, embedder:, before_persist: Engram.config.before_persist,
       persistence_policy: Engram.config.persistence_policy)
       @store = store
@@ -11,24 +13,97 @@ module Engram
       @persistence_policy = persistence_policy
     end
 
-    def add(record)
-      record = prepare(record)
-      @store.add(record) if record
+    def add(record, scope: record.scope)
+      add_prepared(prepare(record), scope: scope)
     end
 
-    def update(id:, record:)
-      record = prepare(record)
-      @store.update(id: id, record: record) if record
+    def update(scope:, id:, record:)
+      update_prepared(scope: scope, id: id, record: prepare(record))
     end
 
-    private
+    # Persists a record already returned by #prepare. These split-phase methods let
+    # orchestrators prepare a complete batch before beginning store mutations.
+    def add_prepared(record, scope: record&.scope)
+      return unless record
+
+      record = canonicalize_provenance!(record)
+      unless Engram::Internal::Scope.record_matches?(record, scope)
+        raise Engram::Error, "cannot move memory across scopes"
+      end
+
+      @store.add(record)
+    end
+
+    def update_prepared(scope:, id:, record:)
+      return unless record
 
+      record = canonicalize_provenance!(record)
+      unless Engram::Internal::Scope.record_matches?(record, scope)
+        raise Engram::Error, "cannot move memory across scopes"
+      end
+
+      @store.update(scope: scope, id: id, record: record)
+    end
+
+    # Applies all write transformations without mutating the store. Input is validated
+    # before callbacks can remove or replace provenance, and final output is validated too.
     def prepare(record)
+      original_provenance = validate_provenance!(record)
       original_content = record.content
       record = @before_persist.call(record) if @before_persist
+      if record
+        transformed_provenance = validate_provenance!(record)
+        validate_provenance_trust!(original_provenance, transformed_provenance) if @before_persist
+      end
       record = @persistence_policy.call(record) if record && @persistence_policy
-      record = record.with(embedding: @embedder.embed(record.content)) if record && record.content != original_content
+      validate_provenance!(record) if record
+      if record && record.content != original_content
+        record = record.with(embedding: @embedder.embed(record.content))
+      end
+      record = Engram::EmbeddingMetadata.attach(record, embedder: @embedder) if record
+      validate_provenance!(record) if record
       record
     end
+
+    # Authorizes a destructive decision without applying write transformations or content
+    # filtering. Policies may opt into this separate contract with #allow_destructive?.
+    # Malformed provenance always fails closed.
+    def allowed?(record)
+      validate_provenance!(record)
+      return true unless @persistence_policy&.respond_to?(:allow_destructive?)
+
+      authorization = @persistence_policy.allow_destructive?(record)
+      unless authorization.equal?(true) || authorization.equal?(false)
+        raise Engram::Error, "persistence policy allow_destructive? must return true or false"
+      end
+      validate_provenance!(record)
+
+      authorization
+    end
+
+    private
+
+    def canonicalize_provenance!(record)
+      metadata = RECORD_METADATA_READER.bind_call(record)
+      canonical_metadata = Engram::Provenance.canonical_metadata_for_persistence(metadata)
+      return record if canonical_metadata.equal?(metadata)
+
+      record.with(metadata: canonical_metadata)
+    rescue TypeError
+      raise Engram::Error, "persistence requires an Engram::Record"
+    end
+
+    def validate_provenance!(record)
+      metadata = RECORD_METADATA_READER.bind_call(record)
+      Engram::Provenance.canonical_integrity_representation_for_persistence(metadata)
+    rescue TypeError
+      raise Engram::Error, "persistence requires an Engram::Record"
+    end
+
+    def validate_provenance_trust!(original, transformed)
+      return if original == transformed
+
+      raise Engram::Error, "before_persist cannot change provenance trust"
+    end
   end
 end

data/lib/engram/persistence_policy.rb

@@ -16,16 +16,26 @@ module Engram
       /\b(?:today|now|this session)\b.*\b(?:fixed|resolved|done|completed|finished)\b/i
     ].freeze
 
-    def initialize(denylist_patterns: [])
+    def initialize(denylist_patterns: [], allow_ungrounded: false)
       @denylist_patterns = denylist_patterns
+      @allow_ungrounded = BasicObject.instance_method(:equal?).bind_call(allow_ungrounded, true)
     end
 
     def call(record)
+      provenance = Engram::Provenance.extract_for_persistence(record.metadata)
+      return nil if provenance&.ungrounded? && !@allow_ungrounded
       return nil if reject?(record.content)
 
       redact(record)
     end
 
+    # Destructive authorization is deliberately independent from write-content filtering and
+    # redaction. Provenance trust still applies, but secret/transient content can always be removed.
+    def allow_destructive?(record)
+      provenance = Engram::Provenance.extract_for_persistence(record.metadata)
+      !provenance&.ungrounded? || @allow_ungrounded
+    end
+
     private
 
     def reject?(content)

data/lib/engram/ports/consolidator.rb

@@ -7,8 +7,13 @@ module Engram
     # dumb pile of embeddings.
     # Implementations: Consolidators::HeuristicConsolidator, Consolidators::LLMConsolidator.
     module Consolidator
-      # Given Array<Record> candidates and a scope, return Array<Decision> (one per
-      # candidate that should result in an action).
+      # Given Array<Record> candidates and a scope, return Array<Decision> (at most one
+      # per candidate occurrence, including NOOP decisions). Each decision must reference
+      # the actual candidate instance from this array, not a copy or replacement. When the
+      # same instance occurs multiple times, it may have no more decisions than occurrences.
+      # Candidates are read-only reconciliation inputs: implementations must not mutate
+      # their state, including nested metadata or embedding values. Decision target IDs
+      # must be plain String or Integer values without singleton behavior or custom state.
       def reconcile_all(candidates:, scope:)
         raise NotImplementedError, "#{self.class} must implement #reconcile_all"
       end

data/lib/engram/ports/extractor.rb

@@ -6,7 +6,7 @@ module Engram
     # Declared now so the differentiator (extract -> consolidate) slots in without
     # reworking the core. Not implemented in v0.1.
     module Extractor
-      # Given conversation messages, return Array<Record> of candidate memories.
+      # Given conversation messages, return an Array whose members are Record or Extraction values.
       def extract(messages:, scope:)
         raise NotImplementedError, "Extractor arrives in v0.2"
       end

data/lib/engram/ports/memory_store.rb

@@ -13,28 +13,46 @@ module Engram
       # Return up to `limit` Records in `scope` nearest to `embedding`,
       # ordered most-relevant first. When `kinds` is provided, only records with
       # those canonical memory kinds are eligible.
-      def search(embedding:, scope:, limit:, kinds: nil)
+      def search(embedding:, scope:, limit:, kinds: nil, embedding_metadata: nil)
         raise NotImplementedError, "#{self.class} must implement #search"
       end
 
       # All Records for a scope (mostly for inspection/tests).
-      def all(scope:)
+      # Supports optional `limit` and `offset` for batching large sweeps.
+      # Returned records are sorted in stable `id` order when batching is used.
+      # Use `after_id` for keyset pagination.
+      def all(scope:, limit: nil, offset: 0, after_id: nil)
         raise NotImplementedError, "#{self.class} must implement #all"
       end
 
+      # Optional performance capability: return the subset of requested ids that
+      # exists in `scope`, preserving each requested value's representation rather
+      # than returning a store-native cast of it. Return at most one requested
+      # representation for each persisted record when the store accepts aliases.
+      # Callers must fall back
+      # to #all for legacy stores that do not expose this method or leave it
+      # unimplemented, so adding it does not break custom MemoryStore adapters.
+      def existing_ids(scope:, ids:)
+        raise NotImplementedError, "#{self.class} does not implement #existing_ids"
+      end
+
       # Replace the content/embedding of an existing memory. Used by consolidation
-      # (UPDATE). Returns the updated Record.
-      def update(id:, record:)
+      # (UPDATE). Returns the updated Record. Raises Engram::Error when the scoped
+      # record does not exist or the replacement would move it to another scope.
+      def update(scope:, id:, record:)
         raise NotImplementedError, "#{self.class} must implement #update"
       end
 
-      # Remove a memory by id. Used by consolidation (FORGET).
-      def delete(id:)
+      # Remove a memory by id. Used by consolidation (FORGET). Returns the number
+      # of affected rows: 1 when deleted, 0 when the scoped record does not exist.
+      def delete(scope:, id:)
         raise NotImplementedError, "#{self.class} must implement #delete"
       end
 
       # Update the last-accessed timestamp of a memory. Used by recency-aware recall.
-      def touch(id:, at: Time.now)
+      # Returns the number of affected rows: 1 when touched, 0 when the scoped record
+      # does not exist.
+      def touch(scope:, id:, at: Time.now)
         raise NotImplementedError, "#{self.class} must implement #touch"
       end
     end

data/lib/engram/ports/processed_turns.rb

@@ -2,18 +2,26 @@
 
 module Engram
   module Ports
-    # Contract for remembering which turns have already been observed, so observation is
-    # idempotent across retries and accidental double-calls.
+    # Scoped lifecycle for observation idempotency. Implementations suppress concurrent claims
+    # while a lease is live. Lease expiry may permit overlap; this contract does not provide
+    # token ownership, fencing, or exactly-once execution.
+    # #claim returns a truthy opaque token when the caller may proceed, or nil when suppressed.
     # Implementations: Adapters::InMemoryProcessedTurns, Rails::CacheProcessedTurns.
     module ProcessedTurns
-      # Has this idempotency key already been processed?
-      def seen?(key)
-        raise NotImplementedError, "#{self.class} must implement #seen?"
+      def claim(scope:, key:)
+        raise NotImplementedError, "#{self.class} must implement #claim"
       end
 
-      # Mark this idempotency key as processed.
-      def record(key)
-        raise NotImplementedError, "#{self.class} must implement #record"
+      def complete(scope:, key:, claim:)
+        raise NotImplementedError, "#{self.class} must implement #complete"
+      end
+
+      def release(scope:, key:, claim:)
+        raise NotImplementedError, "#{self.class} must implement #release"
+      end
+
+      def completed?(scope:, key:)
+        raise NotImplementedError, "#{self.class} must implement #completed?"
       end
     end
   end