engram 0.3.0 → 0.5.0

data/lib/engram/instrumentation.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Engram
+  # Optional ActiveSupport::Notifications integration.
+  #
+  # Engram core remains dependency-free: when ActiveSupport is not loaded, instrumentation
+  # is a no-op around the supplied block.
+  module Instrumentation
+    module_function
+
+    def instrument(event, payload = {})
+      started_at = monotonic_time
+
+      unless notifications?
+        return yield if block_given?
+        return nil
+      end
+
+      ActiveSupport::Notifications.instrument("#{event}.engram", payload) do
+        yield if block_given?
+      ensure
+        payload[:duration_ms] = elapsed_ms(started_at)
+      end
+    end
+
+    def payload(scope: nil, store: nil, **attributes)
+      attributes = attributes.compact
+      attributes[:store_adapter] = adapter_name(store) if store
+      scope_identifier = scope_identifier(scope)
+      attributes[:scope_identifier] = scope_identifier if scope_identifier
+      attributes
+    end
+
+    def notifications?
+      defined?(ActiveSupport::Notifications) && ActiveSupport::Notifications.respond_to?(:instrument)
+    end
+
+    def adapter_name(adapter)
+      adapter.class.name || adapter.class.to_s
+    end
+
+    def scope_identifier(scope)
+      formatter = Engram.config.instrumentation_scope_identifier
+      return nil unless formatter
+
+      formatter.respond_to?(:call) ? formatter.call(scope) : scope.to_s
+    end
+
+    def elapsed_ms(started_at)
+      ((monotonic_time - started_at) * 1000).round(1)
+    end
+
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/engram/memory.rb

@@ -12,21 +12,25 @@ module Engram
       @embedder = embedder
     end
 
-    # Persist a fact. (In v0.2 this is mostly done for you via extract/consolidate.)
-    def add(content, kind: :semantic, importance: 1.0, metadata: {})
-      record = Record.new(
-        content: content,
-        scope: scope,
-        embedding: @embedder.embed(content),
-        kind: kind,
-        importance: importance,
-        metadata: metadata
-      )
-      @store.add(record)
+    # Persist a memory record of the given kind. Returns nil when the configured
+    # persistence policy rejects the record.
+    def add(content, kind: :fact, importance: 1.0, metadata: {})
+      Engram::Instrumentation.instrument("add", Engram::Instrumentation.payload(scope: scope, store: @store, kind: kind)) do
+        embedding = @embedder.embed(content)
+        record = Record.new(
+          content: content,
+          scope: scope,
+          embedding: embedding,
+          kind: kind,
+          importance: importance,
+          metadata: metadata
+        )
+        persist(record)
+      end
     end
 
     # Return the most relevant memories for a query.
-    def recall(query, limit: Engram.config.default_limit)
+    def recall(query, limit: Engram.config.default_limit, kinds: nil)
       UseCases::Recall.new(
         store: @store,
         embedder: @embedder,
@@ -34,12 +38,12 @@ module Engram
         recency_weight: Engram.config.recency_weight,
         recency_halflife: Engram.config.recency_halflife,
         touch: Engram.config.touch_on_recall
-      ).call(query, scope: scope, limit: limit)
+      ).call(query, scope: scope, limit: limit, kinds: kinds)
     end
 
     # Recall, then inject into a prompt string.
-    def inject_into(prompt, query:, limit: Engram.config.default_limit)
-      memories = recall(query, limit: limit)
+    def inject_into(prompt, query:, limit: Engram.config.default_limit, kinds: nil)
+      memories = recall(query, limit: limit, kinds: kinds)
       UseCases::Inject.new.call(prompt: prompt, memories: memories)
     end
 
@@ -55,7 +59,8 @@ module Engram
         store: @store,
         extractor: build_extractor(completion),
         consolidator: build_consolidator(completion),
-        processed_turns: Engram.config.processed_turns
+        processed_turns: Engram.config.processed_turns,
+        embedder: @embedder
       ).call(
         messages: messages,
         scope: scope,
@@ -69,13 +74,29 @@ module Engram
         raise Engram::Error, "observe_later needs ActiveJob (Rails). Use #observe outside Rails."
       end
 
-      Engram::ObserveJob.perform_later(scope, messages)
+      Engram::Instrumentation.instrument(
+        "observe_later",
+        Engram::Instrumentation.payload(scope: scope, store: @store, message_count: messages.size)
+      ) do
+        Engram::ObserveJob.perform_later(scope, messages)
+      end
     end
 
     def all
       @store.all(scope: scope)
     end
 
+    # Recompute embeddings (and embedding metadata) for memories in the scope.
+    # When `stale_only` is true, only records whose current metadata does not match the
+    # active embedder are rebuilt.
+    def rebuild_embeddings(stale_only: true, batch_size: 100)
+      UseCases::RebuildEmbeddings.new(store: @store, embedder: @embedder).call(
+        scope: scope,
+        stale_only: stale_only,
+        batch_size: batch_size
+      )
+    end
+
     # Prune stale memories. `older_than` is a duration in seconds; `min_importance` keeps
     # memories at or above that importance even when old. Returns the forgotten records.
     def forget_stale(older_than:, min_importance: Float::INFINITY)
@@ -85,6 +106,10 @@ module Engram
 
     private
 
+    def persist(record)
+      Persistence.new(store: @store, embedder: @embedder).add(record)
+    end
+
     def build_extractor(completion)
       Extractors::LLMExtractor.new(
         completion: completion,

data/lib/engram/memory_kind.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Engram
+  # Canonical memory categories used by extraction, recall, and policy.
+  module MemoryKind
+    VALID = %i[fact preference instruction episodic].freeze
+    LEGACY_ALIASES = {semantic: :fact}.freeze
+
+    module_function
+
+    def normalize(kind)
+      normalized = kind.to_s.strip.downcase.to_sym
+      normalized = LEGACY_ALIASES.fetch(normalized, normalized)
+      return normalized if VALID.include?(normalized)
+
+      raise ArgumentError, "unknown memory kind #{kind.inspect}; expected one of #{VALID.join(", ")}"
+    end
+  end
+end

data/lib/engram/persistence.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Engram
+  # Applies persistence hooks and policy consistently before writing records.
+  class Persistence
+    def initialize(store:, embedder:, before_persist: Engram.config.before_persist,
+      persistence_policy: Engram.config.persistence_policy)
+      @store = store
+      @embedder = embedder
+      @before_persist = before_persist
+      @persistence_policy = persistence_policy
+    end
+
+    def add(record, scope: record.scope)
+      record = prepare(record)
+      if record && record.scope != scope
+        raise Engram::Error, "cannot move memory across scopes"
+      end
+      @store.add(record) if record
+    end
+
+    def update(scope:, id:, record:)
+      record = prepare(record)
+      @store.update(scope: scope, id: id, record: record) if record
+    end
+
+    private
+
+    def prepare(record)
+      original_content = record.content
+      record = @before_persist.call(record) if @before_persist
+      record = @persistence_policy.call(record) if record && @persistence_policy
+      if record && record.content != original_content
+        record = record.with(embedding: @embedder.embed(record.content))
+      end
+      record ? Engram::EmbeddingMetadata.attach(record, embedder: @embedder) : nil
+    end
+  end
+end

data/lib/engram/persistence_policy.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Engram
+  # Default gate applied before memories are persisted. It keeps obvious secrets and
+  # transient task-progress updates out of durable memory, and can redact caller-supplied
+  # denylist patterns before storage.
+  class PersistencePolicy
+    SECRET_PATTERNS = [
+      /\b(?:api[_ -]?key|token|secret|password)\b\s*(?:is|=|:)\s+(?=\S*[0-9_-])\S{8,}/i,
+      /\bsk-[A-Za-z0-9_-]{6,}\b/,
+      /\b(?:ghp|github_pat)_[A-Za-z0-9_]{10,}\b/
+    ].freeze
+
+    TRANSIENT_PATTERNS = [
+      /\b(?:fixed|resolved|done|completed|finished)\b.*\b(?:today|now|this session)\b/i,
+      /\b(?:today|now|this session)\b.*\b(?:fixed|resolved|done|completed|finished)\b/i
+    ].freeze
+
+    def initialize(denylist_patterns: [])
+      @denylist_patterns = denylist_patterns
+    end
+
+    def call(record)
+      return nil if reject?(record.content)
+
+      redact(record)
+    end
+
+    private
+
+    def reject?(content)
+      SECRET_PATTERNS.any? { |pattern| content.match?(pattern) } ||
+        TRANSIENT_PATTERNS.any? { |pattern| content.match?(pattern) }
+    end
+
+    def redact(record)
+      redacted = @denylist_patterns.reduce(record.content) do |content, pattern|
+        content.gsub(pattern, "[REDACTED]")
+      end
+      return record if redacted == record.content
+
+      record.with(content: redacted)
+    end
+  end
+end

data/lib/engram/ports/memory_store.rb

@@ -11,29 +11,37 @@ module Engram
       end
 
       # Return up to `limit` Records in `scope` nearest to `embedding`,
-      # ordered most-relevant first.
-      def search(embedding:, scope:, limit:)
+      # ordered most-relevant first. When `kinds` is provided, only records with
+      # those canonical memory kinds are eligible.
+      def search(embedding:, scope:, limit:, kinds: nil, embedding_metadata: nil)
         raise NotImplementedError, "#{self.class} must implement #search"
       end
 
       # All Records for a scope (mostly for inspection/tests).
-      def all(scope:)
+      # Supports optional `limit` and `offset` for batching large sweeps.
+      # Returned records are sorted in stable `id` order when batching is used.
+      # Use `after_id` for keyset pagination.
+      def all(scope:, limit: nil, offset: 0, after_id: nil)
         raise NotImplementedError, "#{self.class} must implement #all"
       end
 
       # Replace the content/embedding of an existing memory. Used by consolidation
-      # (UPDATE). Returns the updated Record.
-      def update(id:, record:)
+      # (UPDATE). Returns the updated Record. Raises Engram::Error when the scoped
+      # record does not exist or the replacement would move it to another scope.
+      def update(scope:, id:, record:)
         raise NotImplementedError, "#{self.class} must implement #update"
       end
 
-      # Remove a memory by id. Used by consolidation (FORGET).
-      def delete(id:)
+      # Remove a memory by id. Used by consolidation (FORGET). Returns the number
+      # of affected rows: 1 when deleted, 0 when the scoped record does not exist.
+      def delete(scope:, id:)
         raise NotImplementedError, "#{self.class} must implement #delete"
       end
 
       # Update the last-accessed timestamp of a memory. Used by recency-aware recall.
-      def touch(id:, at: Time.now)
+      # Returns the number of affected rows: 1 when touched, 0 when the scoped record
+      # does not exist.
+      def touch(scope:, id:, at: Time.now)
         raise NotImplementedError, "#{self.class} must implement #touch"
       end
     end

data/lib/engram/ports/processed_turns.rb

@@ -2,18 +2,26 @@
 
 module Engram
   module Ports
-    # Contract for remembering which turns have already been observed, so observation is
-    # idempotent across retries and accidental double-calls.
+    # Scoped lifecycle for observation idempotency. Implementations suppress concurrent claims
+    # while a lease is live. Lease expiry may permit overlap; this contract does not provide
+    # token ownership, fencing, or exactly-once execution.
+    # #claim returns a truthy opaque token when the caller may proceed, or nil when suppressed.
     # Implementations: Adapters::InMemoryProcessedTurns, Rails::CacheProcessedTurns.
     module ProcessedTurns
-      # Has this idempotency key already been processed?
-      def seen?(key)
-        raise NotImplementedError, "#{self.class} must implement #seen?"
+      def claim(scope:, key:)
+        raise NotImplementedError, "#{self.class} must implement #claim"
       end
 
-      # Mark this idempotency key as processed.
-      def record(key)
-        raise NotImplementedError, "#{self.class} must implement #record"
+      def complete(scope:, key:, claim:)
+        raise NotImplementedError, "#{self.class} must implement #complete"
+      end
+
+      def release(scope:, key:, claim:)
+        raise NotImplementedError, "#{self.class} must implement #release"
+      end
+
+      def completed?(scope:, key:)
+        raise NotImplementedError, "#{self.class} must implement #completed?"
       end
     end
   end

data/lib/engram/provenance.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+module Engram
+  # Provider-neutral source provenance stored in Record metadata.
+  #
+  # Offsets are zero-based Unicode codepoint indexes into one host-owned source
+  # message. The end offset is exclusive. Engram stores references and spans,
+  # never the source text itself.
+  class Provenance
+    RESERVED_KEY = "_engram"
+    METADATA_KEY = "provenance"
+    SCHEMA_VERSION = 1
+    ALIGNMENTS = %i[exact normalized inferred ungrounded].freeze
+
+    class Span
+      OFFSET_UNIT = "unicode_codepoint"
+
+      attr_reader :start_offset, :end_offset, :offset_unit
+
+      def initialize(start_offset:, end_offset:, offset_unit: OFFSET_UNIT)
+        unless start_offset.is_a?(Integer) && end_offset.is_a?(Integer) &&
+            start_offset >= 0 && end_offset > start_offset
+          raise ArgumentError, "span offsets must be non-negative integers with end_offset > start_offset"
+        end
+        raise ArgumentError, "unsupported offset unit" unless offset_unit.to_s == OFFSET_UNIT
+
+        @start_offset = start_offset
+        @end_offset = end_offset
+        @offset_unit = OFFSET_UNIT
+        freeze
+      end
+
+      def end_exclusive? = true
+
+      def ==(other)
+        other.is_a?(self.class) && to_h == other.to_h
+      end
+      alias_method :eql?, :==
+
+      def hash = to_h.hash
+
+      def to_h
+        {"start_offset" => start_offset, "end_offset" => end_offset, "offset_unit" => offset_unit}
+      end
+    end
+
+    class Source
+      attr_reader :source_id, :source_type, :message_index, :role, :spans, :alignment
+
+      def initialize(source_id:, source_type:, message_index:, role:, spans:, alignment:)
+        raise ArgumentError, "source_id is required" if source_id.to_s.strip.empty?
+        raise ArgumentError, "source_type is required" if source_type.to_s.strip.empty?
+        unless message_index.is_a?(Integer) && !message_index.negative?
+          raise ArgumentError, "message_index must be a non-negative integer"
+        end
+        raise ArgumentError, "role is required" if role.to_s.strip.empty?
+
+        unless alignment.is_a?(String) || alignment.is_a?(Symbol)
+          raise ArgumentError, "unknown alignment #{alignment.inspect}"
+        end
+        normalized_alignment = alignment.to_sym
+        raise ArgumentError, "unknown alignment #{alignment.inspect}" unless ALIGNMENTS.include?(normalized_alignment)
+
+        @source_id = source_id.to_s.dup.freeze
+        @source_type = source_type.to_s.dup.freeze
+        @message_index = message_index
+        @role = role.to_s.dup.freeze
+        raise ArgumentError, "spans must be an array" unless spans.is_a?(Array)
+        raise ArgumentError, "spans must contain at least one span" if spans.empty?
+        @spans = spans.map do |span|
+          raise ArgumentError, "spans must contain Provenance::Span values" unless span.is_a?(Span)
+          span
+        end.freeze
+        @alignment = normalized_alignment
+        freeze
+      end
+
+      def ==(other)
+        other.is_a?(self.class) && to_h == other.to_h
+      end
+      alias_method :eql?, :==
+
+      def hash = to_h.hash
+
+      def to_h
+        data = {
+          "source_id" => source_id,
+          "source_type" => source_type,
+          "spans" => spans.map(&:to_h),
+          "alignment" => alignment.to_s
+        }
+        data["message_index"] = message_index
+        data["role"] = role
+        data
+      end
+    end
+
+    class Extractor
+      attr_reader :name, :provider, :model
+
+      def initialize(name:, model:, provider: nil)
+        raise ArgumentError, "extractor name is required" if name.to_s.strip.empty?
+        raise ArgumentError, "extractor model is required" if model.to_s.strip.empty?
+
+        @name = name.to_s.dup.freeze
+        @provider = provider&.to_s&.dup&.freeze
+        @model = model.to_s.dup.freeze
+        freeze
+      end
+
+      def ==(other)
+        other.is_a?(self.class) && to_h == other.to_h
+      end
+      alias_method :eql?, :==
+
+      def hash = to_h.hash
+
+      def to_h
+        {"name" => name, "model" => model}.tap do |data|
+          data["provider"] = provider if provider
+        end
+      end
+    end
+
+    attr_reader :sources, :extractor, :confidence
+
+    def initialize(sources:, extractor:, confidence:)
+      raise ArgumentError, "sources must be an array" unless sources.is_a?(Array)
+      raise ArgumentError, "sources must contain at least one source" if sources.empty?
+      @sources = sources.map do |source|
+        raise ArgumentError, "sources must contain Provenance::Source values" unless source.is_a?(Source)
+        source
+      end.freeze
+      raise ArgumentError, "extractor must be a Provenance::Extractor" unless extractor.is_a?(Extractor)
+      valid_confidence = (confidence.is_a?(Integer) || confidence.is_a?(Float)) && confidence.between?(0, 1)
+      unless valid_confidence
+        raise ArgumentError, "confidence must be an Integer or Float between 0 and 1"
+      end
+
+      @extractor = extractor
+      @confidence = confidence
+      freeze
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && to_h == other.to_h
+    end
+    alias_method :eql?, :==
+
+    def hash = to_h.hash
+
+    def to_h
+      {
+        "version" => SCHEMA_VERSION,
+        "sources" => sources.map(&:to_h),
+        "extractor" => extractor.to_h,
+        "confidence" => confidence
+      }
+    end
+
+    class << self
+      def attach(metadata, provenance)
+        raise ArgumentError, "provenance must be a Provenance value" unless provenance.is_a?(self)
+
+        Engram::ReservedMetadata.attach(metadata, METADATA_KEY, provenance.to_h)
+      end
+
+      def extract(metadata)
+        metadata = deep_stringify(metadata || {})
+        return nil unless metadata.is_a?(Hash)
+
+        reserved = metadata[RESERVED_KEY]
+        return nil unless reserved.is_a?(Hash)
+
+        data = reserved[METADATA_KEY]
+        return nil unless data.is_a?(Hash) && data["version"] == SCHEMA_VERSION
+
+        from_h(data)
+      end
+
+      private
+
+      def from_h(data)
+        sources = provenance_array(data["sources"], "_engram.provenance.sources")
+        extractor_data = provenance_hash(data["extractor"], "_engram.provenance.extractor")
+
+        parsed_sources = sources.each_with_index.map do |source, source_index|
+          source_path = "_engram.provenance.sources[#{source_index}]"
+          source = provenance_hash(source, source_path)
+          spans = provenance_array(source["spans"], "#{source_path}.spans")
+          parsed_spans = spans.each_with_index.map do |span, span_index|
+            span_path = "#{source_path}.spans[#{span_index}]"
+            span = provenance_hash(span, span_path)
+            build_provenance_value(span_path) do
+              Span.new(
+                start_offset: span["start_offset"],
+                end_offset: span["end_offset"],
+                offset_unit: span["offset_unit"]
+              )
+            end
+          end
+
+          build_provenance_value(source_path) do
+            Source.new(
+              source_id: source["source_id"],
+              source_type: source["source_type"],
+              message_index: source["message_index"],
+              role: source["role"],
+              spans: parsed_spans,
+              alignment: source["alignment"]
+            )
+          end
+        end
+
+        extractor = build_provenance_value("_engram.provenance.extractor") do
+          Extractor.new(**symbolize_extractor(extractor_data))
+        end
+        build_provenance_value("_engram.provenance") do
+          new(sources: parsed_sources, extractor: extractor, confidence: data["confidence"])
+        end
+      end
+
+      def provenance_hash(value, path)
+        return value if value.is_a?(Hash)
+
+        raise Engram::Error, "malformed provenance at #{path}: expected an object"
+      end
+
+      def provenance_array(value, path)
+        return value if value.is_a?(Array)
+
+        raise Engram::Error, "malformed provenance at #{path}: expected an array"
+      end
+
+      def build_provenance_value(path)
+        yield
+      rescue ArgumentError, TypeError, KeyError, NoMethodError => error
+        raise Engram::Error, "malformed provenance at #{path}: #{error.message}"
+      end
+
+      def symbolize_extractor(data)
+        {name: data["name"], provider: data["provider"], model: data["model"]}
+      end
+
+      def deep_stringify(value)
+        case value
+        when Hash
+          value.each_with_object({}) { |(key, nested), out| out[key.to_s] = deep_stringify(nested) }
+        when Array
+          value.map { |nested| deep_stringify(nested) }
+        else
+          value
+        end
+      end
+    end
+  end
+end