engram 0.3.0 → 0.4.0

data/lib/engram/configuration.rb

@@ -7,7 +7,8 @@ module Engram
   class Configuration
     attr_accessor :store, :embedder, :completion, :default_limit,
       :consolidator, :extraction_min_confidence, :processed_turns,
-      :importance_weight, :recency_weight, :recency_halflife, :touch_on_recall
+      :importance_weight, :recency_weight, :recency_halflife, :touch_on_recall,
+      :persistence_policy, :before_persist, :instrumentation_scope_identifier
 
     def initialize
       @store = Adapters::InMemoryStore.new
@@ -17,6 +18,9 @@ module Engram
       @consolidator = :heuristic # :heuristic (deterministic) or :llm (LLM-as-judge)
       @extraction_min_confidence = 0.5
       @processed_turns = Adapters::InMemoryProcessedTurns.new # idempotency for observe
+      @persistence_policy = PersistencePolicy.new
+      @before_persist = nil
+      @instrumentation_scope_identifier = nil
 
       # Recall ranking. With both weights at 0.0, recall is plain similarity search.
       @importance_weight = 0.0

data/lib/engram/consolidators/llm_consolidator.rb

@@ -21,20 +21,25 @@ module Engram
         Return one decision per candidate, referencing it by its index.
       PROMPT
 
+      # Shaped for OpenAI strict structured outputs: every object sets
+      # additionalProperties: false and lists all of its properties in `required`. target_id
+      # is nullable because add/noop decisions reference no existing memory.
       SCHEMA = {
         type: "object",
+        additionalProperties: false,
         properties: {
           decisions: {
             type: "array",
             items: {
               type: "object",
+              additionalProperties: false,
               properties: {
                 index: {type: "integer"},
                 action: {type: "string", enum: %w[add update forget noop]},
-                target_id: {type: %w[integer string null]},
+                target_id: {type: %w[integer null]},
                 reason: {type: "string"}
               },
-              required: %w[index action]
+              required: %w[index action target_id reason]
             }
           }
         },

data/lib/engram/extractors/llm_extractor.rb

@@ -12,24 +12,32 @@ module Engram
         - Only stable facts about the user (preferences, attributes, decisions, history).
         - Ignore ephemeral chit-chat, questions, and the assistant's own messages.
         - Normalize each fact to a terse third-person statement (e.g. "User is on the Pro plan").
+        - Classify kind as fact, preference, instruction, or episodic.
+        - Do not extract secrets, API keys, passwords, tokens, or transient task progress.
         - Set confidence in [0,1]; importance in [0,1].
         Return an empty list if there is nothing worth remembering.
       PROMPT
 
+      # Shaped for OpenAI strict structured outputs: every object sets
+      # additionalProperties: false and lists all of its properties in `required`. The
+      # extractor still defends against missing/empty fields, so requiring them here only
+      # constrains the model's output, it does not change downstream behaviour.
       SCHEMA = {
         type: "object",
+        additionalProperties: false,
         properties: {
           facts: {
             type: "array",
             items: {
               type: "object",
+              additionalProperties: false,
               properties: {
                 content: {type: "string"},
-                kind: {type: "string", enum: %w[semantic episodic preference]},
+                kind: {type: "string", enum: %w[fact preference instruction episodic semantic]},
                 importance: {type: "number"},
                 confidence: {type: "number"}
               },
-              required: %w[content]
+              required: %w[content kind importance confidence]
             }
           }
         },
@@ -53,8 +61,9 @@ module Engram
           Engram::Record.new(
             content: content,
             scope: scope,
-            kind: (fact["kind"] || "semantic").to_sym,
+            kind: fact["kind"] || "fact",
             importance: (fact["importance"] || 1.0).to_f,
+            metadata: {confidence: (fact["confidence"] || 1.0).to_f},
             embedding: @embedder.embed(content)
           )
         end

data/lib/engram/instrumentation.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Engram
+  # Optional ActiveSupport::Notifications integration.
+  #
+  # Engram core remains dependency-free: when ActiveSupport is not loaded, instrumentation
+  # is a no-op around the supplied block.
+  module Instrumentation
+    module_function
+
+    def instrument(event, payload = {})
+      started_at = monotonic_time
+
+      unless notifications?
+        return yield if block_given?
+        return nil
+      end
+
+      ActiveSupport::Notifications.instrument("#{event}.engram", payload) do
+        yield if block_given?
+      ensure
+        payload[:duration_ms] = elapsed_ms(started_at)
+      end
+    end
+
+    def payload(scope: nil, store: nil, **attributes)
+      attributes = attributes.compact
+      attributes[:store_adapter] = adapter_name(store) if store
+      scope_identifier = scope_identifier(scope)
+      attributes[:scope_identifier] = scope_identifier if scope_identifier
+      attributes
+    end
+
+    def notifications?
+      defined?(ActiveSupport::Notifications) && ActiveSupport::Notifications.respond_to?(:instrument)
+    end
+
+    def adapter_name(adapter)
+      adapter.class.name || adapter.class.to_s
+    end
+
+    def scope_identifier(scope)
+      formatter = Engram.config.instrumentation_scope_identifier
+      return nil unless formatter
+
+      formatter.respond_to?(:call) ? formatter.call(scope) : scope.to_s
+    end
+
+    def elapsed_ms(started_at)
+      ((monotonic_time - started_at) * 1000).round(1)
+    end
+
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/engram/memory.rb

@@ -12,21 +12,24 @@ module Engram
       @embedder = embedder
     end
 
-    # Persist a fact. (In v0.2 this is mostly done for you via extract/consolidate.)
-    def add(content, kind: :semantic, importance: 1.0, metadata: {})
-      record = Record.new(
-        content: content,
-        scope: scope,
-        embedding: @embedder.embed(content),
-        kind: kind,
-        importance: importance,
-        metadata: metadata
-      )
-      @store.add(record)
+    # Persist a memory record of the given kind. Returns nil when the configured
+    # persistence policy rejects the record.
+    def add(content, kind: :fact, importance: 1.0, metadata: {})
+      Engram::Instrumentation.instrument("add", Engram::Instrumentation.payload(scope: scope, store: @store, kind: kind)) do
+        record = Record.new(
+          content: content,
+          scope: scope,
+          embedding: @embedder.embed(content),
+          kind: kind,
+          importance: importance,
+          metadata: metadata
+        )
+        persist(record)
+      end
     end
 
     # Return the most relevant memories for a query.
-    def recall(query, limit: Engram.config.default_limit)
+    def recall(query, limit: Engram.config.default_limit, kinds: nil)
       UseCases::Recall.new(
         store: @store,
         embedder: @embedder,
@@ -34,12 +37,12 @@ module Engram
         recency_weight: Engram.config.recency_weight,
         recency_halflife: Engram.config.recency_halflife,
         touch: Engram.config.touch_on_recall
-      ).call(query, scope: scope, limit: limit)
+      ).call(query, scope: scope, limit: limit, kinds: kinds)
     end
 
     # Recall, then inject into a prompt string.
-    def inject_into(prompt, query:, limit: Engram.config.default_limit)
-      memories = recall(query, limit: limit)
+    def inject_into(prompt, query:, limit: Engram.config.default_limit, kinds: nil)
+      memories = recall(query, limit: limit, kinds: kinds)
       UseCases::Inject.new.call(prompt: prompt, memories: memories)
     end
 
@@ -55,7 +58,8 @@ module Engram
         store: @store,
         extractor: build_extractor(completion),
         consolidator: build_consolidator(completion),
-        processed_turns: Engram.config.processed_turns
+        processed_turns: Engram.config.processed_turns,
+        embedder: @embedder
       ).call(
         messages: messages,
         scope: scope,
@@ -69,7 +73,12 @@ module Engram
         raise Engram::Error, "observe_later needs ActiveJob (Rails). Use #observe outside Rails."
       end
 
-      Engram::ObserveJob.perform_later(scope, messages)
+      Engram::Instrumentation.instrument(
+        "observe_later",
+        Engram::Instrumentation.payload(scope: scope, store: @store, message_count: messages.size)
+      ) do
+        Engram::ObserveJob.perform_later(scope, messages)
+      end
     end
 
     def all
@@ -85,6 +94,10 @@ module Engram
 
     private
 
+    def persist(record)
+      Persistence.new(store: @store, embedder: @embedder).add(record)
+    end
+
     def build_extractor(completion)
       Extractors::LLMExtractor.new(
         completion: completion,

data/lib/engram/memory_kind.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Engram
+  # Canonical memory categories used by extraction, recall, and policy.
+  module MemoryKind
+    VALID = %i[fact preference instruction episodic].freeze
+    LEGACY_ALIASES = {semantic: :fact}.freeze
+
+    module_function
+
+    def normalize(kind)
+      normalized = kind.to_s.strip.downcase.to_sym
+      normalized = LEGACY_ALIASES.fetch(normalized, normalized)
+      return normalized if VALID.include?(normalized)
+
+      raise ArgumentError, "unknown memory kind #{kind.inspect}; expected one of #{VALID.join(", ")}"
+    end
+  end
+end

data/lib/engram/persistence.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Engram
+  # Applies persistence hooks and policy consistently before writing records.
+  class Persistence
+    def initialize(store:, embedder:, before_persist: Engram.config.before_persist,
+      persistence_policy: Engram.config.persistence_policy)
+      @store = store
+      @embedder = embedder
+      @before_persist = before_persist
+      @persistence_policy = persistence_policy
+    end
+
+    def add(record)
+      record = prepare(record)
+      @store.add(record) if record
+    end
+
+    def update(id:, record:)
+      record = prepare(record)
+      @store.update(id: id, record: record) if record
+    end
+
+    private
+
+    def prepare(record)
+      original_content = record.content
+      record = @before_persist.call(record) if @before_persist
+      record = @persistence_policy.call(record) if record && @persistence_policy
+      record = record.with(embedding: @embedder.embed(record.content)) if record && record.content != original_content
+      record
+    end
+  end
+end

data/lib/engram/persistence_policy.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Engram
+  # Default gate applied before memories are persisted. It keeps obvious secrets and
+  # transient task-progress updates out of durable memory, and can redact caller-supplied
+  # denylist patterns before storage.
+  class PersistencePolicy
+    SECRET_PATTERNS = [
+      /\b(?:api[_ -]?key|token|secret|password)\b\s*(?:is|=|:)\s+(?=\S*[0-9_-])\S{8,}/i,
+      /\bsk-[A-Za-z0-9_-]{6,}\b/,
+      /\b(?:ghp|github_pat)_[A-Za-z0-9_]{10,}\b/
+    ].freeze
+
+    TRANSIENT_PATTERNS = [
+      /\b(?:fixed|resolved|done|completed|finished)\b.*\b(?:today|now|this session)\b/i,
+      /\b(?:today|now|this session)\b.*\b(?:fixed|resolved|done|completed|finished)\b/i
+    ].freeze
+
+    def initialize(denylist_patterns: [])
+      @denylist_patterns = denylist_patterns
+    end
+
+    def call(record)
+      return nil if reject?(record.content)
+
+      redact(record)
+    end
+
+    private
+
+    def reject?(content)
+      SECRET_PATTERNS.any? { |pattern| content.match?(pattern) } ||
+        TRANSIENT_PATTERNS.any? { |pattern| content.match?(pattern) }
+    end
+
+    def redact(record)
+      redacted = @denylist_patterns.reduce(record.content) do |content, pattern|
+        content.gsub(pattern, "[REDACTED]")
+      end
+      return record if redacted == record.content
+
+      record.with(content: redacted)
+    end
+  end
+end

data/lib/engram/ports/memory_store.rb

@@ -11,8 +11,9 @@ module Engram
       end
 
       # Return up to `limit` Records in `scope` nearest to `embedding`,
-      # ordered most-relevant first.
-      def search(embedding:, scope:, limit:)
+      # ordered most-relevant first. When `kinds` is provided, only records with
+      # those canonical memory kinds are eligible.
+      def search(embedding:, scope:, limit:, kinds: nil)
         raise NotImplementedError, "#{self.class} must implement #search"
       end
 

data/lib/engram/record.rb

@@ -5,25 +5,30 @@ module Engram
   #
   # `id` is assigned by the store on persistence (nil until then); consolidation uses it
   # to target UPDATE/FORGET. `scope` namespaces memories to an owner (e.g. "user:42").
-  # `kind` is a memory type (semantic / episodic / preference).
+  # `kind` is a memory type (fact / preference / instruction / episodic). The legacy
+  # `semantic` kind is normalized to `fact` for compatibility with pre-1.0 records.
   class Record
     attr_accessor :id, :last_accessed_at
     attr_reader :content, :embedding, :scope, :kind, :importance, :metadata,
       :created_at
 
-    def initialize(content:, scope:, id: nil, embedding: nil, kind: :semantic,
+    def initialize(content:, scope:, id: nil, embedding: nil, kind: :fact,
       importance: 1.0, metadata: {}, created_at: nil, last_accessed_at: nil)
       @id = id
       @content = content
       @scope = scope
       @embedding = embedding
-      @kind = kind
+      @kind = MemoryKind.normalize(kind)
       @importance = importance
       @metadata = metadata
       @created_at = created_at || Time.now
       @last_accessed_at = last_accessed_at
     end
 
+    def with(**attributes)
+      self.class.new(**to_h.merge(attributes))
+    end
+
     def to_h
       {
         id: id, content: content, scope: scope, embedding: embedding, kind: kind,

data/lib/engram/use_cases/inject.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "cgi"
+
 module Engram
   module UseCases
     # Render recalled memories into a prompt as a clearly delimited block.
@@ -12,10 +14,22 @@ module Engram
 
       # Returns a new prompt string. If there are no memories, the prompt is unchanged.
       def call(prompt:, memories:)
-        return prompt if memories.nil? || memories.empty?
+        payload = {memory_count: memories&.size.to_i}
+        Engram::Instrumentation.instrument("inject", payload) do
+          next prompt if memories.nil? || memories.empty?
+
+          block = memories.map { |memory| render_memory(memory) }.join("\n")
+          "#{prompt}\n\n#{@header}:\n<engram-memories>\n#{block}\n</engram-memories>"
+        end
+      end
+
+      private
+
+      def render_memory(memory)
+        kind = CGI.escapeHTML((memory.kind || :fact).to_s)
+        content = CGI.escapeHTML(memory.content.to_s)
 
-        block = memories.map { |m| "- #{m.content}" }.join("\n")
-        "#{prompt}\n\n#{@header}:\n#{block}"
+        %(<engram-memory kind="#{kind}">#{content}</engram-memory>)
       end
     end
   end

data/lib/engram/use_cases/observe.rb

@@ -9,27 +9,45 @@ module Engram
     # When a ProcessedTurns store and an idempotency_key are provided, a turn that was
     # already processed is skipped (no extraction, no duplicate memories).
     class Observe
-      def initialize(store:, extractor:, consolidator:, processed_turns: nil)
+      def initialize(store:, extractor:, consolidator:, processed_turns: nil, embedder: Engram.config.embedder)
         @store = store
         @extractor = extractor
         @consolidator = consolidator
         @processed_turns = processed_turns
+        @embedder = embedder
       end
 
       # Returns the Array<Decision> that were applied (empty if skipped or nothing found).
       def call(messages:, scope:, idempotency_key: nil)
-        return [] if already_processed?(idempotency_key)
+        payload = Engram::Instrumentation.payload(
+          scope: scope,
+          store: @store,
+          message_count: messages.size,
+          idempotency_key_present: !idempotency_key.nil?
+        )
+        Engram::Instrumentation.instrument("observe", payload) do
+          if already_processed?(idempotency_key)
+            payload[:skipped] = true
+            payload[:candidate_count] = 0
+            payload[:decision_count] = 0
+            next []
+          end
 
-        candidates = @extractor.extract(messages: messages, scope: scope)
-        if candidates.empty?
+          candidates = extract(messages: messages, scope: scope)
+          payload[:candidate_count] = candidates.size
+          if candidates.empty?
+            mark_processed(idempotency_key)
+            payload[:decision_count] = 0
+            next []
+          end
+
+          decisions = consolidate(candidates: candidates, scope: scope)
+          applied_decisions = decisions.filter_map { |decision| apply(decision) }
+          payload[:decision_count] = applied_decisions.size
+          payload[:decision_actions] = applied_decisions.map { |decision| decision.action.to_s }
           mark_processed(idempotency_key)
-          return []
+          applied_decisions
         end
-
-        decisions = @consolidator.reconcile_all(candidates: candidates, scope: scope)
-        decisions.each { |decision| apply(decision) }
-        mark_processed(idempotency_key)
-        decisions
       end
 
       private
@@ -42,18 +60,43 @@ module Engram
         @processed_turns.record(key) if key && @processed_turns
       end
 
+      def extract(messages:, scope:)
+        payload = Engram::Instrumentation.payload(scope: scope, store: @store, message_count: messages.size)
+        Engram::Instrumentation.instrument("extract", payload) do
+          candidates = @extractor.extract(messages: messages, scope: scope)
+          payload[:candidate_count] = candidates.size
+          candidates
+        end
+      end
+
+      def consolidate(candidates:, scope:)
+        payload = Engram::Instrumentation.payload(scope: scope, store: @store, candidate_count: candidates.size)
+        Engram::Instrumentation.instrument("consolidate", payload) do
+          decisions = @consolidator.reconcile_all(candidates: candidates, scope: scope)
+          payload[:decision_count] = decisions.size
+          payload[:decision_actions] = decisions.map { |decision| decision.action.to_s }
+          decisions
+        end
+      end
+
       def apply(decision)
         case decision.action
         when :add
-          @store.add(decision.candidate)
+          decision if persistence.add(decision.candidate)
         when :update
-          @store.update(id: decision.target_id, record: decision.candidate) if decision.target_id
+          if decision.target_id && persistence.update(id: decision.target_id, record: decision.candidate)
+            decision
+          end
         when :forget
-          @store.delete(id: decision.target_id) if decision.target_id
+          decision if decision.target_id && @store.delete(id: decision.target_id)
         when :noop
           nil
         end
       end
+
+      def persistence
+        @persistence ||= Persistence.new(store: @store, embedder: @embedder)
+      end
     end
   end
 end

data/lib/engram/use_cases/recall.rb

@@ -24,16 +24,27 @@ module Engram
       end
 
       # Returns Array<Record>, most relevant first.
-      def call(query, scope:, limit: Engram.config.default_limit)
+      def call(query, scope:, limit: Engram.config.default_limit, kinds: nil)
         raise ArgumentError, "query must be a non-empty string" if query.to_s.strip.empty?
 
-        embedding = @embedder.embed(query)
-        pool_limit = reranking? ? limit * @pool_factor : limit
-        pool = @store.search(embedding: embedding, scope: scope, limit: pool_limit)
+        payload = Engram::Instrumentation.payload(
+          scope: scope,
+          store: @store,
+          limit: limit,
+          kinds: Array(kinds).map(&:to_s),
+          reranking: reranking?
+        )
+        Engram::Instrumentation.instrument("recall", payload) do
+          embedding = @embedder.embed(query)
+          pool_limit = reranking? ? limit * @pool_factor : limit
+          pool = @store.search(embedding: embedding, scope: scope, limit: pool_limit, kinds: kinds)
 
-        results = (reranking? ? rerank(pool, embedding) : pool).first(limit)
-        touch(results) if @touch
-        results
+          results = (reranking? ? rerank(pool, embedding) : pool).first(limit)
+          touch(results) if @touch
+          payload[:result_count] = results.size
+          payload[:candidate_count] = pool.size
+          results
+        end
       end
 
       private

data/lib/engram/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Engram
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/engram.rb

@@ -3,9 +3,13 @@
 require_relative "engram/version"
 require_relative "engram/configuration"
 require_relative "engram/math"
+require_relative "engram/memory_kind"
 require_relative "engram/record"
 require_relative "engram/decision"
 require_relative "engram/turn_digest"
+require_relative "engram/persistence_policy"
+require_relative "engram/instrumentation"
+require_relative "engram/persistence"
 
 # Ports (contracts)
 require_relative "engram/ports/memory_store"

data/lib/generators/engram/install_generator.rb

@@ -17,6 +17,10 @@ module Engram
       class_option :dimensions, type: :numeric, default: DEFAULT_DIMENSIONS,
         desc: "Embedding dimensions (match your embedding model)"
 
+      def validate_dimensions
+        validate_dimensions!
+      end
+
       def create_migration_file
         migration_template "create_engram_memories.rb.tt",
           "db/migrate/create_engram_memories.rb"
@@ -40,6 +44,12 @@ module Engram
         options[:dimensions]
       end
 
+      def validate_dimensions!
+        return if dimensions.is_a?(Integer) && dimensions.positive?
+
+        raise ArgumentError, "dimensions must be a positive integer that matches your embedding model"
+      end
+
       def migration_version
         "#{::ActiveRecord::VERSION::MAJOR}.#{::ActiveRecord::VERSION::MINOR}"
       end

data/lib/generators/engram/templates/create_engram_memories.rb.tt

@@ -7,7 +7,7 @@ class CreateEngramMemories < ActiveRecord::Migration[<%= migration_version %>]
     create_table :engram_memories do |t|
       t.string :scope, null: false
       t.text :content, null: false
-      t.string :kind, null: false, default: "semantic"
+      t.string :kind, null: false, default: "fact"
       t.float :importance, null: false, default: 1.0
       t.jsonb :metadata, null: false, default: {}
       t.vector :embedding, limit: <%= dimensions %>
@@ -18,7 +18,14 @@ class CreateEngramMemories < ActiveRecord::Migration[<%= migration_version %>]
 
     add_index :engram_memories, :scope
 
-    # For larger datasets, add an approximate index (requires data present first):
-    # add_index :engram_memories, :embedding, using: :hnsw, opclass: :vector_cosine_ops
+    # For larger datasets, choose one approximate vector index after backfilling data.
+    # HNSW is a strong default for read-heavy recall workloads with frequent inserts.
+    # IVFFlat can be smaller/faster to build, but should be created after enough representative data exists.
+    #
+    # HNSW:
+    #   add_index :engram_memories, :embedding, using: :hnsw, opclass: :vector_cosine_ops
+    # IVFFlat:
+    #   add_index :engram_memories, :embedding, using: :ivfflat, opclass: :vector_cosine_ops
+    # See the Engram README for production index guidance.
   end
 end