llmemory 0.1.17 → 0.2.1

data/lib/llmemory/long_term/graph_based/memory.rb

@@ -6,11 +6,14 @@ require_relative "knowledge_graph"
 require_relative "conflict_resolver"
 require_relative "storage"
 require_relative "../../noise_filter"
+require_relative "../../memory_module"
 
 module Llmemory
   module LongTerm
     module GraphBased
       class Memory
+        include Llmemory::MemoryModule
+
         def initialize(user_id:, storage: nil, vector_store: nil, llm: nil, extractor: nil)
           @user_id = user_id
           @graph_storage = storage || Storages.build
@@ -25,14 +28,95 @@ module Llmemory
           text = Llmemory.configuration.noise_filter_enabled ? NoiseFilter.filter?(conversation_text) : conversation_text.to_s
           return true if text.strip.empty?
 
-          data = @extractor.extract(text) rescue { entities: [], relations: [] }
-          data = { entities: [], relations: [] } unless data.is_a?(Hash)
-          entities = Array(data[:entities] || data["entities"])
-          relations = Array(data[:relations] || data["relations"])
-
+          entities, relations = extract_graph(text)
           return true if entities.empty? && relations.empty?
 
           provenance = Llmemory::Provenance.from_text_fingerprint(text, method: "entity_relation_extraction")
+          ingest(entities, relations, provenance)
+          true
+        end
+
+        def retrieve(query, top_k: 10)
+          results = hybrid_search(query, top_k: top_k)
+          format_as_context(results)
+        end
+
+        def search_candidates(query, user_id: nil, top_k: 20)
+          uid = user_id || @user_id
+          return [] unless uid == @user_id
+          results = hybrid_search(query, top_k: top_k)
+          results.map do |r|
+            {
+              id: r[:id],
+              text: r[:text],
+              timestamp: r[:created_at] || r[:timestamp],
+              score: r[:score] || 1.0,
+              importance: r[:importance]
+            }
+          end
+        end
+
+        attr_reader :user_id
+
+        def storage
+          @graph_storage
+        end
+
+        # Stores a fact produced outside the conversational flow (e.g. a
+        # reflection insight) by extracting entities/relations from `content` and
+        # adding them to the graph, preserving caller-supplied provenance. Lets
+        # the Reflector target graph-based semantic memory.
+        def remember_fact(content:, category: nil, importance: nil, provenance: nil)
+          return nil if content.to_s.strip.empty?
+          entities, relations = extract_graph(content)
+          return nil if entities.empty? && relations.empty?
+          prov = provenance || Llmemory::Provenance.from_text_fingerprint(content, method: "reflection")
+          ingest(entities, relations, prov)
+          true
+        end
+
+        # --- MemoryModule uniform interface ---
+
+        def write(payload, **_meta)
+          memorize(payload)
+        end
+
+        def list(user_id: nil, limit: nil)
+          @graph_storage.list_nodes(user_id || @user_id, limit: limit)
+        end
+
+        def stats(user_id: nil)
+          uid = user_id || @user_id
+          { nodes: @graph_storage.count_nodes(uid), edges: @graph_storage.count_edges(uid) }
+        end
+
+        # Forgets relations by archiving the edges identified by the candidate
+        # ids returned from #read/#search_candidates (edge ids), recording the
+        # removal in the audit log. Edges are soft-archived (archived_at) so they
+        # no longer appear in retrieval; nodes are left in place (a node may still
+        # be referenced by other active edges). Returns the number archived.
+        def forget(ids:, reason: nil)
+          archived = Array(ids).map(&:to_s).select { |edge_id| @kg.archive_edge(edge_id) }
+          forget_log.record(@user_id, memory_type: "graph_based", ids: archived, reason: reason)
+          archived.size
+        end
+
+        private
+
+        def build_vector_store
+          Llmemory::VectorStore.build(source_type: "edge")
+        end
+
+        def extract_graph(text)
+          data = @extractor.extract(text) rescue { entities: [], relations: [] }
+          data = { entities: [], relations: [] } unless data.is_a?(Hash)
+          [Array(data[:entities] || data["entities"]), Array(data[:relations] || data["relations"])]
+        end
+
+        # Adds entities and relations to the graph (nodes, edges, embeddings) with
+        # the given provenance. Shared by memorize (conversation) and
+        # remember_fact (reflection).
+        def ingest(entities, relations, provenance)
           name_to_id = {}
 
           entities.each do |e|
@@ -67,52 +151,12 @@ module Llmemory
             @conflict_resolver.resolve(edge)
             edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: { "provenance" => provenance })
 
-            text = "#{subject} #{predicate} #{object}"
-            embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(text) : nil
+            edge_text = "#{subject} #{predicate} #{object}"
+            embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(edge_text) : nil
             if embedding && @vector_store.respond_to?(:store)
-              @vector_store.store(id: "edge_#{edge_id}", embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
+              @vector_store.store(id: edge_id, embedding: embedding, metadata: { text: edge_text, created_at: Time.now }, user_id: @user_id)
             end
           end
-
-          true
-        end
-
-        def retrieve(query, top_k: 10)
-          results = hybrid_search(query, top_k: top_k)
-          format_as_context(results)
-        end
-
-        def search_candidates(query, user_id: nil, top_k: 20)
-          uid = user_id || @user_id
-          return [] unless uid == @user_id
-          results = hybrid_search(query, top_k: top_k)
-          results.map do |r|
-            {
-              text: r[:text],
-              timestamp: r[:created_at] || r[:timestamp],
-              score: r[:score] || 1.0,
-              importance: r[:importance]
-            }
-          end
-        end
-
-        attr_reader :user_id
-
-        def storage
-          @graph_storage
-        end
-
-        private
-
-        def build_vector_store
-          emb = Llmemory::VectorStore::OpenAIEmbeddings.new
-          store_type = (Llmemory.configuration.long_term_store || :memory).to_s.to_sym
-          if store_type == :active_record || store_type == :activerecord
-            require_relative "../../vector_store/active_record_store"
-            Llmemory::VectorStore::ActiveRecordStore.new(embedding_provider: emb)
-          else
-            Llmemory::VectorStore::MemoryStore.new(embedding_provider: emb)
-          end
         end
 
         def hybrid_search(query, top_k:)
@@ -127,7 +171,7 @@ module Llmemory
           out = vector_results.map do |v|
             id = v[:id] || v["id"]
             meta = v[:metadata] || v["metadata"] || {}
-            { text: meta["text"] || meta[:text] || id.to_s, score: v[:score] || v["score"] || 1.0, created_at: meta["created_at"] || meta[:created_at] }
+            { id: id, text: meta["text"] || meta[:text] || id.to_s, score: v[:score] || v["score"] || 1.0, created_at: meta["created_at"] || meta[:created_at] }
           end
 
           node_ids = out.flat_map { |r| extract_node_ids_from_text(r[:text]) }.compact.uniq
@@ -139,7 +183,7 @@ module Llmemory
               subj = @kg.find_node_by_id(e.subject_id)
               obj = @kg.find_node_by_id(e.target_id)
               edge_text = "#{subj&.name} #{e.predicate} #{obj&.name}"
-              out << { text: edge_text, score: 0.85, created_at: e.created_at } unless out.any? { |o| o[:text] == edge_text }
+              out << { id: e.id, text: edge_text, score: 0.85, created_at: e.created_at } unless out.any? { |o| o[:text] == edge_text }
             end
           end
 
@@ -152,7 +196,7 @@ module Llmemory
               obj = @kg.find_node_by_id(e.target_id)
               next unless subj && obj
               edge_text = "#{subj.name} #{e.predicate} #{obj.name}"
-              out << { text: edge_text, score: 0.7, created_at: e.created_at }
+              out << { id: e.id, text: edge_text, score: 0.7, created_at: e.created_at }
             end
           end
 

data/lib/llmemory/long_term/procedural/memory.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require_relative "skill"
+require_relative "storage"
+require_relative "../../memory_module"
+require_relative "../../vector_store"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      # Procedural long-term memory: a Voyager-style skill library. Agents
+      # register reusable skills (prompts, templates, code), retrieve them by
+      # relevance to the current task, and report outcomes so proven skills are
+      # preferred over unproven ones.
+      #
+      # The success rate of each skill is surfaced as `importance`, so the
+      # retrieval Engine ranks battle-tested skills higher (P3). Semantic
+      # (embedding) retrieval is opt-in via `config.procedural_vector_enabled` or
+      # by injecting a `vector_store:`; when off, search is keyword-only.
+      class Memory
+        include Llmemory::MemoryModule
+
+        attr_reader :user_id, :storage
+
+        def initialize(user_id:, storage: nil, vector_store: nil)
+          @user_id = user_id
+          @storage = storage || Storages.build
+          @vector_store = vector_store
+          @vector_explicit = !vector_store.nil?
+        end
+
+        # Registers a skill. If `version` is omitted and a skill with the same
+        # name exists, the version auto-increments (skill evolution).
+        def register_skill(name:, body:, description: nil, kind: Skill::DEFAULT_KIND, version: nil)
+          version ||= next_version_for(name)
+          skill = Skill.new(
+            id: nil, user_id: @user_id, name: name, body: body,
+            description: description, kind: kind, version: version
+          )
+          id = @storage.save_skill(@user_id, skill.to_h)
+          index_vector(id, skill.searchable_text)
+          id
+        end
+
+        def find_skill(query)
+          raw = @storage.search_skills(@user_id, query).first
+          raw && Skill.from_h(raw)
+        end
+
+        def get_skill(id)
+          raw = @storage.get_skill(@user_id, id)
+          raw && Skill.from_h(raw)
+        end
+
+        def skills(limit: nil)
+          @storage.list_skills(@user_id, limit: limit).map { |s| Skill.from_h(s) }
+        end
+
+        def count
+          @storage.count_skills(@user_id)
+        end
+
+        # Records that applying a skill succeeded or failed. Feeds retrieval
+        # ranking and adaptive retrieval (P8). Returns the updated Skill.
+        def report_outcome(skill_id, success:)
+          raw = @storage.record_outcome(@user_id, skill_id, success: success)
+          raw && Skill.from_h(raw)
+        end
+
+        # Retrieval Engine integration: skills ranked by relevance, recency and
+        # proven utility (success rate exposed as importance). Hybrid (vector +
+        # keyword) when a vector store is active; otherwise keyword-only.
+        def search_candidates(query, user_id: nil, top_k: 20)
+          uid = user_id || @user_id
+          return [] unless uid == @user_id
+
+          keyword = @storage.search_skills(uid, query).first(top_k).map { |raw| candidate_for(raw, 1.0) }
+          vs = vector_store
+          return keyword unless vs
+
+          merge_candidates(vector_candidates(query, top_k, vs), keyword, top_k)
+        end
+
+        # --- MemoryModule uniform interface ---
+
+        def write(name:, body:, description: nil, kind: Skill::DEFAULT_KIND, version: nil, **_meta)
+          register_skill(name: name, body: body, description: description, kind: kind, version: version)
+        end
+
+        def list(user_id: nil, limit: nil)
+          skills(limit: limit)
+        end
+
+        def stats(user_id: nil)
+          { skills: count }
+        end
+
+        def forget(ids:, reason: nil)
+          requested = Array(ids).map(&:to_s)
+          existing = @storage.list_skills(@user_id).map { |s| (s[:id] || s["id"]).to_s }
+          removed = requested & existing
+          @storage.delete_skills(@user_id, removed)
+          forget_log.record(@user_id, memory_type: "procedural", ids: removed, reason: reason)
+          removed.size
+        end
+
+        private
+
+        def next_version_for(name)
+          existing = @storage.find_skills_by_name(@user_id, name)
+          return 1 if existing.empty?
+          existing.map { |s| (s[:version] || s["version"] || 1).to_i }.max + 1
+        end
+
+        # Active vector store: injected, or a config-gated lazy build; nil when
+        # semantic search is disabled (default).
+        def vector_store
+          if @vector_explicit
+            @vector_store
+          elsif Llmemory.configuration.procedural_vector_enabled
+            @vector_store ||= Llmemory::VectorStore.build(source_type: "skill")
+          end
+        end
+
+        # Best-effort embedding indexing; failures never break registration.
+        def index_vector(id, text)
+          vs = vector_store
+          return if vs.nil? || text.to_s.strip.empty?
+          embedding = vs.embed(text)
+          return unless embedding
+          vs.store(id: id, embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
+        rescue StandardError
+          nil
+        end
+
+        def vector_candidates(query, top_k, vs)
+          vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id).filter_map do |r|
+            raw = @storage.get_skill(@user_id, r[:id] || r["id"])
+            raw && candidate_for(raw, (r[:score] || r["score"] || 1.0).to_f)
+          end
+        rescue StandardError
+          []
+        end
+
+        def candidate_for(raw, score)
+          skill = Skill.from_h(raw)
+          {
+            id: skill.id,
+            text: skill.searchable_text,
+            timestamp: skill.created_at,
+            score: score,
+            importance: skill.success_rate,
+            evergreen: false
+          }
+        end
+
+        # Dedup by id keeping the higher score; highest score first, capped.
+        def merge_candidates(primary, secondary, top_k)
+          by_id = {}
+          (primary + secondary).each do |c|
+            key = c[:id] || c[:text]
+            existing = by_id[key]
+            by_id[key] = c if existing.nil? || c[:score].to_f > existing[:score].to_f
+          end
+          by_id.values.sort_by { |c| -c[:score].to_f }.first(top_k)
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/skill.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      # A Skill is a reusable procedure an agent can retrieve and apply: a prompt,
+      # a template or a snippet of code. This is CoALA's "procedural memory" in the
+      # Voyager sense — a growing library of skills the agent learns and reuses.
+      #
+      # Skills track success/failure outcomes so proven skills can be preferred
+      # over unproven ones during retrieval (see #success_rate, and P8 adaptive
+      # retrieval).
+      class Skill
+        KINDS = %w[prompt template code].freeze
+        DEFAULT_KIND = "prompt"
+
+        attr_reader :id, :user_id, :name, :description, :body, :kind, :version,
+                    :success_count, :failure_count, :created_at, :updated_at
+
+        def initialize(id:, user_id:, name:, body:, description: nil, kind: DEFAULT_KIND,
+                       version: 1, success_count: 0, failure_count: 0, created_at: nil, updated_at: nil)
+          @id = id
+          @user_id = user_id
+          @name = name.to_s
+          @description = description
+          @body = body
+          @kind = normalize_kind(kind)
+          @version = version.to_i
+          @success_count = success_count.to_i
+          @failure_count = failure_count.to_i
+          @created_at = created_at || Time.now
+          @updated_at = updated_at || @created_at
+        end
+
+        # Proven utility in [0, 1]. Unproven skills (no outcomes) are neutral.
+        def success_rate
+          total = success_count + failure_count
+          total.zero? ? 0.5 : success_count.to_f / total
+        end
+
+        def searchable_text
+          [name, description, body].compact.map(&:to_s).reject(&:empty?).join("\n")
+        end
+
+        def normalize_kind(kind)
+          k = kind.to_s.strip.downcase
+          KINDS.include?(k) ? k : DEFAULT_KIND
+        end
+
+        def self.from_h(hash)
+          new(
+            id: hash[:id] || hash["id"],
+            user_id: hash[:user_id] || hash["user_id"],
+            name: hash[:name] || hash["name"],
+            description: hash[:description] || hash["description"],
+            body: hash[:body] || hash["body"],
+            kind: hash[:kind] || hash["kind"] || DEFAULT_KIND,
+            version: hash[:version] || hash["version"] || 1,
+            success_count: hash[:success_count] || hash["success_count"] || 0,
+            failure_count: hash[:failure_count] || hash["failure_count"] || 0,
+            created_at: parse_time(hash[:created_at] || hash["created_at"]),
+            updated_at: parse_time(hash[:updated_at] || hash["updated_at"])
+          )
+        end
+
+        def self.parse_time(value)
+          return value if value.nil? || value.is_a?(Time)
+          Time.parse(value.to_s)
+        rescue ArgumentError
+          nil
+        end
+
+        def to_h
+          {
+            id: id,
+            user_id: user_id,
+            name: name,
+            description: description,
+            body: body,
+            kind: kind,
+            version: version,
+            success_count: success_count,
+            failure_count: failure_count,
+            created_at: created_at.respond_to?(:iso8601) ? created_at.iso8601(6) : created_at,
+            updated_at: updated_at.respond_to?(:iso8601) ? updated_at.iso8601(6) : updated_at
+          }
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/storage.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "storages/base"
+require_relative "storages/memory_storage"
+require_relative "storages/file_storage"
+require_relative "storages/database_storage"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      # Backward compatibility: Storage points to the in-memory backend.
+      Storage = Storages::MemoryStorage
+
+      module Storages
+        def self.build(store: nil, base_path: nil, database_url: nil)
+          case (store || Llmemory.configuration.long_term_store).to_s.to_sym
+          when :memory
+            MemoryStorage.new
+          when :file
+            FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
+          when :postgres, :database
+            DatabaseStorage.new(database_url: database_url || Llmemory.configuration.database_url)
+          when :active_record, :activerecord
+            require_relative "storages/active_record_storage"
+            ActiveRecordStorage.new
+          else
+            MemoryStorage.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/storages/active_record_models.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Model for Procedural ActiveRecordStorage. Loaded only when using
+# store: :active_record. JSONB `data` auto-deserializes to a Hash in Rails 5+.
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      module Storages
+        class LlmemorySkill < ::ActiveRecord::Base
+          self.table_name = "llmemory_skills"
+          self.primary_key = "id"
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      module Storages
+        # ActiveRecord backend. Stores each skill as a JSONB `data` document; AR
+        # auto-deserializes jsonb to a Hash (string keys), which Skill.from_h
+        # handles. Mirrors the file-based ActiveRecordStorage pattern.
+        class ActiveRecordStorage < Base
+          def initialize
+            self.class.load_models!
+          end
+
+          def self.load_models!
+            return if @models_loaded
+            require "active_record"
+            require_relative "active_record_models"
+            @models_loaded = true
+          end
+
+          def save_skill(user_id, skill)
+            id = skill[:id] || skill["id"] || "skill_#{SecureRandom.hex(8)}"
+            data = stringify(skill).merge("id" => id, "user_id" => user_id)
+            data["created_at"] ||= Time.now.utc.iso8601
+            rec = LlmemorySkill.find_or_initialize_by(id: id)
+            rec.user_id = user_id
+            rec.data = data
+            rec.search_text = searchable_text(data)
+            rec.created_at ||= Time.current
+            rec.save!
+            id
+          end
+
+          def get_skill(user_id, id)
+            LlmemorySkill.find_by(user_id: user_id, id: id)&.data
+          end
+
+          def list_skills(user_id, limit: nil)
+            scope = LlmemorySkill.where(user_id: user_id).order(created_at: :desc)
+            scope = scope.limit(limit) if limit && limit.to_i.positive?
+            scope.map(&:data)
+          end
+
+          def search_skills(user_id, query)
+            token_scope(LlmemorySkill.where(user_id: user_id), "search_text", query)
+              .order(created_at: :desc).map(&:data)
+          end
+
+          def find_skills_by_name(user_id, name)
+            LlmemorySkill.where(user_id: user_id).where("data->>'name' = ?", name.to_s).map(&:data)
+          end
+
+          def record_outcome(user_id, skill_id, success:)
+            rec = LlmemorySkill.find_by(user_id: user_id, id: skill_id)
+            return nil unless rec
+            data = rec.data || {}
+            key = success ? "success_count" : "failure_count"
+            data[key] = (data[key] || 0).to_i + 1
+            data["updated_at"] = Time.now.utc.iso8601
+            rec.data = data
+            rec.search_text = searchable_text(data)
+            rec.save!
+            data
+          end
+
+          def count_skills(user_id)
+            LlmemorySkill.where(user_id: user_id).count
+          end
+
+          def delete_skills(user_id, ids)
+            LlmemorySkill.where(user_id: user_id, id: Array(ids).map(&:to_s)).delete_all
+          end
+
+          def list_users
+            LlmemorySkill.distinct.pluck(:user_id)
+          end
+
+          private
+
+          def token_scope(scope, column, query)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return scope if tokens.empty?
+            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+            scope.where(clause, *tokens.map { |t| "%#{t}%" })
+          end
+
+          def stringify(hash)
+            JSON.parse(JSON.generate(hash))
+          end
+
+          def searchable_text(data)
+            [data["name"], data["description"], data["body"]].compact.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/storages/base.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      module Storages
+        # Storage contract for procedural memory (skill library). Implementations
+        # persist Skill hashes, support keyword search and name lookup (for
+        # versioning), and record success/failure outcomes.
+        class Base
+          def save_skill(user_id, skill)
+            raise NotImplementedError, "#{self.class}#save_skill must be implemented"
+          end
+
+          def get_skill(user_id, id)
+            raise NotImplementedError, "#{self.class}#get_skill must be implemented"
+          end
+
+          def list_skills(user_id, limit: nil)
+            raise NotImplementedError, "#{self.class}#list_skills must be implemented"
+          end
+
+          def search_skills(user_id, query)
+            raise NotImplementedError, "#{self.class}#search_skills must be implemented"
+          end
+
+          def find_skills_by_name(user_id, name)
+            raise NotImplementedError, "#{self.class}#find_skills_by_name must be implemented"
+          end
+
+          # Increments the success or failure count of a skill and returns the
+          # updated skill hash (or nil if not found).
+          def record_outcome(user_id, skill_id, success:)
+            raise NotImplementedError, "#{self.class}#record_outcome must be implemented"
+          end
+
+          def count_skills(user_id)
+            raise NotImplementedError, "#{self.class}#count_skills must be implemented"
+          end
+
+          # Deletes skills by id. Returns the number actually removed.
+          def delete_skills(user_id, ids)
+            raise NotImplementedError, "#{self.class}#delete_skills must be implemented"
+          end
+
+          def list_users
+            raise NotImplementedError, "#{self.class}#list_users must be implemented"
+          end
+        end
+      end
+    end
+  end
+end