llmemory 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c302656888e6373faedb5732525f76d118fd98fb67ece22278d886faec5ba3cd
-  data.tar.gz: ec0897226ec378e51e86e01cb66e938fbeea3db7e4c49911d7e3d08a08959d07
+  metadata.gz: 80f259aa60090b95d21110bdf1f91c2c91bd5334a1bc4e3effaec444f241f371
+  data.tar.gz: 0b07d9a6ce5e485a69f00dfb2ad7c0cd03a442e95817d310a65d85e40f9def2c
 SHA512:
-  metadata.gz: 6cee9a244e42f198b9fd491d164d85d2524a30d6cf9ca0b4a4da3f1a012f7e2b4643ba1e4dd6838ba7ce32517d5425b77f62df1cb1334ac34d162a273cd6400f
-  data.tar.gz: 6d46031eda16a52b7c8b42b364dc5351a0eb18f0eeaa135271a3976a0ebba39db439e74fafa981b9cea062d70e2bd006b4576ccaccfff29bfd6ec6fe8011a411
+  metadata.gz: 464cc0765650869996ff24a4bf7e2b3ce4d3433a6c73cc9d63167173b04d14deedf0e169ca4579a7d7ac253b5a858f2b88629ce1b888bf48becfa46fad581979
+  data.tar.gz: 7624c8cd607cf882b13d500ee878b5412f49e13282014a33dc836cfa68c9bfdf9f759b89f53b51eccb38c32966f0cc66ae5d23a2297f2c965a67d81ccb8f437f

data/README.md

@@ -74,6 +74,11 @@ Llmemory.configure do |config|
   config.importance_weight = 1.0          # how strongly importance multiplies the score (0 = ignore)
   config.retrieval_feedback_weight = 0.5  # how strongly useful/harmful feedback shifts ranking (0 = ignore)
 
+  # Semantic (embedding) retrieval for episodic/procedural memory (opt-in;
+  # default off keeps them deterministic and network-free)
+  config.episodic_vector_enabled = false
+  config.procedural_vector_enabled = false
+
   # Pre-compaction memory flush (prevents knowledge loss when compacting)
   config.memory_flush_enabled = true
   config.memory_flush_threshold_tokens = 4000
@@ -129,7 +134,7 @@ rails g llmemory:install
 rails db:migrate
 ```
 
-La migración crea las tablas de long-term file-based (resources, items, categories), short-term (checkpoints) y, para graph-based, nodos, aristas y embeddings (`llmemory_nodes`, `llmemory_edges`, `llmemory_embeddings`). Para embeddings se usa pgvector; asegúrate de tener la extensión `vector` en PostgreSQL. Para usar ambas con ActiveRecord:
+La migración crea las tablas de long-term file-based (resources, items, categories), short-term (checkpoints), episódica y procedural (`llmemory_episodes`, `llmemory_skills`; columna `data` JSONB) y, para graph-based, nodos, aristas y embeddings (`llmemory_nodes`, `llmemory_edges`, `llmemory_embeddings`). Para embeddings se usa pgvector; asegúrate de tener la extensión `vector` en PostgreSQL. Para usar ambas con ActiveRecord:
 
 ```ruby
 # config/application.rb o config/initializers/llmemory.rb
@@ -214,7 +219,7 @@ llmemory implements the memory and internal-action concepts from [CoALA — Cogn
 | Learning action | `memorize` / `record_episode` / `register_skill` / reflection |
 | Uniform interface | `Llmemory::MemoryModule` (`read`/`write`/`list`/`stats`/`forget`) |
 
-All three long-term memories below are **additive** — episodic and procedural coexist with semantic memory rather than replacing it. Episodic/procedural ship with `:memory` and `:file` backends (SQL/ActiveRecord and vector search are roadmap items); retrieval there is keyword-based.
+All three long-term memories below are **additive** — episodic and procedural coexist with semantic memory rather than replacing it. They support `:memory`, `:file`, `:postgres` and `:active_record` backends. Retrieval is keyword-based by default (tokenized, so multi-word queries work); semantic (embedding) retrieval is **opt-in** via `config.episodic_vector_enabled` / `config.procedural_vector_enabled` (or by injecting a `vector_store:`), which makes `search_candidates` hybrid (vector + keyword).
 
 ### Working memory (structured, persists across LLM calls)
 

data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb

@@ -30,6 +30,26 @@ class CreateLlmemoryTables < ActiveRecord::Migration[7.0]
     end
     add_index :llmemory_categories, [:user_id, :category_name], unique: true
 
+    # Episodic long-term memory (trajectories) — JSONB document per episode
+    create_table :llmemory_episodes, id: false do |t|
+      t.string :id, null: false, primary_key: true
+      t.string :user_id, null: false
+      t.jsonb :data, null: false, default: {}
+      t.text :search_text
+      t.timestamps
+    end
+    add_index :llmemory_episodes, :user_id
+
+    # Procedural long-term memory (skill library) — JSONB document per skill
+    create_table :llmemory_skills, id: false do |t|
+      t.string :id, null: false, primary_key: true
+      t.string :user_id, null: false
+      t.jsonb :data, null: false, default: {}
+      t.text :search_text
+      t.timestamps
+    end
+    add_index :llmemory_skills, :user_id
+
     create_table :llmemory_checkpoints do |t|
       t.string :user_id, null: false
       t.string :session_id, null: false

data/lib/llmemory/cli/commands/base.rb

@@ -70,6 +70,14 @@ module Llmemory
             Llmemory::LongTerm::GraphBased::Storages::MemoryStorage.new
           end
         end
+
+        def episodic_storage(store_type = nil)
+          Llmemory::LongTerm::Episodic::Storages.build(store: (store_type || Llmemory.configuration.long_term_store))
+        end
+
+        def procedural_storage(store_type = nil)
+          Llmemory::LongTerm::Procedural::Storages.build(store: (store_type || Llmemory.configuration.long_term_store))
+        end
       end
     end
   end

data/lib/llmemory/cli/commands/episodic.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Llmemory
+  module Cli
+    module Commands
+      class Episodic < Commands::Base
+        def option_parser(parser)
+          parser.on("--limit N", Integer, "Max number of episodes (newest first)") { |v| @limit = v }
+          parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
+        end
+
+        def execute(argv, _opts)
+          user_id = argv.first
+          unless user_id
+            $stderr.puts "Usage: llmemory episodes USER_ID [--limit N] [--store TYPE]"
+            exit 1
+          end
+
+          storage = episodic_storage(@store_type)
+          episodes = storage.list_episodes(user_id, limit: @limit)
+
+          if episodes.empty?
+            puts "No episodes for user #{user_id}."
+            return
+          end
+
+          episodes.each do |e|
+            id = e[:id] || e["id"]
+            summary = e[:summary] || e["summary"]
+            outcome = e[:outcome] || e["outcome"]
+            importance = e[:importance] || e["importance"]
+            steps = e[:steps] || e["steps"] || []
+            puts "[#{id}] (importance: #{importance}; outcome: #{outcome || 'n/a'}) #{summary}"
+            puts "  steps: #{Array(steps).size}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/cli/commands/forget_log.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Llmemory
+  module Cli
+    module Commands
+      class ForgetLog < Commands::Base
+        def execute(argv, _opts)
+          user_id = argv.first
+          unless user_id
+            $stderr.puts "Usage: llmemory forget-log USER_ID"
+            exit 1
+          end
+
+          entries = Llmemory::ForgetLog.new(store: short_term_store).entries(user_id)
+
+          if entries.empty?
+            puts "No forget audit entries for user #{user_id}."
+            return
+          end
+
+          entries.each do |e|
+            type = e[:memory_type] || e["memory_type"]
+            count = e[:count] || e["count"]
+            reason = e[:reason] || e["reason"]
+            at = e[:at] || e["at"]
+            ids = e[:ids] || e["ids"] || []
+            reason_str = reason ? " — #{reason}" : ""
+            puts "[#{at}] #{type}: removed #{count} (#{ids.join(', ')})#{reason_str}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/cli/commands/procedural.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Llmemory
+  module Cli
+    module Commands
+      class Procedural < Commands::Base
+        def option_parser(parser)
+          parser.on("--limit N", Integer, "Max number of skills (newest first)") { |v| @limit = v }
+          parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
+        end
+
+        def execute(argv, _opts)
+          user_id = argv.first
+          unless user_id
+            $stderr.puts "Usage: llmemory skills USER_ID [--limit N] [--store TYPE]"
+            exit 1
+          end
+
+          storage = procedural_storage(@store_type)
+          skills = storage.list_skills(user_id, limit: @limit)
+
+          if skills.empty?
+            puts "No skills for user #{user_id}."
+            return
+          end
+
+          skills.each do |s|
+            id = s[:id] || s["id"]
+            name = s[:name] || s["name"]
+            kind = s[:kind] || s["kind"]
+            version = s[:version] || s["version"]
+            succ = (s[:success_count] || s["success_count"] || 0).to_i
+            fail = (s[:failure_count] || s["failure_count"] || 0).to_i
+            total = succ + fail
+            rate = total.zero? ? "n/a" : format("%.2f", succ.to_f / total)
+            puts "[#{id}] #{name} v#{version} (#{kind}) — success rate: #{rate} (#{succ}/#{total})"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/cli/commands/working.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Llmemory
+  module Cli
+    module Commands
+      class Working < Commands::Base
+        def execute(argv, _opts)
+          user_id, session_id = argv
+          unless user_id && session_id
+            $stderr.puts "Usage: llmemory working USER_ID SESSION_ID"
+            exit 1
+          end
+
+          wm = Llmemory::WorkingMemory.new(user_id: user_id, session_id: session_id, store: short_term_store)
+          state = wm.to_h
+
+          if state.empty?
+            puts "Empty working memory for user #{user_id}, session #{session_id}."
+            return
+          end
+
+          state.each do |slot, value|
+            puts "#{slot}: #{value.inspect}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/cli.rb

@@ -5,6 +5,10 @@ require_relative "cli/commands/base"
 require_relative "cli/commands/users"
 require_relative "cli/commands/short_term"
 require_relative "cli/commands/long_term"
+require_relative "cli/commands/episodic"
+require_relative "cli/commands/procedural"
+require_relative "cli/commands/working"
+require_relative "cli/commands/forget_log"
 require_relative "cli/commands/stats"
 require_relative "cli/commands/search"
 require_relative "cli/commands/mcp"
@@ -47,6 +51,10 @@ module Llmemory
         "nodes" => Cli::Commands::LongTerm::Nodes,
         "edges" => Cli::Commands::LongTerm::Edges,
         "graph" => Cli::Commands::LongTerm::Graph,
+        "episodes" => Cli::Commands::Episodic,
+        "skills" => Cli::Commands::Procedural,
+        "working" => Cli::Commands::Working,
+        "forget_log" => Cli::Commands::ForgetLog,
         "search" => Cli::Commands::Search,
         "stats" => Cli::Commands::Stats,
         "mcp" => Cli::Commands::Mcp
@@ -68,6 +76,10 @@ module Llmemory
           nodes USER_ID            List graph nodes (graph-based)
           edges USER_ID            List graph edges (graph-based)
           graph USER_ID            Export graph (--format dot|json)
+          episodes USER_ID         List recorded episodes (episodic memory)
+          skills USER_ID           List registered skills (procedural memory)
+          working USER_ID SESSION  Show working-memory slots for a session
+          forget-log USER_ID       Show audit of forgotten entries
           search USER_ID "query"   Search in memory
           stats [USER_ID]          Show statistics
           mcp [serve]              Start MCP server for LLM agents

data/lib/llmemory/configuration.rb

@@ -11,6 +11,8 @@ module Llmemory
                   :long_term_type,
                   :long_term_store,
                   :long_term_storage_path,
+                  :episodic_vector_enabled,
+                  :procedural_vector_enabled,
                   :database_url,
                   :vector_store,
                   :time_decay_half_life_days,
@@ -55,6 +57,8 @@ module Llmemory
       @long_term_type = :file_based
       @long_term_store = :memory
       @long_term_storage_path = ENV["LLMEMORY_STORAGE_PATH"] || "./llmemory_data"
+      @episodic_vector_enabled = false
+      @procedural_vector_enabled = false
       @database_url = ENV["DATABASE_URL"]
       @vector_store = nil
       @time_decay_half_life_days = 30

data/lib/llmemory/long_term/episodic/memory.rb

@@ -3,6 +3,7 @@
 require_relative "episode"
 require_relative "storage"
 require_relative "../../memory_module"
+require_relative "../../vector_store"
 
 module Llmemory
   module LongTerm
@@ -12,16 +13,19 @@ module Llmemory
       # memory (file/graph), not replace it, and to feed reflection (P2), which
       # distills episodes into semantic knowledge.
       #
-      # Deliberately LLM-free: recording and retrieval are deterministic. Higher
-      # order summarization belongs to reflection.
+      # Recording/retrieval are deterministic and LLM-free by default. Semantic
+      # (embedding) retrieval is opt-in via `config.episodic_vector_enabled` or by
+      # injecting a `vector_store:`; when off, search is keyword-only (unchanged).
       class Memory
         include Llmemory::MemoryModule
 
         attr_reader :user_id, :storage
 
-        def initialize(user_id:, storage: nil)
+        def initialize(user_id:, storage: nil, vector_store: nil)
           @user_id = user_id
           @storage = storage || Storages.build
+          @vector_store = vector_store
+          @vector_explicit = !vector_store.nil?
         end
 
         # Records a trajectory. `steps` is an array of hashes with any of
@@ -39,7 +43,9 @@ module Llmemory
             episode.searchable_text, method: "episode_recording", confidence: episode.importance
           )
           record = episode.to_h.merge(provenance: provenance)
-          @storage.save_episode(@user_id, record)
+          id = @storage.save_episode(@user_id, record)
+          index_vector(id, episode.searchable_text)
+          id
         end
 
         def recent_episodes(limit: 10)
@@ -62,22 +68,17 @@ module Llmemory
         # Retrieval Engine integration. Returns candidates shaped like the other
         # long-term memories so the Engine can rank episodes by relevance,
         # recency (temporal decay) and importance (P3), with provenance (P10).
+        # Hybrid (vector + keyword) when a vector store is active; otherwise
+        # keyword-only.
         def search_candidates(query, user_id: nil, top_k: 20)
           uid = user_id || @user_id
           return [] unless uid == @user_id
 
-          @storage.search_episodes(uid, query).first(top_k).map do |e|
-            episode = Episode.from_h(e)
-            {
-              id: episode.id,
-              text: episode.summary.to_s.empty? ? episode.searchable_text : episode.summary,
-              timestamp: episode.created_at,
-              score: 1.0,
-              importance: episode.importance,
-              evergreen: false,
-              provenance: e[:provenance] || e["provenance"]
-            }
-          end
+          keyword = @storage.search_episodes(uid, query).first(top_k).map { |e| candidate_for(e, 1.0) }
+          vs = vector_store
+          return keyword unless vs
+
+          merge_candidates(vector_candidates(query, top_k, vs), keyword, top_k)
         end
 
         # --- MemoryModule uniform interface ---
@@ -105,6 +106,60 @@ module Llmemory
 
         private
 
+        # Active vector store: the injected one, or a config-gated lazy build.
+        # Returns nil when semantic search is disabled (default).
+        def vector_store
+          if @vector_explicit
+            @vector_store
+          elsif Llmemory.configuration.episodic_vector_enabled
+            @vector_store ||= Llmemory::VectorStore.build(source_type: "episode")
+          end
+        end
+
+        # Best-effort embedding indexing; a failure must never break recording.
+        def index_vector(id, text)
+          vs = vector_store
+          return if vs.nil? || text.to_s.strip.empty?
+          embedding = vs.embed(text)
+          return unless embedding
+          vs.store(id: id, embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
+        rescue StandardError
+          nil
+        end
+
+        def vector_candidates(query, top_k, vs)
+          vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id).filter_map do |r|
+            raw = @storage.get_episode(@user_id, r[:id] || r["id"])
+            raw && candidate_for(raw, (r[:score] || r["score"] || 1.0).to_f)
+          end
+        rescue StandardError
+          []
+        end
+
+        def candidate_for(raw, score)
+          episode = Episode.from_h(raw)
+          {
+            id: episode.id,
+            text: episode.summary.to_s.empty? ? episode.searchable_text : episode.summary,
+            timestamp: episode.created_at,
+            score: score,
+            importance: episode.importance,
+            evergreen: false,
+            provenance: raw[:provenance] || raw["provenance"]
+          }
+        end
+
+        # Dedup by id keeping the higher score; highest score first, capped.
+        def merge_candidates(primary, secondary, top_k)
+          by_id = {}
+          (primary + secondary).each do |c|
+            key = c[:id] || c[:text]
+            existing = by_id[key]
+            by_id[key] = c if existing.nil? || c[:score].to_f > existing[:score].to_f
+          end
+          by_id.values.sort_by { |c| -c[:score].to_f }.first(top_k)
+        end
+
         # Cheap, deterministic summary when the caller does not provide one.
         # LLM-based summarization is reflection's job (P2).
         def derive_summary(steps)

data/lib/llmemory/long_term/episodic/storage.rb

@@ -3,6 +3,7 @@
 require_relative "storages/base"
 require_relative "storages/memory_storage"
 require_relative "storages/file_storage"
+require_relative "storages/database_storage"
 
 module Llmemory
   module LongTerm
@@ -11,16 +12,17 @@ module Llmemory
       Storage = Storages::MemoryStorage
 
       module Storages
-        def self.build(store: nil, base_path: nil)
+        def self.build(store: nil, base_path: nil, database_url: nil)
           case (store || Llmemory.configuration.long_term_store).to_s.to_sym
           when :memory
             MemoryStorage.new
           when :file
             FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
-          when :postgres, :database, :active_record, :activerecord
-            raise NotImplementedError,
-              "Episodic SQL/ActiveRecord storage is not implemented yet; use :memory or :file " \
-              "(or pass an explicit storage instance)."
+          when :postgres, :database
+            DatabaseStorage.new(database_url: database_url || Llmemory.configuration.database_url)
+          when :active_record, :activerecord
+            require_relative "storages/active_record_storage"
+            ActiveRecordStorage.new
           else
             MemoryStorage.new
           end

data/lib/llmemory/long_term/episodic/storages/active_record_models.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Model for Episodic ActiveRecordStorage. Loaded only when using
+# store: :active_record. JSONB `data` auto-deserializes to a Hash in Rails 5+.
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        class LlmemoryEpisode < ::ActiveRecord::Base
+          self.table_name = "llmemory_episodes"
+          self.primary_key = "id"
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        # ActiveRecord backend. Stores each episode as a JSONB `data` document;
+        # AR auto-deserializes jsonb to a Hash (string keys), which Episode.from_h
+        # handles. Mirrors the file-based ActiveRecordStorage pattern.
+        class ActiveRecordStorage < Base
+          def initialize
+            self.class.load_models!
+          end
+
+          def self.load_models!
+            return if @models_loaded
+            require "active_record"
+            require_relative "active_record_models"
+            @models_loaded = true
+          end
+
+          def save_episode(user_id, episode)
+            id = episode[:id] || episode["id"] || "ep_#{SecureRandom.hex(8)}"
+            data = stringify(episode).merge("id" => id, "user_id" => user_id)
+            data["created_at"] ||= Time.now.utc.iso8601
+            rec = LlmemoryEpisode.find_or_initialize_by(id: id)
+            rec.user_id = user_id
+            rec.data = data
+            rec.search_text = searchable_text(data)
+            rec.created_at ||= Time.current
+            rec.save!
+            id
+          end
+
+          def get_episode(user_id, id)
+            rec = LlmemoryEpisode.find_by(user_id: user_id, id: id)
+            rec&.data
+          end
+
+          def list_episodes(user_id, limit: nil)
+            scope = LlmemoryEpisode.where(user_id: user_id).order(created_at: :desc)
+            scope = scope.limit(limit) if limit && limit.to_i.positive?
+            scope.map(&:data)
+          end
+
+          def search_episodes(user_id, query)
+            token_scope(LlmemoryEpisode.where(user_id: user_id), "search_text", query)
+              .order(created_at: :desc).map(&:data)
+          end
+
+          def count_episodes(user_id)
+            LlmemoryEpisode.where(user_id: user_id).count
+          end
+
+          def delete_episodes(user_id, ids)
+            LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s)).delete_all
+          end
+
+          def list_users
+            LlmemoryEpisode.distinct.pluck(:user_id)
+          end
+
+          private
+
+          def token_scope(scope, column, query)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return scope if tokens.empty?
+            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+            scope.where(clause, *tokens.map { |t| "%#{t}%" })
+          end
+
+          def stringify(hash)
+            JSON.parse(JSON.generate(hash))
+          end
+
+          def searchable_text(data)
+            parts = [data["summary"], data["outcome"]]
+            Array(data["steps"]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << s["observation"] << s["action"] << s["result"]
+            end
+            parts.compact.join("\n")
+          end
+        end
+      end
+    end
+  end
+end