llmemory 0.2.0 → 0.2.2

data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      module Storages
+        # ActiveRecord backend. Stores each skill as a JSONB `data` document; AR
+        # auto-deserializes jsonb to a Hash (string keys), which Skill.from_h
+        # handles. Mirrors the file-based ActiveRecordStorage pattern.
+        class ActiveRecordStorage < Base
+          def initialize
+            self.class.load_models!
+          end
+
+          def self.load_models!
+            return if @models_loaded
+            require "active_record"
+            require_relative "active_record_models"
+            @models_loaded = true
+          end
+
+          def save_skill(user_id, skill)
+            id = skill[:id] || skill["id"] || "skill_#{SecureRandom.hex(8)}"
+            data = stringify(skill).merge("id" => id, "user_id" => user_id)
+            data["created_at"] ||= Time.now.utc.iso8601
+            rec = LlmemorySkill.find_or_initialize_by(id: id)
+            rec.user_id = user_id
+            rec.data = data
+            rec.search_text = searchable_text(data)
+            rec.created_at ||= Time.current
+            rec.save!
+            id
+          end
+
+          def get_skill(user_id, id)
+            LlmemorySkill.find_by(user_id: user_id, id: id)&.data
+          end
+
+          def list_skills(user_id, limit: nil, offset: nil)
+            scope = LlmemorySkill.where(user_id: user_id, archived_at: nil).order(created_at: :desc)
+            scope = scope.limit(limit) if limit && limit.to_i.positive?
+            scope = scope.offset(offset) if offset && offset.to_i.positive?
+            scope.map(&:data)
+          end
+
+          def search_skills(user_id, query)
+            token_scope(LlmemorySkill.where(user_id: user_id, archived_at: nil), "search_text", query)
+              .order(created_at: :desc).map(&:data)
+          end
+
+          def find_skills_by_name(user_id, name)
+            LlmemorySkill.where(user_id: user_id, archived_at: nil).where("data->>'name' = ?", name.to_s).map(&:data)
+          end
+
+          def record_outcome(user_id, skill_id, success:)
+            rec = LlmemorySkill.find_by(user_id: user_id, id: skill_id)
+            return nil unless rec
+            data = rec.data || {}
+            key = success ? "success_count" : "failure_count"
+            data[key] = (data[key] || 0).to_i + 1
+            data["updated_at"] = Time.now.utc.iso8601
+            rec.data = data
+            rec.search_text = searchable_text(data)
+            rec.save!
+            data
+          end
+
+          def count_skills(user_id)
+            LlmemorySkill.where(user_id: user_id, archived_at: nil).count
+          end
+
+          def delete_skills(user_id, ids)
+            LlmemorySkill.where(user_id: user_id, id: Array(ids).map(&:to_s)).delete_all
+          end
+
+          def archive_skills(user_id, ids)
+            LlmemorySkill.where(user_id: user_id, id: Array(ids).map(&:to_s), archived_at: nil)
+              .update_all(archived_at: Time.current)
+          end
+
+          def expired_skill_ids(user_id, cutoff:)
+            LlmemorySkill.where(user_id: user_id, archived_at: nil).where("created_at < ?", cutoff).pluck(:id)
+          end
+
+          def list_users
+            LlmemorySkill.distinct.pluck(:user_id)
+          end
+
+          private
+
+          def token_scope(scope, column, query)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return scope if tokens.empty?
+            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+            scope.where(clause, *tokens.map { |t| "%#{t}%" })
+          end
+
+          def stringify(hash)
+            JSON.parse(JSON.generate(hash))
+          end
+
+          def searchable_text(data)
+            [data["name"], data["description"], data["body"]].compact.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/storages/base.rb

@@ -16,7 +16,7 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#get_skill must be implemented"
           end
 
-          def list_skills(user_id, limit: nil)
+          def list_skills(user_id, limit: nil, offset: nil)
             raise NotImplementedError, "#{self.class}#list_skills must be implemented"
           end
 
@@ -43,6 +43,19 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#delete_skills must be implemented"
           end
 
+          # Soft-archives skills by id. Archived skills are excluded from
+          # list_skills / search_skills / count_skills but remain accessible via
+          # get_skill. Returns the number archived.
+          def archive_skills(user_id, ids)
+            raise NotImplementedError, "#{self.class}#archive_skills must be implemented"
+          end
+
+          # Returns skills whose created_at is older than the cutoff and that are
+          # not already archived. Used by the TTL maintenance job.
+          def expired_skill_ids(user_id, cutoff:)
+            raise NotImplementedError, "#{self.class}#expired_skill_ids must be implemented"
+          end
+
           def list_users
             raise NotImplementedError, "#{self.class}#list_users must be implemented"
           end

data/lib/llmemory/long_term/procedural/storages/database_storage.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      module Storages
+        # PostgreSQL backend. Each skill is stored as a JSONB `data` document
+        # (plus id/user_id/created_at and a denormalized search_text), mirroring
+        # the file-based DatabaseStorage pattern.
+        class DatabaseStorage < Base
+          def initialize(database_url: nil)
+            @database_url = database_url || Llmemory.configuration.database_url
+            @connection = nil
+          end
+
+          def save_skill(user_id, skill)
+            ensure_tables!
+            id = skill[:id] || skill["id"] || "skill_#{SecureRandom.hex(8)}"
+            data = symbolize(skill).merge(id: id, user_id: user_id)
+            data[:created_at] ||= Time.now.utc.iso8601
+            conn.exec_params(
+              "INSERT INTO llmemory_skills (id, user_id, data, search_text, created_at) " \
+              "VALUES ($1, $2, $3::jsonb, $4, $5) " \
+              "ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
+              [id, user_id, JSON.generate(data), searchable_text(data), created_at_value(data)]
+            )
+            id
+          end
+
+          def get_skill(user_id, id)
+            ensure_tables!
+            rows = conn.exec_params("SELECT data FROM llmemory_skills WHERE user_id = $1 AND id = $2", [user_id, id])
+            rows.any? ? parse_data(rows.first["data"]) : nil
+          end
+
+          def list_skills(user_id, limit: nil, offset: nil)
+            ensure_tables!
+            sql = "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL ORDER BY created_at DESC"
+            sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
+            sql += " OFFSET #{offset.to_i}" if offset && offset.to_i.positive?
+            conn.exec_params(sql, [user_id]).map { |r| parse_data(r["data"]) }
+          end
+
+          def search_skills(user_id, query)
+            ensure_tables!
+            suffix, params = token_filter("search_text", query, 2)
+            conn.exec_params(
+              "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL#{suffix} ORDER BY created_at DESC",
+              [user_id, *params]
+            ).map { |r| parse_data(r["data"]) }
+          end
+
+          def find_skills_by_name(user_id, name)
+            ensure_tables!
+            conn.exec_params(
+              "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND data->>'name' = $2",
+              [user_id, name.to_s]
+            ).map { |r| parse_data(r["data"]) }
+          end
+
+          def record_outcome(user_id, skill_id, success:)
+            ensure_tables!
+            data = get_skill(user_id, skill_id)
+            return nil unless data
+            key = success ? :success_count : :failure_count
+            data[key] = (data[key] || 0).to_i + 1
+            data[:updated_at] = Time.now.utc.iso8601
+            conn.exec_params(
+              "UPDATE llmemory_skills SET data = $3::jsonb, search_text = $4 WHERE user_id = $1 AND id = $2",
+              [user_id, skill_id, JSON.generate(data), searchable_text(data)]
+            )
+            data
+          end
+
+          def count_skills(user_id)
+            ensure_tables!
+            conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL", [user_id]).first["c"].to_i
+          end
+
+          def delete_skills(user_id, ids)
+            ensure_tables!
+            Array(ids).sum do |id|
+              conn.exec_params("DELETE FROM llmemory_skills WHERE user_id = $1 AND id = $2", [user_id, id]).cmd_tuples
+            end
+          end
+
+          def archive_skills(user_id, ids)
+            ensure_tables!
+            Array(ids).sum do |id|
+              conn.exec_params(
+                "UPDATE llmemory_skills SET archived_at = NOW() WHERE user_id = $1 AND id = $2 AND archived_at IS NULL",
+                [user_id, id]
+              ).cmd_tuples
+            end
+          end
+
+          def expired_skill_ids(user_id, cutoff:)
+            ensure_tables!
+            conn.exec_params(
+              "SELECT id FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND created_at < $2",
+              [user_id, cutoff.iso8601]
+            ).map { |r| r["id"] }
+          end
+
+          def list_users
+            ensure_tables!
+            conn.exec("SELECT DISTINCT user_id FROM llmemory_skills").map { |r| r["user_id"] }
+          end
+
+          private
+
+          def conn
+            @connection ||= begin
+              require "pg"
+              PG.connect(@database_url)
+            end
+          end
+
+          def ensure_tables!
+            conn.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS llmemory_skills (
+                id TEXT NOT NULL PRIMARY KEY,
+                user_id TEXT NOT NULL,
+                data JSONB NOT NULL DEFAULT '{}'::jsonb,
+                search_text TEXT,
+                created_at TIMESTAMPTZ NOT NULL,
+                archived_at TIMESTAMPTZ
+              );
+              CREATE INDEX IF NOT EXISTS idx_llmemory_skills_user_id ON llmemory_skills(user_id);
+              ALTER TABLE llmemory_skills ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
+            SQL
+          end
+
+          def token_filter(column, query, start_index)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return ["", []] if tokens.empty?
+            likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
+            [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
+          end
+
+          def parse_data(value)
+            JSON.parse(value.to_s, symbolize_names: true)
+          rescue JSON::ParserError
+            {}
+          end
+
+          def symbolize(hash)
+            hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
+          end
+
+          def searchable_text(data)
+            [data[:name], data[:description], data[:body]].compact.join("\n")
+          end
+
+          def created_at_value(data)
+            ca = data[:created_at]
+            return Time.now.utc.iso8601 if ca.nil?
+            ca.respond_to?(:iso8601) ? ca.iso8601 : ca.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/storages/file_storage.rb

@@ -29,19 +29,19 @@ module Llmemory
             load_skill(path)
           end
 
-          def list_skills(user_id, limit: nil)
-            sorted = all_skills(user_id).sort_by { |s| s[:created_at] }.reverse
+          def list_skills(user_id, limit: nil, offset: nil)
+            sorted = active_skills(user_id).sort_by { |s| s[:created_at] }.reverse
+            sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
             limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
           end
 
           def search_skills(user_id, query)
-            q = query.to_s.downcase
-            return list_skills(user_id) if q.strip.empty?
-            all_skills(user_id).select { |s| skill_text(s).downcase.include?(q) }
+            return list_skills(user_id) if query.to_s.strip.empty?
+            active_skills(user_id).select { |s| Llmemory::Tokenizer.matches?(skill_text(s), query) }
           end
 
           def find_skills_by_name(user_id, name)
-            all_skills(user_id).select { |s| s[:name].to_s == name.to_s }
+            active_skills(user_id).select { |s| s[:name].to_s == name.to_s }
           end
 
           def record_outcome(user_id, skill_id, success:)
@@ -55,9 +55,7 @@ module Llmemory
           end
 
           def count_skills(user_id)
-            dir = user_path(user_id, "skills")
-            return 0 unless Dir.exist?(dir)
-            Dir.children(dir).count { |f| f.end_with?(".json") }
+            active_skills(user_id).size
           end
 
           def delete_skills(user_id, ids)
@@ -69,6 +67,24 @@ module Llmemory
             end
           end
 
+          def archive_skills(user_id, ids)
+            Array(ids).map(&:to_s).count do |id|
+              path = skill_path(user_id, id)
+              next false unless File.file?(path)
+              data = JSON.parse(File.read(path))
+              next false if data["archived_at"]
+              data["archived_at"] = Time.now.iso8601
+              File.write(path, JSON.generate(data))
+              true
+            end
+          end
+
+          def expired_skill_ids(user_id, cutoff:)
+            active_skills(user_id)
+              .select { |s| (s[:created_at] || Time.now) < cutoff }
+              .map { |s| s[:id].to_s }
+          end
+
           def list_users
             return [] unless Dir.exist?(@base_path)
             Dir.children(@base_path).select { |d| Dir.exist?(File.join(@base_path, d, "skills")) }
@@ -76,6 +92,10 @@ module Llmemory
 
           private
 
+          def active_skills(user_id)
+            all_skills(user_id).reject { |s| s[:archived_at] }
+          end
+
           def all_skills(user_id)
             dir = user_path(user_id, "skills")
             return [] unless Dir.exist?(dir)

data/lib/llmemory/long_term/procedural/storages/memory_storage.rb

@@ -25,19 +25,19 @@ module Llmemory
             @skills[user_id].find { |s| s[:id] == id }
           end
 
-          def list_skills(user_id, limit: nil)
-            sorted = @skills[user_id].sort_by { |s| s[:created_at] }.reverse
+          def list_skills(user_id, limit: nil, offset: nil)
+            sorted = active_skills(user_id).sort_by { |s| as_time(s[:created_at]) }.reverse
+            sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
             limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
           end
 
           def search_skills(user_id, query)
-            q = query.to_s.downcase
-            return list_skills(user_id) if q.strip.empty?
-            @skills[user_id].select { |s| skill_text(s).downcase.include?(q) }
+            return list_skills(user_id) if query.to_s.strip.empty?
+            active_skills(user_id).select { |s| Llmemory::Tokenizer.matches?(skill_text(s), query) }
           end
 
           def find_skills_by_name(user_id, name)
-            @skills[user_id].select { |s| s[:name].to_s == name.to_s }
+            active_skills(user_id).select { |s| s[:name].to_s == name.to_s }
           end
 
           def record_outcome(user_id, skill_id, success:)
@@ -50,7 +50,7 @@ module Llmemory
           end
 
           def count_skills(user_id)
-            @skills[user_id].size
+            active_skills(user_id).size
           end
 
           def delete_skills(user_id, ids)
@@ -60,12 +60,42 @@ module Llmemory
             before - @skills[user_id].size
           end
 
+          def archive_skills(user_id, ids)
+            ids = Array(ids).map(&:to_s)
+            count = 0
+            @skills[user_id].each do |s|
+              next unless ids.include?(s[:id].to_s)
+              next if s[:archived_at]
+              s[:archived_at] = Time.now
+              count += 1
+            end
+            count
+          end
+
+          def expired_skill_ids(user_id, cutoff:)
+            active_skills(user_id)
+              .select { |s| as_time(s[:created_at]) < cutoff }
+              .map { |s| s[:id].to_s }
+          end
+
           def list_users
             @skills.keys
           end
 
           private
 
+          def active_skills(user_id)
+            @skills[user_id].reject { |s| s[:archived_at] }
+          end
+
+          def as_time(value)
+            return Time.now if value.nil?
+            return value if value.is_a?(Time)
+            Time.parse(value.to_s)
+          rescue ArgumentError
+            Time.now
+          end
+
           def symbolize(hash)
             hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
           end

data/lib/llmemory/maintenance/cognitive_pass.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module Maintenance
+    # The cognitive maintenance pass closes CoALA's learning loop in one
+    # scheduled step. Independently, the gem exposes consolidation (short-term ->
+    # semantic), reflection (episodic -> insights), skill mining (episodic ->
+    # procedural) and TTL expiry. This pass orchestrates them so an agent learns
+    # from its experience and keeps its memory healthy without the consumer
+    # wiring each step by hand.
+    #
+    # Designed to run as a maintenance task (cron / Rails Job), per user. Each
+    # step is isolated: a failure in one is captured in the returned report
+    # (`:errors`) and never aborts the others.
+    #
+    # Returns:
+    #   {
+    #     consolidated: true/false/nil,   # nil when no `memory:` was supplied
+    #     insights: [insight_id, ...],
+    #     mined: [proposal_or_skill_id, ...],
+    #     expired: { episodic: N, procedural: M },
+    #     errors: { reflect: "...", mine: "...", ... }   # only failed steps
+    #   }
+    class CognitivePass
+      def self.run!(user_id, **kwargs)
+        new(user_id, **kwargs).run!
+      end
+
+      def initialize(user_id, memory: nil, episodic: nil, procedural: nil, semantic: nil,
+                     llm: nil, reflect: true, mine_skills: nil, expire: true,
+                     reflection_window: 10, mining_window: Llmemory::SkillMining::Miner::DEFAULT_WINDOW)
+        @user_id = user_id
+        @memory = memory
+        @episodic = episodic
+        @procedural = procedural
+        @semantic = semantic
+        @llm = llm
+        @reflect = reflect
+        @mine_skills = mine_skills.nil? ? Llmemory.configuration.skill_mining_enabled : mine_skills
+        @expire = expire
+        @reflection_window = reflection_window
+        @mining_window = mining_window
+      end
+
+      def run!
+        report = { consolidated: nil, insights: [], mined: [], expired: { episodic: 0, procedural: 0 }, errors: {} }
+
+        step(report, :consolidate) { report[:consolidated] = consolidate } if @memory
+        step(report, :reflect)     { report[:insights] = reflect } if @reflect
+        step(report, :mine)        { report[:mined] = mine } if @mine_skills
+        step(report, :expire)      { report[:expired] = expire } if @expire
+
+        report
+      end
+
+      private
+
+      def step(report, name)
+        yield
+      rescue StandardError => e
+        report[:errors][name] = e.message
+      end
+
+      def consolidate
+        @memory.consolidate!
+      end
+
+      def reflect
+        Reflection::Reflector.new(episodic: episodic, semantic: semantic, llm: @llm)
+          .reflect(window: @reflection_window)
+      end
+
+      def mine
+        SkillMining::Miner.new(episodic: episodic, procedural: procedural, llm: @llm)
+          .mine(window: @mining_window, auto_register: true)
+      end
+
+      def expire
+        TTLExpiry.run!(@user_id, episodic: episodic, procedural: procedural)
+      end
+
+      def episodic
+        @episodic ||= @memory&.episodic || Llmemory::LongTerm::Episodic::Memory.new(user_id: @user_id)
+      end
+
+      def procedural
+        @procedural ||= @memory&.procedural || Llmemory::LongTerm::Procedural::Memory.new(user_id: @user_id)
+      end
+
+      def semantic
+        @semantic ||= build_semantic
+      end
+
+      def build_semantic
+        llm_opts = @llm ? { llm: @llm } : {}
+        case (Llmemory.configuration.long_term_type || :file_based).to_s.to_sym
+        when :graph_based
+          Llmemory::LongTerm::GraphBased::Memory.new(
+            user_id: @user_id, storage: Llmemory::LongTerm::GraphBased::Storages.build, **llm_opts
+          )
+        else
+          Llmemory::LongTerm::FileBased::Memory.new(
+            user_id: @user_id, storage: Llmemory::LongTerm::FileBased::Storages.build, **llm_opts
+          )
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/maintenance/ttl_expiry.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module Maintenance
+    # TTL expiry job: soft-archives episodic/procedural entries whose age
+    # exceeds the configured per-type TTL. Designed to run as a maintenance
+    # task (cron / Rails Job). Idempotent — already-archived entries are
+    # skipped by the storage layer.
+    #
+    # Reads `Llmemory.configuration.ttl_episodic_days` and
+    # `Llmemory.configuration.ttl_procedural_days`. A nil/zero TTL disables
+    # expiry for that memory type.
+    #
+    # Returns a hash `{ episodic: N, procedural: M }` with the number of
+    # entries archived per type for the given user.
+    class TTLExpiry
+      DEFAULT_REASON = "ttl_expired"
+
+      def self.run!(user_id, episodic: nil, procedural: nil, reason: DEFAULT_REASON)
+        new(user_id, episodic: episodic, procedural: procedural, reason: reason).run!
+      end
+
+      def initialize(user_id, episodic: nil, procedural: nil, reason: DEFAULT_REASON)
+        @user_id = user_id
+        @episodic = episodic
+        @procedural = procedural
+        @reason = reason
+      end
+
+      def run!
+        {
+          episodic: expire(memory: @episodic ||= Llmemory::LongTerm::Episodic::Memory.new(user_id: @user_id),
+                           ttl_days: Llmemory.configuration.ttl_episodic_days),
+          procedural: expire(memory: @procedural ||= Llmemory::LongTerm::Procedural::Memory.new(user_id: @user_id),
+                             ttl_days: Llmemory.configuration.ttl_procedural_days)
+        }
+      end
+
+      private
+
+      def expire(memory:, ttl_days:)
+        return 0 unless ttl_days && ttl_days.to_f.positive?
+        cutoff = Time.now - (ttl_days.to_f * 86400)
+        ids = memory.expired_ids(cutoff: cutoff)
+        return 0 if ids.empty?
+        memory.forget(ids: ids, reason: @reason, mode: :soft)
+      end
+    end
+  end
+end

data/lib/llmemory/maintenance.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "maintenance/runner"
+require_relative "maintenance/ttl_expiry"
+require_relative "maintenance/cognitive_pass"
 
 module Llmemory
   module Maintenance

data/lib/llmemory/mcp/server.rb

@@ -11,6 +11,14 @@ require_relative "tools/memory_consolidate"
 require_relative "tools/memory_stats"
 require_relative "tools/memory_info"
 require_relative "tools/memory_timeline_context"
+require_relative "tools/memory_episode_record"
+require_relative "tools/memory_episodes"
+require_relative "tools/memory_skill_register"
+require_relative "tools/memory_skill_report"
+require_relative "tools/memory_skills"
+require_relative "tools/memory_forget"
+require_relative "tools/memory_mine_skills"
+require_relative "tools/memory_maintain"
 
 module Llmemory
   module MCP
@@ -157,7 +165,15 @@ module Llmemory
           Tools::MemoryAddMessage,
           Tools::MemoryConsolidate,
           Tools::MemoryStats,
-          Tools::MemoryInfo
+          Tools::MemoryInfo,
+          Tools::MemoryEpisodeRecord,
+          Tools::MemoryEpisodes,
+          Tools::MemorySkillRegister,
+          Tools::MemorySkillReport,
+          Tools::MemorySkills,
+          Tools::MemoryForget,
+          Tools::MemoryMineSkills,
+          Tools::MemoryMaintain
         ]
       end