llmemory 0.2.0 → 0.2.2

data/lib/llmemory/short_term/session_lifecycle.rb

@@ -5,6 +5,18 @@ require_relative "stores"
 module Llmemory
   module ShortTerm
     class SessionLifecycle
+      # Pseudo-sessions used by ForgetLog, FeedbackStore and WorkingMemory share
+      # the short-term K/V store but are not user sessions — they must not be
+      # idle-pruned, stale-pruned, or evicted by enforce_max_entries.
+      PSEUDO_SESSION_PATTERNS = [
+        /\A__[a-z_]+__\z/,    # e.g. "__forget_log__", "__retrieval_feedback__"
+        /:working_memory\z/    # WorkingMemory uses "<session>:working_memory"
+      ].freeze
+
+      def self.pseudo_session?(session_id)
+        PSEUDO_SESSION_PATTERNS.any? { |p| session_id.to_s.match?(p) }
+      end
+
       def initialize(store: nil)
         @store = store || build_store
       end
@@ -14,7 +26,7 @@ module Llmemory
         cutoff = Time.now - (idle_minutes * 60)
         deleted = 0
 
-        @store.list_sessions(user_id: user_id).each do |session_id|
+        user_sessions(user_id).each do |session_id|
           state = @store.load(user_id, session_id)
           next unless state.is_a?(Hash)
 
@@ -36,7 +48,7 @@ module Llmemory
         cutoff = Time.now - (prune_after_days * 86400)
         deleted = 0
 
-        @store.list_sessions(user_id: user_id).each do |session_id|
+        user_sessions(user_id).each do |session_id|
           state = @store.load(user_id, session_id)
           next unless state.is_a?(Hash)
 
@@ -55,7 +67,7 @@ module Llmemory
 
       def enforce_max_entries!(user_id:, max_entries: nil)
         max_entries ||= Llmemory.configuration.session_max_entries_per_user
-        sessions = @store.list_sessions(user_id: user_id)
+        sessions = user_sessions(user_id)
         return 0 if sessions.size <= max_entries
 
         session_ages = sessions.map do |session_id|
@@ -73,6 +85,10 @@ module Llmemory
 
       private
 
+      def user_sessions(user_id)
+        @store.list_sessions(user_id: user_id).reject { |s| self.class.pseudo_session?(s) }
+      end
+
       def build_store
         Stores.build
       end

data/lib/llmemory/skill_mining/miner.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Llmemory
+  module SkillMining
+    # Skill mining scans an agent's recent episodes (episodic memory) for
+    # repeated, successful trajectories and distills them into reusable skills
+    # (procedural memory). This is Voyager's actual contribution: rather than a
+    # passive, hand-written skill library, procedural memory grows from lived
+    # experience.
+    #
+    # Mining is human-in-the-loop by default: `mine` returns skill *proposals*
+    # and writes nothing. Pass `auto_register: true` to register them directly.
+    # Each registered skill carries provenance { method: "skill_mining",
+    # sources: [{ type: "episode", id: ... }] } so it stays traceable to the
+    # experiences it was distilled from.
+    #
+    # `procedural` must respond to:
+    #   register_skill(name:, body:, description:, kind:, provenance:)
+    class Miner
+      DEFAULT_WINDOW = 20
+      DEFAULT_CONFIDENCE = 0.5
+      VALID_KINDS = %w[prompt template code].freeze
+
+      def initialize(episodic:, procedural:, llm: nil)
+        @episodic = episodic
+        @procedural = procedural
+        @llm = llm || Llmemory::LLM.client
+      end
+
+      # Mines the most recent `window` episodes for reusable skills. When
+      # `outcomes` (an allowlist of outcome labels) is given, only episodes whose
+      # outcome is in the set are considered — a deterministic pre-filter.
+      #
+      # Returns an array of proposal hashes
+      # ({ name:, kind:, body:, description:, confidence: }). When
+      # `auto_register: true`, registers each proposal and returns the new skill
+      # ids instead.
+      def mine(window: DEFAULT_WINDOW, outcomes: nil, auto_register: false)
+        result = []
+        Llmemory::Instrumentation.instrument(:mine_skills, window: window, auto_register: auto_register) do
+          episodes = @episodic.recent_episodes(limit: window)
+          episodes = filter_by_outcome(episodes, outcomes) if outcomes
+          next if episodes.empty?
+
+          proposals = distill(episodes)
+          next if proposals.empty?
+
+          result = auto_register ? register(proposals, episodes) : proposals
+        end
+        result
+      end
+
+      private
+
+      def filter_by_outcome(episodes, outcomes)
+        allowed = Array(outcomes).map { |o| o.to_s.strip.downcase }
+        episodes.select { |ep| allowed.include?(ep.outcome.to_s.strip.downcase) }
+      end
+
+      def register(proposals, episodes)
+        sources = episodes.map(&:id).compact.map { |id| { type: "episode", id: id } }
+        proposals.map do |p|
+          provenance = Llmemory::Provenance.build(
+            method: "skill_mining",
+            sources: sources,
+            confidence: p[:confidence]
+          )
+          @procedural.register_skill(
+            name: p[:name],
+            body: p[:body],
+            description: p[:description],
+            kind: p[:kind],
+            provenance: provenance
+          )
+        end
+      end
+
+      def distill(episodes)
+        response = @llm.invoke(build_prompt(episodes))
+        parse_proposals(response)
+      rescue Llmemory::LLMError
+        []
+      end
+
+      def build_prompt(episodes)
+        episodes_text = episodes.each_with_index.map do |ep, i|
+          "Episode #{i + 1} (outcome: #{ep.outcome || 'n/a'}):\n#{ep.searchable_text}"
+        end.join("\n\n")
+
+        <<~PROMPT
+          You are mining an agent's recent experiences for reusable skills. A skill
+          is a repeatable procedure the agent can apply again: a prompt, a template,
+          or a snippet of code. Only propose a skill when you see a SUCCESSFUL
+          pattern that recurs across episodes — generalize the steps into a reusable
+          procedure. Do not propose one-off actions or failures.
+
+          Recent episodes:
+          #{episodes_text}
+
+          Return a JSON array of objects with keys:
+            "name" (short snake_case identifier),
+            "kind" (one of "prompt", "template", "code"),
+            "body" (the reusable procedure itself),
+            "description" (one sentence on when to apply it),
+            "confidence" (0-1).
+          Return an empty array if no reusable skill can be distilled.
+          Example: [{"name": "rollback_on_deploy_failure", "kind": "prompt",
+          "body": "When a deploy fails, roll back to the last known-good release.",
+          "description": "Recover service after a failed deploy", "confidence": 0.8}]
+        PROMPT
+      end
+
+      def parse_proposals(response)
+        json = extract_json_array(response)
+        return [] unless json
+
+        json.filter_map do |item|
+          next nil unless item.is_a?(Hash)
+          name = (item["name"] || item[:name]).to_s.strip
+          body = (item["body"] || item[:body]).to_s.strip
+          next nil if name.empty? || body.empty?
+
+          {
+            name: name,
+            kind: normalize_kind(item["kind"] || item[:kind]),
+            body: body,
+            description: presence(item["description"] || item[:description]),
+            confidence: normalize_confidence(item["confidence"] || item[:confidence])
+          }
+        end
+      end
+
+      def normalize_kind(value)
+        k = value.to_s.strip.downcase
+        VALID_KINDS.include?(k) ? k : "prompt"
+      end
+
+      def normalize_confidence(value)
+        return DEFAULT_CONFIDENCE if value.nil?
+        v = value.to_f
+        v.between?(0, 1) ? v : DEFAULT_CONFIDENCE
+      end
+
+      def presence(value)
+        s = value.to_s.strip
+        s.empty? ? nil : s
+      end
+
+      def extract_json_array(response)
+        response = response.to_s.strip
+        start_idx = response.index("[")
+        end_idx = response.rindex("]")
+        return nil unless start_idx && end_idx
+
+        JSON.parse(response[start_idx..end_idx])
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/llmemory/skill_mining.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "skill_mining/miner"
+
+module Llmemory
+  module SkillMining
+  end
+end

data/lib/llmemory/tokenizer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Llmemory
+  # Shared word tokenizer for keyword search and lexical scoring (BM25, MMR).
+  # Centralizes the tokenization regex that was duplicated across the codebase.
+  module Tokenizer
+    module_function
+
+    WORD = /\b[a-z0-9]{2,}\b/
+
+    def tokenize(text)
+      text.to_s.downcase.scan(WORD)
+    end
+
+    # Lexical match used by storage-level keyword search. A query is split into
+    # tokens and matched as an OR of per-token substrings, so multi-word queries
+    # work (a single contiguous substring of the whole query is no longer
+    # required) while single-term/partial matches are preserved. An empty query
+    # (no tokens) matches everything, keeping prior "return all" behavior.
+    def matches?(text, query)
+      tokens = tokenize(query)
+      return true if tokens.empty?
+      haystack = text.to_s.downcase
+      tokens.any? { |t| haystack.include?(t) }
+    end
+  end
+end

data/lib/llmemory/vector_store/active_record_store.rb

@@ -7,9 +7,10 @@ module Llmemory
     # Persists embeddings in llmemory_embeddings (pgvector).
     # Use when long_term_store is :active_record so hybrid search finds persisted embeddings.
     class ActiveRecordStore < Base
-      def initialize(embedding_provider: nil)
+      def initialize(embedding_provider: nil, source_type: "edge")
         self.class.load_model!
         @embedding_provider = embedding_provider
+        @source_type = source_type.to_s
       end
 
       def self.load_model!
@@ -29,7 +30,7 @@ module Llmemory
         text_content = (metadata || {}).dig("text") || (metadata || {}).dig(:text)
         rec = Llmemory::VectorStore::ActiveRecordEmbedding.find_or_initialize_by(
           user_id: user_id.to_s,
-          source_type: "edge",
+          source_type: @source_type,
           source_id: id.to_s
         )
         rec.embedding = embedding.to_a.map(&:to_f)
@@ -46,7 +47,7 @@ module Llmemory
         sanitized_vec = vec.map { |v| v.finite? ? v : 0.0 }
         vector_literal = "[#{sanitized_vec.join(',')}]"
         # pgvector cosine distance <=> (0 = same, 2 = opposite); score = 1 - distance for similarity
-        scope = Llmemory::VectorStore::ActiveRecordEmbedding.where(user_id: user_id.to_s)
+        scope = Llmemory::VectorStore::ActiveRecordEmbedding.where(user_id: user_id.to_s, source_type: @source_type)
         rows = scope.select(
           Llmemory::VectorStore::ActiveRecordEmbedding.arel_table[Arel.star],
           Arel.sql("(embedding <=> '#{vector_literal}'::vector) AS distance")

data/lib/llmemory/vector_store/openai_embeddings.rb

@@ -54,14 +54,18 @@ module Llmemory
       end
 
       def fetch_embedding(text)
-        response = connection.post("embeddings") do |req|
-          req.headers["Authorization"] = "Bearer #{@api_key}"
-          req.headers["Content-Type"] = "application/json"
-          req.body = { input: text.to_s.strip, model: @model }.to_json
+        result = nil
+        Llmemory::Instrumentation.instrument(:llm_embed, provider: :openai, model: @model, text_chars: text.to_s.length) do
+          response = connection.post("embeddings") do |req|
+            req.headers["Authorization"] = "Bearer #{@api_key}"
+            req.headers["Content-Type"] = "application/json"
+            req.body = { input: text.to_s.strip, model: @model }.to_json
+          end
+          raise Llmemory::LLMError, "OpenAI Embeddings API error: #{response.body}" unless response.success?
+          body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
+          result = body.dig("data", 0, "embedding")&.map(&:to_f) || Array.new(DEFAULT_DIMS, 0.0)
         end
-        raise Llmemory::LLMError, "OpenAI Embeddings API error: #{response.body}" unless response.success?
-        body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
-        body.dig("data", 0, "embedding")&.map(&:to_f) || Array.new(DEFAULT_DIMS, 0.0)
+        result
       end
 
       def connection

data/lib/llmemory/vector_store.rb

@@ -6,5 +6,19 @@ require_relative "vector_store/memory_store"
 
 module Llmemory
   module VectorStore
+    # Builds a vector store wired to OpenAI embeddings, selecting the backend
+    # from config (:active_record persists in llmemory_embeddings; otherwise
+    # in-process). `source_type` namespaces persisted embeddings so different
+    # memory types (edges, episodes, skills) never collide in the shared table.
+    def self.build(source_type: "edge")
+      embeddings = OpenAIEmbeddings.new
+      store_type = (Llmemory.configuration.long_term_store || :memory).to_s.to_sym
+      if store_type == :active_record || store_type == :activerecord
+        require_relative "vector_store/active_record_store"
+        ActiveRecordStore.new(embedding_provider: embeddings, source_type: source_type)
+      else
+        MemoryStore.new(embedding_provider: embeddings)
+      end
+    end
   end
 end

data/lib/llmemory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Llmemory
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

data/lib/llmemory.rb

@@ -3,6 +3,8 @@
 require_relative "llmemory/version"
 require_relative "llmemory/configuration"
 require_relative "llmemory/provenance"
+require_relative "llmemory/tokenizer"
+require_relative "llmemory/instrumentation"
 require_relative "llmemory/memory_module"
 require_relative "llmemory/forget_log"
 require_relative "llmemory/llm"
@@ -14,6 +16,7 @@ require_relative "llmemory/vector_store"
 require_relative "llmemory/maintenance"
 require_relative "llmemory/extractors"
 require_relative "llmemory/reflection"
+require_relative "llmemory/skill_mining"
 require_relative "llmemory/actions"
 require_relative "llmemory/memory"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llmemory
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - llmemory
@@ -106,21 +106,33 @@ files:
 - LICENSE.txt
 - README.md
 - app/controllers/llmemory/dashboard/application_controller.rb
+- app/controllers/llmemory/dashboard/episodic_controller.rb
+- app/controllers/llmemory/dashboard/forget_log_controller.rb
 - app/controllers/llmemory/dashboard/graph_controller.rb
 - app/controllers/llmemory/dashboard/long_term_controller.rb
+- app/controllers/llmemory/dashboard/maintenance_controller.rb
+- app/controllers/llmemory/dashboard/procedural_controller.rb
+- app/controllers/llmemory/dashboard/reflection_controller.rb
 - app/controllers/llmemory/dashboard/search_controller.rb
 - app/controllers/llmemory/dashboard/short_term_controller.rb
 - app/controllers/llmemory/dashboard/stats_controller.rb
 - app/controllers/llmemory/dashboard/users_controller.rb
+- app/controllers/llmemory/dashboard/working_controller.rb
 - app/views/layouts/application.html.erb
+- app/views/llmemory/dashboard/episodic/index.html.erb
+- app/views/llmemory/dashboard/forget_log/show.html.erb
 - app/views/llmemory/dashboard/graph/index.html.erb
 - app/views/llmemory/dashboard/long_term/categories.html.erb
 - app/views/llmemory/dashboard/long_term/index.html.erb
+- app/views/llmemory/dashboard/maintenance/show.html.erb
+- app/views/llmemory/dashboard/procedural/index.html.erb
+- app/views/llmemory/dashboard/reflection/show.html.erb
 - app/views/llmemory/dashboard/search/index.html.erb
 - app/views/llmemory/dashboard/short_term/show.html.erb
 - app/views/llmemory/dashboard/stats/index.html.erb
 - app/views/llmemory/dashboard/users/index.html.erb
 - app/views/llmemory/dashboard/users/show.html.erb
+- app/views/llmemory/dashboard/working/show.html.erb
 - config/routes.rb
 - exe/llmemory
 - exe/llmemory-mcp
@@ -131,6 +143,8 @@ files:
 - lib/llmemory/actions/reason.rb
 - lib/llmemory/cli.rb
 - lib/llmemory/cli/commands/base.rb
+- lib/llmemory/cli/commands/episodic.rb
+- lib/llmemory/cli/commands/forget_log.rb
 - lib/llmemory/cli/commands/long_term.rb
 - lib/llmemory/cli/commands/long_term/categories.rb
 - lib/llmemory/cli/commands/long_term/edges.rb
@@ -138,11 +152,15 @@ files:
 - lib/llmemory/cli/commands/long_term/graph.rb
 - lib/llmemory/cli/commands/long_term/nodes.rb
 - lib/llmemory/cli/commands/long_term/resources.rb
+- lib/llmemory/cli/commands/maintain.rb
 - lib/llmemory/cli/commands/mcp.rb
+- lib/llmemory/cli/commands/mine_skills.rb
+- lib/llmemory/cli/commands/procedural.rb
 - lib/llmemory/cli/commands/search.rb
 - lib/llmemory/cli/commands/short_term.rb
 - lib/llmemory/cli/commands/stats.rb
 - lib/llmemory/cli/commands/users.rb
+- lib/llmemory/cli/commands/working.rb
 - lib/llmemory/configuration.rb
 - lib/llmemory/dashboard.rb
 - lib/llmemory/dashboard/engine.rb
@@ -150,6 +168,7 @@ files:
 - lib/llmemory/extractors/entity_relation_extractor.rb
 - lib/llmemory/extractors/fact_extractor.rb
 - lib/llmemory/forget_log.rb
+- lib/llmemory/instrumentation.rb
 - lib/llmemory/llm.rb
 - lib/llmemory/llm/anthropic.rb
 - lib/llmemory/llm/base.rb
@@ -159,7 +178,10 @@ files:
 - lib/llmemory/long_term/episodic/episode.rb
 - lib/llmemory/long_term/episodic/memory.rb
 - lib/llmemory/long_term/episodic/storage.rb
+- lib/llmemory/long_term/episodic/storages/active_record_models.rb
+- lib/llmemory/long_term/episodic/storages/active_record_storage.rb
 - lib/llmemory/long_term/episodic/storages/base.rb
+- lib/llmemory/long_term/episodic/storages/database_storage.rb
 - lib/llmemory/long_term/episodic/storages/file_storage.rb
 - lib/llmemory/long_term/episodic/storages/memory_storage.rb
 - lib/llmemory/long_term/file_based.rb
@@ -190,23 +212,36 @@ files:
 - lib/llmemory/long_term/procedural/memory.rb
 - lib/llmemory/long_term/procedural/skill.rb
 - lib/llmemory/long_term/procedural/storage.rb
+- lib/llmemory/long_term/procedural/storages/active_record_models.rb
+- lib/llmemory/long_term/procedural/storages/active_record_storage.rb
 - lib/llmemory/long_term/procedural/storages/base.rb
+- lib/llmemory/long_term/procedural/storages/database_storage.rb
 - lib/llmemory/long_term/procedural/storages/file_storage.rb
 - lib/llmemory/long_term/procedural/storages/memory_storage.rb
 - lib/llmemory/maintenance.rb
+- lib/llmemory/maintenance/cognitive_pass.rb
 - lib/llmemory/maintenance/consolidator.rb
 - lib/llmemory/maintenance/reindexer.rb
 - lib/llmemory/maintenance/runner.rb
 - lib/llmemory/maintenance/summarizer.rb
+- lib/llmemory/maintenance/ttl_expiry.rb
 - lib/llmemory/mcp.rb
 - lib/llmemory/mcp/authentication.rb
 - lib/llmemory/mcp/server.rb
 - lib/llmemory/mcp/tools/memory_add_message.rb
 - lib/llmemory/mcp/tools/memory_consolidate.rb
+- lib/llmemory/mcp/tools/memory_episode_record.rb
+- lib/llmemory/mcp/tools/memory_episodes.rb
+- lib/llmemory/mcp/tools/memory_forget.rb
 - lib/llmemory/mcp/tools/memory_info.rb
+- lib/llmemory/mcp/tools/memory_maintain.rb
+- lib/llmemory/mcp/tools/memory_mine_skills.rb
 - lib/llmemory/mcp/tools/memory_retrieve.rb
 - lib/llmemory/mcp/tools/memory_save.rb
 - lib/llmemory/mcp/tools/memory_search.rb
+- lib/llmemory/mcp/tools/memory_skill_register.rb
+- lib/llmemory/mcp/tools/memory_skill_report.rb
+- lib/llmemory/mcp/tools/memory_skills.rb
 - lib/llmemory/mcp/tools/memory_stats.rb
 - lib/llmemory/mcp/tools/memory_timeline.rb
 - lib/llmemory/mcp/tools/memory_timeline_context.rb
@@ -235,6 +270,9 @@ files:
 - lib/llmemory/short_term/stores/memory_store.rb
 - lib/llmemory/short_term/stores/postgres_store.rb
 - lib/llmemory/short_term/stores/redis_store.rb
+- lib/llmemory/skill_mining.rb
+- lib/llmemory/skill_mining/miner.rb
+- lib/llmemory/tokenizer.rb
 - lib/llmemory/vector_store.rb
 - lib/llmemory/vector_store/active_record_embedding.rb
 - lib/llmemory/vector_store/active_record_store.rb