llmemory 0.2.1 → 0.2.3

data/lib/llmemory/skill_mining/miner.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Llmemory
+  module SkillMining
+    # Skill mining scans an agent's recent episodes (episodic memory) for
+    # repeated, successful trajectories and distills them into reusable skills
+    # (procedural memory). This is Voyager's actual contribution: rather than a
+    # passive, hand-written skill library, procedural memory grows from lived
+    # experience.
+    #
+    # Mining is human-in-the-loop by default: `mine` returns skill *proposals*
+    # and writes nothing. Pass `auto_register: true` to register them directly.
+    # Each registered skill carries provenance { method: "skill_mining",
+    # sources: [{ type: "episode", id: ... }] } so it stays traceable to the
+    # experiences it was distilled from.
+    #
+    # `procedural` must respond to:
+    #   register_skill(name:, body:, description:, kind:, provenance:)
+    class Miner
+      DEFAULT_WINDOW = 20
+      DEFAULT_CONFIDENCE = 0.5
+      VALID_KINDS = %w[prompt template code].freeze
+
+      def initialize(episodic:, procedural:, llm: nil)
+        @episodic = episodic
+        @procedural = procedural
+        @llm = llm || Llmemory::LLM.client
+      end
+
+      # Mines the most recent `window` episodes for reusable skills. When
+      # `outcomes` (an allowlist of outcome labels) is given, only episodes whose
+      # outcome is in the set are considered — a deterministic pre-filter.
+      #
+      # Returns an array of proposal hashes
+      # ({ name:, kind:, body:, description:, confidence: }). When
+      # `auto_register: true`, registers each proposal and returns the new skill
+      # ids instead.
+      def mine(window: DEFAULT_WINDOW, outcomes: nil, auto_register: false)
+        result = []
+        Llmemory::Instrumentation.instrument(:mine_skills, window: window, auto_register: auto_register) do
+          episodes = @episodic.recent_episodes(limit: window)
+          episodes = filter_by_outcome(episodes, outcomes) if outcomes
+          next if episodes.empty?
+
+          proposals = distill(episodes)
+          next if proposals.empty?
+
+          result = auto_register ? register(proposals, episodes) : proposals
+        end
+        result
+      end
+
+      private
+
+      def filter_by_outcome(episodes, outcomes)
+        allowed = Array(outcomes).map { |o| o.to_s.strip.downcase }
+        episodes.select { |ep| allowed.include?(ep.outcome.to_s.strip.downcase) }
+      end
+
+      def register(proposals, episodes)
+        sources = episodes.map(&:id).compact.map { |id| { type: "episode", id: id } }
+        proposals.map do |p|
+          provenance = Llmemory::Provenance.build(
+            method: "skill_mining",
+            sources: sources,
+            confidence: p[:confidence]
+          )
+          @procedural.register_skill(
+            name: p[:name],
+            body: p[:body],
+            description: p[:description],
+            kind: p[:kind],
+            provenance: provenance
+          )
+        end
+      end
+
+      def distill(episodes)
+        response = @llm.invoke(build_prompt(episodes))
+        parse_proposals(response)
+      rescue Llmemory::LLMError
+        []
+      end
+
+      def build_prompt(episodes)
+        episodes_text = episodes.each_with_index.map do |ep, i|
+          "Episode #{i + 1} (outcome: #{ep.outcome || 'n/a'}):\n#{ep.searchable_text}"
+        end.join("\n\n")
+
+        <<~PROMPT
+          You are mining an agent's recent experiences for reusable skills. A skill
+          is a repeatable procedure the agent can apply again: a prompt, a template,
+          or a snippet of code. Only propose a skill when you see a SUCCESSFUL
+          pattern that recurs across episodes — generalize the steps into a reusable
+          procedure. Do not propose one-off actions or failures.
+
+          Recent episodes:
+          #{episodes_text}
+
+          Return a JSON array of objects with keys:
+            "name" (short snake_case identifier),
+            "kind" (one of "prompt", "template", "code"),
+            "body" (the reusable procedure itself),
+            "description" (one sentence on when to apply it),
+            "confidence" (0-1).
+          Return an empty array if no reusable skill can be distilled.
+          Example: [{"name": "rollback_on_deploy_failure", "kind": "prompt",
+          "body": "When a deploy fails, roll back to the last known-good release.",
+          "description": "Recover service after a failed deploy", "confidence": 0.8}]
+        PROMPT
+      end
+
+      def parse_proposals(response)
+        json = extract_json_array(response)
+        return [] unless json
+
+        json.filter_map do |item|
+          next nil unless item.is_a?(Hash)
+          name = (item["name"] || item[:name]).to_s.strip
+          body = (item["body"] || item[:body]).to_s.strip
+          next nil if name.empty? || body.empty?
+
+          {
+            name: name,
+            kind: normalize_kind(item["kind"] || item[:kind]),
+            body: body,
+            description: presence(item["description"] || item[:description]),
+            confidence: normalize_confidence(item["confidence"] || item[:confidence])
+          }
+        end
+      end
+
+      def normalize_kind(value)
+        k = value.to_s.strip.downcase
+        VALID_KINDS.include?(k) ? k : "prompt"
+      end
+
+      def normalize_confidence(value)
+        return DEFAULT_CONFIDENCE if value.nil?
+        v = value.to_f
+        v.between?(0, 1) ? v : DEFAULT_CONFIDENCE
+      end
+
+      def presence(value)
+        s = value.to_s.strip
+        s.empty? ? nil : s
+      end
+
+      def extract_json_array(response)
+        response = response.to_s.strip
+        start_idx = response.index("[")
+        end_idx = response.rindex("]")
+        return nil unless start_idx && end_idx
+
+        JSON.parse(response[start_idx..end_idx])
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/llmemory/skill_mining.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "skill_mining/miner"
+
+module Llmemory
+  module SkillMining
+  end
+end

data/lib/llmemory/vector_store/active_record_store.rb

@@ -7,10 +7,11 @@ module Llmemory
     # Persists embeddings in llmemory_embeddings (pgvector).
     # Use when long_term_store is :active_record so hybrid search finds persisted embeddings.
     class ActiveRecordStore < Base
-      def initialize(embedding_provider: nil, source_type: "edge")
+      def initialize(embedding_provider: nil, source_type: "edge", cipher: nil)
         self.class.load_model!
         @embedding_provider = embedding_provider
         @source_type = source_type.to_s
+        @cipher = cipher || Llmemory.build_cipher
       end
 
       def self.load_model!
@@ -34,7 +35,7 @@ module Llmemory
           source_id: id.to_s
         )
         rec.embedding = embedding.to_a.map(&:to_f)
-        rec.text_content = text_content
+        rec.text_content = encrypt_text_content(text_content)
         rec.save!
         id
       end
@@ -58,7 +59,11 @@ module Llmemory
           {
             id: r.source_id,
             score: score,
-            metadata: { "text" => r.text_content, "created_at" => r.created_at, "user_id" => r.user_id }
+            metadata: {
+              "text" => decrypt_text_content(r.text_content),
+              "created_at" => r.created_at,
+              "user_id" => r.user_id
+            }
           }
         end
       end
@@ -69,6 +74,22 @@ module Llmemory
         query_embedding = @embedding_provider.embed(query_text)
         search(query_embedding, top_k: top_k, user_id: user_id)
       end
+
+      private
+
+      def encrypt_text_content(text)
+        return text if text.nil? || text.to_s.empty?
+        return text unless @cipher.enabled?
+
+        @cipher.encrypt(text.to_s)
+      end
+
+      def decrypt_text_content(text)
+        return text if text.nil?
+        return text unless text.is_a?(String) && @cipher.enabled? && @cipher.encrypted?(text)
+
+        @cipher.decrypt(text)
+      end
     end
   end
 end

data/lib/llmemory/vector_store/memory_store.rb

@@ -5,9 +5,10 @@ require_relative "base"
 module Llmemory
   module VectorStore
     class MemoryStore < Base
-      def initialize(embedding_provider: nil)
+      def initialize(embedding_provider: nil, cipher: nil)
         @entries = {}
         @embedding_provider = embedding_provider
+        @cipher = cipher || Llmemory.build_cipher
       end
 
       def embed(text)
@@ -17,7 +18,13 @@ module Llmemory
 
       def store(id:, embedding:, metadata: {}, user_id: nil)
         key = user_id ? "#{user_id}:#{id}" : id.to_s
-        @entries[key] = { embedding: embedding.to_a.map(&:to_f), metadata: (metadata || {}).merge("user_id" => user_id) }
+        meta = (metadata || {}).dup
+        if meta["text"] && @cipher.enabled?
+          meta["text"] = @cipher.encrypt(meta["text"].to_s)
+        elsif meta[:text] && @cipher.enabled?
+          meta[:text] = @cipher.encrypt(meta[:text].to_s)
+        end
+        @entries[key] = { embedding: embedding.to_a.map(&:to_f), metadata: meta.merge("user_id" => user_id) }
         id
       end
 
@@ -27,7 +34,7 @@ module Llmemory
         entries = user_id ? @entries.select { |k, _| k.to_s.start_with?("#{user_id}:") } : @entries
         scores = entries.map do |id, data|
           sim = cosine_similarity(query, data[:embedding])
-          { id: id, score: sim, metadata: data[:metadata] }
+          { id: id, score: sim, metadata: decrypt_metadata(data[:metadata]) }
         end
         scores.sort_by { |s| -s[:score] }.first(top_k)
       end
@@ -40,6 +47,19 @@ module Llmemory
 
       private
 
+      def decrypt_metadata(meta)
+        return meta unless meta.is_a?(Hash) && @cipher.enabled?
+
+        out = meta.dup
+        text = out["text"] || out[:text]
+        if text.is_a?(String) && @cipher.encrypted?(text)
+          decrypted = @cipher.decrypt(text)
+          out["text"] = decrypted
+          out[:text] = decrypted
+        end
+        out
+      end
+
       def cosine_similarity(a, b)
         return 0.0 if a.size != b.size || a.empty?
         dot = a.zip(b).sum { |x, y| x * y }

data/lib/llmemory/vector_store/openai_embeddings.rb

@@ -54,14 +54,18 @@ module Llmemory
       end
 
       def fetch_embedding(text)
-        response = connection.post("embeddings") do |req|
-          req.headers["Authorization"] = "Bearer #{@api_key}"
-          req.headers["Content-Type"] = "application/json"
-          req.body = { input: text.to_s.strip, model: @model }.to_json
+        result = nil
+        Llmemory::Instrumentation.instrument(:llm_embed, provider: :openai, model: @model, text_chars: text.to_s.length) do
+          response = connection.post("embeddings") do |req|
+            req.headers["Authorization"] = "Bearer #{@api_key}"
+            req.headers["Content-Type"] = "application/json"
+            req.body = { input: text.to_s.strip, model: @model }.to_json
+          end
+          raise Llmemory::LLMError, "OpenAI Embeddings API error: #{response.body}" unless response.success?
+          body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
+          result = body.dig("data", 0, "embedding")&.map(&:to_f) || Array.new(DEFAULT_DIMS, 0.0)
         end
-        raise Llmemory::LLMError, "OpenAI Embeddings API error: #{response.body}" unless response.success?
-        body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
-        body.dig("data", 0, "embedding")&.map(&:to_f) || Array.new(DEFAULT_DIMS, 0.0)
+        result
       end
 
       def connection

data/lib/llmemory/vector_store.rb

@@ -10,14 +10,15 @@ module Llmemory
     # from config (:active_record persists in llmemory_embeddings; otherwise
     # in-process). `source_type` namespaces persisted embeddings so different
     # memory types (edges, episodes, skills) never collide in the shared table.
-    def self.build(source_type: "edge")
+    def self.build(source_type: "edge", cipher: nil)
+      resolved_cipher = cipher || Llmemory.build_cipher
       embeddings = OpenAIEmbeddings.new
       store_type = (Llmemory.configuration.long_term_store || :memory).to_s.to_sym
       if store_type == :active_record || store_type == :activerecord
         require_relative "vector_store/active_record_store"
-        ActiveRecordStore.new(embedding_provider: embeddings, source_type: source_type)
+        ActiveRecordStore.new(embedding_provider: embeddings, source_type: source_type, cipher: resolved_cipher)
       else
-        MemoryStore.new(embedding_provider: embeddings)
+        MemoryStore.new(embedding_provider: embeddings, cipher: resolved_cipher)
       end
     end
   end

data/lib/llmemory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Llmemory
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 end

data/lib/llmemory.rb

@@ -4,6 +4,7 @@ require_relative "llmemory/version"
 require_relative "llmemory/configuration"
 require_relative "llmemory/provenance"
 require_relative "llmemory/tokenizer"
+require_relative "llmemory/instrumentation"
 require_relative "llmemory/memory_module"
 require_relative "llmemory/forget_log"
 require_relative "llmemory/llm"
@@ -15,6 +16,7 @@ require_relative "llmemory/vector_store"
 require_relative "llmemory/maintenance"
 require_relative "llmemory/extractors"
 require_relative "llmemory/reflection"
+require_relative "llmemory/skill_mining"
 require_relative "llmemory/actions"
 require_relative "llmemory/memory"
 
@@ -24,3 +26,5 @@ module Llmemory
   class StoreError < Error; end
   class LLMError < Error; end
 end
+
+require_relative "llmemory/crypto/cipher"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llmemory
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - llmemory
@@ -106,21 +106,33 @@ files:
 - LICENSE.txt
 - README.md
 - app/controllers/llmemory/dashboard/application_controller.rb
+- app/controllers/llmemory/dashboard/episodic_controller.rb
+- app/controllers/llmemory/dashboard/forget_log_controller.rb
 - app/controllers/llmemory/dashboard/graph_controller.rb
 - app/controllers/llmemory/dashboard/long_term_controller.rb
+- app/controllers/llmemory/dashboard/maintenance_controller.rb
+- app/controllers/llmemory/dashboard/procedural_controller.rb
+- app/controllers/llmemory/dashboard/reflection_controller.rb
 - app/controllers/llmemory/dashboard/search_controller.rb
 - app/controllers/llmemory/dashboard/short_term_controller.rb
 - app/controllers/llmemory/dashboard/stats_controller.rb
 - app/controllers/llmemory/dashboard/users_controller.rb
+- app/controllers/llmemory/dashboard/working_controller.rb
 - app/views/layouts/application.html.erb
+- app/views/llmemory/dashboard/episodic/index.html.erb
+- app/views/llmemory/dashboard/forget_log/show.html.erb
 - app/views/llmemory/dashboard/graph/index.html.erb
 - app/views/llmemory/dashboard/long_term/categories.html.erb
 - app/views/llmemory/dashboard/long_term/index.html.erb
+- app/views/llmemory/dashboard/maintenance/show.html.erb
+- app/views/llmemory/dashboard/procedural/index.html.erb
+- app/views/llmemory/dashboard/reflection/show.html.erb
 - app/views/llmemory/dashboard/search/index.html.erb
 - app/views/llmemory/dashboard/short_term/show.html.erb
 - app/views/llmemory/dashboard/stats/index.html.erb
 - app/views/llmemory/dashboard/users/index.html.erb
 - app/views/llmemory/dashboard/users/show.html.erb
+- app/views/llmemory/dashboard/working/show.html.erb
 - config/routes.rb
 - exe/llmemory
 - exe/llmemory-mcp
@@ -140,7 +152,9 @@ files:
 - lib/llmemory/cli/commands/long_term/graph.rb
 - lib/llmemory/cli/commands/long_term/nodes.rb
 - lib/llmemory/cli/commands/long_term/resources.rb
+- lib/llmemory/cli/commands/maintain.rb
 - lib/llmemory/cli/commands/mcp.rb
+- lib/llmemory/cli/commands/mine_skills.rb
 - lib/llmemory/cli/commands/procedural.rb
 - lib/llmemory/cli/commands/search.rb
 - lib/llmemory/cli/commands/short_term.rb
@@ -148,12 +162,15 @@ files:
 - lib/llmemory/cli/commands/users.rb
 - lib/llmemory/cli/commands/working.rb
 - lib/llmemory/configuration.rb
+- lib/llmemory/crypto/cipher.rb
+- lib/llmemory/crypto/field_helpers.rb
 - lib/llmemory/dashboard.rb
 - lib/llmemory/dashboard/engine.rb
 - lib/llmemory/extractors.rb
 - lib/llmemory/extractors/entity_relation_extractor.rb
 - lib/llmemory/extractors/fact_extractor.rb
 - lib/llmemory/forget_log.rb
+- lib/llmemory/instrumentation.rb
 - lib/llmemory/llm.rb
 - lib/llmemory/llm/anthropic.rb
 - lib/llmemory/llm/base.rb
@@ -204,10 +221,12 @@ files:
 - lib/llmemory/long_term/procedural/storages/file_storage.rb
 - lib/llmemory/long_term/procedural/storages/memory_storage.rb
 - lib/llmemory/maintenance.rb
+- lib/llmemory/maintenance/cognitive_pass.rb
 - lib/llmemory/maintenance/consolidator.rb
 - lib/llmemory/maintenance/reindexer.rb
 - lib/llmemory/maintenance/runner.rb
 - lib/llmemory/maintenance/summarizer.rb
+- lib/llmemory/maintenance/ttl_expiry.rb
 - lib/llmemory/mcp.rb
 - lib/llmemory/mcp/authentication.rb
 - lib/llmemory/mcp/server.rb
@@ -217,6 +236,8 @@ files:
 - lib/llmemory/mcp/tools/memory_episodes.rb
 - lib/llmemory/mcp/tools/memory_forget.rb
 - lib/llmemory/mcp/tools/memory_info.rb
+- lib/llmemory/mcp/tools/memory_maintain.rb
+- lib/llmemory/mcp/tools/memory_mine_skills.rb
 - lib/llmemory/mcp/tools/memory_retrieve.rb
 - lib/llmemory/mcp/tools/memory_save.rb
 - lib/llmemory/mcp/tools/memory_search.rb
@@ -251,6 +272,8 @@ files:
 - lib/llmemory/short_term/stores/memory_store.rb
 - lib/llmemory/short_term/stores/postgres_store.rb
 - lib/llmemory/short_term/stores/redis_store.rb
+- lib/llmemory/skill_mining.rb
+- lib/llmemory/skill_mining/miner.rb
 - lib/llmemory/tokenizer.rb
 - lib/llmemory/vector_store.rb
 - lib/llmemory/vector_store/active_record_embedding.rb