llmemory 0.1.16 → 0.2.0

data/lib/llmemory/long_term/episodic/storages/file_storage.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        class FileStorage < Base
+          def initialize(base_path: nil)
+            @base_path = base_path || Llmemory.configuration.long_term_storage_path || "./llmemory_data"
+            @base_path = File.expand_path(@base_path)
+          end
+
+          def save_episode(user_id, episode)
+            id = episode[:id] || episode["id"] || "ep_#{next_seq(user_id)}"
+            data = stringify_for_json(episode).merge("id" => id, "user_id" => user_id)
+            data["created_at"] ||= Time.now.iso8601
+            File.write(episode_path(user_id, id), JSON.generate(data))
+            id
+          end
+
+          def get_episode(user_id, id)
+            path = episode_path(user_id, id)
+            return nil unless File.file?(path)
+            load_episode(path)
+          end
+
+          def list_episodes(user_id, limit: nil)
+            sorted = all_episodes(user_id).sort_by { |e| e[:created_at] }.reverse
+            limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
+          end
+
+          def search_episodes(user_id, query)
+            q = query.to_s.downcase
+            return list_episodes(user_id) if q.strip.empty?
+            all_episodes(user_id).select { |e| episode_text(e).downcase.include?(q) }
+          end
+
+          def count_episodes(user_id)
+            dir = user_path(user_id, "episodes")
+            return 0 unless Dir.exist?(dir)
+            Dir.children(dir).count { |f| f.end_with?(".json") }
+          end
+
+          def delete_episodes(user_id, ids)
+            Array(ids).map(&:to_s).count do |id|
+              path = episode_path(user_id, id)
+              next false unless File.file?(path)
+              File.delete(path)
+              true
+            end
+          end
+
+          def list_users
+            return [] unless Dir.exist?(@base_path)
+            Dir.children(@base_path).select { |d| Dir.exist?(File.join(@base_path, d, "episodes")) }
+          end
+
+          private
+
+          def all_episodes(user_id)
+            dir = user_path(user_id, "episodes")
+            return [] unless Dir.exist?(dir)
+            Dir.children(dir).select { |f| f.end_with?(".json") }.map { |f| load_episode(File.join(dir, f)) }.compact
+          end
+
+          def load_episode(path)
+            data = JSON.parse(File.read(path), symbolize_names: true)
+            data[:created_at] = parse_time(data[:created_at])
+            data
+          rescue JSON::ParserError
+            nil
+          end
+
+          def episode_text(episode)
+            parts = [episode[:summary], episode[:outcome]]
+            Array(episode[:steps]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << s[:observation] << s[:action] << s[:result]
+            end
+            parts.compact.join("\n")
+          end
+
+          def stringify_for_json(episode)
+            JSON.parse(JSON.generate(episode))
+          end
+
+          def user_path(user_id, *parts)
+            safe = user_id.to_s.gsub(%r{[^\w\-.]}, "_")
+            File.join(@base_path, safe, *parts)
+          end
+
+          def episode_path(user_id, id)
+            dir = user_path(user_id, "episodes")
+            FileUtils.mkdir_p(dir)
+            File.join(dir, "#{id}.json")
+          end
+
+          def meta_path(user_id)
+            FileUtils.mkdir_p(user_path(user_id))
+            File.join(user_path(user_id), "meta.json")
+          end
+
+          def next_seq(user_id)
+            path = meta_path(user_id)
+            meta = File.file?(path) ? JSON.parse(File.read(path)) : {}
+            meta["episode_id_seq"] = (meta["episode_id_seq"] || 0) + 1
+            File.write(path, JSON.generate(meta))
+            meta["episode_id_seq"]
+          end
+
+          def parse_time(val)
+            return val if val.is_a?(Time)
+            Time.parse(val.to_s)
+          rescue ArgumentError
+            Time.now
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/memory_storage.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        class MemoryStorage < Base
+          def initialize
+            @episodes = Hash.new { |h, k| h[k] = [] }
+            @seq = 0
+          end
+
+          def save_episode(user_id, episode)
+            @seq += 1
+            id = episode[:id] || episode["id"] || "ep_#{@seq}"
+            record = symbolize(episode).merge(id: id, user_id: user_id)
+            record[:created_at] ||= Time.now
+            @episodes[user_id] << record
+            id
+          end
+
+          def get_episode(user_id, id)
+            @episodes[user_id].find { |e| e[:id] == id }
+          end
+
+          def list_episodes(user_id, limit: nil)
+            sorted = @episodes[user_id].sort_by { |e| e[:created_at] }.reverse
+            limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
+          end
+
+          def search_episodes(user_id, query)
+            q = query.to_s.downcase
+            return list_episodes(user_id) if q.strip.empty?
+            @episodes[user_id].select { |e| episode_text(e).downcase.include?(q) }
+          end
+
+          def count_episodes(user_id)
+            @episodes[user_id].size
+          end
+
+          def delete_episodes(user_id, ids)
+            ids = Array(ids).map(&:to_s)
+            before = @episodes[user_id].size
+            @episodes[user_id].reject! { |e| ids.include?(e[:id].to_s) }
+            before - @episodes[user_id].size
+          end
+
+          def list_users
+            @episodes.keys
+          end
+
+          private
+
+          def symbolize(hash)
+            hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
+          end
+
+          def episode_text(episode)
+            parts = [episode[:summary], episode[:outcome]]
+            Array(episode[:steps]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << (s[:observation] || s["observation"])
+              parts << (s[:action] || s["action"])
+              parts << (s[:result] || s["result"])
+            end
+            parts.compact.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "episodic/episode"
+require_relative "episodic/storage"
+require_relative "episodic/memory"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+    end
+  end
+end

data/lib/llmemory/long_term/file_based/memory.rb

@@ -5,11 +5,14 @@ require_relative "item"
 require_relative "category"
 require_relative "storage"
 require_relative "../../noise_filter"
+require_relative "../../memory_module"
 
 module Llmemory
   module LongTerm
     module FileBased
       class Memory
+        include Llmemory::MemoryModule
+
         def initialize(user_id:, storage: nil, llm: nil, extractor: nil)
           @user_id = user_id
           @storage = storage || Storages.build
@@ -63,6 +66,7 @@ module Llmemory
 
           items.first(top_k).each do |i|
             out << {
+              id: i[:id] || i["id"],
               text: i[:content] || i["content"],
               timestamp: i[:created_at] || i["created_at"],
               score: 1.0,
@@ -72,6 +76,7 @@ module Llmemory
           end
           resources.first([top_k - out.size, 0].max).each do |r|
             out << {
+              id: r[:id] || r["id"],
               text: r[:text] || r["text"],
               timestamp: r[:created_at] || r["created_at"],
               score: 0.9
@@ -85,6 +90,47 @@ module Llmemory
           out
         end
 
+        # Stores a single fact produced outside the extraction flow (e.g. by
+        # reflection over episodes), preserving caller-supplied provenance so the
+        # insight remains traceable to its source. Returns the item id.
+        def remember_fact(content:, category: "general", importance: 0.6, provenance: nil)
+          return nil if content.to_s.strip.empty?
+          @storage.save_item(
+            @user_id,
+            category: category.to_s,
+            content: content.to_s,
+            source_resource_id: nil,
+            importance: importance,
+            provenance: provenance
+          )
+        end
+
+        # --- MemoryModule uniform interface ---
+
+        def write(payload, **_meta)
+          memorize(payload)
+        end
+
+        def list(user_id: nil, limit: nil)
+          @storage.list_items(user_id: user_id || @user_id, limit: limit)
+        end
+
+        def stats(user_id: nil)
+          { items: @storage.count_items(user_id: user_id || @user_id) }
+        end
+
+        # Removes items/resources by id and records the removal in the audit log.
+        def forget(ids:, reason: nil)
+          requested = Array(ids).map(&:to_s)
+          existing = (@storage.get_all_items(@user_id) + @storage.get_all_resources(@user_id))
+            .map { |r| (r[:id] || r["id"]).to_s }
+          removed = requested & existing
+          @storage.archive_items(@user_id, removed)
+          @storage.archive_resources(@user_id, removed)
+          forget_log.record(@user_id, memory_type: "file_based", ids: removed, reason: reason)
+          removed.size
+        end
+
         attr_reader :storage, :user_id
 
         private

data/lib/llmemory/long_term/graph_based/memory.rb

@@ -6,11 +6,14 @@ require_relative "knowledge_graph"
 require_relative "conflict_resolver"
 require_relative "storage"
 require_relative "../../noise_filter"
+require_relative "../../memory_module"
 
 module Llmemory
   module LongTerm
     module GraphBased
       class Memory
+        include Llmemory::MemoryModule
+
         def initialize(user_id:, storage: nil, vector_store: nil, llm: nil, extractor: nil)
           @user_id = user_id
           @graph_storage = storage || Storages.build
@@ -88,6 +91,7 @@ module Llmemory
           results = hybrid_search(query, top_k: top_k)
           results.map do |r|
             {
+              id: r[:id],
               text: r[:text],
               timestamp: r[:created_at] || r[:timestamp],
               score: r[:score] || 1.0,
@@ -102,6 +106,29 @@ module Llmemory
           @graph_storage
         end
 
+        # --- MemoryModule uniform interface ---
+
+        def write(payload, **_meta)
+          memorize(payload)
+        end
+
+        def list(user_id: nil, limit: nil)
+          @graph_storage.list_nodes(user_id || @user_id, limit: limit)
+        end
+
+        def stats(user_id: nil)
+          uid = user_id || @user_id
+          { nodes: @graph_storage.count_nodes(uid), edges: @graph_storage.count_edges(uid) }
+        end
+
+        # Forgetting a knowledge graph is not a simple delete-by-id: edges are
+        # soft-archived and nodes can be left orphaned. A dedicated graph
+        # edge/node lifecycle (with orphan handling) is a deliberate follow-up.
+        def forget(ids:, reason: nil)
+          raise NotImplementedError,
+            "Graph forget is not implemented yet; edge/node lifecycle (archival + orphan handling) is a follow-up."
+        end
+
         private
 
         def build_vector_store
@@ -127,7 +154,7 @@ module Llmemory
           out = vector_results.map do |v|
             id = v[:id] || v["id"]
             meta = v[:metadata] || v["metadata"] || {}
-            { text: meta["text"] || meta[:text] || id.to_s, score: v[:score] || v["score"] || 1.0, created_at: meta["created_at"] || meta[:created_at] }
+            { id: id, text: meta["text"] || meta[:text] || id.to_s, score: v[:score] || v["score"] || 1.0, created_at: meta["created_at"] || meta[:created_at] }
           end
 
           node_ids = out.flat_map { |r| extract_node_ids_from_text(r[:text]) }.compact.uniq
@@ -139,7 +166,7 @@ module Llmemory
               subj = @kg.find_node_by_id(e.subject_id)
               obj = @kg.find_node_by_id(e.target_id)
               edge_text = "#{subj&.name} #{e.predicate} #{obj&.name}"
-              out << { text: edge_text, score: 0.85, created_at: e.created_at } unless out.any? { |o| o[:text] == edge_text }
+              out << { id: (e.id ? "edge_#{e.id}" : nil), text: edge_text, score: 0.85, created_at: e.created_at } unless out.any? { |o| o[:text] == edge_text }
             end
           end
 
@@ -152,7 +179,7 @@ module Llmemory
               obj = @kg.find_node_by_id(e.target_id)
               next unless subj && obj
               edge_text = "#{subj.name} #{e.predicate} #{obj.name}"
-              out << { text: edge_text, score: 0.7, created_at: e.created_at }
+              out << { id: (e.id ? "edge_#{e.id}" : nil), text: edge_text, score: 0.7, created_at: e.created_at }
             end
           end
 

data/lib/llmemory/long_term/procedural/memory.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require_relative "skill"
+require_relative "storage"
+require_relative "../../memory_module"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      # Procedural long-term memory: a Voyager-style skill library. Agents
+      # register reusable skills (prompts, templates, code), retrieve them by
+      # relevance to the current task, and report outcomes so proven skills are
+      # preferred over unproven ones.
+      #
+      # Retrieval is keyword-based for now (vector search is a follow-up). The
+      # success rate of each skill is surfaced as `importance`, so the retrieval
+      # Engine ranks battle-tested skills higher (P3 importance weighting).
+      class Memory
+        include Llmemory::MemoryModule
+
+        attr_reader :user_id, :storage
+
+        def initialize(user_id:, storage: nil)
+          @user_id = user_id
+          @storage = storage || Storages.build
+        end
+
+        # Registers a skill. If `version` is omitted and a skill with the same
+        # name exists, the version auto-increments (skill evolution).
+        def register_skill(name:, body:, description: nil, kind: Skill::DEFAULT_KIND, version: nil)
+          version ||= next_version_for(name)
+          skill = Skill.new(
+            id: nil, user_id: @user_id, name: name, body: body,
+            description: description, kind: kind, version: version
+          )
+          @storage.save_skill(@user_id, skill.to_h)
+        end
+
+        def find_skill(query)
+          raw = @storage.search_skills(@user_id, query).first
+          raw && Skill.from_h(raw)
+        end
+
+        def get_skill(id)
+          raw = @storage.get_skill(@user_id, id)
+          raw && Skill.from_h(raw)
+        end
+
+        def skills(limit: nil)
+          @storage.list_skills(@user_id, limit: limit).map { |s| Skill.from_h(s) }
+        end
+
+        def count
+          @storage.count_skills(@user_id)
+        end
+
+        # Records that applying a skill succeeded or failed. Feeds retrieval
+        # ranking and adaptive retrieval (P8). Returns the updated Skill.
+        def report_outcome(skill_id, success:)
+          raw = @storage.record_outcome(@user_id, skill_id, success: success)
+          raw && Skill.from_h(raw)
+        end
+
+        # Retrieval Engine integration: skills ranked by relevance, recency and
+        # proven utility (success rate exposed as importance).
+        def search_candidates(query, user_id: nil, top_k: 20)
+          uid = user_id || @user_id
+          return [] unless uid == @user_id
+
+          @storage.search_skills(uid, query).first(top_k).map do |raw|
+            skill = Skill.from_h(raw)
+            {
+              id: skill.id,
+              text: skill.searchable_text,
+              timestamp: skill.created_at,
+              score: 1.0,
+              importance: skill.success_rate,
+              evergreen: false
+            }
+          end
+        end
+
+        # --- MemoryModule uniform interface ---
+
+        def write(name:, body:, description: nil, kind: Skill::DEFAULT_KIND, version: nil, **_meta)
+          register_skill(name: name, body: body, description: description, kind: kind, version: version)
+        end
+
+        def list(user_id: nil, limit: nil)
+          skills(limit: limit)
+        end
+
+        def stats(user_id: nil)
+          { skills: count }
+        end
+
+        def forget(ids:, reason: nil)
+          requested = Array(ids).map(&:to_s)
+          existing = @storage.list_skills(@user_id).map { |s| (s[:id] || s["id"]).to_s }
+          removed = requested & existing
+          @storage.delete_skills(@user_id, removed)
+          forget_log.record(@user_id, memory_type: "procedural", ids: removed, reason: reason)
+          removed.size
+        end
+
+        private
+
+        def next_version_for(name)
+          existing = @storage.find_skills_by_name(@user_id, name)
+          return 1 if existing.empty?
+          existing.map { |s| (s[:version] || s["version"] || 1).to_i }.max + 1
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/skill.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      # A Skill is a reusable procedure an agent can retrieve and apply: a prompt,
+      # a template or a snippet of code. This is CoALA's "procedural memory" in the
+      # Voyager sense — a growing library of skills the agent learns and reuses.
+      #
+      # Skills track success/failure outcomes so proven skills can be preferred
+      # over unproven ones during retrieval (see #success_rate, and P8 adaptive
+      # retrieval).
+      class Skill
+        KINDS = %w[prompt template code].freeze
+        DEFAULT_KIND = "prompt"
+
+        attr_reader :id, :user_id, :name, :description, :body, :kind, :version,
+                    :success_count, :failure_count, :created_at, :updated_at
+
+        def initialize(id:, user_id:, name:, body:, description: nil, kind: DEFAULT_KIND,
+                       version: 1, success_count: 0, failure_count: 0, created_at: nil, updated_at: nil)
+          @id = id
+          @user_id = user_id
+          @name = name.to_s
+          @description = description
+          @body = body
+          @kind = normalize_kind(kind)
+          @version = version.to_i
+          @success_count = success_count.to_i
+          @failure_count = failure_count.to_i
+          @created_at = created_at || Time.now
+          @updated_at = updated_at || @created_at
+        end
+
+        # Proven utility in [0, 1]. Unproven skills (no outcomes) are neutral.
+        def success_rate
+          total = success_count + failure_count
+          total.zero? ? 0.5 : success_count.to_f / total
+        end
+
+        def searchable_text
+          [name, description, body].compact.map(&:to_s).reject(&:empty?).join("\n")
+        end
+
+        def normalize_kind(kind)
+          k = kind.to_s.strip.downcase
+          KINDS.include?(k) ? k : DEFAULT_KIND
+        end
+
+        def self.from_h(hash)
+          new(
+            id: hash[:id] || hash["id"],
+            user_id: hash[:user_id] || hash["user_id"],
+            name: hash[:name] || hash["name"],
+            description: hash[:description] || hash["description"],
+            body: hash[:body] || hash["body"],
+            kind: hash[:kind] || hash["kind"] || DEFAULT_KIND,
+            version: hash[:version] || hash["version"] || 1,
+            success_count: hash[:success_count] || hash["success_count"] || 0,
+            failure_count: hash[:failure_count] || hash["failure_count"] || 0,
+            created_at: parse_time(hash[:created_at] || hash["created_at"]),
+            updated_at: parse_time(hash[:updated_at] || hash["updated_at"])
+          )
+        end
+
+        def self.parse_time(value)
+          return value if value.nil? || value.is_a?(Time)
+          Time.parse(value.to_s)
+        rescue ArgumentError
+          nil
+        end
+
+        def to_h
+          {
+            id: id,
+            user_id: user_id,
+            name: name,
+            description: description,
+            body: body,
+            kind: kind,
+            version: version,
+            success_count: success_count,
+            failure_count: failure_count,
+            created_at: created_at.respond_to?(:iso8601) ? created_at.iso8601(6) : created_at,
+            updated_at: updated_at.respond_to?(:iso8601) ? updated_at.iso8601(6) : updated_at
+          }
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/storage.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "storages/base"
+require_relative "storages/memory_storage"
+require_relative "storages/file_storage"
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      # Backward compatibility: Storage points to the in-memory backend.
+      Storage = Storages::MemoryStorage
+
+      module Storages
+        def self.build(store: nil, base_path: nil)
+          case (store || Llmemory.configuration.long_term_store).to_s.to_sym
+          when :memory
+            MemoryStorage.new
+          when :file
+            FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
+          when :postgres, :database, :active_record, :activerecord
+            raise NotImplementedError,
+              "Procedural SQL/ActiveRecord storage is not implemented yet; use :memory or :file " \
+              "(or pass an explicit storage instance)."
+          else
+            MemoryStorage.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/procedural/storages/base.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module LongTerm
+    module Procedural
+      module Storages
+        # Storage contract for procedural memory (skill library). Implementations
+        # persist Skill hashes, support keyword search and name lookup (for
+        # versioning), and record success/failure outcomes.
+        class Base
+          def save_skill(user_id, skill)
+            raise NotImplementedError, "#{self.class}#save_skill must be implemented"
+          end
+
+          def get_skill(user_id, id)
+            raise NotImplementedError, "#{self.class}#get_skill must be implemented"
+          end
+
+          def list_skills(user_id, limit: nil)
+            raise NotImplementedError, "#{self.class}#list_skills must be implemented"
+          end
+
+          def search_skills(user_id, query)
+            raise NotImplementedError, "#{self.class}#search_skills must be implemented"
+          end
+
+          def find_skills_by_name(user_id, name)
+            raise NotImplementedError, "#{self.class}#find_skills_by_name must be implemented"
+          end
+
+          # Increments the success or failure count of a skill and returns the
+          # updated skill hash (or nil if not found).
+          def record_outcome(user_id, skill_id, success:)
+            raise NotImplementedError, "#{self.class}#record_outcome must be implemented"
+          end
+
+          def count_skills(user_id)
+            raise NotImplementedError, "#{self.class}#count_skills must be implemented"
+          end
+
+          # Deletes skills by id. Returns the number actually removed.
+          def delete_skills(user_id, ids)
+            raise NotImplementedError, "#{self.class}#delete_skills must be implemented"
+          end
+
+          def list_users
+            raise NotImplementedError, "#{self.class}#list_users must be implemented"
+          end
+        end
+      end
+    end
+  end
+end