llmemory 0.1.15 → 0.1.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 723fae20d0310ccaeaf9ba600148061d17b2a0b29f933d455d1cf656dee85636
-  data.tar.gz: a135ea1661af46e96843bf52744e8004d0ebe7e8d94b0c46a097c36df53d5bc4
+  metadata.gz: b86647810e47140fff5732a066da4a16188704249d09db14720d8c565b6eaf0e
+  data.tar.gz: 7c52c551746d22c29e41015098a68e166788bb614b0e0c2b064fa5fc0824989f
 SHA512:
-  metadata.gz: 256caaee94233d5e57b8d9e6007fe1ced57d35e21d40260ce34b2803ba0ef3593b66668aa06334e647edd103aa431113e38b639776163d71153c4b9bac68c1a1
-  data.tar.gz: 33cd1726e9f7bb3328610bafabca5ebfe51f080e7d34c523fc0b363eb290b353c9109937f2782ed7e60906965236e79229838e32c698d2f0e2f73aa2d421970b
+  metadata.gz: f584d487eca13280b13f7a3e9f4b8eda60ed10ec8dbb45ff72483acbc76b7feb7f79fb5ae572640a31e87ade05b0762f07cacbecb778dffa6d62a7096231fb12
+  data.tar.gz: c68e284c21fc22ccdf20160d9082575a47482b03c5a91fb3b52ec5a6c51b7f5cf13a8915bfd3fe2f5933aa6a52fe8c382e2373393b27d68d8ed1c7ac83766e49

data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb

@@ -17,6 +17,7 @@ class CreateLlmemoryTables < ActiveRecord::Migration[7.0]
       t.text :content, null: false
       t.string :source_resource_id
       t.float :importance, default: 0.7
+      t.jsonb :provenance
       t.timestamps
     end
     add_index :llmemory_items, :user_id

data/lib/llmemory/configuration.rb

@@ -14,6 +14,7 @@ module Llmemory
                   :database_url,
                   :vector_store,
                   :time_decay_half_life_days,
+                  :importance_weight,
                   :max_retrieval_tokens,
                   :prune_after_days,
                   :compact_max_bytes,
@@ -56,6 +57,7 @@ module Llmemory
       @database_url = ENV["DATABASE_URL"]
       @vector_store = nil
       @time_decay_half_life_days = 30
+      @importance_weight = 1.0
       @max_retrieval_tokens = 2000
       @prune_after_days = 90
       @compact_max_bytes = 8192

data/lib/llmemory/long_term/episodic/episode.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      # An Episode is a trajectory of an agent's experience: an ordered list of
+      # steps (observation -> action -> result) plus a summary, an outcome label
+      # and an importance score. This is CoALA's "episodic memory" — distinct
+      # from semantic memory (facts), it stores what happened so it can later be
+      # retrieved as examples or distilled into semantic knowledge (see P2,
+      # reflection).
+      class Episode
+        attr_reader :id, :user_id, :steps, :summary, :outcome, :importance, :provenance, :created_at
+
+        STEP_KEYS = %i[observation action result timestamp].freeze
+
+        def initialize(id:, user_id:, steps: [], summary: nil, outcome: nil, importance: 0.5, provenance: nil, created_at: nil)
+          @id = id
+          @user_id = user_id
+          @steps = self.class.normalize_steps(steps)
+          @summary = summary
+          @outcome = outcome
+          @importance = importance.nil? ? 0.5 : importance.to_f
+          @provenance = provenance
+          @created_at = created_at || Time.now
+        end
+
+        # Flat, searchable representation used for keyword retrieval and, in the
+        # future, embedding. Combines summary, outcome and every step field.
+        def searchable_text
+          parts = [summary, outcome]
+          steps.each do |s|
+            parts << s[:observation]
+            parts << s[:action]
+            parts << s[:result]
+          end
+          parts.compact.map(&:to_s).reject(&:empty?).join("\n")
+        end
+
+        def self.normalize_steps(steps)
+          Array(steps).filter_map do |step|
+            next nil unless step.is_a?(Hash)
+            {
+              observation: step[:observation] || step["observation"],
+              action: step[:action] || step["action"],
+              result: step[:result] || step["result"],
+              timestamp: normalize_time(step[:timestamp] || step["timestamp"])
+            }
+          end
+        end
+
+        def self.normalize_time(value)
+          return nil if value.nil?
+          value.respond_to?(:iso8601) ? value.iso8601 : value.to_s
+        end
+
+        def self.from_h(hash)
+          new(
+            id: hash[:id] || hash["id"],
+            user_id: hash[:user_id] || hash["user_id"],
+            steps: hash[:steps] || hash["steps"] || [],
+            summary: hash[:summary] || hash["summary"],
+            outcome: hash[:outcome] || hash["outcome"],
+            importance: hash[:importance] || hash["importance"] || 0.5,
+            provenance: hash[:provenance] || hash["provenance"],
+            created_at: parse_created_at(hash[:created_at] || hash["created_at"])
+          )
+        end
+
+        def self.parse_created_at(value)
+          return value if value.nil? || value.is_a?(Time)
+          Time.parse(value.to_s)
+        rescue ArgumentError
+          nil
+        end
+
+        def to_h
+          {
+            id: id,
+            user_id: user_id,
+            steps: steps,
+            summary: summary,
+            outcome: outcome,
+            importance: importance,
+            provenance: provenance,
+            created_at: created_at.respond_to?(:iso8601) ? created_at.iso8601(6) : created_at
+          }
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/memory.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative "episode"
+require_relative "storage"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      # Episodic long-term memory: records agent trajectories and retrieves them
+      # by recency, importance and relevance. Designed to coexist with semantic
+      # memory (file/graph), not replace it, and to feed reflection (P2), which
+      # distills episodes into semantic knowledge.
+      #
+      # Deliberately LLM-free: recording and retrieval are deterministic. Higher
+      # order summarization belongs to reflection.
+      class Memory
+        attr_reader :user_id, :storage
+
+        def initialize(user_id:, storage: nil)
+          @user_id = user_id
+          @storage = storage || Storages.build
+        end
+
+        # Records a trajectory. `steps` is an array of hashes with any of
+        # :observation, :action, :result, :timestamp. Returns the episode id.
+        def record_episode(steps:, summary: nil, outcome: nil, importance: 0.5)
+          episode = Episode.new(
+            id: nil,
+            user_id: @user_id,
+            steps: steps,
+            summary: summary || derive_summary(steps),
+            outcome: outcome,
+            importance: importance
+          )
+          provenance = Llmemory::Provenance.from_text_fingerprint(
+            episode.searchable_text, method: "episode_recording", confidence: episode.importance
+          )
+          record = episode.to_h.merge(provenance: provenance)
+          @storage.save_episode(@user_id, record)
+        end
+
+        def recent_episodes(limit: 10)
+          @storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
+        end
+
+        def episodes(limit: nil)
+          @storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
+        end
+
+        def find_episode(id)
+          raw = @storage.get_episode(@user_id, id)
+          raw && Episode.from_h(raw)
+        end
+
+        def count
+          @storage.count_episodes(@user_id)
+        end
+
+        # Retrieval Engine integration. Returns candidates shaped like the other
+        # long-term memories so the Engine can rank episodes by relevance,
+        # recency (temporal decay) and importance (P3), with provenance (P10).
+        def search_candidates(query, user_id: nil, top_k: 20)
+          uid = user_id || @user_id
+          return [] unless uid == @user_id
+
+          @storage.search_episodes(uid, query).first(top_k).map do |e|
+            episode = Episode.from_h(e)
+            {
+              text: episode.summary.to_s.empty? ? episode.searchable_text : episode.summary,
+              timestamp: episode.created_at,
+              score: 1.0,
+              importance: episode.importance,
+              evergreen: false,
+              provenance: e[:provenance] || e["provenance"]
+            }
+          end
+        end
+
+        private
+
+        # Cheap, deterministic summary when the caller does not provide one.
+        # LLM-based summarization is reflection's job (P2).
+        def derive_summary(steps)
+          normalized = Episode.normalize_steps(steps)
+          return nil if normalized.empty?
+          actions = normalized.filter_map { |s| s[:action] }.reject { |a| a.to_s.strip.empty? }
+          return nil if actions.empty?
+          "Episode with #{normalized.size} step(s): #{actions.join(' -> ')}"
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storage.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "storages/base"
+require_relative "storages/memory_storage"
+require_relative "storages/file_storage"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      # Backward compatibility: Storage points to the in-memory backend.
+      Storage = Storages::MemoryStorage
+
+      module Storages
+        def self.build(store: nil, base_path: nil)
+          case (store || Llmemory.configuration.long_term_store).to_s.to_sym
+          when :memory
+            MemoryStorage.new
+          when :file
+            FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
+          when :postgres, :database, :active_record, :activerecord
+            raise NotImplementedError,
+              "Episodic SQL/ActiveRecord storage is not implemented yet; use :memory or :file " \
+              "(or pass an explicit storage instance)."
+          else
+            MemoryStorage.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/base.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        # Storage contract for episodic memory. Implementations persist Episode
+        # hashes and expose recency-ordered listing plus keyword search so the
+        # retrieval Engine can rank episodes alongside other memory types.
+        class Base
+          def save_episode(user_id, episode)
+            raise NotImplementedError, "#{self.class}#save_episode must be implemented"
+          end
+
+          def get_episode(user_id, id)
+            raise NotImplementedError, "#{self.class}#get_episode must be implemented"
+          end
+
+          # Newest first. Optionally capped by limit.
+          def list_episodes(user_id, limit: nil)
+            raise NotImplementedError, "#{self.class}#list_episodes must be implemented"
+          end
+
+          def search_episodes(user_id, query)
+            raise NotImplementedError, "#{self.class}#search_episodes must be implemented"
+          end
+
+          def count_episodes(user_id)
+            raise NotImplementedError, "#{self.class}#count_episodes must be implemented"
+          end
+
+          def list_users
+            raise NotImplementedError, "#{self.class}#list_users must be implemented"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/file_storage.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        class FileStorage < Base
+          def initialize(base_path: nil)
+            @base_path = base_path || Llmemory.configuration.long_term_storage_path || "./llmemory_data"
+            @base_path = File.expand_path(@base_path)
+          end
+
+          def save_episode(user_id, episode)
+            id = episode[:id] || episode["id"] || "ep_#{next_seq(user_id)}"
+            data = stringify_for_json(episode).merge("id" => id, "user_id" => user_id)
+            data["created_at"] ||= Time.now.iso8601
+            File.write(episode_path(user_id, id), JSON.generate(data))
+            id
+          end
+
+          def get_episode(user_id, id)
+            path = episode_path(user_id, id)
+            return nil unless File.file?(path)
+            load_episode(path)
+          end
+
+          def list_episodes(user_id, limit: nil)
+            sorted = all_episodes(user_id).sort_by { |e| e[:created_at] }.reverse
+            limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
+          end
+
+          def search_episodes(user_id, query)
+            q = query.to_s.downcase
+            return list_episodes(user_id) if q.strip.empty?
+            all_episodes(user_id).select { |e| episode_text(e).downcase.include?(q) }
+          end
+
+          def count_episodes(user_id)
+            dir = user_path(user_id, "episodes")
+            return 0 unless Dir.exist?(dir)
+            Dir.children(dir).count { |f| f.end_with?(".json") }
+          end
+
+          def list_users
+            return [] unless Dir.exist?(@base_path)
+            Dir.children(@base_path).select { |d| Dir.exist?(File.join(@base_path, d, "episodes")) }
+          end
+
+          private
+
+          def all_episodes(user_id)
+            dir = user_path(user_id, "episodes")
+            return [] unless Dir.exist?(dir)
+            Dir.children(dir).select { |f| f.end_with?(".json") }.map { |f| load_episode(File.join(dir, f)) }.compact
+          end
+
+          def load_episode(path)
+            data = JSON.parse(File.read(path), symbolize_names: true)
+            data[:created_at] = parse_time(data[:created_at])
+            data
+          rescue JSON::ParserError
+            nil
+          end
+
+          def episode_text(episode)
+            parts = [episode[:summary], episode[:outcome]]
+            Array(episode[:steps]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << s[:observation] << s[:action] << s[:result]
+            end
+            parts.compact.join("\n")
+          end
+
+          def stringify_for_json(episode)
+            JSON.parse(JSON.generate(episode))
+          end
+
+          def user_path(user_id, *parts)
+            safe = user_id.to_s.gsub(%r{[^\w\-.]}, "_")
+            File.join(@base_path, safe, *parts)
+          end
+
+          def episode_path(user_id, id)
+            dir = user_path(user_id, "episodes")
+            FileUtils.mkdir_p(dir)
+            File.join(dir, "#{id}.json")
+          end
+
+          def meta_path(user_id)
+            FileUtils.mkdir_p(user_path(user_id))
+            File.join(user_path(user_id), "meta.json")
+          end
+
+          def next_seq(user_id)
+            path = meta_path(user_id)
+            meta = File.file?(path) ? JSON.parse(File.read(path)) : {}
+            meta["episode_id_seq"] = (meta["episode_id_seq"] || 0) + 1
+            File.write(path, JSON.generate(meta))
+            meta["episode_id_seq"]
+          end
+
+          def parse_time(val)
+            return val if val.is_a?(Time)
+            Time.parse(val.to_s)
+          rescue ArgumentError
+            Time.now
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/memory_storage.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        class MemoryStorage < Base
+          def initialize
+            @episodes = Hash.new { |h, k| h[k] = [] }
+            @seq = 0
+          end
+
+          def save_episode(user_id, episode)
+            @seq += 1
+            id = episode[:id] || episode["id"] || "ep_#{@seq}"
+            record = symbolize(episode).merge(id: id, user_id: user_id)
+            record[:created_at] ||= Time.now
+            @episodes[user_id] << record
+            id
+          end
+
+          def get_episode(user_id, id)
+            @episodes[user_id].find { |e| e[:id] == id }
+          end
+
+          def list_episodes(user_id, limit: nil)
+            sorted = @episodes[user_id].sort_by { |e| e[:created_at] }.reverse
+            limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
+          end
+
+          def search_episodes(user_id, query)
+            q = query.to_s.downcase
+            return list_episodes(user_id) if q.strip.empty?
+            @episodes[user_id].select { |e| episode_text(e).downcase.include?(q) }
+          end
+
+          def count_episodes(user_id)
+            @episodes[user_id].size
+          end
+
+          def list_users
+            @episodes.keys
+          end
+
+          private
+
+          def symbolize(hash)
+            hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
+          end
+
+          def episode_text(episode)
+            parts = [episode[:summary], episode[:outcome]]
+            Array(episode[:steps]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << (s[:observation] || s["observation"])
+              parts << (s[:action] || s["action"])
+              parts << (s[:result] || s["result"])
+            end
+            parts.compact.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "episodic/episode"
+require_relative "episodic/storage"
+require_relative "episodic/memory"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+    end
+  end
+end

data/lib/llmemory/long_term/file_based/item.rb

@@ -4,14 +4,15 @@ module Llmemory
   module LongTerm
     module FileBased
       class Item
-        attr_reader :id, :user_id, :category, :content, :source_resource_id, :created_at
+        attr_reader :id, :user_id, :category, :content, :source_resource_id, :provenance, :created_at
 
-        def initialize(id:, user_id:, category:, content:, source_resource_id: nil, created_at: nil)
+        def initialize(id:, user_id:, category:, content:, source_resource_id: nil, provenance: nil, created_at: nil)
           @id = id
           @user_id = user_id
           @category = category
           @content = content
           @source_resource_id = source_resource_id
+          @provenance = provenance
           @created_at = created_at || Time.now
         end
 
@@ -22,6 +23,7 @@ module Llmemory
             category: category,
             content: content,
             source_resource_id: source_resource_id,
+            provenance: provenance,
             created_at: created_at.iso8601
           }
         end

data/lib/llmemory/long_term/file_based/memory.rb

@@ -65,7 +65,8 @@ module Llmemory
             out << {
               text: i[:content] || i["content"],
               timestamp: i[:created_at] || i["created_at"],
-              score: (i[:importance] || i["importance"] || 1.0).to_f,
+              score: 1.0,
+              importance: (i[:importance] || i["importance"] || 1.0).to_f,
               evergreen: i[:evergreen] || i["evergreen"]
             }
           end
@@ -84,6 +85,21 @@ module Llmemory
           out
         end
 
+        # Stores a single fact produced outside the extraction flow (e.g. by
+        # reflection over episodes), preserving caller-supplied provenance so the
+        # insight remains traceable to its source. Returns the item id.
+        def remember_fact(content:, category: "general", importance: 0.6, provenance: nil)
+          return nil if content.to_s.strip.empty?
+          @storage.save_item(
+            @user_id,
+            category: category.to_s,
+            content: content.to_s,
+            source_resource_id: nil,
+            importance: importance,
+            provenance: provenance
+          )
+        end
+
         attr_reader :storage, :user_id
 
         private
@@ -94,7 +110,10 @@ module Llmemory
 
         def save_item(category:, item:, source_resource_id:, importance: 0.7)
           content = item.is_a?(Hash) ? item["content"] || item[:content] : item.to_s
-          @storage.save_item(@user_id, category: category, content: content, source_resource_id: source_resource_id, importance: importance)
+          provenance = Llmemory::Provenance.from_resource(
+            source_resource_id, method: "fact_extraction", confidence: importance
+          )
+          @storage.save_item(@user_id, category: category, content: content, source_resource_id: source_resource_id, importance: importance, provenance: provenance)
         end
 
         def append_to_daily_log(conversation_text)

data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb

@@ -30,7 +30,7 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             id = "item_#{SecureRandom.hex(8)}"
             attrs = {
               id: id,
@@ -41,6 +41,7 @@ module Llmemory
               created_at: Time.current
             }
             attrs[:importance] = importance if LlmemoryItem.column_names.include?("importance")
+            attrs[:provenance] = provenance if provenance && LlmemoryItem.column_names.include?("provenance")
             LlmemoryItem.create!(attrs)
             id
           end
@@ -189,6 +190,7 @@ module Llmemory
               created_at: r.created_at
             }
             h[:importance] = r.respond_to?(:importance) ? (r.importance || 0.7).to_f : 0.7
+            h[:provenance] = r.provenance if r.respond_to?(:provenance)
             h
           end
 

data/lib/llmemory/long_term/file_based/storages/base.rb

@@ -9,7 +9,7 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#save_resource must be implemented"
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             raise NotImplementedError, "#{self.class}#save_item must be implemented"
           end
 

data/lib/llmemory/long_term/file_based/storages/database_storage.rb

@@ -24,12 +24,12 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             ensure_tables!
             id = "item_#{SecureRandom.hex(8)}"
             conn.exec_params(
-              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, importance, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7)",
-              [id, user_id, category, content, source_resource_id, importance.to_f, Time.now.utc.iso8601]
+              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, importance, provenance, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)",
+              [id, user_id, category, content, source_resource_id, importance.to_f, provenance ? JSON.generate(provenance) : nil, Time.now.utc.iso8601]
             )
             id
           end
@@ -67,7 +67,7 @@ module Llmemory
             ensure_tables!
             pattern = "%#{conn.escape_string(query.to_s.downcase)}%"
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND LOWER(content) LIKE $2",
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND LOWER(content) LIKE $2",
               [user_id, pattern]
             )
             rows_to_items(rows)
@@ -97,7 +97,7 @@ module Llmemory
             ensure_tables!
             cutoff = (Time.now - (days * 86400)).utc.iso8601
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at < $2 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at < $2 ORDER BY created_at",
               [user_id, cutoff]
             )
             rows_to_items(rows)
@@ -106,7 +106,7 @@ module Llmemory
           def get_all_items(user_id)
             ensure_tables!
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 ORDER BY created_at",
               [user_id]
             )
             rows_to_items(rows)
@@ -125,7 +125,7 @@ module Llmemory
             ensure_tables!
             cutoff = (Time.now - (hours * 3600)).utc.iso8601
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at >= $2 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at >= $2 ORDER BY created_at",
               [user_id, cutoff]
             )
             rows_to_items(rows)
@@ -179,7 +179,7 @@ module Llmemory
 
           def list_items(user_id:, category: nil, limit: nil)
             ensure_tables!
-            sql = "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1"
+            sql = "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1"
             params = [user_id]
             if category
               sql += " AND category = $2"
@@ -258,11 +258,13 @@ module Llmemory
                 content TEXT NOT NULL,
                 source_resource_id TEXT,
                 importance REAL DEFAULT 0.7,
+                provenance JSONB,
                 created_at TIMESTAMPTZ NOT NULL
               );
               CREATE INDEX IF NOT EXISTS idx_llmemory_items_user_id ON llmemory_items(user_id);
             SQL
             conn.exec("ALTER TABLE llmemory_items ADD COLUMN IF NOT EXISTS importance REAL DEFAULT 0.7") rescue nil
+            conn.exec("ALTER TABLE llmemory_items ADD COLUMN IF NOT EXISTS provenance JSONB") rescue nil
             conn.exec(<<~SQL)
               CREATE TABLE IF NOT EXISTS llmemory_categories (
                 user_id TEXT NOT NULL,
@@ -282,11 +284,19 @@ module Llmemory
                 content: r["content"],
                 source_resource_id: r["source_resource_id"],
                 importance: (r["importance"] || 0.7).to_f,
+                provenance: parse_provenance(r["provenance"]),
                 created_at: Time.parse(r["created_at"])
               }
             end
           end
 
+          def parse_provenance(value)
+            return nil if value.nil? || value.to_s.strip.empty?
+            JSON.parse(value, symbolize_names: true)
+          rescue JSON::ParserError
+            nil
+          end
+
           def rows_to_resources(rows)
             rows.map do |r|
               {

data/lib/llmemory/long_term/file_based/storages/file_storage.rb

@@ -24,7 +24,7 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             ensure_user_dir(user_id)
             seq = next_seq(user_id, "item_id_seq")
             id = "item_#{seq}"
@@ -35,6 +35,7 @@ module Llmemory
               content: content,
               source_resource_id: source_resource_id,
               importance: importance,
+              provenance: provenance,
               created_at: Time.now.iso8601
             }
             File.write(path, JSON.generate(data))

data/lib/llmemory/long_term/file_based/storages/memory_storage.rb

@@ -22,7 +22,7 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             @item_id_seq += 1
             id = "item_#{@item_id_seq}"
             @items[user_id] << {
@@ -31,6 +31,7 @@ module Llmemory
               content: content,
               source_resource_id: source_resource_id,
               importance: importance,
+              provenance: provenance,
               created_at: Time.now
             }
             id

data/lib/llmemory/long_term/graph_based/edge.rb

@@ -31,6 +31,13 @@ module Llmemory
           !archived_at.nil?
         end
 
+        # Lineage of this edge, stored within properties so it round-trips
+        # through every backend without a schema change. See Llmemory::Provenance.
+        def provenance
+          props = properties || {}
+          props[:provenance] || props["provenance"]
+        end
+
         def to_h
           {
             id: id,

data/lib/llmemory/long_term/graph_based/memory.rb

@@ -32,6 +32,7 @@ module Llmemory
 
           return true if entities.empty? && relations.empty?
 
+          provenance = Llmemory::Provenance.from_text_fingerprint(text, method: "entity_relation_extraction")
           name_to_id = {}
 
           entities.each do |e|
@@ -39,7 +40,7 @@ module Llmemory
             entity_type = e[:type] || e["type"] || "concept"
             name = e[:name] || e["name"]
             next if name.nil? || name.to_s.strip.empty?
-            id = @kg.add_node(entity_type: entity_type, name: name.to_s.strip, properties: {})
+            id = @kg.add_node(entity_type: entity_type, name: name.to_s.strip, properties: { "provenance" => provenance })
             name_to_id[name.to_s.strip] ||= id
           end
 
@@ -50,8 +51,8 @@ module Llmemory
             object = (r[:object] || r["object"]).to_s.strip
             next if subject.empty? || predicate.empty? || object.empty?
 
-            subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: {})
-            object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: {})
+            subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: { "provenance" => provenance })
+            object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: { "provenance" => provenance })
 
             edge = Edge.new(
               id: nil,
@@ -59,12 +60,12 @@ module Llmemory
               subject_id: subject_id,
               predicate: predicate,
               target_id: object_id,
-              properties: {},
+              properties: { "provenance" => provenance },
               created_at: Time.now,
               archived_at: nil
             )
             @conflict_resolver.resolve(edge)
-            edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: {})
+            edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: { "provenance" => provenance })
 
             text = "#{subject} #{predicate} #{object}"
             embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(text) : nil
@@ -89,7 +90,8 @@ module Llmemory
             {
               text: r[:text],
               timestamp: r[:created_at] || r[:timestamp],
-              score: r[:score] || 1.0
+              score: r[:score] || 1.0,
+              importance: r[:importance]
             }
           end
         end

data/lib/llmemory/long_term/graph_based/node.rb

@@ -25,6 +25,13 @@ module Llmemory
           )
         end
 
+        # Lineage of this node, stored within properties so it round-trips
+        # through every backend without a schema change. See Llmemory::Provenance.
+        def provenance
+          props = properties || {}
+          props[:provenance] || props["provenance"]
+        end
+
         def to_h
           {
             id: id,

data/lib/llmemory/long_term.rb

@@ -2,6 +2,7 @@
 
 require_relative "long_term/file_based"
 require_relative "long_term/graph_based"
+require_relative "long_term/episodic"
 
 module Llmemory
   module LongTerm

data/lib/llmemory/provenance.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Llmemory
+  # Provenance records the lineage of a long-term memory item: where it came
+  # from, how it was produced, and with what confidence. It is stored as a
+  # plain JSON-safe Hash so it round-trips through every storage backend
+  # (in-memory, JSON files, SQL columns, jsonb properties) without coupling.
+  #
+  # Shape: { sources: [{ type:, id: }], method:, confidence:, created_at: }
+  #
+  # `method` identifies the producing process (e.g. "fact_extraction",
+  # "entity_relation_extraction", and in the future "reflection"), so a
+  # semantic datum can always be traced back to its raw source.
+  module Provenance
+    module_function
+
+    def build(method:, sources: [], confidence: nil, created_at: nil)
+      {
+        sources: Array(sources).filter_map { |s| normalize_source(s) },
+        method: method&.to_s,
+        confidence: confidence.nil? ? nil : confidence.to_f,
+        created_at: normalize_time(created_at)
+      }
+    end
+
+    # Convenience for the file-based path, where the raw text is persisted as a
+    # Resource and referenced by id.
+    def from_resource(resource_id, method:, confidence: nil, created_at: nil)
+      sources = resource_id ? [{ type: "resource", id: resource_id }] : []
+      build(method: method, sources: sources, confidence: confidence, created_at: created_at)
+    end
+
+    # Convenience for the graph-based path, which does not persist the raw text.
+    # We record a stable fingerprint of the source instead of the document
+    # itself, keeping lineage verifiable without exposing sensitive content.
+    def from_text_fingerprint(text, method:, confidence: nil, created_at: nil)
+      require "digest"
+      sources = []
+      unless text.to_s.strip.empty?
+        sources = [{ type: "text_sha256", id: Digest::SHA256.hexdigest(text.to_s)[0, 16] }]
+      end
+      build(method: method, sources: sources, confidence: confidence, created_at: created_at)
+    end
+
+    def normalize_source(source)
+      return nil if source.nil?
+      if source.is_a?(Hash)
+        type = source[:type] || source["type"]
+        id = source[:id] || source["id"]
+        return nil if id.nil?
+        { type: type.nil? ? "unknown" : type.to_s, id: id }
+      else
+        { type: "unknown", id: source }
+      end
+    end
+
+    def normalize_time(value)
+      value ||= Time.now
+      value.respond_to?(:iso8601) ? value.iso8601 : value.to_s
+    end
+  end
+end

data/lib/llmemory/reflection/reflector.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Llmemory
+  module Reflection
+    # Reflection distills an agent's recent episodes (episodic memory) into
+    # durable, higher-order insights and writes them to semantic memory. This is
+    # CoALA's "updating semantic memory with knowledge" (Reflexion / Generative
+    # Agents): unlike one-shot extraction from raw text, it reasons over lived
+    # experience to generalize lessons and patterns.
+    #
+    # Each insight is stored with provenance { method: "reflection",
+    # sources: [{ type: "episode", id: ... }] } so it stays traceable to the
+    # experiences that produced it.
+    #
+    # `semantic` must respond to:
+    #   remember_fact(content:, category:, importance:, provenance:)
+    # (FileBased::Memory implements this; graph-based is a future target.)
+    class Reflector
+      DEFAULT_CATEGORY = "insights"
+      DEFAULT_IMPORTANCE = 0.6
+
+      def initialize(episodic:, semantic:, llm: nil)
+        @episodic = episodic
+        @semantic = semantic
+        @llm = llm || Llmemory::LLM.client
+      end
+
+      # Reflects over the most recent `window` episodes and writes the resulting
+      # insights to semantic memory. Returns the ids of the stored insights.
+      def reflect(window: 10, category: DEFAULT_CATEGORY)
+        episodes = @episodic.recent_episodes(limit: window)
+        return [] if episodes.empty?
+
+        insights = distill(episodes)
+        return [] if insights.empty?
+
+        sources = episodes.map(&:id).compact.map { |id| { type: "episode", id: id } }
+
+        insights.filter_map do |insight|
+          provenance = Llmemory::Provenance.build(
+            method: "reflection",
+            sources: sources,
+            confidence: insight[:confidence]
+          )
+          @semantic.remember_fact(
+            content: insight[:content],
+            category: category,
+            importance: insight[:confidence] || DEFAULT_IMPORTANCE,
+            provenance: provenance
+          )
+        end
+      end
+
+      private
+
+      def distill(episodes)
+        response = @llm.invoke(build_prompt(episodes))
+        parse_insights(response)
+      rescue Llmemory::LLMError
+        []
+      end
+
+      def build_prompt(episodes)
+        episodes_text = episodes.each_with_index.map do |ep, i|
+          "Episode #{i + 1} (outcome: #{ep.outcome || 'n/a'}):\n#{ep.searchable_text}"
+        end.join("\n\n")
+
+        <<~PROMPT
+          You are reflecting on an agent's recent experiences to distill durable,
+          higher-order insights: lessons learned, recurring patterns, and stable
+          preferences that will help in future situations. Generalize; do not
+          restate raw events.
+
+          Recent episodes:
+          #{episodes_text}
+
+          Return a JSON array of objects with "content" (the insight) and
+          "confidence" (0-1) keys. Return an empty array if nothing durable can
+          be concluded.
+          Example: [{"content": "Rolling back on deploy failure reliably restores service", "confidence": 0.8}]
+        PROMPT
+      end
+
+      def parse_insights(response)
+        json = extract_json_array(response)
+        return [] unless json
+
+        json.filter_map do |item|
+          next nil unless item.is_a?(Hash)
+          content = item["content"] || item[:content]
+          next nil if content.to_s.strip.empty?
+          { content: content.to_s, confidence: normalize_confidence(item["confidence"] || item[:confidence]) }
+        end
+      end
+
+      def normalize_confidence(value)
+        return nil if value.nil?
+        v = value.to_f
+        v.between?(0, 1) ? v : nil
+      end
+
+      def extract_json_array(response)
+        response = response.to_s.strip
+        start_idx = response.index("[")
+        end_idx = response.rindex("]")
+        return nil unless start_idx && end_idx
+
+        JSON.parse(response[start_idx..end_idx])
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/llmemory/reflection.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "reflection/reflector"
+
+module Llmemory
+  module Reflection
+  end
+end

data/lib/llmemory/retrieval/engine.rb

@@ -54,6 +54,7 @@ module Llmemory
             text: c[:text] || c["text"],
             timestamp: parse_timestamp(c[:timestamp] || c["timestamp"] || c[:created_at] || c["created_at"]),
             score: (c[:score] || c["score"] || 1.0).to_f,
+            importance: c[:importance] || c["importance"],
             evergreen: c[:evergreen] || c["evergreen"]
           }
         end

data/lib/llmemory/retrieval/temporal_ranker.rb

@@ -3,12 +3,14 @@
 module Llmemory
   module Retrieval
     class TemporalRanker
-      def initialize(half_life_days: nil)
+      def initialize(half_life_days: nil, importance_weight: nil)
         @half_life_days = half_life_days || Llmemory.configuration.time_decay_half_life_days
+        @importance_weight = importance_weight || Llmemory.configuration.importance_weight
       end
 
       def rank(candidates, now: Time.now)
         lambda_val = Math.log(2) / @half_life_days.to_f
+        weight = [@importance_weight.to_f, 0.0].max
 
         candidates.map do |c|
           score = (c[:score] || c["score"] || 1.0).to_f
@@ -22,10 +24,22 @@ module Llmemory
             Math.exp(-lambda_val * age_days.to_f)
           end
 
-          final_score = score * time_decay
-          c.merge(score: score, temporal_score: final_score, timestamp: timestamp)
+          importance = normalize_importance(c[:importance] || c["importance"])
+          importance_factor = importance**weight
+
+          final_score = score * time_decay * importance_factor
+          c.merge(score: score, importance: importance, temporal_score: final_score, timestamp: timestamp)
         end.sort_by { |c| -(c[:temporal_score] || 0) }
       end
+
+      private
+
+      # Missing importance is neutral (1.0) so candidates that carry no
+      # importance signal (resources, graph edges) are never penalised.
+      def normalize_importance(value)
+        return 1.0 if value.nil?
+        [[value.to_f, 0.0].max, 1.0].min
+      end
     end
   end
 end

data/lib/llmemory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Llmemory
-  VERSION = "0.1.15"
+  VERSION = "0.1.17"
 end

data/lib/llmemory.rb

@@ -2,6 +2,7 @@
 
 require_relative "llmemory/version"
 require_relative "llmemory/configuration"
+require_relative "llmemory/provenance"
 require_relative "llmemory/llm"
 require_relative "llmemory/short_term"
 require_relative "llmemory/long_term"
@@ -9,6 +10,7 @@ require_relative "llmemory/retrieval"
 require_relative "llmemory/vector_store"
 require_relative "llmemory/maintenance"
 require_relative "llmemory/extractors"
+require_relative "llmemory/reflection"
 require_relative "llmemory/memory"
 
 module Llmemory

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llmemory
 version: !ruby/object:Gem::Version
-  version: 0.1.15
+  version: 0.1.17
 platform: ruby
 authors:
 - llmemory
@@ -152,6 +152,13 @@ files:
 - lib/llmemory/llm/base.rb
 - lib/llmemory/llm/openai.rb
 - lib/llmemory/long_term.rb
+- lib/llmemory/long_term/episodic.rb
+- lib/llmemory/long_term/episodic/episode.rb
+- lib/llmemory/long_term/episodic/memory.rb
+- lib/llmemory/long_term/episodic/storage.rb
+- lib/llmemory/long_term/episodic/storages/base.rb
+- lib/llmemory/long_term/episodic/storages/file_storage.rb
+- lib/llmemory/long_term/episodic/storages/memory_storage.rb
 - lib/llmemory/long_term/file_based.rb
 - lib/llmemory/long_term/file_based/category.rb
 - lib/llmemory/long_term/file_based/item.rb
@@ -195,6 +202,9 @@ files:
 - lib/llmemory/mcp/tools/memory_timeline_context.rb
 - lib/llmemory/memory.rb
 - lib/llmemory/noise_filter.rb
+- lib/llmemory/provenance.rb
+- lib/llmemory/reflection.rb
+- lib/llmemory/reflection/reflector.rb
 - lib/llmemory/retrieval.rb
 - lib/llmemory/retrieval/bm25_scorer.rb
 - lib/llmemory/retrieval/context_assembler.rb
@@ -240,7 +250,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Persistent memory system for LLM agents
 test_files: []