llmemory 0.1.15 → 0.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 723fae20d0310ccaeaf9ba600148061d17b2a0b29f933d455d1cf656dee85636
-  data.tar.gz: a135ea1661af46e96843bf52744e8004d0ebe7e8d94b0c46a097c36df53d5bc4
+  metadata.gz: 135c27c05f80b660972a5aa8df5ba315cbdd8235ac68c662ddcfe9249ca79109
+  data.tar.gz: 2e43bacc332ddb44613c79ebd7e4e832091fc68c451a75073860502962d496d9
 SHA512:
-  metadata.gz: 256caaee94233d5e57b8d9e6007fe1ced57d35e21d40260ce34b2803ba0ef3593b66668aa06334e647edd103aa431113e38b639776163d71153c4b9bac68c1a1
-  data.tar.gz: 33cd1726e9f7bb3328610bafabca5ebfe51f080e7d34c523fc0b363eb290b353c9109937f2782ed7e60906965236e79229838e32c698d2f0e2f73aa2d421970b
+  metadata.gz: 9b0d7d67c2647ec0392f993ed31aedc5d51fdc3dadf40bd1a511b90a55b24ce415a9b9a420cc6e676d5a3ba8eb780048e047eff9d0800618dc10e99c4c779a1f
+  data.tar.gz: 10184e66c86c94976d2c164f8717633fbb438e0e95cfb0dbffed4ce52f8bbf8c4266c0b3e48879c792250da15a385464983a886408c69598044c0f60c13f27e7

data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb

@@ -17,6 +17,7 @@ class CreateLlmemoryTables < ActiveRecord::Migration[7.0]
       t.text :content, null: false
       t.string :source_resource_id
       t.float :importance, default: 0.7
+      t.jsonb :provenance
       t.timestamps
     end
     add_index :llmemory_items, :user_id

data/lib/llmemory/configuration.rb

@@ -14,6 +14,7 @@ module Llmemory
                   :database_url,
                   :vector_store,
                   :time_decay_half_life_days,
+                  :importance_weight,
                   :max_retrieval_tokens,
                   :prune_after_days,
                   :compact_max_bytes,
@@ -56,6 +57,7 @@ module Llmemory
       @database_url = ENV["DATABASE_URL"]
       @vector_store = nil
       @time_decay_half_life_days = 30
+      @importance_weight = 1.0
       @max_retrieval_tokens = 2000
       @prune_after_days = 90
       @compact_max_bytes = 8192

data/lib/llmemory/long_term/file_based/item.rb

@@ -4,14 +4,15 @@ module Llmemory
   module LongTerm
     module FileBased
       class Item
-        attr_reader :id, :user_id, :category, :content, :source_resource_id, :created_at
+        attr_reader :id, :user_id, :category, :content, :source_resource_id, :provenance, :created_at
 
-        def initialize(id:, user_id:, category:, content:, source_resource_id: nil, created_at: nil)
+        def initialize(id:, user_id:, category:, content:, source_resource_id: nil, provenance: nil, created_at: nil)
           @id = id
           @user_id = user_id
           @category = category
           @content = content
           @source_resource_id = source_resource_id
+          @provenance = provenance
           @created_at = created_at || Time.now
         end
 
@@ -22,6 +23,7 @@ module Llmemory
             category: category,
             content: content,
             source_resource_id: source_resource_id,
+            provenance: provenance,
             created_at: created_at.iso8601
           }
         end

data/lib/llmemory/long_term/file_based/memory.rb

@@ -65,7 +65,8 @@ module Llmemory
             out << {
               text: i[:content] || i["content"],
               timestamp: i[:created_at] || i["created_at"],
-              score: (i[:importance] || i["importance"] || 1.0).to_f,
+              score: 1.0,
+              importance: (i[:importance] || i["importance"] || 1.0).to_f,
               evergreen: i[:evergreen] || i["evergreen"]
             }
           end
@@ -94,7 +95,10 @@ module Llmemory
 
         def save_item(category:, item:, source_resource_id:, importance: 0.7)
           content = item.is_a?(Hash) ? item["content"] || item[:content] : item.to_s
-          @storage.save_item(@user_id, category: category, content: content, source_resource_id: source_resource_id, importance: importance)
+          provenance = Llmemory::Provenance.from_resource(
+            source_resource_id, method: "fact_extraction", confidence: importance
+          )
+          @storage.save_item(@user_id, category: category, content: content, source_resource_id: source_resource_id, importance: importance, provenance: provenance)
         end
 
         def append_to_daily_log(conversation_text)

data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb

@@ -30,7 +30,7 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             id = "item_#{SecureRandom.hex(8)}"
             attrs = {
               id: id,
@@ -41,6 +41,7 @@ module Llmemory
               created_at: Time.current
             }
             attrs[:importance] = importance if LlmemoryItem.column_names.include?("importance")
+            attrs[:provenance] = provenance if provenance && LlmemoryItem.column_names.include?("provenance")
             LlmemoryItem.create!(attrs)
             id
           end
@@ -189,6 +190,7 @@ module Llmemory
               created_at: r.created_at
             }
             h[:importance] = r.respond_to?(:importance) ? (r.importance || 0.7).to_f : 0.7
+            h[:provenance] = r.provenance if r.respond_to?(:provenance)
             h
           end
 

data/lib/llmemory/long_term/file_based/storages/base.rb

@@ -9,7 +9,7 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#save_resource must be implemented"
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             raise NotImplementedError, "#{self.class}#save_item must be implemented"
           end
 

data/lib/llmemory/long_term/file_based/storages/database_storage.rb

@@ -24,12 +24,12 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             ensure_tables!
             id = "item_#{SecureRandom.hex(8)}"
             conn.exec_params(
-              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, importance, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7)",
-              [id, user_id, category, content, source_resource_id, importance.to_f, Time.now.utc.iso8601]
+              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, importance, provenance, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)",
+              [id, user_id, category, content, source_resource_id, importance.to_f, provenance ? JSON.generate(provenance) : nil, Time.now.utc.iso8601]
             )
             id
           end
@@ -67,7 +67,7 @@ module Llmemory
             ensure_tables!
             pattern = "%#{conn.escape_string(query.to_s.downcase)}%"
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND LOWER(content) LIKE $2",
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND LOWER(content) LIKE $2",
               [user_id, pattern]
             )
             rows_to_items(rows)
@@ -97,7 +97,7 @@ module Llmemory
             ensure_tables!
             cutoff = (Time.now - (days * 86400)).utc.iso8601
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at < $2 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at < $2 ORDER BY created_at",
               [user_id, cutoff]
             )
             rows_to_items(rows)
@@ -106,7 +106,7 @@ module Llmemory
           def get_all_items(user_id)
             ensure_tables!
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 ORDER BY created_at",
               [user_id]
             )
             rows_to_items(rows)
@@ -125,7 +125,7 @@ module Llmemory
             ensure_tables!
             cutoff = (Time.now - (hours * 3600)).utc.iso8601
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at >= $2 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at >= $2 ORDER BY created_at",
               [user_id, cutoff]
             )
             rows_to_items(rows)
@@ -179,7 +179,7 @@ module Llmemory
 
           def list_items(user_id:, category: nil, limit: nil)
             ensure_tables!
-            sql = "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1"
+            sql = "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1"
             params = [user_id]
             if category
               sql += " AND category = $2"
@@ -258,11 +258,13 @@ module Llmemory
                 content TEXT NOT NULL,
                 source_resource_id TEXT,
                 importance REAL DEFAULT 0.7,
+                provenance JSONB,
                 created_at TIMESTAMPTZ NOT NULL
               );
               CREATE INDEX IF NOT EXISTS idx_llmemory_items_user_id ON llmemory_items(user_id);
             SQL
             conn.exec("ALTER TABLE llmemory_items ADD COLUMN IF NOT EXISTS importance REAL DEFAULT 0.7") rescue nil
+            conn.exec("ALTER TABLE llmemory_items ADD COLUMN IF NOT EXISTS provenance JSONB") rescue nil
             conn.exec(<<~SQL)
               CREATE TABLE IF NOT EXISTS llmemory_categories (
                 user_id TEXT NOT NULL,
@@ -282,11 +284,19 @@ module Llmemory
                 content: r["content"],
                 source_resource_id: r["source_resource_id"],
                 importance: (r["importance"] || 0.7).to_f,
+                provenance: parse_provenance(r["provenance"]),
                 created_at: Time.parse(r["created_at"])
               }
             end
           end
 
+          def parse_provenance(value)
+            return nil if value.nil? || value.to_s.strip.empty?
+            JSON.parse(value, symbolize_names: true)
+          rescue JSON::ParserError
+            nil
+          end
+
           def rows_to_resources(rows)
             rows.map do |r|
               {

data/lib/llmemory/long_term/file_based/storages/file_storage.rb

@@ -24,7 +24,7 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             ensure_user_dir(user_id)
             seq = next_seq(user_id, "item_id_seq")
             id = "item_#{seq}"
@@ -35,6 +35,7 @@ module Llmemory
               content: content,
               source_resource_id: source_resource_id,
               importance: importance,
+              provenance: provenance,
               created_at: Time.now.iso8601
             }
             File.write(path, JSON.generate(data))

data/lib/llmemory/long_term/file_based/storages/memory_storage.rb

@@ -22,7 +22,7 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7, provenance: nil)
             @item_id_seq += 1
             id = "item_#{@item_id_seq}"
             @items[user_id] << {
@@ -31,6 +31,7 @@ module Llmemory
               content: content,
               source_resource_id: source_resource_id,
               importance: importance,
+              provenance: provenance,
               created_at: Time.now
             }
             id

data/lib/llmemory/long_term/graph_based/edge.rb

@@ -31,6 +31,13 @@ module Llmemory
           !archived_at.nil?
         end
 
+        # Lineage of this edge, stored within properties so it round-trips
+        # through every backend without a schema change. See Llmemory::Provenance.
+        def provenance
+          props = properties || {}
+          props[:provenance] || props["provenance"]
+        end
+
         def to_h
           {
             id: id,

data/lib/llmemory/long_term/graph_based/memory.rb

@@ -32,6 +32,7 @@ module Llmemory
 
           return true if entities.empty? && relations.empty?
 
+          provenance = Llmemory::Provenance.from_text_fingerprint(text, method: "entity_relation_extraction")
           name_to_id = {}
 
           entities.each do |e|
@@ -39,7 +40,7 @@ module Llmemory
             entity_type = e[:type] || e["type"] || "concept"
             name = e[:name] || e["name"]
             next if name.nil? || name.to_s.strip.empty?
-            id = @kg.add_node(entity_type: entity_type, name: name.to_s.strip, properties: {})
+            id = @kg.add_node(entity_type: entity_type, name: name.to_s.strip, properties: { "provenance" => provenance })
             name_to_id[name.to_s.strip] ||= id
           end
 
@@ -50,8 +51,8 @@ module Llmemory
             object = (r[:object] || r["object"]).to_s.strip
             next if subject.empty? || predicate.empty? || object.empty?
 
-            subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: {})
-            object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: {})
+            subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: { "provenance" => provenance })
+            object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: { "provenance" => provenance })
 
             edge = Edge.new(
               id: nil,
@@ -59,12 +60,12 @@ module Llmemory
               subject_id: subject_id,
               predicate: predicate,
               target_id: object_id,
-              properties: {},
+              properties: { "provenance" => provenance },
               created_at: Time.now,
               archived_at: nil
             )
             @conflict_resolver.resolve(edge)
-            edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: {})
+            edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: { "provenance" => provenance })
 
             text = "#{subject} #{predicate} #{object}"
             embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(text) : nil
@@ -89,7 +90,8 @@ module Llmemory
             {
               text: r[:text],
               timestamp: r[:created_at] || r[:timestamp],
-              score: r[:score] || 1.0
+              score: r[:score] || 1.0,
+              importance: r[:importance]
             }
           end
         end

data/lib/llmemory/long_term/graph_based/node.rb

@@ -25,6 +25,13 @@ module Llmemory
           )
         end
 
+        # Lineage of this node, stored within properties so it round-trips
+        # through every backend without a schema change. See Llmemory::Provenance.
+        def provenance
+          props = properties || {}
+          props[:provenance] || props["provenance"]
+        end
+
         def to_h
           {
             id: id,

data/lib/llmemory/provenance.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Llmemory
+  # Provenance records the lineage of a long-term memory item: where it came
+  # from, how it was produced, and with what confidence. It is stored as a
+  # plain JSON-safe Hash so it round-trips through every storage backend
+  # (in-memory, JSON files, SQL columns, jsonb properties) without coupling.
+  #
+  # Shape: { sources: [{ type:, id: }], method:, confidence:, created_at: }
+  #
+  # `method` identifies the producing process (e.g. "fact_extraction",
+  # "entity_relation_extraction", and in the future "reflection"), so a
+  # semantic datum can always be traced back to its raw source.
+  module Provenance
+    module_function
+
+    def build(method:, sources: [], confidence: nil, created_at: nil)
+      {
+        sources: Array(sources).filter_map { |s| normalize_source(s) },
+        method: method&.to_s,
+        confidence: confidence.nil? ? nil : confidence.to_f,
+        created_at: normalize_time(created_at)
+      }
+    end
+
+    # Convenience for the file-based path, where the raw text is persisted as a
+    # Resource and referenced by id.
+    def from_resource(resource_id, method:, confidence: nil, created_at: nil)
+      sources = resource_id ? [{ type: "resource", id: resource_id }] : []
+      build(method: method, sources: sources, confidence: confidence, created_at: created_at)
+    end
+
+    # Convenience for the graph-based path, which does not persist the raw text.
+    # We record a stable fingerprint of the source instead of the document
+    # itself, keeping lineage verifiable without exposing sensitive content.
+    def from_text_fingerprint(text, method:, confidence: nil, created_at: nil)
+      require "digest"
+      sources = []
+      unless text.to_s.strip.empty?
+        sources = [{ type: "text_sha256", id: Digest::SHA256.hexdigest(text.to_s)[0, 16] }]
+      end
+      build(method: method, sources: sources, confidence: confidence, created_at: created_at)
+    end
+
+    def normalize_source(source)
+      return nil if source.nil?
+      if source.is_a?(Hash)
+        type = source[:type] || source["type"]
+        id = source[:id] || source["id"]
+        return nil if id.nil?
+        { type: type.nil? ? "unknown" : type.to_s, id: id }
+      else
+        { type: "unknown", id: source }
+      end
+    end
+
+    def normalize_time(value)
+      value ||= Time.now
+      value.respond_to?(:iso8601) ? value.iso8601 : value.to_s
+    end
+  end
+end

data/lib/llmemory/retrieval/engine.rb

@@ -54,6 +54,7 @@ module Llmemory
             text: c[:text] || c["text"],
             timestamp: parse_timestamp(c[:timestamp] || c["timestamp"] || c[:created_at] || c["created_at"]),
             score: (c[:score] || c["score"] || 1.0).to_f,
+            importance: c[:importance] || c["importance"],
             evergreen: c[:evergreen] || c["evergreen"]
           }
         end

data/lib/llmemory/retrieval/temporal_ranker.rb

@@ -3,12 +3,14 @@
 module Llmemory
   module Retrieval
     class TemporalRanker
-      def initialize(half_life_days: nil)
+      def initialize(half_life_days: nil, importance_weight: nil)
         @half_life_days = half_life_days || Llmemory.configuration.time_decay_half_life_days
+        @importance_weight = importance_weight || Llmemory.configuration.importance_weight
       end
 
       def rank(candidates, now: Time.now)
         lambda_val = Math.log(2) / @half_life_days.to_f
+        weight = [@importance_weight.to_f, 0.0].max
 
         candidates.map do |c|
           score = (c[:score] || c["score"] || 1.0).to_f
@@ -22,10 +24,22 @@ module Llmemory
             Math.exp(-lambda_val * age_days.to_f)
           end
 
-          final_score = score * time_decay
-          c.merge(score: score, temporal_score: final_score, timestamp: timestamp)
+          importance = normalize_importance(c[:importance] || c["importance"])
+          importance_factor = importance**weight
+
+          final_score = score * time_decay * importance_factor
+          c.merge(score: score, importance: importance, temporal_score: final_score, timestamp: timestamp)
         end.sort_by { |c| -(c[:temporal_score] || 0) }
       end
+
+      private
+
+      # Missing importance is neutral (1.0) so candidates that carry no
+      # importance signal (resources, graph edges) are never penalised.
+      def normalize_importance(value)
+        return 1.0 if value.nil?
+        [[value.to_f, 0.0].max, 1.0].min
+      end
     end
   end
 end

data/lib/llmemory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Llmemory
-  VERSION = "0.1.15"
+  VERSION = "0.1.16"
 end

data/lib/llmemory.rb

@@ -2,6 +2,7 @@
 
 require_relative "llmemory/version"
 require_relative "llmemory/configuration"
+require_relative "llmemory/provenance"
 require_relative "llmemory/llm"
 require_relative "llmemory/short_term"
 require_relative "llmemory/long_term"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llmemory
 version: !ruby/object:Gem::Version
-  version: 0.1.15
+  version: 0.1.16
 platform: ruby
 authors:
 - llmemory
@@ -195,6 +195,7 @@ files:
 - lib/llmemory/mcp/tools/memory_timeline_context.rb
 - lib/llmemory/memory.rb
 - lib/llmemory/noise_filter.rb
+- lib/llmemory/provenance.rb
 - lib/llmemory/retrieval.rb
 - lib/llmemory/retrieval/bm25_scorer.rb
 - lib/llmemory/retrieval/context_assembler.rb
@@ -240,7 +241,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Persistent memory system for LLM agents
 test_files: []