llmemory 0.2.2 → 0.2.4

data/lib/llmemory/long_term/file_based/storages/database_storage.rb

@@ -3,15 +3,19 @@
 require "json"
 require "securerandom"
 require_relative "base"
+require_relative "../../../crypto/field_helpers"
 
 module Llmemory
   module LongTerm
     module FileBased
       module Storages
         class DatabaseStorage < Base
-          def initialize(database_url: nil)
+          include Llmemory::Crypto::FieldHelpers
+
+          def initialize(database_url: nil, cipher: nil)
             @database_url = database_url || Llmemory.configuration.database_url
             @connection = nil
+            @cipher = cipher || Llmemory.build_cipher
           end
 
           def save_resource(user_id, text)
@@ -19,7 +23,7 @@ module Llmemory
             id = "res_#{SecureRandom.hex(8)}"
             conn.exec_params(
               "INSERT INTO llmemory_resources (id, user_id, text, created_at) VALUES ($1, $2, $3, $4)",
-              [id, user_id, text, Time.now.utc.iso8601]
+              [id, user_id, enc(text), Time.now.utc.iso8601]
             )
             id
           end
@@ -29,7 +33,7 @@ module Llmemory
             id = "item_#{SecureRandom.hex(8)}"
             conn.exec_params(
               "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, importance, provenance, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)",
-              [id, user_id, category, content, source_resource_id, importance.to_f, provenance ? JSON.generate(provenance) : nil, Time.now.utc.iso8601]
+              [id, user_id, category, enc(content), source_resource_id, importance.to_f, provenance_json(provenance), Time.now.utc.iso8601]
             )
             id
           end
@@ -40,7 +44,7 @@ module Llmemory
               "SELECT content FROM llmemory_categories WHERE user_id = $1 AND category_name = $2",
               [user_id, category_name]
             )
-            result.any? ? result.first["content"].to_s : ""
+            result.any? ? dec(result.first["content"].to_s) : ""
           end
 
           def save_category(user_id, category_name, content)
@@ -52,7 +56,7 @@ module Llmemory
                 ON CONFLICT (user_id, category_name)
                 DO UPDATE SET content = $3, updated_at = $4
               SQL
-              [user_id, category_name, content, Time.now.utc.iso8601]
+              [user_id, category_name, enc(content), Time.now.utc.iso8601]
             )
             true
           end
@@ -145,7 +149,7 @@ module Llmemory
                 id,
                 user_id,
                 merged_item[:category],
-                merged_item[:content],
+                enc(merged_item[:content]),
                 merged_item[:source_resource_id],
                 created_at
               ]
@@ -283,7 +287,7 @@ module Llmemory
               {
                 id: r["id"],
                 category: r["category"],
-                content: r["content"],
+                content: dec(r["content"]),
                 source_resource_id: r["source_resource_id"],
                 importance: (r["importance"] || 0.7).to_f,
                 provenance: parse_provenance(r["provenance"]),
@@ -304,16 +308,28 @@ module Llmemory
 
           def parse_provenance(value)
             return nil if value.nil? || value.to_s.strip.empty?
+            return value.transform_keys(&:to_sym) if value.is_a?(Hash)
+            return dec_json(value) if value.is_a?(String) && cipher.encrypted?(value)
+
             JSON.parse(value, symbolize_names: true)
           rescue JSON::ParserError
             nil
           end
 
+          def provenance_json(provenance)
+            return nil unless provenance
+            if cipher.enabled?
+              JSON.generate(enc_json(provenance))
+            else
+              JSON.generate(provenance)
+            end
+          end
+
           def rows_to_resources(rows)
             rows.map do |r|
               {
                 id: r["id"],
-                text: r["text"],
+                text: dec(r["text"]),
                 created_at: Time.parse(r["created_at"])
               }
             end

data/lib/llmemory/long_term/file_based/storages/file_storage.rb

@@ -3,15 +3,19 @@
 require "fileutils"
 require "json"
 require_relative "base"
+require_relative "../../../crypto/field_helpers"
 
 module Llmemory
   module LongTerm
     module FileBased
       module Storages
         class FileStorage < Base
-          def initialize(base_path: nil)
+          include Llmemory::Crypto::FieldHelpers
+
+          def initialize(base_path: nil, cipher: nil)
             @base_path = base_path || Llmemory.configuration.long_term_storage_path || "./llmemory_data"
             @base_path = File.expand_path(@base_path)
+            @cipher = cipher || Llmemory.build_cipher
           end
 
           def save_resource(user_id, text)
@@ -19,8 +23,8 @@ module Llmemory
             seq = next_seq(user_id, "resource_id_seq")
             id = "res_#{seq}"
             path = resource_path(user_id, id)
-            data = { text: text, created_at: Time.now.iso8601 }
-            File.write(path, JSON.generate(data))
+            data = { text: enc(text), created_at: Time.now.iso8601 }
+            write_encrypted_file(path, data)
             id
           end
 
@@ -32,26 +36,26 @@ module Llmemory
             data = {
               id: id,
               category: category,
-              content: content,
+              content: enc(content),
               source_resource_id: source_resource_id,
               importance: importance,
-              provenance: provenance,
+              provenance: provenance ? enc_json(provenance) : nil,
               created_at: Time.now.iso8601
             }
-            File.write(path, JSON.generate(data))
+            write_encrypted_file(path, data)
             id
           end
 
           def load_category(user_id, category_name)
             path = category_path(user_id, category_name)
             return "" unless File.file?(path)
-            File.read(path)
+            read_encrypted_text_file(path)
           end
 
           def save_category(user_id, category_name, content)
             ensure_user_dir(user_id, "categories")
             path = category_path(user_id, category_name)
-            File.write(path, content)
+            write_encrypted_text_file(path, content)
             true
           end
 
@@ -83,7 +87,8 @@ module Llmemory
             dir = user_path(user_id, "items")
             return [] unless Dir.exist?(dir)
             Dir.children(dir).select { |f| f.end_with?(".json") }.map do |f|
-              data = JSON.parse(File.read(File.join(dir, f)), symbolize_names: true)
+              data = read_encrypted_file(File.join(dir, f))
+              data = decrypt_item(data)
               data[:created_at] = parse_time(data[:created_at])
               data
             end.sort_by { |i| i[:created_at] }
@@ -93,9 +98,10 @@ module Llmemory
             dir = user_path(user_id, "resources")
             return [] unless Dir.exist?(dir)
             Dir.children(dir).select { |f| f.end_with?(".json") }.map do |f|
-              data = JSON.parse(File.read(File.join(dir, f)), symbolize_names: true)
+              data = read_encrypted_file(File.join(dir, f))
               id = File.basename(f, ".json")
               data[:id] = id
+              data[:text] = dec(data[:text] || data["text"])
               data[:created_at] = parse_time(data[:created_at])
               data
             end.sort_by { |r| r[:created_at] }
@@ -113,7 +119,9 @@ module Llmemory
             id = "item_#{seq}"
             path = item_path(user_id, id)
             data = merged_item.merge(id: id).transform_values { |v| v.respond_to?(:iso8601) ? v.iso8601 : v }
-            File.write(path, JSON.generate(data))
+            data[:content] = enc(data[:content]) if data[:content]
+            data[:provenance] = enc_json(data[:provenance]) if data[:provenance]
+            write_encrypted_file(path, data)
           end
 
           def archive_items(user_id, item_ids)
@@ -127,9 +135,8 @@ module Llmemory
           def save_daily_log_entry(user_id, date, content)
             ensure_user_dir(user_id, "memory")
             path = daily_log_path(user_id, date)
-            existing = File.file?(path) ? File.read(path) : ""
             entry = "#{Time.now.strftime('%H:%M')} #{content}\n"
-            File.write(path, existing + entry)
+            write_encrypted_text_file(path, entry, append: File.file?(path))
             true
           end
 
@@ -143,7 +150,7 @@ module Llmemory
               path = daily_log_path(user_id, d)
               next unless File.file?(path)
 
-              { date: d, content: File.read(path) }
+              { date: d, content: read_encrypted_text_file(path) }
             end
           end
 
@@ -224,6 +231,13 @@ module Llmemory
             return Time.parse(val.to_s) if val
             Time.now
           end
+
+          def decrypt_item(data)
+            data[:content] = dec(data[:content] || data["content"])
+            prov = data[:provenance] || data["provenance"]
+            data[:provenance] = parse_provenance(prov) if prov
+            data
+          end
         end
       end
     end

data/lib/llmemory/long_term/graph_based/memory.rb

@@ -14,9 +14,10 @@ module Llmemory
       class Memory
         include Llmemory::MemoryModule
 
-        def initialize(user_id:, storage: nil, vector_store: nil, llm: nil, extractor: nil)
+        def initialize(user_id:, storage: nil, vector_store: nil, llm: nil, extractor: nil, cipher: nil)
           @user_id = user_id
-          @graph_storage = storage || Storages.build
+          @cipher = cipher || Llmemory.build_cipher
+          @graph_storage = storage || Storages.build(cipher: @cipher)
           @kg = KnowledgeGraph.new(user_id: user_id, storage: @graph_storage)
           @conflict_resolver = ConflictResolver.new(@kg)
           @vector_store = vector_store || build_vector_store
@@ -112,7 +113,7 @@ module Llmemory
         private
 
         def build_vector_store
-          Llmemory::VectorStore.build(source_type: "edge")
+          Llmemory::VectorStore.build(source_type: "edge", cipher: @cipher)
         end
 
         def extract_graph(text)
@@ -161,6 +162,7 @@ module Llmemory
 
             edge_text = "#{subject} #{predicate} #{object}"
             embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(edge_text) : nil
+            record_embed_usage(@vector_store) if embedding
             if embedding && @vector_store.respond_to?(:store)
               @vector_store.store(id: edge_id, embedding: embedding, metadata: { text: edge_text, created_at: Time.now }, user_id: @user_id)
             end
@@ -171,8 +173,10 @@ module Llmemory
           vector_results = []
           if @vector_store.respond_to?(:search_by_text)
             vector_results = @vector_store.search_by_text(query.to_s, top_k: top_k, user_id: @user_id)
+            record_embed_usage(@vector_store)
           elsif @vector_store.respond_to?(:embed) && @vector_store.respond_to?(:search)
             emb = @vector_store.embed(query.to_s)
+            record_embed_usage(@vector_store)
             vector_results = @vector_store.search(emb, top_k: top_k, user_id: @user_id)
           end
 
@@ -230,6 +234,16 @@ module Llmemory
           lines << "=== END MEMORIES ==="
           lines.join("\n")
         end
+
+        def record_embed_usage(vector_store)
+          return unless vector_store
+
+          Llmemory::LLM::UsageRecorder.record_embed_from_store(
+            user_id: @user_id,
+            vector_store: vector_store,
+            store: Llmemory::ShortTerm::Stores.build(cipher: @cipher)
+          )
+        end
       end
     end
   end

data/lib/llmemory/long_term/graph_based/storage.rb

@@ -7,13 +7,14 @@ module Llmemory
   module LongTerm
     module GraphBased
       module Storages
-        def self.build(store: nil)
+        def self.build(store: nil, cipher: nil)
+          resolved_cipher = cipher || Llmemory.build_cipher
           case (store || Llmemory.configuration.long_term_store).to_s.to_sym
           when :memory
             MemoryStorage.new
           when :active_record, :activerecord
             require_relative "storages/active_record_storage"
-            ActiveRecordStorage.new
+            ActiveRecordStorage.new(cipher: resolved_cipher)
           else
             MemoryStorage.new
           end

data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb

@@ -4,13 +4,17 @@ require_relative "base"
 require_relative "active_record_models"
 require_relative "../node"
 require_relative "../edge"
+require_relative "../../../crypto/field_helpers"
 
 module Llmemory
   module LongTerm
     module GraphBased
       module Storages
         class ActiveRecordStorage < Base
-          def initialize
+          include Llmemory::Crypto::FieldHelpers
+
+          def initialize(cipher: nil)
+            @cipher = cipher || Llmemory.build_cipher
             self.class.load_models!
           end
 
@@ -26,17 +30,22 @@ module Llmemory
             rec = if n.id
               LlmemoryGraphNode.find_by(user_id: user_id, id: n.id)
             else
-              LlmemoryGraphNode.find_by(user_id: user_id, entity_type: n.entity_type, name: n.name)
+              LlmemoryGraphNode.find_by(
+                user_id: user_id,
+                entity_type: enc_det(n.entity_type),
+                name: enc_det(n.name)
+              )
             end
+            stored_props = store_properties(n.properties || {})
             if rec
-              rec.update!(properties: n.properties || {}, updated_at: Time.current)
+              rec.update!(properties: stored_props, updated_at: Time.current)
               rec.id
             else
               rec = LlmemoryGraphNode.create!(
                 user_id: user_id,
-                entity_type: n.entity_type.to_s,
-                name: n.name.to_s,
-                properties: n.properties || {}
+                entity_type: enc_det(n.entity_type.to_s),
+                name: enc_det(n.name.to_s),
+                properties: stored_props
               )
               rec.id
             end
@@ -48,13 +57,17 @@ module Llmemory
           end
 
           def find_node_by_name(user_id, entity_type, name)
-            rec = LlmemoryGraphNode.find_by(user_id: user_id, entity_type: entity_type.to_s, name: name.to_s)
+            rec = LlmemoryGraphNode.find_by(
+              user_id: user_id,
+              entity_type: enc_det(entity_type.to_s),
+              name: enc_det(name.to_s)
+            )
             record_to_node(rec) if rec
           end
 
           def list_nodes(user_id, entity_type: nil, limit: nil, offset: nil)
             scope = LlmemoryGraphNode.where(user_id: user_id)
-            scope = scope.where(entity_type: entity_type) if entity_type
+            scope = scope.where(entity_type: enc_det(entity_type.to_s)) if entity_type
             scope = scope.limit(limit) if limit && limit.to_i.positive?
             scope = scope.offset(offset) if offset && offset.to_i.positive?
             scope.map { |r| record_to_node(r) }
@@ -64,24 +77,23 @@ module Llmemory
             e = edge.is_a?(Edge) ? edge : Edge.from_h(edge.to_h)
             rec = if e.id && e.id.is_a?(Integer)
               LlmemoryGraphEdge.find_by(user_id: user_id, id: e.id)
-            else
-              nil
             end
+            stored_props = store_properties(e.properties || {})
             if rec
               rec.update!(
                 subject_id: e.subject_id,
-                predicate: e.predicate,
+                predicate: enc_det(e.predicate),
                 object_id: e.target_id,
-                properties: e.properties || {}
+                properties: stored_props
               )
               rec.id
             else
               rec = LlmemoryGraphEdge.create!(
                 user_id: user_id,
                 subject_id: e.subject_id,
-                predicate: e.predicate,
+                predicate: enc_det(e.predicate),
                 object_id: e.target_id,
-                properties: e.properties || {}
+                properties: stored_props
               )
               rec.id
             end
@@ -91,7 +103,7 @@ module Llmemory
             scope = LlmemoryGraphEdge.where(user_id: user_id)
             scope = scope.where(archived_at: nil) unless include_archived
             scope = scope.where(subject_id: subject_id) if subject_id
-            scope = scope.where(predicate: predicate) if predicate
+            scope = scope.where(predicate: enc_det(predicate.to_s)) if predicate
             scope = scope.where(object_id: object_id) if object_id
             scope.map { |r| record_to_edge(r) }
           end
@@ -110,7 +122,7 @@ module Llmemory
           def list_edges(user_id, subject_id: nil, predicate: nil, limit: nil, offset: nil)
             scope = LlmemoryGraphEdge.where(user_id: user_id, archived_at: nil)
             scope = scope.where(subject_id: subject_id) if subject_id
-            scope = scope.where(predicate: predicate) if predicate
+            scope = scope.where(predicate: enc_det(predicate.to_s)) if predicate
             scope = scope.order(created_at: :desc) if limit && limit.to_i.positive?
             scope = scope.limit(limit) if limit && limit.to_i.positive?
             scope = scope.offset(offset) if offset && offset.to_i.positive?
@@ -153,13 +165,27 @@ module Llmemory
 
           private
 
+          def store_properties(props)
+            return props || {} unless cipher.enabled?
+            return {} if props.nil? || props.empty?
+
+            enc_json(props)
+          end
+
+          def load_properties(raw)
+            return raw || {} if raw.nil? || raw == {}
+            return dec_json(raw) if raw.is_a?(String) && cipher.encrypted?(raw)
+
+            raw
+          end
+
           def record_to_node(r)
             Node.new(
               id: r.id,
               user_id: r.user_id,
-              entity_type: r.entity_type,
-              name: r.name,
-              properties: r.properties || {},
+              entity_type: dec(r.entity_type),
+              name: dec(r.name),
+              properties: load_properties(r.properties),
               created_at: r.created_at,
               updated_at: r.updated_at
             )
@@ -170,9 +196,9 @@ module Llmemory
               id: r.id,
               user_id: r.user_id,
               subject_id: r.subject_id,
-              predicate: r.predicate,
+              predicate: dec(r.predicate),
               target_id: r.object_id,
-              properties: r.properties || {},
+              properties: load_properties(r.properties),
               created_at: r.created_at,
               archived_at: r.archived_at
             )

data/lib/llmemory/long_term/procedural/memory.rb

@@ -22,9 +22,10 @@ module Llmemory
 
         attr_reader :user_id, :storage
 
-        def initialize(user_id:, storage: nil, vector_store: nil)
+        def initialize(user_id:, storage: nil, vector_store: nil, cipher: nil)
           @user_id = user_id
-          @storage = storage || Storages.build
+          @cipher = cipher || Llmemory.build_cipher
+          @storage = storage || Storages.build(cipher: @cipher)
           @vector_store = vector_store
           @vector_explicit = !vector_store.nil?
         end
@@ -131,7 +132,7 @@ module Llmemory
           if @vector_explicit
             @vector_store
           elsif Llmemory.configuration.procedural_vector_enabled
-            @vector_store ||= Llmemory::VectorStore.build(source_type: "skill")
+            @vector_store ||= Llmemory::VectorStore.build(source_type: "skill", cipher: @cipher)
           end
         end
 
@@ -140,6 +141,7 @@ module Llmemory
           vs = vector_store
           return if vs.nil? || text.to_s.strip.empty?
           embedding = vs.embed(text)
+          record_embed_usage(vs)
           return unless embedding
           vs.store(id: id, embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
         rescue StandardError
@@ -147,7 +149,9 @@ module Llmemory
         end
 
         def vector_candidates(query, top_k, vs)
-          vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id).filter_map do |r|
+          results = vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id)
+          record_embed_usage(vs)
+          results.filter_map do |r|
             raw = @storage.get_skill(@user_id, r[:id] || r["id"])
             raw && candidate_for(raw, (r[:score] || r["score"] || 1.0).to_f)
           end
@@ -177,6 +181,14 @@ module Llmemory
           end
           by_id.values.sort_by { |c| -c[:score].to_f }.first(top_k)
         end
+
+        def record_embed_usage(vector_store)
+          Llmemory::LLM::UsageRecorder.record_embed_from_store(
+            user_id: @user_id,
+            vector_store: vector_store,
+            store: Llmemory::ShortTerm::Stores.build(cipher: @cipher)
+          )
+        end
       end
     end
   end

data/lib/llmemory/long_term/procedural/storage.rb

@@ -12,17 +12,24 @@ module Llmemory
       Storage = Storages::MemoryStorage
 
       module Storages
-        def self.build(store: nil, base_path: nil, database_url: nil)
+        def self.build(store: nil, base_path: nil, database_url: nil, cipher: nil)
+          resolved_cipher = cipher || Llmemory.build_cipher
           case (store || Llmemory.configuration.long_term_store).to_s.to_sym
           when :memory
             MemoryStorage.new
           when :file
-            FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
+            FileStorage.new(
+              base_path: base_path || Llmemory.configuration.long_term_storage_path,
+              cipher: resolved_cipher
+            )
           when :postgres, :database
-            DatabaseStorage.new(database_url: database_url || Llmemory.configuration.database_url)
+            DatabaseStorage.new(
+              database_url: database_url || Llmemory.configuration.database_url,
+              cipher: resolved_cipher
+            )
           when :active_record, :activerecord
             require_relative "storages/active_record_storage"
-            ActiveRecordStorage.new
+            ActiveRecordStorage.new(cipher: resolved_cipher)
           else
             MemoryStorage.new
           end

data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb

@@ -4,6 +4,7 @@ require "json"
 require "securerandom"
 require "time"
 require_relative "base"
+require_relative "../../../crypto/field_helpers"
 
 module Llmemory
   module LongTerm
@@ -13,7 +14,10 @@ module Llmemory
         # auto-deserializes jsonb to a Hash (string keys), which Skill.from_h
         # handles. Mirrors the file-based ActiveRecordStorage pattern.
         class ActiveRecordStorage < Base
-          def initialize
+          include Llmemory::Crypto::FieldHelpers
+
+          def initialize(cipher: nil)
+            @cipher = cipher || Llmemory.build_cipher
             self.class.load_models!
           end
 
@@ -30,42 +34,50 @@ module Llmemory
             data["created_at"] ||= Time.now.utc.iso8601
             rec = LlmemorySkill.find_or_initialize_by(id: id)
             rec.user_id = user_id
-            rec.data = data
-            rec.search_text = searchable_text(data)
+            rec.data = cipher.enabled? ? enc_json(data) : data
+            rec.search_text = enc(searchable_text(data))
             rec.created_at ||= Time.current
             rec.save!
             id
           end
 
           def get_skill(user_id, id)
-            LlmemorySkill.find_by(user_id: user_id, id: id)&.data
+            rec = LlmemorySkill.find_by(user_id: user_id, id: id)
+            return nil unless rec
+
+            decode_data(rec.data)
           end
 
           def list_skills(user_id, limit: nil, offset: nil)
             scope = LlmemorySkill.where(user_id: user_id, archived_at: nil).order(created_at: :desc)
             scope = scope.limit(limit) if limit && limit.to_i.positive?
             scope = scope.offset(offset) if offset && offset.to_i.positive?
-            scope.map(&:data)
+            scope.map { |r| decode_data(r.data) }
           end
 
           def search_skills(user_id, query)
             token_scope(LlmemorySkill.where(user_id: user_id, archived_at: nil), "search_text", query)
-              .order(created_at: :desc).map(&:data)
+              .order(created_at: :desc).map { |r| decode_data(r.data) }
           end
 
           def find_skills_by_name(user_id, name)
-            LlmemorySkill.where(user_id: user_id, archived_at: nil).where("data->>'name' = ?", name.to_s).map(&:data)
+            if cipher.enabled?
+              LlmemorySkill.where(user_id: user_id, archived_at: nil).map { |r| decode_data(r.data) }
+                .select { |s| (s[:name] || s["name"]).to_s == name.to_s }
+            else
+              LlmemorySkill.where(user_id: user_id, archived_at: nil).where("data->>'name' = ?", name.to_s).map(&:data)
+            end
           end
 
           def record_outcome(user_id, skill_id, success:)
             rec = LlmemorySkill.find_by(user_id: user_id, id: skill_id)
             return nil unless rec
-            data = rec.data || {}
-            key = success ? "success_count" : "failure_count"
+            data = decode_data(rec.data) || {}
+            key = success ? :success_count : :failure_count
             data[key] = (data[key] || 0).to_i + 1
-            data["updated_at"] = Time.now.utc.iso8601
-            rec.data = data
-            rec.search_text = searchable_text(data)
+            data[:updated_at] = Time.now.utc.iso8601
+            rec.data = cipher.enabled? ? enc_json(data) : data
+            rec.search_text = enc(searchable_text(data))
             rec.save!
             data
           end
@@ -105,7 +117,15 @@ module Llmemory
           end
 
           def searchable_text(data)
-            [data["name"], data["description"], data["body"]].compact.join("\n")
+            h = data.is_a?(Hash) ? data : {}
+            [h["name"] || h[:name], h["description"] || h[:description], h["body"] || h[:body]].compact.join("\n")
+          end
+
+          def decode_data(raw)
+            return raw.transform_keys(&:to_sym) if raw.is_a?(Hash)
+            return dec_json(raw) if raw.is_a?(String) && cipher.encrypted?(raw)
+
+            raw
           end
         end
       end