llmemory 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +78 -1
- data/app/controllers/llmemory/dashboard/application_controller.rb +15 -1
- data/app/controllers/llmemory/dashboard/episodic_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/forget_log_controller.rb +12 -0
- data/app/controllers/llmemory/dashboard/maintenance_controller.rb +92 -0
- data/app/controllers/llmemory/dashboard/procedural_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/reflection_controller.rb +37 -0
- data/app/controllers/llmemory/dashboard/working_controller.rb +14 -0
- data/app/views/llmemory/dashboard/episodic/index.html.erb +37 -0
- data/app/views/llmemory/dashboard/forget_log/show.html.erb +23 -0
- data/app/views/llmemory/dashboard/maintenance/show.html.erb +65 -0
- data/app/views/llmemory/dashboard/procedural/index.html.erb +38 -0
- data/app/views/llmemory/dashboard/reflection/show.html.erb +29 -0
- data/app/views/llmemory/dashboard/users/show.html.erb +16 -0
- data/app/views/llmemory/dashboard/working/show.html.erb +20 -0
- data/config/routes.rb +14 -0
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +2 -0
- data/lib/llmemory/cli/commands/maintain.rb +62 -0
- data/lib/llmemory/cli/commands/mine_skills.rb +50 -0
- data/lib/llmemory/cli.rb +6 -0
- data/lib/llmemory/configuration.rb +28 -2
- data/lib/llmemory/crypto/cipher.rb +147 -0
- data/lib/llmemory/crypto/field_helpers.rb +110 -0
- data/lib/llmemory/instrumentation.rb +33 -0
- data/lib/llmemory/llm/anthropic.rb +21 -16
- data/lib/llmemory/llm/openai.rb +18 -13
- data/lib/llmemory/long_term/episodic/memory.rb +27 -13
- data/lib/llmemory/long_term/episodic/storage.rb +11 -4
- data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +33 -10
- data/lib/llmemory/long_term/episodic/storages/base.rb +15 -2
- data/lib/llmemory/long_term/episodic/storages/database_storage.rb +51 -8
- data/lib/llmemory/long_term/episodic/storages/file_storage.rb +47 -9
- data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +35 -4
- data/lib/llmemory/long_term/file_based/memory.rb +12 -4
- data/lib/llmemory/long_term/file_based/storage.rb +11 -4
- data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +20 -12
- data/lib/llmemory/long_term/file_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/file_based/storages/database_storage.rb +28 -10
- data/lib/llmemory/long_term/file_based/storages/file_storage.rb +32 -16
- data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +4 -2
- data/lib/llmemory/long_term/graph_based/memory.rb +16 -7
- data/lib/llmemory/long_term/graph_based/storage.rb +3 -2
- data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb +51 -23
- data/lib/llmemory/long_term/graph_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/graph_based/storages/memory_storage.rb +4 -2
- data/lib/llmemory/long_term/procedural/memory.rb +30 -16
- data/lib/llmemory/long_term/procedural/skill.rb +6 -2
- data/lib/llmemory/long_term/procedural/storage.rb +11 -4
- data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +47 -17
- data/lib/llmemory/long_term/procedural/storages/base.rb +14 -1
- data/lib/llmemory/long_term/procedural/storages/database_storage.rb +52 -10
- data/lib/llmemory/long_term/procedural/storages/file_storage.rb +49 -11
- data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +36 -5
- data/lib/llmemory/maintenance/cognitive_pass.rb +109 -0
- data/lib/llmemory/maintenance/ttl_expiry.rb +50 -0
- data/lib/llmemory/maintenance.rb +2 -0
- data/lib/llmemory/mcp/server.rb +5 -1
- data/lib/llmemory/mcp/tools/memory_maintain.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_mine_skills.rb +53 -0
- data/lib/llmemory/memory.rb +60 -8
- data/lib/llmemory/memory_module.rb +13 -6
- data/lib/llmemory/reflection/reflector.rb +24 -20
- data/lib/llmemory/retrieval/engine.rb +25 -16
- data/lib/llmemory/short_term/checkpoint.rb +3 -2
- data/lib/llmemory/short_term/stores/active_record_store.rb +12 -10
- data/lib/llmemory/short_term/stores/memory_store.rb +1 -1
- data/lib/llmemory/short_term/stores/postgres_store.rb +11 -5
- data/lib/llmemory/short_term/stores/redis_store.rb +7 -5
- data/lib/llmemory/short_term/stores.rb +7 -6
- data/lib/llmemory/skill_mining/miner.rb +163 -0
- data/lib/llmemory/skill_mining.rb +8 -0
- data/lib/llmemory/vector_store/active_record_store.rb +24 -3
- data/lib/llmemory/vector_store/memory_store.rb +23 -3
- data/lib/llmemory/vector_store/openai_embeddings.rb +11 -7
- data/lib/llmemory/vector_store.rb +4 -3
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory.rb +4 -0
- metadata +24 -1
|
@@ -12,17 +12,24 @@ module Llmemory
|
|
|
12
12
|
Storage = Storages::MemoryStorage
|
|
13
13
|
|
|
14
14
|
module Storages
|
|
15
|
-
def self.build(store: nil, base_path: nil, database_url: nil)
|
|
15
|
+
def self.build(store: nil, base_path: nil, database_url: nil, cipher: nil)
|
|
16
|
+
resolved_cipher = cipher || Llmemory.build_cipher
|
|
16
17
|
case (store || Llmemory.configuration.long_term_store).to_s.to_sym
|
|
17
18
|
when :memory
|
|
18
19
|
MemoryStorage.new
|
|
19
20
|
when :file
|
|
20
|
-
FileStorage.new(
|
|
21
|
+
FileStorage.new(
|
|
22
|
+
base_path: base_path || Llmemory.configuration.long_term_storage_path,
|
|
23
|
+
cipher: resolved_cipher
|
|
24
|
+
)
|
|
21
25
|
when :postgres, :database
|
|
22
|
-
DatabaseStorage.new(
|
|
26
|
+
DatabaseStorage.new(
|
|
27
|
+
database_url: database_url || Llmemory.configuration.database_url,
|
|
28
|
+
cipher: resolved_cipher
|
|
29
|
+
)
|
|
23
30
|
when :active_record, :activerecord
|
|
24
31
|
require_relative "storages/active_record_storage"
|
|
25
|
-
ActiveRecordStorage.new
|
|
32
|
+
ActiveRecordStorage.new(cipher: resolved_cipher)
|
|
26
33
|
else
|
|
27
34
|
MemoryStorage.new
|
|
28
35
|
end
|
|
@@ -4,6 +4,7 @@ require "json"
|
|
|
4
4
|
require "securerandom"
|
|
5
5
|
require "time"
|
|
6
6
|
require_relative "base"
|
|
7
|
+
require_relative "../../../crypto/field_helpers"
|
|
7
8
|
|
|
8
9
|
module Llmemory
|
|
9
10
|
module LongTerm
|
|
@@ -13,7 +14,10 @@ module Llmemory
|
|
|
13
14
|
# AR auto-deserializes jsonb to a Hash (string keys), which Episode.from_h
|
|
14
15
|
# handles. Mirrors the file-based ActiveRecordStorage pattern.
|
|
15
16
|
class ActiveRecordStorage < Base
|
|
16
|
-
|
|
17
|
+
include Llmemory::Crypto::FieldHelpers
|
|
18
|
+
|
|
19
|
+
def initialize(cipher: nil)
|
|
20
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
17
21
|
self.class.load_models!
|
|
18
22
|
end
|
|
19
23
|
|
|
@@ -30,8 +34,8 @@ module Llmemory
|
|
|
30
34
|
data["created_at"] ||= Time.now.utc.iso8601
|
|
31
35
|
rec = LlmemoryEpisode.find_or_initialize_by(id: id)
|
|
32
36
|
rec.user_id = user_id
|
|
33
|
-
rec.data = data
|
|
34
|
-
rec.search_text = searchable_text(data)
|
|
37
|
+
rec.data = cipher.enabled? ? enc_json(data) : data
|
|
38
|
+
rec.search_text = enc(searchable_text(data))
|
|
35
39
|
rec.created_at ||= Time.current
|
|
36
40
|
rec.save!
|
|
37
41
|
id
|
|
@@ -39,28 +43,40 @@ module Llmemory
|
|
|
39
43
|
|
|
40
44
|
def get_episode(user_id, id)
|
|
41
45
|
rec = LlmemoryEpisode.find_by(user_id: user_id, id: id)
|
|
42
|
-
rec
|
|
46
|
+
return nil unless rec
|
|
47
|
+
|
|
48
|
+
decode_data(rec.data)
|
|
43
49
|
end
|
|
44
50
|
|
|
45
|
-
def list_episodes(user_id, limit: nil)
|
|
46
|
-
scope = LlmemoryEpisode.where(user_id: user_id).order(created_at: :desc)
|
|
51
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
52
|
+
scope = LlmemoryEpisode.where(user_id: user_id, archived_at: nil).order(created_at: :desc)
|
|
47
53
|
scope = scope.limit(limit) if limit && limit.to_i.positive?
|
|
48
|
-
scope.
|
|
54
|
+
scope = scope.offset(offset) if offset && offset.to_i.positive?
|
|
55
|
+
scope.map { |r| decode_data(r.data) }
|
|
49
56
|
end
|
|
50
57
|
|
|
51
58
|
def search_episodes(user_id, query)
|
|
52
|
-
token_scope(LlmemoryEpisode.where(user_id: user_id), "search_text", query)
|
|
53
|
-
.order(created_at: :desc).map(
|
|
59
|
+
token_scope(LlmemoryEpisode.where(user_id: user_id, archived_at: nil), "search_text", query)
|
|
60
|
+
.order(created_at: :desc).map { |r| decode_data(r.data) }
|
|
54
61
|
end
|
|
55
62
|
|
|
56
63
|
def count_episodes(user_id)
|
|
57
|
-
LlmemoryEpisode.where(user_id: user_id).count
|
|
64
|
+
LlmemoryEpisode.where(user_id: user_id, archived_at: nil).count
|
|
58
65
|
end
|
|
59
66
|
|
|
60
67
|
def delete_episodes(user_id, ids)
|
|
61
68
|
LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s)).delete_all
|
|
62
69
|
end
|
|
63
70
|
|
|
71
|
+
def archive_episodes(user_id, ids)
|
|
72
|
+
LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s), archived_at: nil)
|
|
73
|
+
.update_all(archived_at: Time.current)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
77
|
+
LlmemoryEpisode.where(user_id: user_id, archived_at: nil).where("created_at < ?", cutoff).pluck(:id)
|
|
78
|
+
end
|
|
79
|
+
|
|
64
80
|
def list_users
|
|
65
81
|
LlmemoryEpisode.distinct.pluck(:user_id)
|
|
66
82
|
end
|
|
@@ -86,6 +102,13 @@ module Llmemory
|
|
|
86
102
|
end
|
|
87
103
|
parts.compact.join("\n")
|
|
88
104
|
end
|
|
105
|
+
|
|
106
|
+
def decode_data(raw)
|
|
107
|
+
return raw.transform_keys(&:to_sym) if raw.is_a?(Hash)
|
|
108
|
+
return dec_json(raw) if raw.is_a?(String) && cipher.encrypted?(raw)
|
|
109
|
+
|
|
110
|
+
raw
|
|
111
|
+
end
|
|
89
112
|
end
|
|
90
113
|
end
|
|
91
114
|
end
|
|
@@ -16,8 +16,8 @@ module Llmemory
|
|
|
16
16
|
raise NotImplementedError, "#{self.class}#get_episode must be implemented"
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
# Newest first. Optionally
|
|
20
|
-
def list_episodes(user_id, limit: nil)
|
|
19
|
+
# Newest first. Optionally paginated with offset/limit.
|
|
20
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
21
21
|
raise NotImplementedError, "#{self.class}#list_episodes must be implemented"
|
|
22
22
|
end
|
|
23
23
|
|
|
@@ -34,6 +34,19 @@ module Llmemory
|
|
|
34
34
|
raise NotImplementedError, "#{self.class}#delete_episodes must be implemented"
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
+
# Soft-archives episodes by id (sets archived_at on the record). Archived
|
|
38
|
+
# episodes are excluded from list_episodes / search_episodes / count_episodes
|
|
39
|
+
# but remain accessible via get_episode. Returns the number archived.
|
|
40
|
+
def archive_episodes(user_id, ids)
|
|
41
|
+
raise NotImplementedError, "#{self.class}#archive_episodes must be implemented"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Returns episodes whose created_at is older than the cutoff and that are
|
|
45
|
+
# not already archived. Used by the TTL maintenance job.
|
|
46
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
47
|
+
raise NotImplementedError, "#{self.class}#expired_episode_ids must be implemented"
|
|
48
|
+
end
|
|
49
|
+
|
|
37
50
|
def list_users
|
|
38
51
|
raise NotImplementedError, "#{self.class}#list_users must be implemented"
|
|
39
52
|
end
|
|
@@ -4,6 +4,7 @@ require "json"
|
|
|
4
4
|
require "securerandom"
|
|
5
5
|
require "time"
|
|
6
6
|
require_relative "base"
|
|
7
|
+
require_relative "../../../crypto/field_helpers"
|
|
7
8
|
|
|
8
9
|
module Llmemory
|
|
9
10
|
module LongTerm
|
|
@@ -13,9 +14,12 @@ module Llmemory
|
|
|
13
14
|
# (plus id/user_id/created_at and a denormalized search_text for keyword
|
|
14
15
|
# search), mirroring the file-based DatabaseStorage pattern.
|
|
15
16
|
class DatabaseStorage < Base
|
|
16
|
-
|
|
17
|
+
include Llmemory::Crypto::FieldHelpers
|
|
18
|
+
|
|
19
|
+
def initialize(database_url: nil, cipher: nil)
|
|
17
20
|
@database_url = database_url || Llmemory.configuration.database_url
|
|
18
21
|
@connection = nil
|
|
22
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
19
23
|
end
|
|
20
24
|
|
|
21
25
|
def save_episode(user_id, episode)
|
|
@@ -23,11 +27,12 @@ module Llmemory
|
|
|
23
27
|
id = episode[:id] || episode["id"] || "ep_#{SecureRandom.hex(8)}"
|
|
24
28
|
data = symbolize(episode).merge(id: id, user_id: user_id)
|
|
25
29
|
data[:created_at] ||= Time.now.utc.iso8601
|
|
30
|
+
search = searchable_text(data)
|
|
26
31
|
conn.exec_params(
|
|
27
32
|
"INSERT INTO llmemory_episodes (id, user_id, data, search_text, created_at) " \
|
|
28
33
|
"VALUES ($1, $2, $3::jsonb, $4, $5) " \
|
|
29
34
|
"ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
|
|
30
|
-
[id, user_id,
|
|
35
|
+
[id, user_id, store_data(data), enc(search), created_at_value(data)]
|
|
31
36
|
)
|
|
32
37
|
id
|
|
33
38
|
end
|
|
@@ -38,10 +43,11 @@ module Llmemory
|
|
|
38
43
|
rows.any? ? parse_data(rows.first["data"]) : nil
|
|
39
44
|
end
|
|
40
45
|
|
|
41
|
-
def list_episodes(user_id, limit: nil)
|
|
46
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
42
47
|
ensure_tables!
|
|
43
|
-
sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 ORDER BY created_at DESC"
|
|
48
|
+
sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL ORDER BY created_at DESC"
|
|
44
49
|
sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
|
|
50
|
+
sql += " OFFSET #{offset.to_i}" if offset && offset.to_i.positive?
|
|
45
51
|
conn.exec_params(sql, [user_id]).map { |r| parse_data(r["data"]) }
|
|
46
52
|
end
|
|
47
53
|
|
|
@@ -49,14 +55,14 @@ module Llmemory
|
|
|
49
55
|
ensure_tables!
|
|
50
56
|
suffix, params = token_filter("search_text", query, 2)
|
|
51
57
|
conn.exec_params(
|
|
52
|
-
"SELECT data FROM llmemory_episodes WHERE user_id = $1#{suffix} ORDER BY created_at DESC",
|
|
58
|
+
"SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL#{suffix} ORDER BY created_at DESC",
|
|
53
59
|
[user_id, *params]
|
|
54
60
|
).map { |r| parse_data(r["data"]) }
|
|
55
61
|
end
|
|
56
62
|
|
|
57
63
|
def count_episodes(user_id)
|
|
58
64
|
ensure_tables!
|
|
59
|
-
conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1", [user_id]).first["c"].to_i
|
|
65
|
+
conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL", [user_id]).first["c"].to_i
|
|
60
66
|
end
|
|
61
67
|
|
|
62
68
|
def delete_episodes(user_id, ids)
|
|
@@ -66,6 +72,24 @@ module Llmemory
|
|
|
66
72
|
end
|
|
67
73
|
end
|
|
68
74
|
|
|
75
|
+
def archive_episodes(user_id, ids)
|
|
76
|
+
ensure_tables!
|
|
77
|
+
Array(ids).sum do |id|
|
|
78
|
+
conn.exec_params(
|
|
79
|
+
"UPDATE llmemory_episodes SET archived_at = NOW() WHERE user_id = $1 AND id = $2 AND archived_at IS NULL",
|
|
80
|
+
[user_id, id]
|
|
81
|
+
).cmd_tuples
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
86
|
+
ensure_tables!
|
|
87
|
+
conn.exec_params(
|
|
88
|
+
"SELECT id FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL AND created_at < $2",
|
|
89
|
+
[user_id, cutoff.iso8601]
|
|
90
|
+
).map { |r| r["id"] }
|
|
91
|
+
end
|
|
92
|
+
|
|
69
93
|
def list_users
|
|
70
94
|
ensure_tables!
|
|
71
95
|
conn.exec("SELECT DISTINCT user_id FROM llmemory_episodes").map { |r| r["user_id"] }
|
|
@@ -87,9 +111,11 @@ module Llmemory
|
|
|
87
111
|
user_id TEXT NOT NULL,
|
|
88
112
|
data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
89
113
|
search_text TEXT,
|
|
90
|
-
created_at TIMESTAMPTZ NOT NULL
|
|
114
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
115
|
+
archived_at TIMESTAMPTZ
|
|
91
116
|
);
|
|
92
117
|
CREATE INDEX IF NOT EXISTS idx_llmemory_episodes_user_id ON llmemory_episodes(user_id);
|
|
118
|
+
ALTER TABLE llmemory_episodes ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
|
|
93
119
|
SQL
|
|
94
120
|
end
|
|
95
121
|
|
|
@@ -103,11 +129,28 @@ module Llmemory
|
|
|
103
129
|
end
|
|
104
130
|
|
|
105
131
|
def parse_data(value)
|
|
106
|
-
|
|
132
|
+
if value.is_a?(Hash)
|
|
133
|
+
return value.transform_keys(&:to_sym)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
str = value.to_s
|
|
137
|
+
if cipher.encrypted?(str)
|
|
138
|
+
cipher.decrypt_json(str)
|
|
139
|
+
else
|
|
140
|
+
JSON.parse(str, symbolize_names: true)
|
|
141
|
+
end
|
|
107
142
|
rescue JSON::ParserError
|
|
108
143
|
{}
|
|
109
144
|
end
|
|
110
145
|
|
|
146
|
+
def store_data(data)
|
|
147
|
+
if cipher.enabled?
|
|
148
|
+
JSON.generate(enc_json(data))
|
|
149
|
+
else
|
|
150
|
+
JSON.generate(data)
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
111
154
|
def symbolize(hash)
|
|
112
155
|
hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
|
|
113
156
|
end
|
|
@@ -4,22 +4,26 @@ require "fileutils"
|
|
|
4
4
|
require "json"
|
|
5
5
|
require "time"
|
|
6
6
|
require_relative "base"
|
|
7
|
+
require_relative "../../../crypto/field_helpers"
|
|
7
8
|
|
|
8
9
|
module Llmemory
|
|
9
10
|
module LongTerm
|
|
10
11
|
module Episodic
|
|
11
12
|
module Storages
|
|
12
13
|
class FileStorage < Base
|
|
13
|
-
|
|
14
|
+
include Llmemory::Crypto::FieldHelpers
|
|
15
|
+
|
|
16
|
+
def initialize(base_path: nil, cipher: nil)
|
|
14
17
|
@base_path = base_path || Llmemory.configuration.long_term_storage_path || "./llmemory_data"
|
|
15
18
|
@base_path = File.expand_path(@base_path)
|
|
19
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
16
20
|
end
|
|
17
21
|
|
|
18
22
|
def save_episode(user_id, episode)
|
|
19
23
|
id = episode[:id] || episode["id"] || "ep_#{next_seq(user_id)}"
|
|
20
24
|
data = stringify_for_json(episode).merge("id" => id, "user_id" => user_id)
|
|
21
25
|
data["created_at"] ||= Time.now.iso8601
|
|
22
|
-
|
|
26
|
+
write_episode_file(episode_path(user_id, id), data)
|
|
23
27
|
id
|
|
24
28
|
end
|
|
25
29
|
|
|
@@ -29,20 +33,19 @@ module Llmemory
|
|
|
29
33
|
load_episode(path)
|
|
30
34
|
end
|
|
31
35
|
|
|
32
|
-
def list_episodes(user_id, limit: nil)
|
|
33
|
-
sorted =
|
|
36
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
37
|
+
sorted = active_episodes(user_id).sort_by { |e| e[:created_at] }.reverse
|
|
38
|
+
sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
|
|
34
39
|
limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
|
|
35
40
|
end
|
|
36
41
|
|
|
37
42
|
def search_episodes(user_id, query)
|
|
38
43
|
return list_episodes(user_id) if query.to_s.strip.empty?
|
|
39
|
-
|
|
44
|
+
active_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
|
|
40
45
|
end
|
|
41
46
|
|
|
42
47
|
def count_episodes(user_id)
|
|
43
|
-
|
|
44
|
-
return 0 unless Dir.exist?(dir)
|
|
45
|
-
Dir.children(dir).count { |f| f.end_with?(".json") }
|
|
48
|
+
active_episodes(user_id).size
|
|
46
49
|
end
|
|
47
50
|
|
|
48
51
|
def delete_episodes(user_id, ids)
|
|
@@ -54,6 +57,24 @@ module Llmemory
|
|
|
54
57
|
end
|
|
55
58
|
end
|
|
56
59
|
|
|
60
|
+
def archive_episodes(user_id, ids)
|
|
61
|
+
Array(ids).map(&:to_s).count do |id|
|
|
62
|
+
path = episode_path(user_id, id)
|
|
63
|
+
next false unless File.file?(path)
|
|
64
|
+
data = load_episode_raw(path)
|
|
65
|
+
next false if data["archived_at"]
|
|
66
|
+
data["archived_at"] = Time.now.iso8601
|
|
67
|
+
write_episode_file(path, data)
|
|
68
|
+
true
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
73
|
+
active_episodes(user_id)
|
|
74
|
+
.select { |e| (e[:created_at] || Time.now) < cutoff }
|
|
75
|
+
.map { |e| e[:id].to_s }
|
|
76
|
+
end
|
|
77
|
+
|
|
57
78
|
def list_users
|
|
58
79
|
return [] unless Dir.exist?(@base_path)
|
|
59
80
|
Dir.children(@base_path).select { |d| Dir.exist?(File.join(@base_path, d, "episodes")) }
|
|
@@ -61,6 +82,10 @@ module Llmemory
|
|
|
61
82
|
|
|
62
83
|
private
|
|
63
84
|
|
|
85
|
+
def active_episodes(user_id)
|
|
86
|
+
all_episodes(user_id).reject { |e| e[:archived_at] }
|
|
87
|
+
end
|
|
88
|
+
|
|
64
89
|
def all_episodes(user_id)
|
|
65
90
|
dir = user_path(user_id, "episodes")
|
|
66
91
|
return [] unless Dir.exist?(dir)
|
|
@@ -68,13 +93,26 @@ module Llmemory
|
|
|
68
93
|
end
|
|
69
94
|
|
|
70
95
|
def load_episode(path)
|
|
71
|
-
data =
|
|
96
|
+
data = load_episode_raw(path)
|
|
97
|
+
return nil unless data
|
|
98
|
+
|
|
72
99
|
data[:created_at] = parse_time(data[:created_at])
|
|
73
100
|
data
|
|
74
101
|
rescue JSON::ParserError
|
|
75
102
|
nil
|
|
76
103
|
end
|
|
77
104
|
|
|
105
|
+
def load_episode_raw(path)
|
|
106
|
+
raw = File.read(path)
|
|
107
|
+
json = cipher.enabled? && cipher.encrypted?(raw) ? cipher.decrypt(raw) : raw
|
|
108
|
+
JSON.parse(json, symbolize_names: true)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def write_episode_file(path, data)
|
|
112
|
+
payload = JSON.generate(data)
|
|
113
|
+
File.write(path, cipher.enabled? ? cipher.encrypt(payload) : payload)
|
|
114
|
+
end
|
|
115
|
+
|
|
78
116
|
def episode_text(episode)
|
|
79
117
|
parts = [episode[:summary], episode[:outcome]]
|
|
80
118
|
Array(episode[:steps]).each do |s|
|
|
@@ -25,18 +25,19 @@ module Llmemory
|
|
|
25
25
|
@episodes[user_id].find { |e| e[:id] == id }
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
-
def list_episodes(user_id, limit: nil)
|
|
29
|
-
sorted =
|
|
28
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
29
|
+
sorted = active_episodes(user_id).sort_by { |e| as_time(e[:created_at]) }.reverse
|
|
30
|
+
sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
|
|
30
31
|
limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
|
|
31
32
|
end
|
|
32
33
|
|
|
33
34
|
def search_episodes(user_id, query)
|
|
34
35
|
return list_episodes(user_id) if query.to_s.strip.empty?
|
|
35
|
-
|
|
36
|
+
active_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
|
|
36
37
|
end
|
|
37
38
|
|
|
38
39
|
def count_episodes(user_id)
|
|
39
|
-
|
|
40
|
+
active_episodes(user_id).size
|
|
40
41
|
end
|
|
41
42
|
|
|
42
43
|
def delete_episodes(user_id, ids)
|
|
@@ -46,12 +47,42 @@ module Llmemory
|
|
|
46
47
|
before - @episodes[user_id].size
|
|
47
48
|
end
|
|
48
49
|
|
|
50
|
+
def archive_episodes(user_id, ids)
|
|
51
|
+
ids = Array(ids).map(&:to_s)
|
|
52
|
+
count = 0
|
|
53
|
+
@episodes[user_id].each do |e|
|
|
54
|
+
next unless ids.include?(e[:id].to_s)
|
|
55
|
+
next if e[:archived_at]
|
|
56
|
+
e[:archived_at] = Time.now
|
|
57
|
+
count += 1
|
|
58
|
+
end
|
|
59
|
+
count
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
63
|
+
active_episodes(user_id)
|
|
64
|
+
.select { |e| as_time(e[:created_at]) < cutoff }
|
|
65
|
+
.map { |e| e[:id].to_s }
|
|
66
|
+
end
|
|
67
|
+
|
|
49
68
|
def list_users
|
|
50
69
|
@episodes.keys
|
|
51
70
|
end
|
|
52
71
|
|
|
53
72
|
private
|
|
54
73
|
|
|
74
|
+
def active_episodes(user_id)
|
|
75
|
+
@episodes[user_id].reject { |e| e[:archived_at] }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def as_time(value)
|
|
79
|
+
return Time.now if value.nil?
|
|
80
|
+
return value if value.is_a?(Time)
|
|
81
|
+
Time.parse(value.to_s)
|
|
82
|
+
rescue ArgumentError
|
|
83
|
+
Time.now
|
|
84
|
+
end
|
|
85
|
+
|
|
55
86
|
def symbolize(hash)
|
|
56
87
|
hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
|
|
57
88
|
end
|
|
@@ -108,11 +108,15 @@ module Llmemory
|
|
|
108
108
|
# --- MemoryModule uniform interface ---
|
|
109
109
|
|
|
110
110
|
def write(payload, **_meta)
|
|
111
|
-
|
|
111
|
+
result = nil
|
|
112
|
+
Llmemory::Instrumentation.instrument(:memory_write, memory_type: "file_based", user_id: @user_id) do
|
|
113
|
+
result = memorize(payload)
|
|
114
|
+
end
|
|
115
|
+
result
|
|
112
116
|
end
|
|
113
117
|
|
|
114
|
-
def list(user_id: nil, limit: nil)
|
|
115
|
-
@storage.list_items(user_id: user_id || @user_id, limit: limit)
|
|
118
|
+
def list(user_id: nil, limit: nil, offset: nil)
|
|
119
|
+
@storage.list_items(user_id: user_id || @user_id, limit: limit, offset: offset)
|
|
116
120
|
end
|
|
117
121
|
|
|
118
122
|
def stats(user_id: nil)
|
|
@@ -120,7 +124,10 @@ module Llmemory
|
|
|
120
124
|
end
|
|
121
125
|
|
|
122
126
|
# Removes items/resources by id and records the removal in the audit log.
|
|
123
|
-
|
|
127
|
+
# Note: file-based storages currently implement `archive_*` as physical
|
|
128
|
+
# removal — `mode: :soft` and `mode: :hard` are functionally equivalent
|
|
129
|
+
# here. Kept for API uniformity.
|
|
130
|
+
def forget(ids:, reason: nil, mode: :soft)
|
|
124
131
|
requested = Array(ids).map(&:to_s)
|
|
125
132
|
existing = (@storage.get_all_items(@user_id) + @storage.get_all_resources(@user_id))
|
|
126
133
|
.map { |r| (r[:id] || r["id"]).to_s }
|
|
@@ -128,6 +135,7 @@ module Llmemory
|
|
|
128
135
|
@storage.archive_items(@user_id, removed)
|
|
129
136
|
@storage.archive_resources(@user_id, removed)
|
|
130
137
|
forget_log.record(@user_id, memory_type: "file_based", ids: removed, reason: reason)
|
|
138
|
+
Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "file_based", user_id: @user_id, count: removed.size, mode: mode)
|
|
131
139
|
removed.size
|
|
132
140
|
end
|
|
133
141
|
|
|
@@ -14,17 +14,24 @@ module Llmemory
|
|
|
14
14
|
Storage = Storages::MemoryStorage
|
|
15
15
|
|
|
16
16
|
module Storages
|
|
17
|
-
def self.build(store: nil, base_path: nil, database_url: nil)
|
|
17
|
+
def self.build(store: nil, base_path: nil, database_url: nil, cipher: nil)
|
|
18
|
+
resolved_cipher = cipher || Llmemory.build_cipher
|
|
18
19
|
case (store || Llmemory.configuration.long_term_store).to_s.to_sym
|
|
19
20
|
when :memory
|
|
20
21
|
MemoryStorage.new
|
|
21
22
|
when :file
|
|
22
|
-
FileStorage.new(
|
|
23
|
+
FileStorage.new(
|
|
24
|
+
base_path: base_path || Llmemory.configuration.long_term_storage_path,
|
|
25
|
+
cipher: resolved_cipher
|
|
26
|
+
)
|
|
23
27
|
when :postgres, :database
|
|
24
|
-
DatabaseStorage.new(
|
|
28
|
+
DatabaseStorage.new(
|
|
29
|
+
database_url: database_url || Llmemory.configuration.database_url,
|
|
30
|
+
cipher: resolved_cipher
|
|
31
|
+
)
|
|
25
32
|
when :active_record, :activerecord
|
|
26
33
|
require_relative "storages/active_record_storage"
|
|
27
|
-
ActiveRecordStorage.new
|
|
34
|
+
ActiveRecordStorage.new(cipher: resolved_cipher)
|
|
28
35
|
else
|
|
29
36
|
MemoryStorage.new
|
|
30
37
|
end
|
|
@@ -2,13 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
require "securerandom"
|
|
4
4
|
require_relative "base"
|
|
5
|
+
require_relative "../../../crypto/field_helpers"
|
|
5
6
|
|
|
6
7
|
module Llmemory
|
|
7
8
|
module LongTerm
|
|
8
9
|
module FileBased
|
|
9
10
|
module Storages
|
|
10
11
|
class ActiveRecordStorage < Base
|
|
11
|
-
|
|
12
|
+
include Llmemory::Crypto::FieldHelpers
|
|
13
|
+
|
|
14
|
+
def initialize(cipher: nil)
|
|
15
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
12
16
|
self.class.load_models!
|
|
13
17
|
end
|
|
14
18
|
|
|
@@ -24,7 +28,7 @@ module Llmemory
|
|
|
24
28
|
LlmemoryResource.create!(
|
|
25
29
|
id: id,
|
|
26
30
|
user_id: user_id,
|
|
27
|
-
text: text,
|
|
31
|
+
text: enc(text),
|
|
28
32
|
created_at: Time.current
|
|
29
33
|
)
|
|
30
34
|
id
|
|
@@ -36,24 +40,26 @@ module Llmemory
|
|
|
36
40
|
id: id,
|
|
37
41
|
user_id: user_id,
|
|
38
42
|
category: category,
|
|
39
|
-
content: content,
|
|
43
|
+
content: enc(content),
|
|
40
44
|
source_resource_id: source_resource_id,
|
|
41
45
|
created_at: Time.current
|
|
42
46
|
}
|
|
43
47
|
attrs[:importance] = importance if LlmemoryItem.column_names.include?("importance")
|
|
44
|
-
|
|
48
|
+
if provenance && LlmemoryItem.column_names.include?("provenance")
|
|
49
|
+
attrs[:provenance] = cipher.enabled? ? enc_json(provenance) : provenance
|
|
50
|
+
end
|
|
45
51
|
LlmemoryItem.create!(attrs)
|
|
46
52
|
id
|
|
47
53
|
end
|
|
48
54
|
|
|
49
55
|
def load_category(user_id, category_name)
|
|
50
56
|
rec = LlmemoryCategory.find_by(user_id: user_id, category_name: category_name)
|
|
51
|
-
rec ? rec.content.to_s : ""
|
|
57
|
+
rec ? dec(rec.content.to_s) : ""
|
|
52
58
|
end
|
|
53
59
|
|
|
54
60
|
def save_category(user_id, category_name, content)
|
|
55
61
|
rec = LlmemoryCategory.find_or_initialize_by(user_id: user_id, category_name: category_name)
|
|
56
|
-
rec.content = content
|
|
62
|
+
rec.content = enc(content)
|
|
57
63
|
rec.updated_at = Time.current
|
|
58
64
|
rec.save!
|
|
59
65
|
true
|
|
@@ -101,7 +107,7 @@ module Llmemory
|
|
|
101
107
|
id: "item_#{SecureRandom.hex(8)}",
|
|
102
108
|
user_id: user_id,
|
|
103
109
|
category: merged_item[:category],
|
|
104
|
-
content: merged_item[:content],
|
|
110
|
+
content: enc(merged_item[:content]),
|
|
105
111
|
source_resource_id: merged_item[:source_resource_id],
|
|
106
112
|
created_at: created_at
|
|
107
113
|
}
|
|
@@ -123,17 +129,19 @@ module Llmemory
|
|
|
123
129
|
LlmemoryCategory.distinct.pluck(:user_id)).uniq
|
|
124
130
|
end
|
|
125
131
|
|
|
126
|
-
def list_resources(user_id:, limit: nil)
|
|
132
|
+
def list_resources(user_id:, limit: nil, offset: nil)
|
|
127
133
|
scope = LlmemoryResource.where(user_id: user_id).order(:created_at)
|
|
128
134
|
scope = scope.limit(limit) if limit && limit.to_i.positive?
|
|
135
|
+
scope = scope.offset(offset) if offset && offset.to_i.positive?
|
|
129
136
|
scope.map { |r| row_to_resource(r) }
|
|
130
137
|
end
|
|
131
138
|
|
|
132
|
-
def list_items(user_id:, category: nil, limit: nil)
|
|
139
|
+
def list_items(user_id:, category: nil, limit: nil, offset: nil)
|
|
133
140
|
scope = LlmemoryItem.where(user_id: user_id)
|
|
134
141
|
scope = scope.where(category: category) if category
|
|
135
142
|
scope = scope.order(:created_at)
|
|
136
143
|
scope = scope.limit(limit) if limit && limit.to_i.positive?
|
|
144
|
+
scope = scope.offset(offset) if offset && offset.to_i.positive?
|
|
137
145
|
scope.map { |r| row_to_item(r) }
|
|
138
146
|
end
|
|
139
147
|
|
|
@@ -192,19 +200,19 @@ module Llmemory
|
|
|
192
200
|
h = {
|
|
193
201
|
id: r.id,
|
|
194
202
|
category: r.category,
|
|
195
|
-
content: r.content,
|
|
203
|
+
content: dec(r.content),
|
|
196
204
|
source_resource_id: r.source_resource_id,
|
|
197
205
|
created_at: r.created_at
|
|
198
206
|
}
|
|
199
207
|
h[:importance] = r.respond_to?(:importance) ? (r.importance || 0.7).to_f : 0.7
|
|
200
|
-
h[:provenance] = r.provenance if r.respond_to?(:provenance)
|
|
208
|
+
h[:provenance] = parse_provenance(r.provenance) if r.respond_to?(:provenance)
|
|
201
209
|
h
|
|
202
210
|
end
|
|
203
211
|
|
|
204
212
|
def row_to_resource(r)
|
|
205
213
|
{
|
|
206
214
|
id: r.id,
|
|
207
|
-
text: r.text,
|
|
215
|
+
text: dec(r.text),
|
|
208
216
|
created_at: r.created_at
|
|
209
217
|
}
|
|
210
218
|
end
|
|
@@ -69,11 +69,11 @@ module Llmemory
|
|
|
69
69
|
raise NotImplementedError, "#{self.class}#list_users must be implemented"
|
|
70
70
|
end
|
|
71
71
|
|
|
72
|
-
def list_resources(user_id:, limit: nil)
|
|
72
|
+
def list_resources(user_id:, limit: nil, offset: nil)
|
|
73
73
|
raise NotImplementedError, "#{self.class}#list_resources must be implemented"
|
|
74
74
|
end
|
|
75
75
|
|
|
76
|
-
def list_items(user_id:, category: nil, limit: nil)
|
|
76
|
+
def list_items(user_id:, category: nil, limit: nil, offset: nil)
|
|
77
77
|
raise NotImplementedError, "#{self.class}#list_items must be implemented"
|
|
78
78
|
end
|
|
79
79
|
|