llmemory 0.1.17 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +178 -1
  3. data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +20 -0
  4. data/lib/llmemory/actions/reason.rb +49 -0
  5. data/lib/llmemory/actions.rb +8 -0
  6. data/lib/llmemory/cli/commands/base.rb +8 -0
  7. data/lib/llmemory/cli/commands/episodic.rb +42 -0
  8. data/lib/llmemory/cli/commands/forget_log.rb +36 -0
  9. data/lib/llmemory/cli/commands/procedural.rb +44 -0
  10. data/lib/llmemory/cli/commands/working.rb +31 -0
  11. data/lib/llmemory/cli.rb +12 -0
  12. data/lib/llmemory/configuration.rb +6 -0
  13. data/lib/llmemory/forget_log.rb +50 -0
  14. data/lib/llmemory/long_term/episodic/memory.rb +97 -15
  15. data/lib/llmemory/long_term/episodic/storage.rb +7 -5
  16. data/lib/llmemory/long_term/episodic/storages/active_record_models.rb +17 -0
  17. data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +93 -0
  18. data/lib/llmemory/long_term/episodic/storages/base.rb +5 -0
  19. data/lib/llmemory/long_term/episodic/storages/database_storage.rb +135 -0
  20. data/lib/llmemory/long_term/episodic/storages/file_storage.rb +11 -3
  21. data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +9 -3
  22. data/lib/llmemory/long_term/file_based/memory.rb +31 -0
  23. data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +11 -4
  24. data/lib/llmemory/long_term/file_based/storages/database_storage.rb +16 -6
  25. data/lib/llmemory/long_term/file_based/storages/file_storage.rb +2 -4
  26. data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +2 -4
  27. data/lib/llmemory/long_term/graph_based/memory.rb +95 -51
  28. data/lib/llmemory/long_term/procedural/memory.rb +170 -0
  29. data/lib/llmemory/long_term/procedural/skill.rb +93 -0
  30. data/lib/llmemory/long_term/procedural/storage.rb +33 -0
  31. data/lib/llmemory/long_term/procedural/storages/active_record_models.rb +17 -0
  32. data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +104 -0
  33. data/lib/llmemory/long_term/procedural/storages/base.rb +53 -0
  34. data/lib/llmemory/long_term/procedural/storages/database_storage.rb +148 -0
  35. data/lib/llmemory/long_term/procedural/storages/file_storage.rb +135 -0
  36. data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +79 -0
  37. data/lib/llmemory/long_term/procedural.rb +12 -0
  38. data/lib/llmemory/long_term.rb +2 -0
  39. data/lib/llmemory/mcp/server.rb +13 -1
  40. data/lib/llmemory/mcp/tools/memory_episode_record.rb +48 -0
  41. data/lib/llmemory/mcp/tools/memory_episodes.rb +43 -0
  42. data/lib/llmemory/mcp/tools/memory_forget.rb +53 -0
  43. data/lib/llmemory/mcp/tools/memory_retrieve.rb +10 -2
  44. data/lib/llmemory/mcp/tools/memory_skill_register.rb +35 -0
  45. data/lib/llmemory/mcp/tools/memory_skill_report.rb +35 -0
  46. data/lib/llmemory/mcp/tools/memory_skills.rb +43 -0
  47. data/lib/llmemory/memory.rb +34 -1
  48. data/lib/llmemory/memory_module.rb +55 -0
  49. data/lib/llmemory/retrieval/bm25_scorer.rb +1 -1
  50. data/lib/llmemory/retrieval/engine.rb +115 -6
  51. data/lib/llmemory/retrieval/feedback_store.rb +50 -0
  52. data/lib/llmemory/retrieval/mmr_reranker.rb +1 -1
  53. data/lib/llmemory/short_term/checkpoint.rb +2 -14
  54. data/lib/llmemory/short_term/session_lifecycle.rb +22 -13
  55. data/lib/llmemory/short_term/stores.rb +27 -0
  56. data/lib/llmemory/tokenizer.rb +27 -0
  57. data/lib/llmemory/vector_store/active_record_store.rb +4 -3
  58. data/lib/llmemory/vector_store.rb +14 -0
  59. data/lib/llmemory/version.rb +1 -1
  60. data/lib/llmemory/working_memory.rb +83 -0
  61. data/lib/llmemory.rb +5 -0
  62. metadata +32 -1
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Llmemory
4
+ module MCP
5
+ module Tools
6
+ class MemorySkills < ::MCP::Tool
7
+ description "List registered skills (procedural memory) for a user, ranked by proven utility when a query is given."
8
+
9
+ input_schema(
10
+ properties: {
11
+ user_id: { type: "string", description: "User identifier" },
12
+ query: { type: "string", description: "Optional keyword to filter skills" },
13
+ limit: { type: "integer", description: "Max skills to return (default 10)" }
14
+ },
15
+ required: ["user_id"]
16
+ )
17
+
18
+ class << self
19
+ def call(user_id:, query: nil, limit: nil, server_context: nil)
20
+ memory = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id)
21
+ cap = (limit || 10).to_i
22
+ skills = if query.to_s.strip.empty?
23
+ memory.skills(limit: cap)
24
+ else
25
+ memory.search_candidates(query, top_k: cap).filter_map { |c| memory.get_skill(c[:id]) }
26
+ end
27
+
28
+ if skills.empty?
29
+ return ::MCP::Tool::Response.new([{ type: "text", text: "No skills for user #{user_id}." }])
30
+ end
31
+
32
+ lines = skills.map do |s|
33
+ "[#{s.id}] #{s.name} v#{s.version} (#{s.kind}) — success rate #{format('%.2f', s.success_rate)} (#{s.success_count}/#{s.success_count + s.failure_count})"
34
+ end
35
+ ::MCP::Tool::Response.new([{ type: "text", text: lines.join("\n") }])
36
+ rescue => e
37
+ ::MCP::Tool::Response.new([{ type: "text", text: "Error listing skills: #{e.message}" }], error: true)
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -10,16 +10,49 @@ module Llmemory
10
10
  DEFAULT_SESSION_ID = "default"
11
11
  STATE_KEY_MESSAGES = :messages
12
12
 
13
- def initialize(user_id:, session_id: DEFAULT_SESSION_ID, checkpoint: nil, long_term: nil, long_term_type: nil, retrieval_engine: nil, api_key: nil)
13
+ def initialize(user_id:, session_id: DEFAULT_SESSION_ID, checkpoint: nil, long_term: nil, long_term_type: nil, retrieval_engine: nil, working_memory: nil, episodic: nil, procedural: nil, api_key: nil)
14
14
  @user_id = user_id
15
15
  @session_id = session_id
16
16
  @checkpoint = checkpoint || ShortTerm::Checkpoint.new(user_id: user_id, session_id: session_id)
17
+ @working_memory = working_memory
18
+ @episodic = episodic
19
+ @procedural = procedural
17
20
  @llm = api_key.to_s.empty? ? nil : Llmemory::LLM.client(api_key: api_key)
18
21
  type = long_term_type || Llmemory.configuration.long_term_type || :file_based
19
22
  @long_term = long_term || build_long_term(type)
20
23
  @retrieval_engine = retrieval_engine || Retrieval::Engine.new(@long_term, llm: @llm)
21
24
  end
22
25
 
26
+ # Structured working memory for this session (CoALA working memory),
27
+ # parallel to the message checkpoint. Lazily built.
28
+ def working_memory
29
+ @working_memory ||= WorkingMemory.new(user_id: @user_id, session_id: @session_id)
30
+ end
31
+
32
+ # Episodic long-term memory (CoALA): records and retrieves agent trajectories.
33
+ # Additive — coexists with the semantic store (file/graph). Lazily built.
34
+ def episodic
35
+ @episodic ||= LongTerm::Episodic::Memory.new(user_id: @user_id)
36
+ end
37
+
38
+ # Procedural long-term memory (Voyager-style skill library). Lazily built.
39
+ def procedural
40
+ @procedural ||= LongTerm::Procedural::Memory.new(user_id: @user_id)
41
+ end
42
+
43
+ # Reflects over recent episodes and writes distilled insights to the
44
+ # semantic store (file/graph) with provenance back to source episodes.
45
+ def reflect!(window: 10, category: "insights")
46
+ Reflection::Reflector.new(episodic: episodic, semantic: @long_term, llm: @llm)
47
+ .reflect(window: window, category: category)
48
+ end
49
+
50
+ # Reasoning action: render a prompt from working memory, call the LLM, write
51
+ # the result back. Composable; does not touch long-term memory.
52
+ def reason(template:, into: Actions::Reason::DEFAULT_SLOT, parse: nil)
53
+ Actions::Reason.call(working_memory: working_memory, template: template, into: into, parse: parse, llm: @llm)
54
+ end
55
+
23
56
  def add_message(role:, content:)
24
57
  msgs = messages
25
58
  msgs << { role: role.to_sym, content: content.to_s }
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Llmemory
4
+ # Uniform contract for queryable long-term memories (file-based, graph-based,
5
+ # episodic). CoALA argues agents should be modular with standardized
6
+ # abstractions; this mixin gives any memory store the same agent-facing
7
+ # surface so frameworks can treat them polymorphically:
8
+ #
9
+ # read(query, user_id:, limit:) -> relevant entries (retrieval)
10
+ # write(payload, ...) -> ingest into the store (learning)
11
+ # list(user_id:, limit:) -> enumerate stored entries
12
+ # stats(user_id:) -> counts and metadata
13
+ #
14
+ # `read` defaults to the de-facto `search_candidates` interface the retrieval
15
+ # Engine already relies on. `write`, `list` and `stats` are implemented by each
16
+ # including class over its native API.
17
+ #
18
+ # Deliberately excluded: session-state stores (Checkpoint, WorkingMemory) are a
19
+ # different abstraction (K/V session state, not a retrievable corpus), and
20
+ # deletion/forgetting semantics are deferred to a coherent unlearning API.
21
+ module MemoryModule
22
+ def read(query, user_id: nil, limit: 20)
23
+ unless respond_to?(:search_candidates)
24
+ raise NotImplementedError, "#{self.class} must implement #read or #search_candidates"
25
+ end
26
+ search_candidates(query, user_id: user_id, top_k: limit)
27
+ end
28
+
29
+ def write(*, **)
30
+ raise NotImplementedError, "#{self.class} must implement #write"
31
+ end
32
+
33
+ def list(user_id: nil, limit: nil)
34
+ raise NotImplementedError, "#{self.class} must implement #list"
35
+ end
36
+
37
+ def stats(user_id: nil)
38
+ raise NotImplementedError, "#{self.class} must implement #stats"
39
+ end
40
+
41
+ # Removes entries by id (the same ids returned by #read) and records the
42
+ # removal in a ForgetLog audit. Returns the number of entries removed.
43
+ # Implemented by stores with a clear deletion model; others may not support
44
+ # it (CoALA-style "unlearning" is understudied; deletion semantics differ).
45
+ def forget(ids:, reason: nil)
46
+ raise NotImplementedError, "#{self.class} does not support #forget"
47
+ end
48
+
49
+ # Shared audit trail for #forget. Lazily built; override or inject by setting
50
+ # @forget_log if a specific backend/store is required.
51
+ def forget_log
52
+ @forget_log ||= Llmemory::ForgetLog.new
53
+ end
54
+ end
55
+ end
@@ -53,7 +53,7 @@ module Llmemory
53
53
  private
54
54
 
55
55
  def tokenize(text)
56
- text.to_s.downcase.scan(/\b[a-z0-9]{2,}\b/)
56
+ Llmemory::Tokenizer.tokenize(text)
57
57
  end
58
58
  end
59
59
  end
@@ -4,34 +4,142 @@ require_relative "temporal_ranker"
4
4
  require_relative "context_assembler"
5
5
  require_relative "bm25_scorer"
6
6
  require_relative "mmr_reranker"
7
+ require_relative "feedback_store"
7
8
 
8
9
  module Llmemory
9
10
  module Retrieval
10
11
  class Engine
11
12
  RELEVANCE_THRESHOLD = 0.7
13
+ FEEDBACK_CAP = 5
12
14
 
13
- def initialize(memory, llm: nil)
15
+ def initialize(memory, llm: nil, feedback: nil)
14
16
  @memory = memory
15
17
  @llm = llm || Llmemory::LLM.client
16
18
  @ranker = TemporalRanker.new
17
19
  @assembler = ContextAssembler.new
18
20
  @bm25_scorer = Bm25Scorer.new
19
21
  @mmr_reranker = MmrReranker.new(lambda: Llmemory.configuration.mmr_lambda)
22
+ @feedback = feedback || FeedbackStore.new
20
23
  end
21
24
 
22
25
  def retrieve_for_inference(user_message, user_id: nil, max_tokens: nil)
23
26
  user_id ||= @memory.respond_to?(:user_id) ? @memory.user_id : nil
24
27
  search_query = generate_query(user_message)
28
+ ranked = ranked_candidates(search_query, user_id, user_message)
29
+ @assembler.assemble(ranked, max_tokens: max_tokens)
30
+ end
31
+
32
+ # Multi-hop retrieval (CoALA: integrating retrieval and reasoning). After
33
+ # each hop, a reasoner inspects what has been retrieved and proposes a
34
+ # follow-up query for the missing piece, enabling multi-hop questions a
35
+ # single retrieval would miss. Candidates accumulate (deduped) across hops.
36
+ #
37
+ # `reasoner` is a callable (user_message, accumulated_candidates, hop) ->
38
+ # next query String, or "DONE"/blank to stop. Defaults to an LLM that
39
+ # proposes the next sub-query. Converges on `max_hops`, "DONE", a blank
40
+ # query, or a repeated query.
41
+ def iterative_retrieve(user_message, user_id: nil, max_tokens: nil, max_hops: 2, reasoner: nil)
42
+ user_id ||= @memory.respond_to?(:user_id) ? @memory.user_id : nil
43
+ reasoner ||= method(:default_followup_query)
44
+
45
+ query = generate_query(user_message)
46
+ seen = []
47
+ accumulated = []
48
+ hop = 0
49
+
50
+ while hop < max_hops && live_query?(query) && !seen.include?(query)
51
+ seen << query
52
+ accumulated = merge_candidates(accumulated, ranked_candidates(query, user_id, query))
53
+ hop += 1
54
+ break if hop >= max_hops
55
+
56
+ query = reasoner.call(user_message, accumulated, hop).to_s.strip
57
+ end
58
+
59
+ final = accumulated.sort_by { |c| -(c[:temporal_score] || c[:score] || 0) }
60
+ @assembler.assemble(final, max_tokens: max_tokens)
61
+ end
62
+
63
+ # Records that previously-retrieved items were useful or harmful for the
64
+ # agent's task. Repeatedly useful items rank higher in future retrievals;
65
+ # noisy ones are dampened. Item ids come from the candidates returned by
66
+ # the memory's #read / #search_candidates.
67
+ def report_feedback(useful_ids: [], harmful_ids: [], user_id: nil)
68
+ user_id ||= @memory.respond_to?(:user_id) ? @memory.user_id : nil
69
+ Array(useful_ids).each { |id| @feedback.record(user_id, id, 1) }
70
+ Array(harmful_ids).each { |id| @feedback.record(user_id, id, -1) }
71
+ true
72
+ end
73
+
74
+ private
75
+
76
+ # One retrieval hop: fetch -> hybrid -> relevance filter -> temporal rank
77
+ # -> feedback adjust -> (optional) MMR. Returns ranked candidates.
78
+ def ranked_candidates(search_query, user_id, relevance_text)
25
79
  candidates = fetch_candidates(search_query, user_id)
26
80
  candidates = apply_hybrid_scoring(candidates, search_query) if Llmemory.configuration.hybrid_search_enabled
27
-
28
- relevant = filter_by_relevance(candidates, user_message)
81
+ relevant = filter_by_relevance(candidates, relevance_text)
29
82
  ranked = @ranker.rank(relevant)
30
- ranked = @mmr_reranker.rerank(ranked) if Llmemory.configuration.mmr_enabled
31
- @assembler.assemble(ranked, max_tokens: max_tokens)
83
+ ranked = apply_feedback(ranked, user_id)
84
+ Llmemory.configuration.mmr_enabled ? @mmr_reranker.rerank(ranked) : ranked
32
85
  end
33
86
 
34
- private
87
+ def live_query?(query)
88
+ !query.nil? && !query.to_s.strip.empty? && query.to_s.strip.upcase != "DONE"
89
+ end
90
+
91
+ def merge_candidates(accumulated, additions)
92
+ by_key = {}
93
+ (accumulated + additions).each do |c|
94
+ key = c[:id] || c[:text]
95
+ current = by_key[key]
96
+ if current.nil? || score_of(c) > score_of(current)
97
+ by_key[key] = c
98
+ end
99
+ end
100
+ by_key.values
101
+ end
102
+
103
+ def score_of(candidate)
104
+ (candidate[:temporal_score] || candidate[:score] || 0).to_f
105
+ end
106
+
107
+ def default_followup_query(user_message, accumulated, _hop)
108
+ context = accumulated.first(10).map { |c| c[:text] }.compact.join("\n")
109
+ prompt = <<~PROMPT
110
+ Question: #{user_message}
111
+ Information retrieved so far:
112
+ #{context}
113
+
114
+ If more information is needed to fully answer the question, reply with a
115
+ single short search query for the missing piece. If what was retrieved is
116
+ sufficient, reply with exactly "DONE".
117
+ PROMPT
118
+ @llm.invoke(prompt.strip).to_s.strip
119
+ rescue Llmemory::LLMError
120
+ "DONE"
121
+ end
122
+
123
+ def apply_feedback(ranked, user_id)
124
+ weight = Llmemory.configuration.retrieval_feedback_weight.to_f
125
+ return ranked if user_id.nil? || weight <= 0
126
+
127
+ adjusted = ranked.map do |c|
128
+ id = c[:id] || c["id"]
129
+ net = id.nil? ? 0 : @feedback.net(user_id, id)
130
+ next c if net.zero?
131
+
132
+ base = (c[:temporal_score] || c[:score] || 0).to_f
133
+ c.merge(temporal_score: base * feedback_factor(net, weight))
134
+ end
135
+ adjusted.sort_by { |c| -(c[:temporal_score] || 0) }
136
+ end
137
+
138
+ # Maps net feedback to a bounded multiplier in [1 - weight, 1 + weight].
139
+ def feedback_factor(net, weight)
140
+ capped = [[net, -FEEDBACK_CAP].max, FEEDBACK_CAP].min
141
+ 1.0 + (weight * (capped.to_f / FEEDBACK_CAP))
142
+ end
35
143
 
36
144
  def generate_query(user_message)
37
145
  return user_message.to_s if user_message.to_s.length <= 100
@@ -51,6 +159,7 @@ module Llmemory
51
159
  raw = @memory.search_candidates(search_query, user_id: user_id, top_k: 20)
52
160
  raw.map do |c|
53
161
  {
162
+ id: c[:id] || c["id"],
54
163
  text: c[:text] || c["text"],
55
164
  timestamp: parse_timestamp(c[:timestamp] || c["timestamp"] || c[:created_at] || c["created_at"]),
56
165
  score: (c[:score] || c["score"] || 1.0).to_f,
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../short_term/stores"
4
+
5
+ module Llmemory
6
+ module Retrieval
7
+ # Persists retrieval feedback: a net utility signal per (user, memory item),
8
+ # accumulated from agents marking retrieved items useful (+1) or harmful (-1).
9
+ #
10
+ # CoALA flags adaptive retrieval — "learning better retrieval procedures" — as
11
+ # understudied. This is the minimal substrate for it: a feedback ledger the
12
+ # Engine consults to boost repeatedly-useful items and dampen noise.
13
+ #
14
+ # Backed by the same pluggable short-term stores as Checkpoint/WorkingMemory,
15
+ # under a per-user pseudo-session key.
16
+ class FeedbackStore
17
+ SESSION_KEY = "__retrieval_feedback__"
18
+
19
+ def initialize(store: nil)
20
+ @store = store || ShortTerm::Stores.build
21
+ end
22
+
23
+ def record(user_id, item_id, delta)
24
+ return if user_id.nil? || item_id.nil?
25
+ state = load(user_id)
26
+ key = item_id.to_s
27
+ state[key] = (state[key] || 0) + delta.to_i
28
+ @store.save(user_id, SESSION_KEY, state)
29
+ state[key]
30
+ end
31
+
32
+ def net(user_id, item_id)
33
+ return 0 if user_id.nil? || item_id.nil?
34
+ load(user_id)[item_id.to_s] || 0
35
+ end
36
+
37
+ def all(user_id)
38
+ load(user_id)
39
+ end
40
+
41
+ private
42
+
43
+ def load(user_id)
44
+ state = @store.load(user_id, SESSION_KEY)
45
+ return {} unless state.is_a?(Hash)
46
+ state.each_with_object({}) { |(k, v), acc| acc[k.to_s] = v.to_i }
47
+ end
48
+ end
49
+ end
50
+ end
@@ -49,7 +49,7 @@ module Llmemory
49
49
  end
50
50
 
51
51
  def tokenize(text)
52
- text.downcase.scan(/\b[a-z0-9]{2,}\b/).uniq
52
+ Llmemory::Tokenizer.tokenize(text).uniq
53
53
  end
54
54
  end
55
55
  end
@@ -1,9 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative "stores/base"
4
- require_relative "stores/memory_store"
5
- require_relative "stores/redis_store"
6
- require_relative "stores/postgres_store"
3
+ require_relative "stores"
7
4
 
8
5
  module Llmemory
9
6
  module ShortTerm
@@ -31,16 +28,7 @@ module Llmemory
31
28
  private
32
29
 
33
30
  def build_store
34
- case Llmemory.configuration.short_term_store.to_sym
35
- when :memory then Stores::MemoryStore.new
36
- when :redis then Stores::RedisStore.new
37
- when :postgres then Stores::PostgresStore.new
38
- when :active_record, :activerecord
39
- require_relative "stores/active_record_store"
40
- Stores::ActiveRecordStore.new
41
- else
42
- Stores::MemoryStore.new
43
- end
31
+ Stores.build
44
32
  end
45
33
  end
46
34
  end
@@ -1,8 +1,22 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "stores"
4
+
3
5
  module Llmemory
4
6
  module ShortTerm
5
7
  class SessionLifecycle
8
+ # Pseudo-sessions used by ForgetLog, FeedbackStore and WorkingMemory share
9
+ # the short-term K/V store but are not user sessions — they must not be
10
+ # idle-pruned, stale-pruned, or evicted by enforce_max_entries.
11
+ PSEUDO_SESSION_PATTERNS = [
12
+ /\A__[a-z_]+__\z/, # e.g. "__forget_log__", "__retrieval_feedback__"
13
+ /:working_memory\z/ # WorkingMemory uses "<session>:working_memory"
14
+ ].freeze
15
+
16
+ def self.pseudo_session?(session_id)
17
+ PSEUDO_SESSION_PATTERNS.any? { |p| session_id.to_s.match?(p) }
18
+ end
19
+
6
20
  def initialize(store: nil)
7
21
  @store = store || build_store
8
22
  end
@@ -12,7 +26,7 @@ module Llmemory
12
26
  cutoff = Time.now - (idle_minutes * 60)
13
27
  deleted = 0
14
28
 
15
- @store.list_sessions(user_id: user_id).each do |session_id|
29
+ user_sessions(user_id).each do |session_id|
16
30
  state = @store.load(user_id, session_id)
17
31
  next unless state.is_a?(Hash)
18
32
 
@@ -34,7 +48,7 @@ module Llmemory
34
48
  cutoff = Time.now - (prune_after_days * 86400)
35
49
  deleted = 0
36
50
 
37
- @store.list_sessions(user_id: user_id).each do |session_id|
51
+ user_sessions(user_id).each do |session_id|
38
52
  state = @store.load(user_id, session_id)
39
53
  next unless state.is_a?(Hash)
40
54
 
@@ -53,7 +67,7 @@ module Llmemory
53
67
 
54
68
  def enforce_max_entries!(user_id:, max_entries: nil)
55
69
  max_entries ||= Llmemory.configuration.session_max_entries_per_user
56
- sessions = @store.list_sessions(user_id: user_id)
70
+ sessions = user_sessions(user_id)
57
71
  return 0 if sessions.size <= max_entries
58
72
 
59
73
  session_ages = sessions.map do |session_id|
@@ -71,17 +85,12 @@ module Llmemory
71
85
 
72
86
  private
73
87
 
88
+ def user_sessions(user_id)
89
+ @store.list_sessions(user_id: user_id).reject { |s| self.class.pseudo_session?(s) }
90
+ end
91
+
74
92
  def build_store
75
- case Llmemory.configuration.short_term_store.to_sym
76
- when :memory then Stores::MemoryStore.new
77
- when :redis then Stores::RedisStore.new
78
- when :postgres then Stores::PostgresStore.new
79
- when :active_record, :activerecord
80
- require_relative "stores/active_record_store"
81
- Stores::ActiveRecordStore.new
82
- else
83
- Stores::MemoryStore.new
84
- end
93
+ Stores.build
85
94
  end
86
95
  end
87
96
  end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "stores/base"
4
+ require_relative "stores/memory_store"
5
+ require_relative "stores/redis_store"
6
+ require_relative "stores/postgres_store"
7
+
8
+ module Llmemory
9
+ module ShortTerm
10
+ module Stores
11
+ # Single source of truth for selecting a short-term store backend.
12
+ # Shared by Checkpoint, SessionLifecycle and WorkingMemory.
13
+ def self.build(store_type = nil)
14
+ case (store_type || Llmemory.configuration.short_term_store).to_sym
15
+ when :memory then MemoryStore.new
16
+ when :redis then RedisStore.new
17
+ when :postgres then PostgresStore.new
18
+ when :active_record, :activerecord
19
+ require_relative "stores/active_record_store"
20
+ ActiveRecordStore.new
21
+ else
22
+ MemoryStore.new
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Llmemory
4
+ # Shared word tokenizer for keyword search and lexical scoring (BM25, MMR).
5
+ # Centralizes the tokenization regex that was duplicated across the codebase.
6
+ module Tokenizer
7
+ module_function
8
+
9
+ WORD = /\b[a-z0-9]{2,}\b/
10
+
11
+ def tokenize(text)
12
+ text.to_s.downcase.scan(WORD)
13
+ end
14
+
15
+ # Lexical match used by storage-level keyword search. A query is split into
16
+ # tokens and matched as an OR of per-token substrings, so multi-word queries
17
+ # work (a single contiguous substring of the whole query is no longer
18
+ # required) while single-term/partial matches are preserved. An empty query
19
+ # (no tokens) matches everything, keeping prior "return all" behavior.
20
+ def matches?(text, query)
21
+ tokens = tokenize(query)
22
+ return true if tokens.empty?
23
+ haystack = text.to_s.downcase
24
+ tokens.any? { |t| haystack.include?(t) }
25
+ end
26
+ end
27
+ end
@@ -7,9 +7,10 @@ module Llmemory
7
7
  # Persists embeddings in llmemory_embeddings (pgvector).
8
8
  # Use when long_term_store is :active_record so hybrid search finds persisted embeddings.
9
9
  class ActiveRecordStore < Base
10
- def initialize(embedding_provider: nil)
10
+ def initialize(embedding_provider: nil, source_type: "edge")
11
11
  self.class.load_model!
12
12
  @embedding_provider = embedding_provider
13
+ @source_type = source_type.to_s
13
14
  end
14
15
 
15
16
  def self.load_model!
@@ -29,7 +30,7 @@ module Llmemory
29
30
  text_content = (metadata || {}).dig("text") || (metadata || {}).dig(:text)
30
31
  rec = Llmemory::VectorStore::ActiveRecordEmbedding.find_or_initialize_by(
31
32
  user_id: user_id.to_s,
32
- source_type: "edge",
33
+ source_type: @source_type,
33
34
  source_id: id.to_s
34
35
  )
35
36
  rec.embedding = embedding.to_a.map(&:to_f)
@@ -46,7 +47,7 @@ module Llmemory
46
47
  sanitized_vec = vec.map { |v| v.finite? ? v : 0.0 }
47
48
  vector_literal = "[#{sanitized_vec.join(',')}]"
48
49
  # pgvector cosine distance <=> (0 = same, 2 = opposite); score = 1 - distance for similarity
49
- scope = Llmemory::VectorStore::ActiveRecordEmbedding.where(user_id: user_id.to_s)
50
+ scope = Llmemory::VectorStore::ActiveRecordEmbedding.where(user_id: user_id.to_s, source_type: @source_type)
50
51
  rows = scope.select(
51
52
  Llmemory::VectorStore::ActiveRecordEmbedding.arel_table[Arel.star],
52
53
  Arel.sql("(embedding <=> '#{vector_literal}'::vector) AS distance")
@@ -6,5 +6,19 @@ require_relative "vector_store/memory_store"
6
6
 
7
7
  module Llmemory
8
8
  module VectorStore
9
+ # Builds a vector store wired to OpenAI embeddings, selecting the backend
10
+ # from config (:active_record persists in llmemory_embeddings; otherwise
11
+ # in-process). `source_type` namespaces persisted embeddings so different
12
+ # memory types (edges, episodes, skills) never collide in the shared table.
13
+ def self.build(source_type: "edge")
14
+ embeddings = OpenAIEmbeddings.new
15
+ store_type = (Llmemory.configuration.long_term_store || :memory).to_s.to_sym
16
+ if store_type == :active_record || store_type == :activerecord
17
+ require_relative "vector_store/active_record_store"
18
+ ActiveRecordStore.new(embedding_provider: embeddings, source_type: source_type)
19
+ else
20
+ MemoryStore.new(embedding_provider: embeddings)
21
+ end
22
+ end
9
23
  end
10
24
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Llmemory
4
- VERSION = "0.1.17"
4
+ VERSION = "0.2.1"
5
5
  end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "short_term/stores"
4
+
5
+ module Llmemory
6
+ # CoALA's "working memory": a structured, symbolic scratch space that persists
7
+ # across LLM calls within a session — distinct from the raw message buffer
8
+ # (Checkpoint). It is the central hub an agent reads from and writes to while
9
+ # reasoning (goals, current task, retrieved context, intermediate reasoning,
10
+ # last observation, free-form scratchpad), plus arbitrary custom slots.
11
+ #
12
+ # Backed by the same pluggable short-term stores as Checkpoint, but under a
13
+ # namespaced session key so working-memory slots never collide with messages.
14
+ class WorkingMemory
15
+ DEFAULT_SESSION_ID = "default"
16
+ SESSION_SUFFIX = ":working_memory"
17
+ SLOTS = %i[goals current_task retrieved_context scratchpad last_observation intermediate_reasoning].freeze
18
+
19
+ attr_reader :user_id, :session_id
20
+
21
+ def initialize(user_id:, session_id: DEFAULT_SESSION_ID, store: nil)
22
+ @user_id = user_id
23
+ @session_id = session_id
24
+ @store_key = "#{session_id}#{SESSION_SUFFIX}"
25
+ @store = store || ShortTerm::Stores.build
26
+ end
27
+
28
+ SLOTS.each do |slot|
29
+ define_method(slot) { read[slot] }
30
+ define_method("#{slot}=") { |value| set(slot, value) }
31
+ end
32
+
33
+ # Read/write an arbitrary slot (typed or custom).
34
+ def get(slot)
35
+ read[slot.to_sym]
36
+ end
37
+
38
+ def set(slot, value)
39
+ state = read
40
+ state[slot.to_sym] = value
41
+ persist(state)
42
+ value
43
+ end
44
+
45
+ # Bulk update in a single write.
46
+ def update(**slots)
47
+ state = read
48
+ slots.each { |k, v| state[k.to_sym] = v }
49
+ persist(state)
50
+ state
51
+ end
52
+
53
+ # Slots set by the caller beyond the predefined typed ones.
54
+ def custom_slots
55
+ read.reject { |k, _| SLOTS.include?(k) }
56
+ end
57
+
58
+ def to_h
59
+ read
60
+ end
61
+
62
+ def clear!
63
+ @store.delete(@user_id, @store_key)
64
+ true
65
+ end
66
+
67
+ private
68
+
69
+ def read
70
+ state = @store.load(@user_id, @store_key)
71
+ return {} unless state.is_a?(Hash)
72
+ symbolize(state)
73
+ end
74
+
75
+ def persist(state)
76
+ @store.save(@user_id, @store_key, state)
77
+ end
78
+
79
+ def symbolize(hash)
80
+ hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
81
+ end
82
+ end
83
+ end