phronomy 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/generators/phronomy/install/templates/create_phronomy_messages.rb.tt +1 -1
- data/lib/phronomy/agent/base.rb +68 -35
- data/lib/phronomy/agent/handoff.rb +6 -2
- data/lib/phronomy/agent/react_agent.rb +57 -31
- data/lib/phronomy/agent/runner.rb +6 -4
- data/lib/phronomy/configuration.rb +6 -0
- data/lib/phronomy/context/assembler.rb +11 -3
- data/lib/phronomy/context/compaction_context.rb +1 -3
- data/lib/phronomy/context/context_version_cache.rb +22 -8
- data/lib/phronomy/context/token_estimator.rb +19 -2
- data/lib/phronomy/eval/eval_result.rb +15 -5
- data/lib/phronomy/eval/runner.rb +46 -11
- data/lib/phronomy/eval/scorer/llm_judge.rb +7 -2
- data/lib/phronomy/graph/compiled_graph.rb +9 -1
- data/lib/phronomy/graph/parallel_node.rb +53 -18
- data/lib/phronomy/graph/state_graph.rb +7 -1
- data/lib/phronomy/guardrail/builtin/pii_pattern_detector.rb +47 -3
- data/lib/phronomy/guardrail/builtin/prompt_injection_detector.rb +15 -1
- data/lib/phronomy/memory/compression/summary.rb +4 -3
- data/lib/phronomy/memory/compression/tool_output_pruner.rb +11 -6
- data/lib/phronomy/memory/conversation_manager.rb +59 -14
- data/lib/phronomy/memory/retrieval/base.rb +4 -3
- data/lib/phronomy/memory/retrieval/composite.rb +5 -4
- data/lib/phronomy/memory/retrieval/recent.rb +4 -3
- data/lib/phronomy/memory/retrieval/semantic.rb +50 -17
- data/lib/phronomy/memory/storage/active_record.rb +18 -13
- data/lib/phronomy/memory/storage/in_memory.rb +25 -16
- data/lib/phronomy/rails/agent_job.rb +20 -3
- data/lib/phronomy/runnable.rb +4 -1
- data/lib/phronomy/state_store/active_record.rb +7 -3
- data/lib/phronomy/state_store/base.rb +16 -2
- data/lib/phronomy/state_store/in_memory.rb +5 -4
- data/lib/phronomy/tool/base.rb +19 -3
- data/lib/phronomy/tool/mcp_tool.rb +67 -9
- data/lib/phronomy/tracing/base.rb +0 -2
- data/lib/phronomy/tracing/langfuse_tracer.rb +24 -4
- data/lib/phronomy/tracing/null_tracer.rb +6 -3
- data/lib/phronomy/trust_pipeline.rb +32 -4
- data/lib/phronomy/vector_store/in_memory.rb +7 -5
- data/lib/phronomy/vector_store/redis_search.rb +30 -23
- data/lib/phronomy/version.rb +1 -1
- data/lib/phronomy.rb +39 -0
- metadata +2 -2
|
@@ -169,7 +169,15 @@ module Phronomy
|
|
|
169
169
|
def next_node(current, state)
|
|
170
170
|
if (cond = @conditional_edges[current])
|
|
171
171
|
result = cond[:condition].call(state)
|
|
172
|
-
|
|
172
|
+
if cond[:mapping]
|
|
173
|
+
unless cond[:mapping].key?(result)
|
|
174
|
+
raise ArgumentError,
|
|
175
|
+
"Conditional edge from #{current.inspect} returned #{result.inspect}, " \
|
|
176
|
+
"which is not present in the mapping (#{cond[:mapping].keys.inspect})"
|
|
177
|
+
end
|
|
178
|
+
return cond[:mapping][result]
|
|
179
|
+
end
|
|
180
|
+
return result
|
|
173
181
|
end
|
|
174
182
|
|
|
175
183
|
edges = @edges[current]
|
|
@@ -64,36 +64,47 @@ module Phronomy
|
|
|
64
64
|
def call(state)
|
|
65
65
|
threads = @branches.map { |branch| Thread.new { branch.call(state) } }
|
|
66
66
|
deadline = @timeout ? (Process.clock_gettime(Process::CLOCK_MONOTONIC) + @timeout) : nil
|
|
67
|
+
state_class = state.class
|
|
67
68
|
|
|
68
69
|
if @on_error == :best_effort
|
|
69
|
-
gather_best_effort(threads, deadline)
|
|
70
|
+
gather_best_effort(threads, deadline, state_class)
|
|
70
71
|
else
|
|
71
|
-
gather_raise(threads, deadline)
|
|
72
|
+
gather_raise(threads, deadline, state_class)
|
|
72
73
|
end
|
|
73
74
|
end
|
|
74
75
|
|
|
75
76
|
private
|
|
76
77
|
|
|
77
78
|
# Joins all threads, enforcing the deadline. Re-raises branch exceptions.
|
|
78
|
-
def gather_raise(threads, deadline)
|
|
79
|
+
def gather_raise(threads, deadline, state_class)
|
|
79
80
|
if deadline
|
|
80
81
|
threads.each do |t|
|
|
81
82
|
remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
82
83
|
next if t.join([remaining, 0].max)
|
|
83
84
|
|
|
84
85
|
# Thread did not finish within the time limit.
|
|
85
|
-
|
|
86
|
+
# Use Thread#raise instead of Thread#kill so that ensure blocks in
|
|
87
|
+
# branches (DB connection return, Mutex release, etc.) are executed.
|
|
88
|
+
timeout_error = Phronomy::Graph::TimeoutError.new(
|
|
89
|
+
"parallel branch timed out after #{@timeout}s"
|
|
90
|
+
)
|
|
91
|
+
threads.each { |thr| thr.raise(timeout_error) unless thr.stop? }
|
|
92
|
+
threads.each do |thr|
|
|
93
|
+
thr.join(0.1)
|
|
94
|
+
rescue
|
|
95
|
+
nil
|
|
96
|
+
end
|
|
86
97
|
raise Phronomy::Graph::TimeoutError,
|
|
87
98
|
"parallel branch timed out after #{@timeout}s"
|
|
88
99
|
end
|
|
89
100
|
end
|
|
90
101
|
|
|
91
102
|
# All threads are done. Thread#value re-raises any stored exception.
|
|
92
|
-
merge_results(threads.map(&:value))
|
|
103
|
+
merge_results(threads.map(&:value), state_class)
|
|
93
104
|
end
|
|
94
105
|
|
|
95
106
|
# Joins all threads, collecting errors instead of re-raising them.
|
|
96
|
-
def gather_best_effort(threads, deadline)
|
|
107
|
+
def gather_best_effort(threads, deadline, state_class)
|
|
97
108
|
errors = []
|
|
98
109
|
results = threads.map do |t|
|
|
99
110
|
if deadline
|
|
@@ -108,7 +119,15 @@ module Phronomy
|
|
|
108
119
|
next nil
|
|
109
120
|
end
|
|
110
121
|
if joined.nil?
|
|
111
|
-
|
|
122
|
+
timeout_error = Phronomy::Graph::TimeoutError.new(
|
|
123
|
+
"branch timed out after #{@timeout}s"
|
|
124
|
+
)
|
|
125
|
+
t.raise(timeout_error) unless t.stop?
|
|
126
|
+
begin
|
|
127
|
+
t.join(0.1)
|
|
128
|
+
rescue
|
|
129
|
+
nil
|
|
130
|
+
end
|
|
112
131
|
errors << Phronomy::Graph::TimeoutError.new(
|
|
113
132
|
"branch timed out after #{@timeout}s"
|
|
114
133
|
)
|
|
@@ -124,33 +143,49 @@ module Phronomy
|
|
|
124
143
|
end
|
|
125
144
|
end
|
|
126
145
|
|
|
127
|
-
merged = merge_results(results) || {}
|
|
146
|
+
merged = merge_results(results, state_class) || {}
|
|
128
147
|
merged[:parallel_errors] = errors unless errors.empty?
|
|
129
148
|
merged.empty? ? nil : merged
|
|
130
149
|
end
|
|
131
150
|
|
|
132
151
|
# Merges an Array of per-branch result Hashes (nils are skipped).
|
|
133
|
-
|
|
152
|
+
# Field merge policy is determined from the State class field declarations:
|
|
153
|
+
# :replace fields — last-write-wins (rightmost branch wins)
|
|
154
|
+
# :append fields — all Arrays are concatenated
|
|
155
|
+
# :merge fields — all Hashes are deep-merged (rightmost wins on conflict)
|
|
156
|
+
# Unknown / undeclared fields fall back to type-based heuristics.
|
|
157
|
+
def merge_results(results, state_class = nil)
|
|
134
158
|
merged = results.compact.each_with_object({}) do |result, acc|
|
|
135
159
|
next unless result.is_a?(Hash)
|
|
136
160
|
|
|
137
161
|
result.each do |key, val|
|
|
138
|
-
acc[key] = acc.key?(key) ? merge_values(acc[key], val) : val
|
|
162
|
+
acc[key] = acc.key?(key) ? merge_values(acc[key], val, state_class&.fields&.dig(key, :type)) : val
|
|
139
163
|
end
|
|
140
164
|
end
|
|
141
165
|
|
|
142
166
|
merged.empty? ? nil : merged
|
|
143
167
|
end
|
|
144
168
|
|
|
145
|
-
# Merges two values
|
|
146
|
-
#
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
old_val.
|
|
152
|
-
|
|
169
|
+
# Merges two values for the same state field key across branches.
|
|
170
|
+
# Uses the declared field policy when available, otherwise falls back to
|
|
171
|
+
# type-based heuristics (Array → concat, Hash → deep-merge, scalar → last-write-wins).
|
|
172
|
+
def merge_values(old_val, new_val, policy = nil)
|
|
173
|
+
case policy
|
|
174
|
+
when :append
|
|
175
|
+
(old_val.is_a?(Array) && new_val.is_a?(Array)) ? old_val + new_val : new_val
|
|
176
|
+
when :merge
|
|
177
|
+
(old_val.is_a?(Hash) && new_val.is_a?(Hash)) ? old_val.merge(new_val) : new_val
|
|
178
|
+
when :replace
|
|
153
179
|
new_val
|
|
180
|
+
else
|
|
181
|
+
# Unknown field or no State class: fall back to type-based heuristic.
|
|
182
|
+
if old_val.is_a?(Array) && new_val.is_a?(Array)
|
|
183
|
+
old_val + new_val
|
|
184
|
+
elsif old_val.is_a?(Hash) && new_val.is_a?(Hash)
|
|
185
|
+
old_val.merge(new_val)
|
|
186
|
+
else
|
|
187
|
+
new_val
|
|
188
|
+
end
|
|
154
189
|
end
|
|
155
190
|
end
|
|
156
191
|
end
|
|
@@ -113,7 +113,8 @@ module Phronomy
|
|
|
113
113
|
def add_subgraph(name, subgraph, input_mapper: nil, output_mapper: nil)
|
|
114
114
|
add_node(name) do |state|
|
|
115
115
|
input = input_mapper ? input_mapper.call(state) : state.to_h
|
|
116
|
-
|
|
116
|
+
sub_thread_id = "#{state.thread_id}/#{name}"
|
|
117
|
+
sub_state = subgraph.invoke(input, config: {thread_id: sub_thread_id})
|
|
117
118
|
output_mapper ? output_mapper.call(sub_state) : sub_state.to_h
|
|
118
119
|
end
|
|
119
120
|
end
|
|
@@ -125,6 +126,11 @@ module Phronomy
|
|
|
125
126
|
# to use for this compiled graph, overriding the global default.
|
|
126
127
|
# @return [CompiledGraph]
|
|
127
128
|
def compile(state_store: nil)
|
|
129
|
+
if @entry_point.nil? && @nodes.size > 1
|
|
130
|
+
raise ArgumentError,
|
|
131
|
+
"set_entry_point was not called; call set_entry_point(:node_name) " \
|
|
132
|
+
"before compile when the graph has multiple nodes"
|
|
133
|
+
end
|
|
128
134
|
CompiledGraph.new(
|
|
129
135
|
state_class: @state_class,
|
|
130
136
|
nodes: @nodes,
|
|
@@ -25,14 +25,20 @@ module Phronomy
|
|
|
25
25
|
# Recognised PII categories and their detection patterns.
|
|
26
26
|
PATTERNS = {
|
|
27
27
|
# Japanese My Number: 12 consecutive or grouped digits (4-4-4).
|
|
28
|
+
# Matched candidates are additionally validated with the official check-digit
|
|
29
|
+
# algorithm (JIS X 0076) to eliminate false positives from arbitrary 12-digit strings.
|
|
28
30
|
my_number: {
|
|
29
31
|
pattern: /(?<!\d)(?<!\d[- ])\d{4}[- ]?\d{4}[- ]?\d{4}(?![- ]?\d)/,
|
|
30
|
-
label: "My Number"
|
|
32
|
+
label: "My Number",
|
|
33
|
+
validate_my_number: true
|
|
31
34
|
},
|
|
32
35
|
# Credit / debit card: 16 digits, optionally separated by spaces or hyphens.
|
|
36
|
+
# Matched candidates are additionally validated with the Luhn algorithm
|
|
37
|
+
# to eliminate false positives from arbitrary 16-digit sequences.
|
|
33
38
|
credit_card: {
|
|
34
39
|
pattern: /\b(?:\d{4}[- ]?){3}\d{4}\b/,
|
|
35
|
-
label: "credit card number"
|
|
40
|
+
label: "credit card number",
|
|
41
|
+
validate_luhn: true
|
|
36
42
|
},
|
|
37
43
|
# Email address (simplified RFC 5322).
|
|
38
44
|
email: {
|
|
@@ -64,9 +70,47 @@ module Phronomy
|
|
|
64
70
|
def check(value)
|
|
65
71
|
text = value.to_s
|
|
66
72
|
@active_patterns.each do |entry|
|
|
67
|
-
|
|
73
|
+
detected = if entry[:validate_luhn]
|
|
74
|
+
# Scan for all candidates then filter by Luhn check-digit validation.
|
|
75
|
+
# This avoids false positives on arbitrary 16-digit strings (e.g. internal IDs).
|
|
76
|
+
text.scan(entry[:pattern]).any? { |m| luhn_valid?(m.gsub(/[- ]/, "")) }
|
|
77
|
+
elsif entry[:validate_my_number]
|
|
78
|
+
# Scan for all candidates then apply the JIS X 0076 check-digit algorithm.
|
|
79
|
+
# This avoids false positives on arbitrary 12-digit strings.
|
|
80
|
+
text.scan(entry[:pattern]).any? { |m| my_number_valid?(m.gsub(/[- ]/, "")) }
|
|
81
|
+
else
|
|
82
|
+
text.match?(entry[:pattern])
|
|
83
|
+
end
|
|
84
|
+
fail!("PII detected in input: #{entry[:label]}") if detected
|
|
68
85
|
end
|
|
69
86
|
end
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
# Returns true when +digits+ (a 12-character string of decimal digits) satisfies
|
|
91
|
+
# the Japanese My Number check-digit algorithm defined in JIS X 0076.
|
|
92
|
+
# The check digit is the 12th digit.
|
|
93
|
+
def my_number_valid?(digits)
|
|
94
|
+
weights = [6, 5, 4, 3, 2, 7, 6, 5, 4, 3, 2]
|
|
95
|
+
total = weights.each_with_index.sum { |w, i| w * digits[i].to_i }
|
|
96
|
+
remainder = total % 11
|
|
97
|
+
check = (remainder <= 1) ? 0 : 11 - remainder
|
|
98
|
+
check == digits[11].to_i
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Returns true when +digits+ (a string of decimal digits) satisfies the
|
|
102
|
+
# Luhn check-digit algorithm used by payment card networks.
|
|
103
|
+
def luhn_valid?(digits)
|
|
104
|
+
digits.chars.reverse.each_with_index.sum do |d, i|
|
|
105
|
+
n = d.to_i
|
|
106
|
+
if i.odd?
|
|
107
|
+
doubled = n * 2
|
|
108
|
+
(doubled > 9) ? (doubled - 9) : doubled
|
|
109
|
+
else
|
|
110
|
+
n
|
|
111
|
+
end
|
|
112
|
+
end % 10 == 0
|
|
113
|
+
end
|
|
70
114
|
end
|
|
71
115
|
end
|
|
72
116
|
end
|
|
@@ -9,6 +9,11 @@ module Phronomy
|
|
|
9
9
|
# {Phronomy::GuardrailError} when any pattern is found in the input string.
|
|
10
10
|
# Additional patterns can be supplied via the +additional_patterns:+ argument.
|
|
11
11
|
#
|
|
12
|
+
# **Limitations**: the built-in patterns cover well-known English and Japanese
|
|
13
|
+
# phrasings. Obfuscated, Base64-encoded, or novel injection phrasing may not
|
|
14
|
+
# be detected. For higher-assurance use cases, combine this guardrail with an
|
|
15
|
+
# LLM-based classifier.
|
|
16
|
+
#
|
|
12
17
|
# @example
|
|
13
18
|
# agent.add_input_guardrail(
|
|
14
19
|
# Phronomy::Guardrail::Builtin::PromptInjectionDetector.new
|
|
@@ -21,6 +26,7 @@ module Phronomy
|
|
|
21
26
|
class PromptInjectionDetector < InputGuardrail
|
|
22
27
|
# Default patterns that signal a prompt injection attempt.
|
|
23
28
|
DEFAULT_PATTERNS = [
|
|
29
|
+
# --- English patterns ---
|
|
24
30
|
/ignore\s+(all\s+)?(previous|prior|above)\s+(instructions?|rules?|prompts?)/i,
|
|
25
31
|
/disregard\s+(all\s+)?(previous|prior|above)\s+(instructions?|rules?|prompts?)/i,
|
|
26
32
|
/forget\s+(all\s+)?(previous|prior|above)\s+(instructions?|rules?|prompts?)/i,
|
|
@@ -30,7 +36,15 @@ module Phronomy
|
|
|
30
36
|
/\bpretend\s+(?:you\s+are|to\s+be)\b/i,
|
|
31
37
|
/\bjailbreak\b/i,
|
|
32
38
|
/\bdan\s*mode\b/i,
|
|
33
|
-
/\bdev(?:eloper)?\s*mode\b/i
|
|
39
|
+
/\bdev(?:eloper)?\s*mode\b/i,
|
|
40
|
+
# --- Japanese patterns ---
|
|
41
|
+
/以前の(指示|ルール|プロンプト)を無視/,
|
|
42
|
+
/指示を無視して/,
|
|
43
|
+
/ルールを無視して/,
|
|
44
|
+
/あなたは今(から)?(?!助けて)/,
|
|
45
|
+
/システムプロンプト/,
|
|
46
|
+
/制約(を|から)無視/,
|
|
47
|
+
/制限(を|から)解除/
|
|
34
48
|
].freeze
|
|
35
49
|
|
|
36
50
|
# @param additional_patterns [Array<Regexp>] extra patterns to check in addition
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "ostruct"
|
|
4
|
-
|
|
5
3
|
module Phronomy
|
|
6
4
|
module Memory
|
|
7
5
|
module Compression
|
|
@@ -64,6 +62,9 @@ module Phronomy
|
|
|
64
62
|
else
|
|
65
63
|
{messages: messages, compaction: nil}
|
|
66
64
|
end
|
|
65
|
+
rescue => e
|
|
66
|
+
warn "[Phronomy] Compression failed (#{e.class}: #{e.message}); saving without compaction."
|
|
67
|
+
{messages: messages, compaction: nil}
|
|
67
68
|
end
|
|
68
69
|
|
|
69
70
|
private
|
|
@@ -98,7 +99,7 @@ module Phronomy
|
|
|
98
99
|
#{text}
|
|
99
100
|
</context>
|
|
100
101
|
CONTEXT
|
|
101
|
-
|
|
102
|
+
RubyLLM::Message.new(role: :system, content: content)
|
|
102
103
|
end
|
|
103
104
|
end
|
|
104
105
|
end
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "ostruct"
|
|
4
|
-
|
|
5
3
|
module Phronomy
|
|
6
4
|
module Memory
|
|
7
5
|
module Compression
|
|
@@ -25,6 +23,11 @@ module Phronomy
|
|
|
25
23
|
class ToolOutputPruner < Base
|
|
26
24
|
TRUNCATION_NOTE = "\n[... output truncated ...]"
|
|
27
25
|
|
|
26
|
+
# Internal value object for cloned messages.
|
|
27
|
+
# Uses Struct (not OpenStruct) so that unknown attribute access raises NoMethodError.
|
|
28
|
+
ClonedMessage = Struct.new(:role, :content, :tool_calls, :model_id, keyword_init: true)
|
|
29
|
+
private_constant :ClonedMessage
|
|
30
|
+
|
|
28
31
|
# @param max_chars [Integer] maximum character length for tool-result content
|
|
29
32
|
def initialize(max_chars: 4000)
|
|
30
33
|
@max_chars = max_chars
|
|
@@ -51,10 +54,12 @@ module Phronomy
|
|
|
51
54
|
private
|
|
52
55
|
|
|
53
56
|
def clone_message(original, new_content)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
ClonedMessage.new(
|
|
58
|
+
role: original.role,
|
|
59
|
+
content: new_content,
|
|
60
|
+
tool_calls: (original.tool_calls if original.respond_to?(:tool_calls)),
|
|
61
|
+
model_id: (original.model_id if original.respond_to?(:model_id))
|
|
62
|
+
)
|
|
58
63
|
end
|
|
59
64
|
end
|
|
60
65
|
end
|
|
@@ -48,6 +48,16 @@ module Phronomy
|
|
|
48
48
|
@retrieval = retrieval
|
|
49
49
|
@compression = compression
|
|
50
50
|
@ttl = ttl
|
|
51
|
+
# Per-thread mutexes allow concurrent saves for different thread_ids while
|
|
52
|
+
# preventing races (duplicate compaction records) within the same thread_id.
|
|
53
|
+
@thread_mutexes = {}
|
|
54
|
+
@thread_mutexes_mutex = Mutex.new
|
|
55
|
+
# Tracks the monotonically increasing next-seq per thread so that TTL
|
|
56
|
+
# purges (which reduce raw.length) do not reset the sequence counter.
|
|
57
|
+
# Protected by a dedicated mutex so concurrent saves for distinct
|
|
58
|
+
# thread_ids do not race on the shared Hash (Issue #60).
|
|
59
|
+
@raw_seq_hwm = {}
|
|
60
|
+
@raw_seq_hwm_mutex = Mutex.new
|
|
51
61
|
end
|
|
52
62
|
|
|
53
63
|
# Load conversation messages for a thread, applying retrieval selection.
|
|
@@ -66,7 +76,7 @@ module Phronomy
|
|
|
66
76
|
def load(thread_id:, query: nil)
|
|
67
77
|
@storage.purge_older_than(thread_id: thread_id, older_than: Time.now - @ttl) if @ttl
|
|
68
78
|
messages = reconstruct(thread_id)
|
|
69
|
-
@retrieval.select(messages, query: query)
|
|
79
|
+
@retrieval.select(messages, query: query, thread_id: thread_id)
|
|
70
80
|
end
|
|
71
81
|
|
|
72
82
|
# Persist new messages for a thread and optionally apply compression.
|
|
@@ -82,8 +92,10 @@ module Phronomy
|
|
|
82
92
|
# @param thread_id [String]
|
|
83
93
|
# @param messages [Array] full conversation history up to this point
|
|
84
94
|
def save(thread_id:, messages:)
|
|
85
|
-
|
|
86
|
-
|
|
95
|
+
thread_mutex(thread_id).synchronize do
|
|
96
|
+
append_new_messages_unlocked(thread_id: thread_id, messages: messages)
|
|
97
|
+
compress_and_save(thread_id: thread_id, messages: messages)
|
|
98
|
+
end
|
|
87
99
|
@retrieval.index(thread_id: thread_id, messages: messages) if @retrieval.respond_to?(:index)
|
|
88
100
|
end
|
|
89
101
|
|
|
@@ -123,18 +135,44 @@ module Phronomy
|
|
|
123
135
|
|
|
124
136
|
private
|
|
125
137
|
|
|
138
|
+
# Returns (or lazily creates) the per-thread mutex for +thread_id+.
|
|
139
|
+
# The outer @thread_mutexes_mutex protects the hash from concurrent creation.
|
|
140
|
+
def thread_mutex(thread_id)
|
|
141
|
+
@thread_mutexes_mutex.synchronize do
|
|
142
|
+
@thread_mutexes[thread_id] ||= Mutex.new
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
126
146
|
# Append messages that are new since the last save to the raw history.
|
|
147
|
+
# Must be called while holding the per-thread mutex (via thread_mutex).
|
|
127
148
|
# Messages are append-only; existing raw entries are never modified.
|
|
128
|
-
|
|
149
|
+
#
|
|
150
|
+
# Uses a per-thread high-water-mark (HWM) to determine the next seq number.
|
|
151
|
+
# The HWM is the maximum of:
|
|
152
|
+
# - The highest seq stored in the raw store (correct after normal appends)
|
|
153
|
+
# - The in-memory HWM (correct after TTL purge empties the raw store)
|
|
154
|
+
# This prevents seq number collisions when TTL purge reduces raw.length.
|
|
155
|
+
def append_new_messages_unlocked(thread_id:, messages:)
|
|
129
156
|
raw = @storage.load_raw(thread_id: thread_id)
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
157
|
+
# Derive the next seq from the raw store's high-water-mark seq when
|
|
158
|
+
# entries are present. Fall back to the in-memory HWM when the raw
|
|
159
|
+
# store has been partially or fully purged by TTL expiry.
|
|
160
|
+
stored_next_seq = raw.any? ? raw.map { |e| e[:seq] }.max + 1 : nil
|
|
161
|
+
hwm = @raw_seq_hwm_mutex.synchronize { @raw_seq_hwm[thread_id] }
|
|
162
|
+
next_seq = [stored_next_seq, hwm].compact.max || 0
|
|
163
|
+
new_messages = messages[next_seq..]
|
|
164
|
+
if new_messages&.any?
|
|
165
|
+
@storage.append_raw(thread_id: thread_id, messages: new_messages, starting_seq: next_seq)
|
|
166
|
+
@raw_seq_hwm_mutex.synchronize { @raw_seq_hwm[thread_id] = next_seq + new_messages.length }
|
|
167
|
+
end
|
|
133
168
|
end
|
|
134
169
|
|
|
135
170
|
# Apply the configured compression strategy and persist the result.
|
|
136
171
|
# When no strategy is configured, saves messages directly to the legacy store.
|
|
137
172
|
# When compression fires, also persists the compaction record.
|
|
173
|
+
# If the compression strategy raises (e.g. LLM timeout), we fall back to
|
|
174
|
+
# saving the messages without compaction so the conversation is never lost
|
|
175
|
+
# due to a transient summarization failure (Issue #58).
|
|
138
176
|
def compress_and_save(thread_id:, messages:)
|
|
139
177
|
unless @compression
|
|
140
178
|
@storage.save(thread_id: thread_id, messages: messages)
|
|
@@ -146,11 +184,16 @@ module Phronomy
|
|
|
146
184
|
all_raw = @storage.load_raw(thread_id: thread_id)
|
|
147
185
|
uncompacted = all_raw.select { |r| r[:seq] >= uncompacted_start_seq }.map { |r| r[:message] }
|
|
148
186
|
|
|
149
|
-
result =
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
187
|
+
result = begin
|
|
188
|
+
@compression.compress(
|
|
189
|
+
thread_id: thread_id,
|
|
190
|
+
messages: uncompacted,
|
|
191
|
+
seq_offset: uncompacted_start_seq
|
|
192
|
+
)
|
|
193
|
+
rescue => e
|
|
194
|
+
warn "[Phronomy] Compression failed (#{e.class}: #{e.message}); saving without compaction."
|
|
195
|
+
{messages: messages, compaction: nil}
|
|
196
|
+
end
|
|
154
197
|
|
|
155
198
|
if result[:compaction]
|
|
156
199
|
@storage.save_compaction(
|
|
@@ -183,14 +226,16 @@ module Phronomy
|
|
|
183
226
|
summary_msgs + uncompacted
|
|
184
227
|
end
|
|
185
228
|
|
|
229
|
+
# Immutable value object used as a summary placeholder in reconstructed context.
|
|
230
|
+
SummaryMessage = Data.define(:role, :content)
|
|
231
|
+
|
|
186
232
|
def summary_message(text)
|
|
187
|
-
require "ostruct"
|
|
188
233
|
content = <<~CONTEXT.chomp
|
|
189
234
|
<context type="summary" source="memory" trusted="false">
|
|
190
235
|
#{text}
|
|
191
236
|
</context>
|
|
192
237
|
CONTEXT
|
|
193
|
-
|
|
238
|
+
SummaryMessage.new(role: :system, content: content)
|
|
194
239
|
end
|
|
195
240
|
end
|
|
196
241
|
end
|
|
@@ -9,10 +9,11 @@ module Phronomy
|
|
|
9
9
|
class Base
|
|
10
10
|
# Select messages to inject into the context from a full chronological history.
|
|
11
11
|
#
|
|
12
|
-
# @param messages
|
|
13
|
-
# @param query
|
|
12
|
+
# @param messages [Array] full history in chronological order
|
|
13
|
+
# @param query [String, nil] current user input for query-aware retrieval
|
|
14
|
+
# @param thread_id [String, nil] active thread identifier for scoped retrieval
|
|
14
15
|
# @return [Array] subset of messages in chronological order
|
|
15
|
-
def select(messages, query: nil)
|
|
16
|
+
def select(messages, query: nil, thread_id: nil)
|
|
16
17
|
raise NotImplementedError, "#{self.class}#select is not implemented"
|
|
17
18
|
end
|
|
18
19
|
end
|
|
@@ -29,15 +29,16 @@ module Phronomy
|
|
|
29
29
|
# Merge results from all child retrievals, deduplicating by role+content.
|
|
30
30
|
# System messages are sorted to the front; others preserve insertion order.
|
|
31
31
|
#
|
|
32
|
-
# @param messages
|
|
33
|
-
# @param query
|
|
32
|
+
# @param messages [Array] full chronological history
|
|
33
|
+
# @param query [String, nil] forwarded to each child retrieval
|
|
34
|
+
# @param thread_id [String, nil] forwarded to each child retrieval
|
|
34
35
|
# @return [Array]
|
|
35
|
-
def select(messages, query: nil)
|
|
36
|
+
def select(messages, query: nil, thread_id: nil)
|
|
36
37
|
all_messages = []
|
|
37
38
|
seen = {}
|
|
38
39
|
|
|
39
40
|
@sources.each do |source|
|
|
40
|
-
source[:retrieval].select(messages, query: query).each do |msg|
|
|
41
|
+
source[:retrieval].select(messages, query: query, thread_id: thread_id).each do |msg|
|
|
41
42
|
key = "#{msg.role}:#{msg.content}"
|
|
42
43
|
next if seen[key]
|
|
43
44
|
|
|
@@ -22,10 +22,11 @@ module Phronomy
|
|
|
22
22
|
|
|
23
23
|
# Returns the last k*2 messages from the history.
|
|
24
24
|
#
|
|
25
|
-
# @param messages
|
|
26
|
-
# @param query
|
|
25
|
+
# @param messages [Array] full chronological history
|
|
26
|
+
# @param query [String, nil] unused for recency-based retrieval
|
|
27
|
+
# @param thread_id [String, nil] unused for recency-based retrieval
|
|
27
28
|
# @return [Array]
|
|
28
|
-
def select(messages, query: nil)
|
|
29
|
+
def select(messages, query: nil, thread_id: nil)
|
|
29
30
|
messages.last(@k * 2)
|
|
30
31
|
end
|
|
31
32
|
end
|
|
@@ -18,26 +18,49 @@ module Phronomy
|
|
|
18
18
|
# @param store [Phronomy::VectorStore::Base] vector store (default InMemory)
|
|
19
19
|
# @param embeddings [Phronomy::Embeddings::Base] embeddings adapter
|
|
20
20
|
# @param k [Integer] number of messages to retrieve
|
|
21
|
-
|
|
21
|
+
# @param max_index_size [Integer, nil] maximum number of entries kept in the
|
|
22
|
+
# local index. When nil, the index grows unboundedly. When exceeded, the
|
|
23
|
+
# oldest entries (by insertion order) are evicted.
|
|
24
|
+
def initialize(embeddings:, store: nil, k: 10, max_index_size: nil)
|
|
22
25
|
@store = store || Phronomy::VectorStore::InMemory.new
|
|
23
26
|
@embeddings = embeddings
|
|
24
27
|
@k = k
|
|
25
|
-
@index = {} # id => message
|
|
28
|
+
@index = {} # id => message (insertion-ordered via Ruby Hash)
|
|
26
29
|
@counter = 0
|
|
30
|
+
@max_index_size = max_index_size
|
|
31
|
+
@mutex = Mutex.new
|
|
32
|
+
@indexed_object_ids = {} # thread_id => { object_id => true }
|
|
27
33
|
end
|
|
28
34
|
|
|
29
35
|
# Index a new batch of messages so they are searchable on future #select calls.
|
|
30
36
|
# Called by ConversationManager#save.
|
|
31
37
|
#
|
|
38
|
+
# Messages are deduplicated by object identity: if a message object has already
|
|
39
|
+
# been indexed for the given thread_id, it is skipped (no duplicate embed call).
|
|
40
|
+
#
|
|
32
41
|
# @param thread_id [String]
|
|
33
42
|
# @param messages [Array]
|
|
34
43
|
def index(thread_id:, messages:)
|
|
35
44
|
messages.each do |msg|
|
|
36
|
-
|
|
37
|
-
@
|
|
45
|
+
# Fast path: skip already-indexed messages without calling embed.
|
|
46
|
+
already_indexed = @mutex.synchronize do
|
|
47
|
+
(@indexed_object_ids[thread_id] ||= {})[msg.object_id]
|
|
48
|
+
end
|
|
49
|
+
next if already_indexed
|
|
50
|
+
|
|
38
51
|
embedding = @embeddings.embed(msg.content.to_s)
|
|
39
|
-
@
|
|
40
|
-
|
|
52
|
+
@mutex.synchronize do
|
|
53
|
+
# Re-check inside lock to handle concurrent callers for the same thread.
|
|
54
|
+
indexed = (@indexed_object_ids[thread_id] ||= {})
|
|
55
|
+
next if indexed[msg.object_id]
|
|
56
|
+
|
|
57
|
+
id = "#{thread_id}:#{@counter}"
|
|
58
|
+
@counter += 1
|
|
59
|
+
@store.add(id: id, embedding: embedding, metadata: {thread_id: thread_id, message: msg})
|
|
60
|
+
@index[id] = msg
|
|
61
|
+
indexed[msg.object_id] = true
|
|
62
|
+
evict_oldest! if @max_index_size && @index.size > @max_index_size
|
|
63
|
+
end
|
|
41
64
|
end
|
|
42
65
|
end
|
|
43
66
|
|
|
@@ -45,24 +68,28 @@ module Phronomy
|
|
|
45
68
|
#
|
|
46
69
|
# @param thread_id [String]
|
|
47
70
|
def clear_index(thread_id:)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
71
|
+
@mutex.synchronize do
|
|
72
|
+
ids = @index.keys.select { |id| id.start_with?("#{thread_id}:") }
|
|
73
|
+
ids.each do |id|
|
|
74
|
+
@index.delete(id)
|
|
75
|
+
@store.remove(id: id)
|
|
76
|
+
end
|
|
77
|
+
@indexed_object_ids.delete(thread_id)
|
|
52
78
|
end
|
|
53
79
|
end
|
|
54
80
|
|
|
55
81
|
# Return semantically relevant messages, or recent messages when query is nil.
|
|
56
82
|
#
|
|
57
|
-
# @param messages
|
|
58
|
-
# @param query
|
|
83
|
+
# @param messages [Array] full history (used as fallback when query is nil)
|
|
84
|
+
# @param query [String, nil] current user input for semantic search
|
|
85
|
+
# @param thread_id [String, nil] when provided, results are filtered to this thread
|
|
59
86
|
# @return [Array]
|
|
60
|
-
def select(messages, query: nil)
|
|
87
|
+
def select(messages, query: nil, thread_id: nil)
|
|
61
88
|
if query && !query.strip.empty?
|
|
62
89
|
query_embedding = @embeddings.embed(query)
|
|
63
|
-
results = @store.search(query_embedding: query_embedding, k: @k * 3)
|
|
90
|
+
results = @mutex.synchronize { @store.search(query_embedding: query_embedding, k: @k * 3) }
|
|
64
91
|
results
|
|
65
|
-
.select { |r| r[:metadata][:thread_id] ==
|
|
92
|
+
.select { |r| thread_id.nil? || r[:metadata][:thread_id] == thread_id }
|
|
66
93
|
.first(@k)
|
|
67
94
|
.map { |r| r[:metadata][:message] }
|
|
68
95
|
else
|
|
@@ -72,8 +99,14 @@ module Phronomy
|
|
|
72
99
|
|
|
73
100
|
private
|
|
74
101
|
|
|
75
|
-
|
|
76
|
-
|
|
102
|
+
# Evicts the oldest index entry to enforce max_index_size.
|
|
103
|
+
# Must be called inside @mutex.synchronize.
|
|
104
|
+
def evict_oldest!
|
|
105
|
+
oldest_id = @index.keys.first
|
|
106
|
+
return unless oldest_id
|
|
107
|
+
|
|
108
|
+
@index.delete(oldest_id)
|
|
109
|
+
@store.remove(id: oldest_id)
|
|
77
110
|
end
|
|
78
111
|
end
|
|
79
112
|
end
|