smart_brain 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/MEMPAL_GUIDE.md +1074 -0
- data/README.en.md +173 -173
- data/README.md +467 -173
- data/config/brain.yml +69 -1
- data/conversation_demo.rb +438 -438
- data/db/migrate/002_turn_events_payload.sql +9 -0
- data/db/migrate/003_tiers_and_lifecycle.sql +28 -0
- data/db/migrate/004_kg_edges.sql +30 -0
- data/db/migrate/005_domains_and_memory_scopes.sql +163 -0
- data/docs/coding_todo.md +139 -0
- data/docs/context_package.md +220 -0
- data/docs/evidence_pack.md +190 -0
- data/docs/gap_vs_mempal.md +161 -0
- data/docs/mcp.md +93 -0
- data/docs/memory_types.md +278 -0
- data/docs/multi_scope_memory_refactor_plan.md +483 -0
- data/docs/multi_scope_migration.md +65 -0
- data/docs/policies.md +308 -0
- data/docs/retrieval_plan.md +231 -0
- data/docs/smartbrain_design.md +299 -0
- data/docs/user_guide.md +546 -0
- data/example.rb +91 -91
- data/examples/01_memory_basic.rb +57 -0
- data/examples/02_governance.rb +63 -0
- data/examples/03_postgres_persistence.rb +63 -0
- data/examples/04_ollama_llm.rb +69 -0
- data/examples/05_smart_rag_integration.rb +79 -0
- data/examples/06_multi_scope_memory.rb +50 -0
- data/examples/README.md +49 -0
- data/exe/smart_brain +168 -0
- data/lib/smart_brain/adapters/smart_rag/direct_client.rb +16 -5
- data/lib/smart_brain/adapters/smart_rag/http_client.rb +16 -5
- data/lib/smart_brain/adapters/smart_rag/null_client.rb +7 -2
- data/lib/smart_brain/adapters/smart_rag/scope_filter.rb +60 -0
- data/lib/smart_brain/configuration.rb +57 -0
- data/lib/smart_brain/consolidator/working_summary.rb +80 -12
- data/lib/smart_brain/context_composer/composer.rb +40 -3
- data/lib/smart_brain/contracts/retrieval_plan.rb +10 -0
- data/lib/smart_brain/contracts/scope_context.rb +46 -0
- data/lib/smart_brain/contracts/scope_ref.rb +25 -0
- data/lib/smart_brain/db.rb +109 -0
- data/lib/smart_brain/event_store/in_memory.rb +6 -2
- data/lib/smart_brain/event_store/postgres.rb +199 -0
- data/lib/smart_brain/fusion/merger.rb +31 -2
- data/lib/smart_brain/governance/briefing.rb +146 -0
- data/lib/smart_brain/governance/fact_check.rb +110 -0
- data/lib/smart_brain/governance/knowledge_graph.rb +60 -0
- data/lib/smart_brain/governance/lifecycle.rb +225 -0
- data/lib/smart_brain/governance/tiers.rb +60 -0
- data/lib/smart_brain/memory_extractor/extractor.rb +25 -7
- data/lib/smart_brain/memory_store/in_memory.rb +202 -17
- data/lib/smart_brain/memory_store/postgres.rb +500 -0
- data/lib/smart_brain/model_provider/base.rb +87 -0
- data/lib/smart_brain/model_provider/factory.rb +49 -0
- data/lib/smart_brain/model_provider/ollama.rb +60 -0
- data/lib/smart_brain/model_provider/openai.rb +60 -0
- data/lib/smart_brain/model_provider/stub.rb +26 -0
- data/lib/smart_brain/model_provider.rb +7 -0
- data/lib/smart_brain/observability/tracker.rb +39 -1
- data/lib/smart_brain/retrievers/exact_retriever.rb +6 -0
- data/lib/smart_brain/retrievers/memory_retriever.rb +59 -5
- data/lib/smart_brain/runtime.rb +288 -16
- data/lib/smart_brain/scopes/conflict_resolver.rb +67 -0
- data/lib/smart_brain/scopes/registry.rb +133 -0
- data/lib/smart_brain/scopes/resolver.rb +32 -0
- data/lib/smart_brain/server/http_app.rb +143 -0
- data/lib/smart_brain/server/mcp_server.rb +385 -0
- data/lib/smart_brain/server/service.rb +129 -0
- data/lib/smart_brain/support/levenshtein.rb +35 -0
- data/lib/smart_brain/version.rb +5 -5
- data/lib/smart_brain.rb +80 -35
- metadata +88 -36
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'securerandom'
|
|
4
|
+
require 'time'
|
|
5
|
+
require_relative '../governance/tiers'
|
|
6
|
+
|
|
7
|
+
module SmartBrain
|
|
8
|
+
module MemoryStore
|
|
9
|
+
# Postgres-backed MemoryStore. Drop-in replacement for MemoryStore::InMemory
|
|
10
|
+
# that additionally:
|
|
11
|
+
# - persists memory_items to Postgres (durable across restarts),
|
|
12
|
+
# - writes memory_chunks with a `simple`-config TSVECTOR so FTS works,
|
|
13
|
+
# - serves #search_memory via ts_rank-ranked full-text search,
|
|
14
|
+
# - persists working summaries to the summaries table.
|
|
15
|
+
class Postgres
|
|
16
|
+
OVERWRITE_TYPES = %w[preferences goals tasks].freeze
|
|
17
|
+
# ts_rank is unbounded and typically tiny (~0.06); scale so FTS scores
|
|
18
|
+
# land in the same band as ExactRetriever's overlap+confidence scores.
|
|
19
|
+
RANK_SCALE = 8.0
|
|
20
|
+
# Match an ASCII word OR a single CJK character. The word class is ASCII-
|
|
21
|
+
# only on purpose: Ruby's [[:alnum:]] is Unicode-aware and would greedy-
|
|
22
|
+
# match a whole CJK run (默认存储) as one token, defeating per-character
|
|
23
|
+
# indexing. Splitting CJK into unigrams lets the 'simple' text-search
|
|
24
|
+
# config (whitespace-only splitter) match at character granularity:
|
|
25
|
+
# indexing 默认存储 as 默 认 存 储 lets a query for 存储 (存 储) hit.
|
|
26
|
+
TOKEN_RE = /[A-Za-z0-9_\-]+|[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]/.freeze
|
|
27
|
+
|
|
28
|
+
def initialize(db:, config:)
|
|
29
|
+
@db = db
|
|
30
|
+
@config = config
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def upsert(extracted)
|
|
34
|
+
session_id = extracted.fetch(:session_id)
|
|
35
|
+
items = extracted.fetch(:items, [])
|
|
36
|
+
written = []
|
|
37
|
+
conflicts = []
|
|
38
|
+
|
|
39
|
+
db.transaction do
|
|
40
|
+
items.each do |item|
|
|
41
|
+
scope_id = item[:scope_id] || legacy_scope_id(session_id)
|
|
42
|
+
existing = active_item(scope_id: scope_id, type: item[:type], key: item[:key])
|
|
43
|
+
|
|
44
|
+
if existing && item[:status] == 'retracted'
|
|
45
|
+
db[:memory_items].where(id: existing[:id]).update(status: 'retracted', updated_at: Sequel::CURRENT_TIMESTAMP)
|
|
46
|
+
conflicts << { type: 'retract', key: item[:key], previous_memory_item_id: existing[:id] }
|
|
47
|
+
next
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
if existing && OVERWRITE_TYPES.include?(item[:type])
|
|
51
|
+
db[:memory_items].where(id: existing[:id]).update(status: 'superseded', updated_at: Sequel::CURRENT_TIMESTAMP)
|
|
52
|
+
conflicts << { type: 'overwrite', key: item[:key], previous_memory_item_id: existing[:id] }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
record = persist_item(session_id: session_id, item: item.merge(scope_id: scope_id))
|
|
56
|
+
written << record.slice(:id, :type, :key, :status, :confidence, :scope_id, :scope, :source_session_id)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
{ count: written.size, items: written, conflicts: conflicts }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def active_items(session_id: nil, scope_ids: nil)
|
|
64
|
+
dataset = db[:memory_items]
|
|
65
|
+
.where(status: 'active', lifecycle_status: Governance::Tiers::CONTEXT_LIFECYCLE)
|
|
66
|
+
.order(:updated_at)
|
|
67
|
+
dataset = scope_filter(dataset, session_id, scope_ids)
|
|
68
|
+
dataset.map { |row| item_from_row(row) }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def entities(session_id: nil, scope_ids: nil)
|
|
72
|
+
dataset = scope_filter(db[:memory_items].where(type: 'entities', status: 'active'), session_id, scope_ids)
|
|
73
|
+
dataset.map do |row|
|
|
74
|
+
value = symbolize(row[:value_json])
|
|
75
|
+
canonical = value[:canonical] || value[:name]
|
|
76
|
+
scope = scope_ref(row[:scope_id])
|
|
77
|
+
{
|
|
78
|
+
id: row[:id],
|
|
79
|
+
name: value[:name] || canonical,
|
|
80
|
+
kind: value[:kind] || 'other',
|
|
81
|
+
canonical: canonical,
|
|
82
|
+
memory_item_id: row[:id],
|
|
83
|
+
scope_id: row[:scope_id],
|
|
84
|
+
scope: scope,
|
|
85
|
+
source_session_id: row[:source_session_id] || row[:session_id]
|
|
86
|
+
}
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Full-text search over memory_chunks (joined to active memory_items),
|
|
91
|
+
# ranked by ts_rank. Returns evidence hashes shaped like ExactRetriever's
|
|
92
|
+
# output so the fusion layer is unchanged.
|
|
93
|
+
#
|
|
94
|
+
# Query terms are OR'd (not AND'd): a CJK query like "数据库 持久化"
|
|
95
|
+
# tokenizes to 数 据 库 持 久 化, and we want any-matching docs recalled
|
|
96
|
+
# (ts_rank then favours docs matching more terms). AND semantics would
|
|
97
|
+
# drop a persistence decision just because it lacks the literal 数据库.
|
|
98
|
+
def search_memory(query:, session_id: nil, scope_ids: nil, limit:)
|
|
99
|
+
or_expr = or_tsquery_expression(query)
|
|
100
|
+
return [] if or_expr.empty?
|
|
101
|
+
|
|
102
|
+
fts = fts_config
|
|
103
|
+
ids = Array(scope_ids).compact
|
|
104
|
+
scope_clause = if ids.empty?
|
|
105
|
+
'm.session_id = ?'
|
|
106
|
+
else
|
|
107
|
+
"m.scope_id IN (#{Array.new(ids.length, '?').join(', ')})"
|
|
108
|
+
end
|
|
109
|
+
sql = <<~SQL
|
|
110
|
+
SELECT c.text, m.id AS item_id, m.type AS memory_type, m.key, m.value_json, m.confidence, m.tier,
|
|
111
|
+
m.scope_id, m.source_session_id, ms.scope_type, ms.external_id,
|
|
112
|
+
ts_rank(c.tsv, to_tsquery(?, ?)) AS rank
|
|
113
|
+
FROM memory_chunks c
|
|
114
|
+
JOIN memory_items m ON m.id = c.memory_item_id
|
|
115
|
+
JOIN memory_scopes ms ON ms.id = m.scope_id
|
|
116
|
+
WHERE #{scope_clause} AND m.status = 'active'
|
|
117
|
+
AND m.lifecycle_status IN ('raw', 'promoted')
|
|
118
|
+
AND c.tsv @@ to_tsquery(?, ?)
|
|
119
|
+
ORDER BY rank DESC
|
|
120
|
+
LIMIT ?
|
|
121
|
+
SQL
|
|
122
|
+
|
|
123
|
+
filter_values = ids.empty? ? [session_id] : ids
|
|
124
|
+
db.fetch(sql, fts, or_expr, *filter_values, fts, or_expr, limit).map do |row|
|
|
125
|
+
value = symbolize(row[:value_json])
|
|
126
|
+
confidence = row[:confidence].to_f
|
|
127
|
+
{
|
|
128
|
+
id: row[:item_id],
|
|
129
|
+
source: 'memory',
|
|
130
|
+
source_uri: "smartbrain://memory/#{row[:item_id]}",
|
|
131
|
+
title: row[:key],
|
|
132
|
+
snippet: flatten_value(value),
|
|
133
|
+
mode: 'fts',
|
|
134
|
+
score: (row[:rank].to_f * RANK_SCALE) + confidence,
|
|
135
|
+
tier: row[:tier] || 'evidence',
|
|
136
|
+
memory_type: row[:memory_type],
|
|
137
|
+
memory_key: row[:key],
|
|
138
|
+
scope_id: row[:scope_id],
|
|
139
|
+
scope: { type: row[:scope_type], id: row[:external_id] },
|
|
140
|
+
source_session_id: row[:source_session_id],
|
|
141
|
+
ref: { memory_item_id: row[:item_id] }
|
|
142
|
+
}
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def save_summary(session_id:, summary:)
|
|
147
|
+
db[:summaries].insert_conflict(
|
|
148
|
+
target: :session_id,
|
|
149
|
+
update: {
|
|
150
|
+
summary_text: summary[:text].to_s,
|
|
151
|
+
summary_version: summary[:summary_version],
|
|
152
|
+
summary_source_turn_range: Sequel.pg_jsonb(symbolizable(summary[:summary_source_turn_range] || {})),
|
|
153
|
+
summary_generated_at: time_from(summary[:summary_generated_at])
|
|
154
|
+
}
|
|
155
|
+
).insert(
|
|
156
|
+
session_id: session_id,
|
|
157
|
+
summary_text: summary[:text].to_s,
|
|
158
|
+
summary_version: summary[:summary_version],
|
|
159
|
+
summary_source_turn_range: Sequel.pg_jsonb(symbolizable(summary[:summary_source_turn_range] || {})),
|
|
160
|
+
summary_generated_at: time_from(summary[:summary_generated_at])
|
|
161
|
+
)
|
|
162
|
+
summary
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def latest_summary(session_id:)
|
|
166
|
+
row = db[:summaries].where(session_id: session_id).first
|
|
167
|
+
return nil unless row
|
|
168
|
+
|
|
169
|
+
row_to_summary(row)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def all_summaries
|
|
173
|
+
db[:summaries].all.each_with_object({}) do |row, h|
|
|
174
|
+
h[row[:session_id]] = row_to_summary(row)
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# --- knowledge lifecycle -------------------------------------------------
|
|
179
|
+
def create_item(session_id:, item:, scope_id: nil, scope: nil)
|
|
180
|
+
persist_item(session_id: session_id, item: item.merge(scope_id: scope_id, scope: scope).compact)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def find_item(id:)
|
|
184
|
+
row = db[:memory_items].where(id: id).first
|
|
185
|
+
row ? item_from_row(row) : nil
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def set_lifecycle(id:, lifecycle_status:, merge_value: nil)
|
|
189
|
+
row = db[:memory_items].where(id: id).first
|
|
190
|
+
return nil unless row
|
|
191
|
+
|
|
192
|
+
updates = { lifecycle_status: lifecycle_status, updated_at: Time.now.utc }
|
|
193
|
+
if merge_value
|
|
194
|
+
merged = symbolize(row[:value_json]).merge(merge_value)
|
|
195
|
+
updates[:value_json] = Sequel.pg_jsonb(symbolizable(merged))
|
|
196
|
+
end
|
|
197
|
+
db[:memory_items].where(id: id).update(updates)
|
|
198
|
+
find_item(id: id)
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def set_status(id:, status:, merge_value: nil)
|
|
202
|
+
row = db[:memory_items].where(id: id).first
|
|
203
|
+
return nil unless row
|
|
204
|
+
|
|
205
|
+
updates = { status: status, updated_at: Time.now.utc }
|
|
206
|
+
updates[:value_json] = Sequel.pg_jsonb(symbolizable(symbolize(row[:value_json]).merge(merge_value))) if merge_value
|
|
207
|
+
db[:memory_items].where(id: id).update(updates)
|
|
208
|
+
find_item(id: id)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def record_event(memory_item_id:, event_type:, from_lifecycle: nil, to_lifecycle: nil,
|
|
212
|
+
reason: nil, reason_type: nil, reviewer: nil, evidence_refs: [])
|
|
213
|
+
id = SecureRandom.uuid
|
|
214
|
+
db[:knowledge_events].insert(
|
|
215
|
+
id: id,
|
|
216
|
+
memory_item_id: memory_item_id,
|
|
217
|
+
event_type: event_type,
|
|
218
|
+
from_lifecycle: from_lifecycle,
|
|
219
|
+
to_lifecycle: to_lifecycle,
|
|
220
|
+
reason: reason,
|
|
221
|
+
reason_type: reason_type,
|
|
222
|
+
reviewer: reviewer,
|
|
223
|
+
evidence_refs: Sequel.pg_jsonb(Array(evidence_refs)),
|
|
224
|
+
created_at: Time.now.utc
|
|
225
|
+
)
|
|
226
|
+
events_for(memory_item_id: memory_item_id).last
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def events_for(memory_item_id:)
|
|
230
|
+
db[:knowledge_events].where(memory_item_id: memory_item_id).order(:created_at).map do |row|
|
|
231
|
+
{
|
|
232
|
+
id: row[:id],
|
|
233
|
+
memory_item_id: row[:memory_item_id],
|
|
234
|
+
event_type: row[:event_type],
|
|
235
|
+
from_lifecycle: row[:from_lifecycle],
|
|
236
|
+
to_lifecycle: row[:to_lifecycle],
|
|
237
|
+
reason: row[:reason],
|
|
238
|
+
reason_type: row[:reason_type],
|
|
239
|
+
reviewer: row[:reviewer],
|
|
240
|
+
evidence_refs: symbolize(row[:evidence_refs]),
|
|
241
|
+
created_at: iso8601(row[:created_at])
|
|
242
|
+
}
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# --- knowledge graph -----------------------------------------------------
|
|
247
|
+
def add_edge(session_id:, edge:, scope_id: nil, scope: nil, source_session_id: nil)
|
|
248
|
+
resolved_scope_id = scope_id || edge[:scope_id] || legacy_scope_id(session_id)
|
|
249
|
+
id = SecureRandom.uuid
|
|
250
|
+
db[:kg_edges].insert(
|
|
251
|
+
id: id,
|
|
252
|
+
session_id: session_id,
|
|
253
|
+
source_session_id: source_session_id || edge[:source_session_id] || session_id,
|
|
254
|
+
scope_id: resolved_scope_id,
|
|
255
|
+
subject: edge[:subject].to_s,
|
|
256
|
+
predicate: edge[:predicate].to_s,
|
|
257
|
+
object: edge[:object].to_s,
|
|
258
|
+
subject_entity_id: resolve_entity_id(nil, edge[:subject], scope_ids: [resolved_scope_id]),
|
|
259
|
+
object_entity_id: resolve_entity_id(nil, edge[:object], scope_ids: [resolved_scope_id]),
|
|
260
|
+
valid_from: edge[:valid_from] ? time_from(edge[:valid_from]) : Time.now.utc,
|
|
261
|
+
valid_to: edge[:valid_to] ? time_from(edge[:valid_to]) : nil,
|
|
262
|
+
source_turn_id: edge[:source_turn_id],
|
|
263
|
+
source_memory_item_id: edge[:source_memory_item_id],
|
|
264
|
+
confidence: edge[:confidence] || 0.6,
|
|
265
|
+
status: edge[:status] || 'active',
|
|
266
|
+
meta_json: Sequel.pg_jsonb(symbolizable(edge[:meta] || {})),
|
|
267
|
+
created_at: Time.now.utc
|
|
268
|
+
)
|
|
269
|
+
find_edge(id: id)
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
def query_edges(session_id: nil, scope_ids: nil, subject: nil, predicate: nil, object: nil, include_invalid: false)
|
|
273
|
+
ds = scope_filter(db[:kg_edges], session_id, scope_ids)
|
|
274
|
+
ds = ds.where(status: 'active') unless include_invalid
|
|
275
|
+
ds = ds.where(Sequel.ilike(:subject, subject.to_s)) if subject
|
|
276
|
+
ds = ds.where(Sequel.ilike(:predicate, predicate.to_s)) if predicate
|
|
277
|
+
ds = ds.where(Sequel.ilike(:object, object.to_s)) if object
|
|
278
|
+
ds.order(:valid_from).map { |row| edge_from_row(row) }
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def edges_for_subject(session_id: nil, scope_ids: nil, subject:)
|
|
282
|
+
scope_filter(db[:kg_edges], session_id, scope_ids)
|
|
283
|
+
.where(Sequel.ilike(:subject, subject.to_s))
|
|
284
|
+
.order(:valid_from).map { |row| edge_from_row(row) }
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
def invalidate_edge(id:, reason: nil)
|
|
288
|
+
edge = find_edge(id: id)
|
|
289
|
+
return nil unless edge
|
|
290
|
+
|
|
291
|
+
meta = (edge[:meta] || {}).merge(invalidated_reason: reason)
|
|
292
|
+
db[:kg_edges].where(id: id).update(
|
|
293
|
+
valid_to: Time.now.utc,
|
|
294
|
+
status: 'invalidated',
|
|
295
|
+
meta_json: Sequel.pg_jsonb(symbolizable(meta))
|
|
296
|
+
)
|
|
297
|
+
find_edge(id: id)
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def edge_stats(session_id: nil, scope_ids: nil)
|
|
301
|
+
scoped = scope_filter(db[:kg_edges], session_id, scope_ids)
|
|
302
|
+
{
|
|
303
|
+
total: scoped.count,
|
|
304
|
+
active: scoped.where(status: 'active').count,
|
|
305
|
+
invalidated: scoped.where(status: 'invalidated').count
|
|
306
|
+
}
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
def find_edge(id:)
|
|
310
|
+
row = db[:kg_edges].where(id: id).first
|
|
311
|
+
row ? edge_from_row(row) : nil
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def resolve_entity_id(session_id, name, scope_ids: nil)
|
|
315
|
+
return nil if name.nil? || name.to_s.empty?
|
|
316
|
+
|
|
317
|
+
match = entities(session_id: session_id, scope_ids: scope_ids).find do |e|
|
|
318
|
+
e[:canonical].to_s.downcase == name.to_s.downcase ||
|
|
319
|
+
e[:name].to_s.downcase == name.to_s.downcase
|
|
320
|
+
end
|
|
321
|
+
match && match[:id]
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
def edge_from_row(row)
|
|
325
|
+
{
|
|
326
|
+
id: row[:id],
|
|
327
|
+
session_id: row[:session_id],
|
|
328
|
+
source_session_id: row[:source_session_id] || row[:session_id],
|
|
329
|
+
scope_id: row[:scope_id],
|
|
330
|
+
scope: scope_ref(row[:scope_id]),
|
|
331
|
+
subject: row[:subject],
|
|
332
|
+
predicate: row[:predicate],
|
|
333
|
+
object: row[:object],
|
|
334
|
+
subject_entity_id: row[:subject_entity_id],
|
|
335
|
+
object_entity_id: row[:object_entity_id],
|
|
336
|
+
valid_from: iso8601(row[:valid_from]),
|
|
337
|
+
valid_to: row[:valid_to] ? iso8601(row[:valid_to]) : nil,
|
|
338
|
+
source_turn_id: row[:source_turn_id],
|
|
339
|
+
source_memory_item_id: row[:source_memory_item_id],
|
|
340
|
+
confidence: row[:confidence].to_f,
|
|
341
|
+
status: row[:status],
|
|
342
|
+
meta: symbolize(row[:meta_json]),
|
|
343
|
+
created_at: iso8601(row[:created_at])
|
|
344
|
+
}
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
private
|
|
348
|
+
|
|
349
|
+
attr_reader :db, :config
|
|
350
|
+
|
|
351
|
+
def active_item(scope_id:, type:, key:)
|
|
352
|
+
db[:memory_items].where(scope_id: scope_id, type: type, key: key, status: 'active').first
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
def persist_item(session_id:, item:)
|
|
356
|
+
id = SecureRandom.uuid
|
|
357
|
+
value_json = item[:value_json] || {}
|
|
358
|
+
now = Time.now.utc
|
|
359
|
+
resolved_scope_id = item[:scope_id] || legacy_scope_id(session_id)
|
|
360
|
+
db[:memory_items].insert(
|
|
361
|
+
id: id,
|
|
362
|
+
session_id: session_id,
|
|
363
|
+
source_session_id: item[:source_session_id] || session_id,
|
|
364
|
+
scope_id: resolved_scope_id,
|
|
365
|
+
type: item[:type],
|
|
366
|
+
key: item[:key],
|
|
367
|
+
value_json: Sequel.pg_jsonb(symbolizable(value_json)),
|
|
368
|
+
confidence: item[:confidence] || 0.6,
|
|
369
|
+
status: item[:status] || 'active',
|
|
370
|
+
tier: item[:tier] || Governance::Tiers.tier_for(item[:type]),
|
|
371
|
+
lifecycle_status: item[:lifecycle_status] || 'raw',
|
|
372
|
+
source_turn_id: item[:source_turn_id],
|
|
373
|
+
source_message_id: item[:source_message_id],
|
|
374
|
+
evidence_refs: Sequel.pg_jsonb([]),
|
|
375
|
+
updated_at: now
|
|
376
|
+
)
|
|
377
|
+
write_chunks(id, item[:key], value_json)
|
|
378
|
+
item.merge(id: id, session_id: session_id, source_session_id: item[:source_session_id] || session_id,
|
|
379
|
+
scope_id: resolved_scope_id, scope: item[:scope] || scope_ref(resolved_scope_id),
|
|
380
|
+
status: item[:status] || 'active', confidence: item[:confidence] || 0.6,
|
|
381
|
+
tier: item[:tier] || Governance::Tiers.tier_for(item[:type]), lifecycle_status: item[:lifecycle_status] || 'raw')
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
def write_chunks(item_id, key, value_json)
|
|
385
|
+
raw = "#{key} #{flatten_value(value_json)}".strip
|
|
386
|
+
tokenized = tokenize_for_fts(raw)
|
|
387
|
+
chunk_text = tokenized.empty? ? raw : "#{tokenized}\n#{raw}"
|
|
388
|
+
fts = fts_config
|
|
389
|
+
db[:memory_chunks].insert(
|
|
390
|
+
id: SecureRandom.uuid,
|
|
391
|
+
memory_item_id: item_id,
|
|
392
|
+
text: chunk_text,
|
|
393
|
+
tsv: Sequel.lit("to_tsvector(?, ?)", fts, chunk_text),
|
|
394
|
+
meta_json: Sequel.pg_jsonb({})
|
|
395
|
+
)
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
def item_from_row(row)
|
|
399
|
+
{
|
|
400
|
+
id: row[:id],
|
|
401
|
+
session_id: row[:session_id],
|
|
402
|
+
source_session_id: row[:source_session_id] || row[:session_id],
|
|
403
|
+
scope_id: row[:scope_id],
|
|
404
|
+
scope: scope_ref(row[:scope_id]),
|
|
405
|
+
type: row[:type],
|
|
406
|
+
key: row[:key],
|
|
407
|
+
value_json: symbolize(row[:value_json]),
|
|
408
|
+
confidence: row[:confidence].to_f,
|
|
409
|
+
status: row[:status],
|
|
410
|
+
tier: row[:tier] || 'evidence',
|
|
411
|
+
lifecycle_status: row[:lifecycle_status] || 'raw',
|
|
412
|
+
source_turn_id: row[:source_turn_id],
|
|
413
|
+
source_message_id: row[:source_message_id],
|
|
414
|
+
evidence_refs: [],
|
|
415
|
+
updated_at: iso8601(row[:updated_at])
|
|
416
|
+
}
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
def scope_filter(dataset, session_id, scope_ids)
|
|
420
|
+
ids = Array(scope_ids).compact
|
|
421
|
+
ids.empty? ? dataset.where(session_id: session_id) : dataset.where(scope_id: ids)
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
def scope_ref(scope_id)
|
|
425
|
+
row = db[:memory_scopes].where(id: scope_id).first
|
|
426
|
+
row ? { type: row[:scope_type], id: row[:external_id] } : nil
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
def legacy_scope_id(session_id)
|
|
430
|
+
session = db[:sessions].where(id: session_id.to_s).first
|
|
431
|
+
row = session && db[:memory_scopes].where(
|
|
432
|
+
domain_id: session[:domain_id], scope_type: 'session', external_id: session_id.to_s
|
|
433
|
+
).first
|
|
434
|
+
raise ArgumentError, "session scope not registered: #{session_id}" unless row
|
|
435
|
+
|
|
436
|
+
row[:id]
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
def fts_config
|
|
440
|
+
cfg = config&.fts_config || 'simple'
|
|
441
|
+
cfg = 'simple' unless cfg.to_s =~ /\A[a-z_]+\z/
|
|
442
|
+
cfg
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
def row_to_summary(row)
|
|
446
|
+
{
|
|
447
|
+
summary_version: row[:summary_version],
|
|
448
|
+
summary_source_turn_range: symbolize(row[:summary_source_turn_range]),
|
|
449
|
+
summary_generated_at: iso8601(row[:summary_generated_at]),
|
|
450
|
+
text: row[:summary_text],
|
|
451
|
+
triggered: true,
|
|
452
|
+
trigger_reason: 'persisted'
|
|
453
|
+
}
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
# Space-joined tokens so CJK matches under the 'simple' text-search config
|
|
457
|
+
# (which only splits on whitespace). Same regex as ExactRetriever#tokenize.
|
|
458
|
+
def tokenize_for_fts(text)
|
|
459
|
+
text.to_s.downcase.scan(TOKEN_RE).uniq.join(' ')
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
# Build an OR tsquery expression ("a | b | c") from query tokens, with each
|
|
463
|
+
# token stripped to tsquery-safe characters. Empty if no usable tokens.
|
|
464
|
+
def or_tsquery_expression(query)
|
|
465
|
+
query.to_s.downcase.scan(TOKEN_RE).uniq.map do |token|
|
|
466
|
+
token.gsub(/[^A-Za-z0-9_\p{Han}]/, '')
|
|
467
|
+
end.reject(&:empty?).join(' | ')
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
def flatten_value(value)
|
|
471
|
+
case value
|
|
472
|
+
when Hash then value.values.map { |v| flatten_value(v) }.join(' ')
|
|
473
|
+
when Array then value.map { |v| flatten_value(v) }.join(' ')
|
|
474
|
+
else value.to_s
|
|
475
|
+
end
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
def time_from(value)
|
|
479
|
+
case value
|
|
480
|
+
when Time then value
|
|
481
|
+
when String then Time.parse(value)
|
|
482
|
+
else Time.now.utc
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
def iso8601(value)
|
|
487
|
+
time = value.is_a?(Time) ? value : (value.nil? ? Time.now.utc : Time.parse(value.to_s))
|
|
488
|
+
time.iso8601
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
def symbolizable(obj)
|
|
492
|
+
SmartBrain::DB.stringify_keys(obj)
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
def symbolize(obj)
|
|
496
|
+
SmartBrain::DB.symbolize_jsonb(obj)
|
|
497
|
+
end
|
|
498
|
+
end
|
|
499
|
+
end
|
|
500
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module SmartBrain
|
|
6
|
+
# ModelProvider 兑现 smartbrain_design.md §9:planner/extractor/summary/rerank 的本地模型调用。
|
|
7
|
+
# 默认 Stub(确定性、零网络);可切 Ollama / OpenAI 兼容 provider 走真实 LLM。
|
|
8
|
+
module ModelProvider
|
|
9
|
+
class Base
|
|
10
|
+
# 真实 LLM 能力(决定 summary 是否走真摘要、merger 是否走 LLM rerank)。
|
|
11
|
+
def llm?
|
|
12
|
+
raise NotImplementedError
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# provider 可用(Stub 永远可用,只是非 LLM)。
|
|
16
|
+
def enabled?
|
|
17
|
+
raise NotImplementedError
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# => { text: String, error?: String }
|
|
21
|
+
def complete(prompt:, system: nil, temperature: nil, max_tokens: nil)
|
|
22
|
+
raise NotImplementedError
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# => [{ index: Integer, score: Float }] 与 documents 等长、按相关性降序
|
|
26
|
+
def rerank(query:, documents:)
|
|
27
|
+
raise NotImplementedError
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
# Strip reasoning blocks emitted by chain-of-thought models (qwen3,
|
|
33
|
+
# deepseek-r1, ...). Keeps downstream parsing (rerank JSON) and summary
|
|
34
|
+
# text clean. Handles both closed <think>...</think> and an unterminated
|
|
35
|
+
# trailing <think> (truncated mid-thought).
|
|
36
|
+
def strip_think(text)
|
|
37
|
+
cleaned = text.to_s.gsub(/<think>.*?<\/think>/m, '')
|
|
38
|
+
cleaned = cleaned.sub(/<think>.*/m, '') if cleaned.include?('<think>')
|
|
39
|
+
cleaned.strip
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# LLM-as-judge rerank:让 complete 模型对候选批量排序。混入有 #complete 的 provider。
|
|
44
|
+
module Rerankable
|
|
45
|
+
def rerank(query:, documents:)
|
|
46
|
+
return documents.each_with_index.map { |_, i| { index: i, score: 1.0 } } if documents.empty?
|
|
47
|
+
|
|
48
|
+
list = documents.each_with_index.map { |d, i| "[#{i}] #{truncate(d)}" }.join("\n")
|
|
49
|
+
prompt = "Rank the following passages by relevance to the query. " \
|
|
50
|
+
"Reply with ONLY a JSON array of integer indices, most relevant first, no prose.\n" \
|
|
51
|
+
"Query: #{query}\nPassages:\n#{list}"
|
|
52
|
+
result = complete(prompt: prompt, temperature: 0.0, max_tokens: 240)
|
|
53
|
+
order = parse_ranking(result[:text], documents.size)
|
|
54
|
+
n = [documents.size, 1].max
|
|
55
|
+
order.map.with_index { |idx, pos| { index: idx, score: (n - pos).to_f / n } }
|
|
56
|
+
rescue StandardError
|
|
57
|
+
documents.each_with_index.map { |_, i| { index: i, score: 1.0 } }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
def truncate(text, limit = 300)
|
|
63
|
+
t = text.to_s.gsub(/\s+/, ' ').strip
|
|
64
|
+
t.length > limit ? "#{t[0, limit]}..." : t
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# 从模型输出里抽出索引顺序:优先 JSON 数组,退而求其次抓所有整数。
|
|
68
|
+
def parse_ranking(text, size)
|
|
69
|
+
if (match = text.to_s.match(/\[[\d\s,]+\]/))
|
|
70
|
+
arr = JSON.parse(match[0])
|
|
71
|
+
return sanitize_order(arr.map(&:to_i), size)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
sanitize_order(text.to_s.scan(/-?\d+/).map(&:to_i), size)
|
|
75
|
+
rescue JSON::ParserError
|
|
76
|
+
sanitize_order(text.to_s.scan(/-?\d+/).map(&:to_i), size)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def sanitize_order(indices, size)
|
|
80
|
+
seen = []
|
|
81
|
+
indices.each { |i| seen << i if i.between?(0, size - 1) && !seen.include?(i) }
|
|
82
|
+
(0...size).each { |i| seen << i unless seen.include?(i) }
|
|
83
|
+
seen
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'base'
|
|
4
|
+
require_relative 'stub'
|
|
5
|
+
require_relative 'ollama'
|
|
6
|
+
require_relative 'openai'
|
|
7
|
+
|
|
8
|
+
module SmartBrain
|
|
9
|
+
module ModelProvider
|
|
10
|
+
module Factory
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# 按 config.llm_provider(stub|ollama|openai)构造 provider。
|
|
14
|
+
def build(config)
|
|
15
|
+
cfg = config.respond_to?(:llm) ? config.llm : (config.fetch(:model_provider, {}))
|
|
16
|
+
provider = (config.respond_to?(:llm_provider) ? config.llm_provider : cfg.fetch(:provider, 'stub')).to_s
|
|
17
|
+
|
|
18
|
+
case provider
|
|
19
|
+
when 'ollama'
|
|
20
|
+
Ollama.new(
|
|
21
|
+
model: cfg[:model] || 'qwen3',
|
|
22
|
+
base_url: ENV.fetch('SMARTBRAIN_LLM_BASE_URL', cfg[:base_url] || 'http://localhost:11434'),
|
|
23
|
+
rerank_model: cfg[:rerank_model],
|
|
24
|
+
temperature: cfg[:temperature] || 0.2,
|
|
25
|
+
timeout_seconds: cfg[:timeout_seconds] || 30,
|
|
26
|
+
think: cfg.fetch(:think, false)
|
|
27
|
+
)
|
|
28
|
+
when 'openai'
|
|
29
|
+
OpenAI.new(
|
|
30
|
+
model: cfg[:model] || 'qwen3',
|
|
31
|
+
base_url: ENV.fetch('SMARTBRAIN_LLM_BASE_URL', cfg[:base_url] || 'http://localhost:11434'),
|
|
32
|
+
api_key: ENV.fetch('SMARTBRAIN_LLM_API_KEY', cfg[:api_key] || ''),
|
|
33
|
+
rerank_model: cfg[:rerank_model],
|
|
34
|
+
temperature: cfg[:temperature] || 0.2,
|
|
35
|
+
timeout_seconds: cfg[:timeout_seconds] || 30
|
|
36
|
+
)
|
|
37
|
+
else
|
|
38
|
+
Stub.new
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
module_function
|
|
44
|
+
|
|
45
|
+
def build(config)
|
|
46
|
+
Factory.build(config)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'faraday'
|
|
4
|
+
require 'json'
|
|
5
|
+
require_relative 'base'
|
|
6
|
+
|
|
7
|
+
module SmartBrain
|
|
8
|
+
module ModelProvider
|
|
9
|
+
# Ollama 本地 provider(OpenAI 兼容)。complete 走 /api/chat;rerank 走 LLM-as-judge。
|
|
10
|
+
class Ollama < Base
|
|
11
|
+
include Rerankable
|
|
12
|
+
|
|
13
|
+
def initialize(model:, base_url:, rerank_model: nil, temperature: 0.2, timeout_seconds: 30, think: false, **_)
|
|
14
|
+
@model = model
|
|
15
|
+
@rerank_model = rerank_model || model
|
|
16
|
+
@temperature = temperature
|
|
17
|
+
@timeout_seconds = timeout_seconds
|
|
18
|
+
@think = think
|
|
19
|
+
@base_url = base_url.to_s.chomp('/')
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def llm?
|
|
23
|
+
true
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def enabled?
|
|
27
|
+
true
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def complete(prompt:, system: nil, temperature: nil, max_tokens: nil, think: nil)
|
|
31
|
+
messages = []
|
|
32
|
+
messages << { 'role' => 'system', 'content' => system } if system && !system.empty?
|
|
33
|
+
messages << { 'role' => 'user', 'content' => prompt }
|
|
34
|
+
|
|
35
|
+
body = { 'model' => @model, 'messages' => messages, 'stream' => false }
|
|
36
|
+
body['think'] = think.nil? ? @think : think
|
|
37
|
+
options = { 'temperature' => temperature || @temperature }
|
|
38
|
+
options['num_predict'] = max_tokens if max_tokens
|
|
39
|
+
body['options'] = options
|
|
40
|
+
|
|
41
|
+
response = connection.post('/api/chat', JSON.generate(body))
|
|
42
|
+
raise "ollama HTTP #{response.status}" unless response.status.between?(200, 299)
|
|
43
|
+
|
|
44
|
+
{ text: strip_think(JSON.parse(response.body).dig('message', 'content').to_s) }
|
|
45
|
+
rescue StandardError => e
|
|
46
|
+
{ text: '', error: "#{e.class}: #{e.message}" }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def connection
|
|
52
|
+
@connection ||= Faraday.new(url: @base_url) do |f|
|
|
53
|
+
f.options.timeout = @timeout_seconds
|
|
54
|
+
f.options.open_timeout = @timeout_seconds
|
|
55
|
+
f.headers['Content-Type'] = 'application/json'
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|