dsh-chat-tide 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/CONTRIBUTING.md +60 -0
- package/LICENSE +21 -0
- package/README.en.md +68 -0
- package/README.md +67 -2
- package/RELEASE.md +55 -0
- package/assets/icon.png +0 -0
- package/cordis.patch.yml +35 -0
- package/lib/ambient.js +44 -0
- package/lib/api.js +199 -0
- package/lib/attention.js +191 -0
- package/lib/budget.js +37 -0
- package/lib/circuit.js +42 -0
- package/lib/client.js +543 -0
- package/lib/config.js +139 -0
- package/lib/conflict.js +44 -0
- package/lib/const.js +24 -0
- package/lib/context.js +232 -0
- package/lib/db.js +325 -0
- package/lib/dedupe.js +29 -0
- package/lib/egress.js +84 -0
- package/lib/embed-child.mjs +41 -0
- package/lib/embed-worker.mjs +35 -0
- package/lib/embed.js +309 -0
- package/lib/events.js +82 -0
- package/lib/exemplar.js +48 -0
- package/lib/extract.js +183 -0
- package/lib/facts.js +311 -0
- package/lib/fts.js +97 -0
- package/lib/images.js +57 -0
- package/lib/index.js +1074 -0
- package/lib/inject.js +107 -0
- package/lib/janitor.js +137 -0
- package/lib/llmroute.js +43 -0
- package/lib/merge.js +54 -0
- package/lib/people.js +55 -0
- package/lib/prefilter.js +74 -0
- package/lib/qqtext.js +42 -0
- package/lib/repair.js +119 -0
- package/lib/route-decision.js +303 -0
- package/lib/route.js +42 -0
- package/lib/schema.js +134 -0
- package/lib/scope.js +121 -0
- package/lib/scrub.js +23 -0
- package/lib/sessions.js +69 -0
- package/lib/settings.js +135 -0
- package/lib/silence.js +92 -0
- package/lib/status-page.js +263 -0
- package/lib/topics.js +72 -0
- package/lib/vec.js +63 -0
- package/package.json +92 -4
- package/presets/README.md +38 -0
- package/presets/attention/example.json +10 -0
- package/presets/config/profiles.json +32 -0
- package/presets/private.example.json +18 -0
- package/presets/rules/generic.json +79 -0
- package/presets/rules.schema.json +32 -0
- package/probes/01-sqlite-vec.mjs +70 -0
- package/probes/02-fts-bigram.mjs +68 -0
- package/probes/03-embedding.mjs +73 -0
- package/probes/04-storage.mjs +118 -0
- package/probes/05-facts.mjs +113 -0
- package/probes/06-events-egress.mjs +72 -0
- package/probes/07-extract.mjs +66 -0
- package/probes/08-context-janitor.mjs +80 -0
- package/probes/09-route-silence.mjs +75 -0
- package/probes/10-vector-pipeline.mjs +77 -0
- package/probes/11-llm-fallback.mjs +28 -0
- package/probes/12-exemplar-budget-merge.mjs +76 -0
- package/probes/13-circuit.mjs +45 -0
- package/probes/14-embed-worker.mjs +41 -0
- package/probes/15-backfill.mjs +57 -0
- package/probes/16-scope-preset.mjs +36 -0
- package/probes/17-prefilter.mjs +33 -0
- package/probes/18-qqtext.mjs +33 -0
- package/probes/19-repair.mjs +47 -0
- package/probes/20-context-message.mjs +49 -0
- package/probes/21-status-page.mjs +75 -0
- package/probes/22-retention.mjs +45 -0
- package/probes/23-ambient-probability.mjs +60 -0
- package/probes/24-inject-delta.mjs +50 -0
- package/probes/25-route-decision.mjs +122 -0
- package/probes/26-attention.mjs +52 -0
- package/probes/27-people-alias.mjs +42 -0
- package/probes/28-wake-gates.mjs +88 -0
- package/probes/29-scope-modes.mjs +53 -0
- package/probes/30-attention-vector.mjs +84 -0
- package/probes/31-group-speakers.mjs +58 -0
- package/probes/32-image-cache.mjs +54 -0
- package/probes/33-topic-growth.mjs +60 -0
- package/probes/34-integrity.mjs +80 -0
- package/probes/35-privacy.mjs +52 -0
- package/probes/36-inject-scopes.mjs +62 -0
- package/probes/37-inject-e2e.mjs +92 -0
- package/probes/38-review-fixes.mjs +74 -0
- package/probes/39-image-only.mjs +31 -0
- package/probes/40-scope-follow.mjs +40 -0
- package/probes/41-attention-label.mjs +35 -0
- package/scripts/attention-sim.mjs +54 -0
- package/scripts/check-dead-config.mjs +27 -0
- package/scripts/check-docs.mjs +71 -0
- package/scripts/check-naming.mjs +48 -0
- package/scripts/curate-report.mjs +228 -0
- package/scripts/doctor.mjs +110 -0
- package/scripts/export-private-rules.mjs +35 -0
- package/scripts/make-icon.mjs +114 -0
- package/scripts/metrics.mjs +94 -0
- package/scripts/pack.mjs +36 -0
- package/scripts/recall-check.mjs +56 -0
- package/scripts/run-checks.mjs +45 -0
- package/scripts/seed.mjs +82 -0
package/lib/db.js
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
// 存储层:schema、迁移、连接。
|
|
2
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
3
|
+
import { mkdirSync } from 'node:fs';
|
|
4
|
+
import { dirname } from 'node:path';
|
|
5
|
+
|
|
6
|
+
export const SCHEMA_VERSION = 13;
|
|
7
|
+
|
|
8
|
+
const SCHEMA_V1 = `
|
|
9
|
+
CREATE TABLE fact (
|
|
10
|
+
fact_id TEXT PRIMARY KEY,
|
|
11
|
+
user_id TEXT NOT NULL,
|
|
12
|
+
entity_id TEXT NOT NULL,
|
|
13
|
+
canonical_key TEXT NOT NULL,
|
|
14
|
+
subtype TEXT NOT NULL,
|
|
15
|
+
status TEXT NOT NULL CHECK (status IN ('active','superseded','deleted','pending_confirmation')),
|
|
16
|
+
current_version INTEGER NOT NULL,
|
|
17
|
+
asserted_by TEXT,
|
|
18
|
+
subject_id TEXT,
|
|
19
|
+
attribution TEXT,
|
|
20
|
+
provenance_tier TEXT NOT NULL CHECK (provenance_tier IN ('human_explicit','source_verifiable','agent_inferred')),
|
|
21
|
+
valid_from INTEGER,
|
|
22
|
+
valid_to INTEGER,
|
|
23
|
+
occurred_at INTEGER,
|
|
24
|
+
observed_at INTEGER,
|
|
25
|
+
updated_at INTEGER,
|
|
26
|
+
source_record_id TEXT,
|
|
27
|
+
conflicts_with TEXT
|
|
28
|
+
);
|
|
29
|
+
CREATE UNIQUE INDEX idx_fact_active ON fact(user_id, entity_id, canonical_key, subtype) WHERE status = 'active';
|
|
30
|
+
CREATE INDEX idx_fact_key ON fact(user_id, entity_id, canonical_key);
|
|
31
|
+
CREATE INDEX idx_fact_status ON fact(status);
|
|
32
|
+
|
|
33
|
+
CREATE TABLE fact_version (
|
|
34
|
+
fact_id TEXT NOT NULL REFERENCES fact(fact_id) ON DELETE CASCADE,
|
|
35
|
+
version INTEGER NOT NULL,
|
|
36
|
+
value TEXT NOT NULL,
|
|
37
|
+
status TEXT NOT NULL,
|
|
38
|
+
valid_from INTEGER,
|
|
39
|
+
valid_to INTEGER,
|
|
40
|
+
occurred_at INTEGER,
|
|
41
|
+
observed_at INTEGER,
|
|
42
|
+
created_at INTEGER NOT NULL,
|
|
43
|
+
embedding_model_version TEXT,
|
|
44
|
+
embedding_pending INTEGER NOT NULL DEFAULT 0,
|
|
45
|
+
PRIMARY KEY (fact_id, version)
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
CREATE TABLE fact_conflict (
|
|
49
|
+
conflict_id TEXT PRIMARY KEY,
|
|
50
|
+
fact_a TEXT NOT NULL,
|
|
51
|
+
fact_b TEXT NOT NULL,
|
|
52
|
+
reason TEXT,
|
|
53
|
+
resolved INTEGER NOT NULL DEFAULT 0,
|
|
54
|
+
created_at INTEGER NOT NULL,
|
|
55
|
+
resolved_at INTEGER,
|
|
56
|
+
resolved_by TEXT
|
|
57
|
+
);
|
|
58
|
+
CREATE INDEX idx_conflict_open ON fact_conflict(resolved, created_at);
|
|
59
|
+
|
|
60
|
+
CREATE TABLE event (
|
|
61
|
+
event_id TEXT PRIMARY KEY,
|
|
62
|
+
user_id TEXT,
|
|
63
|
+
conversation_key TEXT,
|
|
64
|
+
event_type TEXT NOT NULL,
|
|
65
|
+
content TEXT,
|
|
66
|
+
occurred_at INTEGER,
|
|
67
|
+
observed_at INTEGER,
|
|
68
|
+
correlation_id TEXT,
|
|
69
|
+
causation_id TEXT,
|
|
70
|
+
source_record_id TEXT,
|
|
71
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
72
|
+
embedding_model_version TEXT,
|
|
73
|
+
embedding_pending INTEGER NOT NULL DEFAULT 0,
|
|
74
|
+
indexed INTEGER NOT NULL DEFAULT 0
|
|
75
|
+
);
|
|
76
|
+
CREATE INDEX idx_event_type ON event(event_type);
|
|
77
|
+
CREATE INDEX idx_event_corr ON event(correlation_id);
|
|
78
|
+
CREATE INDEX idx_event_conv ON event(conversation_key, occurred_at);
|
|
79
|
+
|
|
80
|
+
CREATE TABLE raw_log (
|
|
81
|
+
record_id TEXT PRIMARY KEY,
|
|
82
|
+
record_type TEXT NOT NULL,
|
|
83
|
+
adapter TEXT,
|
|
84
|
+
peer_id TEXT,
|
|
85
|
+
user_id TEXT,
|
|
86
|
+
conversation_key TEXT,
|
|
87
|
+
platform TEXT,
|
|
88
|
+
correlation_id TEXT,
|
|
89
|
+
causation_id TEXT,
|
|
90
|
+
content_hash TEXT,
|
|
91
|
+
dedupe_key TEXT NOT NULL,
|
|
92
|
+
observed_at INTEGER NOT NULL,
|
|
93
|
+
payload TEXT,
|
|
94
|
+
payload_status TEXT NOT NULL DEFAULT 'available',
|
|
95
|
+
metadata TEXT
|
|
96
|
+
);
|
|
97
|
+
CREATE INDEX idx_raw_conv ON raw_log(conversation_key, observed_at);
|
|
98
|
+
CREATE UNIQUE INDEX idx_raw_dedupe ON raw_log(adapter, peer_id, dedupe_key);
|
|
99
|
+
|
|
100
|
+
CREATE TABLE outbound (
|
|
101
|
+
idempotency_key TEXT PRIMARY KEY,
|
|
102
|
+
message_id TEXT,
|
|
103
|
+
generation_id TEXT,
|
|
104
|
+
send_status TEXT NOT NULL CHECK (send_status IN ('SUBMITTED','SUPPRESSED','FAILED_LOCAL')),
|
|
105
|
+
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
106
|
+
last_error TEXT,
|
|
107
|
+
created_at INTEGER NOT NULL,
|
|
108
|
+
updated_at INTEGER NOT NULL
|
|
109
|
+
);
|
|
110
|
+
CREATE INDEX idx_outbound_gen ON outbound(generation_id);
|
|
111
|
+
|
|
112
|
+
CREATE TABLE job (
|
|
113
|
+
job_id TEXT PRIMARY KEY,
|
|
114
|
+
kind TEXT NOT NULL,
|
|
115
|
+
idem_key TEXT NOT NULL,
|
|
116
|
+
payload TEXT,
|
|
117
|
+
state TEXT NOT NULL DEFAULT 'pending',
|
|
118
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
119
|
+
next_at INTEGER,
|
|
120
|
+
last_error TEXT,
|
|
121
|
+
created_at INTEGER NOT NULL,
|
|
122
|
+
updated_at INTEGER NOT NULL
|
|
123
|
+
);
|
|
124
|
+
CREATE UNIQUE INDEX idx_job_idem ON job(idem_key);
|
|
125
|
+
CREATE INDEX idx_job_state ON job(state, next_at);
|
|
126
|
+
|
|
127
|
+
CREATE TABLE tombstone (
|
|
128
|
+
target_type TEXT NOT NULL,
|
|
129
|
+
target_id TEXT NOT NULL,
|
|
130
|
+
created_at INTEGER NOT NULL,
|
|
131
|
+
PRIMARY KEY (target_type, target_id)
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
-- 全文索引:普通 FTS5 表,存 bigram 变换后的文本;删除按 ref 直接删行(不用外部内容表)
|
|
135
|
+
CREATE VIRTUAL TABLE fact_fts USING fts5(bigram_text, fact_id UNINDEXED, tokenize='unicode61');
|
|
136
|
+
CREATE VIRTUAL TABLE event_fts USING fts5(bigram_text, event_id UNINDEXED, tokenize='unicode61');
|
|
137
|
+
`;
|
|
138
|
+
|
|
139
|
+
export function openDatabase(file) {
|
|
140
|
+
if (file !== ':memory:') mkdirSync(dirname(file), { recursive: true });
|
|
141
|
+
// allowExtension 必须在这里给:node:sqlite 默认禁止加载扩展,sqlite-vec 要靠它
|
|
142
|
+
const db = new DatabaseSync(file, { allowExtension: true });
|
|
143
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
144
|
+
db.exec('PRAGMA foreign_keys = ON');
|
|
145
|
+
db.exec('PRAGMA busy_timeout = 5000');
|
|
146
|
+
db.exec('PRAGMA synchronous = NORMAL');
|
|
147
|
+
migrate(db);
|
|
148
|
+
return db;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const SCHEMA_V2 = `
|
|
152
|
+
CREATE TABLE silence (
|
|
153
|
+
silence_id TEXT PRIMARY KEY,
|
|
154
|
+
scope_type TEXT NOT NULL,
|
|
155
|
+
scope_id TEXT NOT NULL,
|
|
156
|
+
owner_user_id TEXT,
|
|
157
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
158
|
+
started_at INTEGER NOT NULL,
|
|
159
|
+
expires_at INTEGER,
|
|
160
|
+
created_by TEXT,
|
|
161
|
+
updated_by TEXT,
|
|
162
|
+
reason TEXT
|
|
163
|
+
);
|
|
164
|
+
CREATE UNIQUE INDEX idx_silence_active ON silence(scope_type, scope_id) WHERE status = 'active';
|
|
165
|
+
|
|
166
|
+
CREATE TABLE route_policy (
|
|
167
|
+
policy_id TEXT PRIMARY KEY,
|
|
168
|
+
conversation_key TEXT NOT NULL,
|
|
169
|
+
mode TEXT NOT NULL,
|
|
170
|
+
effective_from INTEGER NOT NULL,
|
|
171
|
+
expires_at INTEGER NOT NULL,
|
|
172
|
+
set_by TEXT,
|
|
173
|
+
created_at INTEGER NOT NULL,
|
|
174
|
+
revoked_at INTEGER
|
|
175
|
+
);
|
|
176
|
+
CREATE INDEX idx_policy_active ON route_policy(conversation_key, revoked_at, expires_at);
|
|
177
|
+
`;
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
const SCHEMA_V3 = `
|
|
181
|
+
CREATE TABLE exemplar (
|
|
182
|
+
exemplar_id TEXT PRIMARY KEY,
|
|
183
|
+
text TEXT NOT NULL,
|
|
184
|
+
label TEXT NOT NULL CHECK (label IN ('relevant_ambient','irrelevant_ambient')),
|
|
185
|
+
version TEXT NOT NULL DEFAULT 'v1',
|
|
186
|
+
approved INTEGER NOT NULL DEFAULT 0,
|
|
187
|
+
created_at INTEGER NOT NULL
|
|
188
|
+
);
|
|
189
|
+
CREATE INDEX idx_exemplar_label ON exemplar(label, approved);
|
|
190
|
+
|
|
191
|
+
CREATE TABLE route_budget (
|
|
192
|
+
conversation_key TEXT NOT NULL,
|
|
193
|
+
window_start INTEGER NOT NULL,
|
|
194
|
+
spent INTEGER NOT NULL DEFAULT 0,
|
|
195
|
+
PRIMARY KEY (conversation_key, window_start)
|
|
196
|
+
);
|
|
197
|
+
`;
|
|
198
|
+
|
|
199
|
+
const SCHEMA_V4 = `
|
|
200
|
+
ALTER TABLE fact_version ADD COLUMN surface TEXT;
|
|
201
|
+
UPDATE fact_version SET surface = value WHERE surface IS NULL;
|
|
202
|
+
`;
|
|
203
|
+
|
|
204
|
+
const SCHEMA_V5 = `
|
|
205
|
+
CREATE TABLE IF NOT EXISTS attention (
|
|
206
|
+
attention_id TEXT PRIMARY KEY,
|
|
207
|
+
kind TEXT NOT NULL,
|
|
208
|
+
ref_id TEXT,
|
|
209
|
+
label TEXT NOT NULL,
|
|
210
|
+
note TEXT,
|
|
211
|
+
weight REAL NOT NULL DEFAULT 0.5,
|
|
212
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
213
|
+
hits INTEGER NOT NULL DEFAULT 0,
|
|
214
|
+
last_hit_at INTEGER,
|
|
215
|
+
created_at INTEGER NOT NULL,
|
|
216
|
+
expires_at INTEGER
|
|
217
|
+
);
|
|
218
|
+
CREATE INDEX IF NOT EXISTS idx_attention_status ON attention(status, weight DESC);
|
|
219
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS attention_fts USING fts5(
|
|
220
|
+
attention_id UNINDEXED, text, tokenize='unicode61'
|
|
221
|
+
);
|
|
222
|
+
`;
|
|
223
|
+
|
|
224
|
+
const SCHEMA_V6 = `
|
|
225
|
+
CREATE TABLE IF NOT EXISTS settings (
|
|
226
|
+
key TEXT PRIMARY KEY,
|
|
227
|
+
value TEXT NOT NULL,
|
|
228
|
+
updated_at INTEGER NOT NULL
|
|
229
|
+
);
|
|
230
|
+
`;
|
|
231
|
+
|
|
232
|
+
const SCHEMA_V7 = `
|
|
233
|
+
ALTER TABLE event ADD COLUMN speaker_name TEXT;
|
|
234
|
+
`;
|
|
235
|
+
|
|
236
|
+
const SCHEMA_V8 = `
|
|
237
|
+
ALTER TABLE tombstone ADD COLUMN reason TEXT;
|
|
238
|
+
`;
|
|
239
|
+
|
|
240
|
+
const SCHEMA_V9 = `
|
|
241
|
+
CREATE TABLE IF NOT EXISTS people (
|
|
242
|
+
openid TEXT PRIMARY KEY,
|
|
243
|
+
nickname TEXT,
|
|
244
|
+
relation TEXT,
|
|
245
|
+
first_seen INTEGER,
|
|
246
|
+
last_seen INTEGER
|
|
247
|
+
);
|
|
248
|
+
CREATE TABLE IF NOT EXISTS alias (
|
|
249
|
+
alias_key TEXT PRIMARY KEY,
|
|
250
|
+
canonical_key TEXT NOT NULL
|
|
251
|
+
);
|
|
252
|
+
`;
|
|
253
|
+
|
|
254
|
+
const SCHEMA_V10 = `
|
|
255
|
+
ALTER TABLE event ADD COLUMN scope TEXT;
|
|
256
|
+
-- 群/私聊判定交给调用方(conversation_key 的形态因适配器而异),这里只兜底标记
|
|
257
|
+
UPDATE event SET scope = 'other' WHERE scope IS NULL;
|
|
258
|
+
`;
|
|
259
|
+
|
|
260
|
+
const SCHEMA_V11 = `
|
|
261
|
+
ALTER TABLE fact ADD COLUMN scope TEXT;
|
|
262
|
+
UPDATE fact SET scope = (
|
|
263
|
+
SELECT MAX(e.scope) FROM raw_log r
|
|
264
|
+
JOIN event e ON e.conversation_key = r.conversation_key
|
|
265
|
+
WHERE r.record_id = fact.source_record_id
|
|
266
|
+
) WHERE scope IS NULL;
|
|
267
|
+
UPDATE fact SET scope = 'unknown' WHERE scope IS NULL;
|
|
268
|
+
`;
|
|
269
|
+
|
|
270
|
+
const SCHEMA_V12 = `
|
|
271
|
+
CREATE TABLE IF NOT EXISTS image_cache (
|
|
272
|
+
sha1 TEXT PRIMARY KEY,
|
|
273
|
+
path TEXT,
|
|
274
|
+
media_type TEXT,
|
|
275
|
+
semantic TEXT NOT NULL,
|
|
276
|
+
seen_count INTEGER NOT NULL DEFAULT 1,
|
|
277
|
+
first_seen INTEGER NOT NULL,
|
|
278
|
+
last_seen INTEGER NOT NULL
|
|
279
|
+
);
|
|
280
|
+
`;
|
|
281
|
+
|
|
282
|
+
const SCHEMA_V13 = `
|
|
283
|
+
CREATE TABLE IF NOT EXISTS topic_candidate (
|
|
284
|
+
topic_key TEXT PRIMARY KEY,
|
|
285
|
+
text TEXT NOT NULL,
|
|
286
|
+
count INTEGER NOT NULL DEFAULT 1,
|
|
287
|
+
first_seen INTEGER NOT NULL,
|
|
288
|
+
last_seen INTEGER NOT NULL
|
|
289
|
+
);
|
|
290
|
+
`;
|
|
291
|
+
|
|
292
|
+
const MIGRATIONS = { 1: SCHEMA_V1, 2: SCHEMA_V2, 3: SCHEMA_V3, 4: SCHEMA_V4, 5: SCHEMA_V5, 6: SCHEMA_V6, 7: SCHEMA_V7, 8: SCHEMA_V8, 9: SCHEMA_V9, 10: SCHEMA_V10, 11: SCHEMA_V11, 12: SCHEMA_V12, 13: SCHEMA_V13 };
|
|
293
|
+
|
|
294
|
+
export function migrate(db) {
|
|
295
|
+
const current = db.prepare('PRAGMA user_version').get().user_version ?? 0;
|
|
296
|
+
if (current >= SCHEMA_VERSION) return;
|
|
297
|
+
db.exec('BEGIN IMMEDIATE');
|
|
298
|
+
try {
|
|
299
|
+
for (let v = current + 1; v <= SCHEMA_VERSION; v++) {
|
|
300
|
+
const sql = MIGRATIONS[v];
|
|
301
|
+
if (sql) db.exec(sql);
|
|
302
|
+
}
|
|
303
|
+
db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
|
|
304
|
+
db.exec('COMMIT');
|
|
305
|
+
} catch (err) {
|
|
306
|
+
db.exec('ROLLBACK');
|
|
307
|
+
throw err;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
export function tx(db, fn) {
|
|
312
|
+
db.exec('BEGIN IMMEDIATE');
|
|
313
|
+
try {
|
|
314
|
+
const out = fn();
|
|
315
|
+
db.exec('COMMIT');
|
|
316
|
+
return out;
|
|
317
|
+
} catch (err) {
|
|
318
|
+
try {
|
|
319
|
+
db.exec('ROLLBACK');
|
|
320
|
+
} catch {
|
|
321
|
+
/* 已回滚 */
|
|
322
|
+
}
|
|
323
|
+
throw err;
|
|
324
|
+
}
|
|
325
|
+
}
|
package/lib/dedupe.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// 入站判重:有平台 msgId 用 msgId(强幂等),没有才退内容哈希 + 时间窗。
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
|
|
4
|
+
export const DEFAULT_WINDOW_MS = 10_000;
|
|
5
|
+
|
|
6
|
+
export function contentHash(text) {
|
|
7
|
+
return createHash('sha1').update(String(text ?? '')).digest('hex');
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* 从消息文本里嗅探透传的 msgId 前缀 `[msgid:ROBOT1.0_xxx]`。
|
|
12
|
+
* 传输层若将来把平台 msgId 带进正文,本函数即可自动启用强幂等,零改动。
|
|
13
|
+
*/
|
|
14
|
+
export function sniffPlatformMsgId(text) {
|
|
15
|
+
const m = /\[msgid:([^\]\s]+)\]/.exec(String(text ?? ''));
|
|
16
|
+
return m ? m[1] : null;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* 判重键。
|
|
21
|
+
* - platformMsgId 存在:`msg:<adapter>:<peer>:<id>` —— 同一条平台消息永远同键;
|
|
22
|
+
* - 不存在:`hash:<adapter>:<peer>:<sha1>:<窗口序号>` —— 只在同一时间窗内合并。
|
|
23
|
+
*/
|
|
24
|
+
export function dedupeKey({ adapter, peerId, platformMsgId, text, observedAt, windowMs = DEFAULT_WINDOW_MS }) {
|
|
25
|
+
const id = platformMsgId ?? sniffPlatformMsgId(text);
|
|
26
|
+
if (id) return `msg:${adapter}:${peerId}:${id}`;
|
|
27
|
+
const bucket = Math.floor(observedAt / windowMs);
|
|
28
|
+
return `hash:${adapter}:${peerId}:${contentHash(text)}:${bucket}`;
|
|
29
|
+
}
|
package/lib/egress.js
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
// 出站闸门:generation 状态机 + llm/stream 文本挂起。
|
|
2
|
+
const SILENT_TOOL = 'stay_silent';
|
|
3
|
+
const REPLY_TOOL = 'reply';
|
|
4
|
+
|
|
5
|
+
export function createEgress({ isBindable, cfg, log }) {
|
|
6
|
+
/** sessionId -> 当前 generation */
|
|
7
|
+
const gens = new Map();
|
|
8
|
+
|
|
9
|
+
const newGen = (sessionId, turn) => ({ sessionId, turn, id: `${sessionId}:${turn}`, silent: false, replied: false });
|
|
10
|
+
const current = (sessionId) => gens.get(sessionId) ?? newGen(sessionId, 0);
|
|
11
|
+
|
|
12
|
+
return {
|
|
13
|
+
/** session/event 里调用:turn/start 开新 generation,turn/end 收尾。 */
|
|
14
|
+
observe(sessionId, event) {
|
|
15
|
+
if (!isBindable(sessionId)) return;
|
|
16
|
+
if (event.type === 'turn/start') {
|
|
17
|
+
gens.set(sessionId, newGen(sessionId, event.data?.turn ?? 0));
|
|
18
|
+
} else if (event.type === 'turn/end') {
|
|
19
|
+
const g = gens.get(sessionId);
|
|
20
|
+
if (g?.silent) log?.('info', 'egress', { sessionId, generation: g.id, status: 'SILENT' });
|
|
21
|
+
gens.delete(sessionId);
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
|
|
25
|
+
get(sessionId) {
|
|
26
|
+
return current(sessionId);
|
|
27
|
+
},
|
|
28
|
+
|
|
29
|
+
declareSilent(sessionId) {
|
|
30
|
+
const g = current(sessionId);
|
|
31
|
+
g.silent = true;
|
|
32
|
+
gens.set(sessionId, g);
|
|
33
|
+
return g;
|
|
34
|
+
},
|
|
35
|
+
|
|
36
|
+
declareReply(sessionId) {
|
|
37
|
+
const g = current(sessionId);
|
|
38
|
+
g.replied = true;
|
|
39
|
+
gens.set(sessionId, g);
|
|
40
|
+
return g;
|
|
41
|
+
},
|
|
42
|
+
|
|
43
|
+
/** llm/stream 用:返回异步生成器,静默时丢弃正文。 */
|
|
44
|
+
wrapStream(sessionId, iterable) {
|
|
45
|
+
const bound = isBindable(sessionId);
|
|
46
|
+
if (!bound || cfg.holdTextOnSilence === false) return iterable;
|
|
47
|
+
return (async function* () {
|
|
48
|
+
const textDeltas = [];
|
|
49
|
+
const textEnds = [];
|
|
50
|
+
const toolNames = new Set();
|
|
51
|
+
for await (const chunk of iterable) {
|
|
52
|
+
if (chunk.type === 'text-delta') {
|
|
53
|
+
textDeltas.push(chunk);
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
if (chunk.type === 'block-end' && chunk.block?.type === 'text') {
|
|
57
|
+
textEnds.push(chunk);
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
if (chunk.type === 'tool-call-delta' && chunk.name) toolNames.add(chunk.name);
|
|
61
|
+
if (chunk.type === 'block-end' && chunk.block?.type === 'tool-call' && chunk.block.name) {
|
|
62
|
+
toolNames.add(chunk.block.name);
|
|
63
|
+
}
|
|
64
|
+
yield chunk;
|
|
65
|
+
}
|
|
66
|
+
const silent = toolNames.has(SILENT_TOOL);
|
|
67
|
+
if (silent) {
|
|
68
|
+
// 丢掉正文,只保留一个空文本块,避免下游拿到半截内容
|
|
69
|
+
for (const end of textEnds) yield { ...end, block: { ...end.block, text: '' } };
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
for (const d of textDeltas) yield d;
|
|
73
|
+
for (const end of textEnds) yield end;
|
|
74
|
+
})();
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** 纯函数:从 tool-call 名集合判断本轮是否声明静默(供单测)。 */
|
|
80
|
+
export function isSilentCall(names) {
|
|
81
|
+
return new Set(names).has(SILENT_TOOL);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export { SILENT_TOOL, REPLY_TOOL };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// embedding 子进程:把 transformers/ONNX 放进独立进程,避免与宿主的原生库撞名
|
|
2
|
+
import { env, pipeline } from '@huggingface/transformers';
|
|
3
|
+
|
|
4
|
+
const model = process.env.TM_MODEL || 'Xenova/bge-small-zh-v1.5';
|
|
5
|
+
const dtype = process.env.TM_DTYPE || 'q8';
|
|
6
|
+
const cacheDir = process.env.TM_CACHE_DIR || '';
|
|
7
|
+
const remoteHost = process.env.TM_REMOTE_HOST || '';
|
|
8
|
+
const dim = Number(process.env.TM_DIM || 512);
|
|
9
|
+
|
|
10
|
+
const send = (msg) => {
|
|
11
|
+
try {
|
|
12
|
+
process.send?.(msg);
|
|
13
|
+
} catch {
|
|
14
|
+
/* 父进程已退出 */
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
try {
|
|
19
|
+
if (cacheDir) env.cacheDir = cacheDir;
|
|
20
|
+
if (remoteHost) env.remoteHost = String(remoteHost).replace(/\/+$/, '');
|
|
21
|
+
const pipe = await pipeline('feature-extraction', model, { dtype });
|
|
22
|
+
send({ type: 'ready', dim });
|
|
23
|
+
|
|
24
|
+
process.on('message', async (msg) => {
|
|
25
|
+
if (msg?.type !== 'embed') return;
|
|
26
|
+
try {
|
|
27
|
+
const out = await pipe(Array.from(msg.texts), { pooling: 'mean', normalize: true });
|
|
28
|
+
const rows = out.dims[0];
|
|
29
|
+
const d = out.dims[1];
|
|
30
|
+
const buf = out.data;
|
|
31
|
+
const vectors = [];
|
|
32
|
+
for (let r = 0; r < rows; r++) vectors.push(Array.from(buf.subarray(r * d, (r + 1) * d)));
|
|
33
|
+
send({ type: 'result', id: msg.id, vectors });
|
|
34
|
+
} catch (err) {
|
|
35
|
+
send({ type: 'error', id: msg.id, error: String(err?.message ?? err) });
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
} catch (err) {
|
|
39
|
+
send({ type: 'fatal', error: String(err?.stack ?? err) });
|
|
40
|
+
setTimeout(() => process.exit(1), 50);
|
|
41
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// embedding worker:在独立线程里跑 transformers/ONNX,避开宿主主线程的模块钩子。
|
|
2
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
3
|
+
import { env, pipeline } from '@huggingface/transformers';
|
|
4
|
+
|
|
5
|
+
const { model, dtype, cacheDir, remoteHost, dim = 512 } = workerData ?? {};
|
|
6
|
+
|
|
7
|
+
function post(msg) {
|
|
8
|
+
parentPort.postMessage(msg);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
if (cacheDir) env.cacheDir = cacheDir;
|
|
13
|
+
if (remoteHost) env.remoteHost = String(remoteHost).replace(/\/+$/, '');
|
|
14
|
+
const pipe = await pipeline('feature-extraction', model, { dtype });
|
|
15
|
+
post({ type: 'ready', dim });
|
|
16
|
+
|
|
17
|
+
parentPort.on('message', async (msg) => {
|
|
18
|
+
if (msg?.type !== 'embed') return;
|
|
19
|
+
try {
|
|
20
|
+
const out = await pipe(Array.from(msg.texts), { pooling: 'mean', normalize: true });
|
|
21
|
+
const rows = out.dims[0];
|
|
22
|
+
const d = out.dims[1];
|
|
23
|
+
const buf = out.data;
|
|
24
|
+
const vectors = [];
|
|
25
|
+
for (let r = 0; r < rows; r++) {
|
|
26
|
+
vectors.push(Array.from(buf.subarray(r * d, (r + 1) * d)));
|
|
27
|
+
}
|
|
28
|
+
post({ type: 'result', id: msg.id, vectors });
|
|
29
|
+
} catch (err) {
|
|
30
|
+
post({ type: 'error', id: msg.id, error: String(err?.message ?? err) });
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
} catch (err) {
|
|
34
|
+
post({ type: 'fatal', error: String(err?.stack ?? err?.message ?? err) });
|
|
35
|
+
}
|