@memtensor/memos-local-openclaw-plugin 1.0.8-beta.7 → 1.0.8-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.ts +334 -391
- package/openclaw.plugin.json +1 -1
- package/package.json +3 -4
- package/prebuilds/darwin-arm64/better_sqlite3.node +0 -0
- package/prebuilds/darwin-x64/better_sqlite3.node +0 -0
- package/prebuilds/linux-x64/better_sqlite3.node +0 -0
- package/prebuilds/win32-x64/better_sqlite3.node +0 -0
- package/scripts/postinstall.cjs +25 -59
- package/src/context-engine/index.ts +321 -0
- package/src/hub/server.ts +6 -13
- package/src/ingest/providers/anthropic.ts +6 -9
- package/src/ingest/providers/bedrock.ts +6 -9
- package/src/ingest/providers/gemini.ts +6 -9
- package/src/ingest/providers/index.ts +22 -123
- package/src/ingest/providers/openai.ts +6 -141
- package/src/ingest/task-processor.ts +41 -61
- package/src/ingest/worker.ts +11 -32
- package/src/recall/engine.ts +1 -2
- package/src/sharing/types.ts +0 -1
- package/src/storage/sqlite.ts +11 -194
- package/src/types.ts +0 -3
- package/src/viewer/html.ts +266 -892
- package/src/viewer/server.ts +20 -293
- package/telemetry.credentials.json +5 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import * as fs from "fs";
|
|
2
2
|
import * as path from "path";
|
|
3
|
-
import type { SummarizerConfig, SummaryProvider, Logger
|
|
4
|
-
import { summarizeOpenAI, summarizeTaskOpenAI, generateTaskTitleOpenAI, judgeNewTopicOpenAI,
|
|
5
|
-
import type { FilterResult, DedupResult
|
|
6
|
-
export type { FilterResult, DedupResult
|
|
3
|
+
import type { SummarizerConfig, SummaryProvider, Logger } from "../../types";
|
|
4
|
+
import { summarizeOpenAI, summarizeTaskOpenAI, generateTaskTitleOpenAI, judgeNewTopicOpenAI, filterRelevantOpenAI, judgeDedupOpenAI } from "./openai";
|
|
5
|
+
import type { FilterResult, DedupResult } from "./openai";
|
|
6
|
+
export type { FilterResult, DedupResult } from "./openai";
|
|
7
7
|
import { summarizeAnthropic, summarizeTaskAnthropic, generateTaskTitleAnthropic, judgeNewTopicAnthropic, filterRelevantAnthropic, judgeDedupAnthropic } from "./anthropic";
|
|
8
8
|
import { summarizeGemini, summarizeTaskGemini, generateTaskTitleGemini, judgeNewTopicGemini, filterRelevantGemini, judgeDedupGemini } from "./gemini";
|
|
9
9
|
import { summarizeBedrock, summarizeTaskBedrock, generateTaskTitleBedrock, judgeNewTopicBedrock, filterRelevantBedrock, judgeDedupBedrock } from "./bedrock";
|
|
@@ -287,30 +287,25 @@ export class Summarizer {
|
|
|
287
287
|
}
|
|
288
288
|
|
|
289
289
|
async judgeNewTopic(currentContext: string, newMessage: string): Promise<boolean | null> {
|
|
290
|
-
const
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
);
|
|
295
|
-
return result ?? null;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
async classifyTopic(taskState: string, newMessage: string): Promise<TopicClassifyResult | null> {
|
|
299
|
-
const result = await this.tryChain("classifyTopic", (cfg) =>
|
|
300
|
-
cfg.provider === "openclaw"
|
|
301
|
-
? this.classifyTopicOpenClaw(taskState, newMessage)
|
|
302
|
-
: callTopicClassifier(cfg, taskState, newMessage, this.log),
|
|
303
|
-
);
|
|
304
|
-
return result ?? null;
|
|
305
|
-
}
|
|
290
|
+
const chain: SummarizerConfig[] = [];
|
|
291
|
+
if (this.strongCfg) chain.push(this.strongCfg);
|
|
292
|
+
if (this.fallbackCfg) chain.push(this.fallbackCfg);
|
|
293
|
+
if (chain.length === 0 && this.cfg) chain.push(this.cfg);
|
|
294
|
+
if (chain.length === 0) return null;
|
|
306
295
|
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
296
|
+
for (let i = 0; i < chain.length; i++) {
|
|
297
|
+
const modelInfo = `${chain[i].provider}/${chain[i].model ?? "?"}`;
|
|
298
|
+
try {
|
|
299
|
+
const result = await callTopicJudge(chain[i], currentContext, newMessage, this.log);
|
|
300
|
+
modelHealth.recordSuccess("judgeNewTopic", modelInfo);
|
|
301
|
+
return result;
|
|
302
|
+
} catch (err) {
|
|
303
|
+
const level = i < chain.length - 1 ? "warn" : "error";
|
|
304
|
+
this.log[level](`judgeNewTopic failed (${modelInfo}), ${i < chain.length - 1 ? "trying next" : "no more fallbacks"}: ${err}`);
|
|
305
|
+
modelHealth.recordError("judgeNewTopic", modelInfo, String(err));
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return null;
|
|
314
309
|
}
|
|
315
310
|
|
|
316
311
|
async filterRelevant(
|
|
@@ -351,19 +346,8 @@ export class Summarizer {
|
|
|
351
346
|
|
|
352
347
|
static readonly OPENCLAW_TOPIC_JUDGE_PROMPT = `You are a conversation topic change detector.
|
|
353
348
|
Given a CURRENT CONVERSATION SUMMARY and a NEW USER MESSAGE, decide: has the user started a COMPLETELY NEW topic that is unrelated to the current conversation?
|
|
354
|
-
Default to SAME unless the domain clearly changed. If the new message shares the same person, event, entity, or theme with the current conversation, answer SAME.
|
|
355
|
-
CRITICAL: Short messages (under ~30 characters) that use pronouns (那/这/它/哪些) or ask about tools/details/dimensions of the current topic are almost always follow-ups — answer SAME unless they explicitly name a completely unrelated domain.
|
|
356
349
|
Reply with a single word: "NEW" if topic changed, "SAME" if it continues.`;
|
|
357
350
|
|
|
358
|
-
static readonly OPENCLAW_TOPIC_CLASSIFIER_PROMPT = `Classify if NEW MESSAGE continues current task or starts an unrelated one.
|
|
359
|
-
Output ONLY JSON: {"d":"S"|"N","c":0.0-1.0}
|
|
360
|
-
d=S(same) or N(new). c=confidence. Default S. Only N if completely unrelated domain.
|
|
361
|
-
Sub-questions, tools, methods, details of current topic = S.`;
|
|
362
|
-
|
|
363
|
-
static readonly OPENCLAW_TOPIC_ARBITRATION_PROMPT = `A classifier flagged this message as possibly new topic (low confidence). Is it truly UNRELATED, or a sub-question/follow-up?
|
|
364
|
-
Tools/methods/details of current task = SAME. Shared entity/theme = SAME. Entirely different domain = NEW.
|
|
365
|
-
Reply one word: NEW or SAME`;
|
|
366
|
-
|
|
367
351
|
static readonly OPENCLAW_FILTER_RELEVANT_PROMPT = `You are a memory relevance judge.
|
|
368
352
|
Given a QUERY and CANDIDATE memories, decide: does each candidate help answer the query?
|
|
369
353
|
RULES:
|
|
@@ -449,45 +433,6 @@ Reply with JSON: {"action":"MERGE","mergeTarget":2,"reason":"..."} or {"action":
|
|
|
449
433
|
return answer.startsWith("NEW");
|
|
450
434
|
}
|
|
451
435
|
|
|
452
|
-
private async classifyTopicOpenClaw(taskState: string, newMessage: string): Promise<TopicClassifyResult> {
|
|
453
|
-
this.requireOpenClawAPI();
|
|
454
|
-
const prompt = [
|
|
455
|
-
Summarizer.OPENCLAW_TOPIC_CLASSIFIER_PROMPT,
|
|
456
|
-
``,
|
|
457
|
-
`TASK:\n${taskState}`,
|
|
458
|
-
`\nMSG:\n${newMessage}`,
|
|
459
|
-
].join("\n");
|
|
460
|
-
|
|
461
|
-
const response = await this.openclawAPI!.complete({
|
|
462
|
-
prompt,
|
|
463
|
-
maxTokens: 60,
|
|
464
|
-
temperature: 0,
|
|
465
|
-
model: this.cfg?.model,
|
|
466
|
-
});
|
|
467
|
-
|
|
468
|
-
return parseTopicClassifyResult(response.text.trim(), this.log);
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
private async arbitrateTopicSplitOpenClaw(taskState: string, newMessage: string): Promise<string> {
|
|
472
|
-
this.requireOpenClawAPI();
|
|
473
|
-
const prompt = [
|
|
474
|
-
Summarizer.OPENCLAW_TOPIC_ARBITRATION_PROMPT,
|
|
475
|
-
``,
|
|
476
|
-
`TASK:\n${taskState}`,
|
|
477
|
-
`\nMSG:\n${newMessage}`,
|
|
478
|
-
].join("\n");
|
|
479
|
-
|
|
480
|
-
const response = await this.openclawAPI!.complete({
|
|
481
|
-
prompt,
|
|
482
|
-
maxTokens: 10,
|
|
483
|
-
temperature: 0,
|
|
484
|
-
model: this.cfg?.model,
|
|
485
|
-
});
|
|
486
|
-
|
|
487
|
-
const answer = response.text.trim().toUpperCase();
|
|
488
|
-
return answer.startsWith("NEW") ? "NEW" : "SAME";
|
|
489
|
-
}
|
|
490
|
-
|
|
491
436
|
private async filterRelevantOpenClaw(
|
|
492
437
|
query: string,
|
|
493
438
|
candidates: Array<{ index: number; role: string; content: string; time?: string }>,
|
|
@@ -698,52 +643,6 @@ function callJudgeDedup(cfg: SummarizerConfig, newSummary: string, candidates: A
|
|
|
698
643
|
}
|
|
699
644
|
}
|
|
700
645
|
|
|
701
|
-
function callTopicClassifier(cfg: SummarizerConfig, taskState: string, newMessage: string, log: Logger): Promise<TopicClassifyResult> {
|
|
702
|
-
switch (cfg.provider) {
|
|
703
|
-
case "openai":
|
|
704
|
-
case "openai_compatible":
|
|
705
|
-
case "azure_openai":
|
|
706
|
-
case "zhipu":
|
|
707
|
-
case "siliconflow":
|
|
708
|
-
case "deepseek":
|
|
709
|
-
case "moonshot":
|
|
710
|
-
case "bailian":
|
|
711
|
-
case "cohere":
|
|
712
|
-
case "mistral":
|
|
713
|
-
case "voyage":
|
|
714
|
-
return classifyTopicOpenAI(taskState, newMessage, cfg, log);
|
|
715
|
-
case "anthropic":
|
|
716
|
-
case "gemini":
|
|
717
|
-
case "bedrock":
|
|
718
|
-
return classifyTopicOpenAI(taskState, newMessage, cfg, log);
|
|
719
|
-
default:
|
|
720
|
-
throw new Error(`Unknown summarizer provider: ${cfg.provider}`);
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
function callTopicArbitration(cfg: SummarizerConfig, taskState: string, newMessage: string, log: Logger): Promise<string> {
|
|
725
|
-
switch (cfg.provider) {
|
|
726
|
-
case "openai":
|
|
727
|
-
case "openai_compatible":
|
|
728
|
-
case "azure_openai":
|
|
729
|
-
case "zhipu":
|
|
730
|
-
case "siliconflow":
|
|
731
|
-
case "deepseek":
|
|
732
|
-
case "moonshot":
|
|
733
|
-
case "bailian":
|
|
734
|
-
case "cohere":
|
|
735
|
-
case "mistral":
|
|
736
|
-
case "voyage":
|
|
737
|
-
return arbitrateTopicSplitOpenAI(taskState, newMessage, cfg, log);
|
|
738
|
-
case "anthropic":
|
|
739
|
-
case "gemini":
|
|
740
|
-
case "bedrock":
|
|
741
|
-
return arbitrateTopicSplitOpenAI(taskState, newMessage, cfg, log);
|
|
742
|
-
default:
|
|
743
|
-
throw new Error(`Unknown summarizer provider: ${cfg.provider}`);
|
|
744
|
-
}
|
|
745
|
-
}
|
|
746
|
-
|
|
747
646
|
// ─── Fallbacks ───
|
|
748
647
|
|
|
749
648
|
function ruleFallback(text: string): string {
|
|
@@ -188,26 +188,19 @@ SAME — the new message:
|
|
|
188
188
|
- Reports a result, error, or feedback about the current task
|
|
189
189
|
- Discusses different tools or approaches for the SAME goal (e.g., learning English via BBC → via ChatGPT = SAME)
|
|
190
190
|
- Is a short acknowledgment (ok, thanks, 好的) in response to the current flow
|
|
191
|
-
- Is a follow-up, update, or different angle on the same news event, person, or story
|
|
192
|
-
- Shares the same core entity (person, company, event) even if the specific detail or angle differs
|
|
193
|
-
- Contains pronouns or references (那, 这, 它, 其中, 哪些, those, which, what about, etc.) pointing to items from the current conversation
|
|
194
|
-
- Asks about a sub-topic, tool, detail, dimension, or aspect of the current discussion topic
|
|
195
191
|
|
|
196
192
|
NEW — the new message:
|
|
197
|
-
- Introduces a subject from a
|
|
198
|
-
- Has NO logical connection to what was being discussed
|
|
193
|
+
- Introduces a subject from a DIFFERENT domain than the current task (e.g., tech → cooking, work → personal life, database → travel)
|
|
194
|
+
- Has NO logical connection to what was being discussed
|
|
199
195
|
- Starts a request about a different project, system, or life area
|
|
200
196
|
- Begins with a new greeting/reset followed by a different topic
|
|
201
197
|
|
|
202
198
|
Key principles:
|
|
203
|
-
-
|
|
204
|
-
- CRITICAL: Short messages (under ~30 characters) that use pronouns or ask "what about X" / "哪些" / "那XX呢" are almost always follow-ups referring to the current topic. Only mark them NEW if they explicitly name a completely unrelated domain.
|
|
205
|
-
- If the new message mentions the same person, event, product, or entity as the current task, it is SAME regardless of the angle
|
|
199
|
+
- If the topic domain clearly changed (e.g., server config → recipe, code review → vacation plan), choose NEW
|
|
206
200
|
- Different aspects of the SAME project/system are SAME (e.g., Nginx SSL → Nginx gzip = SAME)
|
|
207
|
-
-
|
|
208
|
-
-
|
|
209
|
-
-
|
|
210
|
-
- Examples: "配置Nginx" → "加gzip压缩" = SAME; "配置Nginx" → "做红烧肉" = NEW; "港股调研" → "那处理系统有哪些" = SAME; "部署服务器" → "年会安排" = NEW
|
|
201
|
+
- Different unrelated technologies discussed independently are NEW (e.g., Redis config → cooking recipe = NEW)
|
|
202
|
+
- When unsure, lean toward SAME for closely related topics, but do NOT hesitate to mark NEW for obvious domain shifts
|
|
203
|
+
- Examples: "配置Nginx" → "加gzip压缩" = SAME; "配置Nginx" → "做红烧肉" = NEW; "MySQL配置" → "K8s部署" in same infra project = SAME; "部署服务器" → "年会安排" = NEW
|
|
211
204
|
|
|
212
205
|
Output exactly one word: NEW or SAME`;
|
|
213
206
|
|
|
@@ -253,134 +246,6 @@ export async function judgeNewTopicOpenAI(
|
|
|
253
246
|
return answer.startsWith("NEW");
|
|
254
247
|
}
|
|
255
248
|
|
|
256
|
-
// ─── Structured Topic Classifier ───
|
|
257
|
-
|
|
258
|
-
export interface TopicClassifyResult {
|
|
259
|
-
decision: "NEW" | "SAME";
|
|
260
|
-
confidence: number;
|
|
261
|
-
boundaryType: string;
|
|
262
|
-
reason: string; // may be empty for compact responses
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
const TOPIC_CLASSIFIER_PROMPT = `Classify if NEW MESSAGE continues current task or starts an unrelated one.
|
|
266
|
-
Output ONLY JSON: {"d":"S"|"N","c":0.0-1.0}
|
|
267
|
-
d=S(same) or N(new). c=confidence. Default S. Only N if completely unrelated domain.
|
|
268
|
-
Sub-questions, tools, methods, details of current topic = S.`;
|
|
269
|
-
|
|
270
|
-
export async function classifyTopicOpenAI(
|
|
271
|
-
taskState: string,
|
|
272
|
-
newMessage: string,
|
|
273
|
-
cfg: SummarizerConfig,
|
|
274
|
-
log: Logger,
|
|
275
|
-
): Promise<TopicClassifyResult> {
|
|
276
|
-
const endpoint = normalizeChatEndpoint(cfg.endpoint ?? "https://api.openai.com/v1/chat/completions");
|
|
277
|
-
const model = cfg.model ?? "gpt-4o-mini";
|
|
278
|
-
const headers: Record<string, string> = {
|
|
279
|
-
"Content-Type": "application/json",
|
|
280
|
-
Authorization: `Bearer ${cfg.apiKey}`,
|
|
281
|
-
...cfg.headers,
|
|
282
|
-
};
|
|
283
|
-
|
|
284
|
-
const userContent = `TASK:\n${taskState}\n\nMSG:\n${newMessage}`;
|
|
285
|
-
|
|
286
|
-
const resp = await fetch(endpoint, {
|
|
287
|
-
method: "POST",
|
|
288
|
-
headers,
|
|
289
|
-
body: JSON.stringify(buildRequestBody(cfg, {
|
|
290
|
-
model,
|
|
291
|
-
temperature: 0,
|
|
292
|
-
max_tokens: 60,
|
|
293
|
-
messages: [
|
|
294
|
-
{ role: "system", content: TOPIC_CLASSIFIER_PROMPT },
|
|
295
|
-
{ role: "user", content: userContent },
|
|
296
|
-
],
|
|
297
|
-
})),
|
|
298
|
-
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
299
|
-
});
|
|
300
|
-
|
|
301
|
-
if (!resp.ok) {
|
|
302
|
-
const body = await resp.text();
|
|
303
|
-
throw new Error(`OpenAI topic-classifier failed (${resp.status}): ${body}`);
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
const json = (await resp.json()) as { choices: Array<{ message: { content: string } }> };
|
|
307
|
-
const raw = json.choices[0]?.message?.content?.trim() ?? "";
|
|
308
|
-
log.debug(`Topic classifier raw: "${raw}"`);
|
|
309
|
-
|
|
310
|
-
return parseTopicClassifyResult(raw, log);
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
const TOPIC_ARBITRATION_PROMPT = `A classifier flagged this message as possibly new topic (low confidence). Is it truly UNRELATED, or a sub-question/follow-up?
|
|
314
|
-
Tools/methods/details of current task = SAME. Shared entity/theme = SAME. Entirely different domain = NEW.
|
|
315
|
-
Reply one word: NEW or SAME`;
|
|
316
|
-
|
|
317
|
-
export async function arbitrateTopicSplitOpenAI(
|
|
318
|
-
taskState: string,
|
|
319
|
-
newMessage: string,
|
|
320
|
-
cfg: SummarizerConfig,
|
|
321
|
-
log: Logger,
|
|
322
|
-
): Promise<string> {
|
|
323
|
-
const endpoint = normalizeChatEndpoint(cfg.endpoint ?? "https://api.openai.com/v1/chat/completions");
|
|
324
|
-
const model = cfg.model ?? "gpt-4o-mini";
|
|
325
|
-
const headers: Record<string, string> = {
|
|
326
|
-
"Content-Type": "application/json",
|
|
327
|
-
Authorization: `Bearer ${cfg.apiKey}`,
|
|
328
|
-
...cfg.headers,
|
|
329
|
-
};
|
|
330
|
-
|
|
331
|
-
const userContent = `TASK:\n${taskState}\n\nMSG:\n${newMessage}`;
|
|
332
|
-
|
|
333
|
-
const resp = await fetch(endpoint, {
|
|
334
|
-
method: "POST",
|
|
335
|
-
headers,
|
|
336
|
-
body: JSON.stringify(buildRequestBody(cfg, {
|
|
337
|
-
model,
|
|
338
|
-
temperature: 0,
|
|
339
|
-
max_tokens: 10,
|
|
340
|
-
messages: [
|
|
341
|
-
{ role: "system", content: TOPIC_ARBITRATION_PROMPT },
|
|
342
|
-
{ role: "user", content: userContent },
|
|
343
|
-
],
|
|
344
|
-
})),
|
|
345
|
-
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
346
|
-
});
|
|
347
|
-
|
|
348
|
-
if (!resp.ok) {
|
|
349
|
-
const body = await resp.text();
|
|
350
|
-
throw new Error(`OpenAI topic-arbitration failed (${resp.status}): ${body}`);
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
const json = (await resp.json()) as { choices: Array<{ message: { content: string } }> };
|
|
354
|
-
const answer = json.choices[0]?.message?.content?.trim().toUpperCase() ?? "";
|
|
355
|
-
log.debug(`Topic arbitration result: "${answer}"`);
|
|
356
|
-
return answer.startsWith("NEW") ? "NEW" : "SAME";
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
export function parseTopicClassifyResult(raw: string, log: Logger): TopicClassifyResult {
|
|
360
|
-
try {
|
|
361
|
-
const jsonMatch = raw.match(/\{[\s\S]*\}/);
|
|
362
|
-
if (jsonMatch) {
|
|
363
|
-
const p = JSON.parse(jsonMatch[0]);
|
|
364
|
-
const decision: "NEW" | "SAME" =
|
|
365
|
-
(p.d === "N" || p.decision === "NEW") ? "NEW" : "SAME";
|
|
366
|
-
const confidence: number =
|
|
367
|
-
typeof p.c === "number" ? p.c : typeof p.confidence === "number" ? p.confidence : 0.5;
|
|
368
|
-
return {
|
|
369
|
-
decision,
|
|
370
|
-
confidence,
|
|
371
|
-
boundaryType: p.boundaryType || "",
|
|
372
|
-
reason: p.reason || "",
|
|
373
|
-
};
|
|
374
|
-
}
|
|
375
|
-
} catch (err) {
|
|
376
|
-
log.debug(`Failed to parse topic classify JSON: ${err}`);
|
|
377
|
-
}
|
|
378
|
-
const upper = raw.toUpperCase();
|
|
379
|
-
if (upper.startsWith("NEW") || upper.startsWith("N"))
|
|
380
|
-
return { decision: "NEW", confidence: 0.5, boundaryType: "", reason: "parse fallback" };
|
|
381
|
-
return { decision: "SAME", confidence: 0.5, boundaryType: "", reason: "parse fallback" };
|
|
382
|
-
}
|
|
383
|
-
|
|
384
249
|
const FILTER_RELEVANT_PROMPT = `You are a memory relevance judge.
|
|
385
250
|
|
|
386
251
|
Given a QUERY and CANDIDATE memories, decide: does each candidate's content contain information that would HELP ANSWER the query?
|
|
@@ -51,9 +51,6 @@ export class TaskProcessor {
|
|
|
51
51
|
* Determines if a new task boundary was crossed and handles transition.
|
|
52
52
|
*/
|
|
53
53
|
async onChunksIngested(sessionKey: string, latestTimestamp: number, owner?: string): Promise<void> {
|
|
54
|
-
if (sessionKey.startsWith("temp:") || sessionKey.startsWith("internal:") || sessionKey.startsWith("system:")) {
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
54
|
const resolvedOwner = owner ?? "agent:main";
|
|
58
55
|
this.ctx.log.debug(`TaskProcessor.onChunksIngested called session=${sessionKey} ts=${latestTimestamp} owner=${resolvedOwner} processing=${this.processing}`);
|
|
59
56
|
this.pendingEvents.push({ sessionKey, latestTimestamp, owner: resolvedOwner });
|
|
@@ -82,19 +79,13 @@ export class TaskProcessor {
|
|
|
82
79
|
}
|
|
83
80
|
}
|
|
84
81
|
|
|
85
|
-
private static extractAgentPrefix(sessionKey: string): string {
|
|
86
|
-
const parts = sessionKey.split(":");
|
|
87
|
-
return parts.length >= 3 ? parts.slice(0, 3).join(":") : sessionKey;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
82
|
private async detectAndProcess(sessionKey: string, latestTimestamp: number, owner: string): Promise<void> {
|
|
91
83
|
this.ctx.log.debug(`TaskProcessor.detectAndProcess session=${sessionKey} owner=${owner}`);
|
|
92
84
|
|
|
93
|
-
const currentAgentPrefix = TaskProcessor.extractAgentPrefix(sessionKey);
|
|
94
85
|
const allActive = this.store.getAllActiveTasks(owner);
|
|
95
86
|
for (const t of allActive) {
|
|
96
|
-
if (t.sessionKey !== sessionKey
|
|
97
|
-
this.ctx.log.info(`Session changed
|
|
87
|
+
if (t.sessionKey !== sessionKey) {
|
|
88
|
+
this.ctx.log.info(`Session changed: finalizing task=${t.id} from session=${t.sessionKey} (owner=${owner})`);
|
|
98
89
|
await this.finalizeTask(t);
|
|
99
90
|
}
|
|
100
91
|
}
|
|
@@ -188,36 +179,26 @@ export class TaskProcessor {
|
|
|
188
179
|
continue;
|
|
189
180
|
}
|
|
190
181
|
|
|
191
|
-
//
|
|
192
|
-
const
|
|
182
|
+
// LLM topic judgment — check this single user message against full task context
|
|
183
|
+
const context = this.buildContextSummary(currentTaskChunks);
|
|
193
184
|
const newMsg = userChunk.content.slice(0, 500);
|
|
194
|
-
this.ctx.log.info(`Topic
|
|
195
|
-
const
|
|
196
|
-
this.ctx.log.info(`Topic
|
|
185
|
+
this.ctx.log.info(`Topic judge: "${newMsg.slice(0, 60)}" vs ${existingUserCount} user turns`);
|
|
186
|
+
const isNew = await this.summarizer.judgeNewTopic(context, newMsg);
|
|
187
|
+
this.ctx.log.info(`Topic judge result: ${isNew === null ? "null(fallback)" : isNew ? "NEW" : "SAME"}`);
|
|
197
188
|
|
|
198
|
-
if (
|
|
189
|
+
if (isNew === null) {
|
|
199
190
|
this.assignChunksToTask(turn, currentTask.id);
|
|
200
191
|
currentTaskChunks = currentTaskChunks.concat(turn);
|
|
201
192
|
continue;
|
|
202
193
|
}
|
|
203
194
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
this.
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
if (!secondResult || secondResult !== "NEW") {
|
|
210
|
-
this.assignChunksToTask(turn, currentTask.id);
|
|
211
|
-
currentTaskChunks = currentTaskChunks.concat(turn);
|
|
212
|
-
continue;
|
|
213
|
-
}
|
|
195
|
+
if (isNew) {
|
|
196
|
+
this.ctx.log.info(`Task boundary at turn ${i}: LLM judged new topic. Msg: "${newMsg.slice(0, 80)}..."`);
|
|
197
|
+
await this.finalizeTask(currentTask);
|
|
198
|
+
currentTask = await this.createNewTaskReturn(sessionKey, userChunk.createdAt, owner);
|
|
199
|
+
currentTaskChunks = [];
|
|
214
200
|
}
|
|
215
201
|
|
|
216
|
-
this.ctx.log.info(`Task boundary at turn ${i}: classifier judged NEW (confidence=${result.confidence}). Msg: "${newMsg.slice(0, 80)}..."`);
|
|
217
|
-
await this.finalizeTask(currentTask);
|
|
218
|
-
currentTask = await this.createNewTaskReturn(sessionKey, userChunk.createdAt, owner);
|
|
219
|
-
currentTaskChunks = [];
|
|
220
|
-
|
|
221
202
|
this.assignChunksToTask(turn, currentTask.id);
|
|
222
203
|
currentTaskChunks = currentTaskChunks.concat(turn);
|
|
223
204
|
}
|
|
@@ -245,39 +226,38 @@ export class TaskProcessor {
|
|
|
245
226
|
}
|
|
246
227
|
|
|
247
228
|
/**
|
|
248
|
-
* Build
|
|
249
|
-
* Includes
|
|
250
|
-
*
|
|
229
|
+
* Build context from existing task chunks for the LLM topic judge.
|
|
230
|
+
* Includes both the task's opening topic and recent exchanges,
|
|
231
|
+
* so the LLM understands both what the task was originally about
|
|
232
|
+
* and where the conversation currently is.
|
|
233
|
+
*
|
|
234
|
+
* For user messages, include full content (up to 500 chars) since
|
|
235
|
+
* they carry the topic signal. For assistant messages, use summary
|
|
236
|
+
* or truncated content since they mostly elaborate.
|
|
251
237
|
*/
|
|
252
|
-
private
|
|
253
|
-
const
|
|
254
|
-
if (
|
|
255
|
-
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
const u = conv[j].summary || conv[j].content.slice(0, 60);
|
|
263
|
-
const nextA = conv[j + 1]?.role === "assistant" ? conv[j + 1] : null;
|
|
264
|
-
const a = nextA ? (nextA.summary || nextA.content.slice(0, 60)) : "";
|
|
265
|
-
turns.push({ u, a });
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
const recent = turns.slice(-3);
|
|
270
|
-
const turnLines = recent.map((t, i) => `${i + 1}. U:${t.u} A:${t.a}`);
|
|
238
|
+
private buildContextSummary(chunks: Chunk[]): string {
|
|
239
|
+
const conversational = chunks.filter((c) => c.role === "user" || c.role === "assistant");
|
|
240
|
+
if (conversational.length === 0) return "";
|
|
241
|
+
|
|
242
|
+
const formatChunk = (c: Chunk) => {
|
|
243
|
+
const label = c.role === "user" ? "User" : "Assistant";
|
|
244
|
+
const maxLen = c.role === "user" ? 500 : 200;
|
|
245
|
+
const text = c.summary || c.content.slice(0, maxLen);
|
|
246
|
+
return `[${label}]: ${text}`;
|
|
247
|
+
};
|
|
271
248
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
const lastA = [...conv].reverse().find((c) => c.role === "assistant");
|
|
275
|
-
if (lastA) snippet = lastA.content.slice(0, 200);
|
|
249
|
+
if (conversational.length <= 10) {
|
|
250
|
+
return conversational.map(formatChunk).join("\n");
|
|
276
251
|
}
|
|
277
252
|
|
|
278
|
-
const
|
|
279
|
-
|
|
280
|
-
return
|
|
253
|
+
const opening = conversational.slice(0, 6).map(formatChunk);
|
|
254
|
+
const recent = conversational.slice(-4).map(formatChunk);
|
|
255
|
+
return [
|
|
256
|
+
"--- Task opening ---",
|
|
257
|
+
...opening,
|
|
258
|
+
"--- Recent exchanges ---",
|
|
259
|
+
...recent,
|
|
260
|
+
].join("\n");
|
|
281
261
|
}
|
|
282
262
|
|
|
283
263
|
private async createNewTaskReturn(sessionKey: string, timestamp: number, owner: string = "agent:main"): Promise<Task> {
|
package/src/ingest/worker.ts
CHANGED
|
@@ -25,14 +25,8 @@ export class IngestWorker {
|
|
|
25
25
|
|
|
26
26
|
getTaskProcessor(): TaskProcessor { return this.taskProcessor; }
|
|
27
27
|
|
|
28
|
-
private static isEphemeralSession(sessionKey: string): boolean {
|
|
29
|
-
return sessionKey.startsWith("temp:") || sessionKey.startsWith("internal:") || sessionKey.startsWith("system:");
|
|
30
|
-
}
|
|
31
|
-
|
|
32
28
|
enqueue(messages: ConversationMessage[]): void {
|
|
33
|
-
|
|
34
|
-
if (filtered.length === 0) return;
|
|
35
|
-
this.queue.push(...filtered);
|
|
29
|
+
this.queue.push(...messages);
|
|
36
30
|
if (!this.processing) {
|
|
37
31
|
this.processQueue().catch((err) => {
|
|
38
32
|
this.ctx.log.error(`Ingest worker error: ${err}`);
|
|
@@ -156,23 +150,14 @@ export class IngestWorker {
|
|
|
156
150
|
let mergeHistory = "[]";
|
|
157
151
|
|
|
158
152
|
// Fast path: exact content_hash match within same owner (agent dimension)
|
|
159
|
-
// Strategy: retire the OLD chunk, keep the NEW one active (latest wins)
|
|
160
153
|
const chunkOwner = msg.owner ?? "agent:main";
|
|
161
154
|
const existingByHash = this.store.findActiveChunkByHash(content, chunkOwner);
|
|
162
155
|
if (existingByHash) {
|
|
163
|
-
this.ctx.log.debug(`Exact-dup (owner=${chunkOwner}): hash match →
|
|
156
|
+
this.ctx.log.debug(`Exact-dup (owner=${chunkOwner}): hash match → existing=${existingByHash}`);
|
|
164
157
|
this.store.recordMergeHit(existingByHash, "DUPLICATE", "exact content hash match");
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
this.store.deleteEmbedding(existingByHash);
|
|
168
|
-
mergedFromOld = existingByHash;
|
|
158
|
+
dedupStatus = "duplicate";
|
|
159
|
+
dedupTarget = existingByHash;
|
|
169
160
|
dedupReason = "exact content hash match";
|
|
170
|
-
if (oldChunk) {
|
|
171
|
-
const oldHistory = JSON.parse(oldChunk.mergeHistory || "[]");
|
|
172
|
-
oldHistory.push({ action: "duplicate_superseded", at: Date.now(), reason: "exact content hash match", sourceChunkId: existingByHash });
|
|
173
|
-
mergeHistory = JSON.stringify(oldHistory);
|
|
174
|
-
mergeCount = (oldChunk.mergeCount || 0) + 1;
|
|
175
|
-
}
|
|
176
161
|
}
|
|
177
162
|
|
|
178
163
|
// Smart dedup: find Top-5 similar chunks, then ask LLM to judge
|
|
@@ -188,9 +173,8 @@ export class IngestWorker {
|
|
|
188
173
|
index: i + 1,
|
|
189
174
|
summary: chunk?.summary ?? "",
|
|
190
175
|
chunkId: s.chunkId,
|
|
191
|
-
role: chunk?.role,
|
|
192
176
|
};
|
|
193
|
-
}).filter(c => c.summary
|
|
177
|
+
}).filter(c => c.summary);
|
|
194
178
|
|
|
195
179
|
if (candidates.length > 0) {
|
|
196
180
|
const dedupResult = await this.summarizer.judgeDedup(summary, candidates);
|
|
@@ -199,18 +183,10 @@ export class IngestWorker {
|
|
|
199
183
|
const targetChunkId = candidates[dedupResult.targetIndex - 1]?.chunkId;
|
|
200
184
|
if (targetChunkId) {
|
|
201
185
|
this.store.recordMergeHit(targetChunkId, "DUPLICATE", dedupResult.reason);
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
this.store.deleteEmbedding(targetChunkId);
|
|
205
|
-
mergedFromOld = targetChunkId;
|
|
186
|
+
dedupStatus = "duplicate";
|
|
187
|
+
dedupTarget = targetChunkId;
|
|
206
188
|
dedupReason = dedupResult.reason;
|
|
207
|
-
|
|
208
|
-
const oldHistory = JSON.parse(oldChunk.mergeHistory || "[]");
|
|
209
|
-
oldHistory.push({ action: "duplicate_superseded", at: Date.now(), reason: dedupResult.reason, sourceChunkId: targetChunkId });
|
|
210
|
-
mergeHistory = JSON.stringify(oldHistory);
|
|
211
|
-
mergeCount = (oldChunk.mergeCount || 0) + 1;
|
|
212
|
-
}
|
|
213
|
-
this.ctx.log.debug(`Smart dedup: DUPLICATE → retiring old=${targetChunkId}, keeping new=${chunkId} active, reason: ${dedupResult.reason}`);
|
|
189
|
+
this.ctx.log.debug(`Smart dedup: DUPLICATE → target=${targetChunkId}, storing with status=duplicate, reason: ${dedupResult.reason}`);
|
|
214
190
|
}
|
|
215
191
|
}
|
|
216
192
|
|
|
@@ -290,6 +266,9 @@ export class IngestWorker {
|
|
|
290
266
|
}
|
|
291
267
|
this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} dedup=${dedupStatus} len=${content.length} hasVec=${!!embedding && dedupStatus === "active"}`);
|
|
292
268
|
|
|
269
|
+
if (dedupStatus === "duplicate") {
|
|
270
|
+
return { action: "duplicate", summary, targetChunkId: dedupTarget ?? undefined, reason: dedupReason ?? undefined };
|
|
271
|
+
}
|
|
293
272
|
if (mergedFromOld) {
|
|
294
273
|
return { action: "merged", chunkId, summary, targetChunkId: mergedFromOld, reason: dedupReason ?? undefined };
|
|
295
274
|
}
|
package/src/recall/engine.ts
CHANGED
|
@@ -77,7 +77,7 @@ export class RecallEngine {
|
|
|
77
77
|
}
|
|
78
78
|
const shortTerms = [...new Set([...spaceSplit, ...cjkBigrams])];
|
|
79
79
|
const patternHits = shortTerms.length > 0
|
|
80
|
-
? this.store.patternSearch(shortTerms, { limit: candidatePool
|
|
80
|
+
? this.store.patternSearch(shortTerms, { limit: candidatePool })
|
|
81
81
|
: [];
|
|
82
82
|
const patternRanked = patternHits.map((h, i) => ({
|
|
83
83
|
id: h.chunkId,
|
|
@@ -234,7 +234,6 @@ export class RecallEngine {
|
|
|
234
234
|
score: Math.round(candidate.score * 1000) / 1000,
|
|
235
235
|
taskId: chunk.taskId,
|
|
236
236
|
skillId: chunk.skillId,
|
|
237
|
-
owner: chunk.owner,
|
|
238
237
|
origin: chunk.owner === "public" ? "local-shared" : "local",
|
|
239
238
|
source: {
|
|
240
239
|
ts: chunk.createdAt,
|