@memtensor/memos-local-openclaw-plugin 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -5
- package/README.md +283 -91
- package/dist/capture/index.d.ts +5 -7
- package/dist/capture/index.d.ts.map +1 -1
- package/dist/capture/index.js +72 -43
- package/dist/capture/index.js.map +1 -1
- package/dist/ingest/dedup.d.ts +8 -0
- package/dist/ingest/dedup.d.ts.map +1 -1
- package/dist/ingest/dedup.js +21 -0
- package/dist/ingest/dedup.js.map +1 -1
- package/dist/ingest/providers/anthropic.d.ts +16 -0
- package/dist/ingest/providers/anthropic.d.ts.map +1 -1
- package/dist/ingest/providers/anthropic.js +214 -1
- package/dist/ingest/providers/anthropic.js.map +1 -1
- package/dist/ingest/providers/bedrock.d.ts +16 -5
- package/dist/ingest/providers/bedrock.d.ts.map +1 -1
- package/dist/ingest/providers/bedrock.js +210 -6
- package/dist/ingest/providers/bedrock.js.map +1 -1
- package/dist/ingest/providers/gemini.d.ts +16 -0
- package/dist/ingest/providers/gemini.d.ts.map +1 -1
- package/dist/ingest/providers/gemini.js +202 -1
- package/dist/ingest/providers/gemini.js.map +1 -1
- package/dist/ingest/providers/index.d.ts +31 -0
- package/dist/ingest/providers/index.d.ts.map +1 -1
- package/dist/ingest/providers/index.js +134 -4
- package/dist/ingest/providers/index.js.map +1 -1
- package/dist/ingest/providers/openai.d.ts +24 -0
- package/dist/ingest/providers/openai.d.ts.map +1 -1
- package/dist/ingest/providers/openai.js +255 -1
- package/dist/ingest/providers/openai.js.map +1 -1
- package/dist/ingest/task-processor.d.ts +65 -0
- package/dist/ingest/task-processor.d.ts.map +1 -0
- package/dist/ingest/task-processor.js +354 -0
- package/dist/ingest/task-processor.js.map +1 -0
- package/dist/ingest/worker.d.ts +3 -1
- package/dist/ingest/worker.d.ts.map +1 -1
- package/dist/ingest/worker.js +131 -23
- package/dist/ingest/worker.js.map +1 -1
- package/dist/recall/engine.d.ts +1 -0
- package/dist/recall/engine.d.ts.map +1 -1
- package/dist/recall/engine.js +22 -11
- package/dist/recall/engine.js.map +1 -1
- package/dist/recall/mmr.d.ts.map +1 -1
- package/dist/recall/mmr.js +3 -1
- package/dist/recall/mmr.js.map +1 -1
- package/dist/skill/bundled-memory-guide.d.ts +6 -0
- package/dist/skill/bundled-memory-guide.d.ts.map +1 -0
- package/dist/skill/bundled-memory-guide.js +95 -0
- package/dist/skill/bundled-memory-guide.js.map +1 -0
- package/dist/skill/evaluator.d.ts +31 -0
- package/dist/skill/evaluator.d.ts.map +1 -0
- package/dist/skill/evaluator.js +194 -0
- package/dist/skill/evaluator.js.map +1 -0
- package/dist/skill/evolver.d.ts +22 -0
- package/dist/skill/evolver.d.ts.map +1 -0
- package/dist/skill/evolver.js +193 -0
- package/dist/skill/evolver.js.map +1 -0
- package/dist/skill/generator.d.ts +25 -0
- package/dist/skill/generator.d.ts.map +1 -0
- package/dist/skill/generator.js +477 -0
- package/dist/skill/generator.js.map +1 -0
- package/dist/skill/installer.d.ts +16 -0
- package/dist/skill/installer.d.ts.map +1 -0
- package/dist/skill/installer.js +89 -0
- package/dist/skill/installer.js.map +1 -0
- package/dist/skill/upgrader.d.ts +19 -0
- package/dist/skill/upgrader.d.ts.map +1 -0
- package/dist/skill/upgrader.js +263 -0
- package/dist/skill/upgrader.js.map +1 -0
- package/dist/skill/validator.d.ts +29 -0
- package/dist/skill/validator.d.ts.map +1 -0
- package/dist/skill/validator.js +227 -0
- package/dist/skill/validator.js.map +1 -0
- package/dist/storage/sqlite.d.ts +141 -1
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +664 -7
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/types.d.ts +93 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +8 -0
- package/dist/types.js.map +1 -1
- package/dist/viewer/html.d.ts +1 -1
- package/dist/viewer/html.d.ts.map +1 -1
- package/dist/viewer/html.js +2391 -159
- package/dist/viewer/html.js.map +1 -1
- package/dist/viewer/server.d.ts +16 -0
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +346 -3
- package/dist/viewer/server.js.map +1 -1
- package/index.ts +572 -89
- package/openclaw.plugin.json +20 -45
- package/package.json +3 -4
- package/skill/memos-memory-guide/SKILL.md +86 -0
- package/src/capture/index.ts +85 -45
- package/src/ingest/dedup.ts +29 -0
- package/src/ingest/providers/anthropic.ts +258 -1
- package/src/ingest/providers/bedrock.ts +256 -6
- package/src/ingest/providers/gemini.ts +252 -1
- package/src/ingest/providers/index.ts +156 -8
- package/src/ingest/providers/openai.ts +304 -1
- package/src/ingest/task-processor.ts +396 -0
- package/src/ingest/worker.ts +145 -34
- package/src/recall/engine.ts +23 -12
- package/src/recall/mmr.ts +3 -1
- package/src/skill/bundled-memory-guide.ts +91 -0
- package/src/skill/evaluator.ts +220 -0
- package/src/skill/evolver.ts +169 -0
- package/src/skill/generator.ts +506 -0
- package/src/skill/installer.ts +59 -0
- package/src/skill/upgrader.ts +257 -0
- package/src/skill/validator.ts +227 -0
- package/src/storage/sqlite.ts +802 -7
- package/src/types.ts +96 -0
- package/src/viewer/html.ts +2391 -159
- package/src/viewer/server.ts +346 -3
- package/SKILL.md +0 -43
- package/www/index.html +0 -632
package/dist/capture/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.captureMessages = captureMessages;
|
|
4
|
-
const types_1 = require("../types");
|
|
5
4
|
const SKIP_ROLES = new Set(["system"]);
|
|
6
5
|
const SELF_TOOLS = new Set([
|
|
7
6
|
"memory_search",
|
|
@@ -9,16 +8,25 @@ const SELF_TOOLS = new Set([
|
|
|
9
8
|
"memory_get",
|
|
10
9
|
"memory_viewer",
|
|
11
10
|
]);
|
|
11
|
+
// OpenClaw inbound metadata sentinels — these are AI-facing prefixes,
|
|
12
|
+
// not user content. Must be stripped before storing as memory.
|
|
13
|
+
const INBOUND_META_SENTINELS = [
|
|
14
|
+
"Conversation info (untrusted metadata):",
|
|
15
|
+
"Sender (untrusted metadata):",
|
|
16
|
+
"Thread starter (untrusted, for context):",
|
|
17
|
+
"Replied message (untrusted, for context):",
|
|
18
|
+
"Forwarded message context (untrusted metadata):",
|
|
19
|
+
"Chat history since last reply (untrusted, for context):",
|
|
20
|
+
];
|
|
21
|
+
const SENTINEL_FAST_RE = new RegExp(INBOUND_META_SENTINELS.map(s => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"));
|
|
12
22
|
/**
|
|
13
|
-
*
|
|
23
|
+
* Extract writable messages from a conversation turn.
|
|
14
24
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* - Truncate long tool results to avoid storage bloat
|
|
19
|
-
* - Strip injected evidence blocks wrapped in [STORED_MEMORY]...[/STORED_MEMORY]
|
|
25
|
+
* Stores the user's actual text — strips only OpenClaw's injected metadata
|
|
26
|
+
* prefixes (Sender info, conversation context, etc.) which are not user content.
|
|
27
|
+
* Only skips: system prompts and our own memory tool results (prevents loop).
|
|
20
28
|
*/
|
|
21
|
-
function captureMessages(messages, sessionKey, turnId,
|
|
29
|
+
function captureMessages(messages, sessionKey, turnId, _evidenceTag, log) {
|
|
22
30
|
const now = Date.now();
|
|
23
31
|
const result = [];
|
|
24
32
|
for (const msg of messages) {
|
|
@@ -27,54 +35,75 @@ function captureMessages(messages, sessionKey, turnId, evidenceTag, log) {
|
|
|
27
35
|
continue;
|
|
28
36
|
if (!msg.content || msg.content.trim().length === 0)
|
|
29
37
|
continue;
|
|
30
|
-
if (role === "tool") {
|
|
31
|
-
|
|
32
|
-
log.debug(`Skipping self-tool result: ${msg.toolName}`);
|
|
33
|
-
continue;
|
|
34
|
-
}
|
|
35
|
-
let content = msg.content.trim();
|
|
36
|
-
const maxChars = types_1.DEFAULTS.toolResultMaxChars;
|
|
37
|
-
if (content.length > maxChars) {
|
|
38
|
-
content = content.slice(0, maxChars) + `\n\n[truncated — original ${content.length} chars]`;
|
|
39
|
-
}
|
|
40
|
-
const toolLabel = msg.toolName ? `[tool:${msg.toolName}] ` : "[tool] ";
|
|
41
|
-
result.push({
|
|
42
|
-
role: "tool",
|
|
43
|
-
content: toolLabel + content,
|
|
44
|
-
timestamp: now,
|
|
45
|
-
turnId,
|
|
46
|
-
sessionKey,
|
|
47
|
-
toolName: msg.toolName,
|
|
48
|
-
});
|
|
38
|
+
if (role === "tool" && msg.toolName && SELF_TOOLS.has(msg.toolName)) {
|
|
39
|
+
log.debug(`Skipping self-tool result: ${msg.toolName}`);
|
|
49
40
|
continue;
|
|
50
41
|
}
|
|
51
|
-
|
|
52
|
-
if (
|
|
42
|
+
let content = msg.content;
|
|
43
|
+
if (role === "user") {
|
|
44
|
+
content = stripInboundMetadata(content);
|
|
45
|
+
}
|
|
46
|
+
if (!content.trim())
|
|
53
47
|
continue;
|
|
54
48
|
result.push({
|
|
55
49
|
role,
|
|
56
|
-
content
|
|
50
|
+
content,
|
|
57
51
|
timestamp: now,
|
|
58
52
|
turnId,
|
|
59
53
|
sessionKey,
|
|
54
|
+
toolName: role === "tool" ? msg.toolName : undefined,
|
|
60
55
|
});
|
|
61
56
|
}
|
|
62
57
|
log.debug(`Captured ${result.length}/${messages.length} messages for session=${sessionKey} turn=${turnId}`);
|
|
63
58
|
return result;
|
|
64
59
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
60
|
+
/**
|
|
61
|
+
* Strip OpenClaw-injected inbound metadata blocks from user messages.
|
|
62
|
+
*
|
|
63
|
+
* These blocks have the shape:
|
|
64
|
+
* Sender (untrusted metadata):
|
|
65
|
+
* ```json
|
|
66
|
+
* { "label": "...", "id": "..." }
|
|
67
|
+
* ```
|
|
68
|
+
*
|
|
69
|
+
* Also strips the envelope timestamp prefix like "[Tue 2026-03-03 21:58 GMT+8] "
|
|
70
|
+
*/
|
|
71
|
+
function stripInboundMetadata(text) {
|
|
72
|
+
if (!SENTINEL_FAST_RE.test(text))
|
|
73
|
+
return text;
|
|
74
|
+
const lines = text.split("\n");
|
|
75
|
+
const result = [];
|
|
76
|
+
let inMetaBlock = false;
|
|
77
|
+
let inFencedJson = false;
|
|
78
|
+
for (let i = 0; i < lines.length; i++) {
|
|
79
|
+
const line = lines[i];
|
|
80
|
+
const trimmed = line.trim();
|
|
81
|
+
if (!inMetaBlock && INBOUND_META_SENTINELS.some(s => s === trimmed)) {
|
|
82
|
+
if (lines[i + 1]?.trim() === "```json") {
|
|
83
|
+
inMetaBlock = true;
|
|
84
|
+
inFencedJson = false;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
// Sentinel without fenced JSON — skip this line only
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (inMetaBlock) {
|
|
91
|
+
if (!inFencedJson && trimmed === "```json") {
|
|
92
|
+
inFencedJson = true;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
if (inFencedJson && trimmed === "```") {
|
|
96
|
+
inMetaBlock = false;
|
|
97
|
+
inFencedJson = false;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
result.push(line);
|
|
77
103
|
}
|
|
78
|
-
|
|
104
|
+
let cleaned = result.join("\n").trim();
|
|
105
|
+
// Strip envelope timestamp prefix: "[Tue 2026-03-03 21:58 GMT+8] actual message"
|
|
106
|
+
cleaned = cleaned.replace(/^\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}(?::\d{2})?\s+[A-Z]{3}[+-]\d{1,2}\]\s*/, "");
|
|
107
|
+
return cleaned;
|
|
79
108
|
}
|
|
80
109
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/capture/index.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/capture/index.ts"],"names":[],"mappings":";;AAiCA,0CAsCC;AArED,MAAM,UAAU,GAAc,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAElD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,eAAe;IACf,iBAAiB;IACjB,YAAY;IACZ,eAAe;CAChB,CAAC,CAAC;AAEH,sEAAsE;AACtE,+DAA+D;AAC/D,MAAM,sBAAsB,GAAG;IAC7B,yCAAyC;IACzC,8BAA8B;IAC9B,0CAA0C;IAC1C,2CAA2C;IAC3C,iDAAiD;IACjD,yDAAyD;CAC1D,CAAC;AAEF,MAAM,gBAAgB,GAAG,IAAI,MAAM,CACjC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CACpF,CAAC;AAEF;;;;;;GAMG;AACH,SAAgB,eAAe,CAC7B,QAAqE,EACrE,UAAkB,EAClB,MAAc,EACd,YAAoB,EACpB,GAAW;IAEX,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAY,CAAC;QAC9B,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QACnC,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE9D,IAAI,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpE,GAAG,CAAC,KAAK,CAAC,8BAA8B,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAC1B,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,OAAO,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;YAAE,SAAS;QAE9B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI;YACJ,OAAO;YACP,SAAS,EAAE,GAAG;YACd,MAAM;YACN,UAAU;YACV,QAAQ,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;SACrD,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,yBAAyB,UAAU,SAAS,MAAM,EAAE,CAAC,CAAC;IAC5G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,oBAAoB,CAAC,IAAY;IACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE9C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5B,IAAI,CAAC,WAAW,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,OAAO,CAAC,EAAE,CAAC;YACpE,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;gBACvC,WAAW,GAAG,IAAI,CAAC;gBACnB,YAAY,GAAG,KAAK,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,qDAAqD;YACrD,SAAS;QACX,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC,YAAY,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC3C,YAAY,GAAG,IAAI,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,IAAI,YAAY,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;gBACtC,WAAW,GAAG,KAAK,CAAC;gBACpB,YAAY,GAAG,KAAK,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAEvC,iFAAiF;IACjF,OAAO,GAAG,OAAO,CAAC,OAAO,CACvB,4GAA4G,EAC5G,EAAE,CACH,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/ingest/dedup.d.ts
CHANGED
|
@@ -8,4 +8,12 @@ import type { Logger } from "../types";
|
|
|
8
8
|
* PRD §4.4: dedup threshold 0.92–0.95
|
|
9
9
|
*/
|
|
10
10
|
export declare function findDuplicate(store: SqliteStore, newVec: number[], threshold: number, log: Logger): string | null;
|
|
11
|
+
/**
|
|
12
|
+
* Find Top-N most similar chunks above a threshold.
|
|
13
|
+
* Used for smart dedup: retrieve candidates, then ask LLM to judge.
|
|
14
|
+
*/
|
|
15
|
+
export declare function findTopSimilar(store: SqliteStore, newVec: number[], threshold: number, topN: number, log: Logger): Array<{
|
|
16
|
+
chunkId: string;
|
|
17
|
+
score: number;
|
|
18
|
+
}>;
|
|
11
19
|
//# sourceMappingURL=dedup.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/ingest/dedup.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,IAAI,CAoBf"}
|
|
1
|
+
{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/ingest/dedup.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,IAAI,CAoBf;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,GACV,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAiB3C"}
|
package/dist/ingest/dedup.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.findDuplicate = findDuplicate;
|
|
4
|
+
exports.findTopSimilar = findTopSimilar;
|
|
4
5
|
const vector_1 = require("../storage/vector");
|
|
5
6
|
/**
|
|
6
7
|
* Check if a new summary embedding is a near-duplicate of any
|
|
@@ -26,4 +27,24 @@ function findDuplicate(store, newVec, threshold, log) {
|
|
|
26
27
|
}
|
|
27
28
|
return null;
|
|
28
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* Find Top-N most similar chunks above a threshold.
|
|
32
|
+
* Used for smart dedup: retrieve candidates, then ask LLM to judge.
|
|
33
|
+
*/
|
|
34
|
+
function findTopSimilar(store, newVec, threshold, topN, log) {
|
|
35
|
+
const all = store.getAllEmbeddings();
|
|
36
|
+
const scored = [];
|
|
37
|
+
for (const { chunkId, vector } of all) {
|
|
38
|
+
const sim = (0, vector_1.cosineSimilarity)(newVec, vector);
|
|
39
|
+
if (sim >= threshold) {
|
|
40
|
+
scored.push({ chunkId, score: sim });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
scored.sort((a, b) => b.score - a.score);
|
|
44
|
+
const result = scored.slice(0, topN);
|
|
45
|
+
if (result.length > 0) {
|
|
46
|
+
log.debug(`findTopSimilar: found ${result.length} candidates above ${threshold} (best=${result[0].score.toFixed(4)})`);
|
|
47
|
+
}
|
|
48
|
+
return result;
|
|
49
|
+
}
|
|
29
50
|
//# sourceMappingURL=dedup.js.map
|
package/dist/ingest/dedup.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/ingest/dedup.ts"],"names":[],"mappings":";;AAWA,sCAyBC;
|
|
1
|
+
{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/ingest/dedup.ts"],"names":[],"mappings":";;AAWA,sCAyBC;AAMD,wCAuBC;AAjED,8CAAqD;AAIrD;;;;;;GAMG;AACH,SAAgB,aAAa,CAC3B,KAAkB,EAClB,MAAgB,EAChB,SAAiB,EACjB,GAAW;IAEX,MAAM,GAAG,GAAG,KAAK,CAAC,gBAAgB,EAAE,CAAC;IAErC,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAA,yBAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7C,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;YACpB,SAAS,GAAG,GAAG,CAAC;YAChB,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;QACrC,GAAG,CAAC,KAAK,CAAC,gCAAgC,MAAM,QAAQ,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAgB,cAAc,CAC5B,KAAkB,EAClB,MAAgB,EAChB,SAAiB,EACjB,IAAY,EACZ,GAAW;IAEX,MAAM,GAAG,GAAG,KAAK,CAAC,gBAAgB,EAAE,CAAC;IACrC,MAAM,MAAM,GAA8C,EAAE,CAAC;IAE7D,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAA,yBAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7C,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,GAAG,CAAC,KAAK,CAAC,yBAAyB,MAAM,CAAC,MAAM,qBAAqB,SAAS,UAAU,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,3 +1,19 @@
|
|
|
1
1
|
import type { SummarizerConfig, Logger } from "../../types";
|
|
2
|
+
export declare function summarizeTaskAnthropic(text: string, cfg: SummarizerConfig, log: Logger): Promise<string>;
|
|
3
|
+
export declare function judgeNewTopicAnthropic(currentContext: string, newMessage: string, cfg: SummarizerConfig, log: Logger): Promise<boolean>;
|
|
4
|
+
import type { FilterResult } from "./openai";
|
|
5
|
+
export type { FilterResult } from "./openai";
|
|
6
|
+
export declare function filterRelevantAnthropic(query: string, candidates: Array<{
|
|
7
|
+
index: number;
|
|
8
|
+
summary: string;
|
|
9
|
+
role: string;
|
|
10
|
+
}>, cfg: SummarizerConfig, log: Logger): Promise<FilterResult>;
|
|
2
11
|
export declare function summarizeAnthropic(text: string, cfg: SummarizerConfig, log: Logger): Promise<string>;
|
|
12
|
+
import type { DedupResult } from "./openai";
|
|
13
|
+
export type { DedupResult } from "./openai";
|
|
14
|
+
export declare function judgeDedupAnthropic(newSummary: string, candidates: Array<{
|
|
15
|
+
index: number;
|
|
16
|
+
summary: string;
|
|
17
|
+
chunkId: string;
|
|
18
|
+
}>, cfg: SummarizerConfig, log: Logger): Promise<DedupResult>;
|
|
3
19
|
//# sourceMappingURL=anthropic.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../../src/ingest/providers/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../../src/ingest/providers/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAyC5D,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CA8BjB;AAeD,wBAAsB,sBAAsB,CAC1C,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,OAAO,CAAC,CAkClB;AAwBD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAC7C,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,EACnE,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,YAAY,CAAC,CAmCvB;AAmBD,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CAgCjB;AAKD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5C,wBAAsB,mBAAmB,CACvC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,EACtE,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,WAAW,CAAC,CAiCtB"}
|
|
@@ -1,7 +1,188 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.summarizeTaskAnthropic = summarizeTaskAnthropic;
|
|
4
|
+
exports.judgeNewTopicAnthropic = judgeNewTopicAnthropic;
|
|
5
|
+
exports.filterRelevantAnthropic = filterRelevantAnthropic;
|
|
3
6
|
exports.summarizeAnthropic = summarizeAnthropic;
|
|
4
|
-
|
|
7
|
+
exports.judgeDedupAnthropic = judgeDedupAnthropic;
|
|
8
|
+
const SYSTEM_PROMPT = `Summarize the text in ONE concise sentence (max 120 characters). IMPORTANT: Use the SAME language as the input text — if the input is Chinese, write Chinese; if English, write English. Preserve exact names, commands, error codes. No bullet points, no preamble — output only the sentence.`;
|
|
9
|
+
const TASK_SUMMARY_PROMPT = `You create a DETAILED task summary from a multi-turn conversation. This summary will be the ONLY record of this conversation, so it must preserve ALL important information.
|
|
10
|
+
|
|
11
|
+
CRITICAL LANGUAGE RULE: You MUST write in the SAME language as the user's messages. Chinese input → Chinese output. English input → English output. NEVER mix languages.
|
|
12
|
+
|
|
13
|
+
Output EXACTLY this structure:
|
|
14
|
+
|
|
15
|
+
📌 Title
|
|
16
|
+
A short, descriptive title (10-30 characters). Like a chat group name.
|
|
17
|
+
|
|
18
|
+
🎯 Goal
|
|
19
|
+
One sentence: what the user wanted to accomplish.
|
|
20
|
+
|
|
21
|
+
📋 Key Steps
|
|
22
|
+
- Describe each meaningful step in detail
|
|
23
|
+
- Include the ACTUAL content produced: code snippets, commands, config blocks, formulas, key paragraphs
|
|
24
|
+
- For code: include the function signature and core logic (up to ~30 lines per block), use fenced code blocks
|
|
25
|
+
- For configs: include the actual config values and structure
|
|
26
|
+
- For lists/instructions: include the actual items, not just "provided a list"
|
|
27
|
+
- Merge only truly trivial back-and-forth (like "ok" / "sure")
|
|
28
|
+
- Do NOT over-summarize: "provided a function" is BAD; show the actual function
|
|
29
|
+
|
|
30
|
+
✅ Result
|
|
31
|
+
What was the final outcome? Include the final version of any code/config/content produced.
|
|
32
|
+
|
|
33
|
+
💡 Key Details
|
|
34
|
+
- Decisions made, trade-offs discussed, caveats noted, alternative approaches mentioned
|
|
35
|
+
- Specific values: numbers, versions, thresholds, URLs, file paths, model names
|
|
36
|
+
- Omit this section only if there truly are no noteworthy details
|
|
37
|
+
|
|
38
|
+
RULES:
|
|
39
|
+
- This summary is a KNOWLEDGE BASE ENTRY, not a brief note. Be thorough.
|
|
40
|
+
- PRESERVE verbatim: code, commands, URLs, file paths, error messages, config values, version numbers, names, amounts
|
|
41
|
+
- DISCARD only: greetings, filler, the assistant explaining what it will do before doing it
|
|
42
|
+
- Replace secrets (API keys, tokens, passwords) with [REDACTED]
|
|
43
|
+
- Target length: 30-50% of the original conversation length. Longer conversations need longer summaries.
|
|
44
|
+
- Output summary only, no preamble.`;
|
|
45
|
+
async function summarizeTaskAnthropic(text, cfg, log) {
|
|
46
|
+
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
47
|
+
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
48
|
+
const headers = {
|
|
49
|
+
"Content-Type": "application/json",
|
|
50
|
+
"x-api-key": cfg.apiKey ?? "",
|
|
51
|
+
"anthropic-version": "2023-06-01",
|
|
52
|
+
...cfg.headers,
|
|
53
|
+
};
|
|
54
|
+
const resp = await fetch(endpoint, {
|
|
55
|
+
method: "POST",
|
|
56
|
+
headers,
|
|
57
|
+
body: JSON.stringify({
|
|
58
|
+
model,
|
|
59
|
+
max_tokens: 4096,
|
|
60
|
+
temperature: cfg.temperature ?? 0.1,
|
|
61
|
+
system: TASK_SUMMARY_PROMPT,
|
|
62
|
+
messages: [{ role: "user", content: text }],
|
|
63
|
+
}),
|
|
64
|
+
signal: AbortSignal.timeout(cfg.timeoutMs ?? 60_000),
|
|
65
|
+
});
|
|
66
|
+
if (!resp.ok) {
|
|
67
|
+
const body = await resp.text();
|
|
68
|
+
throw new Error(`Anthropic task-summarize failed (${resp.status}): ${body}`);
|
|
69
|
+
}
|
|
70
|
+
const json = (await resp.json());
|
|
71
|
+
return json.content.find((c) => c.type === "text")?.text?.trim() ?? "";
|
|
72
|
+
}
|
|
73
|
+
const TOPIC_JUDGE_PROMPT = `You are a conversation topic boundary detector. Given a summary of the CURRENT conversation and a NEW user message, determine if the new message starts a DIFFERENT topic/task.
|
|
74
|
+
|
|
75
|
+
Answer ONLY "NEW" or "SAME".
|
|
76
|
+
|
|
77
|
+
Rules:
|
|
78
|
+
- "NEW" = the new message is about a completely different subject, project, or task
|
|
79
|
+
- "SAME" = the new message continues, follows up on, or is closely related to the current topic
|
|
80
|
+
- Follow-up questions, clarifications, refinements, bug fixes, or next steps on the same task = SAME
|
|
81
|
+
- Greetings or meta-questions like "你好" or "谢谢" without new substance = SAME
|
|
82
|
+
- A clearly unrelated request (e.g., current topic is deployment, new message asks about cooking) = NEW
|
|
83
|
+
|
|
84
|
+
Output exactly one word: NEW or SAME`;
|
|
85
|
+
async function judgeNewTopicAnthropic(currentContext, newMessage, cfg, log) {
|
|
86
|
+
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
87
|
+
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
88
|
+
const headers = {
|
|
89
|
+
"Content-Type": "application/json",
|
|
90
|
+
"x-api-key": cfg.apiKey ?? "",
|
|
91
|
+
"anthropic-version": "2023-06-01",
|
|
92
|
+
...cfg.headers,
|
|
93
|
+
};
|
|
94
|
+
const userContent = `CURRENT CONVERSATION SUMMARY:\n${currentContext}\n\nNEW USER MESSAGE:\n${newMessage}`;
|
|
95
|
+
const resp = await fetch(endpoint, {
|
|
96
|
+
method: "POST",
|
|
97
|
+
headers,
|
|
98
|
+
body: JSON.stringify({
|
|
99
|
+
model,
|
|
100
|
+
max_tokens: 10,
|
|
101
|
+
temperature: 0,
|
|
102
|
+
system: TOPIC_JUDGE_PROMPT,
|
|
103
|
+
messages: [{ role: "user", content: userContent }],
|
|
104
|
+
}),
|
|
105
|
+
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
106
|
+
});
|
|
107
|
+
if (!resp.ok) {
|
|
108
|
+
const body = await resp.text();
|
|
109
|
+
throw new Error(`Anthropic topic-judge failed (${resp.status}): ${body}`);
|
|
110
|
+
}
|
|
111
|
+
const json = (await resp.json());
|
|
112
|
+
const answer = json.content.find((c) => c.type === "text")?.text?.trim().toUpperCase() ?? "";
|
|
113
|
+
log.debug(`Topic judge result: "${answer}"`);
|
|
114
|
+
return answer.startsWith("NEW");
|
|
115
|
+
}
|
|
116
|
+
const FILTER_RELEVANT_PROMPT = `You are a memory relevance judge. Given a user's QUERY and a list of CANDIDATE memory summaries, do two things:
|
|
117
|
+
|
|
118
|
+
1. Select ALL candidates that could be useful for answering the query. When in doubt, INCLUDE the candidate.
|
|
119
|
+
- For questions about lists, history, or "what/where/who" across multiple items (e.g. "which companies did I work at"), include ALL matching items — do NOT stop at the first match.
|
|
120
|
+
- For factual lookups (e.g. "what is the SSH port"), a single direct answer is enough.
|
|
121
|
+
2. Judge whether the selected memories are SUFFICIENT to fully answer the query WITHOUT fetching additional context.
|
|
122
|
+
|
|
123
|
+
IMPORTANT for "sufficient" judgment:
|
|
124
|
+
- sufficient=true ONLY when the memories contain a concrete ANSWER, fact, decision, or actionable information that directly addresses the query.
|
|
125
|
+
- sufficient=false when:
|
|
126
|
+
- The memories only repeat the same question the user asked before (echo, not answer).
|
|
127
|
+
- The memories show related topics but lack the specific detail needed.
|
|
128
|
+
- The memories contain partial information that would benefit from full task context, timeline, or related skills.
|
|
129
|
+
|
|
130
|
+
Output a JSON object with exactly two fields:
|
|
131
|
+
{"relevant":[1,3,5],"sufficient":true}
|
|
132
|
+
|
|
133
|
+
- "relevant": array of candidate numbers that are useful. Empty array [] if none are relevant.
|
|
134
|
+
- "sufficient": true ONLY if the memories contain a direct answer; false otherwise.
|
|
135
|
+
|
|
136
|
+
Output ONLY the JSON object, nothing else.`;
|
|
137
|
+
async function filterRelevantAnthropic(query, candidates, cfg, log) {
|
|
138
|
+
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
139
|
+
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
140
|
+
const headers = {
|
|
141
|
+
"Content-Type": "application/json",
|
|
142
|
+
"x-api-key": cfg.apiKey ?? "",
|
|
143
|
+
"anthropic-version": "2023-06-01",
|
|
144
|
+
...cfg.headers,
|
|
145
|
+
};
|
|
146
|
+
const candidateText = candidates
|
|
147
|
+
.map((c) => `${c.index}. [${c.role}] ${c.summary}`)
|
|
148
|
+
.join("\n");
|
|
149
|
+
const resp = await fetch(endpoint, {
|
|
150
|
+
method: "POST",
|
|
151
|
+
headers,
|
|
152
|
+
body: JSON.stringify({
|
|
153
|
+
model,
|
|
154
|
+
max_tokens: 200,
|
|
155
|
+
temperature: 0,
|
|
156
|
+
system: FILTER_RELEVANT_PROMPT,
|
|
157
|
+
messages: [{ role: "user", content: `QUERY: ${query}\n\nCANDIDATES:\n${candidateText}` }],
|
|
158
|
+
}),
|
|
159
|
+
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
160
|
+
});
|
|
161
|
+
if (!resp.ok) {
|
|
162
|
+
const body = await resp.text();
|
|
163
|
+
throw new Error(`Anthropic filter-relevant failed (${resp.status}): ${body}`);
|
|
164
|
+
}
|
|
165
|
+
const json = (await resp.json());
|
|
166
|
+
const raw = json.content.find((c) => c.type === "text")?.text?.trim() ?? "{}";
|
|
167
|
+
return parseFilterResult(raw, log);
|
|
168
|
+
}
|
|
169
|
+
function parseFilterResult(raw, log) {
|
|
170
|
+
try {
|
|
171
|
+
const match = raw.match(/\{[\s\S]*\}/);
|
|
172
|
+
if (match) {
|
|
173
|
+
const obj = JSON.parse(match[0]);
|
|
174
|
+
if (obj && Array.isArray(obj.relevant)) {
|
|
175
|
+
return {
|
|
176
|
+
relevant: obj.relevant.filter((n) => typeof n === "number"),
|
|
177
|
+
sufficient: obj.sufficient === true,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
catch { }
|
|
183
|
+
log.warn(`filterRelevant: failed to parse LLM output: "${raw}", fallback to all+insufficient`);
|
|
184
|
+
return { relevant: [], sufficient: false };
|
|
185
|
+
}
|
|
5
186
|
async function summarizeAnthropic(text, cfg, log) {
|
|
6
187
|
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
7
188
|
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
@@ -30,4 +211,36 @@ async function summarizeAnthropic(text, cfg, log) {
|
|
|
30
211
|
const json = (await resp.json());
|
|
31
212
|
return json.content.find((c) => c.type === "text")?.text?.trim() ?? "";
|
|
32
213
|
}
|
|
214
|
+
// ─── Smart Dedup ───
|
|
215
|
+
const openai_1 = require("./openai");
|
|
216
|
+
async function judgeDedupAnthropic(newSummary, candidates, cfg, log) {
|
|
217
|
+
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
218
|
+
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
219
|
+
const headers = {
|
|
220
|
+
"Content-Type": "application/json",
|
|
221
|
+
"x-api-key": cfg.apiKey ?? "",
|
|
222
|
+
"anthropic-version": "2023-06-01",
|
|
223
|
+
...cfg.headers,
|
|
224
|
+
};
|
|
225
|
+
const candidateText = candidates.map((c) => `${c.index}. ${c.summary}`).join("\n");
|
|
226
|
+
const resp = await fetch(endpoint, {
|
|
227
|
+
method: "POST",
|
|
228
|
+
headers,
|
|
229
|
+
body: JSON.stringify({
|
|
230
|
+
model,
|
|
231
|
+
max_tokens: 300,
|
|
232
|
+
temperature: 0,
|
|
233
|
+
system: openai_1.DEDUP_JUDGE_PROMPT,
|
|
234
|
+
messages: [{ role: "user", content: `NEW MEMORY:\n${newSummary}\n\nEXISTING MEMORIES:\n${candidateText}` }],
|
|
235
|
+
}),
|
|
236
|
+
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
237
|
+
});
|
|
238
|
+
if (!resp.ok) {
|
|
239
|
+
const body = await resp.text();
|
|
240
|
+
throw new Error(`Anthropic dedup-judge failed (${resp.status}): ${body}`);
|
|
241
|
+
}
|
|
242
|
+
const json = (await resp.json());
|
|
243
|
+
const raw = json.content.find((c) => c.type === "text")?.text?.trim() ?? "{}";
|
|
244
|
+
return (0, openai_1.parseDedupResult)(raw, log);
|
|
245
|
+
}
|
|
33
246
|
//# sourceMappingURL=anthropic.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../../src/ingest/providers/anthropic.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../../src/ingest/providers/anthropic.ts"],"names":[],"mappings":";;AAyCA,wDAkCC;AAeD,wDAuCC;AA2BD,0DAwCC;AAmBD,gDAoCC;AAQD,kDAsCC;AAvSD,MAAM,aAAa,GAAG,iSAAiS,CAAC;AAExT,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCAmCQ,CAAC;AAE9B,KAAK,UAAU,sBAAsB,CAC1C,IAAY,EACZ,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,GAAG;YACnC,MAAM,EAAE,mBAAmB;YAC3B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACzE,CAAC;AAED,MAAM,kBAAkB,GAAG;;;;;;;;;;;qCAWU,CAAC;AAE/B,KAAK,UAAU,sBAAsB,CAC1C,cAAsB,EACtB,UAAkB,EAClB,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,WAAW,GAAG,kCAAkC,cAAc,0BAA0B,UAAU,EAAE,CAAC;IAE3G,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,EAAE;YACd,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,kBAAkB;YAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;SACnD,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC7F,GAAG,CAAC,KAAK,CAAC,wBAAwB,MAAM,GAAG,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,sBAAsB,GAAG;;;;;;;;;;;;;;;;;;;;2CAoBY,CAAC;AAKrC,KAAK,UAAU,uBAAuB,CAC3C,KAAa,EACb,UAAmE,EACnE,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,aAAa,GAAG,UAAU;SAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;SAClD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,sBAAsB;YAC9B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,KAAK,oBAAoB,aAAa,EAAE,EAAE,CAAC;SAC1F,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAC9E,OAAO,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW,EAAE,GAAW;IACjD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACvC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvC,OAAO;oBACL,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;oBAChE,UAAU,EAAE,GAAG,CAAC,UAAU,KAAK,IAAI;iBACpC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IACV,GAAG,CAAC,IAAI,CAAC,gDAAgD,GAAG,iCAAiC,CAAC,CAAC;IAC/F,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAC7C,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,CAAC;YACjC,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAE9B,CAAC;IACF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACzE,CAAC;AAED,sBAAsB;AAEtB,qCAAgE;AAIzD,KAAK,UAAU,mBAAmB,CACvC,UAAkB,EAClB,UAAsE,EACtE,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnF,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,2BAAkB;YAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,UAAU,2BAA2B,aAAa,EAAE,EAAE,CAAC;SAC5G,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAC9E,OAAO,IAAA,yBAAgB,EAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACpC,CAAC"}
|
|
@@ -1,8 +1,19 @@
|
|
|
1
1
|
import type { SummarizerConfig, Logger } from "../../types";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
export declare function summarizeTaskBedrock(text: string, cfg: SummarizerConfig, log: Logger): Promise<string>;
|
|
3
|
+
export declare function judgeNewTopicBedrock(currentContext: string, newMessage: string, cfg: SummarizerConfig, log: Logger): Promise<boolean>;
|
|
4
|
+
import type { FilterResult } from "./openai";
|
|
5
|
+
export type { FilterResult } from "./openai";
|
|
6
|
+
export declare function filterRelevantBedrock(query: string, candidates: Array<{
|
|
7
|
+
index: number;
|
|
8
|
+
summary: string;
|
|
9
|
+
role: string;
|
|
10
|
+
}>, cfg: SummarizerConfig, log: Logger): Promise<FilterResult>;
|
|
7
11
|
export declare function summarizeBedrock(text: string, cfg: SummarizerConfig, log: Logger): Promise<string>;
|
|
12
|
+
import type { DedupResult } from "./openai";
|
|
13
|
+
export type { DedupResult } from "./openai";
|
|
14
|
+
export declare function judgeDedupBedrock(newSummary: string, candidates: Array<{
|
|
15
|
+
index: number;
|
|
16
|
+
summary: string;
|
|
17
|
+
chunkId: string;
|
|
18
|
+
}>, cfg: SummarizerConfig, log: Logger): Promise<DedupResult>;
|
|
8
19
|
//# sourceMappingURL=bedrock.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../../src/ingest/providers/bedrock.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../../src/ingest/providers/bedrock.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAyC5D,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CA+BjB;AAeD,wBAAsB,oBAAoB,CACxC,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,OAAO,CAAC,CAmClB;AAwBD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAC7C,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,EACnE,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,YAAY,CAAC,CAoCvB;AAmBD,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CAoCjB;AAKD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5C,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,EACtE,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,WAAW,CAAC,CA4BtB"}
|