@memtensor/memos-local-openclaw-plugin 1.0.6-beta.9 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/openclaw.plugin.json +1 -1
- package/package.json +3 -5
- package/prebuilds/darwin-arm64/better_sqlite3.node +0 -0
- package/prebuilds/darwin-x64/better_sqlite3.node +0 -0
- package/prebuilds/linux-x64/better_sqlite3.node +0 -0
- package/prebuilds/win32-x64/better_sqlite3.node +0 -0
- package/scripts/postinstall.cjs +44 -44
- package/skill/memos-memory-guide/SKILL.md +20 -0
- package/src/update-check.ts +2 -7
- package/src/viewer/html.ts +4 -4
- package/src/viewer/server.ts +12 -2
- package/telemetry.credentials.json +5 -0
- package/dist/capture/index.d.ts +0 -26
- package/dist/capture/index.d.ts.map +0 -1
- package/dist/capture/index.js +0 -283
- package/dist/capture/index.js.map +0 -1
- package/dist/client/connector.d.ts +0 -34
- package/dist/client/connector.d.ts.map +0 -1
- package/dist/client/connector.js +0 -381
- package/dist/client/connector.js.map +0 -1
- package/dist/client/hub.d.ts +0 -61
- package/dist/client/hub.d.ts.map +0 -1
- package/dist/client/hub.js +0 -174
- package/dist/client/hub.js.map +0 -1
- package/dist/client/skill-sync.d.ts +0 -36
- package/dist/client/skill-sync.d.ts.map +0 -1
- package/dist/client/skill-sync.js +0 -226
- package/dist/client/skill-sync.js.map +0 -1
- package/dist/config.d.ts +0 -5
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -171
- package/dist/config.js.map +0 -1
- package/dist/embedding/index.d.ts +0 -14
- package/dist/embedding/index.d.ts.map +0 -1
- package/dist/embedding/index.js +0 -105
- package/dist/embedding/index.js.map +0 -1
- package/dist/embedding/local.d.ts +0 -3
- package/dist/embedding/local.d.ts.map +0 -1
- package/dist/embedding/local.js +0 -66
- package/dist/embedding/local.js.map +0 -1
- package/dist/embedding/providers/cohere.d.ts +0 -4
- package/dist/embedding/providers/cohere.d.ts.map +0 -1
- package/dist/embedding/providers/cohere.js +0 -57
- package/dist/embedding/providers/cohere.js.map +0 -1
- package/dist/embedding/providers/gemini.d.ts +0 -3
- package/dist/embedding/providers/gemini.d.ts.map +0 -1
- package/dist/embedding/providers/gemini.js +0 -31
- package/dist/embedding/providers/gemini.js.map +0 -1
- package/dist/embedding/providers/mistral.d.ts +0 -3
- package/dist/embedding/providers/mistral.d.ts.map +0 -1
- package/dist/embedding/providers/mistral.js +0 -25
- package/dist/embedding/providers/mistral.js.map +0 -1
- package/dist/embedding/providers/openai.d.ts +0 -3
- package/dist/embedding/providers/openai.d.ts.map +0 -1
- package/dist/embedding/providers/openai.js +0 -35
- package/dist/embedding/providers/openai.js.map +0 -1
- package/dist/embedding/providers/voyage.d.ts +0 -3
- package/dist/embedding/providers/voyage.d.ts.map +0 -1
- package/dist/embedding/providers/voyage.js +0 -25
- package/dist/embedding/providers/voyage.js.map +0 -1
- package/dist/hub/auth.d.ts +0 -19
- package/dist/hub/auth.d.ts.map +0 -1
- package/dist/hub/auth.js +0 -70
- package/dist/hub/auth.js.map +0 -1
- package/dist/hub/server.d.ts +0 -52
- package/dist/hub/server.d.ts.map +0 -1
- package/dist/hub/server.js +0 -1197
- package/dist/hub/server.js.map +0 -1
- package/dist/hub/user-manager.d.ts +0 -40
- package/dist/hub/user-manager.d.ts.map +0 -1
- package/dist/hub/user-manager.js +0 -153
- package/dist/hub/user-manager.js.map +0 -1
- package/dist/index.d.ts +0 -46
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -82
- package/dist/index.js.map +0 -1
- package/dist/ingest/chunker.d.ts +0 -15
- package/dist/ingest/chunker.d.ts.map +0 -1
- package/dist/ingest/chunker.js +0 -192
- package/dist/ingest/chunker.js.map +0 -1
- package/dist/ingest/dedup.d.ts +0 -19
- package/dist/ingest/dedup.d.ts.map +0 -1
- package/dist/ingest/dedup.js +0 -50
- package/dist/ingest/dedup.js.map +0 -1
- package/dist/ingest/providers/anthropic.d.ts +0 -21
- package/dist/ingest/providers/anthropic.d.ts.map +0 -1
- package/dist/ingest/providers/anthropic.js +0 -314
- package/dist/ingest/providers/anthropic.js.map +0 -1
- package/dist/ingest/providers/bedrock.d.ts +0 -21
- package/dist/ingest/providers/bedrock.d.ts.map +0 -1
- package/dist/ingest/providers/bedrock.js +0 -313
- package/dist/ingest/providers/bedrock.js.map +0 -1
- package/dist/ingest/providers/gemini.d.ts +0 -21
- package/dist/ingest/providers/gemini.d.ts.map +0 -1
- package/dist/ingest/providers/gemini.js +0 -298
- package/dist/ingest/providers/gemini.js.map +0 -1
- package/dist/ingest/providers/index.d.ts +0 -68
- package/dist/ingest/providers/index.d.ts.map +0 -1
- package/dist/ingest/providers/index.js +0 -611
- package/dist/ingest/providers/index.js.map +0 -1
- package/dist/ingest/providers/openai.d.ts +0 -30
- package/dist/ingest/providers/openai.d.ts.map +0 -1
- package/dist/ingest/providers/openai.js +0 -387
- package/dist/ingest/providers/openai.js.map +0 -1
- package/dist/ingest/task-processor.d.ts +0 -91
- package/dist/ingest/task-processor.d.ts.map +0 -1
- package/dist/ingest/task-processor.js +0 -478
- package/dist/ingest/task-processor.js.map +0 -1
- package/dist/ingest/worker.d.ts +0 -23
- package/dist/ingest/worker.d.ts.map +0 -1
- package/dist/ingest/worker.js +0 -255
- package/dist/ingest/worker.js.map +0 -1
- package/dist/openclaw-api.d.ts +0 -53
- package/dist/openclaw-api.d.ts.map +0 -1
- package/dist/openclaw-api.js +0 -189
- package/dist/openclaw-api.js.map +0 -1
- package/dist/recall/engine.d.ts +0 -28
- package/dist/recall/engine.d.ts.map +0 -1
- package/dist/recall/engine.js +0 -343
- package/dist/recall/engine.js.map +0 -1
- package/dist/recall/mmr.d.ts +0 -17
- package/dist/recall/mmr.d.ts.map +0 -1
- package/dist/recall/mmr.js +0 -53
- package/dist/recall/mmr.js.map +0 -1
- package/dist/recall/recency.d.ts +0 -20
- package/dist/recall/recency.d.ts.map +0 -1
- package/dist/recall/recency.js +0 -26
- package/dist/recall/recency.js.map +0 -1
- package/dist/recall/rrf.d.ts +0 -16
- package/dist/recall/rrf.d.ts.map +0 -1
- package/dist/recall/rrf.js +0 -15
- package/dist/recall/rrf.js.map +0 -1
- package/dist/shared/llm-call.d.ts +0 -30
- package/dist/shared/llm-call.d.ts.map +0 -1
- package/dist/shared/llm-call.js +0 -253
- package/dist/shared/llm-call.js.map +0 -1
- package/dist/sharing/types.contract.d.ts +0 -2
- package/dist/sharing/types.contract.d.ts.map +0 -1
- package/dist/sharing/types.contract.js +0 -3
- package/dist/sharing/types.contract.js.map +0 -1
- package/dist/sharing/types.d.ts +0 -80
- package/dist/sharing/types.d.ts.map +0 -1
- package/dist/sharing/types.js +0 -3
- package/dist/sharing/types.js.map +0 -1
- package/dist/skill/bundled-memory-guide.d.ts +0 -2
- package/dist/skill/bundled-memory-guide.d.ts.map +0 -1
- package/dist/skill/bundled-memory-guide.js +0 -45
- package/dist/skill/bundled-memory-guide.js.map +0 -1
- package/dist/skill/evaluator.d.ts +0 -28
- package/dist/skill/evaluator.d.ts.map +0 -1
- package/dist/skill/evaluator.js +0 -169
- package/dist/skill/evaluator.js.map +0 -1
- package/dist/skill/evolver.d.ts +0 -48
- package/dist/skill/evolver.d.ts.map +0 -1
- package/dist/skill/evolver.js +0 -406
- package/dist/skill/evolver.js.map +0 -1
- package/dist/skill/generator.d.ts +0 -26
- package/dist/skill/generator.d.ts.map +0 -1
- package/dist/skill/generator.js +0 -521
- package/dist/skill/generator.js.map +0 -1
- package/dist/skill/installer.d.ts +0 -42
- package/dist/skill/installer.d.ts.map +0 -1
- package/dist/skill/installer.js +0 -165
- package/dist/skill/installer.js.map +0 -1
- package/dist/skill/upgrader.d.ts +0 -19
- package/dist/skill/upgrader.d.ts.map +0 -1
- package/dist/skill/upgrader.js +0 -366
- package/dist/skill/upgrader.js.map +0 -1
- package/dist/skill/validator.d.ts +0 -30
- package/dist/skill/validator.d.ts.map +0 -1
- package/dist/skill/validator.js +0 -272
- package/dist/skill/validator.js.map +0 -1
- package/dist/storage/ensure-binding.d.ts +0 -12
- package/dist/storage/ensure-binding.d.ts.map +0 -1
- package/dist/storage/ensure-binding.js +0 -53
- package/dist/storage/ensure-binding.js.map +0 -1
- package/dist/storage/sqlite.d.ts +0 -649
- package/dist/storage/sqlite.d.ts.map +0 -1
- package/dist/storage/sqlite.js +0 -2657
- package/dist/storage/sqlite.js.map +0 -1
- package/dist/storage/vector.d.ts +0 -12
- package/dist/storage/vector.d.ts.map +0 -1
- package/dist/storage/vector.js +0 -34
- package/dist/storage/vector.js.map +0 -1
- package/dist/telemetry.d.ts +0 -47
- package/dist/telemetry.d.ts.map +0 -1
- package/dist/telemetry.js +0 -312
- package/dist/telemetry.js.map +0 -1
- package/dist/tools/index.d.ts +0 -5
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js +0 -12
- package/dist/tools/index.js.map +0 -1
- package/dist/tools/memory-get.d.ts +0 -4
- package/dist/tools/memory-get.d.ts.map +0 -1
- package/dist/tools/memory-get.js +0 -64
- package/dist/tools/memory-get.js.map +0 -1
- package/dist/tools/memory-search.d.ts +0 -7
- package/dist/tools/memory-search.d.ts.map +0 -1
- package/dist/tools/memory-search.js +0 -84
- package/dist/tools/memory-search.js.map +0 -1
- package/dist/tools/memory-timeline.d.ts +0 -4
- package/dist/tools/memory-timeline.d.ts.map +0 -1
- package/dist/tools/memory-timeline.js +0 -73
- package/dist/tools/memory-timeline.js.map +0 -1
- package/dist/tools/network-memory-detail.d.ts +0 -4
- package/dist/tools/network-memory-detail.d.ts.map +0 -1
- package/dist/tools/network-memory-detail.js +0 -34
- package/dist/tools/network-memory-detail.js.map +0 -1
- package/dist/types.d.ts +0 -330
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -38
- package/dist/types.js.map +0 -1
- package/dist/update-check.d.ts +0 -21
- package/dist/update-check.d.ts.map +0 -1
- package/dist/update-check.js +0 -110
- package/dist/update-check.js.map +0 -1
- package/dist/viewer/html.d.ts +0 -2
- package/dist/viewer/html.d.ts.map +0 -1
- package/dist/viewer/html.js +0 -9168
- package/dist/viewer/html.js.map +0 -1
- package/dist/viewer/server.d.ts +0 -205
- package/dist/viewer/server.d.ts.map +0 -1
- package/dist/viewer/server.js +0 -4876
- package/dist/viewer/server.js.map +0 -1
package/dist/ingest/chunker.js
DELETED
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.chunkText = chunkText;
|
|
4
|
-
const MAX_CHUNK_CHARS = 3000;
|
|
5
|
-
const MIN_CHUNK_CHARS = 40;
|
|
6
|
-
const IDEAL_CHUNK_CHARS = 1500;
|
|
7
|
-
const FENCED_CODE_RE = /^(`{3,})[^\n]*\n[\s\S]*?^\1\s*$/gm;
|
|
8
|
-
const FUNC_OPEN_RE = /^[ \t]*(?:(?:export\s+)?(?:async\s+)?(?:function|class|const\s+\w+\s*=\s*(?:\([^)]*\)|[^=])*=>)|(?:def |class )|(?:func |fn |pub\s+fn )|(?:public |private |protected |static )+.*\{)\s*$/;
|
|
9
|
-
const BLOCK_CLOSE_RE = /^[ \t]*[}\]]\s*;?\s*$/;
|
|
10
|
-
const ERROR_STACK_RE = /(?:(?:Error|Exception|Traceback)[^\n]*\n(?:\s+at\s+[^\n]+\n?|.*File "[^\n]+\n?|.*line \d+[^\n]*\n?){2,})/gm;
|
|
11
|
-
const LIST_BLOCK_RE = /(?:^[\s]*[-*•]\s+.+\n?){3,}/gm;
|
|
12
|
-
const COMMAND_LINE_RE = /^(?:\$|>|#)\s+.+$/gm;
|
|
13
|
-
/**
|
|
14
|
-
* Semantic-aware chunking:
|
|
15
|
-
* 1. Extract fenced code blocks as whole units (never split inside)
|
|
16
|
-
* 2. Detect unfenced code regions by brace-matching (functions/classes kept intact)
|
|
17
|
-
* 3. Extract error stacks, list blocks, command lines as separate chunks
|
|
18
|
-
* 4. Split remaining prose at paragraph boundaries (double newline)
|
|
19
|
-
* 5. Merge short adjacent chunks
|
|
20
|
-
*/
|
|
21
|
-
function chunkText(text) {
|
|
22
|
-
let remaining = text;
|
|
23
|
-
const slots = [];
|
|
24
|
-
let counter = 0;
|
|
25
|
-
function ph(content, kind = "paragraph") {
|
|
26
|
-
const tag = `\x00SLOT_${counter++}\x00`;
|
|
27
|
-
slots.push({ placeholder: tag, content: content.trim(), kind });
|
|
28
|
-
return tag;
|
|
29
|
-
}
|
|
30
|
-
remaining = remaining.replace(FENCED_CODE_RE, (m) => ph(m, "code_block"));
|
|
31
|
-
remaining = extractBraceBlocks(remaining, ph);
|
|
32
|
-
const structuralKinds = [
|
|
33
|
-
[ERROR_STACK_RE, "error_stack"],
|
|
34
|
-
[LIST_BLOCK_RE, "list"],
|
|
35
|
-
[COMMAND_LINE_RE, "command"],
|
|
36
|
-
];
|
|
37
|
-
for (const [re, kind] of structuralKinds) {
|
|
38
|
-
remaining = remaining.replace(re, (m) => ph(m, kind));
|
|
39
|
-
}
|
|
40
|
-
const raw = [];
|
|
41
|
-
const sections = remaining.split(/\n{2,}/);
|
|
42
|
-
for (const sec of sections) {
|
|
43
|
-
const trimmed = sec.trim();
|
|
44
|
-
if (!trimmed)
|
|
45
|
-
continue;
|
|
46
|
-
if (trimmed.includes("\x00SLOT_")) {
|
|
47
|
-
const parts = trimmed.split(/(\x00SLOT_\d+\x00)/);
|
|
48
|
-
for (const part of parts) {
|
|
49
|
-
const slot = slots.find((s) => s.placeholder === part);
|
|
50
|
-
if (slot) {
|
|
51
|
-
raw.push({ content: slot.content, kind: slot.kind });
|
|
52
|
-
}
|
|
53
|
-
else if (part.trim().length >= MIN_CHUNK_CHARS) {
|
|
54
|
-
raw.push({ content: part.trim(), kind: "paragraph" });
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
else if (trimmed.length >= MIN_CHUNK_CHARS) {
|
|
59
|
-
raw.push({ content: trimmed, kind: "paragraph" });
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
for (const s of slots) {
|
|
63
|
-
if (!raw.some((c) => c.content === s.content)) {
|
|
64
|
-
raw.push({ content: s.content, kind: s.kind });
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
const merged = mergeSmallChunks(raw);
|
|
68
|
-
const final = splitOversized(merged);
|
|
69
|
-
return final.length > 0 ? final : [{ content: text.trim(), kind: "paragraph" }];
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Detect function/class bodies that aren't inside fenced blocks.
|
|
73
|
-
* Tracks brace depth to keep complete blocks together.
|
|
74
|
-
*/
|
|
75
|
-
function extractBraceBlocks(text, ph) {
|
|
76
|
-
const lines = text.split("\n");
|
|
77
|
-
const result = [];
|
|
78
|
-
let blockLines = [];
|
|
79
|
-
let depth = 0;
|
|
80
|
-
let inBlock = false;
|
|
81
|
-
for (let i = 0; i < lines.length; i++) {
|
|
82
|
-
const line = lines[i];
|
|
83
|
-
if (line.includes("\x00SLOT_")) {
|
|
84
|
-
if (inBlock) {
|
|
85
|
-
blockLines.push(line);
|
|
86
|
-
}
|
|
87
|
-
else {
|
|
88
|
-
result.push(line);
|
|
89
|
-
}
|
|
90
|
-
continue;
|
|
91
|
-
}
|
|
92
|
-
if (!inBlock && FUNC_OPEN_RE.test(line)) {
|
|
93
|
-
inBlock = true;
|
|
94
|
-
blockLines = [line];
|
|
95
|
-
depth = countBraces(line);
|
|
96
|
-
if (depth <= 0)
|
|
97
|
-
depth = 1;
|
|
98
|
-
continue;
|
|
99
|
-
}
|
|
100
|
-
if (inBlock) {
|
|
101
|
-
blockLines.push(line);
|
|
102
|
-
depth += countBraces(line);
|
|
103
|
-
if (depth <= 0 || (BLOCK_CLOSE_RE.test(line) && depth <= 0)) {
|
|
104
|
-
const block = blockLines.join("\n");
|
|
105
|
-
if (block.trim().length >= MIN_CHUNK_CHARS) {
|
|
106
|
-
result.push(ph(block, "code_block"));
|
|
107
|
-
}
|
|
108
|
-
else {
|
|
109
|
-
result.push(block);
|
|
110
|
-
}
|
|
111
|
-
inBlock = false;
|
|
112
|
-
blockLines = [];
|
|
113
|
-
depth = 0;
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
else {
|
|
117
|
-
result.push(line);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
if (blockLines.length > 0) {
|
|
121
|
-
const block = blockLines.join("\n");
|
|
122
|
-
if (block.trim().length >= MIN_CHUNK_CHARS) {
|
|
123
|
-
result.push(ph(block, "code_block"));
|
|
124
|
-
}
|
|
125
|
-
else {
|
|
126
|
-
result.push(block);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
return result.join("\n");
|
|
130
|
-
}
|
|
131
|
-
function countBraces(line) {
|
|
132
|
-
let d = 0;
|
|
133
|
-
for (const ch of line) {
|
|
134
|
-
if (ch === "{" || ch === "(")
|
|
135
|
-
d++;
|
|
136
|
-
else if (ch === "}" || ch === ")")
|
|
137
|
-
d--;
|
|
138
|
-
}
|
|
139
|
-
return d;
|
|
140
|
-
}
|
|
141
|
-
function mergeSmallChunks(chunks) {
|
|
142
|
-
if (chunks.length <= 1)
|
|
143
|
-
return chunks;
|
|
144
|
-
const merged = [];
|
|
145
|
-
let buf = null;
|
|
146
|
-
for (const c of chunks) {
|
|
147
|
-
if (!buf) {
|
|
148
|
-
buf = { ...c };
|
|
149
|
-
continue;
|
|
150
|
-
}
|
|
151
|
-
const bothSmall = buf.content.length < IDEAL_CHUNK_CHARS && c.content.length < IDEAL_CHUNK_CHARS;
|
|
152
|
-
const mergedLen = buf.content.length + c.content.length + 2;
|
|
153
|
-
if (bothSmall && mergedLen <= MAX_CHUNK_CHARS) {
|
|
154
|
-
buf.content = buf.content + "\n\n" + c.content;
|
|
155
|
-
}
|
|
156
|
-
else {
|
|
157
|
-
merged.push(buf);
|
|
158
|
-
buf = { ...c };
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
if (buf)
|
|
162
|
-
merged.push(buf);
|
|
163
|
-
return merged;
|
|
164
|
-
}
|
|
165
|
-
function splitOversized(chunks) {
|
|
166
|
-
const result = [];
|
|
167
|
-
for (const c of chunks) {
|
|
168
|
-
if (c.content.length <= MAX_CHUNK_CHARS) {
|
|
169
|
-
result.push(c);
|
|
170
|
-
continue;
|
|
171
|
-
}
|
|
172
|
-
result.push(...splitAtSentenceBoundary(c.content));
|
|
173
|
-
}
|
|
174
|
-
return result;
|
|
175
|
-
}
|
|
176
|
-
function splitAtSentenceBoundary(text) {
|
|
177
|
-
const sentences = text.match(/[^.!?。!?\n]+(?:[.!?。!?]+|\n{2,})/g) ?? [text];
|
|
178
|
-
const result = [];
|
|
179
|
-
let buf = "";
|
|
180
|
-
for (const s of sentences) {
|
|
181
|
-
if (buf.length + s.length > MAX_CHUNK_CHARS && buf.length > 0) {
|
|
182
|
-
result.push({ content: buf.trim(), kind: "paragraph" });
|
|
183
|
-
buf = "";
|
|
184
|
-
}
|
|
185
|
-
buf += s;
|
|
186
|
-
}
|
|
187
|
-
if (buf.trim().length >= MIN_CHUNK_CHARS) {
|
|
188
|
-
result.push({ content: buf.trim(), kind: "paragraph" });
|
|
189
|
-
}
|
|
190
|
-
return result;
|
|
191
|
-
}
|
|
192
|
-
//# sourceMappingURL=chunker.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/ingest/chunker.ts"],"names":[],"mappings":";;AA8BA,8BAuDC;AA9ED,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,eAAe,GAAG,EAAE,CAAC;AAC3B,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B,MAAM,cAAc,GAAG,mCAAmC,CAAC;AAE3D,MAAM,YAAY,GAChB,2LAA2L,CAAC;AAC9L,MAAM,cAAc,GAAG,uBAAuB,CAAC;AAE/C,MAAM,cAAc,GAClB,4GAA4G,CAAC;AAC/G,MAAM,aAAa,GAAG,+BAA+B,CAAC;AACtD,MAAM,eAAe,GAAG,qBAAqB,CAAC;AAE9C;;;;;;;GAOG;AACH,SAAgB,SAAS,CAAC,IAAY;IACpC,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,MAAM,KAAK,GAAqE,EAAE,CAAC;IACnF,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,SAAS,EAAE,CAAC,OAAe,EAAE,OAAkB,WAAW;QACxD,MAAM,GAAG,GAAG,YAAY,OAAO,EAAE,MAAM,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,OAAO,GAAG,CAAC;IACb,CAAC;IAED,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;IAC1E,SAAS,GAAG,kBAAkB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE9C,MAAM,eAAe,GAA+B;QAClD,CAAC,cAAc,EAAE,aAAa,CAAC;QAC/B,CAAC,aAAa,EAAE,MAAM,CAAC;QACvB,CAAC,eAAe,EAAE,SAAS,CAAC;KAC7B,CAAC;IACF,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,eAAe,EAAE,CAAC;QACzC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE3C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAClD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC;gBACvD,IAAI,IAAI,EAAE,CAAC;oBACT,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvD,CAAC;qBAAM,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;oBACjD,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAErC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AAClF,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CACzB,IAAY,EACZ,EAAiD;IAEjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,UAAU,GAAa,EAAE,CAAC;IAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,IAAI,OAAO,EAAE,CAAC;gBACZ,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,GAAG,IAAI,CAAC;YACf,UAAU,GAAG,CAAC,IAAI,CAAC,CAAC;YACpB,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC1B,IAAI,KAAK,IAAI,CAAC;gBAAE,KAAK,GAAG,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,KAAK,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC5D,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;oBAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;gBACvC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;gBACD,OAAO,GAAG,KAAK,CAAC;gBAChB,UAAU,GAAG,EAAE,CAAC;gBAChB,KAAK,GAAG,CAAC,CAAC;YACZ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;YAAE,CAAC,EAAE,CAAC;aAC7B,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;YAAE,CAAC,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAkB;IAC1C,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IACtC,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,IAAI,GAAG,GAAoB,IAAI,CAAC;IAEhC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;YACf,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,iBAAiB,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC;QACjG,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAE5D,IAAI,SAAS,IAAI,SAAS,IAAI,eAAe,EAAE,CAAC;YAC9C,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,GAAG,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IACD,IAAI,GAAG;QAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,MAAkB;IACxC,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,IAAI,GAAG,GAAG,EAAE,CAAC;IAEb,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,GAAG,eAAe,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YACxD,GAAG,GAAG,EAAE,CAAC;QACX,CAAC;QACD,GAAG,IAAI,CAAC,CAAC;IACX,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/ingest/dedup.d.ts
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import type { SqliteStore } from "../storage/sqlite";
|
|
2
|
-
import type { Logger } from "../types";
|
|
3
|
-
/**
|
|
4
|
-
* Check if a new summary embedding is a near-duplicate of any
|
|
5
|
-
* existing embedding. If similarity >= threshold, return the
|
|
6
|
-
* existing chunk ID to merge/update instead of creating a new entry.
|
|
7
|
-
*
|
|
8
|
-
* PRD §4.4: dedup threshold 0.92–0.95
|
|
9
|
-
*/
|
|
10
|
-
export declare function findDuplicate(store: SqliteStore, newVec: number[], threshold: number, log: Logger, ownerFilter?: string[]): string | null;
|
|
11
|
-
/**
|
|
12
|
-
* Find Top-N most similar chunks above a threshold.
|
|
13
|
-
* Used for smart dedup: retrieve candidates, then ask LLM to judge.
|
|
14
|
-
*/
|
|
15
|
-
export declare function findTopSimilar(store: SqliteStore, newVec: number[], threshold: number, topN: number, log: Logger, ownerFilter?: string[]): Array<{
|
|
16
|
-
chunkId: string;
|
|
17
|
-
score: number;
|
|
18
|
-
}>;
|
|
19
|
-
//# sourceMappingURL=dedup.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/ingest/dedup.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,WAAW,CAAC,EAAE,MAAM,EAAE,GACrB,MAAM,GAAG,IAAI,CAoBf;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,WAAW,CAAC,EAAE,MAAM,EAAE,GACrB,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAiB3C"}
|
package/dist/ingest/dedup.js
DELETED
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.findDuplicate = findDuplicate;
|
|
4
|
-
exports.findTopSimilar = findTopSimilar;
|
|
5
|
-
const vector_1 = require("../storage/vector");
|
|
6
|
-
/**
|
|
7
|
-
* Check if a new summary embedding is a near-duplicate of any
|
|
8
|
-
* existing embedding. If similarity >= threshold, return the
|
|
9
|
-
* existing chunk ID to merge/update instead of creating a new entry.
|
|
10
|
-
*
|
|
11
|
-
* PRD §4.4: dedup threshold 0.92–0.95
|
|
12
|
-
*/
|
|
13
|
-
function findDuplicate(store, newVec, threshold, log, ownerFilter) {
|
|
14
|
-
const all = store.getAllEmbeddings(ownerFilter);
|
|
15
|
-
let bestId = null;
|
|
16
|
-
let bestScore = 0;
|
|
17
|
-
for (const { chunkId, vector } of all) {
|
|
18
|
-
const sim = (0, vector_1.cosineSimilarity)(newVec, vector);
|
|
19
|
-
if (sim > bestScore) {
|
|
20
|
-
bestScore = sim;
|
|
21
|
-
bestId = chunkId;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
if (bestId && bestScore >= threshold) {
|
|
25
|
-
log.debug(`Dedup: found duplicate chunk=${bestId} sim=${bestScore.toFixed(4)}`);
|
|
26
|
-
return bestId;
|
|
27
|
-
}
|
|
28
|
-
return null;
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Find Top-N most similar chunks above a threshold.
|
|
32
|
-
* Used for smart dedup: retrieve candidates, then ask LLM to judge.
|
|
33
|
-
*/
|
|
34
|
-
function findTopSimilar(store, newVec, threshold, topN, log, ownerFilter) {
|
|
35
|
-
const all = store.getAllEmbeddings(ownerFilter);
|
|
36
|
-
const scored = [];
|
|
37
|
-
for (const { chunkId, vector } of all) {
|
|
38
|
-
const sim = (0, vector_1.cosineSimilarity)(newVec, vector);
|
|
39
|
-
if (sim >= threshold) {
|
|
40
|
-
scored.push({ chunkId, score: sim });
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
scored.sort((a, b) => b.score - a.score);
|
|
44
|
-
const result = scored.slice(0, topN);
|
|
45
|
-
if (result.length > 0) {
|
|
46
|
-
log.debug(`findTopSimilar: found ${result.length} candidates above ${threshold} (best=${result[0].score.toFixed(4)})`);
|
|
47
|
-
}
|
|
48
|
-
return result;
|
|
49
|
-
}
|
|
50
|
-
//# sourceMappingURL=dedup.js.map
|
package/dist/ingest/dedup.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/ingest/dedup.ts"],"names":[],"mappings":";;AAWA,sCA0BC;AAMD,wCAwBC;AAnED,8CAAqD;AAIrD;;;;;;GAMG;AACH,SAAgB,aAAa,CAC3B,KAAkB,EAClB,MAAgB,EAChB,SAAiB,EACjB,GAAW,EACX,WAAsB;IAEtB,MAAM,GAAG,GAAG,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAEhD,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAA,yBAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7C,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;YACpB,SAAS,GAAG,GAAG,CAAC;YAChB,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;QACrC,GAAG,CAAC,KAAK,CAAC,gCAAgC,MAAM,QAAQ,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAgB,cAAc,CAC5B,KAAkB,EAClB,MAAgB,EAChB,SAAiB,EACjB,IAAY,EACZ,GAAW,EACX,WAAsB;IAEtB,MAAM,GAAG,GAAG,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,MAAM,GAA8C,EAAE,CAAC;IAE7D,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAA,yBAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7C,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,GAAG,CAAC,KAAK,CAAC,yBAAyB,MAAM,CAAC,MAAM,qBAAqB,SAAS,UAAU,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import type { SummarizerConfig, Logger } from "../../types";
|
|
2
|
-
export declare function summarizeTaskAnthropic(text: string, cfg: SummarizerConfig, log: Logger): Promise<string>;
|
|
3
|
-
export declare function generateTaskTitleAnthropic(text: string, cfg: SummarizerConfig, log: Logger): Promise<string>;
|
|
4
|
-
export declare function judgeNewTopicAnthropic(currentContext: string, newMessage: string, cfg: SummarizerConfig, log: Logger): Promise<boolean>;
|
|
5
|
-
import type { FilterResult } from "./openai";
|
|
6
|
-
export type { FilterResult } from "./openai";
|
|
7
|
-
export declare function filterRelevantAnthropic(query: string, candidates: Array<{
|
|
8
|
-
index: number;
|
|
9
|
-
role: string;
|
|
10
|
-
content: string;
|
|
11
|
-
time?: string;
|
|
12
|
-
}>, cfg: SummarizerConfig, log: Logger): Promise<FilterResult>;
|
|
13
|
-
export declare function summarizeAnthropic(text: string, cfg: SummarizerConfig, log: Logger): Promise<string>;
|
|
14
|
-
import type { DedupResult } from "./openai";
|
|
15
|
-
export type { DedupResult } from "./openai";
|
|
16
|
-
export declare function judgeDedupAnthropic(newSummary: string, candidates: Array<{
|
|
17
|
-
index: number;
|
|
18
|
-
summary: string;
|
|
19
|
-
chunkId: string;
|
|
20
|
-
}>, cfg: SummarizerConfig, log: Logger): Promise<DedupResult>;
|
|
21
|
-
//# sourceMappingURL=anthropic.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../../src/ingest/providers/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAwD5D,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CA8BjB;AAcD,wBAAsB,0BAA0B,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CA8BjB;AA4BD,wBAAsB,sBAAsB,CAC1C,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,OAAO,CAAC,CAkClB;AAoBD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAC7C,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,EAClF,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,YAAY,CAAC,CAuCvB;AAmBD,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CAgCjB;AAKD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5C,wBAAsB,mBAAmB,CACvC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,EACtE,GAAG,EAAE,gBAAgB,EACrB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,WAAW,CAAC,CAiCtB"}
|
|
@@ -1,314 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.summarizeTaskAnthropic = summarizeTaskAnthropic;
|
|
4
|
-
exports.generateTaskTitleAnthropic = generateTaskTitleAnthropic;
|
|
5
|
-
exports.judgeNewTopicAnthropic = judgeNewTopicAnthropic;
|
|
6
|
-
exports.filterRelevantAnthropic = filterRelevantAnthropic;
|
|
7
|
-
exports.summarizeAnthropic = summarizeAnthropic;
|
|
8
|
-
exports.judgeDedupAnthropic = judgeDedupAnthropic;
|
|
9
|
-
const SYSTEM_PROMPT = `You generate a retrieval-friendly title.
|
|
10
|
-
|
|
11
|
-
Return exactly one noun phrase that names the topic AND its key details.
|
|
12
|
-
|
|
13
|
-
Requirements:
|
|
14
|
-
- Same language as input
|
|
15
|
-
- Keep proper nouns, API/function names, specific parameters, versions, error codes
|
|
16
|
-
- Include WHO/WHAT/WHERE details when present (e.g. person name + event, tool name + what it does)
|
|
17
|
-
- Prefer concrete topic words over generic words
|
|
18
|
-
- No verbs unless unavoidable
|
|
19
|
-
- No generic endings like:
|
|
20
|
-
功能说明、使用说明、简介、介绍、用途、summary、overview、basics
|
|
21
|
-
- Chinese: 10-50 characters (aim for 15-30)
|
|
22
|
-
- Non-Chinese: 5-15 words (aim for 8-12)
|
|
23
|
-
- Output title only`;
|
|
24
|
-
const TASK_SUMMARY_PROMPT = `You create a DETAILED task summary from a multi-turn conversation. This summary will be the ONLY record of this conversation, so it must preserve ALL important information.
|
|
25
|
-
|
|
26
|
-
## LANGUAGE RULE (HIGHEST PRIORITY)
|
|
27
|
-
Detect the PRIMARY language of the user's messages. If most user messages are Chinese, ALL output (title, goal, steps, result, details) MUST be in Chinese. If English, output in English. NEVER mix. This rule overrides everything below.
|
|
28
|
-
|
|
29
|
-
Output EXACTLY this structure:
|
|
30
|
-
|
|
31
|
-
📌 Title / 标题
|
|
32
|
-
A short, descriptive title (10-30 characters). Same language as user messages.
|
|
33
|
-
|
|
34
|
-
🎯 Goal / 目标
|
|
35
|
-
One sentence: what the user wanted to accomplish.
|
|
36
|
-
|
|
37
|
-
📋 Key Steps / 关键步骤
|
|
38
|
-
- Describe each meaningful step in detail
|
|
39
|
-
- Include the ACTUAL content produced: code snippets, commands, config blocks, formulas, key paragraphs
|
|
40
|
-
- For code: include the function signature and core logic (up to ~30 lines per block), use fenced code blocks
|
|
41
|
-
- For configs: include the actual config values and structure
|
|
42
|
-
- For lists/instructions: include the actual items, not just "provided a list"
|
|
43
|
-
- Merge only truly trivial back-and-forth (like "ok" / "sure")
|
|
44
|
-
- Do NOT over-summarize: "provided a function" is BAD; show the actual function
|
|
45
|
-
|
|
46
|
-
✅ Result / 结果
|
|
47
|
-
What was the final outcome? Include the final version of any code/config/content produced.
|
|
48
|
-
|
|
49
|
-
💡 Key Details / 关键细节
|
|
50
|
-
- Decisions made, trade-offs discussed, caveats noted, alternative approaches mentioned
|
|
51
|
-
- Specific values: numbers, versions, thresholds, URLs, file paths, model names
|
|
52
|
-
- Omit this section only if there truly are no noteworthy details
|
|
53
|
-
|
|
54
|
-
RULES:
|
|
55
|
-
- This summary is a KNOWLEDGE BASE ENTRY, not a brief note. Be thorough.
|
|
56
|
-
- PRESERVE verbatim: code, commands, URLs, file paths, error messages, config values, version numbers, names, amounts
|
|
57
|
-
- DISCARD only: greetings, filler, the assistant explaining what it will do before doing it
|
|
58
|
-
- Replace secrets (API keys, tokens, passwords) with [REDACTED]
|
|
59
|
-
- Target length: 30-50% of the original conversation length. Longer conversations need longer summaries.
|
|
60
|
-
- Output summary only, no preamble.`;
|
|
61
|
-
async function summarizeTaskAnthropic(text, cfg, log) {
|
|
62
|
-
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
63
|
-
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
64
|
-
const headers = {
|
|
65
|
-
"Content-Type": "application/json",
|
|
66
|
-
"x-api-key": cfg.apiKey ?? "",
|
|
67
|
-
"anthropic-version": "2023-06-01",
|
|
68
|
-
...cfg.headers,
|
|
69
|
-
};
|
|
70
|
-
const resp = await fetch(endpoint, {
|
|
71
|
-
method: "POST",
|
|
72
|
-
headers,
|
|
73
|
-
body: JSON.stringify({
|
|
74
|
-
model,
|
|
75
|
-
max_tokens: 4096,
|
|
76
|
-
temperature: cfg.temperature ?? 0.1,
|
|
77
|
-
system: TASK_SUMMARY_PROMPT,
|
|
78
|
-
messages: [{ role: "user", content: text }],
|
|
79
|
-
}),
|
|
80
|
-
signal: AbortSignal.timeout(cfg.timeoutMs ?? 60_000),
|
|
81
|
-
});
|
|
82
|
-
if (!resp.ok) {
|
|
83
|
-
const body = await resp.text();
|
|
84
|
-
throw new Error(`Anthropic task-summarize failed (${resp.status}): ${body}`);
|
|
85
|
-
}
|
|
86
|
-
const json = (await resp.json());
|
|
87
|
-
return json.content.find((c) => c.type === "text")?.text?.trim() ?? "";
|
|
88
|
-
}
|
|
89
|
-
const TASK_TITLE_PROMPT = `Generate a short title for a conversation task.
|
|
90
|
-
|
|
91
|
-
Input: the first few user messages from a conversation.
|
|
92
|
-
Output: a concise title (5-20 characters for Chinese, 3-8 words for English).
|
|
93
|
-
|
|
94
|
-
Rules:
|
|
95
|
-
- Same language as user messages
|
|
96
|
-
- Describe WHAT the user wanted to do, not system/technical details
|
|
97
|
-
- Ignore system prompts, session startup messages, or boilerplate instructions — focus on the user's actual intent
|
|
98
|
-
- If the user only asked one question, use that question as the title (shortened if needed)
|
|
99
|
-
- Output the title only, no quotes, no prefix, no explanation`;
|
|
100
|
-
async function generateTaskTitleAnthropic(text, cfg, log) {
|
|
101
|
-
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
102
|
-
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
103
|
-
const headers = {
|
|
104
|
-
"Content-Type": "application/json",
|
|
105
|
-
"x-api-key": cfg.apiKey ?? "",
|
|
106
|
-
"anthropic-version": "2023-06-01",
|
|
107
|
-
...cfg.headers,
|
|
108
|
-
};
|
|
109
|
-
const resp = await fetch(endpoint, {
|
|
110
|
-
method: "POST",
|
|
111
|
-
headers,
|
|
112
|
-
body: JSON.stringify({
|
|
113
|
-
model,
|
|
114
|
-
max_tokens: 100,
|
|
115
|
-
temperature: 0,
|
|
116
|
-
system: TASK_TITLE_PROMPT,
|
|
117
|
-
messages: [{ role: "user", content: text }],
|
|
118
|
-
}),
|
|
119
|
-
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
120
|
-
});
|
|
121
|
-
if (!resp.ok) {
|
|
122
|
-
const body = await resp.text();
|
|
123
|
-
throw new Error(`Anthropic task-title failed (${resp.status}): ${body}`);
|
|
124
|
-
}
|
|
125
|
-
const json = (await resp.json());
|
|
126
|
-
return json.content.find((c) => c.type === "text")?.text?.trim() ?? "";
|
|
127
|
-
}
|
|
128
|
-
const TOPIC_JUDGE_PROMPT = `You are a conversation topic boundary detector. Given the CURRENT task context and a NEW user message, decide if the new message belongs to the SAME task or starts a NEW one.
|
|
129
|
-
|
|
130
|
-
Answer ONLY "NEW" or "SAME".
|
|
131
|
-
|
|
132
|
-
SAME — the new message:
|
|
133
|
-
- Continues, follows up on, refines, or corrects the same subject/project/task
|
|
134
|
-
- Asks a clarification or next-step question about what was just discussed
|
|
135
|
-
- Reports a result, error, or feedback about the current task
|
|
136
|
-
- Discusses different tools or approaches for the SAME goal (e.g., learning English via BBC → via ChatGPT = SAME)
|
|
137
|
-
- Is a short acknowledgment (ok, thanks, 好的) in response to the current flow
|
|
138
|
-
|
|
139
|
-
NEW — the new message:
|
|
140
|
-
- Introduces a subject from a DIFFERENT domain than the current task (e.g., tech → cooking, work → personal life, database → travel)
|
|
141
|
-
- Has NO logical connection to what was being discussed
|
|
142
|
-
- Starts a request about a different project, system, or life area
|
|
143
|
-
- Begins with a new greeting/reset followed by a different topic
|
|
144
|
-
|
|
145
|
-
Key principles:
|
|
146
|
-
- If the topic domain clearly changed (e.g., server config → recipe, code review → vacation plan), choose NEW
|
|
147
|
-
- Different aspects of the SAME project/system are SAME (e.g., Nginx SSL → Nginx gzip = SAME)
|
|
148
|
-
- Different unrelated technologies discussed independently are NEW (e.g., Redis config → cooking recipe = NEW)
|
|
149
|
-
- When unsure, lean toward SAME for closely related topics, but do NOT hesitate to mark NEW for obvious domain shifts
|
|
150
|
-
- Examples: "配置Nginx" → "加gzip压缩" = SAME; "配置Nginx" → "做红烧肉" = NEW; "MySQL配置" → "K8s部署" in same infra project = SAME; "部署服务器" → "年会安排" = NEW
|
|
151
|
-
|
|
152
|
-
Output exactly one word: NEW or SAME`;
|
|
153
|
-
async function judgeNewTopicAnthropic(currentContext, newMessage, cfg, log) {
|
|
154
|
-
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
155
|
-
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
156
|
-
const headers = {
|
|
157
|
-
"Content-Type": "application/json",
|
|
158
|
-
"x-api-key": cfg.apiKey ?? "",
|
|
159
|
-
"anthropic-version": "2023-06-01",
|
|
160
|
-
...cfg.headers,
|
|
161
|
-
};
|
|
162
|
-
const userContent = `CURRENT TASK CONTEXT:\n${currentContext}\n\n---\n\nNEW USER MESSAGE:\n${newMessage}`;
|
|
163
|
-
const resp = await fetch(endpoint, {
|
|
164
|
-
method: "POST",
|
|
165
|
-
headers,
|
|
166
|
-
body: JSON.stringify({
|
|
167
|
-
model,
|
|
168
|
-
max_tokens: 10,
|
|
169
|
-
temperature: 0,
|
|
170
|
-
system: TOPIC_JUDGE_PROMPT,
|
|
171
|
-
messages: [{ role: "user", content: userContent }],
|
|
172
|
-
}),
|
|
173
|
-
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
174
|
-
});
|
|
175
|
-
if (!resp.ok) {
|
|
176
|
-
const body = await resp.text();
|
|
177
|
-
throw new Error(`Anthropic topic-judge failed (${resp.status}): ${body}`);
|
|
178
|
-
}
|
|
179
|
-
const json = (await resp.json());
|
|
180
|
-
const answer = json.content.find((c) => c.type === "text")?.text?.trim().toUpperCase() ?? "";
|
|
181
|
-
log.debug(`Topic judge result: "${answer}"`);
|
|
182
|
-
return answer.startsWith("NEW");
|
|
183
|
-
}
|
|
184
|
-
const FILTER_RELEVANT_PROMPT = `You are a memory relevance judge.
|
|
185
|
-
|
|
186
|
-
Given a QUERY and CANDIDATE memories, decide: does each candidate's content contain information that would HELP ANSWER the query?
|
|
187
|
-
|
|
188
|
-
CORE QUESTION: "If I include this memory, will it help produce a better answer?"
|
|
189
|
-
- YES → include
|
|
190
|
-
- NO → exclude
|
|
191
|
-
|
|
192
|
-
RULES:
|
|
193
|
-
1. A candidate is relevant if its content provides facts, context, or data that directly supports answering the query.
|
|
194
|
-
2. A candidate that merely shares the same broad topic/domain but contains NO useful information for answering is NOT relevant.
|
|
195
|
-
3. If NO candidate can help answer the query, return {"relevant":[],"sufficient":false} — do NOT force-pick the "least irrelevant" one.
|
|
196
|
-
|
|
197
|
-
OUTPUT — JSON only:
|
|
198
|
-
{"relevant":[1,3],"sufficient":true}
|
|
199
|
-
- "relevant": candidate numbers whose content helps answer the query. [] if none can help.
|
|
200
|
-
- "sufficient": true only if the selected memories fully answer the query.`;
|
|
201
|
-
async function filterRelevantAnthropic(query, candidates, cfg, log) {
|
|
202
|
-
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
203
|
-
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
204
|
-
const headers = {
|
|
205
|
-
"Content-Type": "application/json",
|
|
206
|
-
"x-api-key": cfg.apiKey ?? "",
|
|
207
|
-
"anthropic-version": "2023-06-01",
|
|
208
|
-
...cfg.headers,
|
|
209
|
-
};
|
|
210
|
-
const candidateText = candidates
|
|
211
|
-
.map((c) => {
|
|
212
|
-
const timeTag = c.time ? ` (${c.time})` : "";
|
|
213
|
-
return `${c.index}. [${c.role}]${timeTag}\n ${c.content}`;
|
|
214
|
-
})
|
|
215
|
-
.join("\n");
|
|
216
|
-
const resp = await fetch(endpoint, {
|
|
217
|
-
method: "POST",
|
|
218
|
-
headers,
|
|
219
|
-
body: JSON.stringify({
|
|
220
|
-
model,
|
|
221
|
-
max_tokens: 200,
|
|
222
|
-
temperature: 0,
|
|
223
|
-
system: FILTER_RELEVANT_PROMPT,
|
|
224
|
-
messages: [{ role: "user", content: `QUERY: ${query}\n\nCANDIDATES:\n${candidateText}` }],
|
|
225
|
-
}),
|
|
226
|
-
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
227
|
-
});
|
|
228
|
-
if (!resp.ok) {
|
|
229
|
-
const body = await resp.text();
|
|
230
|
-
throw new Error(`Anthropic filter-relevant failed (${resp.status}): ${body}`);
|
|
231
|
-
}
|
|
232
|
-
const json = (await resp.json());
|
|
233
|
-
const raw = json.content.find((c) => c.type === "text")?.text?.trim() ?? "{}";
|
|
234
|
-
log.debug(`filterRelevant raw LLM response: "${raw}"`);
|
|
235
|
-
return parseFilterResult(raw, log);
|
|
236
|
-
}
|
|
237
|
-
function parseFilterResult(raw, log) {
|
|
238
|
-
try {
|
|
239
|
-
const match = raw.match(/\{[\s\S]*\}/);
|
|
240
|
-
if (match) {
|
|
241
|
-
const obj = JSON.parse(match[0]);
|
|
242
|
-
if (obj && Array.isArray(obj.relevant)) {
|
|
243
|
-
return {
|
|
244
|
-
relevant: obj.relevant.filter((n) => typeof n === "number"),
|
|
245
|
-
sufficient: obj.sufficient === true,
|
|
246
|
-
};
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
catch { }
|
|
251
|
-
log.warn(`filterRelevant: failed to parse LLM output: "${raw}", fallback to all+insufficient`);
|
|
252
|
-
return { relevant: [], sufficient: false };
|
|
253
|
-
}
|
|
254
|
-
async function summarizeAnthropic(text, cfg, log) {
|
|
255
|
-
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
256
|
-
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
257
|
-
const headers = {
|
|
258
|
-
"Content-Type": "application/json",
|
|
259
|
-
"x-api-key": cfg.apiKey ?? "",
|
|
260
|
-
"anthropic-version": "2023-06-01",
|
|
261
|
-
...cfg.headers,
|
|
262
|
-
};
|
|
263
|
-
const resp = await fetch(endpoint, {
|
|
264
|
-
method: "POST",
|
|
265
|
-
headers,
|
|
266
|
-
body: JSON.stringify({
|
|
267
|
-
model,
|
|
268
|
-
max_tokens: 100,
|
|
269
|
-
temperature: cfg.temperature ?? 0,
|
|
270
|
-
system: SYSTEM_PROMPT,
|
|
271
|
-
messages: [{ role: "user", content: `[TEXT TO SUMMARIZE]\n${text}\n[/TEXT TO SUMMARIZE]` }],
|
|
272
|
-
}),
|
|
273
|
-
signal: AbortSignal.timeout(cfg.timeoutMs ?? 30_000),
|
|
274
|
-
});
|
|
275
|
-
if (!resp.ok) {
|
|
276
|
-
const body = await resp.text();
|
|
277
|
-
throw new Error(`Anthropic summarize failed (${resp.status}): ${body}`);
|
|
278
|
-
}
|
|
279
|
-
const json = (await resp.json());
|
|
280
|
-
return json.content.find((c) => c.type === "text")?.text?.trim() ?? "";
|
|
281
|
-
}
|
|
282
|
-
// ─── Smart Dedup ───
|
|
283
|
-
const openai_1 = require("./openai");
|
|
284
|
-
async function judgeDedupAnthropic(newSummary, candidates, cfg, log) {
|
|
285
|
-
const endpoint = cfg.endpoint ?? "https://api.anthropic.com/v1/messages";
|
|
286
|
-
const model = cfg.model ?? "claude-3-haiku-20240307";
|
|
287
|
-
const headers = {
|
|
288
|
-
"Content-Type": "application/json",
|
|
289
|
-
"x-api-key": cfg.apiKey ?? "",
|
|
290
|
-
"anthropic-version": "2023-06-01",
|
|
291
|
-
...cfg.headers,
|
|
292
|
-
};
|
|
293
|
-
const candidateText = candidates.map((c) => `${c.index}. ${c.summary}`).join("\n");
|
|
294
|
-
const resp = await fetch(endpoint, {
|
|
295
|
-
method: "POST",
|
|
296
|
-
headers,
|
|
297
|
-
body: JSON.stringify({
|
|
298
|
-
model,
|
|
299
|
-
max_tokens: 300,
|
|
300
|
-
temperature: 0,
|
|
301
|
-
system: openai_1.DEDUP_JUDGE_PROMPT,
|
|
302
|
-
messages: [{ role: "user", content: `NEW MEMORY:\n${newSummary}\n\nEXISTING MEMORIES:\n${candidateText}` }],
|
|
303
|
-
}),
|
|
304
|
-
signal: AbortSignal.timeout(cfg.timeoutMs ?? 15_000),
|
|
305
|
-
});
|
|
306
|
-
if (!resp.ok) {
|
|
307
|
-
const body = await resp.text();
|
|
308
|
-
throw new Error(`Anthropic dedup-judge failed (${resp.status}): ${body}`);
|
|
309
|
-
}
|
|
310
|
-
const json = (await resp.json());
|
|
311
|
-
const raw = json.content.find((c) => c.type === "text")?.text?.trim() ?? "{}";
|
|
312
|
-
return (0, openai_1.parseDedupResult)(raw, log);
|
|
313
|
-
}
|
|
314
|
-
//# sourceMappingURL=anthropic.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../../src/ingest/providers/anthropic.ts"],"names":[],"mappings":";;AAwDA,wDAkCC;AAcD,gEAkCC;AA4BD,wDAuCC;AAuBD,0DA4CC;AAmBD,gDAoCC;AAQD,kDAsCC;AAnXD,MAAM,aAAa,GAAG;;;;;;;;;;;;;;oBAcF,CAAC;AAErB,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCAoCQ,CAAC;AAE9B,KAAK,UAAU,sBAAsB,CAC1C,IAAY,EACZ,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,GAAG;YACnC,MAAM,EAAE,mBAAmB;YAC3B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACzE,CAAC;AAED,MAAM,iBAAiB,GAAG;;;;;;;;;;8DAUoC,CAAC;AAExD,KAAK,UAAU,0BAA0B,CAC9C,IAAY,EACZ,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,iBAAiB;YACzB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACzE,CAAC;AAED,MAAM,kBAAkB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;qCAwBU,CAAC;AAE/B,KAAK,UAAU,sBAAsB,CAC1C,cAAsB,EACtB,UAAkB,EAClB,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,WAAW,GAAG,0BAA0B,cAAc,iCAAiC,UAAU,EAAE,CAAC;IAE1G,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,EAAE;YACd,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,kBAAkB;YAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;SACnD,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC7F,GAAG,CAAC,KAAK,CAAC,wBAAwB,MAAM,GAAG,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,sBAAsB,GAAG;;;;;;;;;;;;;;;;2EAgB4C,CAAC;AAKrE,KAAK,UAAU,uBAAuB,CAC3C,KAAa,EACb,UAAkF,EAClF,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,aAAa,GAAG,UAAU;SAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,OAAO,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;IAC9D,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,sBAAsB;YAC9B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,KAAK,oBAAoB,aAAa,EAAE,EAAE,CAAC;SAC1F,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAC9E,GAAG,CAAC,KAAK,CAAC,qCAAqC,GAAG,GAAG,CAAC,CAAC;IACvD,OAAO,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW,EAAE,GAAW;IACjD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACvC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvC,OAAO;oBACL,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;oBAChE,UAAU,EAAE,GAAG,CAAC,UAAU,KAAK,IAAI;iBACpC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IACV,GAAG,CAAC,IAAI,CAAC,gDAAgD,GAAG,iCAAiC,CAAC,CAAC;IAC/F,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAC7C,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,CAAC;YACjC,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,CAAC;SAC5F,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAE9B,CAAC;IACF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACzE,CAAC;AAED,sBAAsB;AAEtB,qCAAgE;AAIzD,KAAK,UAAU,mBAAmB,CACvC,UAAkB,EAClB,UAAsE,EACtE,GAAqB,EACrB,GAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,uCAAuC,CAAC;IACzE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,WAAW,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QAC7B,mBAAmB,EAAE,YAAY;QACjC,GAAG,GAAG,CAAC,OAAO;KACf,CAAC;IAEF,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnF,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,2BAAkB;YAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,UAAU,2BAA2B,aAAa,EAAE,EAAE,CAAC;SAC5G,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC;KACrD,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAuD,CAAC;IACvF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAC9E,OAAO,IAAA,yBAAgB,EAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACpC,CAAC"}
|