@clauderecallhq/cli 0.0.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +33 -0
- package/README.md +543 -3
- package/README.public.md +523 -0
- package/dist/cli.js +362 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands/activate.js +69 -0
- package/dist/commands/activate.js.map +1 -0
- package/dist/commands/audit-secrets.js +103 -0
- package/dist/commands/audit-secrets.js.map +1 -0
- package/dist/commands/blame.js +35 -0
- package/dist/commands/blame.js.map +1 -0
- package/dist/commands/config-verification.js +18 -0
- package/dist/commands/config-verification.js.map +1 -0
- package/dist/commands/context.js +144 -0
- package/dist/commands/context.js.map +1 -0
- package/dist/commands/correlate.js +70 -0
- package/dist/commands/correlate.js.map +1 -0
- package/dist/commands/digest.js +78 -0
- package/dist/commands/digest.js.map +1 -0
- package/dist/commands/health.js +62 -0
- package/dist/commands/health.js.map +1 -0
- package/dist/commands/index.js +247 -0
- package/dist/commands/index.js.map +1 -0
- package/dist/commands/install-extension.js +138 -0
- package/dist/commands/install-extension.js.map +1 -0
- package/dist/commands/installs.js +128 -0
- package/dist/commands/installs.js.map +1 -0
- package/dist/commands/license.js +39 -0
- package/dist/commands/license.js.map +1 -0
- package/dist/commands/list.js +47 -0
- package/dist/commands/list.js.map +1 -0
- package/dist/commands/mcp.js +29 -0
- package/dist/commands/mcp.js.map +1 -0
- package/dist/commands/open.js +28 -0
- package/dist/commands/open.js.map +1 -0
- package/dist/commands/paste.js +154 -0
- package/dist/commands/paste.js.map +1 -0
- package/dist/commands/projects.js +36 -0
- package/dist/commands/projects.js.map +1 -0
- package/dist/commands/search.js +67 -0
- package/dist/commands/search.js.map +1 -0
- package/dist/commands/semantic.js +173 -0
- package/dist/commands/semantic.js.map +1 -0
- package/dist/commands/show.js +121 -0
- package/dist/commands/show.js.map +1 -0
- package/dist/commands/start.js +47 -0
- package/dist/commands/start.js.map +1 -0
- package/dist/commands/stats.js +133 -0
- package/dist/commands/stats.js.map +1 -0
- package/dist/commands/status.js +45 -0
- package/dist/commands/status.js.map +1 -0
- package/dist/commands/stop.js +29 -0
- package/dist/commands/stop.js.map +1 -0
- package/dist/commands/thread.js +396 -0
- package/dist/commands/thread.js.map +1 -0
- package/dist/context/formatter.js +103 -0
- package/dist/context/formatter.js.map +1 -0
- package/dist/daemon/auto-tag-config.js +103 -0
- package/dist/daemon/auto-tag-config.js.map +1 -0
- package/dist/daemon/auto-tag-config.test.js +72 -0
- package/dist/daemon/auto-tag-config.test.js.map +1 -0
- package/dist/daemon/auto-title-config.js +70 -0
- package/dist/daemon/auto-title-config.js.map +1 -0
- package/dist/daemon/bulk-title-jobs.js +170 -0
- package/dist/daemon/bulk-title-jobs.js.map +1 -0
- package/dist/daemon/correlator.js +320 -0
- package/dist/daemon/correlator.js.map +1 -0
- package/dist/daemon/discover.js +316 -0
- package/dist/daemon/discover.js.map +1 -0
- package/dist/daemon/editor-detection.js +186 -0
- package/dist/daemon/editor-detection.js.map +1 -0
- package/dist/daemon/entrypoint.js +55 -0
- package/dist/daemon/entrypoint.js.map +1 -0
- package/dist/daemon/git-correlator.js +256 -0
- package/dist/daemon/git-correlator.js.map +1 -0
- package/dist/daemon/mcp-installer.js +108 -0
- package/dist/daemon/mcp-installer.js.map +1 -0
- package/dist/daemon/onboarding-state.js +140 -0
- package/dist/daemon/onboarding-state.js.map +1 -0
- package/dist/daemon/pidfile.js +57 -0
- package/dist/daemon/pidfile.js.map +1 -0
- package/dist/daemon/ports.js +48 -0
- package/dist/daemon/ports.js.map +1 -0
- package/dist/daemon/scanProgressRegistry.js +62 -0
- package/dist/daemon/scanProgressRegistry.js.map +1 -0
- package/dist/daemon/server.js +2010 -0
- package/dist/daemon/server.js.map +1 -0
- package/dist/daemon/tag-scanner/anthropic-client.js +40 -0
- package/dist/daemon/tag-scanner/anthropic-client.js.map +1 -0
- package/dist/daemon/tag-scanner/autopilot.js +131 -0
- package/dist/daemon/tag-scanner/autopilot.js.map +1 -0
- package/dist/daemon/tag-scanner/claude-cli-driver.js +250 -0
- package/dist/daemon/tag-scanner/claude-cli-driver.js.map +1 -0
- package/dist/daemon/tag-scanner/orchestrator.js +88 -0
- package/dist/daemon/tag-scanner/orchestrator.js.map +1 -0
- package/dist/daemon/tag-scanner/prompt.js +46 -0
- package/dist/daemon/tag-scanner/prompt.js.map +1 -0
- package/dist/daemon/tag-scanner/prompt.test.js +48 -0
- package/dist/daemon/tag-scanner/prompt.test.js.map +1 -0
- package/dist/daemon/tag-scanner/scan-state.js +49 -0
- package/dist/daemon/tag-scanner/scan-state.js.map +1 -0
- package/dist/daemon/tag-scanner/session-fetcher.js +82 -0
- package/dist/daemon/tag-scanner/session-fetcher.js.map +1 -0
- package/dist/daemon/tag-scanner/session-fetcher.test.js +34 -0
- package/dist/daemon/tag-scanner/session-fetcher.test.js.map +1 -0
- package/dist/daemon/tag-scanner/validator.js +50 -0
- package/dist/daemon/tag-scanner/validator.js.map +1 -0
- package/dist/daemon/tag-scanner/validator.test.js +41 -0
- package/dist/daemon/tag-scanner/validator.test.js.map +1 -0
- package/dist/daemon/terminal-registry.js +443 -0
- package/dist/daemon/terminal-registry.js.map +1 -0
- package/dist/daemon/ui.js +64 -0
- package/dist/daemon/ui.js.map +1 -0
- package/dist/daemon/watcher.js +256 -0
- package/dist/daemon/watcher.js.map +1 -0
- package/dist/db/client.js +22 -0
- package/dist/db/client.js.map +1 -0
- package/dist/db/schema.js +496 -0
- package/dist/db/schema.js.map +1 -0
- package/dist/license/api-base.js +13 -0
- package/dist/license/api-base.js.map +1 -0
- package/dist/license/manager.js +43 -0
- package/dist/license/manager.js.map +1 -0
- package/dist/license/public-key.js +19 -0
- package/dist/license/public-key.js.map +1 -0
- package/dist/license/storage.js +27 -0
- package/dist/license/storage.js.map +1 -0
- package/dist/license/verify.js +23 -0
- package/dist/license/verify.js.map +1 -0
- package/dist/mcp/audit.js +126 -0
- package/dist/mcp/audit.js.map +1 -0
- package/dist/mcp/prompts.js +180 -0
- package/dist/mcp/prompts.js.map +1 -0
- package/dist/mcp/server.js +502 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/thread-tools.js +363 -0
- package/dist/mcp/thread-tools.js.map +1 -0
- package/dist/mcp/write-tools.js +239 -0
- package/dist/mcp/write-tools.js.map +1 -0
- package/dist/parser/jsonl.js +150 -0
- package/dist/parser/jsonl.js.map +1 -0
- package/dist/semantic/chunker.js +47 -0
- package/dist/semantic/chunker.js.map +1 -0
- package/dist/semantic/config.js +74 -0
- package/dist/semantic/config.js.map +1 -0
- package/dist/semantic/embedder.js +54 -0
- package/dist/semantic/embedder.js.map +1 -0
- package/dist/semantic/fusion.js +38 -0
- package/dist/semantic/fusion.js.map +1 -0
- package/dist/semantic/model-download.js +69 -0
- package/dist/semantic/model-download.js.map +1 -0
- package/dist/semantic/pipeline.js +375 -0
- package/dist/semantic/pipeline.js.map +1 -0
- package/dist/semantic/query.js +42 -0
- package/dist/semantic/query.js.map +1 -0
- package/dist/semantic/worker.js +78 -0
- package/dist/semantic/worker.js.map +1 -0
- package/dist/stats/backfill.js +151 -0
- package/dist/stats/backfill.js.map +1 -0
- package/dist/stats/health.js +102 -0
- package/dist/stats/health.js.map +1 -0
- package/dist/stats/query.js +385 -0
- package/dist/stats/query.js.map +1 -0
- package/dist/utils/aliases.js +107 -0
- package/dist/utils/aliases.js.map +1 -0
- package/dist/utils/autoCollections.js +635 -0
- package/dist/utils/autoCollections.js.map +1 -0
- package/dist/utils/autoTitle.js +348 -0
- package/dist/utils/autoTitle.js.map +1 -0
- package/dist/utils/collections.js +446 -0
- package/dist/utils/collections.js.map +1 -0
- package/dist/utils/format.js +46 -0
- package/dist/utils/format.js.map +1 -0
- package/dist/utils/notes.js +270 -0
- package/dist/utils/notes.js.map +1 -0
- package/dist/utils/paths.js +50 -0
- package/dist/utils/paths.js.map +1 -0
- package/dist/utils/pricing.js +257 -0
- package/dist/utils/pricing.js.map +1 -0
- package/dist/utils/secret-scanner.js +166 -0
- package/dist/utils/secret-scanner.js.map +1 -0
- package/dist/utils/sessionLabel.js +64 -0
- package/dist/utils/sessionLabel.js.map +1 -0
- package/dist/utils/tags.js +97 -0
- package/dist/utils/tags.js.map +1 -0
- package/dist/utils/thread-context.js +129 -0
- package/dist/utils/thread-context.js.map +1 -0
- package/dist/utils/threadFilter.js +18 -0
- package/dist/utils/threadFilter.js.map +1 -0
- package/dist/utils/threads-titler.js +298 -0
- package/dist/utils/threads-titler.js.map +1 -0
- package/dist/utils/threads.js +383 -0
- package/dist/utils/threads.js.map +1 -0
- package/dist/utils/usage.js +76 -0
- package/dist/utils/usage.js.map +1 -0
- package/dist/verification/compute.js +88 -0
- package/dist/verification/compute.js.map +1 -0
- package/dist/verification/config.js +34 -0
- package/dist/verification/config.js.map +1 -0
- package/dist/web/assets/index-CIr6J4Fw.js +1201 -0
- package/dist/web/assets/index-Ctc8g9Jw.css +1 -0
- package/dist/web/assets/inter-cyrillic-400-normal-HOLc17fK.woff +0 -0
- package/dist/web/assets/inter-cyrillic-400-normal-obahsSVq.woff2 +0 -0
- package/dist/web/assets/inter-cyrillic-500-normal-BasfLYem.woff2 +0 -0
- package/dist/web/assets/inter-cyrillic-500-normal-CxZf_p3X.woff +0 -0
- package/dist/web/assets/inter-cyrillic-600-normal-4D_pXhcN.woff +0 -0
- package/dist/web/assets/inter-cyrillic-600-normal-CWCymEST.woff2 +0 -0
- package/dist/web/assets/inter-cyrillic-700-normal-CjBOestx.woff2 +0 -0
- package/dist/web/assets/inter-cyrillic-700-normal-DrXBdSj3.woff +0 -0
- package/dist/web/assets/inter-cyrillic-ext-400-normal-BQZuk6qB.woff2 +0 -0
- package/dist/web/assets/inter-cyrillic-ext-400-normal-DQukG94-.woff +0 -0
- package/dist/web/assets/inter-cyrillic-ext-500-normal-B0yAr1jD.woff2 +0 -0
- package/dist/web/assets/inter-cyrillic-ext-500-normal-BmqWE9Dz.woff +0 -0
- package/dist/web/assets/inter-cyrillic-ext-600-normal-Bcila6Z-.woff +0 -0
- package/dist/web/assets/inter-cyrillic-ext-600-normal-Dfes3d0z.woff2 +0 -0
- package/dist/web/assets/inter-cyrillic-ext-700-normal-BjwYoWNd.woff2 +0 -0
- package/dist/web/assets/inter-cyrillic-ext-700-normal-LO58E6JB.woff +0 -0
- package/dist/web/assets/inter-greek-400-normal-B4URO6DV.woff2 +0 -0
- package/dist/web/assets/inter-greek-400-normal-q2sYcFCs.woff +0 -0
- package/dist/web/assets/inter-greek-500-normal-BIZE56-Y.woff2 +0 -0
- package/dist/web/assets/inter-greek-500-normal-Xzm54t5V.woff +0 -0
- package/dist/web/assets/inter-greek-600-normal-BZpKdvQh.woff +0 -0
- package/dist/web/assets/inter-greek-600-normal-plRanbMR.woff2 +0 -0
- package/dist/web/assets/inter-greek-700-normal-BUv2fZ6O.woff +0 -0
- package/dist/web/assets/inter-greek-700-normal-C3JjAnD8.woff2 +0 -0
- package/dist/web/assets/inter-greek-ext-400-normal-DGGRlc-M.woff2 +0 -0
- package/dist/web/assets/inter-greek-ext-400-normal-KugGGMne.woff +0 -0
- package/dist/web/assets/inter-greek-ext-500-normal-2j5mBUwD.woff +0 -0
- package/dist/web/assets/inter-greek-ext-500-normal-C4iEst2y.woff2 +0 -0
- package/dist/web/assets/inter-greek-ext-600-normal-B8X0CLgF.woff +0 -0
- package/dist/web/assets/inter-greek-ext-600-normal-DRtmH8MT.woff2 +0 -0
- package/dist/web/assets/inter-greek-ext-700-normal-BoQ6DsYi.woff +0 -0
- package/dist/web/assets/inter-greek-ext-700-normal-qfdV9bQt.woff2 +0 -0
- package/dist/web/assets/inter-latin-400-normal-C38fXH4l.woff2 +0 -0
- package/dist/web/assets/inter-latin-400-normal-CyCys3Eg.woff +0 -0
- package/dist/web/assets/inter-latin-500-normal-BL9OpVg8.woff +0 -0
- package/dist/web/assets/inter-latin-500-normal-Cerq10X2.woff2 +0 -0
- package/dist/web/assets/inter-latin-600-normal-CiBQ2DWP.woff +0 -0
- package/dist/web/assets/inter-latin-600-normal-LgqL8muc.woff2 +0 -0
- package/dist/web/assets/inter-latin-700-normal-BLAVimhd.woff +0 -0
- package/dist/web/assets/inter-latin-700-normal-Yt3aPRUw.woff2 +0 -0
- package/dist/web/assets/inter-latin-ext-400-normal-77YHD8bZ.woff +0 -0
- package/dist/web/assets/inter-latin-ext-400-normal-C1nco2VV.woff2 +0 -0
- package/dist/web/assets/inter-latin-ext-500-normal-BxGbmqWO.woff +0 -0
- package/dist/web/assets/inter-latin-ext-500-normal-CV4jyFjo.woff2 +0 -0
- package/dist/web/assets/inter-latin-ext-600-normal-CIVaiw4L.woff +0 -0
- package/dist/web/assets/inter-latin-ext-600-normal-D2bJ5OIk.woff2 +0 -0
- package/dist/web/assets/inter-latin-ext-700-normal-Ca8adRJv.woff2 +0 -0
- package/dist/web/assets/inter-latin-ext-700-normal-TidjK2hL.woff +0 -0
- package/dist/web/assets/inter-vietnamese-400-normal-Bbgyi5SW.woff +0 -0
- package/dist/web/assets/inter-vietnamese-400-normal-DMkecbls.woff2 +0 -0
- package/dist/web/assets/inter-vietnamese-500-normal-DOriooB6.woff2 +0 -0
- package/dist/web/assets/inter-vietnamese-500-normal-mJboJaSs.woff +0 -0
- package/dist/web/assets/inter-vietnamese-600-normal-BuLX-rYi.woff +0 -0
- package/dist/web/assets/inter-vietnamese-600-normal-Cc8MFFhd.woff2 +0 -0
- package/dist/web/assets/inter-vietnamese-700-normal-BZaoP0fm.woff +0 -0
- package/dist/web/assets/inter-vietnamese-700-normal-DlLaEgI2.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-cyrillic-400-normal-BEIGL1Tu.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-cyrillic-400-normal-ugxPyKxw.woff +0 -0
- package/dist/web/assets/jetbrains-mono-cyrillic-500-normal-DJqRU3vO.woff +0 -0
- package/dist/web/assets/jetbrains-mono-cyrillic-500-normal-DmUKJPL_.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-greek-400-normal-B9oWc5Lo.woff +0 -0
- package/dist/web/assets/jetbrains-mono-greek-400-normal-C190GLew.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-greek-500-normal-D7SFKleX.woff +0 -0
- package/dist/web/assets/jetbrains-mono-greek-500-normal-JpySY46c.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-latin-400-normal-6-qcROiO.woff +0 -0
- package/dist/web/assets/jetbrains-mono-latin-400-normal-V6pRDFza.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-latin-500-normal-BWZEU5yA.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-latin-500-normal-CJOVTJB7.woff +0 -0
- package/dist/web/assets/jetbrains-mono-latin-ext-400-normal-Bc8Ftmh3.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-latin-ext-400-normal-fXTG6kC5.woff +0 -0
- package/dist/web/assets/jetbrains-mono-latin-ext-500-normal-Cut-4mMH.woff2 +0 -0
- package/dist/web/assets/jetbrains-mono-latin-ext-500-normal-ckzbgY84.woff +0 -0
- package/dist/web/assets/jetbrains-mono-vietnamese-400-normal-CqNFfHCs.woff +0 -0
- package/dist/web/assets/jetbrains-mono-vietnamese-500-normal-DNRqzVM1.woff +0 -0
- package/dist/web/favicon.svg +9 -0
- package/dist/web/index.html +15 -0
- package/package.json +80 -9
- package/bin/cli.js +0 -12
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { createReadStream } from 'node:fs';
|
|
2
|
+
import { createInterface } from 'node:readline';
|
|
3
|
+
function num(v) {
|
|
4
|
+
return typeof v === 'number' && Number.isFinite(v) ? v : 0;
|
|
5
|
+
}
|
|
6
|
+
export function extractUsage(msg) {
|
|
7
|
+
const u = msg?.usage;
|
|
8
|
+
if (!u || typeof u !== 'object')
|
|
9
|
+
return null;
|
|
10
|
+
const usage = {
|
|
11
|
+
inputTokens: num(u.input_tokens),
|
|
12
|
+
outputTokens: num(u.output_tokens),
|
|
13
|
+
cacheCreateTokens: num(u.cache_creation_input_tokens),
|
|
14
|
+
cacheReadTokens: num(u.cache_read_input_tokens),
|
|
15
|
+
};
|
|
16
|
+
// Treat an all-zero usage block as "absent" so the rollup doesn't get
|
|
17
|
+
// polluted with meaningless rows.
|
|
18
|
+
if (usage.inputTokens === 0 &&
|
|
19
|
+
usage.outputTokens === 0 &&
|
|
20
|
+
usage.cacheCreateTokens === 0 &&
|
|
21
|
+
usage.cacheReadTokens === 0) {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
return usage;
|
|
25
|
+
}
|
|
26
|
+
// ANSI color escapes that sneak into tool output and render as garbage in HTML.
|
|
27
|
+
// eslint-disable-next-line no-control-regex
|
|
28
|
+
const ANSI_REGEX = /\x1B\[[0-9;]*[a-zA-Z]/g;
|
|
29
|
+
function stripAnsi(s) {
|
|
30
|
+
return s.replace(ANSI_REGEX, '');
|
|
31
|
+
}
|
|
32
|
+
// Cap any single piece of extracted content at this size when building the
|
|
33
|
+
// searchable/displayable `contentText`. The *full* payload is always preserved
|
|
34
|
+
// in `raw_json` — this cap only affects the synthesized text used for display
|
|
35
|
+
// and FTS indexing, so ripping out the DB and re-indexing restores everything.
|
|
36
|
+
const MAX_CHUNK_CHARS = 12_000;
|
|
37
|
+
function truncateChunk(s, label) {
|
|
38
|
+
if (s.length <= MAX_CHUNK_CHARS)
|
|
39
|
+
return s;
|
|
40
|
+
const head = s.slice(0, MAX_CHUNK_CHARS);
|
|
41
|
+
const omitted = s.length - MAX_CHUNK_CHARS;
|
|
42
|
+
return `${head}\n\n⟨… ${omitted.toLocaleString()} more chars in ${label}; see raw JSONL for full content ⟩`;
|
|
43
|
+
}
|
|
44
|
+
function jsonPretty(value) {
|
|
45
|
+
try {
|
|
46
|
+
return JSON.stringify(value, null, 2);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return String(value);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
function extractToolResultContent(block) {
|
|
53
|
+
if (typeof block.content === 'string')
|
|
54
|
+
return block.content;
|
|
55
|
+
if (Array.isArray(block.content)) {
|
|
56
|
+
const parts = [];
|
|
57
|
+
for (const c of block.content) {
|
|
58
|
+
if (c && typeof c === 'object') {
|
|
59
|
+
const cb = c;
|
|
60
|
+
if (cb.type === 'text' && typeof cb.text === 'string')
|
|
61
|
+
parts.push(cb.text);
|
|
62
|
+
else if (cb.type === 'image')
|
|
63
|
+
parts.push('[image]');
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return parts.join('\n');
|
|
67
|
+
}
|
|
68
|
+
return '';
|
|
69
|
+
}
|
|
70
|
+
export function extractContentText(msg) {
|
|
71
|
+
if (!msg)
|
|
72
|
+
return { text: '', toolNames: [] };
|
|
73
|
+
if (typeof msg.content === 'string') {
|
|
74
|
+
return { text: stripAnsi(msg.content), toolNames: [] };
|
|
75
|
+
}
|
|
76
|
+
if (!Array.isArray(msg.content))
|
|
77
|
+
return { text: '', toolNames: [] };
|
|
78
|
+
const parts = [];
|
|
79
|
+
const tools = [];
|
|
80
|
+
for (const block of msg.content) {
|
|
81
|
+
if (!block || typeof block !== 'object')
|
|
82
|
+
continue;
|
|
83
|
+
if (block.type === 'text' && typeof block.text === 'string') {
|
|
84
|
+
parts.push(stripAnsi(block.text));
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (block.type === 'tool_use' && typeof block.name === 'string') {
|
|
88
|
+
tools.push(block.name);
|
|
89
|
+
const input = block.input != null ? jsonPretty(block.input) : '';
|
|
90
|
+
const capped = truncateChunk(input, 'tool input');
|
|
91
|
+
parts.push(`⚡ **Tool call · \`${block.name}\`**\n\n\`\`\`json\n${capped}\n\`\`\``);
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
if (block.type === 'tool_result') {
|
|
95
|
+
const content = stripAnsi(extractToolResultContent(block));
|
|
96
|
+
if (content) {
|
|
97
|
+
const capped = truncateChunk(content, 'tool result');
|
|
98
|
+
// Use a fenced block with no language so we don't mis-highlight raw output.
|
|
99
|
+
parts.push(`**Tool result**\n\n\`\`\`\n${capped}\n\`\`\``);
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
parts.push('_(tool result was empty or image-only)_');
|
|
103
|
+
}
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (block.type === 'image') {
|
|
107
|
+
parts.push('_(image)_');
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
// Unknown block — don't silently drop, show it as JSON so we can debug.
|
|
111
|
+
parts.push(`_(unknown block: ${block.type})_`);
|
|
112
|
+
}
|
|
113
|
+
return { text: parts.join('\n\n'), toolNames: tools };
|
|
114
|
+
}
|
|
115
|
+
export async function* parseSessionFile(filePath) {
|
|
116
|
+
const stream = createReadStream(filePath, { encoding: 'utf8' });
|
|
117
|
+
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
|
118
|
+
for await (const line of rl) {
|
|
119
|
+
if (!line.trim())
|
|
120
|
+
continue;
|
|
121
|
+
let raw;
|
|
122
|
+
try {
|
|
123
|
+
raw = JSON.parse(line);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
if (!raw.uuid || !raw.sessionId)
|
|
129
|
+
continue;
|
|
130
|
+
const { text, toolNames } = extractContentText(raw.message);
|
|
131
|
+
yield {
|
|
132
|
+
uuid: raw.uuid,
|
|
133
|
+
parentUuid: raw.parentUuid ?? null,
|
|
134
|
+
sessionId: raw.sessionId,
|
|
135
|
+
type: raw.type ?? 'unknown',
|
|
136
|
+
role: raw.message?.role ?? null,
|
|
137
|
+
timestamp: raw.timestamp ?? null,
|
|
138
|
+
isSidechain: raw.isSidechain === true,
|
|
139
|
+
cwd: raw.cwd ?? null,
|
|
140
|
+
gitBranch: raw.gitBranch ?? null,
|
|
141
|
+
version: raw.version ?? null,
|
|
142
|
+
contentText: text,
|
|
143
|
+
toolNames,
|
|
144
|
+
raw: line,
|
|
145
|
+
usage: extractUsage(raw.message),
|
|
146
|
+
model: raw.message?.model ?? null,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
//# sourceMappingURL=jsonl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/parser/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAkEhD,SAAS,GAAG,CAAC,CAAU;IACrB,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAA2B;IACtD,MAAM,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC;IACrB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7C,MAAM,KAAK,GAAG;QACZ,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC;QAChC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC;QAClC,iBAAiB,EAAE,GAAG,CAAC,CAAC,CAAC,2BAA2B,CAAC;QACrD,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,uBAAuB,CAAC;KAChD,CAAC;IACF,sEAAsE;IACtE,kCAAkC;IAClC,IACE,KAAK,CAAC,WAAW,KAAK,CAAC;QACvB,KAAK,CAAC,YAAY,KAAK,CAAC;QACxB,KAAK,CAAC,iBAAiB,KAAK,CAAC;QAC7B,KAAK,CAAC,eAAe,KAAK,CAAC,EAC3B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gFAAgF;AAChF,4CAA4C;AAC5C,MAAM,UAAU,GAAG,wBAAwB,CAAC;AAE5C,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,2EAA2E;AAC3E,+EAA+E;AAC/E,8EAA8E;AAC9E,+EAA+E;AAC/E,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B,SAAS,aAAa,CAAC,CAAS,EAAE,KAAa;IAC7C,IAAI,CAAC,CAAC,MAAM,IAAI,eAAe;QAAE,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,GAAG,eAAe,CAAC;IAC3C,OAAO,GAAG,IAAI,UAAU,OAAO,CAAC,cAAc,EAAE,kBAAkB,KAAK,oCAAoC,CAAC;AAC9G,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,KAAmB;IACnD,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAC5D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACjC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,EAAE,GAAG,CAAiB,CAAC;gBAC7B,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,EAAE,CAAC,IAAI,KAAK,QAAQ;oBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;qBACtE,IAAI,EAAE,CAAC,IAAI,KAAK,OAAO;oBAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,GAA2B;IAI5D,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC7C,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAEpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAChC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,SAAS;QAElD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAClC,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAChE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;YAClD,KAAK,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,IAAI,uBAAuB,MAAM,UAAU,CAAC,CAAC;YACnF,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,SAAS,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3D,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;gBACrD,4EAA4E;gBAC5E,KAAK,CAAC,IAAI,CAAC,8BAA8B,MAAM,UAAU,CAAC,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACxD,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxB,SAAS;QACX,CAAC;QAED,wEAAwE;QACxE,KAAK,CAAC,IAAI,CAAC,oBAAoB,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CACrC,QAAgB;IAEhB,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IAChE,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEnE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAC3B,IAAI,GAAa,CAAC;QAClB,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS;YAAE,SAAS;QAE1C,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC5D,MAAM;YACJ,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,IAAI;YAClC,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,SAAS;YAC3B,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,IAAI;YAC/B,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,IAAI;YAChC,WAAW,EAAE,GAAG,CAAC,WAAW,KAAK,IAAI;YACrC,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,IAAI;YACpB,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,IAAI;YAChC,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI;YAC5B,WAAW,EAAE,IAAI;YACjB,SAAS;YACT,GAAG,EAAE,IAAI;YACT,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC;YAChC,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI;SAClC,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { getDb } from '../db/client.js';
|
|
2
|
+
function stripToolJson(text) {
|
|
3
|
+
return text.replace(/```json[\s\S]*?```/g, '[tool-call]').replace(/\{[\s\S]{200,}?\}/g, '[json-object]');
|
|
4
|
+
}
|
|
5
|
+
function buildChunks(messages) {
|
|
6
|
+
const chunks = [];
|
|
7
|
+
const CHUNK_MIN = 3;
|
|
8
|
+
const CHUNK_MAX = 5;
|
|
9
|
+
const CHAR_CAP = 2000;
|
|
10
|
+
let i = 0;
|
|
11
|
+
while (i < messages.length) {
|
|
12
|
+
const chunkMsgs = [];
|
|
13
|
+
let charCount = 0;
|
|
14
|
+
while (chunkMsgs.length < CHUNK_MAX && i < messages.length) {
|
|
15
|
+
const msg = messages[i];
|
|
16
|
+
const cleaned = stripToolJson(msg.content_text ?? '');
|
|
17
|
+
if (charCount + cleaned.length > CHAR_CAP && chunkMsgs.length >= CHUNK_MIN)
|
|
18
|
+
break;
|
|
19
|
+
chunkMsgs.push(msg);
|
|
20
|
+
charCount += cleaned.length;
|
|
21
|
+
i++;
|
|
22
|
+
}
|
|
23
|
+
if (chunkMsgs.length === 0)
|
|
24
|
+
break;
|
|
25
|
+
const text = chunkMsgs.map((m) => {
|
|
26
|
+
const role = m.role ?? 'system';
|
|
27
|
+
const cleaned = stripToolJson(m.content_text ?? '');
|
|
28
|
+
return `[${role}] ${cleaned}`;
|
|
29
|
+
}).join('\n\n');
|
|
30
|
+
chunks.push({ messageUuids: chunkMsgs.map((m) => m.uuid), text });
|
|
31
|
+
if (i < messages.length && chunkMsgs.length >= CHUNK_MIN)
|
|
32
|
+
i = Math.max(i - 1, i - 1);
|
|
33
|
+
}
|
|
34
|
+
return chunks;
|
|
35
|
+
}
|
|
36
|
+
export function chunkSession(sessionId) {
|
|
37
|
+
const db = getDb();
|
|
38
|
+
const messages = db.prepare('SELECT uuid, role, content_text FROM messages WHERE session_id = ? AND is_sidechain = 0 AND content_text IS NOT NULL ORDER BY rowid').all(sessionId);
|
|
39
|
+
return buildChunks(messages);
|
|
40
|
+
}
|
|
41
|
+
export function chunkMessages(sessionId, uuids) {
|
|
42
|
+
const db = getDb();
|
|
43
|
+
const placeholders = uuids.map(() => '?').join(',');
|
|
44
|
+
const messages = db.prepare(`SELECT uuid, role, content_text FROM messages WHERE session_id = ? AND uuid IN (${placeholders}) AND is_sidechain = 0 AND content_text IS NOT NULL ORDER BY rowid`).all(sessionId, ...uuids);
|
|
45
|
+
return buildChunks(messages);
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/semantic/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAaxC,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,eAAe,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,WAAW,CAAC,QAAsB;IACzC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,CAAC,CAAC;IACpB,MAAM,SAAS,GAAG,CAAC,CAAC;IACpB,MAAM,QAAQ,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAiB,EAAE,CAAC;QACnC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,OAAO,SAAS,CAAC,MAAM,GAAG,SAAS,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC3D,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;YACtD,IAAI,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,QAAQ,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS;gBAAE,MAAM;YAClF,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;YAC5B,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAClC,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC;YAChC,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;YACpD,OAAO,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS;YAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CACzB,qIAAqI,CACtI,CAAC,GAAG,CAAC,SAAS,CAAiB,CAAC;IACjC,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE,KAAe;IAC9D,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CACzB,mFAAmF,YAAY,oEAAoE,CACpK,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,KAAK,CAAiB,CAAC;IAC3C,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
/**
|
|
6
|
+
* v0.11 — semantic search config. Persisted as `{ semantic: {...} }` inside
|
|
7
|
+
* `$RECALL_HOME/config.json` (same file as auto-tag config). Off by default.
|
|
8
|
+
*
|
|
9
|
+
* `enabled` — master switch. When true, the watcher fires the pipeline
|
|
10
|
+
* on session close and `recall semantic backfill` works.
|
|
11
|
+
* `model` — claude CLI model passed to `--model`. Falls back to the
|
|
12
|
+
* user's claude default when unset.
|
|
13
|
+
* `ratePerMinute` — token-bucket cap so we never hammer the user's plan. The
|
|
14
|
+
* pipeline waits for capacity rather than dropping work.
|
|
15
|
+
* `lastProcessedSessionId` — backfill cursor for resumability. Updated after
|
|
16
|
+
* each successful session so a re-run picks up where it
|
|
17
|
+
* left off.
|
|
18
|
+
*/
|
|
19
|
+
function recallHome() {
|
|
20
|
+
return process.env.RECALL_HOME ?? join(homedir(), '.recall');
|
|
21
|
+
}
|
|
22
|
+
function ensureHome() {
|
|
23
|
+
const home = recallHome();
|
|
24
|
+
if (!existsSync(home))
|
|
25
|
+
mkdirSync(home, { recursive: true });
|
|
26
|
+
}
|
|
27
|
+
function configFile() {
|
|
28
|
+
return join(recallHome(), 'config.json');
|
|
29
|
+
}
|
|
30
|
+
export const SemanticConfigSchema = z.object({
|
|
31
|
+
enabled: z.boolean().default(false),
|
|
32
|
+
model: z.string().optional(),
|
|
33
|
+
ratePerMinute: z.number().int().min(1).max(600).default(30),
|
|
34
|
+
lastProcessedSessionId: z.string().nullable().default(null),
|
|
35
|
+
backfillPaused: z.boolean().default(false),
|
|
36
|
+
});
|
|
37
|
+
const DEFAULTS = {
|
|
38
|
+
enabled: false,
|
|
39
|
+
ratePerMinute: 30,
|
|
40
|
+
lastProcessedSessionId: null,
|
|
41
|
+
backfillPaused: false,
|
|
42
|
+
};
|
|
43
|
+
function readFileShape() {
|
|
44
|
+
const path = configFile();
|
|
45
|
+
if (!existsSync(path))
|
|
46
|
+
return {};
|
|
47
|
+
try {
|
|
48
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
console.error('[semantic-config] failed to parse config.json, using defaults:', err);
|
|
52
|
+
return {};
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export function readSemanticConfig() {
|
|
56
|
+
const raw = readFileShape().semantic;
|
|
57
|
+
if (!raw)
|
|
58
|
+
return { ...DEFAULTS };
|
|
59
|
+
const parsed = SemanticConfigSchema.safeParse({ ...DEFAULTS, ...raw });
|
|
60
|
+
return parsed.success ? parsed.data : { ...DEFAULTS };
|
|
61
|
+
}
|
|
62
|
+
export function writeSemanticConfig(patch) {
|
|
63
|
+
ensureHome();
|
|
64
|
+
const existing = readFileShape();
|
|
65
|
+
const merged = SemanticConfigSchema.parse({
|
|
66
|
+
...DEFAULTS,
|
|
67
|
+
...(existing.semantic ?? {}),
|
|
68
|
+
...patch,
|
|
69
|
+
});
|
|
70
|
+
const next = { ...existing, semantic: merged };
|
|
71
|
+
writeFileSync(configFile(), JSON.stringify(next, null, 2));
|
|
72
|
+
return merged;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/semantic/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;;;;;;;;;GAaG;AAEH,SAAS,UAAU;IACjB,OAAO,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,UAAU;IACjB,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;IAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,aAAa,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3D,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3D,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;CAC3C,CAAC,CAAC;AAIH,MAAM,QAAQ,GAAmB;IAC/B,OAAO,EAAE,KAAK;IACd,aAAa,EAAE,EAAE;IACjB,sBAAsB,EAAE,IAAI;IAC5B,cAAc,EAAE,KAAK;CACtB,CAAC;AAOF,SAAS,aAAa;IACpB,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;IAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAc,CAAC;IAC7D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,gEAAgE,EAAE,GAAG,CAAC,CAAC;QACrF,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC,QAAQ,CAAC;IACrC,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG,oBAAoB,CAAC,SAAS,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;IACvE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAA8B;IAChE,UAAU,EAAE,CAAC;IACb,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG,oBAAoB,CAAC,KAAK,CAAC;QACxC,GAAG,QAAQ;QACX,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;QAC5B,GAAG,KAAK;KACT,CAAC,CAAC;IACH,MAAM,IAAI,GAAc,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAC1D,aAAa,CAAC,UAAU,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { RECALL_HOME } from '../utils/paths.js';
|
|
3
|
+
const MODEL_ID = 'BAAI/bge-base-en-v1.5';
|
|
4
|
+
const EMBEDDING_DIM = 768;
|
|
5
|
+
const BATCH_SIZE = 16;
|
|
6
|
+
const QUERY_PREFIX = 'Represent this sentence for searching relevant passages: ';
|
|
7
|
+
let pipelineInstance = null;
|
|
8
|
+
let loaded = false;
|
|
9
|
+
function getModelDir() {
|
|
10
|
+
return join(RECALL_HOME, 'models', 'bge-base-en-v1.5');
|
|
11
|
+
}
|
|
12
|
+
export async function loadEmbedder() {
|
|
13
|
+
if (loaded && pipelineInstance)
|
|
14
|
+
return;
|
|
15
|
+
const { pipeline, env } = await import('@huggingface/transformers');
|
|
16
|
+
env.localModelPath = join(RECALL_HOME, 'models');
|
|
17
|
+
env.allowRemoteModels = false;
|
|
18
|
+
const pipe = await pipeline('feature-extraction', MODEL_ID, {
|
|
19
|
+
local_files_only: true,
|
|
20
|
+
cache_dir: getModelDir(),
|
|
21
|
+
});
|
|
22
|
+
pipelineInstance = pipe;
|
|
23
|
+
loaded = true;
|
|
24
|
+
}
|
|
25
|
+
export function getEmbedderStatus() {
|
|
26
|
+
return { loaded, modelId: MODEL_ID, dim: EMBEDDING_DIM };
|
|
27
|
+
}
|
|
28
|
+
export async function embed(texts) {
|
|
29
|
+
if (!pipelineInstance) {
|
|
30
|
+
throw new Error('[embedder] Model not loaded. Call loadEmbedder() before embedding.');
|
|
31
|
+
}
|
|
32
|
+
const results = [];
|
|
33
|
+
for (let i = 0; i < texts.length; i += BATCH_SIZE) {
|
|
34
|
+
const batch = texts.slice(i, i + BATCH_SIZE);
|
|
35
|
+
const output = await pipelineInstance(batch, { pooling: 'cls', normalize: true });
|
|
36
|
+
const nested = output.tolist();
|
|
37
|
+
for (let j = 0; j < batch.length; j++) {
|
|
38
|
+
const row = nested[j];
|
|
39
|
+
const flat = Array.isArray(row[0]) ? row[0] : row;
|
|
40
|
+
results.push(new Float32Array(flat));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return results;
|
|
44
|
+
}
|
|
45
|
+
export async function embedQuery(text) {
|
|
46
|
+
const prefixed = QUERY_PREFIX + text;
|
|
47
|
+
const [result] = await embed([prefixed]);
|
|
48
|
+
return result;
|
|
49
|
+
}
|
|
50
|
+
export function unloadEmbedder() {
|
|
51
|
+
pipelineInstance = null;
|
|
52
|
+
loaded = false;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/semantic/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,QAAQ,GAAG,uBAAuB,CAAC;AACzC,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,YAAY,GAAG,2DAA2D,CAAC;AAOjF,IAAI,gBAAgB,GAAqC,IAAI,CAAC;AAC9D,IAAI,MAAM,GAAG,KAAK,CAAC;AAEnB,SAAS,WAAW;IAClB,OAAO,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,MAAM,IAAI,gBAAgB;QAAE,OAAO;IACvC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;IACpE,GAAG,CAAC,cAAc,GAAG,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IACjD,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC;IAC9B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,EAAE;QAC1D,gBAAgB,EAAE,IAAI;QACtB,SAAS,EAAE,WAAW,EAAE;KACzB,CAAC,CAAC;IACH,gBAAgB,GAAG,IAA4C,CAAC;IAChE,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,KAAe;IACzC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAyB,CAAC,CAAC,CAAE,GAA2B,CAAC;YACpG,OAAO,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,YAAY,GAAG,IAAI,CAAC;IACrC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,gBAAgB,GAAG,IAAI,CAAC;IACxB,MAAM,GAAG,KAAK,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
function getK() {
|
|
2
|
+
const envK = process.env.RECALL_RRF_K;
|
|
3
|
+
if (envK) {
|
|
4
|
+
const parsed = parseInt(envK, 10);
|
|
5
|
+
if (!isNaN(parsed) && parsed >= 1 && parsed <= 1000)
|
|
6
|
+
return parsed;
|
|
7
|
+
}
|
|
8
|
+
return 60;
|
|
9
|
+
}
|
|
10
|
+
export function fuseResults(lanes) {
|
|
11
|
+
const k = getK();
|
|
12
|
+
const scores = new Map();
|
|
13
|
+
for (const lane of lanes) {
|
|
14
|
+
for (let rank = 0; rank < lane.length; rank++) {
|
|
15
|
+
const item = lane[rank];
|
|
16
|
+
const rrfScore = 1 / (k + rank + 1);
|
|
17
|
+
const existing = scores.get(item.id);
|
|
18
|
+
if (existing) {
|
|
19
|
+
existing.score += rrfScore;
|
|
20
|
+
existing.lanes.push(item.lane);
|
|
21
|
+
if (rank + 1 < existing.bestRank) {
|
|
22
|
+
existing.bestRank = rank + 1;
|
|
23
|
+
existing.bestData = item.data;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
scores.set(item.id, { score: rrfScore, lanes: [item.lane], bestRank: rank + 1, bestData: item.data });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
const results = [];
|
|
32
|
+
for (const [id, entry] of scores) {
|
|
33
|
+
results.push({ id, score: entry.score, lanes: entry.lanes, data: entry.bestData });
|
|
34
|
+
}
|
|
35
|
+
results.sort((a, b) => b.score - a.score);
|
|
36
|
+
return results;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=fusion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fusion.js","sourceRoot":"","sources":["../../src/semantic/fusion.ts"],"names":[],"mappings":"AAeA,SAAS,IAAI;IACX,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IACtC,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,IAAI;YAAE,OAAO,MAAM,CAAC;IACrE,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAqB;IAC/C,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;IACjB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiG,CAAC;IACxH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;YACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC;gBAC3B,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/B,IAAI,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC;oBAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;gBAAC,CAAC;YACpG,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1C,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, rmSync, createWriteStream, statSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { readFile } from 'node:fs/promises';
|
|
5
|
+
import { RECALL_HOME } from '../utils/paths.js';
|
|
6
|
+
const BASE_URL = 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/';
|
|
7
|
+
const MODEL_FILES = [
|
|
8
|
+
{ path: 'config.json', sha256: 'TODO_PLACEHOLDER' },
|
|
9
|
+
{ path: 'tokenizer.json', sha256: 'TODO_PLACEHOLDER' },
|
|
10
|
+
{ path: 'tokenizer_config.json', sha256: 'TODO_PLACEHOLDER' },
|
|
11
|
+
{ path: 'onnx/model.onnx', sha256: 'TODO_PLACEHOLDER' },
|
|
12
|
+
];
|
|
13
|
+
export function getModelDir() {
|
|
14
|
+
return join(RECALL_HOME, 'models', 'bge-base-en-v1.5');
|
|
15
|
+
}
|
|
16
|
+
export function isModelInstalled() {
|
|
17
|
+
const dir = getModelDir();
|
|
18
|
+
return MODEL_FILES.every((f) => existsSync(join(dir, f.path)));
|
|
19
|
+
}
|
|
20
|
+
export async function downloadModel(onProgress) {
|
|
21
|
+
const dir = getModelDir();
|
|
22
|
+
mkdirSync(dir, { recursive: true });
|
|
23
|
+
mkdirSync(join(dir, 'onnx'), { recursive: true });
|
|
24
|
+
for (const file of MODEL_FILES) {
|
|
25
|
+
const dest = join(dir, file.path);
|
|
26
|
+
const url = BASE_URL + file.path;
|
|
27
|
+
let startByte = 0;
|
|
28
|
+
if (existsSync(dest))
|
|
29
|
+
startByte = statSync(dest).size;
|
|
30
|
+
const headers = {};
|
|
31
|
+
if (startByte > 0)
|
|
32
|
+
headers['Range'] = `bytes=${startByte}-`;
|
|
33
|
+
const res = await fetch(url, { headers });
|
|
34
|
+
if (!res.ok && res.status !== 206)
|
|
35
|
+
throw new Error(`Failed to download ${file.path}: HTTP ${res.status}`);
|
|
36
|
+
const totalHeader = res.headers.get('content-length');
|
|
37
|
+
const total = totalHeader ? startByte + Number(totalHeader) : 0;
|
|
38
|
+
const body = res.body;
|
|
39
|
+
if (!body)
|
|
40
|
+
throw new Error(`No response body for ${file.path}`);
|
|
41
|
+
const writeStream = createWriteStream(dest, { flags: startByte > 0 ? 'a' : 'w' });
|
|
42
|
+
const reader = body.getReader();
|
|
43
|
+
let downloaded = startByte;
|
|
44
|
+
while (true) {
|
|
45
|
+
const { done, value } = await reader.read();
|
|
46
|
+
if (done)
|
|
47
|
+
break;
|
|
48
|
+
writeStream.write(Buffer.from(value));
|
|
49
|
+
downloaded += value.byteLength;
|
|
50
|
+
onProgress?.(file.path, downloaded, total);
|
|
51
|
+
}
|
|
52
|
+
writeStream.end();
|
|
53
|
+
await new Promise((resolve, reject) => { writeStream.on('finish', resolve); writeStream.on('error', reject); });
|
|
54
|
+
if (file.sha256 !== 'TODO_PLACEHOLDER') {
|
|
55
|
+
const content = await readFile(dest);
|
|
56
|
+
const hash = createHash('sha256').update(content).digest('hex');
|
|
57
|
+
if (hash !== file.sha256) {
|
|
58
|
+
rmSync(dest);
|
|
59
|
+
throw new Error(`SHA-256 mismatch for ${file.path}`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
export function uninstallModel() {
|
|
65
|
+
const dir = getModelDir();
|
|
66
|
+
if (existsSync(dir))
|
|
67
|
+
rmSync(dir, { recursive: true, force: true });
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=model-download.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-download.js","sourceRoot":"","sources":["../../src/semantic/model-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACrF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,QAAQ,GAAG,4DAA4D,CAAC;AAE9E,MAAM,WAAW,GAA4C;IAC3D,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE;IACnD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,kBAAkB,EAAE;IACtD,EAAE,IAAI,EAAE,uBAAuB,EAAE,MAAM,EAAE,kBAAkB,EAAE;IAC7D,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,kBAAkB,EAAE;CACxD,CAAC;AAEF,MAAM,UAAU,WAAW;IACzB,OAAO,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC;IAC1B,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAsE;IAEtE,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC;IAC1B,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,GAAG,GAAG,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;QACjC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;QACtD,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,IAAI,SAAS,GAAG,CAAC;YAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,SAAS,GAAG,CAAC;QAC5D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,IAAI,UAAU,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1G,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAChC,IAAI,UAAU,GAAG,SAAS,CAAC;QAC3B,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACtC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;YAC/B,UAAU,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,WAAW,CAAC,GAAG,EAAE,CAAC;QAClB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtH,IAAI,IAAI,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAChE,IAAI,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;gBAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAAC,CAAC;QACnG,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC;IAC1B,IAAI,UAAU,CAAC,GAAG,CAAC;QAAE,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACrE,CAAC"}
|