codesift-mcp 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +66 -21
- package/README.md +346 -56
- package/dist/cli/args.d.ts +2 -0
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +11 -0
- package/dist/cli/args.js.map +1 -1
- package/dist/cli/commands.d.ts.map +1 -1
- package/dist/cli/commands.js +177 -67
- package/dist/cli/commands.js.map +1 -1
- package/dist/cli/help.d.ts +1 -1
- package/dist/cli/help.d.ts.map +1 -1
- package/dist/cli/help.js +157 -0
- package/dist/cli/help.js.map +1 -1
- package/dist/cli/hooks.d.ts +3 -0
- package/dist/cli/hooks.d.ts.map +1 -0
- package/dist/cli/hooks.js +163 -0
- package/dist/cli/hooks.js.map +1 -0
- package/dist/cli/setup.d.ts +25 -0
- package/dist/cli/setup.d.ts.map +1 -0
- package/dist/cli/setup.js +400 -0
- package/dist/cli/setup.js.map +1 -0
- package/dist/config.d.ts +2 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +2 -0
- package/dist/config.js.map +1 -1
- package/dist/formatters-shortening.d.ts +7 -0
- package/dist/formatters-shortening.d.ts.map +1 -0
- package/dist/formatters-shortening.js +68 -0
- package/dist/formatters-shortening.js.map +1 -0
- package/dist/formatters.d.ts +314 -0
- package/dist/formatters.d.ts.map +1 -0
- package/dist/formatters.js +396 -0
- package/dist/formatters.js.map +1 -0
- package/dist/instructions.d.ts +6 -0
- package/dist/instructions.d.ts.map +1 -0
- package/dist/instructions.js +72 -0
- package/dist/instructions.js.map +1 -0
- package/dist/lsp/lsp-client.d.ts +21 -0
- package/dist/lsp/lsp-client.d.ts.map +1 -0
- package/dist/lsp/lsp-client.js +122 -0
- package/dist/lsp/lsp-client.js.map +1 -0
- package/dist/lsp/lsp-manager.d.ts +12 -0
- package/dist/lsp/lsp-manager.d.ts.map +1 -0
- package/dist/lsp/lsp-manager.js +82 -0
- package/dist/lsp/lsp-manager.js.map +1 -0
- package/dist/lsp/lsp-servers.d.ts +13 -0
- package/dist/lsp/lsp-servers.d.ts.map +1 -0
- package/dist/lsp/lsp-servers.js +57 -0
- package/dist/lsp/lsp-servers.js.map +1 -0
- package/dist/lsp/lsp-tools.d.ts +67 -0
- package/dist/lsp/lsp-tools.d.ts.map +1 -0
- package/dist/lsp/lsp-tools.js +359 -0
- package/dist/lsp/lsp-tools.js.map +1 -0
- package/dist/parser/extractors/_shared.d.ts +11 -0
- package/dist/parser/extractors/_shared.d.ts.map +1 -0
- package/dist/parser/extractors/_shared.js +38 -0
- package/dist/parser/extractors/_shared.js.map +1 -0
- package/dist/parser/extractors/astro.d.ts +15 -0
- package/dist/parser/extractors/astro.d.ts.map +1 -0
- package/dist/parser/extractors/astro.js +104 -0
- package/dist/parser/extractors/astro.js.map +1 -0
- package/dist/parser/extractors/conversation.d.ts +16 -0
- package/dist/parser/extractors/conversation.d.ts.map +1 -0
- package/dist/parser/extractors/conversation.js +196 -0
- package/dist/parser/extractors/conversation.js.map +1 -0
- package/dist/parser/extractors/go.d.ts.map +1 -1
- package/dist/parser/extractors/go.js +22 -45
- package/dist/parser/extractors/go.js.map +1 -1
- package/dist/parser/extractors/python.d.ts +1 -1
- package/dist/parser/extractors/python.d.ts.map +1 -1
- package/dist/parser/extractors/python.js +19 -50
- package/dist/parser/extractors/python.js.map +1 -1
- package/dist/parser/extractors/rust.d.ts +1 -1
- package/dist/parser/extractors/rust.d.ts.map +1 -1
- package/dist/parser/extractors/rust.js +7 -34
- package/dist/parser/extractors/rust.js.map +1 -1
- package/dist/parser/extractors/typescript.d.ts +1 -1
- package/dist/parser/extractors/typescript.d.ts.map +1 -1
- package/dist/parser/extractors/typescript.js +99 -68
- package/dist/parser/extractors/typescript.js.map +1 -1
- package/dist/parser/parser-manager.d.ts.map +1 -1
- package/dist/parser/parser-manager.js +12 -2
- package/dist/parser/parser-manager.js.map +1 -1
- package/dist/parser/symbol-extractor.d.ts +2 -0
- package/dist/parser/symbol-extractor.d.ts.map +1 -1
- package/dist/parser/symbol-extractor.js +2 -0
- package/dist/parser/symbol-extractor.js.map +1 -1
- package/dist/register-tools.d.ts +127 -0
- package/dist/register-tools.d.ts.map +1 -0
- package/dist/register-tools.js +1453 -0
- package/dist/register-tools.js.map +1 -0
- package/dist/retrieval/codebase-retrieval.d.ts +4 -26
- package/dist/retrieval/codebase-retrieval.d.ts.map +1 -1
- package/dist/retrieval/codebase-retrieval.js +105 -403
- package/dist/retrieval/codebase-retrieval.js.map +1 -1
- package/dist/retrieval/retrieval-constants.d.ts +27 -0
- package/dist/retrieval/retrieval-constants.d.ts.map +1 -0
- package/dist/retrieval/retrieval-constants.js +27 -0
- package/dist/retrieval/retrieval-constants.js.map +1 -0
- package/dist/retrieval/retrieval-schemas.d.ts +107 -0
- package/dist/retrieval/retrieval-schemas.d.ts.map +1 -0
- package/dist/retrieval/retrieval-schemas.js +102 -0
- package/dist/retrieval/retrieval-schemas.js.map +1 -0
- package/dist/retrieval/retrieval-utils.d.ts +40 -0
- package/dist/retrieval/retrieval-utils.d.ts.map +1 -0
- package/dist/retrieval/retrieval-utils.js +139 -0
- package/dist/retrieval/retrieval-utils.js.map +1 -0
- package/dist/retrieval/semantic-handlers.d.ts +8 -0
- package/dist/retrieval/semantic-handlers.d.ts.map +1 -0
- package/dist/retrieval/semantic-handlers.js +152 -0
- package/dist/retrieval/semantic-handlers.js.map +1 -0
- package/dist/search/bm25.d.ts +6 -1
- package/dist/search/bm25.d.ts.map +1 -1
- package/dist/search/bm25.js +95 -32
- package/dist/search/bm25.js.map +1 -1
- package/dist/search/chunker.d.ts +10 -0
- package/dist/search/chunker.d.ts.map +1 -1
- package/dist/search/chunker.js +63 -11
- package/dist/search/chunker.js.map +1 -1
- package/dist/search/reranker.d.ts +15 -0
- package/dist/search/reranker.d.ts.map +1 -0
- package/dist/search/reranker.js +126 -0
- package/dist/search/reranker.js.map +1 -0
- package/dist/search/semantic.d.ts +1 -1
- package/dist/search/semantic.d.ts.map +1 -1
- package/dist/search/semantic.js +40 -45
- package/dist/search/semantic.js.map +1 -1
- package/dist/server-helpers.d.ts +29 -0
- package/dist/server-helpers.d.ts.map +1 -0
- package/dist/server-helpers.js +312 -0
- package/dist/server-helpers.js.map +1 -0
- package/dist/server.d.ts +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +11 -271
- package/dist/server.js.map +1 -1
- package/dist/storage/_shared.d.ts +9 -0
- package/dist/storage/_shared.d.ts.map +1 -0
- package/dist/storage/_shared.js +26 -0
- package/dist/storage/_shared.js.map +1 -0
- package/dist/storage/chunk-store.d.ts.map +1 -1
- package/dist/storage/chunk-store.js +23 -63
- package/dist/storage/chunk-store.js.map +1 -1
- package/dist/storage/embedding-store.d.ts +6 -3
- package/dist/storage/embedding-store.d.ts.map +1 -1
- package/dist/storage/embedding-store.js +54 -30
- package/dist/storage/embedding-store.js.map +1 -1
- package/dist/storage/graph-store.d.ts +48 -0
- package/dist/storage/graph-store.d.ts.map +1 -0
- package/dist/storage/graph-store.js +52 -0
- package/dist/storage/graph-store.js.map +1 -0
- package/dist/storage/index-store.d.ts +5 -0
- package/dist/storage/index-store.d.ts.map +1 -1
- package/dist/storage/index-store.js +28 -16
- package/dist/storage/index-store.js.map +1 -1
- package/dist/storage/registry.d.ts +4 -0
- package/dist/storage/registry.d.ts.map +1 -1
- package/dist/storage/registry.js +16 -16
- package/dist/storage/registry.js.map +1 -1
- package/dist/storage/usage-stats.d.ts +6 -0
- package/dist/storage/usage-stats.d.ts.map +1 -1
- package/dist/storage/usage-stats.js +59 -11
- package/dist/storage/usage-stats.js.map +1 -1
- package/dist/storage/usage-tracker.d.ts +3 -0
- package/dist/storage/usage-tracker.d.ts.map +1 -1
- package/dist/storage/usage-tracker.js +50 -132
- package/dist/storage/usage-tracker.js.map +1 -1
- package/dist/storage/watcher.d.ts +2 -1
- package/dist/storage/watcher.d.ts.map +1 -1
- package/dist/storage/watcher.js +16 -16
- package/dist/storage/watcher.js.map +1 -1
- package/dist/tools/ast-query-tools.d.ts +29 -0
- package/dist/tools/ast-query-tools.d.ts.map +1 -0
- package/dist/tools/ast-query-tools.js +110 -0
- package/dist/tools/ast-query-tools.js.map +1 -0
- package/dist/tools/boundary-tools.d.ts +31 -0
- package/dist/tools/boundary-tools.d.ts.map +1 -0
- package/dist/tools/boundary-tools.js +62 -0
- package/dist/tools/boundary-tools.js.map +1 -0
- package/dist/tools/clone-tools.d.ts +35 -0
- package/dist/tools/clone-tools.d.ts.map +1 -0
- package/dist/tools/clone-tools.js +181 -0
- package/dist/tools/clone-tools.js.map +1 -0
- package/dist/tools/community-tools.d.ts +23 -0
- package/dist/tools/community-tools.d.ts.map +1 -0
- package/dist/tools/community-tools.js +297 -0
- package/dist/tools/community-tools.js.map +1 -0
- package/dist/tools/complexity-tools.d.ts +34 -0
- package/dist/tools/complexity-tools.d.ts.map +1 -0
- package/dist/tools/complexity-tools.js +135 -0
- package/dist/tools/complexity-tools.js.map +1 -0
- package/dist/tools/context-tools.d.ts +44 -3
- package/dist/tools/context-tools.d.ts.map +1 -1
- package/dist/tools/context-tools.js +329 -99
- package/dist/tools/context-tools.js.map +1 -1
- package/dist/tools/conversation-tools.d.ts +107 -0
- package/dist/tools/conversation-tools.d.ts.map +1 -0
- package/dist/tools/conversation-tools.js +419 -0
- package/dist/tools/conversation-tools.js.map +1 -0
- package/dist/tools/coordinator-tools.d.ts +73 -0
- package/dist/tools/coordinator-tools.d.ts.map +1 -0
- package/dist/tools/coordinator-tools.js +153 -0
- package/dist/tools/coordinator-tools.js.map +1 -0
- package/dist/tools/cross-repo-tools.d.ts +43 -0
- package/dist/tools/cross-repo-tools.d.ts.map +1 -0
- package/dist/tools/cross-repo-tools.js +55 -0
- package/dist/tools/cross-repo-tools.js.map +1 -0
- package/dist/tools/diff-tools.d.ts +4 -1
- package/dist/tools/diff-tools.d.ts.map +1 -1
- package/dist/tools/diff-tools.js +23 -5
- package/dist/tools/diff-tools.js.map +1 -1
- package/dist/tools/frequency-tools.d.ts +46 -0
- package/dist/tools/frequency-tools.d.ts.map +1 -0
- package/dist/tools/frequency-tools.js +184 -0
- package/dist/tools/frequency-tools.js.map +1 -0
- package/dist/tools/generate-tools.d.ts.map +1 -1
- package/dist/tools/generate-tools.js +13 -2
- package/dist/tools/generate-tools.js.map +1 -1
- package/dist/tools/graph-tools.d.ts +44 -11
- package/dist/tools/graph-tools.d.ts.map +1 -1
- package/dist/tools/graph-tools.js +147 -104
- package/dist/tools/graph-tools.js.map +1 -1
- package/dist/tools/hotspot-tools.d.ts +24 -0
- package/dist/tools/hotspot-tools.d.ts.map +1 -0
- package/dist/tools/hotspot-tools.js +122 -0
- package/dist/tools/hotspot-tools.js.map +1 -0
- package/dist/tools/impact-tools.d.ts +13 -0
- package/dist/tools/impact-tools.d.ts.map +1 -0
- package/dist/tools/impact-tools.js +238 -0
- package/dist/tools/impact-tools.js.map +1 -0
- package/dist/tools/index-tools.d.ts +44 -3
- package/dist/tools/index-tools.d.ts.map +1 -1
- package/dist/tools/index-tools.js +530 -222
- package/dist/tools/index-tools.js.map +1 -1
- package/dist/tools/memory-tools.d.ts +35 -0
- package/dist/tools/memory-tools.d.ts.map +1 -0
- package/dist/tools/memory-tools.js +229 -0
- package/dist/tools/memory-tools.js.map +1 -0
- package/dist/tools/outline-tools.d.ts +24 -13
- package/dist/tools/outline-tools.d.ts.map +1 -1
- package/dist/tools/outline-tools.js +113 -87
- package/dist/tools/outline-tools.js.map +1 -1
- package/dist/tools/pattern-tools.d.ts +32 -0
- package/dist/tools/pattern-tools.d.ts.map +1 -0
- package/dist/tools/pattern-tools.js +116 -0
- package/dist/tools/pattern-tools.js.map +1 -0
- package/dist/tools/report-tools.d.ts +5 -0
- package/dist/tools/report-tools.d.ts.map +1 -0
- package/dist/tools/report-tools.js +167 -0
- package/dist/tools/report-tools.js.map +1 -0
- package/dist/tools/review-diff-tools.d.ts +148 -0
- package/dist/tools/review-diff-tools.d.ts.map +1 -0
- package/dist/tools/review-diff-tools.js +852 -0
- package/dist/tools/review-diff-tools.js.map +1 -0
- package/dist/tools/route-tools.d.ts +32 -0
- package/dist/tools/route-tools.d.ts.map +1 -0
- package/dist/tools/route-tools.js +276 -0
- package/dist/tools/route-tools.js.map +1 -0
- package/dist/tools/search-ranker.d.ts +5 -0
- package/dist/tools/search-ranker.d.ts.map +1 -0
- package/dist/tools/search-ranker.js +142 -0
- package/dist/tools/search-ranker.js.map +1 -0
- package/dist/tools/search-tools.d.ts +24 -1
- package/dist/tools/search-tools.d.ts.map +1 -1
- package/dist/tools/search-tools.js +459 -225
- package/dist/tools/search-tools.js.map +1 -1
- package/dist/tools/secret-tools.d.ts +104 -0
- package/dist/tools/secret-tools.d.ts.map +1 -0
- package/dist/tools/secret-tools.js +410 -0
- package/dist/tools/secret-tools.js.map +1 -0
- package/dist/tools/symbol-tools.d.ts +90 -2
- package/dist/tools/symbol-tools.d.ts.map +1 -1
- package/dist/tools/symbol-tools.js +576 -42
- package/dist/tools/symbol-tools.js.map +1 -1
- package/dist/types.d.ts +34 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/framework-detect.d.ts +5 -0
- package/dist/utils/framework-detect.d.ts.map +1 -0
- package/dist/utils/framework-detect.js +36 -0
- package/dist/utils/framework-detect.js.map +1 -0
- package/dist/utils/glob.d.ts +19 -0
- package/dist/utils/glob.d.ts.map +1 -0
- package/dist/utils/glob.js +74 -0
- package/dist/utils/glob.js.map +1 -0
- package/dist/utils/import-graph.d.ts +29 -0
- package/dist/utils/import-graph.d.ts.map +1 -0
- package/dist/utils/import-graph.js +125 -0
- package/dist/utils/import-graph.js.map +1 -0
- package/dist/utils/test-file.d.ts.map +1 -1
- package/dist/utils/test-file.js +1 -0
- package/dist/utils/test-file.js.map +1 -1
- package/dist/utils/walk.d.ts +45 -0
- package/dist/utils/walk.d.ts.map +1 -0
- package/dist/utils/walk.js +87 -0
- package/dist/utils/walk.js.map +1 -0
- package/package.json +10 -4
- package/rules/codesift.md +187 -0
- package/rules/codesift.mdc +192 -0
- package/rules/codex.md +187 -0
- package/rules/gemini.md +187 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { readFile
|
|
2
|
-
import {
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { atomicWriteFile } from "./_shared.js";
|
|
3
3
|
/**
|
|
4
4
|
* Get the embedding file path from an index path.
|
|
5
5
|
* {hash}.index.json → {hash}.embeddings.ndjson
|
|
@@ -45,26 +45,42 @@ export async function loadEmbeddings(embeddingPath) {
|
|
|
45
45
|
return embeddings;
|
|
46
46
|
}
|
|
47
47
|
/**
|
|
48
|
-
* Save all embeddings
|
|
48
|
+
* Save all embeddings to an ndjson file using streaming writes.
|
|
49
|
+
* Avoids building a single huge string (30K+ symbols × 1536 floats = >300MB).
|
|
49
50
|
*/
|
|
50
51
|
export async function saveEmbeddings(embeddingPath, embeddings) {
|
|
51
|
-
const dir = dirname(embeddingPath);
|
|
52
|
-
await mkdir(dir, { recursive: true });
|
|
53
|
-
const lines = [];
|
|
54
|
-
for (const [id, vec] of embeddings) {
|
|
55
|
-
lines.push(JSON.stringify({ id, vec: Array.from(vec) }));
|
|
56
|
-
}
|
|
57
52
|
const tmpPath = `${embeddingPath}.tmp.${Date.now()}`;
|
|
58
|
-
const
|
|
53
|
+
const { createWriteStream } = await import("node:fs");
|
|
54
|
+
const stream = createWriteStream(tmpPath, { encoding: "utf-8" });
|
|
55
|
+
// Register error listener immediately to prevent unhandled error crash
|
|
56
|
+
let streamError = null;
|
|
57
|
+
stream.on("error", (err) => { streamError = err; });
|
|
59
58
|
try {
|
|
60
|
-
|
|
59
|
+
for (const [id, vec] of embeddings) {
|
|
60
|
+
if (streamError)
|
|
61
|
+
throw streamError;
|
|
62
|
+
const line = JSON.stringify({ id, vec: Array.from(vec) }) + "\n";
|
|
63
|
+
const canContinue = stream.write(line);
|
|
64
|
+
if (!canContinue) {
|
|
65
|
+
await new Promise((resolve) => stream.once("drain", resolve));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (streamError)
|
|
69
|
+
throw streamError;
|
|
70
|
+
await new Promise((resolve, reject) => {
|
|
71
|
+
stream.end(() => streamError ? reject(streamError) : resolve());
|
|
72
|
+
});
|
|
73
|
+
// Atomic rename
|
|
74
|
+
const { rename } = await import("node:fs/promises");
|
|
61
75
|
await rename(tmpPath, embeddingPath);
|
|
62
76
|
}
|
|
63
77
|
catch (err) {
|
|
78
|
+
// Clean up temp file on error
|
|
64
79
|
try {
|
|
80
|
+
const { unlink } = await import("node:fs/promises");
|
|
65
81
|
await unlink(tmpPath);
|
|
66
82
|
}
|
|
67
|
-
catch { /*
|
|
83
|
+
catch { /* ignore */ }
|
|
68
84
|
throw err;
|
|
69
85
|
}
|
|
70
86
|
}
|
|
@@ -72,21 +88,8 @@ export async function saveEmbeddings(embeddingPath, embeddings) {
|
|
|
72
88
|
* Save embedding metadata atomically.
|
|
73
89
|
*/
|
|
74
90
|
export async function saveEmbeddingMeta(metaPath, meta) {
|
|
75
|
-
const dir = dirname(metaPath);
|
|
76
|
-
await mkdir(dir, { recursive: true });
|
|
77
|
-
const tmpPath = `${metaPath}.tmp.${Date.now()}`;
|
|
78
91
|
const data = JSON.stringify(meta);
|
|
79
|
-
|
|
80
|
-
await writeFile(tmpPath, data, "utf-8");
|
|
81
|
-
await rename(tmpPath, metaPath);
|
|
82
|
-
}
|
|
83
|
-
catch (err) {
|
|
84
|
-
try {
|
|
85
|
-
await unlink(tmpPath);
|
|
86
|
-
}
|
|
87
|
-
catch { /* cleanup best-effort */ }
|
|
88
|
-
throw err;
|
|
89
|
-
}
|
|
92
|
+
await atomicWriteFile(metaPath, data);
|
|
90
93
|
}
|
|
91
94
|
/**
|
|
92
95
|
* Load embedding metadata.
|
|
@@ -107,24 +110,41 @@ export async function loadEmbeddingMeta(metaPath) {
|
|
|
107
110
|
return null;
|
|
108
111
|
}
|
|
109
112
|
}
|
|
113
|
+
/** Simple hash for content-change detection (FNV-1a 32-bit). */
|
|
114
|
+
function contentHash(text) {
|
|
115
|
+
let h = 0x811c9dc5;
|
|
116
|
+
for (let i = 0; i < text.length; i++) {
|
|
117
|
+
h ^= text.charCodeAt(i);
|
|
118
|
+
h = Math.imul(h, 0x01000193);
|
|
119
|
+
}
|
|
120
|
+
return h >>> 0;
|
|
121
|
+
}
|
|
122
|
+
/** Track content hashes so we re-embed when symbol content changes. */
|
|
123
|
+
const embeddingContentHashes = new Map();
|
|
110
124
|
/**
|
|
111
125
|
* Batch-embed symbols using the given provider, appending to existing embeddings.
|
|
112
|
-
*
|
|
126
|
+
* Skips symbols whose ID exists AND content hash hasn't changed.
|
|
127
|
+
* Re-embeds symbols whose content changed even if ID is the same.
|
|
113
128
|
*
|
|
114
129
|
* @param symbolTexts - Map of symbolId → text to embed
|
|
115
130
|
* @param existing - Existing embeddings to skip
|
|
116
131
|
* @param embedFn - The provider's embed function
|
|
117
132
|
* @param batchSize - How many texts per API call
|
|
133
|
+
* @param cacheKey - Optional key to track content hashes across calls
|
|
118
134
|
* @returns Map of symbolId → Float32Array (existing + new)
|
|
119
135
|
*/
|
|
120
|
-
export async function batchEmbed(symbolTexts, existing, embedFn, batchSize) {
|
|
136
|
+
export async function batchEmbed(symbolTexts, existing, embedFn, batchSize, cacheKey) {
|
|
121
137
|
const result = new Map(existing);
|
|
122
|
-
|
|
138
|
+
const hashes = cacheKey ? (embeddingContentHashes.get(cacheKey) ?? new Map()) : new Map();
|
|
139
|
+
// Find symbols that need embedding (new or content changed)
|
|
123
140
|
const toEmbed = [];
|
|
124
141
|
for (const [id, text] of symbolTexts) {
|
|
125
|
-
|
|
142
|
+
const hash = contentHash(text);
|
|
143
|
+
const needsEmbed = !existing.has(id) || (cacheKey !== undefined && hashes.get(id) !== hash);
|
|
144
|
+
if (needsEmbed) {
|
|
126
145
|
toEmbed.push({ id, text });
|
|
127
146
|
}
|
|
147
|
+
hashes.set(id, hash);
|
|
128
148
|
}
|
|
129
149
|
// Process in batches (only symbols that need embedding)
|
|
130
150
|
for (let i = 0; i < toEmbed.length; i += batchSize) {
|
|
@@ -143,6 +163,10 @@ export async function batchEmbed(symbolTexts, existing, embedFn, batchSize) {
|
|
|
143
163
|
const stale = [...result.keys()].filter((id) => !symbolTexts.has(id));
|
|
144
164
|
for (const id of stale) {
|
|
145
165
|
result.delete(id);
|
|
166
|
+
hashes.delete(id);
|
|
167
|
+
}
|
|
168
|
+
if (cacheKey) {
|
|
169
|
+
embeddingContentHashes.set(cacheKey, hashes);
|
|
146
170
|
}
|
|
147
171
|
return result;
|
|
148
172
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-store.js","sourceRoot":"","sources":["../../src/storage/embedding-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"embedding-store.js","sourceRoot":"","sources":["../../src/storage/embedding-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAiB;IAChD,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,CAAC;AACnE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IACpD,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;AACtE,CAAC;AAOD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAAqB;IAErB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;IAEnD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC5C,MAAM,KAAK,GAAG,MAAuB,CAAC;gBACtC,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,oCAAoC;IACtC,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAAqB,EACrB,UAAqC;IAErC,MAAM,OAAO,GAAG,GAAG,aAAa,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACrD,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IAEjE,uEAAuE;IACvE,IAAI,WAAW,GAAiB,IAAI,CAAC;IACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,WAAW;gBAAE,MAAM,WAAW,CAAC;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;YACjE,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QACD,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC;QACnC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QACH,gBAAgB;QAChB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;QACpD,MAAM,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,8BAA8B;QAC9B,IAAI,CAAC;YAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QAC1G,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,IAAmB;IAEnB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAExC,IACE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;YAC7C,OAAQ,MAAkC,CAAC,OAAO,CAAC,KAAK,QAAQ;YAChE,OAAQ,MAAkC,CAAC,YAAY,CAAC,KAAK,QAAQ,EACrE,CAAC;YACD,OAAO,MAAuB,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,gEAAgE;AAChE,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,GAAG,UAAU,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED,uEAAuE;AACvE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA+B,CAAC;AAEtE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,WAAgC,EAChC,QAAmC,EACnC,OAAiD,EACjD,SAAiB,EACjB,QAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAC;IAE1H,4DAA4D;IAC5D,MAAM,OAAO,GAAwC,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC;QAC5F,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7B,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,wDAAwD;IACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACtE,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,sBAAsB,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Persistent knowledge graph — caches symbol relationships (import edges,
|
|
3
|
+
* call chains, type usage) across sessions to avoid expensive recomputation.
|
|
4
|
+
*
|
|
5
|
+
* Stored as JSON alongside the index file. Invalidated when the index changes
|
|
6
|
+
* (via content hash comparison).
|
|
7
|
+
*/
|
|
8
|
+
export interface GraphEdge {
|
|
9
|
+
from: string;
|
|
10
|
+
to: string;
|
|
11
|
+
kind: "imports" | "calls" | "uses_type" | "extends" | "implements";
|
|
12
|
+
}
|
|
13
|
+
export interface PersistentGraph {
|
|
14
|
+
/** Hash of the index content — used to detect staleness */
|
|
15
|
+
index_hash: string;
|
|
16
|
+
/** Timestamp of last computation */
|
|
17
|
+
computed_at: number;
|
|
18
|
+
/** Import edges between files */
|
|
19
|
+
edges: GraphEdge[];
|
|
20
|
+
/** Module-level stats */
|
|
21
|
+
modules: Array<{
|
|
22
|
+
path: string;
|
|
23
|
+
symbol_count: number;
|
|
24
|
+
in_degree: number;
|
|
25
|
+
out_degree: number;
|
|
26
|
+
}>;
|
|
27
|
+
/** Detected circular dependencies */
|
|
28
|
+
circular_deps: string[][];
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Get the graph store path from an index path.
|
|
32
|
+
* {hash}.index.json → {hash}.graph.json
|
|
33
|
+
*/
|
|
34
|
+
export declare function getGraphPath(indexPath: string): string;
|
|
35
|
+
/** Compute a hash from index file paths + symbol counts (cheap staleness check). */
|
|
36
|
+
export declare function computeIndexHash(files: Array<{
|
|
37
|
+
path: string;
|
|
38
|
+
symbol_count: number;
|
|
39
|
+
}>): string;
|
|
40
|
+
/**
|
|
41
|
+
* Load a persisted graph. Returns null if not found, corrupted, or stale.
|
|
42
|
+
*/
|
|
43
|
+
export declare function loadGraph(graphPath: string, currentIndexHash: string): Promise<PersistentGraph | null>;
|
|
44
|
+
/**
|
|
45
|
+
* Save a computed graph to disk.
|
|
46
|
+
*/
|
|
47
|
+
export declare function saveGraph(graphPath: string, graph: PersistentGraph): Promise<void>;
|
|
48
|
+
//# sourceMappingURL=graph-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graph-store.d.ts","sourceRoot":"","sources":["../../src/storage/graph-store.ts"],"names":[],"mappings":"AAGA;;;;;;GAMG;AAEH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,SAAS,GAAG,OAAO,GAAG,WAAW,GAAG,SAAS,GAAG,YAAY,CAAC;CACpE;AAED,MAAM,WAAW,eAAe;IAC9B,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,oCAAoC;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,yBAAyB;IACzB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9F,qCAAqC;IACrC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC;CAC3B;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEtD;AAYD,oFAAoF;AACpF,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,MAAM,CAG7F;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAmBjC;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,IAAI,CAAC,CAEf"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { atomicWriteFile } from "./_shared.js";
|
|
3
|
+
/**
|
|
4
|
+
* Get the graph store path from an index path.
|
|
5
|
+
* {hash}.index.json → {hash}.graph.json
|
|
6
|
+
*/
|
|
7
|
+
export function getGraphPath(indexPath) {
|
|
8
|
+
return indexPath.replace(/\.index\.json$/, ".graph.json");
|
|
9
|
+
}
|
|
10
|
+
/** Simple FNV-1a hash for content-change detection. */
|
|
11
|
+
function fnv1a(text) {
|
|
12
|
+
let h = 0x811c9dc5;
|
|
13
|
+
for (let i = 0; i < text.length; i++) {
|
|
14
|
+
h ^= text.charCodeAt(i);
|
|
15
|
+
h = Math.imul(h, 0x01000193);
|
|
16
|
+
}
|
|
17
|
+
return (h >>> 0).toString(16);
|
|
18
|
+
}
|
|
19
|
+
/** Compute a hash from index file paths + symbol counts (cheap staleness check). */
|
|
20
|
+
export function computeIndexHash(files) {
|
|
21
|
+
const fingerprint = files.map((f) => `${f.path}:${f.symbol_count}`).join("\n");
|
|
22
|
+
return fnv1a(fingerprint);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Load a persisted graph. Returns null if not found, corrupted, or stale.
|
|
26
|
+
*/
|
|
27
|
+
export async function loadGraph(graphPath, currentIndexHash) {
|
|
28
|
+
try {
|
|
29
|
+
const raw = await readFile(graphPath, "utf-8");
|
|
30
|
+
const parsed = JSON.parse(raw);
|
|
31
|
+
if (typeof parsed === "object" && parsed !== null &&
|
|
32
|
+
typeof parsed["index_hash"] === "string" &&
|
|
33
|
+
Array.isArray(parsed["edges"])) {
|
|
34
|
+
const graph = parsed;
|
|
35
|
+
// Stale check: index changed since last computation
|
|
36
|
+
if (graph.index_hash !== currentIndexHash)
|
|
37
|
+
return null;
|
|
38
|
+
return graph;
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Save a computed graph to disk.
|
|
48
|
+
*/
|
|
49
|
+
export async function saveGraph(graphPath, graph) {
|
|
50
|
+
await atomicWriteFile(graphPath, JSON.stringify(graph));
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=graph-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graph-store.js","sourceRoot":"","sources":["../../src/storage/graph-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AA6B/C;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC;AAED,uDAAuD;AACvD,SAAS,KAAK,CAAC,IAAY;IACzB,IAAI,CAAC,GAAG,UAAU,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,gBAAgB,CAAC,KAAoD;IACnF,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/E,OAAO,KAAK,CAAC,WAAW,CAAC,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,SAAiB,EACjB,gBAAwB;IAExB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAExC,IACE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;YAC7C,OAAQ,MAAkC,CAAC,YAAY,CAAC,KAAK,QAAQ;YACrE,KAAK,CAAC,OAAO,CAAE,MAAkC,CAAC,OAAO,CAAC,CAAC,EAC3D,CAAC;YACD,MAAM,KAAK,GAAG,MAAyB,CAAC;YACxC,oDAAoD;YACpD,IAAI,KAAK,CAAC,UAAU,KAAK,gBAAgB;gBAAE,OAAO,IAAI,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,SAAiB,EACjB,KAAsB;IAEtB,MAAM,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1D,CAAC"}
|
|
@@ -15,6 +15,11 @@ export declare function loadIndex(indexPath: string): Promise<CodeIndex | null>;
|
|
|
15
15
|
* Serialized per indexPath to prevent read-modify-write races.
|
|
16
16
|
*/
|
|
17
17
|
export declare function saveIncremental(indexPath: string, updatedFile: string, newSymbols: CodeSymbol[], fileEntry?: FileEntry): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Remove all symbols and the file entry for a deleted file.
|
|
20
|
+
* Serialized per indexPath to prevent read-modify-write races.
|
|
21
|
+
*/
|
|
22
|
+
export declare function removeFileFromIndex(indexPath: string, deletedFile: string): Promise<void>;
|
|
18
23
|
/**
|
|
19
24
|
* Derive a deterministic index file path from a repo root.
|
|
20
25
|
* Uses a truncated SHA-256 hash of the root path.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../src/storage/index-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../src/storage/index-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAMpE;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC,CAGf;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAa3B;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,UAAU,EAAE,EACxB,SAAS,CAAC,EAAE,SAAS,GACpB,OAAO,CAAC,IAAI,CAAC,CAgCf;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAOtE"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { readFile
|
|
2
|
-
import { join
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
|
+
import { atomicWriteFile } from "./_shared.js";
|
|
4
5
|
/** Serialize concurrent writes to the same index path. */
|
|
5
6
|
const writeLocks = new Map();
|
|
6
7
|
/**
|
|
@@ -8,21 +9,8 @@ const writeLocks = new Map();
|
|
|
8
9
|
* Writes to a temp file first, then renames to prevent partial reads.
|
|
9
10
|
*/
|
|
10
11
|
export async function saveIndex(indexPath, index) {
|
|
11
|
-
const dir = dirname(indexPath);
|
|
12
|
-
await mkdir(dir, { recursive: true });
|
|
13
|
-
const tmpPath = `${indexPath}.tmp.${Date.now()}.json`;
|
|
14
12
|
const data = JSON.stringify(index);
|
|
15
|
-
|
|
16
|
-
await writeFile(tmpPath, data, "utf-8");
|
|
17
|
-
await rename(tmpPath, indexPath);
|
|
18
|
-
}
|
|
19
|
-
catch (err) {
|
|
20
|
-
try {
|
|
21
|
-
await unlink(tmpPath);
|
|
22
|
-
}
|
|
23
|
-
catch { /* cleanup best-effort */ }
|
|
24
|
-
throw err;
|
|
25
|
-
}
|
|
13
|
+
await atomicWriteFile(indexPath, data);
|
|
26
14
|
}
|
|
27
15
|
/**
|
|
28
16
|
* Load a code index from disk.
|
|
@@ -71,6 +59,30 @@ export async function saveIncremental(indexPath, updatedFile, newSymbols, fileEn
|
|
|
71
59
|
writeLocks.set(indexPath, next.catch(() => { }));
|
|
72
60
|
return next;
|
|
73
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* Remove all symbols and the file entry for a deleted file.
|
|
64
|
+
* Serialized per indexPath to prevent read-modify-write races.
|
|
65
|
+
*/
|
|
66
|
+
export async function removeFileFromIndex(indexPath, deletedFile) {
|
|
67
|
+
const prev = writeLocks.get(indexPath) ?? Promise.resolve();
|
|
68
|
+
const next = prev.then(async () => {
|
|
69
|
+
const existing = await loadIndex(indexPath);
|
|
70
|
+
if (!existing)
|
|
71
|
+
return;
|
|
72
|
+
const hadSymbols = existing.symbols.some((s) => s.file === deletedFile);
|
|
73
|
+
const hadFile = existing.files.some((f) => f.path === deletedFile);
|
|
74
|
+
if (!hadSymbols && !hadFile)
|
|
75
|
+
return;
|
|
76
|
+
existing.symbols = existing.symbols.filter((s) => s.file !== deletedFile);
|
|
77
|
+
existing.symbol_count = existing.symbols.length;
|
|
78
|
+
existing.files = existing.files.filter((f) => f.path !== deletedFile);
|
|
79
|
+
existing.file_count = existing.files.length;
|
|
80
|
+
existing.updated_at = Date.now();
|
|
81
|
+
await saveIndex(indexPath, existing);
|
|
82
|
+
});
|
|
83
|
+
writeLocks.set(indexPath, next.catch(() => { }));
|
|
84
|
+
return next;
|
|
85
|
+
}
|
|
74
86
|
/**
|
|
75
87
|
* Derive a deterministic index file path from a repo root.
|
|
76
88
|
* Uses a truncated SHA-256 hash of the root path.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-store.js","sourceRoot":"","sources":["../../src/storage/index-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"index-store.js","sourceRoot":"","sources":["../../src/storage/index-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,0DAA0D;AAC1D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAyB,CAAC;AAEpD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,SAAiB,EACjB,KAAgB;IAEhB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AACzC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,SAAiB,EACjB,WAAmB,EACnB,UAAwB,EACxB,SAAqB;IAErB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAE5D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QAChC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,mDAAmD,SAAS,EAAE,CAAC,CAAC;QAClF,CAAC;QAED,iBAAiB;QACjB,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CACtC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,CACxC,CAAC;QACF,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC;QAE5C,QAAQ,CAAC,OAAO,GAAG,MAAM,CAAC;QAC1B,QAAQ,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;QACtC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEjC,oCAAoC;QACpC,IAAI,SAAS,EAAE,CAAC;YACd,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;YACtE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,QAAQ,CAAC,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;QAC9C,CAAC;QAED,MAAM,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,gEAAgE;IAChE,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,SAAiB,EACjB,WAAmB;IAEnB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAE5D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QAChC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEtB,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QACnE,IAAI,CAAC,UAAU,IAAI,CAAC,OAAO;YAAE,OAAO;QAEpC,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QAC1E,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;QAChD,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QACtE,QAAQ,CAAC,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5C,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEjC,MAAM,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,QAAgB;IAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC9B,MAAM,CAAC,QAAQ,CAAC;SAChB,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,OAAO,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,aAAa,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAE9D,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,IAAI,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAEjD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -22,6 +22,10 @@ export declare function getRepo(registryPath: string, name: string): Promise<Rep
|
|
|
22
22
|
* List all registered repos.
|
|
23
23
|
*/
|
|
24
24
|
export declare function listRepos(registryPath: string): Promise<RepoMeta[]>;
|
|
25
|
+
/**
|
|
26
|
+
* Partially update a repo's metadata (e.g., last_git_commit after freshness check).
|
|
27
|
+
*/
|
|
28
|
+
export declare function updateRepoMeta(registryPath: string, repoName: string, updates: Partial<Pick<RepoMeta, "last_git_commit" | "symbol_count" | "file_count" | "updated_at">>): Promise<void>;
|
|
25
29
|
/**
|
|
26
30
|
* Remove a repo from the registry.
|
|
27
31
|
* Returns true if the repo existed and was removed, false otherwise.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/storage/registry.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/storage/registry.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGtD;;;GAGG;AACH,wBAAsB,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAa1E;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,QAAQ,GACjB,OAAO,CAAC,IAAI,CAAC,CAGf;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,IAAI,CAAC,CAKf;AAED;;;GAGG;AACH,wBAAsB,OAAO,CAC3B,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAG1B;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAGrB;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,iBAAiB,GAAG,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC,CAAC,GACjG,OAAO,CAAC,IAAI,CAAC,CAOf;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAC9B,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,OAAO,CAAC,CAWlB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEpD"}
|
package/dist/storage/registry.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { readFile
|
|
2
|
-
import {
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { basename } from "node:path";
|
|
3
|
+
import { atomicWriteFile } from "./_shared.js";
|
|
3
4
|
/**
|
|
4
5
|
* Load the multi-repo registry from disk.
|
|
5
6
|
* Returns an empty registry if the file doesn't exist or is invalid.
|
|
@@ -22,21 +23,8 @@ export async function loadRegistry(registryPath) {
|
|
|
22
23
|
* Writes to a temp file first, then renames to prevent partial reads.
|
|
23
24
|
*/
|
|
24
25
|
export async function saveRegistry(registryPath, registry) {
|
|
25
|
-
const dir = dirname(registryPath);
|
|
26
|
-
await mkdir(dir, { recursive: true });
|
|
27
|
-
const tmpPath = `${registryPath}.tmp.${Date.now()}.json`;
|
|
28
26
|
const data = JSON.stringify(registry);
|
|
29
|
-
|
|
30
|
-
await writeFile(tmpPath, data, "utf-8");
|
|
31
|
-
await rename(tmpPath, registryPath);
|
|
32
|
-
}
|
|
33
|
-
catch (err) {
|
|
34
|
-
try {
|
|
35
|
-
await unlink(tmpPath);
|
|
36
|
-
}
|
|
37
|
-
catch { /* cleanup best-effort */ }
|
|
38
|
-
throw err;
|
|
39
|
-
}
|
|
27
|
+
await atomicWriteFile(registryPath, data);
|
|
40
28
|
}
|
|
41
29
|
/**
|
|
42
30
|
* Register or update a repo in the registry.
|
|
@@ -62,6 +50,18 @@ export async function listRepos(registryPath) {
|
|
|
62
50
|
const registry = await loadRegistry(registryPath);
|
|
63
51
|
return Object.values(registry.repos);
|
|
64
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Partially update a repo's metadata (e.g., last_git_commit after freshness check).
|
|
55
|
+
*/
|
|
56
|
+
export async function updateRepoMeta(registryPath, repoName, updates) {
|
|
57
|
+
const registry = await loadRegistry(registryPath);
|
|
58
|
+
const existing = registry.repos[repoName];
|
|
59
|
+
if (!existing)
|
|
60
|
+
return;
|
|
61
|
+
Object.assign(existing, updates);
|
|
62
|
+
registry.updated_at = Date.now();
|
|
63
|
+
await saveRegistry(registryPath, registry);
|
|
64
|
+
}
|
|
65
65
|
/**
|
|
66
66
|
* Remove a repo from the registry.
|
|
67
67
|
* Returns true if the repo existed and was removed, false otherwise.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/storage/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/storage/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,YAAoB;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,OAAO,aAAa,EAAE,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,aAAa,EAAE,CAAC;IACzB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,YAAoB,EACpB,QAAkB;IAElB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,eAAe,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,YAAoB,EACpB,IAAc;IAEd,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,CAAC;IAClD,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACjC,MAAM,YAAY,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,YAAoB,EACpB,IAAY;IAEZ,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,CAAC;IAClD,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,YAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,CAAC;IAClD,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,YAAoB,EACpB,QAAgB,EAChB,OAAkG;IAElG,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACjC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACjC,MAAM,YAAY,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,YAAoB,EACpB,IAAY;IAEZ,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,CAAC;IAElD,IAAI,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACjC,MAAM,YAAY,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IAC3C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,OAAO,SAAS,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,aAAa;IACpB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAE9D,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,IAAI,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC5E,IAAI,OAAO,GAAG,CAAC,YAAY,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAExD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -14,6 +14,11 @@ export interface DailyStats {
|
|
|
14
14
|
call_count: number;
|
|
15
15
|
total_tokens: number;
|
|
16
16
|
}
|
|
17
|
+
export interface QueryTypeStats {
|
|
18
|
+
type: string;
|
|
19
|
+
query_count: number;
|
|
20
|
+
call_count: number;
|
|
21
|
+
}
|
|
17
22
|
export interface UsageStats {
|
|
18
23
|
total_calls: number;
|
|
19
24
|
total_sessions: number;
|
|
@@ -21,6 +26,7 @@ export interface UsageStats {
|
|
|
21
26
|
tools: ToolStats[];
|
|
22
27
|
top_repos: RepoStats[];
|
|
23
28
|
daily: DailyStats[];
|
|
29
|
+
query_types: QueryTypeStats[];
|
|
24
30
|
earliest_ts: number;
|
|
25
31
|
latest_ts: number;
|
|
26
32
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"usage-stats.d.ts","sourceRoot":"","sources":["../../src/storage/usage-stats.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,SAAS,EAAE,SAAS,EAAE,CAAC;IACvB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;
|
|
1
|
+
{"version":3,"file":"usage-stats.d.ts","sourceRoot":"","sources":["../../src/storage/usage-stats.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,SAAS,EAAE,SAAS,EAAE,CAAC;IACvB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAyDD,wBAAsB,aAAa,CAAC,OAAO,CAAC,EAAE;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,UAAU,CAAC,CAuHtB;AAMD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CA0E3D"}
|
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { getCumulativeSavings, getUsagePath } from "./usage-tracker.js";
|
|
3
|
+
import { formatTable } from "../formatters.js";
|
|
4
4
|
// ---------------------------------------------------------------------------
|
|
5
5
|
// Loading
|
|
6
6
|
// ---------------------------------------------------------------------------
|
|
7
|
-
function getUsagePath() {
|
|
8
|
-
const dataDir = process.env["CODESIFT_DATA_DIR"] ?? join(homedir(), ".codesift");
|
|
9
|
-
return join(dataDir, "usage.jsonl");
|
|
10
|
-
}
|
|
11
7
|
function isValidEntry(value) {
|
|
12
8
|
if (typeof value !== "object" || value === null)
|
|
13
9
|
return false;
|
|
@@ -62,6 +58,7 @@ export async function getUsageStats(options) {
|
|
|
62
58
|
tools: [],
|
|
63
59
|
top_repos: [],
|
|
64
60
|
daily: [],
|
|
61
|
+
query_types: [],
|
|
65
62
|
earliest_ts: 0,
|
|
66
63
|
latest_ts: 0,
|
|
67
64
|
};
|
|
@@ -71,6 +68,9 @@ export async function getUsageStats(options) {
|
|
|
71
68
|
const repoMap = new Map();
|
|
72
69
|
const sessionSet = new Set();
|
|
73
70
|
const dailyMap = new Map();
|
|
71
|
+
// codebase_retrieval query type breakdown
|
|
72
|
+
const queryTypeQueries = new Map(); // type → total query count
|
|
73
|
+
const queryTypeCalls = new Map(); // type → calls containing this type
|
|
74
74
|
let earliest = Infinity;
|
|
75
75
|
let latest = 0;
|
|
76
76
|
for (const entry of entries) {
|
|
@@ -92,6 +92,22 @@ export async function getUsageStats(options) {
|
|
|
92
92
|
dayStats.calls += 1;
|
|
93
93
|
dayStats.tokens += entry.result_tokens;
|
|
94
94
|
dailyMap.set(date, dayStats);
|
|
95
|
+
// codebase_retrieval query type breakdown
|
|
96
|
+
if (entry.tool === "codebase_retrieval") {
|
|
97
|
+
const types = entry.args_summary?.["query_types"];
|
|
98
|
+
if (Array.isArray(types)) {
|
|
99
|
+
const seenInCall = new Set();
|
|
100
|
+
for (const t of types) {
|
|
101
|
+
if (typeof t === "string") {
|
|
102
|
+
queryTypeQueries.set(t, (queryTypeQueries.get(t) ?? 0) + 1);
|
|
103
|
+
seenInCall.add(t);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
for (const t of seenInCall) {
|
|
107
|
+
queryTypeCalls.set(t, (queryTypeCalls.get(t) ?? 0) + 1);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
95
111
|
// Time range
|
|
96
112
|
if (entry.ts < earliest)
|
|
97
113
|
earliest = entry.ts;
|
|
@@ -120,6 +136,13 @@ export async function getUsageStats(options) {
|
|
|
120
136
|
}))
|
|
121
137
|
.sort((a, b) => a.date.localeCompare(b.date));
|
|
122
138
|
const totalSessions = sessionSet.size;
|
|
139
|
+
const query_types = [...queryTypeQueries.entries()]
|
|
140
|
+
.map(([type, query_count]) => ({
|
|
141
|
+
type,
|
|
142
|
+
query_count,
|
|
143
|
+
call_count: queryTypeCalls.get(type) ?? 0,
|
|
144
|
+
}))
|
|
145
|
+
.sort((a, b) => b.query_count - a.query_count);
|
|
123
146
|
return {
|
|
124
147
|
total_calls: entries.length,
|
|
125
148
|
total_sessions: totalSessions,
|
|
@@ -127,6 +150,7 @@ export async function getUsageStats(options) {
|
|
|
127
150
|
tools,
|
|
128
151
|
top_repos,
|
|
129
152
|
daily,
|
|
153
|
+
query_types,
|
|
130
154
|
earliest_ts: earliest,
|
|
131
155
|
latest_ts: latest,
|
|
132
156
|
};
|
|
@@ -150,12 +174,32 @@ export function formatUsageReport(stats) {
|
|
|
150
174
|
lines.push("");
|
|
151
175
|
// Tool breakdown
|
|
152
176
|
lines.push("--- Tool Breakdown ---");
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
177
|
+
const toolRows = stats.tools.map((t) => [
|
|
178
|
+
t.tool,
|
|
179
|
+
String(t.total_calls),
|
|
180
|
+
String(t.total_result_tokens),
|
|
181
|
+
String(t.avg_elapsed_ms),
|
|
182
|
+
String(t.avg_result_tokens),
|
|
183
|
+
]);
|
|
184
|
+
lines.push(formatTable(["Tool", "Calls", "Tokens", "Avg ms", "Avg tok"], toolRows));
|
|
158
185
|
lines.push("");
|
|
186
|
+
// codebase_retrieval query type breakdown
|
|
187
|
+
if (stats.query_types.length > 0) {
|
|
188
|
+
const crTool = stats.tools.find((t) => t.tool === "codebase_retrieval");
|
|
189
|
+
const totalQueries = stats.query_types.reduce((sum, qt) => sum + qt.query_count, 0);
|
|
190
|
+
lines.push("--- codebase_retrieval Query Types ---");
|
|
191
|
+
const qtRows = stats.query_types.map((qt) => {
|
|
192
|
+
const pct = totalQueries > 0 ? Math.round((qt.query_count / totalQueries) * 100) : 0;
|
|
193
|
+
return [
|
|
194
|
+
qt.type,
|
|
195
|
+
String(qt.query_count),
|
|
196
|
+
`${pct}%`,
|
|
197
|
+
`${qt.call_count}/${crTool?.total_calls ?? "?"}`,
|
|
198
|
+
];
|
|
199
|
+
});
|
|
200
|
+
lines.push(formatTable(["Type", "Queries", "%", "In calls"], qtRows));
|
|
201
|
+
lines.push("");
|
|
202
|
+
}
|
|
159
203
|
// Top repos
|
|
160
204
|
if (stats.top_repos.length > 0) {
|
|
161
205
|
lines.push("--- Top Repos ---");
|
|
@@ -175,6 +219,10 @@ export function formatUsageReport(stats) {
|
|
|
175
219
|
lines.push(` ... and ${stats.daily.length - 14} earlier days`);
|
|
176
220
|
}
|
|
177
221
|
}
|
|
222
|
+
const saved = getCumulativeSavings();
|
|
223
|
+
if (saved > 0) {
|
|
224
|
+
lines.push(`\nEstimated tokens saved this session: ${saved.toLocaleString()} (~$${(saved * 30 / 1_000_000).toFixed(2)} at Opus rates)`);
|
|
225
|
+
}
|
|
178
226
|
return lines.join("\n");
|
|
179
227
|
}
|
|
180
228
|
//# sourceMappingURL=usage-stats.js.map
|