codesift-mcp 0.8.10 → 0.8.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/hooks.d.ts.map +1 -1
- package/dist/cli/hooks.js +2 -1
- package/dist/cli/hooks.js.map +1 -1
- package/dist/cli/setup.d.ts.map +1 -1
- package/dist/cli/setup.js +52 -9
- package/dist/cli/setup.js.map +1 -1
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/register-tools.d.ts.map +1 -1
- package/dist/register-tools.js +126 -1
- package/dist/register-tools.js.map +1 -1
- package/dist/search/model2vec-tokenize.d.ts +22 -0
- package/dist/search/model2vec-tokenize.d.ts.map +1 -0
- package/dist/search/model2vec-tokenize.js +140 -0
- package/dist/search/model2vec-tokenize.js.map +1 -0
- package/dist/search/semantic.d.ts.map +1 -1
- package/dist/search/semantic.js +7 -0
- package/dist/search/semantic.js.map +1 -1
- package/dist/search/static-embedding-provider.d.ts +24 -0
- package/dist/search/static-embedding-provider.d.ts.map +1 -0
- package/dist/search/static-embedding-provider.js +157 -0
- package/dist/search/static-embedding-provider.js.map +1 -0
- package/dist/storage/_shared.d.ts.map +1 -1
- package/dist/storage/_shared.js +4 -1
- package/dist/storage/_shared.js.map +1 -1
- package/dist/storage/group-registry.d.ts +51 -0
- package/dist/storage/group-registry.d.ts.map +1 -0
- package/dist/storage/group-registry.js +263 -0
- package/dist/storage/group-registry.js.map +1 -0
- package/dist/storage/hash-snapshot.d.ts +36 -0
- package/dist/storage/hash-snapshot.d.ts.map +1 -0
- package/dist/storage/hash-snapshot.js +101 -0
- package/dist/storage/hash-snapshot.js.map +1 -0
- package/dist/storage/usage-stats.d.ts +8 -0
- package/dist/storage/usage-stats.d.ts.map +1 -1
- package/dist/storage/usage-stats.js +74 -24
- package/dist/storage/usage-stats.js.map +1 -1
- package/dist/storage/usage-tracker.d.ts +16 -0
- package/dist/storage/usage-tracker.d.ts.map +1 -1
- package/dist/storage/usage-tracker.js +23 -3
- package/dist/storage/usage-tracker.js.map +1 -1
- package/dist/tools/cross-repo-contract-tools.d.ts +137 -0
- package/dist/tools/cross-repo-contract-tools.d.ts.map +1 -0
- package/dist/tools/cross-repo-contract-tools.js +593 -0
- package/dist/tools/cross-repo-contract-tools.js.map +1 -0
- package/dist/tools/cross-repo-outbound-lexer.d.ts +35 -0
- package/dist/tools/cross-repo-outbound-lexer.d.ts.map +1 -0
- package/dist/tools/cross-repo-outbound-lexer.js +594 -0
- package/dist/tools/cross-repo-outbound-lexer.js.map +1 -0
- package/dist/tools/index-tools.d.ts +31 -2
- package/dist/tools/index-tools.d.ts.map +1 -1
- package/dist/tools/index-tools.js +463 -33
- package/dist/tools/index-tools.js.map +1 -1
- package/dist/tools/pg-introspect-tools.d.ts +147 -0
- package/dist/tools/pg-introspect-tools.d.ts.map +1 -0
- package/dist/tools/pg-introspect-tools.js +396 -0
- package/dist/tools/pg-introspect-tools.js.map +1 -0
- package/dist/types.d.ts +27 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/hf-download-stream.d.ts +21 -0
- package/dist/utils/hf-download-stream.d.ts.map +1 -0
- package/dist/utils/hf-download-stream.js +101 -0
- package/dist/utils/hf-download-stream.js.map +1 -0
- package/dist/utils/hf-hub-download.d.ts +8 -0
- package/dist/utils/hf-hub-download.d.ts.map +1 -0
- package/dist/utils/hf-hub-download.js +149 -0
- package/dist/utils/hf-hub-download.js.map +1 -0
- package/dist/utils/safetensors-loader.d.ts +9 -0
- package/dist/utils/safetensors-loader.d.ts.map +1 -0
- package/dist/utils/safetensors-loader.js +95 -0
- package/dist/utils/safetensors-loader.js.map +1 -0
- package/dist/utils/safetensors-meta-guard.d.ts +7 -0
- package/dist/utils/safetensors-meta-guard.d.ts.map +1 -0
- package/dist/utils/safetensors-meta-guard.js +50 -0
- package/dist/utils/safetensors-meta-guard.js.map +1 -0
- package/package.json +3 -1
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import { readFile, rename } from "node:fs/promises";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { atomicWriteFile } from "./_shared.js";
|
|
4
|
+
/**
|
|
5
|
+
* Return the canonical path for the group registry file inside a data directory.
|
|
6
|
+
*/
|
|
7
|
+
export function getGroupRegistryPath(dataDir) {
|
|
8
|
+
return join(dataDir, "groups.json");
|
|
9
|
+
}
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
// CRITICAL-3: per-path promise chain to serialize R-M-W mutations
|
|
12
|
+
// (mirrors writeLocks pattern from index-store.ts)
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
const writeLocks = new Map();
|
|
15
|
+
function chainMutation(registryPath, fn) {
|
|
16
|
+
// FIX-A: normalize path so "./x/groups.json" and "x/groups.json" share a lock
|
|
17
|
+
const lockKey = resolve(registryPath);
|
|
18
|
+
const prev = writeLocks.get(lockKey) ?? Promise.resolve();
|
|
19
|
+
const next = prev.then(fn);
|
|
20
|
+
// Swallow errors from the chain link so future callers aren't blocked
|
|
21
|
+
writeLocks.set(lockKey, next.catch(() => { }));
|
|
22
|
+
return next;
|
|
23
|
+
}
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
// CRITICAL-2: reserved name guard
|
|
26
|
+
// ---------------------------------------------------------------------------
|
|
27
|
+
const RESERVED_NAMES = /^(__proto__|constructor|prototype)$/;
|
|
28
|
+
function assertValidName(name) {
|
|
29
|
+
if (name.trim() === "") {
|
|
30
|
+
throw new Error(`Invalid group name: name must not be empty or whitespace-only.`);
|
|
31
|
+
}
|
|
32
|
+
if (RESERVED_NAMES.test(name)) {
|
|
33
|
+
throw new Error(`Invalid group name "${name}": reserved prototype/constructor names are not allowed.`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
// CRITICAL-1: split load semantics
|
|
38
|
+
// ENOENT → empty registry (legit first run)
|
|
39
|
+
// JSON/struct bad → quarantine (rename to .corrupt-<ts>), return empty
|
|
40
|
+
// other read error → THROW (prevents silent wipe on EACCES/EMFILE)
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
/**
|
|
43
|
+
* Load the group registry from disk.
|
|
44
|
+
*
|
|
45
|
+
* - ENOENT → returns empty registry (first run)
|
|
46
|
+
* - Corrupt JSON or invalid structure → renames file to groups.json.corrupt-<ts>,
|
|
47
|
+
* logs a warning, and returns empty registry (data is never silently destroyed)
|
|
48
|
+
* - Other I/O error (EACCES, EMFILE, …) → throws so callers can propagate
|
|
49
|
+
*/
|
|
50
|
+
export async function loadGroupRegistry(registryPath) {
|
|
51
|
+
let raw;
|
|
52
|
+
try {
|
|
53
|
+
raw = await readFile(registryPath, "utf-8");
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
const code = err.code;
|
|
57
|
+
if (code === "ENOENT") {
|
|
58
|
+
return emptyGroupRegistry();
|
|
59
|
+
}
|
|
60
|
+
// EACCES, EMFILE, etc. — throw so mutations don't overwrite with empty state
|
|
61
|
+
throw err;
|
|
62
|
+
}
|
|
63
|
+
// File exists — try to parse
|
|
64
|
+
let parsed;
|
|
65
|
+
try {
|
|
66
|
+
parsed = JSON.parse(raw);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
await quarantineCorrupt(registryPath);
|
|
70
|
+
return emptyGroupRegistry();
|
|
71
|
+
}
|
|
72
|
+
if (isValidGroupRegistry(parsed)) {
|
|
73
|
+
return parsed;
|
|
74
|
+
}
|
|
75
|
+
// Structurally invalid (valid JSON but wrong shape)
|
|
76
|
+
await quarantineCorrupt(registryPath);
|
|
77
|
+
return emptyGroupRegistry();
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Rename the corrupt file to a timestamped sibling so data is never lost.
|
|
81
|
+
* Best-effort — if renaming fails we swallow the error (we already know the
|
|
82
|
+
* data is unusable).
|
|
83
|
+
*/
|
|
84
|
+
async function quarantineCorrupt(registryPath) {
|
|
85
|
+
const dest = `${registryPath}.corrupt-${Date.now()}`;
|
|
86
|
+
try {
|
|
87
|
+
await rename(registryPath, dest);
|
|
88
|
+
console.warn(`[codesift] group-registry: corrupt file quarantined as ${dest}`);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
console.warn(`[codesift] group-registry: failed to quarantine corrupt file at ${registryPath}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Save the group registry atomically.
|
|
96
|
+
* Writes to a temp file first, then renames to prevent partial reads.
|
|
97
|
+
*/
|
|
98
|
+
export async function saveGroupRegistry(registryPath, registry) {
|
|
99
|
+
const data = JSON.stringify(registry);
|
|
100
|
+
await atomicWriteFile(registryPath, data);
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Register or update a group. Duplicate repos are deduplicated.
|
|
104
|
+
* On overwrite: preserves created_at, updates updated_at.
|
|
105
|
+
*
|
|
106
|
+
* Mutations are serialized per registry path (CRITICAL-3 R-M-W race fix).
|
|
107
|
+
* Invalid/reserved names are rejected immediately (CRITICAL-2).
|
|
108
|
+
*/
|
|
109
|
+
export async function registerGroup(registryPath, input) {
|
|
110
|
+
assertValidName(input.name);
|
|
111
|
+
// FIX-B: runtime input validation
|
|
112
|
+
if (!Array.isArray(input.repos)) {
|
|
113
|
+
throw new Error(`Invalid repos: expected an array of strings, got ${typeof input.repos}.`);
|
|
114
|
+
}
|
|
115
|
+
for (const r of input.repos) {
|
|
116
|
+
if (typeof r !== "string" || r === "") {
|
|
117
|
+
throw new Error(`Invalid repos: every element must be a non-empty string (got ${JSON.stringify(r)}).`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (input.description !== undefined && typeof input.description !== "string") {
|
|
121
|
+
throw new Error(`Invalid description: expected a string, got ${typeof input.description}.`);
|
|
122
|
+
}
|
|
123
|
+
return chainMutation(registryPath, async () => {
|
|
124
|
+
const registry = await loadGroupRegistry(registryPath);
|
|
125
|
+
const now = Date.now();
|
|
126
|
+
const existing = Object.hasOwn(registry.groups, input.name)
|
|
127
|
+
? registry.groups[input.name]
|
|
128
|
+
: undefined;
|
|
129
|
+
const deduped = [...new Set(input.repos)];
|
|
130
|
+
const group = existing
|
|
131
|
+
? {
|
|
132
|
+
name: input.name,
|
|
133
|
+
repos: deduped,
|
|
134
|
+
...(input.description !== undefined ? { description: input.description } : {}),
|
|
135
|
+
created_at: existing.created_at,
|
|
136
|
+
updated_at: now,
|
|
137
|
+
}
|
|
138
|
+
: {
|
|
139
|
+
name: input.name,
|
|
140
|
+
repos: deduped,
|
|
141
|
+
...(input.description !== undefined ? { description: input.description } : {}),
|
|
142
|
+
created_at: now,
|
|
143
|
+
updated_at: now,
|
|
144
|
+
};
|
|
145
|
+
registry.groups[input.name] = group;
|
|
146
|
+
registry.updated_at = now;
|
|
147
|
+
await saveGroupRegistry(registryPath, registry);
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Get a single group by name. Returns null if not found.
|
|
152
|
+
*
|
|
153
|
+
* Read-only: catches non-fatal load errors and returns null with a warning.
|
|
154
|
+
*/
|
|
155
|
+
export async function getGroup(registryPath, name) {
|
|
156
|
+
let registry;
|
|
157
|
+
try {
|
|
158
|
+
registry = await loadGroupRegistry(registryPath);
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
console.warn(`[codesift] group-registry: could not read registry at ${registryPath}`);
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
return Object.hasOwn(registry.groups, name) ? registry.groups[name] : null;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* List all registered groups.
|
|
168
|
+
*
|
|
169
|
+
* Read-only: catches non-fatal load errors and returns empty array with a warning.
|
|
170
|
+
*/
|
|
171
|
+
export async function listGroups(registryPath) {
|
|
172
|
+
let registry;
|
|
173
|
+
try {
|
|
174
|
+
registry = await loadGroupRegistry(registryPath);
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
console.warn(`[codesift] group-registry: could not read registry at ${registryPath}`);
|
|
178
|
+
return [];
|
|
179
|
+
}
|
|
180
|
+
return Object.keys(registry.groups)
|
|
181
|
+
.filter((k) => Object.hasOwn(registry.groups, k))
|
|
182
|
+
.map((k) => registry.groups[k]);
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Remove a group by name.
|
|
186
|
+
* Returns true if it existed and was removed, false otherwise (idempotent).
|
|
187
|
+
*
|
|
188
|
+
* Mutations are serialized per registry path (CRITICAL-3 R-M-W race fix).
|
|
189
|
+
*/
|
|
190
|
+
export async function removeGroup(registryPath, name) {
|
|
191
|
+
return chainMutation(registryPath, async () => {
|
|
192
|
+
const registry = await loadGroupRegistry(registryPath);
|
|
193
|
+
if (!Object.hasOwn(registry.groups, name)) {
|
|
194
|
+
return false;
|
|
195
|
+
}
|
|
196
|
+
delete registry.groups[name];
|
|
197
|
+
registry.updated_at = Date.now();
|
|
198
|
+
await saveGroupRegistry(registryPath, registry);
|
|
199
|
+
return true;
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
// ---------------------------------------------------------------------------
|
|
203
|
+
// Internal helpers
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
function emptyGroupRegistry() {
|
|
206
|
+
return { groups: {}, updated_at: Date.now() };
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Deep structural guard for a raw parsed GroupRegistry value.
|
|
210
|
+
* Mirrors the Array.isArray improvements from hash-snapshot.ts.
|
|
211
|
+
*/
|
|
212
|
+
function isValidGroupRegistry(value) {
|
|
213
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
const obj = value;
|
|
217
|
+
if (typeof obj["updated_at"] !== "number")
|
|
218
|
+
return false;
|
|
219
|
+
const groups = obj["groups"];
|
|
220
|
+
if (typeof groups !== "object" || groups === null || Array.isArray(groups)) {
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
223
|
+
// FIX-C: validate each group entry, guarding against prototype pollution,
|
|
224
|
+
// key/name mismatch, and reserved keys crafted on disk
|
|
225
|
+
for (const key of Object.keys(groups)) {
|
|
226
|
+
if (!Object.hasOwn(groups, key))
|
|
227
|
+
continue;
|
|
228
|
+
// Reject reserved keys even if they somehow appear in the JSON
|
|
229
|
+
if (RESERVED_NAMES.test(key))
|
|
230
|
+
return false;
|
|
231
|
+
const entry = groups[key];
|
|
232
|
+
if (!isValidRepoGroup(entry))
|
|
233
|
+
return false;
|
|
234
|
+
// key must equal the group's own name field — mismatch = tampered file
|
|
235
|
+
if (entry["name"] !== key)
|
|
236
|
+
return false;
|
|
237
|
+
}
|
|
238
|
+
return true;
|
|
239
|
+
}
|
|
240
|
+
function isValidRepoGroup(value) {
|
|
241
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
242
|
+
return false;
|
|
243
|
+
}
|
|
244
|
+
const obj = value;
|
|
245
|
+
if (typeof obj["name"] !== "string")
|
|
246
|
+
return false;
|
|
247
|
+
if (typeof obj["created_at"] !== "number")
|
|
248
|
+
return false;
|
|
249
|
+
if (typeof obj["updated_at"] !== "number")
|
|
250
|
+
return false;
|
|
251
|
+
const repos = obj["repos"];
|
|
252
|
+
if (!Array.isArray(repos))
|
|
253
|
+
return false;
|
|
254
|
+
for (const r of repos) {
|
|
255
|
+
if (typeof r !== "string")
|
|
256
|
+
return false;
|
|
257
|
+
}
|
|
258
|
+
// description is optional — if present it must be a string
|
|
259
|
+
if (Object.hasOwn(obj, "description") && typeof obj["description"] !== "string")
|
|
260
|
+
return false;
|
|
261
|
+
return true;
|
|
262
|
+
}
|
|
263
|
+
//# sourceMappingURL=group-registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"group-registry.js","sourceRoot":"","sources":["../../src/storage/group-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;AACtC,CAAC;AAED,8EAA8E;AAC9E,kEAAkE;AAClE,mDAAmD;AACnD,8EAA8E;AAC9E,MAAM,UAAU,GAAG,IAAI,GAAG,EAA4B,CAAC;AAEvD,SAAS,aAAa,CAAI,YAAoB,EAAE,EAAoB;IAClE,8EAA8E;IAC9E,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3B,sEAAsE;IACtE,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,kCAAkC;AAClC,8EAA8E;AAC9E,MAAM,cAAc,GAAG,qCAAqC,CAAC;AAE7D,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,uBAAuB,IAAI,0DAA0D,CACtF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mCAAmC;AACnC,wDAAwD;AACxD,0EAA0E;AAC1E,qEAAqE;AACrE,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,YAAoB;IAC1D,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;QACjD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,OAAO,kBAAkB,EAAE,CAAC;QAC9B,CAAC;QACD,6EAA6E;QAC7E,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,6BAA6B;IAC7B,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACtC,OAAO,kBAAkB,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;QACjC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,oDAAoD;IACpD,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACtC,OAAO,kBAAkB,EAAE,CAAC;AAC9B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,iBAAiB,CAAC,YAAoB;IACnD,MAAM,IAAI,GAAG,GAAG,YAAY,YAAY,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CACV,0DAA0D,IAAI,EAAE,CACjE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CACV,mEAAmE,YAAY,EAAE,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,YAAoB,EACpB,QAAuB;IAEvB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,eAAe,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,YAAoB,EACpB,KAA8D;IAE9D,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAE5B,kCAAkC;IAClC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,oDAAoD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IAC7F,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,gEAAgE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CACtF,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QAC7E,MAAM,IAAI,KAAK,CACb,+CAA+C,OAAO,KAAK,CAAC,WAAW,GAAG,CAC3E,CAAC;IACJ,CAAC;IAED,OAAO,aAAa,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC;YACzD,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAE1C,MAAM,KAAK,GAAc,QAAQ;YAC/B,CAAC,CAAC;gBACE,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9E,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,UAAU,EAAE,GAAG;aAChB;YACH,CAAC,CAAC;gBACE,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9E,UAAU,EAAE,GAAG;gBACf,UAAU,EAAE,GAAG;aAChB,CAAC;QAEN,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACpC,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;QAC1B,MAAM,iBAAiB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,YAAoB,EACpB,IAAY;IAEZ,IAAI,QAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,yDAAyD,YAAY,EAAE,CAAC,CAAC;QACtF,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9E,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,YAAoB;IACnD,IAAI,QAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,yDAAyD,YAAY,EAAE,CAAC,CAAC;QACtF,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;AACrC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,YAAoB,EACpB,IAAY;IAEZ,OAAO,aAAa,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAEvD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,MAAM,iBAAiB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,kBAAkB;IACzB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,KAAc;IAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,KAAgC,CAAC;IAE7C,IAAI,OAAO,GAAG,CAAC,YAAY,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAExD,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,OAAO,KAAK,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,uDAAuD;IACvD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAgB,EAAE,GAAG,CAAC;YAAE,SAAS;QACpD,+DAA+D;QAC/D,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAC3C,MAAM,KAAK,GAAI,MAAkC,CAAC,GAAG,CAAC,CAAC;QACvD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC3C,uEAAuE;QACvE,IAAK,KAA4C,CAAC,MAAM,CAAC,KAAK,GAAG;YAAE,OAAO,KAAK,CAAC;IAClF,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,KAAgC,CAAC;IAE7C,IAAI,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClD,IAAI,OAAO,GAAG,CAAC,YAAY,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACxD,IAAI,OAAO,GAAG,CAAC,YAAY,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAExD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;IAC1C,CAAC;IAED,2DAA2D;IAC3D,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,aAAa,CAAC,IAAI,OAAO,GAAG,CAAC,aAAa,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE9F,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export declare const HASH_SNAPSHOT_VERSION = 1;
|
|
2
|
+
export interface FileHashSnapshot {
|
|
3
|
+
version: 1;
|
|
4
|
+
repo: string;
|
|
5
|
+
created_at: number;
|
|
6
|
+
files: Record<string, string>;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Convert an index path to its companion snapshot path.
|
|
10
|
+
* e.g. "/x/abc123.index.json" → "/x/abc123.snapshot.json"
|
|
11
|
+
*
|
|
12
|
+
* Throws if indexPath does not end with ".index.json" (safety guard against
|
|
13
|
+
* accidentally overwriting the wrong file).
|
|
14
|
+
*/
|
|
15
|
+
export declare function getSnapshotPath(indexPath: string): string;
|
|
16
|
+
/**
|
|
17
|
+
* Write a FileHashSnapshot atomically.
|
|
18
|
+
*/
|
|
19
|
+
export declare function saveHashSnapshot(path: string, snap: FileHashSnapshot): Promise<void>;
|
|
20
|
+
/**
|
|
21
|
+
* Load a FileHashSnapshot from disk.
|
|
22
|
+
* Returns null when:
|
|
23
|
+
* - the file does not exist
|
|
24
|
+
* - the file contains invalid JSON
|
|
25
|
+
* - the stored version !== 1
|
|
26
|
+
* - expectedRepo is provided and does not match snapshot.repo
|
|
27
|
+
*
|
|
28
|
+
* Non-ENOENT file read errors are logged as a warning before returning null.
|
|
29
|
+
* JSON parse errors are silent (expected corruption path).
|
|
30
|
+
*/
|
|
31
|
+
export declare function loadHashSnapshot(path: string, expectedRepo?: string): Promise<FileHashSnapshot | null>;
|
|
32
|
+
/**
|
|
33
|
+
* Delete a snapshot file idempotently — never throws if the file is absent.
|
|
34
|
+
*/
|
|
35
|
+
export declare function deleteHashSnapshot(path: string): Promise<void>;
|
|
36
|
+
//# sourceMappingURL=hash-snapshot.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash-snapshot.d.ts","sourceRoot":"","sources":["../../src/storage/hash-snapshot.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,qBAAqB,IAAI,CAAC;AAEvC,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAOzD;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,gBAAgB,GACrB,OAAO,CAAC,IAAI,CAAC,CAGf;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAkClC;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CASpE"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { readFile, unlink } from "node:fs/promises";
|
|
2
|
+
import { atomicWriteFile } from "./_shared.js";
|
|
3
|
+
export const HASH_SNAPSHOT_VERSION = 1;
|
|
4
|
+
/**
|
|
5
|
+
* Convert an index path to its companion snapshot path.
|
|
6
|
+
* e.g. "/x/abc123.index.json" → "/x/abc123.snapshot.json"
|
|
7
|
+
*
|
|
8
|
+
* Throws if indexPath does not end with ".index.json" (safety guard against
|
|
9
|
+
* accidentally overwriting the wrong file).
|
|
10
|
+
*/
|
|
11
|
+
export function getSnapshotPath(indexPath) {
|
|
12
|
+
if (!indexPath.endsWith(".index.json")) {
|
|
13
|
+
throw new Error(`hash-snapshot: expected an .index.json path, got "${indexPath}"`);
|
|
14
|
+
}
|
|
15
|
+
return indexPath.replace(/\.index\.json$/, ".snapshot.json");
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Write a FileHashSnapshot atomically.
|
|
19
|
+
*/
|
|
20
|
+
export async function saveHashSnapshot(path, snap) {
|
|
21
|
+
const data = JSON.stringify(snap);
|
|
22
|
+
await atomicWriteFile(path, data);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Load a FileHashSnapshot from disk.
|
|
26
|
+
* Returns null when:
|
|
27
|
+
* - the file does not exist
|
|
28
|
+
* - the file contains invalid JSON
|
|
29
|
+
* - the stored version !== 1
|
|
30
|
+
* - expectedRepo is provided and does not match snapshot.repo
|
|
31
|
+
*
|
|
32
|
+
* Non-ENOENT file read errors are logged as a warning before returning null.
|
|
33
|
+
* JSON parse errors are silent (expected corruption path).
|
|
34
|
+
*/
|
|
35
|
+
export async function loadHashSnapshot(path, expectedRepo) {
|
|
36
|
+
let raw;
|
|
37
|
+
try {
|
|
38
|
+
raw = await readFile(path, "utf-8");
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
// ENOENT = expected case (file doesn't exist yet). Other errors = log & degrade.
|
|
42
|
+
const errno = err?.code;
|
|
43
|
+
if (errno !== "ENOENT") {
|
|
44
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
45
|
+
console.warn(`[codesift] hash-snapshot load failed (degrading to full re-parse): ${msg}`);
|
|
46
|
+
}
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
let parsed;
|
|
50
|
+
try {
|
|
51
|
+
parsed = JSON.parse(raw);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// JSON parse errors are silent (corruption is expected & recoverable)
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
if (!isValidSnapshot(parsed)) {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
if (expectedRepo !== undefined && parsed.repo !== expectedRepo) {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
return parsed;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Delete a snapshot file idempotently — never throws if the file is absent.
|
|
67
|
+
*/
|
|
68
|
+
export async function deleteHashSnapshot(path) {
|
|
69
|
+
try {
|
|
70
|
+
await unlink(path);
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
// ENOENT = already gone, which is fine. Re-throw anything else.
|
|
74
|
+
if (err.code !== "ENOENT") {
|
|
75
|
+
throw err;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
function isValidSnapshot(value) {
|
|
80
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
const obj = value;
|
|
84
|
+
if (obj["version"] !== HASH_SNAPSHOT_VERSION)
|
|
85
|
+
return false;
|
|
86
|
+
if (typeof obj["repo"] !== "string")
|
|
87
|
+
return false;
|
|
88
|
+
if (typeof obj["created_at"] !== "number")
|
|
89
|
+
return false;
|
|
90
|
+
const files = obj["files"];
|
|
91
|
+
if (typeof files !== "object" || files === null || Array.isArray(files)) {
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
// All values in files must be strings
|
|
95
|
+
for (const v of Object.values(files)) {
|
|
96
|
+
if (typeof v !== "string")
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
return true;
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=hash-snapshot.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash-snapshot.js","sourceRoot":"","sources":["../../src/storage/hash-snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AASvC;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,SAAiB;IAC/C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,qDAAqD,SAAS,GAAG,CAClE,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,IAAsB;IAEtB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,YAAqB;IAErB,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,iFAAiF;QACjF,MAAM,KAAK,GAAI,GAA6B,EAAE,IAAI,CAAC;QACnD,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvB,MAAM,GAAG,GACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CACV,sEAAsE,GAAG,EAAE,CAC5E,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,sEAAsE;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QAC/D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACnD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,gEAAgE;QAChE,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,KAAgC,CAAC;IAE7C,IAAI,GAAG,CAAC,SAAS,CAAC,KAAK,qBAAqB;QAAE,OAAO,KAAK,CAAC;IAC3D,IAAI,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClD,IAAI,OAAO,GAAG,CAAC,YAAY,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAExD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sCAAsC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;IAC1C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -9,6 +9,11 @@ export interface RepoStats {
|
|
|
9
9
|
repo: string;
|
|
10
10
|
call_count: number;
|
|
11
11
|
}
|
|
12
|
+
export interface HostStats {
|
|
13
|
+
host: string;
|
|
14
|
+
call_count: number;
|
|
15
|
+
total_tokens: number;
|
|
16
|
+
}
|
|
12
17
|
export interface DailyStats {
|
|
13
18
|
date: string;
|
|
14
19
|
call_count: number;
|
|
@@ -25,6 +30,8 @@ export interface UsageStats {
|
|
|
25
30
|
avg_calls_per_session: number;
|
|
26
31
|
tools: ToolStats[];
|
|
27
32
|
top_repos: RepoStats[];
|
|
33
|
+
/** Per-machine breakdown — only meaningful once usage-remote/ logs are synced in. */
|
|
34
|
+
hosts: HostStats[];
|
|
28
35
|
daily: DailyStats[];
|
|
29
36
|
query_types: QueryTypeStats[];
|
|
30
37
|
earliest_ts: number;
|
|
@@ -35,6 +42,7 @@ export declare function getUsageStats(options?: {
|
|
|
35
42
|
repo?: string;
|
|
36
43
|
tool?: string;
|
|
37
44
|
session_id?: string;
|
|
45
|
+
host?: string;
|
|
38
46
|
}): Promise<UsageStats>;
|
|
39
47
|
export declare function formatUsageReport(stats: UsageStats): string;
|
|
40
48
|
//# sourceMappingURL=usage-stats.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"usage-stats.d.ts","sourceRoot":"","sources":["../../src/storage/usage-stats.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"usage-stats.d.ts","sourceRoot":"","sources":["../../src/storage/usage-stats.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,SAAS,EAAE,SAAS,EAAE,CAAC;IACvB,qFAAqF;IACrF,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAyFD,wBAAsB,aAAa,CAAC,OAAO,CAAC,EAAE;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,UAAU,CAAC,CAqItB;AAMD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmF3D"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { readFile } from "node:fs/promises";
|
|
2
|
-
import {
|
|
1
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
2
|
+
import { join, basename } from "node:path";
|
|
3
|
+
import { getCumulativeSavings, getUsagePath, getRemoteUsageDir, getLocalHostTag } from "./usage-tracker.js";
|
|
3
4
|
import { formatTable } from "../formatters.js";
|
|
4
5
|
// ---------------------------------------------------------------------------
|
|
5
6
|
// Loading
|
|
@@ -13,42 +14,71 @@ function isValidEntry(value) {
|
|
|
13
14
|
typeof obj["elapsed_ms"] === "number" &&
|
|
14
15
|
typeof obj["session_id"] === "string");
|
|
15
16
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
/** The local log plus any host logs synced into usage-remote/. Each source
|
|
18
|
+
* carries a fallback host tag for entries written before the `host` field
|
|
19
|
+
* existed: the local hostname, or the remote file's name stem. */
|
|
20
|
+
async function listUsageSources() {
|
|
21
|
+
const sources = [{ path: getUsagePath(), fallbackHost: getLocalHostTag() }];
|
|
22
|
+
const remoteDir = getRemoteUsageDir();
|
|
19
23
|
try {
|
|
20
|
-
|
|
24
|
+
for (const name of await readdir(remoteDir)) {
|
|
25
|
+
if (!name.endsWith(".jsonl"))
|
|
26
|
+
continue;
|
|
27
|
+
sources.push({
|
|
28
|
+
path: join(remoteDir, name),
|
|
29
|
+
fallbackHost: basename(name, ".jsonl"),
|
|
30
|
+
});
|
|
31
|
+
}
|
|
21
32
|
}
|
|
22
33
|
catch {
|
|
23
|
-
|
|
34
|
+
// usage-remote/ absent — single-machine setup
|
|
24
35
|
}
|
|
36
|
+
return sources;
|
|
37
|
+
}
|
|
38
|
+
async function loadEntries(options) {
|
|
25
39
|
const sinceTs = options?.since ? new Date(options.since).getTime() : 0;
|
|
26
40
|
const repoFilter = options?.repo ?? null;
|
|
27
41
|
const toolFilter = options?.tool ?? null;
|
|
28
42
|
const sessionFilter = options?.session_id ?? null;
|
|
43
|
+
const hostFilter = options?.host ?? null;
|
|
29
44
|
const entries = [];
|
|
30
|
-
for (const
|
|
31
|
-
|
|
32
|
-
if (!trimmed)
|
|
33
|
-
continue;
|
|
45
|
+
for (const source of await listUsageSources()) {
|
|
46
|
+
let raw;
|
|
34
47
|
try {
|
|
35
|
-
|
|
36
|
-
if (!isValidEntry(parsed))
|
|
37
|
-
continue;
|
|
38
|
-
if (parsed.ts < sinceTs)
|
|
39
|
-
continue;
|
|
40
|
-
if (repoFilter && parsed.repo !== repoFilter)
|
|
41
|
-
continue;
|
|
42
|
-
if (toolFilter && parsed.tool !== toolFilter)
|
|
43
|
-
continue;
|
|
44
|
-
if (sessionFilter && parsed.session_id !== sessionFilter)
|
|
45
|
-
continue;
|
|
46
|
-
entries.push(parsed);
|
|
48
|
+
raw = await readFile(source.path, "utf-8");
|
|
47
49
|
}
|
|
48
50
|
catch {
|
|
49
|
-
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
for (const line of raw.split("\n")) {
|
|
54
|
+
const trimmed = line.trim();
|
|
55
|
+
if (!trimmed)
|
|
56
|
+
continue;
|
|
57
|
+
try {
|
|
58
|
+
const parsed = JSON.parse(trimmed);
|
|
59
|
+
if (!isValidEntry(parsed))
|
|
60
|
+
continue;
|
|
61
|
+
if (parsed.host === undefined)
|
|
62
|
+
parsed.host = source.fallbackHost;
|
|
63
|
+
if (parsed.ts < sinceTs)
|
|
64
|
+
continue;
|
|
65
|
+
if (repoFilter && parsed.repo !== repoFilter)
|
|
66
|
+
continue;
|
|
67
|
+
if (toolFilter && parsed.tool !== toolFilter)
|
|
68
|
+
continue;
|
|
69
|
+
if (sessionFilter && parsed.session_id !== sessionFilter)
|
|
70
|
+
continue;
|
|
71
|
+
if (hostFilter && parsed.host !== hostFilter)
|
|
72
|
+
continue;
|
|
73
|
+
entries.push(parsed);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// Skip malformed lines
|
|
77
|
+
}
|
|
50
78
|
}
|
|
51
79
|
}
|
|
80
|
+
// Sources are per-host append logs — interleave them chronologically.
|
|
81
|
+
entries.sort((a, b) => a.ts - b.ts);
|
|
52
82
|
return entries;
|
|
53
83
|
}
|
|
54
84
|
// ---------------------------------------------------------------------------
|
|
@@ -63,6 +93,7 @@ export async function getUsageStats(options) {
|
|
|
63
93
|
avg_calls_per_session: 0,
|
|
64
94
|
tools: [],
|
|
65
95
|
top_repos: [],
|
|
96
|
+
hosts: [],
|
|
66
97
|
daily: [],
|
|
67
98
|
query_types: [],
|
|
68
99
|
earliest_ts: 0,
|
|
@@ -72,6 +103,7 @@ export async function getUsageStats(options) {
|
|
|
72
103
|
// Per-tool aggregation
|
|
73
104
|
const toolMap = new Map();
|
|
74
105
|
const repoMap = new Map();
|
|
106
|
+
const hostMap = new Map();
|
|
75
107
|
const sessionSet = new Set();
|
|
76
108
|
const dailyMap = new Map();
|
|
77
109
|
// codebase_retrieval query type breakdown
|
|
@@ -90,6 +122,12 @@ export async function getUsageStats(options) {
|
|
|
90
122
|
if (entry.repo) {
|
|
91
123
|
repoMap.set(entry.repo, (repoMap.get(entry.repo) ?? 0) + 1);
|
|
92
124
|
}
|
|
125
|
+
// Host stats (loadEntries guarantees a host on every entry)
|
|
126
|
+
const hostKey = entry.host ?? "unknown";
|
|
127
|
+
const hostStats = hostMap.get(hostKey) ?? { calls: 0, tokens: 0 };
|
|
128
|
+
hostStats.calls += 1;
|
|
129
|
+
hostStats.tokens += entry.result_tokens;
|
|
130
|
+
hostMap.set(hostKey, hostStats);
|
|
93
131
|
// Sessions
|
|
94
132
|
sessionSet.add(entry.session_id);
|
|
95
133
|
// Daily
|
|
@@ -134,6 +172,9 @@ export async function getUsageStats(options) {
|
|
|
134
172
|
.map(([repo, call_count]) => ({ repo, call_count }))
|
|
135
173
|
.sort((a, b) => b.call_count - a.call_count)
|
|
136
174
|
.slice(0, 20);
|
|
175
|
+
const hosts = [...hostMap.entries()]
|
|
176
|
+
.map(([host, stats]) => ({ host, call_count: stats.calls, total_tokens: stats.tokens }))
|
|
177
|
+
.sort((a, b) => b.call_count - a.call_count);
|
|
137
178
|
const daily = [...dailyMap.entries()]
|
|
138
179
|
.map(([date, stats]) => ({
|
|
139
180
|
date,
|
|
@@ -155,6 +196,7 @@ export async function getUsageStats(options) {
|
|
|
155
196
|
avg_calls_per_session: Math.round((entries.length / totalSessions) * 10) / 10,
|
|
156
197
|
tools,
|
|
157
198
|
top_repos,
|
|
199
|
+
hosts,
|
|
158
200
|
daily,
|
|
159
201
|
query_types,
|
|
160
202
|
earliest_ts: earliest,
|
|
@@ -214,6 +256,14 @@ export function formatUsageReport(stats) {
|
|
|
214
256
|
}
|
|
215
257
|
lines.push("");
|
|
216
258
|
}
|
|
259
|
+
// Hosts — only worth showing once logs from more than one machine are merged
|
|
260
|
+
if (stats.hosts.length > 1) {
|
|
261
|
+
lines.push("--- Hosts ---");
|
|
262
|
+
for (const h of stats.hosts) {
|
|
263
|
+
lines.push(` ${h.host}: ${h.call_count} calls, ${h.total_tokens} tokens`);
|
|
264
|
+
}
|
|
265
|
+
lines.push("");
|
|
266
|
+
}
|
|
217
267
|
// Daily breakdown (last 14 days max)
|
|
218
268
|
if (stats.daily.length > 0) {
|
|
219
269
|
lines.push("--- Daily Usage ---");
|