grepmax 0.25.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +5 -0
- package/dist/commands/doctor.js +28 -11
- package/dist/lib/daemon/watcher-manager.js +23 -3
- package/dist/lib/index/batch-processor.js +31 -5
- package/dist/lib/index/cache-coherence.js +75 -0
- package/dist/lib/index/syncer.js +37 -35
- package/dist/lib/index/watcher-batch.js +8 -2
- package/dist/lib/index/watcher.js +14 -3
- package/dist/lib/utils/file-utils.js +2 -30
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.
|
|
13
|
+
"version": "0.26.0",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/README.md
CHANGED
|
@@ -347,6 +347,11 @@ gmax repair --rebuild # guarded whole-corpus rebuild through the d
|
|
|
347
347
|
the previous registry shape. Run `gmax index` in that project to persist exact identity; no reset or
|
|
348
348
|
re-embedding is required when the cached files are unchanged.
|
|
349
349
|
|
|
350
|
+
Cache metadata migrations are lazy and do not require a reset. Catchup stamps compatible legacy
|
|
351
|
+
entries in place, hashes Markdown and MDX by exact bytes, and reprocesses only legacy Markdown or
|
|
352
|
+
paths whose declared vector state disagrees with LanceDB. Files that intentionally produce no
|
|
353
|
+
vectors remain valid cached entries.
|
|
354
|
+
|
|
350
355
|
A model change cannot be applied by a per-project `gmax index --reset`, even when vector widths
|
|
351
356
|
match: one query embedding cannot safely search rows from another embedding space. Stale-generation
|
|
352
357
|
search and sync fail before mutation, preserving the existing index. Run `gmax repair --rebuild` to
|
package/dist/commands/doctor.js
CHANGED
|
@@ -435,21 +435,38 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
435
435
|
if (projects.length > 0) {
|
|
436
436
|
console.log("\nCache Coherence\n");
|
|
437
437
|
try {
|
|
438
|
+
const { reconcileMetaEntry } = yield Promise.resolve().then(() => __importStar(require("../lib/index/cache-coherence")));
|
|
438
439
|
const { MetaCache } = yield Promise.resolve().then(() => __importStar(require("../lib/store/meta-cache")));
|
|
439
440
|
const mc = new MetaCache(config_1.PATHS.lmdbPath);
|
|
440
|
-
|
|
441
|
-
const
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
441
|
+
try {
|
|
442
|
+
for (const project of projects.filter((p) => p.status === "indexed")) {
|
|
443
|
+
const prefix = project.root.endsWith("/")
|
|
444
|
+
? project.root
|
|
445
|
+
: `${project.root}/`;
|
|
446
|
+
const cachedPaths = yield mc.getKeysWithPrefix(prefix);
|
|
447
|
+
const vectorPaths = yield db.getDistinctPathsForPrefix(prefix);
|
|
448
|
+
let pending = 0;
|
|
449
|
+
let legacy = 0;
|
|
450
|
+
for (const cachedPath of cachedPaths) {
|
|
451
|
+
const reconciliation = reconcileMetaEntry(cachedPath, mc.get(cachedPath), vectorPaths.has(cachedPath));
|
|
452
|
+
if (reconciliation.action === "reprocess")
|
|
453
|
+
pending++;
|
|
454
|
+
else if (reconciliation.action === "stamp")
|
|
455
|
+
legacy++;
|
|
456
|
+
}
|
|
457
|
+
for (const vectorPath of vectorPaths) {
|
|
458
|
+
if (!cachedPaths.has(vectorPath))
|
|
459
|
+
pending++;
|
|
460
|
+
}
|
|
461
|
+
if (cachedPaths.size > 0 || vectorPaths.size > 0) {
|
|
462
|
+
const status = pending === 0 ? "ok" : "WARN";
|
|
463
|
+
console.log(`${status} ${project.name || path.basename(project.root)}: ${vectorPaths.size} vector-backed / ${cachedPaths.size} cached; ${pending} pending reconciliation, ${legacy} legacy metadata`);
|
|
464
|
+
}
|
|
450
465
|
}
|
|
451
466
|
}
|
|
452
|
-
|
|
467
|
+
finally {
|
|
468
|
+
yield mc.close();
|
|
469
|
+
}
|
|
453
470
|
}
|
|
454
471
|
catch (_k) { }
|
|
455
472
|
}
|
|
@@ -53,6 +53,7 @@ exports.WatcherManager = void 0;
|
|
|
53
53
|
const path = __importStar(require("node:path"));
|
|
54
54
|
const watcher = __importStar(require("@parcel/watcher"));
|
|
55
55
|
const batch_processor_1 = require("../index/batch-processor");
|
|
56
|
+
const cache_coherence_1 = require("../index/cache-coherence");
|
|
56
57
|
const file_policy_1 = require("../index/file-policy");
|
|
57
58
|
const walker_1 = require("../index/walker");
|
|
58
59
|
const watcher_1 = require("../index/watcher");
|
|
@@ -451,9 +452,12 @@ class WatcherManager {
|
|
|
451
452
|
var _d, _e;
|
|
452
453
|
const { isFileCached } = yield Promise.resolve().then(() => __importStar(require("../utils/cache-check")));
|
|
453
454
|
const metaCache = this.deps.getMetaCache();
|
|
455
|
+
const vectorDb = this.deps.getVectorDb();
|
|
454
456
|
processor.filePolicy.invalidateIgnoreCache();
|
|
455
457
|
const rootPrefix = root.endsWith("/") ? root : `${root}/`;
|
|
456
458
|
const cachedPaths = yield metaCache.getKeysWithPrefix(rootPrefix);
|
|
459
|
+
const vectorPaths = yield vectorDb.getDistinctPathsForPrefix(rootPrefix);
|
|
460
|
+
const knownPaths = new Set([...cachedPaths, ...vectorPaths]);
|
|
457
461
|
if (signal.aborted)
|
|
458
462
|
return false;
|
|
459
463
|
const seenPaths = new Set();
|
|
@@ -486,7 +490,23 @@ class WatcherManager {
|
|
|
486
490
|
continue;
|
|
487
491
|
const stats = classification.stat;
|
|
488
492
|
seenPaths.add(absPath);
|
|
489
|
-
|
|
493
|
+
let cached = metaCache.get(absPath);
|
|
494
|
+
const reconciliation = (0, cache_coherence_1.reconcileMetaEntry)(absPath, cached, vectorPaths.has(absPath));
|
|
495
|
+
if (reconciliation.action === "stamp") {
|
|
496
|
+
cached = reconciliation.entry;
|
|
497
|
+
metaCache.put(absPath, cached);
|
|
498
|
+
}
|
|
499
|
+
else if (reconciliation.action === "reprocess") {
|
|
500
|
+
processor.handleFileEvent("change", absPath, {
|
|
501
|
+
forceReprocess: true,
|
|
502
|
+
});
|
|
503
|
+
queued++;
|
|
504
|
+
if (queued % 500 === 0) {
|
|
505
|
+
(0, logger_1.debug)("catchup", `${path.basename(root)}: throttle pause at ${queued} queued`);
|
|
506
|
+
yield abortableDelay(5000, signal);
|
|
507
|
+
}
|
|
508
|
+
continue;
|
|
509
|
+
}
|
|
490
510
|
if (!isFileCached(cached, stats)) {
|
|
491
511
|
// Fast path: if only mtime changed but size is identical and we have a hash,
|
|
492
512
|
// just verify the hash in-process instead of sending to a worker.
|
|
@@ -545,12 +565,12 @@ class WatcherManager {
|
|
|
545
565
|
(0, logger_1.debug)("catchup", `${path.basename(root)}: ${queued} queued, ${skipped} skipped (cached ok), ${seenPaths.size} total`);
|
|
546
566
|
// Purge files deleted while daemon was offline
|
|
547
567
|
let purged = 0;
|
|
548
|
-
for (const cachedPath of
|
|
568
|
+
for (const cachedPath of knownPaths) {
|
|
549
569
|
if (signal.aborted)
|
|
550
570
|
return false;
|
|
551
571
|
if (!seenPaths.has(cachedPath) &&
|
|
552
572
|
!(0, walker_1.isPathProtectedByWalkState)(cachedPath, walkState)) {
|
|
553
|
-
processor.handleFileEvent("unlink", cachedPath);
|
|
573
|
+
processor.handleFileEvent("unlink", cachedPath, { forceDelete: true });
|
|
554
574
|
purged++;
|
|
555
575
|
}
|
|
556
576
|
}
|
|
@@ -49,6 +49,7 @@ const cache_check_1 = require("../utils/cache-check");
|
|
|
49
49
|
const file_utils_1 = require("../utils/file-utils");
|
|
50
50
|
const logger_1 = require("../utils/logger");
|
|
51
51
|
const pool_1 = require("../workers/pool");
|
|
52
|
+
const cache_coherence_1 = require("./cache-coherence");
|
|
52
53
|
const file_policy_1 = require("./file-policy");
|
|
53
54
|
const watcher_batch_1 = require("./watcher-batch");
|
|
54
55
|
const DEBOUNCE_MS = 2000;
|
|
@@ -61,6 +62,8 @@ class ProjectBatchProcessor {
|
|
|
61
62
|
this.retryCount = new Map();
|
|
62
63
|
this.retryAt = new Map();
|
|
63
64
|
this.terminalFailures = new Set();
|
|
65
|
+
this.forcedReprocess = new Map();
|
|
66
|
+
this.forceGeneration = 0;
|
|
64
67
|
this.debounceTimer = null;
|
|
65
68
|
this.debounceDueMs = 0;
|
|
66
69
|
this.activeBatch = null;
|
|
@@ -89,7 +92,7 @@ class ProjectBatchProcessor {
|
|
|
89
92
|
})();
|
|
90
93
|
this.batchTimeoutMs = Math.max(Math.ceil(taskTimeoutMs * 1.5), 120000);
|
|
91
94
|
}
|
|
92
|
-
handleFileEvent(event, absPath) {
|
|
95
|
+
handleFileEvent(event, absPath, options) {
|
|
93
96
|
var _a, _b;
|
|
94
97
|
if (this.closed)
|
|
95
98
|
return;
|
|
@@ -108,8 +111,13 @@ class ProjectBatchProcessor {
|
|
|
108
111
|
(_a = this.onPolicyChange) === null || _a === void 0 ? void 0 : _a.call(this);
|
|
109
112
|
return;
|
|
110
113
|
}
|
|
111
|
-
if (!(0, file_utils_1.isIndexableFile)(normalized, 1) &&
|
|
114
|
+
if (!(0, file_utils_1.isIndexableFile)(normalized, 1) &&
|
|
115
|
+
!this.metaCache.get(normalized) &&
|
|
116
|
+
!(event === "unlink" && (options === null || options === void 0 ? void 0 : options.forceDelete)))
|
|
112
117
|
return;
|
|
118
|
+
if (options === null || options === void 0 ? void 0 : options.forceReprocess) {
|
|
119
|
+
this.forcedReprocess.set(normalized, ++this.forceGeneration);
|
|
120
|
+
}
|
|
113
121
|
this.retryAt.delete(normalized);
|
|
114
122
|
this.pending.set(normalized, event);
|
|
115
123
|
(_b = this.onActivity) === null || _b === void 0 ? void 0 : _b.call(this);
|
|
@@ -199,12 +207,17 @@ class ProjectBatchProcessor {
|
|
|
199
207
|
return;
|
|
200
208
|
}
|
|
201
209
|
const batch = new Map();
|
|
210
|
+
const batchForceGenerations = new Map();
|
|
202
211
|
const now = Date.now();
|
|
203
212
|
let taken = 0;
|
|
204
213
|
for (const [absPath, event] of this.pending) {
|
|
205
214
|
if (((_a = this.retryAt.get(absPath)) !== null && _a !== void 0 ? _a : 0) > now)
|
|
206
215
|
continue;
|
|
207
216
|
batch.set(absPath, event);
|
|
217
|
+
const forceGeneration = this.forcedReprocess.get(absPath);
|
|
218
|
+
if (forceGeneration !== undefined) {
|
|
219
|
+
batchForceGenerations.set(absPath, forceGeneration);
|
|
220
|
+
}
|
|
208
221
|
taken++;
|
|
209
222
|
if (taken >= MAX_BATCH_SIZE)
|
|
210
223
|
break;
|
|
@@ -292,13 +305,16 @@ class ProjectBatchProcessor {
|
|
|
292
305
|
}
|
|
293
306
|
const stats = classification.stat;
|
|
294
307
|
const cached = this.metaCache.get(absPath);
|
|
295
|
-
|
|
308
|
+
const forceReprocess = batchForceGenerations.has(absPath);
|
|
309
|
+
if (!forceReprocess &&
|
|
310
|
+
(0, cache_coherence_1.isMetaEntryCacheCurrent)(cached, absPath) &&
|
|
311
|
+
(0, cache_check_1.isFileCached)(cached, stats)) {
|
|
296
312
|
completed.add(absPath);
|
|
297
313
|
continue;
|
|
298
314
|
}
|
|
299
315
|
// Fast path: if only mtime changed but size matches and we have a hash,
|
|
300
316
|
// verify in-process instead of dispatching to a worker (~220ms saved).
|
|
301
|
-
if ((cached === null || cached === void 0 ? void 0 : cached.hash) && cached.size === stats.size) {
|
|
317
|
+
if (!forceReprocess && (cached === null || cached === void 0 ? void 0 : cached.hash) && cached.size === stats.size) {
|
|
302
318
|
const snapshot = yield (0, file_utils_1.readFileSnapshot)(absPath, {
|
|
303
319
|
projectRoot: this.projectRoot,
|
|
304
320
|
});
|
|
@@ -341,8 +357,13 @@ class ProjectBatchProcessor {
|
|
|
341
357
|
hash: result.hash,
|
|
342
358
|
mtimeMs: result.mtimeMs,
|
|
343
359
|
size: result.size,
|
|
360
|
+
hashVersion: cache_coherence_1.CURRENT_META_HASH_VERSION,
|
|
361
|
+
hasVectors: result.vectors.length > 0,
|
|
344
362
|
};
|
|
345
|
-
if (
|
|
363
|
+
if (!forceReprocess &&
|
|
364
|
+
(0, cache_coherence_1.isMetaEntryCacheCurrent)(cached, absPath) &&
|
|
365
|
+
cached.hash === result.hash &&
|
|
366
|
+
cached.hasVectors === result.vectors.length > 0) {
|
|
346
367
|
metaUpdates.set(absPath, metaEntry);
|
|
347
368
|
completed.add(absPath);
|
|
348
369
|
continue;
|
|
@@ -441,6 +462,11 @@ class ProjectBatchProcessor {
|
|
|
441
462
|
const remaining = this.pending.size;
|
|
442
463
|
(0, logger_1.log)(this.wtag, `Batch complete: ${batch.size} files, ${reindexed} reindexed (${(duration / 1000).toFixed(1)}s)${remaining > 0 ? ` — ${remaining} remaining` : ""}`);
|
|
443
464
|
for (const absPath of completed) {
|
|
465
|
+
const processedForce = batchForceGenerations.get(absPath);
|
|
466
|
+
if (processedForce !== undefined &&
|
|
467
|
+
this.forcedReprocess.get(absPath) === processedForce) {
|
|
468
|
+
this.forcedReprocess.delete(absPath);
|
|
469
|
+
}
|
|
444
470
|
this.retryCount.delete(absPath);
|
|
445
471
|
this.retryAt.delete(absPath);
|
|
446
472
|
this.terminalFailures.delete(absPath);
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.CURRENT_META_HASH_VERSION = void 0;
|
|
37
|
+
exports.isMetaEntryCacheCurrent = isMetaEntryCacheCurrent;
|
|
38
|
+
exports.reconcileMetaEntry = reconcileMetaEntry;
|
|
39
|
+
const path = __importStar(require("node:path"));
|
|
40
|
+
exports.CURRENT_META_HASH_VERSION = 1;
|
|
41
|
+
function usesLegacyMarkdownHash(filePath, entry) {
|
|
42
|
+
if (entry.hashVersion === exports.CURRENT_META_HASH_VERSION)
|
|
43
|
+
return false;
|
|
44
|
+
const extension = path.extname(filePath).toLowerCase();
|
|
45
|
+
return extension === ".md" || extension === ".mdx";
|
|
46
|
+
}
|
|
47
|
+
function isMetaEntryCacheCurrent(entry, _filePath) {
|
|
48
|
+
return ((entry === null || entry === void 0 ? void 0 : entry.hashVersion) === exports.CURRENT_META_HASH_VERSION &&
|
|
49
|
+
typeof entry.hasVectors === "boolean");
|
|
50
|
+
}
|
|
51
|
+
function reconcileMetaEntry(filePath, entry, vectorPresent) {
|
|
52
|
+
if (!entry) {
|
|
53
|
+
return { action: "reprocess", mustRewriteVectors: vectorPresent };
|
|
54
|
+
}
|
|
55
|
+
const explicitVectorState = typeof entry.hasVectors === "boolean";
|
|
56
|
+
if ((entry.hasVectors === true && !vectorPresent) ||
|
|
57
|
+
(entry.hasVectors === false && vectorPresent) ||
|
|
58
|
+
(!explicitVectorState && !vectorPresent)) {
|
|
59
|
+
return { action: "reprocess", mustRewriteVectors: true };
|
|
60
|
+
}
|
|
61
|
+
if (entry.hashVersion !== undefined &&
|
|
62
|
+
entry.hashVersion !== exports.CURRENT_META_HASH_VERSION) {
|
|
63
|
+
return { action: "reprocess", mustRewriteVectors: false };
|
|
64
|
+
}
|
|
65
|
+
if (usesLegacyMarkdownHash(filePath, entry)) {
|
|
66
|
+
return { action: "reprocess", mustRewriteVectors: false };
|
|
67
|
+
}
|
|
68
|
+
if (entry.hashVersion !== exports.CURRENT_META_HASH_VERSION || !explicitVectorState) {
|
|
69
|
+
return {
|
|
70
|
+
action: "stamp",
|
|
71
|
+
entry: Object.assign(Object.assign({}, entry), { hashVersion: exports.CURRENT_META_HASH_VERSION, hasVectors: vectorPresent }),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
return { action: "current" };
|
|
75
|
+
}
|
package/dist/lib/index/syncer.js
CHANGED
|
@@ -56,12 +56,13 @@ const path = __importStar(require("node:path"));
|
|
|
56
56
|
const config_1 = require("../../config");
|
|
57
57
|
const meta_cache_1 = require("../store/meta-cache");
|
|
58
58
|
const vector_db_1 = require("../store/vector-db");
|
|
59
|
-
|
|
59
|
+
const cache_check_1 = require("../utils/cache-check");
|
|
60
60
|
const lock_1 = require("../utils/lock");
|
|
61
61
|
const logger_1 = require("../utils/logger");
|
|
62
62
|
const project_registry_1 = require("../utils/project-registry");
|
|
63
63
|
const project_root_1 = require("../utils/project-root");
|
|
64
64
|
const pool_1 = require("../workers/pool");
|
|
65
|
+
const cache_coherence_1 = require("./cache-coherence");
|
|
65
66
|
const embedding_generation_1 = require("./embedding-generation");
|
|
66
67
|
const file_policy_1 = require("./file-policy");
|
|
67
68
|
const index_config_1 = require("./index-config");
|
|
@@ -185,33 +186,35 @@ function initialSync(options) {
|
|
|
185
186
|
}
|
|
186
187
|
// At this point metaCache is always initialized (injected, created, or noop)
|
|
187
188
|
const mc = metaCache;
|
|
189
|
+
let vectorPaths = new Set();
|
|
190
|
+
const reprocessPaths = new Set();
|
|
191
|
+
const mustRewritePaths = new Set();
|
|
188
192
|
if (!dryRun) {
|
|
189
193
|
// Scope checks to this project's paths only
|
|
190
194
|
const projectKeys = yield mc.getKeysWithPrefix(rootPrefix);
|
|
191
195
|
(0, logger_1.log)("index", `Cached files: ${projectKeys.size}`);
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
(0, logger_1.log)("index", `Coherence: ${vectorFileCount} vectors / ${projectKeys.size} cached (${pct}%)`);
|
|
200
|
-
}
|
|
201
|
-
if (projectKeys.size > 0 && vectorFileCount === 0) {
|
|
202
|
-
(0, logger_1.log)("index", `Stale cache detected: ${projectKeys.size} cached files but no vectors — clearing cache`);
|
|
203
|
-
for (const key of projectKeys) {
|
|
204
|
-
mc.delete(key);
|
|
196
|
+
vectorPaths = yield vectorDb.getDistinctPathsForPrefix(rootPrefix);
|
|
197
|
+
let stamped = 0;
|
|
198
|
+
for (const key of projectKeys) {
|
|
199
|
+
const reconciliation = (0, cache_coherence_1.reconcileMetaEntry)(key, mc.get(key), vectorPaths.has(key));
|
|
200
|
+
if (reconciliation.action === "stamp") {
|
|
201
|
+
mc.put(key, reconciliation.entry);
|
|
202
|
+
stamped++;
|
|
205
203
|
}
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
(0, logger_1.log)("index", `Partial cache detected: ${vectorFileCount} files in vectors vs ${projectKeys.size} in cache — clearing cache to re-embed missing files`);
|
|
211
|
-
for (const key of projectKeys) {
|
|
212
|
-
mc.delete(key);
|
|
204
|
+
else if (reconciliation.action === "reprocess") {
|
|
205
|
+
reprocessPaths.add(key);
|
|
206
|
+
if (reconciliation.mustRewriteVectors)
|
|
207
|
+
mustRewritePaths.add(key);
|
|
213
208
|
}
|
|
214
|
-
|
|
209
|
+
}
|
|
210
|
+
for (const vectorPath of vectorPaths) {
|
|
211
|
+
if (projectKeys.has(vectorPath))
|
|
212
|
+
continue;
|
|
213
|
+
reprocessPaths.add(vectorPath);
|
|
214
|
+
mustRewritePaths.add(vectorPath);
|
|
215
|
+
}
|
|
216
|
+
if (projectKeys.size > 0 || vectorPaths.size > 0) {
|
|
217
|
+
(0, logger_1.log)("index", `Coherence: ${vectorPaths.size} vector files / ${projectKeys.size} cached; ${stamped} metadata stamps, ${reprocessPaths.size} paths to reconcile`);
|
|
215
218
|
}
|
|
216
219
|
if (reset) {
|
|
217
220
|
forceReprocess = true;
|
|
@@ -431,10 +434,9 @@ function initialSync(options) {
|
|
|
431
434
|
return;
|
|
432
435
|
const stats = classified.stat;
|
|
433
436
|
// Use absolute path as the key for MetaCache
|
|
434
|
-
const cached =
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
cached.size === stats.size) {
|
|
437
|
+
const cached = mc.get(absPath);
|
|
438
|
+
const forcePath = forceReprocess || reprocessPaths.has(absPath);
|
|
439
|
+
if (!forcePath && (0, cache_check_1.isFileCached)(cached, stats)) {
|
|
438
440
|
cacheHits++;
|
|
439
441
|
(0, logger_1.debug)("index", `file ${relPath}: cached`);
|
|
440
442
|
processed += 1;
|
|
@@ -475,6 +477,8 @@ function initialSync(options) {
|
|
|
475
477
|
hash: result.hash,
|
|
476
478
|
mtimeMs: result.mtimeMs,
|
|
477
479
|
size: result.size,
|
|
480
|
+
hashVersion: cache_coherence_1.CURRENT_META_HASH_VERSION,
|
|
481
|
+
hasVectors: result.vectors.length > 0,
|
|
478
482
|
};
|
|
479
483
|
if (result.shouldDelete) {
|
|
480
484
|
(0, logger_1.debug)("index", `file ${relPath}: non-indexable (delete)`);
|
|
@@ -489,7 +493,10 @@ function initialSync(options) {
|
|
|
489
493
|
markProgress(relPath);
|
|
490
494
|
return;
|
|
491
495
|
}
|
|
492
|
-
if (
|
|
496
|
+
if (!forceReprocess &&
|
|
497
|
+
!mustRewritePaths.has(absPath) &&
|
|
498
|
+
(cached === null || cached === void 0 ? void 0 : cached.hash) === result.hash &&
|
|
499
|
+
vectorPaths.has(absPath) === result.vectors.length > 0) {
|
|
493
500
|
(0, logger_1.debug)("index", `file ${relPath}: hash unchanged`);
|
|
494
501
|
if (!dryRun) {
|
|
495
502
|
mc.put(absPath, metaEntry);
|
|
@@ -568,14 +575,9 @@ function initialSync(options) {
|
|
|
568
575
|
: new Error(String(flushError));
|
|
569
576
|
}
|
|
570
577
|
throwIfSyncAborted();
|
|
571
|
-
// Stale cleanup
|
|
572
|
-
//
|
|
573
|
-
|
|
574
|
-
let staleSource = cachedPaths;
|
|
575
|
-
if (staleSource.size === 0 && !dryRun && !shouldSkipCleanup) {
|
|
576
|
-
(0, logger_1.log)("index", "MetaCache empty — querying LanceDB for stale path detection");
|
|
577
|
-
staleSource = yield vectorDb.getDistinctPathsForPrefix(rootPrefix);
|
|
578
|
-
}
|
|
578
|
+
// Stale cleanup uses both stores so vector-only orphans cannot survive
|
|
579
|
+
// merely because another path still has metadata.
|
|
580
|
+
const staleSource = new Set([...cachedPaths, ...vectorPaths]);
|
|
579
581
|
throwIfSyncAborted();
|
|
580
582
|
const staleCandidates = computeStaleFiles(staleSource, seenPaths).filter((candidate) => !(0, walker_1.isPathProtectedByWalkState)(candidate, walkState));
|
|
581
583
|
const stale = [];
|
|
@@ -49,6 +49,7 @@ exports.computeRetryAction = computeRetryAction;
|
|
|
49
49
|
const fs = __importStar(require("node:fs"));
|
|
50
50
|
const cache_check_1 = require("../utils/cache-check");
|
|
51
51
|
const file_utils_1 = require("../utils/file-utils");
|
|
52
|
+
const cache_coherence_1 = require("./cache-coherence");
|
|
52
53
|
function processBatchCore(batch, metaCache, pool, signal) {
|
|
53
54
|
return __awaiter(this, void 0, void 0, function* () {
|
|
54
55
|
let reindexed = 0;
|
|
@@ -80,7 +81,8 @@ function processBatchCore(batch, metaCache, pool, signal) {
|
|
|
80
81
|
continue;
|
|
81
82
|
}
|
|
82
83
|
const cached = metaCache.get(absPath);
|
|
83
|
-
if ((0,
|
|
84
|
+
if ((0, cache_coherence_1.isMetaEntryCacheCurrent)(cached, absPath) &&
|
|
85
|
+
(0, cache_check_1.isFileCached)(cached, stats)) {
|
|
84
86
|
continue;
|
|
85
87
|
}
|
|
86
88
|
const result = yield pool.processFile({
|
|
@@ -91,8 +93,12 @@ function processBatchCore(batch, metaCache, pool, signal) {
|
|
|
91
93
|
hash: result.hash,
|
|
92
94
|
mtimeMs: result.mtimeMs,
|
|
93
95
|
size: result.size,
|
|
96
|
+
hashVersion: cache_coherence_1.CURRENT_META_HASH_VERSION,
|
|
97
|
+
hasVectors: result.vectors.length > 0,
|
|
94
98
|
};
|
|
95
|
-
if (
|
|
99
|
+
if ((0, cache_coherence_1.isMetaEntryCacheCurrent)(cached, absPath) &&
|
|
100
|
+
cached.hash === result.hash &&
|
|
101
|
+
cached.hasVectors === result.vectors.length > 0) {
|
|
96
102
|
metaUpdates.set(absPath, metaEntry);
|
|
97
103
|
continue;
|
|
98
104
|
}
|
|
@@ -54,6 +54,7 @@ exports.startWatcher = startWatcher;
|
|
|
54
54
|
const path = __importStar(require("node:path"));
|
|
55
55
|
const watcher = __importStar(require("@parcel/watcher"));
|
|
56
56
|
const batch_processor_1 = require("./batch-processor");
|
|
57
|
+
const cache_coherence_1 = require("./cache-coherence");
|
|
57
58
|
const file_policy_1 = require("./file-policy");
|
|
58
59
|
const walker_1 = require("./walker");
|
|
59
60
|
// Ignore patterns for @parcel/watcher (micromatch globs + directory names).
|
|
@@ -124,6 +125,8 @@ function startWatcher(opts) {
|
|
|
124
125
|
? projectRoot
|
|
125
126
|
: `${projectRoot}/`;
|
|
126
127
|
const cached = yield opts.metaCache.getKeysWithPrefix(rootPrefix);
|
|
128
|
+
const vectorPaths = yield opts.vectorDb.getDistinctPathsForPrefix(rootPrefix);
|
|
129
|
+
const knownPaths = new Set([...cached, ...vectorPaths]);
|
|
127
130
|
const seen = new Set();
|
|
128
131
|
const state = (0, walker_1.createWalkState)();
|
|
129
132
|
try {
|
|
@@ -138,7 +141,13 @@ function startWatcher(opts) {
|
|
|
138
141
|
return;
|
|
139
142
|
const absolute = path.join(projectRoot, relative);
|
|
140
143
|
seen.add(absolute);
|
|
141
|
-
|
|
144
|
+
const reconciliation = (0, cache_coherence_1.reconcileMetaEntry)(absolute, opts.metaCache.get(absolute), vectorPaths.has(absolute));
|
|
145
|
+
if (reconciliation.action === "stamp") {
|
|
146
|
+
opts.metaCache.put(absolute, reconciliation.entry);
|
|
147
|
+
}
|
|
148
|
+
processor.handleFileEvent("change", absolute, {
|
|
149
|
+
forceReprocess: reconciliation.action === "reprocess",
|
|
150
|
+
});
|
|
142
151
|
}
|
|
143
152
|
}
|
|
144
153
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
@@ -148,12 +157,14 @@ function startWatcher(opts) {
|
|
|
148
157
|
}
|
|
149
158
|
finally { if (e_1) throw e_1.error; }
|
|
150
159
|
}
|
|
151
|
-
for (const cachedPath of
|
|
160
|
+
for (const cachedPath of knownPaths) {
|
|
152
161
|
if (closing)
|
|
153
162
|
return;
|
|
154
163
|
if (!seen.has(cachedPath) &&
|
|
155
164
|
!(0, walker_1.isPathProtectedByWalkState)(cachedPath, state)) {
|
|
156
|
-
processor.handleFileEvent("unlink", cachedPath
|
|
165
|
+
processor.handleFileEvent("unlink", cachedPath, {
|
|
166
|
+
forceDelete: true,
|
|
167
|
+
});
|
|
157
168
|
}
|
|
158
169
|
}
|
|
159
170
|
if (!state.rootComplete || state.errors.length > 0) {
|
|
@@ -43,7 +43,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.computeBufferHash = computeBufferHash;
|
|
46
|
-
exports.stripMarkdownFrontmatter = stripMarkdownFrontmatter;
|
|
47
46
|
exports.computeContentHash = computeContentHash;
|
|
48
47
|
exports.hasNullByte = hasNullByte;
|
|
49
48
|
exports.readFileSnapshot = readFileSnapshot;
|
|
@@ -62,35 +61,8 @@ const path_containment_1 = require("./path-containment");
|
|
|
62
61
|
function computeBufferHash(buffer) {
|
|
63
62
|
return (0, node_crypto_1.createHash)("sha256").update(buffer).digest("hex");
|
|
64
63
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
// so hashing it would force a needless GPU re-embed on every metadata touch.
|
|
68
|
-
const FRONTMATTER_EXTENSIONS = new Set([".md", ".mdx"]);
|
|
69
|
-
// Leading YAML frontmatter: a whole `---` line at the very top (NOT a bare
|
|
70
|
-
// `startsWith("---")`, which would also match a `---` thematic break or `--- x`),
|
|
71
|
-
// through the next whole `---`/`...` line. Anchored at string start (no `m` flag)
|
|
72
|
-
// with an optional BOM, so only true file-leading frontmatter matches; a doc that
|
|
73
|
-
// opens with a thematic break and no closing fence is left untouched.
|
|
74
|
-
const FRONTMATTER_BLOCK = /^\uFEFF?---[ \t]*\r?\n[\s\S]*?\r?\n(?:---|\.\.\.)[ \t]*(?:\r?\n|$)/;
|
|
75
|
-
/** Strip leading YAML frontmatter from markdown bytes; returns input unchanged otherwise. */
|
|
76
|
-
function stripMarkdownFrontmatter(buffer) {
|
|
77
|
-
const text = buffer.toString("utf8");
|
|
78
|
-
const match = text.match(FRONTMATTER_BLOCK);
|
|
79
|
-
if (!match)
|
|
80
|
-
return buffer;
|
|
81
|
-
return Buffer.from(text.slice(match[0].length), "utf8");
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Hash file content for change detection. For markdown, YAML frontmatter is
|
|
85
|
-
* stripped first so a metadata-only edit doesn't bust `isFileCached` and trigger
|
|
86
|
-
* a re-embed; every other file hashes its exact bytes. Must be used identically
|
|
87
|
-
* across the catchup, watcher, and worker hash paths or they will disagree and
|
|
88
|
-
* re-index in a loop.
|
|
89
|
-
*/
|
|
90
|
-
function computeContentHash(buffer, filePath) {
|
|
91
|
-
if (FRONTMATTER_EXTENSIONS.has((0, node_path_1.extname)(filePath).toLowerCase())) {
|
|
92
|
-
return computeBufferHash(stripMarkdownFrontmatter(buffer));
|
|
93
|
-
}
|
|
64
|
+
/** Hash the exact bytes passed to chunking and embedding. */
|
|
65
|
+
function computeContentHash(buffer, _filePath) {
|
|
94
66
|
return computeBufferHash(buffer);
|
|
95
67
|
}
|
|
96
68
|
function hasNullByte(buffer, sampleLength = 1024) {
|
package/package.json
CHANGED