grepmax 0.17.24 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/dead.js +1 -0
- package/dist/commands/doctor.js +44 -0
- package/dist/commands/impact.js +1 -0
- package/dist/commands/peek.js +1 -0
- package/dist/commands/related.js +1 -0
- package/dist/commands/search-output.js +531 -0
- package/dist/commands/search.js +44 -462
- package/dist/commands/similar.js +1 -0
- package/dist/commands/test-find.js +1 -0
- package/dist/commands/trace.js +1 -0
- package/dist/config.js +34 -0
- package/dist/lib/daemon/daemon.js +29 -643
- package/dist/lib/daemon/mlx-server-manager.js +191 -0
- package/dist/lib/daemon/process-manager.js +139 -0
- package/dist/lib/daemon/watcher-manager.js +484 -0
- package/dist/lib/utils/stale-hint.js +115 -13
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -0,0 +1,484 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
45
|
+
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
46
|
+
var m = o[Symbol.asyncIterator], i;
|
|
47
|
+
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
48
|
+
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
49
|
+
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
50
|
+
};
|
|
51
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
|
+
exports.WatcherManager = void 0;
|
|
53
|
+
const watcher = __importStar(require("@parcel/watcher"));
|
|
54
|
+
const fs = __importStar(require("node:fs"));
|
|
55
|
+
const path = __importStar(require("node:path"));
|
|
56
|
+
const batch_processor_1 = require("../index/batch-processor");
|
|
57
|
+
const watcher_1 = require("../index/watcher");
|
|
58
|
+
const logger_1 = require("../utils/logger");
|
|
59
|
+
const project_registry_1 = require("../utils/project-registry");
|
|
60
|
+
const watcher_store_1 = require("../utils/watcher-store");
|
|
61
|
+
// Watcher health windows used for FSEvents auto-recovery.
|
|
62
|
+
const FSEVENTS_RECOVERY_INTERVAL_MS = 60 * 60 * 1000; // try recovery hourly
|
|
63
|
+
const FSEVENTS_HEALTH_WINDOW_MS = 5 * 60 * 1000; // 5 min of quiet = "healthy"
|
|
64
|
+
/**
|
|
65
|
+
* Owns per-project file watching: @parcel/watcher subscriptions, FSEvents
|
|
66
|
+
* overflow recovery, poll-mode fallback, and the offline catchup scan. Holds the
|
|
67
|
+
* watcher health/state maps; shares the `processors` and `subscriptions` maps
|
|
68
|
+
* with the daemon by reference so the streaming index ops and shutdown can still
|
|
69
|
+
* detach a project directly.
|
|
70
|
+
*
|
|
71
|
+
* Extracted from daemon.ts (Phase 2). Behavior-preserving — the daemon keeps
|
|
72
|
+
* thin watchProject/unwatchProject delegators over this manager.
|
|
73
|
+
*/
|
|
74
|
+
class WatcherManager {
|
|
75
|
+
constructor(deps) {
|
|
76
|
+
this.deps = deps;
|
|
77
|
+
this.pendingOps = new Set();
|
|
78
|
+
this.watcherFailCount = new Map();
|
|
79
|
+
this.pollIntervals = new Map();
|
|
80
|
+
this.pollRecoveryTimers = new Map();
|
|
81
|
+
this.lastOverflowMs = new Map();
|
|
82
|
+
this.lastCatchupEndMs = new Map();
|
|
83
|
+
}
|
|
84
|
+
watchProject(root) {
|
|
85
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
86
|
+
if (this.deps.processors.has(root) || this.pendingOps.has(root))
|
|
87
|
+
return;
|
|
88
|
+
const vectorDb = this.deps.getVectorDb();
|
|
89
|
+
const metaCache = this.deps.getMetaCache();
|
|
90
|
+
if (!vectorDb || !metaCache)
|
|
91
|
+
return;
|
|
92
|
+
this.pendingOps.add(root);
|
|
93
|
+
const processor = new batch_processor_1.ProjectBatchProcessor({
|
|
94
|
+
projectRoot: root,
|
|
95
|
+
vectorDb,
|
|
96
|
+
metaCache,
|
|
97
|
+
onReindex: (files, ms) => __awaiter(this, void 0, void 0, function* () {
|
|
98
|
+
console.log(`[daemon:${path.basename(root)}] Reindexed ${files} file${files !== 1 ? "s" : ""} (${(ms / 1000).toFixed(1)}s)`);
|
|
99
|
+
// Update project registry so gmax status shows fresh data
|
|
100
|
+
const proj = (0, project_registry_1.getProject)(root);
|
|
101
|
+
if (proj) {
|
|
102
|
+
let chunkCount = proj.chunkCount;
|
|
103
|
+
try {
|
|
104
|
+
chunkCount = yield vectorDb.countRowsForPath(root);
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
console.warn(`[daemon:${path.basename(root)}] Failed to query chunk count: ${err}`);
|
|
108
|
+
}
|
|
109
|
+
(0, project_registry_1.registerProject)(Object.assign(Object.assign({}, proj), { lastIndexed: new Date().toISOString(), chunkCount }));
|
|
110
|
+
}
|
|
111
|
+
// Back to watching after batch completes
|
|
112
|
+
(0, watcher_store_1.registerWatcher)({
|
|
113
|
+
pid: process.pid,
|
|
114
|
+
projectRoot: root,
|
|
115
|
+
startTime: Date.now(),
|
|
116
|
+
status: "watching",
|
|
117
|
+
lastHeartbeat: Date.now(),
|
|
118
|
+
lastReindex: Date.now(),
|
|
119
|
+
});
|
|
120
|
+
}),
|
|
121
|
+
onActivity: () => {
|
|
122
|
+
this.deps.touchActivity();
|
|
123
|
+
// Mark as syncing while processing
|
|
124
|
+
(0, watcher_store_1.registerWatcher)({
|
|
125
|
+
pid: process.pid,
|
|
126
|
+
projectRoot: root,
|
|
127
|
+
startTime: Date.now(),
|
|
128
|
+
status: "syncing",
|
|
129
|
+
lastHeartbeat: Date.now(),
|
|
130
|
+
});
|
|
131
|
+
},
|
|
132
|
+
});
|
|
133
|
+
this.deps.processors.set(root, processor);
|
|
134
|
+
// Subscribe with @parcel/watcher — native backend, no polling.
|
|
135
|
+
// If the kernel refuses (e.g. FSEvents slots stuck after a prior kill -9),
|
|
136
|
+
// fall straight through to poll mode. The retry/backoff path inside
|
|
137
|
+
// recoverWatcher is for transient overflows, not hard kernel-level
|
|
138
|
+
// subscribe failures, so we skip it on startup by priming failCount past
|
|
139
|
+
// MAX before invoking it.
|
|
140
|
+
try {
|
|
141
|
+
yield this.subscribeWatcher(root, processor);
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
const name = path.basename(root);
|
|
145
|
+
console.error(`[daemon:${name}] Subscribe failed at startup (${err instanceof Error ? err.message : err}) — switching to poll mode`);
|
|
146
|
+
this.watcherFailCount.set(root, 1000); // > MAX_WATCHER_RETRIES
|
|
147
|
+
this.lastOverflowMs.set(root, Date.now());
|
|
148
|
+
this.recoverWatcher(root, processor);
|
|
149
|
+
}
|
|
150
|
+
(0, watcher_store_1.registerWatcher)({
|
|
151
|
+
pid: process.pid,
|
|
152
|
+
projectRoot: root,
|
|
153
|
+
startTime: Date.now(),
|
|
154
|
+
status: "watching",
|
|
155
|
+
lastHeartbeat: Date.now(),
|
|
156
|
+
});
|
|
157
|
+
// Catchup scan — find files changed while daemon was offline
|
|
158
|
+
this.catchupScan(root, processor).catch((err) => {
|
|
159
|
+
console.error(`[daemon:${path.basename(root)}] Catchup scan failed:`, err);
|
|
160
|
+
});
|
|
161
|
+
this.pendingOps.delete(root);
|
|
162
|
+
console.log(`[daemon] Watching ${root}`);
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
subscribeWatcher(root, processor) {
|
|
166
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
167
|
+
const name = path.basename(root);
|
|
168
|
+
// Unsubscribe existing watcher if any (e.g. during recovery)
|
|
169
|
+
const existingSub = this.deps.subscriptions.get(root);
|
|
170
|
+
if (existingSub) {
|
|
171
|
+
try {
|
|
172
|
+
yield existingSub.unsubscribe();
|
|
173
|
+
}
|
|
174
|
+
catch (_a) { }
|
|
175
|
+
this.deps.subscriptions.delete(root);
|
|
176
|
+
}
|
|
177
|
+
const sub = yield watcher.subscribe(root, (err, events) => {
|
|
178
|
+
var _a;
|
|
179
|
+
if (err) {
|
|
180
|
+
console.error(`[daemon:${name}] Watcher error:`, err);
|
|
181
|
+
this.recoverWatcher(root, processor);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
// Only reset fail counter after sustained health (5min since last overflow)
|
|
185
|
+
const lastOverflow = (_a = this.lastOverflowMs.get(root)) !== null && _a !== void 0 ? _a : 0;
|
|
186
|
+
if (Date.now() - lastOverflow > 5 * 60 * 1000) {
|
|
187
|
+
this.watcherFailCount.delete(root);
|
|
188
|
+
}
|
|
189
|
+
for (const event of events) {
|
|
190
|
+
processor.handleFileEvent(event.type === "delete" ? "unlink" : "change", event.path);
|
|
191
|
+
}
|
|
192
|
+
this.deps.touchActivity();
|
|
193
|
+
}, { ignore: watcher_1.WATCHER_IGNORE_GLOBS });
|
|
194
|
+
this.deps.subscriptions.set(root, sub);
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
recoverWatcher(root, processor) {
|
|
198
|
+
var _a;
|
|
199
|
+
const name = path.basename(root);
|
|
200
|
+
if (this.deps.getShuttingDown())
|
|
201
|
+
return;
|
|
202
|
+
// Debounce: avoid multiple overlapping recovery attempts
|
|
203
|
+
const recoveryKey = `recover:${root}`;
|
|
204
|
+
if (this.pendingOps.has(recoveryKey))
|
|
205
|
+
return;
|
|
206
|
+
this.pendingOps.add(recoveryKey);
|
|
207
|
+
const fails = ((_a = this.watcherFailCount.get(root)) !== null && _a !== void 0 ? _a : 0) + 1;
|
|
208
|
+
this.watcherFailCount.set(root, fails);
|
|
209
|
+
this.lastOverflowMs.set(root, Date.now());
|
|
210
|
+
const MAX_WATCHER_RETRIES = 3;
|
|
211
|
+
const POLL_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
|
212
|
+
if (fails > MAX_WATCHER_RETRIES) {
|
|
213
|
+
// FSEvents can't handle this project — degrade to periodic catchup scans
|
|
214
|
+
// Always tear down the broken sub, even if poll mode is already active —
|
|
215
|
+
// this can happen if a recovery attempt resubscribed successfully then
|
|
216
|
+
// re-overflowed during the 5-min health window.
|
|
217
|
+
const sub = this.deps.subscriptions.get(root);
|
|
218
|
+
if (sub) {
|
|
219
|
+
sub.unsubscribe().catch(() => { });
|
|
220
|
+
this.deps.subscriptions.delete(root);
|
|
221
|
+
}
|
|
222
|
+
if (!this.pollIntervals.has(root)) {
|
|
223
|
+
console.error(`[daemon:${name}] FSEvents unreliable after ${fails} failures — switching to poll mode (${POLL_INTERVAL_MS / 60000}min interval)`);
|
|
224
|
+
// Run an immediate catchup, then schedule periodic ones
|
|
225
|
+
this.catchupScan(root, processor).catch((err) => {
|
|
226
|
+
console.error(`[daemon:${name}] Poll catchup failed:`, err);
|
|
227
|
+
});
|
|
228
|
+
const interval = setInterval(() => {
|
|
229
|
+
if (this.deps.getShuttingDown())
|
|
230
|
+
return;
|
|
231
|
+
this.deps.touchActivity();
|
|
232
|
+
this.catchupScan(root, processor).catch((err) => {
|
|
233
|
+
console.error(`[daemon:${name}] Poll catchup failed:`, err);
|
|
234
|
+
});
|
|
235
|
+
}, POLL_INTERVAL_MS);
|
|
236
|
+
this.pollIntervals.set(root, interval);
|
|
237
|
+
// Schedule periodic attempts to climb back to native FSEvents — after
|
|
238
|
+
// a transient burst (large git checkout, npm install) the kernel
|
|
239
|
+
// buffer often calms down within an hour.
|
|
240
|
+
this.schedulePollModeRecovery(root, processor);
|
|
241
|
+
(0, watcher_store_1.registerWatcher)({
|
|
242
|
+
pid: process.pid,
|
|
243
|
+
projectRoot: root,
|
|
244
|
+
startTime: Date.now(),
|
|
245
|
+
status: "watching",
|
|
246
|
+
lastHeartbeat: Date.now(),
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
this.pendingOps.delete(recoveryKey);
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
// Backoff: wait before re-subscribing (3s, 6s, 12s)
|
|
253
|
+
const delayMs = 3000 * Math.pow(2, fails - 1);
|
|
254
|
+
console.error(`[daemon:${name}] Recovering watcher (attempt ${fails}/${MAX_WATCHER_RETRIES}, backoff ${delayMs}ms)...`);
|
|
255
|
+
setTimeout(() => {
|
|
256
|
+
if (this.deps.getShuttingDown()) {
|
|
257
|
+
this.pendingOps.delete(recoveryKey);
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
(() => __awaiter(this, void 0, void 0, function* () {
|
|
261
|
+
var _a;
|
|
262
|
+
try {
|
|
263
|
+
yield this.subscribeWatcher(root, processor);
|
|
264
|
+
const lastCatchup = (_a = this.lastCatchupEndMs.get(root)) !== null && _a !== void 0 ? _a : 0;
|
|
265
|
+
const CATCHUP_COOLDOWN_MS = 60000;
|
|
266
|
+
if (Date.now() - lastCatchup < CATCHUP_COOLDOWN_MS) {
|
|
267
|
+
console.log(`[daemon:${name}] Skipping catchup scan (last completed ${Math.round((Date.now() - lastCatchup) / 1000)}s ago)`);
|
|
268
|
+
}
|
|
269
|
+
else {
|
|
270
|
+
yield this.catchupScan(root, processor);
|
|
271
|
+
}
|
|
272
|
+
console.log(`[daemon:${name}] Watcher recovered`);
|
|
273
|
+
}
|
|
274
|
+
catch (err) {
|
|
275
|
+
console.error(`[daemon:${name}] Watcher recovery failed:`, err);
|
|
276
|
+
}
|
|
277
|
+
finally {
|
|
278
|
+
this.pendingOps.delete(recoveryKey);
|
|
279
|
+
}
|
|
280
|
+
}))();
|
|
281
|
+
}, delayMs);
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Once a project has fallen back to poll mode, periodically try to upgrade
|
|
285
|
+
* back to native FSEvents. The buffer overflows that triggered the fallback
|
|
286
|
+
* are usually transient (big git checkout, npm install, build output) — no
|
|
287
|
+
* point staying in 5-min poll mode forever.
|
|
288
|
+
*/
|
|
289
|
+
schedulePollModeRecovery(root, processor) {
|
|
290
|
+
if (this.pollRecoveryTimers.has(root))
|
|
291
|
+
return;
|
|
292
|
+
const name = path.basename(root);
|
|
293
|
+
const timer = setInterval(() => {
|
|
294
|
+
if (this.deps.getShuttingDown())
|
|
295
|
+
return;
|
|
296
|
+
// Skip if a watcher recovery is already in flight or we're not in poll mode anymore.
|
|
297
|
+
if (!this.pollIntervals.has(root)) {
|
|
298
|
+
const t = this.pollRecoveryTimers.get(root);
|
|
299
|
+
if (t)
|
|
300
|
+
clearInterval(t);
|
|
301
|
+
this.pollRecoveryTimers.delete(root);
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
if (this.pendingOps.has(`recover:${root}`))
|
|
305
|
+
return;
|
|
306
|
+
void (() => __awaiter(this, void 0, void 0, function* () {
|
|
307
|
+
var _a;
|
|
308
|
+
console.log(`[daemon:${name}] Attempting to leave poll mode and reattach FSEvents...`);
|
|
309
|
+
try {
|
|
310
|
+
// Reset failure counter so subscribeWatcher's error path treats this
|
|
311
|
+
// as a fresh start. If it fails again, we'll fall right back into
|
|
312
|
+
// poll mode via the same recoverWatcher path.
|
|
313
|
+
this.watcherFailCount.delete(root);
|
|
314
|
+
yield this.subscribeWatcher(root, processor);
|
|
315
|
+
// Wait one health window — if the new subscription survives without
|
|
316
|
+
// another overflow, we consider it recovered and tear down poll mode.
|
|
317
|
+
yield new Promise((r) => setTimeout(r, FSEVENTS_HEALTH_WINDOW_MS));
|
|
318
|
+
if (this.deps.getShuttingDown())
|
|
319
|
+
return;
|
|
320
|
+
const lastOverflow = (_a = this.lastOverflowMs.get(root)) !== null && _a !== void 0 ? _a : 0;
|
|
321
|
+
if (Date.now() - lastOverflow < FSEVENTS_HEALTH_WINDOW_MS) {
|
|
322
|
+
console.log(`[daemon:${name}] FSEvents recovery aborted — fresh overflow within health window, staying in poll mode`);
|
|
323
|
+
return; // recoverWatcher will have re-armed poll mode if needed
|
|
324
|
+
}
|
|
325
|
+
// Healthy — drop poll mode.
|
|
326
|
+
const pollInterval = this.pollIntervals.get(root);
|
|
327
|
+
if (pollInterval) {
|
|
328
|
+
clearInterval(pollInterval);
|
|
329
|
+
this.pollIntervals.delete(root);
|
|
330
|
+
}
|
|
331
|
+
const recoveryTimer = this.pollRecoveryTimers.get(root);
|
|
332
|
+
if (recoveryTimer) {
|
|
333
|
+
clearInterval(recoveryTimer);
|
|
334
|
+
this.pollRecoveryTimers.delete(root);
|
|
335
|
+
}
|
|
336
|
+
console.log(`[daemon:${name}] FSEvents recovered — poll mode disabled`);
|
|
337
|
+
}
|
|
338
|
+
catch (err) {
|
|
339
|
+
console.error(`[daemon:${name}] Poll-mode recovery attempt failed:`, err);
|
|
340
|
+
}
|
|
341
|
+
}))();
|
|
342
|
+
}, FSEVENTS_RECOVERY_INTERVAL_MS);
|
|
343
|
+
timer.unref();
|
|
344
|
+
this.pollRecoveryTimers.set(root, timer);
|
|
345
|
+
}
|
|
346
|
+
catchupScan(root, processor) {
|
|
347
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
348
|
+
var _a, e_1, _b, _c;
|
|
349
|
+
const { walk } = yield Promise.resolve().then(() => __importStar(require("../index/walker")));
|
|
350
|
+
const { INDEXABLE_EXTENSIONS, MAX_FILE_SIZE_BYTES } = yield Promise.resolve().then(() => __importStar(require("../../config")));
|
|
351
|
+
const { isFileCached } = yield Promise.resolve().then(() => __importStar(require("../utils/cache-check")));
|
|
352
|
+
const metaCache = this.deps.getMetaCache();
|
|
353
|
+
const rootPrefix = root.endsWith("/") ? root : `${root}/`;
|
|
354
|
+
const cachedPaths = yield metaCache.getKeysWithPrefix(rootPrefix);
|
|
355
|
+
const seenPaths = new Set();
|
|
356
|
+
let queued = 0;
|
|
357
|
+
let skipped = 0;
|
|
358
|
+
let debugSamples = 0;
|
|
359
|
+
try {
|
|
360
|
+
for (var _d = true, _e = __asyncValues(walk(root, {
|
|
361
|
+
additionalPatterns: ["**/.git/**", "**/.gmax/**"],
|
|
362
|
+
})), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
|
|
363
|
+
_c = _f.value;
|
|
364
|
+
_d = false;
|
|
365
|
+
const relPath = _c;
|
|
366
|
+
const absPath = path.join(root, relPath);
|
|
367
|
+
const ext = path.extname(absPath).toLowerCase();
|
|
368
|
+
const bn = path.basename(absPath).toLowerCase();
|
|
369
|
+
if (!INDEXABLE_EXTENSIONS.has(ext) && !INDEXABLE_EXTENSIONS.has(bn))
|
|
370
|
+
continue;
|
|
371
|
+
seenPaths.add(absPath);
|
|
372
|
+
try {
|
|
373
|
+
const stats = yield fs.promises.stat(absPath);
|
|
374
|
+
// Skip files that are too large or empty — they'll never be indexed
|
|
375
|
+
if (stats.size === 0 || stats.size > MAX_FILE_SIZE_BYTES)
|
|
376
|
+
continue;
|
|
377
|
+
const cached = metaCache.get(absPath);
|
|
378
|
+
if (!isFileCached(cached, stats)) {
|
|
379
|
+
// Fast path: if only mtime changed but size is identical and we have a hash,
|
|
380
|
+
// just verify the hash in-process instead of sending to a worker.
|
|
381
|
+
if (cached && cached.hash && cached.size === stats.size) {
|
|
382
|
+
const { computeBufferHash } = yield Promise.resolve().then(() => __importStar(require("../utils/file-utils")));
|
|
383
|
+
const buf = yield fs.promises.readFile(absPath);
|
|
384
|
+
const hash = computeBufferHash(buf);
|
|
385
|
+
if (hash === cached.hash) {
|
|
386
|
+
// Content unchanged — update mtime in cache and skip worker
|
|
387
|
+
metaCache.put(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: stats.mtimeMs }));
|
|
388
|
+
skipped++;
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
// Debug: log first few misses to diagnose re-queue loops
|
|
393
|
+
if (debugSamples < 5) {
|
|
394
|
+
(0, logger_1.debug)("catchup", `miss ${relPath}: cached=${cached ? `mtime=${Math.trunc(cached.mtimeMs)} size=${cached.size}` : "null"} stat=mtime=${Math.trunc(stats.mtimeMs)} size=${stats.size}`);
|
|
395
|
+
debugSamples++;
|
|
396
|
+
}
|
|
397
|
+
processor.handleFileEvent("change", absPath);
|
|
398
|
+
queued++;
|
|
399
|
+
// Throttle: pause periodically during large catchup scans to let the
|
|
400
|
+
// batch processor drain and compaction run between bursts.
|
|
401
|
+
if (queued % 500 === 0) {
|
|
402
|
+
(0, logger_1.debug)("catchup", `${path.basename(root)}: throttle pause at ${queued} queued`);
|
|
403
|
+
yield new Promise(r => setTimeout(r, 5000));
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
skipped++;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
catch (_g) { }
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
414
|
+
finally {
|
|
415
|
+
try {
|
|
416
|
+
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
|
|
417
|
+
}
|
|
418
|
+
finally { if (e_1) throw e_1.error; }
|
|
419
|
+
}
|
|
420
|
+
(0, logger_1.debug)("catchup", `${path.basename(root)}: ${queued} queued, ${skipped} skipped (cached ok), ${seenPaths.size} total`);
|
|
421
|
+
// Purge files deleted while daemon was offline
|
|
422
|
+
let purged = 0;
|
|
423
|
+
for (const cachedPath of cachedPaths) {
|
|
424
|
+
if (!seenPaths.has(cachedPath)) {
|
|
425
|
+
processor.handleFileEvent("unlink", cachedPath);
|
|
426
|
+
purged++;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
if (queued > 0 || purged > 0) {
|
|
430
|
+
const parts = [];
|
|
431
|
+
if (queued > 0)
|
|
432
|
+
parts.push(`${queued} changed`);
|
|
433
|
+
if (purged > 0)
|
|
434
|
+
parts.push(`${purged} deleted`);
|
|
435
|
+
console.log(`[daemon:${path.basename(root)}] Catchup: ${parts.join(", ")} file(s) while offline`);
|
|
436
|
+
}
|
|
437
|
+
this.lastCatchupEndMs.set(root, Date.now());
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
unwatchProject(root) {
|
|
441
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
442
|
+
const processor = this.deps.processors.get(root);
|
|
443
|
+
if (!processor)
|
|
444
|
+
return;
|
|
445
|
+
yield processor.close();
|
|
446
|
+
const sub = this.deps.subscriptions.get(root);
|
|
447
|
+
if (sub) {
|
|
448
|
+
yield sub.unsubscribe();
|
|
449
|
+
this.deps.subscriptions.delete(root);
|
|
450
|
+
}
|
|
451
|
+
this.deps.processors.delete(root);
|
|
452
|
+
this.deps.evictSearcher(root);
|
|
453
|
+
this.lastOverflowMs.delete(root);
|
|
454
|
+
this.lastCatchupEndMs.delete(root);
|
|
455
|
+
(0, watcher_store_1.unregisterWatcherByRoot)(root);
|
|
456
|
+
console.log(`[daemon] Unwatched ${root}`);
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Stop all poll intervals + their FSEvents recovery probes and unsubscribe
|
|
461
|
+
* every watcher. Called from the daemon's shutdown after the worker pool is
|
|
462
|
+
* destroyed (the subscriptions map is shared, so it is cleared here).
|
|
463
|
+
*/
|
|
464
|
+
teardown() {
|
|
465
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
466
|
+
for (const interval of this.pollIntervals.values()) {
|
|
467
|
+
clearInterval(interval);
|
|
468
|
+
}
|
|
469
|
+
this.pollIntervals.clear();
|
|
470
|
+
for (const interval of this.pollRecoveryTimers.values()) {
|
|
471
|
+
clearInterval(interval);
|
|
472
|
+
}
|
|
473
|
+
this.pollRecoveryTimers.clear();
|
|
474
|
+
for (const sub of this.deps.subscriptions.values()) {
|
|
475
|
+
try {
|
|
476
|
+
yield sub.unsubscribe();
|
|
477
|
+
}
|
|
478
|
+
catch (_a) { }
|
|
479
|
+
}
|
|
480
|
+
this.deps.subscriptions.clear();
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
exports.WatcherManager = WatcherManager;
|
|
@@ -2,27 +2,36 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports._resetStaleHintForTests = _resetStaleHintForTests;
|
|
4
4
|
exports.maybeWarnStaleChunker = maybeWarnStaleChunker;
|
|
5
|
+
exports.maybeWarnStaleEmbedding = maybeWarnStaleEmbedding;
|
|
6
|
+
exports.maybeWarnCrossProjectDim = maybeWarnCrossProjectDim;
|
|
5
7
|
/**
|
|
6
|
-
* Query-time staleness
|
|
8
|
+
* Query-time staleness nudges.
|
|
7
9
|
*
|
|
8
|
-
* When a graph/search command resolves a project whose index
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* prose so a tool that *does*
|
|
10
|
+
* When a graph/search command resolves a project whose index is out of date —
|
|
11
|
+
* built by an older chunker, or with a different embedding model/dim than the
|
|
12
|
+
* current config — emit a single concise line to STDERR. Writing to stderr
|
|
13
|
+
* (never stdout) keeps `--json` / `--agent` machine output byte-identical —
|
|
14
|
+
* agents parse stdout, so the hint never pollutes it. In `--agent` mode the line
|
|
15
|
+
* is rendered as a structured TSV record instead of prose so a tool that *does*
|
|
16
|
+
* capture stderr can parse it.
|
|
14
17
|
*
|
|
15
|
-
*
|
|
18
|
+
* Each concern fires at most once per process (independent latches, so a stale
|
|
19
|
+
* chunker does not mask a stale embedding). Suppress all with GMAX_NO_STALE_HINT=1.
|
|
16
20
|
*/
|
|
17
21
|
const config_1 = require("../../config");
|
|
22
|
+
const index_config_1 = require("../index/index-config");
|
|
18
23
|
const project_registry_1 = require("./project-registry");
|
|
19
|
-
let
|
|
20
|
-
|
|
24
|
+
let chunkerEmitted = false;
|
|
25
|
+
let embeddingEmitted = false;
|
|
26
|
+
let crossDimEmitted = false;
|
|
27
|
+
/** Reset the once-per-process latches. Test-only. */
|
|
21
28
|
function _resetStaleHintForTests() {
|
|
22
|
-
|
|
29
|
+
chunkerEmitted = false;
|
|
30
|
+
embeddingEmitted = false;
|
|
31
|
+
crossDimEmitted = false;
|
|
23
32
|
}
|
|
24
33
|
function maybeWarnStaleChunker(projectRoot, opts) {
|
|
25
|
-
if (
|
|
34
|
+
if (chunkerEmitted)
|
|
26
35
|
return;
|
|
27
36
|
if (!projectRoot)
|
|
28
37
|
return;
|
|
@@ -42,7 +51,7 @@ function maybeWarnStaleChunker(projectRoot, opts) {
|
|
|
42
51
|
// Latch only once we have something to say, so a suppressed-by-status call
|
|
43
52
|
// does not silence a later command in the same process (CLI is one command
|
|
44
53
|
// per process, but search resolves the root in two places).
|
|
45
|
-
|
|
54
|
+
chunkerEmitted = true;
|
|
46
55
|
const name = project.name || projectRoot;
|
|
47
56
|
if (opts === null || opts === void 0 ? void 0 : opts.agent) {
|
|
48
57
|
const fields = [
|
|
@@ -60,3 +69,96 @@ function maybeWarnStaleChunker(projectRoot, opts) {
|
|
|
60
69
|
const label = gap.severity === "breaking" ? "WARN" : "hint";
|
|
61
70
|
process.stderr.write(`${label} gmax: '${name}' indexed by chunker v${gap.fromVersion} (now v${gap.toVersion}) — ${gap.notes.join(" ")} Run 'gmax index --reset' to refresh. (silence: GMAX_NO_STALE_HINT=1)\n`);
|
|
62
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Nudge when a project's stored embedding model/dim differs from the current
|
|
74
|
+
* global config. A dim change is `breaking` (search scores are invalid until a
|
|
75
|
+
* re-embed); a same-dim model swap is `additive` (results just mix models).
|
|
76
|
+
* Mirrors maybeWarnStaleChunker's discipline: stderr-only, once-per-process,
|
|
77
|
+
* `--agent` TSV, GMAX_NO_STALE_HINT-suppressible.
|
|
78
|
+
*/
|
|
79
|
+
function maybeWarnStaleEmbedding(projectRoot, opts) {
|
|
80
|
+
if (embeddingEmitted)
|
|
81
|
+
return;
|
|
82
|
+
if (!projectRoot)
|
|
83
|
+
return;
|
|
84
|
+
if (process.env.GMAX_NO_STALE_HINT === "1")
|
|
85
|
+
return;
|
|
86
|
+
const project = (0, project_registry_1.getProject)(projectRoot);
|
|
87
|
+
if (!project)
|
|
88
|
+
return;
|
|
89
|
+
if (project.status && project.status !== "indexed")
|
|
90
|
+
return;
|
|
91
|
+
const current = (0, index_config_1.readGlobalConfig)();
|
|
92
|
+
const gap = (0, config_1.describeEmbeddingGap)({ modelTier: project.modelTier, vectorDim: project.vectorDim }, { modelTier: current.modelTier, vectorDim: current.vectorDim });
|
|
93
|
+
if (!gap)
|
|
94
|
+
return;
|
|
95
|
+
embeddingEmitted = true;
|
|
96
|
+
const name = project.name || projectRoot;
|
|
97
|
+
if (opts === null || opts === void 0 ? void 0 : opts.agent) {
|
|
98
|
+
const fields = [
|
|
99
|
+
"stale_embedding",
|
|
100
|
+
`project=${name}`,
|
|
101
|
+
`indexed_model=${gap.fromModel}`,
|
|
102
|
+
`current_model=${gap.toModel}`,
|
|
103
|
+
`indexed_dim=${gap.fromDim}`,
|
|
104
|
+
`current_dim=${gap.toDim}`,
|
|
105
|
+
`dim_changed=${gap.dimChanged}`,
|
|
106
|
+
`severity=${gap.severity}`,
|
|
107
|
+
"fix=gmax index --reset",
|
|
108
|
+
].join("\t");
|
|
109
|
+
process.stderr.write(`${fields}\n`);
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const label = gap.severity === "breaking" ? "WARN" : "hint";
|
|
113
|
+
const detail = gap.dimChanged
|
|
114
|
+
? `vector dim ${gap.fromDim}→${gap.toDim} (incompatible — scores invalid until re-embed)`
|
|
115
|
+
: `model '${gap.fromModel}'→'${gap.toModel}' (same dim; results mix models until re-embed)`;
|
|
116
|
+
process.stderr.write(`${label} gmax: '${name}' indexed with embedding ${detail}. Run 'gmax index --reset'. (silence: GMAX_NO_STALE_HINT=1)\n`);
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Cross-project guard: when a `--all-projects` / `--projects` search spans
|
|
120
|
+
* projects whose stored vector widths disagree (with each other or with the
|
|
121
|
+
* current query width), the shared fixed-dim table cannot answer all of them
|
|
122
|
+
* correctly — mismatched vectors were padded/truncated at insert and score as
|
|
123
|
+
* noise. Warn (stderr) so the caller can re-embed or exclude them, rather than
|
|
124
|
+
* silently returning invalid cross-project rankings.
|
|
125
|
+
*/
|
|
126
|
+
function maybeWarnCrossProjectDim(roots, opts) {
|
|
127
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
128
|
+
if (crossDimEmitted)
|
|
129
|
+
return;
|
|
130
|
+
if (process.env.GMAX_NO_STALE_HINT === "1")
|
|
131
|
+
return;
|
|
132
|
+
if (!roots || roots.length === 0)
|
|
133
|
+
return;
|
|
134
|
+
const current = (0, index_config_1.readGlobalConfig)();
|
|
135
|
+
const currentDim = (_d = (_a = current.vectorDim) !== null && _a !== void 0 ? _a : (_c = config_1.MODEL_TIERS[(_b = current.modelTier) !== null && _b !== void 0 ? _b : config_1.DEFAULT_MODEL_TIER]) === null || _c === void 0 ? void 0 : _c.vectorDim) !== null && _d !== void 0 ? _d : config_1.CONFIG.VECTOR_DIM;
|
|
136
|
+
const byRoot = new Map((0, project_registry_1.listProjects)().map((p) => [p.root, p]));
|
|
137
|
+
const dims = new Set([currentDim]);
|
|
138
|
+
const mismatched = [];
|
|
139
|
+
for (const r of roots) {
|
|
140
|
+
const p = byRoot.get(r.root);
|
|
141
|
+
if (!p)
|
|
142
|
+
continue;
|
|
143
|
+
const dim = (_g = (_e = p.vectorDim) !== null && _e !== void 0 ? _e : (_f = config_1.MODEL_TIERS[p.modelTier]) === null || _f === void 0 ? void 0 : _f.vectorDim) !== null && _g !== void 0 ? _g : currentDim;
|
|
144
|
+
dims.add(dim);
|
|
145
|
+
if (dim !== currentDim)
|
|
146
|
+
mismatched.push({ name: p.name || r.name, dim });
|
|
147
|
+
}
|
|
148
|
+
// All in-scope projects share the query width → nothing to warn about.
|
|
149
|
+
if (dims.size <= 1)
|
|
150
|
+
return;
|
|
151
|
+
crossDimEmitted = true;
|
|
152
|
+
if (opts === null || opts === void 0 ? void 0 : opts.agent) {
|
|
153
|
+
const fields = [
|
|
154
|
+
"stale_embedding_crossdim",
|
|
155
|
+
`query_dim=${currentDim}`,
|
|
156
|
+
`mismatched=${mismatched.map((m) => `${m.name}:${m.dim}`).join(",")}`,
|
|
157
|
+
"fix=re-embed or --exclude-projects",
|
|
158
|
+
].join("\t");
|
|
159
|
+
process.stderr.write(`${fields}\n`);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const list = mismatched.map((m) => `${m.name} (${m.dim}d)`).join(", ");
|
|
163
|
+
process.stderr.write(`WARN gmax: cross-project search mixes embedding dims (query ${currentDim}d) — ${list} return invalid scores. Re-embed them or drop with --exclude-projects. (silence: GMAX_NO_STALE_HINT=1)\n`);
|
|
164
|
+
}
|
package/package.json
CHANGED