grepmax 0.8.2 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/add.js +1 -1
- package/dist/commands/index.js +24 -20
- package/dist/commands/mcp.js +11 -4
- package/dist/commands/remove.js +13 -6
- package/dist/commands/search.js +1 -1
- package/dist/commands/watch.js +85 -12
- package/dist/config.js +1 -0
- package/dist/lib/daemon/daemon.js +314 -0
- package/dist/lib/daemon/ipc-handler.js +53 -0
- package/dist/lib/index/batch-processor.js +286 -0
- package/dist/lib/index/watcher.js +8 -228
- package/dist/lib/utils/daemon-client.js +108 -0
- package/dist/lib/utils/daemon-launcher.js +59 -0
- package/dist/lib/utils/watcher-launcher.js +78 -36
- package/dist/lib/utils/watcher-store.js +42 -0
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/hooks/start.js +7 -2
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.ProjectBatchProcessor = void 0;
|
|
46
|
+
const fs = __importStar(require("node:fs"));
|
|
47
|
+
const path = __importStar(require("node:path"));
|
|
48
|
+
const config_1 = require("../../config");
|
|
49
|
+
const cache_check_1 = require("../utils/cache-check");
|
|
50
|
+
const file_utils_1 = require("../utils/file-utils");
|
|
51
|
+
const logger_1 = require("../utils/logger");
|
|
52
|
+
const lock_1 = require("../utils/lock");
|
|
53
|
+
const pool_1 = require("../workers/pool");
|
|
54
|
+
const watcher_batch_1 = require("./watcher-batch");
|
|
55
|
+
const DEBOUNCE_MS = 2000;
|
|
56
|
+
const FTS_REBUILD_INTERVAL_MS = 5 * 60 * 1000;
|
|
57
|
+
const MAX_RETRIES = 5;
|
|
58
|
+
class ProjectBatchProcessor {
|
|
59
|
+
constructor(opts) {
|
|
60
|
+
this.pending = new Map();
|
|
61
|
+
this.retryCount = new Map();
|
|
62
|
+
this.debounceTimer = null;
|
|
63
|
+
this.processing = false;
|
|
64
|
+
this.closed = false;
|
|
65
|
+
this.consecutiveLockFailures = 0;
|
|
66
|
+
this.currentBatchAc = null;
|
|
67
|
+
this.projectRoot = opts.projectRoot;
|
|
68
|
+
this.vectorDb = opts.vectorDb;
|
|
69
|
+
this.metaCache = opts.metaCache;
|
|
70
|
+
this.dataDir = opts.dataDir;
|
|
71
|
+
this.onReindex = opts.onReindex;
|
|
72
|
+
this.onActivity = opts.onActivity;
|
|
73
|
+
this.wtag = `watch:${path.basename(opts.projectRoot)}`;
|
|
74
|
+
const taskTimeoutMs = (() => {
|
|
75
|
+
var _a;
|
|
76
|
+
const fromEnv = Number.parseInt((_a = process.env.GMAX_WORKER_TASK_TIMEOUT_MS) !== null && _a !== void 0 ? _a : "", 10);
|
|
77
|
+
return Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : 120000;
|
|
78
|
+
})();
|
|
79
|
+
this.batchTimeoutMs = Math.max(Math.ceil(taskTimeoutMs * 1.5), 120000);
|
|
80
|
+
this.ftsInterval = setInterval(() => __awaiter(this, void 0, void 0, function* () {
|
|
81
|
+
if (this.closed || this.processing)
|
|
82
|
+
return;
|
|
83
|
+
try {
|
|
84
|
+
yield this.vectorDb.createFTSIndex();
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
console.error(`[${this.wtag}] FTS rebuild failed:`, err);
|
|
88
|
+
}
|
|
89
|
+
}), FTS_REBUILD_INTERVAL_MS);
|
|
90
|
+
this.ftsInterval.unref();
|
|
91
|
+
}
|
|
92
|
+
handleFileEvent(event, absPath) {
|
|
93
|
+
var _a;
|
|
94
|
+
if (this.closed)
|
|
95
|
+
return;
|
|
96
|
+
if (event !== "unlink") {
|
|
97
|
+
const ext = path.extname(absPath).toLowerCase();
|
|
98
|
+
const bn = path.basename(absPath).toLowerCase();
|
|
99
|
+
if (!config_1.INDEXABLE_EXTENSIONS.has(ext) && !config_1.INDEXABLE_EXTENSIONS.has(bn))
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
this.pending.set(absPath, event);
|
|
103
|
+
(_a = this.onActivity) === null || _a === void 0 ? void 0 : _a.call(this);
|
|
104
|
+
this.scheduleBatch();
|
|
105
|
+
}
|
|
106
|
+
close() {
|
|
107
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
108
|
+
var _a;
|
|
109
|
+
this.closed = true;
|
|
110
|
+
(_a = this.currentBatchAc) === null || _a === void 0 ? void 0 : _a.abort();
|
|
111
|
+
if (this.debounceTimer)
|
|
112
|
+
clearTimeout(this.debounceTimer);
|
|
113
|
+
clearInterval(this.ftsInterval);
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
scheduleBatch() {
|
|
117
|
+
if (this.debounceTimer)
|
|
118
|
+
clearTimeout(this.debounceTimer);
|
|
119
|
+
this.debounceTimer = setTimeout(() => this.processBatch(), DEBOUNCE_MS);
|
|
120
|
+
}
|
|
121
|
+
processBatch() {
|
|
122
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
123
|
+
var _a;
|
|
124
|
+
if (this.closed || this.processing || this.pending.size === 0)
|
|
125
|
+
return;
|
|
126
|
+
this.processing = true;
|
|
127
|
+
const batchAc = new AbortController();
|
|
128
|
+
this.currentBatchAc = batchAc;
|
|
129
|
+
const batchTimeout = setTimeout(() => {
|
|
130
|
+
(0, logger_1.log)(this.wtag, `Batch timed out after ${this.batchTimeoutMs}ms, aborting`);
|
|
131
|
+
batchAc.abort();
|
|
132
|
+
}, this.batchTimeoutMs);
|
|
133
|
+
const batch = new Map(this.pending);
|
|
134
|
+
this.pending.clear();
|
|
135
|
+
const filenames = [...batch.keys()].map((p) => path.basename(p));
|
|
136
|
+
(0, logger_1.log)(this.wtag, `Processing ${batch.size} changed files: ${filenames.join(", ")}`);
|
|
137
|
+
const start = Date.now();
|
|
138
|
+
let reindexed = 0;
|
|
139
|
+
try {
|
|
140
|
+
const lock = yield (0, lock_1.acquireWriterLockWithRetry)(this.dataDir, {
|
|
141
|
+
maxRetries: 3,
|
|
142
|
+
retryDelayMs: 500,
|
|
143
|
+
});
|
|
144
|
+
try {
|
|
145
|
+
const pool = (0, pool_1.getWorkerPool)();
|
|
146
|
+
const deletes = [];
|
|
147
|
+
const vectors = [];
|
|
148
|
+
const metaUpdates = new Map();
|
|
149
|
+
const metaDeletes = [];
|
|
150
|
+
const attempted = new Set();
|
|
151
|
+
for (const [absPath, event] of batch) {
|
|
152
|
+
if (batchAc.signal.aborted)
|
|
153
|
+
break;
|
|
154
|
+
attempted.add(absPath);
|
|
155
|
+
if (event === "unlink") {
|
|
156
|
+
deletes.push(absPath);
|
|
157
|
+
metaDeletes.push(absPath);
|
|
158
|
+
reindexed++;
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
// change or add
|
|
162
|
+
try {
|
|
163
|
+
const stats = yield fs.promises.stat(absPath);
|
|
164
|
+
if (!(0, file_utils_1.isIndexableFile)(absPath, stats.size))
|
|
165
|
+
continue;
|
|
166
|
+
const cached = this.metaCache.get(absPath);
|
|
167
|
+
if ((0, cache_check_1.isFileCached)(cached, stats)) {
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
const result = yield pool.processFile({
|
|
171
|
+
path: absPath,
|
|
172
|
+
absolutePath: absPath,
|
|
173
|
+
}, batchAc.signal);
|
|
174
|
+
const metaEntry = {
|
|
175
|
+
hash: result.hash,
|
|
176
|
+
mtimeMs: result.mtimeMs,
|
|
177
|
+
size: result.size,
|
|
178
|
+
};
|
|
179
|
+
if (cached && cached.hash === result.hash) {
|
|
180
|
+
metaUpdates.set(absPath, metaEntry);
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
if (result.shouldDelete) {
|
|
184
|
+
deletes.push(absPath);
|
|
185
|
+
metaUpdates.set(absPath, metaEntry);
|
|
186
|
+
reindexed++;
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
deletes.push(absPath);
|
|
190
|
+
if (result.vectors.length > 0) {
|
|
191
|
+
vectors.push(...result.vectors);
|
|
192
|
+
}
|
|
193
|
+
metaUpdates.set(absPath, metaEntry);
|
|
194
|
+
reindexed++;
|
|
195
|
+
}
|
|
196
|
+
catch (err) {
|
|
197
|
+
if (batchAc.signal.aborted)
|
|
198
|
+
break;
|
|
199
|
+
const code = err === null || err === void 0 ? void 0 : err.code;
|
|
200
|
+
if (code === "ENOENT") {
|
|
201
|
+
deletes.push(absPath);
|
|
202
|
+
metaDeletes.push(absPath);
|
|
203
|
+
reindexed++;
|
|
204
|
+
}
|
|
205
|
+
else {
|
|
206
|
+
console.error(`[${this.wtag}] Failed to process ${absPath}:`, err);
|
|
207
|
+
if (!pool.isHealthy()) {
|
|
208
|
+
console.error(`[${this.wtag}] Worker pool unhealthy, aborting batch`);
|
|
209
|
+
break;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
// Requeue files that weren't attempted (aborted or pool unhealthy)
|
|
215
|
+
for (const [absPath, event] of batch) {
|
|
216
|
+
if (!attempted.has(absPath) && !this.pending.has(absPath)) {
|
|
217
|
+
this.pending.set(absPath, event);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
// Flush to VectorDB: insert first, then delete old (preserving new)
|
|
221
|
+
const newIds = vectors.map((v) => v.id);
|
|
222
|
+
if (vectors.length > 0) {
|
|
223
|
+
yield this.vectorDb.insertBatch(vectors);
|
|
224
|
+
}
|
|
225
|
+
if (deletes.length > 0) {
|
|
226
|
+
if (newIds.length > 0) {
|
|
227
|
+
yield this.vectorDb.deletePathsExcludingIds(deletes, newIds);
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
yield this.vectorDb.deletePaths(deletes);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
// Update MetaCache
|
|
234
|
+
for (const [p, entry] of metaUpdates) {
|
|
235
|
+
this.metaCache.put(p, entry);
|
|
236
|
+
}
|
|
237
|
+
for (const p of metaDeletes) {
|
|
238
|
+
this.metaCache.delete(p);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
finally {
|
|
242
|
+
yield lock.release();
|
|
243
|
+
}
|
|
244
|
+
const duration = Date.now() - start;
|
|
245
|
+
if (reindexed > 0) {
|
|
246
|
+
(_a = this.onReindex) === null || _a === void 0 ? void 0 : _a.call(this, reindexed, duration);
|
|
247
|
+
}
|
|
248
|
+
(0, logger_1.log)(this.wtag, `Batch complete: ${batch.size} files, ${reindexed} reindexed (${(duration / 1000).toFixed(1)}s)`);
|
|
249
|
+
this.consecutiveLockFailures = 0;
|
|
250
|
+
for (const absPath of batch.keys()) {
|
|
251
|
+
this.retryCount.delete(absPath);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
catch (err) {
|
|
255
|
+
const isLockError = err instanceof Error && err.message.includes("lock already held");
|
|
256
|
+
if (isLockError) {
|
|
257
|
+
this.consecutiveLockFailures++;
|
|
258
|
+
}
|
|
259
|
+
console.error(`[${this.wtag}] Batch processing failed:`, err);
|
|
260
|
+
const { requeued, dropped, backoffMs } = (0, watcher_batch_1.computeRetryAction)(batch, this.retryCount, MAX_RETRIES, isLockError, this.consecutiveLockFailures, DEBOUNCE_MS);
|
|
261
|
+
for (const [absPath, event] of requeued) {
|
|
262
|
+
if (!this.pending.has(absPath)) {
|
|
263
|
+
this.pending.set(absPath, event);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (dropped > 0) {
|
|
267
|
+
console.warn(`[${this.wtag}] Dropped ${dropped} file(s) after ${MAX_RETRIES} failed retries`);
|
|
268
|
+
}
|
|
269
|
+
if (this.pending.size > 0) {
|
|
270
|
+
if (this.debounceTimer)
|
|
271
|
+
clearTimeout(this.debounceTimer);
|
|
272
|
+
this.debounceTimer = setTimeout(() => this.processBatch(), backoffMs);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
finally {
|
|
276
|
+
clearTimeout(batchTimeout);
|
|
277
|
+
this.currentBatchAc = null;
|
|
278
|
+
this.processing = false;
|
|
279
|
+
if (this.pending.size > 0) {
|
|
280
|
+
this.scheduleBatch();
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
exports.ProjectBatchProcessor = ProjectBatchProcessor;
|
|
@@ -44,16 +44,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.WATCHER_IGNORE_PATTERNS = void 0;
|
|
46
46
|
exports.startWatcher = startWatcher;
|
|
47
|
-
const fs = __importStar(require("node:fs"));
|
|
48
47
|
const path = __importStar(require("node:path"));
|
|
49
48
|
const chokidar_1 = require("chokidar");
|
|
50
|
-
const
|
|
51
|
-
const cache_check_1 = require("../utils/cache-check");
|
|
52
|
-
const file_utils_1 = require("../utils/file-utils");
|
|
53
|
-
const logger_1 = require("../utils/logger");
|
|
54
|
-
const lock_1 = require("../utils/lock");
|
|
55
|
-
const pool_1 = require("../workers/pool");
|
|
56
|
-
const watcher_batch_1 = require("./watcher-batch");
|
|
49
|
+
const batch_processor_1 = require("./batch-processor");
|
|
57
50
|
// Chokidar ignored — must exclude heavy directories to keep FD count low.
|
|
58
51
|
// On macOS, chokidar uses FSEvents (single FD) but falls back to fs.watch()
|
|
59
52
|
// (one FD per directory) if FSEvents isn't available or for some subdirs.
|
|
@@ -72,20 +65,10 @@ exports.WATCHER_IGNORE_PATTERNS = [
|
|
|
72
65
|
"**/lancedb/**",
|
|
73
66
|
/(^|[/\\])\../, // dotfiles
|
|
74
67
|
];
|
|
75
|
-
const DEBOUNCE_MS = 2000;
|
|
76
|
-
const FTS_REBUILD_INTERVAL_MS = 5 * 60 * 1000;
|
|
77
68
|
function startWatcher(opts) {
|
|
78
|
-
const { projectRoot
|
|
79
|
-
const
|
|
80
|
-
const
|
|
81
|
-
const pending = new Map();
|
|
82
|
-
const retryCount = new Map();
|
|
83
|
-
let debounceTimer = null;
|
|
84
|
-
let processing = false;
|
|
85
|
-
let closed = false;
|
|
86
|
-
let consecutiveLockFailures = 0;
|
|
87
|
-
let currentBatchAc = null;
|
|
88
|
-
const MAX_RETRIES = 5;
|
|
69
|
+
const { projectRoot } = opts;
|
|
70
|
+
const wtag = `watch:${path.basename(projectRoot)}`;
|
|
71
|
+
const processor = new batch_processor_1.ProjectBatchProcessor(opts);
|
|
89
72
|
// macOS: FSEvents is a single-FD kernel API — no EMFILE risk and no polling.
|
|
90
73
|
// Linux: inotify is event-driven but uses one FD per watch; fall back to
|
|
91
74
|
// polling for monorepos to avoid hitting ulimit.
|
|
@@ -98,215 +81,12 @@ function startWatcher(opts) {
|
|
|
98
81
|
watcher.on("error", (err) => {
|
|
99
82
|
console.error(`[${wtag}] Watcher error:`, err);
|
|
100
83
|
});
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
debounceTimer = setTimeout(() => processBatch(), DEBOUNCE_MS);
|
|
105
|
-
};
|
|
106
|
-
const taskTimeoutMs = (() => {
|
|
107
|
-
var _a;
|
|
108
|
-
const fromEnv = Number.parseInt((_a = process.env.GMAX_WORKER_TASK_TIMEOUT_MS) !== null && _a !== void 0 ? _a : "", 10);
|
|
109
|
-
return Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : 120000;
|
|
110
|
-
})();
|
|
111
|
-
const BATCH_TIMEOUT_MS = Math.max(Math.ceil(taskTimeoutMs * 1.5), 120000);
|
|
112
|
-
const processBatch = () => __awaiter(this, void 0, void 0, function* () {
|
|
113
|
-
if (closed || processing || pending.size === 0)
|
|
114
|
-
return;
|
|
115
|
-
processing = true;
|
|
116
|
-
const batchAc = new AbortController();
|
|
117
|
-
currentBatchAc = batchAc;
|
|
118
|
-
const batchTimeout = setTimeout(() => {
|
|
119
|
-
(0, logger_1.log)(wtag, `Batch timed out after ${BATCH_TIMEOUT_MS}ms, aborting`);
|
|
120
|
-
batchAc.abort();
|
|
121
|
-
}, BATCH_TIMEOUT_MS);
|
|
122
|
-
const batch = new Map(pending);
|
|
123
|
-
pending.clear();
|
|
124
|
-
const filenames = [...batch.keys()].map((p) => path.basename(p));
|
|
125
|
-
(0, logger_1.log)(wtag, `Processing ${batch.size} changed files: ${filenames.join(", ")}`);
|
|
126
|
-
const start = Date.now();
|
|
127
|
-
let reindexed = 0;
|
|
128
|
-
try {
|
|
129
|
-
const lock = yield (0, lock_1.acquireWriterLockWithRetry)(dataDir, {
|
|
130
|
-
maxRetries: 3,
|
|
131
|
-
retryDelayMs: 500,
|
|
132
|
-
});
|
|
133
|
-
try {
|
|
134
|
-
const pool = (0, pool_1.getWorkerPool)();
|
|
135
|
-
const deletes = [];
|
|
136
|
-
const vectors = [];
|
|
137
|
-
const metaUpdates = new Map();
|
|
138
|
-
const metaDeletes = [];
|
|
139
|
-
const attempted = new Set();
|
|
140
|
-
for (const [absPath, event] of batch) {
|
|
141
|
-
if (batchAc.signal.aborted)
|
|
142
|
-
break;
|
|
143
|
-
attempted.add(absPath);
|
|
144
|
-
if (event === "unlink") {
|
|
145
|
-
deletes.push(absPath);
|
|
146
|
-
metaDeletes.push(absPath);
|
|
147
|
-
reindexed++;
|
|
148
|
-
continue;
|
|
149
|
-
}
|
|
150
|
-
// change or add
|
|
151
|
-
try {
|
|
152
|
-
const stats = yield fs.promises.stat(absPath);
|
|
153
|
-
if (!(0, file_utils_1.isIndexableFile)(absPath, stats.size))
|
|
154
|
-
continue;
|
|
155
|
-
// Quick mtime/size check — skip worker pool if unchanged
|
|
156
|
-
const cached = metaCache.get(absPath);
|
|
157
|
-
if ((0, cache_check_1.isFileCached)(cached, stats)) {
|
|
158
|
-
continue;
|
|
159
|
-
}
|
|
160
|
-
const result = yield pool.processFile({
|
|
161
|
-
path: absPath,
|
|
162
|
-
absolutePath: absPath,
|
|
163
|
-
}, batchAc.signal);
|
|
164
|
-
const metaEntry = {
|
|
165
|
-
hash: result.hash,
|
|
166
|
-
mtimeMs: result.mtimeMs,
|
|
167
|
-
size: result.size,
|
|
168
|
-
};
|
|
169
|
-
if (cached && cached.hash === result.hash) {
|
|
170
|
-
metaUpdates.set(absPath, metaEntry);
|
|
171
|
-
continue;
|
|
172
|
-
}
|
|
173
|
-
if (result.shouldDelete) {
|
|
174
|
-
deletes.push(absPath);
|
|
175
|
-
metaUpdates.set(absPath, metaEntry);
|
|
176
|
-
reindexed++;
|
|
177
|
-
continue;
|
|
178
|
-
}
|
|
179
|
-
// Delete old vectors, insert new
|
|
180
|
-
deletes.push(absPath);
|
|
181
|
-
if (result.vectors.length > 0) {
|
|
182
|
-
vectors.push(...result.vectors);
|
|
183
|
-
}
|
|
184
|
-
metaUpdates.set(absPath, metaEntry);
|
|
185
|
-
reindexed++;
|
|
186
|
-
}
|
|
187
|
-
catch (err) {
|
|
188
|
-
if (batchAc.signal.aborted)
|
|
189
|
-
break;
|
|
190
|
-
const code = err === null || err === void 0 ? void 0 : err.code;
|
|
191
|
-
if (code === "ENOENT") {
|
|
192
|
-
deletes.push(absPath);
|
|
193
|
-
metaDeletes.push(absPath);
|
|
194
|
-
reindexed++;
|
|
195
|
-
}
|
|
196
|
-
else {
|
|
197
|
-
console.error(`[${wtag}] Failed to process ${absPath}:`, err);
|
|
198
|
-
if (!pool.isHealthy()) {
|
|
199
|
-
console.error(`[${wtag}] Worker pool unhealthy, aborting batch`);
|
|
200
|
-
break;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
// Requeue files that weren't attempted (aborted or pool unhealthy)
|
|
206
|
-
for (const [absPath, event] of batch) {
|
|
207
|
-
if (!attempted.has(absPath) && !pending.has(absPath)) {
|
|
208
|
-
pending.set(absPath, event);
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
// Flush to VectorDB: insert first, then delete old (preserving new)
|
|
212
|
-
const newIds = vectors.map((v) => v.id);
|
|
213
|
-
if (vectors.length > 0) {
|
|
214
|
-
yield vectorDb.insertBatch(vectors);
|
|
215
|
-
}
|
|
216
|
-
if (deletes.length > 0) {
|
|
217
|
-
if (newIds.length > 0) {
|
|
218
|
-
yield vectorDb.deletePathsExcludingIds(deletes, newIds);
|
|
219
|
-
}
|
|
220
|
-
else {
|
|
221
|
-
yield vectorDb.deletePaths(deletes);
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
// Update MetaCache
|
|
225
|
-
for (const [p, entry] of metaUpdates) {
|
|
226
|
-
metaCache.put(p, entry);
|
|
227
|
-
}
|
|
228
|
-
for (const p of metaDeletes) {
|
|
229
|
-
metaCache.delete(p);
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
finally {
|
|
233
|
-
yield lock.release();
|
|
234
|
-
}
|
|
235
|
-
const duration = Date.now() - start;
|
|
236
|
-
if (reindexed > 0) {
|
|
237
|
-
onReindex === null || onReindex === void 0 ? void 0 : onReindex(reindexed, duration);
|
|
238
|
-
}
|
|
239
|
-
(0, logger_1.log)(wtag, `Batch complete: ${batch.size} files, ${reindexed} reindexed (${(duration / 1000).toFixed(1)}s)`);
|
|
240
|
-
consecutiveLockFailures = 0;
|
|
241
|
-
for (const absPath of batch.keys()) {
|
|
242
|
-
retryCount.delete(absPath);
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
catch (err) {
|
|
246
|
-
const isLockError = err instanceof Error && err.message.includes("lock already held");
|
|
247
|
-
if (isLockError) {
|
|
248
|
-
consecutiveLockFailures++;
|
|
249
|
-
}
|
|
250
|
-
console.error(`[${wtag}] Batch processing failed:`, err);
|
|
251
|
-
const { requeued, dropped, backoffMs } = (0, watcher_batch_1.computeRetryAction)(batch, retryCount, MAX_RETRIES, isLockError, consecutiveLockFailures, DEBOUNCE_MS);
|
|
252
|
-
for (const [absPath, event] of requeued) {
|
|
253
|
-
if (!pending.has(absPath)) {
|
|
254
|
-
pending.set(absPath, event);
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
if (dropped > 0) {
|
|
258
|
-
console.warn(`[${wtag}] Dropped ${dropped} file(s) after ${MAX_RETRIES} failed retries`);
|
|
259
|
-
}
|
|
260
|
-
if (pending.size > 0) {
|
|
261
|
-
if (debounceTimer)
|
|
262
|
-
clearTimeout(debounceTimer);
|
|
263
|
-
debounceTimer = setTimeout(() => processBatch(), backoffMs);
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
finally {
|
|
267
|
-
clearTimeout(batchTimeout);
|
|
268
|
-
currentBatchAc = null;
|
|
269
|
-
processing = false;
|
|
270
|
-
// Process any events that came in while we were processing
|
|
271
|
-
if (pending.size > 0) {
|
|
272
|
-
scheduleBatch();
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
});
|
|
276
|
-
const onFileEvent = (event, absPath) => {
|
|
277
|
-
if (closed)
|
|
278
|
-
return;
|
|
279
|
-
if (event !== "unlink") {
|
|
280
|
-
const ext = path.extname(absPath).toLowerCase();
|
|
281
|
-
const bn = path.basename(absPath).toLowerCase();
|
|
282
|
-
if (!config_1.INDEXABLE_EXTENSIONS.has(ext) && !config_1.INDEXABLE_EXTENSIONS.has(bn))
|
|
283
|
-
return;
|
|
284
|
-
}
|
|
285
|
-
pending.set(absPath, event);
|
|
286
|
-
scheduleBatch();
|
|
287
|
-
};
|
|
288
|
-
watcher.on("add", (p) => onFileEvent("change", p));
|
|
289
|
-
watcher.on("change", (p) => onFileEvent("change", p));
|
|
290
|
-
watcher.on("unlink", (p) => onFileEvent("unlink", p));
|
|
291
|
-
// Periodic FTS rebuild
|
|
292
|
-
const ftsInterval = setInterval(() => __awaiter(this, void 0, void 0, function* () {
|
|
293
|
-
if (closed || processing)
|
|
294
|
-
return;
|
|
295
|
-
try {
|
|
296
|
-
yield vectorDb.createFTSIndex();
|
|
297
|
-
}
|
|
298
|
-
catch (err) {
|
|
299
|
-
console.error(`[${wtag}] FTS rebuild failed:`, err);
|
|
300
|
-
}
|
|
301
|
-
}), FTS_REBUILD_INTERVAL_MS);
|
|
302
|
-
ftsInterval.unref();
|
|
84
|
+
watcher.on("add", (p) => processor.handleFileEvent("change", p));
|
|
85
|
+
watcher.on("change", (p) => processor.handleFileEvent("change", p));
|
|
86
|
+
watcher.on("unlink", (p) => processor.handleFileEvent("unlink", p));
|
|
303
87
|
return {
|
|
304
88
|
close: () => __awaiter(this, void 0, void 0, function* () {
|
|
305
|
-
|
|
306
|
-
currentBatchAc === null || currentBatchAc === void 0 ? void 0 : currentBatchAc.abort();
|
|
307
|
-
if (debounceTimer)
|
|
308
|
-
clearTimeout(debounceTimer);
|
|
309
|
-
clearInterval(ftsInterval);
|
|
89
|
+
yield processor.close();
|
|
310
90
|
yield watcher.close();
|
|
311
91
|
}),
|
|
312
92
|
};
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.sendDaemonCommand = sendDaemonCommand;
|
|
46
|
+
exports.isDaemonRunning = isDaemonRunning;
|
|
47
|
+
const net = __importStar(require("node:net"));
|
|
48
|
+
const config_1 = require("../../config");
|
|
49
|
+
const DEFAULT_TIMEOUT_MS = 5000;
|
|
50
|
+
/**
|
|
51
|
+
* Send a JSON command to the daemon over the Unix domain socket.
|
|
52
|
+
* Returns the parsed response, or {ok: false, error} on failure.
|
|
53
|
+
*/
|
|
54
|
+
function sendDaemonCommand(cmd, opts) {
|
|
55
|
+
var _a;
|
|
56
|
+
const timeout = (_a = opts === null || opts === void 0 ? void 0 : opts.timeoutMs) !== null && _a !== void 0 ? _a : DEFAULT_TIMEOUT_MS;
|
|
57
|
+
return new Promise((resolve) => {
|
|
58
|
+
let settled = false;
|
|
59
|
+
const finish = (resp) => {
|
|
60
|
+
if (settled)
|
|
61
|
+
return;
|
|
62
|
+
settled = true;
|
|
63
|
+
socket.destroy();
|
|
64
|
+
resolve(resp);
|
|
65
|
+
};
|
|
66
|
+
const socket = net.createConnection({ path: config_1.PATHS.daemonSocket });
|
|
67
|
+
const timer = setTimeout(() => {
|
|
68
|
+
finish({ ok: false, error: "timeout" });
|
|
69
|
+
}, timeout);
|
|
70
|
+
socket.on("connect", () => {
|
|
71
|
+
socket.write(`${JSON.stringify(cmd)}\n`);
|
|
72
|
+
});
|
|
73
|
+
let buf = "";
|
|
74
|
+
socket.on("data", (chunk) => {
|
|
75
|
+
buf += chunk.toString();
|
|
76
|
+
const nl = buf.indexOf("\n");
|
|
77
|
+
if (nl !== -1) {
|
|
78
|
+
clearTimeout(timer);
|
|
79
|
+
try {
|
|
80
|
+
finish(JSON.parse(buf.slice(0, nl)));
|
|
81
|
+
}
|
|
82
|
+
catch (_a) {
|
|
83
|
+
finish({ ok: false, error: "invalid response" });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
socket.on("error", (err) => {
|
|
88
|
+
var _a;
|
|
89
|
+
clearTimeout(timer);
|
|
90
|
+
finish({ ok: false, error: (_a = err.code) !== null && _a !== void 0 ? _a : err.message });
|
|
91
|
+
});
|
|
92
|
+
socket.on("close", () => {
|
|
93
|
+
clearTimeout(timer);
|
|
94
|
+
if (!settled) {
|
|
95
|
+
finish({ ok: false, error: "connection closed" });
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Check if the daemon is running by sending a ping.
|
|
102
|
+
*/
|
|
103
|
+
function isDaemonRunning() {
|
|
104
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
105
|
+
const resp = yield sendDaemonCommand({ cmd: "ping" }, { timeoutMs: 2000 });
|
|
106
|
+
return resp.ok === true;
|
|
107
|
+
});
|
|
108
|
+
}
|