grepmax 0.7.29 → 0.7.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/lib/index/syncer.js
CHANGED
|
@@ -50,6 +50,7 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
|
50
50
|
};
|
|
51
51
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
52
|
exports.generateSummaries = generateSummaries;
|
|
53
|
+
exports.computeStaleFiles = computeStaleFiles;
|
|
53
54
|
exports.initialSync = initialSync;
|
|
54
55
|
const fs = __importStar(require("node:fs"));
|
|
55
56
|
const path = __importStar(require("node:path"));
|
|
@@ -178,6 +179,9 @@ function createNoopMetaCache() {
|
|
|
178
179
|
close: () => __awaiter(this, void 0, void 0, function* () { }),
|
|
179
180
|
};
|
|
180
181
|
}
|
|
182
|
+
function computeStaleFiles(cachedPaths, seenPaths) {
|
|
183
|
+
return Array.from(cachedPaths).filter((p) => !seenPaths.has(p));
|
|
184
|
+
}
|
|
181
185
|
function initialSync(options) {
|
|
182
186
|
return __awaiter(this, void 0, void 0, function* () {
|
|
183
187
|
var _a, e_1, _b, _c;
|
|
@@ -465,7 +469,7 @@ function initialSync(options) {
|
|
|
465
469
|
: new Error(String(flushError));
|
|
466
470
|
}
|
|
467
471
|
// Stale cleanup: only remove paths scoped to this project's root
|
|
468
|
-
const stale =
|
|
472
|
+
const stale = computeStaleFiles(cachedPaths, seenPaths);
|
|
469
473
|
if (!dryRun && stale.length > 0 && !shouldSkipCleanup) {
|
|
470
474
|
(0, logger_1.log)("index", `Stale cleanup: ${stale.length} paths`);
|
|
471
475
|
yield vectorDb.deletePaths(stale);
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.processBatchCore = processBatchCore;
|
|
46
|
+
exports.flushBatchToDb = flushBatchToDb;
|
|
47
|
+
exports.computeRetryAction = computeRetryAction;
|
|
48
|
+
const fs = __importStar(require("node:fs"));
|
|
49
|
+
const cache_check_1 = require("../utils/cache-check");
|
|
50
|
+
const file_utils_1 = require("../utils/file-utils");
|
|
51
|
+
function processBatchCore(batch, metaCache, pool) {
|
|
52
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
53
|
+
let reindexed = 0;
|
|
54
|
+
const changedIds = [];
|
|
55
|
+
const deletes = [];
|
|
56
|
+
const vectors = [];
|
|
57
|
+
const metaUpdates = new Map();
|
|
58
|
+
const metaDeletes = [];
|
|
59
|
+
for (const [absPath, event] of batch) {
|
|
60
|
+
if (event === "unlink") {
|
|
61
|
+
deletes.push(absPath);
|
|
62
|
+
metaDeletes.push(absPath);
|
|
63
|
+
reindexed++;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
const stats = yield fs.promises.stat(absPath);
|
|
68
|
+
if (!(0, file_utils_1.isIndexableFile)(absPath, stats.size))
|
|
69
|
+
continue;
|
|
70
|
+
const cached = metaCache.get(absPath);
|
|
71
|
+
if ((0, cache_check_1.isFileCached)(cached, stats)) {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
const result = yield pool.processFile({
|
|
75
|
+
path: absPath,
|
|
76
|
+
absolutePath: absPath,
|
|
77
|
+
});
|
|
78
|
+
const metaEntry = {
|
|
79
|
+
hash: result.hash,
|
|
80
|
+
mtimeMs: result.mtimeMs,
|
|
81
|
+
size: result.size,
|
|
82
|
+
};
|
|
83
|
+
if (cached && cached.hash === result.hash) {
|
|
84
|
+
metaUpdates.set(absPath, metaEntry);
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (result.shouldDelete) {
|
|
88
|
+
deletes.push(absPath);
|
|
89
|
+
metaUpdates.set(absPath, metaEntry);
|
|
90
|
+
reindexed++;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
deletes.push(absPath);
|
|
94
|
+
if (result.vectors.length > 0) {
|
|
95
|
+
vectors.push(...result.vectors);
|
|
96
|
+
for (const v of result.vectors) {
|
|
97
|
+
changedIds.push(v.id);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
metaUpdates.set(absPath, metaEntry);
|
|
101
|
+
reindexed++;
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
const code = err === null || err === void 0 ? void 0 : err.code;
|
|
105
|
+
if (code === "ENOENT") {
|
|
106
|
+
deletes.push(absPath);
|
|
107
|
+
metaDeletes.push(absPath);
|
|
108
|
+
reindexed++;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return { reindexed, changedIds, vectors, deletes, metaUpdates, metaDeletes };
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
function flushBatchToDb(result, vectorDb) {
|
|
116
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
117
|
+
const newIds = result.vectors.map((v) => v.id);
|
|
118
|
+
if (result.vectors.length > 0) {
|
|
119
|
+
yield vectorDb.insertBatch(result.vectors);
|
|
120
|
+
}
|
|
121
|
+
if (result.deletes.length > 0) {
|
|
122
|
+
if (newIds.length > 0) {
|
|
123
|
+
yield vectorDb.deletePathsExcludingIds(result.deletes, newIds);
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
yield vectorDb.deletePaths(result.deletes);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
function computeRetryAction(batch, retryCount, maxRetries, isLockError, consecutiveLockFailures, debounceMs) {
|
|
132
|
+
var _a;
|
|
133
|
+
const requeued = new Map();
|
|
134
|
+
let dropped = 0;
|
|
135
|
+
for (const [absPath, event] of batch) {
|
|
136
|
+
const count = ((_a = retryCount.get(absPath)) !== null && _a !== void 0 ? _a : 0) + 1;
|
|
137
|
+
if (count >= maxRetries) {
|
|
138
|
+
retryCount.delete(absPath);
|
|
139
|
+
dropped++;
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
requeued.set(absPath, event);
|
|
143
|
+
retryCount.set(absPath, count);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
const effectiveFailures = isLockError ? consecutiveLockFailures + 1 : 0;
|
|
147
|
+
const backoffMs = Math.min(debounceMs * Math.pow(2, effectiveFailures), 30000);
|
|
148
|
+
return { requeued, dropped, backoffMs };
|
|
149
|
+
}
|
package/package.json
CHANGED