grepmax 0.26.63 → 0.26.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +12 -2
- package/dist/bin.js +1 -1
- package/dist/commands/recover.js +117 -0
- package/dist/index.js +2 -0
- package/dist/lib/daemon/watcher-manager.js +69 -27
- package/dist/lib/index/watcher-errors.js +16 -0
- package/dist/lib/index/watcher.js +43 -3
- package/dist/lib/store/lance-cleanup.js +425 -0
- package/dist/lib/store/prune-state.js +113 -0
- package/dist/lib/store/recovery-admission.js +92 -0
- package/dist/lib/store/recovery.js +163 -0
- package/dist/lib/store/store-lease.js +119 -4
- package/dist/lib/utils/host-resource.js +3 -3
- package/dist/lib/utils/resource-budget.js +6 -6
- package/lance-maintenance/prune.py +200 -0
- package/lance-maintenance/pyproject.toml +9 -0
- package/lance-maintenance/uv.lock +531 -0
- package/package.json +5 -2
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.recoverStore = recoverStore;
|
|
46
|
+
const fs = __importStar(require("node:fs"));
|
|
47
|
+
const path = __importStar(require("node:path"));
|
|
48
|
+
const resource_budget_1 = require("../utils/resource-budget");
|
|
49
|
+
const lance_cleanup_1 = require("./lance-cleanup");
|
|
50
|
+
const prune_state_1 = require("./prune-state");
|
|
51
|
+
const recovery_admission_1 = require("./recovery-admission");
|
|
52
|
+
const store_lease_1 = require("./store-lease");
|
|
53
|
+
function recoverStore(request) {
|
|
54
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
55
|
+
var _a, _b, _c, _d;
|
|
56
|
+
(_a = request.signal) === null || _a === void 0 ? void 0 : _a.throwIfAborted();
|
|
57
|
+
const input = path.resolve(request.table);
|
|
58
|
+
const stat = fs.lstatSync(input);
|
|
59
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || !input.endsWith(".lance"))
|
|
60
|
+
throw new Error("Recovery requires an existing local Lance table directory, without a table symlink");
|
|
61
|
+
const table = fs.realpathSync(input);
|
|
62
|
+
const store = path.dirname(table);
|
|
63
|
+
const previous = (0, prune_state_1.readPruneState)(store);
|
|
64
|
+
if (!request.prune && !request.check)
|
|
65
|
+
return {
|
|
66
|
+
outcome: "status",
|
|
67
|
+
table,
|
|
68
|
+
state: previous,
|
|
69
|
+
fullTableRewriting: "disabled",
|
|
70
|
+
};
|
|
71
|
+
if (request.prune &&
|
|
72
|
+
(!Number.isSafeInteger(request.version) ||
|
|
73
|
+
request.version < 1 ||
|
|
74
|
+
!request.cutoff ||
|
|
75
|
+
!Number.isSafeInteger(request.cutoff.getTime()) ||
|
|
76
|
+
request.cutoff.getTime() > Date.now()))
|
|
77
|
+
throw new Error("Explicit prune requires a positive --version and a real, nonfuture --cutoff");
|
|
78
|
+
if (request.prune &&
|
|
79
|
+
(previous === null || previous === void 0 ? void 0 : previous.outcome) === "uncertain" &&
|
|
80
|
+
request.acknowledgeUncertain !== previous.attemptId)
|
|
81
|
+
throw new Error(`Previous prune completion is uncertain; inspect retained state, then use --acknowledge-uncertain ${previous.attemptId} for a verified explicit retry`);
|
|
82
|
+
const budget = (0, recovery_admission_1.createRecoveryBudget)();
|
|
83
|
+
(0, recovery_admission_1.assertRecoveryAdmission)(store, budget);
|
|
84
|
+
if (!request.prune) {
|
|
85
|
+
const admission = (0, recovery_admission_1.admitPrune)(store, budget);
|
|
86
|
+
try {
|
|
87
|
+
return {
|
|
88
|
+
outcome: "preflight-passed",
|
|
89
|
+
table,
|
|
90
|
+
state: previous,
|
|
91
|
+
host: admission.snapshot,
|
|
92
|
+
helperReserveMb: resource_budget_1.HELPER_RESOURCE_RESERVE_MB,
|
|
93
|
+
fullTableRewriting: "disabled",
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
admission.close();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
// Setup is admitted and monitored before acquiring store exclusion. No owner
|
|
101
|
+
// is killed and no marker is created or cleared by recovery itself.
|
|
102
|
+
const controller = new AbortController();
|
|
103
|
+
const abort = () => controller.abort();
|
|
104
|
+
(_b = request.signal) === null || _b === void 0 ? void 0 : _b.addEventListener("abort", abort, { once: true });
|
|
105
|
+
if ((_c = request.signal) === null || _c === void 0 ? void 0 : _c.aborted)
|
|
106
|
+
controller.abort();
|
|
107
|
+
let setup;
|
|
108
|
+
let monitor;
|
|
109
|
+
let lease;
|
|
110
|
+
try {
|
|
111
|
+
setup = budget.reserve(resource_budget_1.HELPER_RESOURCE_RESERVE_MB, "prune-runtime-setup");
|
|
112
|
+
monitor = setInterval(() => {
|
|
113
|
+
try {
|
|
114
|
+
(0, recovery_admission_1.assertRecoveryAdmission)(store, budget);
|
|
115
|
+
}
|
|
116
|
+
catch (_a) {
|
|
117
|
+
controller.abort();
|
|
118
|
+
}
|
|
119
|
+
}, 5000);
|
|
120
|
+
const runtime = yield (0, lance_cleanup_1.prepareCleanupRuntime)(controller.signal);
|
|
121
|
+
clearInterval(monitor);
|
|
122
|
+
setup.release();
|
|
123
|
+
(0, recovery_admission_1.assertRecoveryAdmission)(store, budget);
|
|
124
|
+
lease = yield store_lease_1.StoreLease.acquireExclusive({
|
|
125
|
+
storeDir: store,
|
|
126
|
+
timeoutMs: 1000,
|
|
127
|
+
signal: controller.signal,
|
|
128
|
+
role: "explicit-prune-recovery",
|
|
129
|
+
});
|
|
130
|
+
// Another completed/interrupted attempt may have happened during setup.
|
|
131
|
+
const current = (0, prune_state_1.readPruneState)(store);
|
|
132
|
+
if ((current === null || current === void 0 ? void 0 : current.outcome) === "uncertain" &&
|
|
133
|
+
request.acknowledgeUncertain !== current.attemptId)
|
|
134
|
+
throw new Error(`Prune completion is uncertain; inspect and acknowledge attempt ${current.attemptId} before retry`);
|
|
135
|
+
(0, recovery_admission_1.assertRecoveryAdmission)(store, budget);
|
|
136
|
+
const result = yield (0, lance_cleanup_1.pruneVersions)(runtime, table, request.version, request.cutoff, {
|
|
137
|
+
lease,
|
|
138
|
+
signal: controller.signal,
|
|
139
|
+
acknowledgeUncertain: request.acknowledgeUncertain,
|
|
140
|
+
});
|
|
141
|
+
return {
|
|
142
|
+
outcome: "verified",
|
|
143
|
+
table,
|
|
144
|
+
state: (0, prune_state_1.readPruneState)(store),
|
|
145
|
+
result,
|
|
146
|
+
allocatedBytesRecovered: result.allocatedBytesBefore - result.allocatedBytesAfter,
|
|
147
|
+
filesystemFreeBytesChange: result.freeBytesAfter - result.freeBytesBefore,
|
|
148
|
+
fullTableRewriting: "disabled",
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
if (error instanceof store_lease_1.StoreLeaseTimeoutError)
|
|
153
|
+
throw new Error(`Recovery refused: store owners are live or unknown (${error.blockers.map((owner) => `${owner.role} pid=${owner.pid}`).join(", ") || "unverified owner"}); close them separately`);
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
clearInterval(monitor);
|
|
158
|
+
(_d = request.signal) === null || _d === void 0 ? void 0 : _d.removeEventListener("abort", abort);
|
|
159
|
+
setup === null || setup === void 0 ? void 0 : setup.release();
|
|
160
|
+
yield lease === null || lease === void 0 ? void 0 : lease.release();
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
}
|
|
@@ -108,6 +108,14 @@ const defaultProbeDeps = {
|
|
|
108
108
|
}).trim(),
|
|
109
109
|
};
|
|
110
110
|
function defaultProbeOwner(owner, deps = defaultProbeDeps) {
|
|
111
|
+
const parent = probeProcessOwner(owner, deps);
|
|
112
|
+
if ((parent === "dead" || parent === "reused") && owner.activeHelper) {
|
|
113
|
+
const child = probeProcessOwner(Object.assign(Object.assign({}, owner), { pid: owner.activeHelper.pid, processStart: owner.activeHelper.processStart }), deps);
|
|
114
|
+
return child === "same" || child === "unknown" ? child : parent;
|
|
115
|
+
}
|
|
116
|
+
return parent;
|
|
117
|
+
}
|
|
118
|
+
function probeProcessOwner(owner, deps) {
|
|
111
119
|
try {
|
|
112
120
|
deps.signal(owner.pid);
|
|
113
121
|
}
|
|
@@ -124,6 +132,8 @@ function defaultProbeOwner(owner, deps = defaultProbeDeps) {
|
|
|
124
132
|
if (code !== "EPERM")
|
|
125
133
|
return "unknown";
|
|
126
134
|
}
|
|
135
|
+
if (owner.processStart === "unverified")
|
|
136
|
+
return "unknown";
|
|
127
137
|
if (owner.processStart.startsWith(`pid:${owner.pid}:started:`))
|
|
128
138
|
return "same";
|
|
129
139
|
try {
|
|
@@ -138,6 +148,7 @@ function defaultProbeOwner(owner, deps = defaultProbeDeps) {
|
|
|
138
148
|
}
|
|
139
149
|
}
|
|
140
150
|
function readOwner(filePath) {
|
|
151
|
+
var _a;
|
|
141
152
|
try {
|
|
142
153
|
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
143
154
|
if (!parsed || typeof parsed !== "object")
|
|
@@ -147,18 +158,40 @@ function readOwner(filePath) {
|
|
|
147
158
|
typeof owner.processStart === "string" &&
|
|
148
159
|
typeof owner.nonce === "string" &&
|
|
149
160
|
typeof owner.role === "string" &&
|
|
150
|
-
typeof owner.acquiredAt === "number"
|
|
161
|
+
typeof owner.acquiredAt === "number" &&
|
|
162
|
+
(owner.activeHelper === undefined ||
|
|
163
|
+
(Number.isSafeInteger((_a = owner.activeHelper) === null || _a === void 0 ? void 0 : _a.pid) &&
|
|
164
|
+
owner.activeHelper.pid > 0 &&
|
|
165
|
+
typeof owner.activeHelper.processStart === "string" &&
|
|
166
|
+
!!owner.activeHelper.processStart))
|
|
151
167
|
? owner
|
|
152
168
|
: null;
|
|
153
169
|
}
|
|
154
|
-
catch (
|
|
170
|
+
catch (_b) {
|
|
155
171
|
return null;
|
|
156
172
|
}
|
|
157
173
|
}
|
|
158
|
-
function writeOwnerAtomic(filePath, owner) {
|
|
174
|
+
function writeOwnerAtomic(filePath, owner, durable = false) {
|
|
159
175
|
const temp = `${filePath}.${owner.nonce}.tmp`;
|
|
160
|
-
fs.
|
|
176
|
+
const fd = fs.openSync(temp, "wx", 0o600);
|
|
177
|
+
try {
|
|
178
|
+
fs.writeFileSync(fd, `${JSON.stringify(owner)}\n`);
|
|
179
|
+
if (durable)
|
|
180
|
+
fs.fsyncSync(fd);
|
|
181
|
+
}
|
|
182
|
+
finally {
|
|
183
|
+
fs.closeSync(fd);
|
|
184
|
+
}
|
|
161
185
|
fs.renameSync(temp, filePath);
|
|
186
|
+
if (durable) {
|
|
187
|
+
const dir = fs.openSync(path.dirname(filePath), "r");
|
|
188
|
+
try {
|
|
189
|
+
fs.fsyncSync(dir);
|
|
190
|
+
}
|
|
191
|
+
finally {
|
|
192
|
+
fs.closeSync(dir);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
162
195
|
}
|
|
163
196
|
function delay(ms, signal) {
|
|
164
197
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -189,6 +222,7 @@ class StoreLease {
|
|
|
189
222
|
this.storeDir = storeDir;
|
|
190
223
|
this.released = false;
|
|
191
224
|
this.claimedBy = null;
|
|
225
|
+
this.exclusiveUseRunning = false;
|
|
192
226
|
}
|
|
193
227
|
static acquireShared(options) {
|
|
194
228
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -269,6 +303,9 @@ class StoreLease {
|
|
|
269
303
|
}
|
|
270
304
|
release() {
|
|
271
305
|
return __awaiter(this, arguments, void 0, function* (options = {}) {
|
|
306
|
+
if (this.exclusiveUseRunning) {
|
|
307
|
+
throw new Error("Exclusive store operation must finish before releasing its lease");
|
|
308
|
+
}
|
|
272
309
|
if (this.released)
|
|
273
310
|
return;
|
|
274
311
|
if (this.mode === "shared") {
|
|
@@ -306,7 +343,82 @@ class StoreLease {
|
|
|
306
343
|
if (this.claimedBy === holder)
|
|
307
344
|
this.claimedBy = null;
|
|
308
345
|
}
|
|
346
|
+
/** Hold verified cross-process exclusion through a destructive helper and its
|
|
347
|
+
* verification. A local write gate is not a substitute for this ownership. */
|
|
348
|
+
withExclusiveUse(storeDir, operation) {
|
|
349
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
350
|
+
if (this.exclusiveUseRunning) {
|
|
351
|
+
throw new Error("An exclusive store operation is already running");
|
|
352
|
+
}
|
|
353
|
+
const verify = () => {
|
|
354
|
+
if (this.mode !== "exclusive" || this.released) {
|
|
355
|
+
throw new Error("Pruning requires an unreleased exclusive store lease");
|
|
356
|
+
}
|
|
357
|
+
if (fs.realpathSync(storeDir) !== fs.realpathSync(this.storeDir)) {
|
|
358
|
+
throw new Error("Exclusive store lease belongs to a different store");
|
|
359
|
+
}
|
|
360
|
+
const ownerStat = fs.lstatSync(this.paths.intentOwnerFile);
|
|
361
|
+
if (!ownerStat.isFile() || ownerStat.size > 16 * 1024) {
|
|
362
|
+
throw new Error("Exclusive store lease owner is unverified or oversized");
|
|
363
|
+
}
|
|
364
|
+
const current = readOwner(this.paths.intentOwnerFile);
|
|
365
|
+
if ((current === null || current === void 0 ? void 0 : current.nonce) !== this.owner.nonce ||
|
|
366
|
+
current.pid !== this.owner.pid ||
|
|
367
|
+
current.processStart !== this.owner.processStart) {
|
|
368
|
+
throw new Error("Exclusive store lease ownership could not be verified");
|
|
369
|
+
}
|
|
370
|
+
// acquireExclusive has already removed verified stale readers. Refuse any
|
|
371
|
+
// remaining marker rather than assuming ignored/unknown readers are safe.
|
|
372
|
+
if (fs
|
|
373
|
+
.readdirSync(this.paths.readersDir)
|
|
374
|
+
.some((name) => name.endsWith(".json"))) {
|
|
375
|
+
throw new Error("Pruning requires all shared store owners to close");
|
|
376
|
+
}
|
|
377
|
+
};
|
|
378
|
+
verify();
|
|
379
|
+
this.exclusiveUseRunning = true;
|
|
380
|
+
try {
|
|
381
|
+
const result = yield operation();
|
|
382
|
+
verify();
|
|
383
|
+
return result;
|
|
384
|
+
}
|
|
385
|
+
finally {
|
|
386
|
+
this.exclusiveUseRunning = false;
|
|
387
|
+
}
|
|
388
|
+
});
|
|
389
|
+
}
|
|
309
390
|
/** Atomically replace this shared lease with an exclusive lease. */
|
|
391
|
+
pinExclusiveHelper(pid) {
|
|
392
|
+
if (!this.exclusiveUseRunning ||
|
|
393
|
+
this.owner.activeHelper ||
|
|
394
|
+
!Number.isSafeInteger(pid) ||
|
|
395
|
+
pid < 1)
|
|
396
|
+
throw new Error("Helper requires an active exclusive store operation");
|
|
397
|
+
const current = readOwner(this.paths.intentOwnerFile);
|
|
398
|
+
if ((current === null || current === void 0 ? void 0 : current.nonce) !== this.owner.nonce || current.activeHelper)
|
|
399
|
+
throw new Error("Exclusive store ownership changed before helper launch");
|
|
400
|
+
// Publish uncertainty first. A failed identity probe must retain exclusion
|
|
401
|
+
// until this child is confirmed dead, even if the parent is killed.
|
|
402
|
+
this.owner.activeHelper = { pid, processStart: "unverified" };
|
|
403
|
+
// The helper is the primary on-disk owner, so older readers also retain
|
|
404
|
+
// exclusion after parent death. It is blocked on stdin until this identity
|
|
405
|
+
// is durably verified; uncertainty here cannot admit native deletion.
|
|
406
|
+
writeOwnerAtomic(this.paths.intentOwnerFile, Object.assign(Object.assign({}, this.owner), { pid, processStart: "unverified" }), true);
|
|
407
|
+
const start = defaultProbeDeps.processStartOf(pid);
|
|
408
|
+
if (!start)
|
|
409
|
+
throw new Error("Prune helper identity unavailable");
|
|
410
|
+
this.owner.activeHelper.processStart = start;
|
|
411
|
+
writeOwnerAtomic(this.paths.intentOwnerFile, Object.assign(Object.assign({}, this.owner), { pid, processStart: start }), true);
|
|
412
|
+
return () => {
|
|
413
|
+
var _a;
|
|
414
|
+
const latest = readOwner(this.paths.intentOwnerFile);
|
|
415
|
+
if ((latest === null || latest === void 0 ? void 0 : latest.nonce) !== this.owner.nonce ||
|
|
416
|
+
((_a = latest.activeHelper) === null || _a === void 0 ? void 0 : _a.pid) !== pid)
|
|
417
|
+
throw new Error("Exclusive store ownership changed during helper close");
|
|
418
|
+
delete this.owner.activeHelper;
|
|
419
|
+
writeOwnerAtomic(this.paths.intentOwnerFile, this.owner, true);
|
|
420
|
+
};
|
|
421
|
+
}
|
|
310
422
|
upgrade() {
|
|
311
423
|
return __awaiter(this, arguments, void 0, function* (options = {}) {
|
|
312
424
|
if (this.mode !== "shared") {
|
|
@@ -328,6 +440,9 @@ class StoreLease {
|
|
|
328
440
|
/** Replace this exclusive lease with a shared lease without an ownership gap. */
|
|
329
441
|
downgrade() {
|
|
330
442
|
return __awaiter(this, arguments, void 0, function* (options = {}) {
|
|
443
|
+
if (this.exclusiveUseRunning) {
|
|
444
|
+
throw new Error("Exclusive store operation must finish before downgrading its lease");
|
|
445
|
+
}
|
|
331
446
|
if (this.mode !== "exclusive") {
|
|
332
447
|
throw new Error("Only an exclusive store lease can be downgraded");
|
|
333
448
|
}
|
|
@@ -132,7 +132,7 @@ function parseBatchFootprints(output) {
|
|
|
132
132
|
function sampleHostResources(extraPids = [], overrides = {}, extraGroups = []) {
|
|
133
133
|
var _a;
|
|
134
134
|
var _b, _c;
|
|
135
|
-
const deps = Object.assign({ platform: process.platform, pid: process.pid, uid: (_b = (_a = process.getuid) === null || _a === void 0 ? void 0 : _a.call(process)) !== null && _b !== void 0 ? _b : -1, now: Date.now, monotonic: () => performance.now(), freeBytes: os.freemem, run: (command, args, timeoutMs) => (0, node_child_process_1.execFileSync)(command, args, {
|
|
135
|
+
const deps = Object.assign({ platform: process.platform, pid: process.pid, uid: (_b = (_a = process.getuid) === null || _a === void 0 ? void 0 : _a.call(process)) !== null && _b !== void 0 ? _b : -1, now: Date.now, monotonic: () => performance.now(), freeBytes: os.freemem, sampleTimeoutMs: exports.HOST_SAMPLE_TIMEOUT_MS, kernelProbeTimeoutMs: 500, run: (command, args, timeoutMs) => (0, node_child_process_1.execFileSync)(command, args, {
|
|
136
136
|
encoding: "utf8",
|
|
137
137
|
timeout: timeoutMs,
|
|
138
138
|
maxBuffer: 512 * 1024,
|
|
@@ -153,7 +153,7 @@ function sampleHostResources(extraPids = [], overrides = {}, extraGroups = []) {
|
|
|
153
153
|
};
|
|
154
154
|
if (deps.platform !== "darwin")
|
|
155
155
|
return snapshot;
|
|
156
|
-
const deadline = deps.monotonic() +
|
|
156
|
+
const deadline = deps.monotonic() + deps.sampleTimeoutMs;
|
|
157
157
|
const run = (command, args, maximumMs = 500) => {
|
|
158
158
|
const remaining = Math.floor(deadline - deps.monotonic());
|
|
159
159
|
if (remaining <= 0)
|
|
@@ -167,7 +167,7 @@ function sampleHostResources(extraPids = [], overrides = {}, extraGroups = []) {
|
|
|
167
167
|
}
|
|
168
168
|
catch (_d) { }
|
|
169
169
|
try {
|
|
170
|
-
const zone = (0, kernel_zone_1.parseZprintOutput)(run("zprint", ["-L", "data.kalloc.1024"]));
|
|
170
|
+
const zone = (0, kernel_zone_1.parseZprintOutput)(run("zprint", ["-L", "data.kalloc.1024"], deps.kernelProbeTimeoutMs));
|
|
171
171
|
if (zone) {
|
|
172
172
|
const bytes = zone.elements * zone.elementSize;
|
|
173
173
|
if (Number.isSafeInteger(bytes)) {
|
|
@@ -75,7 +75,7 @@ function resolveResourceBudgetMb(raw = process.env.GMAX_RESOURCE_BUDGET_MB) {
|
|
|
75
75
|
throw new ResourceAdmissionError("resource budget must be 256..6144 MiB");
|
|
76
76
|
return n;
|
|
77
77
|
}
|
|
78
|
-
function assertResourceSnapshot(snapshot, chargedMb, budgetMb, now = Date.now()) {
|
|
78
|
+
function assertResourceSnapshot(snapshot, chargedMb, budgetMb, now = Date.now(), sampleMaxAgeMs = 5000) {
|
|
79
79
|
if (snapshot.memoryPressure === "critical" ||
|
|
80
80
|
snapshot.kernelPressure === "critical")
|
|
81
81
|
throw new ResourceAdmissionError("critical host pressure; heavy work refused", true);
|
|
@@ -84,7 +84,7 @@ function assertResourceSnapshot(snapshot, chargedMb, budgetMb, now = Date.now())
|
|
|
84
84
|
if (snapshot.platform !== "darwin")
|
|
85
85
|
return;
|
|
86
86
|
if (now < snapshot.at ||
|
|
87
|
-
now - snapshot.at >
|
|
87
|
+
now - snapshot.at > sampleMaxAgeMs ||
|
|
88
88
|
snapshot.completedAt < snapshot.at ||
|
|
89
89
|
snapshot.completedAt > now)
|
|
90
90
|
throw new ResourceAdmissionError("resource sample stale or clock changed; heavy work paused");
|
|
@@ -107,8 +107,8 @@ class ResourceBudget {
|
|
|
107
107
|
constructor(overrides = {}) {
|
|
108
108
|
this.deps = Object.assign({ root: path.join(config_1.PATHS.sharedRoot, "resource-budget"), platform: process.platform, pid: process.pid, now: Date.now, sample: (extraPids = []) => {
|
|
109
109
|
const records = this.readRecords();
|
|
110
|
-
return (0, host_resource_1.sampleHostResources)([...records.map((r) => r.pid), ...extraPids],
|
|
111
|
-
}, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: safety_latch_1.latchSafetyStop, processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
|
|
110
|
+
return (0, host_resource_1.sampleHostResources)([...records.map((r) => r.pid), ...extraPids], this.deps.samplerOverrides, records.flatMap((r) => (r.groupPid ? [r.groupPid] : [])));
|
|
111
|
+
}, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, requireClientRegistration: true, samplerOverrides: {}, sampleMaxAgeMs: 5000, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: safety_latch_1.latchSafetyStop, processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
|
|
112
112
|
encoding: "utf8",
|
|
113
113
|
timeout: 500,
|
|
114
114
|
maxBuffer: 1024,
|
|
@@ -250,10 +250,10 @@ class ResourceBudget {
|
|
|
250
250
|
}
|
|
251
251
|
checked(snapshot, records) {
|
|
252
252
|
try {
|
|
253
|
-
assertResourceSnapshot(snapshot, this.charge(snapshot, records), resolveResourceBudgetMb(), this.deps.now());
|
|
253
|
+
assertResourceSnapshot(snapshot, this.charge(snapshot, records), resolveResourceBudgetMb(), this.deps.now(), this.deps.sampleMaxAgeMs);
|
|
254
254
|
const legacy = snapshot.processes.filter((p) => p.role === "mcp" &&
|
|
255
255
|
!records.some((r) => r.pid === p.pid && r.kind === "client"));
|
|
256
|
-
if (legacy.length)
|
|
256
|
+
if (legacy.length && this.deps.requireClientRegistration)
|
|
257
257
|
throw new ResourceAdmissionError(`${legacy.length} MCP client(s) need to reconnect after upgrading; heavy work paused`);
|
|
258
258
|
}
|
|
259
259
|
catch (error) {
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""Explicit, exclusively leased Lance 12 prune-only recovery candidate.
|
|
2
|
+
|
|
3
|
+
Never compacts, writes rows, builds indexes or starts a server. This helper is
|
|
4
|
+
not an automatic maintenance path. Native production resource validation is
|
|
5
|
+
required before enabling it on a pressured host.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import importlib.metadata
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
MAX_FILES = 100_000
|
|
18
|
+
MAX_VERSIONS = 100_000
|
|
19
|
+
MAX_PROTECTED_VERSIONS = 256
|
|
20
|
+
CACHE_BYTES = 8 * 1024 * 1024
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def apply_resource_limits():
|
|
24
|
+
# Must run before importing native libraries. Cache limits are not total
|
|
25
|
+
# memory limits. RLIMIT_RSS is not enforceable on macOS; do not claim it is.
|
|
26
|
+
for name in ("RAYON_NUM_THREADS", "TOKIO_WORKER_THREADS", "OMP_NUM_THREADS",
|
|
27
|
+
"OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"):
|
|
28
|
+
os.environ[name] = "1"
|
|
29
|
+
import resource
|
|
30
|
+
resource.setrlimit(resource.RLIMIT_CPU, (90, 91))
|
|
31
|
+
resource.setrlimit(resource.RLIMIT_NOFILE, (256, 256))
|
|
32
|
+
if sys.platform.startswith("linux"):
|
|
33
|
+
resource.setrlimit(resource.RLIMIT_AS, (2 * 1024**3, 2 * 1024**3))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def verify_lease(root, owner_file, nonce):
|
|
37
|
+
expected = Path(str(root.parent) + ".lease") / "exclusive-intent" / "owner.json"
|
|
38
|
+
if Path(owner_file).resolve(strict=True) != expected.resolve(strict=True):
|
|
39
|
+
raise ValueError("Exclusive lease belongs to a different store")
|
|
40
|
+
if expected.is_symlink() or expected.stat().st_size > 16 * 1024:
|
|
41
|
+
raise ValueError("Exclusive lease owner is unverified or oversized")
|
|
42
|
+
owner = json.loads(expected.read_text())
|
|
43
|
+
helper = owner.get("activeHelper")
|
|
44
|
+
expected_pid = os.getpid() if helper and helper.get("pid") == os.getpid() else os.getppid()
|
|
45
|
+
if not nonce or owner.get("nonce") != nonce or owner.get("pid") != expected_pid:
|
|
46
|
+
raise ValueError("Exclusive lease ownership could not be verified")
|
|
47
|
+
readers = expected.parent.parent / "readers"
|
|
48
|
+
if not readers.is_dir() or any(readers.glob("*.json")):
|
|
49
|
+
raise ValueError("All shared store owners must close before pruning")
|
|
50
|
+
return owner
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def measure(root):
|
|
54
|
+
total = allocated = count = 0
|
|
55
|
+
pending = [root]
|
|
56
|
+
while pending:
|
|
57
|
+
directory = pending.pop()
|
|
58
|
+
with os.scandir(directory) as entries:
|
|
59
|
+
for entry in entries:
|
|
60
|
+
count += 1
|
|
61
|
+
if count > MAX_FILES:
|
|
62
|
+
raise ValueError("Recovery metadata scan exceeded bounded file count")
|
|
63
|
+
if entry.is_symlink():
|
|
64
|
+
raise ValueError("Recovery refuses symlinks inside the table")
|
|
65
|
+
if entry.is_dir(follow_symlinks=False):
|
|
66
|
+
pending.append(Path(entry.path))
|
|
67
|
+
elif entry.is_file(follow_symlinks=False):
|
|
68
|
+
stat = entry.stat(follow_symlinks=False)
|
|
69
|
+
total += stat.st_size
|
|
70
|
+
allocated += stat.st_blocks * 512
|
|
71
|
+
else:
|
|
72
|
+
raise ValueError("Recovery refuses nonregular table entries")
|
|
73
|
+
space = os.statvfs(root)
|
|
74
|
+
return total, allocated, space.f_bavail * space.f_frsize
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def tag_versions(dataset):
|
|
78
|
+
return {name: dataset.tags.get_version(name) for name in dataset.tags.list()}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def open_dataset(lance, root):
|
|
82
|
+
return lance.dataset(
|
|
83
|
+
str(root), index_cache_size_bytes=CACHE_BYTES,
|
|
84
|
+
metadata_cache_size_bytes=CACHE_BYTES,
|
|
85
|
+
read_params={"cache_repetition_index": False},
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def snapshot(dataset):
|
|
90
|
+
return (
|
|
91
|
+
[fragment.metadata.to_json() for fragment in dataset.get_fragments()],
|
|
92
|
+
dataset.list_indices(),
|
|
93
|
+
dataset.schema.serialize().to_pybytes(),
|
|
94
|
+
tag_versions(dataset),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def protected_digest(dataset):
|
|
99
|
+
fragments, indices, schema, tags = snapshot(dataset)
|
|
100
|
+
value = [fragments, indices, schema.hex(), tags]
|
|
101
|
+
encoded = json.dumps(value, sort_keys=True, default=lambda item: sorted(item) if isinstance(item, set) else str(item))
|
|
102
|
+
return hashlib.sha256(encoded.encode()).hexdigest()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def open_version(lance, root, version):
|
|
106
|
+
return lance.dataset(str(root), version=version, index_cache_size_bytes=CACHE_BYTES,
|
|
107
|
+
metadata_cache_size_bytes=CACHE_BYTES)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def prune(store, expected_version, cutoff_ms, owner_file, nonce, admission=None):
|
|
111
|
+
root = Path(store).resolve(strict=True)
|
|
112
|
+
if not root.is_dir() or root.suffix != ".lance":
|
|
113
|
+
raise ValueError("Cleanup requires a local Lance table directory")
|
|
114
|
+
owner = verify_lease(root, owner_file, nonce)
|
|
115
|
+
if not isinstance(expected_version, int) or expected_version < 1:
|
|
116
|
+
raise ValueError("Cleanup requires a positive current version")
|
|
117
|
+
if cutoff_ms > time.time_ns() // 1_000_000:
|
|
118
|
+
raise ValueError("Cleanup cutoff must not be in the future")
|
|
119
|
+
if importlib.metadata.version("pylance") != "12.0.0":
|
|
120
|
+
raise ValueError("Cleanup requires pylance 12.0.0")
|
|
121
|
+
before_bytes, before_allocated, before_free = measure(root)
|
|
122
|
+
import lance
|
|
123
|
+
dataset = open_dataset(lance, root)
|
|
124
|
+
if dataset.version != expected_version or dataset.latest_version != expected_version:
|
|
125
|
+
raise ValueError("Table changed before cleanup")
|
|
126
|
+
current = snapshot(dataset)
|
|
127
|
+
all_versions = dataset.versions()
|
|
128
|
+
if len(all_versions) > MAX_VERSIONS:
|
|
129
|
+
raise ValueError("Recovery exceeded bounded version count")
|
|
130
|
+
protected = {expected_version, *current[3].values()}
|
|
131
|
+
versions = []
|
|
132
|
+
for version in all_versions:
|
|
133
|
+
if version["timestamp"].timestamp() * 1000 >= cutoff_ms:
|
|
134
|
+
protected.add(version["version"])
|
|
135
|
+
if version["version"] not in protected and version["timestamp"].timestamp() * 1000 < cutoff_ms:
|
|
136
|
+
versions.append(version["version"])
|
|
137
|
+
if dataset.latest_version != expected_version or verify_lease(root, owner_file, nonce) != owner:
|
|
138
|
+
raise ValueError("Table or exclusive ownership changed during cleanup admission")
|
|
139
|
+
if len(protected) > MAX_PROTECTED_VERSIONS:
|
|
140
|
+
raise ValueError("Protected metadata exceeds bounded recovery version count")
|
|
141
|
+
protected_state = {version: protected_digest(open_version(lance, root, version)) for version in protected}
|
|
142
|
+
if admission is not None:
|
|
143
|
+
admission()
|
|
144
|
+
if dataset.latest_version != expected_version or verify_lease(root, owner_file, nonce) != owner:
|
|
145
|
+
raise ValueError("Table or exclusive ownership changed at deletion boundary")
|
|
146
|
+
# Explicit versions preserve current/tagged/post-cutoff commits. Unverified
|
|
147
|
+
# files stay protected. The native operation cannot rewrite data/index files.
|
|
148
|
+
stats = dataset.cleanup_old_versions(
|
|
149
|
+
versions=versions, delete_unverified=False,
|
|
150
|
+
error_if_tagged_old_versions=True, delete_rate_limit=32,
|
|
151
|
+
) if versions else None
|
|
152
|
+
fresh = open_dataset(lance, root)
|
|
153
|
+
remaining = {version["version"] for version in fresh.versions()}
|
|
154
|
+
if (fresh.version != expected_version or fresh.latest_version != expected_version
|
|
155
|
+
or snapshot(fresh) != current or not protected.issubset(remaining)
|
|
156
|
+
or set(versions).intersection(remaining)
|
|
157
|
+
or verify_lease(root, owner_file, nonce) != owner):
|
|
158
|
+
raise ValueError("Current/protected state or exclusive ownership changed; no retry")
|
|
159
|
+
if any(protected_digest(open_version(lance, root, version)) != digest
|
|
160
|
+
for version, digest in protected_state.items()):
|
|
161
|
+
raise ValueError("Protected state changed; no retry")
|
|
162
|
+
after_bytes, after_allocated, after_free = measure(root)
|
|
163
|
+
return {
|
|
164
|
+
"engine": "12.0.0", "version": expected_version,
|
|
165
|
+
"fragments": len(current[0]),
|
|
166
|
+
"bytesRemoved": stats.bytes_removed if stats else 0,
|
|
167
|
+
"versionsRemoved": stats.old_versions if stats else 0,
|
|
168
|
+
"rewritten": False,
|
|
169
|
+
"fileBytesBefore": before_bytes, "fileBytesAfter": after_bytes,
|
|
170
|
+
"allocatedBytesBefore": before_allocated, "allocatedBytesAfter": after_allocated,
|
|
171
|
+
"freeBytesBefore": before_free, "freeBytesAfter": after_free,
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def main():
|
|
176
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
177
|
+
parser.add_argument("--store", required=True)
|
|
178
|
+
parser.add_argument("--version", required=True, type=int)
|
|
179
|
+
parser.add_argument("--cutoff-ms", required=True, type=int)
|
|
180
|
+
parser.add_argument("--lease-owner", required=True)
|
|
181
|
+
parser.add_argument("--lease-nonce", required=True)
|
|
182
|
+
parser.add_argument("--require-admission", action="store_true")
|
|
183
|
+
args = parser.parse_args()
|
|
184
|
+
if not args.require_admission:
|
|
185
|
+
parser.error("Prune helper requires guarded gmax recovery admission")
|
|
186
|
+
apply_resource_limits()
|
|
187
|
+
if args.require_admission:
|
|
188
|
+
print(json.dumps({"admission": "launch"}), flush=True)
|
|
189
|
+
if sys.stdin.readline(256).strip() != args.lease_nonce:
|
|
190
|
+
raise ValueError("Launch admission unavailable; no cleanup")
|
|
191
|
+
def admission():
|
|
192
|
+
print(json.dumps({"admission": "ready"}), flush=True)
|
|
193
|
+
if sys.stdin.readline(256).strip() != args.lease_nonce:
|
|
194
|
+
raise ValueError("Deletion admission unavailable; no cleanup")
|
|
195
|
+
print(json.dumps(prune(args.store, args.version, args.cutoff_ms, args.lease_owner,
|
|
196
|
+
args.lease_nonce, admission if args.require_admission else None)))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
if __name__ == "__main__":
|
|
200
|
+
main()
|