grepmax 0.26.63 → 0.26.65

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,163 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
36
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
37
+ return new (P || (P = Promise))(function (resolve, reject) {
38
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
39
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
40
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
41
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
42
+ });
43
+ };
44
+ Object.defineProperty(exports, "__esModule", { value: true });
45
+ exports.recoverStore = recoverStore;
46
+ const fs = __importStar(require("node:fs"));
47
+ const path = __importStar(require("node:path"));
48
+ const resource_budget_1 = require("../utils/resource-budget");
49
+ const lance_cleanup_1 = require("./lance-cleanup");
50
+ const prune_state_1 = require("./prune-state");
51
+ const recovery_admission_1 = require("./recovery-admission");
52
+ const store_lease_1 = require("./store-lease");
53
+ function recoverStore(request) {
54
+ return __awaiter(this, void 0, void 0, function* () {
55
+ var _a, _b, _c, _d;
56
+ (_a = request.signal) === null || _a === void 0 ? void 0 : _a.throwIfAborted();
57
+ const input = path.resolve(request.table);
58
+ const stat = fs.lstatSync(input);
59
+ if (!stat.isDirectory() || stat.isSymbolicLink() || !input.endsWith(".lance"))
60
+ throw new Error("Recovery requires an existing local Lance table directory, without a table symlink");
61
+ const table = fs.realpathSync(input);
62
+ const store = path.dirname(table);
63
+ const previous = (0, prune_state_1.readPruneState)(store);
64
+ if (!request.prune && !request.check)
65
+ return {
66
+ outcome: "status",
67
+ table,
68
+ state: previous,
69
+ fullTableRewriting: "disabled",
70
+ };
71
+ if (request.prune &&
72
+ (!Number.isSafeInteger(request.version) ||
73
+ request.version < 1 ||
74
+ !request.cutoff ||
75
+ !Number.isSafeInteger(request.cutoff.getTime()) ||
76
+ request.cutoff.getTime() > Date.now()))
77
+ throw new Error("Explicit prune requires a positive --version and a real, nonfuture --cutoff");
78
+ if (request.prune &&
79
+ (previous === null || previous === void 0 ? void 0 : previous.outcome) === "uncertain" &&
80
+ request.acknowledgeUncertain !== previous.attemptId)
81
+ throw new Error(`Previous prune completion is uncertain; inspect retained state, then use --acknowledge-uncertain ${previous.attemptId} for a verified explicit retry`);
82
+ const budget = (0, recovery_admission_1.createRecoveryBudget)();
83
+ (0, recovery_admission_1.assertRecoveryAdmission)(store, budget);
84
+ if (!request.prune) {
85
+ const admission = (0, recovery_admission_1.admitPrune)(store, budget);
86
+ try {
87
+ return {
88
+ outcome: "preflight-passed",
89
+ table,
90
+ state: previous,
91
+ host: admission.snapshot,
92
+ helperReserveMb: resource_budget_1.HELPER_RESOURCE_RESERVE_MB,
93
+ fullTableRewriting: "disabled",
94
+ };
95
+ }
96
+ finally {
97
+ admission.close();
98
+ }
99
+ }
100
+ // Setup is admitted and monitored before acquiring store exclusion. No owner
101
+ // is killed and no marker is created or cleared by recovery itself.
102
+ const controller = new AbortController();
103
+ const abort = () => controller.abort();
104
+ (_b = request.signal) === null || _b === void 0 ? void 0 : _b.addEventListener("abort", abort, { once: true });
105
+ if ((_c = request.signal) === null || _c === void 0 ? void 0 : _c.aborted)
106
+ controller.abort();
107
+ let setup;
108
+ let monitor;
109
+ let lease;
110
+ try {
111
+ setup = budget.reserve(resource_budget_1.HELPER_RESOURCE_RESERVE_MB, "prune-runtime-setup");
112
+ monitor = setInterval(() => {
113
+ try {
114
+ (0, recovery_admission_1.assertRecoveryAdmission)(store, budget);
115
+ }
116
+ catch (_a) {
117
+ controller.abort();
118
+ }
119
+ }, 5000);
120
+ const runtime = yield (0, lance_cleanup_1.prepareCleanupRuntime)(controller.signal);
121
+ clearInterval(monitor);
122
+ setup.release();
123
+ (0, recovery_admission_1.assertRecoveryAdmission)(store, budget);
124
+ lease = yield store_lease_1.StoreLease.acquireExclusive({
125
+ storeDir: store,
126
+ timeoutMs: 1000,
127
+ signal: controller.signal,
128
+ role: "explicit-prune-recovery",
129
+ });
130
+ // Another completed/interrupted attempt may have happened during setup.
131
+ const current = (0, prune_state_1.readPruneState)(store);
132
+ if ((current === null || current === void 0 ? void 0 : current.outcome) === "uncertain" &&
133
+ request.acknowledgeUncertain !== current.attemptId)
134
+ throw new Error(`Prune completion is uncertain; inspect and acknowledge attempt ${current.attemptId} before retry`);
135
+ (0, recovery_admission_1.assertRecoveryAdmission)(store, budget);
136
+ const result = yield (0, lance_cleanup_1.pruneVersions)(runtime, table, request.version, request.cutoff, {
137
+ lease,
138
+ signal: controller.signal,
139
+ acknowledgeUncertain: request.acknowledgeUncertain,
140
+ });
141
+ return {
142
+ outcome: "verified",
143
+ table,
144
+ state: (0, prune_state_1.readPruneState)(store),
145
+ result,
146
+ allocatedBytesRecovered: result.allocatedBytesBefore - result.allocatedBytesAfter,
147
+ filesystemFreeBytesChange: result.freeBytesAfter - result.freeBytesBefore,
148
+ fullTableRewriting: "disabled",
149
+ };
150
+ }
151
+ catch (error) {
152
+ if (error instanceof store_lease_1.StoreLeaseTimeoutError)
153
+ throw new Error(`Recovery refused: store owners are live or unknown (${error.blockers.map((owner) => `${owner.role} pid=${owner.pid}`).join(", ") || "unverified owner"}); close them separately`);
154
+ throw error;
155
+ }
156
+ finally {
157
+ clearInterval(monitor);
158
+ (_d = request.signal) === null || _d === void 0 ? void 0 : _d.removeEventListener("abort", abort);
159
+ setup === null || setup === void 0 ? void 0 : setup.release();
160
+ yield lease === null || lease === void 0 ? void 0 : lease.release();
161
+ }
162
+ });
163
+ }
@@ -108,6 +108,14 @@ const defaultProbeDeps = {
108
108
  }).trim(),
109
109
  };
110
110
  function defaultProbeOwner(owner, deps = defaultProbeDeps) {
111
+ const parent = probeProcessOwner(owner, deps);
112
+ if ((parent === "dead" || parent === "reused") && owner.activeHelper) {
113
+ const child = probeProcessOwner(Object.assign(Object.assign({}, owner), { pid: owner.activeHelper.pid, processStart: owner.activeHelper.processStart }), deps);
114
+ return child === "same" || child === "unknown" ? child : parent;
115
+ }
116
+ return parent;
117
+ }
118
+ function probeProcessOwner(owner, deps) {
111
119
  try {
112
120
  deps.signal(owner.pid);
113
121
  }
@@ -124,6 +132,8 @@ function defaultProbeOwner(owner, deps = defaultProbeDeps) {
124
132
  if (code !== "EPERM")
125
133
  return "unknown";
126
134
  }
135
+ if (owner.processStart === "unverified")
136
+ return "unknown";
127
137
  if (owner.processStart.startsWith(`pid:${owner.pid}:started:`))
128
138
  return "same";
129
139
  try {
@@ -138,6 +148,7 @@ function defaultProbeOwner(owner, deps = defaultProbeDeps) {
138
148
  }
139
149
  }
140
150
  function readOwner(filePath) {
151
+ var _a;
141
152
  try {
142
153
  const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
143
154
  if (!parsed || typeof parsed !== "object")
@@ -147,18 +158,40 @@ function readOwner(filePath) {
147
158
  typeof owner.processStart === "string" &&
148
159
  typeof owner.nonce === "string" &&
149
160
  typeof owner.role === "string" &&
150
- typeof owner.acquiredAt === "number"
161
+ typeof owner.acquiredAt === "number" &&
162
+ (owner.activeHelper === undefined ||
163
+ (Number.isSafeInteger((_a = owner.activeHelper) === null || _a === void 0 ? void 0 : _a.pid) &&
164
+ owner.activeHelper.pid > 0 &&
165
+ typeof owner.activeHelper.processStart === "string" &&
166
+ !!owner.activeHelper.processStart))
151
167
  ? owner
152
168
  : null;
153
169
  }
154
- catch (_a) {
170
+ catch (_b) {
155
171
  return null;
156
172
  }
157
173
  }
158
- function writeOwnerAtomic(filePath, owner) {
174
+ function writeOwnerAtomic(filePath, owner, durable = false) {
159
175
  const temp = `${filePath}.${owner.nonce}.tmp`;
160
- fs.writeFileSync(temp, `${JSON.stringify(owner)}\n`, { flag: "wx" });
176
+ const fd = fs.openSync(temp, "wx", 0o600);
177
+ try {
178
+ fs.writeFileSync(fd, `${JSON.stringify(owner)}\n`);
179
+ if (durable)
180
+ fs.fsyncSync(fd);
181
+ }
182
+ finally {
183
+ fs.closeSync(fd);
184
+ }
161
185
  fs.renameSync(temp, filePath);
186
+ if (durable) {
187
+ const dir = fs.openSync(path.dirname(filePath), "r");
188
+ try {
189
+ fs.fsyncSync(dir);
190
+ }
191
+ finally {
192
+ fs.closeSync(dir);
193
+ }
194
+ }
162
195
  }
163
196
  function delay(ms, signal) {
164
197
  return __awaiter(this, void 0, void 0, function* () {
@@ -189,6 +222,7 @@ class StoreLease {
189
222
  this.storeDir = storeDir;
190
223
  this.released = false;
191
224
  this.claimedBy = null;
225
+ this.exclusiveUseRunning = false;
192
226
  }
193
227
  static acquireShared(options) {
194
228
  return __awaiter(this, void 0, void 0, function* () {
@@ -269,6 +303,9 @@ class StoreLease {
269
303
  }
270
304
  release() {
271
305
  return __awaiter(this, arguments, void 0, function* (options = {}) {
306
+ if (this.exclusiveUseRunning) {
307
+ throw new Error("Exclusive store operation must finish before releasing its lease");
308
+ }
272
309
  if (this.released)
273
310
  return;
274
311
  if (this.mode === "shared") {
@@ -306,7 +343,82 @@ class StoreLease {
306
343
  if (this.claimedBy === holder)
307
344
  this.claimedBy = null;
308
345
  }
346
+ /** Hold verified cross-process exclusion through a destructive helper and its
347
+ * verification. A local write gate is not a substitute for this ownership. */
348
+ withExclusiveUse(storeDir, operation) {
349
+ return __awaiter(this, void 0, void 0, function* () {
350
+ if (this.exclusiveUseRunning) {
351
+ throw new Error("An exclusive store operation is already running");
352
+ }
353
+ const verify = () => {
354
+ if (this.mode !== "exclusive" || this.released) {
355
+ throw new Error("Pruning requires an unreleased exclusive store lease");
356
+ }
357
+ if (fs.realpathSync(storeDir) !== fs.realpathSync(this.storeDir)) {
358
+ throw new Error("Exclusive store lease belongs to a different store");
359
+ }
360
+ const ownerStat = fs.lstatSync(this.paths.intentOwnerFile);
361
+ if (!ownerStat.isFile() || ownerStat.size > 16 * 1024) {
362
+ throw new Error("Exclusive store lease owner is unverified or oversized");
363
+ }
364
+ const current = readOwner(this.paths.intentOwnerFile);
365
+ if ((current === null || current === void 0 ? void 0 : current.nonce) !== this.owner.nonce ||
366
+ current.pid !== this.owner.pid ||
367
+ current.processStart !== this.owner.processStart) {
368
+ throw new Error("Exclusive store lease ownership could not be verified");
369
+ }
370
+ // acquireExclusive has already removed verified stale readers. Refuse any
371
+ // remaining marker rather than assuming ignored/unknown readers are safe.
372
+ if (fs
373
+ .readdirSync(this.paths.readersDir)
374
+ .some((name) => name.endsWith(".json"))) {
375
+ throw new Error("Pruning requires all shared store owners to close");
376
+ }
377
+ };
378
+ verify();
379
+ this.exclusiveUseRunning = true;
380
+ try {
381
+ const result = yield operation();
382
+ verify();
383
+ return result;
384
+ }
385
+ finally {
386
+ this.exclusiveUseRunning = false;
387
+ }
388
+ });
389
+ }
309
390
  /** Atomically replace this shared lease with an exclusive lease. */
391
+ pinExclusiveHelper(pid) {
392
+ if (!this.exclusiveUseRunning ||
393
+ this.owner.activeHelper ||
394
+ !Number.isSafeInteger(pid) ||
395
+ pid < 1)
396
+ throw new Error("Helper requires an active exclusive store operation");
397
+ const current = readOwner(this.paths.intentOwnerFile);
398
+ if ((current === null || current === void 0 ? void 0 : current.nonce) !== this.owner.nonce || current.activeHelper)
399
+ throw new Error("Exclusive store ownership changed before helper launch");
400
+ // Publish uncertainty first. A failed identity probe must retain exclusion
401
+ // until this child is confirmed dead, even if the parent is killed.
402
+ this.owner.activeHelper = { pid, processStart: "unverified" };
403
+ // The helper is the primary on-disk owner, so older readers also retain
404
+ // exclusion after parent death. It is blocked on stdin until this identity
405
+ // is durably verified; uncertainty here cannot admit native deletion.
406
+ writeOwnerAtomic(this.paths.intentOwnerFile, Object.assign(Object.assign({}, this.owner), { pid, processStart: "unverified" }), true);
407
+ const start = defaultProbeDeps.processStartOf(pid);
408
+ if (!start)
409
+ throw new Error("Prune helper identity unavailable");
410
+ this.owner.activeHelper.processStart = start;
411
+ writeOwnerAtomic(this.paths.intentOwnerFile, Object.assign(Object.assign({}, this.owner), { pid, processStart: start }), true);
412
+ return () => {
413
+ var _a;
414
+ const latest = readOwner(this.paths.intentOwnerFile);
415
+ if ((latest === null || latest === void 0 ? void 0 : latest.nonce) !== this.owner.nonce ||
416
+ ((_a = latest.activeHelper) === null || _a === void 0 ? void 0 : _a.pid) !== pid)
417
+ throw new Error("Exclusive store ownership changed during helper close");
418
+ delete this.owner.activeHelper;
419
+ writeOwnerAtomic(this.paths.intentOwnerFile, this.owner, true);
420
+ };
421
+ }
310
422
  upgrade() {
311
423
  return __awaiter(this, arguments, void 0, function* (options = {}) {
312
424
  if (this.mode !== "shared") {
@@ -328,6 +440,9 @@ class StoreLease {
328
440
  /** Replace this exclusive lease with a shared lease without an ownership gap. */
329
441
  downgrade() {
330
442
  return __awaiter(this, arguments, void 0, function* (options = {}) {
443
+ if (this.exclusiveUseRunning) {
444
+ throw new Error("Exclusive store operation must finish before downgrading its lease");
445
+ }
331
446
  if (this.mode !== "exclusive") {
332
447
  throw new Error("Only an exclusive store lease can be downgraded");
333
448
  }
@@ -132,7 +132,7 @@ function parseBatchFootprints(output) {
132
132
  function sampleHostResources(extraPids = [], overrides = {}, extraGroups = []) {
133
133
  var _a;
134
134
  var _b, _c;
135
- const deps = Object.assign({ platform: process.platform, pid: process.pid, uid: (_b = (_a = process.getuid) === null || _a === void 0 ? void 0 : _a.call(process)) !== null && _b !== void 0 ? _b : -1, now: Date.now, monotonic: () => performance.now(), freeBytes: os.freemem, run: (command, args, timeoutMs) => (0, node_child_process_1.execFileSync)(command, args, {
135
+ const deps = Object.assign({ platform: process.platform, pid: process.pid, uid: (_b = (_a = process.getuid) === null || _a === void 0 ? void 0 : _a.call(process)) !== null && _b !== void 0 ? _b : -1, now: Date.now, monotonic: () => performance.now(), freeBytes: os.freemem, sampleTimeoutMs: exports.HOST_SAMPLE_TIMEOUT_MS, kernelProbeTimeoutMs: 500, run: (command, args, timeoutMs) => (0, node_child_process_1.execFileSync)(command, args, {
136
136
  encoding: "utf8",
137
137
  timeout: timeoutMs,
138
138
  maxBuffer: 512 * 1024,
@@ -153,7 +153,7 @@ function sampleHostResources(extraPids = [], overrides = {}, extraGroups = []) {
153
153
  };
154
154
  if (deps.platform !== "darwin")
155
155
  return snapshot;
156
- const deadline = deps.monotonic() + exports.HOST_SAMPLE_TIMEOUT_MS;
156
+ const deadline = deps.monotonic() + deps.sampleTimeoutMs;
157
157
  const run = (command, args, maximumMs = 500) => {
158
158
  const remaining = Math.floor(deadline - deps.monotonic());
159
159
  if (remaining <= 0)
@@ -167,7 +167,7 @@ function sampleHostResources(extraPids = [], overrides = {}, extraGroups = []) {
167
167
  }
168
168
  catch (_d) { }
169
169
  try {
170
- const zone = (0, kernel_zone_1.parseZprintOutput)(run("zprint", ["-L", "data.kalloc.1024"]));
170
+ const zone = (0, kernel_zone_1.parseZprintOutput)(run("zprint", ["-L", "data.kalloc.1024"], deps.kernelProbeTimeoutMs));
171
171
  if (zone) {
172
172
  const bytes = zone.elements * zone.elementSize;
173
173
  if (Number.isSafeInteger(bytes)) {
@@ -75,7 +75,7 @@ function resolveResourceBudgetMb(raw = process.env.GMAX_RESOURCE_BUDGET_MB) {
75
75
  throw new ResourceAdmissionError("resource budget must be 256..6144 MiB");
76
76
  return n;
77
77
  }
78
- function assertResourceSnapshot(snapshot, chargedMb, budgetMb, now = Date.now()) {
78
+ function assertResourceSnapshot(snapshot, chargedMb, budgetMb, now = Date.now(), sampleMaxAgeMs = 5000) {
79
79
  if (snapshot.memoryPressure === "critical" ||
80
80
  snapshot.kernelPressure === "critical")
81
81
  throw new ResourceAdmissionError("critical host pressure; heavy work refused", true);
@@ -84,7 +84,7 @@ function assertResourceSnapshot(snapshot, chargedMb, budgetMb, now = Date.now())
84
84
  if (snapshot.platform !== "darwin")
85
85
  return;
86
86
  if (now < snapshot.at ||
87
- now - snapshot.at > 5000 ||
87
+ now - snapshot.at > sampleMaxAgeMs ||
88
88
  snapshot.completedAt < snapshot.at ||
89
89
  snapshot.completedAt > now)
90
90
  throw new ResourceAdmissionError("resource sample stale or clock changed; heavy work paused");
@@ -107,8 +107,8 @@ class ResourceBudget {
107
107
  constructor(overrides = {}) {
108
108
  this.deps = Object.assign({ root: path.join(config_1.PATHS.sharedRoot, "resource-budget"), platform: process.platform, pid: process.pid, now: Date.now, sample: (extraPids = []) => {
109
109
  const records = this.readRecords();
110
- return (0, host_resource_1.sampleHostResources)([...records.map((r) => r.pid), ...extraPids], {}, records.flatMap((r) => (r.groupPid ? [r.groupPid] : [])));
111
- }, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: safety_latch_1.latchSafetyStop, processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
110
+ return (0, host_resource_1.sampleHostResources)([...records.map((r) => r.pid), ...extraPids], this.deps.samplerOverrides, records.flatMap((r) => (r.groupPid ? [r.groupPid] : [])));
111
+ }, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, requireClientRegistration: true, samplerOverrides: {}, sampleMaxAgeMs: 5000, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: safety_latch_1.latchSafetyStop, processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
112
112
  encoding: "utf8",
113
113
  timeout: 500,
114
114
  maxBuffer: 1024,
@@ -250,10 +250,10 @@ class ResourceBudget {
250
250
  }
251
251
  checked(snapshot, records) {
252
252
  try {
253
- assertResourceSnapshot(snapshot, this.charge(snapshot, records), resolveResourceBudgetMb(), this.deps.now());
253
+ assertResourceSnapshot(snapshot, this.charge(snapshot, records), resolveResourceBudgetMb(), this.deps.now(), this.deps.sampleMaxAgeMs);
254
254
  const legacy = snapshot.processes.filter((p) => p.role === "mcp" &&
255
255
  !records.some((r) => r.pid === p.pid && r.kind === "client"));
256
- if (legacy.length)
256
+ if (legacy.length && this.deps.requireClientRegistration)
257
257
  throw new ResourceAdmissionError(`${legacy.length} MCP client(s) need to reconnect after upgrading; heavy work paused`);
258
258
  }
259
259
  catch (error) {
@@ -0,0 +1,200 @@
1
+ """Explicit, exclusively leased Lance 12 prune-only recovery candidate.
2
+
3
+ Never compacts, writes rows, builds indexes or starts a server. This helper is
4
+ not an automatic maintenance path. Native production resource validation is
5
+ required before enabling it on a pressured host.
6
+ """
7
+
8
+ import argparse
9
+ import importlib.metadata
10
+ import hashlib
11
+ import json
12
+ import os
13
+ from pathlib import Path
14
+ import sys
15
+ import time
16
+
17
+ MAX_FILES = 100_000
18
+ MAX_VERSIONS = 100_000
19
+ MAX_PROTECTED_VERSIONS = 256
20
+ CACHE_BYTES = 8 * 1024 * 1024
21
+
22
+
23
+ def apply_resource_limits():
24
+ # Must run before importing native libraries. Cache limits are not total
25
+ # memory limits. RLIMIT_RSS is not enforceable on macOS; do not claim it is.
26
+ for name in ("RAYON_NUM_THREADS", "TOKIO_WORKER_THREADS", "OMP_NUM_THREADS",
27
+ "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"):
28
+ os.environ[name] = "1"
29
+ import resource
30
+ resource.setrlimit(resource.RLIMIT_CPU, (90, 91))
31
+ resource.setrlimit(resource.RLIMIT_NOFILE, (256, 256))
32
+ if sys.platform.startswith("linux"):
33
+ resource.setrlimit(resource.RLIMIT_AS, (2 * 1024**3, 2 * 1024**3))
34
+
35
+
36
+ def verify_lease(root, owner_file, nonce):
37
+ expected = Path(str(root.parent) + ".lease") / "exclusive-intent" / "owner.json"
38
+ if Path(owner_file).resolve(strict=True) != expected.resolve(strict=True):
39
+ raise ValueError("Exclusive lease belongs to a different store")
40
+ if expected.is_symlink() or expected.stat().st_size > 16 * 1024:
41
+ raise ValueError("Exclusive lease owner is unverified or oversized")
42
+ owner = json.loads(expected.read_text())
43
+ helper = owner.get("activeHelper")
44
+ expected_pid = os.getpid() if helper and helper.get("pid") == os.getpid() else os.getppid()
45
+ if not nonce or owner.get("nonce") != nonce or owner.get("pid") != expected_pid:
46
+ raise ValueError("Exclusive lease ownership could not be verified")
47
+ readers = expected.parent.parent / "readers"
48
+ if not readers.is_dir() or any(readers.glob("*.json")):
49
+ raise ValueError("All shared store owners must close before pruning")
50
+ return owner
51
+
52
+
53
+ def measure(root):
54
+ total = allocated = count = 0
55
+ pending = [root]
56
+ while pending:
57
+ directory = pending.pop()
58
+ with os.scandir(directory) as entries:
59
+ for entry in entries:
60
+ count += 1
61
+ if count > MAX_FILES:
62
+ raise ValueError("Recovery metadata scan exceeded bounded file count")
63
+ if entry.is_symlink():
64
+ raise ValueError("Recovery refuses symlinks inside the table")
65
+ if entry.is_dir(follow_symlinks=False):
66
+ pending.append(Path(entry.path))
67
+ elif entry.is_file(follow_symlinks=False):
68
+ stat = entry.stat(follow_symlinks=False)
69
+ total += stat.st_size
70
+ allocated += stat.st_blocks * 512
71
+ else:
72
+ raise ValueError("Recovery refuses nonregular table entries")
73
+ space = os.statvfs(root)
74
+ return total, allocated, space.f_bavail * space.f_frsize
75
+
76
+
77
+ def tag_versions(dataset):
78
+ return {name: dataset.tags.get_version(name) for name in dataset.tags.list()}
79
+
80
+
81
+ def open_dataset(lance, root):
82
+ return lance.dataset(
83
+ str(root), index_cache_size_bytes=CACHE_BYTES,
84
+ metadata_cache_size_bytes=CACHE_BYTES,
85
+ read_params={"cache_repetition_index": False},
86
+ )
87
+
88
+
89
+ def snapshot(dataset):
90
+ return (
91
+ [fragment.metadata.to_json() for fragment in dataset.get_fragments()],
92
+ dataset.list_indices(),
93
+ dataset.schema.serialize().to_pybytes(),
94
+ tag_versions(dataset),
95
+ )
96
+
97
+
98
+ def protected_digest(dataset):
99
+ fragments, indices, schema, tags = snapshot(dataset)
100
+ value = [fragments, indices, schema.hex(), tags]
101
+ encoded = json.dumps(value, sort_keys=True, default=lambda item: sorted(item) if isinstance(item, set) else str(item))
102
+ return hashlib.sha256(encoded.encode()).hexdigest()
103
+
104
+
105
+ def open_version(lance, root, version):
106
+ return lance.dataset(str(root), version=version, index_cache_size_bytes=CACHE_BYTES,
107
+ metadata_cache_size_bytes=CACHE_BYTES)
108
+
109
+
110
+ def prune(store, expected_version, cutoff_ms, owner_file, nonce, admission=None):
111
+ root = Path(store).resolve(strict=True)
112
+ if not root.is_dir() or root.suffix != ".lance":
113
+ raise ValueError("Cleanup requires a local Lance table directory")
114
+ owner = verify_lease(root, owner_file, nonce)
115
+ if not isinstance(expected_version, int) or expected_version < 1:
116
+ raise ValueError("Cleanup requires a positive current version")
117
+ if cutoff_ms > time.time_ns() // 1_000_000:
118
+ raise ValueError("Cleanup cutoff must not be in the future")
119
+ if importlib.metadata.version("pylance") != "12.0.0":
120
+ raise ValueError("Cleanup requires pylance 12.0.0")
121
+ before_bytes, before_allocated, before_free = measure(root)
122
+ import lance
123
+ dataset = open_dataset(lance, root)
124
+ if dataset.version != expected_version or dataset.latest_version != expected_version:
125
+ raise ValueError("Table changed before cleanup")
126
+ current = snapshot(dataset)
127
+ all_versions = dataset.versions()
128
+ if len(all_versions) > MAX_VERSIONS:
129
+ raise ValueError("Recovery exceeded bounded version count")
130
+ protected = {expected_version, *current[3].values()}
131
+ versions = []
132
+ for version in all_versions:
133
+ if version["timestamp"].timestamp() * 1000 >= cutoff_ms:
134
+ protected.add(version["version"])
135
+ if version["version"] not in protected and version["timestamp"].timestamp() * 1000 < cutoff_ms:
136
+ versions.append(version["version"])
137
+ if dataset.latest_version != expected_version or verify_lease(root, owner_file, nonce) != owner:
138
+ raise ValueError("Table or exclusive ownership changed during cleanup admission")
139
+ if len(protected) > MAX_PROTECTED_VERSIONS:
140
+ raise ValueError("Protected metadata exceeds bounded recovery version count")
141
+ protected_state = {version: protected_digest(open_version(lance, root, version)) for version in protected}
142
+ if admission is not None:
143
+ admission()
144
+ if dataset.latest_version != expected_version or verify_lease(root, owner_file, nonce) != owner:
145
+ raise ValueError("Table or exclusive ownership changed at deletion boundary")
146
+ # Explicit versions preserve current/tagged/post-cutoff commits. Unverified
147
+ # files stay protected. The native operation cannot rewrite data/index files.
148
+ stats = dataset.cleanup_old_versions(
149
+ versions=versions, delete_unverified=False,
150
+ error_if_tagged_old_versions=True, delete_rate_limit=32,
151
+ ) if versions else None
152
+ fresh = open_dataset(lance, root)
153
+ remaining = {version["version"] for version in fresh.versions()}
154
+ if (fresh.version != expected_version or fresh.latest_version != expected_version
155
+ or snapshot(fresh) != current or not protected.issubset(remaining)
156
+ or set(versions).intersection(remaining)
157
+ or verify_lease(root, owner_file, nonce) != owner):
158
+ raise ValueError("Current/protected state or exclusive ownership changed; no retry")
159
+ if any(protected_digest(open_version(lance, root, version)) != digest
160
+ for version, digest in protected_state.items()):
161
+ raise ValueError("Protected state changed; no retry")
162
+ after_bytes, after_allocated, after_free = measure(root)
163
+ return {
164
+ "engine": "12.0.0", "version": expected_version,
165
+ "fragments": len(current[0]),
166
+ "bytesRemoved": stats.bytes_removed if stats else 0,
167
+ "versionsRemoved": stats.old_versions if stats else 0,
168
+ "rewritten": False,
169
+ "fileBytesBefore": before_bytes, "fileBytesAfter": after_bytes,
170
+ "allocatedBytesBefore": before_allocated, "allocatedBytesAfter": after_allocated,
171
+ "freeBytesBefore": before_free, "freeBytesAfter": after_free,
172
+ }
173
+
174
+
175
+ def main():
176
+ parser = argparse.ArgumentParser(description=__doc__)
177
+ parser.add_argument("--store", required=True)
178
+ parser.add_argument("--version", required=True, type=int)
179
+ parser.add_argument("--cutoff-ms", required=True, type=int)
180
+ parser.add_argument("--lease-owner", required=True)
181
+ parser.add_argument("--lease-nonce", required=True)
182
+ parser.add_argument("--require-admission", action="store_true")
183
+ args = parser.parse_args()
184
+ if not args.require_admission:
185
+ parser.error("Prune helper requires guarded gmax recovery admission")
186
+ apply_resource_limits()
187
+ if args.require_admission:
188
+ print(json.dumps({"admission": "launch"}), flush=True)
189
+ if sys.stdin.readline(256).strip() != args.lease_nonce:
190
+ raise ValueError("Launch admission unavailable; no cleanup")
191
+ def admission():
192
+ print(json.dumps({"admission": "ready"}), flush=True)
193
+ if sys.stdin.readline(256).strip() != args.lease_nonce:
194
+ raise ValueError("Deletion admission unavailable; no cleanup")
195
+ print(json.dumps(prune(args.store, args.version, args.cutoff_ms, args.lease_owner,
196
+ args.lease_nonce, admission if args.require_admission else None)))
197
+
198
+
199
+ if __name__ == "__main__":
200
+ main()
@@ -0,0 +1,9 @@
1
+ [project]
2
+ name = "gmax-lance-maintenance"
3
+ version = "0.1.0"
4
+ description = "Prune-only maintenance for grepmax's pinned Lance 12 store"
5
+ requires-python = ">=3.10"
6
+ dependencies = ["pylance==12.0.0"]
7
+
8
+ [tool.uv]
9
+ package = false