@ai-dossier/sched 0.70.0 → 0.70.1-next.730.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -4
- package/dist/engine-alert.d.ts.map +1 -1
- package/dist/engine-alert.js +13 -3
- package/dist/engine-alert.js.map +1 -1
- package/dist/enqueue.d.ts.map +1 -1
- package/dist/enqueue.js +11 -9
- package/dist/enqueue.js.map +1 -1
- package/dist/persist.d.ts +13 -7
- package/dist/persist.d.ts.map +1 -1
- package/dist/persist.js +335 -94
- package/dist/persist.js.map +1 -1
- package/dist/readiness.d.ts.map +1 -1
- package/dist/readiness.js +7 -6
- package/dist/readiness.js.map +1 -1
- package/dist/state.d.ts +8 -41
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +175 -107
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +4 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +3 -3
package/dist/persist.js
CHANGED
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Crash-safe persistence for the scheduler core.
|
|
4
4
|
*
|
|
5
|
-
* `state.json` is written atomically: contents land in
|
|
5
|
+
* `state.json` is written atomically: contents land in an exclusive temporary file in the
|
|
6
6
|
* SAME directory (rename is atomic only within a filesystem), get fsynced, and
|
|
7
7
|
* are renamed over `state.json`. A process killed between writes therefore
|
|
8
8
|
* leaves either the previous complete state or the new one — never a partial
|
|
9
|
-
* file (AC2).
|
|
10
|
-
*
|
|
9
|
+
* file (AC2). Stray temporary files are ignored; publication never opens an
|
|
10
|
+
* existing temporary pathname that could alias evidence or block on a FIFO.
|
|
11
11
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
12
|
+
* Reads/recovery and mutations share `.sched-lock`. Linux additionally pins a
|
|
13
|
+
* permanent kernel guard, so stale-owner observation and reclamation cannot race
|
|
14
|
+
* a new owner. Other platforms fail closed on stale locks (manual reconciliation).
|
|
15
15
|
*/
|
|
16
16
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
17
17
|
if (k2 === undefined) k2 = k;
|
|
@@ -46,12 +46,17 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
46
46
|
return result;
|
|
47
47
|
};
|
|
48
48
|
})();
|
|
49
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
50
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
51
|
+
};
|
|
49
52
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
50
53
|
exports.SchedStore = exports.CorruptStateError = exports.LockTimeoutError = void 0;
|
|
51
54
|
exports.releaseOwnStateLock = releaseOwnStateLock;
|
|
52
55
|
exports.writeAtomic = writeAtomic;
|
|
53
56
|
exports.isEngineLeaseRace = isEngineLeaseRace;
|
|
54
|
-
const
|
|
57
|
+
const node_child_process_1 = require("node:child_process");
|
|
58
|
+
const node_crypto_1 = require("node:crypto");
|
|
59
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
55
60
|
const path = __importStar(require("node:path"));
|
|
56
61
|
const dispatch_1 = require("./dispatch");
|
|
57
62
|
const journal_1 = require("./journal");
|
|
@@ -67,10 +72,12 @@ const ENGINE_LEASE_HOLDER_FILE = 'holder.json';
|
|
|
67
72
|
/** Thrown when the cross-process lock cannot be acquired in time. */
|
|
68
73
|
class LockTimeoutError extends Error {
|
|
69
74
|
lockPath;
|
|
70
|
-
constructor(lockPath, heldByPid) {
|
|
75
|
+
constructor(lockPath, heldByPid, permanent = false) {
|
|
71
76
|
const holder = heldByPid !== null ? `; held by pid ${heldByPid}` : ' (no pid file found)';
|
|
72
|
-
super(
|
|
73
|
-
`
|
|
77
|
+
super(permanent
|
|
78
|
+
? `Timed out waiting for scheduler kernel guard (${LOCK_TIMEOUT_MS}ms): ${lockPath}. Retry or inspect the state-lock holder; never unlink or replace this permanent guard file.`
|
|
79
|
+
: `Timed out waiting for scheduler lock (${LOCK_TIMEOUT_MS}ms)${holder}. ` +
|
|
80
|
+
`If that process is not running, remove ${lockPath}`);
|
|
74
81
|
this.name = 'LockTimeoutError';
|
|
75
82
|
this.lockPath = lockPath;
|
|
76
83
|
}
|
|
@@ -81,7 +88,8 @@ class CorruptStateError extends Error {
|
|
|
81
88
|
constructor(filePath, cause) {
|
|
82
89
|
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
83
90
|
super(`Scheduler state file is corrupt: ${filePath} (${reason}). ` +
|
|
84
|
-
'
|
|
91
|
+
'Preserve the original file as evidence; repair only the named record/field, then retry. ' +
|
|
92
|
+
'For invalid JSON repair the JSON syntax; for unsupported schemas use a compatible engine; for I/O failures restore file access. Do not reset the queue.');
|
|
85
93
|
this.name = 'CorruptStateError';
|
|
86
94
|
this.filePath = filePath;
|
|
87
95
|
}
|
|
@@ -99,10 +107,10 @@ exports.CorruptStateError = CorruptStateError;
|
|
|
99
107
|
function releaseOwnStateLock(dir) {
|
|
100
108
|
const lockPath = path.join(dir, LOCK_DIR);
|
|
101
109
|
try {
|
|
102
|
-
const pid = Number.parseInt(
|
|
110
|
+
const pid = Number.parseInt(node_fs_1.default.readFileSync(path.join(lockPath, 'pid'), 'utf-8').trim(), 10);
|
|
103
111
|
if (pid !== process.pid)
|
|
104
112
|
return false;
|
|
105
|
-
|
|
113
|
+
node_fs_1.default.rmSync(lockPath, { recursive: true, force: true });
|
|
106
114
|
return true;
|
|
107
115
|
}
|
|
108
116
|
catch {
|
|
@@ -112,93 +120,276 @@ function releaseOwnStateLock(dir) {
|
|
|
112
120
|
/** Write `contents` to `filePath` atomically (tmp + fsync + rename, same directory). */
|
|
113
121
|
function writeAtomic(filePath, contents) {
|
|
114
122
|
const dir = path.dirname(filePath);
|
|
115
|
-
|
|
116
|
-
const tmp =
|
|
117
|
-
const fd =
|
|
123
|
+
node_fs_1.default.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
124
|
+
const tmp = path.join(dir, `.state-write-${(0, node_crypto_1.randomUUID)()}`);
|
|
125
|
+
const fd = node_fs_1.default.openSync(tmp, node_fs_1.default.constants.O_WRONLY | node_fs_1.default.constants.O_CREAT | node_fs_1.default.constants.O_EXCL | node_fs_1.default.constants.O_NOFOLLOW, 0o600);
|
|
118
126
|
try {
|
|
119
|
-
|
|
120
|
-
|
|
127
|
+
try {
|
|
128
|
+
node_fs_1.default.writeFileSync(fd, contents, 'utf8');
|
|
129
|
+
node_fs_1.default.fsyncSync(fd);
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
node_fs_1.default.closeSync(fd);
|
|
133
|
+
}
|
|
134
|
+
node_fs_1.default.renameSync(tmp, filePath);
|
|
121
135
|
}
|
|
122
136
|
finally {
|
|
123
|
-
|
|
137
|
+
node_fs_1.default.rmSync(tmp, { force: true });
|
|
124
138
|
}
|
|
125
|
-
fs.renameSync(tmp, filePath);
|
|
126
139
|
// Best-effort directory fsync so the rename itself is durable. Not available
|
|
127
140
|
// on every platform/fs — a failure here never blocks the write.
|
|
128
141
|
try {
|
|
129
|
-
|
|
142
|
+
fsyncDirectory(dir);
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
// ignore
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const QUARANTINE_DIR = 'quarantine';
|
|
149
|
+
const QUARANTINE_MARKER = 'terminal-quarantine:';
|
|
150
|
+
/** Build an entirely validated inert candidate BEFORE touching any evidence/state files. */
|
|
151
|
+
function isolateTerminalBatches(data, evidence) {
|
|
152
|
+
if (!data || typeof data !== 'object')
|
|
153
|
+
throw new Error('Scheduler state must be an object');
|
|
154
|
+
const obj = data;
|
|
155
|
+
if (!Array.isArray(obj.batches) || !Array.isArray(obj.entries) || !Array.isArray(obj.slots)) {
|
|
156
|
+
throw new Error('Scheduler state must have batches, entries and slots arrays');
|
|
157
|
+
}
|
|
158
|
+
const ids = new Set();
|
|
159
|
+
const records = [];
|
|
160
|
+
const batches = obj.batches.map((batch, index) => {
|
|
161
|
+
// Even a bad record must have one unambiguous identity. Never recover duplicates.
|
|
162
|
+
if (!batch || typeof batch.id !== 'string' || !batch.id || ids.has(batch.id)) {
|
|
163
|
+
throw new Error(`batches[${index}]: missing or duplicate batch id`);
|
|
164
|
+
}
|
|
165
|
+
ids.add(batch.id);
|
|
166
|
+
try {
|
|
167
|
+
(0, state_1.validateBatchRecord)(batch, obj.entries);
|
|
168
|
+
return batch;
|
|
169
|
+
}
|
|
170
|
+
catch (err) {
|
|
171
|
+
const reason = `batches[${index}] (id ${JSON.stringify(batch.id)}): ${err.message}`;
|
|
172
|
+
if (!types_1.TERMINAL_BATCH_STATUSES.has(batch.status))
|
|
173
|
+
throw new Error(reason);
|
|
174
|
+
const references = obj.entries.filter((entry) => entry?.batch === batch.id);
|
|
175
|
+
if (references.some((entry) => !types_1.TERMINAL_ISSUE_STATUSES.has(entry.status))) {
|
|
176
|
+
throw new Error(`${reason}; active/parked entry references this batch — repair only this record`);
|
|
177
|
+
}
|
|
178
|
+
const unit = (0, state_1.batchUnit)(batch.id);
|
|
179
|
+
if (obj.slots.some((slot) => slot?.status !== 'idle' &&
|
|
180
|
+
(slot?.unit === unit ||
|
|
181
|
+
(typeof slot?.unit === 'string' && slot.unit.startsWith(`${unit}#`))))) {
|
|
182
|
+
throw new Error(`${reason}; held batch/member slot — repair only this record`);
|
|
183
|
+
}
|
|
184
|
+
records.push({ index, id: batch.id, reason });
|
|
185
|
+
return {
|
|
186
|
+
...(0, state_1.createBatch)(batch.id, references.map((entry) => entry.issue), new Date(0)),
|
|
187
|
+
status: batch.status,
|
|
188
|
+
blocked_reason: `${QUARANTINE_MARKER}${evidence}; ${reason}; original cleanup obligations require record-local reconciliation`,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
});
|
|
192
|
+
// Also checks schema, entries, slots, referential integrity and all global fields.
|
|
193
|
+
// A terminal exception must never hide independently corrupt active/global state.
|
|
194
|
+
return { state: (0, state_1.validateState)({ ...obj, batches }), records };
|
|
195
|
+
}
|
|
196
|
+
function fsyncDirectory(dir) {
|
|
197
|
+
const fd = node_fs_1.default.openSync(dir, 'r');
|
|
198
|
+
try {
|
|
199
|
+
node_fs_1.default.fsyncSync(fd);
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
node_fs_1.default.closeSync(fd);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
/** Mandatory durable evidence; existing digest files are verified, never overwritten. */
|
|
206
|
+
function preserveQuarantine(dir, digest, raw, records) {
|
|
207
|
+
const evidenceDir = path.join(dir, QUARANTINE_DIR);
|
|
208
|
+
node_fs_1.default.mkdirSync(evidenceDir, { recursive: true, mode: 0o700 });
|
|
209
|
+
if (!node_fs_1.default.lstatSync(evidenceDir).isDirectory() || node_fs_1.default.lstatSync(evidenceDir).isSymbolicLink()) {
|
|
210
|
+
throw new Error(`quarantine directory is not a store-owned directory: ${evidenceDir}`);
|
|
211
|
+
}
|
|
212
|
+
const manifest = Buffer.from(`${JSON.stringify({ digest, state_file: STATE_FILE, records }, null, 2)}\n`);
|
|
213
|
+
for (const [name, bytes] of [
|
|
214
|
+
[`${digest}.json`, raw],
|
|
215
|
+
[`${digest}.manifest.json`, manifest],
|
|
216
|
+
]) {
|
|
217
|
+
const target = path.join(evidenceDir, name);
|
|
218
|
+
let present = true;
|
|
130
219
|
try {
|
|
131
|
-
|
|
220
|
+
node_fs_1.default.lstatSync(target);
|
|
221
|
+
}
|
|
222
|
+
catch (err) {
|
|
223
|
+
if (err.code !== 'ENOENT')
|
|
224
|
+
throw err;
|
|
225
|
+
present = false;
|
|
226
|
+
}
|
|
227
|
+
if (!present) {
|
|
228
|
+
const tmp = path.join(evidenceDir, `.pending-${(0, node_crypto_1.randomUUID)()}`);
|
|
229
|
+
const fd = node_fs_1.default.openSync(tmp, node_fs_1.default.constants.O_WRONLY |
|
|
230
|
+
node_fs_1.default.constants.O_CREAT |
|
|
231
|
+
node_fs_1.default.constants.O_EXCL |
|
|
232
|
+
node_fs_1.default.constants.O_NOFOLLOW, 0o600);
|
|
233
|
+
try {
|
|
234
|
+
try {
|
|
235
|
+
node_fs_1.default.writeFileSync(fd, bytes);
|
|
236
|
+
node_fs_1.default.fsyncSync(fd);
|
|
237
|
+
}
|
|
238
|
+
finally {
|
|
239
|
+
node_fs_1.default.closeSync(fd);
|
|
240
|
+
}
|
|
241
|
+
node_fs_1.default.renameSync(tmp, target);
|
|
242
|
+
}
|
|
243
|
+
finally {
|
|
244
|
+
node_fs_1.default.rmSync(tmp, { force: true });
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
const fd = node_fs_1.default.openSync(target, node_fs_1.default.constants.O_RDONLY | node_fs_1.default.constants.O_NOFOLLOW | node_fs_1.default.constants.O_NONBLOCK);
|
|
248
|
+
try {
|
|
249
|
+
if (!node_fs_1.default.fstatSync(fd).isFile() || !node_fs_1.default.readFileSync(fd).equals(bytes)) {
|
|
250
|
+
throw new Error(`quarantine evidence verification failed: ${target}`);
|
|
251
|
+
}
|
|
252
|
+
node_fs_1.default.fsyncSync(fd);
|
|
132
253
|
}
|
|
133
254
|
finally {
|
|
134
|
-
|
|
255
|
+
node_fs_1.default.closeSync(fd);
|
|
135
256
|
}
|
|
136
257
|
}
|
|
137
|
-
|
|
138
|
-
|
|
258
|
+
fsyncDirectory(evidenceDir);
|
|
259
|
+
fsyncDirectory(dir); // makes the new quarantine directory durable too
|
|
260
|
+
}
|
|
261
|
+
// --- State mutex ---
|
|
262
|
+
const stateGuards = new Map();
|
|
263
|
+
/** Same inherited-open-description flock primitive as zero-trust/lock.ts.
|
|
264
|
+
* That private workspace cannot be a dependency of the published scheduler.
|
|
265
|
+
* The guard inode is permanent: never unlink/replace it, including after a crash.
|
|
266
|
+
*/
|
|
267
|
+
function acquireStateGuard(dir) {
|
|
268
|
+
if (process.platform !== 'linux')
|
|
269
|
+
return null;
|
|
270
|
+
const file = path.join(dir, `${LOCK_DIR}.guard`);
|
|
271
|
+
const fd = node_fs_1.default.openSync(file, node_fs_1.default.constants.O_RDWR | node_fs_1.default.constants.O_CREAT | node_fs_1.default.constants.O_NOFOLLOW | node_fs_1.default.constants.O_NONBLOCK, 0o600);
|
|
272
|
+
try {
|
|
273
|
+
const stat = node_fs_1.default.fstatSync(fd);
|
|
274
|
+
if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.getuid?.()) {
|
|
275
|
+
throw new Error(`Scheduler lock guard must be a private regular file: ${file}`);
|
|
276
|
+
}
|
|
277
|
+
const result = (0, node_child_process_1.spawnSync)('/usr/bin/flock', ['-x', '-w', String(LOCK_TIMEOUT_MS / 1000), '3'], {
|
|
278
|
+
stdio: ['ignore', 'ignore', 'ignore', fd],
|
|
279
|
+
timeout: LOCK_TIMEOUT_MS + 1000,
|
|
280
|
+
});
|
|
281
|
+
if (result.error)
|
|
282
|
+
throw new Error(`Could not acquire scheduler kernel guard (${file}): ${result.error.message}`);
|
|
283
|
+
if (result.status !== 0)
|
|
284
|
+
throw new LockTimeoutError(file, null, true);
|
|
285
|
+
const named = node_fs_1.default.lstatSync(file);
|
|
286
|
+
if (named.dev !== stat.dev || named.ino !== stat.ino)
|
|
287
|
+
throw new Error(`Scheduler lock guard changed: ${file}`);
|
|
288
|
+
return fd;
|
|
289
|
+
}
|
|
290
|
+
catch (err) {
|
|
291
|
+
node_fs_1.default.closeSync(fd);
|
|
292
|
+
throw err;
|
|
139
293
|
}
|
|
140
294
|
}
|
|
141
|
-
// --- Lock (directory mutex, worktree-pool protocol) ---
|
|
142
295
|
function acquireLock(dir) {
|
|
296
|
+
dir = path.resolve(dir);
|
|
143
297
|
const lockPath = path.join(dir, LOCK_DIR);
|
|
144
|
-
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
fs.writeFileSync(path.join(lockPath, 'pid'), String(process.pid), { mode: 0o600 });
|
|
151
|
-
return;
|
|
152
|
-
}
|
|
153
|
-
catch {
|
|
154
|
-
heldByPid = null;
|
|
298
|
+
node_fs_1.default.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
299
|
+
const guard = acquireStateGuard(dir);
|
|
300
|
+
try {
|
|
301
|
+
const start = Date.now();
|
|
302
|
+
let heldByPid = null;
|
|
303
|
+
while (true) {
|
|
155
304
|
try {
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
305
|
+
// Publish ownership atomically; a killed pending writer cannot leave a
|
|
306
|
+
// public PID-less mutex. Pending names are inert and never reclaimed.
|
|
307
|
+
const pending = path.join(dir, `${LOCK_DIR}.pending-${(0, node_crypto_1.randomUUID)()}`);
|
|
308
|
+
node_fs_1.default.mkdirSync(pending, { mode: 0o700 });
|
|
309
|
+
try {
|
|
310
|
+
node_fs_1.default.writeFileSync(path.join(pending, 'pid'), String(process.pid), { mode: 0o600 });
|
|
311
|
+
node_fs_1.default.renameSync(pending, lockPath);
|
|
312
|
+
}
|
|
313
|
+
catch (err) {
|
|
314
|
+
node_fs_1.default.rmSync(pending, { recursive: true, force: true });
|
|
315
|
+
throw err;
|
|
316
|
+
}
|
|
317
|
+
if (guard !== null)
|
|
318
|
+
stateGuards.set(dir, guard);
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
catch (err) {
|
|
322
|
+
// Permission/disk failures are not contention; retrying them hides the
|
|
323
|
+
// real I/O diagnosis and unnecessarily stalls read-only watchers (#1245).
|
|
324
|
+
if (!['EEXIST', 'ENOTEMPTY'].includes(err.code ?? ''))
|
|
325
|
+
throw err;
|
|
326
|
+
heldByPid = null;
|
|
327
|
+
try {
|
|
328
|
+
const pidFile = path.join(lockPath, 'pid');
|
|
329
|
+
if (node_fs_1.default.existsSync(pidFile)) {
|
|
330
|
+
const lockPid = Number.parseInt(node_fs_1.default.readFileSync(pidFile, 'utf-8').trim(), 10);
|
|
331
|
+
heldByPid = Number.isNaN(lockPid) ? null : lockPid;
|
|
176
332
|
try {
|
|
177
|
-
|
|
333
|
+
process.kill(lockPid, 0);
|
|
178
334
|
}
|
|
179
335
|
catch {
|
|
180
|
-
//
|
|
336
|
+
// Without the kernel guard there is no atomic compare-and-remove
|
|
337
|
+
// for a directory owner. Refuse rather than steal a replacement.
|
|
338
|
+
if (guard === null)
|
|
339
|
+
throw new LockTimeoutError(lockPath, heldByPid);
|
|
340
|
+
// Recorded holder is dead. Take the stale lock over by RENAMING it
|
|
341
|
+
// first: with two contenders both seeing the dead pid, only one
|
|
342
|
+
// rename succeeds — the loser's rmSync cannot delete the winner's
|
|
343
|
+
// freshly-acquired lock (the plain rmSync-race the pool protocol
|
|
344
|
+
// inherits from its original mkdir-mutex).
|
|
345
|
+
const stale = `${lockPath}.stale-${process.pid}-${Date.now()}`;
|
|
346
|
+
try {
|
|
347
|
+
node_fs_1.default.renameSync(lockPath, stale);
|
|
348
|
+
}
|
|
349
|
+
catch {
|
|
350
|
+
// another contender already took it over — fall through to retry
|
|
351
|
+
}
|
|
352
|
+
try {
|
|
353
|
+
node_fs_1.default.rmSync(stale, { recursive: true, force: true });
|
|
354
|
+
}
|
|
355
|
+
catch {
|
|
356
|
+
// best effort cleanup
|
|
357
|
+
}
|
|
358
|
+
continue;
|
|
181
359
|
}
|
|
182
|
-
continue;
|
|
183
360
|
}
|
|
184
361
|
}
|
|
362
|
+
catch {
|
|
363
|
+
// retry
|
|
364
|
+
}
|
|
365
|
+
if (Date.now() - start > LOCK_TIMEOUT_MS) {
|
|
366
|
+
throw new LockTimeoutError(lockPath, heldByPid);
|
|
367
|
+
}
|
|
368
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, LOCK_RETRY_MS);
|
|
185
369
|
}
|
|
186
|
-
catch {
|
|
187
|
-
// retry
|
|
188
|
-
}
|
|
189
|
-
if (Date.now() - start > LOCK_TIMEOUT_MS) {
|
|
190
|
-
throw new LockTimeoutError(lockPath, heldByPid);
|
|
191
|
-
}
|
|
192
|
-
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, LOCK_RETRY_MS);
|
|
193
370
|
}
|
|
194
371
|
}
|
|
372
|
+
catch (err) {
|
|
373
|
+
if (guard !== null)
|
|
374
|
+
node_fs_1.default.closeSync(guard);
|
|
375
|
+
throw err;
|
|
376
|
+
}
|
|
195
377
|
}
|
|
196
378
|
function releaseLock(dir) {
|
|
197
|
-
|
|
379
|
+
dir = path.resolve(dir);
|
|
380
|
+
const guard = stateGuards.get(dir);
|
|
381
|
+
try {
|
|
382
|
+
node_fs_1.default.rmSync(path.join(dir, LOCK_DIR), { recursive: true, force: true });
|
|
383
|
+
}
|
|
384
|
+
finally {
|
|
385
|
+
stateGuards.delete(dir);
|
|
386
|
+
if (guard !== undefined)
|
|
387
|
+
node_fs_1.default.closeSync(guard);
|
|
388
|
+
}
|
|
198
389
|
}
|
|
199
390
|
function readEngineLeaseHolder(leasePath) {
|
|
200
391
|
try {
|
|
201
|
-
const raw = JSON.parse(
|
|
392
|
+
const raw = JSON.parse(node_fs_1.default.readFileSync(path.join(leasePath, ENGINE_LEASE_HOLDER_FILE), 'utf8'));
|
|
202
393
|
if (raw === null || typeof raw !== 'object')
|
|
203
394
|
return null;
|
|
204
395
|
const candidate = raw;
|
|
@@ -242,6 +433,7 @@ class SchedStore {
|
|
|
242
433
|
dir;
|
|
243
434
|
/** The lease this store instance acquired (heartbeat target). */
|
|
244
435
|
heldLease = null;
|
|
436
|
+
warnedQuarantines = new Set();
|
|
245
437
|
constructor(dir, userConfigPath = path.resolve(dir, '..', '..', CONFIG_FILE)) {
|
|
246
438
|
this.userConfigPath = userConfigPath;
|
|
247
439
|
this.dir = dir;
|
|
@@ -287,25 +479,25 @@ class SchedStore {
|
|
|
287
479
|
if (readEngineLeaseHolder(leasePath)?.id !== held.id)
|
|
288
480
|
return;
|
|
289
481
|
const next = { ...held, updated_at: now.toISOString() };
|
|
290
|
-
|
|
482
|
+
node_fs_1.default.writeFileSync(tmp, `${JSON.stringify(next)}\n`, { mode: 0o600 });
|
|
291
483
|
// Compare-and-swap on the lease id, as late as the filesystem allows: a
|
|
292
484
|
// reclaim between the first check and here must not be overwritten by a
|
|
293
485
|
// stale heartbeat (rename has no CAS; this narrows the window to the
|
|
294
486
|
// gap between this read and the rename).
|
|
295
487
|
if (readEngineLeaseHolder(leasePath)?.id !== held.id) {
|
|
296
|
-
|
|
488
|
+
node_fs_1.default.rmSync(tmp, { force: true });
|
|
297
489
|
return;
|
|
298
490
|
}
|
|
299
|
-
|
|
491
|
+
node_fs_1.default.renameSync(tmp, path.join(leasePath, ENGINE_LEASE_HOLDER_FILE));
|
|
300
492
|
}
|
|
301
493
|
catch {
|
|
302
|
-
|
|
494
|
+
node_fs_1.default.rmSync(tmp, { force: true });
|
|
303
495
|
// Heartbeat is advisory; the pid-liveness check remains the source of truth.
|
|
304
496
|
}
|
|
305
497
|
}
|
|
306
498
|
/** Acquire the engine lifecycle lease, separate from short-lived state mutation locks. */
|
|
307
499
|
acquireEngineLease() {
|
|
308
|
-
|
|
500
|
+
node_fs_1.default.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
|
|
309
501
|
let reclaimed;
|
|
310
502
|
const leasePath = path.join(this.dir, ENGINE_LEASE_DIR);
|
|
311
503
|
const lease = {
|
|
@@ -319,17 +511,17 @@ class SchedStore {
|
|
|
319
511
|
// A unique pending directory must be created and populated successfully.
|
|
320
512
|
// Permission, quota, or read-only filesystem failures are operational errors,
|
|
321
513
|
// not contention, and must reach the caller.
|
|
322
|
-
|
|
514
|
+
node_fs_1.default.mkdirSync(pending, { mode: 0o700 });
|
|
323
515
|
try {
|
|
324
|
-
|
|
516
|
+
node_fs_1.default.writeFileSync(path.join(pending, ENGINE_LEASE_HOLDER_FILE), `${JSON.stringify(lease)}\n`, {
|
|
325
517
|
mode: 0o600,
|
|
326
518
|
});
|
|
327
|
-
|
|
519
|
+
node_fs_1.default.renameSync(pending, leasePath);
|
|
328
520
|
this.heldLease = lease;
|
|
329
521
|
return { acquired: true, lease, ...(reclaimed ? { reclaimed } : {}) };
|
|
330
522
|
}
|
|
331
523
|
catch (err) {
|
|
332
|
-
|
|
524
|
+
node_fs_1.default.rmSync(pending, { recursive: true, force: true });
|
|
333
525
|
if (!isEngineLeaseRace(err))
|
|
334
526
|
throw err;
|
|
335
527
|
const holder = readEngineLeaseHolder(leasePath);
|
|
@@ -340,7 +532,7 @@ class SchedStore {
|
|
|
340
532
|
// a newly acquired lease after both observed the old holder.
|
|
341
533
|
const stale = `${leasePath}.stale-${lease.id}`;
|
|
342
534
|
try {
|
|
343
|
-
|
|
535
|
+
node_fs_1.default.renameSync(leasePath, stale);
|
|
344
536
|
if (holder !== null) {
|
|
345
537
|
reclaimed = {
|
|
346
538
|
pid: holder.pid,
|
|
@@ -348,7 +540,7 @@ class SchedStore {
|
|
|
348
540
|
updated_at: holder.updated_at ?? null,
|
|
349
541
|
};
|
|
350
542
|
}
|
|
351
|
-
|
|
543
|
+
node_fs_1.default.rmSync(stale, { recursive: true, force: true });
|
|
352
544
|
}
|
|
353
545
|
catch (reclaimErr) {
|
|
354
546
|
if (!isEngineLeaseRace(reclaimErr))
|
|
@@ -362,27 +554,59 @@ class SchedStore {
|
|
|
362
554
|
releaseEngineLease(lease) {
|
|
363
555
|
const leasePath = path.join(this.dir, ENGINE_LEASE_DIR);
|
|
364
556
|
if (readEngineLeaseHolder(leasePath)?.id === lease.id) {
|
|
365
|
-
|
|
557
|
+
node_fs_1.default.rmSync(leasePath, { recursive: true, force: true });
|
|
366
558
|
}
|
|
367
559
|
if (this.heldLease?.id === lease.id)
|
|
368
560
|
this.heldLease = null;
|
|
369
561
|
}
|
|
562
|
+
/** Synchronous, non-reentrant locked read. Requires directory write access even
|
|
563
|
+
* for healthy state, may throw LockTimeoutError, and may publish quarantine
|
|
564
|
+
* evidence/state recovery. Inside withLock use its supplied state, not load(). */
|
|
370
565
|
load() {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
return
|
|
566
|
+
acquireLock(this.dir);
|
|
567
|
+
try {
|
|
568
|
+
return this.loadLocked();
|
|
569
|
+
}
|
|
570
|
+
finally {
|
|
571
|
+
releaseLock(this.dir);
|
|
374
572
|
}
|
|
573
|
+
}
|
|
574
|
+
warnQuarantined(state) {
|
|
575
|
+
for (const batch of state.batches) {
|
|
576
|
+
if (batch.blocked_reason?.startsWith(QUARANTINE_MARKER) &&
|
|
577
|
+
!this.warnedQuarantines.has(batch.blocked_reason)) {
|
|
578
|
+
console.error(`⚠ Scheduler ${this.statePath}: ${batch.blocked_reason}`);
|
|
579
|
+
this.warnedQuarantines.add(batch.blocked_reason);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
loadLocked() {
|
|
584
|
+
const statePath = this.statePath;
|
|
375
585
|
let raw;
|
|
376
586
|
try {
|
|
377
|
-
|
|
587
|
+
const fd = node_fs_1.default.openSync(statePath, node_fs_1.default.constants.O_RDONLY | node_fs_1.default.constants.O_NOFOLLOW | node_fs_1.default.constants.O_NONBLOCK);
|
|
588
|
+
try {
|
|
589
|
+
if (!node_fs_1.default.fstatSync(fd).isFile())
|
|
590
|
+
throw new Error('state must be a regular file, not a special filesystem object');
|
|
591
|
+
raw = node_fs_1.default.readFileSync(fd);
|
|
592
|
+
}
|
|
593
|
+
finally {
|
|
594
|
+
node_fs_1.default.closeSync(fd);
|
|
595
|
+
}
|
|
378
596
|
}
|
|
379
597
|
catch (err) {
|
|
598
|
+
if (err.code === 'ENOENT')
|
|
599
|
+
return (0, state_1.createEmptyState)();
|
|
380
600
|
// An I/O failure (permissions, disk, read-only mount) is NOT corruption —
|
|
381
601
|
// the destructive "rename to reset" advice must not attach to it.
|
|
382
602
|
throw new CorruptStateError(statePath, new Error(`could not READ the file (${err.message}) — this is an I/O failure, not corruption; renaming will not fix it`));
|
|
383
603
|
}
|
|
604
|
+
let parsed;
|
|
384
605
|
try {
|
|
385
|
-
|
|
606
|
+
parsed = JSON.parse(raw.toString('utf8'));
|
|
607
|
+
const state = (0, state_1.validateState)(parsed);
|
|
608
|
+
this.warnQuarantined(state);
|
|
609
|
+
return state;
|
|
386
610
|
}
|
|
387
611
|
catch (err) {
|
|
388
612
|
// A newer-than-installed schema is not corruption (#537) — rethrow as
|
|
@@ -390,7 +614,24 @@ class SchedStore {
|
|
|
390
614
|
// upgrade-the-engine message instead of "rename or remove it".
|
|
391
615
|
if (err instanceof types_1.EngineTooOldError)
|
|
392
616
|
throw err;
|
|
393
|
-
|
|
617
|
+
if (parsed === undefined)
|
|
618
|
+
throw new CorruptStateError(statePath, err);
|
|
619
|
+
const digest = (0, node_crypto_1.createHash)('sha256').update(raw).digest('hex');
|
|
620
|
+
const evidence = path.join(this.dir, QUARANTINE_DIR, `${digest}.json`);
|
|
621
|
+
try {
|
|
622
|
+
const isolated = isolateTerminalBatches(parsed, evidence);
|
|
623
|
+
if (isolated.records.length === 0)
|
|
624
|
+
throw err;
|
|
625
|
+
preserveQuarantine(this.dir, digest, raw, isolated.records);
|
|
626
|
+
this.save(isolated.state);
|
|
627
|
+
this.warnQuarantined(isolated.state);
|
|
628
|
+
return isolated.state;
|
|
629
|
+
}
|
|
630
|
+
catch (recoveryError) {
|
|
631
|
+
if (recoveryError instanceof types_1.EngineTooOldError)
|
|
632
|
+
throw recoveryError;
|
|
633
|
+
throw new CorruptStateError(statePath, new Error(`${err.message}; terminal quarantine refused (${recoveryError.message}); evidence location ${evidence}`));
|
|
634
|
+
}
|
|
394
635
|
}
|
|
395
636
|
}
|
|
396
637
|
save(state) {
|
|
@@ -403,7 +644,7 @@ class SchedStore {
|
|
|
403
644
|
withLock(fn) {
|
|
404
645
|
acquireLock(this.dir);
|
|
405
646
|
try {
|
|
406
|
-
const current = this.
|
|
647
|
+
const current = this.loadLocked();
|
|
407
648
|
const { state, result } = fn(current);
|
|
408
649
|
this.save(state);
|
|
409
650
|
return result;
|
|
@@ -415,7 +656,7 @@ class SchedStore {
|
|
|
415
656
|
loadConfig() {
|
|
416
657
|
const userProfiles = this.loadUserProfiles();
|
|
417
658
|
let config;
|
|
418
|
-
if (!
|
|
659
|
+
if (!node_fs_1.default.existsSync(this.configPath)) {
|
|
419
660
|
config = { max_slots: types_1.DEFAULT_MAX_SLOTS };
|
|
420
661
|
return mergeDispatchProfiles(config, userProfiles);
|
|
421
662
|
}
|
|
@@ -441,7 +682,7 @@ class SchedStore {
|
|
|
441
682
|
/** Parse + strictly validate the project config file (throws on ANY problem). */
|
|
442
683
|
parseConfigFile() {
|
|
443
684
|
let config;
|
|
444
|
-
const parsed = JSON.parse(
|
|
685
|
+
const parsed = JSON.parse(node_fs_1.default.readFileSync(this.configPath, 'utf-8'));
|
|
445
686
|
const version = String(parsed.schema_version);
|
|
446
687
|
if (version !== types_1.CONFIG_SCHEMA_VERSION && !types_1.LEGACY_CONFIG_SCHEMA_VERSIONS.includes(version)) {
|
|
447
688
|
throw new Error(`unsupported schema version ${version}`);
|
|
@@ -507,7 +748,7 @@ class SchedStore {
|
|
|
507
748
|
// are ignored) so it can never block a valid project-config reload; the
|
|
508
749
|
// caller is told, naming the file.
|
|
509
750
|
const userProfiles = this.loadUserProfiles(onUserConfigError);
|
|
510
|
-
if (!
|
|
751
|
+
if (!node_fs_1.default.existsSync(this.configPath)) {
|
|
511
752
|
return mergeDispatchProfiles({ max_slots: types_1.DEFAULT_MAX_SLOTS }, userProfiles);
|
|
512
753
|
}
|
|
513
754
|
return mergeDispatchProfiles(this.parseConfigFile(), userProfiles);
|
|
@@ -519,7 +760,7 @@ class SchedStore {
|
|
|
519
760
|
configFingerprint() {
|
|
520
761
|
const stamp = (file) => {
|
|
521
762
|
try {
|
|
522
|
-
const st =
|
|
763
|
+
const st = node_fs_1.default.statSync(file);
|
|
523
764
|
return `${st.mtimeMs}:${st.size}`;
|
|
524
765
|
}
|
|
525
766
|
catch {
|
|
@@ -573,10 +814,10 @@ class SchedStore {
|
|
|
573
814
|
writeAtomic(this.configPath, `${JSON.stringify(file, null, 2)}\n`);
|
|
574
815
|
}
|
|
575
816
|
loadUserProfiles(onError) {
|
|
576
|
-
if (!
|
|
817
|
+
if (!node_fs_1.default.existsSync(this.userConfigPath))
|
|
577
818
|
return {};
|
|
578
819
|
try {
|
|
579
|
-
const raw = JSON.parse(
|
|
820
|
+
const raw = JSON.parse(node_fs_1.default.readFileSync(this.userConfigPath, 'utf-8'));
|
|
580
821
|
const userConfig = requirePlainObject('user config', raw);
|
|
581
822
|
if (userConfig.dispatch_profiles === undefined)
|
|
582
823
|
return {};
|