@yu_robotics/remote-cli 1.6.255 → 1.6.259
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/MessageHandler.d.ts +5 -12
- package/dist/client/MessageHandler.d.ts.map +1 -1
- package/dist/client/MessageHandler.js +53 -252
- package/dist/client/MessageHandler.js.map +1 -1
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +0 -3
- package/dist/commands/start.js.map +1 -1
- package/dist/executor/AcpExecutor.d.ts.map +1 -1
- package/dist/executor/AcpExecutor.js.map +1 -1
- package/dist/executor/AgyExecutor.d.ts +4 -11
- package/dist/executor/AgyExecutor.d.ts.map +1 -1
- package/dist/executor/AgyExecutor.js +68 -98
- package/dist/executor/AgyExecutor.js.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.d.ts +3 -7
- package/dist/executor/ClaudePersistentExecutor.d.ts.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.js +14 -224
- package/dist/executor/ClaudePersistentExecutor.js.map +1 -1
- package/dist/executor/DshExecutor.d.ts +7 -14
- package/dist/executor/DshExecutor.d.ts.map +1 -1
- package/dist/executor/DshExecutor.js +48 -50
- package/dist/executor/DshExecutor.js.map +1 -1
- package/dist/executor/IExecutor.d.ts +2 -2
- package/dist/executor/IExecutor.d.ts.map +1 -1
- package/dist/executor/acp/AcpClient.d.ts +15 -1
- package/dist/executor/acp/AcpClient.d.ts.map +1 -1
- package/dist/executor/acp/AcpClient.js +85 -7
- package/dist/executor/acp/AcpClient.js.map +1 -1
- package/dist/executor/agy/AgyAcpDiagnostics.d.ts +8 -0
- package/dist/executor/agy/AgyAcpDiagnostics.d.ts.map +1 -0
- package/dist/executor/agy/AgyAcpDiagnostics.js +45 -0
- package/dist/executor/agy/AgyAcpDiagnostics.js.map +1 -0
- package/dist/executor/agy/AgyAcpRuntime.d.ts +9 -16
- package/dist/executor/agy/AgyAcpRuntime.d.ts.map +1 -1
- package/dist/executor/agy/AgyAcpRuntime.js +67 -151
- package/dist/executor/agy/AgyAcpRuntime.js.map +1 -1
- package/dist/executor/agy/AgySessionSelection.d.ts +7 -0
- package/dist/executor/agy/AgySessionSelection.d.ts.map +1 -0
- package/dist/executor/agy/AgySessionSelection.js +60 -0
- package/dist/executor/agy/AgySessionSelection.js.map +1 -0
- package/dist/executor/agy/AgySessionSettings.d.ts +13 -0
- package/dist/executor/agy/AgySessionSettings.d.ts.map +1 -0
- package/dist/executor/agy/AgySessionSettings.js +60 -0
- package/dist/executor/agy/AgySessionSettings.js.map +1 -0
- package/dist/executor/dsh/DshAcpClient.d.ts +1 -1
- package/dist/executor/dsh/DshAcpClient.d.ts.map +1 -1
- package/dist/executor/dsh/DshAcpClient.js +3 -1
- package/dist/executor/dsh/DshAcpClient.js.map +1 -1
- package/dist/executor/index.d.ts +1 -0
- package/dist/executor/index.d.ts.map +1 -1
- package/dist/executor/index.js +1 -0
- package/dist/executor/index.js.map +1 -1
- package/dist/maintenance/release-notes.json +1 -1
- package/dist/scheduling/ScheduleCapabilities.d.ts +2 -2
- package/dist/scheduling/ScheduleCapabilities.js +4 -4
- package/dist/scheduling/ScheduleCapabilities.js.map +1 -1
- package/dist/scheduling/SchedulePurgeVerification.js +1 -1
- package/dist/scheduling/SchedulePurgeVerification.js.map +1 -1
- package/dist/scheduling/ScheduleService.d.ts +10 -29
- package/dist/scheduling/ScheduleService.d.ts.map +1 -1
- package/dist/scheduling/ScheduleService.js +160 -627
- package/dist/scheduling/ScheduleService.js.map +1 -1
- package/dist/scheduling/agy/AgySessionSchedules.d.ts +5 -5
- package/dist/scheduling/agy/AgySessionSchedules.d.ts.map +1 -1
- package/dist/scheduling/agy/AgySessionSchedules.js +20 -17
- package/dist/scheduling/agy/AgySessionSchedules.js.map +1 -1
- package/dist/scheduling/claude/ClaudeScheduleOwnership.d.ts +2 -6
- package/dist/scheduling/claude/ClaudeScheduleOwnership.d.ts.map +1 -1
- package/dist/scheduling/claude/ClaudeScheduleOwnership.js +12 -67
- package/dist/scheduling/claude/ClaudeScheduleOwnership.js.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts +4 -26
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.js +22 -145
- package/dist/scheduling/claude/ClaudeSessionSchedules.js.map +1 -1
- package/dist/scheduling/dsh/ScheduledSessionGuard.d.ts +12 -0
- package/dist/scheduling/dsh/ScheduledSessionGuard.d.ts.map +1 -0
- package/dist/scheduling/dsh/ScheduledSessionGuard.js +96 -0
- package/dist/scheduling/dsh/ScheduledSessionGuard.js.map +1 -0
- package/dist/scheduling/dsh/SessionScheduleBridge.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleBridge.js +0 -25
- package/dist/scheduling/dsh/SessionScheduleBridge.js.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.js +0 -8
- package/dist/scheduling/dsh/SessionScheduleClient.js.map +1 -1
- package/dist/scheduling/dsh/sessionLaunch.js +1 -1
- package/dist/scheduling/dsh/sessionLaunch.js.map +1 -1
- package/dist/scheduling/types.d.ts +26 -77
- package/dist/scheduling/types.d.ts.map +1 -1
- package/dist/scheduling/types.js +1 -10
- package/dist/scheduling/types.js.map +1 -1
- package/dist/service/ServiceManager.js +1 -1
- package/dist/thread/ThreadExecutorPool.d.ts +4 -1
- package/dist/thread/ThreadExecutorPool.d.ts.map +1 -1
- package/dist/thread/ThreadExecutorPool.js +6 -3
- package/dist/thread/ThreadExecutorPool.js.map +1 -1
- package/dist/thread/ThreadManager.d.ts.map +1 -1
- package/dist/thread/ThreadManager.js +23 -2
- package/dist/thread/ThreadManager.js.map +1 -1
- package/native/prebuilds/darwin-arm64/managed_native.node +0 -0
- package/native/prebuilds/darwin-arm64/manifest.json +1 -1
- package/native/prebuilds/darwin-x64/managed_native.node +0 -0
- package/native/prebuilds/darwin-x64/manifest.json +1 -1
- package/native/prebuilds/win32-arm64/managed_native.node +0 -0
- package/native/prebuilds/win32-arm64/manifest.json +1 -1
- package/native/prebuilds/win32-x64/managed_native.node +0 -0
- package/native/prebuilds/win32-x64/manifest.json +1 -1
- package/package.json +1 -1
- package/dist/scheduling/agy/AgyCheckpoint.d.ts +0 -35
- package/dist/scheduling/agy/AgyCheckpoint.d.ts.map +0 -1
- package/dist/scheduling/agy/AgyCheckpoint.js +0 -160
- package/dist/scheduling/agy/AgyCheckpoint.js.map +0 -1
|
@@ -96,11 +96,11 @@ function sameOwner(left, right) {
|
|
|
96
96
|
function sameCleanupOwner(left, right) {
|
|
97
97
|
return left.threadId === right.threadId && left.openId === right.openId;
|
|
98
98
|
}
|
|
99
|
-
/**
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
/** Unknown native mutation/queued reminders remain cleanup obligations after host loss. */
|
|
100
|
+
function unsettledNativeWork(record) {
|
|
101
|
+
return !record.creationRefused && !record.cleanupConfirmed
|
|
102
|
+
&& (['interrupted', 'failed', 'creating'].includes(record.state)
|
|
103
|
+
|| !!record.error?.startsWith('Native process exit unconfirmed:'));
|
|
104
104
|
}
|
|
105
105
|
function deliveryId(scheduleId, occurrenceId) {
|
|
106
106
|
return `${scheduleId}:${occurrenceId}`;
|
|
@@ -137,13 +137,10 @@ class ScheduleService {
|
|
|
137
137
|
initialized = false;
|
|
138
138
|
closing = false;
|
|
139
139
|
destruction;
|
|
140
|
+
pendingCreates = 0;
|
|
140
141
|
fault;
|
|
141
142
|
catalogPath = '';
|
|
142
|
-
recoveryRun;
|
|
143
143
|
catalogGate;
|
|
144
|
-
suspending = false;
|
|
145
|
-
handoffUnresolved = false;
|
|
146
|
-
suspension;
|
|
147
144
|
constructor(options) {
|
|
148
145
|
this.options = { ...options };
|
|
149
146
|
this.now = options.now ?? Date.now;
|
|
@@ -153,9 +150,6 @@ class ScheduleService {
|
|
|
153
150
|
throw new Error('Invalid schedule capacity.');
|
|
154
151
|
return result;
|
|
155
152
|
};
|
|
156
|
-
if (options.recoveryTimeoutMs !== undefined && (!Number.isInteger(options.recoveryTimeoutMs)
|
|
157
|
-
|| options.recoveryTimeoutMs < 1 || options.recoveryTimeoutMs > 60_000))
|
|
158
|
-
throw new Error('Invalid recovery timeout.');
|
|
159
153
|
this.maxLive = limit(options.maxLiveSchedules, 5, 5);
|
|
160
154
|
this.maxRecords = limit(options.maxRecords, 100, 1000);
|
|
161
155
|
this.maxPending = limit(options.maxPendingDeliveries, 100, 1000);
|
|
@@ -180,35 +174,13 @@ class ScheduleService {
|
|
|
180
174
|
await this.compactRefusedCreation(record);
|
|
181
175
|
let changed = false;
|
|
182
176
|
for (const record of this.records.values()) {
|
|
183
|
-
if (
|
|
184
|
-
if (record.recovery?.status === 'pending') {
|
|
185
|
-
record.recovery.status = 'blocked';
|
|
186
|
-
changed = true;
|
|
187
|
-
}
|
|
188
|
-
continue; // Terminal tombstones always win over a stale recovery checkpoint.
|
|
189
|
-
}
|
|
190
|
-
// A consumed future-only attempt or an already blocked interrupted row
|
|
191
|
-
// cannot regain automatic admission after another service restart.
|
|
192
|
-
if (record.recovery?.futureOnly && (record.state === 'creating'
|
|
193
|
-
|| record.state === 'interrupted' && record.recovery.status === 'blocked')) {
|
|
194
|
-
record.state = 'failed';
|
|
177
|
+
if (record.recovery && record.recovery.status !== 'blocked') {
|
|
195
178
|
record.recovery.status = 'blocked';
|
|
196
|
-
record.error ??= 'Interrupted native restoration quarantined; original session and cleanup evidence retained.';
|
|
197
179
|
changed = true;
|
|
198
|
-
continue;
|
|
199
180
|
}
|
|
200
|
-
if (LIVE.has(record.state)
|
|
201
|
-
|| record.state === 'interrupted' && record.recovery?.futureOnly && record.rule.kind === 'cron') {
|
|
202
|
-
const futureOnly = record.recovery?.futureOnly && record.rule.kind === 'cron';
|
|
203
|
-
const recoverable = record.recovery && !record.cancelRequested && this.current(record)
|
|
204
|
-
&& (record.state !== 'running' || futureOnly) && record.state !== 'failed'
|
|
205
|
-
&& (!record.recovery.orderlyOnly || record.recovery.handoff === 'clean' || futureOnly)
|
|
206
|
-
&& (record.state === 'active' || record.recovery.status === 'pending' || futureOnly);
|
|
207
|
-
if (record.recovery)
|
|
208
|
-
record.recovery.status = recoverable ? 'pending' : 'blocked';
|
|
209
|
-
record.error = recoverable ? 'Waiting for native recovery capability and owner readiness.'
|
|
210
|
-
: record.recovery?.orderlyOnly ? 'Dirty shutdown: original native work is unconfirmed; automatic reconstruction is blocked.' : RESTART_REASON;
|
|
181
|
+
if (LIVE.has(record.state)) {
|
|
211
182
|
record.state = 'interrupted';
|
|
183
|
+
record.error = RESTART_REASON;
|
|
212
184
|
changed = true;
|
|
213
185
|
}
|
|
214
186
|
}
|
|
@@ -219,229 +191,16 @@ class ScheduleService {
|
|
|
219
191
|
changed = true;
|
|
220
192
|
}
|
|
221
193
|
}
|
|
222
|
-
if (changed || data === undefined)
|
|
194
|
+
if (changed || data === undefined || data.version !== 6)
|
|
223
195
|
await this.persist();
|
|
224
196
|
this.initialized = true;
|
|
225
197
|
}).catch(error => { this.catalogGate?.close(); this.catalogGate = undefined; throw error; });
|
|
226
198
|
}
|
|
227
|
-
|
|
228
|
-
recover() {
|
|
229
|
-
this.checkReady();
|
|
230
|
-
if (this.recoveryRun)
|
|
231
|
-
return this.recoveryRun;
|
|
232
|
-
const run = this.operation(this.recoverWaiting());
|
|
233
|
-
this.recoveryRun = run;
|
|
234
|
-
void run.finally(() => { if (this.recoveryRun === run)
|
|
235
|
-
this.recoveryRun = undefined; }).catch(() => { });
|
|
236
|
-
return run;
|
|
237
|
-
}
|
|
238
|
-
async recoverWaiting() {
|
|
239
|
-
const candidates = await this.serial(async () => [...this.records.values()]
|
|
240
|
-
.filter(record => record.recovery?.status === 'pending' && record.state === 'interrupted').map(record => record.id));
|
|
241
|
-
// Catalog capacity is bounded, and only one native admission runs at a time.
|
|
242
|
-
for (const id of candidates.slice(0, this.maxLive)) {
|
|
243
|
-
if (this.closing || this.fault)
|
|
244
|
-
return;
|
|
245
|
-
const record = this.records.get(id);
|
|
246
|
-
if (!record)
|
|
247
|
-
continue;
|
|
248
|
-
if (record.cancelRequested || !this.current(record)) {
|
|
249
|
-
await this.blockRecovery(record, 'Recovery owner retired or cancellation was requested.');
|
|
250
|
-
continue;
|
|
251
|
-
}
|
|
252
|
-
if (record.rule.kind === 'once' && Date.parse(record.rule.at) <= this.now()) {
|
|
253
|
-
await this.blockRecovery(record, 'One-shot deadline expired; no late execution was requested.');
|
|
254
|
-
if (record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent)
|
|
255
|
-
await this.serial(async () => { delete record.recovery; await this.persist(); });
|
|
256
|
-
continue;
|
|
257
|
-
}
|
|
258
|
-
const input = { id, threadId: record.threadId, backend: record.backend,
|
|
259
|
-
cwd: record.cwd, prompt: record.prompt, rule: structuredClone(record.rule), ...record.execution,
|
|
260
|
-
recovery: this.checkpoint(record.recovery) };
|
|
261
|
-
if (!this.options.canRecover || !this.options.canRecover(structuredClone(record))) {
|
|
262
|
-
await this.recoveryDiagnostic(record, 'Original owner configuration or transport is not ready for recovery.');
|
|
263
|
-
continue;
|
|
264
|
-
}
|
|
265
|
-
const provider = this.options.provider.recoveryFor?.(input);
|
|
266
|
-
if (!provider || provider.provider !== record.recovery.provider || provider.strategy !== record.recovery.strategy
|
|
267
|
-
|| (provider.handoffPolicy ?? 'drain-all') !== (record.recovery.handoffPolicy ?? 'drain-all')) {
|
|
268
|
-
await this.recoveryDiagnostic(record, 'Native restart recovery is unsupported or provider identity changed.');
|
|
269
|
-
continue;
|
|
270
|
-
}
|
|
271
|
-
if (this.runtimes.has(id))
|
|
272
|
-
continue; // An unresolved/late operation retains exclusive ownership.
|
|
273
|
-
const restoreRecord = structuredClone(record);
|
|
274
|
-
const runtime = this.newRuntime();
|
|
275
|
-
runtime.completed = new Set(record.completedOccurrences);
|
|
276
|
-
const controller = new AbortController();
|
|
277
|
-
runtime.recoveryAbort = controller;
|
|
278
|
-
// Admission contention has not consumed this job's clean checkpoint.
|
|
279
|
-
// Leave it pending so the exact earlier runtime can release the thread.
|
|
280
|
-
try {
|
|
281
|
-
runtime.releaseAdmission = this.options.acquireRecovery?.(structuredClone(record));
|
|
282
|
-
}
|
|
283
|
-
catch (error) {
|
|
284
|
-
await this.recoveryDiagnostic(record, errorText(error));
|
|
285
|
-
continue;
|
|
286
|
-
}
|
|
287
|
-
this.runtimes.set(id, runtime);
|
|
288
|
-
const current = () => !controller.signal.aborted && !this.closing && !this.fault && !record.cancelRequested
|
|
289
|
-
&& this.current(record) && this.options.canRecover?.(structuredClone(record)) === true;
|
|
290
|
-
const emit = (event) => {
|
|
291
|
-
if (this.runtimes.get(id) !== runtime) {
|
|
292
|
-
if (event.type === 'rebound')
|
|
293
|
-
return Promise.reject(new Error('Native rebound owner retired.'));
|
|
294
|
-
return;
|
|
295
|
-
}
|
|
296
|
-
const task = this.onEvent(id, structuredClone(event));
|
|
297
|
-
this.track(task);
|
|
298
|
-
return task;
|
|
299
|
-
};
|
|
300
|
-
let timer;
|
|
301
|
-
let admitted = false;
|
|
302
|
-
let succeeded = false;
|
|
303
|
-
try {
|
|
304
|
-
await this.serial(async () => {
|
|
305
|
-
if (!current())
|
|
306
|
-
throw new Error('Recovery admission expired.');
|
|
307
|
-
// Consume clean proof before mutation. A crash must not replay an orderly-only restore.
|
|
308
|
-
record.state = 'creating';
|
|
309
|
-
delete record.recovery.handoff;
|
|
310
|
-
delete record.recovery.nativeDefinitionAbsent;
|
|
311
|
-
await this.persist();
|
|
312
|
-
});
|
|
313
|
-
admitted = true;
|
|
314
|
-
runtime.recoveryAdmitted = true;
|
|
315
|
-
const admission = Promise.resolve().then(() => provider.reconcile(restoreRecord, { notBefore: this.now(), signal: controller.signal, isCurrent: current,
|
|
316
|
-
isOwnerCurrent: () => !record.cancelRequested && !this.fault && this.current(record),
|
|
317
|
-
canAdmitOccurrence: () => this.canAdmitOccurrence(id, runtime) }, emit));
|
|
318
|
-
const adopt = async (handle) => {
|
|
319
|
-
runtime.handle = handle;
|
|
320
|
-
runtime.resolveReady(handle);
|
|
321
|
-
text(handle.nativeId);
|
|
322
|
-
await this.serial(async () => { record.nativeId = handle.nativeId; if (!this.fault)
|
|
323
|
-
await this.persist(); });
|
|
324
|
-
};
|
|
325
|
-
const receipt = admission.then(async (handle) => {
|
|
326
|
-
await adopt(handle);
|
|
327
|
-
if (!current()) {
|
|
328
|
-
await this.stop(id, runtime);
|
|
329
|
-
throw new Error('Late recovery receipt requires cleanup; no successor was admitted.');
|
|
330
|
-
}
|
|
331
|
-
if (!await handle.list() || !current())
|
|
332
|
-
throw new Error('Recovered native inventory or owner is unconfirmed.');
|
|
333
|
-
return handle;
|
|
334
|
-
}, async (error) => {
|
|
335
|
-
if (error instanceof types_1.NativeScheduleCleanupError) {
|
|
336
|
-
await adopt(error.handle);
|
|
337
|
-
if (controller.signal.aborted)
|
|
338
|
-
await this.stop(id, runtime).catch(() => { });
|
|
339
|
-
}
|
|
340
|
-
runtime.resolveReady(runtime.handle);
|
|
341
|
-
if (!runtime.handle && (error instanceof types_1.NativeScheduleRecoverySettledError
|
|
342
|
-
|| error instanceof types_1.NativeScheduleNotAdmittedError || error instanceof types_1.NativeScheduleAbsentError)) {
|
|
343
|
-
runtime.stopped = true;
|
|
344
|
-
this.releaseRecoveryAdmission(runtime);
|
|
345
|
-
if (this.runtimes.get(id) === runtime)
|
|
346
|
-
this.runtimes.delete(id);
|
|
347
|
-
}
|
|
348
|
-
// A rejection without a handle is not proof that native admission never happened.
|
|
349
|
-
throw error;
|
|
350
|
-
});
|
|
351
|
-
const handle = await Promise.race([receipt, new Promise((_, reject) => {
|
|
352
|
-
timer = setTimeout(() => {
|
|
353
|
-
controller.abort();
|
|
354
|
-
reject(new Error(record.recovery?.futureOnly
|
|
355
|
-
? 'Native recovery timed out; this job will not restart automatically. Native settlement and cleanup remain separate.'
|
|
356
|
-
: 'Native recovery timed out; ownership retained, do not recreate.'));
|
|
357
|
-
}, this.options.recoveryTimeoutMs ?? 30_000);
|
|
358
|
-
})]);
|
|
359
|
-
// Native recovery and inventory validation settled. A later deadline
|
|
360
|
-
// must not abort the adopted handle while its active checkpoint is saved.
|
|
361
|
-
clearTimeout(timer);
|
|
362
|
-
timer = undefined;
|
|
363
|
-
text(handle.nativeId);
|
|
364
|
-
// The provider's reconcile contract includes inventory validation; list is
|
|
365
|
-
// additionally required to verify the returned handle before activating it.
|
|
366
|
-
if (!current())
|
|
367
|
-
throw new Error('Recovered native owner is unconfirmed.');
|
|
368
|
-
await this.serial(async () => {
|
|
369
|
-
if (!current())
|
|
370
|
-
throw new Error('Recovery owner retired before receipt persistence.');
|
|
371
|
-
record.nativeId = handle.nativeId;
|
|
372
|
-
if (handle.recoveryIdentity) {
|
|
373
|
-
text(handle.recoveryIdentity, 8192);
|
|
374
|
-
record.recovery.identity = handle.recoveryIdentity;
|
|
375
|
-
}
|
|
376
|
-
if (record.state === 'creating')
|
|
377
|
-
record.state = 'active';
|
|
378
|
-
record.recovery.status = 'eligible';
|
|
379
|
-
delete record.error;
|
|
380
|
-
await this.persist();
|
|
381
|
-
});
|
|
382
|
-
succeeded = true;
|
|
383
|
-
}
|
|
384
|
-
catch (error) {
|
|
385
|
-
controller.abort();
|
|
386
|
-
if (!admitted || error instanceof types_1.NativeScheduleAbsentError || error instanceof types_1.NativeScheduleNotAdmittedError || error instanceof types_1.NativeScheduleRecoverySettledError) {
|
|
387
|
-
runtime.stopped = true;
|
|
388
|
-
runtime.resolveReady(undefined);
|
|
389
|
-
}
|
|
390
|
-
// Do not retry an ambiguous mutation or release its handle on a timeout.
|
|
391
|
-
if (admitted)
|
|
392
|
-
await this.blockRecovery(record, errorText(error), !!record.recovery?.futureOnly);
|
|
393
|
-
else
|
|
394
|
-
await this.recoveryDiagnostic(record, errorText(error));
|
|
395
|
-
if (runtime.handle)
|
|
396
|
-
await this.stop(id, runtime).catch(() => { });
|
|
397
|
-
else if (runtime.stopped)
|
|
398
|
-
this.runtimes.delete(id);
|
|
399
|
-
}
|
|
400
|
-
finally {
|
|
401
|
-
if (timer)
|
|
402
|
-
clearTimeout(timer);
|
|
403
|
-
if (succeeded || runtime.stopped) {
|
|
404
|
-
this.releaseRecoveryAdmission(runtime);
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
// A handoff containing only skipped/expired one-shots has nothing to
|
|
409
|
-
// reconcile. It must still release the predecessor's local input fence.
|
|
410
|
-
const released = new Set();
|
|
411
|
-
for (const record of this.records.values()) {
|
|
412
|
-
if (released.has(record.threadId) || this.closing || this.fault || this.handoffUnresolved
|
|
413
|
-
|| !this.current(record) || !this.options.canRecover?.(structuredClone(record)))
|
|
414
|
-
continue;
|
|
415
|
-
if ([...this.records.values()].some(other => other.threadId === record.threadId
|
|
416
|
-
&& (LIVE.has(other.state) || recoveryOwnsWork(other) || this.runtimes.has(other.id))))
|
|
417
|
-
continue;
|
|
418
|
-
this.options.provider.recoveryFor?.({ id: record.id, threadId: record.threadId, backend: record.backend,
|
|
419
|
-
cwd: record.cwd, prompt: record.prompt, rule: record.rule, ...record.execution })?.releaseRestartAdmission?.();
|
|
420
|
-
released.add(record.threadId);
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
releaseRecoveryAdmission(runtime) {
|
|
424
|
-
const release = runtime.releaseAdmission;
|
|
425
|
-
runtime.releaseAdmission = undefined;
|
|
426
|
-
release?.();
|
|
427
|
-
if (!release || !runtime.recoveryAdmitted || !runtime.recoveryAbort?.signal.aborted)
|
|
428
|
-
return;
|
|
429
|
-
// Join the current pass before retrying untouched siblings. Failed rows are
|
|
430
|
-
// never candidates, and canRecover/acquireRecovery recheck newer admission.
|
|
431
|
-
const retry = Promise.resolve(this.recoveryRun).catch(() => { }).then(() => {
|
|
432
|
-
if (!this.closing && !this.fault && this.hasPendingRecovery())
|
|
433
|
-
return this.recover();
|
|
434
|
-
});
|
|
435
|
-
this.track(retry);
|
|
436
|
-
}
|
|
437
|
-
checkpoint(value) {
|
|
438
|
-
return { version: 1, provider: value.provider, strategy: value.strategy, identity: value.identity,
|
|
439
|
-
...(value.orderlyOnly && { orderlyOnly: true }), ...(value.futureOnly && { futureOnly: true }), ...(value.handoffPolicy && { handoffPolicy: value.handoffPolicy }), ...(value.handoff && { handoff: value.handoff }),
|
|
440
|
-
...(value.nativeDefinitionAbsent && { nativeDefinitionAbsent: true }) };
|
|
441
|
-
}
|
|
442
|
-
validateCheckpoint(value) {
|
|
199
|
+
validateOwnershipMetadata(value) {
|
|
443
200
|
if (value.futureOnly !== undefined && value.futureOnly !== true)
|
|
444
201
|
throw new Error('Invalid future recovery policy.');
|
|
202
|
+
if (value.abrupt !== undefined && value.abrupt !== true)
|
|
203
|
+
throw new Error('Invalid abrupt recovery capability.');
|
|
445
204
|
if (value.version !== 1 || !['reattach', 'reconstruct'].includes(value.strategy))
|
|
446
205
|
throw new Error('Invalid recovery checkpoint.');
|
|
447
206
|
if (value.orderlyOnly !== undefined && value.orderlyOnly !== true || value.handoff !== undefined && value.handoff !== 'clean')
|
|
@@ -453,22 +212,16 @@ class ScheduleService {
|
|
|
453
212
|
text(value.provider);
|
|
454
213
|
text(value.identity, 8192);
|
|
455
214
|
}
|
|
456
|
-
async recoveryDiagnostic(record, reason) {
|
|
457
|
-
await this.serial(async () => { record.error = reason; if (!this.fault)
|
|
458
|
-
await this.persist(); });
|
|
459
|
-
}
|
|
460
|
-
async blockRecovery(record, reason, quarantine = false) {
|
|
461
|
-
await this.serial(async () => {
|
|
462
|
-
if (!['completed', 'cancelled'].includes(record.state))
|
|
463
|
-
record.state = quarantine ? 'failed' : 'interrupted';
|
|
464
|
-
record.recovery.status = 'blocked';
|
|
465
|
-
record.error = reason;
|
|
466
|
-
if (!this.fault)
|
|
467
|
-
await this.persist();
|
|
468
|
-
});
|
|
469
|
-
}
|
|
470
215
|
create(owner, input) {
|
|
471
|
-
|
|
216
|
+
this.pendingCreates++;
|
|
217
|
+
const creation = this.recordOperation(input.id, this.createSchedule(owner, input).finally(async () => {
|
|
218
|
+
this.pendingCreates--;
|
|
219
|
+
if (this.closing && !this.pendingCreates) {
|
|
220
|
+
await this.tail;
|
|
221
|
+
this.catalogGate?.close();
|
|
222
|
+
this.catalogGate = undefined;
|
|
223
|
+
}
|
|
224
|
+
}));
|
|
472
225
|
return this.operation(creation.catch(async (error) => {
|
|
473
226
|
if (error instanceof types_1.NativeScheduleNotAdmittedError) {
|
|
474
227
|
// The create operation has settled. Concurrent cancel/purge operations
|
|
@@ -506,11 +259,11 @@ class ScheduleService {
|
|
|
506
259
|
for (const value of [input.command, input.model, input.effort])
|
|
507
260
|
if (value !== undefined)
|
|
508
261
|
text(value, 4096);
|
|
509
|
-
const
|
|
510
|
-
if (
|
|
511
|
-
nativeInput.recovery = { version: 1, provider:
|
|
512
|
-
identity:
|
|
513
|
-
this.
|
|
262
|
+
const cleanup = this.options.provider.cleanupFor?.(nativeInput);
|
|
263
|
+
if (cleanup) {
|
|
264
|
+
nativeInput.recovery = { version: 1, provider: cleanup.provider, strategy: cleanup.strategy,
|
|
265
|
+
identity: cleanup.describe(structuredClone(nativeInput), originalOwner) };
|
|
266
|
+
this.validateOwnershipMetadata(nativeInput.recovery);
|
|
514
267
|
}
|
|
515
268
|
const runtime = this.newRuntime();
|
|
516
269
|
await this.serial(async () => {
|
|
@@ -522,28 +275,29 @@ class ScheduleService {
|
|
|
522
275
|
if (this.records.size >= this.maxRecords)
|
|
523
276
|
throw new Error('Schedule catalog is full; history was not pruned.');
|
|
524
277
|
const liveCount = [...this.records.values()].filter(record => LIVE.has(record.state)
|
|
525
|
-
||
|
|
278
|
+
|| this.hasRetainedCleanupRoute(record)
|
|
526
279
|
|| (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)).length;
|
|
527
280
|
if (liveCount >= this.maxLive)
|
|
528
281
|
throw new Error('Live native schedule limit reached.');
|
|
529
282
|
this.records.set(nativeInput.id, { ...originalOwner, id: nativeInput.id, backend,
|
|
530
283
|
cwd: nativeInput.cwd, prompt: nativeInput.prompt, rule: structuredClone(nativeInput.rule),
|
|
531
284
|
createdAt, state: 'creating', execution: { command: input.command, model: input.model, effort: input.effort },
|
|
532
|
-
...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: '
|
|
285
|
+
...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: 'blocked' } }) });
|
|
533
286
|
await this.persist();
|
|
534
287
|
this.runtimes.set(nativeInput.id, runtime);
|
|
535
288
|
});
|
|
536
289
|
try {
|
|
537
290
|
const handle = await this.options.provider.create(structuredClone(nativeInput), event => {
|
|
538
291
|
if (this.runtimes.get(nativeInput.id) !== runtime) {
|
|
539
|
-
if (event.type === 'rebound')
|
|
540
|
-
return Promise.reject(new Error('Native rebound owner retired.'));
|
|
541
292
|
return;
|
|
542
293
|
}
|
|
543
|
-
if (event.type === 'started' || event.type === 'completed'
|
|
294
|
+
if (event.type === 'started' || event.type === 'completed')
|
|
544
295
|
runtime.observedOccurrence = true;
|
|
545
296
|
const task = this.onEvent(nativeInput.id, structuredClone(event));
|
|
546
|
-
|
|
297
|
+
// A rejected pre-CREATE attribution is an admission refusal, not a
|
|
298
|
+
// runtime event fault. persist() still fences actual storage failures.
|
|
299
|
+
if (event.type !== 'attributed')
|
|
300
|
+
this.track(task);
|
|
547
301
|
return task;
|
|
548
302
|
}, () => this.canAdmitOccurrence(nativeInput.id, runtime));
|
|
549
303
|
runtime.handle = handle;
|
|
@@ -552,12 +306,16 @@ class ScheduleService {
|
|
|
552
306
|
await this.serial(async () => {
|
|
553
307
|
const record = this.records.get(nativeInput.id);
|
|
554
308
|
record.nativeId = handle.nativeId;
|
|
555
|
-
if (record.recovery && handle.
|
|
556
|
-
text(handle.
|
|
557
|
-
record.recovery.identity = handle.
|
|
309
|
+
if (record.recovery && handle.cleanupIdentity) {
|
|
310
|
+
text(handle.cleanupIdentity, 8192);
|
|
311
|
+
record.recovery.identity = handle.cleanupIdentity;
|
|
558
312
|
}
|
|
559
313
|
const ownerCurrent = this.current(originalOwner);
|
|
560
|
-
if (this.closing
|
|
314
|
+
if (this.closing) {
|
|
315
|
+
await this.persist();
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (!ownerCurrent) {
|
|
561
319
|
if (LIVE.has(record.state)) {
|
|
562
320
|
record.state = 'interrupted';
|
|
563
321
|
record.error = 'Schedule owner or service retired during creation.';
|
|
@@ -580,6 +338,21 @@ class ScheduleService {
|
|
|
580
338
|
runtime.resolveReady(runtime.handle);
|
|
581
339
|
await this.serial(async () => {
|
|
582
340
|
const record = this.records.get(nativeInput.id);
|
|
341
|
+
if (error instanceof types_1.NativeScheduleCleanupError) {
|
|
342
|
+
if (error.handle.nativeId) {
|
|
343
|
+
text(error.handle.nativeId);
|
|
344
|
+
record.nativeId = error.handle.nativeId;
|
|
345
|
+
}
|
|
346
|
+
if (record.recovery && error.handle.cleanupIdentity) {
|
|
347
|
+
text(error.handle.cleanupIdentity, 8192);
|
|
348
|
+
record.recovery.identity = error.handle.cleanupIdentity;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
if (this.closing) {
|
|
352
|
+
if (!this.fault)
|
|
353
|
+
await this.persist();
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
583
356
|
if (LIVE.has(record.state)) {
|
|
584
357
|
record.state = 'failed';
|
|
585
358
|
record.error = errorText(error);
|
|
@@ -606,13 +379,12 @@ class ScheduleService {
|
|
|
606
379
|
if (record.state !== 'failed' || record.creationRefused !== true || record.nativeId !== undefined
|
|
607
380
|
|| record.cancelRequested || record.cleanupConfirmed || record.completedOccurrences?.length || record.skippedOccurrences?.length
|
|
608
381
|
|| this.records.get(id) !== record || this.cleanup.has(id) || this.purged.has(id)
|
|
609
|
-
|| this.fault || this.closing || this.
|
|
382
|
+
|| this.fault || this.closing || this.refreshes
|
|
610
383
|
|| this.recordOperations.has(id) || this.retainedCancellations.has(id) || this.purges.has(id)
|
|
611
384
|
|| this.delivering.has(id) || [...this.pending.values()].some(item => item.scheduleId === id)
|
|
612
385
|
|| this.options.isNativeMutationPending?.(ownerOf(record))
|
|
613
386
|
|| runtime && (!runtime.stopped || runtime.handle || runtime.observedOccurrence || runtime.running.size || runtime.completed.size
|
|
614
|
-
|| runtime.heldDeliveries.size || runtime.cancelling || runtime.cancelPromise || runtime.stopPromise
|
|
615
|
-
|| runtime.recoveryAbort || runtime.releasing || runtime.releasePromise || runtime.releaseAdmission)
|
|
387
|
+
|| runtime.heldDeliveries.size || runtime.cancelling || runtime.cancelPromise || runtime.stopPromise)
|
|
616
388
|
|| this.purged.size >= MAX_PURGED_IDS)
|
|
617
389
|
return;
|
|
618
390
|
this.purged.set(id, ownerOf(record));
|
|
@@ -717,10 +489,9 @@ class ScheduleService {
|
|
|
717
489
|
record.cancelRequested = true;
|
|
718
490
|
if (record.recovery)
|
|
719
491
|
record.recovery.status = 'blocked';
|
|
720
|
-
this.runtimes.get(id)?.recoveryAbort?.abort();
|
|
721
492
|
changed = true;
|
|
722
493
|
}
|
|
723
|
-
// Persist admission
|
|
494
|
+
// Persist admission denial before any external cleanup attempt.
|
|
724
495
|
if (changed)
|
|
725
496
|
await this.persist();
|
|
726
497
|
return ids;
|
|
@@ -782,12 +553,12 @@ class ScheduleService {
|
|
|
782
553
|
forgetCleanup(owner, id, isCurrent) {
|
|
783
554
|
return this.serial(async () => {
|
|
784
555
|
this.checkReady();
|
|
785
|
-
if (!this.current(owner) || !isCurrent() || this.closing
|
|
556
|
+
if (!this.current(owner) || !isCurrent() || this.closing)
|
|
786
557
|
throw new Error('Schedule cleanup context changed.');
|
|
787
558
|
const prior = this.purged.get(id);
|
|
788
559
|
if (!prior || !sameCleanupOwner(prior, owner) || this.records.has(id))
|
|
789
560
|
throw new Error('Purge owned history before forgetting unresolved cleanup tracking.');
|
|
790
|
-
if (this.options.isNativeMutationPending?.(owner) || this.recordOperations.has(id)
|
|
561
|
+
if (this.options.isNativeMutationPending?.(owner) || this.recordOperations.has(id)
|
|
791
562
|
|| this.refreshes || this.retainedCancellations.has(id) || this.runtimes.has(id))
|
|
792
563
|
return { status: 'pending', reason: 'operation_pending', detail: 'Native work or cleanup is still pending; tracking was not forgotten.' };
|
|
793
564
|
if (this.delivering.has(id))
|
|
@@ -814,7 +585,7 @@ class ScheduleService {
|
|
|
814
585
|
return { status: 'pending', reason: 'operation_pending', detail: 'A native mutation receipt is unsettled; retry after reconciliation.' };
|
|
815
586
|
if (!item || !this.options.withPurgeVerification)
|
|
816
587
|
return { status: 'alreadyRemoved' };
|
|
817
|
-
const current = () => !this.closing && !this.fault &&
|
|
588
|
+
const current = () => !this.closing && !this.fault && isCurrent() && this.current(owner) && this.cleanup.get(id) === item;
|
|
818
589
|
let accepted = false;
|
|
819
590
|
return this.options.withPurgeVerification(structuredClone(item), { isCurrent: current }, async (evidence) => {
|
|
820
591
|
if (accepted)
|
|
@@ -862,11 +633,11 @@ class ScheduleService {
|
|
|
862
633
|
if (this.purged.has(id))
|
|
863
634
|
return this.retryCleanup(owner, id, isCurrent);
|
|
864
635
|
const record = this.owned(owner, id);
|
|
865
|
-
const current = () => !this.closing && !this.fault &&
|
|
636
|
+
const current = () => !this.closing && !this.fault && isCurrent() && this.current(owner)
|
|
866
637
|
&& this.records.get(id) === record && record.cancelRequested === true;
|
|
867
638
|
const busy = () => {
|
|
868
|
-
if (this.options.isNativeMutationPending?.(owner) || this.recordOperations.has(id) || this.
|
|
869
|
-
return pending('operation_pending', 'Creation,
|
|
639
|
+
if (this.options.isNativeMutationPending?.(owner) || this.recordOperations.has(id) || this.refreshes || this.retainedCancellations.has(id)) {
|
|
640
|
+
return pending('operation_pending', 'Creation, inspection or cleanup is still pending. Retry these same IDs.');
|
|
870
641
|
}
|
|
871
642
|
if (this.delivering.has(id))
|
|
872
643
|
return pending('delivery_pending', 'An external delivery is already in flight; its outcome cannot be revoked.');
|
|
@@ -880,7 +651,7 @@ class ScheduleService {
|
|
|
880
651
|
// Do not interpret terminal state, an absent handle or a stopped process as
|
|
881
652
|
// native definition absence. Existing cancelRetained owns its stronger proof.
|
|
882
653
|
let retainedVerified = false;
|
|
883
|
-
const clean = !runtime &&
|
|
654
|
+
const clean = !runtime && record.cleanupConfirmed === true;
|
|
884
655
|
if (runtime?.handle && !runtime.stopped) {
|
|
885
656
|
await this.cancel(owner, id);
|
|
886
657
|
}
|
|
@@ -903,7 +674,7 @@ class ScheduleService {
|
|
|
903
674
|
return unfinished;
|
|
904
675
|
if (runtime?.running.size)
|
|
905
676
|
return pending('occurrence_running', 'Future cancellation does not revoke an already admitted occurrence.');
|
|
906
|
-
if (runtime && (!runtime.stopped
|
|
677
|
+
if (runtime && (!runtime.stopped))
|
|
907
678
|
return pending('operation_pending', 'Native runtime retirement is unconfirmed.');
|
|
908
679
|
const handleProof = runtime?.removalConfirmed && runtime.handle?.nativeDefinitionAbsent !== false;
|
|
909
680
|
const erase = (evidence, unresolved) => this.serial(async () => {
|
|
@@ -969,7 +740,15 @@ class ScheduleService {
|
|
|
969
740
|
/** Includes terminal jobs whose native process exit is still unconfirmed. */
|
|
970
741
|
hasActiveSchedules(threadId) {
|
|
971
742
|
return [...this.records.values()].some(record => (!threadId || record.threadId === threadId)
|
|
972
|
-
&& (LIVE.has(record.state) ||
|
|
743
|
+
&& (LIVE.has(record.state) || this.hasRetainedCleanupRoute(record) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
|
|
744
|
+
}
|
|
745
|
+
/** Read-only admission for purge of an owned runtime, including cancelled jobs. */
|
|
746
|
+
needsNativePurge(owner, id) {
|
|
747
|
+
this.checkReady();
|
|
748
|
+
this.owned(owner, id);
|
|
749
|
+
const runtime = this.runtimes.get(id);
|
|
750
|
+
// Cancellation can hide the card while its native runtime still needs retirement.
|
|
751
|
+
return !!runtime?.handle && !runtime.stopped;
|
|
973
752
|
}
|
|
974
753
|
/** Card eligibility includes retryable cleanup even for a terminal catalog record. */
|
|
975
754
|
canCancel(owner, id) {
|
|
@@ -978,7 +757,7 @@ class ScheduleService {
|
|
|
978
757
|
const runtime = this.runtimes.get(id);
|
|
979
758
|
if (record.state === 'cancelled' && runtime?.handle?.cancellationKeepsRunning && !record.error)
|
|
980
759
|
return false;
|
|
981
|
-
return LIVE.has(record.state) ||
|
|
760
|
+
return LIVE.has(record.state) || this.hasRetainedCleanupRoute(record) || ['interrupted', 'failed'].includes(record.state) && !!record.nativeId && this.supportsRetainedCancellation(record)
|
|
982
761
|
|| Boolean(runtime && !runtime.stopped);
|
|
983
762
|
}
|
|
984
763
|
/** Newly observed cleanup facts never supply historical recovery proof. The optional
|
|
@@ -1004,7 +783,7 @@ class ScheduleService {
|
|
|
1004
783
|
: 'Owner-reviewed native deletion requested; absence and predecessor work remain unconfirmed.';
|
|
1005
784
|
const runtime = this.runtimes.get(id);
|
|
1006
785
|
if (absent && retirementConfirmed?.(record) === true && (!runtime || runtime.stopped && !runtime.running.size)
|
|
1007
|
-
&& !this.recordOperations.has(id) && !this.
|
|
786
|
+
&& !this.recordOperations.has(id) && !this.refreshes && !this.retainedCancellations.has(id)) {
|
|
1008
787
|
record.state = 'cancelled';
|
|
1009
788
|
record.cleanupConfirmed = true;
|
|
1010
789
|
delete record.error;
|
|
@@ -1050,12 +829,12 @@ class ScheduleService {
|
|
|
1050
829
|
continue;
|
|
1051
830
|
const nativeId = record.nativeId;
|
|
1052
831
|
const completed = runtime.completed.size;
|
|
1053
|
-
// Recheck after every read: cancellation
|
|
832
|
+
// Recheck after every read: cancellation or execution may have
|
|
1054
833
|
// advanced while inventory was awaiting its native receipt.
|
|
1055
|
-
const readable = () => !this.closing && !this.fault &&
|
|
834
|
+
const readable = () => !this.closing && !this.fault && this.current(owner)
|
|
1056
835
|
&& this.records.get(id) === record && record.nativeId === nativeId && record.state === 'active' && !record.cancelRequested
|
|
1057
836
|
&& this.runtimes.get(id) === runtime && runtime.handle === handle && !runtime.stopped && !runtime.cancelling
|
|
1058
|
-
&& !runtime.running.size && runtime.completed.size === completed && !runtime.stopPromise
|
|
837
|
+
&& !runtime.running.size && runtime.completed.size === completed && !runtime.stopPromise
|
|
1059
838
|
&& !this.recordOperations.has(id) && !this.retainedCancellations.has(id);
|
|
1060
839
|
if (!readable())
|
|
1061
840
|
continue;
|
|
@@ -1103,18 +882,11 @@ class ScheduleService {
|
|
|
1103
882
|
return;
|
|
1104
883
|
runtime = this.runtimes.get(id);
|
|
1105
884
|
record.cancelRequested = true;
|
|
1106
|
-
runtime?.recoveryAbort?.abort();
|
|
1107
885
|
if (record.recovery)
|
|
1108
886
|
record.recovery.status = 'blocked';
|
|
1109
887
|
await this.persist();
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
delete record.error;
|
|
1113
|
-
await this.persist();
|
|
1114
|
-
}
|
|
1115
|
-
const retainedCleanup = !runtime || runtime.stopped && !runtime.handle?.nativeDefinitionAbsent
|
|
1116
|
-
&& this.supportsRetainedCancellation(record);
|
|
1117
|
-
if (retainedCleanup && (recoveryOwnsWork(record) || record.state === 'failed' && !!record.recovery
|
|
888
|
+
const retainedCleanup = !runtime || runtime.stopped && !runtime.handle?.nativeDefinitionAbsent;
|
|
889
|
+
if (retainedCleanup && (unsettledNativeWork(record) || record.state === 'failed' && !!record.recovery
|
|
1118
890
|
|| ['interrupted', 'failed'].includes(record.state) && !!record.nativeId && this.supportsRetainedCancellation(record))) {
|
|
1119
891
|
operation = Promise.resolve().then(() => this.cancelRetained(record));
|
|
1120
892
|
this.retainedCancellations.set(id, operation);
|
|
@@ -1139,9 +911,6 @@ class ScheduleService {
|
|
|
1139
911
|
}
|
|
1140
912
|
if (!runtime)
|
|
1141
913
|
throw new Error('Native schedule handle is unavailable.');
|
|
1142
|
-
if (runtime.recoveryAbort && !runtime.handle && !runtime.stopped) {
|
|
1143
|
-
throw new Error('Recovery receipt is unresolved; cancellation intent retained until native cleanup can be confirmed.');
|
|
1144
|
-
}
|
|
1145
914
|
runtime.cancelling = true;
|
|
1146
915
|
// Starting outside the state queue also permits synchronous provider callbacks.
|
|
1147
916
|
operation = Promise.resolve().then(() => this.cancelNative(id, runtime));
|
|
@@ -1149,18 +918,24 @@ class ScheduleService {
|
|
|
1149
918
|
});
|
|
1150
919
|
return (existing ?? operation);
|
|
1151
920
|
}
|
|
921
|
+
/** Historical uncertainty is not live work. Only an exact supported cleanup route
|
|
922
|
+
* can keep a retired row in runtime admission and management controls. */
|
|
923
|
+
hasRetainedCleanupRoute(record) {
|
|
924
|
+
return unsettledNativeWork(record) && !!record.recovery && this.supportsRetainedCancellation(record);
|
|
925
|
+
}
|
|
1152
926
|
supportsRetainedCancellation(record) {
|
|
1153
|
-
|
|
927
|
+
const cleanup = this.options.provider.cleanupFor?.({ ...record, ...record.execution });
|
|
928
|
+
return !!record.recovery && !!cleanup?.cancelRetained && cleanup.provider === record.recovery.provider
|
|
929
|
+
&& cleanup.strategy === record.recovery.strategy;
|
|
1154
930
|
}
|
|
1155
931
|
async cancelRetained(record) {
|
|
1156
932
|
const current = () => !this.closing && !this.fault && this.current(record) && record.cancelRequested === true;
|
|
1157
933
|
try {
|
|
1158
|
-
const
|
|
1159
|
-
const provider = this.options.provider.
|
|
934
|
+
const ownership = record.recovery;
|
|
935
|
+
const provider = this.options.provider.cleanupFor?.({ id: record.id, threadId: record.threadId,
|
|
1160
936
|
backend: record.backend, cwd: record.cwd, prompt: record.prompt, rule: record.rule,
|
|
1161
|
-
...record.execution, recovery });
|
|
1162
|
-
if (!
|
|
1163
|
-
|| (provider.handoffPolicy ?? 'drain-all') !== (recovery.handoffPolicy ?? 'drain-all')) {
|
|
937
|
+
...record.execution, recovery: ownership });
|
|
938
|
+
if (!ownership || !provider?.cancelRetained || provider.provider !== ownership.provider || provider.strategy !== ownership.strategy) {
|
|
1164
939
|
throw new Error('Cancellation intent retained; native cleanup is unconfirmed. Original native ownership metadata or cleanup support is unavailable. Preserve the original session. For Claude, use /schedule cleanup review in the bound original thread for explicit deletion-only review; do not recreate this job.');
|
|
1165
940
|
}
|
|
1166
941
|
if (!current())
|
|
@@ -1180,8 +955,14 @@ class ScheduleService {
|
|
|
1180
955
|
return structuredClone(record);
|
|
1181
956
|
}
|
|
1182
957
|
catch (error) {
|
|
1183
|
-
await this.serial(async () => {
|
|
1184
|
-
|
|
958
|
+
await this.serial(async () => {
|
|
959
|
+
// A failed deletion attempt is not a new interruption. Keep the
|
|
960
|
+
// original exit/ownership diagnostic while returning the cleanup error.
|
|
961
|
+
if (record.state !== 'interrupted' || !record.error)
|
|
962
|
+
record.error = errorText(error);
|
|
963
|
+
if (!this.fault)
|
|
964
|
+
await this.persist();
|
|
965
|
+
});
|
|
1185
966
|
throw error;
|
|
1186
967
|
}
|
|
1187
968
|
finally {
|
|
@@ -1194,238 +975,30 @@ class ScheduleService {
|
|
|
1194
975
|
const ids = await this.serial(async () => [...this.pending.keys()]);
|
|
1195
976
|
await this.attemptDeliveries(ids);
|
|
1196
977
|
}
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
return
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
hasPendingRecovery() {
|
|
1203
|
-
return [...this.records.values()].some(record => record.state === 'interrupted' && record.recovery?.status === 'pending');
|
|
1204
|
-
}
|
|
1205
|
-
/** Snapshot eligibility is only admission to a handoff, never permission to install. */
|
|
1206
|
-
canSuspendForRestart() {
|
|
1207
|
-
if (!this.initialized || this.closing || this.fault || this.suspending || this.recoveryRun || this.operations.size || this.pending.size)
|
|
1208
|
-
return false;
|
|
1209
|
-
return [...this.records.values()].filter(record => LIVE.has(record.state) || recoveryOwnsWork(record)
|
|
1210
|
-
|| !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped).every(record => {
|
|
1211
|
-
const runtime = this.runtimes.get(record.id);
|
|
1212
|
-
return record.state === 'active' && !!record.recovery && !record.cancelRequested && !record.error
|
|
1213
|
-
&& !!runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling;
|
|
1214
|
-
});
|
|
1215
|
-
}
|
|
1216
|
-
/** Delete future definitions, drain native work, join owned hosts, then publish restart proof. */
|
|
1217
|
-
suspendForRestart() {
|
|
1218
|
-
if (this.suspension)
|
|
1219
|
-
return this.suspension;
|
|
1220
|
-
if (!this.canSuspendForRestart())
|
|
1221
|
-
return Promise.reject(new Error('Schedules are not ready for restart handoff.'));
|
|
1222
|
-
this.suspending = true;
|
|
1223
|
-
this.handoffUnresolved = true;
|
|
1224
|
-
const entries = [...this.runtimes].filter(([id]) => this.records.get(id)?.state === 'active');
|
|
1225
|
-
const run = (async () => {
|
|
1226
|
-
try {
|
|
1227
|
-
// Checkpoints remain dirty throughout deletion/drain and actual host exit.
|
|
1228
|
-
await this.serial(async () => {
|
|
1229
|
-
for (const [id, runtime] of entries) {
|
|
1230
|
-
if (runtime.handle?.recoveryIdentity) {
|
|
1231
|
-
text(runtime.handle.recoveryIdentity, 8192);
|
|
1232
|
-
this.records.get(id).recovery.identity = runtime.handle.recoveryIdentity;
|
|
1233
|
-
}
|
|
1234
|
-
delete this.records.get(id).recovery.handoff;
|
|
1235
|
-
delete this.records.get(id).recovery.nativeDefinitionAbsent;
|
|
1236
|
-
}
|
|
1237
|
-
await this.persist();
|
|
1238
|
-
});
|
|
1239
|
-
const releases = await Promise.allSettled(entries.map(([, runtime]) => this.boundRecovery(runtime.handle.releaseForRestart(), 'Native schedule drain remains unresolved.')));
|
|
1240
|
-
const failed = releases.find((result) => result.status === 'rejected');
|
|
1241
|
-
if (failed)
|
|
1242
|
-
throw failed.reason;
|
|
1243
|
-
for (const [index, result] of releases.entries()) {
|
|
1244
|
-
const policy = this.records.get(entries[index][0]).recovery.handoffPolicy ?? 'drain-all';
|
|
1245
|
-
if (result.status === 'fulfilled' && (result.value?.policy ?? 'drain-all') !== policy)
|
|
1246
|
-
throw new Error('Native handoff policy receipt changed.');
|
|
1247
|
-
}
|
|
1248
|
-
const absentOneShots = new Set();
|
|
1249
|
-
for (const [index, result] of releases.entries())
|
|
1250
|
-
if (result.status === 'fulfilled' && result.value?.onceDefinitionAlreadyAbsent) {
|
|
1251
|
-
const record = this.records.get(entries[index][0]);
|
|
1252
|
-
if (record.rule.kind !== 'once' || record.recovery?.handoffPolicy !== 'skip-unadmitted')
|
|
1253
|
-
throw new Error('Invalid absent one-shot handoff receipt.');
|
|
1254
|
-
absentOneShots.add(record.id);
|
|
1255
|
-
}
|
|
1256
|
-
const absent = new Set(entries.filter((_, index) => {
|
|
1257
|
-
const result = releases[index];
|
|
1258
|
-
return result.status === 'fulfilled' && result.value?.nativeDefinitionAbsent === true;
|
|
1259
|
-
}).map(([id]) => id));
|
|
1260
|
-
await this.boundRecovery((async () => {
|
|
1261
|
-
while (this.tasks.size)
|
|
1262
|
-
await Promise.allSettled([...this.tasks]);
|
|
1263
|
-
})(), 'Scheduled result drain remains unresolved.');
|
|
1264
|
-
await this.tail;
|
|
1265
|
-
if (this.pending.size)
|
|
1266
|
-
throw new Error('Scheduled result acknowledgement is pending.');
|
|
1267
|
-
if (this.fault || entries.some(([, runtime]) => runtime.running.size))
|
|
1268
|
-
throw this.fault ?? new Error('Native occurrence is still running.');
|
|
1269
|
-
const finished = new Set();
|
|
1270
|
-
for (const [id] of entries) {
|
|
1271
|
-
const record = this.records.get(id);
|
|
1272
|
-
const provider = this.options.provider.recoveryFor?.({ id, threadId: record.threadId, backend: record.backend,
|
|
1273
|
-
cwd: record.cwd, rule: record.rule, prompt: record.prompt, ...record.execution });
|
|
1274
|
-
if (!provider || provider.provider !== record.recovery.provider)
|
|
1275
|
-
throw new Error('Native recovery provider changed during handoff.');
|
|
1276
|
-
const key = `${record.threadId}:${provider.provider}`;
|
|
1277
|
-
if (finished.has(key))
|
|
1278
|
-
continue;
|
|
1279
|
-
if (provider.orderlyOnly && !provider.finishRestart)
|
|
1280
|
-
throw new Error('Owned native host exit cannot be confirmed.');
|
|
1281
|
-
if (provider.finishRestart)
|
|
1282
|
-
await this.boundRecovery(provider.finishRestart(), 'Owned native host exit remains unresolved.');
|
|
1283
|
-
finished.add(key);
|
|
1284
|
-
}
|
|
1285
|
-
await this.boundRecovery((async () => {
|
|
1286
|
-
while (this.tasks.size)
|
|
1287
|
-
await Promise.allSettled([...this.tasks]);
|
|
1288
|
-
})(), 'Scheduled result drain remains unresolved.');
|
|
1289
|
-
await this.serial(async () => {
|
|
1290
|
-
if (this.fault)
|
|
1291
|
-
throw this.fault;
|
|
1292
|
-
for (const [id, runtime] of entries) {
|
|
1293
|
-
const record = this.records.get(id);
|
|
1294
|
-
if (!this.current(record) || record.cancelRequested)
|
|
1295
|
-
throw new Error('Schedule owner retired during handoff.');
|
|
1296
|
-
if (record.state === 'active' && absentOneShots.has(id)) {
|
|
1297
|
-
record.state = 'interrupted';
|
|
1298
|
-
record.error = 'One-shot native definition was already absent at handoff; it will not be replayed.';
|
|
1299
|
-
delete record.recovery;
|
|
1300
|
-
}
|
|
1301
|
-
else if (record.state === 'active') {
|
|
1302
|
-
record.state = 'interrupted';
|
|
1303
|
-
record.recovery.status = 'pending';
|
|
1304
|
-
record.recovery.handoff = 'clean';
|
|
1305
|
-
if (absent.has(id))
|
|
1306
|
-
record.recovery.nativeDefinitionAbsent = true;
|
|
1307
|
-
record.error = 'Native host drained and exited; waiting for automatic recovery.';
|
|
1308
|
-
}
|
|
1309
|
-
else if (record.state === 'interrupted' && record.rule.kind === 'once' && record.recovery?.handoffPolicy === 'skip-unadmitted' && !runtime.running.size) {
|
|
1310
|
-
delete record.recovery;
|
|
1311
|
-
}
|
|
1312
|
-
else if (record.state !== 'completed')
|
|
1313
|
-
throw new Error('Occurrence outcome changed during handoff.');
|
|
1314
|
-
runtime.stopped = true;
|
|
1315
|
-
}
|
|
1316
|
-
await this.persist();
|
|
1317
|
-
for (const [id] of entries)
|
|
1318
|
-
this.runtimes.delete(id);
|
|
1319
|
-
this.handoffUnresolved = false;
|
|
1320
|
-
});
|
|
1321
|
-
}
|
|
1322
|
-
catch (error) {
|
|
1323
|
-
await this.serial(async () => {
|
|
1324
|
-
for (const [id] of entries) {
|
|
1325
|
-
const record = this.records.get(id);
|
|
1326
|
-
delete record.recovery?.handoff;
|
|
1327
|
-
delete record.recovery?.nativeDefinitionAbsent;
|
|
1328
|
-
record.error = `Restart handoff unconfirmed: ${errorText(error)}`;
|
|
1329
|
-
}
|
|
1330
|
-
if (!this.fault)
|
|
1331
|
-
await this.persist();
|
|
1332
|
-
});
|
|
1333
|
-
throw error;
|
|
1334
|
-
}
|
|
1335
|
-
finally {
|
|
1336
|
-
this.suspending = false;
|
|
1337
|
-
}
|
|
1338
|
-
})();
|
|
1339
|
-
this.suspension = run;
|
|
1340
|
-
void run.finally(() => { if (this.suspension === run)
|
|
1341
|
-
this.suspension = undefined; }).catch(() => { });
|
|
1342
|
-
return run;
|
|
1343
|
-
}
|
|
1344
|
-
/** Providers with durable intent must complete handoff before unrelated host teardown. */
|
|
1345
|
-
requiresShutdownHandoff() {
|
|
1346
|
-
return [...this.records.values()].some(record => !!record.recovery
|
|
1347
|
-
&& (!(record.state === 'failed' && record.recovery.futureOnly) || !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped));
|
|
978
|
+
/** Idle future definitions do not block updates. Active work and durable deliveries still do. */
|
|
979
|
+
canRestartWithoutActiveWork() {
|
|
980
|
+
return this.initialized && !this.closing && !this.fault && !this.operations.size
|
|
981
|
+
&& !this.pending.size && !this.delivering.size && [...this.runtimes].every(([id, runtime]) => (!['cancelled', 'failed', 'completed', 'interrupted'].includes(this.records.get(id)?.state ?? '') || runtime.stopped) &&
|
|
982
|
+
!runtime.running.size && !runtime.cancelling) && ![...this.records.values()].some(record => record.state === 'creating');
|
|
1348
983
|
}
|
|
984
|
+
/** Shutdown revokes admission only. It never invokes native schedule RPCs or waits for result ACKs. */
|
|
1349
985
|
destroy() {
|
|
1350
986
|
if (this.destruction)
|
|
1351
987
|
return this.destruction;
|
|
1352
|
-
|
|
1353
|
-
if (!preserve)
|
|
1354
|
-
this.closing = true;
|
|
988
|
+
this.closing = true;
|
|
1355
989
|
return this.destruction = (async () => {
|
|
1356
|
-
if (preserve)
|
|
1357
|
-
await this.suspendForRestart();
|
|
1358
|
-
this.closing = true;
|
|
1359
|
-
for (const runtime of this.runtimes.values())
|
|
1360
|
-
runtime.recoveryAbort?.abort();
|
|
1361
990
|
await this.initialization?.catch(() => { });
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
for (const record of this.records.values()) {
|
|
1365
|
-
if (LIVE.has(record.state)) {
|
|
1366
|
-
const runtime = this.runtimes.get(record.id);
|
|
1367
|
-
if (record.recovery && record.state === 'active' && !record.cancelRequested && !record.error
|
|
1368
|
-
&& runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling) {
|
|
1369
|
-
runtime.releasing = true;
|
|
1370
|
-
record.recovery.status = 'pending';
|
|
1371
|
-
record.error = 'Native restart handoff pending verification.';
|
|
1372
|
-
}
|
|
1373
|
-
else {
|
|
1374
|
-
if (record.recovery)
|
|
1375
|
-
record.recovery.status = 'blocked';
|
|
1376
|
-
record.error = 'Native schedule interrupted by service shutdown.';
|
|
1377
|
-
}
|
|
1378
|
-
record.state = 'interrupted';
|
|
1379
|
-
}
|
|
1380
|
-
}
|
|
1381
|
-
if (this.initialized && !this.fault)
|
|
1382
|
-
await this.persist();
|
|
1383
|
-
}).catch(error => { failure = error; });
|
|
1384
|
-
const stops = await Promise.allSettled([...this.runtimes].map(async ([id, runtime]) => {
|
|
1385
|
-
if (!runtime.releasing)
|
|
1386
|
-
return this.stop(id, runtime);
|
|
1387
|
-
try {
|
|
1388
|
-
runtime.releasePromise ??= runtime.handle.releaseForRestart();
|
|
1389
|
-
let timer;
|
|
1390
|
-
try {
|
|
1391
|
-
await Promise.race([runtime.releasePromise, new Promise((_, reject) => {
|
|
1392
|
-
timer = setTimeout(() => reject(new Error('Native shutdown handoff timed out; ownership retained.')), this.options.recoveryTimeoutMs ?? 30_000);
|
|
1393
|
-
})]);
|
|
1394
|
-
}
|
|
1395
|
-
finally {
|
|
1396
|
-
if (timer)
|
|
1397
|
-
clearTimeout(timer);
|
|
1398
|
-
}
|
|
1399
|
-
await this.serial(async () => {
|
|
1400
|
-
const record = this.records.get(id);
|
|
1401
|
-
if (record.recovery?.status !== 'pending')
|
|
1402
|
-
throw new Error('Occurrence raced native handoff; recovery is blocked.');
|
|
1403
|
-
runtime.stopped = true;
|
|
1404
|
-
record.error = 'Waiting for native recovery after orderly shutdown.';
|
|
1405
|
-
await this.persist();
|
|
1406
|
-
});
|
|
1407
|
-
}
|
|
1408
|
-
catch (error) {
|
|
1409
|
-
await this.blockRecovery(this.records.get(id), errorText(error));
|
|
1410
|
-
throw error;
|
|
1411
|
-
}
|
|
1412
|
-
}));
|
|
1413
|
-
while (this.operations.size)
|
|
1414
|
-
await Promise.allSettled([...this.operations]);
|
|
1415
|
-
while (this.tasks.size)
|
|
1416
|
-
await Promise.allSettled([...this.tasks]);
|
|
1417
|
-
await this.deliveryTail;
|
|
991
|
+
// Complete only writes already in this process's serial filesystem queue.
|
|
992
|
+
// Native requests, results and delivery are independently durable or remain unknown after exit.
|
|
1418
993
|
await this.tail;
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
this.
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
throw error;
|
|
1428
|
-
});
|
|
994
|
+
// An already admitted CREATE may still return a deletion identity. Keep
|
|
995
|
+
// catalog ownership until that receipt is saved, without awaiting native
|
|
996
|
+
// work here. Process exit releases the kernel gate even if it never returns.
|
|
997
|
+
if (!this.pendingCreates) {
|
|
998
|
+
this.catalogGate?.close();
|
|
999
|
+
this.catalogGate = undefined;
|
|
1000
|
+
}
|
|
1001
|
+
})();
|
|
1429
1002
|
}
|
|
1430
1003
|
async cancelNative(id, runtime) {
|
|
1431
1004
|
try {
|
|
@@ -1482,37 +1055,38 @@ class ScheduleService {
|
|
|
1482
1055
|
const runtime = this.runtimes.get(id);
|
|
1483
1056
|
await this.serial(async () => {
|
|
1484
1057
|
if (!runtime || this.runtimes.get(id) !== runtime || !this.records.has(id)) {
|
|
1485
|
-
if (event.type === 'rebound')
|
|
1486
|
-
throw new Error('Native rebound owner retired.');
|
|
1487
1058
|
return;
|
|
1488
1059
|
}
|
|
1489
1060
|
if (this.fault)
|
|
1490
1061
|
throw this.fault;
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
throw new Error('Native handoff exit unconfirmed.');
|
|
1495
|
-
runtime.stopped = true;
|
|
1062
|
+
if (this.closing) {
|
|
1063
|
+
if (event.type === 'attributed')
|
|
1064
|
+
throw new Error('Schedule service closed before native creation attribution.');
|
|
1496
1065
|
return;
|
|
1497
1066
|
}
|
|
1498
|
-
|
|
1067
|
+
const record = this.records.get(id);
|
|
1068
|
+
if (event.type === 'attributed') {
|
|
1069
|
+
if (!record.recovery || record.state !== 'creating' || !this.current(record) || record.cancelRequested) {
|
|
1070
|
+
throw new Error('Native creation attribution is no longer admissible.');
|
|
1071
|
+
}
|
|
1072
|
+
text(event.cleanupIdentity, 8192);
|
|
1073
|
+
record.recovery.identity = event.cleanupIdentity;
|
|
1074
|
+
await this.persist();
|
|
1499
1075
|
return;
|
|
1076
|
+
}
|
|
1500
1077
|
// Process exit evidence is independent of task state, including late exit
|
|
1501
1078
|
// after a failed cancellation cleanup. Never downgrade confirmed exit.
|
|
1502
1079
|
if (event.type === 'stopped' && event.exitConfirmed !== false) {
|
|
1503
1080
|
runtime.stopped = true;
|
|
1504
|
-
this.releaseRecoveryAdmission(runtime);
|
|
1505
1081
|
if (record.error?.startsWith('Native process exit unconfirmed:')) {
|
|
1506
1082
|
delete record.error;
|
|
1507
1083
|
await this.persist();
|
|
1508
1084
|
}
|
|
1509
1085
|
}
|
|
1510
|
-
if (runtime.releasing && event.type === 'stopped' && event.exitConfirmed !== false)
|
|
1511
|
-
return;
|
|
1512
1086
|
const ownerCurrent = this.current(record);
|
|
1513
|
-
if (
|
|
1514
|
-
|
|
1515
|
-
if (
|
|
1087
|
+
if (this.closing)
|
|
1088
|
+
return;
|
|
1089
|
+
if (!ownerCurrent) {
|
|
1516
1090
|
if (record.recovery && (!ownerCurrent || event.type !== 'stopped' || event.exitConfirmed === false))
|
|
1517
1091
|
record.recovery.status = 'blocked';
|
|
1518
1092
|
if (LIVE.has(record.state)) {
|
|
@@ -1527,25 +1101,14 @@ class ScheduleService {
|
|
|
1527
1101
|
await this.persist();
|
|
1528
1102
|
return;
|
|
1529
1103
|
}
|
|
1530
|
-
if (event.type === 'rebound') {
|
|
1531
|
-
// A provider's clean same-conversation handoff changes definition identity,
|
|
1532
|
-
// never occurrence identity. Serialize with cancellation and outbox writes.
|
|
1533
|
-
text(event.nativeId, 8192);
|
|
1534
|
-
text(event.previousNativeId, 8192);
|
|
1535
|
-
if (record.state !== 'active' || record.cancelRequested || runtime.running.size
|
|
1536
|
-
|| record.nativeId !== event.previousNativeId || runtime.handle?.nativeId !== event.nativeId) {
|
|
1537
|
-
throw new Error('Native definition rebound does not match active ownership.');
|
|
1538
|
-
}
|
|
1539
|
-
record.nativeId = event.nativeId;
|
|
1540
|
-
await this.persist();
|
|
1541
|
-
return;
|
|
1542
|
-
}
|
|
1543
1104
|
const finishingFailed = record.state === 'failed' && event.type === 'completed' && runtime.running.has(event.occurrenceId);
|
|
1544
1105
|
const finishingCancelled = record.state === 'cancelled' && runtime.handle?.cancellationKeepsRunning
|
|
1545
1106
|
&& event.type === 'completed' && runtime.running.has(event.occurrenceId);
|
|
1546
1107
|
if ((!finishingCancelled && record.state === 'cancelled') || record.state === 'failed' && !finishingFailed || record.state === 'completed')
|
|
1547
1108
|
return;
|
|
1548
1109
|
if (event.type === 'stopped') {
|
|
1110
|
+
if (record.recovery)
|
|
1111
|
+
record.recovery.status = 'blocked';
|
|
1549
1112
|
if (LIVE.has(record.state)) {
|
|
1550
1113
|
record.state = 'interrupted';
|
|
1551
1114
|
record.error = bounded(event.reason, ERROR_BYTES);
|
|
@@ -1562,18 +1125,6 @@ class ScheduleService {
|
|
|
1562
1125
|
record.error = 'Occurrence tracking limit reached; native schedule stopped.';
|
|
1563
1126
|
cleanup = true;
|
|
1564
1127
|
}
|
|
1565
|
-
else if (event.type === 'skipped') {
|
|
1566
|
-
if (record.recovery?.handoffPolicy !== 'skip-unadmitted' || runtime.running.has(event.occurrenceId))
|
|
1567
|
-
throw new Error('Unnegotiated occurrence skip.');
|
|
1568
|
-
runtime.completed.add(event.occurrenceId);
|
|
1569
|
-
record.completedOccurrences = [...runtime.completed];
|
|
1570
|
-
// No delivery and no invented success. Unseen skipped wakes are not counted.
|
|
1571
|
-
(record.skippedOccurrences ??= []).push(event.occurrenceId);
|
|
1572
|
-
if (record.rule.kind === 'once') {
|
|
1573
|
-
record.state = 'interrupted';
|
|
1574
|
-
record.error = bounded(event.reason, ERROR_BYTES);
|
|
1575
|
-
}
|
|
1576
|
-
}
|
|
1577
1128
|
else if (event.type === 'started') {
|
|
1578
1129
|
runtime.running.add(event.occurrenceId);
|
|
1579
1130
|
record.state = 'running';
|
|
@@ -1606,7 +1157,7 @@ class ScheduleService {
|
|
|
1606
1157
|
else if (record.state !== 'interrupted' && record.state !== 'failed') {
|
|
1607
1158
|
record.state = runtime.running.size ? 'running' : 'active';
|
|
1608
1159
|
if (record.recovery && !runtime.running.size)
|
|
1609
|
-
record.recovery.status = '
|
|
1160
|
+
record.recovery.status = 'blocked';
|
|
1610
1161
|
}
|
|
1611
1162
|
}
|
|
1612
1163
|
}
|
|
@@ -1619,14 +1170,10 @@ class ScheduleService {
|
|
|
1619
1170
|
// stop() records retryable exit failures itself. A cleanup timeout must not
|
|
1620
1171
|
// fault the entire service or prevent a later explicit cancellation retry.
|
|
1621
1172
|
// Persistence failures still set the service fault inside persist().
|
|
1622
|
-
if (cleanup && !runtime.cancelling)
|
|
1173
|
+
if (cleanup && !this.closing && !runtime.cancelling)
|
|
1623
1174
|
this.track(this.stop(id, runtime).catch(() => { }));
|
|
1624
1175
|
if (delivery) {
|
|
1625
|
-
|
|
1626
|
-
if (this.options.deferDelivery)
|
|
1627
|
-
this.track(delivering);
|
|
1628
|
-
else
|
|
1629
|
-
await delivering;
|
|
1176
|
+
await this.attemptDeliveries([delivery], true);
|
|
1630
1177
|
}
|
|
1631
1178
|
}
|
|
1632
1179
|
attemptDeliveries(ids, firstAttempt = false) {
|
|
@@ -1676,25 +1223,27 @@ class ScheduleService {
|
|
|
1676
1223
|
finally {
|
|
1677
1224
|
this.delivering.delete(delivery.schedule.id);
|
|
1678
1225
|
}
|
|
1679
|
-
await this.serial(async () => {
|
|
1226
|
+
await this.serial(async () => {
|
|
1227
|
+
this.pending.delete(id);
|
|
1228
|
+
await this.persist();
|
|
1229
|
+
});
|
|
1680
1230
|
}
|
|
1681
1231
|
});
|
|
1682
1232
|
this.deliveryTail = run.catch(() => { });
|
|
1683
1233
|
return run;
|
|
1684
1234
|
}
|
|
1685
1235
|
stop(id, runtime) {
|
|
1686
|
-
if (this.runtimes.get(id) !== runtime)
|
|
1236
|
+
if (this.closing || this.runtimes.get(id) !== runtime)
|
|
1687
1237
|
return Promise.resolve();
|
|
1688
1238
|
if (runtime.stopped)
|
|
1689
1239
|
return this.rememberCleanup(id, runtime);
|
|
1690
1240
|
if (runtime.stopPromise)
|
|
1691
|
-
return runtime.
|
|
1241
|
+
return runtime.stopPromise;
|
|
1692
1242
|
const operation = (async () => {
|
|
1693
1243
|
try {
|
|
1694
|
-
if (runtime.recoveryAbort?.signal.aborted && !runtime.handle && !runtime.stopped) {
|
|
1695
|
-
throw new Error('Recovery receipt is unresolved; native ownership retained.');
|
|
1696
|
-
}
|
|
1697
1244
|
const handle = await runtime.ready;
|
|
1245
|
+
if (this.closing)
|
|
1246
|
+
return;
|
|
1698
1247
|
if (handle && !runtime.stopped) {
|
|
1699
1248
|
await handle.stop();
|
|
1700
1249
|
// Process/job retirement is not itself proof that persisted timing was deleted.
|
|
@@ -1706,7 +1255,6 @@ class ScheduleService {
|
|
|
1706
1255
|
if (handle?.retirementConfirmed !== false && (!handle?.cancellationKeepsRunning || !runtime.running.size))
|
|
1707
1256
|
runtime.stopped = true;
|
|
1708
1257
|
if (runtime.stopped) {
|
|
1709
|
-
this.releaseRecoveryAdmission(runtime);
|
|
1710
1258
|
await this.rememberCleanup(id, runtime);
|
|
1711
1259
|
}
|
|
1712
1260
|
}
|
|
@@ -1723,7 +1271,7 @@ class ScheduleService {
|
|
|
1723
1271
|
runtime.stopPromise = tracked;
|
|
1724
1272
|
void operation.finally(() => { if (!runtime.stopped)
|
|
1725
1273
|
runtime.stopPromise = undefined; }).catch(() => { });
|
|
1726
|
-
return
|
|
1274
|
+
return tracked;
|
|
1727
1275
|
}
|
|
1728
1276
|
rememberCleanup(id, runtime) {
|
|
1729
1277
|
return this.serial(async () => {
|
|
@@ -1736,22 +1284,10 @@ class ScheduleService {
|
|
|
1736
1284
|
}
|
|
1737
1285
|
});
|
|
1738
1286
|
}
|
|
1739
|
-
async boundRecovery(operation, message) {
|
|
1740
|
-
let timer;
|
|
1741
|
-
try {
|
|
1742
|
-
return await Promise.race([operation, new Promise((_, reject) => {
|
|
1743
|
-
timer = setTimeout(() => reject(new Error(message)), this.options.recoveryTimeoutMs ?? 30_000);
|
|
1744
|
-
})]);
|
|
1745
|
-
}
|
|
1746
|
-
finally {
|
|
1747
|
-
if (timer)
|
|
1748
|
-
clearTimeout(timer);
|
|
1749
|
-
}
|
|
1750
|
-
}
|
|
1751
1287
|
canAdmitOccurrence(id, runtime) {
|
|
1752
1288
|
const record = this.records.get(id);
|
|
1753
1289
|
return !!record && this.runtimes.get(id) === runtime && !this.closing && !this.fault
|
|
1754
|
-
&& !runtime.stopped && !runtime.cancelling && !
|
|
1290
|
+
&& !runtime.stopped && !runtime.cancelling && !record.cancelRequested
|
|
1755
1291
|
&& LIVE.has(record.state) && this.current(record);
|
|
1756
1292
|
}
|
|
1757
1293
|
newRuntime() {
|
|
@@ -1805,7 +1341,7 @@ class ScheduleService {
|
|
|
1805
1341
|
checkReady() {
|
|
1806
1342
|
if (this.fault)
|
|
1807
1343
|
throw this.fault;
|
|
1808
|
-
if (!this.initialized || this.closing
|
|
1344
|
+
if (!this.initialized || this.closing)
|
|
1809
1345
|
throw new Error('Schedule service is not initialized or is closed.');
|
|
1810
1346
|
}
|
|
1811
1347
|
serial(operation) {
|
|
@@ -1856,7 +1392,7 @@ class ScheduleService {
|
|
|
1856
1392
|
}
|
|
1857
1393
|
load(data) {
|
|
1858
1394
|
const catalog = object(data, ['version', 'records', 'pending', 'purged', 'cleanup', 'purgeRequests']);
|
|
1859
|
-
if (![1, 2, 3, 4, 5].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
|
|
1395
|
+
if (![1, 2, 3, 4, 5, 6].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
|
|
1860
1396
|
|| !Array.isArray(catalog.pending) || catalog.pending.length > this.maxPending)
|
|
1861
1397
|
throw new Error('Invalid schedule catalog envelope.');
|
|
1862
1398
|
const records = new Map();
|
|
@@ -1893,8 +1429,8 @@ class ScheduleService {
|
|
|
1893
1429
|
if (record.recovery !== undefined) {
|
|
1894
1430
|
if (catalog.version === 1)
|
|
1895
1431
|
throw new Error('Recovery requires catalog version 2 or newer.');
|
|
1896
|
-
const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'futureOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy']);
|
|
1897
|
-
this.
|
|
1432
|
+
const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'futureOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy', 'abrupt']);
|
|
1433
|
+
this.validateOwnershipMetadata(recovery);
|
|
1898
1434
|
if (!['eligible', 'pending', 'blocked'].includes(recovery.status))
|
|
1899
1435
|
throw new Error('Invalid recovery status.');
|
|
1900
1436
|
}
|
|
@@ -1951,7 +1487,7 @@ class ScheduleService {
|
|
|
1951
1487
|
throw new Error('Purge fences require catalog version 3 or newer.');
|
|
1952
1488
|
const cleanup = new Map();
|
|
1953
1489
|
const purgeRequests = new Map();
|
|
1954
|
-
if (catalog.version === 5) {
|
|
1490
|
+
if (catalog.version === 5 || catalog.version === 6) {
|
|
1955
1491
|
if (!Array.isArray(catalog.cleanup) || catalog.cleanup.length > MAX_CLEANUP_RECORDS)
|
|
1956
1492
|
throw new Error('Invalid cleanup ledger.');
|
|
1957
1493
|
for (const value of catalog.cleanup) {
|
|
@@ -1997,7 +1533,7 @@ class ScheduleService {
|
|
|
1997
1533
|
}
|
|
1998
1534
|
}
|
|
1999
1535
|
else if (catalog.cleanup !== undefined || catalog.purgeRequests !== undefined)
|
|
2000
|
-
throw new Error('Cleanup ledger requires catalog version 5.');
|
|
1536
|
+
throw new Error('Cleanup ledger requires catalog version 5 or newer.');
|
|
2001
1537
|
this.records = records;
|
|
2002
1538
|
this.pending = pending;
|
|
2003
1539
|
this.purged = purged;
|
|
@@ -2023,12 +1559,9 @@ class ScheduleService {
|
|
|
2023
1559
|
}
|
|
2024
1560
|
const handle = await fs.open(temporary, 'wx', 0o600);
|
|
2025
1561
|
try {
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
...(this.cleanup.size || this.purgeRequests.size ? { cleanup: [...this.cleanup.values()],
|
|
2030
|
-
purgeRequests: [...this.purgeRequests].map(([key, receipt]) => ({ key, ...receipt })) } : {}),
|
|
2031
|
-
...(upgraded ? { purged: [...this.purged].map(([id, owner]) => ({ id, ...owner })) } : {}) }), 'utf8');
|
|
1562
|
+
await handle.writeFile(JSON.stringify({ version: 6, records: [...this.records.values()], pending: [...this.pending.values()],
|
|
1563
|
+
cleanup: [...this.cleanup.values()], purgeRequests: [...this.purgeRequests].map(([key, receipt]) => ({ key, ...receipt })),
|
|
1564
|
+
purged: [...this.purged].map(([id, owner]) => ({ id, ...owner })) }), 'utf8');
|
|
2032
1565
|
await handle.sync();
|
|
2033
1566
|
}
|
|
2034
1567
|
finally {
|