@yu_robotics/remote-cli 1.6.253 → 1.6.259
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/MessageHandler.d.ts +5 -12
- package/dist/client/MessageHandler.d.ts.map +1 -1
- package/dist/client/MessageHandler.js +53 -252
- package/dist/client/MessageHandler.js.map +1 -1
- package/dist/client/WebSocketClient.d.ts +0 -3
- package/dist/client/WebSocketClient.d.ts.map +1 -1
- package/dist/client/WebSocketClient.js +1 -3
- package/dist/client/WebSocketClient.js.map +1 -1
- package/dist/commands/start.d.ts +0 -5
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +0 -28
- package/dist/commands/start.js.map +1 -1
- package/dist/executor/AcpExecutor.d.ts.map +1 -1
- package/dist/executor/AcpExecutor.js.map +1 -1
- package/dist/executor/AgyExecutor.d.ts +4 -11
- package/dist/executor/AgyExecutor.d.ts.map +1 -1
- package/dist/executor/AgyExecutor.js +71 -101
- package/dist/executor/AgyExecutor.js.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.d.ts +3 -7
- package/dist/executor/ClaudePersistentExecutor.d.ts.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.js +14 -224
- package/dist/executor/ClaudePersistentExecutor.js.map +1 -1
- package/dist/executor/DshExecutor.d.ts +7 -14
- package/dist/executor/DshExecutor.d.ts.map +1 -1
- package/dist/executor/DshExecutor.js +48 -50
- package/dist/executor/DshExecutor.js.map +1 -1
- package/dist/executor/IExecutor.d.ts +3 -3
- package/dist/executor/IExecutor.d.ts.map +1 -1
- package/dist/executor/acp/AcpClient.d.ts +15 -1
- package/dist/executor/acp/AcpClient.d.ts.map +1 -1
- package/dist/executor/acp/AcpClient.js +85 -7
- package/dist/executor/acp/AcpClient.js.map +1 -1
- package/dist/executor/agy/AgyAcpDiagnostics.d.ts +8 -0
- package/dist/executor/agy/AgyAcpDiagnostics.d.ts.map +1 -0
- package/dist/executor/agy/AgyAcpDiagnostics.js +45 -0
- package/dist/executor/agy/AgyAcpDiagnostics.js.map +1 -0
- package/dist/executor/agy/AgyAcpRuntime.d.ts +9 -16
- package/dist/executor/agy/AgyAcpRuntime.d.ts.map +1 -1
- package/dist/executor/agy/AgyAcpRuntime.js +67 -151
- package/dist/executor/agy/AgyAcpRuntime.js.map +1 -1
- package/dist/executor/agy/AgySessionSelection.d.ts +7 -0
- package/dist/executor/agy/AgySessionSelection.d.ts.map +1 -0
- package/dist/executor/agy/AgySessionSelection.js +60 -0
- package/dist/executor/agy/AgySessionSelection.js.map +1 -0
- package/dist/executor/agy/AgySessionSettings.d.ts +13 -0
- package/dist/executor/agy/AgySessionSettings.d.ts.map +1 -0
- package/dist/executor/agy/AgySessionSettings.js +60 -0
- package/dist/executor/agy/AgySessionSettings.js.map +1 -0
- package/dist/executor/dsh/DshAcpClient.d.ts +1 -1
- package/dist/executor/dsh/DshAcpClient.d.ts.map +1 -1
- package/dist/executor/dsh/DshAcpClient.js +3 -1
- package/dist/executor/dsh/DshAcpClient.js.map +1 -1
- package/dist/executor/index.d.ts +1 -0
- package/dist/executor/index.d.ts.map +1 -1
- package/dist/executor/index.js +1 -0
- package/dist/executor/index.js.map +1 -1
- package/dist/index.js +0 -2
- package/dist/index.js.map +1 -1
- package/dist/maintenance/release-notes.json +1 -1
- package/dist/scheduling/ScheduleCapabilities.d.ts +2 -2
- package/dist/scheduling/ScheduleCapabilities.js +4 -4
- package/dist/scheduling/ScheduleCapabilities.js.map +1 -1
- package/dist/scheduling/SchedulePurgeVerification.js +1 -1
- package/dist/scheduling/SchedulePurgeVerification.js.map +1 -1
- package/dist/scheduling/ScheduleService.d.ts +10 -44
- package/dist/scheduling/ScheduleService.d.ts.map +1 -1
- package/dist/scheduling/ScheduleService.js +164 -715
- package/dist/scheduling/ScheduleService.js.map +1 -1
- package/dist/scheduling/agy/AgySessionSchedules.d.ts +5 -5
- package/dist/scheduling/agy/AgySessionSchedules.d.ts.map +1 -1
- package/dist/scheduling/agy/AgySessionSchedules.js +20 -17
- package/dist/scheduling/agy/AgySessionSchedules.js.map +1 -1
- package/dist/scheduling/claude/ClaudeScheduleOwnership.d.ts +2 -6
- package/dist/scheduling/claude/ClaudeScheduleOwnership.d.ts.map +1 -1
- package/dist/scheduling/claude/ClaudeScheduleOwnership.js +12 -67
- package/dist/scheduling/claude/ClaudeScheduleOwnership.js.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts +4 -26
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.js +22 -145
- package/dist/scheduling/claude/ClaudeSessionSchedules.js.map +1 -1
- package/dist/scheduling/dsh/ScheduledSessionGuard.d.ts +12 -0
- package/dist/scheduling/dsh/ScheduledSessionGuard.d.ts.map +1 -0
- package/dist/scheduling/dsh/ScheduledSessionGuard.js +96 -0
- package/dist/scheduling/dsh/ScheduledSessionGuard.js.map +1 -0
- package/dist/scheduling/dsh/SessionScheduleBridge.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleBridge.js +0 -25
- package/dist/scheduling/dsh/SessionScheduleBridge.js.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.js +0 -8
- package/dist/scheduling/dsh/SessionScheduleClient.js.map +1 -1
- package/dist/scheduling/dsh/sessionLaunch.js +1 -1
- package/dist/scheduling/dsh/sessionLaunch.js.map +1 -1
- package/dist/scheduling/types.d.ts +26 -77
- package/dist/scheduling/types.d.ts.map +1 -1
- package/dist/scheduling/types.js +1 -10
- package/dist/scheduling/types.js.map +1 -1
- package/dist/service/ServiceManager.js +1 -1
- package/dist/thread/ThreadExecutorPool.d.ts +4 -1
- package/dist/thread/ThreadExecutorPool.d.ts.map +1 -1
- package/dist/thread/ThreadExecutorPool.js +6 -3
- package/dist/thread/ThreadExecutorPool.js.map +1 -1
- package/dist/thread/ThreadManager.d.ts.map +1 -1
- package/dist/thread/ThreadManager.js +23 -2
- package/dist/thread/ThreadManager.js.map +1 -1
- package/native/prebuilds/darwin-arm64/managed_native.node +0 -0
- package/native/prebuilds/darwin-arm64/manifest.json +1 -1
- package/native/prebuilds/darwin-x64/managed_native.node +0 -0
- package/native/prebuilds/darwin-x64/manifest.json +1 -1
- package/native/prebuilds/win32-arm64/managed_native.node +0 -0
- package/native/prebuilds/win32-arm64/manifest.json +1 -1
- package/native/prebuilds/win32-x64/managed_native.node +0 -0
- package/native/prebuilds/win32-x64/manifest.json +1 -1
- package/package.json +1 -1
- package/dist/scheduling/agy/AgyCheckpoint.d.ts +0 -35
- package/dist/scheduling/agy/AgyCheckpoint.d.ts.map +0 -1
- package/dist/scheduling/agy/AgyCheckpoint.js +0 -160
- package/dist/scheduling/agy/AgyCheckpoint.js.map +0 -1
- package/dist/service/SessionHostSupervisor.d.ts +0 -63
- package/dist/service/SessionHostSupervisor.d.ts.map +0 -1
- package/dist/service/SessionHostSupervisor.js +0 -323
- package/dist/service/SessionHostSupervisor.js.map +0 -1
- package/dist/sessionHost/HostClient.d.ts +0 -49
- package/dist/sessionHost/HostClient.d.ts.map +0 -1
- package/dist/sessionHost/HostClient.js +0 -101
- package/dist/sessionHost/HostClient.js.map +0 -1
- package/dist/sessionHost/HostProtocol.d.ts +0 -65
- package/dist/sessionHost/HostProtocol.d.ts.map +0 -1
- package/dist/sessionHost/HostProtocol.js +0 -78
- package/dist/sessionHost/HostProtocol.js.map +0 -1
- package/dist/sessionHost/HostServer.d.ts +0 -52
- package/dist/sessionHost/HostServer.d.ts.map +0 -1
- package/dist/sessionHost/HostServer.js +0 -413
- package/dist/sessionHost/HostServer.js.map +0 -1
- package/dist/sessionHost/HostStore.d.ts +0 -43
- package/dist/sessionHost/HostStore.d.ts.map +0 -1
- package/dist/sessionHost/HostStore.js +0 -206
- package/dist/sessionHost/HostStore.js.map +0 -1
- package/dist/sessionHost/RetainedFrontend.d.ts +0 -63
- package/dist/sessionHost/RetainedFrontend.d.ts.map +0 -1
- package/dist/sessionHost/RetainedFrontend.js +0 -244
- package/dist/sessionHost/RetainedFrontend.js.map +0 -1
- package/dist/sessionHost/RetainedHostLauncher.d.ts +0 -25
- package/dist/sessionHost/RetainedHostLauncher.d.ts.map +0 -1
- package/dist/sessionHost/RetainedHostLauncher.js +0 -132
- package/dist/sessionHost/RetainedHostLauncher.js.map +0 -1
- package/dist/sessionHost/RetainedRuntime.d.ts +0 -62
- package/dist/sessionHost/RetainedRuntime.d.ts.map +0 -1
- package/dist/sessionHost/RetainedRuntime.js +0 -478
- package/dist/sessionHost/RetainedRuntime.js.map +0 -1
- package/dist/sessionHost/RuntimeJournal.d.ts +0 -47
- package/dist/sessionHost/RuntimeJournal.d.ts.map +0 -1
- package/dist/sessionHost/RuntimeJournal.js +0 -103
- package/dist/sessionHost/RuntimeJournal.js.map +0 -1
- package/dist/sessionHost/entry.d.ts +0 -13
- package/dist/sessionHost/entry.d.ts.map +0 -1
- package/dist/sessionHost/entry.js +0 -107
- package/dist/sessionHost/entry.js.map +0 -1
- package/dist/update/PinnedHostBundle.d.ts +0 -49
- package/dist/update/PinnedHostBundle.d.ts.map +0 -1
- package/dist/update/PinnedHostBundle.js +0 -309
- package/dist/update/PinnedHostBundle.js.map +0 -1
|
@@ -96,11 +96,11 @@ function sameOwner(left, right) {
|
|
|
96
96
|
function sameCleanupOwner(left, right) {
|
|
97
97
|
return left.threadId === right.threadId && left.openId === right.openId;
|
|
98
98
|
}
|
|
99
|
-
/**
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
/** Unknown native mutation/queued reminders remain cleanup obligations after host loss. */
|
|
100
|
+
function unsettledNativeWork(record) {
|
|
101
|
+
return !record.creationRefused && !record.cleanupConfirmed
|
|
102
|
+
&& (['interrupted', 'failed', 'creating'].includes(record.state)
|
|
103
|
+
|| !!record.error?.startsWith('Native process exit unconfirmed:'));
|
|
104
104
|
}
|
|
105
105
|
function deliveryId(scheduleId, occurrenceId) {
|
|
106
106
|
return `${scheduleId}:${occurrenceId}`;
|
|
@@ -137,15 +137,10 @@ class ScheduleService {
|
|
|
137
137
|
initialized = false;
|
|
138
138
|
closing = false;
|
|
139
139
|
destruction;
|
|
140
|
+
pendingCreates = 0;
|
|
140
141
|
fault;
|
|
141
142
|
catalogPath = '';
|
|
142
|
-
recoveryRun;
|
|
143
143
|
catalogGate;
|
|
144
|
-
suspending = false;
|
|
145
|
-
handoffUnresolved = false;
|
|
146
|
-
suspension;
|
|
147
|
-
hostRetirement;
|
|
148
|
-
hostsRetired = false;
|
|
149
144
|
constructor(options) {
|
|
150
145
|
this.options = { ...options };
|
|
151
146
|
this.now = options.now ?? Date.now;
|
|
@@ -155,9 +150,6 @@ class ScheduleService {
|
|
|
155
150
|
throw new Error('Invalid schedule capacity.');
|
|
156
151
|
return result;
|
|
157
152
|
};
|
|
158
|
-
if (options.recoveryTimeoutMs !== undefined && (!Number.isInteger(options.recoveryTimeoutMs)
|
|
159
|
-
|| options.recoveryTimeoutMs < 1 || options.recoveryTimeoutMs > 60_000))
|
|
160
|
-
throw new Error('Invalid recovery timeout.');
|
|
161
153
|
this.maxLive = limit(options.maxLiveSchedules, 5, 5);
|
|
162
154
|
this.maxRecords = limit(options.maxRecords, 100, 1000);
|
|
163
155
|
this.maxPending = limit(options.maxPendingDeliveries, 100, 1000);
|
|
@@ -182,35 +174,13 @@ class ScheduleService {
|
|
|
182
174
|
await this.compactRefusedCreation(record);
|
|
183
175
|
let changed = false;
|
|
184
176
|
for (const record of this.records.values()) {
|
|
185
|
-
if (
|
|
186
|
-
if (record.recovery?.status === 'pending') {
|
|
187
|
-
record.recovery.status = 'blocked';
|
|
188
|
-
changed = true;
|
|
189
|
-
}
|
|
190
|
-
continue; // Terminal tombstones always win over a stale recovery checkpoint.
|
|
191
|
-
}
|
|
192
|
-
// A consumed future-only attempt or an already blocked interrupted row
|
|
193
|
-
// cannot regain automatic admission after another service restart.
|
|
194
|
-
if (record.recovery?.futureOnly && (record.state === 'creating'
|
|
195
|
-
|| record.state === 'interrupted' && record.recovery.status === 'blocked')) {
|
|
196
|
-
record.state = 'failed';
|
|
177
|
+
if (record.recovery && record.recovery.status !== 'blocked') {
|
|
197
178
|
record.recovery.status = 'blocked';
|
|
198
|
-
record.error ??= 'Interrupted native restoration quarantined; original session and cleanup evidence retained.';
|
|
199
179
|
changed = true;
|
|
200
|
-
continue;
|
|
201
180
|
}
|
|
202
|
-
if (LIVE.has(record.state)
|
|
203
|
-
|| record.state === 'interrupted' && record.recovery?.futureOnly && record.rule.kind === 'cron') {
|
|
204
|
-
const futureOnly = record.recovery?.futureOnly && record.rule.kind === 'cron';
|
|
205
|
-
const recoverable = record.recovery && !record.cancelRequested && this.current(record)
|
|
206
|
-
&& (record.state !== 'running' || futureOnly) && record.state !== 'failed'
|
|
207
|
-
&& (!record.recovery.orderlyOnly || record.recovery.handoff === 'clean' || futureOnly)
|
|
208
|
-
&& (record.state === 'active' || record.recovery.status === 'pending' || futureOnly);
|
|
209
|
-
if (record.recovery)
|
|
210
|
-
record.recovery.status = recoverable ? 'pending' : 'blocked';
|
|
211
|
-
record.error = recoverable ? 'Waiting for native recovery capability and owner readiness.'
|
|
212
|
-
: record.recovery?.orderlyOnly ? 'Dirty shutdown: original native work is unconfirmed; automatic reconstruction is blocked.' : RESTART_REASON;
|
|
181
|
+
if (LIVE.has(record.state)) {
|
|
213
182
|
record.state = 'interrupted';
|
|
183
|
+
record.error = RESTART_REASON;
|
|
214
184
|
changed = true;
|
|
215
185
|
}
|
|
216
186
|
}
|
|
@@ -221,229 +191,16 @@ class ScheduleService {
|
|
|
221
191
|
changed = true;
|
|
222
192
|
}
|
|
223
193
|
}
|
|
224
|
-
if (changed || data === undefined)
|
|
194
|
+
if (changed || data === undefined || data.version !== 6)
|
|
225
195
|
await this.persist();
|
|
226
196
|
this.initialized = true;
|
|
227
197
|
}).catch(error => { this.catalogGate?.close(); this.catalogGate = undefined; throw error; });
|
|
228
198
|
}
|
|
229
|
-
|
|
230
|
-
recover() {
|
|
231
|
-
this.checkReady();
|
|
232
|
-
if (this.recoveryRun)
|
|
233
|
-
return this.recoveryRun;
|
|
234
|
-
const run = this.operation(this.recoverWaiting());
|
|
235
|
-
this.recoveryRun = run;
|
|
236
|
-
void run.finally(() => { if (this.recoveryRun === run)
|
|
237
|
-
this.recoveryRun = undefined; }).catch(() => { });
|
|
238
|
-
return run;
|
|
239
|
-
}
|
|
240
|
-
async recoverWaiting() {
|
|
241
|
-
const candidates = await this.serial(async () => [...this.records.values()]
|
|
242
|
-
.filter(record => record.recovery?.status === 'pending' && record.state === 'interrupted').map(record => record.id));
|
|
243
|
-
// Catalog capacity is bounded, and only one native admission runs at a time.
|
|
244
|
-
for (const id of candidates.slice(0, this.maxLive)) {
|
|
245
|
-
if (this.closing || this.fault)
|
|
246
|
-
return;
|
|
247
|
-
const record = this.records.get(id);
|
|
248
|
-
if (!record)
|
|
249
|
-
continue;
|
|
250
|
-
if (record.cancelRequested || !this.current(record)) {
|
|
251
|
-
await this.blockRecovery(record, 'Recovery owner retired or cancellation was requested.');
|
|
252
|
-
continue;
|
|
253
|
-
}
|
|
254
|
-
if (record.rule.kind === 'once' && Date.parse(record.rule.at) <= this.now()) {
|
|
255
|
-
await this.blockRecovery(record, 'One-shot deadline expired; no late execution was requested.');
|
|
256
|
-
if (record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent)
|
|
257
|
-
await this.serial(async () => { delete record.recovery; await this.persist(); });
|
|
258
|
-
continue;
|
|
259
|
-
}
|
|
260
|
-
const input = { id, threadId: record.threadId, backend: record.backend,
|
|
261
|
-
cwd: record.cwd, prompt: record.prompt, rule: structuredClone(record.rule), ...record.execution,
|
|
262
|
-
recovery: this.checkpoint(record.recovery) };
|
|
263
|
-
if (!this.options.canRecover || !this.options.canRecover(structuredClone(record))) {
|
|
264
|
-
await this.recoveryDiagnostic(record, 'Original owner configuration or transport is not ready for recovery.');
|
|
265
|
-
continue;
|
|
266
|
-
}
|
|
267
|
-
const provider = this.options.provider.recoveryFor?.(input);
|
|
268
|
-
if (!provider || provider.provider !== record.recovery.provider || provider.strategy !== record.recovery.strategy
|
|
269
|
-
|| (provider.handoffPolicy ?? 'drain-all') !== (record.recovery.handoffPolicy ?? 'drain-all')) {
|
|
270
|
-
await this.recoveryDiagnostic(record, 'Native restart recovery is unsupported or provider identity changed.');
|
|
271
|
-
continue;
|
|
272
|
-
}
|
|
273
|
-
if (this.runtimes.has(id))
|
|
274
|
-
continue; // An unresolved/late operation retains exclusive ownership.
|
|
275
|
-
const restoreRecord = structuredClone(record);
|
|
276
|
-
const runtime = this.newRuntime();
|
|
277
|
-
runtime.completed = new Set(record.completedOccurrences);
|
|
278
|
-
const controller = new AbortController();
|
|
279
|
-
runtime.recoveryAbort = controller;
|
|
280
|
-
// Admission contention has not consumed this job's clean checkpoint.
|
|
281
|
-
// Leave it pending so the exact earlier runtime can release the thread.
|
|
282
|
-
try {
|
|
283
|
-
runtime.releaseAdmission = this.options.acquireRecovery?.(structuredClone(record));
|
|
284
|
-
}
|
|
285
|
-
catch (error) {
|
|
286
|
-
await this.recoveryDiagnostic(record, errorText(error));
|
|
287
|
-
continue;
|
|
288
|
-
}
|
|
289
|
-
this.runtimes.set(id, runtime);
|
|
290
|
-
const current = () => !controller.signal.aborted && !this.closing && !this.fault && !record.cancelRequested
|
|
291
|
-
&& this.current(record) && this.options.canRecover?.(structuredClone(record)) === true;
|
|
292
|
-
const emit = (event) => {
|
|
293
|
-
if (this.runtimes.get(id) !== runtime) {
|
|
294
|
-
if (event.type === 'rebound')
|
|
295
|
-
return Promise.reject(new Error('Native rebound owner retired.'));
|
|
296
|
-
return;
|
|
297
|
-
}
|
|
298
|
-
const task = this.onEvent(id, structuredClone(event));
|
|
299
|
-
this.track(task);
|
|
300
|
-
return task;
|
|
301
|
-
};
|
|
302
|
-
let timer;
|
|
303
|
-
let admitted = false;
|
|
304
|
-
let succeeded = false;
|
|
305
|
-
try {
|
|
306
|
-
await this.serial(async () => {
|
|
307
|
-
if (!current())
|
|
308
|
-
throw new Error('Recovery admission expired.');
|
|
309
|
-
// Consume clean proof before mutation. A crash must not replay an orderly-only restore.
|
|
310
|
-
record.state = 'creating';
|
|
311
|
-
delete record.recovery.handoff;
|
|
312
|
-
delete record.recovery.nativeDefinitionAbsent;
|
|
313
|
-
await this.persist();
|
|
314
|
-
});
|
|
315
|
-
admitted = true;
|
|
316
|
-
runtime.recoveryAdmitted = true;
|
|
317
|
-
const admission = Promise.resolve().then(() => provider.reconcile(restoreRecord, { notBefore: this.now(), signal: controller.signal, isCurrent: current,
|
|
318
|
-
isOwnerCurrent: () => !record.cancelRequested && !this.fault && this.current(record),
|
|
319
|
-
canAdmitOccurrence: () => this.canAdmitOccurrence(id, runtime) }, emit));
|
|
320
|
-
const adopt = async (handle) => {
|
|
321
|
-
runtime.handle = handle;
|
|
322
|
-
runtime.resolveReady(handle);
|
|
323
|
-
text(handle.nativeId);
|
|
324
|
-
await this.serial(async () => { record.nativeId = handle.nativeId; if (!this.fault)
|
|
325
|
-
await this.persist(); });
|
|
326
|
-
};
|
|
327
|
-
const receipt = admission.then(async (handle) => {
|
|
328
|
-
await adopt(handle);
|
|
329
|
-
if (!current()) {
|
|
330
|
-
await this.stop(id, runtime);
|
|
331
|
-
throw new Error('Late recovery receipt requires cleanup; no successor was admitted.');
|
|
332
|
-
}
|
|
333
|
-
if (!await handle.list() || !current())
|
|
334
|
-
throw new Error('Recovered native inventory or owner is unconfirmed.');
|
|
335
|
-
return handle;
|
|
336
|
-
}, async (error) => {
|
|
337
|
-
if (error instanceof types_1.NativeScheduleCleanupError) {
|
|
338
|
-
await adopt(error.handle);
|
|
339
|
-
if (controller.signal.aborted)
|
|
340
|
-
await this.stop(id, runtime).catch(() => { });
|
|
341
|
-
}
|
|
342
|
-
runtime.resolveReady(runtime.handle);
|
|
343
|
-
if (!runtime.handle && (error instanceof types_1.NativeScheduleRecoverySettledError
|
|
344
|
-
|| error instanceof types_1.NativeScheduleNotAdmittedError || error instanceof types_1.NativeScheduleAbsentError)) {
|
|
345
|
-
runtime.stopped = true;
|
|
346
|
-
this.releaseRecoveryAdmission(runtime);
|
|
347
|
-
if (this.runtimes.get(id) === runtime)
|
|
348
|
-
this.runtimes.delete(id);
|
|
349
|
-
}
|
|
350
|
-
// A rejection without a handle is not proof that native admission never happened.
|
|
351
|
-
throw error;
|
|
352
|
-
});
|
|
353
|
-
const handle = await Promise.race([receipt, new Promise((_, reject) => {
|
|
354
|
-
timer = setTimeout(() => {
|
|
355
|
-
controller.abort();
|
|
356
|
-
reject(new Error(record.recovery?.futureOnly
|
|
357
|
-
? 'Native recovery timed out; this job will not restart automatically. Native settlement and cleanup remain separate.'
|
|
358
|
-
: 'Native recovery timed out; ownership retained, do not recreate.'));
|
|
359
|
-
}, this.options.recoveryTimeoutMs ?? 30_000);
|
|
360
|
-
})]);
|
|
361
|
-
// Native recovery and inventory validation settled. A later deadline
|
|
362
|
-
// must not abort the adopted handle while its active checkpoint is saved.
|
|
363
|
-
clearTimeout(timer);
|
|
364
|
-
timer = undefined;
|
|
365
|
-
text(handle.nativeId);
|
|
366
|
-
// The provider's reconcile contract includes inventory validation; list is
|
|
367
|
-
// additionally required to verify the returned handle before activating it.
|
|
368
|
-
if (!current())
|
|
369
|
-
throw new Error('Recovered native owner is unconfirmed.');
|
|
370
|
-
await this.serial(async () => {
|
|
371
|
-
if (!current())
|
|
372
|
-
throw new Error('Recovery owner retired before receipt persistence.');
|
|
373
|
-
record.nativeId = handle.nativeId;
|
|
374
|
-
if (handle.recoveryIdentity) {
|
|
375
|
-
text(handle.recoveryIdentity, 8192);
|
|
376
|
-
record.recovery.identity = handle.recoveryIdentity;
|
|
377
|
-
}
|
|
378
|
-
if (record.state === 'creating')
|
|
379
|
-
record.state = 'active';
|
|
380
|
-
record.recovery.status = 'eligible';
|
|
381
|
-
delete record.error;
|
|
382
|
-
await this.persist();
|
|
383
|
-
});
|
|
384
|
-
succeeded = true;
|
|
385
|
-
}
|
|
386
|
-
catch (error) {
|
|
387
|
-
controller.abort();
|
|
388
|
-
if (!admitted || error instanceof types_1.NativeScheduleAbsentError || error instanceof types_1.NativeScheduleNotAdmittedError || error instanceof types_1.NativeScheduleRecoverySettledError) {
|
|
389
|
-
runtime.stopped = true;
|
|
390
|
-
runtime.resolveReady(undefined);
|
|
391
|
-
}
|
|
392
|
-
// Do not retry an ambiguous mutation or release its handle on a timeout.
|
|
393
|
-
if (admitted)
|
|
394
|
-
await this.blockRecovery(record, errorText(error), !!record.recovery?.futureOnly);
|
|
395
|
-
else
|
|
396
|
-
await this.recoveryDiagnostic(record, errorText(error));
|
|
397
|
-
if (runtime.handle)
|
|
398
|
-
await this.stop(id, runtime).catch(() => { });
|
|
399
|
-
else if (runtime.stopped)
|
|
400
|
-
this.runtimes.delete(id);
|
|
401
|
-
}
|
|
402
|
-
finally {
|
|
403
|
-
if (timer)
|
|
404
|
-
clearTimeout(timer);
|
|
405
|
-
if (succeeded || runtime.stopped) {
|
|
406
|
-
this.releaseRecoveryAdmission(runtime);
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
// A handoff containing only skipped/expired one-shots has nothing to
|
|
411
|
-
// reconcile. It must still release the predecessor's local input fence.
|
|
412
|
-
const released = new Set();
|
|
413
|
-
for (const record of this.records.values()) {
|
|
414
|
-
if (released.has(record.threadId) || this.closing || this.fault || this.handoffUnresolved
|
|
415
|
-
|| !this.current(record) || !this.options.canRecover?.(structuredClone(record)))
|
|
416
|
-
continue;
|
|
417
|
-
if ([...this.records.values()].some(other => other.threadId === record.threadId
|
|
418
|
-
&& (LIVE.has(other.state) || recoveryOwnsWork(other) || this.runtimes.has(other.id))))
|
|
419
|
-
continue;
|
|
420
|
-
this.options.provider.recoveryFor?.({ id: record.id, threadId: record.threadId, backend: record.backend,
|
|
421
|
-
cwd: record.cwd, prompt: record.prompt, rule: record.rule, ...record.execution })?.releaseRestartAdmission?.();
|
|
422
|
-
released.add(record.threadId);
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
releaseRecoveryAdmission(runtime) {
|
|
426
|
-
const release = runtime.releaseAdmission;
|
|
427
|
-
runtime.releaseAdmission = undefined;
|
|
428
|
-
release?.();
|
|
429
|
-
if (!release || !runtime.recoveryAdmitted || !runtime.recoveryAbort?.signal.aborted)
|
|
430
|
-
return;
|
|
431
|
-
// Join the current pass before retrying untouched siblings. Failed rows are
|
|
432
|
-
// never candidates, and canRecover/acquireRecovery recheck newer admission.
|
|
433
|
-
const retry = Promise.resolve(this.recoveryRun).catch(() => { }).then(() => {
|
|
434
|
-
if (!this.closing && !this.fault && this.hasPendingRecovery())
|
|
435
|
-
return this.recover();
|
|
436
|
-
});
|
|
437
|
-
this.track(retry);
|
|
438
|
-
}
|
|
439
|
-
checkpoint(value) {
|
|
440
|
-
return { version: 1, provider: value.provider, strategy: value.strategy, identity: value.identity,
|
|
441
|
-
...(value.orderlyOnly && { orderlyOnly: true }), ...(value.futureOnly && { futureOnly: true }), ...(value.handoffPolicy && { handoffPolicy: value.handoffPolicy }), ...(value.handoff && { handoff: value.handoff }),
|
|
442
|
-
...(value.nativeDefinitionAbsent && { nativeDefinitionAbsent: true }) };
|
|
443
|
-
}
|
|
444
|
-
validateCheckpoint(value) {
|
|
199
|
+
validateOwnershipMetadata(value) {
|
|
445
200
|
if (value.futureOnly !== undefined && value.futureOnly !== true)
|
|
446
201
|
throw new Error('Invalid future recovery policy.');
|
|
202
|
+
if (value.abrupt !== undefined && value.abrupt !== true)
|
|
203
|
+
throw new Error('Invalid abrupt recovery capability.');
|
|
447
204
|
if (value.version !== 1 || !['reattach', 'reconstruct'].includes(value.strategy))
|
|
448
205
|
throw new Error('Invalid recovery checkpoint.');
|
|
449
206
|
if (value.orderlyOnly !== undefined && value.orderlyOnly !== true || value.handoff !== undefined && value.handoff !== 'clean')
|
|
@@ -455,22 +212,16 @@ class ScheduleService {
|
|
|
455
212
|
text(value.provider);
|
|
456
213
|
text(value.identity, 8192);
|
|
457
214
|
}
|
|
458
|
-
async recoveryDiagnostic(record, reason) {
|
|
459
|
-
await this.serial(async () => { record.error = reason; if (!this.fault)
|
|
460
|
-
await this.persist(); });
|
|
461
|
-
}
|
|
462
|
-
async blockRecovery(record, reason, quarantine = false) {
|
|
463
|
-
await this.serial(async () => {
|
|
464
|
-
if (!['completed', 'cancelled'].includes(record.state))
|
|
465
|
-
record.state = quarantine ? 'failed' : 'interrupted';
|
|
466
|
-
record.recovery.status = 'blocked';
|
|
467
|
-
record.error = reason;
|
|
468
|
-
if (!this.fault)
|
|
469
|
-
await this.persist();
|
|
470
|
-
});
|
|
471
|
-
}
|
|
472
215
|
create(owner, input) {
|
|
473
|
-
|
|
216
|
+
this.pendingCreates++;
|
|
217
|
+
const creation = this.recordOperation(input.id, this.createSchedule(owner, input).finally(async () => {
|
|
218
|
+
this.pendingCreates--;
|
|
219
|
+
if (this.closing && !this.pendingCreates) {
|
|
220
|
+
await this.tail;
|
|
221
|
+
this.catalogGate?.close();
|
|
222
|
+
this.catalogGate = undefined;
|
|
223
|
+
}
|
|
224
|
+
}));
|
|
474
225
|
return this.operation(creation.catch(async (error) => {
|
|
475
226
|
if (error instanceof types_1.NativeScheduleNotAdmittedError) {
|
|
476
227
|
// The create operation has settled. Concurrent cancel/purge operations
|
|
@@ -508,11 +259,11 @@ class ScheduleService {
|
|
|
508
259
|
for (const value of [input.command, input.model, input.effort])
|
|
509
260
|
if (value !== undefined)
|
|
510
261
|
text(value, 4096);
|
|
511
|
-
const
|
|
512
|
-
if (
|
|
513
|
-
nativeInput.recovery = { version: 1, provider:
|
|
514
|
-
identity:
|
|
515
|
-
this.
|
|
262
|
+
const cleanup = this.options.provider.cleanupFor?.(nativeInput);
|
|
263
|
+
if (cleanup) {
|
|
264
|
+
nativeInput.recovery = { version: 1, provider: cleanup.provider, strategy: cleanup.strategy,
|
|
265
|
+
identity: cleanup.describe(structuredClone(nativeInput), originalOwner) };
|
|
266
|
+
this.validateOwnershipMetadata(nativeInput.recovery);
|
|
516
267
|
}
|
|
517
268
|
const runtime = this.newRuntime();
|
|
518
269
|
await this.serial(async () => {
|
|
@@ -524,28 +275,29 @@ class ScheduleService {
|
|
|
524
275
|
if (this.records.size >= this.maxRecords)
|
|
525
276
|
throw new Error('Schedule catalog is full; history was not pruned.');
|
|
526
277
|
const liveCount = [...this.records.values()].filter(record => LIVE.has(record.state)
|
|
527
|
-
||
|
|
278
|
+
|| this.hasRetainedCleanupRoute(record)
|
|
528
279
|
|| (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)).length;
|
|
529
280
|
if (liveCount >= this.maxLive)
|
|
530
281
|
throw new Error('Live native schedule limit reached.');
|
|
531
282
|
this.records.set(nativeInput.id, { ...originalOwner, id: nativeInput.id, backend,
|
|
532
283
|
cwd: nativeInput.cwd, prompt: nativeInput.prompt, rule: structuredClone(nativeInput.rule),
|
|
533
284
|
createdAt, state: 'creating', execution: { command: input.command, model: input.model, effort: input.effort },
|
|
534
|
-
...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: '
|
|
285
|
+
...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: 'blocked' } }) });
|
|
535
286
|
await this.persist();
|
|
536
287
|
this.runtimes.set(nativeInput.id, runtime);
|
|
537
288
|
});
|
|
538
289
|
try {
|
|
539
290
|
const handle = await this.options.provider.create(structuredClone(nativeInput), event => {
|
|
540
291
|
if (this.runtimes.get(nativeInput.id) !== runtime) {
|
|
541
|
-
if (event.type === 'rebound')
|
|
542
|
-
return Promise.reject(new Error('Native rebound owner retired.'));
|
|
543
292
|
return;
|
|
544
293
|
}
|
|
545
|
-
if (event.type === 'started' || event.type === 'completed'
|
|
294
|
+
if (event.type === 'started' || event.type === 'completed')
|
|
546
295
|
runtime.observedOccurrence = true;
|
|
547
296
|
const task = this.onEvent(nativeInput.id, structuredClone(event));
|
|
548
|
-
|
|
297
|
+
// A rejected pre-CREATE attribution is an admission refusal, not a
|
|
298
|
+
// runtime event fault. persist() still fences actual storage failures.
|
|
299
|
+
if (event.type !== 'attributed')
|
|
300
|
+
this.track(task);
|
|
549
301
|
return task;
|
|
550
302
|
}, () => this.canAdmitOccurrence(nativeInput.id, runtime));
|
|
551
303
|
runtime.handle = handle;
|
|
@@ -554,12 +306,16 @@ class ScheduleService {
|
|
|
554
306
|
await this.serial(async () => {
|
|
555
307
|
const record = this.records.get(nativeInput.id);
|
|
556
308
|
record.nativeId = handle.nativeId;
|
|
557
|
-
if (record.recovery && handle.
|
|
558
|
-
text(handle.
|
|
559
|
-
record.recovery.identity = handle.
|
|
309
|
+
if (record.recovery && handle.cleanupIdentity) {
|
|
310
|
+
text(handle.cleanupIdentity, 8192);
|
|
311
|
+
record.recovery.identity = handle.cleanupIdentity;
|
|
560
312
|
}
|
|
561
313
|
const ownerCurrent = this.current(originalOwner);
|
|
562
|
-
if (this.closing
|
|
314
|
+
if (this.closing) {
|
|
315
|
+
await this.persist();
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (!ownerCurrent) {
|
|
563
319
|
if (LIVE.has(record.state)) {
|
|
564
320
|
record.state = 'interrupted';
|
|
565
321
|
record.error = 'Schedule owner or service retired during creation.';
|
|
@@ -582,6 +338,21 @@ class ScheduleService {
|
|
|
582
338
|
runtime.resolveReady(runtime.handle);
|
|
583
339
|
await this.serial(async () => {
|
|
584
340
|
const record = this.records.get(nativeInput.id);
|
|
341
|
+
if (error instanceof types_1.NativeScheduleCleanupError) {
|
|
342
|
+
if (error.handle.nativeId) {
|
|
343
|
+
text(error.handle.nativeId);
|
|
344
|
+
record.nativeId = error.handle.nativeId;
|
|
345
|
+
}
|
|
346
|
+
if (record.recovery && error.handle.cleanupIdentity) {
|
|
347
|
+
text(error.handle.cleanupIdentity, 8192);
|
|
348
|
+
record.recovery.identity = error.handle.cleanupIdentity;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
if (this.closing) {
|
|
352
|
+
if (!this.fault)
|
|
353
|
+
await this.persist();
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
585
356
|
if (LIVE.has(record.state)) {
|
|
586
357
|
record.state = 'failed';
|
|
587
358
|
record.error = errorText(error);
|
|
@@ -608,13 +379,12 @@ class ScheduleService {
|
|
|
608
379
|
if (record.state !== 'failed' || record.creationRefused !== true || record.nativeId !== undefined
|
|
609
380
|
|| record.cancelRequested || record.cleanupConfirmed || record.completedOccurrences?.length || record.skippedOccurrences?.length
|
|
610
381
|
|| this.records.get(id) !== record || this.cleanup.has(id) || this.purged.has(id)
|
|
611
|
-
|| this.fault || this.closing || this.
|
|
382
|
+
|| this.fault || this.closing || this.refreshes
|
|
612
383
|
|| this.recordOperations.has(id) || this.retainedCancellations.has(id) || this.purges.has(id)
|
|
613
384
|
|| this.delivering.has(id) || [...this.pending.values()].some(item => item.scheduleId === id)
|
|
614
385
|
|| this.options.isNativeMutationPending?.(ownerOf(record))
|
|
615
386
|
|| runtime && (!runtime.stopped || runtime.handle || runtime.observedOccurrence || runtime.running.size || runtime.completed.size
|
|
616
|
-
|| runtime.heldDeliveries.size || runtime.cancelling || runtime.cancelPromise || runtime.stopPromise
|
|
617
|
-
|| runtime.recoveryAbort || runtime.releasing || runtime.releasePromise || runtime.releaseAdmission)
|
|
387
|
+
|| runtime.heldDeliveries.size || runtime.cancelling || runtime.cancelPromise || runtime.stopPromise)
|
|
618
388
|
|| this.purged.size >= MAX_PURGED_IDS)
|
|
619
389
|
return;
|
|
620
390
|
this.purged.set(id, ownerOf(record));
|
|
@@ -719,10 +489,9 @@ class ScheduleService {
|
|
|
719
489
|
record.cancelRequested = true;
|
|
720
490
|
if (record.recovery)
|
|
721
491
|
record.recovery.status = 'blocked';
|
|
722
|
-
this.runtimes.get(id)?.recoveryAbort?.abort();
|
|
723
492
|
changed = true;
|
|
724
493
|
}
|
|
725
|
-
// Persist admission
|
|
494
|
+
// Persist admission denial before any external cleanup attempt.
|
|
726
495
|
if (changed)
|
|
727
496
|
await this.persist();
|
|
728
497
|
return ids;
|
|
@@ -784,12 +553,12 @@ class ScheduleService {
|
|
|
784
553
|
forgetCleanup(owner, id, isCurrent) {
|
|
785
554
|
return this.serial(async () => {
|
|
786
555
|
this.checkReady();
|
|
787
|
-
if (!this.current(owner) || !isCurrent() || this.closing
|
|
556
|
+
if (!this.current(owner) || !isCurrent() || this.closing)
|
|
788
557
|
throw new Error('Schedule cleanup context changed.');
|
|
789
558
|
const prior = this.purged.get(id);
|
|
790
559
|
if (!prior || !sameCleanupOwner(prior, owner) || this.records.has(id))
|
|
791
560
|
throw new Error('Purge owned history before forgetting unresolved cleanup tracking.');
|
|
792
|
-
if (this.options.isNativeMutationPending?.(owner) || this.recordOperations.has(id)
|
|
561
|
+
if (this.options.isNativeMutationPending?.(owner) || this.recordOperations.has(id)
|
|
793
562
|
|| this.refreshes || this.retainedCancellations.has(id) || this.runtimes.has(id))
|
|
794
563
|
return { status: 'pending', reason: 'operation_pending', detail: 'Native work or cleanup is still pending; tracking was not forgotten.' };
|
|
795
564
|
if (this.delivering.has(id))
|
|
@@ -816,7 +585,7 @@ class ScheduleService {
|
|
|
816
585
|
return { status: 'pending', reason: 'operation_pending', detail: 'A native mutation receipt is unsettled; retry after reconciliation.' };
|
|
817
586
|
if (!item || !this.options.withPurgeVerification)
|
|
818
587
|
return { status: 'alreadyRemoved' };
|
|
819
|
-
const current = () => !this.closing && !this.fault &&
|
|
588
|
+
const current = () => !this.closing && !this.fault && isCurrent() && this.current(owner) && this.cleanup.get(id) === item;
|
|
820
589
|
let accepted = false;
|
|
821
590
|
return this.options.withPurgeVerification(structuredClone(item), { isCurrent: current }, async (evidence) => {
|
|
822
591
|
if (accepted)
|
|
@@ -864,11 +633,11 @@ class ScheduleService {
|
|
|
864
633
|
if (this.purged.has(id))
|
|
865
634
|
return this.retryCleanup(owner, id, isCurrent);
|
|
866
635
|
const record = this.owned(owner, id);
|
|
867
|
-
const current = () => !this.closing && !this.fault &&
|
|
636
|
+
const current = () => !this.closing && !this.fault && isCurrent() && this.current(owner)
|
|
868
637
|
&& this.records.get(id) === record && record.cancelRequested === true;
|
|
869
638
|
const busy = () => {
|
|
870
|
-
if (this.options.isNativeMutationPending?.(owner) || this.recordOperations.has(id) || this.
|
|
871
|
-
return pending('operation_pending', 'Creation,
|
|
639
|
+
if (this.options.isNativeMutationPending?.(owner) || this.recordOperations.has(id) || this.refreshes || this.retainedCancellations.has(id)) {
|
|
640
|
+
return pending('operation_pending', 'Creation, inspection or cleanup is still pending. Retry these same IDs.');
|
|
872
641
|
}
|
|
873
642
|
if (this.delivering.has(id))
|
|
874
643
|
return pending('delivery_pending', 'An external delivery is already in flight; its outcome cannot be revoked.');
|
|
@@ -882,7 +651,7 @@ class ScheduleService {
|
|
|
882
651
|
// Do not interpret terminal state, an absent handle or a stopped process as
|
|
883
652
|
// native definition absence. Existing cancelRetained owns its stronger proof.
|
|
884
653
|
let retainedVerified = false;
|
|
885
|
-
const clean = !runtime &&
|
|
654
|
+
const clean = !runtime && record.cleanupConfirmed === true;
|
|
886
655
|
if (runtime?.handle && !runtime.stopped) {
|
|
887
656
|
await this.cancel(owner, id);
|
|
888
657
|
}
|
|
@@ -905,7 +674,7 @@ class ScheduleService {
|
|
|
905
674
|
return unfinished;
|
|
906
675
|
if (runtime?.running.size)
|
|
907
676
|
return pending('occurrence_running', 'Future cancellation does not revoke an already admitted occurrence.');
|
|
908
|
-
if (runtime && (!runtime.stopped
|
|
677
|
+
if (runtime && (!runtime.stopped))
|
|
909
678
|
return pending('operation_pending', 'Native runtime retirement is unconfirmed.');
|
|
910
679
|
const handleProof = runtime?.removalConfirmed && runtime.handle?.nativeDefinitionAbsent !== false;
|
|
911
680
|
const erase = (evidence, unresolved) => this.serial(async () => {
|
|
@@ -971,81 +740,15 @@ class ScheduleService {
|
|
|
971
740
|
/** Includes terminal jobs whose native process exit is still unconfirmed. */
|
|
972
741
|
hasActiveSchedules(threadId) {
|
|
973
742
|
return [...this.records.values()].some(record => (!threadId || record.threadId === threadId)
|
|
974
|
-
&& (LIVE.has(record.state) ||
|
|
975
|
-
}
|
|
976
|
-
/** Admission to explicit process retirement, NOT proof of native exit or queue
|
|
977
|
-
* reconstruction. Only the original live handles can supply deletion receipts.
|
|
978
|
-
*/
|
|
979
|
-
canRetireDrainedHosts() {
|
|
980
|
-
if (!this.initialized || this.closing || this.fault || this.hostRetirement || this.suspending
|
|
981
|
-
|| this.handoffUnresolved || this.recoveryRun || this.operations.size || this.tasks.size
|
|
982
|
-
|| this.recordOperations.size || this.pending.size || this.delivering.size || this.refreshes
|
|
983
|
-
|| this.retainedCancellations.size || this.purges.size || this.cleanup.size)
|
|
984
|
-
return false;
|
|
985
|
-
try {
|
|
986
|
-
this.catalogGate?.verify();
|
|
987
|
-
return [...this.records.values()].every(record => {
|
|
988
|
-
if (record.error || record.recovery || !this.current(record) || this.options.isNativeMutationPending?.(ownerOf(record)))
|
|
989
|
-
return false;
|
|
990
|
-
const runtime = this.runtimes.get(record.id);
|
|
991
|
-
if (!runtime)
|
|
992
|
-
return record.state === 'cancelled' && record.cancelRequested === true && record.cleanupConfirmed === true;
|
|
993
|
-
if (!runtime.handle || runtime.running.size || runtime.heldDeliveries.size || runtime.cancelling || runtime.releasing)
|
|
994
|
-
return false;
|
|
995
|
-
if (record.state === 'completed')
|
|
996
|
-
return record.rule.kind === 'once' && runtime.completed.size > 0;
|
|
997
|
-
return record.state === 'cancelled' && record.cancelRequested === true && runtime.removalConfirmed === true
|
|
998
|
-
&& (runtime.handle.nativeDefinitionAbsent === true || runtime.stopped && record.cleanupConfirmed === true);
|
|
999
|
-
});
|
|
1000
|
-
}
|
|
1001
|
-
catch {
|
|
1002
|
-
return false;
|
|
1003
|
-
}
|
|
743
|
+
&& (LIVE.has(record.state) || this.hasRetainedCleanupRoute(record) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
|
|
1004
744
|
}
|
|
1005
|
-
/**
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
throw new Error('Native schedules are not drained for explicit host retirement.');
|
|
1013
|
-
this.hostRetirement = new Map([...this.runtimes].map(([id, runtime]) => [id,
|
|
1014
|
-
{ runtime, cancelled: this.records.get(id)?.cancelRequested === true && runtime.removalConfirmed === true }]));
|
|
1015
|
-
}
|
|
1016
|
-
/** Called only after the owner joined its original executor groups and drained
|
|
1017
|
-
* native callback ingestion. Never calls stop/cancel/releaseForRestart on a
|
|
1018
|
-
* provider. Failure preserves the same retirement fence, handles and catalog.
|
|
1019
|
-
*/
|
|
1020
|
-
async finishDrainedHostRetirement() {
|
|
1021
|
-
if (!this.hostRetirement)
|
|
1022
|
-
throw new Error('Explicit host retirement was not admitted.');
|
|
1023
|
-
while (this.tasks.size)
|
|
1024
|
-
await Promise.allSettled([...this.tasks]);
|
|
1025
|
-
await this.deliveryTail;
|
|
1026
|
-
await this.serial(async () => {
|
|
1027
|
-
if (this.fault)
|
|
1028
|
-
throw this.fault;
|
|
1029
|
-
if (this.pending.size || this.operations.size || this.recordOperations.size || this.delivering.size)
|
|
1030
|
-
throw new Error('Native retirement still owns pending work or output.');
|
|
1031
|
-
for (const [id, proof] of this.hostRetirement) {
|
|
1032
|
-
const runtime = this.runtimes.get(id), record = this.records.get(id);
|
|
1033
|
-
if (runtime !== proof.runtime || !record || runtime.running.size || runtime.heldDeliveries.size
|
|
1034
|
-
|| !(runtime.stopped || runtime.handle?.retirementConfirmed === true))
|
|
1035
|
-
throw new Error('Exact native host retirement is unconfirmed.');
|
|
1036
|
-
}
|
|
1037
|
-
for (const [id, proof] of this.hostRetirement) {
|
|
1038
|
-
proof.runtime.stopped = true;
|
|
1039
|
-
proof.runtime.releaseAdmission?.();
|
|
1040
|
-
proof.runtime.releaseAdmission = undefined;
|
|
1041
|
-
// AGY invalidates its inventory view after exit. Preserve the exact
|
|
1042
|
-
// deletion receipt captured BEFORE exit rather than querying a new host.
|
|
1043
|
-
if (proof.cancelled)
|
|
1044
|
-
this.records.get(id).cleanupConfirmed = true;
|
|
1045
|
-
}
|
|
1046
|
-
await this.persist();
|
|
1047
|
-
this.hostsRetired = true;
|
|
1048
|
-
});
|
|
745
|
+
/** Read-only admission for purge of an owned runtime, including cancelled jobs. */
|
|
746
|
+
needsNativePurge(owner, id) {
|
|
747
|
+
this.checkReady();
|
|
748
|
+
this.owned(owner, id);
|
|
749
|
+
const runtime = this.runtimes.get(id);
|
|
750
|
+
// Cancellation can hide the card while its native runtime still needs retirement.
|
|
751
|
+
return !!runtime?.handle && !runtime.stopped;
|
|
1049
752
|
}
|
|
1050
753
|
/** Card eligibility includes retryable cleanup even for a terminal catalog record. */
|
|
1051
754
|
canCancel(owner, id) {
|
|
@@ -1054,7 +757,7 @@ class ScheduleService {
|
|
|
1054
757
|
const runtime = this.runtimes.get(id);
|
|
1055
758
|
if (record.state === 'cancelled' && runtime?.handle?.cancellationKeepsRunning && !record.error)
|
|
1056
759
|
return false;
|
|
1057
|
-
return LIVE.has(record.state) ||
|
|
760
|
+
return LIVE.has(record.state) || this.hasRetainedCleanupRoute(record) || ['interrupted', 'failed'].includes(record.state) && !!record.nativeId && this.supportsRetainedCancellation(record)
|
|
1058
761
|
|| Boolean(runtime && !runtime.stopped);
|
|
1059
762
|
}
|
|
1060
763
|
/** Newly observed cleanup facts never supply historical recovery proof. The optional
|
|
@@ -1080,7 +783,7 @@ class ScheduleService {
|
|
|
1080
783
|
: 'Owner-reviewed native deletion requested; absence and predecessor work remain unconfirmed.';
|
|
1081
784
|
const runtime = this.runtimes.get(id);
|
|
1082
785
|
if (absent && retirementConfirmed?.(record) === true && (!runtime || runtime.stopped && !runtime.running.size)
|
|
1083
|
-
&& !this.recordOperations.has(id) && !this.
|
|
786
|
+
&& !this.recordOperations.has(id) && !this.refreshes && !this.retainedCancellations.has(id)) {
|
|
1084
787
|
record.state = 'cancelled';
|
|
1085
788
|
record.cleanupConfirmed = true;
|
|
1086
789
|
delete record.error;
|
|
@@ -1126,12 +829,12 @@ class ScheduleService {
|
|
|
1126
829
|
continue;
|
|
1127
830
|
const nativeId = record.nativeId;
|
|
1128
831
|
const completed = runtime.completed.size;
|
|
1129
|
-
// Recheck after every read: cancellation
|
|
832
|
+
// Recheck after every read: cancellation or execution may have
|
|
1130
833
|
// advanced while inventory was awaiting its native receipt.
|
|
1131
|
-
const readable = () => !this.closing && !this.fault &&
|
|
834
|
+
const readable = () => !this.closing && !this.fault && this.current(owner)
|
|
1132
835
|
&& this.records.get(id) === record && record.nativeId === nativeId && record.state === 'active' && !record.cancelRequested
|
|
1133
836
|
&& this.runtimes.get(id) === runtime && runtime.handle === handle && !runtime.stopped && !runtime.cancelling
|
|
1134
|
-
&& !runtime.running.size && runtime.completed.size === completed && !runtime.stopPromise
|
|
837
|
+
&& !runtime.running.size && runtime.completed.size === completed && !runtime.stopPromise
|
|
1135
838
|
&& !this.recordOperations.has(id) && !this.retainedCancellations.has(id);
|
|
1136
839
|
if (!readable())
|
|
1137
840
|
continue;
|
|
@@ -1179,18 +882,11 @@ class ScheduleService {
|
|
|
1179
882
|
return;
|
|
1180
883
|
runtime = this.runtimes.get(id);
|
|
1181
884
|
record.cancelRequested = true;
|
|
1182
|
-
runtime?.recoveryAbort?.abort();
|
|
1183
885
|
if (record.recovery)
|
|
1184
886
|
record.recovery.status = 'blocked';
|
|
1185
887
|
await this.persist();
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
delete record.error;
|
|
1189
|
-
await this.persist();
|
|
1190
|
-
}
|
|
1191
|
-
const retainedCleanup = !runtime || runtime.stopped && !runtime.handle?.nativeDefinitionAbsent
|
|
1192
|
-
&& this.supportsRetainedCancellation(record);
|
|
1193
|
-
if (retainedCleanup && (recoveryOwnsWork(record) || record.state === 'failed' && !!record.recovery
|
|
888
|
+
const retainedCleanup = !runtime || runtime.stopped && !runtime.handle?.nativeDefinitionAbsent;
|
|
889
|
+
if (retainedCleanup && (unsettledNativeWork(record) || record.state === 'failed' && !!record.recovery
|
|
1194
890
|
|| ['interrupted', 'failed'].includes(record.state) && !!record.nativeId && this.supportsRetainedCancellation(record))) {
|
|
1195
891
|
operation = Promise.resolve().then(() => this.cancelRetained(record));
|
|
1196
892
|
this.retainedCancellations.set(id, operation);
|
|
@@ -1215,9 +911,6 @@ class ScheduleService {
|
|
|
1215
911
|
}
|
|
1216
912
|
if (!runtime)
|
|
1217
913
|
throw new Error('Native schedule handle is unavailable.');
|
|
1218
|
-
if (runtime.recoveryAbort && !runtime.handle && !runtime.stopped) {
|
|
1219
|
-
throw new Error('Recovery receipt is unresolved; cancellation intent retained until native cleanup can be confirmed.');
|
|
1220
|
-
}
|
|
1221
914
|
runtime.cancelling = true;
|
|
1222
915
|
// Starting outside the state queue also permits synchronous provider callbacks.
|
|
1223
916
|
operation = Promise.resolve().then(() => this.cancelNative(id, runtime));
|
|
@@ -1225,18 +918,24 @@ class ScheduleService {
|
|
|
1225
918
|
});
|
|
1226
919
|
return (existing ?? operation);
|
|
1227
920
|
}
|
|
921
|
+
/** Historical uncertainty is not live work. Only an exact supported cleanup route
|
|
922
|
+
* can keep a retired row in runtime admission and management controls. */
|
|
923
|
+
hasRetainedCleanupRoute(record) {
|
|
924
|
+
return unsettledNativeWork(record) && !!record.recovery && this.supportsRetainedCancellation(record);
|
|
925
|
+
}
|
|
1228
926
|
supportsRetainedCancellation(record) {
|
|
1229
|
-
|
|
927
|
+
const cleanup = this.options.provider.cleanupFor?.({ ...record, ...record.execution });
|
|
928
|
+
return !!record.recovery && !!cleanup?.cancelRetained && cleanup.provider === record.recovery.provider
|
|
929
|
+
&& cleanup.strategy === record.recovery.strategy;
|
|
1230
930
|
}
|
|
1231
931
|
async cancelRetained(record) {
|
|
1232
932
|
const current = () => !this.closing && !this.fault && this.current(record) && record.cancelRequested === true;
|
|
1233
933
|
try {
|
|
1234
|
-
const
|
|
1235
|
-
const provider = this.options.provider.
|
|
934
|
+
const ownership = record.recovery;
|
|
935
|
+
const provider = this.options.provider.cleanupFor?.({ id: record.id, threadId: record.threadId,
|
|
1236
936
|
backend: record.backend, cwd: record.cwd, prompt: record.prompt, rule: record.rule,
|
|
1237
|
-
...record.execution, recovery });
|
|
1238
|
-
if (!
|
|
1239
|
-
|| (provider.handoffPolicy ?? 'drain-all') !== (recovery.handoffPolicy ?? 'drain-all')) {
|
|
937
|
+
...record.execution, recovery: ownership });
|
|
938
|
+
if (!ownership || !provider?.cancelRetained || provider.provider !== ownership.provider || provider.strategy !== ownership.strategy) {
|
|
1240
939
|
throw new Error('Cancellation intent retained; native cleanup is unconfirmed. Original native ownership metadata or cleanup support is unavailable. Preserve the original session. For Claude, use /schedule cleanup review in the bound original thread for explicit deletion-only review; do not recreate this job.');
|
|
1241
940
|
}
|
|
1242
941
|
if (!current())
|
|
@@ -1256,8 +955,14 @@ class ScheduleService {
|
|
|
1256
955
|
return structuredClone(record);
|
|
1257
956
|
}
|
|
1258
957
|
catch (error) {
|
|
1259
|
-
await this.serial(async () => {
|
|
1260
|
-
|
|
958
|
+
await this.serial(async () => {
|
|
959
|
+
// A failed deletion attempt is not a new interruption. Keep the
|
|
960
|
+
// original exit/ownership diagnostic while returning the cleanup error.
|
|
961
|
+
if (record.state !== 'interrupted' || !record.error)
|
|
962
|
+
record.error = errorText(error);
|
|
963
|
+
if (!this.fault)
|
|
964
|
+
await this.persist();
|
|
965
|
+
});
|
|
1261
966
|
throw error;
|
|
1262
967
|
}
|
|
1263
968
|
finally {
|
|
@@ -1270,240 +975,30 @@ class ScheduleService {
|
|
|
1270
975
|
const ids = await this.serial(async () => [...this.pending.keys()]);
|
|
1271
976
|
await this.attemptDeliveries(ids);
|
|
1272
977
|
}
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
return
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
hasPendingRecovery() {
|
|
1279
|
-
return [...this.records.values()].some(record => record.state === 'interrupted' && record.recovery?.status === 'pending');
|
|
1280
|
-
}
|
|
1281
|
-
/** Snapshot eligibility is only admission to a handoff, never permission to install. */
|
|
1282
|
-
canSuspendForRestart() {
|
|
1283
|
-
if (!this.initialized || this.closing || this.hostRetirement || this.fault || this.suspending || this.recoveryRun || this.operations.size || this.pending.size)
|
|
1284
|
-
return false;
|
|
1285
|
-
return [...this.records.values()].filter(record => LIVE.has(record.state) || recoveryOwnsWork(record)
|
|
1286
|
-
|| !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped).every(record => {
|
|
1287
|
-
const runtime = this.runtimes.get(record.id);
|
|
1288
|
-
return record.state === 'active' && !!record.recovery && !record.cancelRequested && !record.error
|
|
1289
|
-
&& !!runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling;
|
|
1290
|
-
});
|
|
1291
|
-
}
|
|
1292
|
-
/** Delete future definitions, drain native work, join owned hosts, then publish restart proof. */
|
|
1293
|
-
suspendForRestart() {
|
|
1294
|
-
if (this.suspension)
|
|
1295
|
-
return this.suspension;
|
|
1296
|
-
if (!this.canSuspendForRestart())
|
|
1297
|
-
return Promise.reject(new Error('Schedules are not ready for restart handoff.'));
|
|
1298
|
-
this.suspending = true;
|
|
1299
|
-
this.handoffUnresolved = true;
|
|
1300
|
-
const entries = [...this.runtimes].filter(([id]) => this.records.get(id)?.state === 'active');
|
|
1301
|
-
const run = (async () => {
|
|
1302
|
-
try {
|
|
1303
|
-
// Checkpoints remain dirty throughout deletion/drain and actual host exit.
|
|
1304
|
-
await this.serial(async () => {
|
|
1305
|
-
for (const [id, runtime] of entries) {
|
|
1306
|
-
if (runtime.handle?.recoveryIdentity) {
|
|
1307
|
-
text(runtime.handle.recoveryIdentity, 8192);
|
|
1308
|
-
this.records.get(id).recovery.identity = runtime.handle.recoveryIdentity;
|
|
1309
|
-
}
|
|
1310
|
-
delete this.records.get(id).recovery.handoff;
|
|
1311
|
-
delete this.records.get(id).recovery.nativeDefinitionAbsent;
|
|
1312
|
-
}
|
|
1313
|
-
await this.persist();
|
|
1314
|
-
});
|
|
1315
|
-
const releases = await Promise.allSettled(entries.map(([, runtime]) => this.boundRecovery(runtime.handle.releaseForRestart(), 'Native schedule drain remains unresolved.')));
|
|
1316
|
-
const failed = releases.find((result) => result.status === 'rejected');
|
|
1317
|
-
if (failed)
|
|
1318
|
-
throw failed.reason;
|
|
1319
|
-
for (const [index, result] of releases.entries()) {
|
|
1320
|
-
const policy = this.records.get(entries[index][0]).recovery.handoffPolicy ?? 'drain-all';
|
|
1321
|
-
if (result.status === 'fulfilled' && (result.value?.policy ?? 'drain-all') !== policy)
|
|
1322
|
-
throw new Error('Native handoff policy receipt changed.');
|
|
1323
|
-
}
|
|
1324
|
-
const absentOneShots = new Set();
|
|
1325
|
-
for (const [index, result] of releases.entries())
|
|
1326
|
-
if (result.status === 'fulfilled' && result.value?.onceDefinitionAlreadyAbsent) {
|
|
1327
|
-
const record = this.records.get(entries[index][0]);
|
|
1328
|
-
if (record.rule.kind !== 'once' || record.recovery?.handoffPolicy !== 'skip-unadmitted')
|
|
1329
|
-
throw new Error('Invalid absent one-shot handoff receipt.');
|
|
1330
|
-
absentOneShots.add(record.id);
|
|
1331
|
-
}
|
|
1332
|
-
const absent = new Set(entries.filter((_, index) => {
|
|
1333
|
-
const result = releases[index];
|
|
1334
|
-
return result.status === 'fulfilled' && result.value?.nativeDefinitionAbsent === true;
|
|
1335
|
-
}).map(([id]) => id));
|
|
1336
|
-
await this.boundRecovery((async () => {
|
|
1337
|
-
while (this.tasks.size)
|
|
1338
|
-
await Promise.allSettled([...this.tasks]);
|
|
1339
|
-
})(), 'Scheduled result drain remains unresolved.');
|
|
1340
|
-
await this.tail;
|
|
1341
|
-
if (this.pending.size)
|
|
1342
|
-
throw new Error('Scheduled result acknowledgement is pending.');
|
|
1343
|
-
if (this.fault || entries.some(([, runtime]) => runtime.running.size))
|
|
1344
|
-
throw this.fault ?? new Error('Native occurrence is still running.');
|
|
1345
|
-
const finished = new Set();
|
|
1346
|
-
for (const [id] of entries) {
|
|
1347
|
-
const record = this.records.get(id);
|
|
1348
|
-
const provider = this.options.provider.recoveryFor?.({ id, threadId: record.threadId, backend: record.backend,
|
|
1349
|
-
cwd: record.cwd, rule: record.rule, prompt: record.prompt, ...record.execution });
|
|
1350
|
-
if (!provider || provider.provider !== record.recovery.provider)
|
|
1351
|
-
throw new Error('Native recovery provider changed during handoff.');
|
|
1352
|
-
const key = `${record.threadId}:${provider.provider}`;
|
|
1353
|
-
if (finished.has(key))
|
|
1354
|
-
continue;
|
|
1355
|
-
if (provider.orderlyOnly && !provider.finishRestart)
|
|
1356
|
-
throw new Error('Owned native host exit cannot be confirmed.');
|
|
1357
|
-
if (provider.finishRestart)
|
|
1358
|
-
await this.boundRecovery(provider.finishRestart(), 'Owned native host exit remains unresolved.');
|
|
1359
|
-
finished.add(key);
|
|
1360
|
-
}
|
|
1361
|
-
await this.boundRecovery((async () => {
|
|
1362
|
-
while (this.tasks.size)
|
|
1363
|
-
await Promise.allSettled([...this.tasks]);
|
|
1364
|
-
})(), 'Scheduled result drain remains unresolved.');
|
|
1365
|
-
await this.serial(async () => {
|
|
1366
|
-
if (this.fault)
|
|
1367
|
-
throw this.fault;
|
|
1368
|
-
for (const [id, runtime] of entries) {
|
|
1369
|
-
const record = this.records.get(id);
|
|
1370
|
-
if (!this.current(record) || record.cancelRequested)
|
|
1371
|
-
throw new Error('Schedule owner retired during handoff.');
|
|
1372
|
-
if (record.state === 'active' && absentOneShots.has(id)) {
|
|
1373
|
-
record.state = 'interrupted';
|
|
1374
|
-
record.error = 'One-shot native definition was already absent at handoff; it will not be replayed.';
|
|
1375
|
-
delete record.recovery;
|
|
1376
|
-
}
|
|
1377
|
-
else if (record.state === 'active') {
|
|
1378
|
-
record.state = 'interrupted';
|
|
1379
|
-
record.recovery.status = 'pending';
|
|
1380
|
-
record.recovery.handoff = 'clean';
|
|
1381
|
-
if (absent.has(id))
|
|
1382
|
-
record.recovery.nativeDefinitionAbsent = true;
|
|
1383
|
-
record.error = 'Native host drained and exited; waiting for automatic recovery.';
|
|
1384
|
-
}
|
|
1385
|
-
else if (record.state === 'interrupted' && record.rule.kind === 'once' && record.recovery?.handoffPolicy === 'skip-unadmitted' && !runtime.running.size) {
|
|
1386
|
-
delete record.recovery;
|
|
1387
|
-
}
|
|
1388
|
-
else if (record.state !== 'completed')
|
|
1389
|
-
throw new Error('Occurrence outcome changed during handoff.');
|
|
1390
|
-
runtime.stopped = true;
|
|
1391
|
-
}
|
|
1392
|
-
await this.persist();
|
|
1393
|
-
for (const [id] of entries)
|
|
1394
|
-
this.runtimes.delete(id);
|
|
1395
|
-
this.handoffUnresolved = false;
|
|
1396
|
-
});
|
|
1397
|
-
}
|
|
1398
|
-
catch (error) {
|
|
1399
|
-
await this.serial(async () => {
|
|
1400
|
-
for (const [id] of entries) {
|
|
1401
|
-
const record = this.records.get(id);
|
|
1402
|
-
delete record.recovery?.handoff;
|
|
1403
|
-
delete record.recovery?.nativeDefinitionAbsent;
|
|
1404
|
-
record.error = `Restart handoff unconfirmed: ${errorText(error)}`;
|
|
1405
|
-
}
|
|
1406
|
-
if (!this.fault)
|
|
1407
|
-
await this.persist();
|
|
1408
|
-
});
|
|
1409
|
-
throw error;
|
|
1410
|
-
}
|
|
1411
|
-
finally {
|
|
1412
|
-
this.suspending = false;
|
|
1413
|
-
}
|
|
1414
|
-
})();
|
|
1415
|
-
this.suspension = run;
|
|
1416
|
-
void run.finally(() => { if (this.suspension === run)
|
|
1417
|
-
this.suspension = undefined; }).catch(() => { });
|
|
1418
|
-
return run;
|
|
1419
|
-
}
|
|
1420
|
-
/** Providers with durable intent must complete handoff before unrelated host teardown. */
|
|
1421
|
-
requiresShutdownHandoff() {
|
|
1422
|
-
return [...this.records.values()].some(record => !!record.recovery
|
|
1423
|
-
&& (!(record.state === 'failed' && record.recovery.futureOnly) || !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped));
|
|
978
|
+
/** Idle future definitions do not block updates. Active work and durable deliveries still do. */
|
|
979
|
+
canRestartWithoutActiveWork() {
|
|
980
|
+
return this.initialized && !this.closing && !this.fault && !this.operations.size
|
|
981
|
+
&& !this.pending.size && !this.delivering.size && [...this.runtimes].every(([id, runtime]) => (!['cancelled', 'failed', 'completed', 'interrupted'].includes(this.records.get(id)?.state ?? '') || runtime.stopped) &&
|
|
982
|
+
!runtime.running.size && !runtime.cancelling) && ![...this.records.values()].some(record => record.state === 'creating');
|
|
1424
983
|
}
|
|
984
|
+
/** Shutdown revokes admission only. It never invokes native schedule RPCs or waits for result ACKs. */
|
|
1425
985
|
destroy() {
|
|
1426
|
-
if (this.hostRetirement && !this.hostsRetired)
|
|
1427
|
-
return Promise.reject(new Error('Explicit native host retirement remains unresolved.'));
|
|
1428
986
|
if (this.destruction)
|
|
1429
987
|
return this.destruction;
|
|
1430
|
-
|
|
1431
|
-
if (!preserve)
|
|
1432
|
-
this.closing = true;
|
|
988
|
+
this.closing = true;
|
|
1433
989
|
return this.destruction = (async () => {
|
|
1434
|
-
if (preserve)
|
|
1435
|
-
await this.suspendForRestart();
|
|
1436
|
-
this.closing = true;
|
|
1437
|
-
for (const runtime of this.runtimes.values())
|
|
1438
|
-
runtime.recoveryAbort?.abort();
|
|
1439
990
|
await this.initialization?.catch(() => { });
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
for (const record of this.records.values()) {
|
|
1443
|
-
if (LIVE.has(record.state)) {
|
|
1444
|
-
const runtime = this.runtimes.get(record.id);
|
|
1445
|
-
if (record.recovery && record.state === 'active' && !record.cancelRequested && !record.error
|
|
1446
|
-
&& runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling) {
|
|
1447
|
-
runtime.releasing = true;
|
|
1448
|
-
record.recovery.status = 'pending';
|
|
1449
|
-
record.error = 'Native restart handoff pending verification.';
|
|
1450
|
-
}
|
|
1451
|
-
else {
|
|
1452
|
-
if (record.recovery)
|
|
1453
|
-
record.recovery.status = 'blocked';
|
|
1454
|
-
record.error = 'Native schedule interrupted by service shutdown.';
|
|
1455
|
-
}
|
|
1456
|
-
record.state = 'interrupted';
|
|
1457
|
-
}
|
|
1458
|
-
}
|
|
1459
|
-
if (this.initialized && !this.fault)
|
|
1460
|
-
await this.persist();
|
|
1461
|
-
}).catch(error => { failure = error; });
|
|
1462
|
-
const stops = await Promise.allSettled([...this.runtimes].map(async ([id, runtime]) => {
|
|
1463
|
-
if (!runtime.releasing)
|
|
1464
|
-
return this.stop(id, runtime);
|
|
1465
|
-
try {
|
|
1466
|
-
runtime.releasePromise ??= runtime.handle.releaseForRestart();
|
|
1467
|
-
let timer;
|
|
1468
|
-
try {
|
|
1469
|
-
await Promise.race([runtime.releasePromise, new Promise((_, reject) => {
|
|
1470
|
-
timer = setTimeout(() => reject(new Error('Native shutdown handoff timed out; ownership retained.')), this.options.recoveryTimeoutMs ?? 30_000);
|
|
1471
|
-
})]);
|
|
1472
|
-
}
|
|
1473
|
-
finally {
|
|
1474
|
-
if (timer)
|
|
1475
|
-
clearTimeout(timer);
|
|
1476
|
-
}
|
|
1477
|
-
await this.serial(async () => {
|
|
1478
|
-
const record = this.records.get(id);
|
|
1479
|
-
if (record.recovery?.status !== 'pending')
|
|
1480
|
-
throw new Error('Occurrence raced native handoff; recovery is blocked.');
|
|
1481
|
-
runtime.stopped = true;
|
|
1482
|
-
record.error = 'Waiting for native recovery after orderly shutdown.';
|
|
1483
|
-
await this.persist();
|
|
1484
|
-
});
|
|
1485
|
-
}
|
|
1486
|
-
catch (error) {
|
|
1487
|
-
await this.blockRecovery(this.records.get(id), errorText(error));
|
|
1488
|
-
throw error;
|
|
1489
|
-
}
|
|
1490
|
-
}));
|
|
1491
|
-
while (this.operations.size)
|
|
1492
|
-
await Promise.allSettled([...this.operations]);
|
|
1493
|
-
while (this.tasks.size)
|
|
1494
|
-
await Promise.allSettled([...this.tasks]);
|
|
1495
|
-
await this.deliveryTail;
|
|
991
|
+
// Complete only writes already in this process's serial filesystem queue.
|
|
992
|
+
// Native requests, results and delivery are independently durable or remain unknown after exit.
|
|
1496
993
|
await this.tail;
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
this.
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
throw error;
|
|
1506
|
-
});
|
|
994
|
+
// An already admitted CREATE may still return a deletion identity. Keep
|
|
995
|
+
// catalog ownership until that receipt is saved, without awaiting native
|
|
996
|
+
// work here. Process exit releases the kernel gate even if it never returns.
|
|
997
|
+
if (!this.pendingCreates) {
|
|
998
|
+
this.catalogGate?.close();
|
|
999
|
+
this.catalogGate = undefined;
|
|
1000
|
+
}
|
|
1001
|
+
})();
|
|
1507
1002
|
}
|
|
1508
1003
|
async cancelNative(id, runtime) {
|
|
1509
1004
|
try {
|
|
@@ -1560,41 +1055,38 @@ class ScheduleService {
|
|
|
1560
1055
|
const runtime = this.runtimes.get(id);
|
|
1561
1056
|
await this.serial(async () => {
|
|
1562
1057
|
if (!runtime || this.runtimes.get(id) !== runtime || !this.records.has(id)) {
|
|
1563
|
-
if (event.type === 'rebound')
|
|
1564
|
-
throw new Error('Native rebound owner retired.');
|
|
1565
1058
|
return;
|
|
1566
1059
|
}
|
|
1567
1060
|
if (this.fault)
|
|
1568
1061
|
throw this.fault;
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
throw new Error('Unexpected native work during explicit retirement; retain the native event journal.');
|
|
1573
|
-
}
|
|
1574
|
-
if (this.suspending && event.type === 'stopped') {
|
|
1575
|
-
if (event.exitConfirmed === false)
|
|
1576
|
-
throw new Error('Native handoff exit unconfirmed.');
|
|
1577
|
-
runtime.stopped = true;
|
|
1062
|
+
if (this.closing) {
|
|
1063
|
+
if (event.type === 'attributed')
|
|
1064
|
+
throw new Error('Schedule service closed before native creation attribution.');
|
|
1578
1065
|
return;
|
|
1579
1066
|
}
|
|
1580
|
-
|
|
1067
|
+
const record = this.records.get(id);
|
|
1068
|
+
if (event.type === 'attributed') {
|
|
1069
|
+
if (!record.recovery || record.state !== 'creating' || !this.current(record) || record.cancelRequested) {
|
|
1070
|
+
throw new Error('Native creation attribution is no longer admissible.');
|
|
1071
|
+
}
|
|
1072
|
+
text(event.cleanupIdentity, 8192);
|
|
1073
|
+
record.recovery.identity = event.cleanupIdentity;
|
|
1074
|
+
await this.persist();
|
|
1581
1075
|
return;
|
|
1076
|
+
}
|
|
1582
1077
|
// Process exit evidence is independent of task state, including late exit
|
|
1583
1078
|
// after a failed cancellation cleanup. Never downgrade confirmed exit.
|
|
1584
1079
|
if (event.type === 'stopped' && event.exitConfirmed !== false) {
|
|
1585
1080
|
runtime.stopped = true;
|
|
1586
|
-
this.releaseRecoveryAdmission(runtime);
|
|
1587
1081
|
if (record.error?.startsWith('Native process exit unconfirmed:')) {
|
|
1588
1082
|
delete record.error;
|
|
1589
1083
|
await this.persist();
|
|
1590
1084
|
}
|
|
1591
1085
|
}
|
|
1592
|
-
if (runtime.releasing && event.type === 'stopped' && event.exitConfirmed !== false)
|
|
1593
|
-
return;
|
|
1594
1086
|
const ownerCurrent = this.current(record);
|
|
1595
|
-
if (
|
|
1596
|
-
|
|
1597
|
-
if (
|
|
1087
|
+
if (this.closing)
|
|
1088
|
+
return;
|
|
1089
|
+
if (!ownerCurrent) {
|
|
1598
1090
|
if (record.recovery && (!ownerCurrent || event.type !== 'stopped' || event.exitConfirmed === false))
|
|
1599
1091
|
record.recovery.status = 'blocked';
|
|
1600
1092
|
if (LIVE.has(record.state)) {
|
|
@@ -1609,25 +1101,14 @@ class ScheduleService {
|
|
|
1609
1101
|
await this.persist();
|
|
1610
1102
|
return;
|
|
1611
1103
|
}
|
|
1612
|
-
if (event.type === 'rebound') {
|
|
1613
|
-
// A provider's clean same-conversation handoff changes definition identity,
|
|
1614
|
-
// never occurrence identity. Serialize with cancellation and outbox writes.
|
|
1615
|
-
text(event.nativeId, 8192);
|
|
1616
|
-
text(event.previousNativeId, 8192);
|
|
1617
|
-
if (record.state !== 'active' || record.cancelRequested || runtime.running.size
|
|
1618
|
-
|| record.nativeId !== event.previousNativeId || runtime.handle?.nativeId !== event.nativeId) {
|
|
1619
|
-
throw new Error('Native definition rebound does not match active ownership.');
|
|
1620
|
-
}
|
|
1621
|
-
record.nativeId = event.nativeId;
|
|
1622
|
-
await this.persist();
|
|
1623
|
-
return;
|
|
1624
|
-
}
|
|
1625
1104
|
const finishingFailed = record.state === 'failed' && event.type === 'completed' && runtime.running.has(event.occurrenceId);
|
|
1626
1105
|
const finishingCancelled = record.state === 'cancelled' && runtime.handle?.cancellationKeepsRunning
|
|
1627
1106
|
&& event.type === 'completed' && runtime.running.has(event.occurrenceId);
|
|
1628
1107
|
if ((!finishingCancelled && record.state === 'cancelled') || record.state === 'failed' && !finishingFailed || record.state === 'completed')
|
|
1629
1108
|
return;
|
|
1630
1109
|
if (event.type === 'stopped') {
|
|
1110
|
+
if (record.recovery)
|
|
1111
|
+
record.recovery.status = 'blocked';
|
|
1631
1112
|
if (LIVE.has(record.state)) {
|
|
1632
1113
|
record.state = 'interrupted';
|
|
1633
1114
|
record.error = bounded(event.reason, ERROR_BYTES);
|
|
@@ -1644,18 +1125,6 @@ class ScheduleService {
|
|
|
1644
1125
|
record.error = 'Occurrence tracking limit reached; native schedule stopped.';
|
|
1645
1126
|
cleanup = true;
|
|
1646
1127
|
}
|
|
1647
|
-
else if (event.type === 'skipped') {
|
|
1648
|
-
if (record.recovery?.handoffPolicy !== 'skip-unadmitted' || runtime.running.has(event.occurrenceId))
|
|
1649
|
-
throw new Error('Unnegotiated occurrence skip.');
|
|
1650
|
-
runtime.completed.add(event.occurrenceId);
|
|
1651
|
-
record.completedOccurrences = [...runtime.completed];
|
|
1652
|
-
// No delivery and no invented success. Unseen skipped wakes are not counted.
|
|
1653
|
-
(record.skippedOccurrences ??= []).push(event.occurrenceId);
|
|
1654
|
-
if (record.rule.kind === 'once') {
|
|
1655
|
-
record.state = 'interrupted';
|
|
1656
|
-
record.error = bounded(event.reason, ERROR_BYTES);
|
|
1657
|
-
}
|
|
1658
|
-
}
|
|
1659
1128
|
else if (event.type === 'started') {
|
|
1660
1129
|
runtime.running.add(event.occurrenceId);
|
|
1661
1130
|
record.state = 'running';
|
|
@@ -1688,7 +1157,7 @@ class ScheduleService {
|
|
|
1688
1157
|
else if (record.state !== 'interrupted' && record.state !== 'failed') {
|
|
1689
1158
|
record.state = runtime.running.size ? 'running' : 'active';
|
|
1690
1159
|
if (record.recovery && !runtime.running.size)
|
|
1691
|
-
record.recovery.status = '
|
|
1160
|
+
record.recovery.status = 'blocked';
|
|
1692
1161
|
}
|
|
1693
1162
|
}
|
|
1694
1163
|
}
|
|
@@ -1701,14 +1170,10 @@ class ScheduleService {
|
|
|
1701
1170
|
// stop() records retryable exit failures itself. A cleanup timeout must not
|
|
1702
1171
|
// fault the entire service or prevent a later explicit cancellation retry.
|
|
1703
1172
|
// Persistence failures still set the service fault inside persist().
|
|
1704
|
-
if (cleanup && !runtime.cancelling)
|
|
1173
|
+
if (cleanup && !this.closing && !runtime.cancelling)
|
|
1705
1174
|
this.track(this.stop(id, runtime).catch(() => { }));
|
|
1706
1175
|
if (delivery) {
|
|
1707
|
-
|
|
1708
|
-
if (this.options.deferDelivery)
|
|
1709
|
-
this.track(delivering);
|
|
1710
|
-
else
|
|
1711
|
-
await delivering;
|
|
1176
|
+
await this.attemptDeliveries([delivery], true);
|
|
1712
1177
|
}
|
|
1713
1178
|
}
|
|
1714
1179
|
attemptDeliveries(ids, firstAttempt = false) {
|
|
@@ -1758,25 +1223,27 @@ class ScheduleService {
|
|
|
1758
1223
|
finally {
|
|
1759
1224
|
this.delivering.delete(delivery.schedule.id);
|
|
1760
1225
|
}
|
|
1761
|
-
await this.serial(async () => {
|
|
1226
|
+
await this.serial(async () => {
|
|
1227
|
+
this.pending.delete(id);
|
|
1228
|
+
await this.persist();
|
|
1229
|
+
});
|
|
1762
1230
|
}
|
|
1763
1231
|
});
|
|
1764
1232
|
this.deliveryTail = run.catch(() => { });
|
|
1765
1233
|
return run;
|
|
1766
1234
|
}
|
|
1767
1235
|
stop(id, runtime) {
|
|
1768
|
-
if (this.runtimes.get(id) !== runtime)
|
|
1236
|
+
if (this.closing || this.runtimes.get(id) !== runtime)
|
|
1769
1237
|
return Promise.resolve();
|
|
1770
1238
|
if (runtime.stopped)
|
|
1771
1239
|
return this.rememberCleanup(id, runtime);
|
|
1772
1240
|
if (runtime.stopPromise)
|
|
1773
|
-
return runtime.
|
|
1241
|
+
return runtime.stopPromise;
|
|
1774
1242
|
const operation = (async () => {
|
|
1775
1243
|
try {
|
|
1776
|
-
if (runtime.recoveryAbort?.signal.aborted && !runtime.handle && !runtime.stopped) {
|
|
1777
|
-
throw new Error('Recovery receipt is unresolved; native ownership retained.');
|
|
1778
|
-
}
|
|
1779
1244
|
const handle = await runtime.ready;
|
|
1245
|
+
if (this.closing)
|
|
1246
|
+
return;
|
|
1780
1247
|
if (handle && !runtime.stopped) {
|
|
1781
1248
|
await handle.stop();
|
|
1782
1249
|
// Process/job retirement is not itself proof that persisted timing was deleted.
|
|
@@ -1788,7 +1255,6 @@ class ScheduleService {
|
|
|
1788
1255
|
if (handle?.retirementConfirmed !== false && (!handle?.cancellationKeepsRunning || !runtime.running.size))
|
|
1789
1256
|
runtime.stopped = true;
|
|
1790
1257
|
if (runtime.stopped) {
|
|
1791
|
-
this.releaseRecoveryAdmission(runtime);
|
|
1792
1258
|
await this.rememberCleanup(id, runtime);
|
|
1793
1259
|
}
|
|
1794
1260
|
}
|
|
@@ -1805,7 +1271,7 @@ class ScheduleService {
|
|
|
1805
1271
|
runtime.stopPromise = tracked;
|
|
1806
1272
|
void operation.finally(() => { if (!runtime.stopped)
|
|
1807
1273
|
runtime.stopPromise = undefined; }).catch(() => { });
|
|
1808
|
-
return
|
|
1274
|
+
return tracked;
|
|
1809
1275
|
}
|
|
1810
1276
|
rememberCleanup(id, runtime) {
|
|
1811
1277
|
return this.serial(async () => {
|
|
@@ -1818,22 +1284,10 @@ class ScheduleService {
|
|
|
1818
1284
|
}
|
|
1819
1285
|
});
|
|
1820
1286
|
}
|
|
1821
|
-
async boundRecovery(operation, message) {
|
|
1822
|
-
let timer;
|
|
1823
|
-
try {
|
|
1824
|
-
return await Promise.race([operation, new Promise((_, reject) => {
|
|
1825
|
-
timer = setTimeout(() => reject(new Error(message)), this.options.recoveryTimeoutMs ?? 30_000);
|
|
1826
|
-
})]);
|
|
1827
|
-
}
|
|
1828
|
-
finally {
|
|
1829
|
-
if (timer)
|
|
1830
|
-
clearTimeout(timer);
|
|
1831
|
-
}
|
|
1832
|
-
}
|
|
1833
1287
|
canAdmitOccurrence(id, runtime) {
|
|
1834
1288
|
const record = this.records.get(id);
|
|
1835
|
-
return !!record && this.runtimes.get(id) === runtime && !this.closing && !this.
|
|
1836
|
-
&& !runtime.stopped && !runtime.cancelling && !
|
|
1289
|
+
return !!record && this.runtimes.get(id) === runtime && !this.closing && !this.fault
|
|
1290
|
+
&& !runtime.stopped && !runtime.cancelling && !record.cancelRequested
|
|
1837
1291
|
&& LIVE.has(record.state) && this.current(record);
|
|
1838
1292
|
}
|
|
1839
1293
|
newRuntime() {
|
|
@@ -1846,9 +1300,8 @@ class ScheduleService {
|
|
|
1846
1300
|
void task.catch(error => {
|
|
1847
1301
|
// Persistence/protocol failures fail closed and stop native producers.
|
|
1848
1302
|
this.fault ??= error instanceof Error ? error : new Error('Schedule service failed.');
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
void this.stop(id, runtime).catch(() => { });
|
|
1303
|
+
for (const [id, runtime] of this.runtimes)
|
|
1304
|
+
void this.stop(id, runtime).catch(() => { });
|
|
1852
1305
|
}).finally(() => this.tasks.delete(task));
|
|
1853
1306
|
}
|
|
1854
1307
|
operation(task) {
|
|
@@ -1888,7 +1341,7 @@ class ScheduleService {
|
|
|
1888
1341
|
checkReady() {
|
|
1889
1342
|
if (this.fault)
|
|
1890
1343
|
throw this.fault;
|
|
1891
|
-
if (!this.initialized || this.closing
|
|
1344
|
+
if (!this.initialized || this.closing)
|
|
1892
1345
|
throw new Error('Schedule service is not initialized or is closed.');
|
|
1893
1346
|
}
|
|
1894
1347
|
serial(operation) {
|
|
@@ -1939,7 +1392,7 @@ class ScheduleService {
|
|
|
1939
1392
|
}
|
|
1940
1393
|
load(data) {
|
|
1941
1394
|
const catalog = object(data, ['version', 'records', 'pending', 'purged', 'cleanup', 'purgeRequests']);
|
|
1942
|
-
if (![1, 2, 3, 4, 5].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
|
|
1395
|
+
if (![1, 2, 3, 4, 5, 6].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
|
|
1943
1396
|
|| !Array.isArray(catalog.pending) || catalog.pending.length > this.maxPending)
|
|
1944
1397
|
throw new Error('Invalid schedule catalog envelope.');
|
|
1945
1398
|
const records = new Map();
|
|
@@ -1976,8 +1429,8 @@ class ScheduleService {
|
|
|
1976
1429
|
if (record.recovery !== undefined) {
|
|
1977
1430
|
if (catalog.version === 1)
|
|
1978
1431
|
throw new Error('Recovery requires catalog version 2 or newer.');
|
|
1979
|
-
const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'futureOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy']);
|
|
1980
|
-
this.
|
|
1432
|
+
const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'futureOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy', 'abrupt']);
|
|
1433
|
+
this.validateOwnershipMetadata(recovery);
|
|
1981
1434
|
if (!['eligible', 'pending', 'blocked'].includes(recovery.status))
|
|
1982
1435
|
throw new Error('Invalid recovery status.');
|
|
1983
1436
|
}
|
|
@@ -2034,7 +1487,7 @@ class ScheduleService {
|
|
|
2034
1487
|
throw new Error('Purge fences require catalog version 3 or newer.');
|
|
2035
1488
|
const cleanup = new Map();
|
|
2036
1489
|
const purgeRequests = new Map();
|
|
2037
|
-
if (catalog.version === 5) {
|
|
1490
|
+
if (catalog.version === 5 || catalog.version === 6) {
|
|
2038
1491
|
if (!Array.isArray(catalog.cleanup) || catalog.cleanup.length > MAX_CLEANUP_RECORDS)
|
|
2039
1492
|
throw new Error('Invalid cleanup ledger.');
|
|
2040
1493
|
for (const value of catalog.cleanup) {
|
|
@@ -2080,7 +1533,7 @@ class ScheduleService {
|
|
|
2080
1533
|
}
|
|
2081
1534
|
}
|
|
2082
1535
|
else if (catalog.cleanup !== undefined || catalog.purgeRequests !== undefined)
|
|
2083
|
-
throw new Error('Cleanup ledger requires catalog version 5.');
|
|
1536
|
+
throw new Error('Cleanup ledger requires catalog version 5 or newer.');
|
|
2084
1537
|
this.records = records;
|
|
2085
1538
|
this.pending = pending;
|
|
2086
1539
|
this.purged = purged;
|
|
@@ -2106,12 +1559,9 @@ class ScheduleService {
|
|
|
2106
1559
|
}
|
|
2107
1560
|
const handle = await fs.open(temporary, 'wx', 0o600);
|
|
2108
1561
|
try {
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
...(this.cleanup.size || this.purgeRequests.size ? { cleanup: [...this.cleanup.values()],
|
|
2113
|
-
purgeRequests: [...this.purgeRequests].map(([key, receipt]) => ({ key, ...receipt })) } : {}),
|
|
2114
|
-
...(upgraded ? { purged: [...this.purged].map(([id, owner]) => ({ id, ...owner })) } : {}) }), 'utf8');
|
|
1562
|
+
await handle.writeFile(JSON.stringify({ version: 6, records: [...this.records.values()], pending: [...this.pending.values()],
|
|
1563
|
+
cleanup: [...this.cleanup.values()], purgeRequests: [...this.purgeRequests].map(([key, receipt]) => ({ key, ...receipt })),
|
|
1564
|
+
purged: [...this.purged].map(([id, owner]) => ({ id, ...owner })) }), 'utf8');
|
|
2115
1565
|
await handle.sync();
|
|
2116
1566
|
}
|
|
2117
1567
|
finally {
|
|
@@ -2122,9 +1572,8 @@ class ScheduleService {
|
|
|
2122
1572
|
}
|
|
2123
1573
|
catch (error) {
|
|
2124
1574
|
this.fault = error instanceof Error ? error : new Error('Schedule catalog write failed.');
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
void this.stop(id, runtime).catch(() => { });
|
|
1575
|
+
for (const [id, runtime] of this.runtimes)
|
|
1576
|
+
void this.stop(id, runtime).catch(() => { });
|
|
2128
1577
|
throw this.fault;
|
|
2129
1578
|
}
|
|
2130
1579
|
finally {
|