@yu_robotics/remote-cli 1.6.237 → 1.6.238
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/MessageHandler.d.ts +9 -2
- package/dist/client/MessageHandler.d.ts.map +1 -1
- package/dist/client/MessageHandler.js +202 -21
- package/dist/client/MessageHandler.js.map +1 -1
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +1 -0
- package/dist/commands/start.js.map +1 -1
- package/dist/executor/AcpExecutor.d.ts +2 -0
- package/dist/executor/AcpExecutor.d.ts.map +1 -1
- package/dist/executor/AcpExecutor.js +9 -0
- package/dist/executor/AcpExecutor.js.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.d.ts +17 -7
- package/dist/executor/ClaudePersistentExecutor.d.ts.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.js +362 -58
- package/dist/executor/ClaudePersistentExecutor.js.map +1 -1
- package/dist/executor/DshExecutor.d.ts +13 -1
- package/dist/executor/DshExecutor.d.ts.map +1 -1
- package/dist/executor/DshExecutor.js +39 -1
- package/dist/executor/DshExecutor.js.map +1 -1
- package/dist/executor/IExecutor.d.ts +6 -0
- package/dist/executor/IExecutor.d.ts.map +1 -1
- package/dist/maintenance/release-notes.json +1 -1
- package/dist/scheduling/ScheduleCapabilities.d.ts +1 -1
- package/dist/scheduling/ScheduleCapabilities.d.ts.map +1 -1
- package/dist/scheduling/ScheduleCapabilities.js +3 -2
- package/dist/scheduling/ScheduleCapabilities.js.map +1 -1
- package/dist/scheduling/ScheduleService.d.ts +26 -0
- package/dist/scheduling/ScheduleService.d.ts.map +1 -1
- package/dist/scheduling/ScheduleService.js +549 -17
- package/dist/scheduling/ScheduleService.js.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts +25 -2
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.js +126 -17
- package/dist/scheduling/claude/ClaudeSessionSchedules.js.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleBridge.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleBridge.js +21 -0
- package/dist/scheduling/dsh/SessionScheduleBridge.js.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.js +8 -0
- package/dist/scheduling/dsh/SessionScheduleClient.js.map +1 -1
- package/dist/scheduling/types.d.ts +84 -0
- package/dist/scheduling/types.d.ts.map +1 -1
- package/dist/scheduling/types.js.map +1 -1
- package/dist/settings/SettingsService.d.ts +1 -1
- package/dist/settings/SettingsService.d.ts.map +1 -1
- package/dist/settings/SettingsService.js +3 -3
- package/dist/settings/SettingsService.js.map +1 -1
- package/dist/thread/ThreadExecutorPool.d.ts +1 -1
- package/dist/thread/ThreadExecutorPool.d.ts.map +1 -1
- package/dist/thread/ThreadExecutorPool.js +5 -3
- package/dist/thread/ThreadExecutorPool.js.map +1 -1
- package/dist/update/AutomaticUpdater.d.ts +2 -1
- package/dist/update/AutomaticUpdater.d.ts.map +1 -1
- package/dist/update/AutomaticUpdater.js +11 -3
- package/dist/update/AutomaticUpdater.js.map +1 -1
- package/native/prebuilds/darwin-arm64/managed_native.node +0 -0
- package/native/prebuilds/darwin-arm64/manifest.json +1 -1
- package/native/prebuilds/darwin-x64/managed_native.node +0 -0
- package/native/prebuilds/darwin-x64/manifest.json +1 -1
- package/native/prebuilds/win32-arm64/managed_native.node +0 -0
- package/native/prebuilds/win32-arm64/manifest.json +1 -1
- package/native/prebuilds/win32-x64/managed_native.node +0 -0
- package/native/prebuilds/win32-x64/manifest.json +1 -1
- package/package.json +1 -1
|
@@ -83,6 +83,11 @@ function validateOwner(owner) {
|
|
|
83
83
|
function sameOwner(left, right) {
|
|
84
84
|
return left.threadId === right.threadId && left.openId === right.openId && left.generation === right.generation;
|
|
85
85
|
}
|
|
86
|
+
/** Terminal intent prevents recreation, but uncertain native cleanup still owns admission. */
|
|
87
|
+
function recoveryOwnsWork(record) {
|
|
88
|
+
return !!record.recovery && (!['completed', 'cancelled'].includes(record.state)
|
|
89
|
+
|| !!record.error?.startsWith('Native process exit unconfirmed:'));
|
|
90
|
+
}
|
|
86
91
|
function deliveryId(scheduleId, occurrenceId) {
|
|
87
92
|
return `${scheduleId}:${occurrenceId}`;
|
|
88
93
|
}
|
|
@@ -112,6 +117,10 @@ class ScheduleService {
|
|
|
112
117
|
destruction;
|
|
113
118
|
fault;
|
|
114
119
|
catalogPath = '';
|
|
120
|
+
recoveryRun;
|
|
121
|
+
suspending = false;
|
|
122
|
+
handoffUnresolved = false;
|
|
123
|
+
suspension;
|
|
115
124
|
constructor(options) {
|
|
116
125
|
this.options = { ...options };
|
|
117
126
|
this.now = options.now ?? Date.now;
|
|
@@ -121,6 +130,9 @@ class ScheduleService {
|
|
|
121
130
|
throw new Error('Invalid schedule capacity.');
|
|
122
131
|
return result;
|
|
123
132
|
};
|
|
133
|
+
if (options.recoveryTimeoutMs !== undefined && (!Number.isInteger(options.recoveryTimeoutMs)
|
|
134
|
+
|| options.recoveryTimeoutMs < 1 || options.recoveryTimeoutMs > 60_000))
|
|
135
|
+
throw new Error('Invalid recovery timeout.');
|
|
124
136
|
this.maxLive = limit(options.maxLiveSchedules, 5, 5);
|
|
125
137
|
this.maxRecords = limit(options.maxRecords, 100, 1000);
|
|
126
138
|
this.maxPending = limit(options.maxPendingDeliveries, 100, 1000);
|
|
@@ -138,9 +150,23 @@ class ScheduleService {
|
|
|
138
150
|
this.load(data);
|
|
139
151
|
let changed = false;
|
|
140
152
|
for (const record of this.records.values()) {
|
|
141
|
-
if (
|
|
153
|
+
if (['completed', 'cancelled', 'failed'].includes(record.state)) {
|
|
154
|
+
if (record.recovery?.status === 'pending') {
|
|
155
|
+
record.recovery.status = 'blocked';
|
|
156
|
+
changed = true;
|
|
157
|
+
}
|
|
158
|
+
continue; // Terminal tombstones always win over a stale recovery checkpoint.
|
|
159
|
+
}
|
|
160
|
+
if (LIVE.has(record.state) || record.recovery?.status === 'pending') {
|
|
161
|
+
const recoverable = record.recovery && !record.cancelRequested && this.current(record)
|
|
162
|
+
&& record.state !== 'running' && record.state !== 'failed'
|
|
163
|
+
&& (!record.recovery.orderlyOnly || record.recovery.handoff === 'clean')
|
|
164
|
+
&& (record.state === 'active' || record.recovery.status === 'pending');
|
|
165
|
+
if (record.recovery)
|
|
166
|
+
record.recovery.status = recoverable ? 'pending' : 'blocked';
|
|
167
|
+
record.error = recoverable ? 'Waiting for native recovery capability and owner readiness.'
|
|
168
|
+
: record.recovery?.orderlyOnly ? 'Dirty shutdown: original native work is unconfirmed; automatic reconstruction is blocked.' : RESTART_REASON;
|
|
142
169
|
record.state = 'interrupted';
|
|
143
|
-
record.error = RESTART_REASON;
|
|
144
170
|
changed = true;
|
|
145
171
|
}
|
|
146
172
|
}
|
|
@@ -156,6 +182,193 @@ class ScheduleService {
|
|
|
156
182
|
this.initialized = true;
|
|
157
183
|
});
|
|
158
184
|
}
|
|
185
|
+
/** Sequential startup/reconnect reconciliation. Never drives the native schedule clock. */
|
|
186
|
+
recover() {
|
|
187
|
+
this.checkReady();
|
|
188
|
+
if (this.recoveryRun)
|
|
189
|
+
return this.recoveryRun;
|
|
190
|
+
const run = this.operation(this.recoverWaiting());
|
|
191
|
+
this.recoveryRun = run;
|
|
192
|
+
void run.finally(() => { if (this.recoveryRun === run)
|
|
193
|
+
this.recoveryRun = undefined; }).catch(() => { });
|
|
194
|
+
return run;
|
|
195
|
+
}
|
|
196
|
+
async recoverWaiting() {
|
|
197
|
+
const candidates = await this.serial(async () => [...this.records.values()]
|
|
198
|
+
.filter(record => record.recovery?.status === 'pending' && record.state === 'interrupted').map(record => record.id));
|
|
199
|
+
// Catalog capacity is bounded, and only one native admission runs at a time.
|
|
200
|
+
for (const id of candidates.slice(0, this.maxLive)) {
|
|
201
|
+
if (this.closing || this.fault)
|
|
202
|
+
return;
|
|
203
|
+
const record = this.records.get(id);
|
|
204
|
+
if (record.cancelRequested || !this.current(record)) {
|
|
205
|
+
await this.blockRecovery(record, 'Recovery owner retired or cancellation was requested.');
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
if (record.rule.kind === 'once' && Date.parse(record.rule.at) <= this.now()) {
|
|
209
|
+
await this.blockRecovery(record, 'One-shot deadline expired; no late execution was requested.');
|
|
210
|
+
if (record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent)
|
|
211
|
+
await this.serial(async () => { delete record.recovery; await this.persist(); });
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
const input = { id, threadId: record.threadId, backend: record.backend,
|
|
215
|
+
cwd: record.cwd, prompt: record.prompt, rule: structuredClone(record.rule), ...record.execution,
|
|
216
|
+
recovery: this.checkpoint(record.recovery) };
|
|
217
|
+
if (!this.options.canRecover || !this.options.canRecover(structuredClone(record))) {
|
|
218
|
+
await this.recoveryDiagnostic(record, 'Original owner configuration or transport is not ready for recovery.');
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
const provider = this.options.provider.recoveryFor?.(input);
|
|
222
|
+
if (!provider || provider.provider !== record.recovery.provider || provider.strategy !== record.recovery.strategy
|
|
223
|
+
|| (provider.handoffPolicy ?? 'drain-all') !== (record.recovery.handoffPolicy ?? 'drain-all')) {
|
|
224
|
+
await this.recoveryDiagnostic(record, 'Native restart recovery is unsupported or provider identity changed.');
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
if (this.runtimes.has(id))
|
|
228
|
+
continue; // An unresolved/late operation retains exclusive ownership.
|
|
229
|
+
const restoreRecord = structuredClone(record);
|
|
230
|
+
const runtime = this.newRuntime();
|
|
231
|
+
runtime.completed = new Set(record.completedOccurrences);
|
|
232
|
+
const controller = new AbortController();
|
|
233
|
+
runtime.recoveryAbort = controller;
|
|
234
|
+
this.runtimes.set(id, runtime);
|
|
235
|
+
const current = () => !controller.signal.aborted && !this.closing && !this.fault && !record.cancelRequested
|
|
236
|
+
&& this.current(record) && this.options.canRecover?.(structuredClone(record)) === true;
|
|
237
|
+
const emit = (event) => {
|
|
238
|
+
if (this.runtimes.get(id) !== runtime)
|
|
239
|
+
return;
|
|
240
|
+
this.track(this.onEvent(id, structuredClone(event)));
|
|
241
|
+
};
|
|
242
|
+
let timer;
|
|
243
|
+
let admitted = false;
|
|
244
|
+
let succeeded = false;
|
|
245
|
+
try {
|
|
246
|
+
runtime.releaseAdmission = this.options.acquireRecovery?.(structuredClone(record));
|
|
247
|
+
await this.serial(async () => {
|
|
248
|
+
if (!current())
|
|
249
|
+
throw new Error('Recovery admission expired.');
|
|
250
|
+
// Consume clean proof before mutation. A crash must not replay an orderly-only restore.
|
|
251
|
+
record.state = 'creating';
|
|
252
|
+
delete record.recovery.handoff;
|
|
253
|
+
delete record.recovery.nativeDefinitionAbsent;
|
|
254
|
+
await this.persist();
|
|
255
|
+
});
|
|
256
|
+
admitted = true;
|
|
257
|
+
const admission = Promise.resolve().then(() => provider.reconcile(restoreRecord, { notBefore: this.now(), signal: controller.signal, isCurrent: current,
|
|
258
|
+
isOwnerCurrent: () => !record.cancelRequested && !this.fault && this.current(record) }, emit));
|
|
259
|
+
const adopt = async (handle) => {
|
|
260
|
+
runtime.handle = handle;
|
|
261
|
+
runtime.resolveReady(handle);
|
|
262
|
+
text(handle.nativeId);
|
|
263
|
+
await this.serial(async () => { record.nativeId = handle.nativeId; if (!this.fault)
|
|
264
|
+
await this.persist(); });
|
|
265
|
+
};
|
|
266
|
+
const receipt = admission.then(async (handle) => {
|
|
267
|
+
await adopt(handle);
|
|
268
|
+
if (!current()) {
|
|
269
|
+
await this.stop(id, runtime);
|
|
270
|
+
throw new Error('Late recovery receipt requires cleanup; no successor was admitted.');
|
|
271
|
+
}
|
|
272
|
+
if (!await handle.list() || !current())
|
|
273
|
+
throw new Error('Recovered native inventory or owner is unconfirmed.');
|
|
274
|
+
return handle;
|
|
275
|
+
}, async (error) => {
|
|
276
|
+
if (error instanceof types_1.NativeScheduleCleanupError) {
|
|
277
|
+
await adopt(error.handle);
|
|
278
|
+
if (controller.signal.aborted)
|
|
279
|
+
await this.stop(id, runtime).catch(() => { });
|
|
280
|
+
}
|
|
281
|
+
runtime.resolveReady(runtime.handle);
|
|
282
|
+
// A rejection without a handle is not proof that native admission never happened.
|
|
283
|
+
throw error;
|
|
284
|
+
});
|
|
285
|
+
const handle = await Promise.race([receipt, new Promise((_, reject) => {
|
|
286
|
+
timer = setTimeout(() => { controller.abort(); reject(new Error('Native recovery timed out; ownership retained, do not recreate.')); }, this.options.recoveryTimeoutMs ?? 30_000);
|
|
287
|
+
})]);
|
|
288
|
+
text(handle.nativeId);
|
|
289
|
+
// The provider's reconcile contract includes inventory validation; list is
|
|
290
|
+
// additionally required to verify the returned handle before activating it.
|
|
291
|
+
if (!current())
|
|
292
|
+
throw new Error('Recovered native owner is unconfirmed.');
|
|
293
|
+
await this.serial(async () => {
|
|
294
|
+
if (!current())
|
|
295
|
+
throw new Error('Recovery owner retired before receipt persistence.');
|
|
296
|
+
record.nativeId = handle.nativeId;
|
|
297
|
+
if (record.state === 'creating')
|
|
298
|
+
record.state = 'active';
|
|
299
|
+
record.recovery.status = 'eligible';
|
|
300
|
+
delete record.error;
|
|
301
|
+
await this.persist();
|
|
302
|
+
});
|
|
303
|
+
succeeded = true;
|
|
304
|
+
}
|
|
305
|
+
catch (error) {
|
|
306
|
+
controller.abort();
|
|
307
|
+
if (!admitted) {
|
|
308
|
+
runtime.stopped = true;
|
|
309
|
+
runtime.resolveReady(undefined);
|
|
310
|
+
}
|
|
311
|
+
// Do not retry an ambiguous mutation or release its handle on a timeout.
|
|
312
|
+
await this.blockRecovery(record, errorText(error));
|
|
313
|
+
if (runtime.handle)
|
|
314
|
+
await this.stop(id, runtime).catch(() => { });
|
|
315
|
+
else if (runtime.stopped)
|
|
316
|
+
this.runtimes.delete(id);
|
|
317
|
+
}
|
|
318
|
+
finally {
|
|
319
|
+
if (timer)
|
|
320
|
+
clearTimeout(timer);
|
|
321
|
+
if (succeeded || runtime.stopped) {
|
|
322
|
+
runtime.releaseAdmission?.();
|
|
323
|
+
runtime.releaseAdmission = undefined;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
// A handoff containing only skipped/expired one-shots has nothing to
|
|
328
|
+
// reconcile. It must still release the predecessor's local input fence.
|
|
329
|
+
const released = new Set();
|
|
330
|
+
for (const record of this.records.values()) {
|
|
331
|
+
if (released.has(record.threadId) || this.closing || this.fault || this.handoffUnresolved
|
|
332
|
+
|| !this.current(record) || !this.options.canRecover?.(structuredClone(record)))
|
|
333
|
+
continue;
|
|
334
|
+
if ([...this.records.values()].some(other => other.threadId === record.threadId
|
|
335
|
+
&& (LIVE.has(other.state) || recoveryOwnsWork(other) || this.runtimes.has(other.id))))
|
|
336
|
+
continue;
|
|
337
|
+
this.options.provider.recoveryFor?.({ id: record.id, threadId: record.threadId, backend: record.backend,
|
|
338
|
+
cwd: record.cwd, prompt: record.prompt, rule: record.rule, ...record.execution })?.releaseRestartAdmission?.();
|
|
339
|
+
released.add(record.threadId);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
checkpoint(value) {
|
|
343
|
+
return { version: 1, provider: value.provider, strategy: value.strategy, identity: value.identity,
|
|
344
|
+
...(value.orderlyOnly && { orderlyOnly: true }), ...(value.handoffPolicy && { handoffPolicy: value.handoffPolicy }), ...(value.handoff && { handoff: value.handoff }),
|
|
345
|
+
...(value.nativeDefinitionAbsent && { nativeDefinitionAbsent: true }) };
|
|
346
|
+
}
|
|
347
|
+
validateCheckpoint(value) {
|
|
348
|
+
if (value.version !== 1 || !['reattach', 'reconstruct'].includes(value.strategy))
|
|
349
|
+
throw new Error('Invalid recovery checkpoint.');
|
|
350
|
+
if (value.orderlyOnly !== undefined && value.orderlyOnly !== true || value.handoff !== undefined && value.handoff !== 'clean')
|
|
351
|
+
throw new Error('Invalid handoff evidence.');
|
|
352
|
+
if (value.nativeDefinitionAbsent !== undefined && (value.nativeDefinitionAbsent !== true || value.handoff !== 'clean'))
|
|
353
|
+
throw new Error('Invalid native absence evidence.');
|
|
354
|
+
if (value.handoffPolicy !== undefined && !['drain-all', 'skip-unadmitted'].includes(value.handoffPolicy))
|
|
355
|
+
throw new Error('Invalid handoff policy.');
|
|
356
|
+
text(value.provider);
|
|
357
|
+
text(value.identity, 8192);
|
|
358
|
+
}
|
|
359
|
+
async recoveryDiagnostic(record, reason) {
|
|
360
|
+
await this.serial(async () => { record.error = reason; await this.persist(); });
|
|
361
|
+
}
|
|
362
|
+
async blockRecovery(record, reason) {
|
|
363
|
+
await this.serial(async () => {
|
|
364
|
+
if (!['completed', 'cancelled'].includes(record.state))
|
|
365
|
+
record.state = 'interrupted';
|
|
366
|
+
record.recovery.status = 'blocked';
|
|
367
|
+
record.error = reason;
|
|
368
|
+
if (!this.fault)
|
|
369
|
+
await this.persist();
|
|
370
|
+
});
|
|
371
|
+
}
|
|
159
372
|
create(owner, input) {
|
|
160
373
|
return this.operation(this.createSchedule(owner, input));
|
|
161
374
|
}
|
|
@@ -174,7 +387,18 @@ class ScheduleService {
|
|
|
174
387
|
if (!node_path_1.default.isAbsolute(input.cwd))
|
|
175
388
|
throw new Error('Schedule working directory must be absolute.');
|
|
176
389
|
const originalOwner = ownerOf(owner);
|
|
177
|
-
const nativeInput = { ...structuredClone(input), backend };
|
|
390
|
+
const nativeInput = { ...structuredClone(input), backend, threadId: owner.threadId };
|
|
391
|
+
// Never trust a caller-supplied provider checkpoint.
|
|
392
|
+
delete nativeInput.recovery;
|
|
393
|
+
for (const value of [input.command, input.model, input.effort])
|
|
394
|
+
if (value !== undefined)
|
|
395
|
+
text(value, 4096);
|
|
396
|
+
const recovery = this.options.provider.recoveryFor?.(nativeInput);
|
|
397
|
+
if (recovery) {
|
|
398
|
+
nativeInput.recovery = { version: 1, provider: recovery.provider, strategy: recovery.strategy,
|
|
399
|
+
identity: recovery.describe(structuredClone(nativeInput), originalOwner), ...(recovery.orderlyOnly && { orderlyOnly: true }), ...(recovery.handoffPolicy && { handoffPolicy: recovery.handoffPolicy }) };
|
|
400
|
+
this.validateCheckpoint(nativeInput.recovery);
|
|
401
|
+
}
|
|
178
402
|
const runtime = this.newRuntime();
|
|
179
403
|
await this.serial(async () => {
|
|
180
404
|
this.checkReady();
|
|
@@ -185,17 +409,21 @@ class ScheduleService {
|
|
|
185
409
|
if (this.records.size >= this.maxRecords)
|
|
186
410
|
throw new Error('Schedule catalog is full; history was not pruned.');
|
|
187
411
|
const liveCount = [...this.records.values()].filter(record => LIVE.has(record.state)
|
|
412
|
+
|| recoveryOwnsWork(record)
|
|
188
413
|
|| (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)).length;
|
|
189
414
|
if (liveCount >= this.maxLive)
|
|
190
415
|
throw new Error('Live native schedule limit reached.');
|
|
191
416
|
this.records.set(nativeInput.id, { ...originalOwner, id: nativeInput.id, backend,
|
|
192
417
|
cwd: nativeInput.cwd, prompt: nativeInput.prompt, rule: structuredClone(nativeInput.rule),
|
|
193
|
-
createdAt, state: 'creating' }
|
|
418
|
+
createdAt, state: 'creating', execution: { command: input.command, model: input.model, effort: input.effort },
|
|
419
|
+
...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: 'eligible' } }) });
|
|
194
420
|
await this.persist();
|
|
195
421
|
this.runtimes.set(nativeInput.id, runtime);
|
|
196
422
|
});
|
|
197
423
|
try {
|
|
198
424
|
const handle = await this.options.provider.create(structuredClone(nativeInput), event => {
|
|
425
|
+
if (this.runtimes.get(nativeInput.id) !== runtime)
|
|
426
|
+
return;
|
|
199
427
|
const task = this.onEvent(nativeInput.id, structuredClone(event));
|
|
200
428
|
this.track(task);
|
|
201
429
|
});
|
|
@@ -205,6 +433,10 @@ class ScheduleService {
|
|
|
205
433
|
await this.serial(async () => {
|
|
206
434
|
const record = this.records.get(nativeInput.id);
|
|
207
435
|
record.nativeId = handle.nativeId;
|
|
436
|
+
if (record.recovery && handle.recoveryIdentity) {
|
|
437
|
+
text(handle.recoveryIdentity, 8192);
|
|
438
|
+
record.recovery.identity = handle.recoveryIdentity;
|
|
439
|
+
}
|
|
208
440
|
const ownerCurrent = this.current(originalOwner);
|
|
209
441
|
if (this.closing || !ownerCurrent) {
|
|
210
442
|
if (LIVE.has(record.state)) {
|
|
@@ -252,7 +484,7 @@ class ScheduleService {
|
|
|
252
484
|
/** Includes terminal jobs whose native process exit is still unconfirmed. */
|
|
253
485
|
hasActiveSchedules(threadId) {
|
|
254
486
|
return [...this.records.values()].some(record => (!threadId || record.threadId === threadId)
|
|
255
|
-
&& (LIVE.has(record.state) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
|
|
487
|
+
&& (LIVE.has(record.state) || recoveryOwnsWork(record) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
|
|
256
488
|
}
|
|
257
489
|
/** Card eligibility includes retryable cleanup even for a terminal catalog record. */
|
|
258
490
|
canCancel(owner, id) {
|
|
@@ -261,7 +493,7 @@ class ScheduleService {
|
|
|
261
493
|
const runtime = this.runtimes.get(id);
|
|
262
494
|
if (record.state === 'cancelled' && runtime?.handle?.cancellationKeepsRunning && !record.error)
|
|
263
495
|
return false;
|
|
264
|
-
return LIVE.has(record.state) || Boolean(runtime && !runtime.stopped);
|
|
496
|
+
return LIVE.has(record.state) || recoveryOwnsWork(record) || Boolean(runtime && !runtime.stopped);
|
|
265
497
|
}
|
|
266
498
|
/** Keep admitted native work visible when cleanup cannot safely acquire its host. */
|
|
267
499
|
async retainCleanupError(owner, id, error) {
|
|
@@ -333,11 +565,30 @@ class ScheduleService {
|
|
|
333
565
|
await this.serial(async () => {
|
|
334
566
|
const record = this.owned(owner, id);
|
|
335
567
|
runtime = this.runtimes.get(id);
|
|
568
|
+
record.cancelRequested = true;
|
|
569
|
+
runtime?.recoveryAbort?.abort();
|
|
570
|
+
if (record.recovery)
|
|
571
|
+
record.recovery.status = 'blocked';
|
|
572
|
+
await this.persist();
|
|
573
|
+
if (!runtime && record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent) {
|
|
574
|
+
record.state = 'cancelled';
|
|
575
|
+
delete record.error;
|
|
576
|
+
await this.persist();
|
|
577
|
+
}
|
|
578
|
+
if (!runtime && recoveryOwnsWork(record)) {
|
|
579
|
+
record.error = 'Cancellation intent retained; native cleanup is unconfirmed. Recovery will not recreate this job.';
|
|
580
|
+
await this.persist();
|
|
581
|
+
throw new Error(record.error);
|
|
582
|
+
}
|
|
336
583
|
if (runtime?.cancelPromise) {
|
|
337
584
|
existing = runtime.cancelPromise;
|
|
338
585
|
return;
|
|
339
586
|
}
|
|
340
587
|
if (!LIVE.has(record.state) && !runtime?.cancelling && (!runtime || runtime.stopped)) {
|
|
588
|
+
if (record.recovery && runtime?.stopped) {
|
|
589
|
+
record.state = 'cancelled';
|
|
590
|
+
delete record.error;
|
|
591
|
+
}
|
|
341
592
|
for (const [key, pending] of this.pending) {
|
|
342
593
|
if (pending.scheduleId === id && !pending.afterCancellation)
|
|
343
594
|
this.pending.delete(key);
|
|
@@ -348,6 +599,9 @@ class ScheduleService {
|
|
|
348
599
|
}
|
|
349
600
|
if (!runtime)
|
|
350
601
|
throw new Error('Native schedule handle is unavailable.');
|
|
602
|
+
if (runtime.recoveryAbort && !runtime.handle && !runtime.stopped) {
|
|
603
|
+
throw new Error('Recovery receipt is unresolved; cancellation intent retained until native cleanup can be confirmed.');
|
|
604
|
+
}
|
|
351
605
|
runtime.cancelling = true;
|
|
352
606
|
// Starting outside the state queue also permits synchronous provider callbacks.
|
|
353
607
|
operation = Promise.resolve().then(() => this.cancelNative(id, runtime));
|
|
@@ -361,24 +615,221 @@ class ScheduleService {
|
|
|
361
615
|
const ids = await this.serial(async () => [...this.pending.keys()]);
|
|
362
616
|
await this.attemptDeliveries(ids);
|
|
363
617
|
}
|
|
618
|
+
hasUnresolvedHandoff() { return this.handoffUnresolved; }
|
|
619
|
+
hasUnresolvedRecovery() {
|
|
620
|
+
return [...this.records.values()].some(record => recoveryOwnsWork(record)
|
|
621
|
+
&& record.recovery?.status !== 'eligible');
|
|
622
|
+
}
|
|
623
|
+
hasPendingRecovery() {
|
|
624
|
+
return [...this.records.values()].some(record => record.state === 'interrupted' && record.recovery?.status === 'pending');
|
|
625
|
+
}
|
|
626
|
+
/** Snapshot eligibility is only admission to a handoff, never permission to install. */
|
|
627
|
+
canSuspendForRestart() {
|
|
628
|
+
if (!this.initialized || this.closing || this.fault || this.suspending || this.recoveryRun || this.operations.size || this.pending.size)
|
|
629
|
+
return false;
|
|
630
|
+
return [...this.records.values()].filter(record => LIVE.has(record.state) || recoveryOwnsWork(record)
|
|
631
|
+
|| !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped).every(record => {
|
|
632
|
+
const runtime = this.runtimes.get(record.id);
|
|
633
|
+
return record.state === 'active' && !!record.recovery && !record.cancelRequested && !record.error
|
|
634
|
+
&& !!runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling;
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
/** Delete future definitions, drain native work, join owned hosts, then publish restart proof. */
|
|
638
|
+
suspendForRestart() {
|
|
639
|
+
if (this.suspension)
|
|
640
|
+
return this.suspension;
|
|
641
|
+
if (!this.canSuspendForRestart())
|
|
642
|
+
return Promise.reject(new Error('Schedules are not ready for restart handoff.'));
|
|
643
|
+
this.suspending = true;
|
|
644
|
+
this.handoffUnresolved = true;
|
|
645
|
+
const entries = [...this.runtimes].filter(([id]) => this.records.get(id)?.state === 'active');
|
|
646
|
+
const run = (async () => {
|
|
647
|
+
try {
|
|
648
|
+
// Checkpoints remain dirty throughout deletion/drain and actual host exit.
|
|
649
|
+
await this.serial(async () => {
|
|
650
|
+
for (const [id, runtime] of entries) {
|
|
651
|
+
if (runtime.handle?.recoveryIdentity) {
|
|
652
|
+
text(runtime.handle.recoveryIdentity, 8192);
|
|
653
|
+
this.records.get(id).recovery.identity = runtime.handle.recoveryIdentity;
|
|
654
|
+
}
|
|
655
|
+
delete this.records.get(id).recovery.handoff;
|
|
656
|
+
delete this.records.get(id).recovery.nativeDefinitionAbsent;
|
|
657
|
+
}
|
|
658
|
+
await this.persist();
|
|
659
|
+
});
|
|
660
|
+
const releases = await Promise.allSettled(entries.map(([, runtime]) => this.boundRecovery(runtime.handle.releaseForRestart(), 'Native schedule drain remains unresolved.')));
|
|
661
|
+
const failed = releases.find((result) => result.status === 'rejected');
|
|
662
|
+
if (failed)
|
|
663
|
+
throw failed.reason;
|
|
664
|
+
for (const [index, result] of releases.entries()) {
|
|
665
|
+
const policy = this.records.get(entries[index][0]).recovery.handoffPolicy ?? 'drain-all';
|
|
666
|
+
if (result.status === 'fulfilled' && (result.value?.policy ?? 'drain-all') !== policy)
|
|
667
|
+
throw new Error('Native handoff policy receipt changed.');
|
|
668
|
+
}
|
|
669
|
+
const absentOneShots = new Set();
|
|
670
|
+
for (const [index, result] of releases.entries())
|
|
671
|
+
if (result.status === 'fulfilled' && result.value?.onceDefinitionAlreadyAbsent) {
|
|
672
|
+
const record = this.records.get(entries[index][0]);
|
|
673
|
+
if (record.rule.kind !== 'once' || record.recovery?.handoffPolicy !== 'skip-unadmitted')
|
|
674
|
+
throw new Error('Invalid absent one-shot handoff receipt.');
|
|
675
|
+
absentOneShots.add(record.id);
|
|
676
|
+
}
|
|
677
|
+
const absent = new Set(entries.filter((_, index) => {
|
|
678
|
+
const result = releases[index];
|
|
679
|
+
return result.status === 'fulfilled' && result.value?.nativeDefinitionAbsent === true;
|
|
680
|
+
}).map(([id]) => id));
|
|
681
|
+
await this.boundRecovery((async () => {
|
|
682
|
+
while (this.tasks.size)
|
|
683
|
+
await Promise.allSettled([...this.tasks]);
|
|
684
|
+
})(), 'Scheduled result drain remains unresolved.');
|
|
685
|
+
await this.tail;
|
|
686
|
+
if (this.pending.size)
|
|
687
|
+
throw new Error('Scheduled result acknowledgement is pending.');
|
|
688
|
+
if (this.fault || entries.some(([, runtime]) => runtime.running.size))
|
|
689
|
+
throw this.fault ?? new Error('Native occurrence is still running.');
|
|
690
|
+
const finished = new Set();
|
|
691
|
+
for (const [id] of entries) {
|
|
692
|
+
const record = this.records.get(id);
|
|
693
|
+
const provider = this.options.provider.recoveryFor?.({ id, threadId: record.threadId, backend: record.backend,
|
|
694
|
+
cwd: record.cwd, rule: record.rule, prompt: record.prompt, ...record.execution });
|
|
695
|
+
if (!provider || provider.provider !== record.recovery.provider)
|
|
696
|
+
throw new Error('Native recovery provider changed during handoff.');
|
|
697
|
+
const key = `${record.threadId}:${provider.provider}`;
|
|
698
|
+
if (finished.has(key))
|
|
699
|
+
continue;
|
|
700
|
+
if (provider.orderlyOnly && !provider.finishRestart)
|
|
701
|
+
throw new Error('Owned native host exit cannot be confirmed.');
|
|
702
|
+
if (provider.finishRestart)
|
|
703
|
+
await this.boundRecovery(provider.finishRestart(), 'Owned native host exit remains unresolved.');
|
|
704
|
+
finished.add(key);
|
|
705
|
+
}
|
|
706
|
+
await this.boundRecovery((async () => {
|
|
707
|
+
while (this.tasks.size)
|
|
708
|
+
await Promise.allSettled([...this.tasks]);
|
|
709
|
+
})(), 'Scheduled result drain remains unresolved.');
|
|
710
|
+
await this.serial(async () => {
|
|
711
|
+
if (this.fault)
|
|
712
|
+
throw this.fault;
|
|
713
|
+
for (const [id, runtime] of entries) {
|
|
714
|
+
const record = this.records.get(id);
|
|
715
|
+
if (!this.current(record) || record.cancelRequested)
|
|
716
|
+
throw new Error('Schedule owner retired during handoff.');
|
|
717
|
+
if (record.state === 'active' && absentOneShots.has(id)) {
|
|
718
|
+
record.state = 'interrupted';
|
|
719
|
+
record.error = 'One-shot native definition was already absent at handoff; it will not be replayed.';
|
|
720
|
+
delete record.recovery;
|
|
721
|
+
}
|
|
722
|
+
else if (record.state === 'active') {
|
|
723
|
+
record.state = 'interrupted';
|
|
724
|
+
record.recovery.status = 'pending';
|
|
725
|
+
record.recovery.handoff = 'clean';
|
|
726
|
+
if (absent.has(id))
|
|
727
|
+
record.recovery.nativeDefinitionAbsent = true;
|
|
728
|
+
record.error = 'Native host drained and exited; waiting for automatic recovery.';
|
|
729
|
+
}
|
|
730
|
+
else if (record.state === 'interrupted' && record.rule.kind === 'once' && record.recovery?.handoffPolicy === 'skip-unadmitted' && !runtime.running.size) {
|
|
731
|
+
delete record.recovery;
|
|
732
|
+
}
|
|
733
|
+
else if (record.state !== 'completed')
|
|
734
|
+
throw new Error('Occurrence outcome changed during handoff.');
|
|
735
|
+
runtime.stopped = true;
|
|
736
|
+
}
|
|
737
|
+
await this.persist();
|
|
738
|
+
for (const [id] of entries)
|
|
739
|
+
this.runtimes.delete(id);
|
|
740
|
+
this.handoffUnresolved = false;
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
catch (error) {
|
|
744
|
+
await this.serial(async () => {
|
|
745
|
+
for (const [id] of entries) {
|
|
746
|
+
const record = this.records.get(id);
|
|
747
|
+
delete record.recovery?.handoff;
|
|
748
|
+
delete record.recovery?.nativeDefinitionAbsent;
|
|
749
|
+
record.error = `Restart handoff unconfirmed: ${errorText(error)}`;
|
|
750
|
+
}
|
|
751
|
+
if (!this.fault)
|
|
752
|
+
await this.persist();
|
|
753
|
+
});
|
|
754
|
+
throw error;
|
|
755
|
+
}
|
|
756
|
+
finally {
|
|
757
|
+
this.suspending = false;
|
|
758
|
+
}
|
|
759
|
+
})();
|
|
760
|
+
this.suspension = run;
|
|
761
|
+
void run.finally(() => { if (this.suspension === run)
|
|
762
|
+
this.suspension = undefined; }).catch(() => { });
|
|
763
|
+
return run;
|
|
764
|
+
}
|
|
765
|
+
/** Providers with durable intent must complete handoff before unrelated host teardown. */
|
|
766
|
+
requiresShutdownHandoff() {
|
|
767
|
+
return [...this.records.values()].some(record => !!record.recovery);
|
|
768
|
+
}
|
|
364
769
|
destroy() {
|
|
365
770
|
if (this.destruction)
|
|
366
771
|
return this.destruction;
|
|
367
|
-
this.closing
|
|
772
|
+
const preserve = !this.closing && this.hasActiveSchedules() && this.canSuspendForRestart();
|
|
773
|
+
if (!preserve)
|
|
774
|
+
this.closing = true;
|
|
368
775
|
return this.destruction = (async () => {
|
|
776
|
+
if (preserve)
|
|
777
|
+
await this.suspendForRestart();
|
|
778
|
+
this.closing = true;
|
|
779
|
+
for (const runtime of this.runtimes.values())
|
|
780
|
+
runtime.recoveryAbort?.abort();
|
|
369
781
|
await this.initialization?.catch(() => { });
|
|
370
782
|
let failure;
|
|
371
783
|
await this.serial(async () => {
|
|
372
784
|
for (const record of this.records.values()) {
|
|
373
785
|
if (LIVE.has(record.state)) {
|
|
786
|
+
const runtime = this.runtimes.get(record.id);
|
|
787
|
+
if (record.recovery && record.state === 'active' && !record.cancelRequested && !record.error
|
|
788
|
+
&& runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling) {
|
|
789
|
+
runtime.releasing = true;
|
|
790
|
+
record.recovery.status = 'pending';
|
|
791
|
+
record.error = 'Native restart handoff pending verification.';
|
|
792
|
+
}
|
|
793
|
+
else {
|
|
794
|
+
if (record.recovery)
|
|
795
|
+
record.recovery.status = 'blocked';
|
|
796
|
+
record.error = 'Native schedule interrupted by service shutdown.';
|
|
797
|
+
}
|
|
374
798
|
record.state = 'interrupted';
|
|
375
|
-
record.error = 'Native schedule interrupted by service shutdown.';
|
|
376
799
|
}
|
|
377
800
|
}
|
|
378
801
|
if (this.initialized && !this.fault)
|
|
379
802
|
await this.persist();
|
|
380
803
|
}).catch(error => { failure = error; });
|
|
381
|
-
const stops = await Promise.allSettled([...this.runtimes].map(([id, runtime]) =>
|
|
804
|
+
const stops = await Promise.allSettled([...this.runtimes].map(async ([id, runtime]) => {
|
|
805
|
+
if (!runtime.releasing)
|
|
806
|
+
return this.stop(id, runtime);
|
|
807
|
+
try {
|
|
808
|
+
runtime.releasePromise ??= runtime.handle.releaseForRestart();
|
|
809
|
+
let timer;
|
|
810
|
+
try {
|
|
811
|
+
await Promise.race([runtime.releasePromise, new Promise((_, reject) => {
|
|
812
|
+
timer = setTimeout(() => reject(new Error('Native shutdown handoff timed out; ownership retained.')), this.options.recoveryTimeoutMs ?? 30_000);
|
|
813
|
+
})]);
|
|
814
|
+
}
|
|
815
|
+
finally {
|
|
816
|
+
if (timer)
|
|
817
|
+
clearTimeout(timer);
|
|
818
|
+
}
|
|
819
|
+
await this.serial(async () => {
|
|
820
|
+
const record = this.records.get(id);
|
|
821
|
+
if (record.recovery?.status !== 'pending')
|
|
822
|
+
throw new Error('Occurrence raced native handoff; recovery is blocked.');
|
|
823
|
+
runtime.stopped = true;
|
|
824
|
+
record.error = 'Waiting for native recovery after orderly shutdown.';
|
|
825
|
+
await this.persist();
|
|
826
|
+
});
|
|
827
|
+
}
|
|
828
|
+
catch (error) {
|
|
829
|
+
await this.blockRecovery(this.records.get(id), errorText(error));
|
|
830
|
+
throw error;
|
|
831
|
+
}
|
|
832
|
+
}));
|
|
382
833
|
while (this.operations.size)
|
|
383
834
|
await Promise.allSettled([...this.operations]);
|
|
384
835
|
while (this.tasks.size)
|
|
@@ -449,17 +900,31 @@ class ScheduleService {
|
|
|
449
900
|
if (this.fault)
|
|
450
901
|
throw this.fault;
|
|
451
902
|
const record = this.records.get(id);
|
|
903
|
+
if (this.suspending && event.type === 'stopped') {
|
|
904
|
+
if (event.exitConfirmed === false)
|
|
905
|
+
throw new Error('Native handoff exit unconfirmed.');
|
|
906
|
+
runtime.stopped = true;
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
909
|
+
if (runtime.releasing && runtime.stopped)
|
|
910
|
+
return;
|
|
452
911
|
// Process exit evidence is independent of task state, including late exit
|
|
453
912
|
// after a failed cancellation cleanup. Never downgrade confirmed exit.
|
|
454
913
|
if (event.type === 'stopped' && event.exitConfirmed !== false) {
|
|
455
914
|
runtime.stopped = true;
|
|
915
|
+
runtime.releaseAdmission?.();
|
|
916
|
+
runtime.releaseAdmission = undefined;
|
|
456
917
|
if (record.error?.startsWith('Native process exit unconfirmed:')) {
|
|
457
918
|
delete record.error;
|
|
458
919
|
await this.persist();
|
|
459
920
|
}
|
|
460
921
|
}
|
|
922
|
+
if (runtime.releasing && event.type === 'stopped' && event.exitConfirmed !== false)
|
|
923
|
+
return;
|
|
461
924
|
const ownerCurrent = this.current(record);
|
|
462
925
|
if (this.closing || !ownerCurrent) {
|
|
926
|
+
if (record.recovery && (!ownerCurrent || event.type !== 'stopped' || event.exitConfirmed === false))
|
|
927
|
+
record.recovery.status = 'blocked';
|
|
463
928
|
if (LIVE.has(record.state)) {
|
|
464
929
|
record.state = 'interrupted';
|
|
465
930
|
record.error = 'Schedule owner or service is no longer current.';
|
|
@@ -493,15 +958,30 @@ class ScheduleService {
|
|
|
493
958
|
record.error = 'Occurrence tracking limit reached; native schedule stopped.';
|
|
494
959
|
cleanup = true;
|
|
495
960
|
}
|
|
961
|
+
else if (event.type === 'skipped') {
|
|
962
|
+
if (record.recovery?.handoffPolicy !== 'skip-unadmitted' || runtime.running.has(event.occurrenceId))
|
|
963
|
+
throw new Error('Unnegotiated occurrence skip.');
|
|
964
|
+
runtime.completed.add(event.occurrenceId);
|
|
965
|
+
record.completedOccurrences = [...runtime.completed];
|
|
966
|
+
// No delivery and no invented success. Unseen skipped wakes are not counted.
|
|
967
|
+
(record.skippedOccurrences ??= []).push(event.occurrenceId);
|
|
968
|
+
if (record.rule.kind === 'once') {
|
|
969
|
+
record.state = 'interrupted';
|
|
970
|
+
record.error = bounded(event.reason, ERROR_BYTES);
|
|
971
|
+
}
|
|
972
|
+
}
|
|
496
973
|
else if (event.type === 'started') {
|
|
497
974
|
runtime.running.add(event.occurrenceId);
|
|
498
975
|
record.state = 'running';
|
|
976
|
+
if (record.recovery)
|
|
977
|
+
record.recovery.status = 'blocked';
|
|
499
978
|
}
|
|
500
979
|
else if (event.type === 'completed') {
|
|
501
980
|
if (typeof event.success !== 'boolean')
|
|
502
981
|
throw new Error('Invalid native completion event.');
|
|
503
982
|
runtime.running.delete(event.occurrenceId);
|
|
504
983
|
runtime.completed.add(event.occurrenceId);
|
|
984
|
+
record.completedOccurrences = [...runtime.completed];
|
|
505
985
|
if (this.pending.size >= this.maxPending) {
|
|
506
986
|
record.state = 'interrupted';
|
|
507
987
|
record.error = 'Delivery outbox is full; latest result could not be retained. Native schedule stopped.';
|
|
@@ -519,8 +999,11 @@ class ScheduleService {
|
|
|
519
999
|
record.state = 'completed';
|
|
520
1000
|
cleanup = true;
|
|
521
1001
|
}
|
|
522
|
-
else if (record.state !== 'interrupted')
|
|
1002
|
+
else if (record.state !== 'interrupted') {
|
|
523
1003
|
record.state = runtime.running.size ? 'running' : 'active';
|
|
1004
|
+
if (record.recovery && !runtime.running.size)
|
|
1005
|
+
record.recovery.status = 'eligible';
|
|
1006
|
+
}
|
|
524
1007
|
}
|
|
525
1008
|
}
|
|
526
1009
|
else
|
|
@@ -587,9 +1070,12 @@ class ScheduleService {
|
|
|
587
1070
|
if (runtime.stopped)
|
|
588
1071
|
return Promise.resolve();
|
|
589
1072
|
if (runtime.stopPromise)
|
|
590
|
-
return runtime.stopPromise;
|
|
1073
|
+
return runtime.recoveryAbort ? this.boundRecovery(runtime.stopPromise, 'Native cleanup remains unresolved; ownership retained.') : runtime.stopPromise;
|
|
591
1074
|
const operation = (async () => {
|
|
592
1075
|
try {
|
|
1076
|
+
if (runtime.recoveryAbort?.signal.aborted && !runtime.handle && !runtime.stopped) {
|
|
1077
|
+
throw new Error('Recovery receipt is unresolved; native ownership retained.');
|
|
1078
|
+
}
|
|
593
1079
|
const handle = await runtime.ready;
|
|
594
1080
|
if (handle && !runtime.stopped)
|
|
595
1081
|
await handle.stop();
|
|
@@ -597,6 +1083,10 @@ class ScheduleService {
|
|
|
597
1083
|
// finishes. Keep ownership until its terminal stopped notification.
|
|
598
1084
|
if (!handle?.cancellationKeepsRunning || !runtime.running.size)
|
|
599
1085
|
runtime.stopped = true;
|
|
1086
|
+
if (runtime.stopped) {
|
|
1087
|
+
runtime.releaseAdmission?.();
|
|
1088
|
+
runtime.releaseAdmission = undefined;
|
|
1089
|
+
}
|
|
600
1090
|
}
|
|
601
1091
|
catch (error) {
|
|
602
1092
|
await this.serial(async () => {
|
|
@@ -610,7 +1100,19 @@ class ScheduleService {
|
|
|
610
1100
|
runtime.stopPromise = operation;
|
|
611
1101
|
void operation.finally(() => { if (!runtime.stopped)
|
|
612
1102
|
runtime.stopPromise = undefined; }).catch(() => { });
|
|
613
|
-
return operation;
|
|
1103
|
+
return runtime.recoveryAbort ? this.boundRecovery(operation, 'Native cleanup remains unresolved; ownership retained.') : operation;
|
|
1104
|
+
}
|
|
1105
|
+
async boundRecovery(operation, message) {
|
|
1106
|
+
let timer;
|
|
1107
|
+
try {
|
|
1108
|
+
return await Promise.race([operation, new Promise((_, reject) => {
|
|
1109
|
+
timer = setTimeout(() => reject(new Error(message)), this.options.recoveryTimeoutMs ?? 30_000);
|
|
1110
|
+
})]);
|
|
1111
|
+
}
|
|
1112
|
+
finally {
|
|
1113
|
+
if (timer)
|
|
1114
|
+
clearTimeout(timer);
|
|
1115
|
+
}
|
|
614
1116
|
}
|
|
615
1117
|
newRuntime() {
|
|
616
1118
|
let resolveReady;
|
|
@@ -653,7 +1155,7 @@ class ScheduleService {
|
|
|
653
1155
|
checkReady() {
|
|
654
1156
|
if (this.fault)
|
|
655
1157
|
throw this.fault;
|
|
656
|
-
if (!this.initialized || this.closing)
|
|
1158
|
+
if (!this.initialized || this.closing || this.suspending)
|
|
657
1159
|
throw new Error('Schedule service is not initialized or is closed.');
|
|
658
1160
|
}
|
|
659
1161
|
serial(operation) {
|
|
@@ -676,7 +1178,8 @@ class ScheduleService {
|
|
|
676
1178
|
}
|
|
677
1179
|
try {
|
|
678
1180
|
// JSON can expand one control byte to six bytes (for example, "\\u0001").
|
|
679
|
-
const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024)
|
|
1181
|
+
const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024)
|
|
1182
|
+
+ this.maxRecords * this.maxOccurrences * 515 + 4096;
|
|
680
1183
|
const size = (await handle.stat()).size;
|
|
681
1184
|
if (size > maxBytes)
|
|
682
1185
|
throw new Error('Schedule catalog exceeds its size limit.');
|
|
@@ -702,13 +1205,13 @@ class ScheduleService {
|
|
|
702
1205
|
}
|
|
703
1206
|
load(data) {
|
|
704
1207
|
const catalog = object(data, ['version', 'records', 'pending']);
|
|
705
|
-
if (catalog.version
|
|
1208
|
+
if (![1, 2].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
|
|
706
1209
|
|| !Array.isArray(catalog.pending) || catalog.pending.length > this.maxPending)
|
|
707
1210
|
throw new Error('Invalid schedule catalog envelope.');
|
|
708
1211
|
const records = new Map();
|
|
709
1212
|
const pending = new Map();
|
|
710
1213
|
for (const value of catalog.records) {
|
|
711
|
-
const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation']);
|
|
1214
|
+
const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation', 'execution', 'cancelRequested', 'recovery', 'completedOccurrences', 'skippedOccurrences']);
|
|
712
1215
|
(0, ScheduleCommand_1.validateScheduleId)(record.id);
|
|
713
1216
|
validateOwner(record);
|
|
714
1217
|
(0, ScheduleCommand_1.validateSchedulePrompt)(record.prompt);
|
|
@@ -721,6 +1224,35 @@ class ScheduleService {
|
|
|
721
1224
|
text(record.nativeId);
|
|
722
1225
|
if (record.error !== undefined && (typeof record.error !== 'string' || Buffer.byteLength(record.error) > ERROR_BYTES))
|
|
723
1226
|
throw new Error('Invalid schedule error.');
|
|
1227
|
+
if (record.cancelRequested !== undefined && record.cancelRequested !== true)
|
|
1228
|
+
throw new Error('Invalid cancellation intent.');
|
|
1229
|
+
if (record.execution !== undefined) {
|
|
1230
|
+
const execution = object(record.execution, ['command', 'model', 'effort']);
|
|
1231
|
+
for (const value of Object.values(execution))
|
|
1232
|
+
if (value !== undefined)
|
|
1233
|
+
text(value, 4096);
|
|
1234
|
+
}
|
|
1235
|
+
if (record.recovery !== undefined) {
|
|
1236
|
+
if (catalog.version !== 2)
|
|
1237
|
+
throw new Error('Recovery requires catalog version 2.');
|
|
1238
|
+
const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy']);
|
|
1239
|
+
this.validateCheckpoint(recovery);
|
|
1240
|
+
if (!['eligible', 'pending', 'blocked'].includes(recovery.status))
|
|
1241
|
+
throw new Error('Invalid recovery status.');
|
|
1242
|
+
}
|
|
1243
|
+
if (record.completedOccurrences !== undefined) {
|
|
1244
|
+
if (!Array.isArray(record.completedOccurrences) || record.completedOccurrences.length > this.maxOccurrences
|
|
1245
|
+
|| new Set(record.completedOccurrences).size !== record.completedOccurrences.length)
|
|
1246
|
+
throw new Error('Invalid occurrence history.');
|
|
1247
|
+
record.completedOccurrences.forEach(value => text(value));
|
|
1248
|
+
}
|
|
1249
|
+
if (record.skippedOccurrences !== undefined) {
|
|
1250
|
+
if (!Array.isArray(record.skippedOccurrences) || record.skippedOccurrences.length > this.maxOccurrences
|
|
1251
|
+
|| new Set(record.skippedOccurrences).size !== record.skippedOccurrences.length
|
|
1252
|
+
|| record.skippedOccurrences.some(value => !Array.isArray(record.completedOccurrences) || !record.completedOccurrences.includes(value))) {
|
|
1253
|
+
throw new Error('Invalid skipped occurrence history.');
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
724
1256
|
const rule = object(record.rule, ['kind', 'at', 'expression', 'timezone']);
|
|
725
1257
|
if (rule.kind === 'once')
|
|
726
1258
|
object(rule, ['kind', 'at']);
|
|
@@ -763,7 +1295,7 @@ class ScheduleService {
|
|
|
763
1295
|
}
|
|
764
1296
|
const handle = await fs.open(temporary, 'wx', 0o600);
|
|
765
1297
|
try {
|
|
766
|
-
await handle.writeFile(JSON.stringify({ version:
|
|
1298
|
+
await handle.writeFile(JSON.stringify({ version: 2, records: [...this.records.values()], pending: [...this.pending.values()] }), 'utf8');
|
|
767
1299
|
await handle.sync();
|
|
768
1300
|
}
|
|
769
1301
|
finally {
|