@yu_robotics/remote-cli 1.6.236 → 1.6.238
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/MessageHandler.d.ts +9 -2
- package/dist/client/MessageHandler.d.ts.map +1 -1
- package/dist/client/MessageHandler.js +266 -34
- package/dist/client/MessageHandler.js.map +1 -1
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +1 -0
- package/dist/commands/start.js.map +1 -1
- package/dist/executor/AcpExecutor.d.ts +2 -0
- package/dist/executor/AcpExecutor.d.ts.map +1 -1
- package/dist/executor/AcpExecutor.js +9 -0
- package/dist/executor/AcpExecutor.js.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.d.ts +52 -12
- package/dist/executor/ClaudePersistentExecutor.d.ts.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.js +795 -123
- package/dist/executor/ClaudePersistentExecutor.js.map +1 -1
- package/dist/executor/DshExecutor.d.ts +13 -1
- package/dist/executor/DshExecutor.d.ts.map +1 -1
- package/dist/executor/DshExecutor.js +39 -1
- package/dist/executor/DshExecutor.js.map +1 -1
- package/dist/executor/IExecutor.d.ts +14 -0
- package/dist/executor/IExecutor.d.ts.map +1 -1
- package/dist/executor/claude/TurnLedger.d.ts +33 -0
- package/dist/executor/claude/TurnLedger.d.ts.map +1 -0
- package/dist/executor/claude/TurnLedger.js +126 -0
- package/dist/executor/claude/TurnLedger.js.map +1 -0
- package/dist/maintenance/release-notes.json +1 -1
- package/dist/managed/ManagedToolBridge.d.ts +4 -1
- package/dist/managed/ManagedToolBridge.d.ts.map +1 -1
- package/dist/managed/ManagedToolBridge.js +17 -2
- package/dist/managed/ManagedToolBridge.js.map +1 -1
- package/dist/scheduling/ScheduleCapabilities.d.ts +1 -1
- package/dist/scheduling/ScheduleCapabilities.d.ts.map +1 -1
- package/dist/scheduling/ScheduleCapabilities.js +3 -2
- package/dist/scheduling/ScheduleCapabilities.js.map +1 -1
- package/dist/scheduling/ScheduleService.d.ts +28 -0
- package/dist/scheduling/ScheduleService.d.ts.map +1 -1
- package/dist/scheduling/ScheduleService.js +562 -17
- package/dist/scheduling/ScheduleService.js.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts +28 -2
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.js +158 -18
- package/dist/scheduling/claude/ClaudeSessionSchedules.js.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleBridge.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleBridge.js +21 -0
- package/dist/scheduling/dsh/SessionScheduleBridge.js.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.js +8 -0
- package/dist/scheduling/dsh/SessionScheduleClient.js.map +1 -1
- package/dist/scheduling/types.d.ts +84 -0
- package/dist/scheduling/types.d.ts.map +1 -1
- package/dist/scheduling/types.js.map +1 -1
- package/dist/settings/SettingsService.d.ts +1 -1
- package/dist/settings/SettingsService.d.ts.map +1 -1
- package/dist/settings/SettingsService.js +3 -3
- package/dist/settings/SettingsService.js.map +1 -1
- package/dist/thread/ThreadExecutorPool.d.ts +1 -1
- package/dist/thread/ThreadExecutorPool.d.ts.map +1 -1
- package/dist/thread/ThreadExecutorPool.js +5 -3
- package/dist/thread/ThreadExecutorPool.js.map +1 -1
- package/dist/update/AutomaticUpdater.d.ts +2 -1
- package/dist/update/AutomaticUpdater.d.ts.map +1 -1
- package/dist/update/AutomaticUpdater.js +11 -3
- package/dist/update/AutomaticUpdater.js.map +1 -1
- package/native/prebuilds/darwin-arm64/managed_native.node +0 -0
- package/native/prebuilds/darwin-arm64/manifest.json +1 -1
- package/native/prebuilds/darwin-x64/managed_native.node +0 -0
- package/native/prebuilds/darwin-x64/manifest.json +1 -1
- package/native/prebuilds/win32-arm64/managed_native.node +0 -0
- package/native/prebuilds/win32-arm64/manifest.json +1 -1
- package/native/prebuilds/win32-x64/managed_native.node +0 -0
- package/native/prebuilds/win32-x64/manifest.json +1 -1
- package/package.json +1 -1
|
@@ -83,6 +83,11 @@ function validateOwner(owner) {
|
|
|
83
83
|
function sameOwner(left, right) {
|
|
84
84
|
return left.threadId === right.threadId && left.openId === right.openId && left.generation === right.generation;
|
|
85
85
|
}
|
|
86
|
+
/** Terminal intent prevents recreation, but uncertain native cleanup still owns admission. */
|
|
87
|
+
function recoveryOwnsWork(record) {
|
|
88
|
+
return !!record.recovery && (!['completed', 'cancelled'].includes(record.state)
|
|
89
|
+
|| !!record.error?.startsWith('Native process exit unconfirmed:'));
|
|
90
|
+
}
|
|
86
91
|
function deliveryId(scheduleId, occurrenceId) {
|
|
87
92
|
return `${scheduleId}:${occurrenceId}`;
|
|
88
93
|
}
|
|
@@ -112,6 +117,10 @@ class ScheduleService {
|
|
|
112
117
|
destruction;
|
|
113
118
|
fault;
|
|
114
119
|
catalogPath = '';
|
|
120
|
+
recoveryRun;
|
|
121
|
+
suspending = false;
|
|
122
|
+
handoffUnresolved = false;
|
|
123
|
+
suspension;
|
|
115
124
|
constructor(options) {
|
|
116
125
|
this.options = { ...options };
|
|
117
126
|
this.now = options.now ?? Date.now;
|
|
@@ -121,6 +130,9 @@ class ScheduleService {
|
|
|
121
130
|
throw new Error('Invalid schedule capacity.');
|
|
122
131
|
return result;
|
|
123
132
|
};
|
|
133
|
+
if (options.recoveryTimeoutMs !== undefined && (!Number.isInteger(options.recoveryTimeoutMs)
|
|
134
|
+
|| options.recoveryTimeoutMs < 1 || options.recoveryTimeoutMs > 60_000))
|
|
135
|
+
throw new Error('Invalid recovery timeout.');
|
|
124
136
|
this.maxLive = limit(options.maxLiveSchedules, 5, 5);
|
|
125
137
|
this.maxRecords = limit(options.maxRecords, 100, 1000);
|
|
126
138
|
this.maxPending = limit(options.maxPendingDeliveries, 100, 1000);
|
|
@@ -138,9 +150,23 @@ class ScheduleService {
|
|
|
138
150
|
this.load(data);
|
|
139
151
|
let changed = false;
|
|
140
152
|
for (const record of this.records.values()) {
|
|
141
|
-
if (
|
|
153
|
+
if (['completed', 'cancelled', 'failed'].includes(record.state)) {
|
|
154
|
+
if (record.recovery?.status === 'pending') {
|
|
155
|
+
record.recovery.status = 'blocked';
|
|
156
|
+
changed = true;
|
|
157
|
+
}
|
|
158
|
+
continue; // Terminal tombstones always win over a stale recovery checkpoint.
|
|
159
|
+
}
|
|
160
|
+
if (LIVE.has(record.state) || record.recovery?.status === 'pending') {
|
|
161
|
+
const recoverable = record.recovery && !record.cancelRequested && this.current(record)
|
|
162
|
+
&& record.state !== 'running' && record.state !== 'failed'
|
|
163
|
+
&& (!record.recovery.orderlyOnly || record.recovery.handoff === 'clean')
|
|
164
|
+
&& (record.state === 'active' || record.recovery.status === 'pending');
|
|
165
|
+
if (record.recovery)
|
|
166
|
+
record.recovery.status = recoverable ? 'pending' : 'blocked';
|
|
167
|
+
record.error = recoverable ? 'Waiting for native recovery capability and owner readiness.'
|
|
168
|
+
: record.recovery?.orderlyOnly ? 'Dirty shutdown: original native work is unconfirmed; automatic reconstruction is blocked.' : RESTART_REASON;
|
|
142
169
|
record.state = 'interrupted';
|
|
143
|
-
record.error = RESTART_REASON;
|
|
144
170
|
changed = true;
|
|
145
171
|
}
|
|
146
172
|
}
|
|
@@ -156,6 +182,193 @@ class ScheduleService {
|
|
|
156
182
|
this.initialized = true;
|
|
157
183
|
});
|
|
158
184
|
}
|
|
185
|
+
/** Sequential startup/reconnect reconciliation. Never drives the native schedule clock. */
|
|
186
|
+
recover() {
|
|
187
|
+
this.checkReady();
|
|
188
|
+
if (this.recoveryRun)
|
|
189
|
+
return this.recoveryRun;
|
|
190
|
+
const run = this.operation(this.recoverWaiting());
|
|
191
|
+
this.recoveryRun = run;
|
|
192
|
+
void run.finally(() => { if (this.recoveryRun === run)
|
|
193
|
+
this.recoveryRun = undefined; }).catch(() => { });
|
|
194
|
+
return run;
|
|
195
|
+
}
|
|
196
|
+
async recoverWaiting() {
|
|
197
|
+
const candidates = await this.serial(async () => [...this.records.values()]
|
|
198
|
+
.filter(record => record.recovery?.status === 'pending' && record.state === 'interrupted').map(record => record.id));
|
|
199
|
+
// Catalog capacity is bounded, and only one native admission runs at a time.
|
|
200
|
+
for (const id of candidates.slice(0, this.maxLive)) {
|
|
201
|
+
if (this.closing || this.fault)
|
|
202
|
+
return;
|
|
203
|
+
const record = this.records.get(id);
|
|
204
|
+
if (record.cancelRequested || !this.current(record)) {
|
|
205
|
+
await this.blockRecovery(record, 'Recovery owner retired or cancellation was requested.');
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
if (record.rule.kind === 'once' && Date.parse(record.rule.at) <= this.now()) {
|
|
209
|
+
await this.blockRecovery(record, 'One-shot deadline expired; no late execution was requested.');
|
|
210
|
+
if (record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent)
|
|
211
|
+
await this.serial(async () => { delete record.recovery; await this.persist(); });
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
const input = { id, threadId: record.threadId, backend: record.backend,
|
|
215
|
+
cwd: record.cwd, prompt: record.prompt, rule: structuredClone(record.rule), ...record.execution,
|
|
216
|
+
recovery: this.checkpoint(record.recovery) };
|
|
217
|
+
if (!this.options.canRecover || !this.options.canRecover(structuredClone(record))) {
|
|
218
|
+
await this.recoveryDiagnostic(record, 'Original owner configuration or transport is not ready for recovery.');
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
const provider = this.options.provider.recoveryFor?.(input);
|
|
222
|
+
if (!provider || provider.provider !== record.recovery.provider || provider.strategy !== record.recovery.strategy
|
|
223
|
+
|| (provider.handoffPolicy ?? 'drain-all') !== (record.recovery.handoffPolicy ?? 'drain-all')) {
|
|
224
|
+
await this.recoveryDiagnostic(record, 'Native restart recovery is unsupported or provider identity changed.');
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
if (this.runtimes.has(id))
|
|
228
|
+
continue; // An unresolved/late operation retains exclusive ownership.
|
|
229
|
+
const restoreRecord = structuredClone(record);
|
|
230
|
+
const runtime = this.newRuntime();
|
|
231
|
+
runtime.completed = new Set(record.completedOccurrences);
|
|
232
|
+
const controller = new AbortController();
|
|
233
|
+
runtime.recoveryAbort = controller;
|
|
234
|
+
this.runtimes.set(id, runtime);
|
|
235
|
+
const current = () => !controller.signal.aborted && !this.closing && !this.fault && !record.cancelRequested
|
|
236
|
+
&& this.current(record) && this.options.canRecover?.(structuredClone(record)) === true;
|
|
237
|
+
const emit = (event) => {
|
|
238
|
+
if (this.runtimes.get(id) !== runtime)
|
|
239
|
+
return;
|
|
240
|
+
this.track(this.onEvent(id, structuredClone(event)));
|
|
241
|
+
};
|
|
242
|
+
let timer;
|
|
243
|
+
let admitted = false;
|
|
244
|
+
let succeeded = false;
|
|
245
|
+
try {
|
|
246
|
+
runtime.releaseAdmission = this.options.acquireRecovery?.(structuredClone(record));
|
|
247
|
+
await this.serial(async () => {
|
|
248
|
+
if (!current())
|
|
249
|
+
throw new Error('Recovery admission expired.');
|
|
250
|
+
// Consume clean proof before mutation. A crash must not replay an orderly-only restore.
|
|
251
|
+
record.state = 'creating';
|
|
252
|
+
delete record.recovery.handoff;
|
|
253
|
+
delete record.recovery.nativeDefinitionAbsent;
|
|
254
|
+
await this.persist();
|
|
255
|
+
});
|
|
256
|
+
admitted = true;
|
|
257
|
+
const admission = Promise.resolve().then(() => provider.reconcile(restoreRecord, { notBefore: this.now(), signal: controller.signal, isCurrent: current,
|
|
258
|
+
isOwnerCurrent: () => !record.cancelRequested && !this.fault && this.current(record) }, emit));
|
|
259
|
+
const adopt = async (handle) => {
|
|
260
|
+
runtime.handle = handle;
|
|
261
|
+
runtime.resolveReady(handle);
|
|
262
|
+
text(handle.nativeId);
|
|
263
|
+
await this.serial(async () => { record.nativeId = handle.nativeId; if (!this.fault)
|
|
264
|
+
await this.persist(); });
|
|
265
|
+
};
|
|
266
|
+
const receipt = admission.then(async (handle) => {
|
|
267
|
+
await adopt(handle);
|
|
268
|
+
if (!current()) {
|
|
269
|
+
await this.stop(id, runtime);
|
|
270
|
+
throw new Error('Late recovery receipt requires cleanup; no successor was admitted.');
|
|
271
|
+
}
|
|
272
|
+
if (!await handle.list() || !current())
|
|
273
|
+
throw new Error('Recovered native inventory or owner is unconfirmed.');
|
|
274
|
+
return handle;
|
|
275
|
+
}, async (error) => {
|
|
276
|
+
if (error instanceof types_1.NativeScheduleCleanupError) {
|
|
277
|
+
await adopt(error.handle);
|
|
278
|
+
if (controller.signal.aborted)
|
|
279
|
+
await this.stop(id, runtime).catch(() => { });
|
|
280
|
+
}
|
|
281
|
+
runtime.resolveReady(runtime.handle);
|
|
282
|
+
// A rejection without a handle is not proof that native admission never happened.
|
|
283
|
+
throw error;
|
|
284
|
+
});
|
|
285
|
+
const handle = await Promise.race([receipt, new Promise((_, reject) => {
|
|
286
|
+
timer = setTimeout(() => { controller.abort(); reject(new Error('Native recovery timed out; ownership retained, do not recreate.')); }, this.options.recoveryTimeoutMs ?? 30_000);
|
|
287
|
+
})]);
|
|
288
|
+
text(handle.nativeId);
|
|
289
|
+
// The provider's reconcile contract includes inventory validation; list is
|
|
290
|
+
// additionally required to verify the returned handle before activating it.
|
|
291
|
+
if (!current())
|
|
292
|
+
throw new Error('Recovered native owner is unconfirmed.');
|
|
293
|
+
await this.serial(async () => {
|
|
294
|
+
if (!current())
|
|
295
|
+
throw new Error('Recovery owner retired before receipt persistence.');
|
|
296
|
+
record.nativeId = handle.nativeId;
|
|
297
|
+
if (record.state === 'creating')
|
|
298
|
+
record.state = 'active';
|
|
299
|
+
record.recovery.status = 'eligible';
|
|
300
|
+
delete record.error;
|
|
301
|
+
await this.persist();
|
|
302
|
+
});
|
|
303
|
+
succeeded = true;
|
|
304
|
+
}
|
|
305
|
+
catch (error) {
|
|
306
|
+
controller.abort();
|
|
307
|
+
if (!admitted) {
|
|
308
|
+
runtime.stopped = true;
|
|
309
|
+
runtime.resolveReady(undefined);
|
|
310
|
+
}
|
|
311
|
+
// Do not retry an ambiguous mutation or release its handle on a timeout.
|
|
312
|
+
await this.blockRecovery(record, errorText(error));
|
|
313
|
+
if (runtime.handle)
|
|
314
|
+
await this.stop(id, runtime).catch(() => { });
|
|
315
|
+
else if (runtime.stopped)
|
|
316
|
+
this.runtimes.delete(id);
|
|
317
|
+
}
|
|
318
|
+
finally {
|
|
319
|
+
if (timer)
|
|
320
|
+
clearTimeout(timer);
|
|
321
|
+
if (succeeded || runtime.stopped) {
|
|
322
|
+
runtime.releaseAdmission?.();
|
|
323
|
+
runtime.releaseAdmission = undefined;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
// A handoff containing only skipped/expired one-shots has nothing to
|
|
328
|
+
// reconcile. It must still release the predecessor's local input fence.
|
|
329
|
+
const released = new Set();
|
|
330
|
+
for (const record of this.records.values()) {
|
|
331
|
+
if (released.has(record.threadId) || this.closing || this.fault || this.handoffUnresolved
|
|
332
|
+
|| !this.current(record) || !this.options.canRecover?.(structuredClone(record)))
|
|
333
|
+
continue;
|
|
334
|
+
if ([...this.records.values()].some(other => other.threadId === record.threadId
|
|
335
|
+
&& (LIVE.has(other.state) || recoveryOwnsWork(other) || this.runtimes.has(other.id))))
|
|
336
|
+
continue;
|
|
337
|
+
this.options.provider.recoveryFor?.({ id: record.id, threadId: record.threadId, backend: record.backend,
|
|
338
|
+
cwd: record.cwd, prompt: record.prompt, rule: record.rule, ...record.execution })?.releaseRestartAdmission?.();
|
|
339
|
+
released.add(record.threadId);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
checkpoint(value) {
|
|
343
|
+
return { version: 1, provider: value.provider, strategy: value.strategy, identity: value.identity,
|
|
344
|
+
...(value.orderlyOnly && { orderlyOnly: true }), ...(value.handoffPolicy && { handoffPolicy: value.handoffPolicy }), ...(value.handoff && { handoff: value.handoff }),
|
|
345
|
+
...(value.nativeDefinitionAbsent && { nativeDefinitionAbsent: true }) };
|
|
346
|
+
}
|
|
347
|
+
validateCheckpoint(value) {
|
|
348
|
+
if (value.version !== 1 || !['reattach', 'reconstruct'].includes(value.strategy))
|
|
349
|
+
throw new Error('Invalid recovery checkpoint.');
|
|
350
|
+
if (value.orderlyOnly !== undefined && value.orderlyOnly !== true || value.handoff !== undefined && value.handoff !== 'clean')
|
|
351
|
+
throw new Error('Invalid handoff evidence.');
|
|
352
|
+
if (value.nativeDefinitionAbsent !== undefined && (value.nativeDefinitionAbsent !== true || value.handoff !== 'clean'))
|
|
353
|
+
throw new Error('Invalid native absence evidence.');
|
|
354
|
+
if (value.handoffPolicy !== undefined && !['drain-all', 'skip-unadmitted'].includes(value.handoffPolicy))
|
|
355
|
+
throw new Error('Invalid handoff policy.');
|
|
356
|
+
text(value.provider);
|
|
357
|
+
text(value.identity, 8192);
|
|
358
|
+
}
|
|
359
|
+
async recoveryDiagnostic(record, reason) {
|
|
360
|
+
await this.serial(async () => { record.error = reason; await this.persist(); });
|
|
361
|
+
}
|
|
362
|
+
async blockRecovery(record, reason) {
|
|
363
|
+
await this.serial(async () => {
|
|
364
|
+
if (!['completed', 'cancelled'].includes(record.state))
|
|
365
|
+
record.state = 'interrupted';
|
|
366
|
+
record.recovery.status = 'blocked';
|
|
367
|
+
record.error = reason;
|
|
368
|
+
if (!this.fault)
|
|
369
|
+
await this.persist();
|
|
370
|
+
});
|
|
371
|
+
}
|
|
159
372
|
create(owner, input) {
|
|
160
373
|
return this.operation(this.createSchedule(owner, input));
|
|
161
374
|
}
|
|
@@ -174,7 +387,18 @@ class ScheduleService {
|
|
|
174
387
|
if (!node_path_1.default.isAbsolute(input.cwd))
|
|
175
388
|
throw new Error('Schedule working directory must be absolute.');
|
|
176
389
|
const originalOwner = ownerOf(owner);
|
|
177
|
-
const nativeInput = { ...structuredClone(input), backend };
|
|
390
|
+
const nativeInput = { ...structuredClone(input), backend, threadId: owner.threadId };
|
|
391
|
+
// Never trust a caller-supplied provider checkpoint.
|
|
392
|
+
delete nativeInput.recovery;
|
|
393
|
+
for (const value of [input.command, input.model, input.effort])
|
|
394
|
+
if (value !== undefined)
|
|
395
|
+
text(value, 4096);
|
|
396
|
+
const recovery = this.options.provider.recoveryFor?.(nativeInput);
|
|
397
|
+
if (recovery) {
|
|
398
|
+
nativeInput.recovery = { version: 1, provider: recovery.provider, strategy: recovery.strategy,
|
|
399
|
+
identity: recovery.describe(structuredClone(nativeInput), originalOwner), ...(recovery.orderlyOnly && { orderlyOnly: true }), ...(recovery.handoffPolicy && { handoffPolicy: recovery.handoffPolicy }) };
|
|
400
|
+
this.validateCheckpoint(nativeInput.recovery);
|
|
401
|
+
}
|
|
178
402
|
const runtime = this.newRuntime();
|
|
179
403
|
await this.serial(async () => {
|
|
180
404
|
this.checkReady();
|
|
@@ -185,17 +409,21 @@ class ScheduleService {
|
|
|
185
409
|
if (this.records.size >= this.maxRecords)
|
|
186
410
|
throw new Error('Schedule catalog is full; history was not pruned.');
|
|
187
411
|
const liveCount = [...this.records.values()].filter(record => LIVE.has(record.state)
|
|
412
|
+
|| recoveryOwnsWork(record)
|
|
188
413
|
|| (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)).length;
|
|
189
414
|
if (liveCount >= this.maxLive)
|
|
190
415
|
throw new Error('Live native schedule limit reached.');
|
|
191
416
|
this.records.set(nativeInput.id, { ...originalOwner, id: nativeInput.id, backend,
|
|
192
417
|
cwd: nativeInput.cwd, prompt: nativeInput.prompt, rule: structuredClone(nativeInput.rule),
|
|
193
|
-
createdAt, state: 'creating' }
|
|
418
|
+
createdAt, state: 'creating', execution: { command: input.command, model: input.model, effort: input.effort },
|
|
419
|
+
...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: 'eligible' } }) });
|
|
194
420
|
await this.persist();
|
|
195
421
|
this.runtimes.set(nativeInput.id, runtime);
|
|
196
422
|
});
|
|
197
423
|
try {
|
|
198
424
|
const handle = await this.options.provider.create(structuredClone(nativeInput), event => {
|
|
425
|
+
if (this.runtimes.get(nativeInput.id) !== runtime)
|
|
426
|
+
return;
|
|
199
427
|
const task = this.onEvent(nativeInput.id, structuredClone(event));
|
|
200
428
|
this.track(task);
|
|
201
429
|
});
|
|
@@ -205,6 +433,10 @@ class ScheduleService {
|
|
|
205
433
|
await this.serial(async () => {
|
|
206
434
|
const record = this.records.get(nativeInput.id);
|
|
207
435
|
record.nativeId = handle.nativeId;
|
|
436
|
+
if (record.recovery && handle.recoveryIdentity) {
|
|
437
|
+
text(handle.recoveryIdentity, 8192);
|
|
438
|
+
record.recovery.identity = handle.recoveryIdentity;
|
|
439
|
+
}
|
|
208
440
|
const ownerCurrent = this.current(originalOwner);
|
|
209
441
|
if (this.closing || !ownerCurrent) {
|
|
210
442
|
if (LIVE.has(record.state)) {
|
|
@@ -252,7 +484,7 @@ class ScheduleService {
|
|
|
252
484
|
/** Includes terminal jobs whose native process exit is still unconfirmed. */
|
|
253
485
|
hasActiveSchedules(threadId) {
|
|
254
486
|
return [...this.records.values()].some(record => (!threadId || record.threadId === threadId)
|
|
255
|
-
&& (LIVE.has(record.state) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
|
|
487
|
+
&& (LIVE.has(record.state) || recoveryOwnsWork(record) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
|
|
256
488
|
}
|
|
257
489
|
/** Card eligibility includes retryable cleanup even for a terminal catalog record. */
|
|
258
490
|
canCancel(owner, id) {
|
|
@@ -261,7 +493,20 @@ class ScheduleService {
|
|
|
261
493
|
const runtime = this.runtimes.get(id);
|
|
262
494
|
if (record.state === 'cancelled' && runtime?.handle?.cancellationKeepsRunning && !record.error)
|
|
263
495
|
return false;
|
|
264
|
-
return LIVE.has(record.state) || Boolean(runtime && !runtime.stopped);
|
|
496
|
+
return LIVE.has(record.state) || recoveryOwnsWork(record) || Boolean(runtime && !runtime.stopped);
|
|
497
|
+
}
|
|
498
|
+
/** Keep admitted native work visible when cleanup cannot safely acquire its host. */
|
|
499
|
+
async retainCleanupError(owner, id, error) {
|
|
500
|
+
this.checkReady();
|
|
501
|
+
validateOwner(owner);
|
|
502
|
+
(0, ScheduleCommand_1.validateScheduleId)(id);
|
|
503
|
+
await this.serial(async () => {
|
|
504
|
+
const record = this.owned(owner, id);
|
|
505
|
+
if (this.runtimes.get(id)?.stopped)
|
|
506
|
+
return;
|
|
507
|
+
record.error = errorText(error);
|
|
508
|
+
await this.persist();
|
|
509
|
+
});
|
|
265
510
|
}
|
|
266
511
|
/** User-requested native reconciliation only; no background polling is installed. */
|
|
267
512
|
refresh(owner) {
|
|
@@ -320,11 +565,30 @@ class ScheduleService {
|
|
|
320
565
|
await this.serial(async () => {
|
|
321
566
|
const record = this.owned(owner, id);
|
|
322
567
|
runtime = this.runtimes.get(id);
|
|
568
|
+
record.cancelRequested = true;
|
|
569
|
+
runtime?.recoveryAbort?.abort();
|
|
570
|
+
if (record.recovery)
|
|
571
|
+
record.recovery.status = 'blocked';
|
|
572
|
+
await this.persist();
|
|
573
|
+
if (!runtime && record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent) {
|
|
574
|
+
record.state = 'cancelled';
|
|
575
|
+
delete record.error;
|
|
576
|
+
await this.persist();
|
|
577
|
+
}
|
|
578
|
+
if (!runtime && recoveryOwnsWork(record)) {
|
|
579
|
+
record.error = 'Cancellation intent retained; native cleanup is unconfirmed. Recovery will not recreate this job.';
|
|
580
|
+
await this.persist();
|
|
581
|
+
throw new Error(record.error);
|
|
582
|
+
}
|
|
323
583
|
if (runtime?.cancelPromise) {
|
|
324
584
|
existing = runtime.cancelPromise;
|
|
325
585
|
return;
|
|
326
586
|
}
|
|
327
587
|
if (!LIVE.has(record.state) && !runtime?.cancelling && (!runtime || runtime.stopped)) {
|
|
588
|
+
if (record.recovery && runtime?.stopped) {
|
|
589
|
+
record.state = 'cancelled';
|
|
590
|
+
delete record.error;
|
|
591
|
+
}
|
|
328
592
|
for (const [key, pending] of this.pending) {
|
|
329
593
|
if (pending.scheduleId === id && !pending.afterCancellation)
|
|
330
594
|
this.pending.delete(key);
|
|
@@ -335,6 +599,9 @@ class ScheduleService {
|
|
|
335
599
|
}
|
|
336
600
|
if (!runtime)
|
|
337
601
|
throw new Error('Native schedule handle is unavailable.');
|
|
602
|
+
if (runtime.recoveryAbort && !runtime.handle && !runtime.stopped) {
|
|
603
|
+
throw new Error('Recovery receipt is unresolved; cancellation intent retained until native cleanup can be confirmed.');
|
|
604
|
+
}
|
|
338
605
|
runtime.cancelling = true;
|
|
339
606
|
// Starting outside the state queue also permits synchronous provider callbacks.
|
|
340
607
|
operation = Promise.resolve().then(() => this.cancelNative(id, runtime));
|
|
@@ -348,24 +615,221 @@ class ScheduleService {
|
|
|
348
615
|
const ids = await this.serial(async () => [...this.pending.keys()]);
|
|
349
616
|
await this.attemptDeliveries(ids);
|
|
350
617
|
}
|
|
618
|
+
hasUnresolvedHandoff() { return this.handoffUnresolved; }
|
|
619
|
+
hasUnresolvedRecovery() {
|
|
620
|
+
return [...this.records.values()].some(record => recoveryOwnsWork(record)
|
|
621
|
+
&& record.recovery?.status !== 'eligible');
|
|
622
|
+
}
|
|
623
|
+
hasPendingRecovery() {
|
|
624
|
+
return [...this.records.values()].some(record => record.state === 'interrupted' && record.recovery?.status === 'pending');
|
|
625
|
+
}
|
|
626
|
+
/** Snapshot eligibility is only admission to a handoff, never permission to install. */
|
|
627
|
+
canSuspendForRestart() {
|
|
628
|
+
if (!this.initialized || this.closing || this.fault || this.suspending || this.recoveryRun || this.operations.size || this.pending.size)
|
|
629
|
+
return false;
|
|
630
|
+
return [...this.records.values()].filter(record => LIVE.has(record.state) || recoveryOwnsWork(record)
|
|
631
|
+
|| !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped).every(record => {
|
|
632
|
+
const runtime = this.runtimes.get(record.id);
|
|
633
|
+
return record.state === 'active' && !!record.recovery && !record.cancelRequested && !record.error
|
|
634
|
+
&& !!runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling;
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
/** Delete future definitions, drain native work, join owned hosts, then publish restart proof. */
|
|
638
|
+
suspendForRestart() {
|
|
639
|
+
if (this.suspension)
|
|
640
|
+
return this.suspension;
|
|
641
|
+
if (!this.canSuspendForRestart())
|
|
642
|
+
return Promise.reject(new Error('Schedules are not ready for restart handoff.'));
|
|
643
|
+
this.suspending = true;
|
|
644
|
+
this.handoffUnresolved = true;
|
|
645
|
+
const entries = [...this.runtimes].filter(([id]) => this.records.get(id)?.state === 'active');
|
|
646
|
+
const run = (async () => {
|
|
647
|
+
try {
|
|
648
|
+
// Checkpoints remain dirty throughout deletion/drain and actual host exit.
|
|
649
|
+
await this.serial(async () => {
|
|
650
|
+
for (const [id, runtime] of entries) {
|
|
651
|
+
if (runtime.handle?.recoveryIdentity) {
|
|
652
|
+
text(runtime.handle.recoveryIdentity, 8192);
|
|
653
|
+
this.records.get(id).recovery.identity = runtime.handle.recoveryIdentity;
|
|
654
|
+
}
|
|
655
|
+
delete this.records.get(id).recovery.handoff;
|
|
656
|
+
delete this.records.get(id).recovery.nativeDefinitionAbsent;
|
|
657
|
+
}
|
|
658
|
+
await this.persist();
|
|
659
|
+
});
|
|
660
|
+
const releases = await Promise.allSettled(entries.map(([, runtime]) => this.boundRecovery(runtime.handle.releaseForRestart(), 'Native schedule drain remains unresolved.')));
|
|
661
|
+
const failed = releases.find((result) => result.status === 'rejected');
|
|
662
|
+
if (failed)
|
|
663
|
+
throw failed.reason;
|
|
664
|
+
for (const [index, result] of releases.entries()) {
|
|
665
|
+
const policy = this.records.get(entries[index][0]).recovery.handoffPolicy ?? 'drain-all';
|
|
666
|
+
if (result.status === 'fulfilled' && (result.value?.policy ?? 'drain-all') !== policy)
|
|
667
|
+
throw new Error('Native handoff policy receipt changed.');
|
|
668
|
+
}
|
|
669
|
+
const absentOneShots = new Set();
|
|
670
|
+
for (const [index, result] of releases.entries())
|
|
671
|
+
if (result.status === 'fulfilled' && result.value?.onceDefinitionAlreadyAbsent) {
|
|
672
|
+
const record = this.records.get(entries[index][0]);
|
|
673
|
+
if (record.rule.kind !== 'once' || record.recovery?.handoffPolicy !== 'skip-unadmitted')
|
|
674
|
+
throw new Error('Invalid absent one-shot handoff receipt.');
|
|
675
|
+
absentOneShots.add(record.id);
|
|
676
|
+
}
|
|
677
|
+
const absent = new Set(entries.filter((_, index) => {
|
|
678
|
+
const result = releases[index];
|
|
679
|
+
return result.status === 'fulfilled' && result.value?.nativeDefinitionAbsent === true;
|
|
680
|
+
}).map(([id]) => id));
|
|
681
|
+
await this.boundRecovery((async () => {
|
|
682
|
+
while (this.tasks.size)
|
|
683
|
+
await Promise.allSettled([...this.tasks]);
|
|
684
|
+
})(), 'Scheduled result drain remains unresolved.');
|
|
685
|
+
await this.tail;
|
|
686
|
+
if (this.pending.size)
|
|
687
|
+
throw new Error('Scheduled result acknowledgement is pending.');
|
|
688
|
+
if (this.fault || entries.some(([, runtime]) => runtime.running.size))
|
|
689
|
+
throw this.fault ?? new Error('Native occurrence is still running.');
|
|
690
|
+
const finished = new Set();
|
|
691
|
+
for (const [id] of entries) {
|
|
692
|
+
const record = this.records.get(id);
|
|
693
|
+
const provider = this.options.provider.recoveryFor?.({ id, threadId: record.threadId, backend: record.backend,
|
|
694
|
+
cwd: record.cwd, rule: record.rule, prompt: record.prompt, ...record.execution });
|
|
695
|
+
if (!provider || provider.provider !== record.recovery.provider)
|
|
696
|
+
throw new Error('Native recovery provider changed during handoff.');
|
|
697
|
+
const key = `${record.threadId}:${provider.provider}`;
|
|
698
|
+
if (finished.has(key))
|
|
699
|
+
continue;
|
|
700
|
+
if (provider.orderlyOnly && !provider.finishRestart)
|
|
701
|
+
throw new Error('Owned native host exit cannot be confirmed.');
|
|
702
|
+
if (provider.finishRestart)
|
|
703
|
+
await this.boundRecovery(provider.finishRestart(), 'Owned native host exit remains unresolved.');
|
|
704
|
+
finished.add(key);
|
|
705
|
+
}
|
|
706
|
+
await this.boundRecovery((async () => {
|
|
707
|
+
while (this.tasks.size)
|
|
708
|
+
await Promise.allSettled([...this.tasks]);
|
|
709
|
+
})(), 'Scheduled result drain remains unresolved.');
|
|
710
|
+
await this.serial(async () => {
|
|
711
|
+
if (this.fault)
|
|
712
|
+
throw this.fault;
|
|
713
|
+
for (const [id, runtime] of entries) {
|
|
714
|
+
const record = this.records.get(id);
|
|
715
|
+
if (!this.current(record) || record.cancelRequested)
|
|
716
|
+
throw new Error('Schedule owner retired during handoff.');
|
|
717
|
+
if (record.state === 'active' && absentOneShots.has(id)) {
|
|
718
|
+
record.state = 'interrupted';
|
|
719
|
+
record.error = 'One-shot native definition was already absent at handoff; it will not be replayed.';
|
|
720
|
+
delete record.recovery;
|
|
721
|
+
}
|
|
722
|
+
else if (record.state === 'active') {
|
|
723
|
+
record.state = 'interrupted';
|
|
724
|
+
record.recovery.status = 'pending';
|
|
725
|
+
record.recovery.handoff = 'clean';
|
|
726
|
+
if (absent.has(id))
|
|
727
|
+
record.recovery.nativeDefinitionAbsent = true;
|
|
728
|
+
record.error = 'Native host drained and exited; waiting for automatic recovery.';
|
|
729
|
+
}
|
|
730
|
+
else if (record.state === 'interrupted' && record.rule.kind === 'once' && record.recovery?.handoffPolicy === 'skip-unadmitted' && !runtime.running.size) {
|
|
731
|
+
delete record.recovery;
|
|
732
|
+
}
|
|
733
|
+
else if (record.state !== 'completed')
|
|
734
|
+
throw new Error('Occurrence outcome changed during handoff.');
|
|
735
|
+
runtime.stopped = true;
|
|
736
|
+
}
|
|
737
|
+
await this.persist();
|
|
738
|
+
for (const [id] of entries)
|
|
739
|
+
this.runtimes.delete(id);
|
|
740
|
+
this.handoffUnresolved = false;
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
catch (error) {
|
|
744
|
+
await this.serial(async () => {
|
|
745
|
+
for (const [id] of entries) {
|
|
746
|
+
const record = this.records.get(id);
|
|
747
|
+
delete record.recovery?.handoff;
|
|
748
|
+
delete record.recovery?.nativeDefinitionAbsent;
|
|
749
|
+
record.error = `Restart handoff unconfirmed: ${errorText(error)}`;
|
|
750
|
+
}
|
|
751
|
+
if (!this.fault)
|
|
752
|
+
await this.persist();
|
|
753
|
+
});
|
|
754
|
+
throw error;
|
|
755
|
+
}
|
|
756
|
+
finally {
|
|
757
|
+
this.suspending = false;
|
|
758
|
+
}
|
|
759
|
+
})();
|
|
760
|
+
this.suspension = run;
|
|
761
|
+
void run.finally(() => { if (this.suspension === run)
|
|
762
|
+
this.suspension = undefined; }).catch(() => { });
|
|
763
|
+
return run;
|
|
764
|
+
}
|
|
765
|
+
/** Providers with durable intent must complete handoff before unrelated host teardown. */
|
|
766
|
+
requiresShutdownHandoff() {
|
|
767
|
+
return [...this.records.values()].some(record => !!record.recovery);
|
|
768
|
+
}
|
|
351
769
|
destroy() {
|
|
352
770
|
if (this.destruction)
|
|
353
771
|
return this.destruction;
|
|
354
|
-
this.closing
|
|
772
|
+
const preserve = !this.closing && this.hasActiveSchedules() && this.canSuspendForRestart();
|
|
773
|
+
if (!preserve)
|
|
774
|
+
this.closing = true;
|
|
355
775
|
return this.destruction = (async () => {
|
|
776
|
+
if (preserve)
|
|
777
|
+
await this.suspendForRestart();
|
|
778
|
+
this.closing = true;
|
|
779
|
+
for (const runtime of this.runtimes.values())
|
|
780
|
+
runtime.recoveryAbort?.abort();
|
|
356
781
|
await this.initialization?.catch(() => { });
|
|
357
782
|
let failure;
|
|
358
783
|
await this.serial(async () => {
|
|
359
784
|
for (const record of this.records.values()) {
|
|
360
785
|
if (LIVE.has(record.state)) {
|
|
786
|
+
const runtime = this.runtimes.get(record.id);
|
|
787
|
+
if (record.recovery && record.state === 'active' && !record.cancelRequested && !record.error
|
|
788
|
+
&& runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling) {
|
|
789
|
+
runtime.releasing = true;
|
|
790
|
+
record.recovery.status = 'pending';
|
|
791
|
+
record.error = 'Native restart handoff pending verification.';
|
|
792
|
+
}
|
|
793
|
+
else {
|
|
794
|
+
if (record.recovery)
|
|
795
|
+
record.recovery.status = 'blocked';
|
|
796
|
+
record.error = 'Native schedule interrupted by service shutdown.';
|
|
797
|
+
}
|
|
361
798
|
record.state = 'interrupted';
|
|
362
|
-
record.error = 'Native schedule interrupted by service shutdown.';
|
|
363
799
|
}
|
|
364
800
|
}
|
|
365
801
|
if (this.initialized && !this.fault)
|
|
366
802
|
await this.persist();
|
|
367
803
|
}).catch(error => { failure = error; });
|
|
368
|
-
const stops = await Promise.allSettled([...this.runtimes].map(([id, runtime]) =>
|
|
804
|
+
const stops = await Promise.allSettled([...this.runtimes].map(async ([id, runtime]) => {
|
|
805
|
+
if (!runtime.releasing)
|
|
806
|
+
return this.stop(id, runtime);
|
|
807
|
+
try {
|
|
808
|
+
runtime.releasePromise ??= runtime.handle.releaseForRestart();
|
|
809
|
+
let timer;
|
|
810
|
+
try {
|
|
811
|
+
await Promise.race([runtime.releasePromise, new Promise((_, reject) => {
|
|
812
|
+
timer = setTimeout(() => reject(new Error('Native shutdown handoff timed out; ownership retained.')), this.options.recoveryTimeoutMs ?? 30_000);
|
|
813
|
+
})]);
|
|
814
|
+
}
|
|
815
|
+
finally {
|
|
816
|
+
if (timer)
|
|
817
|
+
clearTimeout(timer);
|
|
818
|
+
}
|
|
819
|
+
await this.serial(async () => {
|
|
820
|
+
const record = this.records.get(id);
|
|
821
|
+
if (record.recovery?.status !== 'pending')
|
|
822
|
+
throw new Error('Occurrence raced native handoff; recovery is blocked.');
|
|
823
|
+
runtime.stopped = true;
|
|
824
|
+
record.error = 'Waiting for native recovery after orderly shutdown.';
|
|
825
|
+
await this.persist();
|
|
826
|
+
});
|
|
827
|
+
}
|
|
828
|
+
catch (error) {
|
|
829
|
+
await this.blockRecovery(this.records.get(id), errorText(error));
|
|
830
|
+
throw error;
|
|
831
|
+
}
|
|
832
|
+
}));
|
|
369
833
|
while (this.operations.size)
|
|
370
834
|
await Promise.allSettled([...this.operations]);
|
|
371
835
|
while (this.tasks.size)
|
|
@@ -436,17 +900,31 @@ class ScheduleService {
|
|
|
436
900
|
if (this.fault)
|
|
437
901
|
throw this.fault;
|
|
438
902
|
const record = this.records.get(id);
|
|
903
|
+
if (this.suspending && event.type === 'stopped') {
|
|
904
|
+
if (event.exitConfirmed === false)
|
|
905
|
+
throw new Error('Native handoff exit unconfirmed.');
|
|
906
|
+
runtime.stopped = true;
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
909
|
+
if (runtime.releasing && runtime.stopped)
|
|
910
|
+
return;
|
|
439
911
|
// Process exit evidence is independent of task state, including late exit
|
|
440
912
|
// after a failed cancellation cleanup. Never downgrade confirmed exit.
|
|
441
913
|
if (event.type === 'stopped' && event.exitConfirmed !== false) {
|
|
442
914
|
runtime.stopped = true;
|
|
915
|
+
runtime.releaseAdmission?.();
|
|
916
|
+
runtime.releaseAdmission = undefined;
|
|
443
917
|
if (record.error?.startsWith('Native process exit unconfirmed:')) {
|
|
444
918
|
delete record.error;
|
|
445
919
|
await this.persist();
|
|
446
920
|
}
|
|
447
921
|
}
|
|
922
|
+
if (runtime.releasing && event.type === 'stopped' && event.exitConfirmed !== false)
|
|
923
|
+
return;
|
|
448
924
|
const ownerCurrent = this.current(record);
|
|
449
925
|
if (this.closing || !ownerCurrent) {
|
|
926
|
+
if (record.recovery && (!ownerCurrent || event.type !== 'stopped' || event.exitConfirmed === false))
|
|
927
|
+
record.recovery.status = 'blocked';
|
|
450
928
|
if (LIVE.has(record.state)) {
|
|
451
929
|
record.state = 'interrupted';
|
|
452
930
|
record.error = 'Schedule owner or service is no longer current.';
|
|
@@ -480,15 +958,30 @@ class ScheduleService {
|
|
|
480
958
|
record.error = 'Occurrence tracking limit reached; native schedule stopped.';
|
|
481
959
|
cleanup = true;
|
|
482
960
|
}
|
|
961
|
+
else if (event.type === 'skipped') {
|
|
962
|
+
if (record.recovery?.handoffPolicy !== 'skip-unadmitted' || runtime.running.has(event.occurrenceId))
|
|
963
|
+
throw new Error('Unnegotiated occurrence skip.');
|
|
964
|
+
runtime.completed.add(event.occurrenceId);
|
|
965
|
+
record.completedOccurrences = [...runtime.completed];
|
|
966
|
+
// No delivery and no invented success. Unseen skipped wakes are not counted.
|
|
967
|
+
(record.skippedOccurrences ??= []).push(event.occurrenceId);
|
|
968
|
+
if (record.rule.kind === 'once') {
|
|
969
|
+
record.state = 'interrupted';
|
|
970
|
+
record.error = bounded(event.reason, ERROR_BYTES);
|
|
971
|
+
}
|
|
972
|
+
}
|
|
483
973
|
else if (event.type === 'started') {
|
|
484
974
|
runtime.running.add(event.occurrenceId);
|
|
485
975
|
record.state = 'running';
|
|
976
|
+
if (record.recovery)
|
|
977
|
+
record.recovery.status = 'blocked';
|
|
486
978
|
}
|
|
487
979
|
else if (event.type === 'completed') {
|
|
488
980
|
if (typeof event.success !== 'boolean')
|
|
489
981
|
throw new Error('Invalid native completion event.');
|
|
490
982
|
runtime.running.delete(event.occurrenceId);
|
|
491
983
|
runtime.completed.add(event.occurrenceId);
|
|
984
|
+
record.completedOccurrences = [...runtime.completed];
|
|
492
985
|
if (this.pending.size >= this.maxPending) {
|
|
493
986
|
record.state = 'interrupted';
|
|
494
987
|
record.error = 'Delivery outbox is full; latest result could not be retained. Native schedule stopped.';
|
|
@@ -506,8 +999,11 @@ class ScheduleService {
|
|
|
506
999
|
record.state = 'completed';
|
|
507
1000
|
cleanup = true;
|
|
508
1001
|
}
|
|
509
|
-
else if (record.state !== 'interrupted')
|
|
1002
|
+
else if (record.state !== 'interrupted') {
|
|
510
1003
|
record.state = runtime.running.size ? 'running' : 'active';
|
|
1004
|
+
if (record.recovery && !runtime.running.size)
|
|
1005
|
+
record.recovery.status = 'eligible';
|
|
1006
|
+
}
|
|
511
1007
|
}
|
|
512
1008
|
}
|
|
513
1009
|
else
|
|
@@ -574,9 +1070,12 @@ class ScheduleService {
|
|
|
574
1070
|
if (runtime.stopped)
|
|
575
1071
|
return Promise.resolve();
|
|
576
1072
|
if (runtime.stopPromise)
|
|
577
|
-
return runtime.stopPromise;
|
|
1073
|
+
return runtime.recoveryAbort ? this.boundRecovery(runtime.stopPromise, 'Native cleanup remains unresolved; ownership retained.') : runtime.stopPromise;
|
|
578
1074
|
const operation = (async () => {
|
|
579
1075
|
try {
|
|
1076
|
+
if (runtime.recoveryAbort?.signal.aborted && !runtime.handle && !runtime.stopped) {
|
|
1077
|
+
throw new Error('Recovery receipt is unresolved; native ownership retained.');
|
|
1078
|
+
}
|
|
580
1079
|
const handle = await runtime.ready;
|
|
581
1080
|
if (handle && !runtime.stopped)
|
|
582
1081
|
await handle.stop();
|
|
@@ -584,6 +1083,10 @@ class ScheduleService {
|
|
|
584
1083
|
// finishes. Keep ownership until its terminal stopped notification.
|
|
585
1084
|
if (!handle?.cancellationKeepsRunning || !runtime.running.size)
|
|
586
1085
|
runtime.stopped = true;
|
|
1086
|
+
if (runtime.stopped) {
|
|
1087
|
+
runtime.releaseAdmission?.();
|
|
1088
|
+
runtime.releaseAdmission = undefined;
|
|
1089
|
+
}
|
|
587
1090
|
}
|
|
588
1091
|
catch (error) {
|
|
589
1092
|
await this.serial(async () => {
|
|
@@ -597,7 +1100,19 @@ class ScheduleService {
|
|
|
597
1100
|
runtime.stopPromise = operation;
|
|
598
1101
|
void operation.finally(() => { if (!runtime.stopped)
|
|
599
1102
|
runtime.stopPromise = undefined; }).catch(() => { });
|
|
600
|
-
return operation;
|
|
1103
|
+
return runtime.recoveryAbort ? this.boundRecovery(operation, 'Native cleanup remains unresolved; ownership retained.') : operation;
|
|
1104
|
+
}
|
|
1105
|
+
async boundRecovery(operation, message) {
|
|
1106
|
+
let timer;
|
|
1107
|
+
try {
|
|
1108
|
+
return await Promise.race([operation, new Promise((_, reject) => {
|
|
1109
|
+
timer = setTimeout(() => reject(new Error(message)), this.options.recoveryTimeoutMs ?? 30_000);
|
|
1110
|
+
})]);
|
|
1111
|
+
}
|
|
1112
|
+
finally {
|
|
1113
|
+
if (timer)
|
|
1114
|
+
clearTimeout(timer);
|
|
1115
|
+
}
|
|
601
1116
|
}
|
|
602
1117
|
newRuntime() {
|
|
603
1118
|
let resolveReady;
|
|
@@ -640,7 +1155,7 @@ class ScheduleService {
|
|
|
640
1155
|
checkReady() {
|
|
641
1156
|
if (this.fault)
|
|
642
1157
|
throw this.fault;
|
|
643
|
-
if (!this.initialized || this.closing)
|
|
1158
|
+
if (!this.initialized || this.closing || this.suspending)
|
|
644
1159
|
throw new Error('Schedule service is not initialized or is closed.');
|
|
645
1160
|
}
|
|
646
1161
|
serial(operation) {
|
|
@@ -663,7 +1178,8 @@ class ScheduleService {
|
|
|
663
1178
|
}
|
|
664
1179
|
try {
|
|
665
1180
|
// JSON can expand one control byte to six bytes (for example, "\\u0001").
|
|
666
|
-
const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024)
|
|
1181
|
+
const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024)
|
|
1182
|
+
+ this.maxRecords * this.maxOccurrences * 515 + 4096;
|
|
667
1183
|
const size = (await handle.stat()).size;
|
|
668
1184
|
if (size > maxBytes)
|
|
669
1185
|
throw new Error('Schedule catalog exceeds its size limit.');
|
|
@@ -689,13 +1205,13 @@ class ScheduleService {
|
|
|
689
1205
|
}
|
|
690
1206
|
load(data) {
|
|
691
1207
|
const catalog = object(data, ['version', 'records', 'pending']);
|
|
692
|
-
if (catalog.version
|
|
1208
|
+
if (![1, 2].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
|
|
693
1209
|
|| !Array.isArray(catalog.pending) || catalog.pending.length > this.maxPending)
|
|
694
1210
|
throw new Error('Invalid schedule catalog envelope.');
|
|
695
1211
|
const records = new Map();
|
|
696
1212
|
const pending = new Map();
|
|
697
1213
|
for (const value of catalog.records) {
|
|
698
|
-
const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation']);
|
|
1214
|
+
const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation', 'execution', 'cancelRequested', 'recovery', 'completedOccurrences', 'skippedOccurrences']);
|
|
699
1215
|
(0, ScheduleCommand_1.validateScheduleId)(record.id);
|
|
700
1216
|
validateOwner(record);
|
|
701
1217
|
(0, ScheduleCommand_1.validateSchedulePrompt)(record.prompt);
|
|
@@ -708,6 +1224,35 @@ class ScheduleService {
|
|
|
708
1224
|
text(record.nativeId);
|
|
709
1225
|
if (record.error !== undefined && (typeof record.error !== 'string' || Buffer.byteLength(record.error) > ERROR_BYTES))
|
|
710
1226
|
throw new Error('Invalid schedule error.');
|
|
1227
|
+
if (record.cancelRequested !== undefined && record.cancelRequested !== true)
|
|
1228
|
+
throw new Error('Invalid cancellation intent.');
|
|
1229
|
+
if (record.execution !== undefined) {
|
|
1230
|
+
const execution = object(record.execution, ['command', 'model', 'effort']);
|
|
1231
|
+
for (const value of Object.values(execution))
|
|
1232
|
+
if (value !== undefined)
|
|
1233
|
+
text(value, 4096);
|
|
1234
|
+
}
|
|
1235
|
+
if (record.recovery !== undefined) {
|
|
1236
|
+
if (catalog.version !== 2)
|
|
1237
|
+
throw new Error('Recovery requires catalog version 2.');
|
|
1238
|
+
const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy']);
|
|
1239
|
+
this.validateCheckpoint(recovery);
|
|
1240
|
+
if (!['eligible', 'pending', 'blocked'].includes(recovery.status))
|
|
1241
|
+
throw new Error('Invalid recovery status.');
|
|
1242
|
+
}
|
|
1243
|
+
if (record.completedOccurrences !== undefined) {
|
|
1244
|
+
if (!Array.isArray(record.completedOccurrences) || record.completedOccurrences.length > this.maxOccurrences
|
|
1245
|
+
|| new Set(record.completedOccurrences).size !== record.completedOccurrences.length)
|
|
1246
|
+
throw new Error('Invalid occurrence history.');
|
|
1247
|
+
record.completedOccurrences.forEach(value => text(value));
|
|
1248
|
+
}
|
|
1249
|
+
if (record.skippedOccurrences !== undefined) {
|
|
1250
|
+
if (!Array.isArray(record.skippedOccurrences) || record.skippedOccurrences.length > this.maxOccurrences
|
|
1251
|
+
|| new Set(record.skippedOccurrences).size !== record.skippedOccurrences.length
|
|
1252
|
+
|| record.skippedOccurrences.some(value => !Array.isArray(record.completedOccurrences) || !record.completedOccurrences.includes(value))) {
|
|
1253
|
+
throw new Error('Invalid skipped occurrence history.');
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
711
1256
|
const rule = object(record.rule, ['kind', 'at', 'expression', 'timezone']);
|
|
712
1257
|
if (rule.kind === 'once')
|
|
713
1258
|
object(rule, ['kind', 'at']);
|
|
@@ -750,7 +1295,7 @@ class ScheduleService {
|
|
|
750
1295
|
}
|
|
751
1296
|
const handle = await fs.open(temporary, 'wx', 0o600);
|
|
752
1297
|
try {
|
|
753
|
-
await handle.writeFile(JSON.stringify({ version:
|
|
1298
|
+
await handle.writeFile(JSON.stringify({ version: 2, records: [...this.records.values()], pending: [...this.pending.values()] }), 'utf8');
|
|
754
1299
|
await handle.sync();
|
|
755
1300
|
}
|
|
756
1301
|
finally {
|