@yu_robotics/remote-cli 1.6.237 → 1.6.238

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/client/MessageHandler.d.ts +9 -2
  2. package/dist/client/MessageHandler.d.ts.map +1 -1
  3. package/dist/client/MessageHandler.js +202 -21
  4. package/dist/client/MessageHandler.js.map +1 -1
  5. package/dist/commands/start.d.ts.map +1 -1
  6. package/dist/commands/start.js +1 -0
  7. package/dist/commands/start.js.map +1 -1
  8. package/dist/executor/AcpExecutor.d.ts +2 -0
  9. package/dist/executor/AcpExecutor.d.ts.map +1 -1
  10. package/dist/executor/AcpExecutor.js +9 -0
  11. package/dist/executor/AcpExecutor.js.map +1 -1
  12. package/dist/executor/ClaudePersistentExecutor.d.ts +17 -7
  13. package/dist/executor/ClaudePersistentExecutor.d.ts.map +1 -1
  14. package/dist/executor/ClaudePersistentExecutor.js +362 -58
  15. package/dist/executor/ClaudePersistentExecutor.js.map +1 -1
  16. package/dist/executor/DshExecutor.d.ts +13 -1
  17. package/dist/executor/DshExecutor.d.ts.map +1 -1
  18. package/dist/executor/DshExecutor.js +39 -1
  19. package/dist/executor/DshExecutor.js.map +1 -1
  20. package/dist/executor/IExecutor.d.ts +6 -0
  21. package/dist/executor/IExecutor.d.ts.map +1 -1
  22. package/dist/maintenance/release-notes.json +1 -1
  23. package/dist/scheduling/ScheduleCapabilities.d.ts +1 -1
  24. package/dist/scheduling/ScheduleCapabilities.d.ts.map +1 -1
  25. package/dist/scheduling/ScheduleCapabilities.js +3 -2
  26. package/dist/scheduling/ScheduleCapabilities.js.map +1 -1
  27. package/dist/scheduling/ScheduleService.d.ts +26 -0
  28. package/dist/scheduling/ScheduleService.d.ts.map +1 -1
  29. package/dist/scheduling/ScheduleService.js +549 -17
  30. package/dist/scheduling/ScheduleService.js.map +1 -1
  31. package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts +25 -2
  32. package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts.map +1 -1
  33. package/dist/scheduling/claude/ClaudeSessionSchedules.js +126 -17
  34. package/dist/scheduling/claude/ClaudeSessionSchedules.js.map +1 -1
  35. package/dist/scheduling/dsh/SessionScheduleBridge.d.ts.map +1 -1
  36. package/dist/scheduling/dsh/SessionScheduleBridge.js +21 -0
  37. package/dist/scheduling/dsh/SessionScheduleBridge.js.map +1 -1
  38. package/dist/scheduling/dsh/SessionScheduleClient.d.ts.map +1 -1
  39. package/dist/scheduling/dsh/SessionScheduleClient.js +8 -0
  40. package/dist/scheduling/dsh/SessionScheduleClient.js.map +1 -1
  41. package/dist/scheduling/types.d.ts +84 -0
  42. package/dist/scheduling/types.d.ts.map +1 -1
  43. package/dist/scheduling/types.js.map +1 -1
  44. package/dist/settings/SettingsService.d.ts +1 -1
  45. package/dist/settings/SettingsService.d.ts.map +1 -1
  46. package/dist/settings/SettingsService.js +3 -3
  47. package/dist/settings/SettingsService.js.map +1 -1
  48. package/dist/thread/ThreadExecutorPool.d.ts +1 -1
  49. package/dist/thread/ThreadExecutorPool.d.ts.map +1 -1
  50. package/dist/thread/ThreadExecutorPool.js +5 -3
  51. package/dist/thread/ThreadExecutorPool.js.map +1 -1
  52. package/dist/update/AutomaticUpdater.d.ts +2 -1
  53. package/dist/update/AutomaticUpdater.d.ts.map +1 -1
  54. package/dist/update/AutomaticUpdater.js +11 -3
  55. package/dist/update/AutomaticUpdater.js.map +1 -1
  56. package/native/prebuilds/darwin-arm64/managed_native.node +0 -0
  57. package/native/prebuilds/darwin-arm64/manifest.json +1 -1
  58. package/native/prebuilds/darwin-x64/managed_native.node +0 -0
  59. package/native/prebuilds/darwin-x64/manifest.json +1 -1
  60. package/native/prebuilds/win32-arm64/managed_native.node +0 -0
  61. package/native/prebuilds/win32-arm64/manifest.json +1 -1
  62. package/native/prebuilds/win32-x64/managed_native.node +0 -0
  63. package/native/prebuilds/win32-x64/manifest.json +1 -1
  64. package/package.json +1 -1
@@ -83,6 +83,11 @@ function validateOwner(owner) {
83
83
  function sameOwner(left, right) {
84
84
  return left.threadId === right.threadId && left.openId === right.openId && left.generation === right.generation;
85
85
  }
86
+ /** Terminal intent prevents recreation, but uncertain native cleanup still owns admission. */
87
+ function recoveryOwnsWork(record) {
88
+ return !!record.recovery && (!['completed', 'cancelled'].includes(record.state)
89
+ || !!record.error?.startsWith('Native process exit unconfirmed:'));
90
+ }
86
91
  function deliveryId(scheduleId, occurrenceId) {
87
92
  return `${scheduleId}:${occurrenceId}`;
88
93
  }
@@ -112,6 +117,10 @@ class ScheduleService {
112
117
  destruction;
113
118
  fault;
114
119
  catalogPath = '';
120
+ recoveryRun;
121
+ suspending = false;
122
+ handoffUnresolved = false;
123
+ suspension;
115
124
  constructor(options) {
116
125
  this.options = { ...options };
117
126
  this.now = options.now ?? Date.now;
@@ -121,6 +130,9 @@ class ScheduleService {
121
130
  throw new Error('Invalid schedule capacity.');
122
131
  return result;
123
132
  };
133
+ if (options.recoveryTimeoutMs !== undefined && (!Number.isInteger(options.recoveryTimeoutMs)
134
+ || options.recoveryTimeoutMs < 1 || options.recoveryTimeoutMs > 60_000))
135
+ throw new Error('Invalid recovery timeout.');
124
136
  this.maxLive = limit(options.maxLiveSchedules, 5, 5);
125
137
  this.maxRecords = limit(options.maxRecords, 100, 1000);
126
138
  this.maxPending = limit(options.maxPendingDeliveries, 100, 1000);
@@ -138,9 +150,23 @@ class ScheduleService {
138
150
  this.load(data);
139
151
  let changed = false;
140
152
  for (const record of this.records.values()) {
141
- if (LIVE.has(record.state)) {
153
+ if (['completed', 'cancelled', 'failed'].includes(record.state)) {
154
+ if (record.recovery?.status === 'pending') {
155
+ record.recovery.status = 'blocked';
156
+ changed = true;
157
+ }
158
+ continue; // Terminal tombstones always win over a stale recovery checkpoint.
159
+ }
160
+ if (LIVE.has(record.state) || record.recovery?.status === 'pending') {
161
+ const recoverable = record.recovery && !record.cancelRequested && this.current(record)
162
+ && record.state !== 'running' && record.state !== 'failed'
163
+ && (!record.recovery.orderlyOnly || record.recovery.handoff === 'clean')
164
+ && (record.state === 'active' || record.recovery.status === 'pending');
165
+ if (record.recovery)
166
+ record.recovery.status = recoverable ? 'pending' : 'blocked';
167
+ record.error = recoverable ? 'Waiting for native recovery capability and owner readiness.'
168
+ : record.recovery?.orderlyOnly ? 'Dirty shutdown: original native work is unconfirmed; automatic reconstruction is blocked.' : RESTART_REASON;
142
169
  record.state = 'interrupted';
143
- record.error = RESTART_REASON;
144
170
  changed = true;
145
171
  }
146
172
  }
@@ -156,6 +182,193 @@ class ScheduleService {
156
182
  this.initialized = true;
157
183
  });
158
184
  }
185
+ /** Sequential startup/reconnect reconciliation. Never drives the native schedule clock. */
186
+ recover() {
187
+ this.checkReady();
188
+ if (this.recoveryRun)
189
+ return this.recoveryRun;
190
+ const run = this.operation(this.recoverWaiting());
191
+ this.recoveryRun = run;
192
+ void run.finally(() => { if (this.recoveryRun === run)
193
+ this.recoveryRun = undefined; }).catch(() => { });
194
+ return run;
195
+ }
196
+ async recoverWaiting() {
197
+ const candidates = await this.serial(async () => [...this.records.values()]
198
+ .filter(record => record.recovery?.status === 'pending' && record.state === 'interrupted').map(record => record.id));
199
+ // Catalog capacity is bounded, and only one native admission runs at a time.
200
+ for (const id of candidates.slice(0, this.maxLive)) {
201
+ if (this.closing || this.fault)
202
+ return;
203
+ const record = this.records.get(id);
204
+ if (record.cancelRequested || !this.current(record)) {
205
+ await this.blockRecovery(record, 'Recovery owner retired or cancellation was requested.');
206
+ continue;
207
+ }
208
+ if (record.rule.kind === 'once' && Date.parse(record.rule.at) <= this.now()) {
209
+ await this.blockRecovery(record, 'One-shot deadline expired; no late execution was requested.');
210
+ if (record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent)
211
+ await this.serial(async () => { delete record.recovery; await this.persist(); });
212
+ continue;
213
+ }
214
+ const input = { id, threadId: record.threadId, backend: record.backend,
215
+ cwd: record.cwd, prompt: record.prompt, rule: structuredClone(record.rule), ...record.execution,
216
+ recovery: this.checkpoint(record.recovery) };
217
+ if (!this.options.canRecover || !this.options.canRecover(structuredClone(record))) {
218
+ await this.recoveryDiagnostic(record, 'Original owner configuration or transport is not ready for recovery.');
219
+ continue;
220
+ }
221
+ const provider = this.options.provider.recoveryFor?.(input);
222
+ if (!provider || provider.provider !== record.recovery.provider || provider.strategy !== record.recovery.strategy
223
+ || (provider.handoffPolicy ?? 'drain-all') !== (record.recovery.handoffPolicy ?? 'drain-all')) {
224
+ await this.recoveryDiagnostic(record, 'Native restart recovery is unsupported or provider identity changed.');
225
+ continue;
226
+ }
227
+ if (this.runtimes.has(id))
228
+ continue; // An unresolved/late operation retains exclusive ownership.
229
+ const restoreRecord = structuredClone(record);
230
+ const runtime = this.newRuntime();
231
+ runtime.completed = new Set(record.completedOccurrences);
232
+ const controller = new AbortController();
233
+ runtime.recoveryAbort = controller;
234
+ this.runtimes.set(id, runtime);
235
+ const current = () => !controller.signal.aborted && !this.closing && !this.fault && !record.cancelRequested
236
+ && this.current(record) && this.options.canRecover?.(structuredClone(record)) === true;
237
+ const emit = (event) => {
238
+ if (this.runtimes.get(id) !== runtime)
239
+ return;
240
+ this.track(this.onEvent(id, structuredClone(event)));
241
+ };
242
+ let timer;
243
+ let admitted = false;
244
+ let succeeded = false;
245
+ try {
246
+ runtime.releaseAdmission = this.options.acquireRecovery?.(structuredClone(record));
247
+ await this.serial(async () => {
248
+ if (!current())
249
+ throw new Error('Recovery admission expired.');
250
+ // Consume clean proof before mutation. A crash must not replay an orderly-only restore.
251
+ record.state = 'creating';
252
+ delete record.recovery.handoff;
253
+ delete record.recovery.nativeDefinitionAbsent;
254
+ await this.persist();
255
+ });
256
+ admitted = true;
257
+ const admission = Promise.resolve().then(() => provider.reconcile(restoreRecord, { notBefore: this.now(), signal: controller.signal, isCurrent: current,
258
+ isOwnerCurrent: () => !record.cancelRequested && !this.fault && this.current(record) }, emit));
259
+ const adopt = async (handle) => {
260
+ runtime.handle = handle;
261
+ runtime.resolveReady(handle);
262
+ text(handle.nativeId);
263
+ await this.serial(async () => { record.nativeId = handle.nativeId; if (!this.fault)
264
+ await this.persist(); });
265
+ };
266
+ const receipt = admission.then(async (handle) => {
267
+ await adopt(handle);
268
+ if (!current()) {
269
+ await this.stop(id, runtime);
270
+ throw new Error('Late recovery receipt requires cleanup; no successor was admitted.');
271
+ }
272
+ if (!await handle.list() || !current())
273
+ throw new Error('Recovered native inventory or owner is unconfirmed.');
274
+ return handle;
275
+ }, async (error) => {
276
+ if (error instanceof types_1.NativeScheduleCleanupError) {
277
+ await adopt(error.handle);
278
+ if (controller.signal.aborted)
279
+ await this.stop(id, runtime).catch(() => { });
280
+ }
281
+ runtime.resolveReady(runtime.handle);
282
+ // A rejection without a handle is not proof that native admission never happened.
283
+ throw error;
284
+ });
285
+ const handle = await Promise.race([receipt, new Promise((_, reject) => {
286
+ timer = setTimeout(() => { controller.abort(); reject(new Error('Native recovery timed out; ownership retained, do not recreate.')); }, this.options.recoveryTimeoutMs ?? 30_000);
287
+ })]);
288
+ text(handle.nativeId);
289
+ // The provider's reconcile contract includes inventory validation; list is
290
+ // additionally required to verify the returned handle before activating it.
291
+ if (!current())
292
+ throw new Error('Recovered native owner is unconfirmed.');
293
+ await this.serial(async () => {
294
+ if (!current())
295
+ throw new Error('Recovery owner retired before receipt persistence.');
296
+ record.nativeId = handle.nativeId;
297
+ if (record.state === 'creating')
298
+ record.state = 'active';
299
+ record.recovery.status = 'eligible';
300
+ delete record.error;
301
+ await this.persist();
302
+ });
303
+ succeeded = true;
304
+ }
305
+ catch (error) {
306
+ controller.abort();
307
+ if (!admitted) {
308
+ runtime.stopped = true;
309
+ runtime.resolveReady(undefined);
310
+ }
311
+ // Do not retry an ambiguous mutation or release its handle on a timeout.
312
+ await this.blockRecovery(record, errorText(error));
313
+ if (runtime.handle)
314
+ await this.stop(id, runtime).catch(() => { });
315
+ else if (runtime.stopped)
316
+ this.runtimes.delete(id);
317
+ }
318
+ finally {
319
+ if (timer)
320
+ clearTimeout(timer);
321
+ if (succeeded || runtime.stopped) {
322
+ runtime.releaseAdmission?.();
323
+ runtime.releaseAdmission = undefined;
324
+ }
325
+ }
326
+ }
327
+ // A handoff containing only skipped/expired one-shots has nothing to
328
+ // reconcile. It must still release the predecessor's local input fence.
329
+ const released = new Set();
330
+ for (const record of this.records.values()) {
331
+ if (released.has(record.threadId) || this.closing || this.fault || this.handoffUnresolved
332
+ || !this.current(record) || !this.options.canRecover?.(structuredClone(record)))
333
+ continue;
334
+ if ([...this.records.values()].some(other => other.threadId === record.threadId
335
+ && (LIVE.has(other.state) || recoveryOwnsWork(other) || this.runtimes.has(other.id))))
336
+ continue;
337
+ this.options.provider.recoveryFor?.({ id: record.id, threadId: record.threadId, backend: record.backend,
338
+ cwd: record.cwd, prompt: record.prompt, rule: record.rule, ...record.execution })?.releaseRestartAdmission?.();
339
+ released.add(record.threadId);
340
+ }
341
+ }
342
+ checkpoint(value) {
343
+ return { version: 1, provider: value.provider, strategy: value.strategy, identity: value.identity,
344
+ ...(value.orderlyOnly && { orderlyOnly: true }), ...(value.handoffPolicy && { handoffPolicy: value.handoffPolicy }), ...(value.handoff && { handoff: value.handoff }),
345
+ ...(value.nativeDefinitionAbsent && { nativeDefinitionAbsent: true }) };
346
+ }
347
+ validateCheckpoint(value) {
348
+ if (value.version !== 1 || !['reattach', 'reconstruct'].includes(value.strategy))
349
+ throw new Error('Invalid recovery checkpoint.');
350
+ if (value.orderlyOnly !== undefined && value.orderlyOnly !== true || value.handoff !== undefined && value.handoff !== 'clean')
351
+ throw new Error('Invalid handoff evidence.');
352
+ if (value.nativeDefinitionAbsent !== undefined && (value.nativeDefinitionAbsent !== true || value.handoff !== 'clean'))
353
+ throw new Error('Invalid native absence evidence.');
354
+ if (value.handoffPolicy !== undefined && !['drain-all', 'skip-unadmitted'].includes(value.handoffPolicy))
355
+ throw new Error('Invalid handoff policy.');
356
+ text(value.provider);
357
+ text(value.identity, 8192);
358
+ }
359
+ async recoveryDiagnostic(record, reason) {
360
+ await this.serial(async () => { record.error = reason; await this.persist(); });
361
+ }
362
+ async blockRecovery(record, reason) {
363
+ await this.serial(async () => {
364
+ if (!['completed', 'cancelled'].includes(record.state))
365
+ record.state = 'interrupted';
366
+ record.recovery.status = 'blocked';
367
+ record.error = reason;
368
+ if (!this.fault)
369
+ await this.persist();
370
+ });
371
+ }
159
372
  create(owner, input) {
160
373
  return this.operation(this.createSchedule(owner, input));
161
374
  }
@@ -174,7 +387,18 @@ class ScheduleService {
174
387
  if (!node_path_1.default.isAbsolute(input.cwd))
175
388
  throw new Error('Schedule working directory must be absolute.');
176
389
  const originalOwner = ownerOf(owner);
177
- const nativeInput = { ...structuredClone(input), backend };
390
+ const nativeInput = { ...structuredClone(input), backend, threadId: owner.threadId };
391
+ // Never trust a caller-supplied provider checkpoint.
392
+ delete nativeInput.recovery;
393
+ for (const value of [input.command, input.model, input.effort])
394
+ if (value !== undefined)
395
+ text(value, 4096);
396
+ const recovery = this.options.provider.recoveryFor?.(nativeInput);
397
+ if (recovery) {
398
+ nativeInput.recovery = { version: 1, provider: recovery.provider, strategy: recovery.strategy,
399
+ identity: recovery.describe(structuredClone(nativeInput), originalOwner), ...(recovery.orderlyOnly && { orderlyOnly: true }), ...(recovery.handoffPolicy && { handoffPolicy: recovery.handoffPolicy }) };
400
+ this.validateCheckpoint(nativeInput.recovery);
401
+ }
178
402
  const runtime = this.newRuntime();
179
403
  await this.serial(async () => {
180
404
  this.checkReady();
@@ -185,17 +409,21 @@ class ScheduleService {
185
409
  if (this.records.size >= this.maxRecords)
186
410
  throw new Error('Schedule catalog is full; history was not pruned.');
187
411
  const liveCount = [...this.records.values()].filter(record => LIVE.has(record.state)
412
+ || recoveryOwnsWork(record)
188
413
  || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)).length;
189
414
  if (liveCount >= this.maxLive)
190
415
  throw new Error('Live native schedule limit reached.');
191
416
  this.records.set(nativeInput.id, { ...originalOwner, id: nativeInput.id, backend,
192
417
  cwd: nativeInput.cwd, prompt: nativeInput.prompt, rule: structuredClone(nativeInput.rule),
193
- createdAt, state: 'creating' });
418
+ createdAt, state: 'creating', execution: { command: input.command, model: input.model, effort: input.effort },
419
+ ...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: 'eligible' } }) });
194
420
  await this.persist();
195
421
  this.runtimes.set(nativeInput.id, runtime);
196
422
  });
197
423
  try {
198
424
  const handle = await this.options.provider.create(structuredClone(nativeInput), event => {
425
+ if (this.runtimes.get(nativeInput.id) !== runtime)
426
+ return;
199
427
  const task = this.onEvent(nativeInput.id, structuredClone(event));
200
428
  this.track(task);
201
429
  });
@@ -205,6 +433,10 @@ class ScheduleService {
205
433
  await this.serial(async () => {
206
434
  const record = this.records.get(nativeInput.id);
207
435
  record.nativeId = handle.nativeId;
436
+ if (record.recovery && handle.recoveryIdentity) {
437
+ text(handle.recoveryIdentity, 8192);
438
+ record.recovery.identity = handle.recoveryIdentity;
439
+ }
208
440
  const ownerCurrent = this.current(originalOwner);
209
441
  if (this.closing || !ownerCurrent) {
210
442
  if (LIVE.has(record.state)) {
@@ -252,7 +484,7 @@ class ScheduleService {
252
484
  /** Includes terminal jobs whose native process exit is still unconfirmed. */
253
485
  hasActiveSchedules(threadId) {
254
486
  return [...this.records.values()].some(record => (!threadId || record.threadId === threadId)
255
- && (LIVE.has(record.state) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
487
+ && (LIVE.has(record.state) || recoveryOwnsWork(record) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
256
488
  }
257
489
  /** Card eligibility includes retryable cleanup even for a terminal catalog record. */
258
490
  canCancel(owner, id) {
@@ -261,7 +493,7 @@ class ScheduleService {
261
493
  const runtime = this.runtimes.get(id);
262
494
  if (record.state === 'cancelled' && runtime?.handle?.cancellationKeepsRunning && !record.error)
263
495
  return false;
264
- return LIVE.has(record.state) || Boolean(runtime && !runtime.stopped);
496
+ return LIVE.has(record.state) || recoveryOwnsWork(record) || Boolean(runtime && !runtime.stopped);
265
497
  }
266
498
  /** Keep admitted native work visible when cleanup cannot safely acquire its host. */
267
499
  async retainCleanupError(owner, id, error) {
@@ -333,11 +565,30 @@ class ScheduleService {
333
565
  await this.serial(async () => {
334
566
  const record = this.owned(owner, id);
335
567
  runtime = this.runtimes.get(id);
568
+ record.cancelRequested = true;
569
+ runtime?.recoveryAbort?.abort();
570
+ if (record.recovery)
571
+ record.recovery.status = 'blocked';
572
+ await this.persist();
573
+ if (!runtime && record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent) {
574
+ record.state = 'cancelled';
575
+ delete record.error;
576
+ await this.persist();
577
+ }
578
+ if (!runtime && recoveryOwnsWork(record)) {
579
+ record.error = 'Cancellation intent retained; native cleanup is unconfirmed. Recovery will not recreate this job.';
580
+ await this.persist();
581
+ throw new Error(record.error);
582
+ }
336
583
  if (runtime?.cancelPromise) {
337
584
  existing = runtime.cancelPromise;
338
585
  return;
339
586
  }
340
587
  if (!LIVE.has(record.state) && !runtime?.cancelling && (!runtime || runtime.stopped)) {
588
+ if (record.recovery && runtime?.stopped) {
589
+ record.state = 'cancelled';
590
+ delete record.error;
591
+ }
341
592
  for (const [key, pending] of this.pending) {
342
593
  if (pending.scheduleId === id && !pending.afterCancellation)
343
594
  this.pending.delete(key);
@@ -348,6 +599,9 @@ class ScheduleService {
348
599
  }
349
600
  if (!runtime)
350
601
  throw new Error('Native schedule handle is unavailable.');
602
+ if (runtime.recoveryAbort && !runtime.handle && !runtime.stopped) {
603
+ throw new Error('Recovery receipt is unresolved; cancellation intent retained until native cleanup can be confirmed.');
604
+ }
351
605
  runtime.cancelling = true;
352
606
  // Starting outside the state queue also permits synchronous provider callbacks.
353
607
  operation = Promise.resolve().then(() => this.cancelNative(id, runtime));
@@ -361,24 +615,221 @@ class ScheduleService {
361
615
  const ids = await this.serial(async () => [...this.pending.keys()]);
362
616
  await this.attemptDeliveries(ids);
363
617
  }
618
+ hasUnresolvedHandoff() { return this.handoffUnresolved; }
619
+ hasUnresolvedRecovery() {
620
+ return [...this.records.values()].some(record => recoveryOwnsWork(record)
621
+ && record.recovery?.status !== 'eligible');
622
+ }
623
+ hasPendingRecovery() {
624
+ return [...this.records.values()].some(record => record.state === 'interrupted' && record.recovery?.status === 'pending');
625
+ }
626
+ /** Snapshot eligibility is only admission to a handoff, never permission to install. */
627
+ canSuspendForRestart() {
628
+ if (!this.initialized || this.closing || this.fault || this.suspending || this.recoveryRun || this.operations.size || this.pending.size)
629
+ return false;
630
+ return [...this.records.values()].filter(record => LIVE.has(record.state) || recoveryOwnsWork(record)
631
+ || !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped).every(record => {
632
+ const runtime = this.runtimes.get(record.id);
633
+ return record.state === 'active' && !!record.recovery && !record.cancelRequested && !record.error
634
+ && !!runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling;
635
+ });
636
+ }
637
+ /** Delete future definitions, drain native work, join owned hosts, then publish restart proof. */
638
+ suspendForRestart() {
639
+ if (this.suspension)
640
+ return this.suspension;
641
+ if (!this.canSuspendForRestart())
642
+ return Promise.reject(new Error('Schedules are not ready for restart handoff.'));
643
+ this.suspending = true;
644
+ this.handoffUnresolved = true;
645
+ const entries = [...this.runtimes].filter(([id]) => this.records.get(id)?.state === 'active');
646
+ const run = (async () => {
647
+ try {
648
+ // Checkpoints remain dirty throughout deletion/drain and actual host exit.
649
+ await this.serial(async () => {
650
+ for (const [id, runtime] of entries) {
651
+ if (runtime.handle?.recoveryIdentity) {
652
+ text(runtime.handle.recoveryIdentity, 8192);
653
+ this.records.get(id).recovery.identity = runtime.handle.recoveryIdentity;
654
+ }
655
+ delete this.records.get(id).recovery.handoff;
656
+ delete this.records.get(id).recovery.nativeDefinitionAbsent;
657
+ }
658
+ await this.persist();
659
+ });
660
+ const releases = await Promise.allSettled(entries.map(([, runtime]) => this.boundRecovery(runtime.handle.releaseForRestart(), 'Native schedule drain remains unresolved.')));
661
+ const failed = releases.find((result) => result.status === 'rejected');
662
+ if (failed)
663
+ throw failed.reason;
664
+ for (const [index, result] of releases.entries()) {
665
+ const policy = this.records.get(entries[index][0]).recovery.handoffPolicy ?? 'drain-all';
666
+ if (result.status === 'fulfilled' && (result.value?.policy ?? 'drain-all') !== policy)
667
+ throw new Error('Native handoff policy receipt changed.');
668
+ }
669
+ const absentOneShots = new Set();
670
+ for (const [index, result] of releases.entries())
671
+ if (result.status === 'fulfilled' && result.value?.onceDefinitionAlreadyAbsent) {
672
+ const record = this.records.get(entries[index][0]);
673
+ if (record.rule.kind !== 'once' || record.recovery?.handoffPolicy !== 'skip-unadmitted')
674
+ throw new Error('Invalid absent one-shot handoff receipt.');
675
+ absentOneShots.add(record.id);
676
+ }
677
+ const absent = new Set(entries.filter((_, index) => {
678
+ const result = releases[index];
679
+ return result.status === 'fulfilled' && result.value?.nativeDefinitionAbsent === true;
680
+ }).map(([id]) => id));
681
+ await this.boundRecovery((async () => {
682
+ while (this.tasks.size)
683
+ await Promise.allSettled([...this.tasks]);
684
+ })(), 'Scheduled result drain remains unresolved.');
685
+ await this.tail;
686
+ if (this.pending.size)
687
+ throw new Error('Scheduled result acknowledgement is pending.');
688
+ if (this.fault || entries.some(([, runtime]) => runtime.running.size))
689
+ throw this.fault ?? new Error('Native occurrence is still running.');
690
+ const finished = new Set();
691
+ for (const [id] of entries) {
692
+ const record = this.records.get(id);
693
+ const provider = this.options.provider.recoveryFor?.({ id, threadId: record.threadId, backend: record.backend,
694
+ cwd: record.cwd, rule: record.rule, prompt: record.prompt, ...record.execution });
695
+ if (!provider || provider.provider !== record.recovery.provider)
696
+ throw new Error('Native recovery provider changed during handoff.');
697
+ const key = `${record.threadId}:${provider.provider}`;
698
+ if (finished.has(key))
699
+ continue;
700
+ if (provider.orderlyOnly && !provider.finishRestart)
701
+ throw new Error('Owned native host exit cannot be confirmed.');
702
+ if (provider.finishRestart)
703
+ await this.boundRecovery(provider.finishRestart(), 'Owned native host exit remains unresolved.');
704
+ finished.add(key);
705
+ }
706
+ await this.boundRecovery((async () => {
707
+ while (this.tasks.size)
708
+ await Promise.allSettled([...this.tasks]);
709
+ })(), 'Scheduled result drain remains unresolved.');
710
+ await this.serial(async () => {
711
+ if (this.fault)
712
+ throw this.fault;
713
+ for (const [id, runtime] of entries) {
714
+ const record = this.records.get(id);
715
+ if (!this.current(record) || record.cancelRequested)
716
+ throw new Error('Schedule owner retired during handoff.');
717
+ if (record.state === 'active' && absentOneShots.has(id)) {
718
+ record.state = 'interrupted';
719
+ record.error = 'One-shot native definition was already absent at handoff; it will not be replayed.';
720
+ delete record.recovery;
721
+ }
722
+ else if (record.state === 'active') {
723
+ record.state = 'interrupted';
724
+ record.recovery.status = 'pending';
725
+ record.recovery.handoff = 'clean';
726
+ if (absent.has(id))
727
+ record.recovery.nativeDefinitionAbsent = true;
728
+ record.error = 'Native host drained and exited; waiting for automatic recovery.';
729
+ }
730
+ else if (record.state === 'interrupted' && record.rule.kind === 'once' && record.recovery?.handoffPolicy === 'skip-unadmitted' && !runtime.running.size) {
731
+ delete record.recovery;
732
+ }
733
+ else if (record.state !== 'completed')
734
+ throw new Error('Occurrence outcome changed during handoff.');
735
+ runtime.stopped = true;
736
+ }
737
+ await this.persist();
738
+ for (const [id] of entries)
739
+ this.runtimes.delete(id);
740
+ this.handoffUnresolved = false;
741
+ });
742
+ }
743
+ catch (error) {
744
+ await this.serial(async () => {
745
+ for (const [id] of entries) {
746
+ const record = this.records.get(id);
747
+ delete record.recovery?.handoff;
748
+ delete record.recovery?.nativeDefinitionAbsent;
749
+ record.error = `Restart handoff unconfirmed: ${errorText(error)}`;
750
+ }
751
+ if (!this.fault)
752
+ await this.persist();
753
+ });
754
+ throw error;
755
+ }
756
+ finally {
757
+ this.suspending = false;
758
+ }
759
+ })();
760
+ this.suspension = run;
761
+ void run.finally(() => { if (this.suspension === run)
762
+ this.suspension = undefined; }).catch(() => { });
763
+ return run;
764
+ }
765
+ /** Providers with durable intent must complete handoff before unrelated host teardown. */
766
+ requiresShutdownHandoff() {
767
+ return [...this.records.values()].some(record => !!record.recovery);
768
+ }
364
769
  destroy() {
365
770
  if (this.destruction)
366
771
  return this.destruction;
367
- this.closing = true;
772
+ const preserve = !this.closing && this.hasActiveSchedules() && this.canSuspendForRestart();
773
+ if (!preserve)
774
+ this.closing = true;
368
775
  return this.destruction = (async () => {
776
+ if (preserve)
777
+ await this.suspendForRestart();
778
+ this.closing = true;
779
+ for (const runtime of this.runtimes.values())
780
+ runtime.recoveryAbort?.abort();
369
781
  await this.initialization?.catch(() => { });
370
782
  let failure;
371
783
  await this.serial(async () => {
372
784
  for (const record of this.records.values()) {
373
785
  if (LIVE.has(record.state)) {
786
+ const runtime = this.runtimes.get(record.id);
787
+ if (record.recovery && record.state === 'active' && !record.cancelRequested && !record.error
788
+ && runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling) {
789
+ runtime.releasing = true;
790
+ record.recovery.status = 'pending';
791
+ record.error = 'Native restart handoff pending verification.';
792
+ }
793
+ else {
794
+ if (record.recovery)
795
+ record.recovery.status = 'blocked';
796
+ record.error = 'Native schedule interrupted by service shutdown.';
797
+ }
374
798
  record.state = 'interrupted';
375
- record.error = 'Native schedule interrupted by service shutdown.';
376
799
  }
377
800
  }
378
801
  if (this.initialized && !this.fault)
379
802
  await this.persist();
380
803
  }).catch(error => { failure = error; });
381
- const stops = await Promise.allSettled([...this.runtimes].map(([id, runtime]) => this.stop(id, runtime)));
804
+ const stops = await Promise.allSettled([...this.runtimes].map(async ([id, runtime]) => {
805
+ if (!runtime.releasing)
806
+ return this.stop(id, runtime);
807
+ try {
808
+ runtime.releasePromise ??= runtime.handle.releaseForRestart();
809
+ let timer;
810
+ try {
811
+ await Promise.race([runtime.releasePromise, new Promise((_, reject) => {
812
+ timer = setTimeout(() => reject(new Error('Native shutdown handoff timed out; ownership retained.')), this.options.recoveryTimeoutMs ?? 30_000);
813
+ })]);
814
+ }
815
+ finally {
816
+ if (timer)
817
+ clearTimeout(timer);
818
+ }
819
+ await this.serial(async () => {
820
+ const record = this.records.get(id);
821
+ if (record.recovery?.status !== 'pending')
822
+ throw new Error('Occurrence raced native handoff; recovery is blocked.');
823
+ runtime.stopped = true;
824
+ record.error = 'Waiting for native recovery after orderly shutdown.';
825
+ await this.persist();
826
+ });
827
+ }
828
+ catch (error) {
829
+ await this.blockRecovery(this.records.get(id), errorText(error));
830
+ throw error;
831
+ }
832
+ }));
382
833
  while (this.operations.size)
383
834
  await Promise.allSettled([...this.operations]);
384
835
  while (this.tasks.size)
@@ -449,17 +900,31 @@ class ScheduleService {
449
900
  if (this.fault)
450
901
  throw this.fault;
451
902
  const record = this.records.get(id);
903
+ if (this.suspending && event.type === 'stopped') {
904
+ if (event.exitConfirmed === false)
905
+ throw new Error('Native handoff exit unconfirmed.');
906
+ runtime.stopped = true;
907
+ return;
908
+ }
909
+ if (runtime.releasing && runtime.stopped)
910
+ return;
452
911
  // Process exit evidence is independent of task state, including late exit
453
912
  // after a failed cancellation cleanup. Never downgrade confirmed exit.
454
913
  if (event.type === 'stopped' && event.exitConfirmed !== false) {
455
914
  runtime.stopped = true;
915
+ runtime.releaseAdmission?.();
916
+ runtime.releaseAdmission = undefined;
456
917
  if (record.error?.startsWith('Native process exit unconfirmed:')) {
457
918
  delete record.error;
458
919
  await this.persist();
459
920
  }
460
921
  }
922
+ if (runtime.releasing && event.type === 'stopped' && event.exitConfirmed !== false)
923
+ return;
461
924
  const ownerCurrent = this.current(record);
462
925
  if (this.closing || !ownerCurrent) {
926
+ if (record.recovery && (!ownerCurrent || event.type !== 'stopped' || event.exitConfirmed === false))
927
+ record.recovery.status = 'blocked';
463
928
  if (LIVE.has(record.state)) {
464
929
  record.state = 'interrupted';
465
930
  record.error = 'Schedule owner or service is no longer current.';
@@ -493,15 +958,30 @@ class ScheduleService {
493
958
  record.error = 'Occurrence tracking limit reached; native schedule stopped.';
494
959
  cleanup = true;
495
960
  }
961
+ else if (event.type === 'skipped') {
962
+ if (record.recovery?.handoffPolicy !== 'skip-unadmitted' || runtime.running.has(event.occurrenceId))
963
+ throw new Error('Unnegotiated occurrence skip.');
964
+ runtime.completed.add(event.occurrenceId);
965
+ record.completedOccurrences = [...runtime.completed];
966
+ // No delivery and no invented success. Unseen skipped wakes are not counted.
967
+ (record.skippedOccurrences ??= []).push(event.occurrenceId);
968
+ if (record.rule.kind === 'once') {
969
+ record.state = 'interrupted';
970
+ record.error = bounded(event.reason, ERROR_BYTES);
971
+ }
972
+ }
496
973
  else if (event.type === 'started') {
497
974
  runtime.running.add(event.occurrenceId);
498
975
  record.state = 'running';
976
+ if (record.recovery)
977
+ record.recovery.status = 'blocked';
499
978
  }
500
979
  else if (event.type === 'completed') {
501
980
  if (typeof event.success !== 'boolean')
502
981
  throw new Error('Invalid native completion event.');
503
982
  runtime.running.delete(event.occurrenceId);
504
983
  runtime.completed.add(event.occurrenceId);
984
+ record.completedOccurrences = [...runtime.completed];
505
985
  if (this.pending.size >= this.maxPending) {
506
986
  record.state = 'interrupted';
507
987
  record.error = 'Delivery outbox is full; latest result could not be retained. Native schedule stopped.';
@@ -519,8 +999,11 @@ class ScheduleService {
519
999
  record.state = 'completed';
520
1000
  cleanup = true;
521
1001
  }
522
- else if (record.state !== 'interrupted')
1002
+ else if (record.state !== 'interrupted') {
523
1003
  record.state = runtime.running.size ? 'running' : 'active';
1004
+ if (record.recovery && !runtime.running.size)
1005
+ record.recovery.status = 'eligible';
1006
+ }
524
1007
  }
525
1008
  }
526
1009
  else
@@ -587,9 +1070,12 @@ class ScheduleService {
587
1070
  if (runtime.stopped)
588
1071
  return Promise.resolve();
589
1072
  if (runtime.stopPromise)
590
- return runtime.stopPromise;
1073
+ return runtime.recoveryAbort ? this.boundRecovery(runtime.stopPromise, 'Native cleanup remains unresolved; ownership retained.') : runtime.stopPromise;
591
1074
  const operation = (async () => {
592
1075
  try {
1076
+ if (runtime.recoveryAbort?.signal.aborted && !runtime.handle && !runtime.stopped) {
1077
+ throw new Error('Recovery receipt is unresolved; native ownership retained.');
1078
+ }
593
1079
  const handle = await runtime.ready;
594
1080
  if (handle && !runtime.stopped)
595
1081
  await handle.stop();
@@ -597,6 +1083,10 @@ class ScheduleService {
597
1083
  // finishes. Keep ownership until its terminal stopped notification.
598
1084
  if (!handle?.cancellationKeepsRunning || !runtime.running.size)
599
1085
  runtime.stopped = true;
1086
+ if (runtime.stopped) {
1087
+ runtime.releaseAdmission?.();
1088
+ runtime.releaseAdmission = undefined;
1089
+ }
600
1090
  }
601
1091
  catch (error) {
602
1092
  await this.serial(async () => {
@@ -610,7 +1100,19 @@ class ScheduleService {
610
1100
  runtime.stopPromise = operation;
611
1101
  void operation.finally(() => { if (!runtime.stopped)
612
1102
  runtime.stopPromise = undefined; }).catch(() => { });
613
- return operation;
1103
+ return runtime.recoveryAbort ? this.boundRecovery(operation, 'Native cleanup remains unresolved; ownership retained.') : operation;
1104
+ }
1105
+ async boundRecovery(operation, message) {
1106
+ let timer;
1107
+ try {
1108
+ return await Promise.race([operation, new Promise((_, reject) => {
1109
+ timer = setTimeout(() => reject(new Error(message)), this.options.recoveryTimeoutMs ?? 30_000);
1110
+ })]);
1111
+ }
1112
+ finally {
1113
+ if (timer)
1114
+ clearTimeout(timer);
1115
+ }
614
1116
  }
615
1117
  newRuntime() {
616
1118
  let resolveReady;
@@ -653,7 +1155,7 @@ class ScheduleService {
653
1155
  checkReady() {
654
1156
  if (this.fault)
655
1157
  throw this.fault;
656
- if (!this.initialized || this.closing)
1158
+ if (!this.initialized || this.closing || this.suspending)
657
1159
  throw new Error('Schedule service is not initialized or is closed.');
658
1160
  }
659
1161
  serial(operation) {
@@ -676,7 +1178,8 @@ class ScheduleService {
676
1178
  }
677
1179
  try {
678
1180
  // JSON can expand one control byte to six bytes (for example, "\\u0001").
679
- const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024) + 4096;
1181
+ const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024)
1182
+ + this.maxRecords * this.maxOccurrences * 515 + 4096;
680
1183
  const size = (await handle.stat()).size;
681
1184
  if (size > maxBytes)
682
1185
  throw new Error('Schedule catalog exceeds its size limit.');
@@ -702,13 +1205,13 @@ class ScheduleService {
702
1205
  }
703
1206
  load(data) {
704
1207
  const catalog = object(data, ['version', 'records', 'pending']);
705
- if (catalog.version !== 1 || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
1208
+ if (![1, 2].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
706
1209
  || !Array.isArray(catalog.pending) || catalog.pending.length > this.maxPending)
707
1210
  throw new Error('Invalid schedule catalog envelope.');
708
1211
  const records = new Map();
709
1212
  const pending = new Map();
710
1213
  for (const value of catalog.records) {
711
- const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation']);
1214
+ const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation', 'execution', 'cancelRequested', 'recovery', 'completedOccurrences', 'skippedOccurrences']);
712
1215
  (0, ScheduleCommand_1.validateScheduleId)(record.id);
713
1216
  validateOwner(record);
714
1217
  (0, ScheduleCommand_1.validateSchedulePrompt)(record.prompt);
@@ -721,6 +1224,35 @@ class ScheduleService {
721
1224
  text(record.nativeId);
722
1225
  if (record.error !== undefined && (typeof record.error !== 'string' || Buffer.byteLength(record.error) > ERROR_BYTES))
723
1226
  throw new Error('Invalid schedule error.');
1227
+ if (record.cancelRequested !== undefined && record.cancelRequested !== true)
1228
+ throw new Error('Invalid cancellation intent.');
1229
+ if (record.execution !== undefined) {
1230
+ const execution = object(record.execution, ['command', 'model', 'effort']);
1231
+ for (const value of Object.values(execution))
1232
+ if (value !== undefined)
1233
+ text(value, 4096);
1234
+ }
1235
+ if (record.recovery !== undefined) {
1236
+ if (catalog.version !== 2)
1237
+ throw new Error('Recovery requires catalog version 2.');
1238
+ const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy']);
1239
+ this.validateCheckpoint(recovery);
1240
+ if (!['eligible', 'pending', 'blocked'].includes(recovery.status))
1241
+ throw new Error('Invalid recovery status.');
1242
+ }
1243
+ if (record.completedOccurrences !== undefined) {
1244
+ if (!Array.isArray(record.completedOccurrences) || record.completedOccurrences.length > this.maxOccurrences
1245
+ || new Set(record.completedOccurrences).size !== record.completedOccurrences.length)
1246
+ throw new Error('Invalid occurrence history.');
1247
+ record.completedOccurrences.forEach(value => text(value));
1248
+ }
1249
+ if (record.skippedOccurrences !== undefined) {
1250
+ if (!Array.isArray(record.skippedOccurrences) || record.skippedOccurrences.length > this.maxOccurrences
1251
+ || new Set(record.skippedOccurrences).size !== record.skippedOccurrences.length
1252
+ || record.skippedOccurrences.some(value => !Array.isArray(record.completedOccurrences) || !record.completedOccurrences.includes(value))) {
1253
+ throw new Error('Invalid skipped occurrence history.');
1254
+ }
1255
+ }
724
1256
  const rule = object(record.rule, ['kind', 'at', 'expression', 'timezone']);
725
1257
  if (rule.kind === 'once')
726
1258
  object(rule, ['kind', 'at']);
@@ -763,7 +1295,7 @@ class ScheduleService {
763
1295
  }
764
1296
  const handle = await fs.open(temporary, 'wx', 0o600);
765
1297
  try {
766
- await handle.writeFile(JSON.stringify({ version: 1, records: [...this.records.values()], pending: [...this.pending.values()] }), 'utf8');
1298
+ await handle.writeFile(JSON.stringify({ version: 2, records: [...this.records.values()], pending: [...this.pending.values()] }), 'utf8');
767
1299
  await handle.sync();
768
1300
  }
769
1301
  finally {