@yu_robotics/remote-cli 1.6.236 → 1.6.238

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/client/MessageHandler.d.ts +9 -2
  2. package/dist/client/MessageHandler.d.ts.map +1 -1
  3. package/dist/client/MessageHandler.js +266 -34
  4. package/dist/client/MessageHandler.js.map +1 -1
  5. package/dist/commands/start.d.ts.map +1 -1
  6. package/dist/commands/start.js +1 -0
  7. package/dist/commands/start.js.map +1 -1
  8. package/dist/executor/AcpExecutor.d.ts +2 -0
  9. package/dist/executor/AcpExecutor.d.ts.map +1 -1
  10. package/dist/executor/AcpExecutor.js +9 -0
  11. package/dist/executor/AcpExecutor.js.map +1 -1
  12. package/dist/executor/ClaudePersistentExecutor.d.ts +52 -12
  13. package/dist/executor/ClaudePersistentExecutor.d.ts.map +1 -1
  14. package/dist/executor/ClaudePersistentExecutor.js +795 -123
  15. package/dist/executor/ClaudePersistentExecutor.js.map +1 -1
  16. package/dist/executor/DshExecutor.d.ts +13 -1
  17. package/dist/executor/DshExecutor.d.ts.map +1 -1
  18. package/dist/executor/DshExecutor.js +39 -1
  19. package/dist/executor/DshExecutor.js.map +1 -1
  20. package/dist/executor/IExecutor.d.ts +14 -0
  21. package/dist/executor/IExecutor.d.ts.map +1 -1
  22. package/dist/executor/claude/TurnLedger.d.ts +33 -0
  23. package/dist/executor/claude/TurnLedger.d.ts.map +1 -0
  24. package/dist/executor/claude/TurnLedger.js +126 -0
  25. package/dist/executor/claude/TurnLedger.js.map +1 -0
  26. package/dist/maintenance/release-notes.json +1 -1
  27. package/dist/managed/ManagedToolBridge.d.ts +4 -1
  28. package/dist/managed/ManagedToolBridge.d.ts.map +1 -1
  29. package/dist/managed/ManagedToolBridge.js +17 -2
  30. package/dist/managed/ManagedToolBridge.js.map +1 -1
  31. package/dist/scheduling/ScheduleCapabilities.d.ts +1 -1
  32. package/dist/scheduling/ScheduleCapabilities.d.ts.map +1 -1
  33. package/dist/scheduling/ScheduleCapabilities.js +3 -2
  34. package/dist/scheduling/ScheduleCapabilities.js.map +1 -1
  35. package/dist/scheduling/ScheduleService.d.ts +28 -0
  36. package/dist/scheduling/ScheduleService.d.ts.map +1 -1
  37. package/dist/scheduling/ScheduleService.js +562 -17
  38. package/dist/scheduling/ScheduleService.js.map +1 -1
  39. package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts +28 -2
  40. package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts.map +1 -1
  41. package/dist/scheduling/claude/ClaudeSessionSchedules.js +158 -18
  42. package/dist/scheduling/claude/ClaudeSessionSchedules.js.map +1 -1
  43. package/dist/scheduling/dsh/SessionScheduleBridge.d.ts.map +1 -1
  44. package/dist/scheduling/dsh/SessionScheduleBridge.js +21 -0
  45. package/dist/scheduling/dsh/SessionScheduleBridge.js.map +1 -1
  46. package/dist/scheduling/dsh/SessionScheduleClient.d.ts.map +1 -1
  47. package/dist/scheduling/dsh/SessionScheduleClient.js +8 -0
  48. package/dist/scheduling/dsh/SessionScheduleClient.js.map +1 -1
  49. package/dist/scheduling/types.d.ts +84 -0
  50. package/dist/scheduling/types.d.ts.map +1 -1
  51. package/dist/scheduling/types.js.map +1 -1
  52. package/dist/settings/SettingsService.d.ts +1 -1
  53. package/dist/settings/SettingsService.d.ts.map +1 -1
  54. package/dist/settings/SettingsService.js +3 -3
  55. package/dist/settings/SettingsService.js.map +1 -1
  56. package/dist/thread/ThreadExecutorPool.d.ts +1 -1
  57. package/dist/thread/ThreadExecutorPool.d.ts.map +1 -1
  58. package/dist/thread/ThreadExecutorPool.js +5 -3
  59. package/dist/thread/ThreadExecutorPool.js.map +1 -1
  60. package/dist/update/AutomaticUpdater.d.ts +2 -1
  61. package/dist/update/AutomaticUpdater.d.ts.map +1 -1
  62. package/dist/update/AutomaticUpdater.js +11 -3
  63. package/dist/update/AutomaticUpdater.js.map +1 -1
  64. package/native/prebuilds/darwin-arm64/managed_native.node +0 -0
  65. package/native/prebuilds/darwin-arm64/manifest.json +1 -1
  66. package/native/prebuilds/darwin-x64/managed_native.node +0 -0
  67. package/native/prebuilds/darwin-x64/manifest.json +1 -1
  68. package/native/prebuilds/win32-arm64/managed_native.node +0 -0
  69. package/native/prebuilds/win32-arm64/manifest.json +1 -1
  70. package/native/prebuilds/win32-x64/managed_native.node +0 -0
  71. package/native/prebuilds/win32-x64/manifest.json +1 -1
  72. package/package.json +1 -1
@@ -83,6 +83,11 @@ function validateOwner(owner) {
83
83
  function sameOwner(left, right) {
84
84
  return left.threadId === right.threadId && left.openId === right.openId && left.generation === right.generation;
85
85
  }
86
+ /** Terminal intent prevents recreation, but uncertain native cleanup still owns admission. */
87
+ function recoveryOwnsWork(record) {
88
+ return !!record.recovery && (!['completed', 'cancelled'].includes(record.state)
89
+ || !!record.error?.startsWith('Native process exit unconfirmed:'));
90
+ }
86
91
  function deliveryId(scheduleId, occurrenceId) {
87
92
  return `${scheduleId}:${occurrenceId}`;
88
93
  }
@@ -112,6 +117,10 @@ class ScheduleService {
112
117
  destruction;
113
118
  fault;
114
119
  catalogPath = '';
120
+ recoveryRun;
121
+ suspending = false;
122
+ handoffUnresolved = false;
123
+ suspension;
115
124
  constructor(options) {
116
125
  this.options = { ...options };
117
126
  this.now = options.now ?? Date.now;
@@ -121,6 +130,9 @@ class ScheduleService {
121
130
  throw new Error('Invalid schedule capacity.');
122
131
  return result;
123
132
  };
133
+ if (options.recoveryTimeoutMs !== undefined && (!Number.isInteger(options.recoveryTimeoutMs)
134
+ || options.recoveryTimeoutMs < 1 || options.recoveryTimeoutMs > 60_000))
135
+ throw new Error('Invalid recovery timeout.');
124
136
  this.maxLive = limit(options.maxLiveSchedules, 5, 5);
125
137
  this.maxRecords = limit(options.maxRecords, 100, 1000);
126
138
  this.maxPending = limit(options.maxPendingDeliveries, 100, 1000);
@@ -138,9 +150,23 @@ class ScheduleService {
138
150
  this.load(data);
139
151
  let changed = false;
140
152
  for (const record of this.records.values()) {
141
- if (LIVE.has(record.state)) {
153
+ if (['completed', 'cancelled', 'failed'].includes(record.state)) {
154
+ if (record.recovery?.status === 'pending') {
155
+ record.recovery.status = 'blocked';
156
+ changed = true;
157
+ }
158
+ continue; // Terminal tombstones always win over a stale recovery checkpoint.
159
+ }
160
+ if (LIVE.has(record.state) || record.recovery?.status === 'pending') {
161
+ const recoverable = record.recovery && !record.cancelRequested && this.current(record)
162
+ && record.state !== 'running' && record.state !== 'failed'
163
+ && (!record.recovery.orderlyOnly || record.recovery.handoff === 'clean')
164
+ && (record.state === 'active' || record.recovery.status === 'pending');
165
+ if (record.recovery)
166
+ record.recovery.status = recoverable ? 'pending' : 'blocked';
167
+ record.error = recoverable ? 'Waiting for native recovery capability and owner readiness.'
168
+ : record.recovery?.orderlyOnly ? 'Dirty shutdown: original native work is unconfirmed; automatic reconstruction is blocked.' : RESTART_REASON;
142
169
  record.state = 'interrupted';
143
- record.error = RESTART_REASON;
144
170
  changed = true;
145
171
  }
146
172
  }
@@ -156,6 +182,193 @@ class ScheduleService {
156
182
  this.initialized = true;
157
183
  });
158
184
  }
185
+ /** Sequential startup/reconnect reconciliation. Never drives the native schedule clock. */
186
+ recover() {
187
+ this.checkReady();
188
+ if (this.recoveryRun)
189
+ return this.recoveryRun;
190
+ const run = this.operation(this.recoverWaiting());
191
+ this.recoveryRun = run;
192
+ void run.finally(() => { if (this.recoveryRun === run)
193
+ this.recoveryRun = undefined; }).catch(() => { });
194
+ return run;
195
+ }
196
+ async recoverWaiting() {
197
+ const candidates = await this.serial(async () => [...this.records.values()]
198
+ .filter(record => record.recovery?.status === 'pending' && record.state === 'interrupted').map(record => record.id));
199
+ // Catalog capacity is bounded, and only one native admission runs at a time.
200
+ for (const id of candidates.slice(0, this.maxLive)) {
201
+ if (this.closing || this.fault)
202
+ return;
203
+ const record = this.records.get(id);
204
+ if (record.cancelRequested || !this.current(record)) {
205
+ await this.blockRecovery(record, 'Recovery owner retired or cancellation was requested.');
206
+ continue;
207
+ }
208
+ if (record.rule.kind === 'once' && Date.parse(record.rule.at) <= this.now()) {
209
+ await this.blockRecovery(record, 'One-shot deadline expired; no late execution was requested.');
210
+ if (record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent)
211
+ await this.serial(async () => { delete record.recovery; await this.persist(); });
212
+ continue;
213
+ }
214
+ const input = { id, threadId: record.threadId, backend: record.backend,
215
+ cwd: record.cwd, prompt: record.prompt, rule: structuredClone(record.rule), ...record.execution,
216
+ recovery: this.checkpoint(record.recovery) };
217
+ if (!this.options.canRecover || !this.options.canRecover(structuredClone(record))) {
218
+ await this.recoveryDiagnostic(record, 'Original owner configuration or transport is not ready for recovery.');
219
+ continue;
220
+ }
221
+ const provider = this.options.provider.recoveryFor?.(input);
222
+ if (!provider || provider.provider !== record.recovery.provider || provider.strategy !== record.recovery.strategy
223
+ || (provider.handoffPolicy ?? 'drain-all') !== (record.recovery.handoffPolicy ?? 'drain-all')) {
224
+ await this.recoveryDiagnostic(record, 'Native restart recovery is unsupported or provider identity changed.');
225
+ continue;
226
+ }
227
+ if (this.runtimes.has(id))
228
+ continue; // An unresolved/late operation retains exclusive ownership.
229
+ const restoreRecord = structuredClone(record);
230
+ const runtime = this.newRuntime();
231
+ runtime.completed = new Set(record.completedOccurrences);
232
+ const controller = new AbortController();
233
+ runtime.recoveryAbort = controller;
234
+ this.runtimes.set(id, runtime);
235
+ const current = () => !controller.signal.aborted && !this.closing && !this.fault && !record.cancelRequested
236
+ && this.current(record) && this.options.canRecover?.(structuredClone(record)) === true;
237
+ const emit = (event) => {
238
+ if (this.runtimes.get(id) !== runtime)
239
+ return;
240
+ this.track(this.onEvent(id, structuredClone(event)));
241
+ };
242
+ let timer;
243
+ let admitted = false;
244
+ let succeeded = false;
245
+ try {
246
+ runtime.releaseAdmission = this.options.acquireRecovery?.(structuredClone(record));
247
+ await this.serial(async () => {
248
+ if (!current())
249
+ throw new Error('Recovery admission expired.');
250
+ // Consume clean proof before mutation. A crash must not replay an orderly-only restore.
251
+ record.state = 'creating';
252
+ delete record.recovery.handoff;
253
+ delete record.recovery.nativeDefinitionAbsent;
254
+ await this.persist();
255
+ });
256
+ admitted = true;
257
+ const admission = Promise.resolve().then(() => provider.reconcile(restoreRecord, { notBefore: this.now(), signal: controller.signal, isCurrent: current,
258
+ isOwnerCurrent: () => !record.cancelRequested && !this.fault && this.current(record) }, emit));
259
+ const adopt = async (handle) => {
260
+ runtime.handle = handle;
261
+ runtime.resolveReady(handle);
262
+ text(handle.nativeId);
263
+ await this.serial(async () => { record.nativeId = handle.nativeId; if (!this.fault)
264
+ await this.persist(); });
265
+ };
266
+ const receipt = admission.then(async (handle) => {
267
+ await adopt(handle);
268
+ if (!current()) {
269
+ await this.stop(id, runtime);
270
+ throw new Error('Late recovery receipt requires cleanup; no successor was admitted.');
271
+ }
272
+ if (!await handle.list() || !current())
273
+ throw new Error('Recovered native inventory or owner is unconfirmed.');
274
+ return handle;
275
+ }, async (error) => {
276
+ if (error instanceof types_1.NativeScheduleCleanupError) {
277
+ await adopt(error.handle);
278
+ if (controller.signal.aborted)
279
+ await this.stop(id, runtime).catch(() => { });
280
+ }
281
+ runtime.resolveReady(runtime.handle);
282
+ // A rejection without a handle is not proof that native admission never happened.
283
+ throw error;
284
+ });
285
+ const handle = await Promise.race([receipt, new Promise((_, reject) => {
286
+ timer = setTimeout(() => { controller.abort(); reject(new Error('Native recovery timed out; ownership retained, do not recreate.')); }, this.options.recoveryTimeoutMs ?? 30_000);
287
+ })]);
288
+ text(handle.nativeId);
289
+ // The provider's reconcile contract includes inventory validation; list is
290
+ // additionally required to verify the returned handle before activating it.
291
+ if (!current())
292
+ throw new Error('Recovered native owner is unconfirmed.');
293
+ await this.serial(async () => {
294
+ if (!current())
295
+ throw new Error('Recovery owner retired before receipt persistence.');
296
+ record.nativeId = handle.nativeId;
297
+ if (record.state === 'creating')
298
+ record.state = 'active';
299
+ record.recovery.status = 'eligible';
300
+ delete record.error;
301
+ await this.persist();
302
+ });
303
+ succeeded = true;
304
+ }
305
+ catch (error) {
306
+ controller.abort();
307
+ if (!admitted) {
308
+ runtime.stopped = true;
309
+ runtime.resolveReady(undefined);
310
+ }
311
+ // Do not retry an ambiguous mutation or release its handle on a timeout.
312
+ await this.blockRecovery(record, errorText(error));
313
+ if (runtime.handle)
314
+ await this.stop(id, runtime).catch(() => { });
315
+ else if (runtime.stopped)
316
+ this.runtimes.delete(id);
317
+ }
318
+ finally {
319
+ if (timer)
320
+ clearTimeout(timer);
321
+ if (succeeded || runtime.stopped) {
322
+ runtime.releaseAdmission?.();
323
+ runtime.releaseAdmission = undefined;
324
+ }
325
+ }
326
+ }
327
+ // A handoff containing only skipped/expired one-shots has nothing to
328
+ // reconcile. It must still release the predecessor's local input fence.
329
+ const released = new Set();
330
+ for (const record of this.records.values()) {
331
+ if (released.has(record.threadId) || this.closing || this.fault || this.handoffUnresolved
332
+ || !this.current(record) || !this.options.canRecover?.(structuredClone(record)))
333
+ continue;
334
+ if ([...this.records.values()].some(other => other.threadId === record.threadId
335
+ && (LIVE.has(other.state) || recoveryOwnsWork(other) || this.runtimes.has(other.id))))
336
+ continue;
337
+ this.options.provider.recoveryFor?.({ id: record.id, threadId: record.threadId, backend: record.backend,
338
+ cwd: record.cwd, prompt: record.prompt, rule: record.rule, ...record.execution })?.releaseRestartAdmission?.();
339
+ released.add(record.threadId);
340
+ }
341
+ }
342
+ checkpoint(value) {
343
+ return { version: 1, provider: value.provider, strategy: value.strategy, identity: value.identity,
344
+ ...(value.orderlyOnly && { orderlyOnly: true }), ...(value.handoffPolicy && { handoffPolicy: value.handoffPolicy }), ...(value.handoff && { handoff: value.handoff }),
345
+ ...(value.nativeDefinitionAbsent && { nativeDefinitionAbsent: true }) };
346
+ }
347
+ validateCheckpoint(value) {
348
+ if (value.version !== 1 || !['reattach', 'reconstruct'].includes(value.strategy))
349
+ throw new Error('Invalid recovery checkpoint.');
350
+ if (value.orderlyOnly !== undefined && value.orderlyOnly !== true || value.handoff !== undefined && value.handoff !== 'clean')
351
+ throw new Error('Invalid handoff evidence.');
352
+ if (value.nativeDefinitionAbsent !== undefined && (value.nativeDefinitionAbsent !== true || value.handoff !== 'clean'))
353
+ throw new Error('Invalid native absence evidence.');
354
+ if (value.handoffPolicy !== undefined && !['drain-all', 'skip-unadmitted'].includes(value.handoffPolicy))
355
+ throw new Error('Invalid handoff policy.');
356
+ text(value.provider);
357
+ text(value.identity, 8192);
358
+ }
359
+ async recoveryDiagnostic(record, reason) {
360
+ await this.serial(async () => { record.error = reason; await this.persist(); });
361
+ }
362
+ async blockRecovery(record, reason) {
363
+ await this.serial(async () => {
364
+ if (!['completed', 'cancelled'].includes(record.state))
365
+ record.state = 'interrupted';
366
+ record.recovery.status = 'blocked';
367
+ record.error = reason;
368
+ if (!this.fault)
369
+ await this.persist();
370
+ });
371
+ }
159
372
  create(owner, input) {
160
373
  return this.operation(this.createSchedule(owner, input));
161
374
  }
@@ -174,7 +387,18 @@ class ScheduleService {
174
387
  if (!node_path_1.default.isAbsolute(input.cwd))
175
388
  throw new Error('Schedule working directory must be absolute.');
176
389
  const originalOwner = ownerOf(owner);
177
- const nativeInput = { ...structuredClone(input), backend };
390
+ const nativeInput = { ...structuredClone(input), backend, threadId: owner.threadId };
391
+ // Never trust a caller-supplied provider checkpoint.
392
+ delete nativeInput.recovery;
393
+ for (const value of [input.command, input.model, input.effort])
394
+ if (value !== undefined)
395
+ text(value, 4096);
396
+ const recovery = this.options.provider.recoveryFor?.(nativeInput);
397
+ if (recovery) {
398
+ nativeInput.recovery = { version: 1, provider: recovery.provider, strategy: recovery.strategy,
399
+ identity: recovery.describe(structuredClone(nativeInput), originalOwner), ...(recovery.orderlyOnly && { orderlyOnly: true }), ...(recovery.handoffPolicy && { handoffPolicy: recovery.handoffPolicy }) };
400
+ this.validateCheckpoint(nativeInput.recovery);
401
+ }
178
402
  const runtime = this.newRuntime();
179
403
  await this.serial(async () => {
180
404
  this.checkReady();
@@ -185,17 +409,21 @@ class ScheduleService {
185
409
  if (this.records.size >= this.maxRecords)
186
410
  throw new Error('Schedule catalog is full; history was not pruned.');
187
411
  const liveCount = [...this.records.values()].filter(record => LIVE.has(record.state)
412
+ || recoveryOwnsWork(record)
188
413
  || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)).length;
189
414
  if (liveCount >= this.maxLive)
190
415
  throw new Error('Live native schedule limit reached.');
191
416
  this.records.set(nativeInput.id, { ...originalOwner, id: nativeInput.id, backend,
192
417
  cwd: nativeInput.cwd, prompt: nativeInput.prompt, rule: structuredClone(nativeInput.rule),
193
- createdAt, state: 'creating' });
418
+ createdAt, state: 'creating', execution: { command: input.command, model: input.model, effort: input.effort },
419
+ ...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: 'eligible' } }) });
194
420
  await this.persist();
195
421
  this.runtimes.set(nativeInput.id, runtime);
196
422
  });
197
423
  try {
198
424
  const handle = await this.options.provider.create(structuredClone(nativeInput), event => {
425
+ if (this.runtimes.get(nativeInput.id) !== runtime)
426
+ return;
199
427
  const task = this.onEvent(nativeInput.id, structuredClone(event));
200
428
  this.track(task);
201
429
  });
@@ -205,6 +433,10 @@ class ScheduleService {
205
433
  await this.serial(async () => {
206
434
  const record = this.records.get(nativeInput.id);
207
435
  record.nativeId = handle.nativeId;
436
+ if (record.recovery && handle.recoveryIdentity) {
437
+ text(handle.recoveryIdentity, 8192);
438
+ record.recovery.identity = handle.recoveryIdentity;
439
+ }
208
440
  const ownerCurrent = this.current(originalOwner);
209
441
  if (this.closing || !ownerCurrent) {
210
442
  if (LIVE.has(record.state)) {
@@ -252,7 +484,7 @@ class ScheduleService {
252
484
  /** Includes terminal jobs whose native process exit is still unconfirmed. */
253
485
  hasActiveSchedules(threadId) {
254
486
  return [...this.records.values()].some(record => (!threadId || record.threadId === threadId)
255
- && (LIVE.has(record.state) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
487
+ && (LIVE.has(record.state) || recoveryOwnsWork(record) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
256
488
  }
257
489
  /** Card eligibility includes retryable cleanup even for a terminal catalog record. */
258
490
  canCancel(owner, id) {
@@ -261,7 +493,20 @@ class ScheduleService {
261
493
  const runtime = this.runtimes.get(id);
262
494
  if (record.state === 'cancelled' && runtime?.handle?.cancellationKeepsRunning && !record.error)
263
495
  return false;
264
- return LIVE.has(record.state) || Boolean(runtime && !runtime.stopped);
496
+ return LIVE.has(record.state) || recoveryOwnsWork(record) || Boolean(runtime && !runtime.stopped);
497
+ }
498
+ /** Keep admitted native work visible when cleanup cannot safely acquire its host. */
499
+ async retainCleanupError(owner, id, error) {
500
+ this.checkReady();
501
+ validateOwner(owner);
502
+ (0, ScheduleCommand_1.validateScheduleId)(id);
503
+ await this.serial(async () => {
504
+ const record = this.owned(owner, id);
505
+ if (this.runtimes.get(id)?.stopped)
506
+ return;
507
+ record.error = errorText(error);
508
+ await this.persist();
509
+ });
265
510
  }
266
511
  /** User-requested native reconciliation only; no background polling is installed. */
267
512
  refresh(owner) {
@@ -320,11 +565,30 @@ class ScheduleService {
320
565
  await this.serial(async () => {
321
566
  const record = this.owned(owner, id);
322
567
  runtime = this.runtimes.get(id);
568
+ record.cancelRequested = true;
569
+ runtime?.recoveryAbort?.abort();
570
+ if (record.recovery)
571
+ record.recovery.status = 'blocked';
572
+ await this.persist();
573
+ if (!runtime && record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent) {
574
+ record.state = 'cancelled';
575
+ delete record.error;
576
+ await this.persist();
577
+ }
578
+ if (!runtime && recoveryOwnsWork(record)) {
579
+ record.error = 'Cancellation intent retained; native cleanup is unconfirmed. Recovery will not recreate this job.';
580
+ await this.persist();
581
+ throw new Error(record.error);
582
+ }
323
583
  if (runtime?.cancelPromise) {
324
584
  existing = runtime.cancelPromise;
325
585
  return;
326
586
  }
327
587
  if (!LIVE.has(record.state) && !runtime?.cancelling && (!runtime || runtime.stopped)) {
588
+ if (record.recovery && runtime?.stopped) {
589
+ record.state = 'cancelled';
590
+ delete record.error;
591
+ }
328
592
  for (const [key, pending] of this.pending) {
329
593
  if (pending.scheduleId === id && !pending.afterCancellation)
330
594
  this.pending.delete(key);
@@ -335,6 +599,9 @@ class ScheduleService {
335
599
  }
336
600
  if (!runtime)
337
601
  throw new Error('Native schedule handle is unavailable.');
602
+ if (runtime.recoveryAbort && !runtime.handle && !runtime.stopped) {
603
+ throw new Error('Recovery receipt is unresolved; cancellation intent retained until native cleanup can be confirmed.');
604
+ }
338
605
  runtime.cancelling = true;
339
606
  // Starting outside the state queue also permits synchronous provider callbacks.
340
607
  operation = Promise.resolve().then(() => this.cancelNative(id, runtime));
@@ -348,24 +615,221 @@ class ScheduleService {
348
615
  const ids = await this.serial(async () => [...this.pending.keys()]);
349
616
  await this.attemptDeliveries(ids);
350
617
  }
618
+ hasUnresolvedHandoff() { return this.handoffUnresolved; }
619
+ hasUnresolvedRecovery() {
620
+ return [...this.records.values()].some(record => recoveryOwnsWork(record)
621
+ && record.recovery?.status !== 'eligible');
622
+ }
623
+ hasPendingRecovery() {
624
+ return [...this.records.values()].some(record => record.state === 'interrupted' && record.recovery?.status === 'pending');
625
+ }
626
+ /** Snapshot eligibility is only admission to a handoff, never permission to install. */
627
+ canSuspendForRestart() {
628
+ if (!this.initialized || this.closing || this.fault || this.suspending || this.recoveryRun || this.operations.size || this.pending.size)
629
+ return false;
630
+ return [...this.records.values()].filter(record => LIVE.has(record.state) || recoveryOwnsWork(record)
631
+ || !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped).every(record => {
632
+ const runtime = this.runtimes.get(record.id);
633
+ return record.state === 'active' && !!record.recovery && !record.cancelRequested && !record.error
634
+ && !!runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling;
635
+ });
636
+ }
637
+ /** Delete future definitions, drain native work, join owned hosts, then publish restart proof. */
638
+ suspendForRestart() {
639
+ if (this.suspension)
640
+ return this.suspension;
641
+ if (!this.canSuspendForRestart())
642
+ return Promise.reject(new Error('Schedules are not ready for restart handoff.'));
643
+ this.suspending = true;
644
+ this.handoffUnresolved = true;
645
+ const entries = [...this.runtimes].filter(([id]) => this.records.get(id)?.state === 'active');
646
+ const run = (async () => {
647
+ try {
648
+ // Checkpoints remain dirty throughout deletion/drain and actual host exit.
649
+ await this.serial(async () => {
650
+ for (const [id, runtime] of entries) {
651
+ if (runtime.handle?.recoveryIdentity) {
652
+ text(runtime.handle.recoveryIdentity, 8192);
653
+ this.records.get(id).recovery.identity = runtime.handle.recoveryIdentity;
654
+ }
655
+ delete this.records.get(id).recovery.handoff;
656
+ delete this.records.get(id).recovery.nativeDefinitionAbsent;
657
+ }
658
+ await this.persist();
659
+ });
660
+ const releases = await Promise.allSettled(entries.map(([, runtime]) => this.boundRecovery(runtime.handle.releaseForRestart(), 'Native schedule drain remains unresolved.')));
661
+ const failed = releases.find((result) => result.status === 'rejected');
662
+ if (failed)
663
+ throw failed.reason;
664
+ for (const [index, result] of releases.entries()) {
665
+ const policy = this.records.get(entries[index][0]).recovery.handoffPolicy ?? 'drain-all';
666
+ if (result.status === 'fulfilled' && (result.value?.policy ?? 'drain-all') !== policy)
667
+ throw new Error('Native handoff policy receipt changed.');
668
+ }
669
+ const absentOneShots = new Set();
670
+ for (const [index, result] of releases.entries())
671
+ if (result.status === 'fulfilled' && result.value?.onceDefinitionAlreadyAbsent) {
672
+ const record = this.records.get(entries[index][0]);
673
+ if (record.rule.kind !== 'once' || record.recovery?.handoffPolicy !== 'skip-unadmitted')
674
+ throw new Error('Invalid absent one-shot handoff receipt.');
675
+ absentOneShots.add(record.id);
676
+ }
677
+ const absent = new Set(entries.filter((_, index) => {
678
+ const result = releases[index];
679
+ return result.status === 'fulfilled' && result.value?.nativeDefinitionAbsent === true;
680
+ }).map(([id]) => id));
681
+ await this.boundRecovery((async () => {
682
+ while (this.tasks.size)
683
+ await Promise.allSettled([...this.tasks]);
684
+ })(), 'Scheduled result drain remains unresolved.');
685
+ await this.tail;
686
+ if (this.pending.size)
687
+ throw new Error('Scheduled result acknowledgement is pending.');
688
+ if (this.fault || entries.some(([, runtime]) => runtime.running.size))
689
+ throw this.fault ?? new Error('Native occurrence is still running.');
690
+ const finished = new Set();
691
+ for (const [id] of entries) {
692
+ const record = this.records.get(id);
693
+ const provider = this.options.provider.recoveryFor?.({ id, threadId: record.threadId, backend: record.backend,
694
+ cwd: record.cwd, rule: record.rule, prompt: record.prompt, ...record.execution });
695
+ if (!provider || provider.provider !== record.recovery.provider)
696
+ throw new Error('Native recovery provider changed during handoff.');
697
+ const key = `${record.threadId}:${provider.provider}`;
698
+ if (finished.has(key))
699
+ continue;
700
+ if (provider.orderlyOnly && !provider.finishRestart)
701
+ throw new Error('Owned native host exit cannot be confirmed.');
702
+ if (provider.finishRestart)
703
+ await this.boundRecovery(provider.finishRestart(), 'Owned native host exit remains unresolved.');
704
+ finished.add(key);
705
+ }
706
+ await this.boundRecovery((async () => {
707
+ while (this.tasks.size)
708
+ await Promise.allSettled([...this.tasks]);
709
+ })(), 'Scheduled result drain remains unresolved.');
710
+ await this.serial(async () => {
711
+ if (this.fault)
712
+ throw this.fault;
713
+ for (const [id, runtime] of entries) {
714
+ const record = this.records.get(id);
715
+ if (!this.current(record) || record.cancelRequested)
716
+ throw new Error('Schedule owner retired during handoff.');
717
+ if (record.state === 'active' && absentOneShots.has(id)) {
718
+ record.state = 'interrupted';
719
+ record.error = 'One-shot native definition was already absent at handoff; it will not be replayed.';
720
+ delete record.recovery;
721
+ }
722
+ else if (record.state === 'active') {
723
+ record.state = 'interrupted';
724
+ record.recovery.status = 'pending';
725
+ record.recovery.handoff = 'clean';
726
+ if (absent.has(id))
727
+ record.recovery.nativeDefinitionAbsent = true;
728
+ record.error = 'Native host drained and exited; waiting for automatic recovery.';
729
+ }
730
+ else if (record.state === 'interrupted' && record.rule.kind === 'once' && record.recovery?.handoffPolicy === 'skip-unadmitted' && !runtime.running.size) {
731
+ delete record.recovery;
732
+ }
733
+ else if (record.state !== 'completed')
734
+ throw new Error('Occurrence outcome changed during handoff.');
735
+ runtime.stopped = true;
736
+ }
737
+ await this.persist();
738
+ for (const [id] of entries)
739
+ this.runtimes.delete(id);
740
+ this.handoffUnresolved = false;
741
+ });
742
+ }
743
+ catch (error) {
744
+ await this.serial(async () => {
745
+ for (const [id] of entries) {
746
+ const record = this.records.get(id);
747
+ delete record.recovery?.handoff;
748
+ delete record.recovery?.nativeDefinitionAbsent;
749
+ record.error = `Restart handoff unconfirmed: ${errorText(error)}`;
750
+ }
751
+ if (!this.fault)
752
+ await this.persist();
753
+ });
754
+ throw error;
755
+ }
756
+ finally {
757
+ this.suspending = false;
758
+ }
759
+ })();
760
+ this.suspension = run;
761
+ void run.finally(() => { if (this.suspension === run)
762
+ this.suspension = undefined; }).catch(() => { });
763
+ return run;
764
+ }
765
+ /** Providers with durable intent must complete handoff before unrelated host teardown. */
766
+ requiresShutdownHandoff() {
767
+ return [...this.records.values()].some(record => !!record.recovery);
768
+ }
351
769
  destroy() {
352
770
  if (this.destruction)
353
771
  return this.destruction;
354
- this.closing = true;
772
+ const preserve = !this.closing && this.hasActiveSchedules() && this.canSuspendForRestart();
773
+ if (!preserve)
774
+ this.closing = true;
355
775
  return this.destruction = (async () => {
776
+ if (preserve)
777
+ await this.suspendForRestart();
778
+ this.closing = true;
779
+ for (const runtime of this.runtimes.values())
780
+ runtime.recoveryAbort?.abort();
356
781
  await this.initialization?.catch(() => { });
357
782
  let failure;
358
783
  await this.serial(async () => {
359
784
  for (const record of this.records.values()) {
360
785
  if (LIVE.has(record.state)) {
786
+ const runtime = this.runtimes.get(record.id);
787
+ if (record.recovery && record.state === 'active' && !record.cancelRequested && !record.error
788
+ && runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling) {
789
+ runtime.releasing = true;
790
+ record.recovery.status = 'pending';
791
+ record.error = 'Native restart handoff pending verification.';
792
+ }
793
+ else {
794
+ if (record.recovery)
795
+ record.recovery.status = 'blocked';
796
+ record.error = 'Native schedule interrupted by service shutdown.';
797
+ }
361
798
  record.state = 'interrupted';
362
- record.error = 'Native schedule interrupted by service shutdown.';
363
799
  }
364
800
  }
365
801
  if (this.initialized && !this.fault)
366
802
  await this.persist();
367
803
  }).catch(error => { failure = error; });
368
- const stops = await Promise.allSettled([...this.runtimes].map(([id, runtime]) => this.stop(id, runtime)));
804
+ const stops = await Promise.allSettled([...this.runtimes].map(async ([id, runtime]) => {
805
+ if (!runtime.releasing)
806
+ return this.stop(id, runtime);
807
+ try {
808
+ runtime.releasePromise ??= runtime.handle.releaseForRestart();
809
+ let timer;
810
+ try {
811
+ await Promise.race([runtime.releasePromise, new Promise((_, reject) => {
812
+ timer = setTimeout(() => reject(new Error('Native shutdown handoff timed out; ownership retained.')), this.options.recoveryTimeoutMs ?? 30_000);
813
+ })]);
814
+ }
815
+ finally {
816
+ if (timer)
817
+ clearTimeout(timer);
818
+ }
819
+ await this.serial(async () => {
820
+ const record = this.records.get(id);
821
+ if (record.recovery?.status !== 'pending')
822
+ throw new Error('Occurrence raced native handoff; recovery is blocked.');
823
+ runtime.stopped = true;
824
+ record.error = 'Waiting for native recovery after orderly shutdown.';
825
+ await this.persist();
826
+ });
827
+ }
828
+ catch (error) {
829
+ await this.blockRecovery(this.records.get(id), errorText(error));
830
+ throw error;
831
+ }
832
+ }));
369
833
  while (this.operations.size)
370
834
  await Promise.allSettled([...this.operations]);
371
835
  while (this.tasks.size)
@@ -436,17 +900,31 @@ class ScheduleService {
436
900
  if (this.fault)
437
901
  throw this.fault;
438
902
  const record = this.records.get(id);
903
+ if (this.suspending && event.type === 'stopped') {
904
+ if (event.exitConfirmed === false)
905
+ throw new Error('Native handoff exit unconfirmed.');
906
+ runtime.stopped = true;
907
+ return;
908
+ }
909
+ if (runtime.releasing && runtime.stopped)
910
+ return;
439
911
  // Process exit evidence is independent of task state, including late exit
440
912
  // after a failed cancellation cleanup. Never downgrade confirmed exit.
441
913
  if (event.type === 'stopped' && event.exitConfirmed !== false) {
442
914
  runtime.stopped = true;
915
+ runtime.releaseAdmission?.();
916
+ runtime.releaseAdmission = undefined;
443
917
  if (record.error?.startsWith('Native process exit unconfirmed:')) {
444
918
  delete record.error;
445
919
  await this.persist();
446
920
  }
447
921
  }
922
+ if (runtime.releasing && event.type === 'stopped' && event.exitConfirmed !== false)
923
+ return;
448
924
  const ownerCurrent = this.current(record);
449
925
  if (this.closing || !ownerCurrent) {
926
+ if (record.recovery && (!ownerCurrent || event.type !== 'stopped' || event.exitConfirmed === false))
927
+ record.recovery.status = 'blocked';
450
928
  if (LIVE.has(record.state)) {
451
929
  record.state = 'interrupted';
452
930
  record.error = 'Schedule owner or service is no longer current.';
@@ -480,15 +958,30 @@ class ScheduleService {
480
958
  record.error = 'Occurrence tracking limit reached; native schedule stopped.';
481
959
  cleanup = true;
482
960
  }
961
+ else if (event.type === 'skipped') {
962
+ if (record.recovery?.handoffPolicy !== 'skip-unadmitted' || runtime.running.has(event.occurrenceId))
963
+ throw new Error('Unnegotiated occurrence skip.');
964
+ runtime.completed.add(event.occurrenceId);
965
+ record.completedOccurrences = [...runtime.completed];
966
+ // No delivery and no invented success. Unseen skipped wakes are not counted.
967
+ (record.skippedOccurrences ??= []).push(event.occurrenceId);
968
+ if (record.rule.kind === 'once') {
969
+ record.state = 'interrupted';
970
+ record.error = bounded(event.reason, ERROR_BYTES);
971
+ }
972
+ }
483
973
  else if (event.type === 'started') {
484
974
  runtime.running.add(event.occurrenceId);
485
975
  record.state = 'running';
976
+ if (record.recovery)
977
+ record.recovery.status = 'blocked';
486
978
  }
487
979
  else if (event.type === 'completed') {
488
980
  if (typeof event.success !== 'boolean')
489
981
  throw new Error('Invalid native completion event.');
490
982
  runtime.running.delete(event.occurrenceId);
491
983
  runtime.completed.add(event.occurrenceId);
984
+ record.completedOccurrences = [...runtime.completed];
492
985
  if (this.pending.size >= this.maxPending) {
493
986
  record.state = 'interrupted';
494
987
  record.error = 'Delivery outbox is full; latest result could not be retained. Native schedule stopped.';
@@ -506,8 +999,11 @@ class ScheduleService {
506
999
  record.state = 'completed';
507
1000
  cleanup = true;
508
1001
  }
509
- else if (record.state !== 'interrupted')
1002
+ else if (record.state !== 'interrupted') {
510
1003
  record.state = runtime.running.size ? 'running' : 'active';
1004
+ if (record.recovery && !runtime.running.size)
1005
+ record.recovery.status = 'eligible';
1006
+ }
511
1007
  }
512
1008
  }
513
1009
  else
@@ -574,9 +1070,12 @@ class ScheduleService {
574
1070
  if (runtime.stopped)
575
1071
  return Promise.resolve();
576
1072
  if (runtime.stopPromise)
577
- return runtime.stopPromise;
1073
+ return runtime.recoveryAbort ? this.boundRecovery(runtime.stopPromise, 'Native cleanup remains unresolved; ownership retained.') : runtime.stopPromise;
578
1074
  const operation = (async () => {
579
1075
  try {
1076
+ if (runtime.recoveryAbort?.signal.aborted && !runtime.handle && !runtime.stopped) {
1077
+ throw new Error('Recovery receipt is unresolved; native ownership retained.');
1078
+ }
580
1079
  const handle = await runtime.ready;
581
1080
  if (handle && !runtime.stopped)
582
1081
  await handle.stop();
@@ -584,6 +1083,10 @@ class ScheduleService {
584
1083
  // finishes. Keep ownership until its terminal stopped notification.
585
1084
  if (!handle?.cancellationKeepsRunning || !runtime.running.size)
586
1085
  runtime.stopped = true;
1086
+ if (runtime.stopped) {
1087
+ runtime.releaseAdmission?.();
1088
+ runtime.releaseAdmission = undefined;
1089
+ }
587
1090
  }
588
1091
  catch (error) {
589
1092
  await this.serial(async () => {
@@ -597,7 +1100,19 @@ class ScheduleService {
597
1100
  runtime.stopPromise = operation;
598
1101
  void operation.finally(() => { if (!runtime.stopped)
599
1102
  runtime.stopPromise = undefined; }).catch(() => { });
600
- return operation;
1103
+ return runtime.recoveryAbort ? this.boundRecovery(operation, 'Native cleanup remains unresolved; ownership retained.') : operation;
1104
+ }
1105
+ async boundRecovery(operation, message) {
1106
+ let timer;
1107
+ try {
1108
+ return await Promise.race([operation, new Promise((_, reject) => {
1109
+ timer = setTimeout(() => reject(new Error(message)), this.options.recoveryTimeoutMs ?? 30_000);
1110
+ })]);
1111
+ }
1112
+ finally {
1113
+ if (timer)
1114
+ clearTimeout(timer);
1115
+ }
601
1116
  }
602
1117
  newRuntime() {
603
1118
  let resolveReady;
@@ -640,7 +1155,7 @@ class ScheduleService {
640
1155
  checkReady() {
641
1156
  if (this.fault)
642
1157
  throw this.fault;
643
- if (!this.initialized || this.closing)
1158
+ if (!this.initialized || this.closing || this.suspending)
644
1159
  throw new Error('Schedule service is not initialized or is closed.');
645
1160
  }
646
1161
  serial(operation) {
@@ -663,7 +1178,8 @@ class ScheduleService {
663
1178
  }
664
1179
  try {
665
1180
  // JSON can expand one control byte to six bytes (for example, "\\u0001").
666
- const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024) + 4096;
1181
+ const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024)
1182
+ + this.maxRecords * this.maxOccurrences * 515 + 4096;
667
1183
  const size = (await handle.stat()).size;
668
1184
  if (size > maxBytes)
669
1185
  throw new Error('Schedule catalog exceeds its size limit.');
@@ -689,13 +1205,13 @@ class ScheduleService {
689
1205
  }
690
1206
  load(data) {
691
1207
  const catalog = object(data, ['version', 'records', 'pending']);
692
- if (catalog.version !== 1 || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
1208
+ if (![1, 2].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
693
1209
  || !Array.isArray(catalog.pending) || catalog.pending.length > this.maxPending)
694
1210
  throw new Error('Invalid schedule catalog envelope.');
695
1211
  const records = new Map();
696
1212
  const pending = new Map();
697
1213
  for (const value of catalog.records) {
698
- const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation']);
1214
+ const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation', 'execution', 'cancelRequested', 'recovery', 'completedOccurrences', 'skippedOccurrences']);
699
1215
  (0, ScheduleCommand_1.validateScheduleId)(record.id);
700
1216
  validateOwner(record);
701
1217
  (0, ScheduleCommand_1.validateSchedulePrompt)(record.prompt);
@@ -708,6 +1224,35 @@ class ScheduleService {
708
1224
  text(record.nativeId);
709
1225
  if (record.error !== undefined && (typeof record.error !== 'string' || Buffer.byteLength(record.error) > ERROR_BYTES))
710
1226
  throw new Error('Invalid schedule error.');
1227
+ if (record.cancelRequested !== undefined && record.cancelRequested !== true)
1228
+ throw new Error('Invalid cancellation intent.');
1229
+ if (record.execution !== undefined) {
1230
+ const execution = object(record.execution, ['command', 'model', 'effort']);
1231
+ for (const value of Object.values(execution))
1232
+ if (value !== undefined)
1233
+ text(value, 4096);
1234
+ }
1235
+ if (record.recovery !== undefined) {
1236
+ if (catalog.version !== 2)
1237
+ throw new Error('Recovery requires catalog version 2.');
1238
+ const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy']);
1239
+ this.validateCheckpoint(recovery);
1240
+ if (!['eligible', 'pending', 'blocked'].includes(recovery.status))
1241
+ throw new Error('Invalid recovery status.');
1242
+ }
1243
+ if (record.completedOccurrences !== undefined) {
1244
+ if (!Array.isArray(record.completedOccurrences) || record.completedOccurrences.length > this.maxOccurrences
1245
+ || new Set(record.completedOccurrences).size !== record.completedOccurrences.length)
1246
+ throw new Error('Invalid occurrence history.');
1247
+ record.completedOccurrences.forEach(value => text(value));
1248
+ }
1249
+ if (record.skippedOccurrences !== undefined) {
1250
+ if (!Array.isArray(record.skippedOccurrences) || record.skippedOccurrences.length > this.maxOccurrences
1251
+ || new Set(record.skippedOccurrences).size !== record.skippedOccurrences.length
1252
+ || record.skippedOccurrences.some(value => !Array.isArray(record.completedOccurrences) || !record.completedOccurrences.includes(value))) {
1253
+ throw new Error('Invalid skipped occurrence history.');
1254
+ }
1255
+ }
711
1256
  const rule = object(record.rule, ['kind', 'at', 'expression', 'timezone']);
712
1257
  if (rule.kind === 'once')
713
1258
  object(rule, ['kind', 'at']);
@@ -750,7 +1295,7 @@ class ScheduleService {
750
1295
  }
751
1296
  const handle = await fs.open(temporary, 'wx', 0o600);
752
1297
  try {
753
- await handle.writeFile(JSON.stringify({ version: 1, records: [...this.records.values()], pending: [...this.pending.values()] }), 'utf8');
1298
+ await handle.writeFile(JSON.stringify({ version: 2, records: [...this.records.values()], pending: [...this.pending.values()] }), 'utf8');
754
1299
  await handle.sync();
755
1300
  }
756
1301
  finally {