@yu_robotics/remote-cli 1.6.237 → 1.6.239
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/MessageHandler.d.ts +9 -2
- package/dist/client/MessageHandler.d.ts.map +1 -1
- package/dist/client/MessageHandler.js +202 -21
- package/dist/client/MessageHandler.js.map +1 -1
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +5 -1
- package/dist/commands/start.js.map +1 -1
- package/dist/executor/AcpExecutor.d.ts +2 -0
- package/dist/executor/AcpExecutor.d.ts.map +1 -1
- package/dist/executor/AcpExecutor.js +9 -0
- package/dist/executor/AcpExecutor.js.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.d.ts +20 -7
- package/dist/executor/ClaudePersistentExecutor.d.ts.map +1 -1
- package/dist/executor/ClaudePersistentExecutor.js +393 -58
- package/dist/executor/ClaudePersistentExecutor.js.map +1 -1
- package/dist/executor/DshExecutor.d.ts +13 -1
- package/dist/executor/DshExecutor.d.ts.map +1 -1
- package/dist/executor/DshExecutor.js +39 -1
- package/dist/executor/DshExecutor.js.map +1 -1
- package/dist/executor/IExecutor.d.ts +6 -0
- package/dist/executor/IExecutor.d.ts.map +1 -1
- package/dist/executor/claude/TurnLedger.js +1 -1
- package/dist/executor/claude/TurnLedger.js.map +1 -1
- package/dist/maintenance/release-notes.json +1 -1
- package/dist/scheduling/ScheduleCapabilities.d.ts +1 -1
- package/dist/scheduling/ScheduleCapabilities.d.ts.map +1 -1
- package/dist/scheduling/ScheduleCapabilities.js +3 -2
- package/dist/scheduling/ScheduleCapabilities.js.map +1 -1
- package/dist/scheduling/ScheduleService.d.ts +27 -0
- package/dist/scheduling/ScheduleService.d.ts.map +1 -1
- package/dist/scheduling/ScheduleService.js +564 -17
- package/dist/scheduling/ScheduleService.js.map +1 -1
- package/dist/scheduling/claude/ClaudeScheduleOwnership.d.ts +27 -0
- package/dist/scheduling/claude/ClaudeScheduleOwnership.d.ts.map +1 -0
- package/dist/scheduling/claude/ClaudeScheduleOwnership.js +122 -0
- package/dist/scheduling/claude/ClaudeScheduleOwnership.js.map +1 -0
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts +30 -2
- package/dist/scheduling/claude/ClaudeSessionSchedules.d.ts.map +1 -1
- package/dist/scheduling/claude/ClaudeSessionSchedules.js +170 -20
- package/dist/scheduling/claude/ClaudeSessionSchedules.js.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleBridge.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleBridge.js +21 -0
- package/dist/scheduling/dsh/SessionScheduleBridge.js.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.d.ts.map +1 -1
- package/dist/scheduling/dsh/SessionScheduleClient.js +8 -0
- package/dist/scheduling/dsh/SessionScheduleClient.js.map +1 -1
- package/dist/scheduling/types.d.ts +90 -0
- package/dist/scheduling/types.d.ts.map +1 -1
- package/dist/scheduling/types.js +5 -1
- package/dist/scheduling/types.js.map +1 -1
- package/dist/service/ServiceManager.d.ts +5 -0
- package/dist/service/ServiceManager.d.ts.map +1 -1
- package/dist/service/ServiceManager.js +86 -9
- package/dist/service/ServiceManager.js.map +1 -1
- package/dist/settings/SettingsService.d.ts +1 -1
- package/dist/settings/SettingsService.d.ts.map +1 -1
- package/dist/settings/SettingsService.js +3 -3
- package/dist/settings/SettingsService.js.map +1 -1
- package/dist/thread/ThreadExecutorPool.d.ts +1 -1
- package/dist/thread/ThreadExecutorPool.d.ts.map +1 -1
- package/dist/thread/ThreadExecutorPool.js +5 -3
- package/dist/thread/ThreadExecutorPool.js.map +1 -1
- package/dist/update/AutomaticUpdater.d.ts +2 -1
- package/dist/update/AutomaticUpdater.d.ts.map +1 -1
- package/dist/update/AutomaticUpdater.js +11 -3
- package/dist/update/AutomaticUpdater.js.map +1 -1
- package/native/prebuilds/darwin-arm64/managed_native.node +0 -0
- package/native/prebuilds/darwin-arm64/manifest.json +1 -1
- package/native/prebuilds/darwin-x64/managed_native.node +0 -0
- package/native/prebuilds/darwin-x64/manifest.json +1 -1
- package/native/prebuilds/win32-arm64/managed_native.node +0 -0
- package/native/prebuilds/win32-arm64/manifest.json +1 -1
- package/native/prebuilds/win32-x64/managed_native.node +0 -0
- package/native/prebuilds/win32-x64/manifest.json +1 -1
- package/package.json +1 -1
|
@@ -37,6 +37,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
39
|
exports.ScheduleService = exports.SCHEDULE_CATALOG_FILE = void 0;
|
|
40
|
+
const OwnershipRuntime_1 = require("../delegation/OwnershipRuntime");
|
|
40
41
|
const node_fs_1 = require("node:fs");
|
|
41
42
|
const fs = __importStar(require("node:fs/promises"));
|
|
42
43
|
const node_path_1 = __importDefault(require("node:path"));
|
|
@@ -83,6 +84,11 @@ function validateOwner(owner) {
|
|
|
83
84
|
function sameOwner(left, right) {
|
|
84
85
|
return left.threadId === right.threadId && left.openId === right.openId && left.generation === right.generation;
|
|
85
86
|
}
|
|
87
|
+
/** Terminal intent prevents recreation, but uncertain native cleanup still owns admission. */
|
|
88
|
+
function recoveryOwnsWork(record) {
|
|
89
|
+
return !!record.recovery && (!['completed', 'cancelled'].includes(record.state)
|
|
90
|
+
|| !!record.error?.startsWith('Native process exit unconfirmed:'));
|
|
91
|
+
}
|
|
86
92
|
function deliveryId(scheduleId, occurrenceId) {
|
|
87
93
|
return `${scheduleId}:${occurrenceId}`;
|
|
88
94
|
}
|
|
@@ -112,6 +118,11 @@ class ScheduleService {
|
|
|
112
118
|
destruction;
|
|
113
119
|
fault;
|
|
114
120
|
catalogPath = '';
|
|
121
|
+
recoveryRun;
|
|
122
|
+
catalogGate;
|
|
123
|
+
suspending = false;
|
|
124
|
+
handoffUnresolved = false;
|
|
125
|
+
suspension;
|
|
115
126
|
constructor(options) {
|
|
116
127
|
this.options = { ...options };
|
|
117
128
|
this.now = options.now ?? Date.now;
|
|
@@ -121,6 +132,9 @@ class ScheduleService {
|
|
|
121
132
|
throw new Error('Invalid schedule capacity.');
|
|
122
133
|
return result;
|
|
123
134
|
};
|
|
135
|
+
if (options.recoveryTimeoutMs !== undefined && (!Number.isInteger(options.recoveryTimeoutMs)
|
|
136
|
+
|| options.recoveryTimeoutMs < 1 || options.recoveryTimeoutMs > 60_000))
|
|
137
|
+
throw new Error('Invalid recovery timeout.');
|
|
124
138
|
this.maxLive = limit(options.maxLiveSchedules, 5, 5);
|
|
125
139
|
this.maxRecords = limit(options.maxRecords, 100, 1000);
|
|
126
140
|
this.maxPending = limit(options.maxPendingDeliveries, 100, 1000);
|
|
@@ -133,14 +147,33 @@ class ScheduleService {
|
|
|
133
147
|
await fs.mkdir(this.options.directory, { recursive: true, mode: 0o700 });
|
|
134
148
|
const directory = await fs.realpath(this.options.directory);
|
|
135
149
|
this.catalogPath = node_path_1.default.join(directory, exports.SCHEDULE_CATALOG_FILE);
|
|
150
|
+
this.catalogGate = (0, OwnershipRuntime_1.kernelGate)(`${this.catalogPath}.lock`);
|
|
151
|
+
if (!this.catalogGate)
|
|
152
|
+
throw new Error('Another client owns the schedule catalog.');
|
|
136
153
|
const data = await this.readCatalog();
|
|
137
154
|
if (data !== undefined)
|
|
138
155
|
this.load(data);
|
|
139
156
|
let changed = false;
|
|
140
157
|
for (const record of this.records.values()) {
|
|
141
|
-
if (
|
|
158
|
+
if (['completed', 'cancelled', 'failed'].includes(record.state)) {
|
|
159
|
+
if (record.recovery?.status === 'pending') {
|
|
160
|
+
record.recovery.status = 'blocked';
|
|
161
|
+
changed = true;
|
|
162
|
+
}
|
|
163
|
+
continue; // Terminal tombstones always win over a stale recovery checkpoint.
|
|
164
|
+
}
|
|
165
|
+
if (LIVE.has(record.state) || record.recovery?.status === 'pending'
|
|
166
|
+
|| record.state === 'interrupted' && record.recovery?.futureOnly && record.rule.kind === 'cron') {
|
|
167
|
+
const futureOnly = record.recovery?.futureOnly && record.rule.kind === 'cron';
|
|
168
|
+
const recoverable = record.recovery && !record.cancelRequested && this.current(record)
|
|
169
|
+
&& (record.state !== 'running' || futureOnly) && record.state !== 'failed'
|
|
170
|
+
&& (!record.recovery.orderlyOnly || record.recovery.handoff === 'clean' || futureOnly)
|
|
171
|
+
&& (record.state === 'active' || record.recovery.status === 'pending' || futureOnly);
|
|
172
|
+
if (record.recovery)
|
|
173
|
+
record.recovery.status = recoverable ? 'pending' : 'blocked';
|
|
174
|
+
record.error = recoverable ? 'Waiting for native recovery capability and owner readiness.'
|
|
175
|
+
: record.recovery?.orderlyOnly ? 'Dirty shutdown: original native work is unconfirmed; automatic reconstruction is blocked.' : RESTART_REASON;
|
|
142
176
|
record.state = 'interrupted';
|
|
143
|
-
record.error = RESTART_REASON;
|
|
144
177
|
changed = true;
|
|
145
178
|
}
|
|
146
179
|
}
|
|
@@ -154,6 +187,195 @@ class ScheduleService {
|
|
|
154
187
|
if (changed || data === undefined)
|
|
155
188
|
await this.persist();
|
|
156
189
|
this.initialized = true;
|
|
190
|
+
}).catch(error => { this.catalogGate?.close(); this.catalogGate = undefined; throw error; });
|
|
191
|
+
}
|
|
192
|
+
/** Sequential startup/reconnect reconciliation. Never drives the native schedule clock. */
|
|
193
|
+
recover() {
|
|
194
|
+
this.checkReady();
|
|
195
|
+
if (this.recoveryRun)
|
|
196
|
+
return this.recoveryRun;
|
|
197
|
+
const run = this.operation(this.recoverWaiting());
|
|
198
|
+
this.recoveryRun = run;
|
|
199
|
+
void run.finally(() => { if (this.recoveryRun === run)
|
|
200
|
+
this.recoveryRun = undefined; }).catch(() => { });
|
|
201
|
+
return run;
|
|
202
|
+
}
|
|
203
|
+
async recoverWaiting() {
|
|
204
|
+
const candidates = await this.serial(async () => [...this.records.values()]
|
|
205
|
+
.filter(record => record.recovery?.status === 'pending' && record.state === 'interrupted').map(record => record.id));
|
|
206
|
+
// Catalog capacity is bounded, and only one native admission runs at a time.
|
|
207
|
+
for (const id of candidates.slice(0, this.maxLive)) {
|
|
208
|
+
if (this.closing || this.fault)
|
|
209
|
+
return;
|
|
210
|
+
const record = this.records.get(id);
|
|
211
|
+
if (record.cancelRequested || !this.current(record)) {
|
|
212
|
+
await this.blockRecovery(record, 'Recovery owner retired or cancellation was requested.');
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
if (record.rule.kind === 'once' && Date.parse(record.rule.at) <= this.now()) {
|
|
216
|
+
await this.blockRecovery(record, 'One-shot deadline expired; no late execution was requested.');
|
|
217
|
+
if (record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent)
|
|
218
|
+
await this.serial(async () => { delete record.recovery; await this.persist(); });
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
const input = { id, threadId: record.threadId, backend: record.backend,
|
|
222
|
+
cwd: record.cwd, prompt: record.prompt, rule: structuredClone(record.rule), ...record.execution,
|
|
223
|
+
recovery: this.checkpoint(record.recovery) };
|
|
224
|
+
if (!this.options.canRecover || !this.options.canRecover(structuredClone(record))) {
|
|
225
|
+
await this.recoveryDiagnostic(record, 'Original owner configuration or transport is not ready for recovery.');
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
const provider = this.options.provider.recoveryFor?.(input);
|
|
229
|
+
if (!provider || provider.provider !== record.recovery.provider || provider.strategy !== record.recovery.strategy
|
|
230
|
+
|| (provider.handoffPolicy ?? 'drain-all') !== (record.recovery.handoffPolicy ?? 'drain-all')) {
|
|
231
|
+
await this.recoveryDiagnostic(record, 'Native restart recovery is unsupported or provider identity changed.');
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
if (this.runtimes.has(id))
|
|
235
|
+
continue; // An unresolved/late operation retains exclusive ownership.
|
|
236
|
+
const restoreRecord = structuredClone(record);
|
|
237
|
+
const runtime = this.newRuntime();
|
|
238
|
+
runtime.completed = new Set(record.completedOccurrences);
|
|
239
|
+
const controller = new AbortController();
|
|
240
|
+
runtime.recoveryAbort = controller;
|
|
241
|
+
this.runtimes.set(id, runtime);
|
|
242
|
+
const current = () => !controller.signal.aborted && !this.closing && !this.fault && !record.cancelRequested
|
|
243
|
+
&& this.current(record) && this.options.canRecover?.(structuredClone(record)) === true;
|
|
244
|
+
const emit = (event) => {
|
|
245
|
+
if (this.runtimes.get(id) !== runtime)
|
|
246
|
+
return;
|
|
247
|
+
this.track(this.onEvent(id, structuredClone(event)));
|
|
248
|
+
};
|
|
249
|
+
let timer;
|
|
250
|
+
let admitted = false;
|
|
251
|
+
let succeeded = false;
|
|
252
|
+
try {
|
|
253
|
+
runtime.releaseAdmission = this.options.acquireRecovery?.(structuredClone(record));
|
|
254
|
+
await this.serial(async () => {
|
|
255
|
+
if (!current())
|
|
256
|
+
throw new Error('Recovery admission expired.');
|
|
257
|
+
// Consume clean proof before mutation. A crash must not replay an orderly-only restore.
|
|
258
|
+
record.state = 'creating';
|
|
259
|
+
delete record.recovery.handoff;
|
|
260
|
+
delete record.recovery.nativeDefinitionAbsent;
|
|
261
|
+
await this.persist();
|
|
262
|
+
});
|
|
263
|
+
admitted = true;
|
|
264
|
+
const admission = Promise.resolve().then(() => provider.reconcile(restoreRecord, { notBefore: this.now(), signal: controller.signal, isCurrent: current,
|
|
265
|
+
isOwnerCurrent: () => !record.cancelRequested && !this.fault && this.current(record) }, emit));
|
|
266
|
+
const adopt = async (handle) => {
|
|
267
|
+
runtime.handle = handle;
|
|
268
|
+
runtime.resolveReady(handle);
|
|
269
|
+
text(handle.nativeId);
|
|
270
|
+
await this.serial(async () => { record.nativeId = handle.nativeId; if (!this.fault)
|
|
271
|
+
await this.persist(); });
|
|
272
|
+
};
|
|
273
|
+
const receipt = admission.then(async (handle) => {
|
|
274
|
+
await adopt(handle);
|
|
275
|
+
if (!current()) {
|
|
276
|
+
await this.stop(id, runtime);
|
|
277
|
+
throw new Error('Late recovery receipt requires cleanup; no successor was admitted.');
|
|
278
|
+
}
|
|
279
|
+
if (!await handle.list() || !current())
|
|
280
|
+
throw new Error('Recovered native inventory or owner is unconfirmed.');
|
|
281
|
+
return handle;
|
|
282
|
+
}, async (error) => {
|
|
283
|
+
if (error instanceof types_1.NativeScheduleCleanupError) {
|
|
284
|
+
await adopt(error.handle);
|
|
285
|
+
if (controller.signal.aborted)
|
|
286
|
+
await this.stop(id, runtime).catch(() => { });
|
|
287
|
+
}
|
|
288
|
+
runtime.resolveReady(runtime.handle);
|
|
289
|
+
// A rejection without a handle is not proof that native admission never happened.
|
|
290
|
+
throw error;
|
|
291
|
+
});
|
|
292
|
+
const handle = await Promise.race([receipt, new Promise((_, reject) => {
|
|
293
|
+
timer = setTimeout(() => { controller.abort(); reject(new Error('Native recovery timed out; ownership retained, do not recreate.')); }, this.options.recoveryTimeoutMs ?? 30_000);
|
|
294
|
+
})]);
|
|
295
|
+
text(handle.nativeId);
|
|
296
|
+
// The provider's reconcile contract includes inventory validation; list is
|
|
297
|
+
// additionally required to verify the returned handle before activating it.
|
|
298
|
+
if (!current())
|
|
299
|
+
throw new Error('Recovered native owner is unconfirmed.');
|
|
300
|
+
await this.serial(async () => {
|
|
301
|
+
if (!current())
|
|
302
|
+
throw new Error('Recovery owner retired before receipt persistence.');
|
|
303
|
+
record.nativeId = handle.nativeId;
|
|
304
|
+
if (record.state === 'creating')
|
|
305
|
+
record.state = 'active';
|
|
306
|
+
record.recovery.status = 'eligible';
|
|
307
|
+
delete record.error;
|
|
308
|
+
await this.persist();
|
|
309
|
+
});
|
|
310
|
+
succeeded = true;
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
controller.abort();
|
|
314
|
+
if (!admitted || error instanceof types_1.NativeScheduleAbsentError) {
|
|
315
|
+
runtime.stopped = true;
|
|
316
|
+
runtime.resolveReady(undefined);
|
|
317
|
+
}
|
|
318
|
+
// Do not retry an ambiguous mutation or release its handle on a timeout.
|
|
319
|
+
await this.blockRecovery(record, errorText(error));
|
|
320
|
+
if (runtime.handle)
|
|
321
|
+
await this.stop(id, runtime).catch(() => { });
|
|
322
|
+
else if (runtime.stopped)
|
|
323
|
+
this.runtimes.delete(id);
|
|
324
|
+
}
|
|
325
|
+
finally {
|
|
326
|
+
if (timer)
|
|
327
|
+
clearTimeout(timer);
|
|
328
|
+
if (succeeded || runtime.stopped) {
|
|
329
|
+
runtime.releaseAdmission?.();
|
|
330
|
+
runtime.releaseAdmission = undefined;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
// A handoff containing only skipped/expired one-shots has nothing to
|
|
335
|
+
// reconcile. It must still release the predecessor's local input fence.
|
|
336
|
+
const released = new Set();
|
|
337
|
+
for (const record of this.records.values()) {
|
|
338
|
+
if (released.has(record.threadId) || this.closing || this.fault || this.handoffUnresolved
|
|
339
|
+
|| !this.current(record) || !this.options.canRecover?.(structuredClone(record)))
|
|
340
|
+
continue;
|
|
341
|
+
if ([...this.records.values()].some(other => other.threadId === record.threadId
|
|
342
|
+
&& (LIVE.has(other.state) || recoveryOwnsWork(other) || this.runtimes.has(other.id))))
|
|
343
|
+
continue;
|
|
344
|
+
this.options.provider.recoveryFor?.({ id: record.id, threadId: record.threadId, backend: record.backend,
|
|
345
|
+
cwd: record.cwd, prompt: record.prompt, rule: record.rule, ...record.execution })?.releaseRestartAdmission?.();
|
|
346
|
+
released.add(record.threadId);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
checkpoint(value) {
|
|
350
|
+
return { version: 1, provider: value.provider, strategy: value.strategy, identity: value.identity,
|
|
351
|
+
...(value.orderlyOnly && { orderlyOnly: true }), ...(value.futureOnly && { futureOnly: true }), ...(value.handoffPolicy && { handoffPolicy: value.handoffPolicy }), ...(value.handoff && { handoff: value.handoff }),
|
|
352
|
+
...(value.nativeDefinitionAbsent && { nativeDefinitionAbsent: true }) };
|
|
353
|
+
}
|
|
354
|
+
validateCheckpoint(value) {
|
|
355
|
+
if (value.futureOnly !== undefined && value.futureOnly !== true)
|
|
356
|
+
throw new Error('Invalid future recovery policy.');
|
|
357
|
+
if (value.version !== 1 || !['reattach', 'reconstruct'].includes(value.strategy))
|
|
358
|
+
throw new Error('Invalid recovery checkpoint.');
|
|
359
|
+
if (value.orderlyOnly !== undefined && value.orderlyOnly !== true || value.handoff !== undefined && value.handoff !== 'clean')
|
|
360
|
+
throw new Error('Invalid handoff evidence.');
|
|
361
|
+
if (value.nativeDefinitionAbsent !== undefined && (value.nativeDefinitionAbsent !== true || value.handoff !== 'clean'))
|
|
362
|
+
throw new Error('Invalid native absence evidence.');
|
|
363
|
+
if (value.handoffPolicy !== undefined && !['drain-all', 'skip-unadmitted'].includes(value.handoffPolicy))
|
|
364
|
+
throw new Error('Invalid handoff policy.');
|
|
365
|
+
text(value.provider);
|
|
366
|
+
text(value.identity, 8192);
|
|
367
|
+
}
|
|
368
|
+
async recoveryDiagnostic(record, reason) {
|
|
369
|
+
await this.serial(async () => { record.error = reason; await this.persist(); });
|
|
370
|
+
}
|
|
371
|
+
async blockRecovery(record, reason) {
|
|
372
|
+
await this.serial(async () => {
|
|
373
|
+
if (!['completed', 'cancelled'].includes(record.state))
|
|
374
|
+
record.state = 'interrupted';
|
|
375
|
+
record.recovery.status = 'blocked';
|
|
376
|
+
record.error = reason;
|
|
377
|
+
if (!this.fault)
|
|
378
|
+
await this.persist();
|
|
157
379
|
});
|
|
158
380
|
}
|
|
159
381
|
create(owner, input) {
|
|
@@ -174,7 +396,18 @@ class ScheduleService {
|
|
|
174
396
|
if (!node_path_1.default.isAbsolute(input.cwd))
|
|
175
397
|
throw new Error('Schedule working directory must be absolute.');
|
|
176
398
|
const originalOwner = ownerOf(owner);
|
|
177
|
-
const nativeInput = { ...structuredClone(input), backend };
|
|
399
|
+
const nativeInput = { ...structuredClone(input), backend, threadId: owner.threadId };
|
|
400
|
+
// Never trust a caller-supplied provider checkpoint.
|
|
401
|
+
delete nativeInput.recovery;
|
|
402
|
+
for (const value of [input.command, input.model, input.effort])
|
|
403
|
+
if (value !== undefined)
|
|
404
|
+
text(value, 4096);
|
|
405
|
+
const recovery = this.options.provider.recoveryFor?.(nativeInput);
|
|
406
|
+
if (recovery) {
|
|
407
|
+
nativeInput.recovery = { version: 1, provider: recovery.provider, strategy: recovery.strategy,
|
|
408
|
+
identity: recovery.describe(structuredClone(nativeInput), originalOwner), ...(recovery.orderlyOnly && { orderlyOnly: true }), ...(recovery.futureOnly && { futureOnly: true }), ...(recovery.handoffPolicy && { handoffPolicy: recovery.handoffPolicy }) };
|
|
409
|
+
this.validateCheckpoint(nativeInput.recovery);
|
|
410
|
+
}
|
|
178
411
|
const runtime = this.newRuntime();
|
|
179
412
|
await this.serial(async () => {
|
|
180
413
|
this.checkReady();
|
|
@@ -185,17 +418,21 @@ class ScheduleService {
|
|
|
185
418
|
if (this.records.size >= this.maxRecords)
|
|
186
419
|
throw new Error('Schedule catalog is full; history was not pruned.');
|
|
187
420
|
const liveCount = [...this.records.values()].filter(record => LIVE.has(record.state)
|
|
421
|
+
|| recoveryOwnsWork(record)
|
|
188
422
|
|| (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)).length;
|
|
189
423
|
if (liveCount >= this.maxLive)
|
|
190
424
|
throw new Error('Live native schedule limit reached.');
|
|
191
425
|
this.records.set(nativeInput.id, { ...originalOwner, id: nativeInput.id, backend,
|
|
192
426
|
cwd: nativeInput.cwd, prompt: nativeInput.prompt, rule: structuredClone(nativeInput.rule),
|
|
193
|
-
createdAt, state: 'creating' }
|
|
427
|
+
createdAt, state: 'creating', execution: { command: input.command, model: input.model, effort: input.effort },
|
|
428
|
+
...(nativeInput.recovery && { recovery: { ...nativeInput.recovery, status: 'eligible' } }) });
|
|
194
429
|
await this.persist();
|
|
195
430
|
this.runtimes.set(nativeInput.id, runtime);
|
|
196
431
|
});
|
|
197
432
|
try {
|
|
198
433
|
const handle = await this.options.provider.create(structuredClone(nativeInput), event => {
|
|
434
|
+
if (this.runtimes.get(nativeInput.id) !== runtime)
|
|
435
|
+
return;
|
|
199
436
|
const task = this.onEvent(nativeInput.id, structuredClone(event));
|
|
200
437
|
this.track(task);
|
|
201
438
|
});
|
|
@@ -205,6 +442,10 @@ class ScheduleService {
|
|
|
205
442
|
await this.serial(async () => {
|
|
206
443
|
const record = this.records.get(nativeInput.id);
|
|
207
444
|
record.nativeId = handle.nativeId;
|
|
445
|
+
if (record.recovery && handle.recoveryIdentity) {
|
|
446
|
+
text(handle.recoveryIdentity, 8192);
|
|
447
|
+
record.recovery.identity = handle.recoveryIdentity;
|
|
448
|
+
}
|
|
208
449
|
const ownerCurrent = this.current(originalOwner);
|
|
209
450
|
if (this.closing || !ownerCurrent) {
|
|
210
451
|
if (LIVE.has(record.state)) {
|
|
@@ -252,7 +493,7 @@ class ScheduleService {
|
|
|
252
493
|
/** Includes terminal jobs whose native process exit is still unconfirmed. */
|
|
253
494
|
hasActiveSchedules(threadId) {
|
|
254
495
|
return [...this.records.values()].some(record => (!threadId || record.threadId === threadId)
|
|
255
|
-
&& (LIVE.has(record.state) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
|
|
496
|
+
&& (LIVE.has(record.state) || recoveryOwnsWork(record) || (this.runtimes.has(record.id) && !this.runtimes.get(record.id).stopped)));
|
|
256
497
|
}
|
|
257
498
|
/** Card eligibility includes retryable cleanup even for a terminal catalog record. */
|
|
258
499
|
canCancel(owner, id) {
|
|
@@ -261,7 +502,7 @@ class ScheduleService {
|
|
|
261
502
|
const runtime = this.runtimes.get(id);
|
|
262
503
|
if (record.state === 'cancelled' && runtime?.handle?.cancellationKeepsRunning && !record.error)
|
|
263
504
|
return false;
|
|
264
|
-
return LIVE.has(record.state) || Boolean(runtime && !runtime.stopped);
|
|
505
|
+
return LIVE.has(record.state) || recoveryOwnsWork(record) || Boolean(runtime && !runtime.stopped);
|
|
265
506
|
}
|
|
266
507
|
/** Keep admitted native work visible when cleanup cannot safely acquire its host. */
|
|
267
508
|
async retainCleanupError(owner, id, error) {
|
|
@@ -333,11 +574,30 @@ class ScheduleService {
|
|
|
333
574
|
await this.serial(async () => {
|
|
334
575
|
const record = this.owned(owner, id);
|
|
335
576
|
runtime = this.runtimes.get(id);
|
|
577
|
+
record.cancelRequested = true;
|
|
578
|
+
runtime?.recoveryAbort?.abort();
|
|
579
|
+
if (record.recovery)
|
|
580
|
+
record.recovery.status = 'blocked';
|
|
581
|
+
await this.persist();
|
|
582
|
+
if (!runtime && record.recovery?.handoff === 'clean' && record.recovery.nativeDefinitionAbsent) {
|
|
583
|
+
record.state = 'cancelled';
|
|
584
|
+
delete record.error;
|
|
585
|
+
await this.persist();
|
|
586
|
+
}
|
|
587
|
+
if (!runtime && recoveryOwnsWork(record)) {
|
|
588
|
+
record.error = 'Cancellation intent retained; native cleanup is unconfirmed. Recovery will not recreate this job.';
|
|
589
|
+
await this.persist();
|
|
590
|
+
throw new Error(record.error);
|
|
591
|
+
}
|
|
336
592
|
if (runtime?.cancelPromise) {
|
|
337
593
|
existing = runtime.cancelPromise;
|
|
338
594
|
return;
|
|
339
595
|
}
|
|
340
596
|
if (!LIVE.has(record.state) && !runtime?.cancelling && (!runtime || runtime.stopped)) {
|
|
597
|
+
if (record.recovery && runtime?.stopped) {
|
|
598
|
+
record.state = 'cancelled';
|
|
599
|
+
delete record.error;
|
|
600
|
+
}
|
|
341
601
|
for (const [key, pending] of this.pending) {
|
|
342
602
|
if (pending.scheduleId === id && !pending.afterCancellation)
|
|
343
603
|
this.pending.delete(key);
|
|
@@ -348,6 +608,9 @@ class ScheduleService {
|
|
|
348
608
|
}
|
|
349
609
|
if (!runtime)
|
|
350
610
|
throw new Error('Native schedule handle is unavailable.');
|
|
611
|
+
if (runtime.recoveryAbort && !runtime.handle && !runtime.stopped) {
|
|
612
|
+
throw new Error('Recovery receipt is unresolved; cancellation intent retained until native cleanup can be confirmed.');
|
|
613
|
+
}
|
|
351
614
|
runtime.cancelling = true;
|
|
352
615
|
// Starting outside the state queue also permits synchronous provider callbacks.
|
|
353
616
|
operation = Promise.resolve().then(() => this.cancelNative(id, runtime));
|
|
@@ -361,24 +624,221 @@ class ScheduleService {
|
|
|
361
624
|
const ids = await this.serial(async () => [...this.pending.keys()]);
|
|
362
625
|
await this.attemptDeliveries(ids);
|
|
363
626
|
}
|
|
627
|
+
hasUnresolvedHandoff() { return this.handoffUnresolved; }
|
|
628
|
+
hasUnresolvedRecovery() {
|
|
629
|
+
return [...this.records.values()].some(record => recoveryOwnsWork(record)
|
|
630
|
+
&& record.recovery?.status !== 'eligible');
|
|
631
|
+
}
|
|
632
|
+
hasPendingRecovery() {
|
|
633
|
+
return [...this.records.values()].some(record => record.state === 'interrupted' && record.recovery?.status === 'pending');
|
|
634
|
+
}
|
|
635
|
+
/** Snapshot eligibility is only admission to a handoff, never permission to install. */
|
|
636
|
+
canSuspendForRestart() {
|
|
637
|
+
if (!this.initialized || this.closing || this.fault || this.suspending || this.recoveryRun || this.operations.size || this.pending.size)
|
|
638
|
+
return false;
|
|
639
|
+
return [...this.records.values()].filter(record => LIVE.has(record.state) || recoveryOwnsWork(record)
|
|
640
|
+
|| !!this.runtimes.get(record.id) && !this.runtimes.get(record.id).stopped).every(record => {
|
|
641
|
+
const runtime = this.runtimes.get(record.id);
|
|
642
|
+
return record.state === 'active' && !!record.recovery && !record.cancelRequested && !record.error
|
|
643
|
+
&& !!runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling;
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
/** Delete future definitions, drain native work, join owned hosts, then publish restart proof. */
|
|
647
|
+
suspendForRestart() {
|
|
648
|
+
if (this.suspension)
|
|
649
|
+
return this.suspension;
|
|
650
|
+
if (!this.canSuspendForRestart())
|
|
651
|
+
return Promise.reject(new Error('Schedules are not ready for restart handoff.'));
|
|
652
|
+
this.suspending = true;
|
|
653
|
+
this.handoffUnresolved = true;
|
|
654
|
+
const entries = [...this.runtimes].filter(([id]) => this.records.get(id)?.state === 'active');
|
|
655
|
+
const run = (async () => {
|
|
656
|
+
try {
|
|
657
|
+
// Checkpoints remain dirty throughout deletion/drain and actual host exit.
|
|
658
|
+
await this.serial(async () => {
|
|
659
|
+
for (const [id, runtime] of entries) {
|
|
660
|
+
if (runtime.handle?.recoveryIdentity) {
|
|
661
|
+
text(runtime.handle.recoveryIdentity, 8192);
|
|
662
|
+
this.records.get(id).recovery.identity = runtime.handle.recoveryIdentity;
|
|
663
|
+
}
|
|
664
|
+
delete this.records.get(id).recovery.handoff;
|
|
665
|
+
delete this.records.get(id).recovery.nativeDefinitionAbsent;
|
|
666
|
+
}
|
|
667
|
+
await this.persist();
|
|
668
|
+
});
|
|
669
|
+
const releases = await Promise.allSettled(entries.map(([, runtime]) => this.boundRecovery(runtime.handle.releaseForRestart(), 'Native schedule drain remains unresolved.')));
|
|
670
|
+
const failed = releases.find((result) => result.status === 'rejected');
|
|
671
|
+
if (failed)
|
|
672
|
+
throw failed.reason;
|
|
673
|
+
for (const [index, result] of releases.entries()) {
|
|
674
|
+
const policy = this.records.get(entries[index][0]).recovery.handoffPolicy ?? 'drain-all';
|
|
675
|
+
if (result.status === 'fulfilled' && (result.value?.policy ?? 'drain-all') !== policy)
|
|
676
|
+
throw new Error('Native handoff policy receipt changed.');
|
|
677
|
+
}
|
|
678
|
+
const absentOneShots = new Set();
|
|
679
|
+
for (const [index, result] of releases.entries())
|
|
680
|
+
if (result.status === 'fulfilled' && result.value?.onceDefinitionAlreadyAbsent) {
|
|
681
|
+
const record = this.records.get(entries[index][0]);
|
|
682
|
+
if (record.rule.kind !== 'once' || record.recovery?.handoffPolicy !== 'skip-unadmitted')
|
|
683
|
+
throw new Error('Invalid absent one-shot handoff receipt.');
|
|
684
|
+
absentOneShots.add(record.id);
|
|
685
|
+
}
|
|
686
|
+
const absent = new Set(entries.filter((_, index) => {
|
|
687
|
+
const result = releases[index];
|
|
688
|
+
return result.status === 'fulfilled' && result.value?.nativeDefinitionAbsent === true;
|
|
689
|
+
}).map(([id]) => id));
|
|
690
|
+
await this.boundRecovery((async () => {
|
|
691
|
+
while (this.tasks.size)
|
|
692
|
+
await Promise.allSettled([...this.tasks]);
|
|
693
|
+
})(), 'Scheduled result drain remains unresolved.');
|
|
694
|
+
await this.tail;
|
|
695
|
+
if (this.pending.size)
|
|
696
|
+
throw new Error('Scheduled result acknowledgement is pending.');
|
|
697
|
+
if (this.fault || entries.some(([, runtime]) => runtime.running.size))
|
|
698
|
+
throw this.fault ?? new Error('Native occurrence is still running.');
|
|
699
|
+
const finished = new Set();
|
|
700
|
+
for (const [id] of entries) {
|
|
701
|
+
const record = this.records.get(id);
|
|
702
|
+
const provider = this.options.provider.recoveryFor?.({ id, threadId: record.threadId, backend: record.backend,
|
|
703
|
+
cwd: record.cwd, rule: record.rule, prompt: record.prompt, ...record.execution });
|
|
704
|
+
if (!provider || provider.provider !== record.recovery.provider)
|
|
705
|
+
throw new Error('Native recovery provider changed during handoff.');
|
|
706
|
+
const key = `${record.threadId}:${provider.provider}`;
|
|
707
|
+
if (finished.has(key))
|
|
708
|
+
continue;
|
|
709
|
+
if (provider.orderlyOnly && !provider.finishRestart)
|
|
710
|
+
throw new Error('Owned native host exit cannot be confirmed.');
|
|
711
|
+
if (provider.finishRestart)
|
|
712
|
+
await this.boundRecovery(provider.finishRestart(), 'Owned native host exit remains unresolved.');
|
|
713
|
+
finished.add(key);
|
|
714
|
+
}
|
|
715
|
+
await this.boundRecovery((async () => {
|
|
716
|
+
while (this.tasks.size)
|
|
717
|
+
await Promise.allSettled([...this.tasks]);
|
|
718
|
+
})(), 'Scheduled result drain remains unresolved.');
|
|
719
|
+
await this.serial(async () => {
|
|
720
|
+
if (this.fault)
|
|
721
|
+
throw this.fault;
|
|
722
|
+
for (const [id, runtime] of entries) {
|
|
723
|
+
const record = this.records.get(id);
|
|
724
|
+
if (!this.current(record) || record.cancelRequested)
|
|
725
|
+
throw new Error('Schedule owner retired during handoff.');
|
|
726
|
+
if (record.state === 'active' && absentOneShots.has(id)) {
|
|
727
|
+
record.state = 'interrupted';
|
|
728
|
+
record.error = 'One-shot native definition was already absent at handoff; it will not be replayed.';
|
|
729
|
+
delete record.recovery;
|
|
730
|
+
}
|
|
731
|
+
else if (record.state === 'active') {
|
|
732
|
+
record.state = 'interrupted';
|
|
733
|
+
record.recovery.status = 'pending';
|
|
734
|
+
record.recovery.handoff = 'clean';
|
|
735
|
+
if (absent.has(id))
|
|
736
|
+
record.recovery.nativeDefinitionAbsent = true;
|
|
737
|
+
record.error = 'Native host drained and exited; waiting for automatic recovery.';
|
|
738
|
+
}
|
|
739
|
+
else if (record.state === 'interrupted' && record.rule.kind === 'once' && record.recovery?.handoffPolicy === 'skip-unadmitted' && !runtime.running.size) {
|
|
740
|
+
delete record.recovery;
|
|
741
|
+
}
|
|
742
|
+
else if (record.state !== 'completed')
|
|
743
|
+
throw new Error('Occurrence outcome changed during handoff.');
|
|
744
|
+
runtime.stopped = true;
|
|
745
|
+
}
|
|
746
|
+
await this.persist();
|
|
747
|
+
for (const [id] of entries)
|
|
748
|
+
this.runtimes.delete(id);
|
|
749
|
+
this.handoffUnresolved = false;
|
|
750
|
+
});
|
|
751
|
+
}
|
|
752
|
+
catch (error) {
|
|
753
|
+
await this.serial(async () => {
|
|
754
|
+
for (const [id] of entries) {
|
|
755
|
+
const record = this.records.get(id);
|
|
756
|
+
delete record.recovery?.handoff;
|
|
757
|
+
delete record.recovery?.nativeDefinitionAbsent;
|
|
758
|
+
record.error = `Restart handoff unconfirmed: ${errorText(error)}`;
|
|
759
|
+
}
|
|
760
|
+
if (!this.fault)
|
|
761
|
+
await this.persist();
|
|
762
|
+
});
|
|
763
|
+
throw error;
|
|
764
|
+
}
|
|
765
|
+
finally {
|
|
766
|
+
this.suspending = false;
|
|
767
|
+
}
|
|
768
|
+
})();
|
|
769
|
+
this.suspension = run;
|
|
770
|
+
void run.finally(() => { if (this.suspension === run)
|
|
771
|
+
this.suspension = undefined; }).catch(() => { });
|
|
772
|
+
return run;
|
|
773
|
+
}
|
|
774
|
+
/** Providers with durable intent must complete handoff before unrelated host teardown. */
|
|
775
|
+
requiresShutdownHandoff() {
|
|
776
|
+
return [...this.records.values()].some(record => !!record.recovery);
|
|
777
|
+
}
|
|
364
778
|
destroy() {
|
|
365
779
|
if (this.destruction)
|
|
366
780
|
return this.destruction;
|
|
367
|
-
this.closing
|
|
781
|
+
const preserve = !this.closing && this.hasActiveSchedules() && this.canSuspendForRestart();
|
|
782
|
+
if (!preserve)
|
|
783
|
+
this.closing = true;
|
|
368
784
|
return this.destruction = (async () => {
|
|
785
|
+
if (preserve)
|
|
786
|
+
await this.suspendForRestart();
|
|
787
|
+
this.closing = true;
|
|
788
|
+
for (const runtime of this.runtimes.values())
|
|
789
|
+
runtime.recoveryAbort?.abort();
|
|
369
790
|
await this.initialization?.catch(() => { });
|
|
370
791
|
let failure;
|
|
371
792
|
await this.serial(async () => {
|
|
372
793
|
for (const record of this.records.values()) {
|
|
373
794
|
if (LIVE.has(record.state)) {
|
|
795
|
+
const runtime = this.runtimes.get(record.id);
|
|
796
|
+
if (record.recovery && record.state === 'active' && !record.cancelRequested && !record.error
|
|
797
|
+
&& runtime?.handle?.releaseForRestart && !runtime.running.size && !runtime.cancelling) {
|
|
798
|
+
runtime.releasing = true;
|
|
799
|
+
record.recovery.status = 'pending';
|
|
800
|
+
record.error = 'Native restart handoff pending verification.';
|
|
801
|
+
}
|
|
802
|
+
else {
|
|
803
|
+
if (record.recovery)
|
|
804
|
+
record.recovery.status = 'blocked';
|
|
805
|
+
record.error = 'Native schedule interrupted by service shutdown.';
|
|
806
|
+
}
|
|
374
807
|
record.state = 'interrupted';
|
|
375
|
-
record.error = 'Native schedule interrupted by service shutdown.';
|
|
376
808
|
}
|
|
377
809
|
}
|
|
378
810
|
if (this.initialized && !this.fault)
|
|
379
811
|
await this.persist();
|
|
380
812
|
}).catch(error => { failure = error; });
|
|
381
|
-
const stops = await Promise.allSettled([...this.runtimes].map(([id, runtime]) =>
|
|
813
|
+
const stops = await Promise.allSettled([...this.runtimes].map(async ([id, runtime]) => {
|
|
814
|
+
if (!runtime.releasing)
|
|
815
|
+
return this.stop(id, runtime);
|
|
816
|
+
try {
|
|
817
|
+
runtime.releasePromise ??= runtime.handle.releaseForRestart();
|
|
818
|
+
let timer;
|
|
819
|
+
try {
|
|
820
|
+
await Promise.race([runtime.releasePromise, new Promise((_, reject) => {
|
|
821
|
+
timer = setTimeout(() => reject(new Error('Native shutdown handoff timed out; ownership retained.')), this.options.recoveryTimeoutMs ?? 30_000);
|
|
822
|
+
})]);
|
|
823
|
+
}
|
|
824
|
+
finally {
|
|
825
|
+
if (timer)
|
|
826
|
+
clearTimeout(timer);
|
|
827
|
+
}
|
|
828
|
+
await this.serial(async () => {
|
|
829
|
+
const record = this.records.get(id);
|
|
830
|
+
if (record.recovery?.status !== 'pending')
|
|
831
|
+
throw new Error('Occurrence raced native handoff; recovery is blocked.');
|
|
832
|
+
runtime.stopped = true;
|
|
833
|
+
record.error = 'Waiting for native recovery after orderly shutdown.';
|
|
834
|
+
await this.persist();
|
|
835
|
+
});
|
|
836
|
+
}
|
|
837
|
+
catch (error) {
|
|
838
|
+
await this.blockRecovery(this.records.get(id), errorText(error));
|
|
839
|
+
throw error;
|
|
840
|
+
}
|
|
841
|
+
}));
|
|
382
842
|
while (this.operations.size)
|
|
383
843
|
await Promise.allSettled([...this.operations]);
|
|
384
844
|
while (this.tasks.size)
|
|
@@ -388,6 +848,8 @@ class ScheduleService {
|
|
|
388
848
|
const rejected = stops.find((result) => result.status === 'rejected');
|
|
389
849
|
if (failure || rejected || this.fault)
|
|
390
850
|
throw failure ?? rejected?.reason ?? this.fault;
|
|
851
|
+
this.catalogGate?.close();
|
|
852
|
+
this.catalogGate = undefined;
|
|
391
853
|
})().catch(error => {
|
|
392
854
|
// Keep admission closed and retain handles, but allow explicit cleanup retries.
|
|
393
855
|
this.destruction = undefined;
|
|
@@ -449,17 +911,31 @@ class ScheduleService {
|
|
|
449
911
|
if (this.fault)
|
|
450
912
|
throw this.fault;
|
|
451
913
|
const record = this.records.get(id);
|
|
914
|
+
if (this.suspending && event.type === 'stopped') {
|
|
915
|
+
if (event.exitConfirmed === false)
|
|
916
|
+
throw new Error('Native handoff exit unconfirmed.');
|
|
917
|
+
runtime.stopped = true;
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
if (runtime.releasing && runtime.stopped)
|
|
921
|
+
return;
|
|
452
922
|
// Process exit evidence is independent of task state, including late exit
|
|
453
923
|
// after a failed cancellation cleanup. Never downgrade confirmed exit.
|
|
454
924
|
if (event.type === 'stopped' && event.exitConfirmed !== false) {
|
|
455
925
|
runtime.stopped = true;
|
|
926
|
+
runtime.releaseAdmission?.();
|
|
927
|
+
runtime.releaseAdmission = undefined;
|
|
456
928
|
if (record.error?.startsWith('Native process exit unconfirmed:')) {
|
|
457
929
|
delete record.error;
|
|
458
930
|
await this.persist();
|
|
459
931
|
}
|
|
460
932
|
}
|
|
933
|
+
if (runtime.releasing && event.type === 'stopped' && event.exitConfirmed !== false)
|
|
934
|
+
return;
|
|
461
935
|
const ownerCurrent = this.current(record);
|
|
462
936
|
if (this.closing || !ownerCurrent) {
|
|
937
|
+
if (record.recovery && (!ownerCurrent || event.type !== 'stopped' || event.exitConfirmed === false))
|
|
938
|
+
record.recovery.status = 'blocked';
|
|
463
939
|
if (LIVE.has(record.state)) {
|
|
464
940
|
record.state = 'interrupted';
|
|
465
941
|
record.error = 'Schedule owner or service is no longer current.';
|
|
@@ -493,15 +969,30 @@ class ScheduleService {
|
|
|
493
969
|
record.error = 'Occurrence tracking limit reached; native schedule stopped.';
|
|
494
970
|
cleanup = true;
|
|
495
971
|
}
|
|
972
|
+
else if (event.type === 'skipped') {
|
|
973
|
+
if (record.recovery?.handoffPolicy !== 'skip-unadmitted' || runtime.running.has(event.occurrenceId))
|
|
974
|
+
throw new Error('Unnegotiated occurrence skip.');
|
|
975
|
+
runtime.completed.add(event.occurrenceId);
|
|
976
|
+
record.completedOccurrences = [...runtime.completed];
|
|
977
|
+
// No delivery and no invented success. Unseen skipped wakes are not counted.
|
|
978
|
+
(record.skippedOccurrences ??= []).push(event.occurrenceId);
|
|
979
|
+
if (record.rule.kind === 'once') {
|
|
980
|
+
record.state = 'interrupted';
|
|
981
|
+
record.error = bounded(event.reason, ERROR_BYTES);
|
|
982
|
+
}
|
|
983
|
+
}
|
|
496
984
|
else if (event.type === 'started') {
|
|
497
985
|
runtime.running.add(event.occurrenceId);
|
|
498
986
|
record.state = 'running';
|
|
987
|
+
if (record.recovery)
|
|
988
|
+
record.recovery.status = 'blocked';
|
|
499
989
|
}
|
|
500
990
|
else if (event.type === 'completed') {
|
|
501
991
|
if (typeof event.success !== 'boolean')
|
|
502
992
|
throw new Error('Invalid native completion event.');
|
|
503
993
|
runtime.running.delete(event.occurrenceId);
|
|
504
994
|
runtime.completed.add(event.occurrenceId);
|
|
995
|
+
record.completedOccurrences = [...runtime.completed];
|
|
505
996
|
if (this.pending.size >= this.maxPending) {
|
|
506
997
|
record.state = 'interrupted';
|
|
507
998
|
record.error = 'Delivery outbox is full; latest result could not be retained. Native schedule stopped.';
|
|
@@ -519,8 +1010,11 @@ class ScheduleService {
|
|
|
519
1010
|
record.state = 'completed';
|
|
520
1011
|
cleanup = true;
|
|
521
1012
|
}
|
|
522
|
-
else if (record.state !== 'interrupted')
|
|
1013
|
+
else if (record.state !== 'interrupted') {
|
|
523
1014
|
record.state = runtime.running.size ? 'running' : 'active';
|
|
1015
|
+
if (record.recovery && !runtime.running.size)
|
|
1016
|
+
record.recovery.status = 'eligible';
|
|
1017
|
+
}
|
|
524
1018
|
}
|
|
525
1019
|
}
|
|
526
1020
|
else
|
|
@@ -587,9 +1081,12 @@ class ScheduleService {
|
|
|
587
1081
|
if (runtime.stopped)
|
|
588
1082
|
return Promise.resolve();
|
|
589
1083
|
if (runtime.stopPromise)
|
|
590
|
-
return runtime.stopPromise;
|
|
1084
|
+
return runtime.recoveryAbort ? this.boundRecovery(runtime.stopPromise, 'Native cleanup remains unresolved; ownership retained.') : runtime.stopPromise;
|
|
591
1085
|
const operation = (async () => {
|
|
592
1086
|
try {
|
|
1087
|
+
if (runtime.recoveryAbort?.signal.aborted && !runtime.handle && !runtime.stopped) {
|
|
1088
|
+
throw new Error('Recovery receipt is unresolved; native ownership retained.');
|
|
1089
|
+
}
|
|
593
1090
|
const handle = await runtime.ready;
|
|
594
1091
|
if (handle && !runtime.stopped)
|
|
595
1092
|
await handle.stop();
|
|
@@ -597,6 +1094,10 @@ class ScheduleService {
|
|
|
597
1094
|
// finishes. Keep ownership until its terminal stopped notification.
|
|
598
1095
|
if (!handle?.cancellationKeepsRunning || !runtime.running.size)
|
|
599
1096
|
runtime.stopped = true;
|
|
1097
|
+
if (runtime.stopped) {
|
|
1098
|
+
runtime.releaseAdmission?.();
|
|
1099
|
+
runtime.releaseAdmission = undefined;
|
|
1100
|
+
}
|
|
600
1101
|
}
|
|
601
1102
|
catch (error) {
|
|
602
1103
|
await this.serial(async () => {
|
|
@@ -610,7 +1111,19 @@ class ScheduleService {
|
|
|
610
1111
|
runtime.stopPromise = operation;
|
|
611
1112
|
void operation.finally(() => { if (!runtime.stopped)
|
|
612
1113
|
runtime.stopPromise = undefined; }).catch(() => { });
|
|
613
|
-
return operation;
|
|
1114
|
+
return runtime.recoveryAbort ? this.boundRecovery(operation, 'Native cleanup remains unresolved; ownership retained.') : operation;
|
|
1115
|
+
}
|
|
1116
|
+
async boundRecovery(operation, message) {
|
|
1117
|
+
let timer;
|
|
1118
|
+
try {
|
|
1119
|
+
return await Promise.race([operation, new Promise((_, reject) => {
|
|
1120
|
+
timer = setTimeout(() => reject(new Error(message)), this.options.recoveryTimeoutMs ?? 30_000);
|
|
1121
|
+
})]);
|
|
1122
|
+
}
|
|
1123
|
+
finally {
|
|
1124
|
+
if (timer)
|
|
1125
|
+
clearTimeout(timer);
|
|
1126
|
+
}
|
|
614
1127
|
}
|
|
615
1128
|
newRuntime() {
|
|
616
1129
|
let resolveReady;
|
|
@@ -653,7 +1166,7 @@ class ScheduleService {
|
|
|
653
1166
|
checkReady() {
|
|
654
1167
|
if (this.fault)
|
|
655
1168
|
throw this.fault;
|
|
656
|
-
if (!this.initialized || this.closing)
|
|
1169
|
+
if (!this.initialized || this.closing || this.suspending)
|
|
657
1170
|
throw new Error('Schedule service is not initialized or is closed.');
|
|
658
1171
|
}
|
|
659
1172
|
serial(operation) {
|
|
@@ -676,7 +1189,8 @@ class ScheduleService {
|
|
|
676
1189
|
}
|
|
677
1190
|
try {
|
|
678
1191
|
// JSON can expand one control byte to six bytes (for example, "\\u0001").
|
|
679
|
-
const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024)
|
|
1192
|
+
const maxBytes = (this.maxRecords + this.maxPending) * (160 * 1024)
|
|
1193
|
+
+ this.maxRecords * this.maxOccurrences * 515 + 4096;
|
|
680
1194
|
const size = (await handle.stat()).size;
|
|
681
1195
|
if (size > maxBytes)
|
|
682
1196
|
throw new Error('Schedule catalog exceeds its size limit.');
|
|
@@ -702,13 +1216,13 @@ class ScheduleService {
|
|
|
702
1216
|
}
|
|
703
1217
|
load(data) {
|
|
704
1218
|
const catalog = object(data, ['version', 'records', 'pending']);
|
|
705
|
-
if (catalog.version
|
|
1219
|
+
if (![1, 2].includes(catalog.version) || !Array.isArray(catalog.records) || catalog.records.length > this.maxRecords
|
|
706
1220
|
|| !Array.isArray(catalog.pending) || catalog.pending.length > this.maxPending)
|
|
707
1221
|
throw new Error('Invalid schedule catalog envelope.');
|
|
708
1222
|
const records = new Map();
|
|
709
1223
|
const pending = new Map();
|
|
710
1224
|
for (const value of catalog.records) {
|
|
711
|
-
const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation']);
|
|
1225
|
+
const record = object(value, ['id', 'backend', 'cwd', 'prompt', 'rule', 'createdAt', 'nativeId', 'state', 'error', 'threadId', 'openId', 'generation', 'execution', 'cancelRequested', 'recovery', 'completedOccurrences', 'skippedOccurrences']);
|
|
712
1226
|
(0, ScheduleCommand_1.validateScheduleId)(record.id);
|
|
713
1227
|
validateOwner(record);
|
|
714
1228
|
(0, ScheduleCommand_1.validateSchedulePrompt)(record.prompt);
|
|
@@ -721,6 +1235,35 @@ class ScheduleService {
|
|
|
721
1235
|
text(record.nativeId);
|
|
722
1236
|
if (record.error !== undefined && (typeof record.error !== 'string' || Buffer.byteLength(record.error) > ERROR_BYTES))
|
|
723
1237
|
throw new Error('Invalid schedule error.');
|
|
1238
|
+
if (record.cancelRequested !== undefined && record.cancelRequested !== true)
|
|
1239
|
+
throw new Error('Invalid cancellation intent.');
|
|
1240
|
+
if (record.execution !== undefined) {
|
|
1241
|
+
const execution = object(record.execution, ['command', 'model', 'effort']);
|
|
1242
|
+
for (const value of Object.values(execution))
|
|
1243
|
+
if (value !== undefined)
|
|
1244
|
+
text(value, 4096);
|
|
1245
|
+
}
|
|
1246
|
+
if (record.recovery !== undefined) {
|
|
1247
|
+
if (catalog.version !== 2)
|
|
1248
|
+
throw new Error('Recovery requires catalog version 2.');
|
|
1249
|
+
const recovery = object(record.recovery, ['version', 'provider', 'strategy', 'identity', 'status', 'orderlyOnly', 'futureOnly', 'handoff', 'nativeDefinitionAbsent', 'handoffPolicy']);
|
|
1250
|
+
this.validateCheckpoint(recovery);
|
|
1251
|
+
if (!['eligible', 'pending', 'blocked'].includes(recovery.status))
|
|
1252
|
+
throw new Error('Invalid recovery status.');
|
|
1253
|
+
}
|
|
1254
|
+
if (record.completedOccurrences !== undefined) {
|
|
1255
|
+
if (!Array.isArray(record.completedOccurrences) || record.completedOccurrences.length > this.maxOccurrences
|
|
1256
|
+
|| new Set(record.completedOccurrences).size !== record.completedOccurrences.length)
|
|
1257
|
+
throw new Error('Invalid occurrence history.');
|
|
1258
|
+
record.completedOccurrences.forEach(value => text(value));
|
|
1259
|
+
}
|
|
1260
|
+
if (record.skippedOccurrences !== undefined) {
|
|
1261
|
+
if (!Array.isArray(record.skippedOccurrences) || record.skippedOccurrences.length > this.maxOccurrences
|
|
1262
|
+
|| new Set(record.skippedOccurrences).size !== record.skippedOccurrences.length
|
|
1263
|
+
|| record.skippedOccurrences.some(value => !Array.isArray(record.completedOccurrences) || !record.completedOccurrences.includes(value))) {
|
|
1264
|
+
throw new Error('Invalid skipped occurrence history.');
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
724
1267
|
const rule = object(record.rule, ['kind', 'at', 'expression', 'timezone']);
|
|
725
1268
|
if (rule.kind === 'once')
|
|
726
1269
|
object(rule, ['kind', 'at']);
|
|
@@ -748,6 +1291,9 @@ class ScheduleService {
|
|
|
748
1291
|
this.pending = pending;
|
|
749
1292
|
}
|
|
750
1293
|
async persist() {
|
|
1294
|
+
if (!this.catalogGate)
|
|
1295
|
+
throw new Error('Schedule catalog ownership is unavailable.');
|
|
1296
|
+
this.catalogGate.verify();
|
|
751
1297
|
if (this.fault)
|
|
752
1298
|
throw this.fault;
|
|
753
1299
|
const temporary = node_path_1.default.join(node_path_1.default.dirname(this.catalogPath), `.schedule-${(0, node_crypto_1.randomUUID)()}.tmp`);
|
|
@@ -763,13 +1309,14 @@ class ScheduleService {
|
|
|
763
1309
|
}
|
|
764
1310
|
const handle = await fs.open(temporary, 'wx', 0o600);
|
|
765
1311
|
try {
|
|
766
|
-
await handle.writeFile(JSON.stringify({ version:
|
|
1312
|
+
await handle.writeFile(JSON.stringify({ version: 2, records: [...this.records.values()], pending: [...this.pending.values()] }), 'utf8');
|
|
767
1313
|
await handle.sync();
|
|
768
1314
|
}
|
|
769
1315
|
finally {
|
|
770
1316
|
await handle.close();
|
|
771
1317
|
}
|
|
772
1318
|
await fs.rename(temporary, this.catalogPath);
|
|
1319
|
+
(0, OwnershipRuntime_1.syncDirectory)(node_path_1.default.dirname(this.catalogPath));
|
|
773
1320
|
}
|
|
774
1321
|
catch (error) {
|
|
775
1322
|
this.fault = error instanceof Error ? error : new Error('Schedule catalog write failed.');
|