@pentoshi/clai 3.8.49 → 3.8.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/responder-context.js +1 -1
- package/dist/agent/responder-context.js.map +1 -1
- package/dist/agent/runner.js +1 -1
- package/dist/agent/runner.js.map +1 -1
- package/dist/agent/tool-call-parser.js +2 -1
- package/dist/agent/tool-call-parser.js.map +1 -1
- package/dist/app/adapters/current-jobs-adapter.js +4 -0
- package/dist/app/adapters/current-jobs-adapter.js.map +1 -1
- package/dist/app/controllers/job-controller.d.ts +4 -0
- package/dist/app/controllers/job-controller.js +12 -0
- package/dist/app/controllers/job-controller.js.map +1 -1
- package/dist/app/controllers/plan-controller.d.ts +1 -0
- package/dist/app/controllers/plan-controller.js +15 -0
- package/dist/app/controllers/plan-controller.js.map +1 -1
- package/dist/app/controllers/session-controller.d.ts +2 -0
- package/dist/app/controllers/session-controller.js +32 -10
- package/dist/app/controllers/session-controller.js.map +1 -1
- package/dist/app/controllers/session-responder.d.ts +19 -23
- package/dist/app/controllers/session-responder.js +157 -262
- package/dist/app/controllers/session-responder.js.map +1 -1
- package/dist/app/ports/jobs-port.d.ts +4 -0
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/repl.js +14 -68
- package/dist/repl.js.map +1 -1
- package/dist/store/history-index.d.ts +43 -0
- package/dist/store/history-index.js +197 -0
- package/dist/store/history-index.js.map +1 -0
- package/dist/store/history.d.ts +6 -5
- package/dist/store/history.js +246 -205
- package/dist/store/history.js.map +1 -1
- package/dist/store/plan.d.ts +1 -1
- package/dist/store/plan.js +17 -8
- package/dist/store/plan.js.map +1 -1
- package/dist/store/responder-settlement.d.ts +3 -0
- package/dist/store/responder-settlement.js +60 -0
- package/dist/store/responder-settlement.js.map +1 -0
- package/dist/tools/definitions.js +16 -20
- package/dist/tools/definitions.js.map +1 -1
- package/dist/tools/jobs.d.ts +25 -6
- package/dist/tools/jobs.js +251 -51
- package/dist/tools/jobs.js.map +1 -1
- package/dist/tools/registry.js +18 -23
- package/dist/tools/registry.js.map +1 -1
- package/dist/tui-v2/app/commands/picker-commands.js +9 -10
- package/dist/tui-v2/app/commands/picker-commands.js.map +1 -1
- package/dist/tui-v2/bootstrap/composition-root.js +9 -0
- package/dist/tui-v2/bootstrap/composition-root.js.map +1 -1
- package/dist/tui-v2/components/jobs/jobs-panel.js +49 -26
- package/dist/tui-v2/components/jobs/jobs-panel.js.map +1 -1
- package/dist/tui-v2/components/status/status-line.d.ts +2 -0
- package/dist/tui-v2/components/status/status-line.js +23 -2
- package/dist/tui-v2/components/status/status-line.js.map +1 -1
- package/dist/tui-v2/state/transcript-hydrate.d.ts +8 -2
- package/dist/tui-v2/state/transcript-hydrate.js +99 -9
- package/dist/tui-v2/state/transcript-hydrate.js.map +1 -1
- package/dist/version.generated.d.ts +2 -2
- package/dist/version.generated.js +2 -2
- package/package.json +1 -1
package/dist/tools/jobs.d.ts
CHANGED
|
@@ -16,6 +16,8 @@ export interface JobLinkMetadata {
|
|
|
16
16
|
parentTaskId?: string | undefined;
|
|
17
17
|
wakeOnCompletion?: boolean | undefined;
|
|
18
18
|
monitor?: JobMonitorMetadata | undefined;
|
|
19
|
+
/** Runtime listening lease that authorized this responder delegation. */
|
|
20
|
+
responderLeaseId?: string | undefined;
|
|
19
21
|
/**
|
|
20
22
|
* Opt-in delegation to the Responder: fire-and-continue, plan subtask +
|
|
21
23
|
* auto-wake on completion, and inclusion in the Responder inbox/UI. When
|
|
@@ -59,7 +61,7 @@ export interface BackgroundJob extends JobLinkMetadata {
|
|
|
59
61
|
target: string;
|
|
60
62
|
expiresAt?: string | undefined;
|
|
61
63
|
} | undefined;
|
|
62
|
-
/**
|
|
64
|
+
/** Accepted when reading legacy registries but never armed or restored. */
|
|
63
65
|
timeoutAt?: string | undefined;
|
|
64
66
|
}
|
|
65
67
|
export declare function formatJobElapsed(job: Pick<BackgroundJob, "startedAt" | "endedAt">, now?: number): string;
|
|
@@ -81,9 +83,12 @@ export interface ResponderNotification {
|
|
|
81
83
|
wakeOnCompletion: boolean;
|
|
82
84
|
responder: boolean;
|
|
83
85
|
monitor?: JobMonitorMetadata | undefined;
|
|
86
|
+
responderLeaseId?: string | undefined;
|
|
84
87
|
deliveredAt?: string | undefined;
|
|
85
88
|
analyzedAt?: string | undefined;
|
|
86
89
|
acknowledgedAt?: string | undefined;
|
|
90
|
+
archivedAt?: string | undefined;
|
|
91
|
+
settledAt?: string | undefined;
|
|
87
92
|
}
|
|
88
93
|
export type JobManagerChange = {
|
|
89
94
|
type: "job";
|
|
@@ -113,7 +118,7 @@ export interface StartJobOptions extends JobLinkMetadata {
|
|
|
113
118
|
ownerSessionId?: string | undefined;
|
|
114
119
|
profile?: string | undefined;
|
|
115
120
|
estimatedSeconds?: number | undefined;
|
|
116
|
-
/**
|
|
121
|
+
/** Legacy compatibility input. Durable jobs no longer have generic deadlines. */
|
|
117
122
|
timeoutMs?: number | undefined;
|
|
118
123
|
authorization?: {
|
|
119
124
|
target: string;
|
|
@@ -127,9 +132,19 @@ export declare class JobManager {
|
|
|
127
132
|
private processes;
|
|
128
133
|
private writers;
|
|
129
134
|
private abortControllers;
|
|
130
|
-
private
|
|
135
|
+
private authorizationTimers;
|
|
136
|
+
private responderLeases;
|
|
137
|
+
private settlementTimers;
|
|
138
|
+
private settlementAttempts;
|
|
131
139
|
private finalizations;
|
|
132
140
|
private listeners;
|
|
141
|
+
/**
|
|
142
|
+
* Tracks the first time a live job (with no ChildProcess handle) failed the
|
|
143
|
+
* liveness check, so we only finalize it as "lost" after a sustained grace
|
|
144
|
+
* window instead of on a single transient miss. Cleared as soon as the job is
|
|
145
|
+
* observed alive again or finalized.
|
|
146
|
+
*/
|
|
147
|
+
private livenessMisses;
|
|
133
148
|
private registryRetryTimer;
|
|
134
149
|
private readonly registryPath;
|
|
135
150
|
private readonly transientV2RegistryPath;
|
|
@@ -140,7 +155,11 @@ export declare class JobManager {
|
|
|
140
155
|
private emit;
|
|
141
156
|
subscribe(listener: JobManagerListener): () => void;
|
|
142
157
|
private matchesSession;
|
|
143
|
-
private
|
|
158
|
+
private clearAuthorizationTimer;
|
|
159
|
+
private scheduleAuthorizationExpiry;
|
|
160
|
+
activateResponderLease(sessionId: string): string;
|
|
161
|
+
getResponderLeaseId(sessionId: string | undefined): string | undefined;
|
|
162
|
+
releaseResponderLease(sessionId: string, leaseId?: string): void;
|
|
144
163
|
private cloneReceipt;
|
|
145
164
|
private notificationForJob;
|
|
146
165
|
private ensureCompletionNotification;
|
|
@@ -148,8 +167,7 @@ export declare class JobManager {
|
|
|
148
167
|
private finalizeJob;
|
|
149
168
|
/** Reconcile a restored process that no longer has a ChildProcess close event. */
|
|
150
169
|
private refreshJobLiveness;
|
|
151
|
-
|
|
152
|
-
private scheduleJobDeadline;
|
|
170
|
+
private scheduleTaskSettlement;
|
|
153
171
|
/**
|
|
154
172
|
* Register an in-flight tool for stall tracking only.
|
|
155
173
|
* Never appears in shell.jobs and never touches the durable registry.
|
|
@@ -166,6 +184,7 @@ export declare class JobManager {
|
|
|
166
184
|
getJob(id: string): BackgroundJob | undefined;
|
|
167
185
|
getPendingNotifications(sessionId?: string): ResponderNotification[];
|
|
168
186
|
pendingNotifications(sessionId?: string): ResponderNotification[];
|
|
187
|
+
claimNextResponderNotification(sessionId: string, leaseId: string): ResponderNotification | undefined;
|
|
169
188
|
markDelivered(notificationId: string): boolean;
|
|
170
189
|
markAnalyzed(notificationId: string): boolean;
|
|
171
190
|
acknowledge(notificationId: string): boolean;
|
package/dist/tools/jobs.js
CHANGED
|
@@ -40,10 +40,21 @@ const TRANSIENT_V2_REGISTRY_FILE = "registry-v2.json";
|
|
|
40
40
|
const MAX_DURABLE_TERMINAL_JOBS = 80;
|
|
41
41
|
/** Drop terminal durable jobs older than this on load/list. */
|
|
42
42
|
const TERMINAL_JOB_MAX_AGE_MS = 48 * 60 * 60 * 1000;
|
|
43
|
+
const ARCHIVED_UNSETTLED_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
44
|
+
const MAX_ARCHIVED_UNSETTLED_NOTIFICATIONS = 40;
|
|
43
45
|
/** Max lines shell.jobs returns to the model (running first, then recent). */
|
|
44
46
|
const LIST_JOBS_MAX_LINES = 40;
|
|
45
47
|
/** Coalesce window for chatty stdout/stderr progress persistence + UI events. */
|
|
46
48
|
const PROGRESS_FLUSH_MS = 250;
|
|
49
|
+
/**
|
|
50
|
+
* A live job with no in-process ChildProcess handle (resumed session, or a
|
|
51
|
+
* handle that was released) is only declared "lost" after failing the liveness
|
|
52
|
+
* check continuously for this long. A single transient `ps`/`kill` hiccup — or
|
|
53
|
+
* an identity read that momentarily fails — must never finalize a job that is
|
|
54
|
+
* actually still running (which produced premature "result ready", a "<1s"
|
|
55
|
+
* elapsed, and a ✗ status=lost that later flipped back to exit=0).
|
|
56
|
+
*/
|
|
57
|
+
const LIVENESS_LOST_GRACE_MS = 8_000;
|
|
47
58
|
/**
|
|
48
59
|
* Number of trailing bytes in `buf` that form an incomplete multi-byte UTF-8
|
|
49
60
|
* sequence (a lead byte whose continuation bytes were cut off by the read
|
|
@@ -73,8 +84,11 @@ function processAlive(pid) {
|
|
|
73
84
|
process.kill(pid, 0);
|
|
74
85
|
return true;
|
|
75
86
|
}
|
|
76
|
-
catch {
|
|
77
|
-
|
|
87
|
+
catch (error) {
|
|
88
|
+
// EPERM means the process EXISTS but we lack permission to signal it
|
|
89
|
+
// (common for detached jobs in another process group) — that is alive, not
|
|
90
|
+
// gone. Only ESRCH ("no such process") proves it is truly dead.
|
|
91
|
+
return error.code === "EPERM";
|
|
78
92
|
}
|
|
79
93
|
}
|
|
80
94
|
function processIdentity(pid) {
|
|
@@ -229,9 +243,19 @@ export class JobManager {
|
|
|
229
243
|
processes = new Map();
|
|
230
244
|
writers = new Map();
|
|
231
245
|
abortControllers = new Map();
|
|
232
|
-
|
|
246
|
+
authorizationTimers = new Map();
|
|
247
|
+
responderLeases = new Map();
|
|
248
|
+
settlementTimers = new Map();
|
|
249
|
+
settlementAttempts = new Map();
|
|
233
250
|
finalizations = new Map();
|
|
234
251
|
listeners = new Set();
|
|
252
|
+
/**
|
|
253
|
+
* Tracks the first time a live job (with no ChildProcess handle) failed the
|
|
254
|
+
* liveness check, so we only finalize it as "lost" after a sustained grace
|
|
255
|
+
* window instead of on a single transient miss. Cleared as soon as the job is
|
|
256
|
+
* observed alive again or finalized.
|
|
257
|
+
*/
|
|
258
|
+
livenessMisses = new Map();
|
|
235
259
|
registryRetryTimer;
|
|
236
260
|
registryPath;
|
|
237
261
|
transientV2RegistryPath;
|
|
@@ -269,11 +293,60 @@ export class JobManager {
|
|
|
269
293
|
return true;
|
|
270
294
|
return job.ownerSessionId === sessionId;
|
|
271
295
|
}
|
|
272
|
-
|
|
273
|
-
const timer = this.
|
|
296
|
+
clearAuthorizationTimer(id) {
|
|
297
|
+
const timer = this.authorizationTimers.get(id);
|
|
274
298
|
if (timer)
|
|
275
299
|
clearTimeout(timer);
|
|
276
|
-
this.
|
|
300
|
+
this.authorizationTimers.delete(id);
|
|
301
|
+
}
|
|
302
|
+
scheduleAuthorizationExpiry(job) {
|
|
303
|
+
this.clearAuthorizationTimer(job.id);
|
|
304
|
+
const raw = job.authorization?.expiresAt;
|
|
305
|
+
if (!raw || !this.isLive(job))
|
|
306
|
+
return;
|
|
307
|
+
const expiresAt = Date.parse(raw);
|
|
308
|
+
if (!Number.isFinite(expiresAt))
|
|
309
|
+
return;
|
|
310
|
+
const timer = setTimeout(() => {
|
|
311
|
+
this.authorizationTimers.delete(job.id);
|
|
312
|
+
this.refreshJobLiveness(job);
|
|
313
|
+
if (this.isLive(job))
|
|
314
|
+
void this.stopJob(job.id, { graceMs: 1_000 });
|
|
315
|
+
}, Math.max(0, expiresAt - Date.now()));
|
|
316
|
+
timer.unref?.();
|
|
317
|
+
this.authorizationTimers.set(job.id, timer);
|
|
318
|
+
}
|
|
319
|
+
activateResponderLease(sessionId) {
|
|
320
|
+
const current = this.responderLeases.get(sessionId);
|
|
321
|
+
if (current)
|
|
322
|
+
return current;
|
|
323
|
+
const leaseId = randomUUID();
|
|
324
|
+
this.responderLeases.set(sessionId, leaseId);
|
|
325
|
+
this.emit({ type: "job", jobId: `responder:${sessionId}` });
|
|
326
|
+
return leaseId;
|
|
327
|
+
}
|
|
328
|
+
getResponderLeaseId(sessionId) {
|
|
329
|
+
return sessionId ? this.responderLeases.get(sessionId) : undefined;
|
|
330
|
+
}
|
|
331
|
+
releaseResponderLease(sessionId, leaseId) {
|
|
332
|
+
const current = this.responderLeases.get(sessionId);
|
|
333
|
+
if (!current || (leaseId && current !== leaseId))
|
|
334
|
+
return;
|
|
335
|
+
this.responderLeases.delete(sessionId);
|
|
336
|
+
const archivedAt = new Date().toISOString();
|
|
337
|
+
let changed = false;
|
|
338
|
+
for (const notification of this.notifications.values()) {
|
|
339
|
+
if (notification.ownerSessionId === sessionId &&
|
|
340
|
+
notification.responderLeaseId === current &&
|
|
341
|
+
!notification.acknowledgedAt &&
|
|
342
|
+
!notification.archivedAt) {
|
|
343
|
+
notification.archivedAt = archivedAt;
|
|
344
|
+
changed = true;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
if (changed && !this.persistSync())
|
|
348
|
+
this.scheduleRegistryRetry();
|
|
349
|
+
this.emit({ type: "job", jobId: `responder:${sessionId}` });
|
|
277
350
|
}
|
|
278
351
|
cloneReceipt(receipt) {
|
|
279
352
|
return { ...receipt, chunks: [...receipt.chunks] };
|
|
@@ -301,6 +374,9 @@ export class JobManager {
|
|
|
301
374
|
JSON.stringify(existing.stderrArtifact) !==
|
|
302
375
|
JSON.stringify(stderrArtifact)));
|
|
303
376
|
const reopenSettlement = Boolean(existing?.acknowledgedAt && completionChanged);
|
|
377
|
+
const responderLeaseId = job.responderLeaseId ?? existing?.responderLeaseId;
|
|
378
|
+
const leaseIsActive = Boolean(responderLeaseId &&
|
|
379
|
+
this.responderLeases.get(job.ownerSessionId) === responderLeaseId);
|
|
304
380
|
const notification = {
|
|
305
381
|
id: existing?.id ??
|
|
306
382
|
`completion:${createHash("sha256").update(`${job.ownerSessionId}\0${job.id}\0${job.startedAt}`).digest("hex").slice(0, 24)}`,
|
|
@@ -320,6 +396,7 @@ export class JobManager {
|
|
|
320
396
|
...(job.exitCode !== undefined ? { exitCode: job.exitCode } : {}),
|
|
321
397
|
...(job.signal !== undefined ? { signal: job.signal } : {}),
|
|
322
398
|
...(job.monitor !== undefined ? { monitor: job.monitor } : {}),
|
|
399
|
+
...(responderLeaseId ? { responderLeaseId } : {}),
|
|
323
400
|
...(existing?.deliveredAt ? { deliveredAt: existing.deliveredAt } : {}),
|
|
324
401
|
...((existing?.analyzedAt ?? existing?.acknowledgedAt)
|
|
325
402
|
? { analyzedAt: existing?.analyzedAt ?? existing?.acknowledgedAt }
|
|
@@ -327,6 +404,12 @@ export class JobManager {
|
|
|
327
404
|
...(!reopenSettlement && existing?.acknowledgedAt
|
|
328
405
|
? { acknowledgedAt: existing.acknowledgedAt }
|
|
329
406
|
: {}),
|
|
407
|
+
...(existing?.settledAt ? { settledAt: existing.settledAt } : {}),
|
|
408
|
+
...(!leaseIsActive
|
|
409
|
+
? { archivedAt: existing?.archivedAt ?? endedAt }
|
|
410
|
+
: existing?.archivedAt
|
|
411
|
+
? { archivedAt: existing.archivedAt }
|
|
412
|
+
: {}),
|
|
330
413
|
};
|
|
331
414
|
const updated = Boolean(existing && JSON.stringify(existing) !== JSON.stringify(notification));
|
|
332
415
|
this.notifications.set(notification.id, notification);
|
|
@@ -347,6 +430,7 @@ export class JobManager {
|
|
|
347
430
|
const correctsLost = job.status === "lost" && status !== "lost";
|
|
348
431
|
if (this.isTerminalStatus(job.status) && !correctsLost) {
|
|
349
432
|
const completion = this.ensureCompletionNotification(job);
|
|
433
|
+
this.scheduleTaskSettlement(job);
|
|
350
434
|
if ((!completion.created && !completion.updated) ||
|
|
351
435
|
!completion.notification) {
|
|
352
436
|
return true;
|
|
@@ -363,7 +447,8 @@ export class JobManager {
|
|
|
363
447
|
return persisted;
|
|
364
448
|
}
|
|
365
449
|
const finalization = (async () => {
|
|
366
|
-
this.
|
|
450
|
+
this.clearAuthorizationTimer(job.id);
|
|
451
|
+
this.livenessMisses.delete(job.id);
|
|
367
452
|
const streamsFlushed = await this.closeWriters(job.id);
|
|
368
453
|
job.status = streamsFlushed ? status : "failed";
|
|
369
454
|
if (details.exitCode !== undefined)
|
|
@@ -377,6 +462,7 @@ export class JobManager {
|
|
|
377
462
|
this.abortControllers.delete(job.id);
|
|
378
463
|
this.processes.delete(job.id);
|
|
379
464
|
const completion = this.ensureCompletionNotification(job);
|
|
465
|
+
this.scheduleTaskSettlement(job);
|
|
380
466
|
this.pruneTerminalJobs();
|
|
381
467
|
const persisted = this.persistSync();
|
|
382
468
|
if (!persisted)
|
|
@@ -402,19 +488,40 @@ export class JobManager {
|
|
|
402
488
|
}
|
|
403
489
|
/** Reconcile a restored process that no longer has a ChildProcess close event. */
|
|
404
490
|
refreshJobLiveness(job) {
|
|
405
|
-
if (!this.isLive(job) || this.processes.has(job.id))
|
|
491
|
+
if (!this.isLive(job) || this.processes.has(job.id)) {
|
|
492
|
+
this.livenessMisses.delete(job.id);
|
|
406
493
|
return;
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
494
|
+
}
|
|
495
|
+
// Primary signal is the pid itself. If the process is alive we keep the job
|
|
496
|
+
// running and only ever declare "lost" on a PROVEN pid reuse (both the
|
|
497
|
+
// stored and the current identity are present AND differ). An identity read
|
|
498
|
+
// that fails or returns nothing is "unknown", never "dead" — treating it as
|
|
499
|
+
// dead is exactly what flipped live jobs to lost mid-run.
|
|
500
|
+
if (processAlive(job.pid)) {
|
|
501
|
+
const identity = processIdentity(job.pid);
|
|
502
|
+
const provenReuse = Boolean(identity && job.processIdentity && identity !== job.processIdentity);
|
|
503
|
+
if (!provenReuse) {
|
|
504
|
+
this.livenessMisses.delete(job.id);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
// The process looks gone (or is a proven pid reuse). Require the miss to
|
|
509
|
+
// persist across a grace window before finalizing — a single transient
|
|
510
|
+
// hiccup must not kill a job whose real close event is still in flight.
|
|
511
|
+
const nowMs = Date.now();
|
|
512
|
+
const firstMiss = this.livenessMisses.get(job.id);
|
|
513
|
+
if (firstMiss === undefined) {
|
|
514
|
+
this.livenessMisses.set(job.id, nowMs);
|
|
412
515
|
return;
|
|
413
516
|
}
|
|
517
|
+
if (nowMs - firstMiss < LIVENESS_LOST_GRACE_MS)
|
|
518
|
+
return;
|
|
519
|
+
this.livenessMisses.delete(job.id);
|
|
414
520
|
job.status = "lost";
|
|
415
521
|
job.endedAt = new Date().toISOString();
|
|
416
|
-
this.
|
|
522
|
+
this.clearAuthorizationTimer(job.id);
|
|
417
523
|
const completion = this.ensureCompletionNotification(job);
|
|
524
|
+
this.scheduleTaskSettlement(job);
|
|
418
525
|
this.pruneTerminalJobs();
|
|
419
526
|
if (!this.persistSync())
|
|
420
527
|
this.scheduleRegistryRetry();
|
|
@@ -428,22 +535,54 @@ export class JobManager {
|
|
|
428
535
|
});
|
|
429
536
|
}
|
|
430
537
|
}
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
this.clearJobDeadline(job.id);
|
|
434
|
-
if (!job.timeoutAt || !this.isLive(job))
|
|
435
|
-
return;
|
|
436
|
-
const timeoutAt = Date.parse(job.timeoutAt);
|
|
437
|
-
if (!Number.isFinite(timeoutAt))
|
|
538
|
+
scheduleTaskSettlement(job, delayMs = 0) {
|
|
539
|
+
if (!job.responder || !this.isTerminalStatus(job.status))
|
|
438
540
|
return;
|
|
541
|
+
const existing = this.settlementTimers.get(job.id);
|
|
542
|
+
if (existing)
|
|
543
|
+
clearTimeout(existing);
|
|
439
544
|
const timer = setTimeout(() => {
|
|
440
|
-
this.
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
545
|
+
this.settlementTimers.delete(job.id);
|
|
546
|
+
void import("../store/responder-settlement.js")
|
|
547
|
+
.then(({ settleResponderJob }) => settleResponderJob({ ...job }))
|
|
548
|
+
.then((result) => {
|
|
549
|
+
if (result === "applied" || result === "noop") {
|
|
550
|
+
this.settlementAttempts.delete(job.id);
|
|
551
|
+
const notification = this.notificationForJob(job.id);
|
|
552
|
+
if (notification && !notification.settledAt) {
|
|
553
|
+
notification.settledAt = new Date().toISOString();
|
|
554
|
+
if (!this.persistSync())
|
|
555
|
+
this.scheduleRegistryRetry();
|
|
556
|
+
this.emit({
|
|
557
|
+
type: "notification",
|
|
558
|
+
jobId: job.id,
|
|
559
|
+
notificationId: notification.id,
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
const attempt = (this.settlementAttempts.get(job.id) ?? 0) + 1;
|
|
565
|
+
this.settlementAttempts.set(job.id, attempt);
|
|
566
|
+
if (attempt < 8) {
|
|
567
|
+
this.scheduleTaskSettlement(job, Math.min(5_000, 50 * 2 ** attempt));
|
|
568
|
+
}
|
|
569
|
+
else {
|
|
570
|
+
this.settlementAttempts.delete(job.id);
|
|
571
|
+
}
|
|
572
|
+
})
|
|
573
|
+
.catch(() => {
|
|
574
|
+
const attempt = (this.settlementAttempts.get(job.id) ?? 0) + 1;
|
|
575
|
+
this.settlementAttempts.set(job.id, attempt);
|
|
576
|
+
if (attempt < 8) {
|
|
577
|
+
this.scheduleTaskSettlement(job, Math.min(5_000, 50 * 2 ** attempt));
|
|
578
|
+
}
|
|
579
|
+
else {
|
|
580
|
+
this.settlementAttempts.delete(job.id);
|
|
581
|
+
}
|
|
582
|
+
});
|
|
583
|
+
}, delayMs);
|
|
445
584
|
timer.unref?.();
|
|
446
|
-
this.
|
|
585
|
+
this.settlementTimers.set(job.id, timer);
|
|
447
586
|
}
|
|
448
587
|
/**
|
|
449
588
|
* Register an in-flight tool for stall tracking only.
|
|
@@ -476,7 +615,7 @@ export class JobManager {
|
|
|
476
615
|
this.abortControllers.delete(id);
|
|
477
616
|
this.processes.delete(id);
|
|
478
617
|
if (!this.isLive(job))
|
|
479
|
-
this.
|
|
618
|
+
this.clearAuthorizationTimer(id);
|
|
480
619
|
if (!this.isDurable(job) && !this.isLive(job)) {
|
|
481
620
|
this.jobs.delete(id);
|
|
482
621
|
this.emit({ type: "job", jobId: id });
|
|
@@ -558,10 +697,8 @@ export class JobManager {
|
|
|
558
697
|
...(options?.wakeOnCompletion !== undefined ? { wakeOnCompletion: options.wakeOnCompletion } : {}),
|
|
559
698
|
...(options?.responder !== undefined ? { responder: options.responder } : {}),
|
|
560
699
|
...(monitor !== undefined ? { monitor } : {}),
|
|
561
|
-
...(options?.
|
|
562
|
-
? {
|
|
563
|
-
timeoutAt: new Date(Date.now() + Math.max(1_000, Math.floor(options.timeoutMs))).toISOString(),
|
|
564
|
-
}
|
|
700
|
+
...(options?.responderLeaseId
|
|
701
|
+
? { responderLeaseId: options.responderLeaseId }
|
|
565
702
|
: {}),
|
|
566
703
|
...(options?.authorization ? { authorization: options.authorization } : {}),
|
|
567
704
|
};
|
|
@@ -670,12 +807,7 @@ export class JobManager {
|
|
|
670
807
|
job.processIdentity = processIdentity(child.pid);
|
|
671
808
|
this.processes.set(id, child);
|
|
672
809
|
this.writers.set(id, { stdout, stderr });
|
|
673
|
-
|
|
674
|
-
const expiryTimer = Number.isFinite(expiresAt) && expiresAt > Date.now()
|
|
675
|
-
? setTimeout(() => { void this.stopJob(id, { graceMs: 1_000 }); }, expiresAt - Date.now())
|
|
676
|
-
: undefined;
|
|
677
|
-
expiryTimer?.unref?.();
|
|
678
|
-
this.scheduleJobDeadline(job);
|
|
810
|
+
this.scheduleAuthorizationExpiry(job);
|
|
679
811
|
let lastProgressFlush = 0;
|
|
680
812
|
let progressDirty = false;
|
|
681
813
|
let progressTimer;
|
|
@@ -725,8 +857,6 @@ export class JobManager {
|
|
|
725
857
|
child.stdout?.on("data", (chunk) => recordOutput(stdout, chunk));
|
|
726
858
|
child.stderr?.on("data", (chunk) => recordOutput(stderr, chunk));
|
|
727
859
|
child.on("close", (code, signal) => {
|
|
728
|
-
if (expiryTimer)
|
|
729
|
-
clearTimeout(expiryTimer);
|
|
730
860
|
stopProgressFlush();
|
|
731
861
|
const status = signal ? "killed" : code === 0 ? "exited" : "failed";
|
|
732
862
|
void this.finalizeJob(job, status, {
|
|
@@ -735,8 +865,6 @@ export class JobManager {
|
|
|
735
865
|
});
|
|
736
866
|
});
|
|
737
867
|
child.on("error", (error) => {
|
|
738
|
-
if (expiryTimer)
|
|
739
|
-
clearTimeout(expiryTimer);
|
|
740
868
|
stopProgressFlush();
|
|
741
869
|
const code = error.code ?? "UNKNOWN";
|
|
742
870
|
try {
|
|
@@ -926,6 +1054,47 @@ export class JobManager {
|
|
|
926
1054
|
pendingNotifications(sessionId) {
|
|
927
1055
|
return this.getPendingNotifications(sessionId);
|
|
928
1056
|
}
|
|
1057
|
+
claimNextResponderNotification(sessionId, leaseId) {
|
|
1058
|
+
if (this.responderLeases.get(sessionId) !== leaseId)
|
|
1059
|
+
return undefined;
|
|
1060
|
+
const archivedAt = new Date().toISOString();
|
|
1061
|
+
let archived = false;
|
|
1062
|
+
for (const notification of this.notifications.values()) {
|
|
1063
|
+
if (notification.ownerSessionId === sessionId &&
|
|
1064
|
+
!notification.acknowledgedAt &&
|
|
1065
|
+
notification.responderLeaseId !== leaseId &&
|
|
1066
|
+
!notification.archivedAt) {
|
|
1067
|
+
notification.archivedAt = archivedAt;
|
|
1068
|
+
archived = true;
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
const notification = [...this.notifications.values()]
|
|
1072
|
+
.filter((candidate) => candidate.ownerSessionId === sessionId &&
|
|
1073
|
+
candidate.responderLeaseId === leaseId &&
|
|
1074
|
+
!candidate.archivedAt &&
|
|
1075
|
+
!candidate.analyzedAt &&
|
|
1076
|
+
!candidate.acknowledgedAt)
|
|
1077
|
+
.sort((left, right) => left.createdAt.localeCompare(right.createdAt))[0];
|
|
1078
|
+
if (!notification) {
|
|
1079
|
+
if (archived && !this.persistSync())
|
|
1080
|
+
this.scheduleRegistryRetry();
|
|
1081
|
+
return undefined;
|
|
1082
|
+
}
|
|
1083
|
+
const previous = notification.deliveredAt;
|
|
1084
|
+
notification.deliveredAt ??= new Date().toISOString();
|
|
1085
|
+
if (!this.persistSync()) {
|
|
1086
|
+
notification.deliveredAt = previous;
|
|
1087
|
+
return undefined;
|
|
1088
|
+
}
|
|
1089
|
+
if (archived)
|
|
1090
|
+
this.persistSync();
|
|
1091
|
+
this.emit({
|
|
1092
|
+
type: "notification",
|
|
1093
|
+
jobId: notification.jobId,
|
|
1094
|
+
notificationId: notification.id,
|
|
1095
|
+
});
|
|
1096
|
+
return notification;
|
|
1097
|
+
}
|
|
929
1098
|
markDelivered(notificationId) {
|
|
930
1099
|
const notification = this.notifications.get(notificationId);
|
|
931
1100
|
if (!notification)
|
|
@@ -1008,6 +1177,7 @@ export class JobManager {
|
|
|
1008
1177
|
if (metadata.responder !== undefined)
|
|
1009
1178
|
job.responder = metadata.responder;
|
|
1010
1179
|
const completion = this.ensureCompletionNotification(job);
|
|
1180
|
+
this.scheduleTaskSettlement(job);
|
|
1011
1181
|
if (completion.notification) {
|
|
1012
1182
|
if (metadata.taskId !== undefined)
|
|
1013
1183
|
completion.notification.taskId = metadata.taskId;
|
|
@@ -1168,7 +1338,7 @@ export class JobManager {
|
|
|
1168
1338
|
const childClose = child
|
|
1169
1339
|
? new Promise((resolve) => child.once("close", () => resolve()))
|
|
1170
1340
|
: undefined;
|
|
1171
|
-
this.
|
|
1341
|
+
this.clearAuthorizationTimer(id);
|
|
1172
1342
|
const previousStatus = job.status;
|
|
1173
1343
|
const previousWakeOnCompletion = job.wakeOnCompletion;
|
|
1174
1344
|
if (options?.suppressWake)
|
|
@@ -1177,8 +1347,7 @@ export class JobManager {
|
|
|
1177
1347
|
if (!this.persistSync()) {
|
|
1178
1348
|
job.status = previousStatus;
|
|
1179
1349
|
job.wakeOnCompletion = previousWakeOnCompletion;
|
|
1180
|
-
|
|
1181
|
-
this.scheduleJobDeadline(job);
|
|
1350
|
+
this.scheduleAuthorizationExpiry(job);
|
|
1182
1351
|
return { ok: false, output: `Failed to persist stop state for job "${id}"; no signal was sent.`, exitCode: 1 };
|
|
1183
1352
|
}
|
|
1184
1353
|
this.emit({ type: "job", jobId: id });
|
|
@@ -1203,8 +1372,7 @@ export class JobManager {
|
|
|
1203
1372
|
};
|
|
1204
1373
|
const restoreRunning = () => {
|
|
1205
1374
|
job.status = "running";
|
|
1206
|
-
|
|
1207
|
-
this.scheduleJobDeadline(job);
|
|
1375
|
+
this.scheduleAuthorizationExpiry(job);
|
|
1208
1376
|
this.persistSync();
|
|
1209
1377
|
this.emit({ type: "job", jobId: id });
|
|
1210
1378
|
};
|
|
@@ -1374,6 +1542,24 @@ export class JobManager {
|
|
|
1374
1542
|
/** Drop stale terminal durable jobs and all leftover ephemeral rows. */
|
|
1375
1543
|
pruneTerminalJobs() {
|
|
1376
1544
|
const now = Date.now();
|
|
1545
|
+
const archivedUnsettled = [...this.notifications.values()]
|
|
1546
|
+
.filter((notification) => Boolean(notification.archivedAt) &&
|
|
1547
|
+
!notification.acknowledgedAt &&
|
|
1548
|
+
!notification.settledAt)
|
|
1549
|
+
.sort((left, right) => (right.archivedAt ?? right.createdAt).localeCompare(left.archivedAt ?? left.createdAt));
|
|
1550
|
+
for (const [index, notification] of archivedUnsettled.entries()) {
|
|
1551
|
+
const archivedAt = Date.parse(notification.archivedAt ?? notification.endedAt ?? notification.createdAt);
|
|
1552
|
+
const expired = Number.isFinite(archivedAt) &&
|
|
1553
|
+
now - archivedAt > ARCHIVED_UNSETTLED_MAX_AGE_MS;
|
|
1554
|
+
if (index < MAX_ARCHIVED_UNSETTLED_NOTIFICATIONS && !expired)
|
|
1555
|
+
continue;
|
|
1556
|
+
this.notifications.delete(notification.id);
|
|
1557
|
+
const timer = this.settlementTimers.get(notification.jobId);
|
|
1558
|
+
if (timer)
|
|
1559
|
+
clearTimeout(timer);
|
|
1560
|
+
this.settlementTimers.delete(notification.jobId);
|
|
1561
|
+
this.settlementAttempts.delete(notification.jobId);
|
|
1562
|
+
}
|
|
1377
1563
|
for (const [id, notification] of this.notifications) {
|
|
1378
1564
|
const job = this.jobs.get(notification.jobId);
|
|
1379
1565
|
if (!job ||
|
|
@@ -1392,8 +1578,11 @@ export class JobManager {
|
|
|
1392
1578
|
if (this.isLive(job))
|
|
1393
1579
|
continue;
|
|
1394
1580
|
const notification = this.notificationForJob(job.id);
|
|
1395
|
-
if (notification &&
|
|
1581
|
+
if (notification &&
|
|
1582
|
+
!notification.acknowledgedAt &&
|
|
1583
|
+
!(notification.archivedAt && notification.settledAt)) {
|
|
1396
1584
|
continue;
|
|
1585
|
+
}
|
|
1397
1586
|
const ended = job.endedAt ? Date.parse(job.endedAt) : Date.parse(job.startedAt);
|
|
1398
1587
|
if (Number.isFinite(ended) && now - ended > TERMINAL_JOB_MAX_AGE_MS) {
|
|
1399
1588
|
this.jobs.delete(id);
|
|
@@ -1433,6 +1622,9 @@ export class JobManager {
|
|
|
1433
1622
|
if (notifiedJobs.has(notification.jobId))
|
|
1434
1623
|
continue;
|
|
1435
1624
|
notifiedJobs.add(notification.jobId);
|
|
1625
|
+
if (!notification.acknowledgedAt && !notification.archivedAt) {
|
|
1626
|
+
notification.archivedAt = new Date().toISOString();
|
|
1627
|
+
}
|
|
1436
1628
|
this.notifications.set(notification.id, notification);
|
|
1437
1629
|
}
|
|
1438
1630
|
}
|
|
@@ -1441,8 +1633,14 @@ export class JobManager {
|
|
|
1441
1633
|
continue;
|
|
1442
1634
|
job.kind = "durable";
|
|
1443
1635
|
if (["starting", "running", "stopping"].includes(job.status)) {
|
|
1444
|
-
const
|
|
1445
|
-
|
|
1636
|
+
const alive = processAlive(job.pid);
|
|
1637
|
+
const identity = alive ? processIdentity(job.pid) : undefined;
|
|
1638
|
+
// Only declare lost when the process is truly gone, or when we can
|
|
1639
|
+
// PROVE a pid reuse (stored + current identity both known and
|
|
1640
|
+
// different). An alive pid whose identity is unknown/unreadable is
|
|
1641
|
+
// kept running — never kill a live process over a failed `ps` read.
|
|
1642
|
+
const provenReuse = Boolean(identity && job.processIdentity && identity !== job.processIdentity);
|
|
1643
|
+
if (!alive || provenReuse) {
|
|
1446
1644
|
job.status = "lost";
|
|
1447
1645
|
job.endedAt = new Date().toISOString();
|
|
1448
1646
|
}
|
|
@@ -1451,9 +1649,11 @@ export class JobManager {
|
|
|
1451
1649
|
job.heartbeatAt = new Date().toISOString();
|
|
1452
1650
|
}
|
|
1453
1651
|
}
|
|
1652
|
+
delete job.timeoutAt;
|
|
1454
1653
|
this.jobs.set(job.id, job);
|
|
1455
1654
|
this.ensureCompletionNotification(job);
|
|
1456
|
-
this.
|
|
1655
|
+
this.scheduleAuthorizationExpiry(job);
|
|
1656
|
+
this.scheduleTaskSettlement(job);
|
|
1457
1657
|
}
|
|
1458
1658
|
for (const [id, notification] of this.notifications) {
|
|
1459
1659
|
const job = this.jobs.get(notification.jobId);
|