@ours.network/fleet 0.17.1 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -2
- package/dist/application/role-removal-service.js +1 -1
- package/dist/application/session-control.d.ts +14 -10
- package/dist/application/session-control.js +14 -3
- package/dist/atomic-file.d.ts +7 -1
- package/dist/atomic-file.js +33 -5
- package/dist/build-info.json +10 -0
- package/dist/capabilities.d.ts +20 -0
- package/dist/capabilities.js +21 -0
- package/dist/cli.js +98 -10
- package/dist/config.d.ts +9 -2
- package/dist/config.js +16 -2
- package/dist/creation.d.ts +16 -0
- package/dist/creation.js +28 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +70 -4
- package/dist/doctor.d.ts +5 -0
- package/dist/doctor.js +87 -2
- package/dist/harness/acp-agent.d.ts +3 -0
- package/dist/harness/acp-agent.js +4 -1
- package/dist/harness/codex-app-server-proxy.d.ts +4 -0
- package/dist/harness/codex-app-server-proxy.js +133 -0
- package/dist/harness/codex.js +79 -11
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -0
- package/dist/loops/manager.d.ts +42 -1
- package/dist/loops/manager.js +115 -16
- package/dist/loops/state.d.ts +46 -2
- package/dist/loops/state.js +81 -3
- package/dist/monitor.d.ts +21 -0
- package/dist/monitor.js +42 -0
- package/dist/ops.d.ts +6 -0
- package/dist/ops.js +46 -1
- package/dist/owner-channel/channel.d.ts +18 -2
- package/dist/owner-channel/channel.js +146 -2
- package/dist/owner-channel/commands.d.ts +2 -2
- package/dist/owner-channel/commands.js +7 -2
- package/dist/owner-channel/notices.d.ts +2 -0
- package/dist/owner-channel/notices.js +3 -0
- package/dist/provenance.d.ts +77 -0
- package/dist/provenance.js +283 -0
- package/dist/runner.d.ts +7 -1
- package/dist/runner.js +100 -14
- package/dist/session/acp.d.ts +40 -4
- package/dist/session/acp.js +157 -30
- package/dist/session/arbiter.d.ts +28 -2
- package/dist/session/arbiter.js +75 -4
- package/dist/session/control.js +12 -6
- package/dist/session/event-log.d.ts +109 -0
- package/dist/session/event-log.js +247 -0
- package/dist/session/events.d.ts +21 -0
- package/dist/session/events.js +105 -26
- package/dist/session/tmux.d.ts +3 -2
- package/dist/session/tmux.js +2 -0
- package/dist/session/types.d.ts +39 -2
- package/dist/session/types.js +11 -1
- package/dist/spawn.d.ts +3 -3
- package/dist/spawn.js +40 -14
- package/dist/temp-lifecycle.d.ts +62 -0
- package/dist/temp-lifecycle.js +437 -0
- package/package.json +5 -3
package/dist/session/acp.js
CHANGED
|
@@ -7,8 +7,9 @@ import * as acp from '@agentclientprotocol/sdk';
|
|
|
7
7
|
import { normalizeSessionUpdate } from './conversation-normalizer.js';
|
|
8
8
|
import { ConversationEventStore } from './conversation-store.js';
|
|
9
9
|
import { SessionEvents } from './events.js';
|
|
10
|
-
import { SessionControlError, classifyChildExit, turnResult } from './types.js';
|
|
10
|
+
import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, classifyChildExit, turnResult, } from './types.js';
|
|
11
11
|
const CANCEL_SETTLE_GRACE_MS = 15_000;
|
|
12
|
+
const CANCEL_TERMINATE_GRACE_MS = 5_000;
|
|
12
13
|
/** A permission no human answered is eventually a decision nobody made. */
|
|
13
14
|
const PERMISSION_TIMEOUT_MS = 10 * 60_000;
|
|
14
15
|
/** Spec §4.3: 10-15 s before a vanished controller triggers the unattended policy. */
|
|
@@ -214,6 +215,15 @@ export class AcpSession {
|
|
|
214
215
|
/** Armed when the last controller detaches; unattended policy applies on fire. */
|
|
215
216
|
controllerGrace;
|
|
216
217
|
cancelEscalation;
|
|
218
|
+
cancelForceKill;
|
|
219
|
+
cancelRecoveryReason;
|
|
220
|
+
/**
|
|
221
|
+
* Rejects the moment the adapter process is gone. Every in-flight ACP request
|
|
222
|
+
* races it, so a dead adapter can never leave a turn — and therefore a
|
|
223
|
+
* scheduled run's `activeRunId` or an admission claim — unsettled forever.
|
|
224
|
+
*/
|
|
225
|
+
terminated;
|
|
226
|
+
terminate;
|
|
217
227
|
/** ACP-authenticated in-flight calls, including independently reserved permissions. */
|
|
218
228
|
activeToolCalls = new Map();
|
|
219
229
|
toolBoundaryWaiters = new Set();
|
|
@@ -228,16 +238,29 @@ export class AcpSession {
|
|
|
228
238
|
roleId: options.name, log: line => options.log(`[${options.name}] ${line}`),
|
|
229
239
|
});
|
|
230
240
|
this.sessionFile = join(options.stateDir, '.acp-session-id');
|
|
241
|
+
this.terminated = new Promise((_resolve, reject) => { this.terminate = reject; });
|
|
242
|
+
// Nothing awaits this promise until a request races it; an unobserved
|
|
243
|
+
// rejection here must never take the whole runner down.
|
|
244
|
+
this.terminated.catch(() => undefined);
|
|
231
245
|
child.stderr.on('data', chunk => options.log(`[${options.name}] acp: ${String(chunk).trimEnd()}`));
|
|
232
246
|
child.once('exit', (code, signal) => {
|
|
247
|
+
if (this.cancelForceKill)
|
|
248
|
+
clearTimeout(this.cancelForceKill);
|
|
249
|
+
this.cancelForceKill = undefined;
|
|
233
250
|
// Record the child's real exit code/signal. The tmux path can only see a
|
|
234
251
|
// shell's `$?`; here the truth is available, so keep it.
|
|
235
|
-
|
|
252
|
+
const classified = classifyChildExit(code, signal);
|
|
253
|
+
this.exit = this.cancelRecoveryReason
|
|
254
|
+
? { ...classified, detail: `${this.cancelRecoveryReason}; ${classified.detail}` }
|
|
255
|
+
: classified;
|
|
236
256
|
options.log(`[${options.name}] acp: agent exited (${code ?? signal ?? 'unknown'})`);
|
|
237
257
|
if (this.readiness !== 'failed') {
|
|
238
258
|
this.readiness = 'failed';
|
|
239
259
|
this.lastError = `ACP agent ${this.exit.detail}`;
|
|
240
260
|
}
|
|
261
|
+
// A request whose peer no longer exists will never answer. Fail it here
|
|
262
|
+
// rather than trusting the transport to notice the closed stream.
|
|
263
|
+
this.terminate(new SessionControlError('offline', `ACP agent ${this.exit.detail}`));
|
|
241
264
|
this.events.emit('state', { status: 'failed', text: this.lastError });
|
|
242
265
|
this.conversation.appendSafe({
|
|
243
266
|
kind: 'session.state', sessionGeneration: this.sessionGeneration,
|
|
@@ -319,7 +342,10 @@ export class AcpSession {
|
|
|
319
342
|
}
|
|
320
343
|
}
|
|
321
344
|
isAlive() {
|
|
322
|
-
|
|
345
|
+
// child.killed only means kill() successfully SENT a signal. A process may
|
|
346
|
+
// ignore SIGTERM and remain alive. exitCode or signalCode is the actual
|
|
347
|
+
// terminal fact (signal exits deliberately leave exitCode null).
|
|
348
|
+
return this.child.exitCode === null && (this.child.signalCode ?? null) === null;
|
|
323
349
|
}
|
|
324
350
|
snapshot() {
|
|
325
351
|
return {
|
|
@@ -425,8 +451,28 @@ export class AcpSession {
|
|
|
425
451
|
});
|
|
426
452
|
}
|
|
427
453
|
/**
|
|
428
|
-
*
|
|
429
|
-
*
|
|
454
|
+
* Steering is an optional admission fast path, not the only safe way to
|
|
455
|
+
* deliver a wake. Codex can reject `_session/steering` while a long-running
|
|
456
|
+
* turn is between tools. Queue one ordinary, non-cancelling prompt in that
|
|
457
|
+
* case and wait for its terminal result. This keeps the monitor's cursor
|
|
458
|
+
* uncommitted until the wake really runs and, critically, keeps one rejected
|
|
459
|
+
* steering response from becoming a tight replay loop.
|
|
460
|
+
*/
|
|
461
|
+
async steerOrQueueWake(text, options) {
|
|
462
|
+
const steered = await this.steerPrompt(text);
|
|
463
|
+
if (steered.accepted || steered.detail !== 'ACP steering failed'
|
|
464
|
+
|| this.closing || !this.isAlive())
|
|
465
|
+
return steered;
|
|
466
|
+
const queued = await this.submitPrompt(text, { ...options, interrupt: false, steer: false });
|
|
467
|
+
return {
|
|
468
|
+
...queued,
|
|
469
|
+
detail: `steering rejected; queued delivery ${queued.detail ?? queued.outcome}`,
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
473
|
+
* Monitor-only safe-boundary delivery. Steering is the preferred live
|
|
474
|
+
* insertion and rejected steering is queued: this path never calls
|
|
475
|
+
* session/cancel and never resolves a pending permission.
|
|
430
476
|
*/
|
|
431
477
|
async submitPromptAfterTool(text, options = {}) {
|
|
432
478
|
if (this.closing || !this.isAlive())
|
|
@@ -443,7 +489,7 @@ export class AcpSession {
|
|
|
443
489
|
}
|
|
444
490
|
if (initialToolCount === 0) {
|
|
445
491
|
this.recordAfterToolDelivery('direct', 0, 0);
|
|
446
|
-
const result = await this.
|
|
492
|
+
const result = await this.steerOrQueueWake(text, options);
|
|
447
493
|
return { ...result, safeBoundary: { state: 'direct', waitedMs: 0, activeToolCount: 0 } };
|
|
448
494
|
}
|
|
449
495
|
this.recordAfterToolDelivery('deferred', initialToolCount, 0);
|
|
@@ -464,7 +510,7 @@ export class AcpSession {
|
|
|
464
510
|
const state = atBoundary ? 'after_tool' : 'timeout';
|
|
465
511
|
const remainingToolCount = this.activeToolCalls.size;
|
|
466
512
|
this.recordAfterToolDelivery(state, remainingToolCount, waitedMs);
|
|
467
|
-
const result = await this.
|
|
513
|
+
const result = await this.steerOrQueueWake(text, options);
|
|
468
514
|
return {
|
|
469
515
|
...result,
|
|
470
516
|
safeBoundary: { state, waitedMs, activeToolCount: remainingToolCount },
|
|
@@ -476,7 +522,9 @@ export class AcpSession {
|
|
|
476
522
|
* for it is what turned a busy agent into a timeout and then into "dead".
|
|
477
523
|
*/
|
|
478
524
|
async queuePrompt(text, options = {}) {
|
|
479
|
-
if (
|
|
525
|
+
if (this.cancelRecoveryReason)
|
|
526
|
+
throw new SessionControlError('control-unavailable', 'ACP adapter restart is in progress after the cancellation deadline', ACP_CANCEL_DEADLINE_EXCEEDED);
|
|
527
|
+
if (this.closing || !this.sessionId || !this.isAlive())
|
|
480
528
|
throw new SessionControlError('offline', this.lastError ?? 'ACP session is offline');
|
|
481
529
|
if (options.interrupt)
|
|
482
530
|
await this.cancelActive(options.interruptSource ?? 'local-console');
|
|
@@ -570,8 +618,24 @@ export class AcpSession {
|
|
|
570
618
|
throw error;
|
|
571
619
|
}
|
|
572
620
|
}
|
|
621
|
+
/**
|
|
622
|
+
* Explicit cancellation on behalf of a human or an operator. Forced recovery
|
|
623
|
+
* is reported as an outcome, never as a thrown failure: by the time this
|
|
624
|
+
* resolves the turn is over either way, and only the durable-ingress path
|
|
625
|
+
* (`queuePrompt({ interrupt: true })`) needs the typed error, because only it
|
|
626
|
+
* still owes an undelivered message a replay.
|
|
627
|
+
*/
|
|
573
628
|
async interrupt(source = 'local-console') {
|
|
574
|
-
|
|
629
|
+
try {
|
|
630
|
+
await this.cancelActive(source);
|
|
631
|
+
return { state: 'settled' };
|
|
632
|
+
}
|
|
633
|
+
catch (error) {
|
|
634
|
+
if (error instanceof SessionControlError
|
|
635
|
+
&& error.reasonCode === ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
636
|
+
return { state: 'forced', reasonCode: ACP_CANCEL_DEADLINE_EXCEEDED };
|
|
637
|
+
throw error;
|
|
638
|
+
}
|
|
575
639
|
}
|
|
576
640
|
async cancelActive(source) {
|
|
577
641
|
if (!this.sessionId)
|
|
@@ -594,21 +658,66 @@ export class AcpSession {
|
|
|
594
658
|
acpSessionId: this.sessionId, promptId: active.id, turnId: active.id,
|
|
595
659
|
payload: { cancellationSource: source },
|
|
596
660
|
});
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
661
|
+
}
|
|
662
|
+
for (const [permissionId, pending] of [...this.pendingPermissions])
|
|
663
|
+
this.settlePendingAutomatically(permissionId, pending, 'cancelled', undefined, 'the turn was cancelled while this request was pending');
|
|
664
|
+
if (active && this.activeTurn === active) {
|
|
665
|
+
active.cancellationWait ??= this.awaitCancellationSettlement(active);
|
|
666
|
+
await active.cancellationWait;
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Do not admit work behind a turn whose adapter may already require restart.
|
|
671
|
+
* A cooperative adapter settles this promise immediately through runPrompt's
|
|
672
|
+
* finally block. A stubborn adapter receives SIGTERM at the deadline and
|
|
673
|
+
* SIGKILL after one more bounded grace; callers get a typed recovery error so
|
|
674
|
+
* durable ingress can leave the next request replayable for the resumed run.
|
|
675
|
+
*/
|
|
676
|
+
awaitCancellationSettlement(active) {
|
|
677
|
+
const settleMs = this.options.cancelGraceMs ?? CANCEL_SETTLE_GRACE_MS;
|
|
678
|
+
const deadline = new Promise((resolve, reject) => {
|
|
600
679
|
this.cancelEscalation = setTimeout(() => {
|
|
601
|
-
|
|
680
|
+
// Both bail-outs must SETTLE the race. Returning silently once left the
|
|
681
|
+
// caller — and the arbiter's exclusive tail behind it — awaiting a
|
|
682
|
+
// promise nothing would ever resolve.
|
|
683
|
+
if (this.activeTurn !== active) {
|
|
684
|
+
resolve();
|
|
602
685
|
return;
|
|
603
|
-
|
|
604
|
-
this.
|
|
605
|
-
|
|
686
|
+
}
|
|
687
|
+
if (!this.isAlive()) {
|
|
688
|
+
resolve();
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
active.cancellationDeadlineExceeded = true;
|
|
692
|
+
this.cancelRecoveryReason = ACP_CANCEL_DEADLINE_EXCEEDED;
|
|
693
|
+
this.readiness = 'failed';
|
|
694
|
+
this.lastError = `${ACP_CANCEL_DEADLINE_EXCEEDED}: ACP turn did not settle within ${settleMs}ms`;
|
|
695
|
+
this.options.log(`[${this.options.name}] ${this.lastError}; restarting adapter with resume`);
|
|
696
|
+
this.events.emit('error', {
|
|
697
|
+
turnId: active.id, origin: active.origin, status: ACP_CANCEL_DEADLINE_EXCEEDED,
|
|
698
|
+
text: this.lastError,
|
|
699
|
+
});
|
|
700
|
+
this.conversation.appendSafe({
|
|
701
|
+
kind: 'session.state', sessionGeneration: this.sessionGeneration,
|
|
702
|
+
acpSessionId: this.sessionId,
|
|
703
|
+
promptId: active.id, turnId: active.id,
|
|
704
|
+
payload: { status: 'failed', detail: this.lastError },
|
|
705
|
+
});
|
|
606
706
|
this.child.kill('SIGTERM');
|
|
607
|
-
|
|
707
|
+
const terminateMs = this.options.cancelTerminateGraceMs ?? CANCEL_TERMINATE_GRACE_MS;
|
|
708
|
+
this.cancelForceKill = setTimeout(() => {
|
|
709
|
+
if (this.child.exitCode === null) {
|
|
710
|
+
this.options.log(`[${this.options.name}] ${ACP_CANCEL_DEADLINE_EXCEEDED}: `
|
|
711
|
+
+ `adapter ignored SIGTERM for ${terminateMs}ms; sending SIGKILL`);
|
|
712
|
+
this.child.kill('SIGKILL');
|
|
713
|
+
}
|
|
714
|
+
}, terminateMs);
|
|
715
|
+
this.cancelForceKill.unref?.();
|
|
716
|
+
reject(new SessionControlError('control-unavailable', 'ACP adapter restart is in progress after the cancellation deadline', ACP_CANCEL_DEADLINE_EXCEEDED));
|
|
717
|
+
}, settleMs);
|
|
608
718
|
this.cancelEscalation.unref?.();
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
this.settlePendingAutomatically(permissionId, pending, 'cancelled', undefined, 'the turn was cancelled while this request was pending');
|
|
719
|
+
});
|
|
720
|
+
return Promise.race([active.settled, deadline]);
|
|
612
721
|
}
|
|
613
722
|
respondPermission(permissionId, optionId) {
|
|
614
723
|
const pending = this.pendingPermissions.get(permissionId);
|
|
@@ -740,6 +849,9 @@ export class AcpSession {
|
|
|
740
849
|
if (this.cancelEscalation)
|
|
741
850
|
clearTimeout(this.cancelEscalation);
|
|
742
851
|
this.cancelEscalation = undefined;
|
|
852
|
+
if (this.cancelForceKill)
|
|
853
|
+
clearTimeout(this.cancelForceKill);
|
|
854
|
+
this.cancelForceKill = undefined;
|
|
743
855
|
if (this.controllerGrace)
|
|
744
856
|
clearTimeout(this.controllerGrace);
|
|
745
857
|
this.controllerGrace = undefined;
|
|
@@ -752,6 +864,8 @@ export class AcpSession {
|
|
|
752
864
|
this.connection.close();
|
|
753
865
|
if (this.isAlive())
|
|
754
866
|
this.child.kill('SIGTERM');
|
|
867
|
+
// The transport is gone: nothing still awaiting an ACP answer can get one.
|
|
868
|
+
this.terminate(new SessionControlError('offline', 'the ACP session was closed'));
|
|
755
869
|
this.conversation.appendSafe({
|
|
756
870
|
kind: 'session.state', sessionGeneration: this.sessionGeneration,
|
|
757
871
|
acpSessionId: this.sessionId, payload: { status: 'offline' },
|
|
@@ -839,7 +953,9 @@ export class AcpSession {
|
|
|
839
953
|
if (!this.sessionId || !this.isAlive())
|
|
840
954
|
return turnResult(false, 'failed', this.lastError ?? 'ACP session is offline');
|
|
841
955
|
this.readiness = 'running';
|
|
842
|
-
|
|
956
|
+
let settle;
|
|
957
|
+
const settled = new Promise(resolve => { settle = resolve; });
|
|
958
|
+
this.activeTurn = { id: turnId, output: '', origin, settled, settle };
|
|
843
959
|
this.events.emit('state', { turnId, status: 'running', origin });
|
|
844
960
|
this.conversation.appendSafe({
|
|
845
961
|
kind: 'prompt.started', sessionGeneration: this.sessionGeneration,
|
|
@@ -847,10 +963,13 @@ export class AcpSession {
|
|
|
847
963
|
source: conversationSource(origin).source, payload: {},
|
|
848
964
|
});
|
|
849
965
|
try {
|
|
850
|
-
const response = await
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
966
|
+
const response = await Promise.race([
|
|
967
|
+
this.connection.agent.request(acp.methods.agent.session.prompt, {
|
|
968
|
+
sessionId: this.sessionId,
|
|
969
|
+
prompt: promptContentBlocks(text, origin),
|
|
970
|
+
}),
|
|
971
|
+
this.terminated,
|
|
972
|
+
]);
|
|
854
973
|
this.readiness = 'idle';
|
|
855
974
|
const cancellationSource = this.activeTurn?.id === turnId
|
|
856
975
|
? this.activeTurn.cancellationSource : undefined;
|
|
@@ -872,7 +991,11 @@ export class AcpSession {
|
|
|
872
991
|
return turnResult(true, classifyStopReason(response.stopReason), response.stopReason, this.activeTurn?.id === turnId ? this.activeTurn.output : undefined, this.activeTurn?.id === turnId ? this.activeTurn.cancellationSource : undefined);
|
|
873
992
|
}
|
|
874
993
|
catch (error) {
|
|
875
|
-
const
|
|
994
|
+
const escalated = this.activeTurn?.id === turnId
|
|
995
|
+
&& this.activeTurn.cancellationDeadlineExceeded;
|
|
996
|
+
const detail = escalated
|
|
997
|
+
? this.lastError ?? ACP_CANCEL_DEADLINE_EXCEEDED
|
|
998
|
+
: error?.message ?? String(error);
|
|
876
999
|
this.lastError = origin?.kind === 'scheduled-loop' ? 'scheduled-loop turn failed' : detail;
|
|
877
1000
|
this.readiness = this.isAlive() ? 'idle' : 'failed';
|
|
878
1001
|
this.events.emit('error', {
|
|
@@ -891,6 +1014,7 @@ export class AcpSession {
|
|
|
891
1014
|
finally {
|
|
892
1015
|
this.releaseAllTools();
|
|
893
1016
|
if (this.activeTurn?.id === turnId) {
|
|
1017
|
+
this.activeTurn.settle();
|
|
894
1018
|
if (this.cancelEscalation)
|
|
895
1019
|
clearTimeout(this.cancelEscalation);
|
|
896
1020
|
this.cancelEscalation = undefined;
|
|
@@ -902,10 +1026,13 @@ export class AcpSession {
|
|
|
902
1026
|
if (!this.sessionId || !this.isAlive())
|
|
903
1027
|
return turnResult(false, 'failed', this.lastError ?? 'ACP session is offline');
|
|
904
1028
|
try {
|
|
905
|
-
const response = await
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
1029
|
+
const response = await Promise.race([
|
|
1030
|
+
this.connection.agent.request('_session/steering', {
|
|
1031
|
+
sessionId: this.sessionId,
|
|
1032
|
+
prompt: [{ type: 'text', text }],
|
|
1033
|
+
}),
|
|
1034
|
+
this.terminated,
|
|
1035
|
+
]);
|
|
909
1036
|
if (response.outcome === 'failed')
|
|
910
1037
|
return turnResult(false, 'failed', 'ACP steering failed');
|
|
911
1038
|
return turnResult(true, 'inconclusive', response.outcome);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ConversationHandlePage, ExitRecord, PromptOrigin, QueuedPrompt, SessionEvent, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnCancellationSource, TurnResult } from './types.js';
|
|
1
|
+
import type { ConversationHandlePage, ExitRecord, InterruptOutcome, PromptOrigin, QueuedPrompt, SessionEvent, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnCancellationSource, TurnResult } from './types.js';
|
|
2
2
|
import type { ConversationEventV1, ConversationSnapshot, PromptReceipt, SubmitPromptCommand } from './conversation-types.js';
|
|
3
3
|
export type ScheduledAttempt = {
|
|
4
4
|
state: 'started';
|
|
@@ -22,9 +22,35 @@ export declare class RoleTurnArbiter implements SessionHandle {
|
|
|
22
22
|
private tail;
|
|
23
23
|
private unsettled;
|
|
24
24
|
private stopping;
|
|
25
|
+
/** Bumped by `retireStalledAdmission`; every claim carries the one it was made under. */
|
|
26
|
+
private generation;
|
|
27
|
+
/** Resolves the instant the CURRENT generation is retired. */
|
|
28
|
+
private retirement;
|
|
25
29
|
constructor(session: SessionHandle);
|
|
30
|
+
/**
|
|
31
|
+
* Waiters race the tail against their own generation's retirement, so a
|
|
32
|
+
* single operation that never settles cannot own the boundary forever. A
|
|
33
|
+
* woken waiter from a retired generation never runs its operation: the
|
|
34
|
+
* ordering it was promised no longer exists, and admitting it silently would
|
|
35
|
+
* be the one thing worse than telling the caller it was not admitted.
|
|
36
|
+
*/
|
|
26
37
|
private exclusive;
|
|
27
38
|
private track;
|
|
39
|
+
/**
|
|
40
|
+
* Release the admission boundary after a cancellation that never settled.
|
|
41
|
+
* Nothing can make a hung ACP call return, so the stuck operation keeps its
|
|
42
|
+
* own promise — it just stops owning `tail` and stops holding a claim.
|
|
43
|
+
*
|
|
44
|
+
* Without this, one unsettled `interrupt` left every later producer queued
|
|
45
|
+
* behind it forever: a scheduled poll never returned, so the loop was never
|
|
46
|
+
* rescheduled and reported neither `started` nor `skipped_busy`, and an owner
|
|
47
|
+
* prompt stayed pending with nothing to resolve it.
|
|
48
|
+
*
|
|
49
|
+
* One-active-run safety is unchanged. The arbiter only orders admission;
|
|
50
|
+
* `tryScheduled` still asks the session itself whether it is idle, and the
|
|
51
|
+
* session still refuses to run two turns at once.
|
|
52
|
+
*/
|
|
53
|
+
retireStalledAdmission(): void;
|
|
28
54
|
queuePrompt(text: string, options?: SubmitPromptOptions): Promise<QueuedPrompt>;
|
|
29
55
|
submitPrompt(text: string, options?: SubmitPromptOptions): Promise<TurnResult>;
|
|
30
56
|
/**
|
|
@@ -45,7 +71,7 @@ export declare class RoleTurnArbiter implements SessionHandle {
|
|
|
45
71
|
conversationSnapshot(): ConversationSnapshot;
|
|
46
72
|
subscribeConversation(listener: (event: ConversationEventV1) => void): () => void;
|
|
47
73
|
submitPromptBrowser(command: SubmitPromptCommand): Promise<PromptReceipt>;
|
|
48
|
-
interrupt(source?: TurnCancellationSource): Promise<
|
|
74
|
+
interrupt(source?: TurnCancellationSource): Promise<InterruptOutcome>;
|
|
49
75
|
respondPermission(permissionId: string, optionId: string): boolean;
|
|
50
76
|
respondPermissionV2(permissionId: string, optionId: string, sessionGeneration: string): 'accepted' | 'stale';
|
|
51
77
|
eventsSince(seq: number): SessionEvent[];
|
package/dist/session/arbiter.js
CHANGED
|
@@ -1,3 +1,20 @@
|
|
|
1
|
+
import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, interruptOutcome, } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Raised for admissions that were still waiting on a generation which had to be
|
|
4
|
+
* retired. It carries `ACP_CANCEL_DEADLINE_EXCEEDED` because that is the only
|
|
5
|
+
* thing that ever retires a generation, and because durable owner ingress
|
|
6
|
+
* already treats that reason as "never delivered — replay it".
|
|
7
|
+
*/
|
|
8
|
+
class AdmissionRetiredError extends SessionControlError {
|
|
9
|
+
constructor() {
|
|
10
|
+
super('control-unavailable', 'admission was retired after a cancellation that never settled', ACP_CANCEL_DEADLINE_EXCEEDED);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function retirementSignal() {
|
|
14
|
+
let retire;
|
|
15
|
+
const promise = new Promise(resolve => { retire = resolve; });
|
|
16
|
+
return { promise, retire };
|
|
17
|
+
}
|
|
1
18
|
/**
|
|
2
19
|
* One in-process admission boundary for every producer targeting a role.
|
|
3
20
|
* Scheduled callers get an atomic idle recheck plus submission; ordinary
|
|
@@ -11,21 +28,64 @@ export class RoleTurnArbiter {
|
|
|
11
28
|
tail = Promise.resolve();
|
|
12
29
|
unsettled = 0;
|
|
13
30
|
stopping = false;
|
|
31
|
+
/** Bumped by `retireStalledAdmission`; every claim carries the one it was made under. */
|
|
32
|
+
generation = 0;
|
|
33
|
+
/** Resolves the instant the CURRENT generation is retired. */
|
|
34
|
+
retirement = retirementSignal();
|
|
14
35
|
constructor(session) {
|
|
15
36
|
this.session = session;
|
|
16
37
|
this.backend = session.backend;
|
|
17
38
|
this.pid = session.pid;
|
|
18
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* Waiters race the tail against their own generation's retirement, so a
|
|
42
|
+
* single operation that never settles cannot own the boundary forever. A
|
|
43
|
+
* woken waiter from a retired generation never runs its operation: the
|
|
44
|
+
* ordering it was promised no longer exists, and admitting it silently would
|
|
45
|
+
* be the one thing worse than telling the caller it was not admitted.
|
|
46
|
+
*/
|
|
19
47
|
exclusive(operation) {
|
|
20
|
-
const
|
|
48
|
+
const generation = this.generation;
|
|
49
|
+
const run = Promise.race([this.tail, this.retirement.promise]).then(() => {
|
|
50
|
+
if (generation !== this.generation)
|
|
51
|
+
throw new AdmissionRetiredError();
|
|
52
|
+
return operation();
|
|
53
|
+
});
|
|
21
54
|
this.tail = run.then(() => undefined, () => undefined);
|
|
22
55
|
return run;
|
|
23
56
|
}
|
|
24
57
|
track(queued) {
|
|
58
|
+
const generation = this.generation;
|
|
25
59
|
this.unsettled++;
|
|
26
|
-
const completion = queued.completion.finally(() => {
|
|
60
|
+
const completion = queued.completion.finally(() => {
|
|
61
|
+
// A retired generation's claims were released in one step already.
|
|
62
|
+
if (generation === this.generation)
|
|
63
|
+
this.unsettled = Math.max(0, this.unsettled - 1);
|
|
64
|
+
});
|
|
27
65
|
return { ...queued, completion };
|
|
28
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* Release the admission boundary after a cancellation that never settled.
|
|
69
|
+
* Nothing can make a hung ACP call return, so the stuck operation keeps its
|
|
70
|
+
* own promise — it just stops owning `tail` and stops holding a claim.
|
|
71
|
+
*
|
|
72
|
+
* Without this, one unsettled `interrupt` left every later producer queued
|
|
73
|
+
* behind it forever: a scheduled poll never returned, so the loop was never
|
|
74
|
+
* rescheduled and reported neither `started` nor `skipped_busy`, and an owner
|
|
75
|
+
* prompt stayed pending with nothing to resolve it.
|
|
76
|
+
*
|
|
77
|
+
* One-active-run safety is unchanged. The arbiter only orders admission;
|
|
78
|
+
* `tryScheduled` still asks the session itself whether it is idle, and the
|
|
79
|
+
* session still refuses to run two turns at once.
|
|
80
|
+
*/
|
|
81
|
+
retireStalledAdmission() {
|
|
82
|
+
const retired = this.retirement;
|
|
83
|
+
this.generation++;
|
|
84
|
+
this.retirement = retirementSignal();
|
|
85
|
+
this.tail = Promise.resolve();
|
|
86
|
+
this.unsettled = 0;
|
|
87
|
+
retired.retire();
|
|
88
|
+
}
|
|
29
89
|
queuePrompt(text, options = {}) {
|
|
30
90
|
return this.exclusive(async () => this.track(await this.session.queuePrompt(text, options)));
|
|
31
91
|
}
|
|
@@ -54,13 +114,24 @@ export class RoleTurnArbiter {
|
|
|
54
114
|
try {
|
|
55
115
|
await beforeQueue?.();
|
|
56
116
|
const queued = await this.session.queuePrompt(text, { interrupt: false, origin });
|
|
57
|
-
|
|
117
|
+
// The race is lost, but the prompt was admitted and WILL run. Keep the
|
|
118
|
+
// claim so idle accounting matches reality instead of under-counting.
|
|
119
|
+
if (queued.queuedBehind > 0) {
|
|
120
|
+
this.track(queued);
|
|
58
121
|
return { state: 'unavailable', error: 'scheduled admission race' };
|
|
122
|
+
}
|
|
59
123
|
return { state: 'started', queued: this.track(queued) };
|
|
60
124
|
}
|
|
61
125
|
catch (error) {
|
|
62
126
|
return { state: 'unavailable', error: error?.message ?? String(error) };
|
|
63
127
|
}
|
|
128
|
+
}).catch(error => {
|
|
129
|
+
// A retired attempt was never submitted. Reporting that as an outcome
|
|
130
|
+
// instead of a rejection is what keeps the poll — and with it the loop's
|
|
131
|
+
// own rescheduling — from being abandoned mid-tick.
|
|
132
|
+
if (error instanceof AdmissionRetiredError)
|
|
133
|
+
return { state: 'unavailable', error: error.message };
|
|
134
|
+
throw error;
|
|
64
135
|
});
|
|
65
136
|
}
|
|
66
137
|
stopScheduledAdmission() { this.stopping = true; }
|
|
@@ -87,7 +158,7 @@ export class RoleTurnArbiter {
|
|
|
87
158
|
return this.exclusive(() => this.session.submitPromptBrowser(command));
|
|
88
159
|
}
|
|
89
160
|
interrupt(source = 'local-console') {
|
|
90
|
-
return this.exclusive(() => this.session.interrupt(source));
|
|
161
|
+
return this.exclusive(async () => interruptOutcome(await this.session.interrupt(source)));
|
|
91
162
|
}
|
|
92
163
|
respondPermission(permissionId, optionId) {
|
|
93
164
|
return this.session.respondPermission(permissionId, optionId);
|
package/dist/session/control.js
CHANGED
|
@@ -2,7 +2,7 @@ import { randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
|
|
|
2
2
|
import { chmodSync, existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
3
|
import { createConnection, createServer } from 'node:net';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import { SessionControlError } from './types.js';
|
|
5
|
+
import { SessionControlError, interruptOutcome } from './types.js';
|
|
6
6
|
const MAX_LINE_BYTES = 64 * 1024;
|
|
7
7
|
/** Commands that require protocol version 3. */
|
|
8
8
|
const V3_COMMANDS = new Set([
|
|
@@ -253,10 +253,13 @@ export class RoleControlServer {
|
|
|
253
253
|
});
|
|
254
254
|
return;
|
|
255
255
|
}
|
|
256
|
-
case 'interrupt':
|
|
257
|
-
|
|
258
|
-
|
|
256
|
+
case 'interrupt': {
|
|
257
|
+
// Forced recovery cancelled the turn just as surely as a cooperative
|
|
258
|
+
// stop did. Report HOW, never as a failed operation.
|
|
259
|
+
const outcome = interruptOutcome(await this.session.interrupt('local-console'));
|
|
260
|
+
this.write(socket, { version: 1, id: request.id, ok: true, result: outcome });
|
|
259
261
|
return;
|
|
262
|
+
}
|
|
260
263
|
case 'loop_status': {
|
|
261
264
|
if (!this.loopManager)
|
|
262
265
|
throw new SessionControlError('rejected', 'scheduled loops are unavailable for this role');
|
|
@@ -396,8 +399,11 @@ export class RoleControlServer {
|
|
|
396
399
|
this.write(socket, { version: 1, id: request.id, ok: true, result: existing });
|
|
397
400
|
return;
|
|
398
401
|
}
|
|
399
|
-
await this.session.interrupt('local-console');
|
|
400
|
-
const receipt = {
|
|
402
|
+
const outcome = interruptOutcome(await this.session.interrupt('local-console'));
|
|
403
|
+
const receipt = {
|
|
404
|
+
accepted: true, commandId: request.commandId, at: new Date().toISOString(),
|
|
405
|
+
...outcome,
|
|
406
|
+
};
|
|
401
407
|
this.interruptCommands.set(request.commandId, receipt);
|
|
402
408
|
if (this.interruptCommands.size > 200) {
|
|
403
409
|
const oldest = this.interruptCommands.keys().next().value;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Crash-safe JSONL primitives for the session event stream.
|
|
3
|
+
*
|
|
4
|
+
* The 06:32 disk-full incident showed what an unguarded `appendFileSync` does
|
|
5
|
+
* under ENOSPC: the kernel short-writes, the already-written prefix stays on
|
|
6
|
+
* disk, and the next successful append lands directly on it — fusing a partial
|
|
7
|
+
* record with a valid later one into a single unparseable line. These helpers
|
|
8
|
+
* make an append all-or-nothing, keep a damaged byte range from ever swallowing
|
|
9
|
+
* a later record, and describe damage rather than quietly dropping it.
|
|
10
|
+
*
|
|
11
|
+
* Single-writer assumption: rollback truncates back to the size observed at the
|
|
12
|
+
* start of the append, so exactly one process may append to a given path.
|
|
13
|
+
*/
|
|
14
|
+
/** The syscall surface an append needs, injectable so faults can be forced deterministically. */
|
|
15
|
+
export interface LogIo {
|
|
16
|
+
openSync(path: string, flags: string, mode?: number): number;
|
|
17
|
+
fstatSync(fd: number): {
|
|
18
|
+
size: number;
|
|
19
|
+
};
|
|
20
|
+
readSync(fd: number, buffer: Buffer, offset: number, length: number, position: number): number;
|
|
21
|
+
writeSync(fd: number, buffer: Buffer, offset: number, length: number): number;
|
|
22
|
+
ftruncateSync(fd: number, length: number): void;
|
|
23
|
+
closeSync(fd: number): void;
|
|
24
|
+
}
|
|
25
|
+
export declare const nodeLogIo: LogIo;
|
|
26
|
+
export declare class AtomicAppendError extends Error {
|
|
27
|
+
readonly rollbackFailed: boolean;
|
|
28
|
+
readonly cause?: unknown | undefined;
|
|
29
|
+
constructor(message: string, rollbackFailed: boolean, cause?: unknown | undefined);
|
|
30
|
+
}
|
|
31
|
+
export interface AppendResult {
|
|
32
|
+
/** True when a dangling partial record was closed off before this line was written. */
|
|
33
|
+
repairedBoundary: boolean;
|
|
34
|
+
}
|
|
35
|
+
export interface AppendOptions {
|
|
36
|
+
io?: LogIo;
|
|
37
|
+
/** Total attempts, including the first. Bounded so a full disk cannot spin. */
|
|
38
|
+
maxAttempts?: number;
|
|
39
|
+
mode?: number;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Append one line, all or nothing. A failed write is rolled back to the byte
|
|
43
|
+
* length observed before it started, so a partial record never persists; if the
|
|
44
|
+
* file already ends mid-record (damage from before this fix), the line is put on
|
|
45
|
+
* a fresh line so it cannot fuse with the damaged bytes.
|
|
46
|
+
*/
|
|
47
|
+
export declare function appendLineAtomic(path: string, line: string, options?: AppendOptions): AppendResult;
|
|
48
|
+
export type DamageReason = 'truncated_tail' | 'interior_corruption';
|
|
49
|
+
export interface DamagedLine {
|
|
50
|
+
/** 1-based line number in the file as read. */
|
|
51
|
+
lineNumber: number;
|
|
52
|
+
/** True byte length of the damaged line, even when `raw` is capped. */
|
|
53
|
+
bytes: number;
|
|
54
|
+
reason: DamageReason;
|
|
55
|
+
/** The damaged bytes, capped for storage; `truncatedEvidence` says when. */
|
|
56
|
+
raw: string;
|
|
57
|
+
truncatedEvidence: boolean;
|
|
58
|
+
}
|
|
59
|
+
/** A run of sequence numbers that is simply gone — never recoverable, only reportable. */
|
|
60
|
+
export interface SequenceGap {
|
|
61
|
+
afterSeq: number;
|
|
62
|
+
beforeSeq: number;
|
|
63
|
+
missing: number;
|
|
64
|
+
}
|
|
65
|
+
export interface LogRecord {
|
|
66
|
+
version: 1;
|
|
67
|
+
seq: number;
|
|
68
|
+
[key: string]: unknown;
|
|
69
|
+
}
|
|
70
|
+
export interface ReadLogResult {
|
|
71
|
+
/** Records this version understands and can replay. */
|
|
72
|
+
records: LogRecord[];
|
|
73
|
+
/**
|
|
74
|
+
* Sequence numbers claimed by every readable entry in file order, whatever
|
|
75
|
+
* its version. A record written by a newer version is not replayable here,
|
|
76
|
+
* but it did occupy its number: counting it is what keeps a forward-compatible
|
|
77
|
+
* file from looking like it has a hole, and keeps continuation past it.
|
|
78
|
+
*/
|
|
79
|
+
sequence: number[];
|
|
80
|
+
/** Highest sequence still readable anywhere in the file, for monotonic continuation. */
|
|
81
|
+
maxSeq: number;
|
|
82
|
+
damaged: DamagedLine[];
|
|
83
|
+
gaps: SequenceGap[];
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Read the log line by line. One bad line costs exactly that line: everything
|
|
87
|
+
* before and after it is kept, and the damage is described rather than dropped.
|
|
88
|
+
*/
|
|
89
|
+
export declare function readLog(path: string): ReadLogResult;
|
|
90
|
+
/**
|
|
91
|
+
* Missing sequence numbers between consecutive records. Records that parse
|
|
92
|
+
* perfectly still leave a hole when the writes between them never landed, so
|
|
93
|
+
* this is computed over whatever ordered run the caller cares about — including
|
|
94
|
+
* a rotated stream followed by its live successor, where the hole falls on the
|
|
95
|
+
* boundary and neither file can see it alone.
|
|
96
|
+
*/
|
|
97
|
+
export declare function findSequenceGaps(sequence: number[]): SequenceGap[];
|
|
98
|
+
export interface QuarantineResult {
|
|
99
|
+
quarantined: boolean;
|
|
100
|
+
/** Present when the sidecar could not be written; the source file is untouched either way. */
|
|
101
|
+
error?: string;
|
|
102
|
+
sidecar: string;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Copy damaged bytes to a sidecar for later forensics. The damaged file itself
|
|
106
|
+
* is never read-modify-written, and a sidecar that cannot be created is reported
|
|
107
|
+
* rather than retried into the event writer.
|
|
108
|
+
*/
|
|
109
|
+
export declare function quarantineDamage(path: string, damaged: DamagedLine[], source?: string): QuarantineResult;
|