@scotthuang/agent-knock-knock 0.12.10 → 0.12.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/src/claude-local-transcript-provider.d.ts +21 -1
- package/dist/src/claude-local-transcript-provider.js +121 -15
- package/dist/src/claude-local-transcript-provider.js.map +1 -1
- package/dist/src/cli-core.js +1311 -129
- package/dist/src/cli-core.js.map +1 -1
- package/dist/src/terminal-process-source.d.ts +7 -0
- package/dist/src/terminal-process-source.js +50 -1
- package/dist/src/terminal-process-source.js.map +1 -1
- package/package.json +1 -1
package/dist/src/cli-core.js
CHANGED
|
@@ -8,7 +8,7 @@ import { fileURLToPath } from "node:url";
|
|
|
8
8
|
import { createCodexTerminalAgentAdapter, detectCodexDurableCompletion } from "./codex-terminal-agent-adapter.js";
|
|
9
9
|
import { codexLifecycleBehaviorProfile } from "./codex-lifecycle-compatibility.js";
|
|
10
10
|
import { createClaudeTerminalAgentAdapter } from "./claude-terminal-agent-adapter.js";
|
|
11
|
-
import { captureClaudeTranscriptAnchor, defaultClaudeHome, detectClaudeTranscriptAcceptance, detectClaudeTranscriptCompletion, detectClaudeTranscriptPendingApproval, listClaudeThreadLifecycleCandidates, revalidateClaudeThreadLifecycleCandidate } from "./claude-local-transcript-provider.js";
|
|
11
|
+
import { captureClaudeTranscriptAnchor, defaultClaudeHome, detectClaudeTranscriptAcceptance, detectClaudeTranscriptCompletion, detectClaudeTranscriptPendingApproval, listClaudeThreadLifecycleCandidates, observeClaudeDeadProcessTranscriptCompletion, revalidateClaudeThreadLifecycleCandidate } from "./claude-local-transcript-provider.js";
|
|
12
12
|
import { captureCodexCandidateSetRolloutAcceptanceAnchor, captureCodexRolloutAcceptanceAnchor, detectCodexBoundRolloutCompletion, detectCodexCandidateSetRolloutAcceptance, detectCodexRolloutAcceptance, terminalSubmissionReplayReceipt, validateCodexRolloutAcceptanceAnchor, validateTerminalSubmissionAcceptanceEvidence } from "./terminal-submission-acceptance.js";
|
|
13
13
|
import { CodexLocalSessionProvider } from "./codex-local-session-provider.js";
|
|
14
14
|
import { CodexStoreAdapter } from "./codex-store-adapter.js";
|
|
@@ -6199,7 +6199,8 @@ function availableListActions(entry, { conversation } = {}) {
|
|
|
6199
6199
|
}
|
|
6200
6200
|
if (terminalBridgeReady &&
|
|
6201
6201
|
entry.status === "stalled" &&
|
|
6202
|
-
terminalBridgeSubmission(conversation)?.status !== "uncertain"
|
|
6202
|
+
terminalBridgeSubmission(conversation)?.status !== "uncertain" &&
|
|
6203
|
+
!isVerifiedDeadTerminalAgentProcess(conversation ?? {})) {
|
|
6203
6204
|
actions.renew = {
|
|
6204
6205
|
tool: "agent_knock_knock_renew",
|
|
6205
6206
|
arguments: targetArguments
|
|
@@ -6596,6 +6597,245 @@ function codexProcessIncarnationForPid(pid) {
|
|
|
6596
6597
|
evidence: "codex_process_birth"
|
|
6597
6598
|
};
|
|
6598
6599
|
}
|
|
6600
|
+
function observeExactCodexDeadProcessRolloutCompletion({ options, conversation, terminalControl }) {
|
|
6601
|
+
const takeover = isRecord(conversation.native_session_takeover)
|
|
6602
|
+
? conversation.native_session_takeover
|
|
6603
|
+
: undefined;
|
|
6604
|
+
const submission = terminalBridgeSubmission(conversation);
|
|
6605
|
+
const acceptanceEvidence = isRecord(submission?.acceptance_evidence)
|
|
6606
|
+
? submission.acceptance_evidence
|
|
6607
|
+
: undefined;
|
|
6608
|
+
const anchor = isRecord(takeover?.codex_rollout_acceptance_anchor)
|
|
6609
|
+
? takeover.codex_rollout_acceptance_anchor
|
|
6610
|
+
: undefined;
|
|
6611
|
+
if (submission?.status !== "agent_accepted" ||
|
|
6612
|
+
!requiresExactBoundCodexCompletion(conversation, options) ||
|
|
6613
|
+
acceptanceEvidence?.source !== "codex_rollout" ||
|
|
6614
|
+
!anchor) {
|
|
6615
|
+
return {
|
|
6616
|
+
status: "unverifiable",
|
|
6617
|
+
reason: "Codex has no exact accepted rollout authority for the dead process"
|
|
6618
|
+
};
|
|
6619
|
+
}
|
|
6620
|
+
const runtime = terminalRuntimeIdentityForConversation(conversation, terminalControl);
|
|
6621
|
+
const nativeRollout = isRecord(runtime.nativeRollout)
|
|
6622
|
+
? runtime.nativeRollout
|
|
6623
|
+
: undefined;
|
|
6624
|
+
const request = terminalDurableRequestForConversation(conversation, terminalControl);
|
|
6625
|
+
const result = detectCodexBoundRolloutCompletion({
|
|
6626
|
+
anchor: anchor,
|
|
6627
|
+
acceptanceEvidence: acceptanceEvidence,
|
|
6628
|
+
currentIdentity: {
|
|
6629
|
+
sessionId: runtime.nativeSessionId ?? runtime.sessionId ?? "",
|
|
6630
|
+
processUuid: runtime.nativeProcessUuid,
|
|
6631
|
+
processBirth: runtime.nativeProcessBirth,
|
|
6632
|
+
...(nativeRollout
|
|
6633
|
+
? {
|
|
6634
|
+
rollout: {
|
|
6635
|
+
fd: String(nativeRollout.fd ?? ""),
|
|
6636
|
+
device: String(nativeRollout.device ?? ""),
|
|
6637
|
+
inode: String(nativeRollout.inode ?? ""),
|
|
6638
|
+
path: String(nativeRollout.path ?? "")
|
|
6639
|
+
}
|
|
6640
|
+
}
|
|
6641
|
+
: {})
|
|
6642
|
+
},
|
|
6643
|
+
requestHash: stringValue(request.requestHash) ??
|
|
6644
|
+
stringValue(takeover?.terminal_bridge_request_hash) ??
|
|
6645
|
+
""
|
|
6646
|
+
});
|
|
6647
|
+
if (result.status === "completed") {
|
|
6648
|
+
return {
|
|
6649
|
+
status: "present",
|
|
6650
|
+
completion: {
|
|
6651
|
+
...result.completion,
|
|
6652
|
+
metadata: {
|
|
6653
|
+
...result.completion.metadata,
|
|
6654
|
+
context_match: "exact_bound_rollout",
|
|
6655
|
+
detector_code: result.diagnostics.code
|
|
6656
|
+
}
|
|
6657
|
+
}
|
|
6658
|
+
};
|
|
6659
|
+
}
|
|
6660
|
+
if (result.status === "pending" &&
|
|
6661
|
+
result.diagnostics.code === "exact_turn_not_complete") {
|
|
6662
|
+
return { status: "absent" };
|
|
6663
|
+
}
|
|
6664
|
+
return {
|
|
6665
|
+
status: "unverifiable",
|
|
6666
|
+
reason: `[codex_exact_bound_rollout:${result.diagnostics.code}] ` +
|
|
6667
|
+
(result.diagnostics.detail ??
|
|
6668
|
+
"the exact bound rollout is not safely inspectable")
|
|
6669
|
+
};
|
|
6670
|
+
}
|
|
6671
|
+
async function observeDurableCompletionBeforeDeadStall({ options, conversation, terminalControl }) {
|
|
6672
|
+
try {
|
|
6673
|
+
const executor = executorForConversation(conversation);
|
|
6674
|
+
const takeover = isRecord(conversation.native_session_takeover)
|
|
6675
|
+
? conversation.native_session_takeover
|
|
6676
|
+
: undefined;
|
|
6677
|
+
if (executor.kind === "claude") {
|
|
6678
|
+
const anchor = isRecord(takeover?.claude_transcript_anchor)
|
|
6679
|
+
? takeover.claude_transcript_anchor
|
|
6680
|
+
: undefined;
|
|
6681
|
+
const pid = Number(anchor?.pid);
|
|
6682
|
+
const startedAt = Number(anchor?.agent_started_at_ms);
|
|
6683
|
+
const sessionId = stringValue(anchor?.session_id);
|
|
6684
|
+
const cwd = stringValue(anchor?.cwd);
|
|
6685
|
+
if (!Number.isSafeInteger(pid) ||
|
|
6686
|
+
pid <= 1 ||
|
|
6687
|
+
!Number.isSafeInteger(startedAt) ||
|
|
6688
|
+
startedAt <= 0 ||
|
|
6689
|
+
!sessionId ||
|
|
6690
|
+
!cwd ||
|
|
6691
|
+
pid !== Number(takeover?.terminal_agent_pid) ||
|
|
6692
|
+
sessionId !== stringValue(takeover?.terminal_agent_session_id) ||
|
|
6693
|
+
path.resolve(cwd) !== path.resolve(conversation.workspace)) {
|
|
6694
|
+
return {
|
|
6695
|
+
status: "unverifiable",
|
|
6696
|
+
reason: "Claude has no exact immutable transcript anchor for the dead process"
|
|
6697
|
+
};
|
|
6698
|
+
}
|
|
6699
|
+
const processUuid = stringValue(takeover?.terminal_agent_process_uuid);
|
|
6700
|
+
if (processUuid !== `claude-pid:${pid}:started:${startedAt}`) {
|
|
6701
|
+
return {
|
|
6702
|
+
status: "unverifiable",
|
|
6703
|
+
reason: "Claude transcript anchor does not match the exact process incarnation"
|
|
6704
|
+
};
|
|
6705
|
+
}
|
|
6706
|
+
const submission = terminalBridgeSubmission(conversation);
|
|
6707
|
+
const observation = observeClaudeDeadProcessTranscriptCompletion(terminalDurableRequestForConversation(conversation, terminalControl), {
|
|
6708
|
+
claudeHome: expandHome(options.claudeHome),
|
|
6709
|
+
// Process death freezes this exact historical incarnation. The
|
|
6710
|
+
// transcript detector still revalidates the immutable PID/session/
|
|
6711
|
+
// cwd/start anchor and the no-follow file boundary before reading.
|
|
6712
|
+
agentRows: [{
|
|
6713
|
+
pid,
|
|
6714
|
+
startedAt,
|
|
6715
|
+
sessionId,
|
|
6716
|
+
cwd,
|
|
6717
|
+
kind: "interactive",
|
|
6718
|
+
status: "idle"
|
|
6719
|
+
}],
|
|
6720
|
+
acceptanceEvidence: submission?.acceptance_evidence
|
|
6721
|
+
});
|
|
6722
|
+
return observation;
|
|
6723
|
+
}
|
|
6724
|
+
return observeExactCodexDeadProcessRolloutCompletion({
|
|
6725
|
+
options,
|
|
6726
|
+
conversation,
|
|
6727
|
+
terminalControl
|
|
6728
|
+
});
|
|
6729
|
+
}
|
|
6730
|
+
catch (error) {
|
|
6731
|
+
return {
|
|
6732
|
+
status: "unverifiable",
|
|
6733
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
6734
|
+
};
|
|
6735
|
+
}
|
|
6736
|
+
}
|
|
6737
|
+
async function observeBoundTerminalAgentProcess({ options, conversation, terminalControl }) {
|
|
6738
|
+
try {
|
|
6739
|
+
assertTurnBindingCurrent(conversation, "verify the bound agent process for");
|
|
6740
|
+
const storeDir = managedSessionStoreDirForConversation(conversation);
|
|
6741
|
+
if (!storeDir) {
|
|
6742
|
+
return {
|
|
6743
|
+
status: "unverifiable",
|
|
6744
|
+
reason: "the managed Store is unavailable"
|
|
6745
|
+
};
|
|
6746
|
+
}
|
|
6747
|
+
const session = loadManagedSession(storeDir, sessionIdForConversation(conversation));
|
|
6748
|
+
const binding = session.binding;
|
|
6749
|
+
const takeover = isRecord(conversation.native_session_takeover)
|
|
6750
|
+
? conversation.native_session_takeover
|
|
6751
|
+
: undefined;
|
|
6752
|
+
const pid = Number(takeover?.terminal_agent_pid);
|
|
6753
|
+
const bindingId = stringValue(conversation.terminal_binding_id);
|
|
6754
|
+
const bindingGeneration = Number(conversation.terminal_binding_generation);
|
|
6755
|
+
const processUuid = stringValue(takeover?.terminal_agent_process_uuid);
|
|
6756
|
+
const processBirth = stringValue(takeover?.terminal_agent_process_birth);
|
|
6757
|
+
const messageId = stringValue(takeover?.terminal_bridge_message_id);
|
|
6758
|
+
const submission = terminalBridgeSubmission(conversation);
|
|
6759
|
+
if (session.status !== "bound" ||
|
|
6760
|
+
!binding ||
|
|
6761
|
+
!Number.isSafeInteger(pid) ||
|
|
6762
|
+
pid <= 1 ||
|
|
6763
|
+
!bindingId ||
|
|
6764
|
+
!Number.isSafeInteger(bindingGeneration) ||
|
|
6765
|
+
bindingGeneration < 1 ||
|
|
6766
|
+
!processUuid ||
|
|
6767
|
+
!messageId ||
|
|
6768
|
+
!hasCanonicalTerminalEndpoint(terminalControl) ||
|
|
6769
|
+
binding.native_process.pid !== pid ||
|
|
6770
|
+
!terminalControlsShareIncarnation(binding.terminal_control, terminalControl) ||
|
|
6771
|
+
binding.binding_id !== bindingId ||
|
|
6772
|
+
binding.generation !== bindingGeneration ||
|
|
6773
|
+
session.agent !== executorForConversation(conversation).kind ||
|
|
6774
|
+
path.resolve(session.workspace) !== path.resolve(conversation.workspace) ||
|
|
6775
|
+
binding.native_thread_id !==
|
|
6776
|
+
(stringValue(conversation.native_thread_id) ??
|
|
6777
|
+
stringValue(takeover?.terminal_agent_session_id)) ||
|
|
6778
|
+
binding.native_process.process_uuid !== processUuid ||
|
|
6779
|
+
stringValue(takeover?.terminal_binding_id) !== bindingId ||
|
|
6780
|
+
Number(takeover?.terminal_binding_generation) !== bindingGeneration ||
|
|
6781
|
+
submission?.status !== "agent_accepted" ||
|
|
6782
|
+
stringValue(submission.session_id) !==
|
|
6783
|
+
sessionIdForConversation(conversation) ||
|
|
6784
|
+
stringValue(submission.turn_id) !== turnIdForConversation(conversation) ||
|
|
6785
|
+
stringValue(submission.message_id) !== messageId ||
|
|
6786
|
+
stringValue(submission.binding_id) !== bindingId ||
|
|
6787
|
+
Number(submission.binding_generation) !== bindingGeneration ||
|
|
6788
|
+
(binding.native_process.process_birth ?? undefined) !== processBirth) {
|
|
6789
|
+
return {
|
|
6790
|
+
status: "unverifiable",
|
|
6791
|
+
reason: "the Turn and Session no longer share one exact process binding"
|
|
6792
|
+
};
|
|
6793
|
+
}
|
|
6794
|
+
const processSource = createTerminalProcessSource(options);
|
|
6795
|
+
if (processSource.completeInventoryAuthority !== true) {
|
|
6796
|
+
return {
|
|
6797
|
+
status: "unverifiable",
|
|
6798
|
+
reason: "the configured process source is not complete process-death authority"
|
|
6799
|
+
};
|
|
6800
|
+
}
|
|
6801
|
+
const snapshots = await processSource.listProcessSnapshots((snapshot) => snapshot.pid === pid, { includeCwd: false, includeAncestors: false });
|
|
6802
|
+
const exact = snapshots.filter((snapshot) => snapshot.pid === pid);
|
|
6803
|
+
if (exact.length === 1) {
|
|
6804
|
+
return { status: "alive", pid };
|
|
6805
|
+
}
|
|
6806
|
+
if (exact.length !== 0) {
|
|
6807
|
+
return {
|
|
6808
|
+
status: "unverifiable",
|
|
6809
|
+
reason: `the process inventory returned ${exact.length} rows for pid ${pid}`
|
|
6810
|
+
};
|
|
6811
|
+
}
|
|
6812
|
+
return {
|
|
6813
|
+
status: "verified_dead",
|
|
6814
|
+
proof: {
|
|
6815
|
+
kind: "exact_pid_absent_from_complete_process_inventory",
|
|
6816
|
+
agent: session.agent,
|
|
6817
|
+
pid,
|
|
6818
|
+
process_uuid: processUuid,
|
|
6819
|
+
process_birth: binding.native_process.process_birth,
|
|
6820
|
+
conversation_id: conversation.conversation_id,
|
|
6821
|
+
session_id: sessionIdForConversation(conversation),
|
|
6822
|
+
turn_id: turnIdForConversation(conversation),
|
|
6823
|
+
terminal_control: terminalControl,
|
|
6824
|
+
terminal_endpoint: terminalControlEvidence(terminalControl),
|
|
6825
|
+
binding_id: binding.binding_id,
|
|
6826
|
+
binding_generation: binding.generation,
|
|
6827
|
+
message_id: messageId,
|
|
6828
|
+
observed_at: cliNow().toISOString()
|
|
6829
|
+
}
|
|
6830
|
+
};
|
|
6831
|
+
}
|
|
6832
|
+
catch (error) {
|
|
6833
|
+
return {
|
|
6834
|
+
status: "unverifiable",
|
|
6835
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
6836
|
+
};
|
|
6837
|
+
}
|
|
6838
|
+
}
|
|
6599
6839
|
function processIncarnationRelationship({ binding, livePid, liveProcessUuid, liveProcessBirth }) {
|
|
6600
6840
|
if (binding.native_process.pid !== livePid) {
|
|
6601
6841
|
return "different";
|
|
@@ -17539,6 +17779,10 @@ async function runRenew(options) {
|
|
|
17539
17779
|
if (conversation.status !== "stalled") {
|
|
17540
17780
|
throw new Error(`cannot renew ${conversation.conversation_id}; conversation is ${conversation.status}, not stalled`);
|
|
17541
17781
|
}
|
|
17782
|
+
if (isVerifiedDeadTerminalAgentProcess(conversation)) {
|
|
17783
|
+
throw new Error(`cannot renew ${conversation.conversation_id}; its bound terminal agent ` +
|
|
17784
|
+
"process is verified dead. Close this orphaned Turn instead.");
|
|
17785
|
+
}
|
|
17542
17786
|
if (terminalBridgeSubmission(conversation)?.status === "uncertain") {
|
|
17543
17787
|
throw new Error(`cannot renew ${conversation.conversation_id}; its terminal submission ` +
|
|
17544
17788
|
"is uncertain and cannot be attributed by monitoring. Inspect the pane " +
|
|
@@ -17570,6 +17814,10 @@ async function runRenew(options) {
|
|
|
17570
17814
|
if (current.status !== "stalled") {
|
|
17571
17815
|
throw new Error(`cannot renew ${current.conversation_id}; conversation is ${current.status}, not stalled`);
|
|
17572
17816
|
}
|
|
17817
|
+
if (isVerifiedDeadTerminalAgentProcess(current)) {
|
|
17818
|
+
throw new Error(`cannot renew ${current.conversation_id}; its bound terminal agent ` +
|
|
17819
|
+
"process is verified dead");
|
|
17820
|
+
}
|
|
17573
17821
|
if (terminalBridgeSubmission(current)?.status === "uncertain") {
|
|
17574
17822
|
throw new Error(`cannot renew ${current.conversation_id}; its terminal submission is uncertain`);
|
|
17575
17823
|
}
|
|
@@ -17780,6 +18028,47 @@ async function reconcileMonitors(options, { includeCallbackRecovery, reason, con
|
|
|
17780
18028
|
logPath,
|
|
17781
18029
|
options
|
|
17782
18030
|
});
|
|
18031
|
+
const deadProcessStall = await stallAcceptedTurnForVerifiedDeadAgent({
|
|
18032
|
+
options,
|
|
18033
|
+
storeDir,
|
|
18034
|
+
statePath,
|
|
18035
|
+
logPath,
|
|
18036
|
+
expectedConversationId: initialConversation.conversation_id,
|
|
18037
|
+
expectedMessageId: stringValue(isRecord(initialConversation.native_session_takeover)
|
|
18038
|
+
? initialConversation.native_session_takeover
|
|
18039
|
+
.terminal_bridge_message_id
|
|
18040
|
+
: undefined)
|
|
18041
|
+
});
|
|
18042
|
+
if (deadProcessStall.completionPreparation) {
|
|
18043
|
+
const completionPreparation = deadProcessStall.completionPreparation;
|
|
18044
|
+
skipped += 1;
|
|
18045
|
+
if (!completionPreparation.claimed) {
|
|
18046
|
+
items.push({
|
|
18047
|
+
conversation_id: initialConversation.conversation_id,
|
|
18048
|
+
status: "skipped",
|
|
18049
|
+
reason: completionPreparation.reason
|
|
18050
|
+
});
|
|
18051
|
+
}
|
|
18052
|
+
else {
|
|
18053
|
+
const completionResult = runPreparedCallback(completionPreparation.prepared, { emit: false });
|
|
18054
|
+
items.push({
|
|
18055
|
+
conversation_id: initialConversation.conversation_id,
|
|
18056
|
+
status: "recovered",
|
|
18057
|
+
reason: "bound_agent_process_dead_completion_recovered",
|
|
18058
|
+
delivered: completionResult.delivered
|
|
18059
|
+
});
|
|
18060
|
+
}
|
|
18061
|
+
continue;
|
|
18062
|
+
}
|
|
18063
|
+
if (deadProcessStall.stalled) {
|
|
18064
|
+
skipped += 1;
|
|
18065
|
+
items.push({
|
|
18066
|
+
conversation_id: initialConversation.conversation_id,
|
|
18067
|
+
status: "stalled",
|
|
18068
|
+
reason: deadProcessStall.reason
|
|
18069
|
+
});
|
|
18070
|
+
continue;
|
|
18071
|
+
}
|
|
17783
18072
|
const initialTakeover = isRecord(initialConversation.native_session_takeover)
|
|
17784
18073
|
? initialConversation.native_session_takeover
|
|
17785
18074
|
: undefined;
|
|
@@ -18270,116 +18559,710 @@ function terminalBridgeReconciliationEligibility(conversation) {
|
|
|
18270
18559
|
hardDeadlineAtMs
|
|
18271
18560
|
};
|
|
18272
18561
|
}
|
|
18273
|
-
function
|
|
18274
|
-
|
|
18275
|
-
|
|
18276
|
-
? initial.native_session_takeover
|
|
18562
|
+
function terminalAgentProcessDisposition(conversation) {
|
|
18563
|
+
return isRecord(conversation.terminal_agent_process_disposition)
|
|
18564
|
+
? conversation.terminal_agent_process_disposition
|
|
18277
18565
|
: undefined;
|
|
18278
|
-
|
|
18279
|
-
|
|
18566
|
+
}
|
|
18567
|
+
function isVerifiedDeadTerminalAgentProcess(conversation) {
|
|
18568
|
+
return terminalAgentProcessDisposition(conversation)?.status ===
|
|
18569
|
+
"verified_dead";
|
|
18570
|
+
}
|
|
18571
|
+
function verifiedDeadTerminalAgentProcessEvidenceId(proof) {
|
|
18572
|
+
return createHash("sha256")
|
|
18573
|
+
.update(canonicalJson({
|
|
18574
|
+
kind: proof.kind,
|
|
18575
|
+
agent: proof.agent,
|
|
18576
|
+
pid: proof.pid,
|
|
18577
|
+
process_uuid: proof.process_uuid,
|
|
18578
|
+
process_birth: proof.process_birth ?? null,
|
|
18579
|
+
conversation_id: proof.conversation_id,
|
|
18580
|
+
session_id: proof.session_id,
|
|
18581
|
+
turn_id: proof.turn_id,
|
|
18582
|
+
terminal_endpoint: proof.terminal_endpoint,
|
|
18583
|
+
binding_id: proof.binding_id,
|
|
18584
|
+
binding_generation: proof.binding_generation,
|
|
18585
|
+
message_id: proof.message_id
|
|
18586
|
+
}))
|
|
18587
|
+
.digest("hex");
|
|
18588
|
+
}
|
|
18589
|
+
function storedVerifiedDeadTerminalAgentProcessProof({ conversation, storeDir, terminalControl }) {
|
|
18590
|
+
const disposition = terminalAgentProcessDisposition(conversation);
|
|
18591
|
+
if (disposition?.status !== "verified_dead") {
|
|
18592
|
+
return { status: "absent" };
|
|
18593
|
+
}
|
|
18594
|
+
const proof = isRecord(disposition.proof)
|
|
18595
|
+
? disposition.proof
|
|
18280
18596
|
: undefined;
|
|
18281
|
-
|
|
18282
|
-
|
|
18283
|
-
|
|
18597
|
+
const evidenceId = stringValue(disposition.evidence_id);
|
|
18598
|
+
const recordedAt = stringValue(disposition.recorded_at);
|
|
18599
|
+
const takeover = isRecord(conversation.native_session_takeover)
|
|
18600
|
+
? conversation.native_session_takeover
|
|
18601
|
+
: undefined;
|
|
18602
|
+
const submission = terminalBridgeSubmission(conversation);
|
|
18603
|
+
const session = tryLoadManagedSession(storeDir, sessionIdForConversation(conversation));
|
|
18604
|
+
const binding = session?.binding;
|
|
18605
|
+
const candidate = proof;
|
|
18606
|
+
if (!candidate ||
|
|
18607
|
+
candidate.kind !==
|
|
18608
|
+
"exact_pid_absent_from_complete_process_inventory" ||
|
|
18609
|
+
!session ||
|
|
18610
|
+
session.status !== "bound" ||
|
|
18611
|
+
!binding ||
|
|
18612
|
+
candidate.agent !== session.agent ||
|
|
18613
|
+
session.agent !== executorForConversation(conversation).kind ||
|
|
18614
|
+
path.resolve(session.workspace) !== path.resolve(conversation.workspace) ||
|
|
18615
|
+
candidate.pid !== Number(takeover?.terminal_agent_pid) ||
|
|
18616
|
+
candidate.pid !== binding.native_process.pid ||
|
|
18617
|
+
candidate.process_uuid !==
|
|
18618
|
+
stringValue(takeover?.terminal_agent_process_uuid) ||
|
|
18619
|
+
candidate.process_uuid !== binding.native_process.process_uuid ||
|
|
18620
|
+
(candidate.process_birth ?? undefined) !==
|
|
18621
|
+
(stringValue(takeover?.terminal_agent_process_birth) ?? undefined) ||
|
|
18622
|
+
(candidate.process_birth ?? undefined) !==
|
|
18623
|
+
(binding.native_process.process_birth ?? undefined) ||
|
|
18624
|
+
candidate.conversation_id !== conversation.conversation_id ||
|
|
18625
|
+
candidate.session_id !== sessionIdForConversation(conversation) ||
|
|
18626
|
+
candidate.turn_id !== turnIdForConversation(conversation) ||
|
|
18627
|
+
candidate.binding_id !== stringValue(conversation.terminal_binding_id) ||
|
|
18628
|
+
candidate.binding_id !== binding.binding_id ||
|
|
18629
|
+
candidate.binding_generation !==
|
|
18630
|
+
Number(conversation.terminal_binding_generation) ||
|
|
18631
|
+
candidate.binding_generation !== binding.generation ||
|
|
18632
|
+
!terminalControlsShareIncarnation(binding.terminal_control, terminalControl) ||
|
|
18633
|
+
candidate.binding_id !== stringValue(takeover?.terminal_binding_id) ||
|
|
18634
|
+
candidate.binding_generation !==
|
|
18635
|
+
Number(takeover?.terminal_binding_generation) ||
|
|
18636
|
+
candidate.message_id !==
|
|
18637
|
+
stringValue(takeover?.terminal_bridge_message_id) ||
|
|
18638
|
+
candidate.message_id !== stringValue(submission?.message_id) ||
|
|
18639
|
+
submission?.status !== "agent_accepted" ||
|
|
18640
|
+
stringValue(submission?.session_id) !== candidate.session_id ||
|
|
18641
|
+
stringValue(submission?.turn_id) !== candidate.turn_id ||
|
|
18642
|
+
stringValue(submission?.binding_id) !== candidate.binding_id ||
|
|
18643
|
+
Number(submission?.binding_generation) !==
|
|
18644
|
+
candidate.binding_generation ||
|
|
18645
|
+
!isRecord(candidate.terminal_control) ||
|
|
18646
|
+
!terminalControlsShareIncarnation(candidate.terminal_control, terminalControl) ||
|
|
18647
|
+
!terminalControlEvidenceMatches(candidate.terminal_endpoint, terminalControl, { requireProcessAnchor: true }) ||
|
|
18648
|
+
Number(candidate.terminal_endpoint.process_anchor_pid) !==
|
|
18649
|
+
terminalEndpointFromControlRef(terminalControl).processAnchorPid ||
|
|
18650
|
+
validTimestampMs(candidate.observed_at) === undefined ||
|
|
18651
|
+
!recordedAt ||
|
|
18652
|
+
validTimestampMs(recordedAt) === undefined ||
|
|
18653
|
+
recordedAt !== candidate.observed_at ||
|
|
18654
|
+
!evidenceId ||
|
|
18655
|
+
evidenceId !== verifiedDeadTerminalAgentProcessEvidenceId(candidate)) {
|
|
18284
18656
|
return {
|
|
18285
|
-
|
|
18286
|
-
|
|
18287
|
-
reason: "local_completion_not_applicable"
|
|
18657
|
+
status: "invalid",
|
|
18658
|
+
reason: "the persisted process-death proof no longer matches the exact Turn, Session, terminal, or submission binding"
|
|
18288
18659
|
};
|
|
18289
18660
|
}
|
|
18290
|
-
|
|
18291
|
-
|
|
18292
|
-
|
|
18661
|
+
return {
|
|
18662
|
+
status: "valid",
|
|
18663
|
+
proof: candidate,
|
|
18664
|
+
evidenceId,
|
|
18665
|
+
recordedAt
|
|
18666
|
+
};
|
|
18667
|
+
}
|
|
18668
|
+
function eventVerifiedDeadTerminalAgentProcessProof({ conversation, storeDir, terminalControl, logPath }) {
|
|
18669
|
+
const candidates = readExistingEvents(logPath).filter((event) => event.event === "terminal_agent_process_verified_dead" &&
|
|
18670
|
+
event.conversation_id === conversation.conversation_id);
|
|
18671
|
+
if (candidates.length === 0) {
|
|
18672
|
+
return { status: "absent" };
|
|
18293
18673
|
}
|
|
18294
|
-
|
|
18295
|
-
|
|
18296
|
-
|
|
18297
|
-
|
|
18298
|
-
|
|
18299
|
-
|
|
18300
|
-
|
|
18301
|
-
|
|
18302
|
-
|
|
18303
|
-
|
|
18304
|
-
|
|
18305
|
-
|
|
18306
|
-
|
|
18307
|
-
|
|
18308
|
-
|
|
18309
|
-
|
|
18310
|
-
|
|
18311
|
-
|
|
18312
|
-
|
|
18313
|
-
|
|
18314
|
-
|
|
18315
|
-
|
|
18316
|
-
|
|
18317
|
-
|
|
18318
|
-
|
|
18319
|
-
|
|
18320
|
-
|
|
18321
|
-
|
|
18322
|
-
|
|
18323
|
-
|
|
18324
|
-
|
|
18325
|
-
|
|
18326
|
-
|
|
18327
|
-
|
|
18328
|
-
|
|
18329
|
-
|
|
18330
|
-
|
|
18331
|
-
|
|
18332
|
-
|
|
18333
|
-
|
|
18334
|
-
|
|
18335
|
-
|
|
18336
|
-
|
|
18337
|
-
|
|
18338
|
-
|
|
18339
|
-
|
|
18340
|
-
|
|
18341
|
-
|
|
18342
|
-
|
|
18343
|
-
|
|
18344
|
-
|
|
18345
|
-
|
|
18346
|
-
|
|
18347
|
-
|
|
18348
|
-
|
|
18349
|
-
|
|
18350
|
-
|
|
18351
|
-
|
|
18352
|
-
|
|
18353
|
-
|
|
18354
|
-
|
|
18355
|
-
|
|
18356
|
-
|
|
18357
|
-
|
|
18358
|
-
|
|
18359
|
-
|
|
18360
|
-
|
|
18361
|
-
|
|
18362
|
-
|
|
18363
|
-
|
|
18364
|
-
|
|
18365
|
-
|
|
18366
|
-
|
|
18367
|
-
|
|
18368
|
-
|
|
18369
|
-
|
|
18370
|
-
|
|
18371
|
-
|
|
18372
|
-
|
|
18373
|
-
|
|
18374
|
-
|
|
18375
|
-
|
|
18376
|
-
|
|
18377
|
-
|
|
18378
|
-
|
|
18379
|
-
|
|
18380
|
-
|
|
18381
|
-
|
|
18382
|
-
|
|
18674
|
+
if (candidates.length !== 1) {
|
|
18675
|
+
return {
|
|
18676
|
+
status: "invalid",
|
|
18677
|
+
reason: "the process-death event history is ambiguous"
|
|
18678
|
+
};
|
|
18679
|
+
}
|
|
18680
|
+
const event = candidates[0];
|
|
18681
|
+
const proof = isRecord(event.proof)
|
|
18682
|
+
? event.proof
|
|
18683
|
+
: undefined;
|
|
18684
|
+
const evidenceId = stringValue(event.evidence_id);
|
|
18685
|
+
const recordedAt = stringValue(event.ts);
|
|
18686
|
+
if (!proof ||
|
|
18687
|
+
!evidenceId ||
|
|
18688
|
+
!recordedAt ||
|
|
18689
|
+
proof.observed_at !== recordedAt ||
|
|
18690
|
+
verifiedDeadTerminalAgentProcessEvidenceId(proof) !== evidenceId) {
|
|
18691
|
+
return {
|
|
18692
|
+
status: "invalid",
|
|
18693
|
+
reason: "the process-death event proof is malformed"
|
|
18694
|
+
};
|
|
18695
|
+
}
|
|
18696
|
+
const synthetic = {
|
|
18697
|
+
...conversation,
|
|
18698
|
+
terminal_agent_process_disposition: {
|
|
18699
|
+
status: "verified_dead",
|
|
18700
|
+
proof,
|
|
18701
|
+
evidence_id: evidenceId,
|
|
18702
|
+
recorded_at: recordedAt
|
|
18703
|
+
}
|
|
18704
|
+
};
|
|
18705
|
+
return storedVerifiedDeadTerminalAgentProcessProof({
|
|
18706
|
+
conversation: synthetic,
|
|
18707
|
+
storeDir,
|
|
18708
|
+
terminalControl
|
|
18709
|
+
});
|
|
18710
|
+
}
|
|
18711
|
+
function exactVerifiedDeadTerminalAgentProcessAuthority({ conversation, storeDir, terminalControl, logPath }) {
|
|
18712
|
+
const stored = storedVerifiedDeadTerminalAgentProcessProof({
|
|
18713
|
+
conversation,
|
|
18714
|
+
storeDir,
|
|
18715
|
+
terminalControl
|
|
18716
|
+
});
|
|
18717
|
+
if (stored.status === "invalid") {
|
|
18718
|
+
return stored;
|
|
18719
|
+
}
|
|
18720
|
+
const event = eventVerifiedDeadTerminalAgentProcessProof({
|
|
18721
|
+
conversation,
|
|
18722
|
+
storeDir,
|
|
18723
|
+
terminalControl,
|
|
18724
|
+
logPath
|
|
18725
|
+
});
|
|
18726
|
+
if (stored.status === "absent") {
|
|
18727
|
+
return event;
|
|
18728
|
+
}
|
|
18729
|
+
if (event.status !== "valid" ||
|
|
18730
|
+
event.evidenceId !== stored.evidenceId ||
|
|
18731
|
+
canonicalJson(event.proof) !== canonicalJson(stored.proof)) {
|
|
18732
|
+
return {
|
|
18733
|
+
status: "invalid",
|
|
18734
|
+
reason: "the persisted process-death disposition has no exact append-only event"
|
|
18735
|
+
};
|
|
18736
|
+
}
|
|
18737
|
+
return stored;
|
|
18738
|
+
}
|
|
18739
|
+
function ensureVerifiedDeadTerminalAgentProcessEvent({ logPath, proof, action }) {
|
|
18740
|
+
const evidenceId = verifiedDeadTerminalAgentProcessEvidenceId(proof);
|
|
18741
|
+
const existing = readExistingEvents(logPath).find((event) => event.event === "terminal_agent_process_verified_dead" &&
|
|
18742
|
+
event.conversation_id === proof.conversation_id &&
|
|
18743
|
+
event.evidence_id === evidenceId);
|
|
18744
|
+
if (existing) {
|
|
18745
|
+
const existingProof = isRecord(existing.proof)
|
|
18746
|
+
? existing.proof
|
|
18747
|
+
: undefined;
|
|
18748
|
+
const existingAt = stringValue(existing.ts);
|
|
18749
|
+
if (!existingProof ||
|
|
18750
|
+
!existingAt ||
|
|
18751
|
+
validTimestampMs(existingAt) === undefined ||
|
|
18752
|
+
existingProof.observed_at !== existingAt ||
|
|
18753
|
+
verifiedDeadTerminalAgentProcessEvidenceId(existingProof) !== evidenceId) {
|
|
18754
|
+
throw new Error(`verified-dead process event ${evidenceId} is inconsistent`);
|
|
18755
|
+
}
|
|
18756
|
+
return {
|
|
18757
|
+
proof: existingProof,
|
|
18758
|
+
evidenceId,
|
|
18759
|
+
recordedAt: existingAt
|
|
18760
|
+
};
|
|
18761
|
+
}
|
|
18762
|
+
appendEvent(logPath, {
|
|
18763
|
+
ts: proof.observed_at,
|
|
18764
|
+
conversation_id: proof.conversation_id,
|
|
18765
|
+
event: "terminal_agent_process_verified_dead",
|
|
18766
|
+
evidence_id: evidenceId,
|
|
18767
|
+
status: "verified_dead",
|
|
18768
|
+
proof,
|
|
18769
|
+
action
|
|
18770
|
+
});
|
|
18771
|
+
return {
|
|
18772
|
+
proof,
|
|
18773
|
+
evidenceId,
|
|
18774
|
+
recordedAt: proof.observed_at
|
|
18775
|
+
};
|
|
18776
|
+
}
|
|
18777
|
+
function ensureVerifiedDeadConversationStalledEvent({ logPath, proof, evidenceId, reason, terminalControl, completionObservation }) {
|
|
18778
|
+
const existing = readExistingEvents(logPath).find((event) => event.event === "conversation_stalled" &&
|
|
18779
|
+
event.conversation_id === proof.conversation_id &&
|
|
18780
|
+
event.evidence_id === evidenceId);
|
|
18781
|
+
if (existing) {
|
|
18782
|
+
if (existing.ts !== proof.observed_at ||
|
|
18783
|
+
existing.reason !== reason ||
|
|
18784
|
+
existing.disposition !== "verified_dead_agent_process" ||
|
|
18785
|
+
existing.completion_observation !== completionObservation) {
|
|
18786
|
+
throw new Error(`verified-dead stalled event ${evidenceId} is inconsistent`);
|
|
18787
|
+
}
|
|
18788
|
+
return;
|
|
18789
|
+
}
|
|
18790
|
+
appendEvent(logPath, {
|
|
18791
|
+
ts: proof.observed_at,
|
|
18792
|
+
conversation_id: proof.conversation_id,
|
|
18793
|
+
event: "conversation_stalled",
|
|
18794
|
+
evidence_id: evidenceId,
|
|
18795
|
+
status: "stalled",
|
|
18796
|
+
reason,
|
|
18797
|
+
terminal_bridge: true,
|
|
18798
|
+
terminal_control: terminalControl,
|
|
18799
|
+
disposition: "verified_dead_agent_process",
|
|
18800
|
+
completion_observation: completionObservation
|
|
18801
|
+
});
|
|
18802
|
+
}
|
|
18803
|
+
function verifiedDeadConversationStalledEventDecision({ logPath, proof, evidenceId, reason }) {
|
|
18804
|
+
const candidates = readExistingEvents(logPath).filter((event) => event.event === "conversation_stalled" &&
|
|
18805
|
+
event.conversation_id === proof.conversation_id &&
|
|
18806
|
+
event.evidence_id === evidenceId);
|
|
18807
|
+
if (candidates.length === 0) {
|
|
18808
|
+
return { status: "absent" };
|
|
18809
|
+
}
|
|
18810
|
+
if (candidates.length !== 1) {
|
|
18811
|
+
return {
|
|
18812
|
+
status: "invalid",
|
|
18813
|
+
reason: "the verified-dead stalled event history is ambiguous"
|
|
18814
|
+
};
|
|
18815
|
+
}
|
|
18816
|
+
const event = candidates[0];
|
|
18817
|
+
const completionObservation = stringValue(event.completion_observation);
|
|
18818
|
+
if (event.ts !== proof.observed_at ||
|
|
18819
|
+
event.status !== "stalled" ||
|
|
18820
|
+
event.reason !== reason ||
|
|
18821
|
+
event.disposition !== "verified_dead_agent_process" ||
|
|
18822
|
+
!["absent", "unverifiable"].includes(completionObservation ?? "")) {
|
|
18823
|
+
return {
|
|
18824
|
+
status: "invalid",
|
|
18825
|
+
reason: "the verified-dead stalled event decision is inconsistent"
|
|
18826
|
+
};
|
|
18827
|
+
}
|
|
18828
|
+
return {
|
|
18829
|
+
status: "valid",
|
|
18830
|
+
completionObservation: completionObservation
|
|
18831
|
+
};
|
|
18832
|
+
}
|
|
18833
|
+
function ensureVerifiedDeadConversationClosedEvent({ logPath, conversation, evidenceId }) {
|
|
18834
|
+
const closedAt = required(stringValue(conversation.closed_at), "verified-dead closed Turn has no closed_at timestamp");
|
|
18835
|
+
const reason = required(stringValue(conversation.close_reason), "verified-dead closed Turn has no close reason");
|
|
18836
|
+
if (conversation.status !== "closed" ||
|
|
18837
|
+
validTimestampMs(closedAt) === undefined) {
|
|
18838
|
+
throw new Error("verified-dead closed Turn state is inconsistent");
|
|
18839
|
+
}
|
|
18840
|
+
const existing = readExistingEvents(logPath).find((event) => event.event === "conversation_closed" &&
|
|
18841
|
+
event.conversation_id === conversation.conversation_id &&
|
|
18842
|
+
event.evidence_id === evidenceId);
|
|
18843
|
+
if (existing) {
|
|
18844
|
+
if (existing.ts !== closedAt ||
|
|
18845
|
+
existing.status !== "closed" ||
|
|
18846
|
+
existing.reason !== reason ||
|
|
18847
|
+
existing.disposition !== "verified_dead_agent_process") {
|
|
18848
|
+
throw new Error(`verified-dead close event ${evidenceId} is inconsistent`);
|
|
18849
|
+
}
|
|
18850
|
+
return;
|
|
18851
|
+
}
|
|
18852
|
+
appendEvent(logPath, {
|
|
18853
|
+
ts: closedAt,
|
|
18854
|
+
conversation_id: conversation.conversation_id,
|
|
18855
|
+
event: "conversation_closed",
|
|
18856
|
+
evidence_id: evidenceId,
|
|
18857
|
+
status: "closed",
|
|
18858
|
+
reason,
|
|
18859
|
+
disposition: "verified_dead_agent_process"
|
|
18860
|
+
});
|
|
18861
|
+
}
|
|
18862
|
+
function acceptedTurnCanBeStalledForDeadAgent({ storeDir, conversation }) {
|
|
18863
|
+
if (conversation.status !== "waiting_for_agent") {
|
|
18864
|
+
return false;
|
|
18865
|
+
}
|
|
18866
|
+
const takeover = isRecord(conversation.native_session_takeover)
|
|
18867
|
+
? conversation.native_session_takeover
|
|
18868
|
+
: undefined;
|
|
18869
|
+
const messageId = stringValue(takeover?.terminal_bridge_message_id);
|
|
18870
|
+
const submission = terminalBridgeSubmission(conversation);
|
|
18871
|
+
if (takeover?.terminal_bridge !== true ||
|
|
18872
|
+
!messageId ||
|
|
18873
|
+
submission?.status !== "agent_accepted" ||
|
|
18874
|
+
stringValue(submission.message_id) !== messageId) {
|
|
18875
|
+
return false;
|
|
18876
|
+
}
|
|
18877
|
+
const transferId = stringValue(takeover.deferred_foreground_transfer_id);
|
|
18878
|
+
if (!transferId) {
|
|
18879
|
+
return true;
|
|
18880
|
+
}
|
|
18881
|
+
const transfer = loadDeferredForegroundTransfer(storeDir, transferId);
|
|
18882
|
+
return transfer.status === "resolved";
|
|
18883
|
+
}
|
|
18884
|
+
async function stallAcceptedTurnForVerifiedDeadAgent({ options, storeDir, statePath, logPath, expectedConversationId, expectedMessageId }) {
|
|
18885
|
+
const canonicalStoreDir = pathsForConversationDir(path.dirname(statePath)).storeDir;
|
|
18886
|
+
if (path.resolve(storeDir) !== path.resolve(canonicalStoreDir)) {
|
|
18887
|
+
const conversation = loadState(statePath);
|
|
18888
|
+
return {
|
|
18889
|
+
stalled: false,
|
|
18890
|
+
conversation,
|
|
18891
|
+
reason: "dead_process_stall_store_mismatch"
|
|
18892
|
+
};
|
|
18893
|
+
}
|
|
18894
|
+
const initial = loadState(statePath);
|
|
18895
|
+
const initialTakeover = isRecord(initial.native_session_takeover)
|
|
18896
|
+
? initial.native_session_takeover
|
|
18897
|
+
: undefined;
|
|
18898
|
+
const terminalControl = terminalControlFromTakeover(initialTakeover);
|
|
18899
|
+
if (initial.conversation_id !== expectedConversationId ||
|
|
18900
|
+
!terminalControl ||
|
|
18901
|
+
!acceptedTurnCanBeStalledForDeadAgent({ storeDir, conversation: initial })) {
|
|
18902
|
+
return {
|
|
18903
|
+
stalled: false,
|
|
18904
|
+
conversation: initial,
|
|
18905
|
+
reason: "dead_process_stall_not_applicable"
|
|
18906
|
+
};
|
|
18907
|
+
}
|
|
18908
|
+
const releaseTerminalLock = acquireTerminalBridgeSendLock(storeDir, terminalControl, { timeoutMs: 30000 });
|
|
18909
|
+
try {
|
|
18910
|
+
return await withStoreWriterLeaseAsync(storeDir, async () => {
|
|
18911
|
+
const releaseStateLock = acquireFileLock(`${statePath}.lock`);
|
|
18912
|
+
let stateLockReleased = false;
|
|
18913
|
+
try {
|
|
18914
|
+
const current = loadState(statePath);
|
|
18915
|
+
const takeover = isRecord(current.native_session_takeover)
|
|
18916
|
+
? current.native_session_takeover
|
|
18917
|
+
: undefined;
|
|
18918
|
+
const currentControl = terminalControlFromTakeover(takeover);
|
|
18919
|
+
const messageId = stringValue(takeover?.terminal_bridge_message_id);
|
|
18920
|
+
if (current.conversation_id !== expectedConversationId ||
|
|
18921
|
+
(expectedMessageId !== undefined && messageId !== expectedMessageId) ||
|
|
18922
|
+
!currentControl ||
|
|
18923
|
+
!terminalControlsShareIncarnation(currentControl, terminalControl) ||
|
|
18924
|
+
!acceptedTurnCanBeStalledForDeadAgent({
|
|
18925
|
+
storeDir,
|
|
18926
|
+
conversation: current
|
|
18927
|
+
})) {
|
|
18928
|
+
return {
|
|
18929
|
+
stalled: false,
|
|
18930
|
+
conversation: current,
|
|
18931
|
+
reason: "dead_process_stall_generation_changed"
|
|
18932
|
+
};
|
|
18933
|
+
}
|
|
18934
|
+
assertTurnBindingCurrent(current, "stall a verified-dead agent Turn for");
|
|
18935
|
+
const ledger = loadTerminalBridgeDispatchLedger(currentControl);
|
|
18936
|
+
if (!ledger ||
|
|
18937
|
+
stringValue(ledger.status) !== "agent_accepted" ||
|
|
18938
|
+
!sameCanonicalStatePath(ledger.state_path, statePath) ||
|
|
18939
|
+
path.resolve(stringValue(ledger.store_dir) ?? "") !==
|
|
18940
|
+
path.resolve(storeDir) ||
|
|
18941
|
+
path.resolve(stringValue(ledger.event_log_path) ?? "") !==
|
|
18942
|
+
path.resolve(logPath) ||
|
|
18943
|
+
stringValue(ledger.conversation_id) !== current.conversation_id ||
|
|
18944
|
+
stringValue(ledger.session_id) !== sessionIdForConversation(current) ||
|
|
18945
|
+
stringValue(ledger.turn_id) !== turnIdForConversation(current) ||
|
|
18946
|
+
stringValue(ledger.message_id) !== messageId ||
|
|
18947
|
+
!terminalDispatchRecordMatchesControl(ledger, currentControl, {
|
|
18948
|
+
requireProcessAnchor: true
|
|
18949
|
+
}) ||
|
|
18950
|
+
terminalDispatchRecordProcessAnchor(ledger) !==
|
|
18951
|
+
terminalEndpointFromControlRef(currentControl).processAnchorPid ||
|
|
18952
|
+
stringValue(ledger.binding_id) !==
|
|
18953
|
+
stringValue(current.terminal_binding_id) ||
|
|
18954
|
+
Number(ledger.binding_generation) !==
|
|
18955
|
+
Number(current.terminal_binding_generation)) {
|
|
18956
|
+
return {
|
|
18957
|
+
stalled: false,
|
|
18958
|
+
conversation: current,
|
|
18959
|
+
reason: "dead_process_stall_dispatch_changed"
|
|
18960
|
+
};
|
|
18961
|
+
}
|
|
18962
|
+
try {
|
|
18963
|
+
assertVerifiedDeadTerminalBridgeDispatchAuthority({
|
|
18964
|
+
terminalControl: currentControl,
|
|
18965
|
+
conversation: current,
|
|
18966
|
+
storeDir,
|
|
18967
|
+
statePath,
|
|
18968
|
+
logPath,
|
|
18969
|
+
expectedMessageId: required(messageId, "verified-dead stall has no terminal message id")
|
|
18970
|
+
});
|
|
18971
|
+
}
|
|
18972
|
+
catch (error) {
|
|
18973
|
+
return {
|
|
18974
|
+
stalled: false,
|
|
18975
|
+
conversation: current,
|
|
18976
|
+
reason: "dead_process_stall_dispatch_changed: " +
|
|
18977
|
+
(error instanceof Error ? error.message : String(error))
|
|
18978
|
+
};
|
|
18979
|
+
}
|
|
18980
|
+
const persistedAuthority = exactVerifiedDeadTerminalAgentProcessAuthority({
|
|
18981
|
+
conversation: current,
|
|
18982
|
+
storeDir,
|
|
18983
|
+
terminalControl: currentControl,
|
|
18984
|
+
logPath
|
|
18985
|
+
});
|
|
18986
|
+
if (persistedAuthority.status === "invalid") {
|
|
18987
|
+
return {
|
|
18988
|
+
stalled: false,
|
|
18989
|
+
conversation: current,
|
|
18990
|
+
reason: `bound_agent_process_evidence_invalid: ${persistedAuthority.reason}`
|
|
18991
|
+
};
|
|
18992
|
+
}
|
|
18993
|
+
let proof;
|
|
18994
|
+
if (persistedAuthority.status === "valid") {
|
|
18995
|
+
proof = persistedAuthority.proof;
|
|
18996
|
+
}
|
|
18997
|
+
else {
|
|
18998
|
+
const observation = await observeBoundTerminalAgentProcess({
|
|
18999
|
+
options,
|
|
19000
|
+
conversation: current,
|
|
19001
|
+
terminalControl: currentControl
|
|
19002
|
+
});
|
|
19003
|
+
if (observation.status !== "verified_dead") {
|
|
19004
|
+
return {
|
|
19005
|
+
stalled: false,
|
|
19006
|
+
conversation: current,
|
|
19007
|
+
reason: observation.status === "alive"
|
|
19008
|
+
? "bound_agent_process_alive"
|
|
19009
|
+
: `bound_agent_process_unverifiable: ${observation.reason}`
|
|
19010
|
+
};
|
|
19011
|
+
}
|
|
19012
|
+
proof = observation.proof;
|
|
19013
|
+
}
|
|
19014
|
+
const stalledReason = "bound terminal agent process is verified dead";
|
|
19015
|
+
const priorStalledDecision = verifiedDeadConversationStalledEventDecision({
|
|
19016
|
+
logPath,
|
|
19017
|
+
proof,
|
|
19018
|
+
evidenceId: verifiedDeadTerminalAgentProcessEvidenceId(proof),
|
|
19019
|
+
reason: stalledReason
|
|
19020
|
+
});
|
|
19021
|
+
if (priorStalledDecision.status === "invalid") {
|
|
19022
|
+
return {
|
|
19023
|
+
stalled: false,
|
|
19024
|
+
conversation: current,
|
|
19025
|
+
reason: `bound_agent_process_evidence_invalid: ${priorStalledDecision.reason}`
|
|
19026
|
+
};
|
|
19027
|
+
}
|
|
19028
|
+
if (priorStalledDecision.status === "valid") {
|
|
19029
|
+
if (persistedAuthority.status !== "valid") {
|
|
19030
|
+
return {
|
|
19031
|
+
stalled: false,
|
|
19032
|
+
conversation: current,
|
|
19033
|
+
reason: "bound_agent_process_evidence_invalid: the stalled decision has no exact death event"
|
|
19034
|
+
};
|
|
19035
|
+
}
|
|
19036
|
+
const stalled = {
|
|
19037
|
+
...current,
|
|
19038
|
+
status: "stalled",
|
|
19039
|
+
stalled_at: persistedAuthority.recordedAt,
|
|
19040
|
+
stalled_reason: stalledReason,
|
|
19041
|
+
terminal_agent_process_disposition: {
|
|
19042
|
+
status: "verified_dead",
|
|
19043
|
+
proof: persistedAuthority.proof,
|
|
19044
|
+
evidence_id: persistedAuthority.evidenceId,
|
|
19045
|
+
recorded_at: persistedAuthority.recordedAt,
|
|
19046
|
+
completion_observation: {
|
|
19047
|
+
status: priorStalledDecision.completionObservation
|
|
19048
|
+
}
|
|
19049
|
+
},
|
|
19050
|
+
updated_at: persistedAuthority.recordedAt
|
|
19051
|
+
};
|
|
19052
|
+
saveState(statePath, stalled);
|
|
19053
|
+
return {
|
|
19054
|
+
stalled: true,
|
|
19055
|
+
conversation: stalled,
|
|
19056
|
+
reason: priorStalledDecision.completionObservation === "unverifiable"
|
|
19057
|
+
? "bound_agent_process_verified_dead_completion_unverifiable"
|
|
19058
|
+
: "bound_agent_process_verified_dead"
|
|
19059
|
+
};
|
|
19060
|
+
}
|
|
19061
|
+
const durableCompletion = await observeDurableCompletionBeforeDeadStall({
|
|
19062
|
+
options,
|
|
19063
|
+
conversation: current,
|
|
19064
|
+
terminalControl: currentControl
|
|
19065
|
+
});
|
|
19066
|
+
if (durableCompletion.status === "present") {
|
|
19067
|
+
// Completion claiming re-enters the Turn state lock. Keep the
|
|
19068
|
+
// canonical terminal and Store-writer locks, but release the state
|
|
19069
|
+
// lock before preparing the callback/local completion atomically.
|
|
19070
|
+
releaseStateLock();
|
|
19071
|
+
stateLockReleased = true;
|
|
19072
|
+
const completionPreparation = prepareTerminalBridgeCompletionCallbackWithLocksHeld({
|
|
19073
|
+
options,
|
|
19074
|
+
statePath,
|
|
19075
|
+
logPath,
|
|
19076
|
+
conversation: current,
|
|
19077
|
+
executor: executorForConversation(current),
|
|
19078
|
+
terminalControl: currentControl,
|
|
19079
|
+
terminalMessageId: required(messageId, "verified-dead completion has no terminal message id"),
|
|
19080
|
+
completion: durableCompletion.completion
|
|
19081
|
+
});
|
|
19082
|
+
return {
|
|
19083
|
+
stalled: false,
|
|
19084
|
+
conversation: completionPreparation.conversation,
|
|
19085
|
+
reason: completionPreparation.claimed
|
|
19086
|
+
? "bound_agent_process_dead_completion_prepared"
|
|
19087
|
+
: `bound_agent_process_dead_completion_${completionPreparation.reason}`,
|
|
19088
|
+
completionPreparation
|
|
19089
|
+
};
|
|
19090
|
+
}
|
|
19091
|
+
const completionUnverifiable = durableCompletion.status === "unverifiable";
|
|
19092
|
+
// Keep the durable lifecycle reason independent of a transient
|
|
19093
|
+
// completion-read outcome. If the process crashes between the
|
|
19094
|
+
// append-only events and state save, a later reconciliation must be
|
|
19095
|
+
// able to reuse the same death evidence even when the transcript or
|
|
19096
|
+
// rollout has become readable in the meantime.
|
|
19097
|
+
const reason = stalledReason;
|
|
19098
|
+
const audit = ensureVerifiedDeadTerminalAgentProcessEvent({
|
|
19099
|
+
logPath,
|
|
19100
|
+
proof,
|
|
19101
|
+
action: "monitor_reconciliation"
|
|
19102
|
+
});
|
|
19103
|
+
ensureVerifiedDeadConversationStalledEvent({
|
|
19104
|
+
logPath,
|
|
19105
|
+
proof: audit.proof,
|
|
19106
|
+
evidenceId: audit.evidenceId,
|
|
19107
|
+
reason,
|
|
19108
|
+
terminalControl: currentControl,
|
|
19109
|
+
completionObservation: completionUnverifiable ? "unverifiable" : "absent"
|
|
19110
|
+
});
|
|
19111
|
+
if (cliEnv().AKK_TEST_EXIT_AFTER_VERIFIED_DEAD_STALL_EVENTS === "1") {
|
|
19112
|
+
cliExit(86);
|
|
19113
|
+
}
|
|
19114
|
+
const stalled = {
|
|
19115
|
+
...current,
|
|
19116
|
+
status: "stalled",
|
|
19117
|
+
stalled_at: audit.recordedAt,
|
|
19118
|
+
stalled_reason: reason,
|
|
19119
|
+
terminal_agent_process_disposition: {
|
|
19120
|
+
status: "verified_dead",
|
|
19121
|
+
proof: audit.proof,
|
|
19122
|
+
evidence_id: audit.evidenceId,
|
|
19123
|
+
recorded_at: audit.recordedAt,
|
|
19124
|
+
completion_observation: {
|
|
19125
|
+
status: completionUnverifiable ? "unverifiable" : "absent"
|
|
19126
|
+
}
|
|
19127
|
+
},
|
|
19128
|
+
updated_at: audit.recordedAt
|
|
19129
|
+
};
|
|
19130
|
+
saveState(statePath, stalled);
|
|
19131
|
+
runtimeLog("warn", "terminal_agent_process_verified_dead", {
|
|
19132
|
+
conversation_id: current.conversation_id,
|
|
19133
|
+
terminal_target: currentControl.target,
|
|
19134
|
+
pid: audit.proof.pid,
|
|
19135
|
+
reason
|
|
19136
|
+
});
|
|
19137
|
+
return {
|
|
19138
|
+
stalled: true,
|
|
19139
|
+
conversation: stalled,
|
|
19140
|
+
reason: completionUnverifiable
|
|
19141
|
+
? "bound_agent_process_verified_dead_completion_unverifiable"
|
|
19142
|
+
: "bound_agent_process_verified_dead"
|
|
19143
|
+
};
|
|
19144
|
+
}
|
|
19145
|
+
finally {
|
|
19146
|
+
if (!stateLockReleased) {
|
|
19147
|
+
releaseStateLock();
|
|
19148
|
+
}
|
|
19149
|
+
}
|
|
19150
|
+
});
|
|
19151
|
+
}
|
|
19152
|
+
finally {
|
|
19153
|
+
releaseTerminalLock();
|
|
19154
|
+
}
|
|
19155
|
+
}
|
|
19156
|
+
function settleLocalTerminalBridgeCompletionClaim({ storeDir, statePath, logPath }) {
|
|
19157
|
+
const initial = loadState(statePath);
|
|
19158
|
+
const initialTakeover = isRecord(initial.native_session_takeover)
|
|
19159
|
+
? initial.native_session_takeover
|
|
19160
|
+
: undefined;
|
|
19161
|
+
const initialClaim = isRecord(initialTakeover?.terminal_bridge_completion_claim)
|
|
19162
|
+
? initialTakeover.terminal_bridge_completion_claim
|
|
19163
|
+
: undefined;
|
|
19164
|
+
if (stringValue(initial.gateway_method) ||
|
|
19165
|
+
!initialClaim ||
|
|
19166
|
+
!["idle", "failed"].includes(String(initial.status))) {
|
|
19167
|
+
return {
|
|
19168
|
+
handled: false,
|
|
19169
|
+
recovered: false,
|
|
19170
|
+
reason: "local_completion_not_applicable"
|
|
19171
|
+
};
|
|
19172
|
+
}
|
|
19173
|
+
const initialTerminalControl = terminalControlFromTakeover(initialTakeover);
|
|
19174
|
+
if (!initialTerminalControl) {
|
|
19175
|
+
throw new Error(`local terminal completion ${initial.conversation_id} lost its terminal authority`);
|
|
19176
|
+
}
|
|
19177
|
+
const releaseTerminalLock = acquireTerminalBridgeSendLock(storeDir, initialTerminalControl, { timeoutMs: 30000 });
|
|
19178
|
+
try {
|
|
19179
|
+
return withStoreWriterLease(storeDir, () => {
|
|
19180
|
+
const releaseStateLock = acquireFileLock(`${statePath}.lock`);
|
|
19181
|
+
try {
|
|
19182
|
+
const conversation = loadState(statePath);
|
|
19183
|
+
const takeover = isRecord(conversation.native_session_takeover)
|
|
19184
|
+
? conversation.native_session_takeover
|
|
19185
|
+
: undefined;
|
|
19186
|
+
const claim = isRecord(takeover?.terminal_bridge_completion_claim)
|
|
19187
|
+
? takeover.terminal_bridge_completion_claim
|
|
19188
|
+
: undefined;
|
|
19189
|
+
const terminalControl = terminalControlFromTakeover(takeover);
|
|
19190
|
+
const terminalMessageId = stringValue(takeover?.terminal_bridge_message_id);
|
|
19191
|
+
const callbackMessageId = stringValue(claim?.callback_message_id);
|
|
19192
|
+
const completionFingerprint = stringValue(claim?.completion_fingerprint);
|
|
19193
|
+
const completionId = stringValue(claim?.completion_id);
|
|
19194
|
+
const claimedAt = stringValue(claim?.claimed_at);
|
|
19195
|
+
const outcome = stringValue(claim?.outcome);
|
|
19196
|
+
const expectedMessageType = outcome === "success"
|
|
19197
|
+
? "done"
|
|
19198
|
+
: outcome === "failure"
|
|
19199
|
+
? "error"
|
|
19200
|
+
: undefined;
|
|
19201
|
+
if (stringValue(conversation.gateway_method) ||
|
|
19202
|
+
conversation.callback_delivery !== undefined ||
|
|
19203
|
+
!claim ||
|
|
19204
|
+
!terminalControl ||
|
|
19205
|
+
!terminalControlsShareIncarnation(terminalControl, initialTerminalControl) ||
|
|
19206
|
+
!terminalMessageId ||
|
|
19207
|
+
!callbackMessageId ||
|
|
19208
|
+
!completionFingerprint ||
|
|
19209
|
+
!completionId ||
|
|
19210
|
+
!claimedAt ||
|
|
19211
|
+
validTimestampMs(claimedAt) === undefined ||
|
|
19212
|
+
!expectedMessageType ||
|
|
19213
|
+
(outcome === "success" && conversation.status !== "idle") ||
|
|
19214
|
+
(outcome === "failure" && conversation.status !== "failed") ||
|
|
19215
|
+
deterministicTerminalCallbackMessageId({
|
|
19216
|
+
conversationId: conversation.conversation_id,
|
|
19217
|
+
terminalMessageId,
|
|
19218
|
+
completionFingerprint,
|
|
19219
|
+
outcome
|
|
19220
|
+
}) !== callbackMessageId) {
|
|
19221
|
+
throw new Error(`local terminal completion ${conversation.conversation_id} has ` +
|
|
19222
|
+
"inconsistent claim, Turn phase, or callback authority");
|
|
19223
|
+
}
|
|
19224
|
+
const submission = terminalBridgeSubmission(conversation);
|
|
19225
|
+
if (submission?.status !== "agent_accepted" ||
|
|
19226
|
+
stringValue(submission.message_id) !== terminalMessageId ||
|
|
19227
|
+
stringValue(submission.session_id) !==
|
|
19228
|
+
sessionIdForConversation(conversation) ||
|
|
19229
|
+
stringValue(submission.turn_id) !== turnIdForConversation(conversation)) {
|
|
19230
|
+
throw new Error(`local terminal completion ${conversation.conversation_id} is ` +
|
|
19231
|
+
"not tied to one accepted terminal submission");
|
|
19232
|
+
}
|
|
19233
|
+
const events = readExistingEvents(logPath);
|
|
19234
|
+
const claimed = events.some((event) => event.event === "terminal_bridge_completion_claimed" &&
|
|
19235
|
+
event.conversation_id === conversation.conversation_id &&
|
|
19236
|
+
event.terminal_bridge_message_id === terminalMessageId &&
|
|
19237
|
+
event.completion_fingerprint === completionFingerprint &&
|
|
19238
|
+
event.completion_id === completionId &&
|
|
19239
|
+
event.callback_message_id === callbackMessageId &&
|
|
19240
|
+
event.outcome === outcome &&
|
|
19241
|
+
event.ts === claimedAt);
|
|
19242
|
+
const detected = events.some((event) => event.event === "terminal_bridge_completion_detected" &&
|
|
19243
|
+
event.conversation_id === conversation.conversation_id &&
|
|
19244
|
+
event.terminal_bridge_message_id === terminalMessageId &&
|
|
19245
|
+
event.completion_id === completionId &&
|
|
19246
|
+
event.callback_message_id === callbackMessageId &&
|
|
19247
|
+
event.completion_outcome === outcome);
|
|
19248
|
+
const messageRecorded = events.some((event) => {
|
|
19249
|
+
const message = isRecord(event.message) ? event.message : undefined;
|
|
19250
|
+
const metadata = isRecord(message?.metadata)
|
|
19251
|
+
? message.metadata
|
|
19252
|
+
: undefined;
|
|
19253
|
+
return event.event === "message" &&
|
|
19254
|
+
event.conversation_id === conversation.conversation_id &&
|
|
19255
|
+
event.session_id === sessionIdForConversation(conversation) &&
|
|
19256
|
+
event.turn_id === turnIdForConversation(conversation) &&
|
|
19257
|
+
message?.id === callbackMessageId &&
|
|
19258
|
+
message?.type === expectedMessageType &&
|
|
19259
|
+
message?.to === "openclaw" &&
|
|
19260
|
+
message?.requires_response === false &&
|
|
19261
|
+
metadata?.terminal_bridge_message_id === terminalMessageId;
|
|
19262
|
+
});
|
|
19263
|
+
if (!claimed || !detected || !messageRecorded) {
|
|
19264
|
+
throw new Error(`local terminal completion ${conversation.conversation_id} is ` +
|
|
19265
|
+
"missing exact claim, detection, or message evidence");
|
|
18383
19266
|
}
|
|
18384
19267
|
const ledger = loadTerminalBridgeDispatchLedger(terminalControl);
|
|
18385
19268
|
const expectedBindingId = stringValue(conversation.terminal_binding_id);
|
|
@@ -18913,6 +19796,7 @@ async function runClose(options) {
|
|
|
18913
19796
|
const releaseStateLock = acquireFileLock(`${statePath}.lock`);
|
|
18914
19797
|
try {
|
|
18915
19798
|
const conversation = loadState(statePath);
|
|
19799
|
+
let verifiedDeadProcess;
|
|
18916
19800
|
assertConversationHasNoNonterminalDeferredForegroundTransfer({
|
|
18917
19801
|
storeDir: closeStoreDir,
|
|
18918
19802
|
conversation,
|
|
@@ -18926,34 +19810,120 @@ async function runClose(options) {
|
|
|
18926
19810
|
!terminalControlsShareIncarnation(currentTerminalControl, terminalControl)) {
|
|
18927
19811
|
throw new Error("terminal control changed after the close action was listed; refresh AKK list");
|
|
18928
19812
|
}
|
|
18929
|
-
if (
|
|
18930
|
-
|
|
18931
|
-
options,
|
|
18932
|
-
storeDir: closeStoreDir,
|
|
19813
|
+
if (conversation.status === "closed" && currentTerminalControl) {
|
|
19814
|
+
const recoveredAuthority = exactVerifiedDeadTerminalAgentProcessAuthority({
|
|
18933
19815
|
conversation,
|
|
18934
|
-
|
|
19816
|
+
storeDir: closeStoreDir,
|
|
19817
|
+
terminalControl: currentTerminalControl,
|
|
19818
|
+
logPath
|
|
18935
19819
|
});
|
|
19820
|
+
if (recoveredAuthority.status === "invalid") {
|
|
19821
|
+
throw new Error(`cannot finish close recovery for ${conversation.conversation_id}; ` +
|
|
19822
|
+
recoveredAuthority.reason);
|
|
19823
|
+
}
|
|
19824
|
+
if (recoveredAuthority.status === "valid") {
|
|
19825
|
+
const audit = ensureVerifiedDeadTerminalAgentProcessEvent({
|
|
19826
|
+
logPath,
|
|
19827
|
+
proof: recoveredAuthority.proof,
|
|
19828
|
+
action: "managed_close"
|
|
19829
|
+
});
|
|
19830
|
+
const expectedMessageId = required(stringValue(currentTakeover?.terminal_bridge_message_id), "verified-dead close recovery has no terminal message id");
|
|
19831
|
+
const dispatchLedgerResolved = resolveVerifiedDeadTerminalBridgeDispatchLedger({
|
|
19832
|
+
terminalControl: currentTerminalControl,
|
|
19833
|
+
conversation,
|
|
19834
|
+
storeDir: closeStoreDir,
|
|
19835
|
+
statePath,
|
|
19836
|
+
logPath,
|
|
19837
|
+
expectedMessageId,
|
|
19838
|
+
reason: "conversation explicitly closed by request"
|
|
19839
|
+
});
|
|
19840
|
+
ensureVerifiedDeadConversationClosedEvent({
|
|
19841
|
+
logPath,
|
|
19842
|
+
conversation,
|
|
19843
|
+
evidenceId: audit.evidenceId
|
|
19844
|
+
});
|
|
19845
|
+
printJson({
|
|
19846
|
+
conversation,
|
|
19847
|
+
closed: true,
|
|
19848
|
+
recovered: true,
|
|
19849
|
+
terminal_dispatch_resolved: dispatchLedgerResolved
|
|
19850
|
+
});
|
|
19851
|
+
return;
|
|
19852
|
+
}
|
|
19853
|
+
}
|
|
19854
|
+
if (terminalControl && currentTerminalControl) {
|
|
19855
|
+
verifiedDeadProcess =
|
|
19856
|
+
await assertGenericCloseDoesNotBypassObservedHandoff({
|
|
19857
|
+
options,
|
|
19858
|
+
storeDir: closeStoreDir,
|
|
19859
|
+
conversation,
|
|
19860
|
+
terminalControl: currentTerminalControl
|
|
19861
|
+
});
|
|
18936
19862
|
}
|
|
18937
19863
|
const now = cliNow().toISOString();
|
|
18938
19864
|
const closeReason = options.reason ?? "closed by request";
|
|
19865
|
+
const verifiedDeadAudit = verifiedDeadProcess
|
|
19866
|
+
? ensureVerifiedDeadTerminalAgentProcessEvent({
|
|
19867
|
+
logPath,
|
|
19868
|
+
proof: verifiedDeadProcess,
|
|
19869
|
+
action: "managed_close"
|
|
19870
|
+
})
|
|
19871
|
+
: undefined;
|
|
19872
|
+
const verifiedDeadMessageId = verifiedDeadAudit
|
|
19873
|
+
? required(stringValue(currentTakeover?.terminal_bridge_message_id), "verified-dead close has no terminal message id")
|
|
19874
|
+
: undefined;
|
|
19875
|
+
if (verifiedDeadAudit && currentTerminalControl) {
|
|
19876
|
+
assertVerifiedDeadTerminalBridgeDispatchAuthority({
|
|
19877
|
+
terminalControl: currentTerminalControl,
|
|
19878
|
+
conversation,
|
|
19879
|
+
storeDir: closeStoreDir,
|
|
19880
|
+
statePath,
|
|
19881
|
+
logPath,
|
|
19882
|
+
expectedMessageId: verifiedDeadMessageId
|
|
19883
|
+
});
|
|
19884
|
+
}
|
|
18939
19885
|
const closed = {
|
|
18940
19886
|
...conversation,
|
|
18941
19887
|
status: "closed",
|
|
18942
19888
|
closed_at: now,
|
|
18943
19889
|
close_reason: closeReason,
|
|
19890
|
+
...(verifiedDeadAudit
|
|
19891
|
+
? {
|
|
19892
|
+
terminal_agent_process_disposition: {
|
|
19893
|
+
status: "verified_dead",
|
|
19894
|
+
proof: verifiedDeadAudit.proof,
|
|
19895
|
+
evidence_id: verifiedDeadAudit.evidenceId,
|
|
19896
|
+
recorded_at: verifiedDeadAudit.recordedAt
|
|
19897
|
+
}
|
|
19898
|
+
}
|
|
19899
|
+
: {}),
|
|
18944
19900
|
updated_at: now
|
|
18945
19901
|
};
|
|
18946
19902
|
saveState(statePath, closed);
|
|
19903
|
+
if (verifiedDeadAudit &&
|
|
19904
|
+
cliEnv().AKK_TEST_EXIT_AFTER_VERIFIED_DEAD_CLOSE_STATE === "1") {
|
|
19905
|
+
cliExit(86);
|
|
19906
|
+
}
|
|
18947
19907
|
let dispatchLedgerResolved = false;
|
|
18948
19908
|
let dispatchLedgerWarning;
|
|
18949
19909
|
if (currentTerminalControl) {
|
|
18950
19910
|
try {
|
|
18951
|
-
dispatchLedgerResolved =
|
|
18952
|
-
|
|
18953
|
-
|
|
18954
|
-
|
|
18955
|
-
|
|
18956
|
-
|
|
19911
|
+
dispatchLedgerResolved = verifiedDeadAudit
|
|
19912
|
+
? resolveVerifiedDeadTerminalBridgeDispatchLedger({
|
|
19913
|
+
terminalControl: currentTerminalControl,
|
|
19914
|
+
conversation: closed,
|
|
19915
|
+
storeDir: closeStoreDir,
|
|
19916
|
+
statePath,
|
|
19917
|
+
logPath,
|
|
19918
|
+
expectedMessageId: verifiedDeadMessageId,
|
|
19919
|
+
reason: "conversation explicitly closed by request"
|
|
19920
|
+
})
|
|
19921
|
+
: resolveTerminalBridgeDispatchLedger({
|
|
19922
|
+
terminalControl: currentTerminalControl,
|
|
19923
|
+
conversation: closed,
|
|
19924
|
+
expectedMessageId: stringValue(currentTakeover?.terminal_bridge_message_id),
|
|
19925
|
+
reason: "conversation explicitly closed by request"
|
|
19926
|
+
});
|
|
18957
19927
|
}
|
|
18958
19928
|
catch (error) {
|
|
18959
19929
|
dispatchLedgerWarning =
|
|
@@ -18965,13 +19935,22 @@ async function runClose(options) {
|
|
|
18965
19935
|
});
|
|
18966
19936
|
}
|
|
18967
19937
|
}
|
|
18968
|
-
|
|
18969
|
-
|
|
18970
|
-
|
|
18971
|
-
|
|
18972
|
-
|
|
18973
|
-
|
|
18974
|
-
}
|
|
19938
|
+
if (verifiedDeadAudit) {
|
|
19939
|
+
ensureVerifiedDeadConversationClosedEvent({
|
|
19940
|
+
logPath,
|
|
19941
|
+
conversation: closed,
|
|
19942
|
+
evidenceId: verifiedDeadAudit.evidenceId
|
|
19943
|
+
});
|
|
19944
|
+
}
|
|
19945
|
+
else {
|
|
19946
|
+
appendEvent(logPath, {
|
|
19947
|
+
ts: now,
|
|
19948
|
+
conversation_id: conversation.conversation_id,
|
|
19949
|
+
event: "conversation_closed",
|
|
19950
|
+
status: "closed",
|
|
19951
|
+
reason: closed.close_reason
|
|
19952
|
+
});
|
|
19953
|
+
}
|
|
18975
19954
|
runtimeLog("info", "conversation_closed", {
|
|
18976
19955
|
conversation_id: conversation.conversation_id,
|
|
18977
19956
|
status: "closed",
|
|
@@ -19017,6 +19996,19 @@ async function assertGenericCloseDoesNotBypassObservedHandoff({ options, storeDi
|
|
|
19017
19996
|
!terminalControlsShareIncarnation(sourceSession.binding.terminal_control, terminalControl)) {
|
|
19018
19997
|
return;
|
|
19019
19998
|
}
|
|
19999
|
+
const storedProof = exactVerifiedDeadTerminalAgentProcessAuthority({
|
|
20000
|
+
conversation,
|
|
20001
|
+
storeDir,
|
|
20002
|
+
terminalControl,
|
|
20003
|
+
logPath: stringValue(conversation.event_log_path) ??
|
|
20004
|
+
logPathForStatePath(required(stringValue(conversation.state_path), "managed Turn state path is unavailable"))
|
|
20005
|
+
});
|
|
20006
|
+
if (storedProof.status === "valid") {
|
|
20007
|
+
return storedProof.proof;
|
|
20008
|
+
}
|
|
20009
|
+
if (storedProof.status === "invalid") {
|
|
20010
|
+
throw new Error(`cannot close ${conversation.conversation_id}; ${storedProof.reason}`);
|
|
20011
|
+
}
|
|
19020
20012
|
const takeover = isRecord(conversation.native_session_takeover)
|
|
19021
20013
|
? conversation.native_session_takeover
|
|
19022
20014
|
: undefined;
|
|
@@ -19030,12 +20022,24 @@ async function assertGenericCloseDoesNotBypassObservedHandoff({ options, storeDi
|
|
|
19030
20022
|
terminal = await bridge.resolveStoredTerminal(sourceSession.agent, pid, terminalControl, { pid });
|
|
19031
20023
|
}
|
|
19032
20024
|
catch (error) {
|
|
19033
|
-
if (
|
|
19034
|
-
|
|
20025
|
+
if (error instanceof TerminalControlUnavailableError) {
|
|
20026
|
+
// An unavailable terminal cannot prove a live A -> B handoff. Preserve the
|
|
20027
|
+
// explicit Store-only close path used to retire unavailable Turn history.
|
|
20028
|
+
return;
|
|
19035
20029
|
}
|
|
19036
|
-
|
|
19037
|
-
|
|
19038
|
-
|
|
20030
|
+
const processObservation = await observeBoundTerminalAgentProcess({
|
|
20031
|
+
options,
|
|
20032
|
+
conversation,
|
|
20033
|
+
terminalControl
|
|
20034
|
+
});
|
|
20035
|
+
if (processObservation.status === "verified_dead") {
|
|
20036
|
+
return processObservation.proof;
|
|
20037
|
+
}
|
|
20038
|
+
if (processObservation.status === "unverifiable") {
|
|
20039
|
+
throw new Error(`cannot close ${conversation.conversation_id}; the bound agent process ` +
|
|
20040
|
+
`death is unverifiable: ${processObservation.reason}`);
|
|
20041
|
+
}
|
|
20042
|
+
throw error;
|
|
19039
20043
|
}
|
|
19040
20044
|
const companions = sourceSession.agent === "codex"
|
|
19041
20045
|
? codexAllowedCompanionSetForManagedSession({
|
|
@@ -20739,6 +21743,46 @@ async function runTerminalBridgeMonitorWithLock(options, lifecycle, expectedTerm
|
|
|
20739
21743
|
runPreparedCallback(preparedCompletion.prepared);
|
|
20740
21744
|
return;
|
|
20741
21745
|
}
|
|
21746
|
+
// Durable completion wins over process-death cleanup. A coding-agent
|
|
21747
|
+
// process may exit immediately after committing its final transcript, so
|
|
21748
|
+
// only classify the Turn as orphaned after this poll (and an independent
|
|
21749
|
+
// durable probe under the mutation locks) both find no completion.
|
|
21750
|
+
if (!completion) {
|
|
21751
|
+
const deadProcessStall = await stallAcceptedTurnForVerifiedDeadAgent({
|
|
21752
|
+
options,
|
|
21753
|
+
storeDir: pathsForConversationDir(path.dirname(statePath)).storeDir,
|
|
21754
|
+
statePath,
|
|
21755
|
+
logPath,
|
|
21756
|
+
expectedConversationId: conversation.conversation_id,
|
|
21757
|
+
expectedMessageId: currentMessageId
|
|
21758
|
+
});
|
|
21759
|
+
if (deadProcessStall.completionPreparation) {
|
|
21760
|
+
const completionPreparation = deadProcessStall.completionPreparation;
|
|
21761
|
+
if (!completionPreparation.claimed) {
|
|
21762
|
+
printJson({
|
|
21763
|
+
conversation: completionPreparation.conversation,
|
|
21764
|
+
monitored: true,
|
|
21765
|
+
terminal_bridge: true,
|
|
21766
|
+
completed: false,
|
|
21767
|
+
duplicate: true,
|
|
21768
|
+
reason: completionPreparation.reason
|
|
21769
|
+
});
|
|
21770
|
+
return;
|
|
21771
|
+
}
|
|
21772
|
+
runPreparedCallback(completionPreparation.prepared);
|
|
21773
|
+
return;
|
|
21774
|
+
}
|
|
21775
|
+
if (deadProcessStall.stalled) {
|
|
21776
|
+
printJson({
|
|
21777
|
+
conversation: deadProcessStall.conversation,
|
|
21778
|
+
monitored: true,
|
|
21779
|
+
terminal_bridge: true,
|
|
21780
|
+
stalled: true,
|
|
21781
|
+
reason: deadProcessStall.reason
|
|
21782
|
+
});
|
|
21783
|
+
return;
|
|
21784
|
+
}
|
|
21785
|
+
}
|
|
20742
21786
|
// A concrete approval or completion observed on this poll wins over a timeout boundary.
|
|
20743
21787
|
const nowMs = cliNowMs();
|
|
20744
21788
|
if (Number.isFinite(hardTimeoutMinutes) &&
|
|
@@ -24461,6 +25505,144 @@ function resolveTerminalBridgeDispatchLedger({ terminalControl, conversation, ex
|
|
|
24461
25505
|
});
|
|
24462
25506
|
return true;
|
|
24463
25507
|
}
|
|
25508
|
+
function assertVerifiedDeadTerminalBridgeDispatchAuthority({ terminalControl, conversation, storeDir, statePath, logPath, expectedMessageId }) {
|
|
25509
|
+
assertTurnBindingCurrent(conversation, "resolve a verified-dead agent dispatch for");
|
|
25510
|
+
const takeover = isRecord(conversation.native_session_takeover)
|
|
25511
|
+
? conversation.native_session_takeover
|
|
25512
|
+
: undefined;
|
|
25513
|
+
const submission = terminalBridgeSubmission(conversation);
|
|
25514
|
+
const rawStateReceiptHistory = takeover?.terminal_bridge_submission_receipts;
|
|
25515
|
+
const rawStateReceipts = Array.isArray(rawStateReceiptHistory)
|
|
25516
|
+
? rawStateReceiptHistory.filter((receipt) => isRecord(receipt) &&
|
|
25517
|
+
stringValue(receipt.message_id) === expectedMessageId)
|
|
25518
|
+
: [];
|
|
25519
|
+
const validatedStateReceipts = terminalBridgeSubmissionReceipts(conversation).filter((receipt) => stringValue(receipt.message_id) === expectedMessageId);
|
|
25520
|
+
const requestText = String(takeover?.terminal_bridge_request_text ?? conversation.user_request ?? "");
|
|
25521
|
+
const requestHash = terminalBridgeRequestFingerprint(requestText);
|
|
25522
|
+
const sessionId = sessionIdForConversation(conversation);
|
|
25523
|
+
const turnId = turnIdForConversation(conversation);
|
|
25524
|
+
const bindingId = stringValue(conversation.terminal_binding_id);
|
|
25525
|
+
const bindingGeneration = Number(conversation.terminal_binding_generation);
|
|
25526
|
+
const nativeThreadId = stringValue(conversation.native_thread_id) ??
|
|
25527
|
+
stringValue(takeover?.terminal_agent_session_id);
|
|
25528
|
+
const endpointAnchor = terminalEndpointFromControlRef(terminalControl).processAnchorPid;
|
|
25529
|
+
if (!expectedMessageId ||
|
|
25530
|
+
!Array.isArray(rawStateReceiptHistory) ||
|
|
25531
|
+
rawStateReceipts.length !== 1 ||
|
|
25532
|
+
validatedStateReceipts.length !== 1 ||
|
|
25533
|
+
canonicalJson(rawStateReceipts[0]) !== canonicalJson(submission) ||
|
|
25534
|
+
canonicalJson(validatedStateReceipts[0]) !== canonicalJson(submission) ||
|
|
25535
|
+
stringValue(takeover?.terminal_bridge_message_id) !== expectedMessageId ||
|
|
25536
|
+
submission?.status !== "agent_accepted" ||
|
|
25537
|
+
stringValue(submission.message_id) !== expectedMessageId ||
|
|
25538
|
+
stringValue(submission.session_id) !== sessionId ||
|
|
25539
|
+
stringValue(submission.turn_id) !== turnId ||
|
|
25540
|
+
stringValue(submission.binding_id) !== bindingId ||
|
|
25541
|
+
Number(submission.binding_generation) !== bindingGeneration ||
|
|
25542
|
+
!requestHash ||
|
|
25543
|
+
stringValue(submission.request_hash) !== requestHash ||
|
|
25544
|
+
!bindingId ||
|
|
25545
|
+
!Number.isSafeInteger(bindingGeneration) ||
|
|
25546
|
+
bindingGeneration < 1 ||
|
|
25547
|
+
!nativeThreadId ||
|
|
25548
|
+
!Number.isSafeInteger(endpointAnchor) ||
|
|
25549
|
+
Number(endpointAnchor) < 1 ||
|
|
25550
|
+
!sameCanonicalStatePath(conversation.state_path, statePath) ||
|
|
25551
|
+
path.resolve(stringValue(conversation.event_log_path) ?? "") !==
|
|
25552
|
+
path.resolve(logPath) ||
|
|
25553
|
+
path.resolve(managedSessionStoreDirForConversation(conversation) ?? "") !==
|
|
25554
|
+
path.resolve(storeDir)) {
|
|
25555
|
+
throw new Error(`verified-dead Turn ${conversation.conversation_id} has no exact accepted submission authority`);
|
|
25556
|
+
}
|
|
25557
|
+
const ledger = loadTerminalBridgeDispatchLedger(terminalControl);
|
|
25558
|
+
const rawReceiptHistory = ledger?.terminal_submission_receipts;
|
|
25559
|
+
const receipts = Array.isArray(rawReceiptHistory)
|
|
25560
|
+
? rawReceiptHistory.filter((receipt) => isRecord(receipt) &&
|
|
25561
|
+
stringValue(receipt.message_id) === expectedMessageId)
|
|
25562
|
+
: [];
|
|
25563
|
+
// Parse the entire append-only history as well as selecting the one exact
|
|
25564
|
+
// receipt. A synthesized top-level candidate is not sufficient cleanup
|
|
25565
|
+
// authority for a verified-dead process.
|
|
25566
|
+
const validatedReceiptHistory = terminalLedgerReceiptHistory(ledger);
|
|
25567
|
+
const validatedReceipts = validatedReceiptHistory.filter((receipt) => stringValue(receipt.message_id) === expectedMessageId);
|
|
25568
|
+
const receipt = receipts[0];
|
|
25569
|
+
const recordMatches = (record, expectedStatus) => Boolean(record &&
|
|
25570
|
+
record.status === expectedStatus &&
|
|
25571
|
+
stringValue(record.generation_id) === expectedMessageId &&
|
|
25572
|
+
stringValue(record.conversation_id) === conversation.conversation_id &&
|
|
25573
|
+
stringValue(record.session_id) === sessionId &&
|
|
25574
|
+
stringValue(record.turn_id) === turnId &&
|
|
25575
|
+
stringValue(record.message_id) === expectedMessageId &&
|
|
25576
|
+
stringValue(record.request_hash) === requestHash &&
|
|
25577
|
+
sameCanonicalStatePath(record.state_path, statePath) &&
|
|
25578
|
+
path.resolve(stringValue(record.store_dir) ?? "") ===
|
|
25579
|
+
path.resolve(storeDir) &&
|
|
25580
|
+
path.resolve(stringValue(record.event_log_path) ?? "") ===
|
|
25581
|
+
path.resolve(logPath) &&
|
|
25582
|
+
stringValue(record.binding_id) === bindingId &&
|
|
25583
|
+
Number(record.binding_generation) === bindingGeneration &&
|
|
25584
|
+
stringValue(record.native_thread_id) === nativeThreadId &&
|
|
25585
|
+
stringValue(record.executor_kind) ===
|
|
25586
|
+
executorForConversation(conversation).kind &&
|
|
25587
|
+
(stringValue(record.openclaw_session) ?? undefined) ===
|
|
25588
|
+
(stringValue(conversation.openclaw_session) ?? undefined) &&
|
|
25589
|
+
Boolean(record.callback_expected) === Boolean(conversation.gateway_method) &&
|
|
25590
|
+
(stringValue(record.message_type) ?? undefined) ===
|
|
25591
|
+
(stringValue(submission.message_type) ?? undefined) &&
|
|
25592
|
+
(stringValue(record.message_body_hash) ?? undefined) ===
|
|
25593
|
+
(stringValue(submission.message_body_hash) ?? undefined) &&
|
|
25594
|
+
isRecord(record.terminal_endpoint) &&
|
|
25595
|
+
terminalDispatchRecordMatchesControl(record, terminalControl, {
|
|
25596
|
+
requireProcessAnchor: true
|
|
25597
|
+
}) &&
|
|
25598
|
+
terminalDispatchRecordProcessAnchor(record) === endpointAnchor);
|
|
25599
|
+
if (!ledger ||
|
|
25600
|
+
terminalDispatchLedgerLooksLifecycle(ledger) ||
|
|
25601
|
+
!["agent_accepted", "resolved"].includes(String(ledger.status)) ||
|
|
25602
|
+
!recordMatches(ledger, ledger.status === "resolved" ? "resolved" : "agent_accepted") ||
|
|
25603
|
+
receipts.length !== 1 ||
|
|
25604
|
+
validatedReceipts.length !== 1 ||
|
|
25605
|
+
canonicalJson(receipts[0]) !== canonicalJson(validatedReceipts[0]) ||
|
|
25606
|
+
!recordMatches(receipt, "agent_accepted") ||
|
|
25607
|
+
(ledger.status === "resolved" &&
|
|
25608
|
+
validTimestampMs(ledger.resolved_at) === undefined)) {
|
|
25609
|
+
throw new Error(`verified-dead Turn ${conversation.conversation_id} no longer owns one exact terminal dispatch receipt`);
|
|
25610
|
+
}
|
|
25611
|
+
let stateAcceptance;
|
|
25612
|
+
let ledgerAcceptance;
|
|
25613
|
+
let receiptAcceptance;
|
|
25614
|
+
try {
|
|
25615
|
+
stateAcceptance = terminalAcceptanceEvidenceForConversation(conversation, requestText, submission.acceptance_evidence);
|
|
25616
|
+
ledgerAcceptance = terminalAcceptanceEvidenceForConversation(conversation, requestText, ledger.acceptance_evidence);
|
|
25617
|
+
receiptAcceptance = terminalAcceptanceEvidenceForConversation(conversation, requestText, receipt.acceptance_evidence);
|
|
25618
|
+
}
|
|
25619
|
+
catch (error) {
|
|
25620
|
+
throw new Error(`verified-dead Turn ${conversation.conversation_id} has invalid native acceptance evidence: ` +
|
|
25621
|
+
(error instanceof Error ? error.message : String(error)));
|
|
25622
|
+
}
|
|
25623
|
+
if (canonicalJson(stateAcceptance) !== canonicalJson(ledgerAcceptance) ||
|
|
25624
|
+
canonicalJson(stateAcceptance) !== canonicalJson(receiptAcceptance)) {
|
|
25625
|
+
throw new Error(`verified-dead Turn ${conversation.conversation_id} has conflicting native acceptance receipts`);
|
|
25626
|
+
}
|
|
25627
|
+
return { ledger, resolved: ledger.status === "resolved" };
|
|
25628
|
+
}
|
|
25629
|
+
function resolveVerifiedDeadTerminalBridgeDispatchLedger(options) {
|
|
25630
|
+
const authority = assertVerifiedDeadTerminalBridgeDispatchAuthority(options);
|
|
25631
|
+
if (authority.resolved) {
|
|
25632
|
+
return true;
|
|
25633
|
+
}
|
|
25634
|
+
saveTerminalBridgeDispatchLedger(options.terminalControl, {
|
|
25635
|
+
...authority.ledger,
|
|
25636
|
+
status: "resolved",
|
|
25637
|
+
resolved_at: cliNow().toISOString(),
|
|
25638
|
+
reason: options.reason
|
|
25639
|
+
});
|
|
25640
|
+
const resolved = assertVerifiedDeadTerminalBridgeDispatchAuthority(options);
|
|
25641
|
+
if (!resolved.resolved) {
|
|
25642
|
+
throw new Error(`verified-dead Turn ${options.conversation.conversation_id} dispatch did not resolve`);
|
|
25643
|
+
}
|
|
25644
|
+
return true;
|
|
25645
|
+
}
|
|
24464
25646
|
function terminalBridgeRequestFingerprint(value) {
|
|
24465
25647
|
const text = String(value ?? "");
|
|
24466
25648
|
return text ? createHash("sha256").update(text).digest("hex") : undefined;
|