@gajae-code/coding-agent 0.18.0 → 0.18.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/dist/types/cli/stats-cli.d.ts +0 -5
- package/dist/types/config/autorouting-tier-map.d.ts +29 -2
- package/dist/types/config/model-registry.d.ts +16 -1
- package/dist/types/config/settings-schema.d.ts +2 -2
- package/dist/types/exa/mcp-client.d.ts +1 -5
- package/dist/types/exec/bash-shell-guardian.d.ts +26 -0
- package/dist/types/gjc-runtime/session-state-sidecar.d.ts +1 -0
- package/dist/types/hashline/parser.d.ts +28 -5
- package/dist/types/modes/acp/acp-agent.d.ts +6 -1
- package/dist/types/sdk/acp/adapter.d.ts +3 -1
- package/dist/types/sdk/broker/lifecycle.d.ts +6 -1
- package/dist/types/sdk/broker/recovery-backoff.d.ts +62 -0
- package/dist/types/sdk/broker/runtime.d.ts +20 -0
- package/dist/types/sdk/broker/startup-budget.d.ts +5 -1
- package/dist/types/sdk/bus/index.d.ts +1 -0
- package/dist/types/sdk/bus/telegram-daemon-contract.d.ts +1 -1
- package/dist/types/sdk/client/client.d.ts +5 -1
- package/dist/types/sdk/host/session-runtime.d.ts +7 -0
- package/dist/types/sdk/prompt-watchdog.d.ts +1 -1
- package/dist/types/session/agent-session.d.ts +2 -0
- package/dist/types/task/index.d.ts +0 -1
- package/dist/types/tools/bash.d.ts +1 -0
- package/dist/types/tools/descriptors.d.ts +5 -0
- package/dist/types/tools/eval-backends.d.ts +37 -0
- package/dist/types/tools/eval.d.ts +1 -6
- package/dist/types/tools/index.d.ts +1 -36
- package/dist/types/tools/session-descriptions.d.ts +22 -0
- package/package.json +7 -7
- package/scripts/generate-sdk-operation-inventory.ts +2 -0
- package/src/cli/doctor/managed-link.ts +4 -4
- package/src/cli/doctor/repairs.ts +9 -1
- package/src/cli/stats-cli.ts +4 -36
- package/src/cli-main.ts +4 -8
- package/src/cli.ts +4 -8
- package/src/commands/commit.ts +2 -2
- package/src/commands/sdk.ts +1 -2
- package/src/config/autorouting-tier-map.ts +20 -2
- package/src/config/model-preset-registry.ts +17 -2
- package/src/config/model-profile-activation.ts +3 -1
- package/src/config/model-profiles.ts +11 -3
- package/src/config/model-registry.ts +45 -17
- package/src/config/settings-schema.ts +8 -2
- package/src/exa/mcp-client.ts +41 -8
- package/src/exec/bash-shell-guardian.ts +214 -48
- package/src/gjc-runtime/session-state-sidecar.ts +88 -58
- package/src/hashline/execute.ts +51 -3
- package/src/hashline/parser.ts +63 -16
- package/src/internal-urls/docs-index.generated.ts +25 -25
- package/src/modes/acp/acp-agent.ts +136 -20
- package/src/modes/controllers/ssh-command-controller.ts +3 -1
- package/src/modes/interactive-mode.ts +10 -0
- package/src/modes/utils/context-usage.ts +2 -6
- package/src/prompts/tools/task.md +8 -27
- package/src/sdk/acp/adapter.ts +38 -3
- package/src/sdk/broker/doctor-restart.ts +38 -3
- package/src/sdk/broker/lifecycle.ts +408 -218
- package/src/sdk/broker/recovery-backoff.ts +135 -0
- package/src/sdk/broker/runtime.ts +83 -0
- package/src/sdk/broker/startup-budget.ts +6 -1
- package/src/sdk/bus/chat-daemon-control.ts +2 -2
- package/src/sdk/bus/index.ts +308 -22
- package/src/sdk/bus/telegram-daemon-contract.ts +1 -1
- package/src/sdk/client/client.ts +53 -35
- package/src/sdk/host/session-runtime.ts +99 -3
- package/src/sdk/prompt-watchdog.ts +17 -4
- package/src/sdk/protocol/operation-inventory.generated.json +11 -0
- package/src/session/agent-session.ts +79 -17
- package/src/session/internal/managed-session-storage.ts +23 -0
- package/src/task/index.ts +21 -66
- package/src/tools/bash.ts +39 -0
- package/src/tools/descriptors.ts +27 -1
- package/src/tools/eval-backends.ts +95 -0
- package/src/tools/eval.ts +3 -15
- package/src/tools/index.ts +9 -88
- package/src/tools/search.ts +2 -7
- package/src/tools/session-descriptions.ts +135 -0
- package/src/tools/tool-catalog.generated.ts +1 -1
- package/dist/types/cli/grep-cli.d.ts +0 -17
- package/dist/types/cli/shell-cli.d.ts +0 -8
- package/dist/types/commands/grep.d.ts +0 -42
- package/dist/types/commands/shell.d.ts +0 -21
- package/src/cli/grep-cli.ts +0 -160
- package/src/cli/shell-cli.ts +0 -176
- package/src/commands/grep.ts +0 -48
- package/src/commands/shell.ts +0 -29
|
@@ -97,6 +97,7 @@ import {
|
|
|
97
97
|
} from "./session-index";
|
|
98
98
|
import {
|
|
99
99
|
cancellableSleep,
|
|
100
|
+
DEFAULT_BROKER_PRESPAWN_PREPARATION_TIMEOUT_MS,
|
|
100
101
|
DEFAULT_READINESS_TIMEOUT_MS,
|
|
101
102
|
deriveLifecycleOuterDeadlines,
|
|
102
103
|
isValidReadinessTimeoutMs,
|
|
@@ -517,6 +518,7 @@ type LifecycleCommandResolver = () => LifecycleCommand;
|
|
|
517
518
|
const lifecycleCommandResolversForTest = new WeakMap<Broker, LifecycleCommandResolver>();
|
|
518
519
|
const lifecycleCleanupHooksForTest = new WeakMap<Broker, () => void>();
|
|
519
520
|
const startupAdmittedInputs = new WeakSet<Input>();
|
|
521
|
+
const startupBrokerDerivedAdmissions = new WeakMap<Input, { admittedAt: number; preSpawnDeadlineAt: number }>();
|
|
520
522
|
const startupLaunchInputs = new WeakMap<Input, SessionLaunch>();
|
|
521
523
|
type EnsureLaunchWorktreeForTest = (
|
|
522
524
|
plan: GjcLaunchWorktreePlan,
|
|
@@ -627,7 +629,10 @@ export function hasValidLifecycleDeadlines(value: LifecycleDeadlines, now = Date
|
|
|
627
629
|
type Input = Record<string, unknown>;
|
|
628
630
|
// The admitted launch deadline must survive the response phase: executeLifecycle
|
|
629
631
|
// receives the caller's original input after startup admission has expanded it.
|
|
630
|
-
type LifecycleEffectIntentWithDeadline = LifecycleEffectIntent & {
|
|
632
|
+
type LifecycleEffectIntentWithDeadline = LifecycleEffectIntent & {
|
|
633
|
+
lifecycleCleanupDeadlineAt?: number;
|
|
634
|
+
admissionCleanupDeadlineAt?: number;
|
|
635
|
+
};
|
|
631
636
|
export const isCanonicalSessionId = (value: string): boolean => /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value);
|
|
632
637
|
const defaultStateRoot = (cwd: string) => path.join(path.resolve(cwd), ".gjc", "state");
|
|
633
638
|
const hasDefaultStateRoot = (cwd: string, root: string) => path.resolve(root) === defaultStateRoot(cwd);
|
|
@@ -4828,6 +4833,7 @@ async function terminateSpawnedChild(
|
|
|
4828
4833
|
terminationStartDeadlineAt: number,
|
|
4829
4834
|
expected: EffectMarker | undefined,
|
|
4830
4835
|
timing: LifecycleTiming,
|
|
4836
|
+
reapUnregisteredCutoff = false,
|
|
4831
4837
|
): Promise<boolean> {
|
|
4832
4838
|
const pid = child.pid;
|
|
4833
4839
|
if (!pid || (expected && pid !== expected.pid)) return false;
|
|
@@ -4841,100 +4847,226 @@ async function terminateSpawnedChild(
|
|
|
4841
4847
|
await broker.index.refresh();
|
|
4842
4848
|
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4843
4849
|
const registered = expected ? broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker) : false;
|
|
4844
|
-
//
|
|
4845
|
-
// cleanup
|
|
4846
|
-
|
|
4847
|
-
const processExitDeadlineAt = expected && !registered ? deadline -
|
|
4850
|
+
// Reserve a quarter of the termination phase for the exact post-exit proof
|
|
4851
|
+
// and ledger-backed artifact cleanup. Registered hosts retain their deadline.
|
|
4852
|
+
const postExitProofReserveMs = Math.max(POLL_MS, Math.floor((deadline - terminationStartDeadlineAt) / 4));
|
|
4853
|
+
const processExitDeadlineAt = expected && !registered ? deadline - postExitProofReserveMs : deadline;
|
|
4848
4854
|
const unregisteredTerminationDeadlineAt =
|
|
4849
4855
|
processExitDeadlineAt - Math.max(POLL_MS, Math.floor((processExitDeadlineAt - terminationStartDeadlineAt) / 2));
|
|
4850
4856
|
const observe = (): ProcessObservation =>
|
|
4851
|
-
child.exitCode !== null
|
|
4857
|
+
child.exitCode !== null || child.signalCode !== null
|
|
4852
4858
|
? "exited"
|
|
4853
4859
|
: observeProcess(pid, incarnation, value => processIncarnationForBroker(broker, value));
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
}
|
|
4860
|
-
|
|
4861
|
-
return observation;
|
|
4860
|
+
let ownedExitObserved = child.exitCode !== null || child.signalCode !== null;
|
|
4861
|
+
const ownedExit = Promise.withResolvers<void>();
|
|
4862
|
+
const observeOwnedExit = (): void => {
|
|
4863
|
+
ownedExitObserved = true;
|
|
4864
|
+
ownedExit.resolve();
|
|
4862
4865
|
};
|
|
4863
|
-
|
|
4864
|
-
|
|
4865
|
-
|
|
4866
|
-
if (observation !== "uncertain") return;
|
|
4867
|
-
// This direct ChildProcess is owned by this broker invocation. A process-incarnation
|
|
4868
|
-
// read can briefly lag its exit event, so recheck only this owned child before failing.
|
|
4869
|
-
observation = await waitForExit(processExitDeadlineAt);
|
|
4866
|
+
child.once("exit", observeOwnedExit);
|
|
4867
|
+
const cleanupOwnedExitListeners = (): void => {
|
|
4868
|
+
child.removeListener("exit", observeOwnedExit);
|
|
4870
4869
|
};
|
|
4871
|
-
|
|
4872
|
-
|
|
4873
|
-
|
|
4874
|
-
|
|
4875
|
-
|
|
4876
|
-
|
|
4877
|
-
|
|
4878
|
-
|
|
4879
|
-
|
|
4880
|
-
|
|
4870
|
+
try {
|
|
4871
|
+
const waitForExit = async (until: number): Promise<ProcessObservation> => {
|
|
4872
|
+
let observation = observe();
|
|
4873
|
+
while (observation !== "exited" && timing.now() < until) {
|
|
4874
|
+
await Promise.race([ownedExit.promise, timing.sleep(Math.max(0, Math.min(POLL_MS, until - timing.now())))]);
|
|
4875
|
+
if (ownedExitObserved) observation = "exited";
|
|
4876
|
+
else observation = observe();
|
|
4877
|
+
}
|
|
4878
|
+
|
|
4879
|
+
return observation;
|
|
4880
|
+
};
|
|
4881
|
+
|
|
4882
|
+
let observation = observe();
|
|
4883
|
+
const recheckOwnedExitObservation = async (): Promise<void> => {
|
|
4884
|
+
if (observation !== "uncertain") return;
|
|
4885
|
+
// This direct ChildProcess is owned by this broker invocation. A process-incarnation
|
|
4886
|
+
// read can briefly lag its exit event, so recheck only this owned child before failing.
|
|
4887
|
+
observation = await waitForExit(processExitDeadlineAt);
|
|
4888
|
+
};
|
|
4889
|
+
if (observation === "alive") {
|
|
4890
|
+
if (expected && !registered) {
|
|
4891
|
+
await waitUntil(timing, terminationStartDeadlineAt);
|
|
4892
|
+
// A child with a published cutoff receipt owns its rollback. Without one,
|
|
4893
|
+
// waiting for a blocked module to publish is wasted signal/exit time.
|
|
4894
|
+
const failure = await readLifecycleFailureArtifact(
|
|
4895
|
+
lifecycleFailurePath(root, id, expected.effectMarker),
|
|
4896
|
+
expected,
|
|
4897
|
+
);
|
|
4898
|
+
if (failure) await waitUntil(timing, unregisteredTerminationDeadlineAt);
|
|
4899
|
+
} else {
|
|
4900
|
+
await waitUntil(timing, terminationStartDeadlineAt);
|
|
4881
4901
|
}
|
|
4882
|
-
} else {
|
|
4883
|
-
await waitUntil(timing, terminationStartDeadlineAt);
|
|
4884
|
-
}
|
|
4885
|
-
observation = observe();
|
|
4886
|
-
}
|
|
4887
|
-
if (observation === "alive") {
|
|
4888
|
-
if (!(await signalVerifiedSession({ locator: { stateRoot: root }, pid }, id, "SIGTERM", expected))) {
|
|
4889
4902
|
observation = observe();
|
|
4890
|
-
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4903
|
+
}
|
|
4904
|
+
if (observation === "alive") {
|
|
4905
|
+
const signalResult =
|
|
4906
|
+
expected && !registered
|
|
4907
|
+
? child.kill("SIGTERM")
|
|
4908
|
+
: await signalVerifiedSession({ locator: { stateRoot: root }, pid }, id, "SIGTERM", expected);
|
|
4909
|
+
if (!signalResult) {
|
|
4910
|
+
observation = observe();
|
|
4911
|
+
await recheckOwnedExitObservation();
|
|
4912
|
+
if (observation !== "exited") {
|
|
4913
|
+
await recordTerminalUncertain(broker, id, root, pid);
|
|
4914
|
+
return false;
|
|
4915
|
+
}
|
|
4916
|
+
} else {
|
|
4917
|
+
const remaining = Math.max(0, processExitDeadlineAt - timing.now());
|
|
4918
|
+
const gracefulDeadline =
|
|
4919
|
+
timing.now() +
|
|
4920
|
+
Math.min(CLOSE_TIMEOUT_MS, Math.floor(remaining * (expected && !registered ? 0.15 : 0.5)));
|
|
4921
|
+
observation = await waitForExit(gracefulDeadline);
|
|
4894
4922
|
}
|
|
4895
|
-
} else {
|
|
4896
|
-
const remaining = Math.max(0, processExitDeadlineAt - timing.now());
|
|
4897
|
-
const gracefulDeadline = timing.now() + Math.min(CLOSE_TIMEOUT_MS, Math.floor(remaining / 2));
|
|
4898
|
-
observation = await waitForExit(gracefulDeadline);
|
|
4899
4923
|
}
|
|
4900
|
-
|
|
4901
|
-
|
|
4902
|
-
|
|
4903
|
-
|
|
4904
|
-
|
|
4905
|
-
if (
|
|
4906
|
-
|
|
4907
|
-
|
|
4924
|
+
if (observation === "alive") {
|
|
4925
|
+
const signalResult =
|
|
4926
|
+
expected && !registered
|
|
4927
|
+
? child.kill("SIGKILL")
|
|
4928
|
+
: await signalVerifiedSession({ locator: { stateRoot: root }, pid }, id, "SIGKILL", expected);
|
|
4929
|
+
if (!signalResult) {
|
|
4930
|
+
observation = observe();
|
|
4931
|
+
await recheckOwnedExitObservation();
|
|
4932
|
+
if (observation !== "exited") {
|
|
4933
|
+
await recordTerminalUncertain(broker, id, root, pid);
|
|
4934
|
+
return false;
|
|
4935
|
+
}
|
|
4936
|
+
} else {
|
|
4937
|
+
observation = await waitForExit(processExitDeadlineAt);
|
|
4908
4938
|
}
|
|
4909
|
-
} else {
|
|
4910
|
-
observation = await waitForExit(processExitDeadlineAt);
|
|
4911
4939
|
}
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
failure
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4940
|
+
await recheckOwnedExitObservation();
|
|
4941
|
+
if (observation !== "exited" || !lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4942
|
+
let rollbackGeneration: number | null | undefined;
|
|
4943
|
+
if (expected) {
|
|
4944
|
+
const failure = await readLifecycleFailureArtifact(
|
|
4945
|
+
lifecycleFailurePath(root, id, expected.effectMarker),
|
|
4946
|
+
expected,
|
|
4947
|
+
);
|
|
4948
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4949
|
+
const rollbackComplete =
|
|
4950
|
+
failure?.artifact.rollback.fenced === true &&
|
|
4951
|
+
failure.artifact.rollback.runtimeRemoved &&
|
|
4952
|
+
failure.artifact.rollback.hostStopped &&
|
|
4953
|
+
failure.artifact.rollback.brokerRegistrationReleased;
|
|
4954
|
+
if (failure && expected && !rollbackComplete) {
|
|
4955
|
+
await broker.index.refresh();
|
|
4956
|
+
const hasRegistration = broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker);
|
|
4957
|
+
if (!hasRegistration) {
|
|
4958
|
+
const artifactsRemoved = await removeOwnedLifecycleArtifacts(
|
|
4959
|
+
root,
|
|
4960
|
+
id,
|
|
4961
|
+
expected,
|
|
4962
|
+
undefined,
|
|
4963
|
+
undefined,
|
|
4964
|
+
proofBudget,
|
|
4965
|
+
);
|
|
4966
|
+
const endpointGone = await endpointRemoved(root, id);
|
|
4967
|
+
// A signal-delivery error can be emitted while the child still runs. Recheck
|
|
4968
|
+
// exact process absence after cleanup before reporting a proven reap.
|
|
4969
|
+
if (
|
|
4970
|
+
artifactsRemoved &&
|
|
4971
|
+
endpointGone &&
|
|
4972
|
+
(observe() === "exited" || ownedExitObserved) &&
|
|
4973
|
+
lifecycleProofWithinDeadline(proofBudget)
|
|
4974
|
+
)
|
|
4975
|
+
return true;
|
|
4976
|
+
return failClosed();
|
|
4977
|
+
}
|
|
4978
|
+
}
|
|
4979
|
+
if (!rollbackComplete) {
|
|
4980
|
+
if (!readyThenExitToleranceEnabled()) {
|
|
4981
|
+
// A child that exited with a non-zero status before publishing any
|
|
4982
|
+
// lifecycle receipt has no durable rollback to wait for. Prove the
|
|
4983
|
+
// owned artifacts are gone and release its registration, but retain
|
|
4984
|
+
// terminal uncertainty for timeouts, clean exits, and child-authored
|
|
4985
|
+
// startup receipts whose rollback is incomplete.
|
|
4986
|
+
await broker.index.refresh();
|
|
4987
|
+
const uncertainRegistration = broker.index
|
|
4988
|
+
.listSessions()
|
|
4989
|
+
.sessions.some(
|
|
4990
|
+
session => session.sessionId === id && session.pid === pid && session.terminalUncertain === true,
|
|
4991
|
+
);
|
|
4992
|
+
if (
|
|
4993
|
+
failure ||
|
|
4994
|
+
uncertainRegistration ||
|
|
4995
|
+
(child.exitCode === null && child.signalCode === null) ||
|
|
4996
|
+
child.exitCode === 0
|
|
4997
|
+
)
|
|
4998
|
+
return failClosed();
|
|
4999
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5000
|
+
const publishedReady = (await probePublishedReadyAuthority(root, id, expected)).kind === "matched";
|
|
5001
|
+
if (publishedReady) return failClosed();
|
|
5002
|
+
if (reapUnregisteredCutoff && !(await endpointRemoved(root, id))) return failClosed();
|
|
5003
|
+
const artifactsRemoved = await removeOwnedLifecycleArtifacts(
|
|
5004
|
+
root,
|
|
5005
|
+
id,
|
|
5006
|
+
expected,
|
|
5007
|
+
undefined,
|
|
5008
|
+
undefined,
|
|
5009
|
+
proofBudget,
|
|
5010
|
+
);
|
|
5011
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5012
|
+
await broker.index.refresh();
|
|
5013
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5014
|
+
const registered = broker.index
|
|
5015
|
+
.listSessions()
|
|
5016
|
+
.sessions.find(session => session.sessionId === id && session.pid === pid);
|
|
5017
|
+
let registrationReleased =
|
|
5018
|
+
!broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker) ||
|
|
5019
|
+
registered?.terminal === true ||
|
|
5020
|
+
registered?.terminalUncertain === true;
|
|
5021
|
+
if (registered && !registrationReleased) {
|
|
5022
|
+
registrationReleased = await broker.index.unregisterIfCurrent(registered);
|
|
5023
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5024
|
+
await broker.index.refresh();
|
|
5025
|
+
registrationReleased =
|
|
5026
|
+
registrationReleased ||
|
|
5027
|
+
!broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker);
|
|
5028
|
+
}
|
|
5029
|
+
const stillExited =
|
|
5030
|
+
observeProcess(pid, expected.incarnation, value => processIncarnationForBroker(broker, value)) ===
|
|
5031
|
+
"exited";
|
|
5032
|
+
const endpointGone = await endpointRemoved(root, id);
|
|
5033
|
+
if (
|
|
5034
|
+
stillExited &&
|
|
5035
|
+
artifactsRemoved &&
|
|
5036
|
+
endpointGone &&
|
|
5037
|
+
registrationReleased &&
|
|
5038
|
+
registered?.terminalUncertain !== true &&
|
|
5039
|
+
lifecycleProofWithinDeadline(proofBudget)
|
|
5040
|
+
) {
|
|
5041
|
+
if (!reapUnregisteredCutoff) return true;
|
|
5042
|
+
try {
|
|
5043
|
+
await writeSessionLifecycleFailure(
|
|
5044
|
+
root,
|
|
5045
|
+
id,
|
|
5046
|
+
expected.effectMarker,
|
|
5047
|
+
{ phase: "startup", reason: "failed", message: `Session ${id} exited at readiness cutoff.` },
|
|
5048
|
+
{
|
|
5049
|
+
endpointGeneration: null,
|
|
5050
|
+
fenced: true,
|
|
5051
|
+
runtimeRemoved: true,
|
|
5052
|
+
hostStopped: true,
|
|
5053
|
+
brokerRegistrationReleased: true,
|
|
5054
|
+
},
|
|
5055
|
+
undefined,
|
|
5056
|
+
expected.incarnation,
|
|
5057
|
+
expected.pid,
|
|
5058
|
+
);
|
|
5059
|
+
} catch {
|
|
5060
|
+
return failClosed();
|
|
5061
|
+
}
|
|
5062
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5063
|
+
return true;
|
|
5064
|
+
}
|
|
5065
|
+
return failClosed();
|
|
5066
|
+
}
|
|
4935
5067
|
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4936
5068
|
const publishedReady = (await probePublishedReadyAuthority(root, id, expected)).kind === "matched";
|
|
4937
|
-
if (
|
|
5069
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4938
5070
|
const artifactsRemoved = await removeOwnedLifecycleArtifacts(
|
|
4939
5071
|
root,
|
|
4940
5072
|
id,
|
|
@@ -4949,21 +5081,50 @@ async function terminateSpawnedChild(
|
|
|
4949
5081
|
const registered = broker.index
|
|
4950
5082
|
.listSessions()
|
|
4951
5083
|
.sessions.find(session => session.sessionId === id && session.pid === pid);
|
|
5084
|
+
const registeredRowTerminal = registered?.terminal === true || registered?.terminalUncertain === true;
|
|
4952
5085
|
let registrationReleased =
|
|
4953
|
-
!broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker) ||
|
|
4954
|
-
registered?.terminal === true ||
|
|
4955
|
-
registered?.terminalUncertain === true;
|
|
5086
|
+
!broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker) || registeredRowTerminal;
|
|
4956
5087
|
if (registered && !registrationReleased) {
|
|
5088
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4957
5089
|
registrationReleased = await broker.index.unregisterIfCurrent(registered);
|
|
4958
5090
|
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4959
5091
|
await broker.index.refresh();
|
|
5092
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4960
5093
|
registrationReleased =
|
|
4961
5094
|
registrationReleased || !broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker);
|
|
4962
5095
|
}
|
|
4963
5096
|
const stillExited =
|
|
4964
5097
|
observeProcess(pid, expected.incarnation, value => processIncarnationForBroker(broker, value)) ===
|
|
4965
5098
|
"exited";
|
|
5099
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4966
5100
|
const endpointGone = await endpointRemoved(root, id);
|
|
5101
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5102
|
+
try {
|
|
5103
|
+
if (publishedReady && !failure) {
|
|
5104
|
+
await writeSessionLifecycleFailure(
|
|
5105
|
+
root,
|
|
5106
|
+
id,
|
|
5107
|
+
expected.effectMarker,
|
|
5108
|
+
{
|
|
5109
|
+
phase: "startup",
|
|
5110
|
+
reason: "failed",
|
|
5111
|
+
message: `Session ${id} ${READY_THEN_EXIT_MESSAGE}.`,
|
|
5112
|
+
},
|
|
5113
|
+
{
|
|
5114
|
+
endpointGeneration: registered?.endpointGeneration ?? null,
|
|
5115
|
+
fenced: stillExited && registrationReleased,
|
|
5116
|
+
runtimeRemoved: artifactsRemoved && endpointGone,
|
|
5117
|
+
hostStopped: stillExited,
|
|
5118
|
+
brokerRegistrationReleased: registrationReleased,
|
|
5119
|
+
},
|
|
5120
|
+
undefined,
|
|
5121
|
+
expected.incarnation,
|
|
5122
|
+
expected.pid,
|
|
5123
|
+
);
|
|
5124
|
+
}
|
|
5125
|
+
} catch {
|
|
5126
|
+
// A missing broker-authored receipt must not hide a proven dead child.
|
|
5127
|
+
}
|
|
4967
5128
|
if (
|
|
4968
5129
|
stillExited &&
|
|
4969
5130
|
artifactsRemoved &&
|
|
@@ -4974,102 +5135,35 @@ async function terminateSpawnedChild(
|
|
|
4974
5135
|
return true;
|
|
4975
5136
|
return failClosed();
|
|
4976
5137
|
}
|
|
4977
|
-
|
|
4978
|
-
|
|
4979
|
-
|
|
4980
|
-
|
|
4981
|
-
|
|
4982
|
-
|
|
4983
|
-
|
|
4984
|
-
|
|
4985
|
-
|
|
4986
|
-
|
|
4987
|
-
)
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
4993
|
-
.
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
4997
|
-
|
|
4998
|
-
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
4999
|
-
registrationReleased = await broker.index.unregisterIfCurrent(registered);
|
|
5000
|
-
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5001
|
-
await broker.index.refresh();
|
|
5002
|
-
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5003
|
-
registrationReleased =
|
|
5004
|
-
registrationReleased || !broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker);
|
|
5005
|
-
}
|
|
5006
|
-
const stillExited =
|
|
5007
|
-
observeProcess(pid, expected.incarnation, value => processIncarnationForBroker(broker, value)) === "exited";
|
|
5008
|
-
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5009
|
-
const endpointGone = await endpointRemoved(root, id);
|
|
5010
|
-
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5011
|
-
try {
|
|
5012
|
-
if (publishedReady && !failure) {
|
|
5013
|
-
await writeSessionLifecycleFailure(
|
|
5014
|
-
root,
|
|
5015
|
-
id,
|
|
5016
|
-
expected.effectMarker,
|
|
5017
|
-
{
|
|
5018
|
-
phase: "startup",
|
|
5019
|
-
reason: "failed",
|
|
5020
|
-
message: `Session ${id} ${READY_THEN_EXIT_MESSAGE}.`,
|
|
5021
|
-
},
|
|
5022
|
-
{
|
|
5023
|
-
endpointGeneration: registered?.endpointGeneration ?? null,
|
|
5024
|
-
fenced: stillExited && registrationReleased,
|
|
5025
|
-
runtimeRemoved: artifactsRemoved && endpointGone,
|
|
5026
|
-
hostStopped: stillExited,
|
|
5027
|
-
brokerRegistrationReleased: registrationReleased,
|
|
5028
|
-
},
|
|
5029
|
-
undefined,
|
|
5030
|
-
expected.incarnation,
|
|
5031
|
-
expected.pid,
|
|
5032
|
-
);
|
|
5033
|
-
}
|
|
5034
|
-
} catch {
|
|
5035
|
-
// A missing broker-authored receipt must not hide a proven dead child.
|
|
5036
|
-
}
|
|
5037
|
-
if (
|
|
5038
|
-
stillExited &&
|
|
5039
|
-
artifactsRemoved &&
|
|
5040
|
-
endpointGone &&
|
|
5041
|
-
registrationReleased &&
|
|
5042
|
-
lifecycleProofWithinDeadline(proofBudget)
|
|
5043
|
-
)
|
|
5044
|
-
return true;
|
|
5138
|
+
rollbackGeneration = failure.artifact.rollback.endpointGeneration;
|
|
5139
|
+
}
|
|
5140
|
+
if (expected && !(await removeOwnedLifecycleArtifacts(root, id, expected, undefined, undefined, proofBudget)))
|
|
5141
|
+
return failClosed();
|
|
5142
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5143
|
+
await broker.index.refresh();
|
|
5144
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5145
|
+
if (
|
|
5146
|
+
rollbackGeneration === null &&
|
|
5147
|
+
expected &&
|
|
5148
|
+
!broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker)
|
|
5149
|
+
)
|
|
5150
|
+
return lifecycleProofWithinDeadline(proofBudget) ? true : failClosed();
|
|
5151
|
+
const registeredBeforeTermination =
|
|
5152
|
+
rollbackGeneration === undefined || rollbackGeneration === null
|
|
5153
|
+
? undefined
|
|
5154
|
+
: broker.index.findHostRegistration(id, rollbackGeneration, pid, expected?.effectMarker);
|
|
5155
|
+
const unregistered = registeredBeforeTermination
|
|
5156
|
+
? broker.index.hostUnregisteredAfter(registeredBeforeTermination)
|
|
5157
|
+
: undefined;
|
|
5158
|
+
if (!registeredBeforeTermination || !unregistered) {
|
|
5045
5159
|
return failClosed();
|
|
5046
5160
|
}
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
await broker.index.refresh();
|
|
5053
|
-
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5054
|
-
if (
|
|
5055
|
-
rollbackGeneration === null &&
|
|
5056
|
-
expected &&
|
|
5057
|
-
!broker.index.hasHostRegistrationForLifecycle(id, pid, expected.effectMarker)
|
|
5058
|
-
)
|
|
5059
|
-
return lifecycleProofWithinDeadline(proofBudget) ? true : failClosed();
|
|
5060
|
-
const registeredBeforeTermination =
|
|
5061
|
-
rollbackGeneration === undefined || rollbackGeneration === null
|
|
5062
|
-
? undefined
|
|
5063
|
-
: broker.index.findHostRegistration(id, rollbackGeneration, pid, expected?.effectMarker);
|
|
5064
|
-
const unregistered = registeredBeforeTermination
|
|
5065
|
-
? broker.index.hostUnregisteredAfter(registeredBeforeTermination)
|
|
5066
|
-
: undefined;
|
|
5067
|
-
if (!registeredBeforeTermination || !unregistered) {
|
|
5068
|
-
return failClosed();
|
|
5161
|
+
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5162
|
+
const endpointGone = await endpointRemoved(root, id);
|
|
5163
|
+
return endpointGone && lifecycleProofWithinDeadline(proofBudget) ? true : failClosed();
|
|
5164
|
+
} finally {
|
|
5165
|
+
cleanupOwnedExitListeners();
|
|
5069
5166
|
}
|
|
5070
|
-
if (!lifecycleProofWithinDeadline(proofBudget)) return failClosed();
|
|
5071
|
-
const endpointGone = await endpointRemoved(root, id);
|
|
5072
|
-
return endpointGone && lifecycleProofWithinDeadline(proofBudget) ? true : failClosed();
|
|
5073
5167
|
}
|
|
5074
5168
|
|
|
5075
5169
|
async function signalVerifiedSession(
|
|
@@ -6414,6 +6508,7 @@ async function executeLifecycleResponse(
|
|
|
6414
6508
|
const timing = lifecycleTiming(broker);
|
|
6415
6509
|
const admissionGranted = startupAdmittedInputs.has(input);
|
|
6416
6510
|
if (!admissionGranted) {
|
|
6511
|
+
const startupReceivedAt = timing.now();
|
|
6417
6512
|
const suppliedDeadlineFields = [
|
|
6418
6513
|
input.receivedAt,
|
|
6419
6514
|
input.requestedReadinessTimeoutMs,
|
|
@@ -6441,15 +6536,26 @@ async function executeLifecycleResponse(
|
|
|
6441
6536
|
);
|
|
6442
6537
|
}
|
|
6443
6538
|
const admitted = await broker.runStartup(queueWaitMs, timing, async admittedAt => {
|
|
6444
|
-
const
|
|
6445
|
-
|
|
6446
|
-
|
|
6539
|
+
const callerSuppliedDeadlines = suppliedDeadlineFields.some(value => value !== undefined);
|
|
6540
|
+
let admittedInput: Input;
|
|
6541
|
+
if (launch.worktreePlan) admittedInput = { ...input, admittedAt, requestedReadinessTimeoutMs };
|
|
6542
|
+
else if (callerSuppliedDeadlines) admittedInput = { ...input };
|
|
6543
|
+
else admittedInput = { ...input, ...deriveLifecycleDeadlines(admittedAt, requestedReadinessTimeoutMs) };
|
|
6447
6544
|
startupAdmittedInputs.add(admittedInput);
|
|
6545
|
+
if (!launch.worktreePlan && !callerSuppliedDeadlines)
|
|
6546
|
+
startupBrokerDerivedAdmissions.set(admittedInput, {
|
|
6547
|
+
admittedAt,
|
|
6548
|
+
preSpawnDeadlineAt: Math.min(
|
|
6549
|
+
admittedAt + DEFAULT_BROKER_PRESPAWN_PREPARATION_TIMEOUT_MS,
|
|
6550
|
+
startupReceivedAt + queueWaitMs,
|
|
6551
|
+
),
|
|
6552
|
+
});
|
|
6448
6553
|
startupLaunchInputs.set(admittedInput, launch);
|
|
6449
6554
|
try {
|
|
6450
6555
|
return await executeLifecycleResponse(broker, operation, admittedInput, identity, cleanup);
|
|
6451
6556
|
} finally {
|
|
6452
6557
|
startupLaunchInputs.delete(admittedInput);
|
|
6558
|
+
startupBrokerDerivedAdmissions.delete(admittedInput);
|
|
6453
6559
|
startupAdmittedInputs.delete(admittedInput);
|
|
6454
6560
|
}
|
|
6455
6561
|
});
|
|
@@ -6477,6 +6583,8 @@ async function executeLifecycleResponse(
|
|
|
6477
6583
|
let readinessDeadline: number;
|
|
6478
6584
|
let terminationStartDeadline: number;
|
|
6479
6585
|
let outerCleanupDeadlineAt: number | undefined;
|
|
6586
|
+
const brokerAdmission = startupBrokerDerivedAdmissions.get(input);
|
|
6587
|
+
const preSpawnDeadlineAt = brokerAdmission?.preSpawnDeadlineAt;
|
|
6480
6588
|
if (launch.worktreePlan) {
|
|
6481
6589
|
const prepTimeouts = readPreparationTimeouts(input);
|
|
6482
6590
|
if (!prepTimeouts.ok) return fail("invalid_input", PREPARATION_TIMEOUT_INVALID_MESSAGE);
|
|
@@ -6507,6 +6615,10 @@ async function executeLifecycleResponse(
|
|
|
6507
6615
|
lifecycleDeadline = deadlines.lifecycleCleanupDeadlineAt;
|
|
6508
6616
|
readinessDeadline = deadlines.semanticReadyDeadlineAt;
|
|
6509
6617
|
terminationStartDeadline = deadlines.terminationStartDeadlineAt;
|
|
6618
|
+
if (preSpawnDeadlineAt !== undefined) {
|
|
6619
|
+
outerCleanupDeadlineAt = preSpawnDeadlineAt + deadlines.requestedReadinessTimeoutMs;
|
|
6620
|
+
lifecycleDeadline = outerCleanupDeadlineAt;
|
|
6621
|
+
}
|
|
6510
6622
|
}
|
|
6511
6623
|
|
|
6512
6624
|
if (!hasProcessIncarnationAuthority())
|
|
@@ -6530,6 +6642,9 @@ async function executeLifecycleResponse(
|
|
|
6530
6642
|
childOwnershipEstablished: false,
|
|
6531
6643
|
lifecycleCleanupDeadlineAt:
|
|
6532
6644
|
outerCleanupDeadlineAt ?? (childDeadlines as LifecycleDeadlines).lifecycleCleanupDeadlineAt,
|
|
6645
|
+
...(brokerAdmission
|
|
6646
|
+
? { admissionCleanupDeadlineAt: (childDeadlines as LifecycleDeadlines).lifecycleCleanupDeadlineAt }
|
|
6647
|
+
: {}),
|
|
6533
6648
|
...(plannedWorktreeIntent ? { worktree: plannedWorktreeIntent } : {}),
|
|
6534
6649
|
};
|
|
6535
6650
|
|
|
@@ -6631,7 +6746,22 @@ async function executeLifecycleResponse(
|
|
|
6631
6746
|
}
|
|
6632
6747
|
return mapPreparationFailure(error);
|
|
6633
6748
|
}
|
|
6634
|
-
if (
|
|
6749
|
+
if (preSpawnDeadlineAt !== undefined && brokerAdmission !== undefined) {
|
|
6750
|
+
const prepFinishedAt = timing.now();
|
|
6751
|
+
if (prepFinishedAt >= preSpawnDeadlineAt)
|
|
6752
|
+
return fail(
|
|
6753
|
+
"readiness_timeout",
|
|
6754
|
+
`Broker pre-spawn preparation exceeded its ${Math.max(0, preSpawnDeadlineAt - brokerAdmission.admittedAt)} ms allowance after ${Math.max(0, prepFinishedAt - brokerAdmission.admittedAt)} ms.`,
|
|
6755
|
+
);
|
|
6756
|
+
// Only broker-derived tuples restart here; exact caller deadlines remain unchanged.
|
|
6757
|
+
childDeadlines = deriveLifecycleDeadlines(
|
|
6758
|
+
prepFinishedAt,
|
|
6759
|
+
(childDeadlines as LifecycleDeadlines).requestedReadinessTimeoutMs,
|
|
6760
|
+
);
|
|
6761
|
+
readinessDeadline = childDeadlines.semanticReadyDeadlineAt;
|
|
6762
|
+
terminationStartDeadline = childDeadlines.terminationStartDeadlineAt;
|
|
6763
|
+
lifecycleDeadline = childDeadlines.lifecycleCleanupDeadlineAt;
|
|
6764
|
+
} else if (!launch.worktreePlan && timing.now() >= readinessDeadline)
|
|
6635
6765
|
return fail(
|
|
6636
6766
|
"readiness_timeout",
|
|
6637
6767
|
"Lifecycle preparation exhausted the semantic readiness deadline before spawning.",
|
|
@@ -6863,6 +6993,7 @@ async function executeLifecycleResponse(
|
|
|
6863
6993
|
knownSecrets,
|
|
6864
6994
|
)
|
|
6865
6995
|
: undefined;
|
|
6996
|
+
const ledgerUncertain = broker.ledger.get(identity)?.state === "terminal_uncertain";
|
|
6866
6997
|
const terminated = await terminateSpawnedChild(
|
|
6867
6998
|
child,
|
|
6868
6999
|
broker,
|
|
@@ -6872,8 +7003,8 @@ async function executeLifecycleResponse(
|
|
|
6872
7003
|
terminationStartDeadline,
|
|
6873
7004
|
spawnedAuthority,
|
|
6874
7005
|
timing,
|
|
7006
|
+
readiness.kind === "timeout",
|
|
6875
7007
|
);
|
|
6876
|
-
|
|
6877
7008
|
if (!terminated)
|
|
6878
7009
|
return readiness.kind === "ready_probe_failed"
|
|
6879
7010
|
? fail(
|
|
@@ -6884,6 +7015,11 @@ async function executeLifecycleResponse(
|
|
|
6884
7015
|
"terminal_uncertain",
|
|
6885
7016
|
`Session ${launch.id} did not become ready and its spawned process could not be verified dead. ${diagnostic ?? "stage=readiness waiting_for=session_ready child=uncertain exit=not_observed signal=not_observed stderr=none"}`,
|
|
6886
7017
|
);
|
|
7018
|
+
if (ledgerUncertain)
|
|
7019
|
+
return fail(
|
|
7020
|
+
"terminal_uncertain",
|
|
7021
|
+
`Lifecycle startup cleanup could not be proven; retained artifacts require reconciliation. Original launch failure: Session ${launch.id} did not become ready and its spawned process could not be verified dead.`,
|
|
7022
|
+
);
|
|
6887
7023
|
return readiness.kind === "startup_failed"
|
|
6888
7024
|
? fail(
|
|
6889
7025
|
readiness.failure.code ?? "spawn_failed",
|
|
@@ -7880,6 +8016,7 @@ async function exactCleanupProof(
|
|
|
7880
8016
|
evidence: { artifact: LifecycleFailureArtifact } | undefined,
|
|
7881
8017
|
proofBudget?: LifecycleProofBudget,
|
|
7882
8018
|
allowObservedWindowsReadyExit = false,
|
|
8019
|
+
allowObservedUnregisteredCutoff = false,
|
|
7883
8020
|
): Promise<LifecycleCleanupProof | undefined> {
|
|
7884
8021
|
const rollback = evidence?.artifact.rollback;
|
|
7885
8022
|
const durableRollbackComplete =
|
|
@@ -7887,6 +8024,16 @@ async function exactCleanupProof(
|
|
|
7887
8024
|
rollback.runtimeRemoved &&
|
|
7888
8025
|
rollback.hostStopped &&
|
|
7889
8026
|
rollback.brokerRegistrationReleased;
|
|
8027
|
+
// An owned child reaped before registration cannot finish its cutoff receipt.
|
|
8028
|
+
// The broker may retire that receipt only after independently proving process,
|
|
8029
|
+
// endpoint, and registration absence through this exact cleanup transaction.
|
|
8030
|
+
const observedUnregisteredCutoff =
|
|
8031
|
+
allowObservedUnregisteredCutoff &&
|
|
8032
|
+
rollback?.endpointGeneration === null &&
|
|
8033
|
+
rollback.fenced === false &&
|
|
8034
|
+
rollback.runtimeRemoved === false &&
|
|
8035
|
+
rollback.hostStopped === false &&
|
|
8036
|
+
rollback.brokerRegistrationReleased === false;
|
|
7890
8037
|
// Windows deliberately keeps the failure receipt incomplete because its
|
|
7891
8038
|
// directory-entry publication cannot be flushed. A broker-observed
|
|
7892
8039
|
// ready-then-exit can still authorize the separate ledger-backed exact
|
|
@@ -7904,7 +8051,7 @@ async function exactCleanupProof(
|
|
|
7904
8051
|
!root ||
|
|
7905
8052
|
!id ||
|
|
7906
8053
|
!expected ||
|
|
7907
|
-
(!durableRollbackComplete && !observedWindowsReadyExit) ||
|
|
8054
|
+
(!durableRollbackComplete && !observedWindowsReadyExit && !observedUnregisteredCutoff) ||
|
|
7908
8055
|
observeProcess(expected.pid, expected.incarnation, value => processIncarnationForBroker(broker, value)) !==
|
|
7909
8056
|
"exited"
|
|
7910
8057
|
)
|
|
@@ -7991,6 +8138,26 @@ export interface LifecycleExecutionOutcome {
|
|
|
7991
8138
|
deferredArtifactCleanup?: () => Promise<void>;
|
|
7992
8139
|
}
|
|
7993
8140
|
|
|
8141
|
+
export function classifyLateAdmissionSpawnFailure(
|
|
8142
|
+
response: BrokerResponse,
|
|
8143
|
+
effectIntent: LifecycleEffectIntentWithDeadline | undefined,
|
|
8144
|
+
now: number,
|
|
8145
|
+
messageSource: BrokerResponse,
|
|
8146
|
+
): BrokerResponse {
|
|
8147
|
+
if (
|
|
8148
|
+
response.ok ||
|
|
8149
|
+
response.error.code !== "spawn_failed" ||
|
|
8150
|
+
effectIntent?.childOwnershipEstablished !== true ||
|
|
8151
|
+
effectIntent.admissionCleanupDeadlineAt === undefined ||
|
|
8152
|
+
now < effectIntent.admissionCleanupDeadlineAt
|
|
8153
|
+
)
|
|
8154
|
+
return response;
|
|
8155
|
+
return {
|
|
8156
|
+
...response,
|
|
8157
|
+
error: { code: "terminal_uncertain", message: terminalUncertainStartupMessage(messageSource) },
|
|
8158
|
+
};
|
|
8159
|
+
}
|
|
8160
|
+
|
|
7994
8161
|
/** Returns the response together with every durable lifecycle fact needed for truthful replay. */
|
|
7995
8162
|
export async function executeLifecycle(
|
|
7996
8163
|
broker: Broker,
|
|
@@ -7999,6 +8166,7 @@ export async function executeLifecycle(
|
|
|
7999
8166
|
identity: string,
|
|
8000
8167
|
cleanup?: CleanupEvidence,
|
|
8001
8168
|
): Promise<LifecycleExecutionOutcome> {
|
|
8169
|
+
const timing = lifecycleTiming(broker);
|
|
8002
8170
|
let proofBudget = lifecycleProofBudgetFromInput(broker, input);
|
|
8003
8171
|
if (!proofBudget)
|
|
8004
8172
|
proofBudget = lifecycleProofBudgetFromEffectIntent(broker, broker.ledger.get(identity)?.effectIntent);
|
|
@@ -8052,16 +8220,25 @@ export async function executeLifecycle(
|
|
|
8052
8220
|
const binding = validateLifecycleDeleteMetadataBinding(broker, operation, input, identity, cleanup);
|
|
8053
8221
|
if (binding) return { response: binding };
|
|
8054
8222
|
}
|
|
8223
|
+
const reconciled = await reconcileLifecycleCleanup(
|
|
8224
|
+
broker,
|
|
8225
|
+
identity,
|
|
8226
|
+
cleanup,
|
|
8227
|
+
operation === "session.delete"
|
|
8228
|
+
? { ok: true, result: { sessionId: cleanup.sessionId } }
|
|
8229
|
+
: fail("spawn_failed", "No ready SDK endpoint remains available."),
|
|
8230
|
+
proofBudget,
|
|
8231
|
+
);
|
|
8055
8232
|
return {
|
|
8056
|
-
response:
|
|
8057
|
-
broker,
|
|
8058
|
-
identity,
|
|
8059
|
-
cleanup,
|
|
8233
|
+
response:
|
|
8060
8234
|
operation === "session.delete"
|
|
8061
|
-
?
|
|
8062
|
-
:
|
|
8063
|
-
|
|
8064
|
-
|
|
8235
|
+
? reconciled
|
|
8236
|
+
: classifyLateAdmissionSpawnFailure(
|
|
8237
|
+
reconciled,
|
|
8238
|
+
broker.ledger.get(identity)?.effectIntent,
|
|
8239
|
+
timing.now(),
|
|
8240
|
+
reconciled,
|
|
8241
|
+
),
|
|
8065
8242
|
};
|
|
8066
8243
|
}
|
|
8067
8244
|
const response = await executeLifecycleResponse(broker, operation, input, identity, cleanup);
|
|
@@ -8090,6 +8267,7 @@ export async function executeLifecycle(
|
|
|
8090
8267
|
evidence,
|
|
8091
8268
|
proofBudget,
|
|
8092
8269
|
!response.ok && response.error.code === "ready_then_exited",
|
|
8270
|
+
!response.ok && response.error.code === "readiness_timeout",
|
|
8093
8271
|
);
|
|
8094
8272
|
const startupFailure: LifecycleStartupFailureReceipt | undefined = evidence
|
|
8095
8273
|
? {
|
|
@@ -8189,47 +8367,59 @@ export async function executeLifecycle(
|
|
|
8189
8367
|
? response
|
|
8190
8368
|
: response.error.code === "endpoint_unreadable"
|
|
8191
8369
|
? response
|
|
8192
|
-
:
|
|
8193
|
-
|
|
8194
|
-
|
|
8195
|
-
|
|
8370
|
+
: response.error.code === "readiness_timeout" &&
|
|
8371
|
+
cleanupProof &&
|
|
8372
|
+
lifecycleCleanupResponse &&
|
|
8373
|
+
!lifecycleCleanupResponse.ok &&
|
|
8374
|
+
lifecycleCleanupResponse.error.code === "spawn_failed"
|
|
8196
8375
|
? {
|
|
8197
8376
|
...response,
|
|
8198
8377
|
...(durableEffects ? { durableEffects } : {}),
|
|
8199
8378
|
...(startupFailure ? { startupFailure } : {}),
|
|
8200
8379
|
}
|
|
8201
|
-
:
|
|
8380
|
+
: // An unreadable/corrupt endpoint is already the fail-closed honest
|
|
8381
|
+
// terminal classification (it names the artifact, not the
|
|
8382
|
+
// child); the reconciliation wrapper must not erase it.
|
|
8383
|
+
response.error.code === "ready_then_exited" && (cleanupProof || provenDeadCleanup)
|
|
8202
8384
|
? {
|
|
8203
|
-
|
|
8204
|
-
error: startupFailure.code
|
|
8205
|
-
? {
|
|
8206
|
-
code: startupFailure.code,
|
|
8207
|
-
message: startupFailure.message,
|
|
8208
|
-
details: startupFailure.details!,
|
|
8209
|
-
endpoint: "unavailable" as const,
|
|
8210
|
-
}
|
|
8211
|
-
: {
|
|
8212
|
-
code: "spawn_failed",
|
|
8213
|
-
message: startupFailure.message,
|
|
8214
|
-
endpoint: "unavailable" as const,
|
|
8215
|
-
},
|
|
8385
|
+
...response,
|
|
8216
8386
|
...(durableEffects ? { durableEffects } : {}),
|
|
8217
|
-
startupFailure,
|
|
8387
|
+
...(startupFailure ? { startupFailure } : {}),
|
|
8218
8388
|
}
|
|
8219
|
-
:
|
|
8220
|
-
?
|
|
8221
|
-
: {
|
|
8389
|
+
: startupFailure && cleanupProof
|
|
8390
|
+
? {
|
|
8222
8391
|
ok: false,
|
|
8223
|
-
error:
|
|
8224
|
-
|
|
8225
|
-
|
|
8226
|
-
|
|
8392
|
+
error: startupFailure.code
|
|
8393
|
+
? {
|
|
8394
|
+
code: startupFailure.code,
|
|
8395
|
+
message: startupFailure.message,
|
|
8396
|
+
details: startupFailure.details!,
|
|
8397
|
+
endpoint: "unavailable" as const,
|
|
8398
|
+
}
|
|
8399
|
+
: {
|
|
8400
|
+
code: "spawn_failed",
|
|
8401
|
+
message: startupFailure.message,
|
|
8402
|
+
endpoint: "unavailable" as const,
|
|
8403
|
+
},
|
|
8227
8404
|
...(durableEffects ? { durableEffects } : {}),
|
|
8228
|
-
|
|
8405
|
+
startupFailure,
|
|
8229
8406
|
}
|
|
8407
|
+
: provenDeadCleanup && response.error.code !== "terminal_uncertain"
|
|
8408
|
+
? response
|
|
8409
|
+
: {
|
|
8410
|
+
ok: false,
|
|
8411
|
+
error: {
|
|
8412
|
+
code: "terminal_uncertain",
|
|
8413
|
+
message: terminalUncertainStartupMessage(response),
|
|
8414
|
+
},
|
|
8415
|
+
...(durableEffects ? { durableEffects } : {}),
|
|
8416
|
+
...(startupFailure ? { startupFailure } : {}),
|
|
8417
|
+
}
|
|
8230
8418
|
: response;
|
|
8419
|
+
// The extended proof window may reconcile artifacts after the original
|
|
8420
|
+
// admission deadline, but cannot retroactively prove a timely spawn failure.
|
|
8231
8421
|
return {
|
|
8232
|
-
response: terminalResponse,
|
|
8422
|
+
response: classifyLateAdmissionSpawnFailure(terminalResponse, entry?.effectIntent, timing.now(), response),
|
|
8233
8423
|
...(durableEffects ? { durableEffects } : {}),
|
|
8234
8424
|
...(startupFailure ? { startupFailure } : {}),
|
|
8235
8425
|
};
|