@sellable/install 0.1.800 → 0.1.802
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/container/Dockerfile +4 -4
- package/container/README.md +1 -1
- package/lib/sellable-agent/fly-customer-image/Dockerfile +1 -1
- package/lib/sellable-agent/fly-customer-image/customer-runtime.mjs +267 -129
- package/lib/sellable-agent/fly-customer-image/endpoint-model-selection.d.mts +49 -2
- package/lib/sellable-agent/fly-customer-image/endpoint-model-selection.mjs +150 -14
- package/lib/sellable-agent/fly-customer-image/runs-api-health-check.mjs +19 -8
- package/lib/sellable-agent/hermes-memory-snapshot.mjs +37 -323
- package/lib/sellable-agent/host-bootstrap.mjs +1 -1
- package/lib/sellable-agent/host-worker.mjs +1 -1
- package/lib/sellable-agent/profile-materializer.mjs +1 -1
- package/lib/sellable-agent/provisioning-adapter.mjs +1 -1
- package/package.json +1 -1
package/container/Dockerfile
CHANGED
|
@@ -3,12 +3,12 @@ FROM ${HERMES_IMAGE}
|
|
|
3
3
|
|
|
4
4
|
USER root
|
|
5
5
|
|
|
6
|
-
ARG INSTALLER_PACKAGE=@sellable/install@0.1.
|
|
6
|
+
ARG INSTALLER_PACKAGE=@sellable/install@0.1.802
|
|
7
7
|
ARG MCP_PACKAGE=@sellable/mcp@0.1.957
|
|
8
8
|
ARG INSTALLER_INTEGRITY
|
|
9
9
|
ARG MCP_INTEGRITY=sha512-tIZuO5IEanT/afamDIWdorUNMdqZA6xnjL9jGAmGqYIlIqkXdHB9slWiwfLfmM44iEp7OT98IBHJlvbpMThk6A==
|
|
10
10
|
|
|
11
|
-
RUN test "${INSTALLER_PACKAGE}" = "@sellable/install@0.1.
|
|
11
|
+
RUN test "${INSTALLER_PACKAGE}" = "@sellable/install@0.1.802" \
|
|
12
12
|
&& test "${MCP_PACKAGE}" = "@sellable/mcp@0.1.957" \
|
|
13
13
|
&& test -n "${INSTALLER_INTEGRITY}" \
|
|
14
14
|
&& test "$(npm view "${INSTALLER_PACKAGE}" dist.integrity)" = "${INSTALLER_INTEGRITY}" \
|
|
@@ -23,9 +23,9 @@ RUN test "${INSTALLER_PACKAGE}" = "@sellable/install@0.1.800" \
|
|
|
23
23
|
&& mkdir -p /opt/sellable-agent/install-root /opt/sellable-agent/mcp-root \
|
|
24
24
|
&& npm install --prefix /opt/sellable-agent/install-root --omit=dev --ignore-scripts --no-audit --no-fund "${INSTALLER_PACKAGE}" \
|
|
25
25
|
&& npm install --prefix /opt/sellable-agent/mcp-root --omit=dev --ignore-scripts --no-audit --no-fund "${MCP_PACKAGE}" \
|
|
26
|
-
&& node --input-type=module -e "import { readFileSync } from 'node:fs'; const install = JSON.parse(readFileSync('/opt/sellable-agent/install-root/node_modules/@sellable/install/package.json')); const mcp = JSON.parse(readFileSync('/opt/sellable-agent/mcp-root/node_modules/@sellable/mcp/package.json')); if (install.name !== '@sellable/install' || install.version !== '0.1.
|
|
26
|
+
&& node --input-type=module -e "import { readFileSync } from 'node:fs'; const install = JSON.parse(readFileSync('/opt/sellable-agent/install-root/node_modules/@sellable/install/package.json')); const mcp = JSON.parse(readFileSync('/opt/sellable-agent/mcp-root/node_modules/@sellable/mcp/package.json')); if (install.name !== '@sellable/install' || install.version !== '0.1.802' || mcp.name !== '@sellable/mcp' || mcp.version !== '0.1.957') throw new Error('package identity rejected');" \
|
|
27
27
|
&& node --input-type=module -e "import { buildExternalRuntimeClosure } from '/opt/sellable-agent/install-root/node_modules/@sellable/install/lib/sellable-agent/external-runtime-builder.mjs'; const built = buildExternalRuntimeClosure({ mcpPackageRoot: '/opt/sellable-agent/mcp-root/node_modules/@sellable/mcp', mcpNodeModulesRoot: '/opt/sellable-agent/mcp-root/node_modules', hermesSourceRoot: '/opt/hermes', ownerUid: 0, ownerGid: 0 }); if (!built.closureDigest) throw new Error('runtime closure rejected');" \
|
|
28
|
-
&& node --input-type=module -e "import { chmodSync, writeFileSync } from 'node:fs'; const release = { installerPackage: '@sellable/install@0.1.
|
|
28
|
+
&& node --input-type=module -e "import { chmodSync, writeFileSync } from 'node:fs'; const release = { installerPackage: '@sellable/install@0.1.802', installerIntegrity: process.argv[1], mcpPackage: '@sellable/mcp@0.1.957', mcpIntegrity: process.argv[2] }; writeFileSync('/opt/sellable-agent/release.json', JSON.stringify(release) + '\\n', { mode: 0o444 }); chmodSync('/opt/sellable-agent/release.json', 0o444);" "${INSTALLER_INTEGRITY}" "${MCP_INTEGRITY}" \
|
|
29
29
|
&& npm cache clean --force
|
|
30
30
|
|
|
31
31
|
COPY entrypoint.sh /usr/local/bin/sellable-agent-container-entrypoint
|
package/container/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
This image is a dedicated Agent worker/runtime boundary beside an existing Hermes dashboard. It never replaces or reconfigures the dashboard container.
|
|
4
4
|
|
|
5
5
|
Build from this directory after
|
|
6
|
-
`@sellable/install@0.1.
|
|
6
|
+
`@sellable/install@0.1.802` is
|
|
7
7
|
published:
|
|
8
8
|
|
|
9
9
|
```sh
|
|
@@ -8,7 +8,7 @@ ENV PLAYWRIGHT_BROWSERS_PATH=/opt/hermes/.playwright
|
|
|
8
8
|
|
|
9
9
|
ARG MCP_PACKAGE=@sellable/mcp@0.1.957
|
|
10
10
|
ARG MCP_INTEGRITY=sha512-tIZuO5IEanT/afamDIWdorUNMdqZA6xnjL9jGAmGqYIlIqkXdHB9slWiwfLfmM44iEp7OT98IBHJlvbpMThk6A==
|
|
11
|
-
ARG INSTALLER_PACKAGE=@sellable/install@0.1.
|
|
11
|
+
ARG INSTALLER_PACKAGE=@sellable/install@0.1.802
|
|
12
12
|
ARG ADMIN_MCP_PACKAGE=@sellable/admin-mcp@0.1.202
|
|
13
13
|
ARG ADMIN_MCP_INTEGRITY=sha512-ym3aye/0vQ8tfULJlKz9JFOxvIQsl1G+n3np+TTgK/tecwNblBo/j3AkTnRqH9iomKzPbZzP+n4tEQsvsU8ByA==
|
|
14
14
|
ARG TUS_JS_CLIENT_VERSION=4.3.1
|
|
@@ -88,6 +88,11 @@ import {
|
|
|
88
88
|
} from "../profile-materializer.mjs";
|
|
89
89
|
import { reconcilePublicHermesSkills } from "../public-skills.mjs";
|
|
90
90
|
import { createCustomerRuntimePhaseTelemetry } from "./customer-runtime-phase-telemetry.mjs";
|
|
91
|
+
import {
|
|
92
|
+
endpointChainStepAdmissionDeadline,
|
|
93
|
+
endpointModelSelectionChain,
|
|
94
|
+
nextEndpointChainStep,
|
|
95
|
+
} from "./endpoint-model-selection.mjs";
|
|
91
96
|
import {
|
|
92
97
|
holdEndpointUatNativeCompletion,
|
|
93
98
|
validateEndpointUatCompletionControl,
|
|
@@ -98,11 +103,11 @@ import {
|
|
|
98
103
|
validateEndpointModelSelection,
|
|
99
104
|
validateOpenerOutputContract,
|
|
100
105
|
} from "./hermes-runs-client.mjs";
|
|
101
|
-
import { createConfiguredRunsClient } from "./runs-api-connections.mjs";
|
|
102
106
|
import {
|
|
103
107
|
createOpenerStageExecution,
|
|
104
108
|
usesOpenerDraftStrategy,
|
|
105
109
|
} from "./opener-stage-execution.mjs";
|
|
110
|
+
import { createConfiguredRunsClient } from "./runs-api-connections.mjs";
|
|
106
111
|
|
|
107
112
|
const DATA_ROOT = "/opt/data";
|
|
108
113
|
const PROFILES_ROOT = join(DATA_ROOT, "profiles");
|
|
@@ -126,7 +131,7 @@ const RUNTIME_GENERATION = Number(
|
|
|
126
131
|
const BOOT_SESSION_ID = randomUUID();
|
|
127
132
|
const HERMES_VERSION = process.env.SELLABLE_AGENT_HERMES_VERSION ?? "0.21.5";
|
|
128
133
|
const INSTALLER_PACKAGE =
|
|
129
|
-
process.env.SELLABLE_AGENT_INSTALLER_PACKAGE ?? "@sellable/install@0.1.
|
|
134
|
+
process.env.SELLABLE_AGENT_INSTALLER_PACKAGE ?? "@sellable/install@0.1.802";
|
|
130
135
|
const MCP_PACKAGE =
|
|
131
136
|
process.env.SELLABLE_AGENT_MCP_PACKAGE ?? "@sellable/mcp@0.1.957";
|
|
132
137
|
export const ENDPOINT_MAX_CONCURRENT_RUNS = 10;
|
|
@@ -1074,6 +1079,10 @@ function endpointEventEnvelope(event, ordinal) {
|
|
|
1074
1079
|
outputDigest: event.outputDigest ?? null,
|
|
1075
1080
|
errorDigest: event.errorDigest ?? null,
|
|
1076
1081
|
evidenceDigest: event.evidence?.digest ?? null,
|
|
1082
|
+
// Which model of an endpoint's fallback chain the event belongs to.
|
|
1083
|
+
...(Number.isSafeInteger(event.chainStep)
|
|
1084
|
+
? { chainStep: event.chainStep }
|
|
1085
|
+
: {}),
|
|
1077
1086
|
...(event.servedRuntime || event.usage
|
|
1078
1087
|
? {
|
|
1079
1088
|
servedProvider: event.servedRuntime?.provider ?? null,
|
|
@@ -1402,10 +1411,19 @@ export function endpointOpenerOutputContract(claim) {
|
|
|
1402
1411
|
// A single-run opener on a caller-named model (Opus on OpenRouter) is held to
|
|
1403
1412
|
// the same strict schema, which the bridge sends as the provider's json_schema
|
|
1404
1413
|
// so the model cannot return malformed JSON. The machine's default model has
|
|
1405
|
-
// no strict transport, so it gets no contract.
|
|
1406
|
-
|
|
1414
|
+
// no strict transport, so it gets no contract. Neither does a Codex step: the
|
|
1415
|
+
// bridge enforces the strict format only on OpenRouter. In a fallback chain
|
|
1416
|
+
// the server attaches the schema for its custom Runs steps, so a Hermes step
|
|
1417
|
+
// (Codex or OpenRouter) runs without it, as it would on its own.
|
|
1418
|
+
export function endpointStrictOutputContract(
|
|
1419
|
+
claim,
|
|
1420
|
+
modelSelection,
|
|
1421
|
+
{ chained = false } = {}
|
|
1422
|
+
) {
|
|
1407
1423
|
if (
|
|
1408
1424
|
!modelSelection ||
|
|
1425
|
+
(!modelSelection.runsApiBaseUrl &&
|
|
1426
|
+
(chained || modelSelection.provider === "openai-codex")) ||
|
|
1409
1427
|
usesOpenerDraftStrategy(claim) ||
|
|
1410
1428
|
claim?.run?.productExecution?.workKind !== "OPENER" ||
|
|
1411
1429
|
!claim.run.productExecution.providerOutputSchema
|
|
@@ -1954,7 +1972,22 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
1954
1972
|
throw new Error("customer_endpoint_mcp_descriptor_rejected");
|
|
1955
1973
|
}
|
|
1956
1974
|
const apiKeyFile = getApiKeyFile({ claim, activeRuntime });
|
|
1957
|
-
|
|
1975
|
+
// The models this run tries, in order. Each step opens its own client from
|
|
1976
|
+
// a claim carrying exactly that step's selection, so the trusted
|
|
1977
|
+
// connection checks are the same as for a single-model run.
|
|
1978
|
+
const chain = endpointModelSelectionChain(modelSelection);
|
|
1979
|
+
const chained = chain.length > 1;
|
|
1980
|
+
// A one-model run is a chain of one; no selection is the Agent default.
|
|
1981
|
+
const steps = chain.length ? chain : [null];
|
|
1982
|
+
const stepClaim = (index) =>
|
|
1983
|
+
chain.length === 0
|
|
1984
|
+
? claim
|
|
1985
|
+
: { ...claim, run: { ...claim.run, modelSelection: chain[index] } };
|
|
1986
|
+
const runs = createRunsClient({
|
|
1987
|
+
apiKeyFile,
|
|
1988
|
+
claim: stepClaim(0),
|
|
1989
|
+
activeRuntime,
|
|
1990
|
+
});
|
|
1958
1991
|
const controller = new AbortController();
|
|
1959
1992
|
const abortFromOuter = () => controller.abort(outerSignal.reason);
|
|
1960
1993
|
outerSignal.addEventListener("abort", abortFromOuter, { once: true });
|
|
@@ -2056,7 +2089,6 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2056
2089
|
await abortablePause(20_000, controller.signal, sleepImpl);
|
|
2057
2090
|
}
|
|
2058
2091
|
})();
|
|
2059
|
-
let terminalEvent = null;
|
|
2060
2092
|
let ordinal = 0;
|
|
2061
2093
|
const pendingStreamEvents = [];
|
|
2062
2094
|
const flushStreamEvents = async () => {
|
|
@@ -2066,104 +2098,31 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2066
2098
|
pendingStreamEvents.splice(0, batch.length);
|
|
2067
2099
|
held.committedSequence = sequences.at(-1);
|
|
2068
2100
|
};
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
append(claim, identity, event, ordinal++)
|
|
2086
|
-
);
|
|
2087
|
-
return stageEventChain;
|
|
2088
|
-
};
|
|
2089
|
-
held.openerStages = createOpenerStageExecution({
|
|
2090
|
-
runs,
|
|
2091
|
-
input: claim.run.input,
|
|
2092
|
-
instructions: endpointRunInstructions(claim),
|
|
2093
|
-
outputContract: endpointOpenerOutputContract(claim),
|
|
2094
|
-
correlation: `${claim.attempt.conversationId}:${claim.attempt.runId}:${claim.attempt.id}:${randomUUID()}`,
|
|
2095
|
-
deadlineAt: new Date(deadlineMs),
|
|
2096
|
-
signal: controller.signal,
|
|
2097
|
-
now,
|
|
2098
|
-
validateOutput: (output) =>
|
|
2099
|
-
endpointProductProjection(claim.run.productExecution, output).ok,
|
|
2100
|
-
onSubmitted: async ({ stage, runId, requestDigest }) => {
|
|
2101
|
-
const first = held.nativeRunId === null;
|
|
2102
|
-
held.nativeRunId ??= runId;
|
|
2103
|
-
await appendStageEvent({
|
|
2104
|
-
kind: first
|
|
2105
|
-
? "NATIVE_SUBMIT_ACCEPTED"
|
|
2106
|
-
: "OPENER_STAGE_SUBMIT_ACCEPTED",
|
|
2107
|
-
stage,
|
|
2108
|
-
requestDigest,
|
|
2109
|
-
nativeRunIdDigest: sha256(runId),
|
|
2110
|
-
});
|
|
2111
|
-
if (held.pendingControl) await queueControl(held.pendingControl);
|
|
2112
|
-
},
|
|
2113
|
-
onReceipt: async (receipt) => {
|
|
2114
|
-
await appendStageEvent({
|
|
2115
|
-
kind: "OPENER_STAGE_COMPLETED",
|
|
2116
|
-
...receipt,
|
|
2101
|
+
const done = (result) => ({ done: true, result });
|
|
2102
|
+
// Runs one model of the chain. A final outcome has already completed the
|
|
2103
|
+
// attempt (`done`); a clean pre-output failure returns its class and a
|
|
2104
|
+
// `finish` that completes the attempt as a single model would have.
|
|
2105
|
+
const runNativeStep = async (stepIndex, stepRuns) => {
|
|
2106
|
+
const selection = chain[stepIndex] ?? null;
|
|
2107
|
+
// Events of a chained run name the model they belong to.
|
|
2108
|
+
const tag = (event) =>
|
|
2109
|
+
chained ? { ...event, chainStep: stepIndex } : event;
|
|
2110
|
+
if (!stepRuns) {
|
|
2111
|
+
return {
|
|
2112
|
+
failure: "STEP_UNAVAILABLE",
|
|
2113
|
+
finish: async () => {
|
|
2114
|
+
await complete(claim, identity, {
|
|
2115
|
+
attemptStatus: "FAILED",
|
|
2116
|
+
status: "FAILED",
|
|
2117
2117
|
});
|
|
2118
|
+
return { ok: false, code: "FAILED" };
|
|
2118
2119
|
},
|
|
2119
|
-
}
|
|
2120
|
-
const selected = await held.openerStages.run();
|
|
2121
|
-
const observed = await holdEndpointUatNativeCompletion({
|
|
2122
|
-
control: uatCompletionControl,
|
|
2123
|
-
claim,
|
|
2124
|
-
event: {
|
|
2125
|
-
kind: "RUN_COMPLETED",
|
|
2126
|
-
terminal: true,
|
|
2127
|
-
output: selected.output,
|
|
2128
|
-
outputDigest: selected.outputDigest,
|
|
2129
|
-
timestamp: selected.timestamp,
|
|
2130
|
-
nativeEventIdentity: selected.nativeEventIdentity,
|
|
2131
|
-
},
|
|
2132
|
-
now,
|
|
2133
|
-
sleep: sleepImpl,
|
|
2134
|
-
signal: controller.signal,
|
|
2135
|
-
});
|
|
2136
|
-
const late = deadlineExpired || now().getTime() >= deadlineMs;
|
|
2137
|
-
await append(
|
|
2138
|
-
claim,
|
|
2139
|
-
identity,
|
|
2140
|
-
late
|
|
2141
|
-
? {
|
|
2142
|
-
kind: "LATE_COMPLETION_NOTICE",
|
|
2143
|
-
outputDigest: observed.outputDigest,
|
|
2144
|
-
timestamp: observed.timestamp,
|
|
2145
|
-
nativeEventIdentity: observed.nativeEventIdentity,
|
|
2146
|
-
}
|
|
2147
|
-
: observed,
|
|
2148
|
-
ordinal++
|
|
2149
|
-
);
|
|
2150
|
-
if (late) return { ok: true, code: "LATE_NOTICE_ONLY" };
|
|
2151
|
-
const projection = endpointProductProjection(
|
|
2152
|
-
claim.run.productExecution,
|
|
2153
|
-
selected.output
|
|
2154
|
-
);
|
|
2155
|
-
if (!projection.ok) throw new Error("OPENER_STAGE_OUTPUT_INVALID");
|
|
2156
|
-
const completion = await complete(claim, identity, {
|
|
2157
|
-
attemptStatus: "SUCCEEDED",
|
|
2158
|
-
status: "SUCCEEDED",
|
|
2159
|
-
output: selected.output,
|
|
2160
|
-
...projection.completion,
|
|
2161
|
-
});
|
|
2162
|
-
return completion?.accepted === true
|
|
2163
|
-
? { ok: true, code: "SUCCEEDED" }
|
|
2164
|
-
: { ok: false, code: "OPENER_STAGE_COMPLETION_REJECTED" };
|
|
2120
|
+
};
|
|
2165
2121
|
}
|
|
2122
|
+
held.runs = stepRuns;
|
|
2123
|
+
held.nativeRunId = null;
|
|
2166
2124
|
let submitted;
|
|
2125
|
+
let terminalEvent = null;
|
|
2167
2126
|
const submitRequest = {
|
|
2168
2127
|
input: claim.run.input,
|
|
2169
2128
|
instructions: endpointRunInstructions(claim),
|
|
@@ -2171,17 +2130,26 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2171
2130
|
claim.run.conversationHistory
|
|
2172
2131
|
),
|
|
2173
2132
|
correlation: `${claim.attempt.conversationId}:${claim.attempt.runId}:${claim.attempt.id}`,
|
|
2174
|
-
...(
|
|
2133
|
+
...(selection ? { modelSelection: selection } : {}),
|
|
2175
2134
|
};
|
|
2176
2135
|
const strictOutputContract = endpointStrictOutputContract(
|
|
2177
2136
|
claim,
|
|
2178
|
-
|
|
2137
|
+
selection,
|
|
2138
|
+
{ chained }
|
|
2179
2139
|
);
|
|
2180
2140
|
if (strictOutputContract)
|
|
2181
2141
|
submitRequest.outputContract = strictOutputContract;
|
|
2142
|
+
// A busy step stops waiting at its own budget so the next model still
|
|
2143
|
+
// has time; the last step (or a single model) waits until the deadline.
|
|
2144
|
+
const admissionDeadlineMs = endpointChainStepAdmissionDeadline({
|
|
2145
|
+
startedAtMs: now().getTime(),
|
|
2146
|
+
deadlineMs,
|
|
2147
|
+
stepIndex,
|
|
2148
|
+
stepCount: steps.length,
|
|
2149
|
+
});
|
|
2182
2150
|
while (!submitted) {
|
|
2183
2151
|
try {
|
|
2184
|
-
submitted = await
|
|
2152
|
+
submitted = await stepRuns.submit(submitRequest, {
|
|
2185
2153
|
signal: controller.signal,
|
|
2186
2154
|
});
|
|
2187
2155
|
} catch (error) {
|
|
@@ -2190,22 +2158,27 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2190
2158
|
attemptStatus: "UNCERTAIN",
|
|
2191
2159
|
status: "UNCERTAIN",
|
|
2192
2160
|
});
|
|
2193
|
-
return { ok: false, code: "SUBMIT_AMBIGUOUS" };
|
|
2161
|
+
return done({ ok: false, code: "SUBMIT_AMBIGUOUS" });
|
|
2194
2162
|
}
|
|
2195
2163
|
if (error?.classification !== "SUBMIT_NOT_ADMITTED") throw error;
|
|
2196
|
-
const
|
|
2197
|
-
if (deadlineExpired || remainingMs <= 0) {
|
|
2164
|
+
const notAdmittedTimeout = async () => {
|
|
2198
2165
|
await complete(claim, identity, {
|
|
2199
2166
|
attemptStatus: "TIMED_OUT",
|
|
2200
2167
|
status: "TIMED_OUT",
|
|
2201
2168
|
});
|
|
2202
2169
|
return { ok: false, code: "SUBMIT_NOT_ADMITTED_TIMEOUT" };
|
|
2203
|
-
}
|
|
2170
|
+
};
|
|
2171
|
+
const remainingMs = deadlineMs - now().getTime();
|
|
2172
|
+
if (deadlineExpired || remainingMs <= 0)
|
|
2173
|
+
return done(await notAdmittedTimeout());
|
|
2174
|
+
const admissionRemainingMs = admissionDeadlineMs - now().getTime();
|
|
2175
|
+
if (admissionRemainingMs <= 0)
|
|
2176
|
+
return { failure: "NOT_ADMITTED", finish: notAdmittedTimeout };
|
|
2204
2177
|
const retryAfterMs = Number.isFinite(error.retryAfterMs)
|
|
2205
2178
|
? Math.max(100, Math.min(5_000, error.retryAfterMs))
|
|
2206
2179
|
: 1_000;
|
|
2207
2180
|
await abortablePause(
|
|
2208
|
-
Math.min(retryAfterMs,
|
|
2181
|
+
Math.min(retryAfterMs, admissionRemainingMs),
|
|
2209
2182
|
controller.signal,
|
|
2210
2183
|
sleepImpl
|
|
2211
2184
|
);
|
|
@@ -2219,17 +2192,17 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2219
2192
|
await append(
|
|
2220
2193
|
claim,
|
|
2221
2194
|
identity,
|
|
2222
|
-
{
|
|
2195
|
+
tag({
|
|
2223
2196
|
kind: "NATIVE_SUBMIT_ACCEPTED",
|
|
2224
2197
|
requestDigest: submitted.requestDigest,
|
|
2225
2198
|
nativeRunIdDigest: sha256(submitted.runId),
|
|
2226
|
-
},
|
|
2199
|
+
}),
|
|
2227
2200
|
ordinal++
|
|
2228
2201
|
);
|
|
2229
2202
|
if (held.pendingControl) await queueControl(held.pendingControl);
|
|
2230
2203
|
let streamed;
|
|
2231
2204
|
try {
|
|
2232
|
-
streamed = await
|
|
2205
|
+
streamed = await stepRuns.streamEvents(submitted.runId, {
|
|
2233
2206
|
signal: controller.signal,
|
|
2234
2207
|
onEvent: async (event) => {
|
|
2235
2208
|
const observedEvent = await holdEndpointUatNativeCompletion({
|
|
@@ -2259,7 +2232,7 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2259
2232
|
}
|
|
2260
2233
|
: observedEvent;
|
|
2261
2234
|
pendingStreamEvents.push({
|
|
2262
|
-
event: durableEvent,
|
|
2235
|
+
event: tag(durableEvent),
|
|
2263
2236
|
ordinal: ordinal++,
|
|
2264
2237
|
});
|
|
2265
2238
|
const approvalBoundary =
|
|
@@ -2283,7 +2256,7 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2283
2256
|
}
|
|
2284
2257
|
}
|
|
2285
2258
|
if (!terminalEvent && !controller.signal.aborted) {
|
|
2286
|
-
const status = await
|
|
2259
|
+
const status = await stepRuns.status(submitted.runId, {
|
|
2287
2260
|
signal: controller.signal,
|
|
2288
2261
|
});
|
|
2289
2262
|
if (status.terminal) {
|
|
@@ -2300,7 +2273,7 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2300
2273
|
late: deadlineExpired || now().getTime() >= deadlineMs,
|
|
2301
2274
|
};
|
|
2302
2275
|
} else if (status.ok === false) {
|
|
2303
|
-
await append(claim, identity, status, ordinal++);
|
|
2276
|
+
await append(claim, identity, tag(status), ordinal++);
|
|
2304
2277
|
}
|
|
2305
2278
|
}
|
|
2306
2279
|
if (
|
|
@@ -2315,18 +2288,24 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2315
2288
|
// Server timeout is already canonical. The committed digest-only
|
|
2316
2289
|
// late notice above is the sole mutation; never replace timeout with
|
|
2317
2290
|
// a second completion or expose the late output as canonical.
|
|
2318
|
-
return { ok: true, code: "LATE_NOTICE_ONLY" };
|
|
2291
|
+
return done({ ok: true, code: "LATE_NOTICE_ONLY" });
|
|
2319
2292
|
}
|
|
2320
2293
|
const productProjection = endpointProductProjection(
|
|
2321
2294
|
claim.run.productExecution,
|
|
2322
2295
|
terminalEvent.output
|
|
2323
2296
|
);
|
|
2324
2297
|
if (!productProjection.ok) {
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2298
|
+
// The output was never used, so another model may still write.
|
|
2299
|
+
return {
|
|
2300
|
+
failure: "OUTPUT_CONTRACT_FAILED",
|
|
2301
|
+
finish: async () => {
|
|
2302
|
+
await complete(claim, identity, {
|
|
2303
|
+
attemptStatus: "SUCCEEDED",
|
|
2304
|
+
status: "OUTPUT_CONTRACT_FAILED",
|
|
2305
|
+
});
|
|
2306
|
+
return { ok: false, code: "OUTPUT_CONTRACT_FAILED" };
|
|
2307
|
+
},
|
|
2308
|
+
};
|
|
2330
2309
|
}
|
|
2331
2310
|
await complete(claim, identity, {
|
|
2332
2311
|
attemptStatus: "SUCCEEDED",
|
|
@@ -2334,7 +2313,7 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2334
2313
|
output: terminalEvent.output,
|
|
2335
2314
|
...productProjection.completion,
|
|
2336
2315
|
});
|
|
2337
|
-
return { ok: true, code: "SUCCEEDED" };
|
|
2316
|
+
return done({ ok: true, code: "SUCCEEDED" });
|
|
2338
2317
|
}
|
|
2339
2318
|
if (
|
|
2340
2319
|
terminalEvent?.classification === "CANCELLED_POSITIVE" ||
|
|
@@ -2345,26 +2324,185 @@ export function createCustomerEndpointRelay(options = {}) {
|
|
|
2345
2324
|
status: "CANCELLED_PRE_EFFECT",
|
|
2346
2325
|
positiveCancellation: true,
|
|
2347
2326
|
});
|
|
2348
|
-
return { ok: true, code: "CANCELLED" };
|
|
2327
|
+
return done({ ok: true, code: "CANCELLED" });
|
|
2349
2328
|
}
|
|
2350
2329
|
if (terminalEvent) {
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2330
|
+
// A native failure carries no output (Claude Code usage limit and
|
|
2331
|
+
// sign-in refusals arrive this way too).
|
|
2332
|
+
return {
|
|
2333
|
+
failure: "NATIVE_FAILED",
|
|
2334
|
+
finish: async () => {
|
|
2335
|
+
await complete(claim, identity, {
|
|
2336
|
+
attemptStatus: "FAILED",
|
|
2337
|
+
status: "FAILED",
|
|
2338
|
+
});
|
|
2339
|
+
return { ok: false, code: "FAILED" };
|
|
2340
|
+
},
|
|
2341
|
+
};
|
|
2356
2342
|
}
|
|
2357
2343
|
const deadline = deadlineExpired || now().getTime() >= deadlineMs;
|
|
2358
2344
|
await complete(claim, identity, {
|
|
2359
2345
|
attemptStatus: deadline ? "TIMED_OUT_UNCERTAIN" : "UNCERTAIN",
|
|
2360
2346
|
status: deadline ? "TIMED_OUT_UNCERTAIN" : "UNCERTAIN",
|
|
2361
2347
|
});
|
|
2362
|
-
return {
|
|
2348
|
+
return done({
|
|
2363
2349
|
ok: false,
|
|
2364
2350
|
code:
|
|
2365
2351
|
streamed?.classification ??
|
|
2366
2352
|
(deadline ? "TIMED_OUT_UNCERTAIN" : "NATIVE_LOST_UNCERTAIN"),
|
|
2367
|
-
};
|
|
2353
|
+
});
|
|
2354
|
+
};
|
|
2355
|
+
try {
|
|
2356
|
+
if (usesOpenerDraftStrategy(claim) && modelSelection) {
|
|
2357
|
+
// The three-draft stages are pinned to the Codex output-format path in
|
|
2358
|
+
// the Hermes bridge; refuse rather than silently serve the default.
|
|
2359
|
+
await complete(claim, identity, {
|
|
2360
|
+
attemptStatus: "FAILED",
|
|
2361
|
+
status: "FAILED",
|
|
2362
|
+
});
|
|
2363
|
+
return { ok: false, code: "OPENER_STAGE_MODEL_SELECTION_UNSUPPORTED" };
|
|
2364
|
+
}
|
|
2365
|
+
if (usesOpenerDraftStrategy(claim)) {
|
|
2366
|
+
// Phase 206: writers are private native sessions within this one fenced
|
|
2367
|
+
// claim. Only the editor's validated result reaches durable completion.
|
|
2368
|
+
let stageEventChain = Promise.resolve();
|
|
2369
|
+
const appendStageEvent = (event) => {
|
|
2370
|
+
stageEventChain = stageEventChain.then(() =>
|
|
2371
|
+
append(claim, identity, event, ordinal++)
|
|
2372
|
+
);
|
|
2373
|
+
return stageEventChain;
|
|
2374
|
+
};
|
|
2375
|
+
held.openerStages = createOpenerStageExecution({
|
|
2376
|
+
runs,
|
|
2377
|
+
input: claim.run.input,
|
|
2378
|
+
instructions: endpointRunInstructions(claim),
|
|
2379
|
+
outputContract: endpointOpenerOutputContract(claim),
|
|
2380
|
+
correlation: `${claim.attempt.conversationId}:${claim.attempt.runId}:${claim.attempt.id}:${randomUUID()}`,
|
|
2381
|
+
deadlineAt: new Date(deadlineMs),
|
|
2382
|
+
signal: controller.signal,
|
|
2383
|
+
now,
|
|
2384
|
+
validateOutput: (output) =>
|
|
2385
|
+
endpointProductProjection(claim.run.productExecution, output).ok,
|
|
2386
|
+
onSubmitted: async ({ stage, runId, requestDigest }) => {
|
|
2387
|
+
const first = held.nativeRunId === null;
|
|
2388
|
+
held.nativeRunId ??= runId;
|
|
2389
|
+
await appendStageEvent({
|
|
2390
|
+
kind: first
|
|
2391
|
+
? "NATIVE_SUBMIT_ACCEPTED"
|
|
2392
|
+
: "OPENER_STAGE_SUBMIT_ACCEPTED",
|
|
2393
|
+
stage,
|
|
2394
|
+
requestDigest,
|
|
2395
|
+
nativeRunIdDigest: sha256(runId),
|
|
2396
|
+
});
|
|
2397
|
+
if (held.pendingControl) await queueControl(held.pendingControl);
|
|
2398
|
+
},
|
|
2399
|
+
onReceipt: async (receipt) => {
|
|
2400
|
+
await appendStageEvent({
|
|
2401
|
+
kind: "OPENER_STAGE_COMPLETED",
|
|
2402
|
+
...receipt,
|
|
2403
|
+
});
|
|
2404
|
+
},
|
|
2405
|
+
});
|
|
2406
|
+
const selected = await held.openerStages.run();
|
|
2407
|
+
const observed = await holdEndpointUatNativeCompletion({
|
|
2408
|
+
control: uatCompletionControl,
|
|
2409
|
+
claim,
|
|
2410
|
+
event: {
|
|
2411
|
+
kind: "RUN_COMPLETED",
|
|
2412
|
+
terminal: true,
|
|
2413
|
+
output: selected.output,
|
|
2414
|
+
outputDigest: selected.outputDigest,
|
|
2415
|
+
timestamp: selected.timestamp,
|
|
2416
|
+
nativeEventIdentity: selected.nativeEventIdentity,
|
|
2417
|
+
},
|
|
2418
|
+
now,
|
|
2419
|
+
sleep: sleepImpl,
|
|
2420
|
+
signal: controller.signal,
|
|
2421
|
+
});
|
|
2422
|
+
const late = deadlineExpired || now().getTime() >= deadlineMs;
|
|
2423
|
+
await append(
|
|
2424
|
+
claim,
|
|
2425
|
+
identity,
|
|
2426
|
+
late
|
|
2427
|
+
? {
|
|
2428
|
+
kind: "LATE_COMPLETION_NOTICE",
|
|
2429
|
+
outputDigest: observed.outputDigest,
|
|
2430
|
+
timestamp: observed.timestamp,
|
|
2431
|
+
nativeEventIdentity: observed.nativeEventIdentity,
|
|
2432
|
+
}
|
|
2433
|
+
: observed,
|
|
2434
|
+
ordinal++
|
|
2435
|
+
);
|
|
2436
|
+
if (late) return { ok: true, code: "LATE_NOTICE_ONLY" };
|
|
2437
|
+
const projection = endpointProductProjection(
|
|
2438
|
+
claim.run.productExecution,
|
|
2439
|
+
selected.output
|
|
2440
|
+
);
|
|
2441
|
+
if (!projection.ok) throw new Error("OPENER_STAGE_OUTPUT_INVALID");
|
|
2442
|
+
const completion = await complete(claim, identity, {
|
|
2443
|
+
attemptStatus: "SUCCEEDED",
|
|
2444
|
+
status: "SUCCEEDED",
|
|
2445
|
+
output: selected.output,
|
|
2446
|
+
...projection.completion,
|
|
2447
|
+
});
|
|
2448
|
+
return completion?.accepted === true
|
|
2449
|
+
? { ok: true, code: "SUCCEEDED" }
|
|
2450
|
+
: { ok: false, code: "OPENER_STAGE_COMPLETION_REJECTED" };
|
|
2451
|
+
}
|
|
2452
|
+
// A step that failed cleanly before producing any output hands the run
|
|
2453
|
+
// to the next model; every other outcome completes this attempt once.
|
|
2454
|
+
// Only when the failed step cannot have changed anything: a product draft
|
|
2455
|
+
// run has no effects by construction, and an endpoint without tools has
|
|
2456
|
+
// nothing to call. Any other run may already have used a tool, so another
|
|
2457
|
+
// model must not repeat the turn.
|
|
2458
|
+
const chainMayAdvance =
|
|
2459
|
+
claim.run.productExecution != null ||
|
|
2460
|
+
(descriptorBinding.expected.allowedToolNames ?? []).length === 0;
|
|
2461
|
+
const history = [];
|
|
2462
|
+
let stepIndex = 0;
|
|
2463
|
+
let stepRuns = runs;
|
|
2464
|
+
for (;;) {
|
|
2465
|
+
const outcome = await runNativeStep(stepIndex, stepRuns);
|
|
2466
|
+
if (outcome.done) return outcome.result;
|
|
2467
|
+
// A stop, a gateway fence or a lost lease (which aborts the
|
|
2468
|
+
// controller) ends the run as a single model would have ended it.
|
|
2469
|
+
const stopRequested =
|
|
2470
|
+
held.stopRequested ||
|
|
2471
|
+
held.pendingControl?.kind === "STOP_PENDING" ||
|
|
2472
|
+
controller.signal.aborted;
|
|
2473
|
+
const decision =
|
|
2474
|
+
stopRequested || !chainMayAdvance
|
|
2475
|
+
? { advance: false }
|
|
2476
|
+
: nextEndpointChainStep(
|
|
2477
|
+
steps,
|
|
2478
|
+
history,
|
|
2479
|
+
outcome.failure,
|
|
2480
|
+
deadlineMs - now().getTime()
|
|
2481
|
+
);
|
|
2482
|
+
if (!decision.advance) return await outcome.finish();
|
|
2483
|
+
await append(
|
|
2484
|
+
claim,
|
|
2485
|
+
identity,
|
|
2486
|
+
{
|
|
2487
|
+
kind: "CHAIN_STEP_ADVANCED",
|
|
2488
|
+
classification: outcome.failure,
|
|
2489
|
+
chainStep: stepIndex,
|
|
2490
|
+
},
|
|
2491
|
+
ordinal++
|
|
2492
|
+
);
|
|
2493
|
+
history.push(outcome.failure);
|
|
2494
|
+
stepIndex = decision.stepIndex;
|
|
2495
|
+
try {
|
|
2496
|
+
stepRuns = createRunsClient({
|
|
2497
|
+
apiKeyFile,
|
|
2498
|
+
claim: stepClaim(stepIndex),
|
|
2499
|
+
activeRuntime,
|
|
2500
|
+
});
|
|
2501
|
+
} catch {
|
|
2502
|
+
// Nothing was sent for this step; it counts as a clean failure.
|
|
2503
|
+
stepRuns = null;
|
|
2504
|
+
}
|
|
2505
|
+
}
|
|
2368
2506
|
} catch (error) {
|
|
2369
2507
|
if (heartbeatFailure) throw heartbeatFailure;
|
|
2370
2508
|
if (
|