@sellable/install 0.1.606 → 0.1.608-wip.phase188.20260812.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/container/Dockerfile +4 -4
- package/container/README.md +2 -2
- package/lib/sellable-agent/admin-runtime-reconciler.mjs +24 -0
- package/lib/sellable-agent/fly-admin-image/Dockerfile +16 -0
- package/lib/sellable-agent/fly-admin-image/admin-runtime.mjs +10 -7
- package/lib/sellable-agent/fly-customer-image/Dockerfile +1 -1
- package/lib/sellable-agent/fly-customer-image/customer-runtime.mjs +460 -95
- package/lib/sellable-agent/fly-runtime-identity.mjs +10 -1
- package/lib/sellable-agent/hermes-memory-snapshot.mjs +115 -1
- package/lib/sellable-agent/host-bootstrap.mjs +1 -1
- package/lib/sellable-agent/host-worker.mjs +1 -1
- package/lib/sellable-agent/profile-materializer.mjs +1 -1
- package/lib/sellable-agent/provisioning-adapter.mjs +1 -1
- package/package.json +1 -1
package/container/Dockerfile
CHANGED
|
@@ -3,12 +3,12 @@ FROM ${HERMES_IMAGE}
|
|
|
3
3
|
|
|
4
4
|
USER root
|
|
5
5
|
|
|
6
|
-
ARG INSTALLER_PACKAGE=@sellable/install@0.1.
|
|
6
|
+
ARG INSTALLER_PACKAGE=@sellable/install@0.1.608-wip.phase188.20260812.1
|
|
7
7
|
ARG MCP_PACKAGE=@sellable/mcp@0.1.864
|
|
8
8
|
ARG INSTALLER_INTEGRITY
|
|
9
9
|
ARG MCP_INTEGRITY=sha512-UevVd71zQL2FgnqYaeHBkTi73iUV0Lg5BchM5JNhUbvaFxAFQ2M1n9uSS18qVuM9Km0aGZ7a0k8M9e2AVPboAw==
|
|
10
10
|
|
|
11
|
-
RUN test "${INSTALLER_PACKAGE}" = "@sellable/install@0.1.
|
|
11
|
+
RUN test "${INSTALLER_PACKAGE}" = "@sellable/install@0.1.608-wip.phase188.20260812.1" \
|
|
12
12
|
&& test "${MCP_PACKAGE}" = "@sellable/mcp@0.1.864" \
|
|
13
13
|
&& test -n "${INSTALLER_INTEGRITY}" \
|
|
14
14
|
&& test "$(npm view "${INSTALLER_PACKAGE}" dist.integrity)" = "${INSTALLER_INTEGRITY}" \
|
|
@@ -23,9 +23,9 @@ RUN test "${INSTALLER_PACKAGE}" = "@sellable/install@0.1.606" \
|
|
|
23
23
|
&& mkdir -p /opt/sellable-agent/install-root /opt/sellable-agent/mcp-root \
|
|
24
24
|
&& npm install --prefix /opt/sellable-agent/install-root --omit=dev --ignore-scripts --no-audit --no-fund "${INSTALLER_PACKAGE}" \
|
|
25
25
|
&& npm install --prefix /opt/sellable-agent/mcp-root --omit=dev --ignore-scripts --no-audit --no-fund "${MCP_PACKAGE}" \
|
|
26
|
-
&& node --input-type=module -e "import { readFileSync } from 'node:fs'; const install = JSON.parse(readFileSync('/opt/sellable-agent/install-root/node_modules/@sellable/install/package.json')); const mcp = JSON.parse(readFileSync('/opt/sellable-agent/mcp-root/node_modules/@sellable/mcp/package.json')); if (install.name !== '@sellable/install' || install.version !== '0.1.
|
|
26
|
+
&& node --input-type=module -e "import { readFileSync } from 'node:fs'; const install = JSON.parse(readFileSync('/opt/sellable-agent/install-root/node_modules/@sellable/install/package.json')); const mcp = JSON.parse(readFileSync('/opt/sellable-agent/mcp-root/node_modules/@sellable/mcp/package.json')); if (install.name !== '@sellable/install' || install.version !== '0.1.608-wip.phase188.20260812.1' || mcp.name !== '@sellable/mcp' || mcp.version !== '0.1.864') throw new Error('package identity rejected');" \
|
|
27
27
|
&& node --input-type=module -e "import { buildExternalRuntimeClosure } from '/opt/sellable-agent/install-root/node_modules/@sellable/install/lib/sellable-agent/external-runtime-builder.mjs'; const built = buildExternalRuntimeClosure({ mcpPackageRoot: '/opt/sellable-agent/mcp-root/node_modules/@sellable/mcp', mcpNodeModulesRoot: '/opt/sellable-agent/mcp-root/node_modules', hermesSourceRoot: '/opt/hermes', ownerUid: 0, ownerGid: 0 }); if (!built.closureDigest) throw new Error('runtime closure rejected');" \
|
|
28
|
-
&& node --input-type=module -e "import { chmodSync, writeFileSync } from 'node:fs'; const release = { installerPackage: '@sellable/install@0.1.
|
|
28
|
+
&& node --input-type=module -e "import { chmodSync, writeFileSync } from 'node:fs'; const release = { installerPackage: '@sellable/install@0.1.608-wip.phase188.20260812.1', installerIntegrity: process.argv[1], mcpPackage: '@sellable/mcp@0.1.864', mcpIntegrity: process.argv[2] }; writeFileSync('/opt/sellable-agent/release.json', JSON.stringify(release) + '\\n', { mode: 0o444 }); chmodSync('/opt/sellable-agent/release.json', 0o444);" "${INSTALLER_INTEGRITY}" "${MCP_INTEGRITY}" \
|
|
29
29
|
&& npm cache clean --force
|
|
30
30
|
|
|
31
31
|
COPY entrypoint.sh /usr/local/bin/sellable-agent-container-entrypoint
|
package/container/README.md
CHANGED
|
@@ -3,13 +3,13 @@
|
|
|
3
3
|
This image is a dedicated Agent worker/runtime boundary beside an existing Hermes dashboard. It never replaces or reconfigures the dashboard container.
|
|
4
4
|
|
|
5
5
|
Build from this directory after
|
|
6
|
-
`@sellable/install@0.1.
|
|
6
|
+
`@sellable/install@0.1.608-wip.phase188.20260812.1` is
|
|
7
7
|
published:
|
|
8
8
|
|
|
9
9
|
```sh
|
|
10
10
|
docker build --platform linux/amd64 \
|
|
11
11
|
--build-arg INSTALLER_INTEGRITY='sha512-…' \
|
|
12
|
-
--tag sellable-agent-host:0.1.
|
|
12
|
+
--tag sellable-agent-host:0.1.608-wip.phase188.20260812.1-wip.hermes-desktop.20260811160530 .
|
|
13
13
|
```
|
|
14
14
|
|
|
15
15
|
Create a private `bootstrap/` directory owned by root with mode `0700`. It must contain `bootstrap.json`, `provider-auth`, and `slack-app`, each owned by root with mode `0600`. `bootstrap.json` contains only non-secret identities, hashes, package integrities, and the control-plane URL. The other two files contain the provider JSON and global Slack app token whose hashes are pinned by `bootstrap.json`.
|
|
@@ -80,6 +80,30 @@ export function validateAdminRuntimeTuple(tuple) {
|
|
|
80
80
|
};
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
|
+
if (tuple.profileContractTemplate !== undefined) {
|
|
84
|
+
const templatePackages = tuple.profileContractTemplate?.packages;
|
|
85
|
+
const profilePackageByTupleName = {
|
|
86
|
+
"@sellable/mcp": templatePackages?.sellable?.mcp,
|
|
87
|
+
"@sellable/install": templatePackages?.sellable?.install,
|
|
88
|
+
"@sellable/admin-mcp": templatePackages?.["sellable-admin"]?.mcp,
|
|
89
|
+
"@sellable/admin-install": templatePackages?.["sellable-admin"]?.install,
|
|
90
|
+
};
|
|
91
|
+
for (const name of ADMIN_RUNTIME_TUPLE_MEMBERS) {
|
|
92
|
+
const tuplePackage = tuple.packages[name];
|
|
93
|
+
const profilePackage = profilePackageByTupleName[name];
|
|
94
|
+
if (
|
|
95
|
+
profilePackage?.name !== tuplePackage.name ||
|
|
96
|
+
profilePackage?.version !== tuplePackage.version ||
|
|
97
|
+
profilePackage?.integrity !== tuplePackage.integrity
|
|
98
|
+
) {
|
|
99
|
+
return {
|
|
100
|
+
ok: false,
|
|
101
|
+
code: "admin_runtime_tuple_profile_package_mismatch",
|
|
102
|
+
packageName: name,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
83
107
|
if (
|
|
84
108
|
!SHA256.test(tuple.lockedTreeDigest ?? "") ||
|
|
85
109
|
!SHA256.test(tuple.artifactDigest ?? "")
|
|
@@ -1,4 +1,17 @@
|
|
|
1
1
|
ARG HERMES_IMAGE=nousresearch/hermes-agent@sha256:c0cab4e3711bcb27a312be1b3776254fc06fd50d5f7a6b8017915fc7171cb39e
|
|
2
|
+
ARG SLACK_PP_GO_IMAGE=golang@sha256:386d475a660466863d9f8c766fec64d7fdad3edac2c6a05020c09534d71edb4b
|
|
3
|
+
|
|
4
|
+
FROM ${SLACK_PP_GO_IMAGE} AS slack-pp-cli-builder
|
|
5
|
+
|
|
6
|
+
ARG SLACK_PP_CLI_PACKAGE=github.com/mvanhorn/printing-press-library/library/productivity/slack/cmd/slack-pp-cli@v0.0.0-20260519210111-529e9728df37
|
|
7
|
+
ARG SLACK_PP_CLI_SHA256=2d7289d9e0d7f6ab85e2f650dfb95d1841ea6a1e97c31f74b91e9f0b71b39a8e
|
|
8
|
+
|
|
9
|
+
RUN set -eux; \
|
|
10
|
+
install -d -m 0755 /out; \
|
|
11
|
+
env GOBIN=/out CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
|
12
|
+
go install -trimpath "${SLACK_PP_CLI_PACKAGE}"; \
|
|
13
|
+
test "$(sha256sum /out/slack-pp-cli | cut -d ' ' -f 1)" = "${SLACK_PP_CLI_SHA256}"
|
|
14
|
+
|
|
2
15
|
FROM ${HERMES_IMAGE}
|
|
3
16
|
|
|
4
17
|
USER root
|
|
@@ -20,6 +33,8 @@ ARG TUS_JS_CLIENT_VERSION=4.3.1
|
|
|
20
33
|
ARG STRIPE_CLI_PACKAGE=@stripe/cli-linux-x64@1.45.0
|
|
21
34
|
ARG STRIPE_CLI_INTEGRITY=sha512-1ZhoPpoweYfynqsvhCLlTjZ7lPz5IKtxqAMT5MBy6zrwLRTP9x0T6r8h/jLYL8lRH1c7zC/6X6/XUu2V+r9Bog==
|
|
22
35
|
|
|
36
|
+
COPY --from=slack-pp-cli-builder --chmod=0555 /out/slack-pp-cli /usr/local/bin/slack-pp-cli
|
|
37
|
+
|
|
23
38
|
RUN set -eux; \
|
|
24
39
|
test -n "${ADMIN_PACKAGE_LOCK_B64}"; \
|
|
25
40
|
test -n "${ADMIN_PROFILE_CONTRACT_TEMPLATE_B64}"; \
|
|
@@ -65,6 +80,7 @@ RUN set -eux; \
|
|
|
65
80
|
stripe_cli_version="$(node -p "require('/usr/local/lib/sellable-agent/stripe-cli/node_modules/@stripe/cli-linux-x64/package.json').version")"; \
|
|
66
81
|
test "$(stripe version | head -n 1)" = "stripe version ${stripe_cli_version}"; \
|
|
67
82
|
test "$(node -p "require('/usr/local/lib/sellable-agent/node_modules/tus-js-client/package.json').version")" = "${TUS_JS_CLIENT_VERSION}"; \
|
|
83
|
+
slack-pp-cli --help >/dev/null; \
|
|
68
84
|
test -f /usr/local/lib/sellable-agent/admin-artifact/node_modules/@sellable/install/lib/sellable-agent/admin-runtime-reconciler.mjs; \
|
|
69
85
|
test -f /usr/local/lib/sellable-agent/admin-artifact/node_modules/@sellable/admin-mcp/dist/index.js; \
|
|
70
86
|
node -e 'const p=require("/usr/local/lib/sellable-agent/admin-artifact/node_modules/@sellable/admin-mcp/package.json"); if(p.main!=="dist/index.js" || p.bin?.["sellable-admin-mcp"]!=="dist/index.js") process.exit(1);'; \
|
|
@@ -70,6 +70,7 @@ import {
|
|
|
70
70
|
ensureHermesSnapshotCron,
|
|
71
71
|
markHermesSnapshotDirty,
|
|
72
72
|
mergeHermesState,
|
|
73
|
+
migrateLegacyHermesUserState,
|
|
73
74
|
reconcileHermesSnapshot,
|
|
74
75
|
restoreLatestHermesSnapshot,
|
|
75
76
|
} from "../hermes-memory-snapshot.mjs";
|
|
@@ -382,18 +383,14 @@ function installAdminHome({ settingsRoot, layout }) {
|
|
|
382
383
|
} catch {
|
|
383
384
|
throw new Error("admin_runtime_admin_home_wrappers_rejected");
|
|
384
385
|
}
|
|
385
|
-
// The
|
|
386
|
-
//
|
|
387
|
-
// not carry. Every wrapper it generates from local bytes must still be ready.
|
|
386
|
+
// The sealed image carries the checksum-pinned Printing Press binary, so
|
|
387
|
+
// every wrapper generated from local bytes must be immediately usable.
|
|
388
388
|
if (
|
|
389
389
|
generated.error ||
|
|
390
390
|
summary?.adminHome !== layout.adminHome ||
|
|
391
391
|
!Array.isArray(summary.wrappers) ||
|
|
392
392
|
summary.wrappers.length < 1 ||
|
|
393
|
-
summary.wrappers.some(
|
|
394
|
-
(wrapper) =>
|
|
395
|
-
wrapper?.kind !== "external-binary" && wrapper?.ready !== true
|
|
396
|
-
)
|
|
393
|
+
summary.wrappers.some((wrapper) => wrapper?.ready !== true)
|
|
397
394
|
) {
|
|
398
395
|
throw new Error("admin_runtime_admin_home_wrappers_rejected");
|
|
399
396
|
}
|
|
@@ -3170,6 +3167,12 @@ async function runAdminProduction() {
|
|
|
3170
3167
|
fetchLatest: () => client.getMemorySnapshot(),
|
|
3171
3168
|
});
|
|
3172
3169
|
if (!memoryRestore.ok) throw new Error(memoryRestore.code);
|
|
3170
|
+
const legacyUserState = migrateLegacyHermesUserState({
|
|
3171
|
+
profileId: basename(profileContract.profileHome),
|
|
3172
|
+
profileRoot: profileContract.profileHome,
|
|
3173
|
+
hermesRoot: DATA_ROOT,
|
|
3174
|
+
});
|
|
3175
|
+
if (!legacyUserState.ok) throw new Error(legacyUserState.code);
|
|
3173
3176
|
const profile = await materializeAdminProfile({
|
|
3174
3177
|
profileContract,
|
|
3175
3178
|
settingsRoot: imported.receipt.generationRoot,
|
|
@@ -6,7 +6,7 @@ USER root
|
|
|
6
6
|
|
|
7
7
|
ARG MCP_PACKAGE=@sellable/mcp@0.1.864
|
|
8
8
|
ARG MCP_INTEGRITY=sha512-UevVd71zQL2FgnqYaeHBkTi73iUV0Lg5BchM5JNhUbvaFxAFQ2M1n9uSS18qVuM9Km0aGZ7a0k8M9e2AVPboAw==
|
|
9
|
-
ARG INSTALLER_PACKAGE=@sellable/install@0.1.
|
|
9
|
+
ARG INSTALLER_PACKAGE=@sellable/install@0.1.608-wip.phase188.20260812.1
|
|
10
10
|
ARG ADMIN_MCP_PACKAGE=@sellable/admin-mcp@0.1.186-wip.phase158.20260802034727
|
|
11
11
|
ARG ADMIN_MCP_INTEGRITY=sha512-qYNew5wd2IfgVcp1UYOG9QvpF/8Mx19YJ5HQtMKzvSfUf3evePjtaKsoXLf2C/GUJ5QlLlHJo84VWOI3V01+9A==
|
|
12
12
|
ARG TUS_JS_CLIENT_VERSION=4.3.1
|
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
import {
|
|
45
45
|
ensureHermesSnapshotCron,
|
|
46
46
|
markHermesSnapshotDirty,
|
|
47
|
+
migrateLegacyHermesUserState,
|
|
47
48
|
reconcileHermesSnapshot,
|
|
48
49
|
restoreLatestHermesSnapshot,
|
|
49
50
|
} from "../hermes-memory-snapshot.mjs";
|
|
@@ -95,7 +96,7 @@ const RUNTIME_GENERATION = Number(
|
|
|
95
96
|
const BOOT_SESSION_ID = randomUUID();
|
|
96
97
|
const HERMES_VERSION = process.env.SELLABLE_AGENT_HERMES_VERSION ?? "0.20.0";
|
|
97
98
|
const INSTALLER_PACKAGE =
|
|
98
|
-
process.env.SELLABLE_AGENT_INSTALLER_PACKAGE ?? "@sellable/install@0.1.
|
|
99
|
+
process.env.SELLABLE_AGENT_INSTALLER_PACKAGE ?? "@sellable/install@0.1.608-wip.phase188.20260812.1";
|
|
99
100
|
const MCP_PACKAGE =
|
|
100
101
|
process.env.SELLABLE_AGENT_MCP_PACKAGE ?? "@sellable/mcp@0.1.864";
|
|
101
102
|
|
|
@@ -258,6 +259,10 @@ function compileDesired(claim, trustRootPem) {
|
|
|
258
259
|
if (
|
|
259
260
|
!operation ||
|
|
260
261
|
operation.targetRevisionId !== claim?.revision?.id ||
|
|
262
|
+
!Number.isSafeInteger(operation.configGeneration) ||
|
|
263
|
+
operation.configGeneration < 1 ||
|
|
264
|
+
!Number.isSafeInteger(operation.lifecycleGeneration) ||
|
|
265
|
+
operation.lifecycleGeneration < 0 ||
|
|
261
266
|
!slack ||
|
|
262
267
|
!service ||
|
|
263
268
|
!model ||
|
|
@@ -2136,26 +2141,83 @@ async function startNativeGateway({ desired, runtimeConfig, secrets }) {
|
|
|
2136
2141
|
|
|
2137
2142
|
export const API_SERVER_READINESS_DEADLINE_MS = 240_000;
|
|
2138
2143
|
const API_SERVER_READINESS_RETRY_MS = 1_000;
|
|
2144
|
+
export const NATIVE_GATEWAY_SERVING_PROOF_INTERVAL_MS = 30_000;
|
|
2145
|
+
export const NATIVE_GATEWAY_RECOVERY_COOLDOWN_MS = 60_000;
|
|
2146
|
+
export const NATIVE_GATEWAY_RECOVERY_MAX_ATTEMPTS = 3;
|
|
2147
|
+
const NATIVE_GATEWAY_RECOVERY_BUDGET_MS = 30_000;
|
|
2148
|
+
const NATIVE_GATEWAY_RECOVERY_BACKOFF_MS = 500;
|
|
2149
|
+
const NATIVE_GATEWAY_RECOVERY_BACKOFF_CAP_MS = 2_000;
|
|
2150
|
+
|
|
2151
|
+
const NATIVE_GATEWAY_SERVING_REASONS = new Set([
|
|
2152
|
+
"GATEWAY_STOPPED",
|
|
2153
|
+
"SLACK_DISCONNECTED",
|
|
2154
|
+
"API_UNAVAILABLE",
|
|
2155
|
+
"AUTH_UNAVAILABLE",
|
|
2156
|
+
"CAPABILITIES_UNAVAILABLE",
|
|
2157
|
+
"IDENTITY_MISMATCH",
|
|
2158
|
+
"RECOVERY_FAILED",
|
|
2159
|
+
"RECOVERY_IN_PROGRESS",
|
|
2160
|
+
]);
|
|
2161
|
+
|
|
2162
|
+
function nativeGatewayServingFailure(reasonCode) {
|
|
2163
|
+
const error = new Error(
|
|
2164
|
+
`customer_native_gateway_serving_rejected:${reasonCode}`
|
|
2165
|
+
);
|
|
2166
|
+
error.reasonCode = reasonCode;
|
|
2167
|
+
return error;
|
|
2168
|
+
}
|
|
2169
|
+
|
|
2170
|
+
function nativeGatewayServingFailureReason(error) {
|
|
2171
|
+
return NATIVE_GATEWAY_SERVING_REASONS.has(error?.reasonCode)
|
|
2172
|
+
? error.reasonCode
|
|
2173
|
+
: "RECOVERY_FAILED";
|
|
2174
|
+
}
|
|
2175
|
+
|
|
2176
|
+
function throwIfAborted(signal) {
|
|
2177
|
+
if (!signal?.aborted) return;
|
|
2178
|
+
throw signal.reason instanceof Error
|
|
2179
|
+
? signal.reason
|
|
2180
|
+
: new DOMException("Aborted", "AbortError");
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
function nativeGatewayRuntimeStatusReason(
|
|
2184
|
+
status,
|
|
2185
|
+
{ expectedPid, expectedHome }
|
|
2186
|
+
) {
|
|
2187
|
+
if (!status) return "API_UNAVAILABLE";
|
|
2188
|
+
if (
|
|
2189
|
+
!Number.isSafeInteger(expectedPid) ||
|
|
2190
|
+
expectedPid <= 1 ||
|
|
2191
|
+
status.pid !== expectedPid ||
|
|
2192
|
+
status.kind !== "hermes-gateway" ||
|
|
2193
|
+
!Array.isArray(status.argv) ||
|
|
2194
|
+
!status.argv.includes("gateway") ||
|
|
2195
|
+
!status.argv.includes("run") ||
|
|
2196
|
+
!Number.isSafeInteger(status.start_time) ||
|
|
2197
|
+
status.start_time <= 0 ||
|
|
2198
|
+
status.hermes_home !== expectedHome
|
|
2199
|
+
) {
|
|
2200
|
+
return "IDENTITY_MISMATCH";
|
|
2201
|
+
}
|
|
2202
|
+
if (status.gateway_state !== "running") return "GATEWAY_STOPPED";
|
|
2203
|
+
if (status.platforms?.slack?.state !== "connected") {
|
|
2204
|
+
return "SLACK_DISCONNECTED";
|
|
2205
|
+
}
|
|
2206
|
+
if (status.platforms?.api_server?.state !== "connected") {
|
|
2207
|
+
return "API_UNAVAILABLE";
|
|
2208
|
+
}
|
|
2209
|
+
return null;
|
|
2210
|
+
}
|
|
2139
2211
|
|
|
2140
2212
|
export function nativeGatewayRuntimeStatusAccepted(
|
|
2141
2213
|
status,
|
|
2142
2214
|
{ expectedPid, expectedHome }
|
|
2143
2215
|
) {
|
|
2144
|
-
return
|
|
2145
|
-
status
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
status.kind === "hermes-gateway" &&
|
|
2150
|
-
Array.isArray(status.argv) &&
|
|
2151
|
-
status.argv.includes("gateway") &&
|
|
2152
|
-
status.argv.includes("run") &&
|
|
2153
|
-
Number.isSafeInteger(status.start_time) &&
|
|
2154
|
-
status.start_time > 0 &&
|
|
2155
|
-
status.hermes_home === expectedHome &&
|
|
2156
|
-
status.gateway_state === "running" &&
|
|
2157
|
-
status.platforms?.slack?.state === "connected" &&
|
|
2158
|
-
status.platforms?.api_server?.state === "connected"
|
|
2216
|
+
return (
|
|
2217
|
+
nativeGatewayRuntimeStatusReason(status, {
|
|
2218
|
+
expectedPid,
|
|
2219
|
+
expectedHome,
|
|
2220
|
+
}) === null
|
|
2159
2221
|
);
|
|
2160
2222
|
}
|
|
2161
2223
|
|
|
@@ -2178,86 +2240,140 @@ function readNativeGatewayRuntimeStatus(runtimeHome) {
|
|
|
2178
2240
|
}
|
|
2179
2241
|
}
|
|
2180
2242
|
|
|
2181
|
-
async function
|
|
2243
|
+
export async function proveNativeGatewayServing({
|
|
2182
2244
|
desired,
|
|
2183
2245
|
apiServerKey,
|
|
2184
2246
|
gateway,
|
|
2185
2247
|
runtimeConfig,
|
|
2248
|
+
signal,
|
|
2249
|
+
fetchImpl = fetch,
|
|
2250
|
+
readStatus = readNativeGatewayRuntimeStatus,
|
|
2186
2251
|
}) {
|
|
2187
2252
|
const origin = "http://127.0.0.1:8642";
|
|
2253
|
+
const requestSignal = () =>
|
|
2254
|
+
signal
|
|
2255
|
+
? AbortSignal.any([signal, AbortSignal.timeout(5_000)])
|
|
2256
|
+
: AbortSignal.timeout(5_000);
|
|
2188
2257
|
const request = (authorization) =>
|
|
2189
|
-
|
|
2258
|
+
fetchImpl(`${origin}/v1/capabilities`, {
|
|
2190
2259
|
headers: authorization ? { authorization } : {},
|
|
2191
|
-
signal:
|
|
2260
|
+
signal: requestSignal(),
|
|
2192
2261
|
});
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2262
|
+
throwIfAborted(signal);
|
|
2263
|
+
const gatewayReadback = gateway?.readback();
|
|
2264
|
+
if (
|
|
2265
|
+
!gatewayReadback?.running ||
|
|
2266
|
+
!Number.isSafeInteger(gateway?.child?.pid) ||
|
|
2267
|
+
gateway.child.pid <= 1
|
|
2268
|
+
) {
|
|
2269
|
+
throw nativeGatewayServingFailure("GATEWAY_STOPPED");
|
|
2270
|
+
}
|
|
2271
|
+
const statusReason = nativeGatewayRuntimeStatusReason(
|
|
2272
|
+
readStatus(runtimeConfig.runtimeHome),
|
|
2273
|
+
{
|
|
2274
|
+
expectedPid: gateway.child.pid,
|
|
2275
|
+
expectedHome: runtimeConfig.runtimeHome,
|
|
2202
2276
|
}
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2277
|
+
);
|
|
2278
|
+
if (statusReason) throw nativeGatewayServingFailure(statusReason);
|
|
2279
|
+
let health;
|
|
2280
|
+
let missing;
|
|
2281
|
+
let wrong;
|
|
2282
|
+
let correct;
|
|
2283
|
+
try {
|
|
2284
|
+
[health, missing, wrong, correct] = await Promise.all([
|
|
2285
|
+
fetchImpl(`${origin}/health`, { signal: requestSignal() }),
|
|
2286
|
+
request(null),
|
|
2287
|
+
request("Bearer phase166-readiness-intentionally-wrong"),
|
|
2288
|
+
request(`Bearer ${apiServerKey}`),
|
|
2289
|
+
]);
|
|
2290
|
+
} catch (error) {
|
|
2291
|
+
throwIfAborted(signal);
|
|
2292
|
+
throw nativeGatewayServingFailure("API_UNAVAILABLE");
|
|
2293
|
+
}
|
|
2294
|
+
throwIfAborted(signal);
|
|
2295
|
+
if (health.status !== 200) {
|
|
2296
|
+
throw nativeGatewayServingFailure("API_UNAVAILABLE");
|
|
2297
|
+
}
|
|
2298
|
+
if (
|
|
2299
|
+
missing.status !== 401 ||
|
|
2300
|
+
wrong.status !== 401 ||
|
|
2301
|
+
correct.status !== 200
|
|
2302
|
+
) {
|
|
2303
|
+
throw nativeGatewayServingFailure("AUTH_UNAVAILABLE");
|
|
2304
|
+
}
|
|
2305
|
+
let capabilities;
|
|
2306
|
+
try {
|
|
2307
|
+
capabilities = await correct.json();
|
|
2308
|
+
} catch {
|
|
2309
|
+
throw nativeGatewayServingFailure("CAPABILITIES_UNAVAILABLE");
|
|
2310
|
+
}
|
|
2311
|
+
const finalStatusReason = nativeGatewayRuntimeStatusReason(
|
|
2312
|
+
readStatus(runtimeConfig.runtimeHome),
|
|
2313
|
+
{
|
|
2314
|
+
expectedPid: gateway.child.pid,
|
|
2315
|
+
expectedHome: runtimeConfig.runtimeHome,
|
|
2214
2316
|
}
|
|
2317
|
+
);
|
|
2318
|
+
if (finalStatusReason) {
|
|
2319
|
+
throw nativeGatewayServingFailure(finalStatusReason);
|
|
2320
|
+
}
|
|
2321
|
+
if (
|
|
2322
|
+
capabilities?.auth?.required !== true ||
|
|
2323
|
+
capabilities?.features?.run_submission !== true ||
|
|
2324
|
+
capabilities?.features?.run_events_sse !== true
|
|
2325
|
+
) {
|
|
2326
|
+
throw nativeGatewayServingFailure("CAPABILITIES_UNAVAILABLE");
|
|
2327
|
+
}
|
|
2328
|
+
return Object.freeze({
|
|
2329
|
+
host: "127.0.0.1",
|
|
2330
|
+
port: 8642,
|
|
2331
|
+
cors: false,
|
|
2332
|
+
keyGeneration: desired.apiServerKeyGeneration,
|
|
2333
|
+
keyFingerprint: desired.apiServerKeyFingerprint,
|
|
2334
|
+
authStatusDigest: sha256(
|
|
2335
|
+
JSON.stringify({
|
|
2336
|
+
health: health.status,
|
|
2337
|
+
missing: missing.status,
|
|
2338
|
+
wrong: wrong.status,
|
|
2339
|
+
correct: correct.status,
|
|
2340
|
+
})
|
|
2341
|
+
),
|
|
2342
|
+
capabilitiesDigest: sha256(JSON.stringify(capabilities)),
|
|
2343
|
+
fixtureDigest: HERMES_RUNS_FIXTURE_IDENTITY.fixtureDigest,
|
|
2344
|
+
relayCapability: "fixture_bound_runs_relay_v1",
|
|
2345
|
+
runtimeGeneration: desired.runtimeGeneration,
|
|
2346
|
+
gatewayPid: gateway.child.pid,
|
|
2347
|
+
});
|
|
2348
|
+
}
|
|
2349
|
+
|
|
2350
|
+
async function proveApiServerReadiness({
|
|
2351
|
+
desired,
|
|
2352
|
+
apiServerKey,
|
|
2353
|
+
gateway,
|
|
2354
|
+
runtimeConfig,
|
|
2355
|
+
signal,
|
|
2356
|
+
}) {
|
|
2357
|
+
const deadlineAt = Date.now() + API_SERVER_READINESS_DEADLINE_MS;
|
|
2358
|
+
while (Date.now() < deadlineAt) {
|
|
2215
2359
|
try {
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
expectedHome: runtimeConfig.runtimeHome,
|
|
2233
|
-
}
|
|
2234
|
-
) &&
|
|
2235
|
-
capabilities?.auth?.required === true &&
|
|
2236
|
-
capabilities?.features?.run_submission === true &&
|
|
2237
|
-
capabilities?.features?.run_events_sse === true
|
|
2238
|
-
) {
|
|
2239
|
-
return Object.freeze({
|
|
2240
|
-
host: "127.0.0.1",
|
|
2241
|
-
port: 8642,
|
|
2242
|
-
cors: false,
|
|
2243
|
-
keyGeneration: desired.apiServerKeyGeneration,
|
|
2244
|
-
keyFingerprint: desired.apiServerKeyFingerprint,
|
|
2245
|
-
authStatusDigest: sha256(
|
|
2246
|
-
JSON.stringify({
|
|
2247
|
-
health: health.status,
|
|
2248
|
-
missing: missing.status,
|
|
2249
|
-
wrong: wrong.status,
|
|
2250
|
-
correct: correct.status,
|
|
2251
|
-
})
|
|
2252
|
-
),
|
|
2253
|
-
capabilitiesDigest: sha256(JSON.stringify(capabilities)),
|
|
2254
|
-
fixtureDigest: HERMES_RUNS_FIXTURE_IDENTITY.fixtureDigest,
|
|
2255
|
-
relayCapability: "fixture_bound_runs_relay_v1",
|
|
2256
|
-
runtimeGeneration: desired.runtimeGeneration,
|
|
2257
|
-
gatewayPid: gateway.child.pid,
|
|
2258
|
-
});
|
|
2360
|
+
return await proveNativeGatewayServing({
|
|
2361
|
+
desired,
|
|
2362
|
+
apiServerKey,
|
|
2363
|
+
gateway,
|
|
2364
|
+
runtimeConfig,
|
|
2365
|
+
signal,
|
|
2366
|
+
});
|
|
2367
|
+
} catch (error) {
|
|
2368
|
+
throwIfAborted(signal);
|
|
2369
|
+
if (nativeGatewayServingFailureReason(error) === "GATEWAY_STOPPED") {
|
|
2370
|
+
const gatewayReadback = gateway?.readback();
|
|
2371
|
+
throw new Error(
|
|
2372
|
+
`customer_native_gateway_start_rejected:${redactRuntimeDiagnostic(
|
|
2373
|
+
gatewayReadback?.output ?? "candidate exited before readiness"
|
|
2374
|
+
)}`
|
|
2375
|
+
);
|
|
2259
2376
|
}
|
|
2260
|
-
} catch {
|
|
2261
2377
|
// Hermes can hold its singleton lock before the local HTTP listener is
|
|
2262
2378
|
// accepting connections. Keep the lifecycle claim fenced while the
|
|
2263
2379
|
// bounded readiness window converges instead of replacing it again.
|
|
@@ -2274,10 +2390,12 @@ export async function startProvenNativeGateway({
|
|
|
2274
2390
|
startGateway = startNativeGateway,
|
|
2275
2391
|
proveReadiness = proveApiServerReadiness,
|
|
2276
2392
|
stopGateway = stopNativeGateway,
|
|
2393
|
+
signal,
|
|
2277
2394
|
}) {
|
|
2278
2395
|
let gateway = null;
|
|
2279
2396
|
try {
|
|
2280
|
-
gateway = await startGateway({ desired, runtimeConfig, secrets });
|
|
2397
|
+
gateway = await startGateway({ desired, runtimeConfig, secrets, signal });
|
|
2398
|
+
throwIfAborted(signal);
|
|
2281
2399
|
const gatewayReadback = gateway.readback();
|
|
2282
2400
|
if (!gatewayReadback.running) {
|
|
2283
2401
|
throw new Error("customer_native_gateway_readback_rejected");
|
|
@@ -2287,6 +2405,7 @@ export async function startProvenNativeGateway({
|
|
|
2287
2405
|
apiServerKey: secrets.hermesApiServer,
|
|
2288
2406
|
gateway,
|
|
2289
2407
|
runtimeConfig,
|
|
2408
|
+
signal,
|
|
2290
2409
|
});
|
|
2291
2410
|
return { gateway, gatewayReadback, apiServerReadiness };
|
|
2292
2411
|
} catch (error) {
|
|
@@ -2309,6 +2428,230 @@ export async function startProvenNativeGateway({
|
|
|
2309
2428
|
}
|
|
2310
2429
|
}
|
|
2311
2430
|
|
|
2431
|
+
function nativeGatewayServingObservation({
|
|
2432
|
+
runtime,
|
|
2433
|
+
bootSessionId,
|
|
2434
|
+
flyGeneration,
|
|
2435
|
+
state,
|
|
2436
|
+
reasonCode,
|
|
2437
|
+
checkedAt,
|
|
2438
|
+
}) {
|
|
2439
|
+
const gatewayPid = runtime.gateway?.child?.pid;
|
|
2440
|
+
const desired = runtime.desired;
|
|
2441
|
+
const authority = runtime.servingAuthority;
|
|
2442
|
+
if (
|
|
2443
|
+
!Number.isSafeInteger(gatewayPid) ||
|
|
2444
|
+
gatewayPid <= 1 ||
|
|
2445
|
+
authority?.runtimeId !== desired.runtimeId ||
|
|
2446
|
+
authority?.profileId !== desired.profileId ||
|
|
2447
|
+
!Number.isSafeInteger(authority?.runtimeGeneration) ||
|
|
2448
|
+
authority.runtimeGeneration < 1 ||
|
|
2449
|
+
!Number.isSafeInteger(authority?.configGeneration) ||
|
|
2450
|
+
authority.configGeneration < 1 ||
|
|
2451
|
+
!Number.isSafeInteger(authority?.lifecycleGeneration) ||
|
|
2452
|
+
authority.lifecycleGeneration < 0
|
|
2453
|
+
) {
|
|
2454
|
+
throw new Error("customer_native_gateway_observation_rejected");
|
|
2455
|
+
}
|
|
2456
|
+
return Object.freeze({
|
|
2457
|
+
version: 1,
|
|
2458
|
+
runtimeId: authority.runtimeId,
|
|
2459
|
+
flyGeneration,
|
|
2460
|
+
runtimeGeneration: authority.runtimeGeneration,
|
|
2461
|
+
configGeneration: authority.configGeneration,
|
|
2462
|
+
lifecycleGeneration: authority.lifecycleGeneration,
|
|
2463
|
+
bootSessionId,
|
|
2464
|
+
profileId: desired.profileId,
|
|
2465
|
+
hermesHomeId: flyRuntimeDigest({
|
|
2466
|
+
schemaVersion: "sellable-agent-hermes-home/v1",
|
|
2467
|
+
root: DATA_ROOT,
|
|
2468
|
+
profileId: desired.profileId,
|
|
2469
|
+
}),
|
|
2470
|
+
gatewayPid,
|
|
2471
|
+
checkedAt: new Date(checkedAt).toISOString(),
|
|
2472
|
+
state,
|
|
2473
|
+
reasonCode,
|
|
2474
|
+
});
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
/**
|
|
2478
|
+
* One in-process, single-flight owner beside the existing signed claim loop.
|
|
2479
|
+
* It creates no timer: the lifecycle loop decides when to call poll().
|
|
2480
|
+
*/
|
|
2481
|
+
export function createNativeGatewayServingSupervisor({
|
|
2482
|
+
runtime,
|
|
2483
|
+
getActiveRuntime,
|
|
2484
|
+
bootSessionId,
|
|
2485
|
+
flyGeneration,
|
|
2486
|
+
now = Date.now,
|
|
2487
|
+
random = Math.random,
|
|
2488
|
+
pause = (ms, signal) => abortablePause(ms, signal, sleep),
|
|
2489
|
+
proveServing = proveNativeGatewayServing,
|
|
2490
|
+
startProvenGateway = startProvenNativeGateway,
|
|
2491
|
+
stopGateway = stopNativeGateway,
|
|
2492
|
+
}) {
|
|
2493
|
+
let stopped = false;
|
|
2494
|
+
let episode = null;
|
|
2495
|
+
let episodeController = null;
|
|
2496
|
+
let nextEligibleAt = now() + NATIVE_GATEWAY_SERVING_PROOF_INTERVAL_MS;
|
|
2497
|
+
let current = null;
|
|
2498
|
+
const outgoing = [];
|
|
2499
|
+
|
|
2500
|
+
const authorityCurrent = () =>
|
|
2501
|
+
!stopped &&
|
|
2502
|
+
getActiveRuntime() === runtime &&
|
|
2503
|
+
runtime.servingAuthority?.flyGeneration === flyGeneration;
|
|
2504
|
+
|
|
2505
|
+
const publish = (state, reasonCode) => {
|
|
2506
|
+
if (!authorityCurrent()) return null;
|
|
2507
|
+
const observation = nativeGatewayServingObservation({
|
|
2508
|
+
runtime,
|
|
2509
|
+
bootSessionId,
|
|
2510
|
+
flyGeneration,
|
|
2511
|
+
state,
|
|
2512
|
+
reasonCode,
|
|
2513
|
+
checkedAt: now(),
|
|
2514
|
+
});
|
|
2515
|
+
current = observation;
|
|
2516
|
+
if (state !== "READY") {
|
|
2517
|
+
while (outgoing[0]?.state === "READY") outgoing.shift();
|
|
2518
|
+
}
|
|
2519
|
+
const previous = outgoing.at(-1);
|
|
2520
|
+
if (
|
|
2521
|
+
previous?.state === observation.state &&
|
|
2522
|
+
previous?.reasonCode === observation.reasonCode &&
|
|
2523
|
+
previous?.gatewayPid === observation.gatewayPid
|
|
2524
|
+
) {
|
|
2525
|
+
outgoing[outgoing.length - 1] = observation;
|
|
2526
|
+
} else {
|
|
2527
|
+
outgoing.push(observation);
|
|
2528
|
+
}
|
|
2529
|
+
return observation;
|
|
2530
|
+
};
|
|
2531
|
+
|
|
2532
|
+
publish("READY", null);
|
|
2533
|
+
|
|
2534
|
+
const runEpisode = async (signal) => {
|
|
2535
|
+
let failureReason = "RECOVERY_FAILED";
|
|
2536
|
+
try {
|
|
2537
|
+
await proveServing({
|
|
2538
|
+
desired: runtime.desired,
|
|
2539
|
+
apiServerKey: runtime.secrets.hermesApiServer,
|
|
2540
|
+
gateway: runtime.gateway,
|
|
2541
|
+
runtimeConfig: runtime.runtimeConfig,
|
|
2542
|
+
signal,
|
|
2543
|
+
});
|
|
2544
|
+
if (!authorityCurrent()) return;
|
|
2545
|
+
publish("READY", null);
|
|
2546
|
+
nextEligibleAt = now() + NATIVE_GATEWAY_SERVING_PROOF_INTERVAL_MS;
|
|
2547
|
+
return;
|
|
2548
|
+
} catch (error) {
|
|
2549
|
+
throwIfAborted(signal);
|
|
2550
|
+
failureReason = nativeGatewayServingFailureReason(error);
|
|
2551
|
+
}
|
|
2552
|
+
|
|
2553
|
+
publish("RECOVERING", "RECOVERY_IN_PROGRESS");
|
|
2554
|
+
const deadlineAt = now() + NATIVE_GATEWAY_RECOVERY_BUDGET_MS;
|
|
2555
|
+
for (
|
|
2556
|
+
let attempt = 1;
|
|
2557
|
+
attempt <= NATIVE_GATEWAY_RECOVERY_MAX_ATTEMPTS;
|
|
2558
|
+
attempt += 1
|
|
2559
|
+
) {
|
|
2560
|
+
const backoff = Math.round(
|
|
2561
|
+
Math.min(
|
|
2562
|
+
NATIVE_GATEWAY_RECOVERY_BACKOFF_CAP_MS,
|
|
2563
|
+
NATIVE_GATEWAY_RECOVERY_BACKOFF_MS * 2 ** (attempt - 1)
|
|
2564
|
+
) *
|
|
2565
|
+
(0.75 + random() * 0.5)
|
|
2566
|
+
);
|
|
2567
|
+
await pause(backoff, signal);
|
|
2568
|
+
throwIfAborted(signal);
|
|
2569
|
+
if (!authorityCurrent()) return;
|
|
2570
|
+
if (now() >= deadlineAt) {
|
|
2571
|
+
failureReason = "RECOVERY_FAILED";
|
|
2572
|
+
break;
|
|
2573
|
+
}
|
|
2574
|
+
try {
|
|
2575
|
+
await stopGateway(runtime.gateway);
|
|
2576
|
+
throwIfAborted(signal);
|
|
2577
|
+
if (!authorityCurrent()) return;
|
|
2578
|
+
if (now() >= deadlineAt) {
|
|
2579
|
+
failureReason = "RECOVERY_FAILED";
|
|
2580
|
+
break;
|
|
2581
|
+
}
|
|
2582
|
+
const proven = await startProvenGateway({
|
|
2583
|
+
desired: runtime.desired,
|
|
2584
|
+
runtimeConfig: runtime.runtimeConfig,
|
|
2585
|
+
secrets: runtime.secrets,
|
|
2586
|
+
signal,
|
|
2587
|
+
proveReadiness: (input) => proveServing({ ...input, signal }),
|
|
2588
|
+
});
|
|
2589
|
+
throwIfAborted(signal);
|
|
2590
|
+
if (!authorityCurrent()) {
|
|
2591
|
+
await stopGateway(proven.gateway);
|
|
2592
|
+
return;
|
|
2593
|
+
}
|
|
2594
|
+
if (now() >= deadlineAt) {
|
|
2595
|
+
await stopGateway(proven.gateway);
|
|
2596
|
+
failureReason = "RECOVERY_FAILED";
|
|
2597
|
+
break;
|
|
2598
|
+
}
|
|
2599
|
+
runtime.gateway = proven.gateway;
|
|
2600
|
+
publish("READY", null);
|
|
2601
|
+
nextEligibleAt = now() + NATIVE_GATEWAY_SERVING_PROOF_INTERVAL_MS;
|
|
2602
|
+
return;
|
|
2603
|
+
} catch (error) {
|
|
2604
|
+
throwIfAborted(signal);
|
|
2605
|
+
failureReason = nativeGatewayServingFailureReason(error);
|
|
2606
|
+
}
|
|
2607
|
+
}
|
|
2608
|
+
if (!authorityCurrent()) return;
|
|
2609
|
+
publish("DEGRADED", failureReason);
|
|
2610
|
+
nextEligibleAt = now() + NATIVE_GATEWAY_RECOVERY_COOLDOWN_MS;
|
|
2611
|
+
};
|
|
2612
|
+
|
|
2613
|
+
return Object.freeze({
|
|
2614
|
+
pendingObservation() {
|
|
2615
|
+
return outgoing[0] ?? current;
|
|
2616
|
+
},
|
|
2617
|
+
observationSent(observation) {
|
|
2618
|
+
if (outgoing[0] === observation) outgoing.shift();
|
|
2619
|
+
},
|
|
2620
|
+
poll({ force = false } = {}) {
|
|
2621
|
+
if (
|
|
2622
|
+
stopped ||
|
|
2623
|
+
episode ||
|
|
2624
|
+
!authorityCurrent() ||
|
|
2625
|
+
(!force && now() < nextEligibleAt)
|
|
2626
|
+
) {
|
|
2627
|
+
return episode;
|
|
2628
|
+
}
|
|
2629
|
+
episodeController = new AbortController();
|
|
2630
|
+
const activeEpisode = runEpisode(episodeController.signal)
|
|
2631
|
+
.catch((error) => {
|
|
2632
|
+
if (error?.name !== "AbortError") throw error;
|
|
2633
|
+
})
|
|
2634
|
+
.finally(() => {
|
|
2635
|
+
if (episode === activeEpisode) {
|
|
2636
|
+
episode = null;
|
|
2637
|
+
episodeController = null;
|
|
2638
|
+
}
|
|
2639
|
+
});
|
|
2640
|
+
episode = activeEpisode;
|
|
2641
|
+
return activeEpisode;
|
|
2642
|
+
},
|
|
2643
|
+
async stop() {
|
|
2644
|
+
stopped = true;
|
|
2645
|
+
episodeController?.abort();
|
|
2646
|
+
try {
|
|
2647
|
+
await episode;
|
|
2648
|
+
} catch (error) {
|
|
2649
|
+
if (error?.name !== "AbortError") throw error;
|
|
2650
|
+
}
|
|
2651
|
+
},
|
|
2652
|
+
});
|
|
2653
|
+
}
|
|
2654
|
+
|
|
2312
2655
|
async function proveProvisionedApplication({
|
|
2313
2656
|
desired,
|
|
2314
2657
|
runtimeConfig,
|
|
@@ -2674,6 +3017,12 @@ async function processOperation({ claim, exchange, client, identity }) {
|
|
|
2674
3017
|
fetchLatest: () => client.getMemorySnapshot(),
|
|
2675
3018
|
});
|
|
2676
3019
|
if (!memoryRestore.ok) throw new Error(memoryRestore.code);
|
|
3020
|
+
const legacyUserState = migrateLegacyHermesUserState({
|
|
3021
|
+
profileId: desired.profileId,
|
|
3022
|
+
profileRoot,
|
|
3023
|
+
hermesRoot: DATA_ROOT,
|
|
3024
|
+
});
|
|
3025
|
+
if (!legacyUserState.ok) throw new Error(legacyUserState.code);
|
|
2677
3026
|
const materialized = materializeNativeSlackCustomerProfile({
|
|
2678
3027
|
desired,
|
|
2679
3028
|
profilesRoot: PROFILES_ROOT,
|
|
@@ -2809,6 +3158,14 @@ async function processOperation({ claim, exchange, client, identity }) {
|
|
|
2809
3158
|
runtimeConfig,
|
|
2810
3159
|
trustRootPem,
|
|
2811
3160
|
action: claim.operation.action,
|
|
3161
|
+
servingAuthority: Object.freeze({
|
|
3162
|
+
runtimeId: RUNTIME_ID,
|
|
3163
|
+
flyGeneration: RUNTIME_GENERATION,
|
|
3164
|
+
runtimeGeneration: RUNTIME_GENERATION,
|
|
3165
|
+
configGeneration: claim.operation.configGeneration,
|
|
3166
|
+
lifecycleGeneration: claim.operation.lifecycleGeneration,
|
|
3167
|
+
profileId: desired.profileId,
|
|
3168
|
+
}),
|
|
2812
3169
|
gateway: proof.gateway ?? null,
|
|
2813
3170
|
modelHealthReporter,
|
|
2814
3171
|
};
|
|
@@ -2858,6 +3215,7 @@ async function main() {
|
|
|
2858
3215
|
});
|
|
2859
3216
|
const controller = new AbortController();
|
|
2860
3217
|
let activeRuntime = null;
|
|
3218
|
+
let servingSupervisor = null;
|
|
2861
3219
|
let shutdownMemorySnapshot = null;
|
|
2862
3220
|
const endpointRelay = createCustomerEndpointRelay({
|
|
2863
3221
|
getControlClient: () => client,
|
|
@@ -2871,6 +3229,8 @@ async function main() {
|
|
|
2871
3229
|
const runtime = activeRuntime;
|
|
2872
3230
|
shutdownMemorySnapshot = (async () => {
|
|
2873
3231
|
try {
|
|
3232
|
+
await servingSupervisor?.stop();
|
|
3233
|
+
servingSupervisor = null;
|
|
2874
3234
|
await endpointRelay.fenceActive("RUNTIME_SHUTDOWN");
|
|
2875
3235
|
await stopNativeGateway(runtime.gateway);
|
|
2876
3236
|
runtime.gateway = null;
|
|
@@ -2912,6 +3272,10 @@ async function main() {
|
|
|
2912
3272
|
callerProfileId: activeRuntime.desired.profileId,
|
|
2913
3273
|
}),
|
|
2914
3274
|
leaseSeconds: 60,
|
|
3275
|
+
servingObservation:
|
|
3276
|
+
servingSupervisor?.pendingObservation() ?? undefined,
|
|
3277
|
+
onServingObservationSent: (observation) =>
|
|
3278
|
+
servingSupervisor?.observationSent(observation),
|
|
2915
3279
|
})
|
|
2916
3280
|
: await client.claim(60);
|
|
2917
3281
|
} catch (error) {
|
|
@@ -2956,20 +3320,12 @@ async function main() {
|
|
|
2956
3320
|
),
|
|
2957
3321
|
});
|
|
2958
3322
|
}
|
|
2959
|
-
|
|
2960
|
-
activeRuntime?.action !== "INSTANTIATE" &&
|
|
2961
|
-
activeRuntime?.gateway &&
|
|
2962
|
-
!activeRuntime.gateway.readback().running
|
|
2963
|
-
) {
|
|
2964
|
-
activeRuntime.gateway = await startNativeGateway({
|
|
2965
|
-
desired: activeRuntime.desired,
|
|
2966
|
-
runtimeConfig: activeRuntime.runtimeConfig,
|
|
2967
|
-
secrets: activeRuntime.secrets,
|
|
2968
|
-
});
|
|
2969
|
-
}
|
|
3323
|
+
void servingSupervisor?.poll();
|
|
2970
3324
|
await sleep(1_000);
|
|
2971
3325
|
continue;
|
|
2972
3326
|
}
|
|
3327
|
+
await servingSupervisor?.stop();
|
|
3328
|
+
servingSupervisor = null;
|
|
2973
3329
|
operationHeartbeat = startOperationHeartbeat({
|
|
2974
3330
|
claim,
|
|
2975
3331
|
exchange,
|
|
@@ -2984,6 +3340,15 @@ async function main() {
|
|
|
2984
3340
|
client,
|
|
2985
3341
|
identity,
|
|
2986
3342
|
});
|
|
3343
|
+
servingSupervisor =
|
|
3344
|
+
activeRuntime.action === "INSTANTIATE"
|
|
3345
|
+
? null
|
|
3346
|
+
: createNativeGatewayServingSupervisor({
|
|
3347
|
+
runtime: activeRuntime,
|
|
3348
|
+
getActiveRuntime: () => activeRuntime,
|
|
3349
|
+
bootSessionId: BOOT_SESSION_ID,
|
|
3350
|
+
flyGeneration: RUNTIME_GENERATION,
|
|
3351
|
+
});
|
|
2987
3352
|
soulClaimTracker = createSoulClaimObservationTracker({
|
|
2988
3353
|
runtimeId: RUNTIME_ID,
|
|
2989
3354
|
profileId: activeRuntime.desired.profileId,
|
|
@@ -784,6 +784,8 @@ export async function claimWithSoulObservation({
|
|
|
784
784
|
tracker,
|
|
785
785
|
readSoul,
|
|
786
786
|
leaseSeconds = 60,
|
|
787
|
+
servingObservation,
|
|
788
|
+
onServingObservationSent,
|
|
787
789
|
}) {
|
|
788
790
|
// Deliver lifecycle work before any potentially slow local loader proof.
|
|
789
791
|
// Reconciliation uses the same cadence only after the server confirms that
|
|
@@ -791,7 +793,9 @@ export async function claimWithSoulObservation({
|
|
|
791
793
|
const tick = await client.claimTick({
|
|
792
794
|
leaseSeconds,
|
|
793
795
|
soulObservation: tracker.pending(),
|
|
796
|
+
...(servingObservation ? { servingObservation } : {}),
|
|
794
797
|
});
|
|
798
|
+
if (servingObservation) onServingObservationSent?.(servingObservation);
|
|
795
799
|
if (tick?.claim) return tick.claim;
|
|
796
800
|
|
|
797
801
|
const acknowledgement = tick?.soulAcknowledgement;
|
|
@@ -1074,10 +1078,15 @@ export function createFlyCustomerControlClient({
|
|
|
1074
1078
|
}
|
|
1075
1079
|
return { ...result, signedUrl: undefined, bytes };
|
|
1076
1080
|
};
|
|
1077
|
-
const claimTick = async ({
|
|
1081
|
+
const claimTick = async ({
|
|
1082
|
+
leaseSeconds = 60,
|
|
1083
|
+
soulObservation,
|
|
1084
|
+
servingObservation,
|
|
1085
|
+
} = {}) =>
|
|
1078
1086
|
post("claim", "/api/v3/sellable-agent/worker/claim", {
|
|
1079
1087
|
leaseSeconds,
|
|
1080
1088
|
...(soulObservation ? { soulObservation } : {}),
|
|
1089
|
+
...(servingObservation ? { servingObservation } : {}),
|
|
1081
1090
|
});
|
|
1082
1091
|
return Object.freeze({
|
|
1083
1092
|
claimTick,
|
|
@@ -25,7 +25,7 @@ import { withProfileSoulLock } from "./fly-soul-bridge.mjs";
|
|
|
25
25
|
export const HERMES_SNAPSHOT_HOOKS = Object.freeze({
|
|
26
26
|
post_tool_call: Object.freeze([
|
|
27
27
|
Object.freeze({
|
|
28
|
-
matcher: "^(memory|skill_manage)$",
|
|
28
|
+
matcher: "^(cronjob|memory|skill_manage)$",
|
|
29
29
|
command: "/opt/sellable/bin/hermes-memory-dirty",
|
|
30
30
|
// Node startup can exceed one second when a shared-CPU customer is also
|
|
31
31
|
// serving Slack and an operator Desktop. Missing this hook defers a real
|
|
@@ -125,6 +125,8 @@ const SNAPSHOT_TRANSIENT_DIRECTORIES = new Set([
|
|
|
125
125
|
"venv",
|
|
126
126
|
]);
|
|
127
127
|
const SOUL_LOCK_ROOT = "soul-locks";
|
|
128
|
+
const LEGACY_USER_STATE_MIGRATION_SCHEMA =
|
|
129
|
+
"sellable-hermes-legacy-user-state-migration/v1";
|
|
128
130
|
const RESTORE_STATE_SCHEMA = "sellable-hermes-restore-state/v1";
|
|
129
131
|
const RETRY_STATE_SCHEMA = "sellable-hermes-snapshot-retry/v1";
|
|
130
132
|
const snapshotReconciles = new Map();
|
|
@@ -396,6 +398,118 @@ function stageHermesUserState({ profileId, profileRoot, hermesRoot }) {
|
|
|
396
398
|
return stagingRoot;
|
|
397
399
|
}
|
|
398
400
|
|
|
401
|
+
function legacyUserStateMigrationMarker(profileRoot) {
|
|
402
|
+
return join(
|
|
403
|
+
profileRoot,
|
|
404
|
+
".sellable-agent",
|
|
405
|
+
"migrations",
|
|
406
|
+
"legacy-root-user-state-v1.json"
|
|
407
|
+
);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function readLegacyUserStateMigration(path, profileId) {
|
|
411
|
+
regularFile(path);
|
|
412
|
+
const value = JSON.parse(readFileSync(path, "utf8"));
|
|
413
|
+
if (
|
|
414
|
+
value?.schemaVersion !== LEGACY_USER_STATE_MIGRATION_SCHEMA ||
|
|
415
|
+
value.profileId !== profileId ||
|
|
416
|
+
!Array.isArray(value.entries) ||
|
|
417
|
+
value.entries.some(
|
|
418
|
+
(name) =>
|
|
419
|
+
typeof name !== "string" ||
|
|
420
|
+
!HERMES_SNAPSHOT_USER_STATE_ENTRIES.includes(name)
|
|
421
|
+
) ||
|
|
422
|
+
new Set(value.entries).size !== value.entries.length
|
|
423
|
+
) {
|
|
424
|
+
throw new Error("hermes_legacy_user_state_migration_rejected");
|
|
425
|
+
}
|
|
426
|
+
return value;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Move the pre-profile Hermes layout into the current profile exactly once.
|
|
431
|
+
*
|
|
432
|
+
* Older Admin Machines stored durable user state directly under HERMES_HOME
|
|
433
|
+
* (for example /opt/data/skills and /opt/data/cron). Profile isolation moved
|
|
434
|
+
* those paths under profiles/<id>. Snapshot restore cannot discover that
|
|
435
|
+
* unscoped state because it intentionally restores profile archives only.
|
|
436
|
+
* Stage only the same bounded user-state allowlist used by snapshots, merge
|
|
437
|
+
* without overwriting newer profile bytes, then mark the resulting profile
|
|
438
|
+
* dirty so the next portable snapshot becomes self-contained.
|
|
439
|
+
*/
|
|
440
|
+
export function migrateLegacyHermesUserState({
|
|
441
|
+
profileId,
|
|
442
|
+
profileRoot,
|
|
443
|
+
hermesRoot,
|
|
444
|
+
}) {
|
|
445
|
+
if (!validProfilePath(profileId, profileRoot, hermesRoot)) {
|
|
446
|
+
return failed("hermes_legacy_user_state_identity_rejected");
|
|
447
|
+
}
|
|
448
|
+
const marker = legacyUserStateMigrationMarker(profileRoot);
|
|
449
|
+
try {
|
|
450
|
+
if (existsSync(marker)) {
|
|
451
|
+
const receipt = readLegacyUserStateMigration(marker, profileId);
|
|
452
|
+
return {
|
|
453
|
+
ok: true,
|
|
454
|
+
status: "REUSED",
|
|
455
|
+
profileId,
|
|
456
|
+
entries: receipt.entries,
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
const entries = HERMES_SNAPSHOT_USER_STATE_ENTRIES.filter((name) =>
|
|
460
|
+
existsSync(join(hermesRoot, name))
|
|
461
|
+
);
|
|
462
|
+
if (entries.length === 0) {
|
|
463
|
+
return { ok: true, status: "NO_LEGACY_STATE", profileId, entries: [] };
|
|
464
|
+
}
|
|
465
|
+
const backupsRoot = join(hermesRoot, "backups");
|
|
466
|
+
mkdirSync(backupsRoot, { recursive: true, mode: 0o700 });
|
|
467
|
+
const stagingRoot = mkdtempSync(join(backupsRoot, ".legacy-user-state-"));
|
|
468
|
+
chmodSync(stagingRoot, 0o700);
|
|
469
|
+
try {
|
|
470
|
+
for (const name of entries) {
|
|
471
|
+
const source = join(hermesRoot, name);
|
|
472
|
+
const stat = lstatSync(source);
|
|
473
|
+
if (stat.isSymbolicLink()) {
|
|
474
|
+
throw new Error("hermes_legacy_user_state_symlink_rejected");
|
|
475
|
+
}
|
|
476
|
+
stageUserStateEntry(source, join(stagingRoot, name));
|
|
477
|
+
if (name.endsWith(".db")) {
|
|
478
|
+
for (const suffix of ["-wal", "-shm", "-journal"]) {
|
|
479
|
+
const sidecar = `${source}${suffix}`;
|
|
480
|
+
if (existsSync(sidecar)) {
|
|
481
|
+
stageUserStateEntry(
|
|
482
|
+
sidecar,
|
|
483
|
+
join(stagingRoot, `${name}${suffix}`)
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
mkdirSync(profileRoot, { recursive: true, mode: 0o700 });
|
|
490
|
+
mergeHermesState(stagingRoot, profileRoot);
|
|
491
|
+
if (entries.some((name) => !existsSync(join(profileRoot, name)))) {
|
|
492
|
+
throw new Error("hermes_legacy_user_state_readback_rejected");
|
|
493
|
+
}
|
|
494
|
+
atomicWrite(
|
|
495
|
+
marker,
|
|
496
|
+
`${JSON.stringify({
|
|
497
|
+
schemaVersion: LEGACY_USER_STATE_MIGRATION_SCHEMA,
|
|
498
|
+
profileId,
|
|
499
|
+
entries,
|
|
500
|
+
})}\n`,
|
|
501
|
+
0o600
|
|
502
|
+
);
|
|
503
|
+
markHermesSnapshotDirty(profileRoot);
|
|
504
|
+
return { ok: true, status: "MIGRATED", profileId, entries };
|
|
505
|
+
} finally {
|
|
506
|
+
rmSync(stagingRoot, { recursive: true, force: true });
|
|
507
|
+
}
|
|
508
|
+
} catch {
|
|
509
|
+
return failed("hermes_legacy_user_state_migration_failed");
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
399
513
|
export function markHermesSnapshotDirty(profileRoot) {
|
|
400
514
|
if (
|
|
401
515
|
!isAbsolute(profileRoot) ||
|
|
@@ -38,7 +38,7 @@ import { installAgentHostServices } from "./service-installer.mjs";
|
|
|
38
38
|
const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
|
39
39
|
const VERSION = "sellable-agent-host-bootstrap/v1";
|
|
40
40
|
const RECEIPT_VERSION = "sellable-agent-host-registration/v1";
|
|
41
|
-
const INSTALLER_PACKAGE = "@sellable/install@0.1.
|
|
41
|
+
const INSTALLER_PACKAGE = "@sellable/install@0.1.608-wip.phase188.20260812.1";
|
|
42
42
|
const MCP_PACKAGE = "@sellable/mcp@0.1.864";
|
|
43
43
|
const HERMES_VERSION = "0.18.0";
|
|
44
44
|
const SHA256 = /^[a-f0-9]{64}$/;
|
|
@@ -1546,7 +1546,7 @@ export function compileClaimToProfileDesired(activeClaim, config) {
|
|
|
1546
1546
|
]) ||
|
|
1547
1547
|
pinned.hermesCli !== "hermes" ||
|
|
1548
1548
|
pinned.hermesVersion !== "0.18.0" ||
|
|
1549
|
-
pinned.installerPackage !== "@sellable/install@0.1.
|
|
1549
|
+
pinned.installerPackage !== "@sellable/install@0.1.608-wip.phase188.20260812.1" ||
|
|
1550
1550
|
pinned.mcpPackage !== "@sellable/mcp@0.1.864" ||
|
|
1551
1551
|
!Array.isArray(policy.toolInclude) ||
|
|
1552
1552
|
policy.toolInclude.length === 0 ||
|
|
@@ -278,7 +278,7 @@ function validateDesired(desired) {
|
|
|
278
278
|
desired.serviceCredentialGeneration < 1 ||
|
|
279
279
|
desired.hermesCli !== "hermes" ||
|
|
280
280
|
desired.hermesVersion !== "0.18.0" ||
|
|
281
|
-
desired.installerPackage !== "@sellable/install@0.1.
|
|
281
|
+
desired.installerPackage !== "@sellable/install@0.1.608-wip.phase188.20260812.1" ||
|
|
282
282
|
desired.mcpPackage !== "@sellable/mcp@0.1.864" ||
|
|
283
283
|
!Array.isArray(desired.toolInclude) ||
|
|
284
284
|
desired.toolInclude.length === 0 ||
|
|
@@ -25,7 +25,7 @@ import { fileURLToPath } from "node:url";
|
|
|
25
25
|
import { deriveContainedProfileId } from "./profile-materializer.mjs";
|
|
26
26
|
|
|
27
27
|
export const PROVISIONING_ACTION = "PROVISION_HERMES_PROFILE";
|
|
28
|
-
export const PINNED_INSTALL_PACKAGE = "@sellable/install@0.1.
|
|
28
|
+
export const PINNED_INSTALL_PACKAGE = "@sellable/install@0.1.608-wip.phase188.20260812.1";
|
|
29
29
|
export const PINNED_MCP_PACKAGE = "@sellable/mcp@0.1.864";
|
|
30
30
|
const PINNED_INSTALL_VERSION = PINNED_INSTALL_PACKAGE.slice(
|
|
31
31
|
PINNED_INSTALL_PACKAGE.lastIndexOf("@") + 1
|