@evident-ai/runner-cdk 0.1.1-dev.da88f8e → 3.4.1-dev.1856549
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -13
- package/dist/evident-scale-to-zero-construct.js +4 -5
- package/dist/microvm/constants.js +2 -3
- package/dist/microvm/construct.d.ts +16 -2
- package/dist/microvm/construct.js +8 -0
- package/dist/microvm/controller/handle-doorbell.js +2 -2
- package/dist/microvm/controller/microvm-client.d.ts +1 -1
- package/dist/microvm-image-context/Dockerfile +10 -7
- package/dist/microvm-image-context/hook-server.js +2 -6
- package/dist/microvm-image-context/hooks/common.sh +184 -21
- package/dist/microvm-image-context/hooks/resume +1 -1
- package/dist/microvm-image-context/hooks/run +9 -3
- package/dist/waker/construct.js +1 -1
- package/package.json +10 -8
package/README.md
CHANGED
|
@@ -9,9 +9,9 @@ Reusable AWS CDK constructs for running an [Evident](https://evident.run) agent
|
|
|
9
9
|
`desiredCount 0`, and an HMAC-authenticated waker Lambda scales it back up on the next
|
|
10
10
|
message. You pay for the time the agent is actually working.
|
|
11
11
|
- **`EvidentMicrovmConstruct`** — a per-session AWS Lambda MicroVM that boots on demand
|
|
12
|
-
and suspends between messages. Installing this package is all you
|
|
13
|
-
context ships inside it (see [The MicroVM image](#the-microvm-image)
|
|
14
|
-
no checkout of `sroze/evident` involved. See
|
|
12
|
+
and suspends between messages. Installing the `@dev` tag of this package is all you
|
|
13
|
+
need: the image build context ships inside it (see [The MicroVM image](#the-microvm-image)
|
|
14
|
+
below), so there is no checkout of `sroze/evident` involved. See
|
|
15
15
|
[the AWS runner doc](https://evident.run/docs/aws-runner) for the full picture of both
|
|
16
16
|
strategies.
|
|
17
17
|
|
|
@@ -34,8 +34,9 @@ cluster can host several agents.
|
|
|
34
34
|
## Getting started
|
|
35
35
|
|
|
36
36
|
For a full end-to-end walkthrough — shared infra, secrets without SOPS/KMS, the image,
|
|
37
|
-
the two grants, deploy, and wiring the wake webhook — see
|
|
38
|
-
|
|
37
|
+
the two grants, deploy, and wiring the wake webhook — see `GETTING-STARTED.md` in this
|
|
38
|
+
package's directory (not published to npm; read it from a checkout of the repo). The
|
|
39
|
+
snippet below is the short version.
|
|
39
40
|
|
|
40
41
|
```ts
|
|
41
42
|
import { EvidentScaleToZeroConstruct } from "@evident-ai/runner-cdk";
|
|
@@ -111,7 +112,7 @@ the construct's own dogfooding consumer.
|
|
|
111
112
|
|
|
112
113
|
This construct never builds an image. Pass any `ecs.ContainerImage` — from a registry, or
|
|
113
114
|
built from a `Dockerfile` you control. The generic runner image lives separately in
|
|
114
|
-
`
|
|
115
|
+
`runner/docker-images/fargate`, so you can adopt the image, the construct, or both.
|
|
115
116
|
|
|
116
117
|
## The MicroVM image
|
|
117
118
|
|
|
@@ -126,6 +127,7 @@ the per-phase hook scripts and the bundled hook server, published inside the tar
|
|
|
126
127
|
the agent's workspace. `stageMicrovmImageContext()` puts the two together:
|
|
127
128
|
|
|
128
129
|
```ts
|
|
130
|
+
import path from 'node:path';
|
|
129
131
|
import {
|
|
130
132
|
EvidentMicrovmConstruct,
|
|
131
133
|
HOOKS_PORT,
|
|
@@ -149,6 +151,8 @@ new EvidentMicrovmConstruct(this, 'Runner', {
|
|
|
149
151
|
baseImageArn,
|
|
150
152
|
baseImageVersion,
|
|
151
153
|
doorbellSecret,
|
|
154
|
+
runnerSecret,
|
|
155
|
+
runnerOpencodeConfigPath: 'opencode.runner.jsonc',
|
|
152
156
|
});
|
|
153
157
|
```
|
|
154
158
|
|
|
@@ -156,14 +160,19 @@ Your repo does not have to be a pnpm workspace. If it commits a `pnpm-lock.yaml`
|
|
|
156
160
|
pre-installs dependencies at build time (a faster first boot); if not, that step is skipped
|
|
157
161
|
and the agent installs on first use.
|
|
158
162
|
|
|
163
|
+
| MicroVM prop | Omitted behavior |
|
|
164
|
+
| --- | --- |
|
|
165
|
+
| `runnerSecret` | No GitHub or MCP credentials are exported at `/run`. |
|
|
166
|
+
| `runnerOpencodeConfigPath` | OpenCode uses the baked project configuration. Relative paths resolve from the workspace; absolute paths resolve in the image. |
|
|
167
|
+
| `gitUserName` / `gitUserEmail` | The hook uses the Evident bot defaults for git identity. |
|
|
168
|
+
|
|
159
169
|
## Status / limitations
|
|
160
170
|
|
|
161
|
-
- **Published to npm** as `@evident-ai/runner-cdk` (MIT) —
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
dependency for dogfooding.
|
|
171
|
+
- **Published to npm** as `@evident-ai/runner-cdk` (MIT). Install the `@dev` tag —
|
|
172
|
+
`npm install @evident-ai/runner-cdk@dev` (or `pnpm add`/`yarn add`); no checkout of
|
|
173
|
+
this repo is required. It builds to a self-contained `dist/` (`pnpm --filter
|
|
174
|
+
@evident-ai/runner-cdk build`) with no `workspace:`/`@evident/*` runtime dependency.
|
|
175
|
+
This repo's own consumer (`infrastructure/evident-runner`) still consumes it as a
|
|
176
|
+
`workspace:*` dependency for dogfooding.
|
|
168
177
|
- **AWS/ECS-specific.** Non-AWS clouds are an open question on the epic, not a supported
|
|
169
178
|
path.
|
|
@@ -101,7 +101,7 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
|
|
|
101
101
|
// --endpoint / --tunnel). The agent is resolved from EVIDENT_AGENT_KEY.
|
|
102
102
|
EVIDENT_API_URL: evidentApiUrl,
|
|
103
103
|
EVIDENT_TUNNEL_URL: evidentTunnelUrl,
|
|
104
|
-
// Litestream replica target (read by
|
|
104
|
+
// Litestream replica target (read by runner/synchroniser, which
|
|
105
105
|
// generates /etc/evident/litestream.yml at boot).
|
|
106
106
|
LITESTREAM_BUCKET: replicaBucket.bucketName,
|
|
107
107
|
LITESTREAM_PREFIX: this.replicaPrefix,
|
|
@@ -115,10 +115,9 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
|
|
|
115
115
|
// window (protected/active sessions are never touched). Tightened from 7d
|
|
116
116
|
// to 24h per #537: at 7d, opencode.db was observed growing ~11x (77 MB ->
|
|
117
117
|
// 849 MB) in ~24h, so 7d was far too permissive at the observed growth
|
|
118
|
-
// rate. 24h is
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
// interval and never touches a protected/active session.
|
|
118
|
+
// rate. 24h is the tighter mitigation that came out of that (see also the
|
|
119
|
+
// OPENCODE_VERSION pin above). Sweep runs on the default 1h interval and
|
|
120
|
+
// never touches a protected/active session.
|
|
122
121
|
EVIDENT_SESSION_CLEANUP_MAX_AGE: '24h',
|
|
123
122
|
CLUSTER: cluster.clusterName,
|
|
124
123
|
SERVICE: serviceName,
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// Shared literals for the MicroVM controller and for the image it launches.
|
|
3
3
|
// Every module imports from here rather than restating a value that must match
|
|
4
|
-
// across them
|
|
5
|
-
// image
|
|
6
|
-
// `scripts/build.ts`, so these values reach the image as inlined constants.
|
|
4
|
+
// across them. The image template pins matching copies, guarded by
|
|
5
|
+
// `microvm/image/dockerfile.test.ts`.
|
|
7
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
7
|
exports.MICROVM_MAX_RUN_SECONDS = exports.SUSPENDING_POLL_INTERVAL_MS = exports.SUSPENDING_POLL_ATTEMPTS = exports.RUN_HOOK_PAYLOAD_MAX_BYTES = exports.HOOK_TIMEOUT_SECONDS = exports.HOOKS_DIR = exports.HOOKS_PORT = void 0;
|
|
9
8
|
// The port the image's hook server binds, baked into the published Dockerfile
|
|
@@ -12,7 +12,7 @@ export interface EvidentMicrovmConstructProps {
|
|
|
12
12
|
readonly imageSource: string;
|
|
13
13
|
/**
|
|
14
14
|
* ARN of the Lambda-managed base MicroVM image, from
|
|
15
|
-
* `aws lambda list-managed-microvm-images`. A string, not a `CfnParameter`
|
|
15
|
+
* `aws lambda-microvms list-managed-microvm-images`. A string, not a `CfnParameter`
|
|
16
16
|
* built here: the caller creates the parameter at ITS OWN stack scope
|
|
17
17
|
* (preserving its logical id) and passes its resolved value down — a
|
|
18
18
|
* construct must not mint parameters into a consumer's stack.
|
|
@@ -20,7 +20,7 @@ export interface EvidentMicrovmConstructProps {
|
|
|
20
20
|
readonly baseImageArn: string;
|
|
21
21
|
/**
|
|
22
22
|
* Version of the Lambda-managed base MicroVM image, from
|
|
23
|
-
* `aws lambda list-managed-microvm-images`. See `baseImageArn` for why this
|
|
23
|
+
* `aws lambda-microvms list-managed-microvm-images`. See `baseImageArn` for why this
|
|
24
24
|
* is a plain string.
|
|
25
25
|
*/
|
|
26
26
|
readonly baseImageVersion: string;
|
|
@@ -41,6 +41,20 @@ export interface EvidentMicrovmConstructProps {
|
|
|
41
41
|
* in its own scope, and passes the result in either case.
|
|
42
42
|
*/
|
|
43
43
|
readonly doorbellSecret: secretsmanager.ISecret;
|
|
44
|
+
/**
|
|
45
|
+
* Optional runner secret whose JSON values are exported by `/run`. Omitting it
|
|
46
|
+
* leaves GitHub and MCP credentials unavailable to a generic consumer.
|
|
47
|
+
*/
|
|
48
|
+
readonly runnerSecret?: secretsmanager.ISecret;
|
|
49
|
+
/**
|
|
50
|
+
* Optional runner OpenCode overlay. Absolute paths are image paths; relative
|
|
51
|
+
* paths are resolved from the baked workspace by the `/run` hook.
|
|
52
|
+
*/
|
|
53
|
+
readonly runnerOpencodeConfigPath?: string;
|
|
54
|
+
/** Optional git identity the `/run` hook configures for agent commits. */
|
|
55
|
+
readonly gitUserName?: string;
|
|
56
|
+
/** Optional git email the `/run` hook configures for agent commits. */
|
|
57
|
+
readonly gitUserEmail?: string;
|
|
44
58
|
/**
|
|
45
59
|
* TCP port the image's hook server listens on, baked into both the image
|
|
46
60
|
* (`HOOKS_PORT` env var) and the `MicrovmImage`'s `hooks.port` — a mismatch
|
|
@@ -121,6 +121,12 @@ class EvidentMicrovmConstruct extends constructs_1.Construct {
|
|
|
121
121
|
// already bakes in for its own runners (evident-scale-to-zero-construct.ts:191).
|
|
122
122
|
// Safe to bake: identical for every VM from this image version.
|
|
123
123
|
EVIDENT_SESSION_CLEANUP_MAX_AGE: '24h',
|
|
124
|
+
...(props.runnerOpencodeConfigPath
|
|
125
|
+
? { RUNNER_OPENCODE_CONFIG: props.runnerOpencodeConfigPath }
|
|
126
|
+
: {}),
|
|
127
|
+
...(props.runnerSecret ? { RUNNER_SECRET_ARN: props.runnerSecret.secretArn } : {}),
|
|
128
|
+
...(props.gitUserName ? { GIT_USER_NAME: props.gitUserName } : {}),
|
|
129
|
+
...(props.gitUserEmail ? { GIT_USER_EMAIL: props.gitUserEmail } : {}),
|
|
124
130
|
};
|
|
125
131
|
// ...and the hooks configuration every shape's image is built with.
|
|
126
132
|
const imageHooks = {
|
|
@@ -179,6 +185,8 @@ class EvidentMicrovmConstruct extends constructs_1.Construct {
|
|
|
179
185
|
// customer account, and what Phase 4 (#263) has to tighten when this goes
|
|
180
186
|
// multi-tenant.
|
|
181
187
|
durableState.grantReadWrite(defaultImage.executionRole);
|
|
188
|
+
// The image receives only an ARN; `/run` reads the secret value at runtime.
|
|
189
|
+
props.runnerSecret?.grantRead(defaultImage.executionRole);
|
|
182
190
|
// The shape catalogue the controller resolves a doorbell's `shape` field
|
|
183
191
|
// against, and advertises over the describe channel (#723). One JSON
|
|
184
192
|
// array, built with `Stack.toJsonString` — not `JSON.stringify` — because
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.handleDoorbell = handleDoorbell;
|
|
4
4
|
const node_crypto_1 = require("node:crypto");
|
|
5
|
-
const
|
|
5
|
+
const sdk_1 = require("@evident/sdk");
|
|
6
6
|
const constants_1 = require("../constants");
|
|
7
7
|
const doorbell_1 = require("./doorbell");
|
|
8
8
|
const throttle_retry_1 = require("./throttle-retry");
|
|
@@ -175,7 +175,7 @@ async function shouldRecreateForNewerImage(doorbell, shape, runningVersion, micr
|
|
|
175
175
|
return compareImageVersions(running, latest) < 0;
|
|
176
176
|
}
|
|
177
177
|
async function handleDoorbell({ rawBody, signatureHeader, doorbellSecret, shapes, microvm, sleep, random, }) {
|
|
178
|
-
if (!(0,
|
|
178
|
+
if (!(0, sdk_1.verifyEvidentSignature)(rawBody, signatureHeader, doorbellSecret)) {
|
|
179
179
|
return decide({
|
|
180
180
|
statusCode: 401,
|
|
181
181
|
action: 'rejected',
|
|
@@ -49,7 +49,7 @@ export interface MicrovmClient {
|
|
|
49
49
|
* (docs/spikes/lambda-microvms-phase0/README.md). AWS's own Claude-agents
|
|
50
50
|
* guidance recommending `maxIdleDurationSeconds: 120` targets VMs reached
|
|
51
51
|
* inbound and does not apply here. Idle is instead decided IN the VM by
|
|
52
|
-
* `evident run --idle-timeout` (
|
|
52
|
+
* `evident run --idle-timeout` (runner/docker-images/microvm/hooks/common.sh's
|
|
53
53
|
* `IDLE_TIMEOUT_SECONDS`), whose clean exit drives this
|
|
54
54
|
* client's own `suspend()` via the `runner.suspend_requested` doorbell.
|
|
55
55
|
*/
|
|
@@ -37,7 +37,7 @@ RUN corepack enable && corepack prepare pnpm@9.15.0 --activate
|
|
|
37
37
|
# binary assertion further down fails the build if any required binary is gone.
|
|
38
38
|
RUN apt-get update \
|
|
39
39
|
&& apt-get install -y --no-install-recommends \
|
|
40
|
-
ca-certificates curl git jq less procps ripgrep \
|
|
40
|
+
ca-certificates curl git jq less procps psmisc ripgrep \
|
|
41
41
|
postgresql postgresql-contrib direnv \
|
|
42
42
|
gnupg unzip \
|
|
43
43
|
# --- GitHub CLI (`gh`) from the official apt repo ---
|
|
@@ -71,8 +71,8 @@ RUN apt-get update \
|
|
|
71
71
|
# restore` attempt — then starts a backgrounded `litestream replicate` once
|
|
72
72
|
# opencode has opened the DB (WI-3); `/suspend`/`/resume`/`/terminate` flush,
|
|
73
73
|
# restart and stop it in turn (WI-4). Do not remove as dead code.
|
|
74
|
-
ARG LITESTREAM_VERSION=0.5.
|
|
75
|
-
ARG LITESTREAM_SHA256=
|
|
74
|
+
ARG LITESTREAM_VERSION=0.5.16
|
|
75
|
+
ARG LITESTREAM_SHA256=678022e4103145302598e35d37f8718392d42e153feeb1e2d4a64dd0cd3aaf10
|
|
76
76
|
RUN curl -fsSL -o /tmp/litestream.tar.gz \
|
|
77
77
|
"https://github.com/benbjohnson/litestream/releases/download/v${LITESTREAM_VERSION}/litestream-${LITESTREAM_VERSION}-linux-arm64.tar.gz" \
|
|
78
78
|
&& echo "${LITESTREAM_SHA256} /tmp/litestream.tar.gz" | sha256sum -c - \
|
|
@@ -171,10 +171,10 @@ ENV HOME=/home/runner
|
|
|
171
171
|
# permanent no-op and every VM from this snapshot shares one machine id. Empty
|
|
172
172
|
# is the correct unset state — it is the absence of an identity, so nothing
|
|
173
173
|
# per-VM-unique enters the shared snapshot.
|
|
174
|
-
RUN mkdir -p /var/lib/runner-pg /var/lib/dbus \
|
|
174
|
+
RUN mkdir -p /var/lib/runner-pg /var/lib/dbus /home/runner/.local/state/evident \
|
|
175
175
|
&& install -o runner -g runner -m 0644 /dev/null /etc/machine-id \
|
|
176
176
|
&& install -o runner -g runner -m 0644 /dev/null /var/lib/dbus/machine-id \
|
|
177
|
-
|
|
177
|
+
&& chown -R runner:runner /var/lib/runner-pg /home/runner
|
|
178
178
|
COPY --chown=10001:10001 repo ${WORKSPACE}
|
|
179
179
|
WORKDIR ${WORKSPACE}
|
|
180
180
|
USER runner
|
|
@@ -204,9 +204,12 @@ RUN git reset --quiet
|
|
|
204
204
|
# database stay per-VM (see entrypoint.sh's pre-warm in evident-runner).
|
|
205
205
|
RUN if [ -f pnpm-lock.yaml ]; then \
|
|
206
206
|
pnpm install --frozen-lockfile \
|
|
207
|
-
&& pnpm run build --filter='./packages/*'
|
|
207
|
+
&& pnpm run build --filter='./packages/*' \
|
|
208
|
+
--filter='./aws/runner-cdk' \
|
|
209
|
+
--filter='./aws/lambda-microvm-cdk' \
|
|
210
|
+
--filter='./aws/lambda-microvm-runtime'; \
|
|
208
211
|
else \
|
|
209
|
-
echo "[workspace-prep] no pnpm-lock.yaml in the baked repository — skipping the dependency install and the
|
|
212
|
+
echo "[workspace-prep] no pnpm-lock.yaml in the baked repository — skipping the dependency install and the workspace build; the agent installs on first use."; \
|
|
210
213
|
fi
|
|
211
214
|
|
|
212
215
|
# Data dir for the local dev Postgres cluster; listens on 5433 to match this
|
|
@@ -275,14 +275,10 @@ var require_dist = __commonJS({
|
|
|
275
275
|
}
|
|
276
276
|
});
|
|
277
277
|
|
|
278
|
-
//
|
|
278
|
+
// ../../runner/docker-images/microvm/hook-server.ts
|
|
279
279
|
var import_lambda_microvm_runtime = __toESM(require_dist());
|
|
280
|
-
|
|
281
|
-
// src/microvm/constants.ts
|
|
282
|
-
var HOOKS_PORT = 8080;
|
|
283
280
|
var HOOKS_DIR = "/etc/evident/hooks";
|
|
284
|
-
|
|
285
|
-
// src/microvm/image/hook-server.ts
|
|
281
|
+
var HOOKS_PORT = 8080;
|
|
286
282
|
var server = (0, import_lambda_microvm_runtime.createNodeRuntime)({ hooksDir: HOOKS_DIR });
|
|
287
283
|
var port = Number(process.env.HOOKS_PORT ?? HOOKS_PORT);
|
|
288
284
|
server.listen(port, "0.0.0.0", () => {
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
# Sourced by every hook script. Nothing here runs at image build time.
|
|
3
3
|
#
|
|
4
4
|
# These hooks are the SECOND shell speaking the runner-synchroniser CLI contract
|
|
5
|
-
# (
|
|
6
|
-
#
|
|
5
|
+
# (runner/docker-images/fargate/entrypoint.sh is the first), so both are held to it by
|
|
6
|
+
# runner/synchroniser/src/shell-contract.test.ts — which derives what
|
|
7
7
|
# `run_synchroniser` below must handle from the subcommands this shell calls.
|
|
8
8
|
|
|
9
9
|
# Installed from npm by the image (docker/Dockerfile's ARG
|
|
@@ -50,14 +50,47 @@ LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
|
|
|
50
50
|
# earlier boot cannot silently disable replication for the VM's whole life.
|
|
51
51
|
SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
53
|
+
hook_name() { printf '%s' "${0##*/}"; }
|
|
54
|
+
log() { echo "[hook:$(hook_name)] $*"; }
|
|
55
|
+
warn() { echo "[hook:$(hook_name)] $*" >&2; }
|
|
56
|
+
error() { echo "[hook:$(hook_name)] ERROR: $*" >&2; }
|
|
57
|
+
|
|
58
|
+
session_db_recovery_report_path() {
|
|
59
|
+
local path="${EVIDENT_SESSION_DB_RECOVERY_REPORT:-}"
|
|
60
|
+
path="${path#"${path%%[![:space:]]*}"}"
|
|
61
|
+
path="${path%"${path##*[![:space:]]}"}"
|
|
62
|
+
if [ -n "${path}" ]; then
|
|
63
|
+
printf '%s\n' "${path}"
|
|
64
|
+
return 0
|
|
65
|
+
fi
|
|
66
|
+
local home="${HOME:-}"
|
|
67
|
+
home="${home#"${home%%[![:space:]]*}"}"
|
|
68
|
+
home="${home%"${home##*[![:space:]]}"}"
|
|
69
|
+
[ -n "${home}" ] || return 1
|
|
70
|
+
printf '%s\n' "${home}/.local/state/evident/session-db-recovery.jsonl"
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
report_session_db_giveup() {
|
|
74
|
+
local reason="$1" exit_code="${2:-null}" outcome path at
|
|
75
|
+
case "${reason}" in
|
|
76
|
+
synchroniser_config_unresolved|synchroniser_config_unevaluable|synchroniser_config_incomplete|litestream_config_unavailable|classification_fatal) outcome="restore_misconfigured" ;;
|
|
77
|
+
restore_deadline_exceeded|restore_tool_unusable|classification_unrecognised) outcome="fresh_session_db" ;;
|
|
78
|
+
*) warn "SESSION-DB-RECOVERY-REPORT-UNKNOWN-REASON: ${reason}"; return 0 ;;
|
|
79
|
+
esac
|
|
80
|
+
if ! path="$(session_db_recovery_report_path)"; then
|
|
81
|
+
warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
|
|
82
|
+
return 0
|
|
83
|
+
fi
|
|
84
|
+
at="${EPOCHREALTIME:-}"
|
|
85
|
+
[ -n "${at}" ] || warn "SESSION-DB-RECOVERY-REPORT-TIMESTAMP-UNAVAILABLE"
|
|
86
|
+
printf '{"v":1,"event":"session_db_recovery","at":"%s","stage":"restore","outcome":"%s","severity":"error","reason":"%s","litestream_exit_code":%s,"attempt":null,"replica_objects":null,"replica_bytes":null,"quarantine_destination":null,"quarantined_objects":null,"quarantine_failed_objects":null,"quarantined_bytes":null,"verified_restore_point":null,"restore_points_tried":null,"replication_suspended":true}\n' "${at}" "${outcome}" "${reason}" "${exit_code}" >>"${path}" || warn "SESSION-DB-RECOVERY-REPORT-APPEND-FAILED"
|
|
87
|
+
return 0
|
|
88
|
+
}
|
|
56
89
|
|
|
57
90
|
# Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
|
|
58
91
|
# corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
|
|
59
92
|
# `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
|
|
60
|
-
# unusable)/32 (retry) — see `
|
|
93
|
+
# unusable)/32 (retry) — see `runner/synchroniser/src/cli.ts`'s own
|
|
61
94
|
# comment for what each means, not restated here. Any OTHER non-zero status
|
|
62
95
|
# means the tool itself broke, which is the only case worth an ERROR here —
|
|
63
96
|
# EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
|
|
@@ -171,7 +204,7 @@ timed_synchroniser() {
|
|
|
171
204
|
}
|
|
172
205
|
|
|
173
206
|
# Exports what `runner-synchroniser` resolves its object-store location from
|
|
174
|
-
# (
|
|
207
|
+
# (runner/synchroniser/src/config.ts). It treats either being empty as
|
|
175
208
|
# "persistence disabled" and then reports every restore as a WARNING it still
|
|
176
209
|
# exits 0 for — so an unset value has to be caught HERE, where it can still be
|
|
177
210
|
# told apart from "the store is simply empty".
|
|
@@ -211,13 +244,13 @@ load_state_config() {
|
|
|
211
244
|
# `model-auth-ready` therefore stays a boot-time diagnostic: its only output is
|
|
212
245
|
# a log line, since the runtime forwards no hook stderr to the doorbell caller.
|
|
213
246
|
|
|
214
|
-
# How long restore_credentials' shared step budget is, covering
|
|
215
|
-
# its synchroniser calls (restore claude, restore opencode,
|
|
216
|
-
# as one window rather than a timeout apiece — a fixed per-call cap at the same
|
|
247
|
+
# How long restore_credentials' shared step budget is, covering the runner-secret
|
|
248
|
+
# fetch plus its three synchroniser calls (restore claude, restore opencode,
|
|
249
|
+
# model-auth-ready) as one window rather than a timeout apiece — a fixed per-call cap at the same
|
|
217
250
|
# total would false-fire on the ordinary case of one slow call (a slow S3 GET),
|
|
218
251
|
# which is precisely the boot this exists to keep healthy. From #930's 3-boot
|
|
219
|
-
# sample: the step measured ~3-5s total (~1.2s per call, three
|
|
220
|
-
# starts of a 1.8 MB bundle) — a WEAK estimate this file's own
|
|
252
|
+
# sample: the step measured ~3-5s total (~1.2s per synchroniser call, three
|
|
253
|
+
# node cold starts of a 1.8 MB bundle) — a WEAK estimate this file's own
|
|
221
254
|
# SYNCHRONISER-TIMING lines are what will sharpen for real. Too tight and a
|
|
222
255
|
# routine slow call boots this VM with no model credentials until an operator
|
|
223
256
|
# reconnects it; too loose and a hung call burns more of the hook's own
|
|
@@ -229,6 +262,7 @@ CREDENTIAL_RESTORE_DEADLINE_SECONDS="${EVIDENT_CREDENTIAL_RESTORE_DEADLINE_SECON
|
|
|
229
262
|
# and a call that ignored it would be unbounded again). Hard ceiling on the
|
|
230
263
|
# step: DEADLINE + this = 10s, once — not per call.
|
|
231
264
|
CREDENTIAL_RESTORE_KILL_GRACE_SECONDS=2
|
|
265
|
+
GITHUB_PROBE_DEADLINE_SECONDS="${EVIDENT_GITHUB_PROBE_DEADLINE_SECONDS:-10}"
|
|
232
266
|
|
|
233
267
|
# What is left of the shared step budget, in whole seconds, `step_started_s`
|
|
234
268
|
# seconds after it began. `SECONDS` (a bash builtin with no failure mode,
|
|
@@ -268,12 +302,66 @@ bounded_restore_call() {
|
|
|
268
302
|
return 0
|
|
269
303
|
}
|
|
270
304
|
|
|
305
|
+
fetch_runner_secret() {
|
|
306
|
+
local step_started_s="$1" remaining rc=0 payload value stderr_file started_ms populated=0 missing=0
|
|
307
|
+
if [ -z "${RUNNER_SECRET_ARN:-}" ]; then
|
|
308
|
+
log "runner secret is not configured; continuing without GitHub and MCP credentials"
|
|
309
|
+
return 0
|
|
310
|
+
fi
|
|
311
|
+
remaining="$(remaining_credential_budget "${step_started_s}")"
|
|
312
|
+
if [ "${remaining}" -lt 1 ]; then
|
|
313
|
+
warn "CREDENTIAL-RESTORE-SKIPPED: runner-secret skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
|
|
314
|
+
return 0
|
|
315
|
+
fi
|
|
316
|
+
if ! stderr_file="$(mktemp /dev/shm/runner-secret-stderr.XXXXXX)"; then
|
|
317
|
+
warn "RUNNER-SECRET-STDERR-UNAVAILABLE: could not allocate diagnostic storage; continuing without runner credentials"
|
|
318
|
+
return 0
|
|
319
|
+
fi
|
|
320
|
+
started_ms="$(now_ms)"
|
|
321
|
+
payload="$(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${remaining}" aws secretsmanager get-secret-value --secret-id "${RUNNER_SECRET_ARN}" --query SecretString --output text 2>"${stderr_file}")" || rc=$?
|
|
322
|
+
log_elapsed_since runner-secret-fetch "${started_ms}" "${rc}"
|
|
323
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then
|
|
324
|
+
warn "CREDENTIAL-RESTORE-TIMEOUT: runner-secret did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
|
|
325
|
+
rm -f "${stderr_file}"
|
|
326
|
+
return 0
|
|
327
|
+
fi
|
|
328
|
+
if [ "${rc}" -ne 0 ]; then
|
|
329
|
+
warn "RUNNER-SECRET-UNREADABLE: $(<"${stderr_file}")"
|
|
330
|
+
rm -f "${stderr_file}"
|
|
331
|
+
return 0
|
|
332
|
+
fi
|
|
333
|
+
rm -f "${stderr_file}"
|
|
334
|
+
if ! jq -e 'type == "object"' >/dev/null 2>&1 <<<"${payload}"; then
|
|
335
|
+
warn "RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object"
|
|
336
|
+
return 0
|
|
337
|
+
fi
|
|
338
|
+
local -a populated_keys=()
|
|
339
|
+
for key in GH_TOKEN BRAVE_API_KEY CLOUDFLARE_API_TOKEN NEON_API_KEY; do
|
|
340
|
+
value="$(jq -r --arg k "${key}" '.[$k] // empty' <<<"${payload}")"
|
|
341
|
+
if [ -n "${value}" ]; then
|
|
342
|
+
export "${key}=${value}"
|
|
343
|
+
populated=$((populated + 1))
|
|
344
|
+
populated_keys+=("${key}")
|
|
345
|
+
else
|
|
346
|
+
missing=$((missing + 1))
|
|
347
|
+
fi
|
|
348
|
+
done
|
|
349
|
+
if [ "${populated}" -eq 0 ]; then
|
|
350
|
+
warn "RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-microvm/README.md"
|
|
351
|
+
else
|
|
352
|
+
log "RUNNER-SECRET-OK: populated ${populated_keys[*]}; ${missing} allow-listed keys empty or absent"
|
|
353
|
+
fi
|
|
354
|
+
return 0
|
|
355
|
+
}
|
|
356
|
+
|
|
271
357
|
restore_credentials() {
|
|
272
358
|
local step_started_ms step_started_s
|
|
273
359
|
step_started_ms="$(now_ms)"
|
|
274
360
|
step_started_s="${SECONDS}"
|
|
275
361
|
|
|
276
362
|
load_state_config || return 1
|
|
363
|
+
# This shares the existing bounded window so /run's worst-case duration does not grow.
|
|
364
|
+
fetch_runner_secret "${step_started_s}"
|
|
277
365
|
bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
|
|
278
366
|
bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
|
|
279
367
|
|
|
@@ -313,7 +401,7 @@ infrastructure/evident-microvm/README.md."
|
|
|
313
401
|
fi
|
|
314
402
|
# The step total, in the SAME greppable shape as the per-call lines above, so
|
|
315
403
|
# one query answers both "what does credential restore cost?" and "which of
|
|
316
|
-
# its
|
|
404
|
+
# its four operations cost it". Bounded now by CREDENTIAL_RESTORE_DEADLINE_SECONDS
|
|
317
405
|
# + CREDENTIAL_RESTORE_KILL_GRACE_SECONDS as one shared window (see the
|
|
318
406
|
# comment above that constant) rather than per call. `rc=0` is deliberate
|
|
319
407
|
# here, not an absence of failure modes: a timed-out call already warned by
|
|
@@ -323,6 +411,55 @@ infrastructure/evident-microvm/README.md."
|
|
|
323
411
|
return 0
|
|
324
412
|
}
|
|
325
413
|
|
|
414
|
+
apply_runner_opencode_config() {
|
|
415
|
+
if [ -z "${RUNNER_OPENCODE_CONFIG:-}" ]; then
|
|
416
|
+
log "runner OpenCode config is not configured; using the baked project config"
|
|
417
|
+
return 0
|
|
418
|
+
fi
|
|
419
|
+
local source="${RUNNER_OPENCODE_CONFIG}" target="opencode.json"
|
|
420
|
+
[[ "${source}" = /* ]] || source="${WORKSPACE}/${source}"
|
|
421
|
+
[ -f "${WORKSPACE}/opencode.jsonc" ] && target="opencode.jsonc"
|
|
422
|
+
if [ ! -f "${source}" ]; then
|
|
423
|
+
error "RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)"
|
|
424
|
+
return 0
|
|
425
|
+
fi
|
|
426
|
+
cp "${source}" "${WORKSPACE}/${target}"
|
|
427
|
+
git -C "${WORKSPACE}" update-index --skip-worktree "${target}" 2>/dev/null \
|
|
428
|
+
|| warn "could not mark ${target} skip-worktree; it may show as a local change"
|
|
429
|
+
log "Applied runner OpenCode config ${source} to ${WORKSPACE}/${target}"
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
configure_github_access() {
|
|
433
|
+
if [ -z "${GH_TOKEN:-}" ]; then
|
|
434
|
+
warn "GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above"
|
|
435
|
+
return 0
|
|
436
|
+
fi
|
|
437
|
+
export GIT_CONFIG_GLOBAL=/tmp/gitconfig
|
|
438
|
+
if ! : >"${GIT_CONFIG_GLOBAL}" ||
|
|
439
|
+
! git config --global user.name "${GIT_USER_NAME:-evident-bot}" ||
|
|
440
|
+
! git config --global user.email "${GIT_USER_EMAIL:-evident-bot@users.noreply.github.com}" ||
|
|
441
|
+
! git config --global init.defaultBranch main ||
|
|
442
|
+
! printf '%s\n' '#!/usr/bin/env bash' '[ "$1" = get ] || exit 0' 'echo username=x-access-token' 'echo "password=${GH_TOKEN}"' >/tmp/git-credential-helper.sh ||
|
|
443
|
+
! chmod 0700 /tmp/git-credential-helper.sh ||
|
|
444
|
+
! git config --global credential."https://github.com".helper /tmp/git-credential-helper.sh; then
|
|
445
|
+
warn "GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access"
|
|
446
|
+
return 0
|
|
447
|
+
fi
|
|
448
|
+
(
|
|
449
|
+
local output rc=0 login repo_url repo
|
|
450
|
+
output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api user --jq .login 2>&1)" || rc=$?
|
|
451
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
|
|
452
|
+
if [ "${rc}" -ne 0 ]; then warn "GITHUB-AUTH-REJECTED: ${output}"; return; fi
|
|
453
|
+
log "GITHUB-AUTH-OK: ${output}"
|
|
454
|
+
repo_url="$(git -C "${WORKSPACE}" remote get-url origin 2>/dev/null || true)"
|
|
455
|
+
repo="$(printf '%s' "${repo_url}" | sed -E 's#(https://github.com/|git@github.com:)##; s#\.git$##')"
|
|
456
|
+
[ -n "${repo}" ] || return
|
|
457
|
+
output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api "repos/${repo}" --jq .full_name 2>&1)" || rc=$?
|
|
458
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
|
|
459
|
+
[ "${rc}" -eq 0 ] || warn "GITHUB-REPO-INACCESSIBLE: ${repo}: ${output}"
|
|
460
|
+
) &
|
|
461
|
+
}
|
|
462
|
+
|
|
326
463
|
# Best-effort by design: /suspend must still drop the tunnel and /terminate must
|
|
327
464
|
# still clean up, so a flush that cannot happen is loud but never fatal.
|
|
328
465
|
sync_credentials() {
|
|
@@ -337,7 +474,7 @@ sync_credentials() {
|
|
|
337
474
|
# Writes NOTHING to S3 itself; replication back to S3 is started separately by
|
|
338
475
|
# `start_litestream` (below) from the `run`/`resume` hooks. `ensure_litestream_config`/
|
|
339
476
|
# `restore_session_db` are the MicroVM side of the same contract
|
|
340
|
-
#
|
|
477
|
+
# runner/docker-images/fargate/entrypoint.sh's inlined restore already speaks, going
|
|
341
478
|
# through the SAME runner-synchroniser CLI.
|
|
342
479
|
|
|
343
480
|
# Reads the ~30 MB litestream binary into the page cache, in the background, so
|
|
@@ -491,6 +628,24 @@ restore_session_db() {
|
|
|
491
628
|
# suspend/resume and a /run retry, so a marker left by an earlier boot must
|
|
492
629
|
# never silently disable replication for the rest of this VM's life.
|
|
493
630
|
rm -f "${SESSION_DB_NO_REPLICATE_MARKER}"
|
|
631
|
+
local report_path
|
|
632
|
+
if report_path="$(session_db_recovery_report_path)"; then
|
|
633
|
+
# The default report directory is provisioned in the image, so this is a
|
|
634
|
+
# no-op there; a shared-path override (EVIDENT_SESSION_DB_RECOVERY_REPORT)
|
|
635
|
+
# can name a directory that isn't, and the typed synchroniser writer
|
|
636
|
+
# already creates it for that same override (session-db-recovery-report.ts).
|
|
637
|
+
# Best-effort only: report_session_db_giveup below still warns and
|
|
638
|
+
# continues if this didn't leave a writable path. `${path%/*}` leaves a
|
|
639
|
+
# bare relative filename (no `/`) unchanged rather than reducing to `.`
|
|
640
|
+
# like dirname(1) — without the explicit check, mkdir would create a
|
|
641
|
+
# directory AT the report path itself.
|
|
642
|
+
local report_dir="${report_path%/*}"
|
|
643
|
+
[ "${report_dir}" = "${report_path}" ] && report_dir="."
|
|
644
|
+
[ -d "${report_dir}" ] || mkdir -p "${report_dir}" 2>/dev/null || true
|
|
645
|
+
: >"${report_path}" || warn "SESSION-DB-RECOVERY-REPORT-TRUNCATE-FAILED"
|
|
646
|
+
else
|
|
647
|
+
warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
|
|
648
|
+
fi
|
|
494
649
|
|
|
495
650
|
# Config resolved ONCE here, in the synchroniser (config.ts owns
|
|
496
651
|
# OPENCODE_DB_PATH; no second copy of that path here), exactly as
|
|
@@ -507,6 +662,7 @@ restore_session_db() {
|
|
|
507
662
|
log_elapsed_since session-db-env "${env_started_ms}" "${env_rc}"
|
|
508
663
|
if [ "${env_rc}" -ne 0 ]; then
|
|
509
664
|
mark_no_replicate "could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
665
|
+
report_session_db_giveup synchroniser_config_unresolved
|
|
510
666
|
return 0
|
|
511
667
|
fi
|
|
512
668
|
# Guarded for the SAME reason as the substitution above, which is easy to miss:
|
|
@@ -517,10 +673,11 @@ restore_session_db() {
|
|
|
517
673
|
# it is the whole lifecycle transition, so Q3 makes it a give-up instead.
|
|
518
674
|
eval "${synchroniser_env}" || {
|
|
519
675
|
mark_no_replicate "the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
676
|
+
report_session_db_giveup synchroniser_config_unevaluable
|
|
520
677
|
return 0
|
|
521
678
|
}
|
|
522
679
|
|
|
523
|
-
# #931's exact lesson, one shell over (
|
|
680
|
+
# #931's exact lesson, one shell over (runner/docker-images/fargate/entrypoint.sh):
|
|
524
681
|
# the `|| { ... }` above only catches a non-zero EXIT — an `env` that exits
|
|
525
682
|
# 0 with an INCOMPLETE contract (a runner-synchroniser version/build skew)
|
|
526
683
|
# would otherwise abort right here under `set -u` the moment
|
|
@@ -531,6 +688,7 @@ restore_session_db() {
|
|
|
531
688
|
# be actively wrong, not merely absent).
|
|
532
689
|
if [ -z "${OPENCODE_DB_PATH+x}" ]; then
|
|
533
690
|
mark_no_replicate "run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
691
|
+
report_session_db_giveup synchroniser_config_incomplete
|
|
534
692
|
return 0
|
|
535
693
|
fi
|
|
536
694
|
|
|
@@ -548,6 +706,7 @@ restore_session_db() {
|
|
|
548
706
|
|
|
549
707
|
if ! ensure_litestream_config; then
|
|
550
708
|
mark_no_replicate "could not generate ${LITESTREAM_CONFIG_FILE} (see the error above)"
|
|
709
|
+
report_session_db_giveup litestream_config_unavailable
|
|
551
710
|
return 0
|
|
552
711
|
fi
|
|
553
712
|
|
|
@@ -566,18 +725,20 @@ restore_session_db() {
|
|
|
566
725
|
124 | 137)
|
|
567
726
|
discard_session_db_debris
|
|
568
727
|
mark_no_replicate "SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within the ${SESSION_DB_RESTORE_DEADLINE_SECONDS}s deadline (+${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}s kill grace), ${SECONDS}s into the hook; opencode starts with a fresh session DB and nothing is replicated this boot"
|
|
728
|
+
report_session_db_giveup restore_deadline_exceeded "${restore_rc}"
|
|
569
729
|
return 0
|
|
570
730
|
;;
|
|
571
731
|
125 | 126 | 127)
|
|
572
732
|
error "litestream restore could not even run (timeout exited ${restore_rc})"
|
|
573
733
|
discard_session_db_debris
|
|
574
734
|
mark_no_replicate "restore tool is broken (timeout exited ${restore_rc}); opencode starts with a fresh session DB and nothing is replicated this boot"
|
|
735
|
+
report_session_db_giveup restore_tool_unusable "${restore_rc}"
|
|
575
736
|
return 0
|
|
576
737
|
;;
|
|
577
738
|
esac
|
|
578
739
|
|
|
579
740
|
local classify_rc=0
|
|
580
|
-
run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave || classify_rc=$?
|
|
741
|
+
run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave --fresh-db-fallback || classify_rc=$?
|
|
581
742
|
case "${classify_rc}" in
|
|
582
743
|
0) ;; # restored, or no replica yet — the CLI already logged which
|
|
583
744
|
31)
|
|
@@ -587,8 +748,8 @@ restore_session_db() {
|
|
|
587
748
|
# half-restored DB, so the marker stays; a 31 is structurally guaranteed
|
|
588
749
|
# to be a FRESH one, and replicating it starts a new backup chain instead
|
|
589
750
|
# of leaving this boot with a zero-width backup window. Same reasoning,
|
|
590
|
-
# and same one-line change, as
|
|
591
|
-
warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and
|
|
751
|
+
# and same one-line change, as runner/docker-images/fargate/entrypoint.sh's `31)`.
|
|
752
|
+
warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot (see the WARNING above)"
|
|
592
753
|
;;
|
|
593
754
|
32)
|
|
594
755
|
# The CLI's own contract for 32 is "re-run litestream restore and ask
|
|
@@ -602,11 +763,13 @@ restore_session_db() {
|
|
|
602
763
|
# fresh rather than fail /run.
|
|
603
764
|
discard_session_db_debris
|
|
604
765
|
mark_no_replicate "session-db-classify returned fatal (30); see the FATAL message above"
|
|
766
|
+
report_session_db_giveup classification_fatal
|
|
605
767
|
;;
|
|
606
768
|
*)
|
|
607
769
|
# run_synchroniser already logged the "tool broke" ERROR for this.
|
|
608
770
|
discard_session_db_debris
|
|
609
771
|
mark_no_replicate "session-db-classify exited ${classify_rc}, which is none of its documented answers"
|
|
772
|
+
report_session_db_giveup classification_unrecognised
|
|
610
773
|
;;
|
|
611
774
|
esac
|
|
612
775
|
|
|
@@ -1018,7 +1181,7 @@ flush_session_db() {
|
|
|
1018
1181
|
# returned its 200, so it is deliberately outside that arithmetic.
|
|
1019
1182
|
#
|
|
1020
1183
|
# EVIDENT_IDLE_TIMEOUT_SECONDS, deliberately the SAME env var name
|
|
1021
|
-
#
|
|
1184
|
+
# runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
|
|
1022
1185
|
# idle-timeout flag, so an operator who knows one knows the other. A
|
|
1023
1186
|
# non-numeric override must never silently DROP the flag — that degrades to
|
|
1024
1187
|
# an always-on VM burning ~$2.40/day, exactly the bug this closes — so it
|
|
@@ -1057,7 +1220,7 @@ start_tunnel() {
|
|
|
1057
1220
|
# into a MicroVM, so it is the ONLY way to seed model credentials on a runner
|
|
1058
1221
|
# that is already up — the S3 credential store is read at /run and /resume,
|
|
1059
1222
|
# never mid-life. The allow-list stays a single directory, as on ECS
|
|
1060
|
-
# (
|
|
1223
|
+
# (runner/docker-images/fargate/entrypoint.sh): the flag is repeatable, but every
|
|
1061
1224
|
# extra entry widens what Evident can write into a VM that executes agent
|
|
1062
1225
|
# code. ${HOME} is set by the image (ENV HOME=/home/runner) and `set -u` makes
|
|
1063
1226
|
# an unset one abort rather than silently allow-list "/.claude".
|
|
@@ -1104,7 +1267,7 @@ CLI_SHUTDOWN_CEILING_SECONDS=32
|
|
|
1104
1267
|
# everything else in the hook. 10 s is ~4x the measured shutdown and gives a
|
|
1105
1268
|
# provable worst case of 10 + the 10 s `suspend` spends in sync_credentials
|
|
1106
1269
|
# before us = 20 s, inside the ~55 s the in-VM server allows this script
|
|
1107
|
-
# (DEFAULT_TIMEOUT_SECONDS,
|
|
1270
|
+
# (DEFAULT_TIMEOUT_SECONDS, aws/lambda-microvm-runtime/src/runtime.ts),
|
|
1108
1271
|
# itself inside AWS's 60 s (HOOK_TIMEOUT_SECONDS, src/constants.ts) —
|
|
1109
1272
|
# overrunning that fails the whole lifecycle transition, which is worse than the
|
|
1110
1273
|
# kill this replaces. The residual cost is a drain longer than 10 s being
|
|
@@ -61,7 +61,7 @@ start_litestream
|
|
|
61
61
|
# measures in ~2 s (see the waker note in common.sh).
|
|
62
62
|
#
|
|
63
63
|
# Safe to leave running past the hook's own exit: the runtime waits on the
|
|
64
|
-
# script's `'exit'`, not `'close'` (
|
|
64
|
+
# script's `'exit'`, not `'close'` (aws/lambda-microvm-runtime/src/
|
|
65
65
|
# runtime.ts), so a lingering child never delays the HTTP response. Its stdio
|
|
66
66
|
# is deliberately NOT redirected — that inherited fd is how the RUNNER-KEY-*
|
|
67
67
|
# line reaches CloudWatch, which is the whole point of running it at all.
|
|
@@ -79,7 +79,13 @@ check_runner_key "${runner_key}" "${api_url}" || exit 1
|
|
|
79
79
|
restore_session_db
|
|
80
80
|
log "session DB restore done ${SECONDS}s into the hook"
|
|
81
81
|
|
|
82
|
-
# 7 —
|
|
82
|
+
# 7 — apply the overlay before OpenCode resolves its project configuration.
|
|
83
|
+
apply_runner_opencode_config
|
|
84
|
+
|
|
85
|
+
# 8 — configure git after credentials are restored and before agent shells start.
|
|
86
|
+
configure_github_access
|
|
87
|
+
|
|
88
|
+
# 9 — opencode. Started here, not at build time: a warm process in the shared
|
|
83
89
|
# snapshot would carry its installation id and database into every VM. Not
|
|
84
90
|
# waited on: a slow opencode boot is not a reason to fail /run (the tunnel CLI
|
|
85
91
|
# auto-starts opencode when it finds none healthy,
|
|
@@ -87,14 +93,14 @@ log "session DB restore done ${SECONDS}s into the hook"
|
|
|
87
93
|
# reclaims a runner that never comes online).
|
|
88
94
|
start_opencode
|
|
89
95
|
|
|
90
|
-
#
|
|
96
|
+
# 10 — begin replicating the session DB (#812 WI-3), now that opencode has
|
|
91
97
|
# opened it and before any work can arrive over the tunnel. Bare, like
|
|
92
98
|
# restore_session_db above: start_litestream never returns non-zero (every
|
|
93
99
|
# guard inside it is its own `return 0`), so there is nothing here for
|
|
94
100
|
# `set -e` to abort on.
|
|
95
101
|
start_litestream
|
|
96
102
|
|
|
97
|
-
#
|
|
103
|
+
# 11 — the first per-VM identity on the wire. The subshell's umask makes the file
|
|
98
104
|
# unreadable to anyone else from the moment it exists, before the key is in it.
|
|
99
105
|
(
|
|
100
106
|
umask 077
|
package/dist/waker/construct.js
CHANGED
|
@@ -56,7 +56,7 @@ class EvidentWaker extends constructs_1.Construct {
|
|
|
56
56
|
runtime: lambda.Runtime.NODEJS_22_X,
|
|
57
57
|
// Pre-bundled at PACKAGE build time (`pnpm --filter @evident-ai/runner-cdk
|
|
58
58
|
// build`, see scripts/build.ts), not at synth time: esbuild inlines
|
|
59
|
-
// @evident/
|
|
59
|
+
// @evident/sdk's HMAC verifier into
|
|
60
60
|
// dist/waker-lambda/handler.js, so neither this package's published npm
|
|
61
61
|
// artifact (which ships dist/ already built) nor a consuming app needs
|
|
62
62
|
// that private workspace package, or esbuild, on synth's PATH. Requires
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@evident-ai/runner-cdk",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.4.1-dev.1856549",
|
|
4
4
|
"description": "Reusable CDK constructs for an Evident agent runner: a single scale-to-zero Fargate runner (task + service + per-agent self-stop role + waker Lambda), or a per-session AWS Lambda MicroVM that boots on demand and suspends between messages. Instantiate once per agent from your own stack.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -15,13 +15,14 @@
|
|
|
15
15
|
],
|
|
16
16
|
"scripts": {
|
|
17
17
|
"build": "pnpm run build-bundled-deps && ts-node scripts/build.ts",
|
|
18
|
-
"//build-bundled-deps": "scripts/build.ts esbuild-bundles
|
|
19
|
-
"build-bundled-deps": "pnpm --filter @evident-ai/lambda-microvm-runtime build",
|
|
18
|
+
"//build-bundled-deps": "scripts/build.ts esbuild-bundles runner/docker-images/microvm/hook-server.ts, which imports @evident-ai/lambda-microvm-runtime by its built `main`. @evident-ai/lambda-microvm-cdk is a workspace:* devDependency whose types resolve through its own dist/, so tsc --project tsconfig.build.json cannot compile src/microvm/{shapes,construct}.ts without it. Chained into `build` rather than left to turbo's `^build` because publish-runner-cdk.yaml and infrastructure/evident-runner's build-runner-cdk-dep invoke `pnpm --filter @evident-ai/runner-cdk build` directly, which bypasses turbo entirely.",
|
|
19
|
+
"build-bundled-deps": "pnpm --filter @evident-ai/lambda-microvm-cdk build && pnpm --filter @evident-ai/lambda-microvm-runtime build",
|
|
20
20
|
"typecheck": "tsc --noEmit",
|
|
21
21
|
"test": "node --test --require ts-node/register 'src/**/*.test.ts'",
|
|
22
22
|
"format": "prettier --write 'src/**/*.ts'",
|
|
23
23
|
"lint": "eslint 'src/**/*.ts' --max-warnings=0"
|
|
24
24
|
},
|
|
25
|
+
"//peerDependencies": "@evident-ai/lambda-microvm-cdk is a registry range here, never workspace:*: publish-runner-cdk.yaml ships this package with `npm publish`, not `pnpm publish`, so pnpm's workspace-protocol substitution never runs and a workspace: range would reach npm literally, breaking every external install. Enforced by src/published-surface.test.ts. The matching devDependency is workspace:* so this monorepo's own build compiles against aws/lambda-microvm-cdk's source.",
|
|
25
26
|
"peerDependencies": {
|
|
26
27
|
"@evident-ai/lambda-microvm-cdk": "^0.1.1",
|
|
27
28
|
"aws-cdk-lib": "^2.240.0",
|
|
@@ -31,16 +32,17 @@
|
|
|
31
32
|
"@aws-sdk/client-ecs": "^3.682.0",
|
|
32
33
|
"@aws-sdk/client-lambda-microvms": "^3.1095.0",
|
|
33
34
|
"@aws-sdk/client-secrets-manager": "^3.682.0",
|
|
34
|
-
"@evident-ai/lambda-microvm-cdk": "
|
|
35
|
+
"@evident-ai/lambda-microvm-cdk": "workspace:*",
|
|
35
36
|
"@evident-ai/lambda-microvm-runtime": "workspace:*",
|
|
36
|
-
"@evident/
|
|
37
|
+
"@evident/sdk": "workspace:*",
|
|
37
38
|
"@types/node": "^22",
|
|
38
39
|
"aws-cdk-lib": "^2.240.0",
|
|
39
40
|
"constructs": "^10.5.0",
|
|
40
41
|
"esbuild": "^0.25.2",
|
|
41
42
|
"prettier": "^3.3.3",
|
|
42
43
|
"ts-node": "^10.9.2",
|
|
43
|
-
"typescript": "^5.6.3"
|
|
44
|
+
"typescript": "^5.6.3",
|
|
45
|
+
"runner-microvm-image": "workspace:*"
|
|
44
46
|
},
|
|
45
47
|
"publishConfig": {
|
|
46
48
|
"access": "public"
|
|
@@ -48,9 +50,9 @@
|
|
|
48
50
|
"repository": {
|
|
49
51
|
"type": "git",
|
|
50
52
|
"url": "https://github.com/sroze/evident.git",
|
|
51
|
-
"directory": "
|
|
53
|
+
"directory": "aws/runner-cdk"
|
|
52
54
|
},
|
|
53
55
|
"homepage": "https://evident.run",
|
|
54
|
-
"author": "Evident <
|
|
56
|
+
"author": "Evident <hello@evident.run>",
|
|
55
57
|
"license": "MIT"
|
|
56
58
|
}
|