@evident-ai/runner-cdk 0.1.1-dev.736307e → 0.1.1-dev.77d5cb6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -13
- package/dist/evident-scale-to-zero-construct.js +4 -5
- package/dist/microvm/constants.js +2 -3
- package/dist/microvm/construct.d.ts +16 -2
- package/dist/microvm/construct.js +8 -0
- package/dist/microvm/controller/handle-doorbell.js +2 -2
- package/dist/microvm/controller/microvm-client.d.ts +1 -1
- package/dist/microvm-image-context/Dockerfile +8 -5
- package/dist/microvm-image-context/hook-server.js +2 -6
- package/dist/microvm-image-context/hooks/common.sh +122 -18
- package/dist/microvm-image-context/hooks/resume +1 -1
- package/dist/microvm-image-context/hooks/run +9 -3
- package/dist/waker/construct.js +1 -1
- package/package.json +9 -7
package/README.md
CHANGED
|
@@ -9,9 +9,9 @@ Reusable AWS CDK constructs for running an [Evident](https://evident.run) agent
|
|
|
9
9
|
`desiredCount 0`, and an HMAC-authenticated waker Lambda scales it back up on the next
|
|
10
10
|
message. You pay for the time the agent is actually working.
|
|
11
11
|
- **`EvidentMicrovmConstruct`** — a per-session AWS Lambda MicroVM that boots on demand
|
|
12
|
-
and suspends between messages. Installing this package is all you
|
|
13
|
-
context ships inside it (see [The MicroVM image](#the-microvm-image)
|
|
14
|
-
no checkout of `sroze/evident` involved. See
|
|
12
|
+
and suspends between messages. Installing the `@dev` tag of this package is all you
|
|
13
|
+
need: the image build context ships inside it (see [The MicroVM image](#the-microvm-image)
|
|
14
|
+
below), so there is no checkout of `sroze/evident` involved. See
|
|
15
15
|
[the AWS runner doc](https://evident.run/docs/aws-runner) for the full picture of both
|
|
16
16
|
strategies.
|
|
17
17
|
|
|
@@ -34,8 +34,9 @@ cluster can host several agents.
|
|
|
34
34
|
## Getting started
|
|
35
35
|
|
|
36
36
|
For a full end-to-end walkthrough — shared infra, secrets without SOPS/KMS, the image,
|
|
37
|
-
the two grants, deploy, and wiring the wake webhook — see
|
|
38
|
-
|
|
37
|
+
the two grants, deploy, and wiring the wake webhook — see `GETTING-STARTED.md` in this
|
|
38
|
+
package's directory (not published to npm; read it from a checkout of the repo). The
|
|
39
|
+
snippet below is the short version.
|
|
39
40
|
|
|
40
41
|
```ts
|
|
41
42
|
import { EvidentScaleToZeroConstruct } from "@evident-ai/runner-cdk";
|
|
@@ -111,7 +112,7 @@ the construct's own dogfooding consumer.
|
|
|
111
112
|
|
|
112
113
|
This construct never builds an image. Pass any `ecs.ContainerImage` — from a registry, or
|
|
113
114
|
built from a `Dockerfile` you control. The generic runner image lives separately in
|
|
114
|
-
`
|
|
115
|
+
`runner/docker-images/fargate`, so you can adopt the image, the construct, or both.
|
|
115
116
|
|
|
116
117
|
## The MicroVM image
|
|
117
118
|
|
|
@@ -126,6 +127,7 @@ the per-phase hook scripts and the bundled hook server, published inside the tar
|
|
|
126
127
|
the agent's workspace. `stageMicrovmImageContext()` puts the two together:
|
|
127
128
|
|
|
128
129
|
```ts
|
|
130
|
+
import path from 'node:path';
|
|
129
131
|
import {
|
|
130
132
|
EvidentMicrovmConstruct,
|
|
131
133
|
HOOKS_PORT,
|
|
@@ -149,6 +151,8 @@ new EvidentMicrovmConstruct(this, 'Runner', {
|
|
|
149
151
|
baseImageArn,
|
|
150
152
|
baseImageVersion,
|
|
151
153
|
doorbellSecret,
|
|
154
|
+
runnerSecret,
|
|
155
|
+
runnerOpencodeConfigPath: 'opencode.runner.jsonc',
|
|
152
156
|
});
|
|
153
157
|
```
|
|
154
158
|
|
|
@@ -156,14 +160,19 @@ Your repo does not have to be a pnpm workspace. If it commits a `pnpm-lock.yaml`
|
|
|
156
160
|
pre-installs dependencies at build time (a faster first boot); if not, that step is skipped
|
|
157
161
|
and the agent installs on first use.
|
|
158
162
|
|
|
163
|
+
| MicroVM prop | Omitted behavior |
|
|
164
|
+
| --- | --- |
|
|
165
|
+
| `runnerSecret` | No GitHub or MCP credentials are exported at `/run`. |
|
|
166
|
+
| `runnerOpencodeConfigPath` | OpenCode uses the baked project configuration. Relative paths resolve from the workspace; absolute paths resolve in the image. |
|
|
167
|
+
| `gitUserName` / `gitUserEmail` | The hook uses the Evident bot defaults for git identity. |
|
|
168
|
+
|
|
159
169
|
## Status / limitations
|
|
160
170
|
|
|
161
|
-
- **Published to npm** as `@evident-ai/runner-cdk` (MIT) —
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
dependency for dogfooding.
|
|
171
|
+
- **Published to npm** as `@evident-ai/runner-cdk` (MIT). Install the `@dev` tag —
|
|
172
|
+
`npm install @evident-ai/runner-cdk@dev` (or `pnpm add`/`yarn add`); no checkout of
|
|
173
|
+
this repo is required. It builds to a self-contained `dist/` (`pnpm --filter
|
|
174
|
+
@evident-ai/runner-cdk build`) with no `workspace:`/`@evident/*` runtime dependency.
|
|
175
|
+
This repo's own consumer (`infrastructure/evident-runner`) still consumes it as a
|
|
176
|
+
`workspace:*` dependency for dogfooding.
|
|
168
177
|
- **AWS/ECS-specific.** Non-AWS clouds are an open question on the epic, not a supported
|
|
169
178
|
path.
|
|
@@ -101,7 +101,7 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
|
|
|
101
101
|
// --endpoint / --tunnel). The agent is resolved from EVIDENT_AGENT_KEY.
|
|
102
102
|
EVIDENT_API_URL: evidentApiUrl,
|
|
103
103
|
EVIDENT_TUNNEL_URL: evidentTunnelUrl,
|
|
104
|
-
// Litestream replica target (read by
|
|
104
|
+
// Litestream replica target (read by runner/synchroniser, which
|
|
105
105
|
// generates /etc/evident/litestream.yml at boot).
|
|
106
106
|
LITESTREAM_BUCKET: replicaBucket.bucketName,
|
|
107
107
|
LITESTREAM_PREFIX: this.replicaPrefix,
|
|
@@ -115,10 +115,9 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
|
|
|
115
115
|
// window (protected/active sessions are never touched). Tightened from 7d
|
|
116
116
|
// to 24h per #537: at 7d, opencode.db was observed growing ~11x (77 MB ->
|
|
117
117
|
// 849 MB) in ~24h, so 7d was far too permissive at the observed growth
|
|
118
|
-
// rate. 24h is
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
// interval and never touches a protected/active session.
|
|
118
|
+
// rate. 24h is the tighter mitigation that came out of that (see also the
|
|
119
|
+
// OPENCODE_VERSION pin above). Sweep runs on the default 1h interval and
|
|
120
|
+
// never touches a protected/active session.
|
|
122
121
|
EVIDENT_SESSION_CLEANUP_MAX_AGE: '24h',
|
|
123
122
|
CLUSTER: cluster.clusterName,
|
|
124
123
|
SERVICE: serviceName,
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// Shared literals for the MicroVM controller and for the image it launches.
|
|
3
3
|
// Every module imports from here rather than restating a value that must match
|
|
4
|
-
// across them
|
|
5
|
-
// image
|
|
6
|
-
// `scripts/build.ts`, so these values reach the image as inlined constants.
|
|
4
|
+
// across them. The image template pins matching copies, guarded by
|
|
5
|
+
// `microvm/image/dockerfile.test.ts`.
|
|
7
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
7
|
exports.MICROVM_MAX_RUN_SECONDS = exports.SUSPENDING_POLL_INTERVAL_MS = exports.SUSPENDING_POLL_ATTEMPTS = exports.RUN_HOOK_PAYLOAD_MAX_BYTES = exports.HOOK_TIMEOUT_SECONDS = exports.HOOKS_DIR = exports.HOOKS_PORT = void 0;
|
|
9
8
|
// The port the image's hook server binds, baked into the published Dockerfile
|
|
@@ -12,7 +12,7 @@ export interface EvidentMicrovmConstructProps {
|
|
|
12
12
|
readonly imageSource: string;
|
|
13
13
|
/**
|
|
14
14
|
* ARN of the Lambda-managed base MicroVM image, from
|
|
15
|
-
* `aws lambda list-managed-microvm-images`. A string, not a `CfnParameter`
|
|
15
|
+
* `aws lambda-microvms list-managed-microvm-images`. A string, not a `CfnParameter`
|
|
16
16
|
* built here: the caller creates the parameter at ITS OWN stack scope
|
|
17
17
|
* (preserving its logical id) and passes its resolved value down — a
|
|
18
18
|
* construct must not mint parameters into a consumer's stack.
|
|
@@ -20,7 +20,7 @@ export interface EvidentMicrovmConstructProps {
|
|
|
20
20
|
readonly baseImageArn: string;
|
|
21
21
|
/**
|
|
22
22
|
* Version of the Lambda-managed base MicroVM image, from
|
|
23
|
-
* `aws lambda list-managed-microvm-images`. See `baseImageArn` for why this
|
|
23
|
+
* `aws lambda-microvms list-managed-microvm-images`. See `baseImageArn` for why this
|
|
24
24
|
* is a plain string.
|
|
25
25
|
*/
|
|
26
26
|
readonly baseImageVersion: string;
|
|
@@ -41,6 +41,20 @@ export interface EvidentMicrovmConstructProps {
|
|
|
41
41
|
* in its own scope, and passes the result in either case.
|
|
42
42
|
*/
|
|
43
43
|
readonly doorbellSecret: secretsmanager.ISecret;
|
|
44
|
+
/**
|
|
45
|
+
* Optional runner secret whose JSON values are exported by `/run`. Omitting it
|
|
46
|
+
* leaves GitHub and MCP credentials unavailable to a generic consumer.
|
|
47
|
+
*/
|
|
48
|
+
readonly runnerSecret?: secretsmanager.ISecret;
|
|
49
|
+
/**
|
|
50
|
+
* Optional runner OpenCode overlay. Absolute paths are image paths; relative
|
|
51
|
+
* paths are resolved from the baked workspace by the `/run` hook.
|
|
52
|
+
*/
|
|
53
|
+
readonly runnerOpencodeConfigPath?: string;
|
|
54
|
+
/** Optional git identity the `/run` hook configures for agent commits. */
|
|
55
|
+
readonly gitUserName?: string;
|
|
56
|
+
/** Optional git email the `/run` hook configures for agent commits. */
|
|
57
|
+
readonly gitUserEmail?: string;
|
|
44
58
|
/**
|
|
45
59
|
* TCP port the image's hook server listens on, baked into both the image
|
|
46
60
|
* (`HOOKS_PORT` env var) and the `MicrovmImage`'s `hooks.port` — a mismatch
|
|
@@ -121,6 +121,12 @@ class EvidentMicrovmConstruct extends constructs_1.Construct {
|
|
|
121
121
|
// already bakes in for its own runners (evident-scale-to-zero-construct.ts:191).
|
|
122
122
|
// Safe to bake: identical for every VM from this image version.
|
|
123
123
|
EVIDENT_SESSION_CLEANUP_MAX_AGE: '24h',
|
|
124
|
+
...(props.runnerOpencodeConfigPath
|
|
125
|
+
? { RUNNER_OPENCODE_CONFIG: props.runnerOpencodeConfigPath }
|
|
126
|
+
: {}),
|
|
127
|
+
...(props.runnerSecret ? { RUNNER_SECRET_ARN: props.runnerSecret.secretArn } : {}),
|
|
128
|
+
...(props.gitUserName ? { GIT_USER_NAME: props.gitUserName } : {}),
|
|
129
|
+
...(props.gitUserEmail ? { GIT_USER_EMAIL: props.gitUserEmail } : {}),
|
|
124
130
|
};
|
|
125
131
|
// ...and the hooks configuration every shape's image is built with.
|
|
126
132
|
const imageHooks = {
|
|
@@ -179,6 +185,8 @@ class EvidentMicrovmConstruct extends constructs_1.Construct {
|
|
|
179
185
|
// customer account, and what Phase 4 (#263) has to tighten when this goes
|
|
180
186
|
// multi-tenant.
|
|
181
187
|
durableState.grantReadWrite(defaultImage.executionRole);
|
|
188
|
+
// The image receives only an ARN; `/run` reads the secret value at runtime.
|
|
189
|
+
props.runnerSecret?.grantRead(defaultImage.executionRole);
|
|
182
190
|
// The shape catalogue the controller resolves a doorbell's `shape` field
|
|
183
191
|
// against, and advertises over the describe channel (#723). One JSON
|
|
184
192
|
// array, built with `Stack.toJsonString` — not `JSON.stringify` — because
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.handleDoorbell = handleDoorbell;
|
|
4
4
|
const node_crypto_1 = require("node:crypto");
|
|
5
|
-
const
|
|
5
|
+
const sdk_1 = require("@evident/sdk");
|
|
6
6
|
const constants_1 = require("../constants");
|
|
7
7
|
const doorbell_1 = require("./doorbell");
|
|
8
8
|
const throttle_retry_1 = require("./throttle-retry");
|
|
@@ -175,7 +175,7 @@ async function shouldRecreateForNewerImage(doorbell, shape, runningVersion, micr
|
|
|
175
175
|
return compareImageVersions(running, latest) < 0;
|
|
176
176
|
}
|
|
177
177
|
async function handleDoorbell({ rawBody, signatureHeader, doorbellSecret, shapes, microvm, sleep, random, }) {
|
|
178
|
-
if (!(0,
|
|
178
|
+
if (!(0, sdk_1.verifyEvidentSignature)(rawBody, signatureHeader, doorbellSecret)) {
|
|
179
179
|
return decide({
|
|
180
180
|
statusCode: 401,
|
|
181
181
|
action: 'rejected',
|
|
@@ -49,7 +49,7 @@ export interface MicrovmClient {
|
|
|
49
49
|
* (docs/spikes/lambda-microvms-phase0/README.md). AWS's own Claude-agents
|
|
50
50
|
* guidance recommending `maxIdleDurationSeconds: 120` targets VMs reached
|
|
51
51
|
* inbound and does not apply here. Idle is instead decided IN the VM by
|
|
52
|
-
* `evident run --idle-timeout` (
|
|
52
|
+
* `evident run --idle-timeout` (runner/docker-images/microvm/hooks/common.sh's
|
|
53
53
|
* `IDLE_TIMEOUT_SECONDS`), whose clean exit drives this
|
|
54
54
|
* client's own `suspend()` via the `runner.suspend_requested` doorbell.
|
|
55
55
|
*/
|
|
@@ -37,7 +37,7 @@ RUN corepack enable && corepack prepare pnpm@9.15.0 --activate
|
|
|
37
37
|
# binary assertion further down fails the build if any required binary is gone.
|
|
38
38
|
RUN apt-get update \
|
|
39
39
|
&& apt-get install -y --no-install-recommends \
|
|
40
|
-
ca-certificates curl git jq less procps ripgrep \
|
|
40
|
+
ca-certificates curl git jq less procps psmisc ripgrep \
|
|
41
41
|
postgresql postgresql-contrib direnv \
|
|
42
42
|
gnupg unzip \
|
|
43
43
|
# --- GitHub CLI (`gh`) from the official apt repo ---
|
|
@@ -71,8 +71,8 @@ RUN apt-get update \
|
|
|
71
71
|
# restore` attempt — then starts a backgrounded `litestream replicate` once
|
|
72
72
|
# opencode has opened the DB (WI-3); `/suspend`/`/resume`/`/terminate` flush,
|
|
73
73
|
# restart and stop it in turn (WI-4). Do not remove as dead code.
|
|
74
|
-
ARG LITESTREAM_VERSION=0.5.
|
|
75
|
-
ARG LITESTREAM_SHA256=
|
|
74
|
+
ARG LITESTREAM_VERSION=0.5.16
|
|
75
|
+
ARG LITESTREAM_SHA256=678022e4103145302598e35d37f8718392d42e153feeb1e2d4a64dd0cd3aaf10
|
|
76
76
|
RUN curl -fsSL -o /tmp/litestream.tar.gz \
|
|
77
77
|
"https://github.com/benbjohnson/litestream/releases/download/v${LITESTREAM_VERSION}/litestream-${LITESTREAM_VERSION}-linux-arm64.tar.gz" \
|
|
78
78
|
&& echo "${LITESTREAM_SHA256} /tmp/litestream.tar.gz" | sha256sum -c - \
|
|
@@ -204,9 +204,12 @@ RUN git reset --quiet
|
|
|
204
204
|
# database stay per-VM (see entrypoint.sh's pre-warm in evident-runner).
|
|
205
205
|
RUN if [ -f pnpm-lock.yaml ]; then \
|
|
206
206
|
pnpm install --frozen-lockfile \
|
|
207
|
-
&& pnpm run build --filter='./packages/*'
|
|
207
|
+
&& pnpm run build --filter='./packages/*' \
|
|
208
|
+
--filter='./aws/runner-cdk' \
|
|
209
|
+
--filter='./aws/lambda-microvm-cdk' \
|
|
210
|
+
--filter='./aws/lambda-microvm-runtime'; \
|
|
208
211
|
else \
|
|
209
|
-
echo "[workspace-prep] no pnpm-lock.yaml in the baked repository — skipping the dependency install and the
|
|
212
|
+
echo "[workspace-prep] no pnpm-lock.yaml in the baked repository — skipping the dependency install and the workspace build; the agent installs on first use."; \
|
|
210
213
|
fi
|
|
211
214
|
|
|
212
215
|
# Data dir for the local dev Postgres cluster; listens on 5433 to match this
|
|
@@ -275,14 +275,10 @@ var require_dist = __commonJS({
|
|
|
275
275
|
}
|
|
276
276
|
});
|
|
277
277
|
|
|
278
|
-
//
|
|
278
|
+
// ../../runner/docker-images/microvm/hook-server.ts
|
|
279
279
|
var import_lambda_microvm_runtime = __toESM(require_dist());
|
|
280
|
-
|
|
281
|
-
// src/microvm/constants.ts
|
|
282
|
-
var HOOKS_PORT = 8080;
|
|
283
280
|
var HOOKS_DIR = "/etc/evident/hooks";
|
|
284
|
-
|
|
285
|
-
// src/microvm/image/hook-server.ts
|
|
281
|
+
var HOOKS_PORT = 8080;
|
|
286
282
|
var server = (0, import_lambda_microvm_runtime.createNodeRuntime)({ hooksDir: HOOKS_DIR });
|
|
287
283
|
var port = Number(process.env.HOOKS_PORT ?? HOOKS_PORT);
|
|
288
284
|
server.listen(port, "0.0.0.0", () => {
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
# Sourced by every hook script. Nothing here runs at image build time.
|
|
3
3
|
#
|
|
4
4
|
# These hooks are the SECOND shell speaking the runner-synchroniser CLI contract
|
|
5
|
-
# (
|
|
6
|
-
#
|
|
5
|
+
# (runner/docker-images/fargate/entrypoint.sh is the first), so both are held to it by
|
|
6
|
+
# runner/synchroniser/src/shell-contract.test.ts — which derives what
|
|
7
7
|
# `run_synchroniser` below must handle from the subcommands this shell calls.
|
|
8
8
|
|
|
9
9
|
# Installed from npm by the image (docker/Dockerfile's ARG
|
|
@@ -57,7 +57,7 @@ error() { echo "[hook:$(basename "$0")] ERROR: $*" >&2; }
|
|
|
57
57
|
# Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
|
|
58
58
|
# corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
|
|
59
59
|
# `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
|
|
60
|
-
# unusable)/32 (retry) — see `
|
|
60
|
+
# unusable)/32 (retry) — see `runner/synchroniser/src/cli.ts`'s own
|
|
61
61
|
# comment for what each means, not restated here. Any OTHER non-zero status
|
|
62
62
|
# means the tool itself broke, which is the only case worth an ERROR here —
|
|
63
63
|
# EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
|
|
@@ -171,7 +171,7 @@ timed_synchroniser() {
|
|
|
171
171
|
}
|
|
172
172
|
|
|
173
173
|
# Exports what `runner-synchroniser` resolves its object-store location from
|
|
174
|
-
# (
|
|
174
|
+
# (runner/synchroniser/src/config.ts). It treats either being empty as
|
|
175
175
|
# "persistence disabled" and then reports every restore as a WARNING it still
|
|
176
176
|
# exits 0 for — so an unset value has to be caught HERE, where it can still be
|
|
177
177
|
# told apart from "the store is simply empty".
|
|
@@ -211,13 +211,13 @@ load_state_config() {
|
|
|
211
211
|
# `model-auth-ready` therefore stays a boot-time diagnostic: its only output is
|
|
212
212
|
# a log line, since the runtime forwards no hook stderr to the doorbell caller.
|
|
213
213
|
|
|
214
|
-
# How long restore_credentials' shared step budget is, covering
|
|
215
|
-
# its synchroniser calls (restore claude, restore opencode,
|
|
216
|
-
# as one window rather than a timeout apiece — a fixed per-call cap at the same
|
|
214
|
+
# How long restore_credentials' shared step budget is, covering the runner-secret
|
|
215
|
+
# fetch plus its three synchroniser calls (restore claude, restore opencode,
|
|
216
|
+
# model-auth-ready) as one window rather than a timeout apiece — a fixed per-call cap at the same
|
|
217
217
|
# total would false-fire on the ordinary case of one slow call (a slow S3 GET),
|
|
218
218
|
# which is precisely the boot this exists to keep healthy. From #930's 3-boot
|
|
219
|
-
# sample: the step measured ~3-5s total (~1.2s per call, three
|
|
220
|
-
# starts of a 1.8 MB bundle) — a WEAK estimate this file's own
|
|
219
|
+
# sample: the step measured ~3-5s total (~1.2s per synchroniser call, three
|
|
220
|
+
# node cold starts of a 1.8 MB bundle) — a WEAK estimate this file's own
|
|
221
221
|
# SYNCHRONISER-TIMING lines are what will sharpen for real. Too tight and a
|
|
222
222
|
# routine slow call boots this VM with no model credentials until an operator
|
|
223
223
|
# reconnects it; too loose and a hung call burns more of the hook's own
|
|
@@ -229,6 +229,7 @@ CREDENTIAL_RESTORE_DEADLINE_SECONDS="${EVIDENT_CREDENTIAL_RESTORE_DEADLINE_SECON
|
|
|
229
229
|
# and a call that ignored it would be unbounded again). Hard ceiling on the
|
|
230
230
|
# step: DEADLINE + this = 10s, once — not per call.
|
|
231
231
|
CREDENTIAL_RESTORE_KILL_GRACE_SECONDS=2
|
|
232
|
+
GITHUB_PROBE_DEADLINE_SECONDS="${EVIDENT_GITHUB_PROBE_DEADLINE_SECONDS:-10}"
|
|
232
233
|
|
|
233
234
|
# What is left of the shared step budget, in whole seconds, `step_started_s`
|
|
234
235
|
# seconds after it began. `SECONDS` (a bash builtin with no failure mode,
|
|
@@ -268,12 +269,66 @@ bounded_restore_call() {
|
|
|
268
269
|
return 0
|
|
269
270
|
}
|
|
270
271
|
|
|
272
|
+
fetch_runner_secret() {
|
|
273
|
+
local step_started_s="$1" remaining rc=0 payload value stderr_file started_ms populated=0 missing=0
|
|
274
|
+
if [ -z "${RUNNER_SECRET_ARN:-}" ]; then
|
|
275
|
+
log "runner secret is not configured; continuing without GitHub and MCP credentials"
|
|
276
|
+
return 0
|
|
277
|
+
fi
|
|
278
|
+
remaining="$(remaining_credential_budget "${step_started_s}")"
|
|
279
|
+
if [ "${remaining}" -lt 1 ]; then
|
|
280
|
+
warn "CREDENTIAL-RESTORE-SKIPPED: runner-secret skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
|
|
281
|
+
return 0
|
|
282
|
+
fi
|
|
283
|
+
if ! stderr_file="$(mktemp /dev/shm/runner-secret-stderr.XXXXXX)"; then
|
|
284
|
+
warn "RUNNER-SECRET-STDERR-UNAVAILABLE: could not allocate diagnostic storage; continuing without runner credentials"
|
|
285
|
+
return 0
|
|
286
|
+
fi
|
|
287
|
+
started_ms="$(now_ms)"
|
|
288
|
+
payload="$(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${remaining}" aws secretsmanager get-secret-value --secret-id "${RUNNER_SECRET_ARN}" --query SecretString --output text 2>"${stderr_file}")" || rc=$?
|
|
289
|
+
log_elapsed_since runner-secret-fetch "${started_ms}" "${rc}"
|
|
290
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then
|
|
291
|
+
warn "CREDENTIAL-RESTORE-TIMEOUT: runner-secret did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
|
|
292
|
+
rm -f "${stderr_file}"
|
|
293
|
+
return 0
|
|
294
|
+
fi
|
|
295
|
+
if [ "${rc}" -ne 0 ]; then
|
|
296
|
+
warn "RUNNER-SECRET-UNREADABLE: $(<"${stderr_file}")"
|
|
297
|
+
rm -f "${stderr_file}"
|
|
298
|
+
return 0
|
|
299
|
+
fi
|
|
300
|
+
rm -f "${stderr_file}"
|
|
301
|
+
if ! jq -e 'type == "object"' >/dev/null 2>&1 <<<"${payload}"; then
|
|
302
|
+
warn "RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object"
|
|
303
|
+
return 0
|
|
304
|
+
fi
|
|
305
|
+
local -a populated_keys=()
|
|
306
|
+
for key in GH_TOKEN BRAVE_API_KEY CLOUDFLARE_API_TOKEN NEON_API_KEY; do
|
|
307
|
+
value="$(jq -r --arg k "${key}" '.[$k] // empty' <<<"${payload}")"
|
|
308
|
+
if [ -n "${value}" ]; then
|
|
309
|
+
export "${key}=${value}"
|
|
310
|
+
populated=$((populated + 1))
|
|
311
|
+
populated_keys+=("${key}")
|
|
312
|
+
else
|
|
313
|
+
missing=$((missing + 1))
|
|
314
|
+
fi
|
|
315
|
+
done
|
|
316
|
+
if [ "${populated}" -eq 0 ]; then
|
|
317
|
+
warn "RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-microvm/README.md"
|
|
318
|
+
else
|
|
319
|
+
log "RUNNER-SECRET-OK: populated ${populated_keys[*]}; ${missing} allow-listed keys empty or absent"
|
|
320
|
+
fi
|
|
321
|
+
return 0
|
|
322
|
+
}
|
|
323
|
+
|
|
271
324
|
restore_credentials() {
|
|
272
325
|
local step_started_ms step_started_s
|
|
273
326
|
step_started_ms="$(now_ms)"
|
|
274
327
|
step_started_s="${SECONDS}"
|
|
275
328
|
|
|
276
329
|
load_state_config || return 1
|
|
330
|
+
# This shares the existing bounded window so /run's worst-case duration does not grow.
|
|
331
|
+
fetch_runner_secret "${step_started_s}"
|
|
277
332
|
bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
|
|
278
333
|
bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
|
|
279
334
|
|
|
@@ -313,7 +368,7 @@ infrastructure/evident-microvm/README.md."
|
|
|
313
368
|
fi
|
|
314
369
|
# The step total, in the SAME greppable shape as the per-call lines above, so
|
|
315
370
|
# one query answers both "what does credential restore cost?" and "which of
|
|
316
|
-
# its
|
|
371
|
+
# its four operations cost it". Bounded now by CREDENTIAL_RESTORE_DEADLINE_SECONDS
|
|
317
372
|
# + CREDENTIAL_RESTORE_KILL_GRACE_SECONDS as one shared window (see the
|
|
318
373
|
# comment above that constant) rather than per call. `rc=0` is deliberate
|
|
319
374
|
# here, not an absence of failure modes: a timed-out call already warned by
|
|
@@ -323,6 +378,55 @@ infrastructure/evident-microvm/README.md."
|
|
|
323
378
|
return 0
|
|
324
379
|
}
|
|
325
380
|
|
|
381
|
+
apply_runner_opencode_config() {
|
|
382
|
+
if [ -z "${RUNNER_OPENCODE_CONFIG:-}" ]; then
|
|
383
|
+
log "runner OpenCode config is not configured; using the baked project config"
|
|
384
|
+
return 0
|
|
385
|
+
fi
|
|
386
|
+
local source="${RUNNER_OPENCODE_CONFIG}" target="opencode.json"
|
|
387
|
+
[[ "${source}" = /* ]] || source="${WORKSPACE}/${source}"
|
|
388
|
+
[ -f "${WORKSPACE}/opencode.jsonc" ] && target="opencode.jsonc"
|
|
389
|
+
if [ ! -f "${source}" ]; then
|
|
390
|
+
error "RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)"
|
|
391
|
+
return 0
|
|
392
|
+
fi
|
|
393
|
+
cp "${source}" "${WORKSPACE}/${target}"
|
|
394
|
+
git -C "${WORKSPACE}" update-index --skip-worktree "${target}" 2>/dev/null \
|
|
395
|
+
|| warn "could not mark ${target} skip-worktree; it may show as a local change"
|
|
396
|
+
log "Applied runner OpenCode config ${source} to ${WORKSPACE}/${target}"
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
configure_github_access() {
|
|
400
|
+
if [ -z "${GH_TOKEN:-}" ]; then
|
|
401
|
+
warn "GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above"
|
|
402
|
+
return 0
|
|
403
|
+
fi
|
|
404
|
+
export GIT_CONFIG_GLOBAL=/tmp/gitconfig
|
|
405
|
+
if ! : >"${GIT_CONFIG_GLOBAL}" ||
|
|
406
|
+
! git config --global user.name "${GIT_USER_NAME:-evident-bot}" ||
|
|
407
|
+
! git config --global user.email "${GIT_USER_EMAIL:-evident-bot@users.noreply.github.com}" ||
|
|
408
|
+
! git config --global init.defaultBranch main ||
|
|
409
|
+
! printf '%s\n' '#!/usr/bin/env bash' '[ "$1" = get ] || exit 0' 'echo username=x-access-token' 'echo "password=${GH_TOKEN}"' >/tmp/git-credential-helper.sh ||
|
|
410
|
+
! chmod 0700 /tmp/git-credential-helper.sh ||
|
|
411
|
+
! git config --global credential."https://github.com".helper /tmp/git-credential-helper.sh; then
|
|
412
|
+
warn "GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access"
|
|
413
|
+
return 0
|
|
414
|
+
fi
|
|
415
|
+
(
|
|
416
|
+
local output rc=0 login repo_url repo
|
|
417
|
+
output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api user --jq .login 2>&1)" || rc=$?
|
|
418
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
|
|
419
|
+
if [ "${rc}" -ne 0 ]; then warn "GITHUB-AUTH-REJECTED: ${output}"; return; fi
|
|
420
|
+
log "GITHUB-AUTH-OK: ${output}"
|
|
421
|
+
repo_url="$(git -C "${WORKSPACE}" remote get-url origin 2>/dev/null || true)"
|
|
422
|
+
repo="$(printf '%s' "${repo_url}" | sed -E 's#(https://github.com/|git@github.com:)##; s#\.git$##')"
|
|
423
|
+
[ -n "${repo}" ] || return
|
|
424
|
+
output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api "repos/${repo}" --jq .full_name 2>&1)" || rc=$?
|
|
425
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
|
|
426
|
+
[ "${rc}" -eq 0 ] || warn "GITHUB-REPO-INACCESSIBLE: ${repo}: ${output}"
|
|
427
|
+
) &
|
|
428
|
+
}
|
|
429
|
+
|
|
326
430
|
# Best-effort by design: /suspend must still drop the tunnel and /terminate must
|
|
327
431
|
# still clean up, so a flush that cannot happen is loud but never fatal.
|
|
328
432
|
sync_credentials() {
|
|
@@ -337,7 +441,7 @@ sync_credentials() {
|
|
|
337
441
|
# Writes NOTHING to S3 itself; replication back to S3 is started separately by
|
|
338
442
|
# `start_litestream` (below) from the `run`/`resume` hooks. `ensure_litestream_config`/
|
|
339
443
|
# `restore_session_db` are the MicroVM side of the same contract
|
|
340
|
-
#
|
|
444
|
+
# runner/docker-images/fargate/entrypoint.sh's inlined restore already speaks, going
|
|
341
445
|
# through the SAME runner-synchroniser CLI.
|
|
342
446
|
|
|
343
447
|
# Reads the ~30 MB litestream binary into the page cache, in the background, so
|
|
@@ -520,7 +624,7 @@ restore_session_db() {
|
|
|
520
624
|
return 0
|
|
521
625
|
}
|
|
522
626
|
|
|
523
|
-
# #931's exact lesson, one shell over (
|
|
627
|
+
# #931's exact lesson, one shell over (runner/docker-images/fargate/entrypoint.sh):
|
|
524
628
|
# the `|| { ... }` above only catches a non-zero EXIT — an `env` that exits
|
|
525
629
|
# 0 with an INCOMPLETE contract (a runner-synchroniser version/build skew)
|
|
526
630
|
# would otherwise abort right here under `set -u` the moment
|
|
@@ -577,7 +681,7 @@ restore_session_db() {
|
|
|
577
681
|
esac
|
|
578
682
|
|
|
579
683
|
local classify_rc=0
|
|
580
|
-
run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave || classify_rc=$?
|
|
684
|
+
run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave --fresh-db-fallback || classify_rc=$?
|
|
581
685
|
case "${classify_rc}" in
|
|
582
686
|
0) ;; # restored, or no replica yet — the CLI already logged which
|
|
583
687
|
31)
|
|
@@ -587,8 +691,8 @@ restore_session_db() {
|
|
|
587
691
|
# half-restored DB, so the marker stays; a 31 is structurally guaranteed
|
|
588
692
|
# to be a FRESH one, and replicating it starts a new backup chain instead
|
|
589
693
|
# of leaving this boot with a zero-width backup window. Same reasoning,
|
|
590
|
-
# and same one-line change, as
|
|
591
|
-
warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and
|
|
694
|
+
# and same one-line change, as runner/docker-images/fargate/entrypoint.sh's `31)`.
|
|
695
|
+
warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot (see the WARNING above)"
|
|
592
696
|
;;
|
|
593
697
|
32)
|
|
594
698
|
# The CLI's own contract for 32 is "re-run litestream restore and ask
|
|
@@ -1018,7 +1122,7 @@ flush_session_db() {
|
|
|
1018
1122
|
# returned its 200, so it is deliberately outside that arithmetic.
|
|
1019
1123
|
#
|
|
1020
1124
|
# EVIDENT_IDLE_TIMEOUT_SECONDS, deliberately the SAME env var name
|
|
1021
|
-
#
|
|
1125
|
+
# runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
|
|
1022
1126
|
# idle-timeout flag, so an operator who knows one knows the other. A
|
|
1023
1127
|
# non-numeric override must never silently DROP the flag — that degrades to
|
|
1024
1128
|
# an always-on VM burning ~$2.40/day, exactly the bug this closes — so it
|
|
@@ -1057,7 +1161,7 @@ start_tunnel() {
|
|
|
1057
1161
|
# into a MicroVM, so it is the ONLY way to seed model credentials on a runner
|
|
1058
1162
|
# that is already up — the S3 credential store is read at /run and /resume,
|
|
1059
1163
|
# never mid-life. The allow-list stays a single directory, as on ECS
|
|
1060
|
-
# (
|
|
1164
|
+
# (runner/docker-images/fargate/entrypoint.sh): the flag is repeatable, but every
|
|
1061
1165
|
# extra entry widens what Evident can write into a VM that executes agent
|
|
1062
1166
|
# code. ${HOME} is set by the image (ENV HOME=/home/runner) and `set -u` makes
|
|
1063
1167
|
# an unset one abort rather than silently allow-list "/.claude".
|
|
@@ -1104,7 +1208,7 @@ CLI_SHUTDOWN_CEILING_SECONDS=32
|
|
|
1104
1208
|
# everything else in the hook. 10 s is ~4x the measured shutdown and gives a
|
|
1105
1209
|
# provable worst case of 10 + the 10 s `suspend` spends in sync_credentials
|
|
1106
1210
|
# before us = 20 s, inside the ~55 s the in-VM server allows this script
|
|
1107
|
-
# (DEFAULT_TIMEOUT_SECONDS,
|
|
1211
|
+
# (DEFAULT_TIMEOUT_SECONDS, aws/lambda-microvm-runtime/src/runtime.ts),
|
|
1108
1212
|
# itself inside AWS's 60 s (HOOK_TIMEOUT_SECONDS, src/constants.ts) —
|
|
1109
1213
|
# overrunning that fails the whole lifecycle transition, which is worse than the
|
|
1110
1214
|
# kill this replaces. The residual cost is a drain longer than 10 s being
|
|
@@ -61,7 +61,7 @@ start_litestream
|
|
|
61
61
|
# measures in ~2 s (see the waker note in common.sh).
|
|
62
62
|
#
|
|
63
63
|
# Safe to leave running past the hook's own exit: the runtime waits on the
|
|
64
|
-
# script's `'exit'`, not `'close'` (
|
|
64
|
+
# script's `'exit'`, not `'close'` (aws/lambda-microvm-runtime/src/
|
|
65
65
|
# runtime.ts), so a lingering child never delays the HTTP response. Its stdio
|
|
66
66
|
# is deliberately NOT redirected — that inherited fd is how the RUNNER-KEY-*
|
|
67
67
|
# line reaches CloudWatch, which is the whole point of running it at all.
|
|
@@ -79,7 +79,13 @@ check_runner_key "${runner_key}" "${api_url}" || exit 1
|
|
|
79
79
|
restore_session_db
|
|
80
80
|
log "session DB restore done ${SECONDS}s into the hook"
|
|
81
81
|
|
|
82
|
-
# 7 —
|
|
82
|
+
# 7 — apply the overlay before OpenCode resolves its project configuration.
|
|
83
|
+
apply_runner_opencode_config
|
|
84
|
+
|
|
85
|
+
# 8 — configure git after credentials are restored and before agent shells start.
|
|
86
|
+
configure_github_access
|
|
87
|
+
|
|
88
|
+
# 9 — opencode. Started here, not at build time: a warm process in the shared
|
|
83
89
|
# snapshot would carry its installation id and database into every VM. Not
|
|
84
90
|
# waited on: a slow opencode boot is not a reason to fail /run (the tunnel CLI
|
|
85
91
|
# auto-starts opencode when it finds none healthy,
|
|
@@ -87,14 +93,14 @@ log "session DB restore done ${SECONDS}s into the hook"
|
|
|
87
93
|
# reclaims a runner that never comes online).
|
|
88
94
|
start_opencode
|
|
89
95
|
|
|
90
|
-
#
|
|
96
|
+
# 10 — begin replicating the session DB (#812 WI-3), now that opencode has
|
|
91
97
|
# opened it and before any work can arrive over the tunnel. Bare, like
|
|
92
98
|
# restore_session_db above: start_litestream never returns non-zero (every
|
|
93
99
|
# guard inside it is its own `return 0`), so there is nothing here for
|
|
94
100
|
# `set -e` to abort on.
|
|
95
101
|
start_litestream
|
|
96
102
|
|
|
97
|
-
#
|
|
103
|
+
# 11 — the first per-VM identity on the wire. The subshell's umask makes the file
|
|
98
104
|
# unreadable to anyone else from the moment it exists, before the key is in it.
|
|
99
105
|
(
|
|
100
106
|
umask 077
|
package/dist/waker/construct.js
CHANGED
|
@@ -56,7 +56,7 @@ class EvidentWaker extends constructs_1.Construct {
|
|
|
56
56
|
runtime: lambda.Runtime.NODEJS_22_X,
|
|
57
57
|
// Pre-bundled at PACKAGE build time (`pnpm --filter @evident-ai/runner-cdk
|
|
58
58
|
// build`, see scripts/build.ts), not at synth time: esbuild inlines
|
|
59
|
-
// @evident/
|
|
59
|
+
// @evident/sdk's HMAC verifier into
|
|
60
60
|
// dist/waker-lambda/handler.js, so neither this package's published npm
|
|
61
61
|
// artifact (which ships dist/ already built) nor a consuming app needs
|
|
62
62
|
// that private workspace package, or esbuild, on synth's PATH. Requires
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@evident-ai/runner-cdk",
|
|
3
|
-
"version": "0.1.1-dev.
|
|
3
|
+
"version": "0.1.1-dev.77d5cb6",
|
|
4
4
|
"description": "Reusable CDK constructs for an Evident agent runner: a single scale-to-zero Fargate runner (task + service + per-agent self-stop role + waker Lambda), or a per-session AWS Lambda MicroVM that boots on demand and suspends between messages. Instantiate once per agent from your own stack.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -15,13 +15,14 @@
|
|
|
15
15
|
],
|
|
16
16
|
"scripts": {
|
|
17
17
|
"build": "pnpm run build-bundled-deps && ts-node scripts/build.ts",
|
|
18
|
-
"//build-bundled-deps": "scripts/build.ts esbuild-bundles
|
|
19
|
-
"build-bundled-deps": "pnpm --filter @evident-ai/lambda-microvm-runtime build",
|
|
18
|
+
"//build-bundled-deps": "scripts/build.ts esbuild-bundles runner/docker-images/microvm/hook-server.ts, which imports @evident-ai/lambda-microvm-runtime by its built `main`. @evident-ai/lambda-microvm-cdk is a workspace:* devDependency whose types resolve through its own dist/, so tsc --project tsconfig.build.json cannot compile src/microvm/{shapes,construct}.ts without it. Chained into `build` rather than left to turbo's `^build` because publish-runner-cdk.yaml and infrastructure/evident-runner's build-runner-cdk-dep invoke `pnpm --filter @evident-ai/runner-cdk build` directly, which bypasses turbo entirely.",
|
|
19
|
+
"build-bundled-deps": "pnpm --filter @evident-ai/lambda-microvm-cdk build && pnpm --filter @evident-ai/lambda-microvm-runtime build",
|
|
20
20
|
"typecheck": "tsc --noEmit",
|
|
21
21
|
"test": "node --test --require ts-node/register 'src/**/*.test.ts'",
|
|
22
22
|
"format": "prettier --write 'src/**/*.ts'",
|
|
23
23
|
"lint": "eslint 'src/**/*.ts' --max-warnings=0"
|
|
24
24
|
},
|
|
25
|
+
"//peerDependencies": "@evident-ai/lambda-microvm-cdk is a registry range here, never workspace:*: publish-runner-cdk.yaml ships this package with `npm publish`, not `pnpm publish`, so pnpm's workspace-protocol substitution never runs and a workspace: range would reach npm literally, breaking every external install. Enforced by src/published-surface.test.ts. The matching devDependency is workspace:* so this monorepo's own build compiles against aws/lambda-microvm-cdk's source.",
|
|
25
26
|
"peerDependencies": {
|
|
26
27
|
"@evident-ai/lambda-microvm-cdk": "^0.1.1",
|
|
27
28
|
"aws-cdk-lib": "^2.240.0",
|
|
@@ -31,16 +32,17 @@
|
|
|
31
32
|
"@aws-sdk/client-ecs": "^3.682.0",
|
|
32
33
|
"@aws-sdk/client-lambda-microvms": "^3.1095.0",
|
|
33
34
|
"@aws-sdk/client-secrets-manager": "^3.682.0",
|
|
34
|
-
"@evident-ai/lambda-microvm-cdk": "
|
|
35
|
+
"@evident-ai/lambda-microvm-cdk": "workspace:*",
|
|
35
36
|
"@evident-ai/lambda-microvm-runtime": "workspace:*",
|
|
36
|
-
"@evident/
|
|
37
|
+
"@evident/sdk": "workspace:*",
|
|
37
38
|
"@types/node": "^22",
|
|
38
39
|
"aws-cdk-lib": "^2.240.0",
|
|
39
40
|
"constructs": "^10.5.0",
|
|
40
41
|
"esbuild": "^0.25.2",
|
|
41
42
|
"prettier": "^3.3.3",
|
|
42
43
|
"ts-node": "^10.9.2",
|
|
43
|
-
"typescript": "^5.6.3"
|
|
44
|
+
"typescript": "^5.6.3",
|
|
45
|
+
"runner-microvm-image": "workspace:*"
|
|
44
46
|
},
|
|
45
47
|
"publishConfig": {
|
|
46
48
|
"access": "public"
|
|
@@ -48,7 +50,7 @@
|
|
|
48
50
|
"repository": {
|
|
49
51
|
"type": "git",
|
|
50
52
|
"url": "https://github.com/sroze/evident.git",
|
|
51
|
-
"directory": "
|
|
53
|
+
"directory": "aws/runner-cdk"
|
|
52
54
|
},
|
|
53
55
|
"homepage": "https://evident.run",
|
|
54
56
|
"author": "Evident <hello@evident.run>",
|