@evident-ai/runner-cdk 0.1.1-dev.da88f8e → 3.4.1-dev.2f1b44b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -14
- package/dist/controller-lambda/handler.js +1 -1
- package/dist/evident-scale-to-zero-construct.d.ts +16 -4
- package/dist/evident-scale-to-zero-construct.js +19 -14
- package/dist/microvm/constants.js +2 -3
- package/dist/microvm/construct.d.ts +16 -2
- package/dist/microvm/construct.js +12 -0
- package/dist/microvm/controller/handle-doorbell.js +2 -2
- package/dist/microvm/controller/microvm-client.d.ts +1 -1
- package/dist/microvm/controller/shape-catalogue.d.ts +1 -1
- package/dist/microvm/controller/shape-catalogue.js +1 -1
- package/dist/microvm-image-context/Dockerfile +10 -7
- package/dist/microvm-image-context/hook-server.js +2 -6
- package/dist/microvm-image-context/hooks/common.sh +431 -23
- package/dist/microvm-image-context/hooks/resume +6 -2
- package/dist/microvm-image-context/hooks/run +24 -3
- package/dist/microvm-image-context/hooks/suspend +3 -0
- package/dist/microvm-image-context/hooks/terminate +3 -0
- package/dist/waker/construct.js +1 -1
- package/package.json +10 -8
package/README.md
CHANGED
|
@@ -9,9 +9,9 @@ Reusable AWS CDK constructs for running an [Evident](https://evident.run) agent
|
|
|
9
9
|
`desiredCount 0`, and an HMAC-authenticated waker Lambda scales it back up on the next
|
|
10
10
|
message. You pay for the time the agent is actually working.
|
|
11
11
|
- **`EvidentMicrovmConstruct`** — a per-session AWS Lambda MicroVM that boots on demand
|
|
12
|
-
and suspends between messages. Installing this package is all you
|
|
13
|
-
context ships inside it (see [The MicroVM image](#the-microvm-image)
|
|
14
|
-
no checkout of `sroze/evident` involved. See
|
|
12
|
+
and suspends between messages. Installing the `@dev` tag of this package is all you
|
|
13
|
+
need: the image build context ships inside it (see [The MicroVM image](#the-microvm-image)
|
|
14
|
+
below), so there is no checkout of `sroze/evident` involved. See
|
|
15
15
|
[the AWS runner doc](https://evident.run/docs/aws-runner) for the full picture of both
|
|
16
16
|
strategies.
|
|
17
17
|
|
|
@@ -34,8 +34,9 @@ cluster can host several agents.
|
|
|
34
34
|
## Getting started
|
|
35
35
|
|
|
36
36
|
For a full end-to-end walkthrough — shared infra, secrets without SOPS/KMS, the image,
|
|
37
|
-
the two grants, deploy, and wiring the wake webhook — see
|
|
38
|
-
|
|
37
|
+
the two grants, deploy, and wiring the wake webhook — see `GETTING-STARTED.md` in this
|
|
38
|
+
package's directory (not published to npm; read it from a checkout of the repo). The
|
|
39
|
+
snippet below is the short version.
|
|
39
40
|
|
|
40
41
|
```ts
|
|
41
42
|
import { EvidentScaleToZeroConstruct } from "@evident-ai/runner-cdk";
|
|
@@ -96,7 +97,7 @@ the construct's own dogfooding consumer.
|
|
|
96
97
|
| `cluster`, `securityGroup`, `taskRole`, `logGroup`, `replicaBucket` | interfaces | Shared infra you create and pass in. |
|
|
97
98
|
| `image` | `ecs.ContainerImage` | Your runner image. |
|
|
98
99
|
| `agentSecret`, `wakerSecret` | `secretsmanager.ISecret` | Any Secrets Manager secret. |
|
|
99
|
-
| `availableSecretKeys` | `Set<string>` | Keys present in the agent secret.
|
|
100
|
+
| `availableSecretKeys` | `Set<string>` | Keys present in the agent secret. Every listed key is injected; `EVIDENT_AGENT_KEY` and `GH_TOKEN` are also injected for compatibility with adopters that do not enumerate their secret. |
|
|
100
101
|
|
|
101
102
|
### Two deliberate "no default" choices
|
|
102
103
|
|
|
@@ -111,7 +112,7 @@ the construct's own dogfooding consumer.
|
|
|
111
112
|
|
|
112
113
|
This construct never builds an image. Pass any `ecs.ContainerImage` — from a registry, or
|
|
113
114
|
built from a `Dockerfile` you control. The generic runner image lives separately in
|
|
114
|
-
`
|
|
115
|
+
`runner/docker-images/fargate`, so you can adopt the image, the construct, or both.
|
|
115
116
|
|
|
116
117
|
## The MicroVM image
|
|
117
118
|
|
|
@@ -126,6 +127,7 @@ the per-phase hook scripts and the bundled hook server, published inside the tar
|
|
|
126
127
|
the agent's workspace. `stageMicrovmImageContext()` puts the two together:
|
|
127
128
|
|
|
128
129
|
```ts
|
|
130
|
+
import path from 'node:path';
|
|
129
131
|
import {
|
|
130
132
|
EvidentMicrovmConstruct,
|
|
131
133
|
HOOKS_PORT,
|
|
@@ -149,6 +151,8 @@ new EvidentMicrovmConstruct(this, 'Runner', {
|
|
|
149
151
|
baseImageArn,
|
|
150
152
|
baseImageVersion,
|
|
151
153
|
doorbellSecret,
|
|
154
|
+
runnerSecret,
|
|
155
|
+
runnerOpencodeConfigPath: 'opencode.runner.jsonc',
|
|
152
156
|
});
|
|
153
157
|
```
|
|
154
158
|
|
|
@@ -156,14 +160,19 @@ Your repo does not have to be a pnpm workspace. If it commits a `pnpm-lock.yaml`
|
|
|
156
160
|
pre-installs dependencies at build time (a faster first boot); if not, that step is skipped
|
|
157
161
|
and the agent installs on first use.
|
|
158
162
|
|
|
163
|
+
| MicroVM prop | Omitted behavior |
|
|
164
|
+
| --- | --- |
|
|
165
|
+
| `runnerSecret` | No GitHub or MCP credentials are exported at `/run`. When supplied, `/run` reads the JSON secret and exports every non-empty value whose key is a valid environment-variable name. |
|
|
166
|
+
| `runnerOpencodeConfigPath` | OpenCode uses the baked project configuration. Relative paths resolve from the workspace; absolute paths resolve in the image. |
|
|
167
|
+
| `gitUserName` / `gitUserEmail` | The hook uses the Evident bot defaults for git identity. |
|
|
168
|
+
|
|
159
169
|
## Status / limitations
|
|
160
170
|
|
|
161
|
-
- **Published to npm** as `@evident-ai/runner-cdk` (MIT) —
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
dependency for dogfooding.
|
|
171
|
+
- **Published to npm** as `@evident-ai/runner-cdk` (MIT). Install the `@dev` tag —
|
|
172
|
+
`npm install @evident-ai/runner-cdk@dev` (or `pnpm add`/`yarn add`); no checkout of
|
|
173
|
+
this repo is required. It builds to a self-contained `dist/` (`pnpm --filter
|
|
174
|
+
@evident-ai/runner-cdk build`) with no `workspace:`/`@evident/*` runtime dependency.
|
|
175
|
+
This repo's own consumer (`infrastructure/evident-runner`) still consumes it as a
|
|
176
|
+
`workspace:*` dependency for dogfooding.
|
|
168
177
|
- **AWS/ECS-specific.** Non-AWS clouds are an open question on the epic, not a supported
|
|
169
178
|
path.
|
|
@@ -42866,7 +42866,7 @@ Reference: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sso.ht
|
|
|
42866
42866
|
// ../../node_modules/.pnpm/@aws-sdk+signature-v4-multi-region@3.996.42/node_modules/@aws-sdk/signature-v4-multi-region/dist-cjs/index.js
|
|
42867
42867
|
var require_dist_cjs23 = __commonJS({
|
|
42868
42868
|
"../../node_modules/.pnpm/@aws-sdk+signature-v4-multi-region@3.996.42/node_modules/@aws-sdk/signature-v4-multi-region/dist-cjs/index.js"(exports2) {
|
|
42869
|
-
var { SignatureV4: SignatureV43, signatureV4aContainer } =
|
|
42869
|
+
var { SignatureV4: SignatureV43, signatureV4aContainer } = require_dist_cjs2();
|
|
42870
42870
|
var signatureV4CrtContainer = {
|
|
42871
42871
|
CrtSignerV4: null
|
|
42872
42872
|
};
|
|
@@ -61,13 +61,25 @@ export type EvidentScaleToZeroConstructProps = {
|
|
|
61
61
|
replicaBucket: s3.IBucket;
|
|
62
62
|
image: ecs.ContainerImage;
|
|
63
63
|
/**
|
|
64
|
-
* Container secrets.
|
|
65
|
-
*
|
|
66
|
-
*
|
|
64
|
+
* Container secrets. `availableSecretKeys` are injected by name; EVIDENT_AGENT_KEY
|
|
65
|
+
* and GH_TOKEN remain available to external adopters that do not enumerate keys.
|
|
66
|
+
*
|
|
67
|
+
* `agentSecret` always supplies EVIDENT_AGENT_KEY. When `capabilitySecret` is
|
|
68
|
+
* ALSO given (#1868 WI-2), every capability key (GH_TOKEN included) is injected
|
|
69
|
+
* from THAT secret instead, and `availableSecretKeys` is ignored — the two
|
|
70
|
+
* deploy targets (this one and the MicroVM stack) then read the same
|
|
71
|
+
* capability keys from the same shared secret rather than each carrying its
|
|
72
|
+
* own copy. When `capabilitySecret` is absent, behavior is unchanged: GH_TOKEN
|
|
73
|
+
* plus every `availableSecretKeys` entry, all from `agentSecret` — an external
|
|
74
|
+
* adopter passing only `agentSecret`/`availableSecretKeys` is unaffected.
|
|
67
75
|
*/
|
|
68
76
|
agentSecret: secretsmanager.ISecret;
|
|
69
|
-
/** Key names present in the agent secret
|
|
77
|
+
/** Key names present in the agent secret. Every listed key is injected into the container. */
|
|
70
78
|
availableSecretKeys: Set<string>;
|
|
79
|
+
/** The shared capability secret (#1868 WI-2). See `agentSecret`'s doc above. */
|
|
80
|
+
capabilitySecret?: secretsmanager.ISecret;
|
|
81
|
+
/** Key names present in `capabilitySecret`. Ignored when `capabilitySecret` is absent. */
|
|
82
|
+
capabilityKeys?: Set<string>;
|
|
71
83
|
/** Waker secret (EVIDENT_WAKE_SECRET), consumed by the waker at runtime. */
|
|
72
84
|
wakerSecret: secretsmanager.ISecret;
|
|
73
85
|
};
|
|
@@ -61,7 +61,7 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
|
|
|
61
61
|
service;
|
|
62
62
|
constructor(scope, id, props) {
|
|
63
63
|
super(scope, id);
|
|
64
|
-
const { agentName, evidentAgentId, gitRepo, gitBranch, idleTimeoutSeconds, cpu, memoryLimitMiB, resourcePrefix = DEFAULT_RESOURCE_PREFIX, envName, evidentApiUrl, evidentTunnelUrl, extraEnvironment, cluster, securityGroup, taskRole, logGroup, replicaBucket, image, agentSecret, availableSecretKeys, wakerSecret, } = props;
|
|
64
|
+
const { agentName, evidentAgentId, gitRepo, gitBranch, idleTimeoutSeconds, cpu, memoryLimitMiB, resourcePrefix = DEFAULT_RESOURCE_PREFIX, envName, evidentApiUrl, evidentTunnelUrl, extraEnvironment, cluster, securityGroup, taskRole, logGroup, replicaBucket, image, agentSecret, availableSecretKeys, capabilitySecret, capabilityKeys, wakerSecret, } = props;
|
|
65
65
|
this.replicaPrefix = `agents/${evidentAgentId}`;
|
|
66
66
|
// A plain STRING (not service.serviceName) so the container env is static and
|
|
67
67
|
// has no construct-ordering dependency on the service.
|
|
@@ -101,7 +101,7 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
|
|
|
101
101
|
// --endpoint / --tunnel). The agent is resolved from EVIDENT_AGENT_KEY.
|
|
102
102
|
EVIDENT_API_URL: evidentApiUrl,
|
|
103
103
|
EVIDENT_TUNNEL_URL: evidentTunnelUrl,
|
|
104
|
-
// Litestream replica target (read by
|
|
104
|
+
// Litestream replica target (read by runner/synchroniser, which
|
|
105
105
|
// generates /etc/evident/litestream.yml at boot).
|
|
106
106
|
LITESTREAM_BUCKET: replicaBucket.bucketName,
|
|
107
107
|
LITESTREAM_PREFIX: this.replicaPrefix,
|
|
@@ -115,10 +115,9 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
|
|
|
115
115
|
// window (protected/active sessions are never touched). Tightened from 7d
|
|
116
116
|
// to 24h per #537: at 7d, opencode.db was observed growing ~11x (77 MB ->
|
|
117
117
|
// 849 MB) in ~24h, so 7d was far too permissive at the observed growth
|
|
118
|
-
// rate. 24h is
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
// interval and never touches a protected/active session.
|
|
118
|
+
// rate. 24h is the tighter mitigation that came out of that (see also the
|
|
119
|
+
// OPENCODE_VERSION pin above). Sweep runs on the default 1h interval and
|
|
120
|
+
// never touches a protected/active session.
|
|
122
121
|
EVIDENT_SESSION_CLEANUP_MAX_AGE: '24h',
|
|
123
122
|
CLUSTER: cluster.clusterName,
|
|
124
123
|
SERVICE: serviceName,
|
|
@@ -128,16 +127,22 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
|
|
|
128
127
|
...extraEnvironment,
|
|
129
128
|
};
|
|
130
129
|
const secrets = {
|
|
130
|
+
// Identifies the runner to Evident. Always agentSecret — capabilitySecret
|
|
131
|
+
// never carries this key (#1868 WI-2: the two are a deliberate split, not
|
|
132
|
+
// a duplication of the same data).
|
|
131
133
|
EVIDENT_AGENT_KEY: ecs.Secret.fromSecretsManager(agentSecret, 'EVIDENT_AGENT_KEY'),
|
|
132
|
-
GH_TOKEN: ecs.Secret.fromSecretsManager(agentSecret, 'GH_TOKEN'),
|
|
133
134
|
};
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
135
|
+
if (capabilitySecret !== undefined) {
|
|
136
|
+
for (const key of capabilityKeys ?? new Set()) {
|
|
137
|
+
secrets[key] = ecs.Secret.fromSecretsManager(capabilitySecret, key);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
// Legacy shape, unchanged: GH_TOKEN (required to clone this construct's
|
|
142
|
+
// workspace at boot) plus every availableSecretKeys entry, all from
|
|
143
|
+
// agentSecret — what every external adopter still gets.
|
|
144
|
+
secrets.GH_TOKEN = ecs.Secret.fromSecretsManager(agentSecret, 'GH_TOKEN');
|
|
145
|
+
for (const key of availableSecretKeys) {
|
|
141
146
|
secrets[key] = ecs.Secret.fromSecretsManager(agentSecret, key);
|
|
142
147
|
}
|
|
143
148
|
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// Shared literals for the MicroVM controller and for the image it launches.
|
|
3
3
|
// Every module imports from here rather than restating a value that must match
|
|
4
|
-
// across them
|
|
5
|
-
// image
|
|
6
|
-
// `scripts/build.ts`, so these values reach the image as inlined constants.
|
|
4
|
+
// across them. The image template pins matching copies, guarded by
|
|
5
|
+
// `microvm/image/dockerfile.test.ts`.
|
|
7
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
7
|
exports.MICROVM_MAX_RUN_SECONDS = exports.SUSPENDING_POLL_INTERVAL_MS = exports.SUSPENDING_POLL_ATTEMPTS = exports.RUN_HOOK_PAYLOAD_MAX_BYTES = exports.HOOK_TIMEOUT_SECONDS = exports.HOOKS_DIR = exports.HOOKS_PORT = void 0;
|
|
9
8
|
// The port the image's hook server binds, baked into the published Dockerfile
|
|
@@ -12,7 +12,7 @@ export interface EvidentMicrovmConstructProps {
|
|
|
12
12
|
readonly imageSource: string;
|
|
13
13
|
/**
|
|
14
14
|
* ARN of the Lambda-managed base MicroVM image, from
|
|
15
|
-
* `aws lambda list-managed-microvm-images`. A string, not a `CfnParameter`
|
|
15
|
+
* `aws lambda-microvms list-managed-microvm-images`. A string, not a `CfnParameter`
|
|
16
16
|
* built here: the caller creates the parameter at ITS OWN stack scope
|
|
17
17
|
* (preserving its logical id) and passes its resolved value down — a
|
|
18
18
|
* construct must not mint parameters into a consumer's stack.
|
|
@@ -20,7 +20,7 @@ export interface EvidentMicrovmConstructProps {
|
|
|
20
20
|
readonly baseImageArn: string;
|
|
21
21
|
/**
|
|
22
22
|
* Version of the Lambda-managed base MicroVM image, from
|
|
23
|
-
* `aws lambda list-managed-microvm-images`. See `baseImageArn` for why this
|
|
23
|
+
* `aws lambda-microvms list-managed-microvm-images`. See `baseImageArn` for why this
|
|
24
24
|
* is a plain string.
|
|
25
25
|
*/
|
|
26
26
|
readonly baseImageVersion: string;
|
|
@@ -41,6 +41,20 @@ export interface EvidentMicrovmConstructProps {
|
|
|
41
41
|
* in its own scope, and passes the result in either case.
|
|
42
42
|
*/
|
|
43
43
|
readonly doorbellSecret: secretsmanager.ISecret;
|
|
44
|
+
/**
|
|
45
|
+
* Optional runner secret whose JSON values are exported by `/run`. Omitting it
|
|
46
|
+
* leaves GitHub and MCP credentials unavailable to a generic consumer.
|
|
47
|
+
*/
|
|
48
|
+
readonly runnerSecret?: secretsmanager.ISecret;
|
|
49
|
+
/**
|
|
50
|
+
* Optional runner OpenCode overlay. Absolute paths are image paths; relative
|
|
51
|
+
* paths are resolved from the baked workspace by the `/run` hook.
|
|
52
|
+
*/
|
|
53
|
+
readonly runnerOpencodeConfigPath?: string;
|
|
54
|
+
/** Optional git identity the `/run` hook configures for agent commits. */
|
|
55
|
+
readonly gitUserName?: string;
|
|
56
|
+
/** Optional git email the `/run` hook configures for agent commits. */
|
|
57
|
+
readonly gitUserEmail?: string;
|
|
44
58
|
/**
|
|
45
59
|
* TCP port the image's hook server listens on, baked into both the image
|
|
46
60
|
* (`HOOKS_PORT` env var) and the `MicrovmImage`'s `hooks.port` — a mismatch
|
|
@@ -121,6 +121,16 @@ class EvidentMicrovmConstruct extends constructs_1.Construct {
|
|
|
121
121
|
// already bakes in for its own runners (evident-scale-to-zero-construct.ts:191).
|
|
122
122
|
// Safe to bake: identical for every VM from this image version.
|
|
123
123
|
EVIDENT_SESSION_CLEANUP_MAX_AGE: '24h',
|
|
124
|
+
...(props.runnerOpencodeConfigPath
|
|
125
|
+
? { RUNNER_OPENCODE_CONFIG: props.runnerOpencodeConfigPath }
|
|
126
|
+
: {}),
|
|
127
|
+
...(props.runnerSecret
|
|
128
|
+
? {
|
|
129
|
+
RUNNER_SECRET_ARN: props.runnerSecret.secretArn,
|
|
130
|
+
}
|
|
131
|
+
: {}),
|
|
132
|
+
...(props.gitUserName ? { GIT_USER_NAME: props.gitUserName } : {}),
|
|
133
|
+
...(props.gitUserEmail ? { GIT_USER_EMAIL: props.gitUserEmail } : {}),
|
|
124
134
|
};
|
|
125
135
|
// ...and the hooks configuration every shape's image is built with.
|
|
126
136
|
const imageHooks = {
|
|
@@ -179,6 +189,8 @@ class EvidentMicrovmConstruct extends constructs_1.Construct {
|
|
|
179
189
|
// customer account, and what Phase 4 (#263) has to tighten when this goes
|
|
180
190
|
// multi-tenant.
|
|
181
191
|
durableState.grantReadWrite(defaultImage.executionRole);
|
|
192
|
+
// The image receives only an ARN; `/run` reads the secret value at runtime.
|
|
193
|
+
props.runnerSecret?.grantRead(defaultImage.executionRole);
|
|
182
194
|
// The shape catalogue the controller resolves a doorbell's `shape` field
|
|
183
195
|
// against, and advertises over the describe channel (#723). One JSON
|
|
184
196
|
// array, built with `Stack.toJsonString` — not `JSON.stringify` — because
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.handleDoorbell = handleDoorbell;
|
|
4
4
|
const node_crypto_1 = require("node:crypto");
|
|
5
|
-
const
|
|
5
|
+
const sdk_1 = require("@evident/sdk");
|
|
6
6
|
const constants_1 = require("../constants");
|
|
7
7
|
const doorbell_1 = require("./doorbell");
|
|
8
8
|
const throttle_retry_1 = require("./throttle-retry");
|
|
@@ -175,7 +175,7 @@ async function shouldRecreateForNewerImage(doorbell, shape, runningVersion, micr
|
|
|
175
175
|
return compareImageVersions(running, latest) < 0;
|
|
176
176
|
}
|
|
177
177
|
async function handleDoorbell({ rawBody, signatureHeader, doorbellSecret, shapes, microvm, sleep, random, }) {
|
|
178
|
-
if (!(0,
|
|
178
|
+
if (!(0, sdk_1.verifyEvidentSignature)(rawBody, signatureHeader, doorbellSecret)) {
|
|
179
179
|
return decide({
|
|
180
180
|
statusCode: 401,
|
|
181
181
|
action: 'rejected',
|
|
@@ -49,7 +49,7 @@ export interface MicrovmClient {
|
|
|
49
49
|
* (docs/spikes/lambda-microvms-phase0/README.md). AWS's own Claude-agents
|
|
50
50
|
* guidance recommending `maxIdleDurationSeconds: 120` targets VMs reached
|
|
51
51
|
* inbound and does not apply here. Idle is instead decided IN the VM by
|
|
52
|
-
* `evident run --idle-timeout` (
|
|
52
|
+
* `evident run --idle-timeout` (runner/docker-images/microvm/hooks/common.sh's
|
|
53
53
|
* `IDLE_TIMEOUT_SECONDS`), whose clean exit drives this
|
|
54
54
|
* client's own `suspend()` via the `runner.suspend_requested` doorbell.
|
|
55
55
|
*/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Parses the `MICROVM_SHAPES` environment variable (a JSON array, written by
|
|
3
|
-
* CloudFormation from `
|
|
3
|
+
* CloudFormation from `aws/runner-cdk/src/microvm/shapes.ts` at
|
|
4
4
|
* deploy time — see D6/D7 in the plan) into a `ShapeCatalogue` the pure
|
|
5
5
|
* decision core can query, without that core ever touching `process.env`
|
|
6
6
|
* itself.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
3
|
* Parses the `MICROVM_SHAPES` environment variable (a JSON array, written by
|
|
4
|
-
* CloudFormation from `
|
|
4
|
+
* CloudFormation from `aws/runner-cdk/src/microvm/shapes.ts` at
|
|
5
5
|
* deploy time — see D6/D7 in the plan) into a `ShapeCatalogue` the pure
|
|
6
6
|
* decision core can query, without that core ever touching `process.env`
|
|
7
7
|
* itself.
|
|
@@ -37,7 +37,7 @@ RUN corepack enable && corepack prepare pnpm@9.15.0 --activate
|
|
|
37
37
|
# binary assertion further down fails the build if any required binary is gone.
|
|
38
38
|
RUN apt-get update \
|
|
39
39
|
&& apt-get install -y --no-install-recommends \
|
|
40
|
-
ca-certificates curl git jq less procps ripgrep \
|
|
40
|
+
ca-certificates curl git jq less procps psmisc ripgrep \
|
|
41
41
|
postgresql postgresql-contrib direnv \
|
|
42
42
|
gnupg unzip \
|
|
43
43
|
# --- GitHub CLI (`gh`) from the official apt repo ---
|
|
@@ -71,8 +71,8 @@ RUN apt-get update \
|
|
|
71
71
|
# restore` attempt — then starts a backgrounded `litestream replicate` once
|
|
72
72
|
# opencode has opened the DB (WI-3); `/suspend`/`/resume`/`/terminate` flush,
|
|
73
73
|
# restart and stop it in turn (WI-4). Do not remove as dead code.
|
|
74
|
-
ARG LITESTREAM_VERSION=0.5.
|
|
75
|
-
ARG LITESTREAM_SHA256=
|
|
74
|
+
ARG LITESTREAM_VERSION=0.5.16
|
|
75
|
+
ARG LITESTREAM_SHA256=678022e4103145302598e35d37f8718392d42e153feeb1e2d4a64dd0cd3aaf10
|
|
76
76
|
RUN curl -fsSL -o /tmp/litestream.tar.gz \
|
|
77
77
|
"https://github.com/benbjohnson/litestream/releases/download/v${LITESTREAM_VERSION}/litestream-${LITESTREAM_VERSION}-linux-arm64.tar.gz" \
|
|
78
78
|
&& echo "${LITESTREAM_SHA256} /tmp/litestream.tar.gz" | sha256sum -c - \
|
|
@@ -171,10 +171,10 @@ ENV HOME=/home/runner
|
|
|
171
171
|
# permanent no-op and every VM from this snapshot shares one machine id. Empty
|
|
172
172
|
# is the correct unset state — it is the absence of an identity, so nothing
|
|
173
173
|
# per-VM-unique enters the shared snapshot.
|
|
174
|
-
RUN mkdir -p /var/lib/runner-pg /var/lib/dbus \
|
|
174
|
+
RUN mkdir -p /var/lib/runner-pg /var/lib/dbus /home/runner/.local/state/evident \
|
|
175
175
|
&& install -o runner -g runner -m 0644 /dev/null /etc/machine-id \
|
|
176
176
|
&& install -o runner -g runner -m 0644 /dev/null /var/lib/dbus/machine-id \
|
|
177
|
-
|
|
177
|
+
&& chown -R runner:runner /var/lib/runner-pg /home/runner
|
|
178
178
|
COPY --chown=10001:10001 repo ${WORKSPACE}
|
|
179
179
|
WORKDIR ${WORKSPACE}
|
|
180
180
|
USER runner
|
|
@@ -204,9 +204,12 @@ RUN git reset --quiet
|
|
|
204
204
|
# database stay per-VM (see entrypoint.sh's pre-warm in evident-runner).
|
|
205
205
|
RUN if [ -f pnpm-lock.yaml ]; then \
|
|
206
206
|
pnpm install --frozen-lockfile \
|
|
207
|
-
&& pnpm run build --filter='./packages/*'
|
|
207
|
+
&& pnpm run build --filter='./packages/*' \
|
|
208
|
+
--filter='./aws/runner-cdk' \
|
|
209
|
+
--filter='./aws/lambda-microvm-cdk' \
|
|
210
|
+
--filter='./aws/lambda-microvm-runtime'; \
|
|
208
211
|
else \
|
|
209
|
-
echo "[workspace-prep] no pnpm-lock.yaml in the baked repository — skipping the dependency install and the
|
|
212
|
+
echo "[workspace-prep] no pnpm-lock.yaml in the baked repository — skipping the dependency install and the workspace build; the agent installs on first use."; \
|
|
210
213
|
fi
|
|
211
214
|
|
|
212
215
|
# Data dir for the local dev Postgres cluster; listens on 5433 to match this
|
|
@@ -275,14 +275,10 @@ var require_dist = __commonJS({
|
|
|
275
275
|
}
|
|
276
276
|
});
|
|
277
277
|
|
|
278
|
-
//
|
|
278
|
+
// ../../runner/docker-images/microvm/hook-server.ts
|
|
279
279
|
var import_lambda_microvm_runtime = __toESM(require_dist());
|
|
280
|
-
|
|
281
|
-
// src/microvm/constants.ts
|
|
282
|
-
var HOOKS_PORT = 8080;
|
|
283
280
|
var HOOKS_DIR = "/etc/evident/hooks";
|
|
284
|
-
|
|
285
|
-
// src/microvm/image/hook-server.ts
|
|
281
|
+
var HOOKS_PORT = 8080;
|
|
286
282
|
var server = (0, import_lambda_microvm_runtime.createNodeRuntime)({ hooksDir: HOOKS_DIR });
|
|
287
283
|
var port = Number(process.env.HOOKS_PORT ?? HOOKS_PORT);
|
|
288
284
|
server.listen(port, "0.0.0.0", () => {
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
# Sourced by every hook script. Nothing here runs at image build time.
|
|
3
3
|
#
|
|
4
4
|
# These hooks are the SECOND shell speaking the runner-synchroniser CLI contract
|
|
5
|
-
# (
|
|
6
|
-
#
|
|
5
|
+
# (runner/docker-images/fargate/entrypoint.sh is the first), so both are held to it by
|
|
6
|
+
# runner/synchroniser/src/shell-contract.test.ts — which derives what
|
|
7
7
|
# `run_synchroniser` below must handle from the subcommands this shell calls.
|
|
8
8
|
|
|
9
9
|
# Installed from npm by the image (docker/Dockerfile's ARG
|
|
@@ -24,6 +24,8 @@ CONTEXT_FILE="/dev/shm/evident-run-context"
|
|
|
24
24
|
TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
|
|
25
25
|
OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
|
|
26
26
|
LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
|
|
27
|
+
CREDS_SYNC_PID_FILE="/dev/shm/evident-creds-sync.pid"
|
|
28
|
+
CREDS_SYNC_LAST_ERROR_FILE="/dev/shm/evident-creds-sync.last-error"
|
|
27
29
|
|
|
28
30
|
# Where the runner's credential store lives inside the durable-state bucket.
|
|
29
31
|
# The BUCKET is the same for every VM from an image version, so the stack bakes
|
|
@@ -50,14 +52,49 @@ LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
|
|
|
50
52
|
# earlier boot cannot silently disable replication for the VM's whole life.
|
|
51
53
|
SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
|
|
52
54
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
55
|
+
hook_name() { printf '%s' "${0##*/}"; }
|
|
56
|
+
log() { echo "[hook:$(hook_name)] $*"; }
|
|
57
|
+
warn() { echo "[hook:$(hook_name)] $*" >&2; }
|
|
58
|
+
error() { echo "[hook:$(hook_name)] ERROR: $*" >&2; }
|
|
59
|
+
|
|
60
|
+
session_db_recovery_report_path() {
|
|
61
|
+
local path="${EVIDENT_SESSION_DB_RECOVERY_REPORT:-}"
|
|
62
|
+
path="${path#"${path%%[![:space:]]*}"}"
|
|
63
|
+
path="${path%"${path##*[![:space:]]}"}"
|
|
64
|
+
if [ -n "${path}" ]; then
|
|
65
|
+
printf '%s\n' "${path}"
|
|
66
|
+
return 0
|
|
67
|
+
fi
|
|
68
|
+
local home="${HOME:-}"
|
|
69
|
+
home="${home#"${home%%[![:space:]]*}"}"
|
|
70
|
+
home="${home%"${home##*[![:space:]]}"}"
|
|
71
|
+
[ -n "${home}" ] || return 1
|
|
72
|
+
printf '%s\n' "${home}/.local/state/evident/session-db-recovery.jsonl"
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
report_session_db_giveup() {
|
|
76
|
+
local reason="$1" exit_code="${2:-null}" outcome path at
|
|
77
|
+
case "${reason}" in
|
|
78
|
+
synchroniser_config_unresolved|synchroniser_config_unevaluable|synchroniser_config_incomplete|litestream_config_unavailable|classification_fatal) outcome="restore_misconfigured" ;;
|
|
79
|
+
restore_deadline_exceeded|restore_tool_unusable|classification_unrecognised) outcome="fresh_session_db" ;;
|
|
80
|
+
*) warn "SESSION-DB-RECOVERY-REPORT-UNKNOWN-REASON: ${reason}"; return 0 ;;
|
|
81
|
+
esac
|
|
82
|
+
if ! path="$(session_db_recovery_report_path)"; then
|
|
83
|
+
warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
|
|
84
|
+
return 0
|
|
85
|
+
fi
|
|
86
|
+
at="${EPOCHREALTIME:-}"
|
|
87
|
+
[ -n "${at}" ] || warn "SESSION-DB-RECOVERY-REPORT-TIMESTAMP-UNAVAILABLE"
|
|
88
|
+
printf '{"v":1,"event":"session_db_recovery","at":"%s","stage":"restore","outcome":"%s","severity":"error","reason":"%s","litestream_exit_code":%s,"attempt":null,"replica_objects":null,"replica_bytes":null,"quarantine_destination":null,"quarantined_objects":null,"quarantine_failed_objects":null,"quarantined_bytes":null,"verified_restore_point":null,"restore_points_tried":null,"replication_suspended":true}\n' "${at}" "${outcome}" "${reason}" "${exit_code}" >>"${path}" || warn "SESSION-DB-RECOVERY-REPORT-APPEND-FAILED"
|
|
89
|
+
return 0
|
|
90
|
+
}
|
|
56
91
|
|
|
57
92
|
# Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
|
|
58
93
|
# corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
|
|
59
94
|
# `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
|
|
60
|
-
# unusable)/32 (retry)
|
|
95
|
+
# unusable)/32 (retry), extended by `session-db-verify`'s 33 (integrity
|
|
96
|
+
# exhausted, replica separated and local disposed) / 34 (could not prove
|
|
97
|
+
# separation or disposal) — see `runner/synchroniser/src/cli.ts`'s own
|
|
61
98
|
# comment for what each means, not restated here. Any OTHER non-zero status
|
|
62
99
|
# means the tool itself broke, which is the only case worth an ERROR here —
|
|
63
100
|
# EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
|
|
@@ -83,7 +120,7 @@ run_synchroniser() {
|
|
|
83
120
|
[ -n "${deadline}" ] && launcher=(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${deadline}")
|
|
84
121
|
"${launcher[@]}" "${SYNCHRONISER}" "${call_args[@]}" || rc=$?
|
|
85
122
|
case "${rc}" in
|
|
86
|
-
0 | 10 | 30 | 31 | 32) ;;
|
|
123
|
+
0 | 10 | 30 | 31 | 32 | 33 | 34) ;;
|
|
87
124
|
124 | 137)
|
|
88
125
|
[ -n "${deadline}" ] || error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw"
|
|
89
126
|
;;
|
|
@@ -171,7 +208,7 @@ timed_synchroniser() {
|
|
|
171
208
|
}
|
|
172
209
|
|
|
173
210
|
# Exports what `runner-synchroniser` resolves its object-store location from
|
|
174
|
-
# (
|
|
211
|
+
# (runner/synchroniser/src/config.ts). It treats either being empty as
|
|
175
212
|
# "persistence disabled" and then reports every restore as a WARNING it still
|
|
176
213
|
# exits 0 for — so an unset value has to be caught HERE, where it can still be
|
|
177
214
|
# told apart from "the store is simply empty".
|
|
@@ -211,13 +248,13 @@ load_state_config() {
|
|
|
211
248
|
# `model-auth-ready` therefore stays a boot-time diagnostic: its only output is
|
|
212
249
|
# a log line, since the runtime forwards no hook stderr to the doorbell caller.
|
|
213
250
|
|
|
214
|
-
# How long restore_credentials' shared step budget is, covering
|
|
215
|
-
# its synchroniser calls (restore claude, restore opencode,
|
|
216
|
-
# as one window rather than a timeout apiece — a fixed per-call cap at the same
|
|
251
|
+
# How long restore_credentials' shared step budget is, covering the runner-secret
|
|
252
|
+
# fetch plus its three synchroniser calls (restore claude, restore opencode,
|
|
253
|
+
# model-auth-ready) as one window rather than a timeout apiece — a fixed per-call cap at the same
|
|
217
254
|
# total would false-fire on the ordinary case of one slow call (a slow S3 GET),
|
|
218
255
|
# which is precisely the boot this exists to keep healthy. From #930's 3-boot
|
|
219
|
-
# sample: the step measured ~3-5s total (~1.2s per call, three
|
|
220
|
-
# starts of a 1.8 MB bundle) — a WEAK estimate this file's own
|
|
256
|
+
# sample: the step measured ~3-5s total (~1.2s per synchroniser call, three
|
|
257
|
+
# node cold starts of a 1.8 MB bundle) — a WEAK estimate this file's own
|
|
221
258
|
# SYNCHRONISER-TIMING lines are what will sharpen for real. Too tight and a
|
|
222
259
|
# routine slow call boots this VM with no model credentials until an operator
|
|
223
260
|
# reconnects it; too loose and a hung call burns more of the hook's own
|
|
@@ -229,6 +266,7 @@ CREDENTIAL_RESTORE_DEADLINE_SECONDS="${EVIDENT_CREDENTIAL_RESTORE_DEADLINE_SECON
|
|
|
229
266
|
# and a call that ignored it would be unbounded again). Hard ceiling on the
|
|
230
267
|
# step: DEADLINE + this = 10s, once — not per call.
|
|
231
268
|
CREDENTIAL_RESTORE_KILL_GRACE_SECONDS=2
|
|
269
|
+
GITHUB_PROBE_DEADLINE_SECONDS="${EVIDENT_GITHUB_PROBE_DEADLINE_SECONDS:-10}"
|
|
232
270
|
|
|
233
271
|
# What is left of the shared step budget, in whole seconds, `step_started_s`
|
|
234
272
|
# seconds after it began. `SECONDS` (a bash builtin with no failure mode,
|
|
@@ -268,12 +306,64 @@ bounded_restore_call() {
|
|
|
268
306
|
return 0
|
|
269
307
|
}
|
|
270
308
|
|
|
309
|
+
fetch_runner_secret() {
|
|
310
|
+
local step_started_s="$1" remaining rc=0 payload stderr_file started_ms populated=0 skipped=0 key value
|
|
311
|
+
if [ -z "${RUNNER_SECRET_ARN:-}" ]; then
|
|
312
|
+
log "runner secret is not configured; continuing without GitHub and MCP credentials"
|
|
313
|
+
return 0
|
|
314
|
+
fi
|
|
315
|
+
remaining="$(remaining_credential_budget "${step_started_s}")"
|
|
316
|
+
if [ "${remaining}" -lt 1 ]; then
|
|
317
|
+
warn "CREDENTIAL-RESTORE-SKIPPED: runner-secret skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
|
|
318
|
+
return 0
|
|
319
|
+
fi
|
|
320
|
+
if ! stderr_file="$(mktemp /dev/shm/runner-secret-stderr.XXXXXX)"; then
|
|
321
|
+
warn "RUNNER-SECRET-STDERR-UNAVAILABLE: could not allocate diagnostic storage; continuing without runner credentials"
|
|
322
|
+
return 0
|
|
323
|
+
fi
|
|
324
|
+
started_ms="$(now_ms)"
|
|
325
|
+
payload="$(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${remaining}" aws secretsmanager get-secret-value --secret-id "${RUNNER_SECRET_ARN}" --query SecretString --output text 2>"${stderr_file}")" || rc=$?
|
|
326
|
+
log_elapsed_since runner-secret-fetch "${started_ms}" "${rc}"
|
|
327
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then
|
|
328
|
+
warn "CREDENTIAL-RESTORE-TIMEOUT: runner-secret did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
|
|
329
|
+
rm -f "${stderr_file}"
|
|
330
|
+
return 0
|
|
331
|
+
fi
|
|
332
|
+
if [ "${rc}" -ne 0 ]; then
|
|
333
|
+
warn "RUNNER-SECRET-UNREADABLE: $(<"${stderr_file}")"
|
|
334
|
+
rm -f "${stderr_file}"
|
|
335
|
+
return 0
|
|
336
|
+
fi
|
|
337
|
+
rm -f "${stderr_file}"
|
|
338
|
+
if ! jq -e 'type == "object"' >/dev/null 2>&1 <<<"${payload}"; then
|
|
339
|
+
warn "RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object"
|
|
340
|
+
return 0
|
|
341
|
+
fi
|
|
342
|
+
while IFS= read -r -d '' key && IFS= read -r -d '' value; do
|
|
343
|
+
if [[ ! "${key}" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
|
|
344
|
+
warn "RUNNER-SECRET-KEY-SKIPPED: ${key@Q} is not a valid environment variable name"
|
|
345
|
+
skipped=$((skipped + 1))
|
|
346
|
+
continue
|
|
347
|
+
fi
|
|
348
|
+
export "${key}=${value}"
|
|
349
|
+
populated=$((populated + 1))
|
|
350
|
+
done < <(jq -j 'to_entries[] | select(.value | type == "string" and length > 0) | .key, "\u0000", .value, "\u0000"' <<<"${payload}")
|
|
351
|
+
if [ "${populated}" -eq 0 ]; then
|
|
352
|
+
warn "RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md"
|
|
353
|
+
else
|
|
354
|
+
log "RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names"
|
|
355
|
+
fi
|
|
356
|
+
return 0
|
|
357
|
+
}
|
|
358
|
+
|
|
271
359
|
restore_credentials() {
|
|
272
360
|
local step_started_ms step_started_s
|
|
273
361
|
step_started_ms="$(now_ms)"
|
|
274
362
|
step_started_s="${SECONDS}"
|
|
275
363
|
|
|
276
364
|
load_state_config || return 1
|
|
365
|
+
# This shares the existing bounded window so /run's worst-case duration does not grow.
|
|
366
|
+
fetch_runner_secret "${step_started_s}"
|
|
277
367
|
bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
|
|
278
368
|
bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
|
|
279
369
|
|
|
@@ -297,7 +387,7 @@ restore_credentials() {
|
|
|
297
387
|
(neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither \
|
|
298
388
|
ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs \
|
|
299
389
|
a model provider fails until one is connected. See 'Seeding a credential store' in \
|
|
300
|
-
infrastructure/evident-
|
|
390
|
+
infrastructure/evident-runner/MICROVM.md."
|
|
301
391
|
;;
|
|
302
392
|
124 | 137)
|
|
303
393
|
warn "CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within its ${auth_remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
|
|
@@ -313,7 +403,7 @@ infrastructure/evident-microvm/README.md."
|
|
|
313
403
|
fi
|
|
314
404
|
# The step total, in the SAME greppable shape as the per-call lines above, so
|
|
315
405
|
# one query answers both "what does credential restore cost?" and "which of
|
|
316
|
-
# its
|
|
406
|
+
# its four operations cost it". Bounded now by CREDENTIAL_RESTORE_DEADLINE_SECONDS
|
|
317
407
|
# + CREDENTIAL_RESTORE_KILL_GRACE_SECONDS as one shared window (see the
|
|
318
408
|
# comment above that constant) rather than per call. `rc=0` is deliberate
|
|
319
409
|
# here, not an absence of failure modes: a timed-out call already warned by
|
|
@@ -323,6 +413,55 @@ infrastructure/evident-microvm/README.md."
|
|
|
323
413
|
return 0
|
|
324
414
|
}
|
|
325
415
|
|
|
416
|
+
apply_runner_opencode_config() {
|
|
417
|
+
if [ -z "${RUNNER_OPENCODE_CONFIG:-}" ]; then
|
|
418
|
+
log "runner OpenCode config is not configured; using the baked project config"
|
|
419
|
+
return 0
|
|
420
|
+
fi
|
|
421
|
+
local source="${RUNNER_OPENCODE_CONFIG}" target="opencode.json"
|
|
422
|
+
[[ "${source}" = /* ]] || source="${WORKSPACE}/${source}"
|
|
423
|
+
[ -f "${WORKSPACE}/opencode.jsonc" ] && target="opencode.jsonc"
|
|
424
|
+
if [ ! -f "${source}" ]; then
|
|
425
|
+
error "RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)"
|
|
426
|
+
return 0
|
|
427
|
+
fi
|
|
428
|
+
cp "${source}" "${WORKSPACE}/${target}"
|
|
429
|
+
git -C "${WORKSPACE}" update-index --skip-worktree "${target}" 2>/dev/null \
|
|
430
|
+
|| warn "could not mark ${target} skip-worktree; it may show as a local change"
|
|
431
|
+
log "Applied runner OpenCode config ${source} to ${WORKSPACE}/${target}"
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
configure_github_access() {
|
|
435
|
+
if [ -z "${GH_TOKEN:-}" ]; then
|
|
436
|
+
warn "GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above"
|
|
437
|
+
return 0
|
|
438
|
+
fi
|
|
439
|
+
export GIT_CONFIG_GLOBAL=/tmp/gitconfig
|
|
440
|
+
if ! : >"${GIT_CONFIG_GLOBAL}" ||
|
|
441
|
+
! git config --global user.name "${GIT_USER_NAME:-evident-bot}" ||
|
|
442
|
+
! git config --global user.email "${GIT_USER_EMAIL:-evident-bot@users.noreply.github.com}" ||
|
|
443
|
+
! git config --global init.defaultBranch main ||
|
|
444
|
+
! printf '%s\n' '#!/usr/bin/env bash' '[ "$1" = get ] || exit 0' 'echo username=x-access-token' 'echo "password=${GH_TOKEN}"' >/tmp/git-credential-helper.sh ||
|
|
445
|
+
! chmod 0700 /tmp/git-credential-helper.sh ||
|
|
446
|
+
! git config --global credential."https://github.com".helper /tmp/git-credential-helper.sh; then
|
|
447
|
+
warn "GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access"
|
|
448
|
+
return 0
|
|
449
|
+
fi
|
|
450
|
+
(
|
|
451
|
+
local output rc=0 login repo_url repo
|
|
452
|
+
output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api user --jq .login 2>&1)" || rc=$?
|
|
453
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
|
|
454
|
+
if [ "${rc}" -ne 0 ]; then warn "GITHUB-AUTH-REJECTED: ${output}"; return; fi
|
|
455
|
+
log "GITHUB-AUTH-OK: ${output}"
|
|
456
|
+
repo_url="$(git -C "${WORKSPACE}" remote get-url origin 2>/dev/null || true)"
|
|
457
|
+
repo="$(printf '%s' "${repo_url}" | sed -E 's#(https://github.com/|git@github.com:)##; s#\.git$##')"
|
|
458
|
+
[ -n "${repo}" ] || return
|
|
459
|
+
output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api "repos/${repo}" --jq .full_name 2>&1)" || rc=$?
|
|
460
|
+
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
|
|
461
|
+
[ "${rc}" -eq 0 ] || warn "GITHUB-REPO-INACCESSIBLE: ${repo}: ${output}"
|
|
462
|
+
) &
|
|
463
|
+
}
|
|
464
|
+
|
|
326
465
|
# Best-effort by design: /suspend must still drop the tunnel and /terminate must
|
|
327
466
|
# still clean up, so a flush that cannot happen is loud but never fatal.
|
|
328
467
|
sync_credentials() {
|
|
@@ -337,7 +476,7 @@ sync_credentials() {
|
|
|
337
476
|
# Writes NOTHING to S3 itself; replication back to S3 is started separately by
|
|
338
477
|
# `start_litestream` (below) from the `run`/`resume` hooks. `ensure_litestream_config`/
|
|
339
478
|
# `restore_session_db` are the MicroVM side of the same contract
|
|
340
|
-
#
|
|
479
|
+
# runner/docker-images/fargate/entrypoint.sh's inlined restore already speaks, going
|
|
341
480
|
# through the SAME runner-synchroniser CLI.
|
|
342
481
|
|
|
343
482
|
# Reads the ~30 MB litestream binary into the page cache, in the background, so
|
|
@@ -491,6 +630,24 @@ restore_session_db() {
|
|
|
491
630
|
# suspend/resume and a /run retry, so a marker left by an earlier boot must
|
|
492
631
|
# never silently disable replication for the rest of this VM's life.
|
|
493
632
|
rm -f "${SESSION_DB_NO_REPLICATE_MARKER}"
|
|
633
|
+
local report_path
|
|
634
|
+
if report_path="$(session_db_recovery_report_path)"; then
|
|
635
|
+
# The default report directory is provisioned in the image, so this is a
|
|
636
|
+
# no-op there; a shared-path override (EVIDENT_SESSION_DB_RECOVERY_REPORT)
|
|
637
|
+
# can name a directory that isn't, and the typed synchroniser writer
|
|
638
|
+
# already creates it for that same override (session-db-recovery-report.ts).
|
|
639
|
+
# Best-effort only: report_session_db_giveup below still warns and
|
|
640
|
+
# continues if this didn't leave a writable path. `${path%/*}` leaves a
|
|
641
|
+
# bare relative filename (no `/`) unchanged rather than reducing to `.`
|
|
642
|
+
# like dirname(1) — without the explicit check, mkdir would create a
|
|
643
|
+
# directory AT the report path itself.
|
|
644
|
+
local report_dir="${report_path%/*}"
|
|
645
|
+
[ "${report_dir}" = "${report_path}" ] && report_dir="."
|
|
646
|
+
[ -d "${report_dir}" ] || mkdir -p "${report_dir}" 2>/dev/null || true
|
|
647
|
+
: >"${report_path}" || warn "SESSION-DB-RECOVERY-REPORT-TRUNCATE-FAILED"
|
|
648
|
+
else
|
|
649
|
+
warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
|
|
650
|
+
fi
|
|
494
651
|
|
|
495
652
|
# Config resolved ONCE here, in the synchroniser (config.ts owns
|
|
496
653
|
# OPENCODE_DB_PATH; no second copy of that path here), exactly as
|
|
@@ -507,6 +664,7 @@ restore_session_db() {
|
|
|
507
664
|
log_elapsed_since session-db-env "${env_started_ms}" "${env_rc}"
|
|
508
665
|
if [ "${env_rc}" -ne 0 ]; then
|
|
509
666
|
mark_no_replicate "could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
667
|
+
report_session_db_giveup synchroniser_config_unresolved
|
|
510
668
|
return 0
|
|
511
669
|
fi
|
|
512
670
|
# Guarded for the SAME reason as the substitution above, which is easy to miss:
|
|
@@ -517,10 +675,11 @@ restore_session_db() {
|
|
|
517
675
|
# it is the whole lifecycle transition, so Q3 makes it a give-up instead.
|
|
518
676
|
eval "${synchroniser_env}" || {
|
|
519
677
|
mark_no_replicate "the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
678
|
+
report_session_db_giveup synchroniser_config_unevaluable
|
|
520
679
|
return 0
|
|
521
680
|
}
|
|
522
681
|
|
|
523
|
-
# #931's exact lesson, one shell over (
|
|
682
|
+
# #931's exact lesson, one shell over (runner/docker-images/fargate/entrypoint.sh):
|
|
524
683
|
# the `|| { ... }` above only catches a non-zero EXIT — an `env` that exits
|
|
525
684
|
# 0 with an INCOMPLETE contract (a runner-synchroniser version/build skew)
|
|
526
685
|
# would otherwise abort right here under `set -u` the moment
|
|
@@ -531,6 +690,7 @@ restore_session_db() {
|
|
|
531
690
|
# be actively wrong, not merely absent).
|
|
532
691
|
if [ -z "${OPENCODE_DB_PATH+x}" ]; then
|
|
533
692
|
mark_no_replicate "run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
693
|
+
report_session_db_giveup synchroniser_config_incomplete
|
|
534
694
|
return 0
|
|
535
695
|
fi
|
|
536
696
|
|
|
@@ -548,6 +708,7 @@ restore_session_db() {
|
|
|
548
708
|
|
|
549
709
|
if ! ensure_litestream_config; then
|
|
550
710
|
mark_no_replicate "could not generate ${LITESTREAM_CONFIG_FILE} (see the error above)"
|
|
711
|
+
report_session_db_giveup litestream_config_unavailable
|
|
551
712
|
return 0
|
|
552
713
|
fi
|
|
553
714
|
|
|
@@ -566,18 +727,20 @@ restore_session_db() {
|
|
|
566
727
|
124 | 137)
|
|
567
728
|
discard_session_db_debris
|
|
568
729
|
mark_no_replicate "SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within the ${SESSION_DB_RESTORE_DEADLINE_SECONDS}s deadline (+${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}s kill grace), ${SECONDS}s into the hook; opencode starts with a fresh session DB and nothing is replicated this boot"
|
|
730
|
+
report_session_db_giveup restore_deadline_exceeded "${restore_rc}"
|
|
569
731
|
return 0
|
|
570
732
|
;;
|
|
571
733
|
125 | 126 | 127)
|
|
572
734
|
error "litestream restore could not even run (timeout exited ${restore_rc})"
|
|
573
735
|
discard_session_db_debris
|
|
574
736
|
mark_no_replicate "restore tool is broken (timeout exited ${restore_rc}); opencode starts with a fresh session DB and nothing is replicated this boot"
|
|
737
|
+
report_session_db_giveup restore_tool_unusable "${restore_rc}"
|
|
575
738
|
return 0
|
|
576
739
|
;;
|
|
577
740
|
esac
|
|
578
741
|
|
|
579
742
|
local classify_rc=0
|
|
580
|
-
run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave || classify_rc=$?
|
|
743
|
+
run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave --fresh-db-fallback || classify_rc=$?
|
|
581
744
|
case "${classify_rc}" in
|
|
582
745
|
0) ;; # restored, or no replica yet — the CLI already logged which
|
|
583
746
|
31)
|
|
@@ -587,8 +750,8 @@ restore_session_db() {
|
|
|
587
750
|
# half-restored DB, so the marker stays; a 31 is structurally guaranteed
|
|
588
751
|
# to be a FRESH one, and replicating it starts a new backup chain instead
|
|
589
752
|
# of leaving this boot with a zero-width backup window. Same reasoning,
|
|
590
|
-
# and same one-line change, as
|
|
591
|
-
warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and
|
|
753
|
+
# and same one-line change, as runner/docker-images/fargate/entrypoint.sh's `31)`.
|
|
754
|
+
warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot (see the WARNING above)"
|
|
592
755
|
;;
|
|
593
756
|
32)
|
|
594
757
|
# The CLI's own contract for 32 is "re-run litestream restore and ask
|
|
@@ -602,11 +765,13 @@ restore_session_db() {
|
|
|
602
765
|
# fresh rather than fail /run.
|
|
603
766
|
discard_session_db_debris
|
|
604
767
|
mark_no_replicate "session-db-classify returned fatal (30); see the FATAL message above"
|
|
768
|
+
report_session_db_giveup classification_fatal
|
|
605
769
|
;;
|
|
606
770
|
*)
|
|
607
771
|
# run_synchroniser already logged the "tool broke" ERROR for this.
|
|
608
772
|
discard_session_db_debris
|
|
609
773
|
mark_no_replicate "session-db-classify exited ${classify_rc}, which is none of its documented answers"
|
|
774
|
+
report_session_db_giveup classification_unrecognised
|
|
610
775
|
;;
|
|
611
776
|
esac
|
|
612
777
|
|
|
@@ -614,6 +779,99 @@ restore_session_db() {
|
|
|
614
779
|
}
|
|
615
780
|
# --- Session DB restore (end) -----------------------------------------------
|
|
616
781
|
|
|
782
|
+
# --- Session DB integrity verification (#1868 WI-4) --------------------------
|
|
783
|
+
#
|
|
784
|
+
# The restore loop above only proves litestream could WRITE a file; it never
|
|
785
|
+
# proves the file is intact (#1345: a leaf-page-corrupt-but-openable DB
|
|
786
|
+
# re-restores unchanged forever). `session-db-verify` runs a real `PRAGMA
|
|
787
|
+
# integrity_check` and, on failure, walks retained restore points back until
|
|
788
|
+
# one passes — the MicroVM side of the identical check
|
|
789
|
+
# runner/docker-images/fargate/entrypoint.sh already runs after ITS restore
|
|
790
|
+
# loop, through the same runner-synchroniser CLI.
|
|
791
|
+
|
|
792
|
+
# Bounded by `timed_synchroniser`, not an external `timeout` around
|
|
793
|
+
# `run_synchroniser` itself (a shell function, not an exported binary — that
|
|
794
|
+
# would fail with rc 127, silently mis-triggering the allowlist's "tool
|
|
795
|
+
# broke" branch). The grace period is `CREDENTIAL_RESTORE_KILL_GRACE_SECONDS`
|
|
796
|
+
# (2s): `run_synchroniser`'s `timeout -k` hardcodes that one constant for
|
|
797
|
+
# every `timed_synchroniser` caller, not a value unique to this step, so the
|
|
798
|
+
# real worst case is DEADLINE + 2s, once — accounted for in
|
|
799
|
+
# hook-scripts.test.ts's budget-ladder test.
|
|
800
|
+
#
|
|
801
|
+
# 4s, NOT measured on this platform: #930's synchroniser-call sample put a
|
|
802
|
+
# single call at ~1.2s (a node cold start), and this ONE call also runs a
|
|
803
|
+
# PRAGMA integrity_check that scales with the restored DB's size, which #930
|
|
804
|
+
# never exercised. The 124/137 branch below is the DESIGNED-FOR outcome on a
|
|
805
|
+
# large DB, not an edge case — SYNCHRONISER-TIMING op=session-db-verify is
|
|
806
|
+
# what should actually size this once real boots report it.
|
|
807
|
+
SESSION_DB_VERIFY_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_VERIFY_DEADLINE_SECONDS:-4}"
|
|
808
|
+
|
|
809
|
+
verify_session_db() {
|
|
810
|
+
# Guards mirror start_litestream's first three, in the same order and for
|
|
811
|
+
# the same reason: verifying a DB this boot already decided not to
|
|
812
|
+
# replicate (or never restored at all) changes nothing about how /run
|
|
813
|
+
# proceeds, and would spend budget only to report on a moot outcome.
|
|
814
|
+
if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
|
|
815
|
+
log "skipping session-DB verification: persistence is disabled"
|
|
816
|
+
return 0
|
|
817
|
+
fi
|
|
818
|
+
|
|
819
|
+
if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
|
|
820
|
+
log "skipping session-DB verification: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
|
|
821
|
+
return 0
|
|
822
|
+
fi
|
|
823
|
+
|
|
824
|
+
if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
|
|
825
|
+
log "skipping session-DB verification: no usable ${LITESTREAM_CONFIG_FILE}"
|
|
826
|
+
return 0
|
|
827
|
+
fi
|
|
828
|
+
|
|
829
|
+
# Exported below this step's own deadline so the CLI's in-process walkback
|
|
830
|
+
# loop gives up on its own before the process-group kill lands — its 180s
|
|
831
|
+
# default (config.ts) is far outside this one step's slice of the hook's
|
|
832
|
+
# SIGTERM budget.
|
|
833
|
+
local verify_rc=0
|
|
834
|
+
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS="${SESSION_DB_VERIFY_DEADLINE_SECONDS}" \
|
|
835
|
+
timed_synchroniser session-db-verify "${SESSION_DB_VERIFY_DEADLINE_SECONDS}" \
|
|
836
|
+
session-db-verify "${LITESTREAM_CONFIG_FILE}" || verify_rc=$?
|
|
837
|
+
|
|
838
|
+
case "${verify_rc}" in
|
|
839
|
+
0) ;; # verified intact, or nothing to verify yet — the CLI already logged which
|
|
840
|
+
33)
|
|
841
|
+
# Integrity exhausted, but the classifier already proved the corrupt
|
|
842
|
+
# replica separated and the local copy disposed of — booting with a
|
|
843
|
+
# fresh DB and a new backup chain is safe, exactly like ECS's own `33)`.
|
|
844
|
+
log "SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated."
|
|
845
|
+
;;
|
|
846
|
+
34)
|
|
847
|
+
# The ONE deliberate exception to "nothing about the session DB may
|
|
848
|
+
# ever fail /run" (Q3): a 34 means separation/disposal could NOT be
|
|
849
|
+
# proven, so continuing would hand opencode a DB it may not be safe to
|
|
850
|
+
# open or write — the same evidence-quality bar `check_runner_key`
|
|
851
|
+
# already applies one step earlier in this hook (contrary evidence,
|
|
852
|
+
# not absent evidence, is what's fatal). The CLI already logged
|
|
853
|
+
# SESSION-DB-REPLICA-SEPARATION-UNVERIFIED / SESSION-DB-LOCAL-DISCARD-FAILED.
|
|
854
|
+
error "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk, so nothing will be started (see the ERROR above)."
|
|
855
|
+
return 1
|
|
856
|
+
;;
|
|
857
|
+
124 | 137)
|
|
858
|
+
# The designed-for outcome on a large DB (see the deadline comment
|
|
859
|
+
# above), not a broken tool: continue with the restored DB exactly as
|
|
860
|
+
# ECS's own `*)` branch does for an unexpected code.
|
|
861
|
+
warn "SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_DEADLINE_SECONDS}s deadline; continuing with the restored opencode.db as-is, unverified"
|
|
862
|
+
;;
|
|
863
|
+
*)
|
|
864
|
+
# run_synchroniser already logged the "tool broke" ERROR for this. A
|
|
865
|
+
# broken verifier must not turn a boot that works today into a
|
|
866
|
+
# crash-loop.
|
|
867
|
+
warn "SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${verify_rc}, which is none of its documented answers; continuing with the restored opencode.db as-is"
|
|
868
|
+
;;
|
|
869
|
+
esac
|
|
870
|
+
|
|
871
|
+
return 0
|
|
872
|
+
}
|
|
873
|
+
# --- Session DB integrity verification (end) ---------------------------------
|
|
874
|
+
|
|
617
875
|
# `kill -0` answers "does this pid exist", which is not the question any caller
|
|
618
876
|
# here is asking. A process that has exited but has not been reaped — a zombie —
|
|
619
877
|
# still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
|
|
@@ -706,6 +964,7 @@ regenerate_machine_id() {
|
|
|
706
964
|
tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
|
|
707
965
|
opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
|
|
708
966
|
litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
|
|
967
|
+
creds_sync_is_running() { is_running "${CREDS_SYNC_PID_FILE}"; }
|
|
709
968
|
|
|
710
969
|
# `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
|
|
711
970
|
# and an interpolation of a missing field is still a non-empty string, so a
|
|
@@ -926,6 +1185,155 @@ kill_litestream() {
|
|
|
926
1185
|
}
|
|
927
1186
|
# --- litestream replicate (end) ----------------------------------------------
|
|
928
1187
|
|
|
1188
|
+
# --- credential sync loop (#1868 WI-3, ECS parity) ---------------------------
|
|
1189
|
+
#
|
|
1190
|
+
# sync_credentials (above) covers the three boundary flushes /run's restore,
|
|
1191
|
+
# /suspend and /terminate already call. What it does NOT cover is a VM that
|
|
1192
|
+
# runs for a long time between those boundaries: a provider re-authenticated
|
|
1193
|
+
# through the proxied UI hours into a run would sit unflushed until the next
|
|
1194
|
+
# suspend/terminate, and a VM that dies without one (a crash, an OOM kill)
|
|
1195
|
+
# loses everything since boot. runner/docker-images/fargate/entrypoint.sh's
|
|
1196
|
+
# own sync_credentials_loop is the ECS side of the identical gap; this is the
|
|
1197
|
+
# same fix, backgrounded the same way as start_opencode/start_litestream so it
|
|
1198
|
+
# outlives this hook process, `( … ) &` rather than `setsid`: a plain
|
|
1199
|
+
# backgrounded subshell is reparented to init and keeps running once its
|
|
1200
|
+
# parent hook script exits (verified: PPID=1, still alive, with no controlling
|
|
1201
|
+
# terminal in this image to send it a stray SIGHUP), and it inherits every
|
|
1202
|
+
# function this file defines, so it can call run_synchroniser directly with no
|
|
1203
|
+
# re-exec.
|
|
1204
|
+
|
|
1205
|
+
# Bounded confirmation window `stop_credential_sync` polls after signalling the
|
|
1206
|
+
# loop, sized against the SIGTERM budget ladder (#812 WI-4's
|
|
1207
|
+
# hook-scripts.test.ts): /terminate's own steps already use 49 of the 55s
|
|
1208
|
+
# ceiling, leaving 6s of headroom — this matches the *_KILL_GRACE_SECONDS
|
|
1209
|
+
# convention (CREDENTIAL_RESTORE_KILL_GRACE_SECONDS,
|
|
1210
|
+
# SESSION_DB_RESTORE_KILL_GRACE_SECONDS, both 2s) rather than a longer
|
|
1211
|
+
# drain-style wait, since the loop's current child is one fast
|
|
1212
|
+
# `run_synchroniser sync-once` call (#930: ~1.2s measured), not a writer
|
|
1213
|
+
# needing a graceful drain.
|
|
1214
|
+
CREDS_SYNC_STOP_WAIT_SECONDS=2
|
|
1215
|
+
|
|
1216
|
+
# Best-effort per tick, exactly like sync_credentials above: a failed tick
|
|
1217
|
+
# must never end the loop, or a single transient S3 error would silently
|
|
1218
|
+
# disable sync for the rest of the VM's life.
|
|
1219
|
+
start_credential_sync() {
|
|
1220
|
+
if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
|
|
1221
|
+
warn "CREDS-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot."
|
|
1222
|
+
return 0
|
|
1223
|
+
fi
|
|
1224
|
+
|
|
1225
|
+
if creds_sync_is_running; then
|
|
1226
|
+
warn "credential sync loop already running (pid $(cat "${CREDS_SYNC_PID_FILE}")); reusing it"
|
|
1227
|
+
return 0
|
|
1228
|
+
fi
|
|
1229
|
+
|
|
1230
|
+
# CREDS_SYNC_INTERVAL is exported by restore_session_db's `eval "$(run_synchroniser env)"`
|
|
1231
|
+
# on /run (config.ts's own default is 60s), but that eval can fail or be skipped by an
|
|
1232
|
+
# earlier give-up — never leave the loop unbound under set -u for a value with a safe,
|
|
1233
|
+
# named fallback (unlike OPENCODE_DB_PATH, a guessed sync cadence is not actively wrong).
|
|
1234
|
+
local interval="${CREDS_SYNC_INTERVAL:-60}"
|
|
1235
|
+
if [ -z "${CREDS_SYNC_INTERVAL:-}" ]; then
|
|
1236
|
+
warn "CREDS-SYNC-INTERVAL-DEFAULTED: CREDS_SYNC_INTERVAL was not set by run_synchroniser env; using ${interval}s"
|
|
1237
|
+
fi
|
|
1238
|
+
|
|
1239
|
+
rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
1240
|
+
|
|
1241
|
+
(
|
|
1242
|
+
# Releases the fds this subshell inherited from the hook process before
|
|
1243
|
+
# settling in for the VM's whole remaining life: nothing here writes to
|
|
1244
|
+
# them (every synchroniser call already redirects its own), so there is
|
|
1245
|
+
# no reason to keep holding the hook's original stdout/stderr open. A
|
|
1246
|
+
# long-lived process that instead inherited a pipe's write end (a test
|
|
1247
|
+
# harness reading the hook's own output, for one) would keep that pipe
|
|
1248
|
+
# from ever reporting EOF — testing-guide.mdc's own lesson, and the same
|
|
1249
|
+
# reason start_opencode/start_litestream never inherit stdio either. That
|
|
1250
|
+
# redirect also means `warn`/`log`/`error` calls in here go nowhere, so a
|
|
1251
|
+
# failed sync-once is instead recorded to CREDS_SYNC_LAST_ERROR_FILE and
|
|
1252
|
+
# surfaced by stop_credential_sync, which DOES have live stdio.
|
|
1253
|
+
exec >/dev/null 2>&1 </dev/null
|
|
1254
|
+
|
|
1255
|
+
# A TERM this subshell receives (from stop_credential_sync, below) only
|
|
1256
|
+
# kills THIS wrapper by default — its currently-running child (`sleep`,
|
|
1257
|
+
# or a `run_synchroniser sync-once` call) is a separate process that
|
|
1258
|
+
# would otherwise be orphaned and keep running, free to upload STALE
|
|
1259
|
+
# credentials to S3 after the boundary flush that /suspend and
|
|
1260
|
+
# /terminate perform immediately following the stop. Tracking the
|
|
1261
|
+
# current child explicitly and forwarding the signal closes that race.
|
|
1262
|
+
creds_sync_child_pid=""
|
|
1263
|
+
trap 'trap - TERM; [ -n "${creds_sync_child_pid}" ] && kill -TERM "${creds_sync_child_pid}" 2>/dev/null; exit 0' TERM
|
|
1264
|
+
|
|
1265
|
+
while true; do
|
|
1266
|
+
sleep "${interval}" &
|
|
1267
|
+
creds_sync_child_pid=$!
|
|
1268
|
+
wait "${creds_sync_child_pid}" 2>/dev/null
|
|
1269
|
+
creds_sync_child_pid=""
|
|
1270
|
+
|
|
1271
|
+
run_synchroniser sync-once claude &
|
|
1272
|
+
creds_sync_child_pid=$!
|
|
1273
|
+
wait "${creds_sync_child_pid}" 2>/dev/null || echo "claude" >"${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
1274
|
+
creds_sync_child_pid=""
|
|
1275
|
+
|
|
1276
|
+
run_synchroniser sync-once opencode &
|
|
1277
|
+
creds_sync_child_pid=$!
|
|
1278
|
+
wait "${creds_sync_child_pid}" 2>/dev/null || echo "opencode" >"${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
1279
|
+
creds_sync_child_pid=""
|
|
1280
|
+
done
|
|
1281
|
+
) &
|
|
1282
|
+
|
|
1283
|
+
echo $! >"${CREDS_SYNC_PID_FILE}"
|
|
1284
|
+
log "CREDS-SYNC-STARTED: pid=$! interval=${interval}s"
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
# Signals the loop, then confirms (bounded — see CREDS_SYNC_STOP_WAIT_SECONDS)
|
|
1288
|
+
# that it and its current child are actually gone before returning: /suspend
|
|
1289
|
+
# and /terminate start their own boundary flush immediately after this call,
|
|
1290
|
+
# and an orphaned in-flight sync-once surviving past that point can overwrite
|
|
1291
|
+
# fresher credentials with stale ones. The TERM trap inside the loop (above)
|
|
1292
|
+
# forwards the signal to its current child almost instantly — this poll is a
|
|
1293
|
+
# defensive confirmation, not the primary mechanism, so it stays short; a
|
|
1294
|
+
# SIGKILL backstop covers a child that ignores TERM entirely.
|
|
1295
|
+
#
|
|
1296
|
+
# The DIED branch is a liveness report, not a no-op: every recovery/no-op path
|
|
1297
|
+
# must say what it found (development-workflow.mdc) — a stopped-before-called
|
|
1298
|
+
# loop and a died-on-its-own loop are different facts an operator needs told
|
|
1299
|
+
# apart, not the same "nothing to stop" line.
|
|
1300
|
+
stop_credential_sync() {
|
|
1301
|
+
if [ ! -s "${CREDS_SYNC_PID_FILE}" ]; then
|
|
1302
|
+
log "CREDS-SYNC-NOT-RUNNING: no credential sync loop to stop"
|
|
1303
|
+
return 0
|
|
1304
|
+
fi
|
|
1305
|
+
|
|
1306
|
+
local pid
|
|
1307
|
+
pid="$(cat "${CREDS_SYNC_PID_FILE}")"
|
|
1308
|
+
if ! process_is_alive "${pid}"; then
|
|
1309
|
+
rm -f "${CREDS_SYNC_PID_FILE}"
|
|
1310
|
+
warn "CREDS-SYNC-DIED: credential sync loop (pid=${pid}) had already exited before this stop"
|
|
1311
|
+
return 0
|
|
1312
|
+
fi
|
|
1313
|
+
|
|
1314
|
+
kill -TERM "${pid}" 2>/dev/null || true
|
|
1315
|
+
rm -f "${CREDS_SYNC_PID_FILE}"
|
|
1316
|
+
|
|
1317
|
+
local waited_ms=0
|
|
1318
|
+
while process_is_alive "${pid}" && [ "${waited_ms}" -lt $((CREDS_SYNC_STOP_WAIT_SECONDS * 1000)) ]; do
|
|
1319
|
+
sleep 0.1
|
|
1320
|
+
waited_ms=$((waited_ms + 100))
|
|
1321
|
+
done
|
|
1322
|
+
|
|
1323
|
+
if process_is_alive "${pid}"; then
|
|
1324
|
+
kill -KILL "${pid}" 2>/dev/null || true
|
|
1325
|
+
warn "CREDS-SYNC-STOP-TIMEOUT: pid=${pid} still alive after ${CREDS_SYNC_STOP_WAIT_SECONDS}s; sent SIGKILL"
|
|
1326
|
+
fi
|
|
1327
|
+
|
|
1328
|
+
if [ -s "${CREDS_SYNC_LAST_ERROR_FILE}" ]; then
|
|
1329
|
+
warn "CREDS-SYNC-HAD-FAILURES: sync-once failed at least once for: $(tr '\n' ' ' <"${CREDS_SYNC_LAST_ERROR_FILE}")"
|
|
1330
|
+
rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
1331
|
+
fi
|
|
1332
|
+
|
|
1333
|
+
log "CREDS-SYNC-STOPPED: pid=${pid}"
|
|
1334
|
+
}
|
|
1335
|
+
# --- credential sync loop (end) -----------------------------------------------
|
|
1336
|
+
|
|
929
1337
|
# --- flush_session_db (#812 WI-4) -------------------------------------------
|
|
930
1338
|
#
|
|
931
1339
|
# The checked, synchronous flush /suspend and /terminate need before they
|
|
@@ -1018,7 +1426,7 @@ flush_session_db() {
|
|
|
1018
1426
|
# returned its 200, so it is deliberately outside that arithmetic.
|
|
1019
1427
|
#
|
|
1020
1428
|
# EVIDENT_IDLE_TIMEOUT_SECONDS, deliberately the SAME env var name
|
|
1021
|
-
#
|
|
1429
|
+
# runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
|
|
1022
1430
|
# idle-timeout flag, so an operator who knows one knows the other. A
|
|
1023
1431
|
# non-numeric override must never silently DROP the flag — that degrades to
|
|
1024
1432
|
# an always-on VM burning ~$2.40/day, exactly the bug this closes — so it
|
|
@@ -1057,7 +1465,7 @@ start_tunnel() {
|
|
|
1057
1465
|
# into a MicroVM, so it is the ONLY way to seed model credentials on a runner
|
|
1058
1466
|
# that is already up — the S3 credential store is read at /run and /resume,
|
|
1059
1467
|
# never mid-life. The allow-list stays a single directory, as on ECS
|
|
1060
|
-
# (
|
|
1468
|
+
# (runner/docker-images/fargate/entrypoint.sh): the flag is repeatable, but every
|
|
1061
1469
|
# extra entry widens what Evident can write into a VM that executes agent
|
|
1062
1470
|
# code. ${HOME} is set by the image (ENV HOME=/home/runner) and `set -u` makes
|
|
1063
1471
|
# an unset one abort rather than silently allow-list "/.claude".
|
|
@@ -1104,7 +1512,7 @@ CLI_SHUTDOWN_CEILING_SECONDS=32
|
|
|
1104
1512
|
# everything else in the hook. 10 s is ~4x the measured shutdown and gives a
|
|
1105
1513
|
# provable worst case of 10 + the 10 s `suspend` spends in sync_credentials
|
|
1106
1514
|
# before us = 20 s, inside the ~55 s the in-VM server allows this script
|
|
1107
|
-
# (DEFAULT_TIMEOUT_SECONDS,
|
|
1515
|
+
# (DEFAULT_TIMEOUT_SECONDS, aws/lambda-microvm-runtime/src/runtime.ts),
|
|
1108
1516
|
# itself inside AWS's 60 s (HOOK_TIMEOUT_SECONDS, src/constants.ts) —
|
|
1109
1517
|
# overrunning that fails the whole lifecycle transition, which is worse than the
|
|
1110
1518
|
# kill this replaces. The residual cost is a drain longer than 10 s being
|
|
@@ -5,7 +5,10 @@
|
|
|
5
5
|
# session-DB replicator /suspend stopped before the snapshot (#812 WI-4):
|
|
6
6
|
# litestream does not survive a suspend/resume freeze on this design (Q2) —
|
|
7
7
|
# /suspend stops it and /resume starts a fresh one, the same pattern already
|
|
8
|
-
# proven for the tunnel.
|
|
8
|
+
# proven for the tunnel. The interval credential sync loop (#1868 WI-3) is the
|
|
9
|
+
# same story one function over: /suspend stops it too, so a resumed VM that
|
|
10
|
+
# never restarted it here would never sync credentials again for the rest of
|
|
11
|
+
# its life.
|
|
9
12
|
set -euo pipefail
|
|
10
13
|
|
|
11
14
|
# shellcheck source=./common.sh
|
|
@@ -46,6 +49,7 @@ fi
|
|
|
46
49
|
# already-running) handle the rest — this needs no logic of its own.
|
|
47
50
|
load_state_config || warn "could not resolve durable-state config; the session DB will not resume replicating"
|
|
48
51
|
start_litestream
|
|
52
|
+
start_credential_sync
|
|
49
53
|
|
|
50
54
|
# Diagnostic-only, unlike /run's gate: a failed resume costs the user their
|
|
51
55
|
# whole session, so this never exits — it only converts a silent "resumed but
|
|
@@ -61,7 +65,7 @@ start_litestream
|
|
|
61
65
|
# measures in ~2 s (see the waker note in common.sh).
|
|
62
66
|
#
|
|
63
67
|
# Safe to leave running past the hook's own exit: the runtime waits on the
|
|
64
|
-
# script's `'exit'`, not `'close'` (
|
|
68
|
+
# script's `'exit'`, not `'close'` (aws/lambda-microvm-runtime/src/
|
|
65
69
|
# runtime.ts), so a lingering child never delays the HTTP response. Its stdio
|
|
66
70
|
# is deliberately NOT redirected — that inherited fd is how the RUNNER-KEY-*
|
|
67
71
|
# line reaches CloudWatch, which is the whole point of running it at all.
|
|
@@ -28,6 +28,7 @@ cleanup() {
|
|
|
28
28
|
stop_tunnel || warn "stop_tunnel failed while cleaning up"
|
|
29
29
|
stop_opencode || warn "stop_opencode failed while cleaning up"
|
|
30
30
|
kill_litestream || warn "kill_litestream failed while cleaning up"
|
|
31
|
+
stop_credential_sync || warn "stop_credential_sync failed while cleaning up"
|
|
31
32
|
}
|
|
32
33
|
trap cleanup EXIT
|
|
33
34
|
|
|
@@ -79,7 +80,22 @@ check_runner_key "${runner_key}" "${api_url}" || exit 1
|
|
|
79
80
|
restore_session_db
|
|
80
81
|
log "session DB restore done ${SECONDS}s into the hook"
|
|
81
82
|
|
|
82
|
-
# 7 —
|
|
83
|
+
# 7 — integrity-check the restored DB (#1868 WI-4), after the config file
|
|
84
|
+
# exists (verify_session_db reads it) and before anything opens the DB —
|
|
85
|
+
# the only window in which that's true. The ONE step in this sequence that
|
|
86
|
+
# can still fail /run past the runner-key gate: a 34 means the corrupt DB's
|
|
87
|
+
# separation/disposal could not be proven safe (see verify_session_db's own
|
|
88
|
+
# comment for why that's a deliberate exception to "nothing about the
|
|
89
|
+
# session DB may ever fail /run").
|
|
90
|
+
verify_session_db || exit 1
|
|
91
|
+
|
|
92
|
+
# 8 — apply the overlay before OpenCode resolves its project configuration.
|
|
93
|
+
apply_runner_opencode_config
|
|
94
|
+
|
|
95
|
+
# 9 — configure git after credentials are restored and before agent shells start.
|
|
96
|
+
configure_github_access
|
|
97
|
+
|
|
98
|
+
# 10 — opencode. Started here, not at build time: a warm process in the shared
|
|
83
99
|
# snapshot would carry its installation id and database into every VM. Not
|
|
84
100
|
# waited on: a slow opencode boot is not a reason to fail /run (the tunnel CLI
|
|
85
101
|
# auto-starts opencode when it finds none healthy,
|
|
@@ -87,14 +103,19 @@ log "session DB restore done ${SECONDS}s into the hook"
|
|
|
87
103
|
# reclaims a runner that never comes online).
|
|
88
104
|
start_opencode
|
|
89
105
|
|
|
90
|
-
#
|
|
106
|
+
# 11 — begin replicating the session DB (#812 WI-3), now that opencode has
|
|
91
107
|
# opened it and before any work can arrive over the tunnel. Bare, like
|
|
92
108
|
# restore_session_db above: start_litestream never returns non-zero (every
|
|
93
109
|
# guard inside it is its own `return 0`), so there is nothing here for
|
|
94
110
|
# `set -e` to abort on.
|
|
95
111
|
start_litestream
|
|
96
112
|
|
|
97
|
-
#
|
|
113
|
+
# 11a — the interval credential sync (#1868 WI-3), matching ECS's own
|
|
114
|
+
# post-litestream position. Bare for the same reason: every guard inside
|
|
115
|
+
# start_credential_sync is its own `return 0`.
|
|
116
|
+
start_credential_sync
|
|
117
|
+
|
|
118
|
+
# 12 — the first per-VM identity on the wire. The subshell's umask makes the file
|
|
98
119
|
# unreadable to anyone else from the moment it exists, before the key is in it.
|
|
99
120
|
(
|
|
100
121
|
umask 077
|
|
@@ -8,6 +8,9 @@ set -euo pipefail
|
|
|
8
8
|
# shellcheck source=./common.sh
|
|
9
9
|
source "$(dirname "$0")/common.sh"
|
|
10
10
|
|
|
11
|
+
# Stopped BEFORE the boundary flush (#1868 WI-3): the interval loop and this
|
|
12
|
+
# flush must not race each other on the same credential stores.
|
|
13
|
+
stop_credential_sync
|
|
11
14
|
sync_credentials
|
|
12
15
|
stop_tunnel
|
|
13
16
|
|
|
@@ -8,6 +8,9 @@ set -uo pipefail
|
|
|
8
8
|
# shellcheck source=./common.sh
|
|
9
9
|
source "$(dirname "$0")/common.sh"
|
|
10
10
|
|
|
11
|
+
# Stopped BEFORE the boundary flush (#1868 WI-3): the interval loop and this
|
|
12
|
+
# flush must not race each other on the same credential stores.
|
|
13
|
+
stop_credential_sync
|
|
11
14
|
sync_credentials
|
|
12
15
|
stop_tunnel
|
|
13
16
|
|
package/dist/waker/construct.js
CHANGED
|
@@ -56,7 +56,7 @@ class EvidentWaker extends constructs_1.Construct {
|
|
|
56
56
|
runtime: lambda.Runtime.NODEJS_22_X,
|
|
57
57
|
// Pre-bundled at PACKAGE build time (`pnpm --filter @evident-ai/runner-cdk
|
|
58
58
|
// build`, see scripts/build.ts), not at synth time: esbuild inlines
|
|
59
|
-
// @evident/
|
|
59
|
+
// @evident/sdk's HMAC verifier into
|
|
60
60
|
// dist/waker-lambda/handler.js, so neither this package's published npm
|
|
61
61
|
// artifact (which ships dist/ already built) nor a consuming app needs
|
|
62
62
|
// that private workspace package, or esbuild, on synth's PATH. Requires
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@evident-ai/runner-cdk",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.4.1-dev.2f1b44b",
|
|
4
4
|
"description": "Reusable CDK constructs for an Evident agent runner: a single scale-to-zero Fargate runner (task + service + per-agent self-stop role + waker Lambda), or a per-session AWS Lambda MicroVM that boots on demand and suspends between messages. Instantiate once per agent from your own stack.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -15,13 +15,14 @@
|
|
|
15
15
|
],
|
|
16
16
|
"scripts": {
|
|
17
17
|
"build": "pnpm run build-bundled-deps && ts-node scripts/build.ts",
|
|
18
|
-
"//build-bundled-deps": "scripts/build.ts esbuild-bundles
|
|
19
|
-
"build-bundled-deps": "pnpm --filter @evident-ai/lambda-microvm-runtime build",
|
|
18
|
+
"//build-bundled-deps": "scripts/build.ts esbuild-bundles runner/docker-images/microvm/hook-server.ts, which imports @evident-ai/lambda-microvm-runtime by its built `main`. @evident-ai/lambda-microvm-cdk is a workspace:* devDependency whose types resolve through its own dist/, so tsc --project tsconfig.build.json cannot compile src/microvm/{shapes,construct}.ts without it. Chained into `build` rather than left to turbo's `^build` because publish-runner-cdk.yaml and infrastructure/evident-runner's build-runner-cdk-dep invoke `pnpm --filter @evident-ai/runner-cdk build` directly, which bypasses turbo entirely.",
|
|
19
|
+
"build-bundled-deps": "pnpm --filter @evident-ai/lambda-microvm-cdk build && pnpm --filter @evident-ai/lambda-microvm-runtime build",
|
|
20
20
|
"typecheck": "tsc --noEmit",
|
|
21
21
|
"test": "node --test --require ts-node/register 'src/**/*.test.ts'",
|
|
22
22
|
"format": "prettier --write 'src/**/*.ts'",
|
|
23
23
|
"lint": "eslint 'src/**/*.ts' --max-warnings=0"
|
|
24
24
|
},
|
|
25
|
+
"//peerDependencies": "@evident-ai/lambda-microvm-cdk is a registry range here, never workspace:*: publish-runner-cdk.yaml ships this package with `npm publish`, not `pnpm publish`, so pnpm's workspace-protocol substitution never runs and a workspace: range would reach npm literally, breaking every external install. Enforced by src/published-surface.test.ts. The matching devDependency is workspace:* so this monorepo's own build compiles against aws/lambda-microvm-cdk's source.",
|
|
25
26
|
"peerDependencies": {
|
|
26
27
|
"@evident-ai/lambda-microvm-cdk": "^0.1.1",
|
|
27
28
|
"aws-cdk-lib": "^2.240.0",
|
|
@@ -31,16 +32,17 @@
|
|
|
31
32
|
"@aws-sdk/client-ecs": "^3.682.0",
|
|
32
33
|
"@aws-sdk/client-lambda-microvms": "^3.1095.0",
|
|
33
34
|
"@aws-sdk/client-secrets-manager": "^3.682.0",
|
|
34
|
-
"@evident-ai/lambda-microvm-cdk": "
|
|
35
|
+
"@evident-ai/lambda-microvm-cdk": "workspace:*",
|
|
35
36
|
"@evident-ai/lambda-microvm-runtime": "workspace:*",
|
|
36
|
-
"@evident/
|
|
37
|
+
"@evident/sdk": "workspace:*",
|
|
37
38
|
"@types/node": "^22",
|
|
38
39
|
"aws-cdk-lib": "^2.240.0",
|
|
39
40
|
"constructs": "^10.5.0",
|
|
40
41
|
"esbuild": "^0.25.2",
|
|
41
42
|
"prettier": "^3.3.3",
|
|
42
43
|
"ts-node": "^10.9.2",
|
|
43
|
-
"typescript": "^5.6.3"
|
|
44
|
+
"typescript": "^5.6.3",
|
|
45
|
+
"runner-microvm-image": "workspace:*"
|
|
44
46
|
},
|
|
45
47
|
"publishConfig": {
|
|
46
48
|
"access": "public"
|
|
@@ -48,9 +50,9 @@
|
|
|
48
50
|
"repository": {
|
|
49
51
|
"type": "git",
|
|
50
52
|
"url": "https://github.com/sroze/evident.git",
|
|
51
|
-
"directory": "
|
|
53
|
+
"directory": "aws/runner-cdk"
|
|
52
54
|
},
|
|
53
55
|
"homepage": "https://evident.run",
|
|
54
|
-
"author": "Evident <
|
|
56
|
+
"author": "Evident <hello@evident.run>",
|
|
55
57
|
"license": "MIT"
|
|
56
58
|
}
|