@evident-ai/runner-synchroniser 0.1.0 → 0.1.1-dev.0a5718b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -29
- package/dist/cli.js +396 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -15,8 +15,10 @@ the boot script gates on. It also renders the `litestream.yml` that covers the s
|
|
|
15
15
|
database, so both halves read their bucket, prefix and paths from one config module
|
|
16
16
|
instead of two.
|
|
17
17
|
|
|
18
|
-
It is a CLI, not a library
|
|
19
|
-
|
|
18
|
+
It is a CLI, not a library, invoked by two consumers that differ in how they get the
|
|
19
|
+
binary: the Fargate runner image installs the published package and invokes it by name
|
|
20
|
+
(`runner-synchroniser <command>`); the MicroVM still builds it from source and runs the
|
|
21
|
+
bundle directly (`node cli.js <command>`).
|
|
20
22
|
|
|
21
23
|
## Install
|
|
22
24
|
|
|
@@ -30,11 +32,25 @@ npx @evident-ai/runner-synchroniser@dev env
|
|
|
30
32
|
`dist/cli.js` is a single self-contained ESM bundle with **no runtime dependencies** —
|
|
31
33
|
the AWS SDK is bundled at build time, so installing it pulls nothing else in.
|
|
32
34
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
### Versioning
|
|
36
|
+
|
|
37
|
+
The package's version **tracks the product release version**, exactly like
|
|
38
|
+
`@evident-ai/cli`. `latest` is published on a product release, at the release
|
|
39
|
+
version. `dev` is published on every merge to `main` that touches this package or
|
|
40
|
+
the root `package.json`, at `<root major>.<minor>.<patch+1>-dev.<sha>`, so `dev`
|
|
41
|
+
stays strictly ahead of `latest`.
|
|
42
|
+
|
|
43
|
+
The version committed in this package's `package.json` is **vestigial** — CI
|
|
44
|
+
overwrites it before every publish. This is why `latest` jumped straight from
|
|
45
|
+
`0.1.0` to `3.x.y`: the `0.1.x` line predates the policy, not a missing
|
|
46
|
+
major-version story, and because npm publishes are immutable that jump is
|
|
47
|
+
permanent.
|
|
48
|
+
|
|
49
|
+
The two consumers still differ in how they get the binary: the Fargate runner
|
|
50
|
+
image installs the published package
|
|
51
|
+
(`ARG RUNNER_SYNCHRONISER_VERSION=dev`, resolved to a concrete version by the
|
|
52
|
+
deploy workflow); the MicroVM still builds it from source and reads no npm
|
|
53
|
+
version at all.
|
|
38
54
|
|
|
39
55
|
## Commands
|
|
40
56
|
|
|
@@ -47,9 +63,9 @@ the AWS SDK is bundled at build time, so installing it pulls nothing else in.
|
|
|
47
63
|
| `model-auth-ready` | — | `0` = ready, `10` = not ready; other = tool broken |
|
|
48
64
|
| `self-stop` | — | `0` = stopped, `20` = keep the task; other = tool broken |
|
|
49
65
|
|
|
50
|
-
`self-stop` scales this agent's own ECS service to `desiredCount=0` on a clean idle
|
|
51
|
-
|
|
52
|
-
|
|
66
|
+
`self-stop` scales this agent's own ECS service to `desiredCount=0` on a clean idle exit
|
|
67
|
+
(see Configuration for `CLUSTER`/`SERVICE`/`EVIDENT_SELFSTOP_ROLE_ARN`). It exits `0`
|
|
68
|
+
**only** on a confirmed `desiredCount` of 0.
|
|
53
69
|
|
|
54
70
|
Everything the operator needs to read goes to **stderr**, prefixed `[auth-persistence]`,
|
|
55
71
|
so a caller can safely capture stdout. The prefix now covers more than credentials — the
|
|
@@ -96,23 +112,28 @@ real violation also rejects correct rewrites.
|
|
|
96
112
|
|
|
97
113
|
## Configuration
|
|
98
114
|
|
|
99
|
-
|
|
115
|
+
Configuration is environment variables; `src/config.ts` resolves the shared paths, keys
|
|
116
|
+
and settings below once for every command. The three variables marked † are read
|
|
117
|
+
directly by `self-stop` instead — they configure that one command, not a shared path or
|
|
118
|
+
key, so they never appear in `env`'s output or `litestream.yml`.
|
|
119
|
+
|
|
120
|
+
| Variable | Meaning | When absent |
|
|
121
|
+
| ------------------------------------------ | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- |
|
|
122
|
+
| `HOME` | Every credential and database path is derived from it. | Throws; logged `FATAL:`, exits `1`. Affects every command. |
|
|
123
|
+
| `LITESTREAM_BUCKET` / `LITESTREAM_PREFIX` | Object-store bucket and key prefix. | Persistence off (needs **both**): `restore`/`sync-once` no-op, `env` warns. |
|
|
124
|
+
| `AWS_REGION` | Region for the AWS clients (S3, and the ECS/STS calls `self-stop` makes). | Left to the SDK's own resolution. |
|
|
125
|
+
| `CREDS_SYNC_INTERVAL` | Seconds between sync ticks, reported by `env` for the caller's loop. | Falls back to `60`; also on unparseable or non-positive values. |
|
|
126
|
+
| `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` | Presence alone counts as configured model auth (see `model-auth-ready`). | No API-key fallback; `model-auth-ready` then depends solely on the credential files. |
|
|
127
|
+
| `CLUSTER` † / `SERVICE` † | ECS cluster/service `self-stop` scales to `desiredCount=0`. | Warns "cannot self-stop" and exits `20` (keep the task). Ignored by every other command. |
|
|
128
|
+
| `EVIDENT_SELFSTOP_ROLE_ARN` † | Role `self-stop` assumes for its ECS calls. | Optional: falls back to the task role's own credentials. A failed/incomplete assume-role → `20`. |
|
|
100
129
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
| `HOME` | **Required.** Every credential and database path is derived from it; unset is a fatal error. |
|
|
104
|
-
| `LITESTREAM_BUCKET` | Object-store bucket. |
|
|
105
|
-
| `LITESTREAM_PREFIX` | Key prefix within the bucket. |
|
|
106
|
-
| `AWS_REGION` | Region for the S3 client. Unset leaves it to the SDK's own resolution. |
|
|
107
|
-
| `CREDS_SYNC_INTERVAL` | Seconds between sync ticks, reported by `env` for the caller's loop. Defaults to `60`. |
|
|
108
|
-
| `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` | Presence alone counts as configured model auth (see `model-auth-ready`). |
|
|
130
|
+
`packages/runner-image/README.md` documents these same three from the deployment side —
|
|
131
|
+
keep them in sync.
|
|
109
132
|
|
|
110
|
-
**
|
|
111
|
-
|
|
112
|
-
loudly that credentials will be lost when the task is replaced — half a location can
|
|
113
|
-
never produce a malformed key.
|
|
133
|
+
Requiring **both** `LITESTREAM_BUCKET` and `LITESTREAM_PREFIX` (never just one) means
|
|
134
|
+
half a location can never produce a malformed key.
|
|
114
135
|
|
|
115
|
-
The paths and keys that follow from
|
|
136
|
+
The paths and keys that follow from the bucket/prefix:
|
|
116
137
|
|
|
117
138
|
| Store | Local path | Object key |
|
|
118
139
|
| ----------- | ----------------------------------------- | ----------------------------------- |
|
|
@@ -121,7 +142,7 @@ The paths and keys that follow from that:
|
|
|
121
142
|
| OpenCode DB | `$HOME/.local/share/opencode/opencode.db` | `<prefix>/opencode.db` (litestream) |
|
|
122
143
|
|
|
123
144
|
The session database is replicated by litestream itself; this package only generates its
|
|
124
|
-
config. `env` emits `
|
|
145
|
+
config. `env` emits `PERSISTENCE_BUCKET`, `CLAUDE_CREDS`, `OPENCODE_DB_PATH` and
|
|
125
146
|
`CREDS_SYNC_INTERVAL`, each single-quoted so a value containing shell metacharacters is
|
|
126
147
|
inert when `eval`'d.
|
|
127
148
|
|
|
@@ -171,10 +192,11 @@ pnpm lint
|
|
|
171
192
|
|
|
172
193
|
From the repository root, prefix with `pnpm --filter @evident-ai/runner-synchroniser`.
|
|
173
194
|
|
|
174
|
-
The bundle must stay a **single file**:
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
195
|
+
The bundle must stay a **single file**: both consumers need a self-contained
|
|
196
|
+
`dist/cli.js` with no sibling chunks — the npm `files: ["dist"]` install ships
|
|
197
|
+
whatever `dist` contains, and the MicroVM runs `dist/cli.js` directly with no
|
|
198
|
+
`node_modules` alongside it. `tsup.config.ts` disables splitting and bundles the
|
|
199
|
+
AWS SDK so neither consumer needs anything else.
|
|
178
200
|
|
|
179
201
|
The restore/sync/CLI tests need neither AWS nor a real filesystem — rather than shimming
|
|
180
202
|
a fake `aws` binary onto `PATH`, they inject two ports: `InMemoryObjectStore` (in
|
package/dist/cli.js
CHANGED
|
@@ -39606,6 +39606,87 @@ function describeRestoreOutcome(label, path2, outcome) {
|
|
|
39606
39606
|
return assertNever(outcome);
|
|
39607
39607
|
}
|
|
39608
39608
|
}
|
|
39609
|
+
function assertNeverSessionDb(value) {
|
|
39610
|
+
throw new Error(`Unhandled session-DB restore outcome: ${JSON.stringify(value)}`);
|
|
39611
|
+
}
|
|
39612
|
+
function assertNeverUnusableReplicaReason(value) {
|
|
39613
|
+
throw new Error(`Unhandled unusable-replica reason: ${JSON.stringify(value)}`);
|
|
39614
|
+
}
|
|
39615
|
+
function assertNeverFatalCause(value) {
|
|
39616
|
+
throw new Error(`Unhandled fatal cause: ${JSON.stringify(value)}`);
|
|
39617
|
+
}
|
|
39618
|
+
function sessionDbExitCode(outcome) {
|
|
39619
|
+
switch (outcome.kind) {
|
|
39620
|
+
case "disabled":
|
|
39621
|
+
case "restored":
|
|
39622
|
+
case "noReplica":
|
|
39623
|
+
return 0;
|
|
39624
|
+
case "retryTransient":
|
|
39625
|
+
case "recovered":
|
|
39626
|
+
return 32;
|
|
39627
|
+
// re-run `litestream restore` and ask again
|
|
39628
|
+
case "unusableReplica":
|
|
39629
|
+
return 31;
|
|
39630
|
+
// booted with a fresh DB; the shell must skip `litestream replicate`
|
|
39631
|
+
case "fatal":
|
|
39632
|
+
return 30;
|
|
39633
|
+
// shared by a genuine misconfig and a deliberate `crash` choice
|
|
39634
|
+
default:
|
|
39635
|
+
return assertNeverSessionDb(outcome);
|
|
39636
|
+
}
|
|
39637
|
+
}
|
|
39638
|
+
function describeUnusableReplica(reason) {
|
|
39639
|
+
const base = "SESSION-DB-REPLICA-UNUSABLE: booting with a FRESH opencode.db; prior session history is lost until the replica is fixed.";
|
|
39640
|
+
switch (reason) {
|
|
39641
|
+
case "strategy":
|
|
39642
|
+
return `${base} (--on-unusable-replica=leave was selected; nothing in S3 was touched.)`;
|
|
39643
|
+
case "attemptsExhausted":
|
|
39644
|
+
return `${base} (recovery did not make the replica usable after retrying; giving up.)`;
|
|
39645
|
+
case "layoutMismatch":
|
|
39646
|
+
return `${base} (no level-0 LTX object was found under the replica prefix, so --on-unusable-replica=prune had nothing matching its expected layout to delete.)`;
|
|
39647
|
+
case "recoveryFailed":
|
|
39648
|
+
return `${base} (the recovery delete itself failed; see the warning above for the S3 error.)`;
|
|
39649
|
+
case "targetHealthy":
|
|
39650
|
+
return `${base} (--on-unusable-replica=prune found a newest L0 object but could not confirm it is corrupt \u2014 it either passed the LTX structural check or could not be re-read \u2014 so it declined to delete it; nothing in S3 was touched.)`;
|
|
39651
|
+
default:
|
|
39652
|
+
return assertNeverUnusableReplicaReason(reason);
|
|
39653
|
+
}
|
|
39654
|
+
}
|
|
39655
|
+
function describeSessionDbFatal(outcome, config) {
|
|
39656
|
+
switch (outcome.cause) {
|
|
39657
|
+
// Names the bucket/region and the probe error — never the crash flag.
|
|
39658
|
+
case "misconfig":
|
|
39659
|
+
return `FATAL: could not verify the opencode.db replica in S3 is reachable/authorised (bucket=${config.bucket ?? "<unset>"}, region=${config.region ?? "<unset>"}): ${outcome.detail}`;
|
|
39660
|
+
// Names the flag and states nothing was touched — never bucket/region, so an
|
|
39661
|
+
// operator reading this is never sent chasing IAM over their own flag.
|
|
39662
|
+
case "deliberate":
|
|
39663
|
+
return `FATAL: ${outcome.detail}`;
|
|
39664
|
+
default:
|
|
39665
|
+
return assertNeverFatalCause(outcome.cause);
|
|
39666
|
+
}
|
|
39667
|
+
}
|
|
39668
|
+
function describeSessionDbClassification(outcome, config) {
|
|
39669
|
+
switch (outcome.kind) {
|
|
39670
|
+
case "disabled":
|
|
39671
|
+
return "No opencode.db restore attempted; credential persistence is disabled (LITESTREAM_BUCKET/LITESTREAM_PREFIX unset).";
|
|
39672
|
+
case "restored":
|
|
39673
|
+
return "opencode.db restored from the object store.";
|
|
39674
|
+
case "noReplica":
|
|
39675
|
+
return "No opencode.db replica found in the object store; opencode will create a fresh session DB.";
|
|
39676
|
+
case "retryTransient":
|
|
39677
|
+
return "opencode.db replica restore failed (transient); retrying.";
|
|
39678
|
+
case "recovered": {
|
|
39679
|
+
const objects = outcome.deleted.length === 0 ? "no objects (see the warnings above for what failed)" : `${outcome.deleted.length} object${outcome.deleted.length === 1 ? "" : "s"} (${outcome.deleted.join(", ")})`;
|
|
39680
|
+
return `Attempted opencode.db replica recovery via --on-unusable-replica=${outcome.strategy}, deleting ${objects}; retrying the restore.` + (outcome.strategy === "clear" ? " Prior session history is lost." : "");
|
|
39681
|
+
}
|
|
39682
|
+
case "unusableReplica":
|
|
39683
|
+
return describeUnusableReplica(outcome.reason);
|
|
39684
|
+
case "fatal":
|
|
39685
|
+
return describeSessionDbFatal(outcome, config);
|
|
39686
|
+
default:
|
|
39687
|
+
return assertNeverSessionDb(outcome);
|
|
39688
|
+
}
|
|
39689
|
+
}
|
|
39609
39690
|
|
|
39610
39691
|
// src/file-ops.ts
|
|
39611
39692
|
init_esm_shims();
|
|
@@ -39708,6 +39789,48 @@ async function modelAuthReady(config, fileOps) {
|
|
|
39708
39789
|
return config.hasModelApiKey;
|
|
39709
39790
|
}
|
|
39710
39791
|
|
|
39792
|
+
// src/replica-keys.ts
|
|
39793
|
+
init_esm_shims();
|
|
39794
|
+
var DEFAULT_UNUSABLE_REPLICA_STRATEGY = "prune";
|
|
39795
|
+
var STRATEGIES = ["prune", "leave", "clear", "crash"];
|
|
39796
|
+
function parseUnusableReplicaStrategy(value) {
|
|
39797
|
+
if (value === void 0) return DEFAULT_UNUSABLE_REPLICA_STRATEGY;
|
|
39798
|
+
return STRATEGIES.includes(value) ? value : null;
|
|
39799
|
+
}
|
|
39800
|
+
function replicaDbPrefix(prefix) {
|
|
39801
|
+
return `${prefix}/opencode.db/`;
|
|
39802
|
+
}
|
|
39803
|
+
var LTX_FILENAME = /^([0-9a-fA-F]+)-([0-9a-fA-F]+)\.ltx$/;
|
|
39804
|
+
function parseLtxKey(prefix, key) {
|
|
39805
|
+
if (key.split("/").includes("..")) return null;
|
|
39806
|
+
const root12 = `${replicaDbPrefix(prefix)}ltx/`;
|
|
39807
|
+
if (!key.startsWith(root12)) return null;
|
|
39808
|
+
const [levelPart, filename, ...rest] = key.slice(root12.length).split("/");
|
|
39809
|
+
if (rest.length > 0 || filename === void 0 || !/^\d+$/.test(levelPart)) return null;
|
|
39810
|
+
const match = LTX_FILENAME.exec(filename);
|
|
39811
|
+
if (!match) return null;
|
|
39812
|
+
return {
|
|
39813
|
+
level: Number.parseInt(levelPart, 10),
|
|
39814
|
+
minTxid: BigInt(`0x${match[1]}`),
|
|
39815
|
+
maxTxid: BigInt(`0x${match[2]}`)
|
|
39816
|
+
};
|
|
39817
|
+
}
|
|
39818
|
+
function newestL0Key(prefix, keys) {
|
|
39819
|
+
let newest = null;
|
|
39820
|
+
for (const key of keys) {
|
|
39821
|
+
const parsed = parseLtxKey(prefix, key);
|
|
39822
|
+
if (parsed === null || parsed.level !== 0) continue;
|
|
39823
|
+
if (newest === null || parsed.maxTxid > newest.maxTxid) {
|
|
39824
|
+
newest = { key, maxTxid: parsed.maxTxid };
|
|
39825
|
+
}
|
|
39826
|
+
}
|
|
39827
|
+
return newest?.key ?? null;
|
|
39828
|
+
}
|
|
39829
|
+
function isDeletableReplicaKey(prefix, key) {
|
|
39830
|
+
if (!key.startsWith(replicaDbPrefix(prefix))) return false;
|
|
39831
|
+
return !key.split("/").includes("..");
|
|
39832
|
+
}
|
|
39833
|
+
|
|
39711
39834
|
// src/restore.ts
|
|
39712
39835
|
init_esm_shims();
|
|
39713
39836
|
import { dirname as dirname3 } from "path";
|
|
@@ -43273,6 +43396,10 @@ var _ep4 = {
|
|
|
43273
43396
|
DisableS3ExpressSessionAuth: { type: "staticContextParams", value: true },
|
|
43274
43397
|
Bucket: { type: "contextParams", name: "Bucket" }
|
|
43275
43398
|
};
|
|
43399
|
+
var _ep8 = {
|
|
43400
|
+
Bucket: { type: "contextParams", name: "Bucket" },
|
|
43401
|
+
Prefix: { type: "contextParams", name: "Prefix" }
|
|
43402
|
+
};
|
|
43276
43403
|
var _mw011 = (Command3, cs, config, o7) => [
|
|
43277
43404
|
getThrow200ExceptionsPlugin(config)
|
|
43278
43405
|
];
|
|
@@ -49549,11 +49676,21 @@ var S3Client = class extends Client {
|
|
|
49549
49676
|
}
|
|
49550
49677
|
};
|
|
49551
49678
|
|
|
49679
|
+
// ../../node_modules/.pnpm/@aws-sdk+client-s3@3.1095.0/node_modules/@aws-sdk/client-s3/dist-es/commands/DeleteObjectCommand.js
|
|
49680
|
+
init_esm_shims();
|
|
49681
|
+
var DeleteObjectCommand = class extends command11(_ep011, _mw011, "DeleteObject", DeleteObject$) {
|
|
49682
|
+
};
|
|
49683
|
+
|
|
49552
49684
|
// ../../node_modules/.pnpm/@aws-sdk+client-s3@3.1095.0/node_modules/@aws-sdk/client-s3/dist-es/commands/GetObjectCommand.js
|
|
49553
49685
|
init_esm_shims();
|
|
49554
49686
|
var GetObjectCommand = class extends command11(_ep011, _mw7, "GetObject", GetObject$) {
|
|
49555
49687
|
};
|
|
49556
49688
|
|
|
49689
|
+
// ../../node_modules/.pnpm/@aws-sdk+client-s3@3.1095.0/node_modules/@aws-sdk/client-s3/dist-es/commands/ListObjectsV2Command.js
|
|
49690
|
+
init_esm_shims();
|
|
49691
|
+
var ListObjectsV2Command = class extends command11(_ep8, _mw011, "ListObjectsV2", ListObjectsV2$) {
|
|
49692
|
+
};
|
|
49693
|
+
|
|
49557
49694
|
// ../../node_modules/.pnpm/@aws-sdk+client-s3@3.1095.0/node_modules/@aws-sdk/client-s3/dist-es/commands/PutObjectCommand.js
|
|
49558
49695
|
init_esm_shims();
|
|
49559
49696
|
var PutObjectCommand = class extends command11(_ep011, _mw11, "PutObject", PutObject$) {
|
|
@@ -49588,8 +49725,190 @@ var S3ObjectStore = class {
|
|
|
49588
49725
|
async put(key, body) {
|
|
49589
49726
|
await this.client.send(new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: body }));
|
|
49590
49727
|
}
|
|
49728
|
+
async list(prefix) {
|
|
49729
|
+
const keys = [];
|
|
49730
|
+
let continuationToken;
|
|
49731
|
+
do {
|
|
49732
|
+
const response = await this.client.send(
|
|
49733
|
+
new ListObjectsV2Command({
|
|
49734
|
+
Bucket: this.bucket,
|
|
49735
|
+
Prefix: prefix,
|
|
49736
|
+
ContinuationToken: continuationToken
|
|
49737
|
+
})
|
|
49738
|
+
);
|
|
49739
|
+
for (const object of response.Contents ?? []) {
|
|
49740
|
+
if (object.Key) keys.push(object.Key);
|
|
49741
|
+
}
|
|
49742
|
+
continuationToken = response.NextContinuationToken;
|
|
49743
|
+
} while (continuationToken);
|
|
49744
|
+
return keys;
|
|
49745
|
+
}
|
|
49746
|
+
async delete(key) {
|
|
49747
|
+
try {
|
|
49748
|
+
await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: key }));
|
|
49749
|
+
} catch (error) {
|
|
49750
|
+
if (isNotFound(error)) return;
|
|
49751
|
+
throw error;
|
|
49752
|
+
}
|
|
49753
|
+
}
|
|
49591
49754
|
};
|
|
49592
49755
|
|
|
49756
|
+
// src/session-db.ts
|
|
49757
|
+
init_esm_shims();
|
|
49758
|
+
|
|
49759
|
+
// src/replica-recovery.ts
|
|
49760
|
+
init_esm_shims();
|
|
49761
|
+
|
|
49762
|
+
// src/ltx-format.ts
|
|
49763
|
+
init_esm_shims();
|
|
49764
|
+
var LTX_HEADER_SIZE = 100;
|
|
49765
|
+
var LTX_MAGIC = "LTX1";
|
|
49766
|
+
function isStructurallySoundLtxObject(bytes) {
|
|
49767
|
+
if (bytes.length < LTX_HEADER_SIZE) return false;
|
|
49768
|
+
return bytes.subarray(0, LTX_MAGIC.length).toString("ascii") === LTX_MAGIC;
|
|
49769
|
+
}
|
|
49770
|
+
|
|
49771
|
+
// src/replica-recovery.ts
|
|
49772
|
+
async function probeReplica(store, prefix, log) {
|
|
49773
|
+
const root12 = replicaDbPrefix(prefix);
|
|
49774
|
+
try {
|
|
49775
|
+
const keys = await store.list(root12);
|
|
49776
|
+
return { ok: true, keys };
|
|
49777
|
+
} catch (error) {
|
|
49778
|
+
const detail = describeError(error);
|
|
49779
|
+
log(`WARNING: could not list the replica prefix ${root12}: ${detail}`);
|
|
49780
|
+
return { ok: false, detail };
|
|
49781
|
+
}
|
|
49782
|
+
}
|
|
49783
|
+
async function pruneNewestL0(store, prefix, keys, log) {
|
|
49784
|
+
const target = newestL0Key(prefix, keys);
|
|
49785
|
+
if (target === null || !isDeletableReplicaKey(prefix, target)) {
|
|
49786
|
+
if (target !== null) {
|
|
49787
|
+
log(`WARNING: refusing to prune ${target}: failed the replica-key guard.`);
|
|
49788
|
+
}
|
|
49789
|
+
return { deleted: null, reason: "layoutMismatch" };
|
|
49790
|
+
}
|
|
49791
|
+
let bytes;
|
|
49792
|
+
try {
|
|
49793
|
+
bytes = await store.get(target);
|
|
49794
|
+
} catch (error) {
|
|
49795
|
+
log(
|
|
49796
|
+
`WARNING: could not re-read ${target} to check whether it is actually corrupt before pruning it: ${describeError(error)}. Declining to prune without positive evidence.`
|
|
49797
|
+
);
|
|
49798
|
+
return { deleted: null, reason: "targetHealthy" };
|
|
49799
|
+
}
|
|
49800
|
+
if (bytes === null || isStructurallySoundLtxObject(bytes)) {
|
|
49801
|
+
log(
|
|
49802
|
+
`INFO: the newest L0 replica object ${target} ` + (bytes === null ? "is already gone" : "looks structurally sound") + "; declining to prune it without positive evidence it is the corrupt one."
|
|
49803
|
+
);
|
|
49804
|
+
return { deleted: null, reason: "targetHealthy" };
|
|
49805
|
+
}
|
|
49806
|
+
log(`INFO: pruning the newest L0 replica object ${target} (fails the LTX structural check).`);
|
|
49807
|
+
try {
|
|
49808
|
+
await store.delete(target);
|
|
49809
|
+
} catch (error) {
|
|
49810
|
+
log(`WARNING: failed to prune ${target}: ${describeError(error)}`);
|
|
49811
|
+
return { deleted: null, reason: "deleteFailed" };
|
|
49812
|
+
}
|
|
49813
|
+
return { deleted: target };
|
|
49814
|
+
}
|
|
49815
|
+
async function clearReplica(store, prefix, keys, log) {
|
|
49816
|
+
const deleted = [];
|
|
49817
|
+
const failed = [];
|
|
49818
|
+
for (const key of keys) {
|
|
49819
|
+
if (!isDeletableReplicaKey(prefix, key)) {
|
|
49820
|
+
log(`WARNING: refusing to delete ${key}: outside the replica prefix.`);
|
|
49821
|
+
continue;
|
|
49822
|
+
}
|
|
49823
|
+
try {
|
|
49824
|
+
await store.delete(key);
|
|
49825
|
+
deleted.push(key);
|
|
49826
|
+
} catch (error) {
|
|
49827
|
+
log(`WARNING: failed to delete ${key}: ${describeError(error)}`);
|
|
49828
|
+
failed.push(key);
|
|
49829
|
+
}
|
|
49830
|
+
}
|
|
49831
|
+
return { deleted, failed };
|
|
49832
|
+
}
|
|
49833
|
+
|
|
49834
|
+
// src/session-db.ts
|
|
49835
|
+
function assertNeverStrategy(value) {
|
|
49836
|
+
throw new Error(`Unhandled unusable-replica strategy: ${JSON.stringify(value)}`);
|
|
49837
|
+
}
|
|
49838
|
+
async function classifySessionDbRestore(params) {
|
|
49839
|
+
const outcome = await decideSessionDbRestore(params);
|
|
49840
|
+
if (outcome.kind === "unusableReplica") {
|
|
49841
|
+
await discardLocalDebris(params.fileOps, params.config.opencodeDbPath, params.log);
|
|
49842
|
+
}
|
|
49843
|
+
return outcome;
|
|
49844
|
+
}
|
|
49845
|
+
async function decideSessionDbRestore(params) {
|
|
49846
|
+
const { exitCode, attempt, strategy, config, fileOps, store, log } = params;
|
|
49847
|
+
if (config.bucket === null || config.prefix === null) {
|
|
49848
|
+
return { kind: "disabled" };
|
|
49849
|
+
}
|
|
49850
|
+
const prefix = config.prefix;
|
|
49851
|
+
if (exitCode === 0) {
|
|
49852
|
+
const stat2 = await fileOps.stat(config.opencodeDbPath);
|
|
49853
|
+
return stat2 !== null && stat2.size > 0 ? { kind: "restored" } : { kind: "noReplica" };
|
|
49854
|
+
}
|
|
49855
|
+
if (store === null) {
|
|
49856
|
+
return {
|
|
49857
|
+
kind: "fatal",
|
|
49858
|
+
cause: "misconfig",
|
|
49859
|
+
detail: "persistence is enabled (LITESTREAM_BUCKET/LITESTREAM_PREFIX are set) but no object store is available to verify the replica is reachable"
|
|
49860
|
+
};
|
|
49861
|
+
}
|
|
49862
|
+
if (attempt === 1) {
|
|
49863
|
+
return { kind: "retryTransient" };
|
|
49864
|
+
}
|
|
49865
|
+
if (attempt >= 3) {
|
|
49866
|
+
return { kind: "unusableReplica", reason: "attemptsExhausted" };
|
|
49867
|
+
}
|
|
49868
|
+
const probe = await probeReplica(store, prefix, log);
|
|
49869
|
+
if (!probe.ok) {
|
|
49870
|
+
return {
|
|
49871
|
+
kind: "fatal",
|
|
49872
|
+
cause: "misconfig",
|
|
49873
|
+
detail: `could not verify the session-DB replica is reachable: ${probe.detail}`
|
|
49874
|
+
};
|
|
49875
|
+
}
|
|
49876
|
+
const keys = probe.keys;
|
|
49877
|
+
switch (strategy) {
|
|
49878
|
+
case "crash":
|
|
49879
|
+
return {
|
|
49880
|
+
kind: "fatal",
|
|
49881
|
+
cause: "deliberate",
|
|
49882
|
+
detail: "the replica is unusable and --on-unusable-replica=crash was selected; no S3 object was touched"
|
|
49883
|
+
};
|
|
49884
|
+
case "leave":
|
|
49885
|
+
return { kind: "unusableReplica", reason: "strategy" };
|
|
49886
|
+
case "prune": {
|
|
49887
|
+
const result = await pruneNewestL0(store, prefix, keys, log);
|
|
49888
|
+
if (result.deleted === null) {
|
|
49889
|
+
const reason = result.reason === "deleteFailed" ? "recoveryFailed" : result.reason;
|
|
49890
|
+
return { kind: "unusableReplica", reason };
|
|
49891
|
+
}
|
|
49892
|
+
return { kind: "recovered", strategy: "prune", deleted: [result.deleted] };
|
|
49893
|
+
}
|
|
49894
|
+
case "clear": {
|
|
49895
|
+
const result = await clearReplica(store, prefix, keys, log);
|
|
49896
|
+
return { kind: "recovered", strategy: "clear", deleted: result.deleted };
|
|
49897
|
+
}
|
|
49898
|
+
default:
|
|
49899
|
+
return assertNeverStrategy(strategy);
|
|
49900
|
+
}
|
|
49901
|
+
}
|
|
49902
|
+
async function discardLocalDebris(fileOps, dbPath, log) {
|
|
49903
|
+
for (const path2 of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) {
|
|
49904
|
+
try {
|
|
49905
|
+
await fileOps.remove(path2);
|
|
49906
|
+
} catch (error) {
|
|
49907
|
+
log(`WARNING: could not remove local session-DB debris at ${path2}: ${describeError(error)}`);
|
|
49908
|
+
}
|
|
49909
|
+
}
|
|
49910
|
+
}
|
|
49911
|
+
|
|
49593
49912
|
// src/self-stop.ts
|
|
49594
49913
|
init_esm_shims();
|
|
49595
49914
|
function isComplete(credentials) {
|
|
@@ -49666,7 +49985,15 @@ async function selfStop({
|
|
|
49666
49985
|
|
|
49667
49986
|
// src/shell-contract.json
|
|
49668
49987
|
var shell_contract_default = {
|
|
49669
|
-
commands: [
|
|
49988
|
+
commands: [
|
|
49989
|
+
"env",
|
|
49990
|
+
"litestream-config",
|
|
49991
|
+
"restore",
|
|
49992
|
+
"sync-once",
|
|
49993
|
+
"model-auth-ready",
|
|
49994
|
+
"self-stop",
|
|
49995
|
+
"session-db-classify"
|
|
49996
|
+
]
|
|
49670
49997
|
};
|
|
49671
49998
|
|
|
49672
49999
|
// src/sync.ts
|
|
@@ -49718,6 +50045,9 @@ var EXIT_NOT_READY = 10;
|
|
|
49718
50045
|
var EXIT_KEEP_TASK = 20;
|
|
49719
50046
|
var EXIT_USAGE = 2;
|
|
49720
50047
|
var EXIT_BROKEN = 1;
|
|
50048
|
+
var EXIT_SESSION_DB_FATAL = 30;
|
|
50049
|
+
var EXIT_SESSION_DB_UNUSABLE = 31;
|
|
50050
|
+
var EXIT_SESSION_DB_RETRY = 32;
|
|
49721
50051
|
var STATE_SUFFIX = ".synchash";
|
|
49722
50052
|
var COMMANDS = shell_contract_default.commands;
|
|
49723
50053
|
var USAGE = `Usage: runner-synchroniser <command> [args]
|
|
@@ -49729,6 +50059,12 @@ var USAGE = `Usage: runner-synchroniser <command> [args]
|
|
|
49729
50059
|
model-auth-ready exit 0 when some model auth is configured, ${EXIT_NOT_READY} when not
|
|
49730
50060
|
self-stop scale this agent's own ECS service to 0; exit 0 only
|
|
49731
50061
|
when desiredCount is confirmed 0, ${EXIT_KEEP_TASK} to keep the task
|
|
50062
|
+
session-db-classify <litestream-restore-exit-code> <attempt>
|
|
50063
|
+
[--on-unusable-replica=<prune|leave|clear|crash>]
|
|
50064
|
+
classify a just-run \`litestream restore\` of opencode.db;
|
|
50065
|
+
exit 0 restored/no-replica/disabled, ${EXIT_SESSION_DB_RETRY} re-run
|
|
50066
|
+
and ask again, ${EXIT_SESSION_DB_UNUSABLE} unusable (booted fresh,
|
|
50067
|
+
skip replicate), ${EXIT_SESSION_DB_FATAL} fatal
|
|
49732
50068
|
`;
|
|
49733
50069
|
function shellQuote(value) {
|
|
49734
50070
|
return `'${value.replaceAll("'", `'\\''`)}'`;
|
|
@@ -49742,11 +50078,39 @@ function awsSelfStopAccessFor(config) {
|
|
|
49742
50078
|
function parseStoreName(value) {
|
|
49743
50079
|
return value === "claude" || value === "opencode" ? value : null;
|
|
49744
50080
|
}
|
|
50081
|
+
var ON_UNUSABLE_REPLICA_FLAG = "--on-unusable-replica=";
|
|
50082
|
+
function parseNonNegativeInt(value) {
|
|
50083
|
+
return /^\d+$/.test(value) ? Number.parseInt(value, 10) : null;
|
|
50084
|
+
}
|
|
50085
|
+
function parseSessionDbClassifyArgs(args) {
|
|
50086
|
+
let strategyRaw;
|
|
50087
|
+
const positional = [];
|
|
50088
|
+
for (const arg of args) {
|
|
50089
|
+
if (arg.startsWith(ON_UNUSABLE_REPLICA_FLAG)) {
|
|
50090
|
+
if (strategyRaw !== void 0) return null;
|
|
50091
|
+
strategyRaw = arg.slice(ON_UNUSABLE_REPLICA_FLAG.length);
|
|
50092
|
+
} else if (arg.startsWith("--")) {
|
|
50093
|
+
return null;
|
|
50094
|
+
} else {
|
|
50095
|
+
positional.push(arg);
|
|
50096
|
+
}
|
|
50097
|
+
}
|
|
50098
|
+
if (positional.length !== 2) return null;
|
|
50099
|
+
const exitCode = parseNonNegativeInt(positional[0]);
|
|
50100
|
+
const attempt = parseNonNegativeInt(positional[1]);
|
|
50101
|
+
const strategy = parseUnusableReplicaStrategy(strategyRaw);
|
|
50102
|
+
if (exitCode === null || attempt === null || attempt < 1 || strategy === null) return null;
|
|
50103
|
+
return { exitCode, attempt, strategy };
|
|
50104
|
+
}
|
|
49745
50105
|
function renderEnv(config) {
|
|
49746
50106
|
return [
|
|
49747
|
-
//
|
|
49748
|
-
//
|
|
49749
|
-
|
|
50107
|
+
// The bucket NAME, kept (not reduced to a boolean) so an operator can
|
|
50108
|
+
// read it; empty exactly when persistence is disabled (both
|
|
50109
|
+
// LITESTREAM_BUCKET and LITESTREAM_PREFIX must be set — see config.ts's
|
|
50110
|
+
// persistenceEnabled). entrypoint.sh gates BOTH the credential-sync
|
|
50111
|
+
// loop and whether it starts litestream replicate on this being
|
|
50112
|
+
// non-empty.
|
|
50113
|
+
`PERSISTENCE_BUCKET=${shellQuote(config.bucket ?? "")}`,
|
|
49750
50114
|
`CLAUDE_CREDS=${shellQuote(config.claude.path)}`,
|
|
49751
50115
|
`OPENCODE_DB_PATH=${shellQuote(config.opencodeDbPath)}`,
|
|
49752
50116
|
`CREDS_SYNC_INTERVAL=${shellQuote(String(config.syncIntervalSeconds))}`
|
|
@@ -49812,6 +50176,20 @@ async function commandSyncOnce(name, config, deps) {
|
|
|
49812
50176
|
await writeLastHash(fileOps, statePath, result.hash, log);
|
|
49813
50177
|
}
|
|
49814
50178
|
}
|
|
50179
|
+
async function commandSessionDbClassify(args, config, deps) {
|
|
50180
|
+
const { fileOps, log } = deps;
|
|
50181
|
+
const outcome = await classifySessionDbRestore({
|
|
50182
|
+
exitCode: args.exitCode,
|
|
50183
|
+
attempt: args.attempt,
|
|
50184
|
+
strategy: args.strategy,
|
|
50185
|
+
config: { bucket: config.bucket, prefix: config.prefix, opencodeDbPath: config.opencodeDbPath },
|
|
50186
|
+
fileOps,
|
|
50187
|
+
store: (deps.objectStoreFor ?? s3ObjectStoreFor)(config),
|
|
50188
|
+
log
|
|
50189
|
+
});
|
|
50190
|
+
log(describeSessionDbClassification(outcome, { bucket: config.bucket, region: config.region }));
|
|
50191
|
+
return sessionDbExitCode(outcome);
|
|
50192
|
+
}
|
|
49815
50193
|
async function main(argv, deps) {
|
|
49816
50194
|
const { env: env3, fileOps, log, out } = deps;
|
|
49817
50195
|
const [command12, ...rest] = argv;
|
|
@@ -49847,6 +50225,17 @@ ${USAGE}`);
|
|
|
49847
50225
|
});
|
|
49848
50226
|
return stopped ? 0 : EXIT_KEEP_TASK;
|
|
49849
50227
|
}
|
|
50228
|
+
if (command12 === "session-db-classify") {
|
|
50229
|
+
const args = parseSessionDbClassifyArgs(rest);
|
|
50230
|
+
if (args === null) {
|
|
50231
|
+
log(
|
|
50232
|
+
`'session-db-classify' needs <litestream-restore-exit-code> <attempt> and an optional --on-unusable-replica=<prune|leave|clear|crash>.
|
|
50233
|
+
${USAGE}`
|
|
50234
|
+
);
|
|
50235
|
+
return EXIT_USAGE;
|
|
50236
|
+
}
|
|
50237
|
+
return commandSessionDbClassify(args, config, deps);
|
|
50238
|
+
}
|
|
49850
50239
|
const name = parseStoreName(rest[0]);
|
|
49851
50240
|
if (name === null) {
|
|
49852
50241
|
log(`'${command12}' needs a store name: claude or opencode.
|
|
@@ -49886,6 +50275,9 @@ export {
|
|
|
49886
50275
|
EXIT_BROKEN,
|
|
49887
50276
|
EXIT_KEEP_TASK,
|
|
49888
50277
|
EXIT_NOT_READY,
|
|
50278
|
+
EXIT_SESSION_DB_FATAL,
|
|
50279
|
+
EXIT_SESSION_DB_RETRY,
|
|
50280
|
+
EXIT_SESSION_DB_UNUSABLE,
|
|
49889
50281
|
EXIT_USAGE,
|
|
49890
50282
|
main
|
|
49891
50283
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@evident-ai/runner-synchroniser",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1-dev.0a5718b",
|
|
4
4
|
"description": "Restores and syncs the Evident runner's OpenCode credential stores (and litestream config) to an object store, so a runner survives task replacement with almost no state loss.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/cli.js",
|