@evident-ai/runner-synchroniser 3.5.2-dev.1c27282 → 3.5.2-dev.4f7b3ac

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +83 -16
  2. package/dist/cli.js +553 -115
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -56,7 +56,7 @@ build time.
56
56
  | `env` | resolved config as shell-eval'able vars | `0` = ran; non-zero = tool broken |
57
57
  | `litestream-config` | the generated `litestream.yml` | `0` = ran; non-zero = tool broken |
58
58
  | `restore <claude\|opencode>` | — | `0` = ran; non-zero = tool broken |
59
- | `sync-once <claude\|opencode>` | — | `0` = uploaded/unchanged/absent/disabled; `40` = not persisted; other = tool broken |
59
+ | `sync-once <claude\|opencode>` | — | `0` = uploaded (per-runner) or published/adopted/unchanged/absent/disabled (shared vault); `40` = not persisted this tick; other = tool broken |
60
60
  | `model-auth-ready` | — | `0` = ready, `10` = not ready; other = tool broken |
61
61
  | `self-stop` | — | `0` = stopped, `20` = keep the task; other = tool broken |
62
62
  | `session-db-classify <litestream-restore-exit-code> <attempt> [--on-unusable-replica=<prune\|leave\|clear\|crash>] [--recovery-occurred] [--fresh-db-fallback]` | — | `0` = restored/no-replica/disabled, `32` = re-run and ask again, `31` = unusable (booted fresh, replicates into the existing prefix), `30` = fatal; other = usage/tool broken |
@@ -278,8 +278,13 @@ nothing had ever run a real `PRAGMA integrity_check`).
278
278
  then the prefix is re-listed as empty before the corrupt local DB and sidecars are
279
279
  discarded. Only both proofs permit a fresh boot. Otherwise exit `34` stops the boot before
280
280
  OpenCode or Litestream starts; the backup history remains readable at its quarantine
281
- destination or original key, and surviving local files remain in place without a process
282
- opening or writing them.
281
+ destination or original key, and surviving local files remain in place without a process
282
+ opening or writing them.
283
+
284
+ Each restore-point listing and each individual candidate restore has its own hard timeout.
285
+ Replica separation also has a wall-clock budget. When that budget expires, the synchroniser
286
+ reports the number moved and remaining, leaves the active prefix unverified, and exits `34`;
287
+ the next refused start can continue moving the remaining prefix.
283
288
 
284
289
  A walked-back boot keeps replicating into the **same** replica — no S3 mutation, no new
285
290
  prefix. litestream re-bases to the replica's high-water mark and continues the txid
@@ -311,6 +316,7 @@ before looking at everything" would report the wrong conclusion to the operator.
311
316
  | `WARNING: SESSION-DB-INTEGRITY-FAILED` | `verifySessionDb` (`session-db-verify.ts`) | The local DB fails its check, before the walkback starts | — (prose carries the PRAGMA's own detail) |
312
317
  | `WARNING: SESSION-DB-WALKBACK-ENUMERATION-FAILED` | the walkback loop (`session-db-verify.ts`) | `litestream ltx` exited non-zero (prose carries its stderr — missing binary, bad `-config`, S3 error) or answered with a listing that could not be parsed | — (prose carries litestream's own stderr) |
313
318
  | `WARNING: SESSION-DB-WALKBACK-CANDIDATE-FAILED` | the walkback loop (`session-db-verify.ts`) | A candidate restore point fails to restore, or restores but also fails its own check | `txid` |
319
+ | `WARNING: SESSION-DB-REPLICA-QUARANTINED` | `quarantineReplica` (`replica-recovery.ts`) | A corrupt replica is moved aside, whether the move finishes or its disposal budget stops it early | `moved`, `remaining`, quarantine destination |
314
320
  | `INFO: SESSION-DB-WALKBACK-ADOPTED` | `adopt` (`session-db-verify.ts`) | A candidate passes and is adopted over `opencode.db` | `txid`, `candidates`, `bytes` |
315
321
  | `SESSION-DB-INTEGRITY` | `describeSessionDbVerification` (`diagnostics.ts`), logged once per invocation as the final outcome | The local DB passed its check as-is | `bytes` |
316
322
  | `SESSION-DB-INTEGRITY-WALKBACK` | `describeSessionDbVerification` (`diagnostics.ts`), logged once per invocation as the final outcome | Same event as `SESSION-DB-WALKBACK-ADOPTED` above, restated as the command's outcome | `txid`, `candidates`, `bytes` |
@@ -318,19 +324,45 @@ before looking at everything" would report the wrong conclusion to the operator.
318
324
  | `SESSION-DB-REPLICA-SEPARATED` | `separateCorruptReplica` (`replica-recovery.ts`) | The active prefix was re-listed empty after separation | replica root and quarantine destination |
319
325
  | `SESSION-DB-REPLICA-SEPARATION-UNVERIFIED` | `separateCorruptReplica` (`replica-recovery.ts`) | A separation could not be proven | remaining objects or list error |
320
326
  | `SESSION-DB-LOCAL-DISCARD-FAILED` | `verifySessionDb` (`session-db-verify.ts`) | The local DB or a sidecar could not be proven gone | removed and surviving paths |
321
-
322
- The first four fire only while the walkback loop runs; the last three are always the
323
- one message this command logs as its own final word on the outcome — `entrypoint.sh`
324
- reuses `SESSION-DB-INTEGRITY-EXHAUSTED` verbatim when it logs its own `33` handling, the
325
- same way it already reuses `SESSION-DB-REPLICA-UNUSABLE` for `session-db-classify`'s `31`.
327
+ | `SESSION-DB-VERIFY-TIMEOUT` | the boot wrapper around `session-db-verify` | Verification did not finish before its outer deadline; the message says whether corruption was already proven and whether separation had started | timeout and refusal reason |
328
+
329
+ The walkback markers describe each attempt and its progress. The final outcome markers
330
+ describe whether the active prefix and local files were proven safe; `entrypoint.sh` reuses
331
+ `SESSION-DB-INTEGRITY-EXHAUSTED` verbatim when it logs its own `33` handling, the same way it
332
+ already reuses `SESSION-DB-REPLICA-UNUSABLE` for `session-db-classify`'s `31`. The CLI wrapper
333
+ emits `SESSION-DB-VERIFY-TIMEOUT` when the outer verification deadline is reached.
334
+
335
+ #### Refused verification outcomes
336
+
337
+ The recovery report records `replica_separation_unproven` when separation did not prove the
338
+ active prefix empty, `local_discard_failed` when local session-database files could not be
339
+ proven removed, `verification_timeout_before_separation` when verification timed out after
340
+ integrity failure but before replica separation began, and
341
+ `verification_timeout_during_separation` when the timeout occurred after separation had
342
+ started. The timeout reasons point to `SESSION-DB-VERIFY-TIMEOUT` and do not claim that
343
+ objects were removed.
344
+
345
+ On a refused boot, wait while the reported `remaining` count falls: each start moves another
346
+ bounded slice and the runner comes online after the active prefix drains. If it stops falling,
347
+ move or delete the objects under `<prefix>/opencode.db/` manually; the next start then uses a
348
+ fresh session database. Set-aside history remains at
349
+ `<prefix>/quarantine/opencode.db/<stamp>/`, with one stamp for each start that moved anything.
350
+ Do not use `EVIDENT_ON_UNUSABLE_REPLICA` for this case; that variable controls the earlier
351
+ restore-failure classification path, not post-restore verification.
352
+
353
+ A store too large for one candidate restore to finish inside
354
+ `EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS` is refused and drained rather than
355
+ adopting an older restore point.
326
356
 
327
357
  ### Why the contract is asymmetric
328
358
 
329
359
  `restore` logs domain outcomes and still exits `0`: a missing remote object or a
330
360
  restore failure should not abort a boot, because a runner with no credentials yet
331
361
  is a runner a human can still log into. `sync-once` returns `40` when the current
332
- credential file is not persisted (`failed`, `hashFailed`, or `localInvalid`), while
333
- `absent` and `disabled` remain legitimate `0` outcomes.
362
+ credential file is not persisted. In per-runner mode, that means `failed`, `hashFailed`,
363
+ or `localInvalid`; in shared-vault mode it also includes `remoteFailed`, `remoteInvalid`,
364
+ `noEtag`, `contended`, and `adoptFailed`. `published` (created or replaced), `adopted`,
365
+ `unchanged`, `absent`, and `disabled` are legitimate `0` outcomes.
334
366
 
335
367
  The consequence is the point: **any non-zero status from `restore` means the tool itself
336
368
  broke** — and `sync-once` uses `40` for its typed persistence answer; other non-zero
@@ -379,10 +411,14 @@ key, so they never appear in `env`'s output or `litestream.yml`.
379
411
  | ------------------------------------------ | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- |
380
412
  | `HOME` | Every credential and database path is derived from it. | Throws; logged `FATAL:`, exits `1`. Affects every command. |
381
413
  | `LITESTREAM_BUCKET` / `LITESTREAM_PREFIX` | Object-store bucket and key prefix. | Persistence off (needs **both**): `restore`/`sync-once` no-op, `env` warns. |
414
+ | `CREDENTIAL_VAULT_PREFIX` | Shared prefix for both credential objects; it does not redirect the session database. | Unset or blank: use this runner's `LITESTREAM_PREFIX`; ignored when persistence is disabled. |
382
415
  | `AWS_REGION` | Region for the AWS clients (S3, and the ECS/STS calls `self-stop` makes). | Left to the SDK's own resolution. |
383
416
  | `CREDS_SYNC_INTERVAL` | Seconds between sync ticks, reported by `env` for the caller's loop. | Falls back to `60`; also on unparseable or non-positive values. |
384
417
  | `EVIDENT_SESSION_DB_WALKBACK_MAX_POINTS` | Distinct restore points `session-db-verify`'s walkback will try before giving up. | Falls back to `10`; also on unparseable or non-positive values. |
385
418
  | `EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS` | Wall-clock budget, in seconds, for the whole walkback loop. | Falls back to `180`; also on unparseable or non-positive values. |
419
+ | `EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS` | Hard timeout, in seconds, for one `litestream ltx` restore-point listing. | Falls back to `120`; also on unparseable or non-positive values. |
420
+ | `EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS` | Hard timeout, in seconds, for each individual `litestream restore` walkback candidate. | Falls back to `300`; also on unparseable or non-positive values. |
421
+ | `EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS` | Wall-clock budget, in seconds, for moving a corrupt replica aside. | Falls back to `900`; also on unparseable or non-positive values. |
386
422
  | `EVIDENT_SESSION_DB_RECOVERY_REPORT` | JSONL report drained once into runner activity after authentication. | `$HOME/.local/state/evident/session-db-recovery.jsonl`; blank values use the default. |
387
423
  | `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` | Presence alone counts as configured model auth (see `model-auth-ready`). | No API-key fallback; `model-auth-ready` then depends solely on the credential files. |
388
424
  | `CLUSTER` † / `SERVICE` † | ECS cluster/service `self-stop` scales to `desiredCount=0`. | Warns "cannot self-stop" and exits `20` (keep the task). Ignored by every other command. |
@@ -402,6 +438,11 @@ The paths and keys that follow from the bucket/prefix:
402
438
  | OpenCode | `$HOME/.local/share/opencode/auth.json` | `<prefix>/opencode/auth.json` |
403
439
  | OpenCode DB | `$HOME/.local/share/opencode/opencode.db` | `<prefix>/opencode.db` (litestream) |
404
440
 
441
+ When persistence is enabled and `CREDENTIAL_VAULT_PREFIX` is set, both credential keys
442
+ use that shared prefix instead of `<prefix>`. The OpenCode database continues replicating
443
+ under `LITESTREAM_PREFIX`; the vault prefix only redirects the two credential objects. The
444
+ Evident CLI resolves this value from the runner's vault binding.
445
+
405
446
  The session database is replicated by litestream itself; this package only generates its
406
447
  config. `env` emits `PERSISTENCE_BUCKET`, `CLAUDE_CREDS`, `OPENCODE_DB_PATH` and
407
448
  `CREDS_SYNC_INTERVAL`, each single-quoted so a value containing shell metacharacters is
@@ -435,18 +476,44 @@ running that version — the runner installs this package from a floating npm ta
435
476
  - **Nothing throws out of `restore`/`sync`.** `sync-once` runs on an unsupervised timer,
436
477
  where an unguarded failure would silently stop persisting credentials for the rest of
437
478
  the task's life. Failures degrade to a warning and are retried on the next tick.
438
- - **`sync-once` uploads only on change.** Each store's last uploaded hash (a sha256, not
439
- the credentials) is kept in a state file next to it, so every invocation is a
440
- self-contained process.
479
+ - **Per-runner `sync-once` uploads only on change.** Its last uploaded hash (a sha256, not
480
+ the credentials) is kept in a state file next to the credential file, so every invocation
481
+ is a self-contained process.
482
+ - **Shared-vault mode is two-way and conditional.** `sync-once` adopts a changed shared
483
+ credential or publishes a locally changed one with a conditional write. A writer that
484
+ loses the race adopts the winner instead of overwriting it. A `40` exit is an expected
485
+ "not persisted this tick" result, not a broken tool; the polling loop accepts it and
486
+ retries on the next tick.
441
487
  - **Not-found is distinguished from failed.** A missing object is reported as "nothing to
442
488
  restore"; an access or network error is reported as a failure, because that tells the
443
489
  operator their persisted credentials may still be recoverable.
444
490
 
491
+ ### Shared-vault log markers
492
+
493
+ In shared-vault mode, these markers identify the synchronization result without logging
494
+ credential bytes. Hash values are short prefixes and ETags remain opaque:
495
+
496
+ | Marker | Meaning |
497
+ | ------ | ------- |
498
+ | `VAULT-MODE` | Both credential stores use the shared vault prefix. |
499
+ | `VAULT-PUBLISHED` | A credential was created in, or conditionally replaced in, the vault. |
500
+ | `VAULT-ADOPTED` | A valid vault credential was installed locally. |
501
+ | `VAULT-CONTENDED` | Another writer won the conditional write, so this runner is adopting its result. |
502
+ | `VAULT-CONTENDED-UNRESOLVED` | The winning object could not be read; the next tick will retry. |
503
+ | `VAULT-REMOTE-INVALID` | The vault object was not valid JSON and was either replaced or left untouched. |
504
+ | `VAULT-REMOTE-UNREADABLE` | The vault read failed; the next tick will retry. |
505
+ | `VAULT-NO-ETAG` | Publication was skipped because a remote object had no ETag. |
506
+ | `VAULT-ADOPT-FAILED` | A valid vault credential could not be installed locally; the next tick will retry. |
507
+
445
508
  ## The `ObjectStore` port
446
509
 
447
- `restore`/`sync` and `session-db-classify` talk to a four-method interface: `get(key)`
448
- returning `null` when the object is absent, `put(key, body)`, `list(prefix)` and
449
- `delete(key)` for session-DB recovery. `list(prefix)` returns `{ key, size }` entries
510
+ The `ObjectStore` port has seven methods. `get(key)` returns `{ body, etag }` or `null`
511
+ when the object is absent; `etag` is an opaque compare-and-swap token and may be `null`
512
+ when the store did not report one. `put(key, body)` is the unconditional write used by
513
+ per-runner synchronization. `putIfAbsent(key, body)` creates with a conditional
514
+ `If-None-Match: *` write, and `putIfMatch(key, body, etag)` replaces with a conditional
515
+ `If-Match: <etag>` write. Both return `'written'` or `'conflict'`; only contention
516
+ returns `'conflict'`, while every other error throws. `list(prefix)` returns `{ key, size }` entries
450
517
  (`ObjectSummary[]`) — every object under `prefix`, size in bytes. It throws on any error
451
518
  and never maps one to `[]` (an empty array means "reached the store, nothing there"); the
452
519
  adapter owns pagination, and `delete` is idempotent. S3 vocabulary (`@aws-sdk/client-s3`,
package/dist/cli.js CHANGED
@@ -50887,6 +50887,9 @@ init_esm_shims();
50887
50887
  var DEFAULT_SYNC_INTERVAL_SECONDS = 60;
50888
50888
  var DEFAULT_WALKBACK_MAX_POINTS = 10;
50889
50889
  var DEFAULT_WALKBACK_BUDGET_SECONDS = 180;
50890
+ var DEFAULT_RESTORE_POINTS_TIMEOUT_SECONDS = 120;
50891
+ var DEFAULT_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 300;
50892
+ var DEFAULT_DISPOSAL_BUDGET_SECONDS = 900;
50890
50893
  function positiveIntOr(value, defaultValue) {
50891
50894
  const parsed = Number.parseInt(value ?? "", 10);
50892
50895
  return Number.isFinite(parsed) && parsed > 0 ? parsed : defaultValue;
@@ -50903,22 +50906,24 @@ function resolveConfig(env4) {
50903
50906
  const bucket = nonEmpty(env4.LITESTREAM_BUCKET);
50904
50907
  const prefix = nonEmpty(env4.LITESTREAM_PREFIX);
50905
50908
  const persistenceEnabled = bucket !== null && prefix !== null;
50909
+ const vaultPrefix = persistenceEnabled ? nonEmpty(env4.CREDENTIAL_VAULT_PREFIX)?.replace(/\/+$/, "") || null : null;
50906
50910
  return {
50907
50911
  homeDir,
50908
50912
  claude: {
50909
50913
  label: "Claude",
50910
50914
  path: `${homeDir}/.claude/.credentials.json`,
50911
- key: persistenceEnabled ? `${prefix}/claude/credentials.json` : null
50915
+ key: persistenceEnabled ? `${vaultPrefix ?? prefix}/claude/credentials.json` : null
50912
50916
  },
50913
50917
  opencode: {
50914
50918
  label: "OpenCode",
50915
50919
  path: `${homeDir}/.local/share/opencode/auth.json`,
50916
- key: persistenceEnabled ? `${prefix}/opencode/auth.json` : null
50920
+ key: persistenceEnabled ? `${vaultPrefix ?? prefix}/opencode/auth.json` : null
50917
50921
  },
50918
50922
  opencodeDbPath: `${homeDir}/.local/share/opencode/opencode.db`,
50919
50923
  sessionDbRecoveryReportPath: nonEmpty(env4.EVIDENT_SESSION_DB_RECOVERY_REPORT) ?? `${homeDir}/.local/state/evident/session-db-recovery.jsonl`,
50920
50924
  bucket: persistenceEnabled ? bucket : null,
50921
50925
  prefix: persistenceEnabled ? prefix : null,
50926
+ vaultPrefix,
50922
50927
  region: nonEmpty(env4.AWS_REGION),
50923
50928
  syncIntervalSeconds: positiveIntOr(env4.CREDS_SYNC_INTERVAL, DEFAULT_SYNC_INTERVAL_SECONDS),
50924
50929
  walkbackMaxPoints: positiveIntOr(
@@ -50929,6 +50934,18 @@ function resolveConfig(env4) {
50929
50934
  env4.EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS,
50930
50935
  DEFAULT_WALKBACK_BUDGET_SECONDS
50931
50936
  ),
50937
+ restorePointsTimeoutSeconds: positiveIntOr(
50938
+ env4.EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS,
50939
+ DEFAULT_RESTORE_POINTS_TIMEOUT_SECONDS
50940
+ ),
50941
+ restoreCandidateTimeoutSeconds: positiveIntOr(
50942
+ env4.EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS,
50943
+ DEFAULT_RESTORE_CANDIDATE_TIMEOUT_SECONDS
50944
+ ),
50945
+ disposalBudgetSeconds: positiveIntOr(
50946
+ env4.EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS,
50947
+ DEFAULT_DISPOSAL_BUDGET_SECONDS
50948
+ ),
50932
50949
  hasModelApiKey: nonEmpty(env4.ANTHROPIC_API_KEY) !== null || nonEmpty(env4.OPENAI_API_KEY) !== null,
50933
50950
  runnerKey: nonEmpty(env4.EVIDENT_RUNNER_KEY) ?? nonEmpty(env4.EVIDENT_AGENT_KEY),
50934
50951
  apiUrl: nonEmpty(env4.EVIDENT_API_URL)
@@ -51239,49 +51256,88 @@ function stderrOf(error) {
51239
51256
  }
51240
51257
  return describeError(error);
51241
51258
  }
51259
+ function commandDetail(error, operation4, timeoutSeconds) {
51260
+ if (typeof error === "object" && error !== null) {
51261
+ const processError = error;
51262
+ const signal = typeof processError.signal === "string" ? processError.signal : null;
51263
+ if (processError.killed === true || signal === "SIGKILL") {
51264
+ return `${operation4} timed out after ${timeoutSeconds}s (process killed with ${signal ?? "SIGKILL"})`;
51265
+ }
51266
+ }
51267
+ return stderrOf(error);
51268
+ }
51242
51269
  var LitestreamCli = class {
51243
- constructor(configPath, dbPath) {
51270
+ constructor(configPath, dbPath, timeouts) {
51244
51271
  this.configPath = configPath;
51245
51272
  this.dbPath = dbPath;
51273
+ this.timeouts = timeouts;
51246
51274
  }
51247
51275
  async listRestorePoints() {
51248
51276
  try {
51249
- const { stdout, stderr } = await execFileAsync("litestream", [
51250
- "ltx",
51251
- "-config",
51252
- this.configPath,
51253
- // MEASURED: `-level` defaults to 0, which only shows L0 — `all` is
51254
- // required to see the L1/L2/L9 compaction levels too (plan §2.3).
51255
- "-level",
51256
- "all",
51257
- "-json",
51258
- this.dbPath
51259
- ]);
51277
+ const { stdout, stderr } = await execFileAsync(
51278
+ "litestream",
51279
+ [
51280
+ "ltx",
51281
+ "-config",
51282
+ this.configPath,
51283
+ // MEASURED: `-level` defaults to 0, which only shows L0 — `all` is
51284
+ // required to see the L1/L2/L9 compaction levels too (plan §2.3).
51285
+ "-level",
51286
+ "all",
51287
+ "-json",
51288
+ this.dbPath
51289
+ ],
51290
+ {
51291
+ timeout: this.timeouts.restorePointsTimeoutSeconds * 1e3,
51292
+ killSignal: "SIGKILL"
51293
+ }
51294
+ );
51260
51295
  return { stdout, detail: stderr };
51261
51296
  } catch (error) {
51262
- return { stdout: null, detail: stderrOf(error) };
51297
+ return {
51298
+ stdout: null,
51299
+ detail: commandDetail(
51300
+ error,
51301
+ "litestream restore-point enumeration",
51302
+ this.timeouts.restorePointsTimeoutSeconds
51303
+ )
51304
+ };
51263
51305
  }
51264
51306
  }
51265
51307
  async restoreAt(txid, outputPath) {
51266
51308
  try {
51267
- const { stderr } = await execFileAsync("litestream", [
51268
- "restore",
51269
- "-config",
51270
- this.configPath,
51271
- "-txid",
51272
- txid,
51273
- // MEASURED: without `-force`, restoring into a path that already
51274
- // exists exits 1 with "output path already exists and is not
51275
- // empty" (plan §2.4) — the caller always restores into a scratch
51276
- // path it just cleared, so overwriting it is always intended.
51277
- "-force",
51278
- "-o",
51279
- outputPath,
51280
- this.dbPath
51281
- ]);
51309
+ const { stderr } = await execFileAsync(
51310
+ "litestream",
51311
+ [
51312
+ "restore",
51313
+ "-config",
51314
+ this.configPath,
51315
+ "-txid",
51316
+ txid,
51317
+ // MEASURED: without `-force`, restoring into a path that already
51318
+ // exists exits 1 with "output path already exists and is not
51319
+ // empty" (plan §2.4) — the caller always restores into a scratch
51320
+ // path it just cleared, so overwriting it is always intended.
51321
+ "-force",
51322
+ "-o",
51323
+ outputPath,
51324
+ this.dbPath
51325
+ ],
51326
+ {
51327
+ timeout: this.timeouts.restoreCandidateTimeoutSeconds * 1e3,
51328
+ killSignal: "SIGKILL"
51329
+ }
51330
+ );
51282
51331
  return { ok: true, detail: stderr };
51283
51332
  } catch (error) {
51284
- return { ok: false, detail: stderrOf(error) };
51333
+ return {
51334
+ ok: false,
51335
+ detail: commandDetail(
51336
+ error,
51337
+ `litestream restore of txid=${txid}`,
51338
+ this.timeouts.restoreCandidateTimeoutSeconds
51339
+ )
51340
+ };
51285
51341
  }
51286
51342
  }
51287
51343
  };
@@ -51317,20 +51373,23 @@ init_esm_shims();
51317
51373
 
51318
51374
  // src/valid-json.ts
51319
51375
  init_esm_shims();
51320
- async function isValidJson(fileOps, path) {
51321
- let contents;
51376
+ function isValidJsonBytes(contents) {
51377
+ if (contents.length === 0) return false;
51322
51378
  try {
51323
- contents = await fileOps.readFile(path);
51379
+ const value = JSON.parse(contents.toString("utf8"));
51380
+ return value !== null && value !== false;
51324
51381
  } catch {
51325
51382
  return false;
51326
51383
  }
51327
- if (contents.length === 0) return false;
51384
+ }
51385
+ async function isValidJson(fileOps, path) {
51386
+ let contents;
51328
51387
  try {
51329
- const value = JSON.parse(contents.toString("utf8"));
51330
- return value !== null && value !== false;
51388
+ contents = await fileOps.readFile(path);
51331
51389
  } catch {
51332
51390
  return false;
51333
51391
  }
51392
+ return isValidJsonBytes(contents);
51334
51393
  }
51335
51394
 
51336
51395
  // src/model-auth.ts
@@ -51387,55 +51446,20 @@ function isDeletableReplicaKey(prefix, key) {
51387
51446
 
51388
51447
  // src/restore.ts
51389
51448
  init_esm_shims();
51449
+ import { dirname as dirname4 } from "node:path";
51450
+
51451
+ // src/credential-install.ts
51452
+ init_esm_shims();
51390
51453
  import { dirname as dirname3 } from "node:path";
51391
- async function restore(params) {
51392
- const { label, path, bucket, key, store, fileOps, log } = params;
51454
+ async function installCredentialBytes({
51455
+ label,
51456
+ path,
51457
+ body,
51458
+ source,
51459
+ fileOps,
51460
+ log
51461
+ }) {
51393
51462
  const dir = dirname3(path);
51394
- try {
51395
- await fileOps.mkdirp(dir);
51396
- } catch (error) {
51397
- log(`WARNING: could not create ${label} credential directory ${dir}: ${describeError(error)}`);
51398
- return { kind: "failed", reason: `could not create ${dir}: ${describeError(error)}` };
51399
- }
51400
- try {
51401
- await fileOps.chmod(dir, 448);
51402
- } catch (error) {
51403
- log(
51404
- `WARNING: could not restrict ${label} credential directory ${dir}: ${describeError(error)}`
51405
- );
51406
- }
51407
- const local = await fileOps.stat(path);
51408
- if (local !== null && local.size > 0 && !await isValidJson(fileOps, path)) {
51409
- log(
51410
- `WARNING: local ${label} credentials at ${path} are invalid or unreadable; keeping them and skipping restore.`
51411
- );
51412
- return { kind: "localInvalid" };
51413
- }
51414
- if (local !== null && local.size === 0) {
51415
- log(
51416
- `Local ${label} credentials at ${path} are empty; will attempt to restore from S3 instead of protecting them.`
51417
- );
51418
- }
51419
- if (await isValidJson(fileOps, path)) {
51420
- await chmodOrWarn(fileOps, log, path, `local ${label} credentials at ${path}`);
51421
- log(`Keeping valid local ${label} credentials at ${path}; no restore.`);
51422
- return { kind: "local" };
51423
- }
51424
- if (store === null || bucket === null || key === null) {
51425
- return { kind: "disabled" };
51426
- }
51427
- const location = `s3://${bucket}/${key}`;
51428
- let body;
51429
- try {
51430
- body = await store.get(key);
51431
- } catch (error) {
51432
- log(`WARNING: ${label} credentials download failed from ${location}: ${describeError(error)}`);
51433
- return { kind: "failed", reason: describeError(error) };
51434
- }
51435
- if (body === null) {
51436
- log(`No persisted ${label} credentials at ${location}; nothing to restore.`);
51437
- return { kind: "missing" };
51438
- }
51439
51463
  const temp = `${dir}/.credentials.${Math.random().toString(36).slice(2, 8)}`;
51440
51464
  try {
51441
51465
  await fileOps.createExclusive(temp);
@@ -51464,12 +51488,12 @@ async function restore(params) {
51464
51488
  await fileOps.writeFile(temp, body);
51465
51489
  } catch (error) {
51466
51490
  await fileOps.remove(temp);
51467
- log(`WARNING: ${label} credentials download failed from ${location}: ${describeError(error)}`);
51491
+ log(`WARNING: ${label} credentials download failed from ${source}: ${describeError(error)}`);
51468
51492
  return { kind: "failed", reason: describeError(error) };
51469
51493
  }
51470
51494
  if (!await isValidJson(fileOps, temp)) {
51471
51495
  await fileOps.remove(temp);
51472
- log(`WARNING: discarded invalid ${label} credentials from ${location}.`);
51496
+ log(`WARNING: discarded invalid ${label} credentials from ${source}.`);
51473
51497
  return { kind: "invalid" };
51474
51498
  }
51475
51499
  try {
@@ -51482,8 +51506,7 @@ async function restore(params) {
51482
51506
  return { kind: "failed", reason: `could not install ${path}: ${describeError(error)}` };
51483
51507
  }
51484
51508
  await chmodOrWarn(fileOps, log, path, `restored ${label} credentials at ${path}`);
51485
- log(`Restored ${label} credentials from ${location} to ${path}.`);
51486
- return { kind: "restored" };
51509
+ return { kind: "installed" };
51487
51510
  }
51488
51511
  async function chmodOrWarn(fileOps, log, path, subject) {
51489
51512
  try {
@@ -51493,6 +51516,71 @@ async function chmodOrWarn(fileOps, log, path, subject) {
51493
51516
  }
51494
51517
  }
51495
51518
 
51519
+ // src/restore.ts
51520
+ async function restore(params) {
51521
+ const { label, path, bucket, key, store, fileOps, log } = params;
51522
+ const dir = dirname4(path);
51523
+ try {
51524
+ await fileOps.mkdirp(dir);
51525
+ } catch (error) {
51526
+ log(`WARNING: could not create ${label} credential directory ${dir}: ${describeError(error)}`);
51527
+ return { kind: "failed", reason: `could not create ${dir}: ${describeError(error)}` };
51528
+ }
51529
+ try {
51530
+ await fileOps.chmod(dir, 448);
51531
+ } catch (error) {
51532
+ log(
51533
+ `WARNING: could not restrict ${label} credential directory ${dir}: ${describeError(error)}`
51534
+ );
51535
+ }
51536
+ const local = await fileOps.stat(path);
51537
+ if (local !== null && local.size > 0 && !await isValidJson(fileOps, path)) {
51538
+ log(
51539
+ `WARNING: local ${label} credentials at ${path} are invalid or unreadable; keeping them and skipping restore.`
51540
+ );
51541
+ return { kind: "localInvalid" };
51542
+ }
51543
+ if (local !== null && local.size === 0) {
51544
+ log(
51545
+ `Local ${label} credentials at ${path} are empty; will attempt to restore from S3 instead of protecting them.`
51546
+ );
51547
+ }
51548
+ if (await isValidJson(fileOps, path)) {
51549
+ await chmodOrWarn(fileOps, log, path, `local ${label} credentials at ${path}`);
51550
+ log(`Keeping valid local ${label} credentials at ${path}; no restore.`);
51551
+ return { kind: "local" };
51552
+ }
51553
+ if (store === null || bucket === null || key === null) {
51554
+ return { kind: "disabled" };
51555
+ }
51556
+ const location = `s3://${bucket}/${key}`;
51557
+ let object;
51558
+ try {
51559
+ object = await store.get(key);
51560
+ } catch (error) {
51561
+ log(`WARNING: ${label} credentials download failed from ${location}: ${describeError(error)}`);
51562
+ return { kind: "failed", reason: describeError(error) };
51563
+ }
51564
+ if (object === null) {
51565
+ log(`No persisted ${label} credentials at ${location}; nothing to restore.`);
51566
+ return { kind: "missing" };
51567
+ }
51568
+ const installation = await installCredentialBytes({
51569
+ label,
51570
+ path,
51571
+ body: object.body,
51572
+ source: location,
51573
+ fileOps,
51574
+ log
51575
+ });
51576
+ if (installation.kind === "installed") {
51577
+ log(`Restored ${label} credentials from ${location} to ${path}.`);
51578
+ return { kind: "restored" };
51579
+ }
51580
+ if (installation.kind === "invalid") return { kind: "invalid" };
51581
+ return { kind: "failed", reason: installation.reason };
51582
+ }
51583
+
51496
51584
  // src/s3-object-store.ts
51497
51585
  init_esm_shims();
51498
51586
 
@@ -61271,6 +61359,11 @@ function isNotFound(error) {
61271
61359
  if (candidate.name === "NoSuchKey" || candidate.name === "NotFound") return true;
61272
61360
  return candidate.$metadata?.httpStatusCode === 404;
61273
61361
  }
61362
+ function isConditionalConflict(error) {
61363
+ if (typeof error !== "object" || error === null) return false;
61364
+ const candidate = error;
61365
+ return candidate.name === "PreconditionFailed" || candidate.name === "ConditionalRequestConflict" || candidate.$metadata?.httpStatusCode === 412 || candidate.$metadata?.httpStatusCode === 409;
61366
+ }
61274
61367
  var S3ObjectStore = class {
61275
61368
  constructor(bucket, region = null, client) {
61276
61369
  this.bucket = bucket;
@@ -61283,7 +61376,10 @@ var S3ObjectStore = class {
61283
61376
  new GetObjectCommand({ Bucket: this.bucket, Key: key })
61284
61377
  );
61285
61378
  if (!response.Body) return null;
61286
- return Buffer.from(await response.Body.transformToByteArray());
61379
+ return {
61380
+ body: Buffer.from(await response.Body.transformToByteArray()),
61381
+ etag: response.ETag ?? null
61382
+ };
61287
61383
  } catch (error) {
61288
61384
  if (isNotFound(error)) return null;
61289
61385
  throw error;
@@ -61292,6 +61388,38 @@ var S3ObjectStore = class {
61292
61388
  async put(key, body) {
61293
61389
  await this.client.send(new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: body }));
61294
61390
  }
61391
+ async putIfAbsent(key, body) {
61392
+ try {
61393
+ await this.client.send(
61394
+ new PutObjectCommand({
61395
+ Bucket: this.bucket,
61396
+ Key: key,
61397
+ Body: body,
61398
+ IfNoneMatch: "*"
61399
+ })
61400
+ );
61401
+ return "written";
61402
+ } catch (error) {
61403
+ if (isConditionalConflict(error)) return "conflict";
61404
+ throw error;
61405
+ }
61406
+ }
61407
+ async putIfMatch(key, body, etag) {
61408
+ try {
61409
+ await this.client.send(
61410
+ new PutObjectCommand({
61411
+ Bucket: this.bucket,
61412
+ Key: key,
61413
+ Body: body,
61414
+ IfMatch: etag
61415
+ })
61416
+ );
61417
+ return "written";
61418
+ } catch (error) {
61419
+ if (isConditionalConflict(error) || isNotFound(error)) return "conflict";
61420
+ throw error;
61421
+ }
61422
+ }
61295
61423
  async list(prefix) {
61296
61424
  const objects = [];
61297
61425
  let continuationToken;
@@ -61370,7 +61498,9 @@ async function probeReplica(store, prefix, log) {
61370
61498
  return { ok: true, objects };
61371
61499
  } catch (error) {
61372
61500
  const detail = describeError(error);
61373
- log(`WARNING: could not list the replica prefix ${root12}: ${detail}`);
61501
+ log(
61502
+ `WARNING: SESSION-DB-REPLICA-SEPARATION-UNVERIFIED: could not list the replica prefix ${root12}: ${detail}`
61503
+ );
61374
61504
  return { ok: false, detail };
61375
61505
  }
61376
61506
  }
@@ -61399,7 +61529,8 @@ async function pruneNewestL0(store, prefix, keys, log) {
61399
61529
  }
61400
61530
  let bytes;
61401
61531
  try {
61402
- bytes = await store.get(target);
61532
+ const object = await store.get(target);
61533
+ bytes = object === null ? null : object.body;
61403
61534
  } catch (error) {
61404
61535
  log(
61405
61536
  `WARNING: could not re-read ${target} to check whether it is actually corrupt before pruning it: ${describeError(error)}. Declining to prune without positive evidence.`
@@ -61424,13 +61555,15 @@ async function pruneNewestL0(store, prefix, keys, log) {
61424
61555
  function quarantineRoot(prefix, stamp) {
61425
61556
  return `${prefix}/quarantine/opencode.db/${stamp}/`;
61426
61557
  }
61427
- async function quarantineReplica(store, prefix, objects, log, stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")) {
61558
+ async function quarantineReplica(store, prefix, objects, log, stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-"), budgetSeconds = DEFAULT_DISPOSAL_BUDGET_SECONDS, now = Date.now) {
61428
61559
  const destination = quarantineRoot(prefix, stamp);
61429
61560
  const root12 = replicaDbPrefix(prefix);
61430
61561
  const moved = [];
61431
61562
  const failed = [];
61432
61563
  let movedBytes = 0;
61564
+ const deadline = now() + budgetSeconds * 1e3;
61433
61565
  for (const object of objects) {
61566
+ if (now() >= deadline) break;
61434
61567
  const { key } = object;
61435
61568
  if (!isDeletableReplicaKey(prefix, key)) {
61436
61569
  log(`WARNING: refusing to quarantine ${key}: outside the replica prefix.`);
@@ -61458,15 +61591,15 @@ async function quarantineReplica(store, prefix, objects, log, stamp = (/* @__PUR
61458
61591
  moved.push(key);
61459
61592
  movedBytes += object.size;
61460
61593
  }
61461
- logQuarantined(destination, moved, failed, log);
61594
+ logQuarantined(destination, moved, failed, objects.length - moved.length, log);
61462
61595
  return { destination, moved, failed, movedBytes };
61463
61596
  }
61464
- function logQuarantined(destination, moved, failed, log) {
61597
+ function logQuarantined(destination, moved, failed, remaining, log) {
61465
61598
  log(
61466
- `WARNING: SESSION-DB-REPLICA-QUARANTINED: moved ${moved.length} unusable replica object(s) aside to ${destination} (${failed.length} could not be moved). Moved history remains readable at that destination; failed moves remain at their original keys.` + (failed.length > 0 ? " The active replica prefix is not clear." : "") + `See SESSION-DB-REPLICA-SIZE above for how much was set aside.`
61599
+ `WARNING: SESSION-DB-REPLICA-QUARANTINED: moved ${moved.length} unusable replica object(s) aside to ${destination} (${failed.length} could not be moved). Moved history remains readable at that destination; failed moves remain at their original keys.` + (remaining > 0 ? ` The active replica prefix still has ${remaining} object(s).` : "") + ` Progress: moved=${moved.length} remaining=${remaining}. See SESSION-DB-REPLICA-SIZE above for how much was set aside.`
61467
61600
  );
61468
61601
  }
61469
- async function separateCorruptReplica(store, prefix, log, stamp) {
61602
+ async function separateCorruptReplica(store, prefix, log, stamp, disposalBudgetSeconds = DEFAULT_DISPOSAL_BUDGET_SECONDS, now = Date.now) {
61470
61603
  const probe = await probeReplica(store, prefix, log);
61471
61604
  if (!probe.ok) return { kind: "unreachable", detail: probe.detail };
61472
61605
  const root12 = replicaDbPrefix(prefix);
@@ -61476,7 +61609,15 @@ async function separateCorruptReplica(store, prefix, log, stamp) {
61476
61609
  );
61477
61610
  return { kind: "alreadyEmpty" };
61478
61611
  }
61479
- const result = await quarantineReplica(store, prefix, probe.objects, log, stamp);
61612
+ const result = await quarantineReplica(
61613
+ store,
61614
+ prefix,
61615
+ probe.objects,
61616
+ log,
61617
+ stamp,
61618
+ disposalBudgetSeconds,
61619
+ now
61620
+ );
61480
61621
  try {
61481
61622
  const remaining = await store.list(root12);
61482
61623
  if (remaining.length > 0) {
@@ -61628,7 +61769,7 @@ async function discardLocalDebris(fileOps, dbPath, log) {
61628
61769
 
61629
61770
  // src/session-db-recovery-report.ts
61630
61771
  init_esm_shims();
61631
- import { dirname as dirname4 } from "node:path";
61772
+ import { dirname as dirname5 } from "node:path";
61632
61773
  function base(at2, stage) {
61633
61774
  return {
61634
61775
  v: 1,
@@ -61736,7 +61877,7 @@ async function appendSessionDbRecoveryRecord(record, path, fileOps, log) {
61736
61877
  );
61737
61878
  return;
61738
61879
  }
61739
- await fileOps.mkdirp(dirname4(path));
61880
+ await fileOps.mkdirp(dirname5(path));
61740
61881
  await fileOps.appendFile(path, line);
61741
61882
  } catch (error) {
61742
61883
  log(
@@ -61807,7 +61948,14 @@ async function verifySessionDb(params) {
61807
61948
  );
61808
61949
  const decision = await walkBack(params);
61809
61950
  if (decision.kind !== "exhausted") return decision;
61810
- const separation = params.store === null || config.prefix === null ? { kind: "notConfigured" } : await separateCorruptReplica(params.store, config.prefix, log);
61951
+ const separation = params.store === null || config.prefix === null ? { kind: "notConfigured" } : await separateCorruptReplica(
61952
+ params.store,
61953
+ config.prefix,
61954
+ log,
61955
+ void 0,
61956
+ params.disposalBudgetSeconds,
61957
+ params.now
61958
+ );
61811
61959
  const localDb = separation.kind === "notConfigured" || separation.kind === "alreadyEmpty" || separation.kind === "quarantined" ? await discardCorruptDb(fileOps, dbPath, log) : { kind: "retained", reason: "separationUnproven" };
61812
61960
  return { ...decision, separation, localDb };
61813
61961
  }
@@ -62083,21 +62231,25 @@ var nodeSqliteIntegrity = {
62083
62231
 
62084
62232
  // src/sync.ts
62085
62233
  init_esm_shims();
62086
- async function sync(params) {
62087
- const { label, path, bucket, key, store, lastHash, fileOps, log } = params;
62088
- if (store === null || bucket === null || key === null) {
62089
- return { outcome: { kind: "disabled" }, hash: lastHash };
62090
- }
62234
+
62235
+ // src/local-credential.ts
62236
+ init_esm_shims();
62237
+ async function readLocalCredential({
62238
+ label,
62239
+ path,
62240
+ fileOps,
62241
+ log
62242
+ }) {
62091
62243
  const exists = await fileOps.stat(path) !== null;
62092
62244
  const valid = await isValidJson(fileOps, path);
62093
62245
  if (exists && !valid) {
62094
62246
  log(
62095
62247
  `WARNING: local ${label} credentials at ${path} are invalid or unreadable; keeping them and skipping sync.`
62096
62248
  );
62097
- return { outcome: { kind: "localInvalid" }, hash: lastHash };
62249
+ return { kind: "invalid" };
62098
62250
  }
62099
62251
  if (!valid) {
62100
- return { outcome: { kind: "absent" }, hash: lastHash };
62252
+ return { kind: "absent" };
62101
62253
  }
62102
62254
  try {
62103
62255
  await fileOps.chmod(path, 384);
@@ -62113,8 +62265,28 @@ async function sync(params) {
62113
62265
  log(
62114
62266
  `WARNING: could not hash local ${label} credentials at ${path} (${describeError(error)}); skipping this sync tick.`
62115
62267
  );
62268
+ return { kind: "hashFailed" };
62269
+ }
62270
+ return { kind: "ok", hash };
62271
+ }
62272
+
62273
+ // src/sync.ts
62274
+ async function sync(params) {
62275
+ const { label, path, bucket, key, store, lastHash, fileOps, log } = params;
62276
+ if (store === null || bucket === null || key === null) {
62277
+ return { outcome: { kind: "disabled" }, hash: lastHash };
62278
+ }
62279
+ const local = await readLocalCredential({ label, path, fileOps, log });
62280
+ if (local.kind === "invalid") {
62281
+ return { outcome: { kind: "localInvalid" }, hash: lastHash };
62282
+ }
62283
+ if (local.kind === "absent") {
62284
+ return { outcome: { kind: "absent" }, hash: lastHash };
62285
+ }
62286
+ if (local.kind === "hashFailed") {
62116
62287
  return { outcome: { kind: "hashFailed" }, hash: lastHash };
62117
62288
  }
62289
+ const { hash } = local;
62118
62290
  if (hash === lastHash) {
62119
62291
  return { outcome: { kind: "unchanged" }, hash: lastHash };
62120
62292
  }
@@ -62129,6 +62301,256 @@ async function sync(params) {
62129
62301
  return { outcome: { kind: "uploaded" }, hash };
62130
62302
  }
62131
62303
 
62304
+ // src/vault-sync.ts
62305
+ init_esm_shims();
62306
+ import { createHash as createHash10 } from "node:crypto";
62307
+ import { dirname as dirname6 } from "node:path";
62308
+ async function syncVault(params) {
62309
+ const { label, path, bucket, key, store, lastHash, fileOps, log } = params;
62310
+ if (store === null || bucket === null || key === null) {
62311
+ return { outcome: { kind: "disabled" }, hash: lastHash };
62312
+ }
62313
+ const location = `s3://${bucket}/${key}`;
62314
+ let remote;
62315
+ try {
62316
+ remote = await store.get(key);
62317
+ } catch (error) {
62318
+ const reason = describeError(error);
62319
+ log(
62320
+ `WARNING: VAULT-REMOTE-UNREADABLE: could not read the shared ${label} credential at ${location}: ${reason}`
62321
+ );
62322
+ return { outcome: { kind: "remoteFailed", reason }, hash: lastHash };
62323
+ }
62324
+ let local;
62325
+ try {
62326
+ local = await readLocalCredential({ label, path, fileOps, log });
62327
+ } catch (error) {
62328
+ log(
62329
+ `WARNING: local ${label} credentials at ${path} are invalid or unreadable; keeping them and skipping sync. (${describeError(error)})`
62330
+ );
62331
+ return { outcome: { kind: "localInvalid" }, hash: lastHash };
62332
+ }
62333
+ if (local.kind === "invalid") {
62334
+ return { outcome: { kind: "localInvalid" }, hash: lastHash };
62335
+ }
62336
+ if (local.kind === "hashFailed") {
62337
+ return { outcome: { kind: "hashFailed" }, hash: lastHash };
62338
+ }
62339
+ if (remote === null) {
62340
+ if (local.kind === "absent") return { outcome: { kind: "absent" }, hash: lastHash };
62341
+ let body;
62342
+ try {
62343
+ body = await fileOps.readFile(path);
62344
+ } catch (error) {
62345
+ log(
62346
+ `WARNING: local ${label} credentials at ${path} are invalid or unreadable; keeping them and skipping sync. (${describeError(error)})`
62347
+ );
62348
+ return { outcome: { kind: "localInvalid" }, hash: lastHash };
62349
+ }
62350
+ return publish(
62351
+ { ...params, bucket, key, store, lastHash, log },
62352
+ body,
62353
+ local.hash,
62354
+ null,
62355
+ location
62356
+ );
62357
+ }
62358
+ const remoteHash = hashBytes(remote.body);
62359
+ const remoteValid = isValidJsonBytes(remote.body);
62360
+ if (local.kind === "absent") {
62361
+ if (!remoteValid) {
62362
+ log(
62363
+ `WARNING: VAULT-REMOTE-INVALID: the shared ${label} credential at ${location} is not valid JSON; keeping this runner's local file and not adopting it.`
62364
+ );
62365
+ return { outcome: { kind: "remoteInvalid" }, hash: lastHash };
62366
+ }
62367
+ return adopt2(
62368
+ { ...params, bucket, key, store, lastHash, log },
62369
+ remote.body,
62370
+ remoteHash,
62371
+ location
62372
+ );
62373
+ }
62374
+ let localBody;
62375
+ try {
62376
+ localBody = await fileOps.readFile(path);
62377
+ } catch (error) {
62378
+ log(
62379
+ `WARNING: local ${label} credentials at ${path} are invalid or unreadable; keeping them and skipping sync. (${describeError(error)})`
62380
+ );
62381
+ return { outcome: { kind: "localInvalid" }, hash: lastHash };
62382
+ }
62383
+ if (remoteValid && remoteHash === local.hash) {
62384
+ return { outcome: { kind: "unchanged" }, hash: remoteHash };
62385
+ }
62386
+ if (!remoteValid) {
62387
+ if (remote.etag === null) {
62388
+ return noEtag({ label, location, lastHash, log });
62389
+ }
62390
+ log(
62391
+ `WARNING: VAULT-REMOTE-INVALID: the shared ${label} credential at ${location} is not valid JSON; replacing it with this runner's local credential.`
62392
+ );
62393
+ return publish(
62394
+ { ...params, bucket, key, store, lastHash, log },
62395
+ localBody,
62396
+ local.hash,
62397
+ remote,
62398
+ location
62399
+ );
62400
+ }
62401
+ if (remoteHash !== lastHash) {
62402
+ return adopt2(
62403
+ { ...params, bucket, key, store, lastHash, log },
62404
+ remote.body,
62405
+ remoteHash,
62406
+ location
62407
+ );
62408
+ }
62409
+ if (remote.etag === null) {
62410
+ return noEtag({ label, location, lastHash, log });
62411
+ }
62412
+ return publish(
62413
+ { ...params, bucket, key, store, lastHash, log },
62414
+ localBody,
62415
+ local.hash,
62416
+ remote,
62417
+ location
62418
+ );
62419
+ }
62420
+ function isVaultCredentialPersisted(outcome) {
62421
+ switch (outcome.kind) {
62422
+ case "disabled":
62423
+ case "absent":
62424
+ case "unchanged":
62425
+ case "published":
62426
+ case "adopted":
62427
+ return true;
62428
+ case "localInvalid":
62429
+ case "hashFailed":
62430
+ case "adoptFailed":
62431
+ case "contended":
62432
+ case "remoteFailed":
62433
+ case "remoteInvalid":
62434
+ case "noEtag":
62435
+ return false;
62436
+ default:
62437
+ return assertNever(outcome, "vault sync outcome");
62438
+ }
62439
+ }
62440
+ async function publish(params, body, hash, remote, location) {
62441
+ const { label, key, store, lastHash, log } = params;
62442
+ if (store === null || key === null) return { outcome: { kind: "disabled" }, hash: lastHash };
62443
+ if (remote !== null && remote.etag === null) {
62444
+ return noEtag({ label, location, lastHash, log });
62445
+ }
62446
+ let result;
62447
+ try {
62448
+ if (remote === null) {
62449
+ result = await store.putIfAbsent(key, body);
62450
+ } else {
62451
+ const { etag } = remote;
62452
+ if (etag === null) return noEtag({ label, location, lastHash, log });
62453
+ result = await store.putIfMatch(key, body, etag);
62454
+ }
62455
+ } catch (error) {
62456
+ const reason = describeError(error);
62457
+ log(`WARNING: ${label} credentials upload failed to ${location}: ${reason}`);
62458
+ return { outcome: { kind: "remoteFailed", reason }, hash: lastHash };
62459
+ }
62460
+ if (result === "written") {
62461
+ const mode = remote === null ? "created" : "replaced";
62462
+ const previous = remote === null ? "" : ` previous=${shortHash(lastHash ?? hashBytes(remote.body))}`;
62463
+ log(
62464
+ `VAULT-PUBLISHED: ${mode === "created" ? "created" : "replaced"} the shared ${label} credential at ${location} hash=${shortHash(hash)}${previous}`
62465
+ );
62466
+ return { outcome: { kind: "published", mode }, hash };
62467
+ }
62468
+ log(
62469
+ `WARNING: VAULT-CONTENDED: another writer changed the shared ${label} credential at ${location} first; adopting theirs instead of overwriting it.`
62470
+ );
62471
+ let winner;
62472
+ try {
62473
+ winner = await store.get(key);
62474
+ } catch (error) {
62475
+ return unresolvedContention(params, location, describeError(error));
62476
+ }
62477
+ if (winner === null)
62478
+ return unresolvedContention(params, location, "the winning object is absent");
62479
+ if (!isValidJsonBytes(winner.body)) {
62480
+ return unresolvedContention(params, location, "the winning object is not valid JSON");
62481
+ }
62482
+ const winnerHash = hashBytes(winner.body);
62483
+ if (winnerHash === hash) return { outcome: { kind: "unchanged" }, hash: winnerHash };
62484
+ return adopt2(params, winner.body, winnerHash, location);
62485
+ }
62486
+ async function adopt2(params, body, hash, location) {
62487
+ const { label, path, fileOps, log, lastHash } = params;
62488
+ const dir = dirname6(path);
62489
+ try {
62490
+ await fileOps.mkdirp(dir);
62491
+ } catch (error) {
62492
+ return adoptFailed(params, location, describeError(error));
62493
+ }
62494
+ let installation;
62495
+ try {
62496
+ installation = await installCredentialBytes({
62497
+ label,
62498
+ path,
62499
+ body,
62500
+ source: location,
62501
+ fileOps,
62502
+ log
62503
+ });
62504
+ } catch (error) {
62505
+ return adoptFailed(params, location, describeError(error));
62506
+ }
62507
+ if (installation.kind === "installed") {
62508
+ log(
62509
+ `VAULT-ADOPTED: installed the shared ${label} credential from ${location} hash=${shortHash(hash)} previous=${lastHash === null ? "none" : shortHash(lastHash)}`
62510
+ );
62511
+ return { outcome: { kind: "adopted" }, hash };
62512
+ }
62513
+ return adoptFailed(
62514
+ params,
62515
+ location,
62516
+ installation.kind === "invalid" ? "the winning object was not valid JSON" : installation.reason
62517
+ );
62518
+ }
62519
+ function adoptFailed(params, location, reason) {
62520
+ const { label, log, lastHash } = params;
62521
+ log(
62522
+ `WARNING: VAULT-ADOPT-FAILED: could not install the shared ${label} credential from ${location} (${reason}); keeping the local file and retrying on the next tick.`
62523
+ );
62524
+ return { outcome: { kind: "adoptFailed" }, hash: lastHash };
62525
+ }
62526
+ function unresolvedContention(params, location, reason) {
62527
+ const { label, log, lastHash } = params;
62528
+ log(
62529
+ `WARNING: VAULT-CONTENDED-UNRESOLVED: could not read the winning ${label} credential at ${location} (${reason}); retrying on the next tick.`
62530
+ );
62531
+ return { outcome: { kind: "contended" }, hash: lastHash };
62532
+ }
62533
+ function noEtag({
62534
+ label,
62535
+ location,
62536
+ lastHash,
62537
+ log
62538
+ }) {
62539
+ log(
62540
+ `WARNING: VAULT-NO-ETAG: the shared ${label} credential at ${location} came back without an ETag; skipping publication this tick rather than overwriting it unconditionally.`
62541
+ );
62542
+ return {
62543
+ outcome: { kind: "noEtag" },
62544
+ hash: lastHash
62545
+ };
62546
+ }
62547
+ function hashBytes(body) {
62548
+ return createHash10("sha256").update(body).digest("hex");
62549
+ }
62550
+ function shortHash(hash) {
62551
+ return hash.slice(0, 12);
62552
+ }
62553
+
62132
62554
  // src/cli.ts
62133
62555
  var EXIT_NOT_READY = 10;
62134
62556
  var EXIT_KEEP_TASK = 20;
@@ -62147,7 +62569,8 @@ var USAGE = `Usage: runner-synchroniser <command> [args]
62147
62569
  env print the resolved config as shell-sourceable lines
62148
62570
  litestream-config print the generated litestream.yml
62149
62571
  restore <claude|opencode> restore a credential store from the object store
62150
- sync-once <claude|opencode> upload a credential store if it changed
62572
+ sync-once <claude|opencode> synchronize a credential store; shared-vault mode may
62573
+ adopt or conditionally publish credentials
62151
62574
  model-auth-ready exit 0 when some model auth is configured, ${EXIT_NOT_READY} when not
62152
62575
  self-stop scale this agent's own ECS service to 0; exit 0 only
62153
62576
  when desiredCount is confirmed 0, ${EXIT_KEEP_TASK} to keep the task
@@ -62177,8 +62600,11 @@ function awsSelfStopAccessFor(config) {
62177
62600
  function nodeSqliteIntegrityFor() {
62178
62601
  return nodeSqliteIntegrity;
62179
62602
  }
62180
- function litestreamCliFor(configPath, dbPath) {
62181
- return new LitestreamCli(configPath, dbPath);
62603
+ function litestreamCliFor(configPath, dbPath, config) {
62604
+ return new LitestreamCli(configPath, dbPath, {
62605
+ restorePointsTimeoutSeconds: config.restorePointsTimeoutSeconds,
62606
+ restoreCandidateTimeoutSeconds: config.restoreCandidateTimeoutSeconds
62607
+ });
62182
62608
  }
62183
62609
  function parseStoreName(value) {
62184
62610
  return value === "claude" || value === "opencode" ? value : null;
@@ -62277,7 +62703,7 @@ async function commandSyncOnce(name, config, deps) {
62277
62703
  const store = storeOf(config, name);
62278
62704
  const statePath = `${store.path}${STATE_SUFFIX}`;
62279
62705
  const lastHash = await readLastHash(fileOps, statePath, log);
62280
- const result = await sync({
62706
+ const params = {
62281
62707
  label: store.label,
62282
62708
  path: store.path,
62283
62709
  bucket: config.bucket,
@@ -62286,10 +62712,14 @@ async function commandSyncOnce(name, config, deps) {
62286
62712
  lastHash,
62287
62713
  fileOps,
62288
62714
  log
62289
- });
62715
+ };
62716
+ const result = config.vaultPrefix === null ? await sync(params) : await syncVault(params);
62290
62717
  if (result.hash !== null && result.hash !== lastHash) {
62291
62718
  await writeLastHash(fileOps, statePath, result.hash, log);
62292
62719
  }
62720
+ if (config.vaultPrefix !== null) {
62721
+ return isVaultCredentialPersisted(result.outcome) ? 0 : EXIT_SYNC_NOT_PERSISTED;
62722
+ }
62293
62723
  return result.outcome.kind === "failed" || result.outcome.kind === "hashFailed" || result.outcome.kind === "localInvalid" ? EXIT_SYNC_NOT_PERSISTED : 0;
62294
62724
  }
62295
62725
  async function commandSessionDbClassify(args, config, deps) {
@@ -62334,7 +62764,8 @@ async function commandSessionDbVerify(litestreamConfigPath, config, deps) {
62334
62764
  const { fileOps, log } = deps;
62335
62765
  const litestream = (deps.litestreamFor ?? litestreamCliFor)(
62336
62766
  litestreamConfigPath,
62337
- config.opencodeDbPath
62767
+ config.opencodeDbPath,
62768
+ config
62338
62769
  );
62339
62770
  const outcome = await verifySessionDb({
62340
62771
  config: { opencodeDbPath: config.opencodeDbPath, prefix: config.prefix },
@@ -62344,7 +62775,8 @@ async function commandSessionDbVerify(litestreamConfigPath, config, deps) {
62344
62775
  litestream,
62345
62776
  log,
62346
62777
  maxPoints: config.walkbackMaxPoints,
62347
- budgetSeconds: config.walkbackBudgetSeconds
62778
+ budgetSeconds: config.walkbackBudgetSeconds,
62779
+ disposalBudgetSeconds: config.disposalBudgetSeconds
62348
62780
  });
62349
62781
  log(describeSessionDbVerification(outcome));
62350
62782
  if (outcome.kind === "exhausted" && sessionDbVerifyExitCode(outcome) === 34) {
@@ -62377,8 +62809,14 @@ ${USAGE}`);
62377
62809
  const config = resolveConfig(env4);
62378
62810
  if (command12 === "env") {
62379
62811
  if (config.bucket === null) {
62812
+ const vaultPrefixIgnored = (env4.CREDENTIAL_VAULT_PREFIX?.trim() ?? "") !== "";
62813
+ log(
62814
+ "WARNING: credential persistence is DISABLED (LITESTREAM_BUCKET/LITESTREAM_PREFIX unset). Credentials stay on this task's ephemeral disk and are LOST when it is replaced." + (vaultPrefixIgnored ? " The requested CREDENTIAL_VAULT_PREFIX is ignored because persistence requires both LITESTREAM_BUCKET and LITESTREAM_PREFIX." : "")
62815
+ );
62816
+ }
62817
+ if (config.vaultPrefix !== null) {
62380
62818
  log(
62381
- "WARNING: credential persistence is DISABLED (LITESTREAM_BUCKET/LITESTREAM_PREFIX unset). Credentials stay on this task's ephemeral disk and are LOST when it is replaced."
62819
+ `VAULT-MODE: both Claude and OpenCode credential files are shared through bucket=${config.bucket} prefix=${config.vaultPrefix} instead of this runner's own prefix.`
62382
62820
  );
62383
62821
  }
62384
62822
  out(renderEnv(config));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@evident-ai/runner-synchroniser",
3
- "version": "3.5.2-dev.1c27282",
3
+ "version": "3.5.2-dev.4f7b3ac",
4
4
  "description": "Restores and syncs the Evident runner's OpenCode credential stores (and litestream config) to an object store, so a runner survives task replacement with almost no state loss.",
5
5
  "type": "module",
6
6
  "main": "./dist/cli.js",