@edgehero/pi-dispatch 1.10.3 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/.env.example +303 -150
  2. package/README.md +52 -0
  3. package/deploy/com.pi-dispatch.worker.plist +10 -4
  4. package/deploy/docker-compose.yml +49 -16
  5. package/deploy/egress-proxy.conf +32 -2
  6. package/deploy/nssm-install.cmd +12 -6
  7. package/deploy/pi-dispatch-egress-out.network +10 -0
  8. package/deploy/pi-dispatch-egress-proxy.container +50 -0
  9. package/deploy/pi-dispatch-netns-keeper.container +80 -0
  10. package/deploy/pi-dispatch-netns-keeper.network +18 -0
  11. package/deploy/pi-dispatch-valkey.container +51 -0
  12. package/deploy/pi-dispatch-valkey.network +16 -0
  13. package/deploy/receiver.service +6 -0
  14. package/deploy/worker-env-wrapper.cmd +12 -1
  15. package/deploy/worker-env-wrapper.sh +63 -37
  16. package/deploy/worker.service +18 -8
  17. package/package.json +15 -5
  18. package/src/azure-host.mjs +19 -0
  19. package/src/azure-identity.mjs +18 -2
  20. package/src/backend-conformance.mjs +71 -18
  21. package/src/backend-local.mjs +637 -21
  22. package/src/backend-podman.mjs +1168 -0
  23. package/src/backend-registry.mjs +86 -3
  24. package/src/backends.mjs +489 -37
  25. package/src/branch.mjs +7 -2
  26. package/src/cancel-cli.mjs +174 -0
  27. package/src/cancel-state.mjs +125 -0
  28. package/src/cli.mjs +188 -90
  29. package/src/config.mjs +503 -43
  30. package/src/connection.mjs +374 -8
  31. package/src/container-spec.mjs +102 -7
  32. package/src/daemon-facts.mjs +167 -0
  33. package/src/deployment-venue.mjs +158 -0
  34. package/src/docker-run.mjs +146 -15
  35. package/src/doctor.mjs +4756 -394
  36. package/src/egress-conf-copy.mjs +166 -0
  37. package/src/egress-proxy-state.mjs +151 -0
  38. package/src/egress.mjs +456 -25
  39. package/src/entry.mjs +27 -0
  40. package/src/env-allowlist.mjs +245 -40
  41. package/src/env-file.mjs +1869 -33
  42. package/src/exit-code.mjs +15 -0
  43. package/src/flow-gate.mjs +5 -3
  44. package/src/forgejo-host.mjs +19 -0
  45. package/src/forgejo-identity.mjs +21 -2
  46. package/src/get-token.mjs +67 -18
  47. package/src/git-dirty.mjs +9 -1
  48. package/src/git-hardening.mjs +33 -0
  49. package/src/github-app-setup.mjs +29 -12
  50. package/src/github-prompt.mjs +4 -1
  51. package/src/gitlab-host.mjs +19 -0
  52. package/src/gitlab-identity.mjs +19 -2
  53. package/src/host-pi.mjs +19 -3
  54. package/src/host-registry.mjs +29 -2
  55. package/src/identity.mjs +29 -4
  56. package/src/image-preflight.mjs +46 -11
  57. package/src/image-ref.mjs +21 -0
  58. package/src/index.mjs +363 -13
  59. package/src/init.mjs +197 -38
  60. package/src/job-user.mjs +252 -0
  61. package/src/json-duplicates.mjs +204 -0
  62. package/src/live-probes.mjs +1020 -0
  63. package/src/materialize.mjs +4 -11
  64. package/src/netns-keeper.mjs +264 -0
  65. package/src/on-failure.mjs +119 -0
  66. package/src/outbox.mjs +7 -0
  67. package/src/packages.mjs +2 -2
  68. package/src/podman-stack.mjs +1304 -0
  69. package/src/prepare-github.mjs +6 -6
  70. package/src/prepare-local.mjs +51 -17
  71. package/src/prepare.mjs +27 -6
  72. package/src/pricing.mjs +9 -5
  73. package/src/processor.mjs +506 -26
  74. package/src/provider-key.mjs +66 -0
  75. package/src/provider-steering.mjs +185 -0
  76. package/src/queue.mjs +35 -8
  77. package/src/redact.mjs +84 -0
  78. package/src/reserved-env.mjs +7 -3
  79. package/src/retention-sweep.mjs +178 -0
  80. package/src/run-container.mjs +181 -14
  81. package/src/run-history.mjs +105 -16
  82. package/src/runtime-observations.mjs +1152 -0
  83. package/src/runtime-settings.mjs +13 -8
  84. package/src/sandbox-cli.mjs +100 -95
  85. package/src/sandbox-store.mjs +612 -45
  86. package/src/sandbox.mjs +1459 -37
  87. package/src/schedules.mjs +16 -3
  88. package/src/secret-profiles.mjs +2 -1
  89. package/src/secrets.mjs +24 -6
  90. package/src/service-env.mjs +247 -0
  91. package/src/service.mjs +618 -28
  92. package/src/session-store.mjs +678 -53
  93. package/src/start.mjs +1348 -326
  94. package/src/subscriptions.mjs +7 -3
  95. package/src/transient.mjs +240 -0
  96. package/src/triggers-file.mjs +71 -15
  97. package/src/triggers.mjs +179 -19
  98. package/src/up.mjs +1399 -85
  99. package/src/valkey-auth.mjs +529 -0
  100. package/src/valkey-endpoint.mjs +367 -0
  101. package/src/watch-closer.mjs +158 -0
@@ -1,8 +1,12 @@
1
1
  /**
2
2
  * Operator-declared subscription plans (issue #53): the one place a flat-rate plan's real price can be
3
- * stated. Subscription-backed providers ship all-zero rate tables (pi-ai's kimi-coding and zai-coding-cn
4
- * both do), so their runs record cost 0 and read as FREE when they are PREPAID -- and the env boundary
5
- * REFUSES OAuth/subscription logins on purpose (env-allowlist.mjs: an expiring token cannot power an
3
+ * stated. pi-ai's rate table for a subscription-backed provider does not state the plan's price: at 0.80.7
4
+ * kimi-coding and zai-coding-cn shipped all-zero tables, so their runs recorded cost 0 and read as FREE
5
+ * when they were PREPAID; at the 0.99.1 pin most of their models carry an implied API-equivalent rate
6
+ * instead (issue #509), so the same prepaid runs now record a POSITIVE cost and read as METERED spend.
7
+ * Either way the table is the wrong price, and a zero rate is no longer even a hint that a plan exists
8
+ * (the qwen-token-plan family is all-zero, and so are free models of metered providers). And the env
9
+ * boundary REFUSES OAuth/subscription logins on purpose (env-allowlist.mjs: an expiring token cannot power an
6
10
  * unattended service), so no credential ever reaches the worker that could name the plan. An operator-side
7
11
  * declaration is therefore the only honest price source, and `subscriptions.json` is that declaration.
8
12
  *
@@ -0,0 +1,240 @@
1
+ /**
2
+ * One stated rule for the question `CONST-RETRY-INFRA-ONLY` asks at every failure site: would the same
3
+ * job, with the same inputs, refuse identically an hour from now with nobody touching anything?
4
+ *
5
+ * Absence, a bad credential, a malformed file, a URL that will always redirect, a certificate the host
6
+ * will always distrust: yes, determinate, and the operator has to fix it. A rate limit, a busy or
7
+ * unreachable filesystem, a network timeout, a forge that answered 502: no.
8
+ *
9
+ * WHY THIS EXISTS AS A MODULE (issue #316). The distinction was drawn twice, differently, one file apart:
10
+ * `gitlab-host.mjs` and `forgejo-host.mjs` classify a fetch rejection, a non-ok status and an unparseable
11
+ * body as `InfraRetry`, while `gitlab-identity.mjs`, `forgejo-identity.mjs`, `azure-identity.mjs` and
12
+ * `identity.mjs` tagged those same three conditions `piDispatchConfig`. Two precedents and no rule, which
13
+ * is how `classifyAppMintError` came to send a GitHub primary rate limit to the determinate side. #310
14
+ * made that expensive: the tag now means never retried, budget refunded, and a PUBLIC comment telling the
15
+ * issue author that the operator's deployment is misconfigured.
16
+ *
17
+ * IMPORT-FREE, and it has to stay that way. The identity modules are re-exported to the receiver through
18
+ * `worker/package.json`'s export map, and the receiver has no business importing `processor.mjs` to reach
19
+ * `InfraRetry`. So the rule is a set of predicates over a status, an errno and a rejection, and each
20
+ * caller pairs it with the throw its own layer can afford: `InfraRetry` where the processor will retry, an
21
+ * UNTAGGED throw at boot, where `entryExitCode` maps untagged to 1 (the supervisor restarts) and tagged to
22
+ * `EXIT_POLICY` (2), which `RestartPreventExitStatus=2` and `AppExit 2 Exit` deliberately leave stopped.
23
+ */
24
+
25
+ /**
26
+ * Filesystem and spawn errnos that mean the thing is not there, or can never be reached by that name.
27
+ *
28
+ * An ALLOW-LIST, not a deny-list, which is the shape `env-allowlist.mjs`'s `credentialFromPiAuth` already
29
+ * argues for and the reason the two agree: the expensive direction is the false determinate, so a code
30
+ * nobody has thought about lands on the retryable side. `EACCES` is transient on purpose and it is the
31
+ * case that motivates the whole set: a not-yet-mounted autofs path, a directory whose permissions a
32
+ * deploy is mid-way through changing, and a genuinely wrong chmod are indistinguishable from one stat,
33
+ * and only the last one is the operator's to fix.
34
+ *
35
+ * `ENOTDIR` joins `ENOENT` because a path component that is a file is the same answer as absence. `ELOOP`
36
+ * and `ENAMETOOLONG` are properties of the PATH rather than of the filesystem's mood: a symlink cycle and
37
+ * an over-long name resolve identically forever, and retrying either is paying to be told so twice.
38
+ */
39
+ export const DETERMINATE_FS_CODES = new Set(["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"]);
40
+
41
+ /** True when this errno means the thing is absent or unreachable by that name, rather than out of reach now. */
42
+ export function isDeterminateFsCode(code) {
43
+ return DETERMINATE_FS_CODES.has(code);
44
+ }
45
+
46
+ /**
47
+ * HTTP statuses that are transient on their own, regardless of headers.
48
+ *
49
+ * 429 is the obvious one. 408 (request timeout) and 425 (too early) are the two the old catch-all in
50
+ * `classifyAppMintError` swallowed alongside it, both of them by definition about timing.
51
+ */
52
+ export const TRANSIENT_STATUSES = new Set([408, 425, 429]);
53
+
54
+ /**
55
+ * The two 5xx that are NOT about load. "This server does not implement that method" and "it does not
56
+ * speak that HTTP version" answer identically forever, and both are what a misconfigured reverse proxy in
57
+ * front of a self-hosted forge actually returns.
58
+ */
59
+ export const DETERMINATE_5XX = new Set([501, 505]);
60
+
61
+ /**
62
+ * Classify an HTTP status, with the headers available where they change the answer.
63
+ *
64
+ * `getHeader` is a FUNCTION rather than a headers object because the two callers hold two different
65
+ * shapes: octokit exposes `error.response.headers` as a lowercase-keyed plain object, and `fetch` exposes
66
+ * `res.headers.get(name)`. A shared helper that took one of them would make the other call site adapt at
67
+ * the point where a mistake is silent.
68
+ *
69
+ * The 403 arm is the defect this module was written for. GitHub answers a PRIMARY rate limit with 403 and
70
+ * `x-ratelimit-remaining: 0`, sending `x-ratelimit-reset` and NO `retry-after`; only the secondary limit
71
+ * usually sends `retry-after`. The old code tested `retry-after` alone, so the primary limit -- the one an
72
+ * ordinary busy deployment actually hits, and which clears within the hour -- fell through to the
73
+ * catch-all and was reported to the issue author as a misconfiguration.
74
+ *
75
+ * **A SECONDARY limit can arrive with NEITHER header**, which is why GitHub's own guidance is a three-rung
76
+ * ladder ending in "otherwise wait at least a minute", and why octokit's throttling plugin detects it by
77
+ * matching the response BODY rather than the headers. Headers alone are therefore not enough, and the
78
+ * callers that hold a message pass it here; see `saysRateLimited`.
79
+ *
80
+ * A status that is absent or is not a positive integer is INDETERMINATE and therefore transient. That is
81
+ * not hypothetical: `@octokit/request-error` sets `status = 0` when the status will not parse, and `0` is
82
+ * a number, so the old `status !== undefined` catch-all turned "we could not tell what happened" into a
83
+ * permanent refusal. The same split `makeImagePreflight` draws with `docker info`: a determinate negative
84
+ * is a policy answer, an unanswered probe is not.
85
+ */
86
+ export function isTransientStatus(status, getHeader = () => undefined, message = undefined) {
87
+ if (!Number.isInteger(status) || status <= 0) return true;
88
+ if (DETERMINATE_5XX.has(status)) return false;
89
+ if (status >= 500) return true;
90
+ if (TRANSIENT_STATUSES.has(status)) return true;
91
+ if (status === 403) {
92
+ // Secondary rate limit by header, then primary by quota, then the secondary limit that announces
93
+ // itself only in its body. Anything else answering 403 is a scope or credential problem, which is
94
+ // determinate.
95
+ const retryAfter = getHeader("retry-after");
96
+ if (retryAfter !== undefined && retryAfter !== null) return true;
97
+ if (String(getHeader("x-ratelimit-remaining") ?? "") === "0") return true;
98
+ if (saysRateLimited(message)) return true;
99
+ // A 403 whose body is not JSON did not come from the forge's API at all: it is a WAF or proxy
100
+ // interstitial in front of it, which is indeterminate rather than a refusal we can read.
101
+ if (getHeader("content-type") !== undefined && !isJsonContentType(getHeader)) return true;
102
+ }
103
+ return false;
104
+ }
105
+
106
+ /**
107
+ * Whether a response claims to carry JSON.
108
+ *
109
+ * The discriminator for an unparseable body, which is otherwise ambiguous in the expensive direction. A
110
+ * TRUNCATED JSON body is transient. An HTML page is not: it is a Cloudflare Access or OIDC portal
111
+ * answering instead of the forge, or Azure DevOps answering an expired PAT with **203 and a sign-in
112
+ * page** -- 203 is `ok`, so it reaches the parse rather than the status check, and an expired PAT is
113
+ * about as determinate as a fault gets.
114
+ */
115
+ export function isJsonContentType(getHeader) {
116
+ const type = String(getHeader("content-type") ?? "").toLowerCase();
117
+ return type.includes("application/json") || type.includes("+json");
118
+ }
119
+
120
+ /**
121
+ * Whether an error body says it is a rate limit, for the 403s where the headers do not.
122
+ *
123
+ * The vocabulary is GitHub's own and matches what octokit's throttling plugin looks for. Kept as a
124
+ * message test rather than a status rule because that is the only channel this case has.
125
+ */
126
+ export function saysRateLimited(message) {
127
+ return typeof message === "string" && /secondary rate limit|abuse detection|rate limit exceeded/i.test(message);
128
+ }
129
+
130
+ /**
131
+ * TLS and protocol faults that are determinate, and therefore the one family of connection failure that
132
+ * stays tagged.
133
+ *
134
+ * A CONNECTION is transient by default: refused, reset, timed out, DNS that did not answer. A TRUST
135
+ * failure is not. An instance behind a private CA fails the handshake identically forever until the
136
+ * operator sets `NODE_EXTRA_CA_CERTS`, which is the single commonest self-hosted GitLab misconfiguration
137
+ * and the reason `fetchFailureReason` unwraps the cause chain at all: the whole point of that unwrapping
138
+ * is to put "unable to verify the first certificate" in front of an operator so they can act on it, and
139
+ * reclassifying it as transient would answer that with a restart loop instead.
140
+ */
141
+ export const DETERMINATE_TLS_CODES = new Set([
142
+ "UNABLE_TO_GET_ISSUER_CERT",
143
+ "UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
144
+ "UNABLE_TO_VERIFY_LEAF_SIGNATURE",
145
+ "SELF_SIGNED_CERT_IN_CHAIN",
146
+ "DEPTH_ZERO_SELF_SIGNED_CERT",
147
+ "CERT_UNTRUSTED",
148
+ "CERT_HAS_EXPIRED",
149
+ "CERT_NOT_YET_VALID",
150
+ "ERR_TLS_CERT_ALTNAME_INVALID",
151
+ // `ERR_SSL_*` is matched by PREFIX beside this set (see `isDeterminateFetchFailure`); this one is
152
+ // named as the worked example of why, and because it is the commonest of them.
153
+ "ERR_SSL_WRONG_VERSION_NUMBER",
154
+ ]);
155
+
156
+ /**
157
+ * True when a rejection names a fault an operator has to fix, rather than one that may clear on its own.
158
+ *
159
+ * Walks the cause chain the way `fetchFailureReason` does, and `AggregateError.errors` beside it, because
160
+ * `fetch` reports a multi-address connect failure as an aggregate and a cause walk alone would step past
161
+ * the only frame carrying a code.
162
+ *
163
+ * TWO SIGNALS, and the second is deliberately narrow. The CODE is the real one, and every TLS failure
164
+ * Node actually produces carries it. The message is only consulted when a frame has NO code at all, for
165
+ * two cases where Node gives nothing else: a cause that lost its code on the way through a wrapper, and
166
+ * `redirect: "error"`, which every client here passes and which rejects with the bare string "unexpected
167
+ * redirect". A redirect is a URL-shaped fault wearing a connection failure's clothes: an `http://` URL for
168
+ * an https instance, or a host that 302s to an SSO portal, redirects identically forever.
169
+ *
170
+ * The code guard on the message test is not decoration. Node's DNS errors embed the HOSTNAME
171
+ * (`getaddrinfo ENOTFOUND certificates.corp`), so an unguarded `/certificate/i` over the whole chain makes
172
+ * every DNS failure at a host whose name contains that word look like a trust problem, which is the
173
+ * expensive direction.
174
+ *
175
+ * Deliberately NOT determinate: `ENOTFOUND` and `ECONNREFUSED`. A typo'd hostname and a wrong port are
176
+ * permanent, and are indistinguishable from a DNS outage or a forge that is down. They stay on the
177
+ * retryable side under this module's own allow-list rule, and that residual is named here rather than
178
+ * hidden.
179
+ */
180
+ export function isDeterminateFetchFailure(err) {
181
+ const seen = new Set();
182
+ const walk = (e, depth) => {
183
+ if (!e || depth > 6 || seen.has(e)) return false;
184
+ seen.add(e);
185
+ if (DETERMINATE_TLS_CODES.has(e.code)) return true;
186
+ // Every OpenSSL protocol alert, not just the two spelled out above. An appliance that only speaks
187
+ // TLS 1.0 answers `ERR_SSL_TLSV1_ALERT_PROTOCOL_VERSION`, which is as determinate as a distrusted
188
+ // certificate and was missed by a name list; the prefix is the derivable version of that list.
189
+ if (typeof e.code === "string" && e.code.startsWith("ERR_SSL_")) return true;
190
+ if (e.code === undefined && typeof e.message === "string") {
191
+ if (/certificate/i.test(e.message)) return true;
192
+ if (e.message === "unexpected redirect") return true;
193
+ }
194
+ if (Array.isArray(e.errors) && e.errors.some((inner) => walk(inner, depth + 1))) return true;
195
+ return walk(e.cause, depth + 1);
196
+ };
197
+ return walk(err, 0);
198
+ }
199
+
200
+ /**
201
+ * A transient failure, reported UNTAGGED.
202
+ *
203
+ * The absence of the tag is the whole payload: `cli.mjs` and `receiver/src/cli.mjs` both map a tagged
204
+ * throw to `EXIT_POLICY` (2), which `RestartPreventExitStatus=2` and nssm's `AppExit 2 Exit` deliberately
205
+ * leave stopped, and map everything else to 1, which restarts. A boot-path caller therefore cannot use
206
+ * `InfraRetry` (it lives in `processor.mjs`, which the receiver must not import) and does not need to:
207
+ * plain is exactly right, and naming the constructor keeps that a decision rather than an omission.
208
+ */
209
+ export function transientError(message, cause) {
210
+ return new Error(message, cause ? { cause } : undefined);
211
+ }
212
+
213
+ /**
214
+ * Read one header out of whatever shape the caller happens to hold.
215
+ *
216
+ * Both readers accept a plain object, a `Headers` and a `Map`, and the reason is that failing to read a
217
+ * header falls to the DETERMINATE side, which is the expensive one: a genuine rate limit read through the
218
+ * wrong accessor becomes a public accusation. `@octokit/request` v7 builds a plain object today, and
219
+ * `fetch` gives a `Headers`, but neither is guaranteed by anything in this repo, and a silent `undefined`
220
+ * is not a failure mode worth keeping for the sake of two fewer branches.
221
+ *
222
+ * `Object.hasOwn` matters: a bare `headers[name]` reads through `Object.prototype`, so a lookup for a
223
+ * header called `constructor` would answer with a function.
224
+ */
225
+ function headerReaderFor(headers) {
226
+ if (!headers) return () => undefined;
227
+ if (typeof headers.get === "function") return (name) => headers.get(name) ?? undefined;
228
+ if (typeof headers === "object") return (name) => (Object.hasOwn(headers, name) ? headers[name] : undefined);
229
+ return () => undefined;
230
+ }
231
+
232
+ /** Read a header off an octokit rejection's `error.response.headers` bag. */
233
+ export function octokitHeaderReader(error) {
234
+ return headerReaderFor(error?.response?.headers);
235
+ }
236
+
237
+ /** Read a header off a WHATWG `Response`. `redirect: "error"` responses still carry one. */
238
+ export function responseHeaderReader(res) {
239
+ return headerReaderFor(res?.headers);
240
+ }
@@ -13,11 +13,15 @@
13
13
  * every write now takes `<path>.lock` via exclusive create (`wx`), the session-store's idiom with its
14
14
  * two doctrines kept verbatim: EEXIST is the ONLY failure that means locked (anything else failed to
15
15
  * create the lock for its own reason and is reported as that reason), and a leaked lock is logged,
16
- * never thrown. What is NEW here, with no in-repo precedent, is the STALE TAKEOVER: a lock whose
17
- * mtime is older than LOCK_STALE_MS is unlinked and retaken once. The session store can afford to
18
- * discard on contention and let its reaper sweep a leak; this file cannot -- a crashed writer's lock
19
- * would otherwise wedge every trigger add, edit, delete and disarm on the deployment forever, and
20
- * there is no reaper whose beat covers it. The residual is the classic one: unlink-then-create is not
16
+ * never thrown. The STALE TAKEOVER started here and no longer has to argue for itself: a lock whose
17
+ * mtime is older than LOCK_STALE_MS is unlinked and retaken once, and the session store took the same
18
+ * idiom under issue #336, which refuted the premise this comment used to rest on -- that the store could
19
+ * afford to discard on contention and let its reaper sweep a leak. It could not: its reaper keys on a
20
+ * transcript, so a key whose first promotion died before one landed was never swept at all. The reason
21
+ * this file needed it FIRST still stands: a crashed writer's lock would wedge every trigger add, edit,
22
+ * delete and disarm on the deployment forever, and there is no reaper whose beat covers it. The two
23
+ * thresholds differ by 360 times because the work under the two locks does (see the session store's own
24
+ * constant). The residual is the classic one: unlink-then-create is not
21
25
  * atomic, so two writers racing a stale takeover can interleave in a window of milliseconds. That
22
26
  * window replaces today's always-open one, and the loser's write still validated through the shared
23
27
  * parser, so the file stays loadable; the lost update is one disarm or one edit, and the disarm
@@ -42,6 +46,7 @@
42
46
 
43
47
  import nodeFs from "node:fs";
44
48
  import { parseTriggers } from "./triggers.mjs";
49
+ import { findDuplicateKey } from "./json-duplicates.mjs";
45
50
 
46
51
  /**
47
52
  * A lock older than this is a crashed writer's, not a live one's: every write under it is a read,
@@ -63,8 +68,31 @@ function lockPathFor(triggersPath) {
63
68
  * Take the lock, with one stale takeover. Returns an fd, or null when a LIVE writer holds it.
64
69
  * Throws only for non-EEXIST failures -- the session-store doctrine: reporting a read-only dir or a
65
70
  * full disk as "locked" sends an operator hunting for a stuck lock file that does not exist.
71
+ *
72
+ * `now` is INJECTED and defaults to the real clock, so every production path is byte-identical. It
73
+ * exists for two reasons and the test is only the second of them.
74
+ *
75
+ * The staleness threshold had NO test, because pinning it meant a ten-second sleep; with a clock it is
76
+ * a two-line assertion, and a bound nothing exercises is a bound that drifts.
77
+ *
78
+ * And the comparison it feeds is between TWO DIFFERENT CLOCKS. `Date.now()` is this process's;
79
+ * `mtimeMs` is the FILESYSTEM's, which on a network mount is a server's. `OQ-031` already records two
80
+ * hosts sharing one working tree as a live hazard, and `triggers.json` is exactly the file such a
81
+ * deployment shares. BOTH signs of the skew are bad, and differently:
82
+ *
83
+ * - A server more than LOCK_STALE_MS BEHIND makes every LIVE lock read as stale, so the takeover fires
84
+ * on every attempt and the millisecond-wide double-take window this module concedes stops being a
85
+ * rare race and becomes the normal case. Worse than it sounds, because `releaseLock` unlinks by PATH
86
+ * rather than by fd: once A's lock is stolen, A's release deletes B's, so under sustained skew the
87
+ * lock is not merely racy, it is functionally absent.
88
+ * - A server AHEAD makes the difference negative, so the comparison is always true and a genuinely
89
+ * crashed writer's lock is NEVER swept. `writeTriggers` then refuses forever and `disarmTrigger`
90
+ * exhausts its retries, which means a spent one-shot never records its disarm and can fire again.
91
+ *
92
+ * Nothing here closes either; the seam is what let a test demonstrate the skew, and issue #293's
93
+ * clock-shifted CI run is what found it.
66
94
  */
67
- function takeLock(triggersPath, fs, log) {
95
+ function takeLock(triggersPath, fs, log, now = () => Date.now()) {
68
96
  const lock = lockPathFor(triggersPath);
69
97
  let sweptAgeMs = null;
70
98
  for (let attempt = 0; attempt < 2; attempt++) {
@@ -84,13 +112,13 @@ function takeLock(triggersPath, fs, log) {
84
112
  // The holder released between our open and our stat: the next loop iteration takes it.
85
113
  continue;
86
114
  }
87
- if (Date.now() - mtimeMs <= LOCK_STALE_MS) return null; // live writer; caller decides
115
+ if (now() - mtimeMs <= LOCK_STALE_MS) return null; // live writer; caller decides
88
116
  try {
89
117
  fs.unlinkSync(lock);
90
118
  } catch {
91
119
  // Someone else swept it first; the retry create answers who won.
92
120
  }
93
- sweptAgeMs = Math.round(Date.now() - mtimeMs);
121
+ sweptAgeMs = Math.round(now() - mtimeMs);
94
122
  }
95
123
  }
96
124
  return null;
@@ -165,8 +193,8 @@ function discardTmp(fs, tmp) {
165
193
  * Returns `{ ok: true }`, or `{ invalid }` for a validation failure OR a held lock; fs failures throw,
166
194
  * the contract this function has always had.
167
195
  */
168
- export function writeTriggers({ triggersPath, mutate, fs = nodeFs, log = () => {} }) {
169
- const fd = takeLock(triggersPath, fs, log);
196
+ export function writeTriggers({ triggersPath, mutate, fs = nodeFs, log = () => {}, now = () => Date.now() }) {
197
+ const fd = takeLock(triggersPath, fs, log, now);
170
198
  if (fd === null) {
171
199
  // Immediate, not retried: the callers sit on the pi TUI event loop, and the holder is a write
172
200
  // that finishes in milliseconds. The operator re-presses; the file was never touched.
@@ -174,12 +202,30 @@ export function writeTriggers({ triggersPath, mutate, fs = nodeFs, log = () => {
174
202
  }
175
203
  try {
176
204
  let current = [];
205
+ // The text ONLY IF THE PARSE ACCEPTED IT, which is the scanner's whole contract: it never has to
206
+ // decide whether malformed input is malformed, so it is not asked. Assigning it before the parse
207
+ // would hand it a truncated file and get back a confident answer about a key, when what happened is
208
+ // that the file was cut in half.
209
+ let parsedText = null;
177
210
  try {
178
- const raw = JSON.parse(fs.readFileSync(triggersPath, "utf8"));
179
- if (Array.isArray(raw?.triggers)) current = raw.triggers;
211
+ const raw = fs.readFileSync(triggersPath, "utf8");
212
+ const value = JSON.parse(raw);
213
+ parsedText = raw;
214
+ if (Array.isArray(value?.triggers)) current = value.triggers;
180
215
  } catch {
181
216
  // Missing/invalid file: start from empty; the validated atomic write below repairs it.
182
217
  }
218
+ // The repair posture above cannot extend to a duplicate key (issue #313), and the two are exactly
219
+ // separated by the parse. A file that does not parse is repairable and stays repairable. A file that
220
+ // PARSES is not missing, so "start from empty" would not repair it, it would delete the operator's
221
+ // trigger set; and rebuilding from `current` writes back the winning value with the shadowed one
222
+ // silently gone, which turns a divergence a reviewer could still find into one nobody ever can.
223
+ if (parsedText !== null) {
224
+ const duplicate = findDuplicateKey(parsedText);
225
+ if (duplicate) {
226
+ return { invalid: `triggers file has a duplicate key ${JSON.stringify(duplicate.key)} at ${duplicate.at}; refusing to rewrite a file whose reviewed value and running value differ: ${triggersPath}` };
227
+ }
228
+ }
183
229
  const next = mutate(current.map((t) => ({ ...t })));
184
230
  const text = serialize(next);
185
231
  try {
@@ -224,11 +270,11 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
224
270
  * not failure), or `{ invalid }` with an operator-actionable reason. NEVER throws, and NEVER repairs:
225
271
  * an unreadable file is `{ invalid }` with the bytes untouched.
226
272
  */
227
- export async function disarmTrigger({ triggersPath, index, number, flow, command, jobId, at, fs = nodeFs, log = () => {} }) {
273
+ export async function disarmTrigger({ triggersPath, index, number, flow, command, jobId, at, fs = nodeFs, log = () => {}, now = () => Date.now() }) {
228
274
  for (let attempt = 0; attempt < DISARM_LOCK_ATTEMPTS; attempt++) {
229
275
  let fd;
230
276
  try {
231
- fd = takeLock(triggersPath, fs, log);
277
+ fd = takeLock(triggersPath, fs, log, now);
232
278
  } catch (err) {
233
279
  return { invalid: `triggers file lock failed (${err?.code ?? "lock-error"}): ${triggersPath}` };
234
280
  }
@@ -238,14 +284,24 @@ export async function disarmTrigger({ triggersPath, index, number, flow, command
238
284
  }
239
285
  try {
240
286
  let raw;
287
+ let currentText;
241
288
  try {
242
- raw = JSON.parse(fs.readFileSync(triggersPath, "utf8"));
289
+ currentText = fs.readFileSync(triggersPath, "utf8");
290
+ raw = JSON.parse(currentText);
243
291
  } catch (err) {
244
292
  // NEVER repair-from-empty here: overwriting a file we could not read, to record one
245
293
  // disarm, would destroy the operator's trigger set. writeTriggers' repair posture is for
246
294
  // the operator CRUD path, where "missing" means "first trigger".
247
295
  return { invalid: `triggers file unreadable (${err?.code ?? "parse-error"}), disarm not written: ${triggersPath}` };
248
296
  }
297
+ // And a duplicate key is the destruction that comment thought it had closed (issue #313). This
298
+ // path parses, collapses the duplicate to its last value, and then rewrites the WHOLE file from
299
+ // the parsed array -- so the shadowed key is gone from disk, permanently, to record one disarm.
300
+ // A file that says two things is not a file to rewrite from one of them.
301
+ const duplicate = findDuplicateKey(currentText);
302
+ if (duplicate) {
303
+ return { invalid: `triggers file has a duplicate key ${JSON.stringify(duplicate.key)} at ${duplicate.at}, disarm not written: ${triggersPath}` };
304
+ }
249
305
  const entries = Array.isArray(raw?.triggers) ? raw.triggers : null;
250
306
  const entry = entries?.[index];
251
307
  if (!entry || typeof entry !== "object") {