@edgehero/pi-dispatch 1.10.3 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +303 -150
- package/README.md +52 -0
- package/deploy/com.pi-dispatch.worker.plist +10 -4
- package/deploy/docker-compose.yml +49 -16
- package/deploy/egress-proxy.conf +32 -2
- package/deploy/nssm-install.cmd +12 -6
- package/deploy/pi-dispatch-egress-out.network +10 -0
- package/deploy/pi-dispatch-egress-proxy.container +50 -0
- package/deploy/pi-dispatch-netns-keeper.container +80 -0
- package/deploy/pi-dispatch-netns-keeper.network +18 -0
- package/deploy/pi-dispatch-valkey.container +51 -0
- package/deploy/pi-dispatch-valkey.network +16 -0
- package/deploy/receiver.service +6 -0
- package/deploy/worker-env-wrapper.cmd +12 -1
- package/deploy/worker-env-wrapper.sh +63 -37
- package/deploy/worker.service +18 -8
- package/package.json +15 -5
- package/src/azure-host.mjs +19 -0
- package/src/azure-identity.mjs +18 -2
- package/src/backend-conformance.mjs +71 -18
- package/src/backend-local.mjs +637 -21
- package/src/backend-podman.mjs +1168 -0
- package/src/backend-registry.mjs +86 -3
- package/src/backends.mjs +489 -37
- package/src/branch.mjs +7 -2
- package/src/cancel-cli.mjs +174 -0
- package/src/cancel-state.mjs +125 -0
- package/src/cli.mjs +188 -90
- package/src/config.mjs +503 -43
- package/src/connection.mjs +374 -8
- package/src/container-spec.mjs +102 -7
- package/src/daemon-facts.mjs +167 -0
- package/src/deployment-venue.mjs +158 -0
- package/src/docker-run.mjs +146 -15
- package/src/doctor.mjs +4756 -394
- package/src/egress-conf-copy.mjs +166 -0
- package/src/egress-proxy-state.mjs +151 -0
- package/src/egress.mjs +456 -25
- package/src/entry.mjs +27 -0
- package/src/env-allowlist.mjs +245 -40
- package/src/env-file.mjs +1869 -33
- package/src/exit-code.mjs +15 -0
- package/src/flow-gate.mjs +5 -3
- package/src/forgejo-host.mjs +19 -0
- package/src/forgejo-identity.mjs +21 -2
- package/src/get-token.mjs +67 -18
- package/src/git-dirty.mjs +9 -1
- package/src/git-hardening.mjs +33 -0
- package/src/github-app-setup.mjs +29 -12
- package/src/github-prompt.mjs +4 -1
- package/src/gitlab-host.mjs +19 -0
- package/src/gitlab-identity.mjs +19 -2
- package/src/host-pi.mjs +19 -3
- package/src/host-registry.mjs +29 -2
- package/src/identity.mjs +29 -4
- package/src/image-preflight.mjs +46 -11
- package/src/image-ref.mjs +21 -0
- package/src/index.mjs +363 -13
- package/src/init.mjs +197 -38
- package/src/job-user.mjs +252 -0
- package/src/json-duplicates.mjs +204 -0
- package/src/live-probes.mjs +1020 -0
- package/src/materialize.mjs +4 -11
- package/src/netns-keeper.mjs +264 -0
- package/src/on-failure.mjs +119 -0
- package/src/outbox.mjs +7 -0
- package/src/packages.mjs +2 -2
- package/src/podman-stack.mjs +1304 -0
- package/src/prepare-github.mjs +6 -6
- package/src/prepare-local.mjs +51 -17
- package/src/prepare.mjs +27 -6
- package/src/pricing.mjs +9 -5
- package/src/processor.mjs +506 -26
- package/src/provider-key.mjs +66 -0
- package/src/provider-steering.mjs +185 -0
- package/src/queue.mjs +35 -8
- package/src/redact.mjs +84 -0
- package/src/reserved-env.mjs +7 -3
- package/src/retention-sweep.mjs +178 -0
- package/src/run-container.mjs +181 -14
- package/src/run-history.mjs +105 -16
- package/src/runtime-observations.mjs +1152 -0
- package/src/runtime-settings.mjs +13 -8
- package/src/sandbox-cli.mjs +100 -95
- package/src/sandbox-store.mjs +612 -45
- package/src/sandbox.mjs +1459 -37
- package/src/schedules.mjs +16 -3
- package/src/secret-profiles.mjs +2 -1
- package/src/secrets.mjs +24 -6
- package/src/service-env.mjs +247 -0
- package/src/service.mjs +618 -28
- package/src/session-store.mjs +678 -53
- package/src/start.mjs +1348 -326
- package/src/subscriptions.mjs +7 -3
- package/src/transient.mjs +240 -0
- package/src/triggers-file.mjs +71 -15
- package/src/triggers.mjs +179 -19
- package/src/up.mjs +1399 -85
- package/src/valkey-auth.mjs +529 -0
- package/src/valkey-endpoint.mjs +367 -0
- package/src/watch-closer.mjs +158 -0
package/src/subscriptions.mjs
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Operator-declared subscription plans (issue #53): the one place a flat-rate plan's real price can be
|
|
3
|
-
* stated.
|
|
4
|
-
*
|
|
5
|
-
*
|
|
3
|
+
* stated. pi-ai's rate table for a subscription-backed provider does not state the plan's price: at 0.80.7
|
|
4
|
+
* kimi-coding and zai-coding-cn shipped all-zero tables, so their runs recorded cost 0 and read as FREE
|
|
5
|
+
* when they were PREPAID; at the 0.99.1 pin most of their models carry an implied API-equivalent rate
|
|
6
|
+
* instead (issue #509), so the same prepaid runs now record a POSITIVE cost and read as METERED spend.
|
|
7
|
+
* Either way the table is the wrong price, and a zero rate is no longer even a hint that a plan exists
|
|
8
|
+
* (the qwen-token-plan family is all-zero, and so are free models of metered providers). And the env
|
|
9
|
+
* boundary REFUSES OAuth/subscription logins on purpose (env-allowlist.mjs: an expiring token cannot power an
|
|
6
10
|
* unattended service), so no credential ever reaches the worker that could name the plan. An operator-side
|
|
7
11
|
* declaration is therefore the only honest price source, and `subscriptions.json` is that declaration.
|
|
8
12
|
*
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One stated rule for the question `CONST-RETRY-INFRA-ONLY` asks at every failure site: would the same
|
|
3
|
+
* job, with the same inputs, refuse identically an hour from now with nobody touching anything?
|
|
4
|
+
*
|
|
5
|
+
* Absence, a bad credential, a malformed file, a URL that will always redirect, a certificate the host
|
|
6
|
+
* will always distrust: yes, determinate, and the operator has to fix it. A rate limit, a busy or
|
|
7
|
+
* unreachable filesystem, a network timeout, a forge that answered 502: no.
|
|
8
|
+
*
|
|
9
|
+
* WHY THIS EXISTS AS A MODULE (issue #316). The distinction was drawn twice, differently, one file apart:
|
|
10
|
+
* `gitlab-host.mjs` and `forgejo-host.mjs` classify a fetch rejection, a non-ok status and an unparseable
|
|
11
|
+
* body as `InfraRetry`, while `gitlab-identity.mjs`, `forgejo-identity.mjs`, `azure-identity.mjs` and
|
|
12
|
+
* `identity.mjs` tagged those same three conditions `piDispatchConfig`. Two precedents and no rule, which
|
|
13
|
+
* is how `classifyAppMintError` came to send a GitHub primary rate limit to the determinate side. #310
|
|
14
|
+
* made that expensive: the tag now means never retried, budget refunded, and a PUBLIC comment telling the
|
|
15
|
+
* issue author that the operator's deployment is misconfigured.
|
|
16
|
+
*
|
|
17
|
+
* IMPORT-FREE, and it has to stay that way. The identity modules are re-exported to the receiver through
|
|
18
|
+
* `worker/package.json`'s export map, and the receiver has no business importing `processor.mjs` to reach
|
|
19
|
+
* `InfraRetry`. So the rule is a set of predicates over a status, an errno and a rejection, and each
|
|
20
|
+
* caller pairs it with the throw its own layer can afford: `InfraRetry` where the processor will retry, an
|
|
21
|
+
* UNTAGGED throw at boot, where `entryExitCode` maps untagged to 1 (the supervisor restarts) and tagged to
|
|
22
|
+
* `EXIT_POLICY` (2), which `RestartPreventExitStatus=2` and `AppExit 2 Exit` deliberately leave stopped.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Filesystem and spawn errnos that mean the thing is not there, or can never be reached by that name.
|
|
27
|
+
*
|
|
28
|
+
* An ALLOW-LIST, not a deny-list, which is the shape `env-allowlist.mjs`'s `credentialFromPiAuth` already
|
|
29
|
+
* argues for and the reason the two agree: the expensive direction is the false determinate, so a code
|
|
30
|
+
* nobody has thought about lands on the retryable side. `EACCES` is transient on purpose and it is the
|
|
31
|
+
* case that motivates the whole set: a not-yet-mounted autofs path, a directory whose permissions a
|
|
32
|
+
* deploy is mid-way through changing, and a genuinely wrong chmod are indistinguishable from one stat,
|
|
33
|
+
* and only the last one is the operator's to fix.
|
|
34
|
+
*
|
|
35
|
+
* `ENOTDIR` joins `ENOENT` because a path component that is a file is the same answer as absence. `ELOOP`
|
|
36
|
+
* and `ENAMETOOLONG` are properties of the PATH rather than of the filesystem's mood: a symlink cycle and
|
|
37
|
+
* an over-long name resolve identically forever, and retrying either is paying to be told so twice.
|
|
38
|
+
*/
|
|
39
|
+
export const DETERMINATE_FS_CODES = new Set(["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"]);
|
|
40
|
+
|
|
41
|
+
/** True when this errno means the thing is absent or unreachable by that name, rather than out of reach now. */
|
|
42
|
+
export function isDeterminateFsCode(code) {
|
|
43
|
+
return DETERMINATE_FS_CODES.has(code);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* HTTP statuses that are transient on their own, regardless of headers.
|
|
48
|
+
*
|
|
49
|
+
* 429 is the obvious one. 408 (request timeout) and 425 (too early) are the two the old catch-all in
|
|
50
|
+
* `classifyAppMintError` swallowed alongside it, both of them by definition about timing.
|
|
51
|
+
*/
|
|
52
|
+
export const TRANSIENT_STATUSES = new Set([408, 425, 429]);
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The two 5xx that are NOT about load. "This server does not implement that method" and "it does not
|
|
56
|
+
* speak that HTTP version" answer identically forever, and both are what a misconfigured reverse proxy in
|
|
57
|
+
* front of a self-hosted forge actually returns.
|
|
58
|
+
*/
|
|
59
|
+
export const DETERMINATE_5XX = new Set([501, 505]);
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Classify an HTTP status, with the headers available where they change the answer.
|
|
63
|
+
*
|
|
64
|
+
* `getHeader` is a FUNCTION rather than a headers object because the two callers hold two different
|
|
65
|
+
* shapes: octokit exposes `error.response.headers` as a lowercase-keyed plain object, and `fetch` exposes
|
|
66
|
+
* `res.headers.get(name)`. A shared helper that took one of them would make the other call site adapt at
|
|
67
|
+
* the point where a mistake is silent.
|
|
68
|
+
*
|
|
69
|
+
* The 403 arm is the defect this module was written for. GitHub answers a PRIMARY rate limit with 403 and
|
|
70
|
+
* `x-ratelimit-remaining: 0`, sending `x-ratelimit-reset` and NO `retry-after`; only the secondary limit
|
|
71
|
+
* usually sends `retry-after`. The old code tested `retry-after` alone, so the primary limit -- the one an
|
|
72
|
+
* ordinary busy deployment actually hits, and which clears within the hour -- fell through to the
|
|
73
|
+
* catch-all and was reported to the issue author as a misconfiguration.
|
|
74
|
+
*
|
|
75
|
+
* **A SECONDARY limit can arrive with NEITHER header**, which is why GitHub's own guidance is a three-rung
|
|
76
|
+
* ladder ending in "otherwise wait at least a minute", and why octokit's throttling plugin detects it by
|
|
77
|
+
* matching the response BODY rather than the headers. Headers alone are therefore not enough, and the
|
|
78
|
+
* callers that hold a message pass it here; see `saysRateLimited`.
|
|
79
|
+
*
|
|
80
|
+
* A status that is absent or is not a positive integer is INDETERMINATE and therefore transient. That is
|
|
81
|
+
* not hypothetical: `@octokit/request-error` sets `status = 0` when the status will not parse, and `0` is
|
|
82
|
+
* a number, so the old `status !== undefined` catch-all turned "we could not tell what happened" into a
|
|
83
|
+
* permanent refusal. The same split `makeImagePreflight` draws with `docker info`: a determinate negative
|
|
84
|
+
* is a policy answer, an unanswered probe is not.
|
|
85
|
+
*/
|
|
86
|
+
export function isTransientStatus(status, getHeader = () => undefined, message = undefined) {
|
|
87
|
+
if (!Number.isInteger(status) || status <= 0) return true;
|
|
88
|
+
if (DETERMINATE_5XX.has(status)) return false;
|
|
89
|
+
if (status >= 500) return true;
|
|
90
|
+
if (TRANSIENT_STATUSES.has(status)) return true;
|
|
91
|
+
if (status === 403) {
|
|
92
|
+
// Secondary rate limit by header, then primary by quota, then the secondary limit that announces
|
|
93
|
+
// itself only in its body. Anything else answering 403 is a scope or credential problem, which is
|
|
94
|
+
// determinate.
|
|
95
|
+
const retryAfter = getHeader("retry-after");
|
|
96
|
+
if (retryAfter !== undefined && retryAfter !== null) return true;
|
|
97
|
+
if (String(getHeader("x-ratelimit-remaining") ?? "") === "0") return true;
|
|
98
|
+
if (saysRateLimited(message)) return true;
|
|
99
|
+
// A 403 whose body is not JSON did not come from the forge's API at all: it is a WAF or proxy
|
|
100
|
+
// interstitial in front of it, which is indeterminate rather than a refusal we can read.
|
|
101
|
+
if (getHeader("content-type") !== undefined && !isJsonContentType(getHeader)) return true;
|
|
102
|
+
}
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Whether a response claims to carry JSON.
|
|
108
|
+
*
|
|
109
|
+
* The discriminator for an unparseable body, which is otherwise ambiguous in the expensive direction. A
|
|
110
|
+
* TRUNCATED JSON body is transient. An HTML page is not: it is a Cloudflare Access or OIDC portal
|
|
111
|
+
* answering instead of the forge, or Azure DevOps answering an expired PAT with **203 and a sign-in
|
|
112
|
+
* page** -- 203 is `ok`, so it reaches the parse rather than the status check, and an expired PAT is
|
|
113
|
+
* about as determinate as a fault gets.
|
|
114
|
+
*/
|
|
115
|
+
export function isJsonContentType(getHeader) {
|
|
116
|
+
const type = String(getHeader("content-type") ?? "").toLowerCase();
|
|
117
|
+
return type.includes("application/json") || type.includes("+json");
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Whether an error body says it is a rate limit, for the 403s where the headers do not.
|
|
122
|
+
*
|
|
123
|
+
* The vocabulary is GitHub's own and matches what octokit's throttling plugin looks for. Kept as a
|
|
124
|
+
* message test rather than a status rule because that is the only channel this case has.
|
|
125
|
+
*/
|
|
126
|
+
export function saysRateLimited(message) {
|
|
127
|
+
return typeof message === "string" && /secondary rate limit|abuse detection|rate limit exceeded/i.test(message);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* TLS and protocol faults that are determinate, and therefore the one family of connection failure that
|
|
132
|
+
* stays tagged.
|
|
133
|
+
*
|
|
134
|
+
* A CONNECTION is transient by default: refused, reset, timed out, DNS that did not answer. A TRUST
|
|
135
|
+
* failure is not. An instance behind a private CA fails the handshake identically forever until the
|
|
136
|
+
* operator sets `NODE_EXTRA_CA_CERTS`, which is the single commonest self-hosted GitLab misconfiguration
|
|
137
|
+
* and the reason `fetchFailureReason` unwraps the cause chain at all: the whole point of that unwrapping
|
|
138
|
+
* is to put "unable to verify the first certificate" in front of an operator so they can act on it, and
|
|
139
|
+
* reclassifying it as transient would answer that with a restart loop instead.
|
|
140
|
+
*/
|
|
141
|
+
export const DETERMINATE_TLS_CODES = new Set([
|
|
142
|
+
"UNABLE_TO_GET_ISSUER_CERT",
|
|
143
|
+
"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
|
|
144
|
+
"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
|
|
145
|
+
"SELF_SIGNED_CERT_IN_CHAIN",
|
|
146
|
+
"DEPTH_ZERO_SELF_SIGNED_CERT",
|
|
147
|
+
"CERT_UNTRUSTED",
|
|
148
|
+
"CERT_HAS_EXPIRED",
|
|
149
|
+
"CERT_NOT_YET_VALID",
|
|
150
|
+
"ERR_TLS_CERT_ALTNAME_INVALID",
|
|
151
|
+
// `ERR_SSL_*` is matched by PREFIX beside this set (see `isDeterminateFetchFailure`); this one is
|
|
152
|
+
// named as the worked example of why, and because it is the commonest of them.
|
|
153
|
+
"ERR_SSL_WRONG_VERSION_NUMBER",
|
|
154
|
+
]);
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* True when a rejection names a fault an operator has to fix, rather than one that may clear on its own.
|
|
158
|
+
*
|
|
159
|
+
* Walks the cause chain the way `fetchFailureReason` does, and `AggregateError.errors` beside it, because
|
|
160
|
+
* `fetch` reports a multi-address connect failure as an aggregate and a cause walk alone would step past
|
|
161
|
+
* the only frame carrying a code.
|
|
162
|
+
*
|
|
163
|
+
* TWO SIGNALS, and the second is deliberately narrow. The CODE is the real one, and every TLS failure
|
|
164
|
+
* Node actually produces carries it. The message is only consulted when a frame has NO code at all, for
|
|
165
|
+
* two cases where Node gives nothing else: a cause that lost its code on the way through a wrapper, and
|
|
166
|
+
* `redirect: "error"`, which every client here passes and which rejects with the bare string "unexpected
|
|
167
|
+
* redirect". A redirect is a URL-shaped fault wearing a connection failure's clothes: an `http://` URL for
|
|
168
|
+
* an https instance, or a host that 302s to an SSO portal, redirects identically forever.
|
|
169
|
+
*
|
|
170
|
+
* The code guard on the message test is not decoration. Node's DNS errors embed the HOSTNAME
|
|
171
|
+
* (`getaddrinfo ENOTFOUND certificates.corp`), so an unguarded `/certificate/i` over the whole chain makes
|
|
172
|
+
* every DNS failure at a host whose name contains that word look like a trust problem, which is the
|
|
173
|
+
* expensive direction.
|
|
174
|
+
*
|
|
175
|
+
* Deliberately NOT determinate: `ENOTFOUND` and `ECONNREFUSED`. A typo'd hostname and a wrong port are
|
|
176
|
+
* permanent, and are indistinguishable from a DNS outage or a forge that is down. They stay on the
|
|
177
|
+
* retryable side under this module's own allow-list rule, and that residual is named here rather than
|
|
178
|
+
* hidden.
|
|
179
|
+
*/
|
|
180
|
+
export function isDeterminateFetchFailure(err) {
|
|
181
|
+
const seen = new Set();
|
|
182
|
+
const walk = (e, depth) => {
|
|
183
|
+
if (!e || depth > 6 || seen.has(e)) return false;
|
|
184
|
+
seen.add(e);
|
|
185
|
+
if (DETERMINATE_TLS_CODES.has(e.code)) return true;
|
|
186
|
+
// Every OpenSSL protocol alert, not just the two spelled out above. An appliance that only speaks
|
|
187
|
+
// TLS 1.0 answers `ERR_SSL_TLSV1_ALERT_PROTOCOL_VERSION`, which is as determinate as a distrusted
|
|
188
|
+
// certificate and was missed by a name list; the prefix is the derivable version of that list.
|
|
189
|
+
if (typeof e.code === "string" && e.code.startsWith("ERR_SSL_")) return true;
|
|
190
|
+
if (e.code === undefined && typeof e.message === "string") {
|
|
191
|
+
if (/certificate/i.test(e.message)) return true;
|
|
192
|
+
if (e.message === "unexpected redirect") return true;
|
|
193
|
+
}
|
|
194
|
+
if (Array.isArray(e.errors) && e.errors.some((inner) => walk(inner, depth + 1))) return true;
|
|
195
|
+
return walk(e.cause, depth + 1);
|
|
196
|
+
};
|
|
197
|
+
return walk(err, 0);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* A transient failure, reported UNTAGGED.
|
|
202
|
+
*
|
|
203
|
+
* The absence of the tag is the whole payload: `cli.mjs` and `receiver/src/cli.mjs` both map a tagged
|
|
204
|
+
* throw to `EXIT_POLICY` (2), which `RestartPreventExitStatus=2` and nssm's `AppExit 2 Exit` deliberately
|
|
205
|
+
* leave stopped, and map everything else to 1, which restarts. A boot-path caller therefore cannot use
|
|
206
|
+
* `InfraRetry` (it lives in `processor.mjs`, which the receiver must not import) and does not need to:
|
|
207
|
+
* plain is exactly right, and naming the constructor keeps that a decision rather than an omission.
|
|
208
|
+
*/
|
|
209
|
+
export function transientError(message, cause) {
|
|
210
|
+
return new Error(message, cause ? { cause } : undefined);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Read one header out of whatever shape the caller happens to hold.
|
|
215
|
+
*
|
|
216
|
+
* Both readers accept a plain object, a `Headers` and a `Map`, and the reason is that failing to read a
|
|
217
|
+
* header falls to the DETERMINATE side, which is the expensive one: a genuine rate limit read through the
|
|
218
|
+
* wrong accessor becomes a public accusation. `@octokit/request` v7 builds a plain object today, and
|
|
219
|
+
* `fetch` gives a `Headers`, but neither is guaranteed by anything in this repo, and a silent `undefined`
|
|
220
|
+
* is not a failure mode worth keeping for the sake of two fewer branches.
|
|
221
|
+
*
|
|
222
|
+
* `Object.hasOwn` matters: a bare `headers[name]` reads through `Object.prototype`, so a lookup for a
|
|
223
|
+
* header called `constructor` would answer with a function.
|
|
224
|
+
*/
|
|
225
|
+
function headerReaderFor(headers) {
|
|
226
|
+
if (!headers) return () => undefined;
|
|
227
|
+
if (typeof headers.get === "function") return (name) => headers.get(name) ?? undefined;
|
|
228
|
+
if (typeof headers === "object") return (name) => (Object.hasOwn(headers, name) ? headers[name] : undefined);
|
|
229
|
+
return () => undefined;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Read a header off an octokit rejection's `error.response.headers` bag. */
|
|
233
|
+
export function octokitHeaderReader(error) {
|
|
234
|
+
return headerReaderFor(error?.response?.headers);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Read a header off a WHATWG `Response`. `redirect: "error"` responses still carry one. */
|
|
238
|
+
export function responseHeaderReader(res) {
|
|
239
|
+
return headerReaderFor(res?.headers);
|
|
240
|
+
}
|
package/src/triggers-file.mjs
CHANGED
|
@@ -13,11 +13,15 @@
|
|
|
13
13
|
* every write now takes `<path>.lock` via exclusive create (`wx`), the session-store's idiom with its
|
|
14
14
|
* two doctrines kept verbatim: EEXIST is the ONLY failure that means locked (anything else failed to
|
|
15
15
|
* create the lock for its own reason and is reported as that reason), and a leaked lock is logged,
|
|
16
|
-
* never thrown.
|
|
17
|
-
* mtime is older than LOCK_STALE_MS is unlinked and retaken once
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
16
|
+
* never thrown. The STALE TAKEOVER started here and no longer has to argue for itself: a lock whose
|
|
17
|
+
* mtime is older than LOCK_STALE_MS is unlinked and retaken once, and the session store took the same
|
|
18
|
+
* idiom under issue #336, which refuted the premise this comment used to rest on -- that the store could
|
|
19
|
+
* afford to discard on contention and let its reaper sweep a leak. It could not: its reaper keys on a
|
|
20
|
+
* transcript, so a key whose first promotion died before one landed was never swept at all. The reason
|
|
21
|
+
* this file needed it FIRST still stands: a crashed writer's lock would wedge every trigger add, edit,
|
|
22
|
+
* delete and disarm on the deployment forever, and there is no reaper whose beat covers it. The two
|
|
23
|
+
* thresholds differ by 360 times because the work under the two locks does (see the session store's own
|
|
24
|
+
* constant). The residual is the classic one: unlink-then-create is not
|
|
21
25
|
* atomic, so two writers racing a stale takeover can interleave in a window of milliseconds. That
|
|
22
26
|
* window replaces today's always-open one, and the loser's write still validated through the shared
|
|
23
27
|
* parser, so the file stays loadable; the lost update is one disarm or one edit, and the disarm
|
|
@@ -42,6 +46,7 @@
|
|
|
42
46
|
|
|
43
47
|
import nodeFs from "node:fs";
|
|
44
48
|
import { parseTriggers } from "./triggers.mjs";
|
|
49
|
+
import { findDuplicateKey } from "./json-duplicates.mjs";
|
|
45
50
|
|
|
46
51
|
/**
|
|
47
52
|
* A lock older than this is a crashed writer's, not a live one's: every write under it is a read,
|
|
@@ -63,8 +68,31 @@ function lockPathFor(triggersPath) {
|
|
|
63
68
|
* Take the lock, with one stale takeover. Returns an fd, or null when a LIVE writer holds it.
|
|
64
69
|
* Throws only for non-EEXIST failures -- the session-store doctrine: reporting a read-only dir or a
|
|
65
70
|
* full disk as "locked" sends an operator hunting for a stuck lock file that does not exist.
|
|
71
|
+
*
|
|
72
|
+
* `now` is INJECTED and defaults to the real clock, so every production path is byte-identical. It
|
|
73
|
+
* exists for two reasons and the test is only the second of them.
|
|
74
|
+
*
|
|
75
|
+
* The staleness threshold had NO test, because pinning it meant a ten-second sleep; with a clock it is
|
|
76
|
+
* a two-line assertion, and a bound nothing exercises is a bound that drifts.
|
|
77
|
+
*
|
|
78
|
+
* And the comparison it feeds is between TWO DIFFERENT CLOCKS. `Date.now()` is this process's;
|
|
79
|
+
* `mtimeMs` is the FILESYSTEM's, which on a network mount is a server's. `OQ-031` already records two
|
|
80
|
+
* hosts sharing one working tree as a live hazard, and `triggers.json` is exactly the file such a
|
|
81
|
+
* deployment shares. BOTH signs of the skew are bad, and differently:
|
|
82
|
+
*
|
|
83
|
+
* - A server more than LOCK_STALE_MS BEHIND makes every LIVE lock read as stale, so the takeover fires
|
|
84
|
+
* on every attempt and the millisecond-wide double-take window this module concedes stops being a
|
|
85
|
+
* rare race and becomes the normal case. Worse than it sounds, because `releaseLock` unlinks by PATH
|
|
86
|
+
* rather than by fd: once A's lock is stolen, A's release deletes B's, so under sustained skew the
|
|
87
|
+
* lock is not merely racy, it is functionally absent.
|
|
88
|
+
* - A server AHEAD makes the difference negative, so the comparison is always true and a genuinely
|
|
89
|
+
* crashed writer's lock is NEVER swept. `writeTriggers` then refuses forever and `disarmTrigger`
|
|
90
|
+
* exhausts its retries, which means a spent one-shot never records its disarm and can fire again.
|
|
91
|
+
*
|
|
92
|
+
* Nothing here closes either; the seam is what let a test demonstrate the skew, and issue #293's
|
|
93
|
+
* clock-shifted CI run is what found it.
|
|
66
94
|
*/
|
|
67
|
-
function takeLock(triggersPath, fs, log) {
|
|
95
|
+
function takeLock(triggersPath, fs, log, now = () => Date.now()) {
|
|
68
96
|
const lock = lockPathFor(triggersPath);
|
|
69
97
|
let sweptAgeMs = null;
|
|
70
98
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
@@ -84,13 +112,13 @@ function takeLock(triggersPath, fs, log) {
|
|
|
84
112
|
// The holder released between our open and our stat: the next loop iteration takes it.
|
|
85
113
|
continue;
|
|
86
114
|
}
|
|
87
|
-
if (
|
|
115
|
+
if (now() - mtimeMs <= LOCK_STALE_MS) return null; // live writer; caller decides
|
|
88
116
|
try {
|
|
89
117
|
fs.unlinkSync(lock);
|
|
90
118
|
} catch {
|
|
91
119
|
// Someone else swept it first; the retry create answers who won.
|
|
92
120
|
}
|
|
93
|
-
sweptAgeMs = Math.round(
|
|
121
|
+
sweptAgeMs = Math.round(now() - mtimeMs);
|
|
94
122
|
}
|
|
95
123
|
}
|
|
96
124
|
return null;
|
|
@@ -165,8 +193,8 @@ function discardTmp(fs, tmp) {
|
|
|
165
193
|
* Returns `{ ok: true }`, or `{ invalid }` for a validation failure OR a held lock; fs failures throw,
|
|
166
194
|
* the contract this function has always had.
|
|
167
195
|
*/
|
|
168
|
-
export function writeTriggers({ triggersPath, mutate, fs = nodeFs, log = () => {} }) {
|
|
169
|
-
const fd = takeLock(triggersPath, fs, log);
|
|
196
|
+
export function writeTriggers({ triggersPath, mutate, fs = nodeFs, log = () => {}, now = () => Date.now() }) {
|
|
197
|
+
const fd = takeLock(triggersPath, fs, log, now);
|
|
170
198
|
if (fd === null) {
|
|
171
199
|
// Immediate, not retried: the callers sit on the pi TUI event loop, and the holder is a write
|
|
172
200
|
// that finishes in milliseconds. The operator re-presses; the file was never touched.
|
|
@@ -174,12 +202,30 @@ export function writeTriggers({ triggersPath, mutate, fs = nodeFs, log = () => {
|
|
|
174
202
|
}
|
|
175
203
|
try {
|
|
176
204
|
let current = [];
|
|
205
|
+
// The text ONLY IF THE PARSE ACCEPTED IT, which is the scanner's whole contract: it never has to
|
|
206
|
+
// decide whether malformed input is malformed, so it is not asked. Assigning it before the parse
|
|
207
|
+
// would hand it a truncated file and get back a confident answer about a key, when what happened is
|
|
208
|
+
// that the file was cut in half.
|
|
209
|
+
let parsedText = null;
|
|
177
210
|
try {
|
|
178
|
-
const raw =
|
|
179
|
-
|
|
211
|
+
const raw = fs.readFileSync(triggersPath, "utf8");
|
|
212
|
+
const value = JSON.parse(raw);
|
|
213
|
+
parsedText = raw;
|
|
214
|
+
if (Array.isArray(value?.triggers)) current = value.triggers;
|
|
180
215
|
} catch {
|
|
181
216
|
// Missing/invalid file: start from empty; the validated atomic write below repairs it.
|
|
182
217
|
}
|
|
218
|
+
// The repair posture above cannot extend to a duplicate key (issue #313), and the two are exactly
|
|
219
|
+
// separated by the parse. A file that does not parse is repairable and stays repairable. A file that
|
|
220
|
+
// PARSES is not missing, so "start from empty" would not repair it, it would delete the operator's
|
|
221
|
+
// trigger set; and rebuilding from `current` writes back the winning value with the shadowed one
|
|
222
|
+
// silently gone, which turns a divergence a reviewer could still find into one nobody ever can.
|
|
223
|
+
if (parsedText !== null) {
|
|
224
|
+
const duplicate = findDuplicateKey(parsedText);
|
|
225
|
+
if (duplicate) {
|
|
226
|
+
return { invalid: `triggers file has a duplicate key ${JSON.stringify(duplicate.key)} at ${duplicate.at}; refusing to rewrite a file whose reviewed value and running value differ: ${triggersPath}` };
|
|
227
|
+
}
|
|
228
|
+
}
|
|
183
229
|
const next = mutate(current.map((t) => ({ ...t })));
|
|
184
230
|
const text = serialize(next);
|
|
185
231
|
try {
|
|
@@ -224,11 +270,11 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
224
270
|
* not failure), or `{ invalid }` with an operator-actionable reason. NEVER throws, and NEVER repairs:
|
|
225
271
|
* an unreadable file is `{ invalid }` with the bytes untouched.
|
|
226
272
|
*/
|
|
227
|
-
export async function disarmTrigger({ triggersPath, index, number, flow, command, jobId, at, fs = nodeFs, log = () => {} }) {
|
|
273
|
+
export async function disarmTrigger({ triggersPath, index, number, flow, command, jobId, at, fs = nodeFs, log = () => {}, now = () => Date.now() }) {
|
|
228
274
|
for (let attempt = 0; attempt < DISARM_LOCK_ATTEMPTS; attempt++) {
|
|
229
275
|
let fd;
|
|
230
276
|
try {
|
|
231
|
-
fd = takeLock(triggersPath, fs, log);
|
|
277
|
+
fd = takeLock(triggersPath, fs, log, now);
|
|
232
278
|
} catch (err) {
|
|
233
279
|
return { invalid: `triggers file lock failed (${err?.code ?? "lock-error"}): ${triggersPath}` };
|
|
234
280
|
}
|
|
@@ -238,14 +284,24 @@ export async function disarmTrigger({ triggersPath, index, number, flow, command
|
|
|
238
284
|
}
|
|
239
285
|
try {
|
|
240
286
|
let raw;
|
|
287
|
+
let currentText;
|
|
241
288
|
try {
|
|
242
|
-
|
|
289
|
+
currentText = fs.readFileSync(triggersPath, "utf8");
|
|
290
|
+
raw = JSON.parse(currentText);
|
|
243
291
|
} catch (err) {
|
|
244
292
|
// NEVER repair-from-empty here: overwriting a file we could not read, to record one
|
|
245
293
|
// disarm, would destroy the operator's trigger set. writeTriggers' repair posture is for
|
|
246
294
|
// the operator CRUD path, where "missing" means "first trigger".
|
|
247
295
|
return { invalid: `triggers file unreadable (${err?.code ?? "parse-error"}), disarm not written: ${triggersPath}` };
|
|
248
296
|
}
|
|
297
|
+
// And a duplicate key is the destruction that comment thought it had closed (issue #313). This
|
|
298
|
+
// path parses, collapses the duplicate to its last value, and then rewrites the WHOLE file from
|
|
299
|
+
// the parsed array -- so the shadowed key is gone from disk, permanently, to record one disarm.
|
|
300
|
+
// A file that says two things is not a file to rewrite from one of them.
|
|
301
|
+
const duplicate = findDuplicateKey(currentText);
|
|
302
|
+
if (duplicate) {
|
|
303
|
+
return { invalid: `triggers file has a duplicate key ${JSON.stringify(duplicate.key)} at ${duplicate.at}, disarm not written: ${triggersPath}` };
|
|
304
|
+
}
|
|
249
305
|
const entries = Array.isArray(raw?.triggers) ? raw.triggers : null;
|
|
250
306
|
const entry = entries?.[index];
|
|
251
307
|
if (!entry || typeof entry !== "object") {
|