@edgehero/pi-dispatch 1.10.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +300 -148
- package/README.md +50 -0
- package/deploy/com.pi-dispatch.worker.plist +9 -3
- package/deploy/docker-compose.yml +49 -16
- package/deploy/egress-proxy.conf +32 -2
- package/deploy/nssm-install.cmd +12 -6
- package/deploy/pi-dispatch-egress-out.network +10 -0
- package/deploy/pi-dispatch-egress-proxy.container +50 -0
- package/deploy/pi-dispatch-netns-keeper.container +80 -0
- package/deploy/pi-dispatch-netns-keeper.network +18 -0
- package/deploy/pi-dispatch-valkey.container +51 -0
- package/deploy/pi-dispatch-valkey.network +16 -0
- package/deploy/receiver.service +6 -0
- package/deploy/worker-env-wrapper.cmd +11 -0
- package/deploy/worker-env-wrapper.sh +60 -34
- package/deploy/worker.service +18 -8
- package/package.json +14 -4
- package/src/azure-host.mjs +19 -0
- package/src/azure-identity.mjs +18 -2
- package/src/backend-conformance.mjs +71 -18
- package/src/backend-local.mjs +637 -21
- package/src/backend-podman.mjs +1168 -0
- package/src/backend-registry.mjs +86 -3
- package/src/backends.mjs +489 -37
- package/src/branch.mjs +7 -2
- package/src/cancel-cli.mjs +174 -0
- package/src/cancel-state.mjs +125 -0
- package/src/cli.mjs +188 -90
- package/src/config.mjs +503 -43
- package/src/connection.mjs +374 -8
- package/src/container-spec.mjs +102 -7
- package/src/daemon-facts.mjs +167 -0
- package/src/deployment-venue.mjs +158 -0
- package/src/docker-run.mjs +146 -15
- package/src/doctor.mjs +4701 -414
- package/src/egress-conf-copy.mjs +166 -0
- package/src/egress-proxy-state.mjs +151 -0
- package/src/egress.mjs +455 -25
- package/src/entry.mjs +27 -0
- package/src/env-allowlist.mjs +222 -40
- package/src/env-file.mjs +1869 -33
- package/src/exit-code.mjs +15 -0
- package/src/flow-gate.mjs +5 -3
- package/src/forgejo-host.mjs +19 -0
- package/src/forgejo-identity.mjs +21 -2
- package/src/get-token.mjs +67 -18
- package/src/git-dirty.mjs +9 -1
- package/src/git-hardening.mjs +33 -0
- package/src/github-app-setup.mjs +29 -12
- package/src/github-prompt.mjs +4 -1
- package/src/gitlab-host.mjs +19 -0
- package/src/gitlab-identity.mjs +19 -2
- package/src/host-registry.mjs +32 -5
- package/src/identity.mjs +29 -4
- package/src/image-preflight.mjs +46 -11
- package/src/image-ref.mjs +21 -0
- package/src/index.mjs +387 -17
- package/src/init.mjs +197 -38
- package/src/job-user.mjs +252 -0
- package/src/json-duplicates.mjs +204 -0
- package/src/live-probes.mjs +1020 -0
- package/src/materialize.mjs +4 -11
- package/src/netns-keeper.mjs +264 -0
- package/src/on-failure.mjs +119 -0
- package/src/outbox.mjs +7 -0
- package/src/podman-stack.mjs +1304 -0
- package/src/prepare-github.mjs +6 -6
- package/src/prepare-local.mjs +51 -17
- package/src/prepare.mjs +27 -6
- package/src/processor.mjs +505 -26
- package/src/provider-key.mjs +41 -0
- package/src/provider-steering.mjs +144 -0
- package/src/queue.mjs +35 -8
- package/src/redact.mjs +84 -0
- package/src/reserved-env.mjs +7 -3
- package/src/retention-sweep.mjs +178 -0
- package/src/run-container.mjs +181 -14
- package/src/run-history.mjs +105 -16
- package/src/runtime-observations.mjs +1152 -0
- package/src/runtime-settings.mjs +13 -8
- package/src/sandbox-cli.mjs +100 -95
- package/src/sandbox-store.mjs +612 -45
- package/src/sandbox.mjs +1459 -37
- package/src/schedules.mjs +16 -3
- package/src/secret-profiles.mjs +2 -1
- package/src/secrets.mjs +23 -6
- package/src/service-env.mjs +247 -0
- package/src/service.mjs +618 -28
- package/src/session-store.mjs +678 -53
- package/src/start.mjs +1395 -268
- package/src/transient.mjs +240 -0
- package/src/triggers-file.mjs +71 -15
- package/src/triggers.mjs +176 -19
- package/src/up.mjs +1399 -85
- package/src/valkey-auth.mjs +529 -0
- package/src/valkey-endpoint.mjs +367 -0
- package/src/watch-closer.mjs +158 -0
package/src/env-allowlist.mjs
CHANGED
|
@@ -1,9 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the EXACT environment a job container receives. Never a pass-through.
|
|
3
|
+
*
|
|
4
|
+
* `no-broad-env-into-container` is a BLOCKER, and for good reason: `ANTHROPIC_OAUTH_TOKEN`
|
|
5
|
+
* silently outranks `ANTHROPIC_API_KEY`, so one stray host variable would redirect which
|
|
6
|
+
* credential every job spends, with no error and no log line. So we forward a closed set.
|
|
7
|
+
*
|
|
8
|
+
* The provider key variable is DERIVED from pi's own table, not hardcoded. Deriving it means any of
|
|
9
|
+
* pi's ~30 providers works with no code change here, and the list cannot drift when pi adds one, as a
|
|
10
|
+
* hand-copied table would. `getApiKeyEnvVars` (that table) is intentionally NOT exported by pi, which
|
|
11
|
+
* is why `providerKeyCandidates` below has to recover it a different way.
|
|
12
|
+
*
|
|
13
|
+
* **There was a second wrapper here, `providerKeyVars`, and it is GONE (issue #309).** It was a
|
|
14
|
+
* one-line pass-through to `findEnvKeys(provider, hostEnv)`, which filters pi's list by PRESENCE and
|
|
15
|
+
* returns a single `undefined` for both "no such provider" and "known provider, nothing set". Every
|
|
16
|
+
* caller it ever had asked it the wrong question: `resolveProviderCredential` wanted the names and
|
|
17
|
+
* read the values elsewhere (issue #311), and `secrets.mjs`'s reserved-name gate wanted what a trigger
|
|
18
|
+
* may not NAME, which is a property of the provider and not of this host. The presence filter also
|
|
19
|
+
* leaks: pi's `getProviderEnvValue` is `env?.[name] || process.env[name]`, so it reports names the
|
|
20
|
+
* MACHINE carries as well. It is deleted rather than re-documented because an exported helper that
|
|
21
|
+
* answers a subtly wrong question does not stay uncalled; both defects in this cluster were somebody
|
|
22
|
+
* reaching for it. pi's own behaviour is still pinned, against `findEnvKeys` directly, in
|
|
23
|
+
* `worker/test/env-allowlist.test.mjs`.
|
|
24
|
+
*/
|
|
25
|
+
|
|
1
26
|
import { readFileSync } from "node:fs";
|
|
2
27
|
import { homedir } from "node:os";
|
|
3
28
|
import { join } from "node:path";
|
|
4
29
|
import { findEnvKeys } from "@earendil-works/pi-ai/compat";
|
|
30
|
+
import { getBuiltinProviders } from "@earendil-works/pi-ai/providers/all";
|
|
5
31
|
import { egressEnv } from "./egress.mjs";
|
|
6
32
|
import { forgeSpec } from "./forges.mjs";
|
|
33
|
+
import { apiKeyVariable } from "./provider-key.mjs";
|
|
34
|
+
import { isDeterminateFsCode } from "./transient.mjs";
|
|
7
35
|
|
|
8
36
|
function configError(message) {
|
|
9
37
|
const error = new Error(message);
|
|
@@ -12,25 +40,62 @@ function configError(message) {
|
|
|
12
40
|
}
|
|
13
41
|
|
|
14
42
|
/**
|
|
15
|
-
*
|
|
43
|
+
* An environment in which EVERY variable is set, used only to interrogate pi.
|
|
16
44
|
*
|
|
17
|
-
* `
|
|
18
|
-
*
|
|
19
|
-
*
|
|
45
|
+
* `findEnvKeys(provider, env)` filters pi's own candidate list by PRESENCE, so against an env where
|
|
46
|
+
* nothing is absent the filter is the identity and the return value IS pi's table for that provider,
|
|
47
|
+
* in pi's own precedence order. That is how `providerKeyCandidates` recovers a list pi deliberately
|
|
48
|
+
* does not export (`getApiKeyEnvVars` is module-private). It is a dependency on pi filtering rather
|
|
49
|
+
* than short-circuiting, which is stated here rather than discovered later: it holds at the pin, it is
|
|
50
|
+
* round-tripped for every provider by env-allowlist.test.mjs, and CONST-PI-VERSION-PINNED is what
|
|
51
|
+
* makes that test the upgrade gate.
|
|
52
|
+
*
|
|
53
|
+
* A Proxy rather than a literal, and the second reason is the load-bearing one:
|
|
54
|
+
* - the candidate NAMES are the thing being asked for, so they cannot be enumerated in advance;
|
|
55
|
+
* - `getProviderEnvValue` (pi-ai/dist/utils/provider-env.js) falls back to the REAL `process.env`
|
|
56
|
+
* for any name the injected env has no value for. A plain `{}` therefore answers with whatever the
|
|
57
|
+
* machine happens to export -- a doctor run on a laptop would disagree with the server it is
|
|
58
|
+
* diagnosing, and every test injecting a fake env would be non-hermetic. A trap that always returns
|
|
59
|
+
* a non-empty string short-circuits that `||` before the fallback is reached.
|
|
60
|
+
* Strings only: a symbol read (Symbol.toPrimitive and friends) answering with a string would make this
|
|
61
|
+
* object lie about being one.
|
|
62
|
+
*/
|
|
63
|
+
const EVERY_VAR_SET = new Proxy({}, { get: (_target, name) => (typeof name === "string" ? "set" : undefined) });
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Every variable pi reads this provider's key from, in pi's precedence order, set or not. `[]` means pi
|
|
67
|
+
* reads no API-key variable for this id -- either there is no such provider, or it authenticates some
|
|
68
|
+
* other way. `piProviders` is what tells those two apart.
|
|
20
69
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
70
|
+
* This is the ONLY question this module asks pi about variable names, and that is the point (#286, #311,
|
|
71
|
+
* #309). Asking `findEnvKeys(provider, hostEnv)` instead answers "what does this HOST have", conflates
|
|
72
|
+
* "unknown provider" with "known provider, nothing set" in one `undefined`, and decides presence partly
|
|
73
|
+
* from the real `process.env`. Three callers wanted three different things from it and all three were
|
|
74
|
+
* better served by the candidate list plus a presence test of their own: `doctor` NAMES a variable an
|
|
75
|
+
* operator should set, `resolveProviderCredential` reads values out of the env it was handed, and
|
|
76
|
+
* `secrets.mjs` reserves what a trigger may not name.
|
|
77
|
+
*/
|
|
78
|
+
export function providerKeyCandidates(provider) {
|
|
79
|
+
// The string filter is on the RESULT, not just on the trap. pi looks its provider up in a plain object
|
|
80
|
+
// literal, so `__proto__` and `constructor` resolve up the prototype chain and hand back a non-string
|
|
81
|
+
// "variable name"; against a real environment that candidate reads as undefined and pi drops it, but
|
|
82
|
+
// an environment where everything is present keeps it, and doctor would print `set [object Object] in
|
|
83
|
+
// .env`. pricing.mjs guards pi's other generated lookups the same way and for the same reason.
|
|
84
|
+
return (findEnvKeys(provider, EVERY_VAR_SET) ?? []).filter((name) => typeof name === "string");
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* pi's provider catalog, through the same side-effect-free specifier `pricing.mjs` uses -- never
|
|
89
|
+
* `getProviders` from "/compat", which is a @deprecated alias for this exact function and reaching it
|
|
90
|
+
* means loading compat's module-scope provider registration.
|
|
28
91
|
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
92
|
+
* Only ever consulted AFTER `providerKeyCandidates` comes back empty, and the order is load-bearing:
|
|
93
|
+
* the catalog is NOT a superset of the ids `findEnvKeys` answers for. `radius` is a purely dynamic
|
|
94
|
+
* provider with a real key variable (PI_GATEWAY_API_KEY) and no catalog entry, so asking this first
|
|
95
|
+
* would call a working configuration unknown.
|
|
31
96
|
*/
|
|
32
|
-
export function
|
|
33
|
-
return
|
|
97
|
+
export function piProviders() {
|
|
98
|
+
return getBuiltinProviders();
|
|
34
99
|
}
|
|
35
100
|
|
|
36
101
|
/**
|
|
@@ -44,13 +109,31 @@ export function providerKeyVars(provider, hostEnv) {
|
|
|
44
109
|
* it, and it is not the credential for an unattended service). Throws a config-tagged error (pre-spend
|
|
45
110
|
* refusal) when neither source yields a credential.
|
|
46
111
|
*/
|
|
47
|
-
export function resolveProviderCredential({ provider, hostEnv, authFromPi = false, agentDir, readFile = readFileSync }) {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
112
|
+
export function resolveProviderCredential({ provider, hostEnv, authFromPi = false, agentDir, readFile = readFileSync, forwardEnv = [] }) {
|
|
113
|
+
// The NAMES come from pi, the VALUES from `hostEnv`, and this asks each question of the thing that can
|
|
114
|
+
// answer it (issue #311). It used to be one `providerKeyVars(provider, hostEnv)` call, which conflates
|
|
115
|
+
// them: that is `findEnvKeys`, whose presence test falls back to the REAL `process.env` for any name the
|
|
116
|
+
// given env lacks, so a name could be "present" on the strength of the machine and then be read out of
|
|
117
|
+
// an env that does not carry it, yielding `{ NAME: undefined }` -- a credential-shaped answer with no
|
|
118
|
+
// credential in it, and no fall through to the auth.json login that would have worked.
|
|
119
|
+
//
|
|
120
|
+
// `providerKeyCandidates` is the hermetic half of the same oracle, so the leaky question is not asked at
|
|
121
|
+
// all rather than asked and corrected. The result is identical by construction: a name truthy in
|
|
122
|
+
// `hostEnv` is always one `findEnvKeys` would have returned, and pi's precedence order is preserved.
|
|
123
|
+
// Truthiness matches pi's own filter, so a name kept here is a name pi would read.
|
|
124
|
+
//
|
|
125
|
+
// Unreachable in the shipped worker, where `hostEnv` IS `process.env` and the two agree by identity. It
|
|
126
|
+
// is the DI seam that was dishonest, which is worth fixing where the seam is the whole test surface.
|
|
127
|
+
// ONE read per name, checked and returned, for the same reason: two reads could decide on one value and
|
|
128
|
+
// ship another.
|
|
129
|
+
const held = providerKeyCandidates(provider)
|
|
130
|
+
.map((name) => [name, hostEnv[name]])
|
|
131
|
+
.filter(([, value]) => value);
|
|
132
|
+
if (held.length > 0) {
|
|
133
|
+
return Object.fromEntries(held);
|
|
51
134
|
}
|
|
52
135
|
if (authFromPi) {
|
|
53
|
-
const { name, value } = credentialFromPiAuth(provider, agentDir ?? defaultAgentDir(hostEnv), readFile);
|
|
136
|
+
const { name, value } = credentialFromPiAuth(provider, agentDir ?? defaultAgentDir(hostEnv), readFile, { hostEnv, forwardEnv });
|
|
54
137
|
return { [name]: value };
|
|
55
138
|
}
|
|
56
139
|
throw configError(`provider ${provider} has no configured credential in the worker environment`);
|
|
@@ -61,13 +144,30 @@ function defaultAgentDir(hostEnv) {
|
|
|
61
144
|
return hostEnv.PI_CODING_AGENT_DIR || join(homedir(), ".pi", "agent");
|
|
62
145
|
}
|
|
63
146
|
|
|
64
|
-
function credentialFromPiAuth(provider, agentDir, readFile) {
|
|
147
|
+
function credentialFromPiAuth(provider, agentDir, readFile, { hostEnv = {}, forwardEnv = [] } = {}) {
|
|
65
148
|
const path = join(agentDir, "auth.json");
|
|
66
149
|
let auth;
|
|
67
150
|
try {
|
|
68
151
|
auth = JSON.parse(readFile(path, "utf8"));
|
|
69
|
-
} catch {
|
|
70
|
-
|
|
152
|
+
} catch (error) {
|
|
153
|
+
// ONLY the determinate failures are tagged. This used to be a bare `catch {}` that relabelled every
|
|
154
|
+
// read failure as "no pi login", and issue #310 is what made that expensive: the processor now turns
|
|
155
|
+
// a `piDispatchConfig` error into a refusal that is never retried, refunds the reserve, and posts
|
|
156
|
+
// publicly that the deployment is misconfigured. Under the old catch an EMFILE from fd exhaustion, an
|
|
157
|
+
// EIO on a network filesystem, an EACCES, an EISDIR, or a torn read while `pi login` rewrites the file
|
|
158
|
+
// all produced that verdict, on a deployment that was correctly configured the microsecond before and
|
|
159
|
+
// after. A determinate refusal is one the same inputs reproduce; these are not.
|
|
160
|
+
//
|
|
161
|
+
// Absent is determinate (there is no login), and so is unparseable (the file is there and wrong, and
|
|
162
|
+
// an operator has to fix it). Everything else propagates untagged, which is `CONST-RETRY-INFRA-ONLY`
|
|
163
|
+
// putting it back where it belongs: an infrastructure fault reported as itself, with its own message.
|
|
164
|
+
// The codes come from the shared rule rather than a second copy of them here (issue #316): this
|
|
165
|
+
// file's argument IS that rule, and restating it was how the identity modules came to disagree
|
|
166
|
+
// with the host modules about the same three conditions.
|
|
167
|
+
const absent = isDeterminateFsCode(error?.code);
|
|
168
|
+
if (!absent && !(error instanceof SyntaxError)) throw error;
|
|
169
|
+
const why = absent ? `nothing readable at ${path}` : `the pi login at ${path} is not valid JSON`;
|
|
170
|
+
throw configError(`no credential for provider "${provider}": not in the worker environment, and ${why} — set the key in .env, or run \`pi login\``);
|
|
71
171
|
}
|
|
72
172
|
const cred = auth?.[provider];
|
|
73
173
|
if (!cred) throw configError(`no credential for provider "${provider}": not in the worker environment, and ${path} has no "${provider}" login — set the key in .env, or run \`pi login\``);
|
|
@@ -77,24 +177,84 @@ function credentialFromPiAuth(provider, agentDir, readFile) {
|
|
|
77
177
|
);
|
|
78
178
|
}
|
|
79
179
|
if (cred.type !== "api_key" || !cred.key) throw configError(`unsupported pi credential for "${provider}" in ${path} — set an API key in .env`);
|
|
80
|
-
|
|
180
|
+
// `typeof`, not truthiness. A hand-edited auth.json can hold a number, an array or an object here (pi
|
|
181
|
+
// parses the file and validates no schema), and the old `!cred.key` guard passed all three: an object
|
|
182
|
+
// became `-e ANTHROPIC_API_KEY=[object Object]` in a paid container. Issue #311 made this reachable for
|
|
183
|
+
// twelve more providers, so it is guarded here rather than left to the coercion.
|
|
184
|
+
if (typeof cred.key !== "string") throw configError(`the pi login for "${provider}" in ${path} does not hold a string API key — run \`pi login\` again, or set the key in .env`);
|
|
185
|
+
// **The worker forwards this value; pi RESOLVES it.** `auth-storage.js` reads the stored key through
|
|
186
|
+
// `resolveConfigValue(cred.key, cred.env)`, a grammar where a leading "!" runs the rest as a SHELL
|
|
187
|
+
// COMMAND and takes stdout, "$VAR"/"${VAR}" interpolate from the environment, and "$$"/"$!" escape. An
|
|
188
|
+
// environment variable is read raw, through no such grammar, so a login stored in any of those forms
|
|
189
|
+
// arrives in the container as its own source text and every job spends a container to fail auth, with
|
|
190
|
+
// `doctor` green because the field is a non-empty string.
|
|
191
|
+
//
|
|
192
|
+
// This REFUSES rather than resolving. Running the command host-side is not on the table (it would
|
|
193
|
+
// execute operator shell out of a credential file, per job, in the worker); interpolating "$VAR" would
|
|
194
|
+
// be reimplementing a grammar pi does not export, which is `no-reimplementing-pi` and the same trap
|
|
195
|
+
// #311 is about. The test is deliberately a superset of pi's parse: any "$" at all, not a parse of one.
|
|
196
|
+
// A key that pi would have passed through unchanged contains neither character, and over-refusing
|
|
197
|
+
// pre-spend costs nothing, which is exactly the trade `CONST-BUDGET-BEFORE-TOKENS` asks for.
|
|
198
|
+
if (cred.key.startsWith("!") || cred.key.includes("$")) {
|
|
199
|
+
throw configError(
|
|
200
|
+
`the pi login for "${provider}" in ${path} is a command or a variable reference, not a literal key. pi resolves that form itself; this service forwards the value to a container, where it is read as-is. Resolve it and set the key in .env instead.`,
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
// The credential's companion config (`cred.env`), which is NOT a variable-name hint -- it is a
|
|
204
|
+
// `Record<string, string>` of provider settings, and for both Cloudflare providers pi returns NO auth at
|
|
205
|
+
// all without `CLOUDFLARE_ACCOUNT_ID` (and `CLOUDFLARE_GATEWAY_ID` for the gateway). The container env
|
|
206
|
+
// is a closed set, so these names do not ride along with the key; pi does fall back to the ambient
|
|
207
|
+
// environment for them, which makes `PI_FORWARD_ENV` a real answer rather than a shrug. Refuse only for
|
|
208
|
+
// the names that will NOT arrive, so a deployment that already forwards them is untouched.
|
|
209
|
+
const missing = Object.keys(cred.env ?? {}).filter((n) => !(forwardEnv.includes(n) && hostEnv?.[n]));
|
|
210
|
+
if (missing.length > 0) {
|
|
211
|
+
throw configError(
|
|
212
|
+
`the pi login for "${provider}" carries provider settings the container will not receive (${missing.join(", ")}). The job env is a closed set. Set ${missing.join(" and ")} in the worker environment and list ${missing.length > 1 ? "them" : "it"} in PI_FORWARD_ENV.`,
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
const name = resolveEnvName(provider);
|
|
81
216
|
if (!name) {
|
|
82
|
-
|
|
217
|
+
// Two different facts, and they need different fixes -- the same split `doctor` makes, in the same
|
|
218
|
+
// order (candidates first, catalog second: `radius` has a key variable and no catalog entry, so
|
|
219
|
+
// asking membership first would call a working configuration unknown). The old message said "set it
|
|
220
|
+
// in the worker environment manually" for BOTH, which for the first is advice `doctor` correctly
|
|
221
|
+
// calls impossible: there is no variable to set.
|
|
222
|
+
if (piProviders().includes(provider)) {
|
|
223
|
+
throw configError(
|
|
224
|
+
`pi authenticates "${provider}" without an API-key environment variable (an AWS profile or an OAuth login), and the container env is a closed set of variables, so it has no way in. Configure a provider whose credential is a single environment variable.`,
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
throw configError(`could not determine the environment variable pi expects for provider "${provider}" — pi has no such provider, so check PI_PROVIDER against pi's own ids`);
|
|
83
228
|
}
|
|
84
229
|
return { name, value: cred.key };
|
|
85
230
|
}
|
|
86
231
|
|
|
87
232
|
/**
|
|
88
|
-
* The env var name
|
|
89
|
-
*
|
|
90
|
-
*
|
|
233
|
+
* The env var name to write this provider's `auth.json` api key under. `null` when pi reads no key
|
|
234
|
+
* variable for the provider at all, which `credentialFromPiAuth` turns into a refusal.
|
|
235
|
+
*
|
|
236
|
+
* The oracle instinct in the old version of this comment was right and its execution was not, which is
|
|
237
|
+
* worth recording rather than deleting (issue #311). It asked pi's own `findEnvKeys` rather than keeping a
|
|
238
|
+
* provider→var table, but the CANDIDATES it asked with were hand-generated: the credential's `env` field
|
|
239
|
+
* plus a conventional `<PROVIDER>_API_KEY`/`_KEY`. That convention is the part that drifts, and it was
|
|
240
|
+
* wrong for 13 of the 34 provider ids that have a key variable at the pin -- `google` is
|
|
241
|
+
* `GEMINI_API_KEY`, `huggingface` is `HF_TOKEN`, `moonshotai` is `MOONSHOT_API_KEY`, `github-copilot` is
|
|
242
|
+
* `COPILOT_GITHUB_TOKEN`, `radius` is `PI_GATEWAY_API_KEY` -- so for those pi recognized nothing and a
|
|
243
|
+
* valid `pi login` refused every job. The convention happening to be right for the other 21 is what let
|
|
244
|
+
* this survive: `anthropic` and `openai` are both in that set.
|
|
245
|
+
*
|
|
246
|
+
* The `env` candidate was dead on arrival besides: at the pin `ApiKeyCredential.env` is a
|
|
247
|
+
* `Record<string, string>` of provider config (Cloudflare account and gateway ids), never a variable
|
|
248
|
+
* name, so the string filter dropped it on every call.
|
|
249
|
+
*
|
|
250
|
+
* `providerKeyCandidates` is pi's whole list, so there is nothing left to guess, and it asks against an
|
|
251
|
+
* environment where every name is present -- which also closes a hermeticity hole the synthetic object
|
|
252
|
+
* had: pi's `getProviderEnvValue` falls back to the real `process.env`, so on a host that exported
|
|
253
|
+
* `ANTHROPIC_OAUTH_TOKEN` the old code resolved to THAT name. `apiKeyVariable` then picks the same
|
|
254
|
+
* variable `doctor` names, from the same module, so the two cannot diverge again.
|
|
91
255
|
*/
|
|
92
|
-
function resolveEnvName(provider
|
|
93
|
-
|
|
94
|
-
const candidates = [cred.env, `${upper}_API_KEY`, `${upper}_KEY`].filter((s) => typeof s === "string" && s.length > 0);
|
|
95
|
-
const synthetic = Object.fromEntries(candidates.map((name) => [name, cred.key]));
|
|
96
|
-
const recognized = findEnvKeys(provider, synthetic);
|
|
97
|
-
return recognized?.[0] ?? null;
|
|
256
|
+
function resolveEnvName(provider) {
|
|
257
|
+
return apiKeyVariable(providerKeyCandidates(provider));
|
|
98
258
|
}
|
|
99
259
|
|
|
100
260
|
/**
|
|
@@ -103,7 +263,9 @@ function resolveEnvName(provider, cred) {
|
|
|
103
263
|
* run.github).
|
|
104
264
|
*
|
|
105
265
|
* Throws if the provider is not configured -- a deterministic misconfiguration the caller maps to
|
|
106
|
-
* a pre-spend refusal, never a launched-then-failed container.
|
|
266
|
+
* a pre-spend refusal, never a launched-then-failed container. Both halves of that are real as of issue
|
|
267
|
+
* #310: the processor probes this resolution among its free gates, and classifies the throw if one still
|
|
268
|
+
* reaches it.
|
|
107
269
|
*
|
|
108
270
|
* `packagePaths` is the operator-staged pi package set for THIS job: already-resolved absolute
|
|
109
271
|
* CONTAINER paths under the :ro overlay, empty for a job whose trigger opted OUT (or when nothing is staged).
|
|
@@ -111,11 +273,12 @@ function resolveEnvName(provider, cred) {
|
|
|
111
273
|
* `allowGlobalExtensions` defaults to TRUE here, matching loadConfig's default (REQ-GLOBAL-PI-OVERLAY): a
|
|
112
274
|
* caller that says nothing gets the operator's staged setup, and only an explicit `false` withholds it.
|
|
113
275
|
*/
|
|
114
|
-
export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId, githubToken, forgeKind, forgeHosts = {}, hostEnv, allowGlobalExtensions = true, packagePaths = [], forwardEnv = [], secrets = {}, sessionFile = null, flow = null, command = null, authFromPi = false, egress = false, egressProxy, agentDir, readFile = readFileSync }) {
|
|
276
|
+
export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId, githubToken, forgeKind, forgeHosts = {}, hostEnv, allowGlobalExtensions = true, packagePaths = [], forwardEnv = [], secrets = {}, sessionFile = null, flow = null, command = null, excludeTools = [], authFromPi = false, egress = false, egressProxy, agentDir, home = null, readFile = readFileSync }) {
|
|
115
277
|
// The provider credential(s), by pi's expected variable name(s) -- from the worker env, or (when
|
|
116
278
|
// PI_AUTH_FROM_PI is set and the env has none) host-side from pi's auth.json. Throws (config) if
|
|
117
|
-
// neither source yields one, which the processor turns into a
|
|
118
|
-
|
|
279
|
+
// neither source yields one, which the processor turns into a policy refusal that refunds any reserve
|
|
280
|
+
// (issue #310); the same call is made by its free credential gate, before anything is reserved at all.
|
|
281
|
+
const credEnv = resolveProviderCredential({ provider, hostEnv, authFromPi, agentDir, readFile, forwardEnv });
|
|
119
282
|
|
|
120
283
|
const env = {
|
|
121
284
|
PI_PROVIDER: provider,
|
|
@@ -163,6 +326,13 @@ export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId,
|
|
|
163
326
|
// exclusive by parse (command XOR flow); that is deliberately NOT re-enforced here -- a second
|
|
164
327
|
// validator is a second place to disagree with the first.
|
|
165
328
|
PI_COMMAND: command || undefined,
|
|
329
|
+
// The trigger's run.excludeTools, STRUCTURALLY (issue #291): the pi tool names the runner must
|
|
330
|
+
// withhold from createAgentSession, so a "read-only" trigger stops being prompt text. Comma is a
|
|
331
|
+
// safe delimiter because the loader admits only names from its pinned built-in set, none of which
|
|
332
|
+
// carries one -- that is a load-time guarantee, deliberately NOT re-enforced here (PI_COMMAND's
|
|
333
|
+
// second-validator rule directly above). Absent means the full pinned default set, never an empty
|
|
334
|
+
// string, for PI_PACKAGES' reason.
|
|
335
|
+
PI_EXCLUDE_TOOLS: excludeTools.length > 0 ? excludeTools.join(",") : undefined,
|
|
166
336
|
// Kill switch for job-time package installation, UNCONDITIONAL for every job. pi's resolver shells out
|
|
167
337
|
// to a REAL `npm install` for any npm:/git: source unless offline mode is on, and `~/.pi/agent` IS
|
|
168
338
|
// writable in the container. We emit only local paths, so nothing should reach that branch -- this
|
|
@@ -177,9 +347,14 @@ export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId,
|
|
|
177
347
|
// Operator-declared extra vars (PI_FORWARD_ENV), forwarded by EXACT name -- the allowlist
|
|
178
348
|
// no-broad-env-into-container prescribes, not a host pass-through. This is how a CUSTOM provider's
|
|
179
349
|
// key (one pi's findEnvKeys table does not know) reaches the container. A name whose value is unset
|
|
180
|
-
// on the host is skipped, never forwarded as empty.
|
|
350
|
+
// on the host is skipped, never forwarded as empty. That second half was a claim rather than a check
|
|
351
|
+
// until issue #311: the guard tested `!== undefined`, so a name set to "" WAS forwarded, and since this
|
|
352
|
+
// loop runs after the credential assign above, `PI_FORWARD_ENV=<the provider's key variable>` with a
|
|
353
|
+
// blank value in the host env blanked a working auth.json credential and the job spent a container to
|
|
354
|
+
// fail auth. Same emptiness rule as the secrets loop below, which had it right.
|
|
181
355
|
for (const name of forwardEnv) {
|
|
182
|
-
|
|
356
|
+
const value = hostEnv[name];
|
|
357
|
+
if (value !== undefined && value !== "") env[name] = value;
|
|
183
358
|
}
|
|
184
359
|
|
|
185
360
|
// The trigger's own secrets (REQ-TRIGGER-SECRETS), resolved HOST-SIDE by the processor before anything
|
|
@@ -226,6 +401,13 @@ export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId,
|
|
|
226
401
|
// turn. It rides the closed map, never PI_FORWARD_ENV, so arming the policy cannot half-work.
|
|
227
402
|
Object.assign(env, egressEnv({ proxy: egressProxy, armed: egress }));
|
|
228
403
|
|
|
404
|
+
// Issue #341: the job's HOME, only when the worker runs it under `--user` (`home` is passed then and only then).
|
|
405
|
+
// AFTER the PI_FORWARD_ENV and secrets loops, for the egress variables' reason: a forwarded or secret HOME must
|
|
406
|
+
// not win, because a uid with no passwd entry in the image would otherwise get `HOME=/` (Docker) or
|
|
407
|
+
// `HOME=/workspace` (Podman, measured), and pi's auth.json lands nowhere or in the operator's repository.
|
|
408
|
+
// env-internal HOME: written into the job's closed env map here, never read from the worker's environment.
|
|
409
|
+
if (typeof home === "string" && home !== "") env.HOME = home;
|
|
410
|
+
|
|
229
411
|
// Forge-backed jobs, and local cron jobs that opted in via run.github. Other local-folder jobs have
|
|
230
412
|
// no token (CONST-TOKEN-SCOPED-PER-JOB). The mint goes into BOTH of its forge's variables because
|
|
231
413
|
// each CLI has its own preference -- gh prefers GH_TOKEN over GITHUB_TOKEN, glab prefers GITLAB_TOKEN
|