@edgehero/pi-dispatch 2.1.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +41 -5
- package/README.md +11 -5
- package/deploy/docker-compose.yml +12 -0
- package/deploy/egress-proxy.conf +28 -3
- package/deploy/pi-dispatch-egress-proxy.container +8 -2
- package/package.json +9 -2
- package/src/allocation.mjs +731 -0
- package/src/backends.mjs +243 -0
- package/src/budget.mjs +40 -4
- package/src/cli.mjs +222 -11
- package/src/config.mjs +126 -5
- package/src/daemon-facts.mjs +3 -0
- package/src/deployment-venue.mjs +1 -0
- package/src/doctor.mjs +2280 -195
- package/src/dollar-budget.mjs +373 -0
- package/src/dollar-fingerprint.mjs +83 -0
- package/src/egress-cli.mjs +316 -0
- package/src/egress-proxy-state.mjs +35 -5
- package/src/egress.mjs +12 -0
- package/src/env-allowlist.mjs +125 -6
- package/src/env-file.mjs +194 -25
- package/src/envelope.mjs +413 -0
- package/src/exit-code.mjs +22 -0
- package/src/fleet-lease.mjs +85 -25
- package/src/get-token.mjs +16 -5
- package/src/git-dirty.mjs +67 -0
- package/src/github-app-setup.mjs +6 -3
- package/src/github-host.mjs +5 -3
- package/src/host-pi.mjs +1 -1
- package/src/identity.mjs +2 -1
- package/src/image-preflight.mjs +98 -24
- package/src/image-ref.mjs +37 -0
- package/src/import-pi.mjs +4 -2
- package/src/index.mjs +407 -62
- package/src/init.mjs +18 -0
- package/src/job-id.mjs +26 -3
- package/src/live-probes.mjs +24 -9
- package/src/model-catalog.mjs +297 -0
- package/src/model-endpoints.mjs +671 -0
- package/src/model-ref.mjs +151 -0
- package/src/models-json.mjs +268 -0
- package/src/money.mjs +144 -0
- package/src/octokit-log.mjs +65 -0
- package/src/outbox-plan.mjs +218 -0
- package/src/outbox.mjs +29 -9
- package/src/output-cap.mjs +157 -0
- package/src/pause-windows.mjs +81 -2
- package/src/pi-model-loader.mjs +77 -0
- package/src/podman-stack.mjs +16 -3
- package/src/portfolio-snapshot.mjs +304 -0
- package/src/prepare-local.mjs +247 -12
- package/src/prepare.mjs +35 -3
- package/src/priorities.mjs +569 -0
- package/src/processor.mjs +599 -170
- package/src/project-id.mjs +17 -0
- package/src/projects.mjs +238 -0
- package/src/provider-steering.mjs +179 -65
- package/src/queue.mjs +111 -6
- package/src/reserved-env.mjs +31 -0
- package/src/run-container.mjs +59 -5
- package/src/run-history.mjs +379 -24
- package/src/run-mirror.mjs +30 -0
- package/src/runtime-settings.mjs +104 -9
- package/src/schedules.mjs +33 -1
- package/src/scoped-limits.mjs +447 -27
- package/src/service.mjs +15 -4
- package/src/session-store.mjs +131 -6
- package/src/start.mjs +528 -40
- package/src/triggers-file.mjs +65 -4
- package/src/triggers.mjs +141 -11
- package/src/up.mjs +308 -34
- package/src/valkey-endpoint.mjs +3 -2
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The project id rule (issue #499, INT-PROJECTS-FILE-CONTRACT), in a module with NO imports. `run-history.mjs` checks a
|
|
3
|
+
* record's `project` against it, and the admin loads run-history inside pi, so the record module must not drag the
|
|
4
|
+
* projects parser's graph (config.mjs and its fs, os and child_process) in with it: the rule `model-ref.mjs` keeps for
|
|
5
|
+
* the same reader. `projects.mjs` re-exports both names.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A project id: lowercase, 1 to 32 characters, free of `:`, `#` and `/`, so it can enter a run record and a Valkey key
|
|
10
|
+
* without escaping.
|
|
11
|
+
*/
|
|
12
|
+
export const PROJECT_ID_RE = /^[a-z0-9][a-z0-9-]{0,31}$/;
|
|
13
|
+
|
|
14
|
+
/** Is this a well-formed project id? The record path's charset check. */
|
|
15
|
+
export function isProjectId(value) {
|
|
16
|
+
return typeof value === "string" && PROJECT_ID_RE.test(value);
|
|
17
|
+
}
|
package/src/projects.mjs
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Projects (issue #499, INT-PROJECTS-FILE-CONTRACT): a named group of repos and folders. One `projects.json` of
|
|
3
|
+
* `{ version: 1, projects: [{ id, name?, members }] }`. A job whose scope is a member belongs to that project, and the
|
|
4
|
+
* project's id is written into the job's run record (`project`), so spend can later be read and capped per project.
|
|
5
|
+
*
|
|
6
|
+
* This module is pure and fs-injectable, on the scoped-limits.mjs pattern: `parseProjects` validates the file TEXT
|
|
7
|
+
* fail-loud, `loadProjects` layers the one fs read on top, and `projectOf` is what the pickup gate and the record path
|
|
8
|
+
* consume. The worker holds the parsed list in a watched ref with a last-good copy (start.mjs).
|
|
9
|
+
*
|
|
10
|
+
* `version` is REQUIRED and a newer one is refused: which project a scope belongs to decides which cap applies to it
|
|
11
|
+
* (issue #499 part B), so this is a money file, and a field an old build silently drops could widen a cap.
|
|
12
|
+
*
|
|
13
|
+
* `name` is display text for the panel. It never enters a record or a log line: the record carries the `id`, which is
|
|
14
|
+
* charset-checked, and so stays free of personal data by construction. For the same reason no error message here
|
|
15
|
+
* quotes a name, and an invalid-JSON error does not quote the parser's message, which carries file text.
|
|
16
|
+
*
|
|
17
|
+
* Custom: projects validated inline per scoped-limits.mjs precedent; zod not in deps
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { existsSync as fsExistsSync, readFileSync as fsReadFileSync } from "node:fs";
|
|
21
|
+
import { isAbsolute, resolve } from "node:path";
|
|
22
|
+
import { configError } from "./config.mjs";
|
|
23
|
+
import { fingerprint } from "./fingerprint.mjs";
|
|
24
|
+
import { hash16 } from "./fleet-lease.mjs";
|
|
25
|
+
import { PROJECT_ID_RE, isProjectId } from "./project-id.mjs";
|
|
26
|
+
import { parseScopeString, qualifiedScopeOf } from "./pause-windows.mjs";
|
|
27
|
+
import { canonicalScope } from "./scoped-limits.mjs";
|
|
28
|
+
|
|
29
|
+
/** The highest schema version this build reads. A file declaring a higher one is refused loudly. */
|
|
30
|
+
export const PROJECTS_VERSION = 1;
|
|
31
|
+
|
|
32
|
+
// The id rule lives in an import-free module, so the run record can check an id without this module's graph.
|
|
33
|
+
export { PROJECT_ID_RE, isProjectId };
|
|
34
|
+
|
|
35
|
+
/** The longest `name` accepted, in UTF-16 code units. A display label, not a document. */
|
|
36
|
+
const NAME_MAX = 120;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Parse, validate and normalize the projects file TEXT. Returns the normalized list: each project rebuilt as an
|
|
40
|
+
* explicit `{ id, name, members }` literal (`name` null when absent, members in their stored spelling, unknown fields
|
|
41
|
+
* dropped by the operator-file policy). Throws `configError` on anything malformed. `path` is for messages only.
|
|
42
|
+
*
|
|
43
|
+
* Members use the scope grammar of `parseScopeString` (issue #498):
|
|
44
|
+
* - a forge-qualified scope (`github:acme/web`), stored as `<kind>:<repo>`;
|
|
45
|
+
* - an absolute folder (`/srv/shop-tools`), stored resolved, the spelling `canonicalScope` gives a local job.
|
|
46
|
+
* A bare `owner/name` is refused: it names that repo on every forge, and this file has no legacy to keep. A relative
|
|
47
|
+
* folder, a drive path on a host where it is not absolute, `*` and globs are refused too: each would be a member no
|
|
48
|
+
* job's scope can ever equal.
|
|
49
|
+
*
|
|
50
|
+
* Refused across the file: a duplicate id, a scope claimed by two projects (both ids named, one project per scope),
|
|
51
|
+
* a scope listed twice in one project, and an empty `members`.
|
|
52
|
+
*/
|
|
53
|
+
export function parseProjects(text, path) {
|
|
54
|
+
try {
|
|
55
|
+
return parseProjectsText(text, path);
|
|
56
|
+
} catch (error) {
|
|
57
|
+
// Every refusal ESCAPED at the source (PR #569's review): a folder member may hold a C1 or bidi character, and this
|
|
58
|
+
// message reaches the worker's `projects_reload_invalid` log line, doctor and an admin tool's error. JSON quoting
|
|
59
|
+
// keeps C0 visible but leaves C1, bidi and zero-width characters raw.
|
|
60
|
+
if (error?.piDispatchConfig === true) error.message = escapeControls(error.message);
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The characters `escapeControls` writes out, the SAME set the admin panel's `escapeInterpreted` escapes (PR #569's
|
|
67
|
+
* second review; `admin/test/projects.test.mjs` compares the two over every code point): the controls (C0, DEL, C1),
|
|
68
|
+
* every format character (bidi controls and isolates, zero-width and invisible ones, the tag block) but the two joiners
|
|
69
|
+
* that compose (U+200C, U+200D), the line and paragraph separators, every blank that is not U+0020 but draws one
|
|
70
|
+
* column (the no-break and fixed-width spaces, U+2800), the Hangul fillers, and the unassigned code points the
|
|
71
|
+
* terminal draws as nothing (U+2065, U+FFF0-U+FFF8, the special-purpose plane outside its variation selectors).
|
|
72
|
+
* U+3000 is kept, as the panel keeps it: it draws two columns, an ordinary full-width space.
|
|
73
|
+
*/
|
|
74
|
+
const ESCAPED = /(?![\u200c\u200d\u3000\u{e0100}-\u{e01ef}])[\p{Cc}\p{Cf}\p{Zl}\p{Zp}\p{Zs}\u2800\u115f\u1160\u3164\uffa0\u2065\ufff0-\ufff8\u{e0000}-\u{e0fff}]/gu;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Text with the characters that change what a reader SEES written out as `\\u{XXXX}` (`ESCAPED`), U+0020 kept. For a
|
|
78
|
+
* message about operator text that reaches a log line or a terminal. The worker's twin of the panel's
|
|
79
|
+
* `escapeInterpreted`; it imports nothing.
|
|
80
|
+
*/
|
|
81
|
+
export function escapeControls(text) {
|
|
82
|
+
return String(text ?? "").replace(ESCAPED, (ch) => (ch === " " ? ch : `\\u{${ch.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}}`));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function parseProjectsText(text, path) {
|
|
86
|
+
let parsed;
|
|
87
|
+
try {
|
|
88
|
+
parsed = JSON.parse(text);
|
|
89
|
+
} catch (error) {
|
|
90
|
+
// Not the parser's own message: it quotes the file's text around the fault, and that text may be a `name`.
|
|
91
|
+
const at = /position (\d+)/.exec(String(error?.message))?.[1];
|
|
92
|
+
throw configError(`projects file is not valid JSON${at === undefined ? "" : ` (at character ${at})`}: ${path}`);
|
|
93
|
+
}
|
|
94
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
95
|
+
throw configError(`projects file must be an object with "version" and "projects": ${path}`);
|
|
96
|
+
}
|
|
97
|
+
const version = parsed.version;
|
|
98
|
+
if (!Number.isInteger(version) || version < 1) {
|
|
99
|
+
throw configError(`projects file must have "version": ${PROJECTS_VERSION} (an integer >= 1): ${path}`);
|
|
100
|
+
}
|
|
101
|
+
if (version > PROJECTS_VERSION) {
|
|
102
|
+
throw configError(`projects file written by a newer pi-dispatch (version ${version}; this build understands ${PROJECTS_VERSION}): ${path}`);
|
|
103
|
+
}
|
|
104
|
+
if (!Array.isArray(parsed.projects)) throw configError(`projects file must have a "projects" array: ${path}`);
|
|
105
|
+
const projects = parsed.projects.map((entry, index) => normalizeProject(entry, index, path));
|
|
106
|
+
const ids = new Map();
|
|
107
|
+
const owners = new Map();
|
|
108
|
+
projects.forEach((project, index) => {
|
|
109
|
+
if (ids.has(project.id)) {
|
|
110
|
+
throw configError(`project at index ${index}: duplicate id "${project.id}" (first at index ${ids.get(project.id)}): ${path}`);
|
|
111
|
+
}
|
|
112
|
+
ids.set(project.id, index);
|
|
113
|
+
for (const member of project.members) {
|
|
114
|
+
const owner = owners.get(member);
|
|
115
|
+
// One project per scope: a job in two projects would have two project ledgers and an unclear refusal.
|
|
116
|
+
if (owner !== undefined) {
|
|
117
|
+
throw configError(`project at index ${index}: ${JSON.stringify(member)} is claimed by both "${owner}" and "${project.id}" (a scope belongs to one project): ${path}`);
|
|
118
|
+
}
|
|
119
|
+
owners.set(member, project.id);
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
return projects;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function normalizeProject(entry, index, path) {
|
|
126
|
+
const at = `project at index ${index}`;
|
|
127
|
+
if (entry === null || typeof entry !== "object" || Array.isArray(entry)) throw configError(`${at}: must be an object: ${path}`);
|
|
128
|
+
if (!isProjectId(entry.id)) {
|
|
129
|
+
throw configError(`${at}: id must match ${PROJECT_ID_RE.source} (lowercase letters, digits and "-", 1 to 32 characters, starting with a letter or digit): ${path}`);
|
|
130
|
+
}
|
|
131
|
+
const id = entry.id;
|
|
132
|
+
let name = null;
|
|
133
|
+
if (entry.name !== undefined && entry.name !== null) {
|
|
134
|
+
// The name's own text is never echoed: it is free text, and this message can reach a log line.
|
|
135
|
+
if (typeof entry.name !== "string" || entry.name.trim() === "" || entry.name.length > NAME_MAX || /[\u0000-\u001f\u007f]/.test(entry.name)) {
|
|
136
|
+
throw configError(`${at} ("${id}"): name, when given, must be a string of 1 to ${NAME_MAX} characters with no control characters: ${path}`);
|
|
137
|
+
}
|
|
138
|
+
name = entry.name.trim();
|
|
139
|
+
}
|
|
140
|
+
if (!Array.isArray(entry.members) || entry.members.length === 0) {
|
|
141
|
+
throw configError(`${at} ("${id}"): members must be a non-empty array of scopes (github:owner/name or an absolute folder): ${path}`);
|
|
142
|
+
}
|
|
143
|
+
const members = [];
|
|
144
|
+
entry.members.forEach((raw, m) => {
|
|
145
|
+
const member = normalizeMember(raw, `${at} ("${id}") member ${m}`, path);
|
|
146
|
+
if (members.includes(member)) throw configError(`${at} ("${id}") member ${m}: ${JSON.stringify(member)} is listed twice: ${path}`);
|
|
147
|
+
members.push(member);
|
|
148
|
+
});
|
|
149
|
+
return { id, name, members };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** One member, in its stored spelling, or the refusal naming where it sits. */
|
|
153
|
+
function normalizeMember(raw, at, path) {
|
|
154
|
+
if (typeof raw !== "string" || raw.trim() === "") throw configError(`${at}: must be a non-empty string: ${path}`);
|
|
155
|
+
const trimmed = raw.trim().normalize("NFC");
|
|
156
|
+
if (trimmed.includes("*")) throw configError(`${at}: members match exactly; a scope containing "*" is refused (no globs): ${path}`);
|
|
157
|
+
let form;
|
|
158
|
+
try {
|
|
159
|
+
form = parseScopeString(trimmed);
|
|
160
|
+
} catch (error) {
|
|
161
|
+
throw configError(`${at}: ${error.message}: ${path}`);
|
|
162
|
+
}
|
|
163
|
+
if (form.type === "qualified") return `${form.kind}:${form.repo}`;
|
|
164
|
+
if (form.type === "bare") {
|
|
165
|
+
throw configError(`${at}: ${JSON.stringify(trimmed)} is a bare repo, which names that repo on every forge; write it with its forge, such as github:${trimmed}, or give an absolute folder: ${path}`);
|
|
166
|
+
}
|
|
167
|
+
// `local`: platform-native isAbsolute, so a drive path on a POSIX host is refused rather than kept as a member no
|
|
168
|
+
// job here can have (scoped limits keep such a row verbatim and inert; a new file need not).
|
|
169
|
+
if (!isAbsolute(trimmed)) throw configError(`${at}: ${JSON.stringify(trimmed)} is not an absolute path on this host: ${path}`);
|
|
170
|
+
return resolve(trimmed);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Load and validate the projects file named by `config.projectsFile`. Returns `[]` when it is unset (no projects, a
|
|
175
|
+
* valid deployment). An empty string is a value, so it reaches `existsSync` and is refused, as the scoped-limits key's
|
|
176
|
+
* is. `readFileSync`/`existsSync` are injectable for tests.
|
|
177
|
+
*/
|
|
178
|
+
export function loadProjects(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync } = {}) {
|
|
179
|
+
const path = config.projectsFile;
|
|
180
|
+
if (path === null || path === undefined) return [];
|
|
181
|
+
if (!existsSync(path)) throw configError(`projects file does not exist: ${escapeControls(path)}`);
|
|
182
|
+
return parseProjects(readFileSync(path, "utf8"), path);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* The id of the project this job belongs to, or null. `job` is job data (`kind`, `repo`, `folder`). A forge job is
|
|
187
|
+
* matched by its forge-qualified scope (`github:acme/web`), never by its bare repo, so a GitHub job and a Forgejo job
|
|
188
|
+
* for `acme/web` can sit in different projects. A local job is matched by its resolved folder (`canonicalScope`), the
|
|
189
|
+
* spelling a member is stored in, so `/srv/shop-tools/` matches the member `/srv/shop-tools`.
|
|
190
|
+
*/
|
|
191
|
+
export function projectOf(job, projects) {
|
|
192
|
+
if (!Array.isArray(projects) || projects.length === 0) return null;
|
|
193
|
+
const scope = memberScopeOf(job);
|
|
194
|
+
if (typeof scope !== "string" || scope === "") return null;
|
|
195
|
+
for (const project of projects) {
|
|
196
|
+
if (Array.isArray(project?.members) && project.members.includes(scope)) return project.id;
|
|
197
|
+
}
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* The scope a job is matched to a member by (`projectOf`): its forge-qualified scope, or for a local job its resolved
|
|
203
|
+
* folder. When `projectOf` found a project, this IS the member it matched, in the stored spelling a priorities plan
|
|
204
|
+
* names by `scopeRef` (issue #504 part B), so the repo share a plan gave it needs no second matching rule.
|
|
205
|
+
*/
|
|
206
|
+
export function memberScopeOf(job) {
|
|
207
|
+
return job?.kind === "local" ? canonicalScope(job) : qualifiedScopeOf(job);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* What `fpProjects` hashes (issue #499 part C, INT-HOST-REGISTRY-CONTRACT), exported so a test can read it: one entry
|
|
212
|
+
* per project, sorted by id, as `{ id, members }` where `members` is the sorted 16-hex hash of each member. Never a
|
|
213
|
+
* `name`, which is free text, and never a member in clear: a folder member is a host path and a repo member a
|
|
214
|
+
* repository name, and the registry's content rule keeps both out of a Valkey value, even inside a digest's input.
|
|
215
|
+
* The id is charset-checked operator text, the same admissibility a run record gives it.
|
|
216
|
+
*
|
|
217
|
+
* Membership is what the comparison is about: two hosts that put one scope in two projects, or a scope in a project on
|
|
218
|
+
* one host and in none on the other, record different projects and count the scope against different project rows.
|
|
219
|
+
* A `name` decides nothing, so renaming the display text on one host is not a disagreement.
|
|
220
|
+
*/
|
|
221
|
+
export function projectsFingerprintInput(projects) {
|
|
222
|
+
return (Array.isArray(projects) ? projects : [])
|
|
223
|
+
.filter((p) => isProjectId(p?.id))
|
|
224
|
+
.map((p) => ({ id: p.id, members: (Array.isArray(p.members) ? p.members : []).map((m) => hash16(String(m))).sort() }))
|
|
225
|
+
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* The 16-hex fingerprint `fpProjects` of a host's live projects (`fingerprint.mjs`). It never abstains: a host with no
|
|
230
|
+
* projects file publishes the fingerprint of no projects, because a host that puts a repo in no project while a peer
|
|
231
|
+
* puts it in one is the disagreement worth seeing.
|
|
232
|
+
*/
|
|
233
|
+
export function projectsFingerprint(projects) {
|
|
234
|
+
return fingerprint(projectsFingerprintInput(projects));
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** The fingerprint of no projects: how a reader tells whether projects are in use anywhere on the fleet. */
|
|
238
|
+
export const EMPTY_PROJECTS_FINGERPRINT = projectsFingerprint([]);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The environment variables pi and
|
|
3
|
-
*
|
|
2
|
+
* The environment variables pi and the packages it runs read to CONFIGURE a provider or pi itself: where the
|
|
3
|
+
* request goes, which credentials it carries, and where pi reads its own configuration (issues #314, #511).
|
|
4
4
|
*
|
|
5
5
|
* WHY THIS IS A REFUSAL. `run.secrets` lets a trigger name an environment variable, and the existing gates
|
|
6
6
|
* refuse the names the worker writes and the ones pi reads the resolved provider's KEY from. They refuse
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
* resolved provider, and the bound that gate keeps is deliberate and documented -- an `anthropic` job may
|
|
22
22
|
* bind `OPENAI_API_KEY` for a flow that talks to OpenAI itself, because refusing it would be this project
|
|
23
23
|
* claiming a namespace it does not own. Including them here would have broken that bound ARBITRARILY: only
|
|
24
|
-
*
|
|
24
|
+
* six of pi's thirty-eight key variables (at the 0.99.1 pin) happen to be read by name in a scanned
|
|
25
25
|
* artifact, so `OPENAI_API_KEY` would refuse while `GROQ_API_KEY` and `HF_TOKEN` stayed bindable, and the
|
|
26
26
|
* documented rule would be false for reasons no operator could predict. They are subtracted, and the bolt
|
|
27
27
|
* subtracts them the same way rather than by hand. ONE is kept, by name and for a stated reason, in
|
|
@@ -31,70 +31,93 @@
|
|
|
31
31
|
* them and they are read by the SDK as provider configuration, which is exactly what this set is.
|
|
32
32
|
*
|
|
33
33
|
* DERIVED, AND THE DERIVATION IS BOLTED IN BOTH DIRECTIONS by `worker/test/provider-steering.test.mjs`,
|
|
34
|
-
* from the pinned artifacts rather than from a second copy of them
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
34
|
+
* from the pinned artifacts rather than from a second copy of them. Every name the scan finds is in, with
|
|
35
|
+
* no judgement about which ones matter; the only subtractions are named and asserted there.
|
|
36
|
+
*
|
|
37
|
+
* TWO HOPS, THROUGH DECLARED DEPENDENCIES. Hop 1 is every package pi-ai's own dist imports, discovered
|
|
38
|
+
* from its import statements rather than listed, so a pi bump that adds an SDK fails the bolt. Hop 2 is,
|
|
39
|
+
* for each of those, the packages its sources import AND its package.json declares in `dependencies`,
|
|
40
|
+
* which is where google-auth-library, `@smithy/core` and the AWS credential chain are. The declared-
|
|
41
|
+
* dependency filter keeps out an optional peer that resolves only by an accident of layout (`openai`
|
|
42
|
+
* imports `undici` without declaring it).
|
|
43
|
+
*
|
|
44
|
+
* EVERY OCCURRENCE COUNTED, not a list of accessor spellings (a list kept missing the next one). After
|
|
45
|
+
* stripping comments, the bolt counts every occurrence that can reach the environment in every scanned
|
|
46
|
+
* file: each `env` token (so `process.env`, `ctx.env`, a parameter named `env`, `{ env: e } = process`),
|
|
47
|
+
* each `process["env"]`, each use of an alias of one (`const v = env()`, `const e = process.env`), and
|
|
48
|
+
* each call of a helper. Helpers are DERIVED: any named function one of whose own parameters is the key of
|
|
49
|
+
* an environment read (`resolveEnvConfigValue(name, env)`), so a new call of an existing helper counts. An occurrence either NAMES a variable (`.X`, `["X"]`, a key
|
|
50
|
+
* resolved through a string constant, a call `("X")`, `"X" in env`, `{ X } = env`, a selector argument
|
|
51
|
+
* beside its key) or it is a SITE, and the bolt pins every file's sites with a COUNT
|
|
52
|
+
* (`worker/test/fixtures/provider-steering-sites.json`). So a new occurrence anywhere either names
|
|
53
|
+
* something the equality sees or changes a count: a new helper reading `process.env[name]`, a key built
|
|
54
|
+
* at runtime, a new alias. A write is a site, not a read.
|
|
55
|
+
* The rule starts from an `env` token, so a form with none is NOT seen: `process["e" + "nv"]`,
|
|
56
|
+
* `const E = "env"; process[E]`, `{ ["env"]: e } = process`, `Reflect.get(process, "env")`,
|
|
57
|
+
* `require("process")["env"]`. None occurs in the pinned sources.
|
|
58
|
+
*
|
|
59
|
+
* BOTH COPIES. The hoisted pi-ai this workspace resolves, and the one the runner dispatches through,
|
|
60
|
+
* nested under pi-coding-agent with its own google-auth-library (10.6.2 there, 10.9.1 hoisted). The union
|
|
61
|
+
* of the two is reserved.
|
|
62
|
+
*
|
|
63
|
+
* LOWERCASE TWINS ARE LITERAL MEMBERS, and matching stays EXACT. google-auth-library reads
|
|
64
|
+
* `google_application_credentials`, `gcloud_project` and `google_cloud_project` beside the uppercase
|
|
65
|
+
* forms, so the scan finds them as reads and they are in. Folding case instead would refuse names no
|
|
66
|
+
* pinned source reads (`openai_base_url`, `anthropic_api_key`), which is the other half of the rule: an
|
|
67
|
+
* operator's own name stays bindable unless something pinned reads it.
|
|
68
|
+
*
|
|
69
|
+
* PI'S OWN NAMESPACE, in `PI_OWN_READS` below. Every `PI_*` name pi-coding-agent's dist and the pi
|
|
70
|
+
* packages it declares read (pi-tui runs in the same process). The agent and session directory keys are
|
|
71
|
+
* built at runtime from pi's app name, so the bolt imports pi's `config.js` to evaluate them. Two sets are
|
|
72
|
+
* subtracted: the names the worker writes into the container (`CONTAINER_ENV_NAMES`, reserved already),
|
|
73
|
+
* and the ones the runner assigns before pi runs, derived from `image/runner/src` (`PI_OFFLINE`,
|
|
74
|
+
* `PI_TELEMETRY`). `dist/bundle/`, the vendored single-file build, is not the code the runner loads and is
|
|
75
|
+
* not scanned.
|
|
76
|
+
*
|
|
77
|
+
* THE LIMITS, stated because a set like this is only worth what its boundary is honest about.
|
|
78
|
+
*
|
|
79
|
+
* Hop 3 is not followed, and these are read there and NOT reserved: in gcp-metadata (under
|
|
80
|
+
* google-auth-library) `GCE_METADATA_HOST`, `GCE_METADATA_IP`, `METADATA_SERVER_DETECTION` and `K_SERVICE`;
|
|
81
|
+
* in the AWS credential providers (under `@aws-sdk/credential-provider-node`) `AWS_ROLE_ARN` and
|
|
82
|
+
* `AWS_ROLE_SESSION_NAME` (web identity), `AWS_CONTAINER_AUTHORIZATION_TOKEN` and its `_FILE` form (http,
|
|
83
|
+
* useless without the reserved `AWS_CONTAINER_CREDENTIALS_FULL_URI`), and `AWS_ACCOUNT_ID`,
|
|
84
|
+
* `AWS_CREDENTIAL_SCOPE` and `AWS_CREDENTIAL_EXPIRATION` (env). The metadata host is the sharpest of them:
|
|
85
|
+
* it moves where google-auth-library asks for a token when no other credential is configured.
|
|
86
|
+
*
|
|
87
|
+
* pi-coding-agent's own third-party dependencies are not scanned either (undici, jiti, yaml, semver,
|
|
88
|
+
* cross-spawn, chalk and the rest). At the 0.99.1 pin they read the proxy variables (reserved by the egress
|
|
89
|
+
* policy or listed below) and tooling switches: jiti's `NODE_DEBUG` and Babel flags, chalk's
|
|
90
|
+
* `FORCE_COLOR`, yaml's `LOG_TOKENS`.
|
|
91
|
+
*
|
|
92
|
+
* Names pi reads outside its `PI_*` namespace are found and left out, and the bolt pins the list: the
|
|
93
|
+
* terminal, the OS, the editor, and the `llama.cpp` extension's `LLAMA_BASE_URL`, which pi reads both
|
|
94
|
+
* through `ctx.env` and through `process.env`. The worker cannot dispatch to that provider, and the bolt
|
|
95
|
+
* asserts it still cannot.
|
|
54
96
|
*
|
|
55
97
|
* `NODE_OPTIONS`, `NODE_EXTRA_CA_CERTS`, `SSL_CERT_FILE` and `NODE_TLS_REJECT_UNAUTHORIZED` are NOT here.
|
|
56
98
|
* They subvert any provider call, but they are properties of the RUNTIME rather than of a provider, they
|
|
57
99
|
* predate this gate, and `NODE_OPTIONS` additionally needs a file the attacker can place. They belong to
|
|
58
100
|
* whatever closes the runtime-hijack question, not to a set derived from what pi reads about providers.
|
|
59
101
|
* The same holds for `HOME`, `PATH`, `APPDATA`, `USERPROFILE` and `XDG_CONFIG_HOME`, which the scan DOES
|
|
60
|
-
* reach
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* vault to hold a useful string at a reference the
|
|
70
|
-
*
|
|
71
|
-
* set rather than distinguishing parts of it.
|
|
102
|
+
* reach: the Anthropic SDK reads the four directory variables only to locate its default config directory
|
|
103
|
+
* (`core/credentials`), and `PATH` inside its agent toolset, which pi does not use. `HOMEDRIVE` and
|
|
104
|
+
* `HOMEPATH` join them (issue #511): `@smithy/core` reads them in the same home-directory helper. The bolt subtracts them
|
|
105
|
+
* by name, with that reason, and asserts the scan still finds each, so a subtraction that stopped being
|
|
106
|
+
* needed is removed rather than carried. `HOME` is reserved anyway, by `reserved-env.mjs`, because the
|
|
107
|
+
* worker writes it (issue #341). The Anthropic-specific switch for the same directory,
|
|
108
|
+
* `ANTHROPIC_CONFIG_DIR`, IS in the set: it names a credential location outright.
|
|
109
|
+
*
|
|
110
|
+
* And a bound on the whole family: a trigger author picks the variable NAME and a vault REFERENCE, never a
|
|
111
|
+
* value. Exploiting any of these needs the operator's own vault to hold a useful string at a reference the
|
|
112
|
+
* author is allowed to name. That is equally true of `AZURE_OPENAI_BASE_URL`, the variable #314 was filed
|
|
113
|
+
* about, so it bounds the severity of the whole set rather than distinguishing parts of it. An operator
|
|
114
|
+
* who needs one of these names in a job puts it on `PI_FORWARD_ENV`, the operator's own list.
|
|
72
115
|
*
|
|
73
116
|
* IMPORT-FREE, like `reserved-env.mjs` and `provider-key.mjs` beside it: `triggers.mjs` is the shared
|
|
74
117
|
* validator, the receiver loads it, and `admin/build.mjs` inlines it into the published console.
|
|
75
118
|
*/
|
|
76
119
|
|
|
77
120
|
|
|
78
|
-
/**
|
|
79
|
-
* The steering variables a literal scan of the packages pi imports cannot reach.
|
|
80
|
-
*
|
|
81
|
-
* Named rather than quietly absent, because "derived, never curated" would otherwise be a claim the bolt
|
|
82
|
-
* cannot keep. Two reasons they are unreachable, and the test asserts BOTH still hold.
|
|
83
|
-
*
|
|
84
|
-
* The AWS four are read inside `@smithy/core`, one dependency hop past this scan's boundary, and two of
|
|
85
|
-
* them are additionally read through a key the resolver builds at runtime
|
|
86
|
-
* (`AWS_ENDPOINT_URL_<SERVICEID>`). They matter because pi stops pinning the Bedrock endpoint itself as
|
|
87
|
-
* soon as `AWS_REGION` or `AWS_PROFILE` is present, which is the ordinary way to configure Bedrock. Both
|
|
88
|
-
* measured: `AWS_ENDPOINT_URL` redirects the call, and `AWS_SHARED_CREDENTIALS_FILE` replaces the
|
|
89
|
-
* credential it is signed with. `AWS_CONFIG_FILE` does both, and can also name a `credential_process`
|
|
90
|
-
* shell command.
|
|
91
|
-
*
|
|
92
|
-
* The proxy spellings are read by pi's own `getProxyEnv`, which lowercases and uppercases the key it is
|
|
93
|
-
* given and asks for both. `EGRESS_ENV_VARS` owns `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` and keeps
|
|
94
|
-
* them; what is added here is the spellings it does not have. pi reads the LOWERCASE form FIRST, so
|
|
95
|
-
* `https_proxy` outranks the egress policy's own variable in pi's reader. Only the schemes a provider call
|
|
96
|
-
* can use are listed: `ws_proxy` and the rest are reachable in `getProxyEnv` but not from an HTTPS request.
|
|
97
|
-
*/
|
|
98
121
|
/**
|
|
99
122
|
* Provider KEY variables that stay reserved here although the bolt subtracts key variables in general.
|
|
100
123
|
*
|
|
@@ -109,16 +132,55 @@
|
|
|
109
132
|
*/
|
|
110
133
|
const RETAINED_KEY_VARIABLES = ["ANTHROPIC_AUTH_TOKEN"];
|
|
111
134
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
135
|
+
/**
|
|
136
|
+
* The steering variables the scan cannot reach.
|
|
137
|
+
*
|
|
138
|
+
* Named rather than quietly absent, because "derived, never curated" would otherwise be a claim the bolt
|
|
139
|
+
* cannot keep. Four of them, each read through a key built at runtime, and the bolt pins the key that
|
|
140
|
+
* builds it among the unresolved ones.
|
|
141
|
+
*
|
|
142
|
+
* `AWS_ENDPOINT_URL_BEDROCK_RUNTIME` is `@smithy/core`'s `AWS_ENDPOINT_URL_<SERVICE>`. It matters because pi
|
|
143
|
+
* stops pinning the Bedrock endpoint itself as soon as `AWS_REGION` or `AWS_PROFILE` is present, which is
|
|
144
|
+
* the ordinary way to configure Bedrock. (`AWS_ENDPOINT_URL`, `AWS_CONFIG_FILE` and
|
|
145
|
+
* `AWS_SHARED_CREDENTIALS_FILE` were here until issue #511; the scan now names them.)
|
|
146
|
+
*
|
|
147
|
+
* The proxy spellings are read by pi's own `getProxyEnv`, which lowercases and uppercases the key it is
|
|
148
|
+
* given and asks for both, and builds `${protocol}_proxy`. `all_proxy` and `no_proxy` it names literally,
|
|
149
|
+
* so the scan finds those. `EGRESS_ENV_VARS` owns `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY`; what is left
|
|
150
|
+
* is here. pi reads the LOWERCASE form FIRST, so `https_proxy` outranks the egress policy's own variable in
|
|
151
|
+
* pi's reader. Only the schemes a provider call can use are listed.
|
|
152
|
+
*/
|
|
153
|
+
const UNREACHABLE_BY_SCAN = ["AWS_ENDPOINT_URL_BEDROCK_RUNTIME", "ALL_PROXY", "http_proxy", "https_proxy"];
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* pi's own reads in its `PI_*` namespace (issue #511), minus what the worker and the runner write.
|
|
157
|
+
*
|
|
158
|
+
* `PI_CODING_AGENT_DIR` points pi at another agent directory, and with it another `auth.json`, settings
|
|
159
|
+
* and models file; `PI_RADIUS_GATEWAY`, `PI_SHARE_VIEWER_URL` and `PI_INSTALLER_API_BASE` are URLs pi
|
|
160
|
+
* calls. The rest are here because the scan finds them and the rule has no judgement in it: a name the
|
|
161
|
+
* pinned pi reads for its own configuration is not a name a trigger author picks.
|
|
162
|
+
*/
|
|
163
|
+
const PI_OWN_READS = [
|
|
164
|
+
"PI_CLEAR_ON_SHRINK",
|
|
165
|
+
"PI_CODING_AGENT_DIR",
|
|
166
|
+
"PI_CODING_AGENT_SESSION_DIR",
|
|
167
|
+
"PI_EXPERIMENTAL",
|
|
168
|
+
"PI_HARDWARE_CURSOR",
|
|
169
|
+
"PI_HYPERLINKS",
|
|
170
|
+
"PI_IMAGE_PROTOCOL",
|
|
171
|
+
"PI_INSTALLER_API_BASE",
|
|
172
|
+
"PI_MANAGED_INSTALL_ROOT",
|
|
173
|
+
"PI_PACKAGE_DIR",
|
|
174
|
+
"PI_RADIUS_GATEWAY",
|
|
175
|
+
"PI_SHARE_VIEWER_URL",
|
|
176
|
+
"PI_SKIP_VERSION_CHECK",
|
|
177
|
+
"PI_STARTUP_BENCHMARK",
|
|
178
|
+
"PI_TIMING",
|
|
179
|
+
"PI_TRUE_COLOR",
|
|
180
|
+
"PI_TUI_DEBUG",
|
|
181
|
+
"PI_TUI_DEBUG_REDRAW",
|
|
182
|
+
"PI_TUI_ESC_TIMEOUT",
|
|
183
|
+
"PI_TUI_WRITE_LOG",
|
|
122
184
|
];
|
|
123
185
|
|
|
124
186
|
export const PROVIDER_STEERING_VARS = new Set([
|
|
@@ -141,35 +203,75 @@ export const PROVIDER_STEERING_VARS = new Set([
|
|
|
141
203
|
"ANTHROPIC_WORK_ID",
|
|
142
204
|
"ANTHROPIC_WORK_SECRET",
|
|
143
205
|
"AWS_ACCESS_KEY_ID",
|
|
206
|
+
"AWS_ACCOUNT_ID_ENDPOINT_MODE",
|
|
207
|
+
"AWS_AUTH_SCHEME_PREFERENCE",
|
|
144
208
|
"AWS_BEARER_TOKEN_BEDROCK",
|
|
145
209
|
"AWS_BEDROCK_BASE_URL",
|
|
146
210
|
"AWS_BEDROCK_FORCE_CACHE",
|
|
147
211
|
"AWS_BEDROCK_FORCE_HTTP1",
|
|
148
212
|
"AWS_BEDROCK_SKIP_AUTH",
|
|
213
|
+
"AWS_CONFIG_FILE",
|
|
149
214
|
"AWS_CONTAINER_CREDENTIALS_FULL_URI",
|
|
150
215
|
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
|
|
216
|
+
"AWS_DEFAULTS_MODE",
|
|
151
217
|
"AWS_DEFAULT_REGION",
|
|
218
|
+
"AWS_DISABLE_CLOCK_SKEW_CORRECTION",
|
|
219
|
+
"AWS_EC2_METADATA_DISABLED",
|
|
220
|
+
"AWS_EC2_METADATA_SERVICE_ENDPOINT",
|
|
221
|
+
"AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
|
|
222
|
+
"AWS_ENDPOINT_URL",
|
|
223
|
+
"AWS_EXECUTION_ENV",
|
|
224
|
+
"AWS_IGNORE_CONFIGURED_ENDPOINT_URLS",
|
|
225
|
+
"AWS_LAMBDA_FUNCTION_NAME",
|
|
226
|
+
"AWS_MAX_ATTEMPTS",
|
|
227
|
+
"AWS_NEW_RETRIES_2026",
|
|
152
228
|
"AWS_PROFILE",
|
|
153
229
|
"AWS_REGION",
|
|
230
|
+
"AWS_RETRY_MODE",
|
|
231
|
+
"AWS_SDK_JS_NODE_VERSION_SUPPORT_WARNING_DISABLED",
|
|
232
|
+
"AWS_SDK_UA_APP_ID",
|
|
154
233
|
"AWS_SECRET_ACCESS_KEY",
|
|
155
234
|
"AWS_SESSION_TOKEN",
|
|
235
|
+
"AWS_SHARED_CREDENTIALS_FILE",
|
|
236
|
+
"AWS_SIGV4A_SIGNING_REGION_SET",
|
|
237
|
+
"AWS_USE_DUALSTACK_ENDPOINT",
|
|
238
|
+
"AWS_USE_FIPS_ENDPOINT",
|
|
156
239
|
"AWS_WEB_IDENTITY_TOKEN_FILE",
|
|
157
240
|
"AZURE_OPENAI_API_VERSION",
|
|
158
241
|
"AZURE_OPENAI_BASE_URL",
|
|
159
242
|
"AZURE_OPENAI_DEPLOYMENT_NAME_MAP",
|
|
160
243
|
"AZURE_OPENAI_ENDPOINT",
|
|
161
244
|
"AZURE_OPENAI_RESOURCE_NAME",
|
|
245
|
+
"CLOUDFLARE_ACCOUNT_ID",
|
|
246
|
+
"CLOUDFLARE_GATEWAY_ID",
|
|
247
|
+
"CLOUDSDK_CONFIG",
|
|
248
|
+
"CLOUD_RUN_JOB",
|
|
249
|
+
"DEBUG",
|
|
250
|
+
"FUNCTION_NAME",
|
|
251
|
+
"FUNCTION_TARGET",
|
|
252
|
+
"GAE_MODULE_NAME",
|
|
253
|
+
"GAE_SERVICE",
|
|
162
254
|
"GCLOUD_PROJECT",
|
|
255
|
+
"GOOGLE_API_CERTIFICATE_CONFIG",
|
|
163
256
|
"GOOGLE_API_KEY",
|
|
164
257
|
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
165
258
|
"GOOGLE_CLOUD_LOCATION",
|
|
166
259
|
"GOOGLE_CLOUD_PROJECT",
|
|
260
|
+
"GOOGLE_CLOUD_QUOTA_PROJECT",
|
|
261
|
+
"GOOGLE_EXTERNAL_ACCOUNT_ALLOW_EXECUTABLES",
|
|
167
262
|
"GOOGLE_GEMINI_BASE_URL",
|
|
263
|
+
"GOOGLE_GENAI_ACCESS_TOKEN",
|
|
264
|
+
"GOOGLE_GENAI_API_KEY",
|
|
265
|
+
"GOOGLE_GENAI_API_VERSION",
|
|
266
|
+
"GOOGLE_GENAI_DEBUG",
|
|
267
|
+
"GOOGLE_GENAI_USER_PROJECT",
|
|
168
268
|
"GOOGLE_GENAI_USE_ENTERPRISE",
|
|
169
269
|
"GOOGLE_GENAI_USE_VERTEXAI",
|
|
170
270
|
"GOOGLE_VERTEX_BASE_URL",
|
|
271
|
+
"HOSTNAME",
|
|
171
272
|
"KIMI_CODE_OAUTH_HOST",
|
|
172
273
|
"KIMI_OAUTH_HOST",
|
|
274
|
+
"K_CONFIGURATION",
|
|
173
275
|
"OPENAI_ADMIN_KEY",
|
|
174
276
|
"OPENAI_API_VERSION",
|
|
175
277
|
"OPENAI_BASE_URL",
|
|
@@ -180,6 +282,18 @@ export const PROVIDER_STEERING_VARS = new Set([
|
|
|
180
282
|
"OPENAI_WEBHOOK_SECRET",
|
|
181
283
|
"PI_CACHE_RETENTION",
|
|
182
284
|
"PI_OAUTH_CALLBACK_HOST",
|
|
285
|
+
"SMITHY_NEW_RETRIES_2026",
|
|
286
|
+
"WS_NO_BUFFER_UTIL",
|
|
287
|
+
"WS_NO_UTF_8_VALIDATE",
|
|
288
|
+
"_X_AMZN_TRACE_ID",
|
|
289
|
+
// The lowercase twins google-auth-library reads beside the uppercase forms, and the two proxy spellings
|
|
290
|
+
// pi's getProxyEnv names literally (issue #511).
|
|
291
|
+
"all_proxy",
|
|
292
|
+
"gcloud_project",
|
|
293
|
+
"google_application_credentials",
|
|
294
|
+
"google_cloud_project",
|
|
295
|
+
"no_proxy",
|
|
296
|
+
...PI_OWN_READS,
|
|
183
297
|
...RETAINED_KEY_VARIABLES,
|
|
184
298
|
...UNREACHABLE_BY_SCAN,
|
|
185
299
|
]);
|