@edgehero/pi-dispatch 2.1.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +41 -5
- package/README.md +11 -5
- package/deploy/docker-compose.yml +12 -0
- package/deploy/egress-proxy.conf +28 -3
- package/deploy/pi-dispatch-egress-proxy.container +8 -2
- package/package.json +9 -2
- package/src/allocation.mjs +731 -0
- package/src/backends.mjs +243 -0
- package/src/budget.mjs +40 -4
- package/src/cli.mjs +222 -11
- package/src/config.mjs +126 -5
- package/src/daemon-facts.mjs +3 -0
- package/src/deployment-venue.mjs +1 -0
- package/src/doctor.mjs +2280 -195
- package/src/dollar-budget.mjs +373 -0
- package/src/dollar-fingerprint.mjs +83 -0
- package/src/egress-cli.mjs +316 -0
- package/src/egress-proxy-state.mjs +35 -5
- package/src/egress.mjs +12 -0
- package/src/env-allowlist.mjs +125 -6
- package/src/env-file.mjs +194 -25
- package/src/envelope.mjs +413 -0
- package/src/exit-code.mjs +22 -0
- package/src/fleet-lease.mjs +85 -25
- package/src/get-token.mjs +16 -5
- package/src/git-dirty.mjs +67 -0
- package/src/github-app-setup.mjs +6 -3
- package/src/github-host.mjs +5 -3
- package/src/host-pi.mjs +1 -1
- package/src/identity.mjs +2 -1
- package/src/image-preflight.mjs +98 -24
- package/src/image-ref.mjs +37 -0
- package/src/import-pi.mjs +4 -2
- package/src/index.mjs +407 -62
- package/src/init.mjs +18 -0
- package/src/job-id.mjs +26 -3
- package/src/live-probes.mjs +24 -9
- package/src/model-catalog.mjs +297 -0
- package/src/model-endpoints.mjs +671 -0
- package/src/model-ref.mjs +151 -0
- package/src/models-json.mjs +268 -0
- package/src/money.mjs +144 -0
- package/src/octokit-log.mjs +65 -0
- package/src/outbox-plan.mjs +218 -0
- package/src/outbox.mjs +29 -9
- package/src/output-cap.mjs +157 -0
- package/src/pause-windows.mjs +81 -2
- package/src/pi-model-loader.mjs +77 -0
- package/src/podman-stack.mjs +16 -3
- package/src/portfolio-snapshot.mjs +304 -0
- package/src/prepare-local.mjs +247 -12
- package/src/prepare.mjs +35 -3
- package/src/priorities.mjs +569 -0
- package/src/processor.mjs +599 -170
- package/src/project-id.mjs +17 -0
- package/src/projects.mjs +238 -0
- package/src/provider-steering.mjs +179 -65
- package/src/queue.mjs +111 -6
- package/src/reserved-env.mjs +31 -0
- package/src/run-container.mjs +59 -5
- package/src/run-history.mjs +379 -24
- package/src/run-mirror.mjs +30 -0
- package/src/runtime-settings.mjs +104 -9
- package/src/schedules.mjs +33 -1
- package/src/scoped-limits.mjs +447 -27
- package/src/service.mjs +15 -4
- package/src/session-store.mjs +131 -6
- package/src/start.mjs +528 -40
- package/src/triggers-file.mjs +65 -4
- package/src/triggers.mjs +141 -11
- package/src/up.mjs +308 -34
- package/src/valkey-endpoint.mjs +3 -2
package/src/get-token.mjs
CHANGED
|
@@ -27,6 +27,7 @@ import { createAppAuth as realCreateAppAuth } from "@octokit/auth-app";
|
|
|
27
27
|
import { Octokit as RealOctokit } from "@octokit/rest";
|
|
28
28
|
import { configError } from "./config.mjs";
|
|
29
29
|
import { resolveSelfId } from "./identity.mjs";
|
|
30
|
+
import { octokitLogOptions } from "./octokit-log.mjs";
|
|
30
31
|
import { InfraRetry } from "./processor.mjs";
|
|
31
32
|
import { isDeterminateFetchFailure, isDeterminateFsCode, isTransientStatus, octokitHeaderReader } from "./transient.mjs";
|
|
32
33
|
|
|
@@ -54,14 +55,18 @@ export async function makeGitHubAuth(cfg, deps = {}) {
|
|
|
54
55
|
execFile = realExecFile,
|
|
55
56
|
readFile = realReadFile,
|
|
56
57
|
env = process.env,
|
|
58
|
+
// Issue #530: the caller's `(event, fields)` logger, which a GitHub client's own warnings go through. Every client
|
|
59
|
+
// here is built with `octokitLog`, so none prints Octokit's plain `GET /user - 401 ...` line into a JSON stream.
|
|
60
|
+
log,
|
|
57
61
|
} = deps;
|
|
62
|
+
const quiet = octokitLogOptions(log);
|
|
58
63
|
|
|
59
64
|
const source = cfg?.source;
|
|
60
65
|
|
|
61
66
|
if (source === "pat") {
|
|
62
67
|
const patVar = cfg.patVar ?? "GITHUB_PAT";
|
|
63
68
|
const token = requireToken(env[patVar], patVar);
|
|
64
|
-
const octokit = new Octokit({ auth: token });
|
|
69
|
+
const octokit = new Octokit({ auth: token, ...quiet });
|
|
65
70
|
const selfId = await resolveSelfId({ source: "pat", octokit });
|
|
66
71
|
const mintToken = async () => requireToken(token, patVar);
|
|
67
72
|
return { mintToken, selfId, source };
|
|
@@ -71,7 +76,7 @@ export async function makeGitHubAuth(cfg, deps = {}) {
|
|
|
71
76
|
const execFileAsync = promisify(execFile);
|
|
72
77
|
const ghToken = () => runGhAuthToken(execFileAsync);
|
|
73
78
|
// Resolve identity from a token minted once here; each job mints its own below.
|
|
74
|
-
const octokit = new Octokit({ auth: await ghToken() });
|
|
79
|
+
const octokit = new Octokit({ auth: await ghToken(), ...quiet });
|
|
75
80
|
const selfId = await resolveSelfId({ source: "gh", octokit });
|
|
76
81
|
const mintToken = async (job) => {
|
|
77
82
|
assertResumeAllowedOnGhSource(job, allowGhResume);
|
|
@@ -89,10 +94,16 @@ export async function makeGitHubAuth(cfg, deps = {}) {
|
|
|
89
94
|
// The key is either already in hand (GITHUB_APP_PRIVATE_KEY, normalised and shape-checked at config
|
|
90
95
|
// load) or on disk. loadGitHubAuth refuses both-set, so this is a fallback, never a precedence rule.
|
|
91
96
|
const privateKey = cfg.privateKey ?? (await readPem(readFile, cfg.privateKeyPath));
|
|
92
|
-
|
|
97
|
+
// `log` rides in the auth options too: the per-job `createAppAuth` below builds no Octokit, and
|
|
98
|
+
// `@octokit/auth-app` otherwise defaults its own warnings to `console.warn`.
|
|
99
|
+
const auth = { appId: cfg.appId, privateKey, installationId: cfg.installationId, log: quiet.log };
|
|
100
|
+
// And its `request`: auth-app's default one is a bare `@octokit/request`, whose deprecation warning goes to
|
|
101
|
+
// `console`. An unauthenticated Octokit's `request` carries the quiet logger and adds no auth hook; the identity
|
|
102
|
+
// client below is not changed, since Octokit already hands its own request to its auth strategy.
|
|
103
|
+
const mintRequest = new Octokit({ ...quiet }).request;
|
|
93
104
|
|
|
94
105
|
// App-JWT client: resolveSelfId's app path reads GET /app then GET /users/{slug}[bot].
|
|
95
|
-
const octokit = new Octokit({ authStrategy: createAppAuth, auth });
|
|
106
|
+
const octokit = new Octokit({ authStrategy: createAppAuth, auth, ...quiet });
|
|
96
107
|
const selfId = await resolveSelfId({ source: "app", octokit });
|
|
97
108
|
|
|
98
109
|
const mintToken = async (job) => {
|
|
@@ -112,7 +123,7 @@ export async function makeGitHubAuth(cfg, deps = {}) {
|
|
|
112
123
|
const permissions = job?.permissions;
|
|
113
124
|
let minted;
|
|
114
125
|
try {
|
|
115
|
-
const appAuth = createAppAuth(auth);
|
|
126
|
+
const appAuth = createAppAuth({ ...auth, request: mintRequest });
|
|
116
127
|
minted = await appAuth({ type: "installation", repositoryNames, ...(permissions && { permissions }) });
|
|
117
128
|
} catch (error) {
|
|
118
129
|
throw classifyAppMintError(error);
|
package/src/git-dirty.mjs
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
2
4
|
import { GIT_READ_FLAGS } from "./git-hardening.mjs";
|
|
3
5
|
|
|
4
6
|
/**
|
|
@@ -22,3 +24,68 @@ export function gitDirty(folder, { exec = execFileSync } = {}) {
|
|
|
22
24
|
return null;
|
|
23
25
|
}
|
|
24
26
|
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Why `folder` cannot be a local job's folder, as one sentence with its fix, or null when it can (issue #524).
|
|
30
|
+
*
|
|
31
|
+
* The worker's own rule, stated here so `pi-dispatch run` refuses before anything is queued. `prepare-local.mjs`
|
|
32
|
+
* requires `.git` AT the folder (a directory, or the file a worktree or submodule has) and reads the job's
|
|
33
|
+
* instructions from HEAD. So a subfolder of a repository is refused like a plain folder, and a repository with no
|
|
34
|
+
* commit yet is refused too, as the worker's `local-folder-no-commit`. Before this the CLI skipped every check when
|
|
35
|
+
* `.git` was missing, the worker refused the job as a generic `config-refused`, and nothing said which folder or why.
|
|
36
|
+
*
|
|
37
|
+
* Only the folder's root counts, and that is the worker's requirement rather than a choice made here. When the folder
|
|
38
|
+
* has no `.git`, git itself is asked whether the folder sits inside a repository (`rev-parse --show-toplevel`,
|
|
39
|
+
* hardened), and a root is named only when git confirms one. A walk up the parents for a `.git` entry was refuted in
|
|
40
|
+
* PR #528's review: an empty `.git` directory is not a repository, and it named one anyway. Even a confirmed root is
|
|
41
|
+
* never offered as the only fix. A dotfiles repository (`git init` in the home directory, `*` ignored) contains
|
|
42
|
+
* every folder under home, so "run it on <home>" reads clean and hands the agent the whole home directory. Both
|
|
43
|
+
* options are always named, and the operator picks.
|
|
44
|
+
*
|
|
45
|
+
* `exists` and `exec` are seams; `exec` runs HARDENED for the same reason `gitDirty` does.
|
|
46
|
+
*/
|
|
47
|
+
export function localRepoProblem(folder, { exists = existsSync, exec = execFileSync } = {}) {
|
|
48
|
+
if (!exists(join(folder, ".git"))) {
|
|
49
|
+
const root = enclosingRepoRoot(folder, exec);
|
|
50
|
+
if (root) return `${folder} is not the root of a git repository, and a local job needs one. Run \`git init\` here and commit, or, if this folder belongs to the repository at ${root}, run it on ${root}.`;
|
|
51
|
+
return `${folder} is not a git repository. A local job needs one: run \`git init\` there and commit, then run again.`;
|
|
52
|
+
}
|
|
53
|
+
const git = (args) => exec("git", [...GIT_READ_FLAGS, "-C", folder, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
54
|
+
try {
|
|
55
|
+
git(["rev-parse", "--verify", "--quiet", "HEAD^{commit}"]);
|
|
56
|
+
} catch (error) {
|
|
57
|
+
// A missing `git` is not a missing commit, and the fix is a different one.
|
|
58
|
+
if (error?.code === "ENOENT") return "git is not installed or not on PATH, so the folder cannot be checked. Install git, then run again.";
|
|
59
|
+
// Exit 1 is also a missing commit object, a garbage branch ref or an unreadable `refs/heads` (PR #528's review,
|
|
60
|
+
// measured): "commit once" is said only for an unborn HEAD, the worker's own test (`prepare-local.mjs`).
|
|
61
|
+
if (error?.status === 1 && unbornHead(git)) return `${folder} is a git repository with no commit yet. A local job reads its instructions from HEAD: commit once, then run again.`;
|
|
62
|
+
return `${folder} is not a usable git repository: git could not read it. Check it with \`git -C ${folder} status\`, then run again.`;
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** An unborn HEAD: HEAD names no object at all, and names a branch (the worker's `unbornHead`, by the same exit codes). */
|
|
68
|
+
function unbornHead(git) {
|
|
69
|
+
try {
|
|
70
|
+
git(["rev-parse", "--verify", "--quiet", "HEAD"]);
|
|
71
|
+
return false;
|
|
72
|
+
} catch (error) {
|
|
73
|
+
if (error?.status !== 1) return false;
|
|
74
|
+
}
|
|
75
|
+
try {
|
|
76
|
+
git(["symbolic-ref", "-q", "HEAD"]);
|
|
77
|
+
return true;
|
|
78
|
+
} catch {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The repository git says `folder` is inside, or null when git confirms none (or cannot run). Hardened like the rest. */
|
|
84
|
+
function enclosingRepoRoot(folder, exec) {
|
|
85
|
+
try {
|
|
86
|
+
const top = String(exec("git", [...GIT_READ_FLAGS, "-C", folder, "rev-parse", "--show-toplevel"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] })).trim();
|
|
87
|
+
return top === "" ? null : top;
|
|
88
|
+
} catch {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
}
|
package/src/github-app-setup.mjs
CHANGED
|
@@ -33,14 +33,17 @@
|
|
|
33
33
|
* defaulting to the real thing — the up.mjs convention — so the whole flow is testable offline.
|
|
34
34
|
*/
|
|
35
35
|
import { createSign } from "node:crypto";
|
|
36
|
-
import {
|
|
36
|
+
import { existsSync, writeFileSync } from "node:fs";
|
|
37
37
|
import { hostname } from "node:os";
|
|
38
38
|
import { join } from "node:path";
|
|
39
39
|
import { parseArgs } from "node:util";
|
|
40
|
-
import { updateEnvFile } from "./env-file.mjs";
|
|
40
|
+
import { ENV_WRITER_FS, updateEnvFile } from "./env-file.mjs";
|
|
41
41
|
import { openBrowser as defaultOpenBrowser } from "./open-browser.mjs";
|
|
42
42
|
import { defaultPrompt } from "./up.mjs";
|
|
43
43
|
|
|
44
|
+
/** This command's default fs seam: its own calls, and every call the `.env` writer makes (`ENV_WRITER_FS`, issue #522). */
|
|
45
|
+
export const SETUP_GITHUB_FS = { existsSync, ...ENV_WRITER_FS };
|
|
46
|
+
|
|
44
47
|
const API_ROOT = "https://api.github.com";
|
|
45
48
|
// GitHub rejects requests without a User-Agent, and node's fetch does not always send one.
|
|
46
49
|
const USER_AGENT = "pi-dispatch-setup-github";
|
|
@@ -69,7 +72,7 @@ export async function runGithubAppSetup(argv = [], deps = {}) {
|
|
|
69
72
|
openBrowser = defaultOpenBrowser,
|
|
70
73
|
out = (s) => process.stdout.write(s),
|
|
71
74
|
prompt = defaultPrompt,
|
|
72
|
-
fs =
|
|
75
|
+
fs = SETUP_GITHUB_FS,
|
|
73
76
|
cwd = process.cwd(),
|
|
74
77
|
now = () => Date.now(),
|
|
75
78
|
codeTimeoutMs = 600000, // ~10 min for a human to click through GitHub's create-app page
|
package/src/github-host.mjs
CHANGED
|
@@ -25,13 +25,15 @@
|
|
|
25
25
|
|
|
26
26
|
import { Octokit } from "@octokit/rest";
|
|
27
27
|
import { configError } from "./config.mjs";
|
|
28
|
+
import { octokitLogOptions } from "./octokit-log.mjs";
|
|
28
29
|
import { InfraRetry } from "./processor.mjs";
|
|
29
30
|
|
|
30
31
|
/**
|
|
31
|
-
* Build the host surface. `octokitFor` is `(token) => new Octokit({ auth: token })`; inject a fake
|
|
32
|
-
* to test offline.
|
|
32
|
+
* Build the host surface. `octokitFor` is `(token) => new Octokit({ auth: token, ...octokitLogOptions(log) })`; inject a fake
|
|
33
|
+
* to test offline. `log` is the caller's `(event, fields)` logger for the client's own warnings
|
|
34
|
+
* (`octokitLog`, issue #530). Returns `{ resolveDefaultBranchSha, isDefaultBranchProtected, postStatusComment }`.
|
|
33
35
|
*/
|
|
34
|
-
export function makeGitHubHost({ octokitFor = (token) => new Octokit({ auth: token }) } = {}) {
|
|
36
|
+
export function makeGitHubHost({ log, octokitFor = (token) => new Octokit({ auth: token, ...octokitLogOptions(log) }) } = {}) {
|
|
35
37
|
/**
|
|
36
38
|
* Resolve the default branch and its tip SHA with FRESH API calls only -- never a webhook field.
|
|
37
39
|
* `GET /repos/{owner}/{repo}` yields `default_branch`; `GET .../branches/{branch}` yields the tip.
|
package/src/host-pi.mjs
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* What it must never become. Nothing on the worker's BOOT path may import this file. It reads host paths and
|
|
16
16
|
* may spawn a package manager, and neither belongs anywhere near `start.mjs`.
|
|
17
17
|
*
|
|
18
|
-
* Everything here MIRRORS a private detail of the pinned pi (0.
|
|
18
|
+
* Everything here MIRRORS a private detail of the pinned pi (1.0.3) rather than calling it: pi exports no
|
|
19
19
|
* public answer to "where is this package installed" or "is this resource enabled", and importing the whole
|
|
20
20
|
* coding-agent SDK to read two well-known paths is not worth the weight. That mirroring is a real risk --
|
|
21
21
|
* pi could change the grammar and we would silently start staging something the operator turned off -- so it
|
package/src/identity.mjs
CHANGED
|
@@ -72,7 +72,8 @@ export async function resolveSelfId(auth) {
|
|
|
72
72
|
if (!isDeterminateFetchFailure(error) && status !== undefined && isTransientStatus(status, octokitHeaderReader(error), error?.message)) {
|
|
73
73
|
throw transientError(`resolveSelfId: could not reach GitHub to resolve self identity: ${error.message}`, error);
|
|
74
74
|
}
|
|
75
|
-
|
|
75
|
+
// The RequestError rides as `cause` (issue #530's review), so the failure line can name GitHub's request id.
|
|
76
|
+
throw Object.assign(configError(`resolveSelfId: could not resolve self identity: ${error.message}`), { cause: error });
|
|
76
77
|
}
|
|
77
78
|
|
|
78
79
|
if (!Number.isInteger(id)) {
|
package/src/image-preflight.mjs
CHANGED
|
@@ -42,10 +42,9 @@ export function resolveJobImage(job, defaultImage) {
|
|
|
42
42
|
* { missing: image } -- the daemon answered and does not have it => POLICY, refuse, do not retry
|
|
43
43
|
* { unavailable: image} -- docker itself did not answer => INFRA, retry
|
|
44
44
|
* { forgeUnsupported } -- present, but declares it cannot serve this job's forge => POLICY, refuse
|
|
45
|
-
* {
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
* carries exclusions => POLICY
|
|
45
|
+
* { <gate.result>, declared } -- present, but does not declare a capability this job needs, one key per
|
|
46
|
+
* CAPABILITY_GATES row (replicaUnsupported, commandUnsupported,
|
|
47
|
+
* excludeToolsUnsupported, modelPolicyUnsupported, costCapUnsupported) => POLICY
|
|
49
48
|
*
|
|
50
49
|
* A non-zero `docker image inspect` is AMBIGUOUS -- an absent image and an unreachable daemon both exit 1 --
|
|
51
50
|
* so the failure path disambiguates POSITIVELY with `docker info` rather than by matching docker's stderr.
|
|
@@ -97,26 +96,13 @@ export function makeImagePreflight({ image, spawnFn = spawn, bin = "docker" }) {
|
|
|
97
96
|
// asymmetry is deliberate rather than an oversight. `forges` is an EXCLUSION list, so no claim
|
|
98
97
|
// excludes nothing; `capabilities` is an INCLUSION list, so no claim includes nothing. One rule
|
|
99
98
|
// underlies both -- an image that declares nothing gets no benefit of the doubt about what it
|
|
100
|
-
// contains -- and neither costs an UNFLAGGED job anything:
|
|
101
|
-
// job actually carries
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
// the bare `/name args` prompt reaches the model as PROSE -- no handler runs, the agent
|
|
108
|
-
// improvises, and the queue records a clean exit 0. Unreachable for a commandless job, so the
|
|
109
|
-
// existing fleet pays nothing for it.
|
|
110
|
-
if (job?.command !== undefined && !(capabilities ?? []).includes("commands")) {
|
|
111
|
-
return { commandUnsupported: wanted, declared: capabilities ?? [] };
|
|
112
|
-
}
|
|
113
|
-
// Same inclusion-list polarity again, and here the stale-image failure is a PERMISSION quietly
|
|
114
|
-
// not enforced (issue #291): a runner that predates run.excludeTools reads no PI_EXCLUDE_TOOLS,
|
|
115
|
-
// so a "read-only" trigger's job would run with every tool the file says to remove and record a
|
|
116
|
-
// clean exit -- the silent fail-open this field exists to close. Unreachable for a job without
|
|
117
|
-
// exclusions, so the existing fleet pays nothing for it.
|
|
118
|
-
if (job?.excludeTools !== undefined && !(capabilities ?? []).includes("excludeTools")) {
|
|
119
|
-
return { excludeToolsUnsupported: wanted, declared: capabilities ?? [] };
|
|
99
|
+
// contains -- and neither costs an UNFLAGGED job anything: each gate below is unreachable unless
|
|
100
|
+
// the job actually carries the feature it guards. The gates, their order and why each exists are
|
|
101
|
+
// the CAPABILITY_GATES table; the first one this job needs and the image lacks refuses.
|
|
102
|
+
for (const gate of CAPABILITY_GATES) {
|
|
103
|
+
if (gate.needed(job) && !(capabilities ?? []).includes(gate.token)) {
|
|
104
|
+
return { [gate.result]: wanted, declared: capabilities ?? [] };
|
|
105
|
+
}
|
|
120
106
|
}
|
|
121
107
|
// `capabilities` rides the ok result for issue #341's job-user gate, which needs `anyUid` and runs after
|
|
122
108
|
// this preflight; an image that declares nothing reads as [].
|
|
@@ -127,6 +113,94 @@ export function makeImagePreflight({ image, spawnFn = spawn, bin = "docker" }) {
|
|
|
127
113
|
};
|
|
128
114
|
}
|
|
129
115
|
|
|
116
|
+
/**
|
|
117
|
+
* The image capability gates, ONE table (issues #501, #502 made it one; each row predates them). A row is a
|
|
118
|
+
* feature a job may carry and the `dev.pi-dispatch.capabilities` token an image must declare before a job
|
|
119
|
+
* carrying it runs there, because an image built before the feature silently ignores it: the runner reads no
|
|
120
|
+
* such variable and the job records a clean exit having done something else. The preflight refuses the first
|
|
121
|
+
* row whose `needed(job)` holds and whose `token` the image does not declare, before any spend.
|
|
122
|
+
*
|
|
123
|
+
* token the capability the image must declare
|
|
124
|
+
* needed whether this job carries the feature (a job that does not pays nothing for the row)
|
|
125
|
+
* result the key the preflight result names the image under, which the processor branches on
|
|
126
|
+
* reason the run record's reason for the refusal
|
|
127
|
+
* event the worker log event
|
|
128
|
+
* comment the forge comment, given the image and the `declared` phrase the processor builds; it names
|
|
129
|
+
* the FIX rather than the label that noticed it, because an operator reading "rebuild the image"
|
|
130
|
+
* is already where they need to be
|
|
131
|
+
*
|
|
132
|
+
* A new image-gated feature (`costCap`, `modelPolicy`) is a row here, not another branch in two files. Order
|
|
133
|
+
* is the order the branches had, so a job carrying several features is refused for the same one as before.
|
|
134
|
+
*/
|
|
135
|
+
export const CAPABILITY_GATES = Object.freeze([
|
|
136
|
+
// REQ-REPLICA-RUNS: an image built before it bakes a HARD_RULES.md whose rule 3 hard-codes `pi/issue-<n>`
|
|
137
|
+
// as a SYSTEM rule, which the model treats as authoritative over the user prompt naming `pi/issue-<n>-r2`.
|
|
138
|
+
// Both replicas would push to one branch: the push race the feature exists to avoid, two runs billed.
|
|
139
|
+
Object.freeze({
|
|
140
|
+
token: "replicas",
|
|
141
|
+
needed: (job) => job?.replica !== undefined,
|
|
142
|
+
result: "replicaUnsupported",
|
|
143
|
+
reason: "job-image-replicas-unsupported",
|
|
144
|
+
event: "refused_image_replicas_unsupported",
|
|
145
|
+
comment: (image, declared) => `Refused: the job image "${image}" does not declare replica support (\`dev.pi-dispatch.capabilities\` ${declared}), so its baked guardrails would name the wrong branch. Rebuild the image from a version that has this feature. Not run.`,
|
|
146
|
+
}),
|
|
147
|
+
// Issue #189: a runner that predates run.command reads no PI_COMMAND, so the bare `/name args` prompt
|
|
148
|
+
// reaches the model as PROSE -- no handler runs, the agent improvises, and the queue records a clean exit 0.
|
|
149
|
+
// The runner's own command-unregistered refusal does not exist on such an image, so the host refuses first.
|
|
150
|
+
Object.freeze({
|
|
151
|
+
token: "commands",
|
|
152
|
+
needed: (job) => job?.command !== undefined,
|
|
153
|
+
result: "commandUnsupported",
|
|
154
|
+
reason: "job-image-commands-unsupported",
|
|
155
|
+
event: "refused_image_commands_unsupported",
|
|
156
|
+
comment: (image, declared) => `Refused: the job image "${image}" does not declare command support (\`dev.pi-dispatch.capabilities\` ${declared}), so its runner would not dispatch \`run.command\`. Rebuild the image from a version that has this feature. Not run.`,
|
|
157
|
+
}),
|
|
158
|
+
// Issue #291: a runner that predates run.excludeTools reads no PI_EXCLUDE_TOOLS, so a "read-only" trigger's
|
|
159
|
+
// job would run with every tool the file says to remove and record a clean exit: a PERMISSION quietly not
|
|
160
|
+
// enforced, the silent fail-open this repo brands the worst outcome available.
|
|
161
|
+
Object.freeze({
|
|
162
|
+
token: "excludeTools",
|
|
163
|
+
needed: (job) => job?.excludeTools !== undefined,
|
|
164
|
+
result: "excludeToolsUnsupported",
|
|
165
|
+
reason: "job-image-exclude-tools-unsupported",
|
|
166
|
+
event: "refused_image_exclude_tools_unsupported",
|
|
167
|
+
comment: (image, declared) => `Refused: the job image "${image}" does not declare exclude-tools support (\`dev.pi-dispatch.capabilities\` ${declared}), so its runner would ignore \`run.excludeTools\` and run this trigger with every tool it says to remove. Rebuild the image from a version that has this feature. Not run.`,
|
|
168
|
+
}),
|
|
169
|
+
// Issue #502: a runner that predates the model guard either ignores PI_ALLOWED_MODELS (an image older than the
|
|
170
|
+
// policy seams) or refuses it before any call as `model-policy-unenforceable`, after the budget was reserved. The
|
|
171
|
+
// first runs every model the list forbids on a clean exit, the second spends a slot to learn what the label
|
|
172
|
+
// already says. `needed` reads the EFFECTIVE list (index.mjs `effectiveJobOf`), so a deployment-wide
|
|
173
|
+
// PI_ALLOWED_MODELS is gated the same as a trigger's `run.models`.
|
|
174
|
+
Object.freeze({
|
|
175
|
+
token: "modelPolicy",
|
|
176
|
+
needed: (job) => Array.isArray(job?.models) && job.models.length > 0,
|
|
177
|
+
result: "modelPolicyUnsupported",
|
|
178
|
+
reason: "job-image-model-policy-unsupported",
|
|
179
|
+
event: "refused_image_model_policy_unsupported",
|
|
180
|
+
comment: (image, declared) => `Refused: the job image "${image}" does not declare allowed-model support (\`dev.pi-dispatch.capabilities\` ${declared}), so its runner cannot hold this job to the models it may use. Rebuild the image from a version that has this feature. Not run.`,
|
|
181
|
+
}),
|
|
182
|
+
// Issue #501: a runner that predates the cost guard reads no PI_MAX_COST_MICROS, so a job with a dollar cap
|
|
183
|
+
// would run UNCAPPED and record a clean exit: a money limit quietly not kept. Needed whenever the job carries
|
|
184
|
+
// a cap, 0 included (`!== null`, buildContainerEnv's rule: 0 is the tightest cap, not none). The refusal is
|
|
185
|
+
// pre-spend like every row: nothing minted, nothing cloned, no budget reserved (`budgetReserved: false`).
|
|
186
|
+
Object.freeze({
|
|
187
|
+
token: "costCap",
|
|
188
|
+
needed: (job) => job?.maxCostMicros !== null && job?.maxCostMicros !== undefined,
|
|
189
|
+
result: "costCapUnsupported",
|
|
190
|
+
reason: "job-image-cost-cap-unsupported",
|
|
191
|
+
event: "refused_image_cost_cap_unsupported",
|
|
192
|
+
comment: (image, declared) => `Refused: the job image "${image}" does not declare cost-cap support (\`dev.pi-dispatch.capabilities\` ${declared}), so its runner would ignore the per-job dollar cap and run this job with no cap at all. Rebuild the image from a version that has this feature. Not run.`,
|
|
193
|
+
}),
|
|
194
|
+
]);
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* The capability an image declares when its runner signs its exit line with a per-job key read from stdin, and runs
|
|
198
|
+
* under an exec-only node its job's tools cannot read (issue #545). NOT a gate: no job is refused for lacking it. The
|
|
199
|
+
* processor hands a key only to an image that declares it, and reads only a signed exit line from such an image; an
|
|
200
|
+
* image without it runs and is read exactly as before, under the #542 trust rule alone.
|
|
201
|
+
*/
|
|
202
|
+
export const EXIT_AUTH_CAPABILITY = "exitAuth";
|
|
203
|
+
|
|
130
204
|
/**
|
|
131
205
|
* Exported so the test cannot drift from the format string above. `|` because an image id is
|
|
132
206
|
* `sha256:<hex>` and a version is a version, so neither can contain one -- and because a literal control
|
package/src/image-ref.mjs
CHANGED
|
@@ -19,3 +19,40 @@ export function imageRefProblem(image) {
|
|
|
19
19
|
if (/[\u0000-\u001f\u007f-\u009f]/.test(image)) return { code: "control", reason: "must not hold a control character" };
|
|
20
20
|
return null;
|
|
21
21
|
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Whether an image reference names its registry (issue #523), by the docker reference grammar: the first path segment
|
|
25
|
+
* is a registry host only when there is a `/` after it and it holds a `.` or a `:`, or is `localhost`. Everything else
|
|
26
|
+
* (`pi-job:2.1.0`, `team/job:7`) is a short name the runtime resolves on a public registry (Docker Hub, or podman's
|
|
27
|
+
* search registries), where anyone may publish that name.
|
|
28
|
+
*/
|
|
29
|
+
export function registryQualified(image) {
|
|
30
|
+
const slash = image.indexOf("/");
|
|
31
|
+
if (slash < 0) return false;
|
|
32
|
+
const first = image.slice(0, slash);
|
|
33
|
+
return first.includes(".") || first.includes(":") || first === "localhost";
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Whether a missing image may be offered as a pull (issue #523, review round 2): a registry-qualified name other than
|
|
38
|
+
* `localhost/...`. That prefix is qualified for the "never Docker Hub" rule, but it is podman's name for an image built
|
|
39
|
+
* on this host, with no registry behind it, so a pull of it can only fail or reach whatever listens on this host.
|
|
40
|
+
* `localhost:<port>/...` names a real registry on a port and is pullable.
|
|
41
|
+
*/
|
|
42
|
+
export function pullOffered(image) {
|
|
43
|
+
return registryQualified(image) && !image.startsWith("localhost/");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* How to provide a missing deployment job image with `bin` (docker or podman), ONE text for `up` and doctor's fix lines
|
|
48
|
+
* (issue #523, review): the worker's default `pi-job:latest` is ghcr's latest pulled and re-tagged, since that local
|
|
49
|
+
* name has no registry behind it; a registry-qualified name is pulled as it is named; a short name is never offered as
|
|
50
|
+
* a pull, since a public registry answers for it, nor a `localhost/` name (`pullOffered`), so the text says how to
|
|
51
|
+
* provide it instead.
|
|
52
|
+
*/
|
|
53
|
+
export function jobImageFix(bin, image) {
|
|
54
|
+
if (image === "pi-job:latest") return `${bin} pull ghcr.io/edgehero/pi-job:latest && ${bin} tag ghcr.io/edgehero/pi-job:latest pi-job:latest`;
|
|
55
|
+
if (pullOffered(image)) return `${bin} pull ${image}`;
|
|
56
|
+
if (registryQualified(image)) return `${image} is a locally built name (localhost/ is no registry to pull from): build it on this host, or \`${bin} tag\` an image you have as ${image}`;
|
|
57
|
+
return `${image} names no registry host, so a pull would fetch whatever a public registry holds under that name: build it on this host, \`${bin} tag\` an image you have as ${image}, or set PI_JOB_IMAGE to a registry-qualified name (such as ghcr.io/edgehero/pi-job:<version>)`;
|
|
58
|
+
}
|
package/src/import-pi.mjs
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
*/
|
|
26
26
|
import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync, lstatSync, statSync, copyFileSync, renameSync, rmSync } from "node:fs";
|
|
27
27
|
import { execFile } from "node:child_process";
|
|
28
|
-
import { join } from "node:path";
|
|
28
|
+
import { join, resolve } from "node:path";
|
|
29
29
|
import { promisify } from "node:util";
|
|
30
30
|
import { FROM_HOST, PACKAGES_SUBDIR, RESOURCE_DIRS, STAGE_MANIFEST, mergeHostPackages, parsePackagesFile } from "./packages.mjs";
|
|
31
31
|
import { agentDirFrom, readHostPi } from "./host-pi.mjs";
|
|
@@ -121,7 +121,9 @@ export async function runImportPi(argv = [], deps = {}) {
|
|
|
121
121
|
// `--host-packages` is accepted as a no-op for symmetry with `--with-extensions`.
|
|
122
122
|
const withHostPackages = withPackages && !argv.includes("--no-host-packages");
|
|
123
123
|
const from = flagValue(argv, "--from") ?? defaultFrom(env);
|
|
124
|
-
|
|
124
|
+
// Absolute, always (PR #553's review): the next steps print it as the PI_GLOBAL_PI_DIR to set, and the worker
|
|
125
|
+
// refuses a relative one, since the worker and the container runtime resolve it differently.
|
|
126
|
+
const to = resolve(cwd, flagValue(argv, "--to") ?? "pi-global");
|
|
125
127
|
const packagesFile = flagValue(argv, "--packages-file") ?? env.PI_PACKAGES_FILE ?? join(cwd, "pi-packages.json");
|
|
126
128
|
|
|
127
129
|
if (!fs.existsSync(from)) {
|