@edgehero/pi-dispatch 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +44 -7
- package/README.md +14 -6
- package/deploy/com.pi-dispatch.worker.plist +1 -1
- package/deploy/docker-compose.yml +12 -0
- package/deploy/egress-proxy.conf +28 -3
- package/deploy/pi-dispatch-egress-proxy.container +8 -2
- package/deploy/worker-env-wrapper.cmd +1 -1
- package/deploy/worker-env-wrapper.sh +3 -3
- package/package.json +9 -2
- package/src/allocation.mjs +731 -0
- package/src/backends.mjs +243 -0
- package/src/budget.mjs +40 -4
- package/src/cli.mjs +222 -11
- package/src/config.mjs +126 -5
- package/src/daemon-facts.mjs +3 -0
- package/src/deployment-venue.mjs +1 -0
- package/src/doctor.mjs +2316 -183
- package/src/dollar-budget.mjs +373 -0
- package/src/dollar-fingerprint.mjs +83 -0
- package/src/egress-cli.mjs +316 -0
- package/src/egress-proxy-state.mjs +35 -5
- package/src/egress.mjs +16 -3
- package/src/env-allowlist.mjs +142 -18
- package/src/env-file.mjs +194 -25
- package/src/envelope.mjs +413 -0
- package/src/exit-code.mjs +22 -0
- package/src/fleet-lease.mjs +85 -25
- package/src/get-token.mjs +16 -5
- package/src/git-dirty.mjs +67 -0
- package/src/github-app-setup.mjs +6 -3
- package/src/github-host.mjs +5 -3
- package/src/host-pi.mjs +19 -3
- package/src/identity.mjs +2 -1
- package/src/image-preflight.mjs +98 -24
- package/src/image-ref.mjs +37 -0
- package/src/import-pi.mjs +4 -2
- package/src/index.mjs +407 -62
- package/src/init.mjs +18 -0
- package/src/job-id.mjs +26 -3
- package/src/live-probes.mjs +24 -9
- package/src/model-catalog.mjs +297 -0
- package/src/model-endpoints.mjs +649 -0
- package/src/model-ref.mjs +151 -0
- package/src/models-json.mjs +262 -0
- package/src/money.mjs +144 -0
- package/src/octokit-log.mjs +65 -0
- package/src/outbox-plan.mjs +218 -0
- package/src/outbox.mjs +29 -9
- package/src/output-cap.mjs +157 -0
- package/src/packages.mjs +2 -2
- package/src/pause-windows.mjs +81 -2
- package/src/pi-model-loader.mjs +77 -0
- package/src/podman-stack.mjs +16 -3
- package/src/portfolio-snapshot.mjs +304 -0
- package/src/prepare-local.mjs +247 -12
- package/src/prepare.mjs +35 -3
- package/src/pricing.mjs +9 -5
- package/src/priorities.mjs +569 -0
- package/src/processor.mjs +603 -173
- package/src/project-id.mjs +17 -0
- package/src/projects.mjs +238 -0
- package/src/provider-key.mjs +32 -7
- package/src/provider-steering.mjs +214 -59
- package/src/queue.mjs +111 -6
- package/src/reserved-env.mjs +30 -0
- package/src/run-container.mjs +59 -5
- package/src/run-history.mjs +379 -24
- package/src/run-mirror.mjs +30 -0
- package/src/runtime-settings.mjs +104 -9
- package/src/schedules.mjs +33 -1
- package/src/scoped-limits.mjs +447 -27
- package/src/secrets.mjs +2 -1
- package/src/service.mjs +15 -4
- package/src/session-store.mjs +131 -6
- package/src/start.mjs +528 -40
- package/src/subscriptions.mjs +7 -3
- package/src/triggers-file.mjs +65 -4
- package/src/triggers.mjs +140 -9
- package/src/up.mjs +308 -34
- package/src/valkey-endpoint.mjs +3 -2
package/src/prepare-local.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { GIT_READ_FLAGS } from "./git-hardening.mjs";
|
|
2
2
|
import { execFile } from "node:child_process";
|
|
3
3
|
import * as realFs from "node:fs";
|
|
4
|
-
import { basename, join } from "node:path";
|
|
4
|
+
import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
|
|
5
|
+
import { envelopeProtectedIdentities, fileIdentity } from "./envelope.mjs";
|
|
5
6
|
import { promisify } from "node:util";
|
|
6
7
|
import { materializePiDir } from "./materialize.mjs";
|
|
7
8
|
import { InfraRetry } from "./processor.mjs";
|
|
@@ -9,6 +10,22 @@ import { isDeterminateFsCode } from "./transient.mjs";
|
|
|
9
10
|
|
|
10
11
|
const exec = promisify(execFile);
|
|
11
12
|
|
|
13
|
+
/** The run-record reason for a local job whose folder has no `.git` at its root (issue #524). */
|
|
14
|
+
export const LOCAL_FOLDER_NOT_A_REPO = "local-folder-not-a-repo";
|
|
15
|
+
/** The run-record reason for a local job whose repository has no commit at HEAD yet (issue #524). */
|
|
16
|
+
export const LOCAL_FOLDER_NO_COMMIT = "local-folder-no-commit";
|
|
17
|
+
/** The run-record reason for a local job whose repository git cannot resolve HEAD in, other than an unborn HEAD (issue #524). */
|
|
18
|
+
export const LOCAL_FOLDER_UNREADABLE_REPO = "local-folder-unreadable-repo";
|
|
19
|
+
/**
|
|
20
|
+
* The run-record reason for a local job whose folder is NAMED inside a job path (a `PI_DISPATCH_RUN_ROOTS` root or a cron
|
|
21
|
+
* trigger's `run.folder`) and RESOLVES, at prepare, to a directory outside it (issue #504 part B). A job container can
|
|
22
|
+
* write inside its own folder, so it can swap a link below a job path after the folder was checked; such a folder is
|
|
23
|
+
* refused rather than mounted wherever the link now points.
|
|
24
|
+
*/
|
|
25
|
+
export const LOCAL_FOLDER_ESCAPED = "local-folder-escaped";
|
|
26
|
+
/** The run-record reason for a local job whose resolved folder is the envelope's folder or a directory above it (issue #504 part B). */
|
|
27
|
+
export const LOCAL_FOLDER_HOLDS_ENVELOPE = "local-folder-holds-envelope";
|
|
28
|
+
|
|
12
29
|
/**
|
|
13
30
|
* Prepare a LOCAL-FOLDER job. This is the zero-GitHub path: no token, no clone, no PR. The folder
|
|
14
31
|
* on the operator's own machine becomes /workspace (bind-mounted read-write), edited in place.
|
|
@@ -27,11 +44,62 @@ const exec = promisify(execFile);
|
|
|
27
44
|
* default keeps a directly-constructed call (tests, older wiring) honest: a job with no derived
|
|
28
45
|
* context is a manual run.
|
|
29
46
|
*/
|
|
30
|
-
export async function prepareLocalWorkspace({ folder, task, jobDir, git = defaultGit, event = { source: "manual" }, fs = realFs }) {
|
|
31
|
-
requirePath(fs,
|
|
32
|
-
|
|
47
|
+
export async function prepareLocalWorkspace({ folder: named, task, jobDir, git = defaultGit, event = { source: "manual" }, fs = realFs, jobPaths = null, envelopeFile = null, portfolio = null }) {
|
|
48
|
+
requirePath(fs, named, `local folder does not exist: ${named}`, "the local folder");
|
|
49
|
+
// Issue #504 part B: the folder is RESOLVED here, once, and the resolved path is the one everything below reads and the
|
|
50
|
+
// one the container mounts (`workspace`). The raw string was mounted before, so a link below a job path swapped after
|
|
51
|
+
// the folder was checked (by the admin's run-root check at enqueue, or by nothing at all for a chained child) would
|
|
52
|
+
// have mounted whatever it pointed to, read-write. Resolving narrows the window to the moments between this line and
|
|
53
|
+
// the container start, and the two checks below judge the folder that will be mounted. Residual, named: a link above
|
|
54
|
+
// the resolved folder swapped inside that window is followed by the runtime when it mounts.
|
|
55
|
+
const folder = resolveFolder(fs, named);
|
|
56
|
+
const placement = judgePlacement(fs, named, folder, { jobPaths: typeof jobPaths === "function" ? jobPaths() : jobPaths, envelopeFile });
|
|
57
|
+
if (placement) return { outcome: "policy", reason: placement };
|
|
58
|
+
// Issue #524: a folder that is not a repository is its own determinate refusal, RETURNED with its own reason
|
|
59
|
+
// (CONST-RETRY-INFRA-ONLY), not a config throw. As a throw it reached the processor's config arm, which can only
|
|
60
|
+
// say "this deployment is misconfigured" (`config-refused`), about a folder the operator can fix with `git init`.
|
|
61
|
+
// Returned here it is a prepare-stage policy refusal like `sha-gone`: before the budget reserve, so it spends
|
|
62
|
+
// nothing, and `prepare.mjs` removes the job dir it made. The reason is a fixed token and carries no path, so the
|
|
63
|
+
// worker log and the run record stay as PII-free as they are. `pi-dispatch run` refuses the same folder before
|
|
64
|
+
// queueing; this is the check for a folder that stopped being a repository between queue and pickup, and for a
|
|
65
|
+
// cron trigger's folder, which nothing checks when it fires.
|
|
66
|
+
if (!presentAt(fs, join(folder, ".git"), "the local folder's .git")) return { outcome: "policy", reason: LOCAL_FOLDER_NOT_A_REPO };
|
|
33
67
|
|
|
34
|
-
|
|
68
|
+
// Issue #524: a HEAD git cannot resolve escaped as an untagged throw, a failed job with no reason. It is now
|
|
69
|
+
// refused by name, RETURNED before the reserve and before anything is written, like the not-a-repo one above,
|
|
70
|
+
// but only once the files git needs have been read back under `presentAt`'s rule (PR #528's review, round 2).
|
|
71
|
+
// git's exit code says "could not", never "why": `rev-parse --verify --quiet` exits 1 for an unborn HEAD AND for
|
|
72
|
+
// a missing commit object, a garbage branch ref, an unreadable `refs/heads`; it exits 128 for an empty `.git`
|
|
73
|
+
// AND for an EACCES on `.git/HEAD` or `.git/objects`, with the same "not a git repository" either way (all
|
|
74
|
+
// measured, git 2.x). Its text is no help either: a locale translates it, and it names host paths.
|
|
75
|
+
// - `local-folder-no-commit` only for a truly unborn HEAD: the follow-up `rev-parse --verify --quiet HEAD`
|
|
76
|
+
// also exits 1 AND `symbolic-ref -q HEAD` succeeds (HEAD names a branch that does not exist yet). A HEAD
|
|
77
|
+
// that resolves to a missing object or a non-commit, or a branch ref that cannot be read, is not "commit
|
|
78
|
+
// once" and must not be told so.
|
|
79
|
+
// - `local-folder-unreadable-repo` for every other 1 or 128.
|
|
80
|
+
// Before either refusal, `assertGitFilesReadable` reads what git needed. An error outside the determinate
|
|
81
|
+
// allow-list (`transient.mjs`: EACCES and EIO are deliberately transient) throws InfraRetry, so a filesystem
|
|
82
|
+
// that cannot answer right now is retried rather than refused for good. Any other git failure (a missing git
|
|
83
|
+
// binary, a signal) still throws as it did.
|
|
84
|
+
let sha;
|
|
85
|
+
try {
|
|
86
|
+
sha = (await git(folder, ["rev-parse", "--verify", "--quiet", "HEAD^{commit}"])).trim();
|
|
87
|
+
} catch (error) {
|
|
88
|
+
if (error?.code !== 1 && error?.code !== 128) throw error;
|
|
89
|
+
assertGitFilesReadable(fs, folder);
|
|
90
|
+
if (error.code === 1 && (await unbornHead(git, folder))) return { outcome: "policy", reason: LOCAL_FOLDER_NO_COMMIT };
|
|
91
|
+
return { outcome: "policy", reason: LOCAL_FOLDER_UNREADABLE_REPO };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Issue #505: a confirmed portfolio job's snapshot, built BEFORE anything is written, so a refusal
|
|
95
|
+
// (`portfolio-snapshot-oversize`) or an InfraRetry (Valkey) leaves nothing behind but the directory prepare.mjs removes.
|
|
96
|
+
// Null from the builder means the live file no longer flags the job (or this host lost its envelope): no snapshot, and
|
|
97
|
+
// the job runs as an ordinary cron job.
|
|
98
|
+
let snapshot = null;
|
|
99
|
+
if (typeof portfolio === "function") {
|
|
100
|
+
snapshot = await portfolio();
|
|
101
|
+
if (snapshot?.outcome === "policy") return snapshot;
|
|
102
|
+
}
|
|
35
103
|
|
|
36
104
|
fs.mkdirSync(jobDir, { recursive: true });
|
|
37
105
|
// The outbox is the container's only signal channel back to the worker (INT-OUTBOX-CONTRACT). It is
|
|
@@ -57,14 +125,108 @@ export async function prepareLocalWorkspace({ folder, task, jobDir, git = defaul
|
|
|
57
125
|
const eventBody = {
|
|
58
126
|
source: event.source,
|
|
59
127
|
...(event.trigger ? { trigger: event.trigger } : {}),
|
|
60
|
-
folder: basename(
|
|
128
|
+
folder: basename(named),
|
|
61
129
|
sha,
|
|
62
130
|
...(event.source === "cron" ? { scheduledFor: event.scheduledFor ?? null, previousRunAt: event.previousRunAt ?? null } : {}),
|
|
63
131
|
};
|
|
64
132
|
fs.writeFileSync(join(jobDir, "event.json"), JSON.stringify(eventBody, null, 2), { mode: 0o444 });
|
|
133
|
+
// /job/portfolio.json (INT-CONTAINER-JOB-INPUTS), 0o444 beside event.json, so it reaches the container on the existing
|
|
134
|
+
// read-only /job mount with no new mount. Its presence is also what composes the portfolio persona in the runner.
|
|
135
|
+
if (typeof snapshot?.body === "string") fs.writeFileSync(join(jobDir, "portfolio.json"), snapshot.body, { mode: 0o444 });
|
|
65
136
|
|
|
66
137
|
// The folder itself is /workspace (rw). No clone: local jobs edit in place.
|
|
67
|
-
|
|
138
|
+
// `portfolio: true` only when the snapshot was written (issue #505): the plan collector requires prepare to have
|
|
139
|
+
// agreed, beside the pickup and the live file at collection, so a job that ran without the facts writes no plan.
|
|
140
|
+
return { workspace: folder, jobDir, outboxDir, sha, materialised: written, ...(typeof snapshot?.body === "string" ? { portfolio: true } : {}) };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The folder's real path (`realpathSync.native`). A determinate absence here (the folder vanished, or a link in it now
|
|
145
|
+
* dangles) is the "does not exist" config refusal `requirePath` gives; any other error is retried, `presentAt`'s rule.
|
|
146
|
+
*/
|
|
147
|
+
function resolveFolder(fs, named) {
|
|
148
|
+
try {
|
|
149
|
+
return (fs.realpathSync?.native ?? fs.realpathSync)(named);
|
|
150
|
+
} catch (error) {
|
|
151
|
+
if (isDeterminateFsCode(error?.code)) {
|
|
152
|
+
const refusal = new Error(`local folder does not exist: ${named}`);
|
|
153
|
+
refusal.piDispatchConfig = true;
|
|
154
|
+
throw refusal;
|
|
155
|
+
}
|
|
156
|
+
throw new InfraRetry(`could not resolve the local folder (${error?.code ?? "unknown"}): ${basename(named)}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** `p` is `root` or lies below it, as text. Both absolute and resolved. */
|
|
161
|
+
function insideOrEqual(p, root) {
|
|
162
|
+
return p === root || p.startsWith(root.endsWith(sep) ? root : `${root}${sep}`);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** The identity of a path (`dev:ino`, links followed), or null when it cannot be read. */
|
|
166
|
+
function identityAt(fs, path) {
|
|
167
|
+
try {
|
|
168
|
+
return fileIdentity(fs.statSync(path, { bigint: true }));
|
|
169
|
+
} catch {
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Where the RESOLVED folder may be mounted from (issue #504 part B), or the refusal reason:
|
|
176
|
+
* - `local-folder-escaped`: the folder is named inside a job path (its text, or its text against a job path's real
|
|
177
|
+
* path, lies inside a run root or a cron folder, or an ancestor of its text has a job path's identity) and its
|
|
178
|
+
* resolved path lies inside none of those job paths, judged
|
|
179
|
+
* by identity up the resolved path, so a firmlink or a case variant is the directory it names. A job path is a
|
|
180
|
+
* place a job can write; a link a job planted there must not carry another job's folder out of it. A folder named
|
|
181
|
+
* outside every job path (an operator's own `pi-dispatch run`) is not held to them, since nothing there is a job's;
|
|
182
|
+
* - `local-folder-holds-envelope`: the resolved folder is the envelope file's folder or a directory above it
|
|
183
|
+
* (`envelopeProtectedIdentities`), so the job could write its own bounds.
|
|
184
|
+
* `jobPaths` is `{ runRoots, cronFolders }` (schedules.mjs `envelopeJobPaths`), or null for none.
|
|
185
|
+
*/
|
|
186
|
+
function judgePlacement(fs, named, folder, { jobPaths, envelopeFile }) {
|
|
187
|
+
const real = (p) => {
|
|
188
|
+
try {
|
|
189
|
+
return (fs.realpathSync?.native ?? fs.realpathSync)(p);
|
|
190
|
+
} catch {
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
const areas = [...(jobPaths?.runRoots ?? []), ...(jobPaths?.cronFolders ?? [])].filter((p) => typeof p === "string" && p.trim() !== "" && isAbsolute(p));
|
|
195
|
+
const text = resolve(named);
|
|
196
|
+
// The identities of the named path's own ancestors, walked up its TEXT (each stat follows links, as the kernel does):
|
|
197
|
+
// "named inside" is decided by identity too, so a case variant of a run root on a case-insensitive volume, or a
|
|
198
|
+
// firmlink spelling of it, is inside it, where a string comparison let it skip the check (PR #574's review).
|
|
199
|
+
const namedAncestors = new Set();
|
|
200
|
+
for (let p = text; areas.length > 0; p = dirname(p)) {
|
|
201
|
+
const id = identityAt(fs, p);
|
|
202
|
+
if (id !== null) namedAncestors.add(id);
|
|
203
|
+
if (dirname(p) === p) break;
|
|
204
|
+
}
|
|
205
|
+
const holding = areas.filter((area) => {
|
|
206
|
+
const r = real(area);
|
|
207
|
+
if (insideOrEqual(text, resolve(area)) || (r !== null && insideOrEqual(text, r))) return true;
|
|
208
|
+
const id = identityAt(fs, area);
|
|
209
|
+
return id !== null && namedAncestors.has(id);
|
|
210
|
+
});
|
|
211
|
+
if (holding.length > 0) {
|
|
212
|
+
const ids = new Set(holding.map((area) => identityAt(fs, area)).filter((id) => id !== null));
|
|
213
|
+
let inside = false;
|
|
214
|
+
for (let p = folder; ; p = dirname(p)) {
|
|
215
|
+
const id = identityAt(fs, p);
|
|
216
|
+
if (id !== null && ids.has(id)) {
|
|
217
|
+
inside = true;
|
|
218
|
+
break;
|
|
219
|
+
}
|
|
220
|
+
if (dirname(p) === p) break;
|
|
221
|
+
}
|
|
222
|
+
if (!inside) return LOCAL_FOLDER_ESCAPED;
|
|
223
|
+
}
|
|
224
|
+
if (envelopeFile !== null && envelopeFile !== undefined) {
|
|
225
|
+
const protectedIds = envelopeProtectedIdentities(envelopeFile, { realpathSync: fs.realpathSync?.native ?? fs.realpathSync, statSync: fs.statSync });
|
|
226
|
+
const id = identityAt(fs, folder);
|
|
227
|
+
if (id !== null && protectedIds.has(id)) return LOCAL_FOLDER_HOLDS_ENVELOPE;
|
|
228
|
+
}
|
|
229
|
+
return null;
|
|
68
230
|
}
|
|
69
231
|
|
|
70
232
|
/**
|
|
@@ -82,14 +244,23 @@ export async function prepareLocalWorkspace({ folder, task, jobDir, git = defaul
|
|
|
82
244
|
* `.git` is a FILE in a worktree and in a submodule, and a symlinked project folder is ordinary.
|
|
83
245
|
*/
|
|
84
246
|
function requirePath(fs, path, absentMessage, what) {
|
|
247
|
+
if (presentAt(fs, path, what)) return;
|
|
248
|
+
const refusal = new Error(absentMessage);
|
|
249
|
+
refusal.piDispatchConfig = true;
|
|
250
|
+
throw refusal;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* True when something is at `path`, false when it is determinately absent, and a THROWN InfraRetry when the
|
|
255
|
+
* filesystem could not answer. The second half of `requirePath`, split out so the `.git` check can return its own
|
|
256
|
+
* refusal instead of throwing one, with the same rule about which errors mean absent (issue #316, below).
|
|
257
|
+
*/
|
|
258
|
+
function presentAt(fs, path, what) {
|
|
85
259
|
try {
|
|
86
260
|
fs.statSync(path);
|
|
261
|
+
return true;
|
|
87
262
|
} catch (error) {
|
|
88
|
-
if (isDeterminateFsCode(error?.code))
|
|
89
|
-
const refusal = new Error(absentMessage);
|
|
90
|
-
refusal.piDispatchConfig = true;
|
|
91
|
-
throw refusal;
|
|
92
|
-
}
|
|
263
|
+
if (isDeterminateFsCode(error?.code)) return false;
|
|
93
264
|
// Not absent, just unreachable right now. Throw the retryable class so the queue tries again
|
|
94
265
|
// instead of spending the delivery on a verdict that is wrong by the time it is posted.
|
|
95
266
|
// BASENAME, not the path (issue #289): an InfraRetry survives retries and its message becomes the
|
|
@@ -101,6 +272,70 @@ function requirePath(fs, path, absentMessage, what) {
|
|
|
101
272
|
}
|
|
102
273
|
}
|
|
103
274
|
|
|
275
|
+
/**
|
|
276
|
+
* True only for an unborn HEAD: one that names a branch with no commit yet (issue #524, PR #528's review). Both
|
|
277
|
+
* answers are git's exit codes. `rev-parse --verify --quiet HEAD` exits 0 when HEAD names SOME object, which after
|
|
278
|
+
* `HEAD^{commit}` failed means a missing or non-commit object; `symbolic-ref -q HEAD` fails for a detached HEAD and
|
|
279
|
+
* for a branch ref git cannot read (measured: a garbage ref and an EACCES `refs/heads` both exit 128 there).
|
|
280
|
+
*/
|
|
281
|
+
async function unbornHead(git, folder) {
|
|
282
|
+
try {
|
|
283
|
+
await git(folder, ["rev-parse", "--verify", "--quiet", "HEAD"]);
|
|
284
|
+
return false;
|
|
285
|
+
} catch (error) {
|
|
286
|
+
if (error?.code !== 1) return false;
|
|
287
|
+
}
|
|
288
|
+
try {
|
|
289
|
+
await git(folder, ["symbolic-ref", "-q", "HEAD"]);
|
|
290
|
+
return true;
|
|
291
|
+
} catch {
|
|
292
|
+
return false;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Read back the files git needs to resolve HEAD, under `presentAt`'s rule: an absence the allow-list calls
|
|
298
|
+
* determinate is fine (the refusal stands), anything else throws InfraRetry so the job is retried (PR #528's review,
|
|
299
|
+
* round 2). `.git`; when it is a file (a worktree or submodule), the gitdir it names; `<gitdir>/HEAD`;
|
|
300
|
+
* `<gitdir>/config` when present; and the branch ref HEAD names, which is where an unreadable `refs/heads` shows.
|
|
301
|
+
* A ref that is absent is ordinary (an unborn branch, or one only in packed-refs). Messages carry basenames only.
|
|
302
|
+
*/
|
|
303
|
+
function assertGitFilesReadable(fs, folder) {
|
|
304
|
+
const dotGit = join(folder, ".git");
|
|
305
|
+
const st = statOrRetry(fs, dotGit);
|
|
306
|
+
if (!st) return;
|
|
307
|
+
let gitDir = dotGit;
|
|
308
|
+
if (st.isFile?.()) {
|
|
309
|
+
const pointer = readOrRetry(fs, dotGit);
|
|
310
|
+
const named = pointer === null ? null : /^gitdir:\s*(.+)$/m.exec(pointer)?.[1]?.trim();
|
|
311
|
+
if (!named) return;
|
|
312
|
+
gitDir = resolve(folder, named);
|
|
313
|
+
if (!statOrRetry(fs, gitDir)) return;
|
|
314
|
+
}
|
|
315
|
+
const head = readOrRetry(fs, join(gitDir, "HEAD"));
|
|
316
|
+
readOrRetry(fs, join(gitDir, "config"));
|
|
317
|
+
const ref = head === null ? null : /^ref:\s*(refs\/\S+)\s*$/m.exec(head)?.[1];
|
|
318
|
+
if (ref && !ref.split("/").includes("..")) readOrRetry(fs, join(gitDir, ref));
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function statOrRetry(fs, path) {
|
|
322
|
+
try {
|
|
323
|
+
return fs.statSync(path);
|
|
324
|
+
} catch (error) {
|
|
325
|
+
if (isDeterminateFsCode(error?.code)) return null;
|
|
326
|
+
throw new InfraRetry(`could not read the local folder's git files (${error?.code ?? "unknown"}): ${basename(path)}`);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function readOrRetry(fs, path) {
|
|
331
|
+
try {
|
|
332
|
+
return fs.readFileSync(path, "utf8");
|
|
333
|
+
} catch (error) {
|
|
334
|
+
if (isDeterminateFsCode(error?.code)) return null;
|
|
335
|
+
throw new InfraRetry(`could not read the local folder's git files (${error?.code ?? "unknown"}): ${basename(path)}`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
104
339
|
async function defaultGit(gitDir, args) {
|
|
105
340
|
// This was the one copy of seven missing `core.fsmonitor=false`, which is why the flags are imported
|
|
106
341
|
// now rather than restated (issue #286's sweep). Nothing was exploitable -- the only command below is
|
package/src/prepare.mjs
CHANGED
|
@@ -59,6 +59,14 @@ export function makePrepareWorkspace({
|
|
|
59
59
|
// construction -- prepares exactly what it always did: no /session mount, nothing on disk.
|
|
60
60
|
resolveSession = () => null,
|
|
61
61
|
prepareLocal = prepareLocalWorkspace,
|
|
62
|
+
// Issue #504 part B: where a local job's resolved folder may lie. `{ jobPaths, envelopeFile }`: `jobPaths()` gives the
|
|
63
|
+
// run roots and cron folders a folder named inside must stay inside, and `envelopeFile` the file whose folder no job
|
|
64
|
+
// may mount. Null (a bare dispatcher) passes neither, and prepareLocal then judges nothing but the folder itself.
|
|
65
|
+
localPlacement = null,
|
|
66
|
+
// Issue #505: `(job) => null | { body } | { outcome: "policy", reason }`, the portfolio snapshot builder
|
|
67
|
+
// (portfolio-snapshot.mjs `makePortfolioSnapshot`). Asked only for a local job the processor confirmed as a portfolio
|
|
68
|
+
// job (`portfolio: true` below); null (a bare dispatcher) writes no snapshot for any job.
|
|
69
|
+
portfolioSnapshot = null,
|
|
62
70
|
// Keyed by `job.kind`, so a new forge is one entry rather than a new `if`. A kind with no entry falls
|
|
63
71
|
// through to the throw below, which is what makes an unrouted job loud instead of a silent no-op.
|
|
64
72
|
preparers = { github: prepareGithubWorkspace },
|
|
@@ -70,11 +78,32 @@ export function makePrepareWorkspace({
|
|
|
70
78
|
// not only here: a temp cleaner (systemd-tmpfiles ages /tmp) can remove an idle jobs dir, and a name under the temp
|
|
71
79
|
// dir that is gone is one any account can create next.
|
|
72
80
|
ensureDir = (dir) => ensureJobsDir(dir),
|
|
81
|
+
// Issue #524: how a thrown preparer's job dir is removed. A seam so the guard around it is testable.
|
|
82
|
+
removeDir = (dir) => rmSync(dir, { recursive: true, force: true }),
|
|
73
83
|
}) {
|
|
74
84
|
ensureDir(jobsDir);
|
|
75
|
-
return async function prepareWorkspace(job, token, { queueJobId, piVersion = null, jobUser = null, podmanStore = null } = {}) {
|
|
85
|
+
return async function prepareWorkspace(job, token, { queueJobId, piVersion = null, jobUser = null, podmanStore = null, portfolio = false } = {}) {
|
|
76
86
|
ensureDir(jobsDir);
|
|
77
87
|
const jobDir = mkdtempSync(join(jobsDir, "job-"));
|
|
88
|
+
// Issue #524: a THROW out of anything below leaves `jobDir` to nobody. The processor tears down only what
|
|
89
|
+
// it was handed, and a throw hands it nothing, so every config refusal raised in a preparer (a local folder
|
|
90
|
+
// that does not exist, a forge kind with no preparer) and every infrastructure throw (a clone that failed)
|
|
91
|
+
// left one empty `jobs/job-*` per attempt, forever. `discardOnPolicy` covers the RETURNED refusals; this
|
|
92
|
+
// covers the thrown ones, in one place for every kind, rather than in each preparer that can throw.
|
|
93
|
+
// A retry makes a fresh directory, so nothing is lost by removing this one.
|
|
94
|
+
try {
|
|
95
|
+
return await prepareInto(job, token, jobDir, { queueJobId, piVersion, jobUser, podmanStore, portfolio });
|
|
96
|
+
} catch (error) {
|
|
97
|
+
// GUARDED: a removal that fails (a busy mount, a permission flipped mid-job) must never replace the error
|
|
98
|
+
// that is the job's actual outcome. The directory is then left, which is what happened before this catch.
|
|
99
|
+
try {
|
|
100
|
+
removeDir(jobDir);
|
|
101
|
+
} catch {}
|
|
102
|
+
throw error;
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
async function prepareInto(job, token, jobDir, { queueJobId, piVersion, jobUser, podmanStore, portfolio }) {
|
|
78
107
|
// The trigger's injected skills (REQ-PER-TRIGGER-SKILLS, issue #60), COPIED here rather than
|
|
79
108
|
// mounted, and copied ONCE for every job kind because this is where local and forge converge.
|
|
80
109
|
//
|
|
@@ -128,7 +157,10 @@ export function makePrepareWorkspace({
|
|
|
128
157
|
? `Use the "${job.flow}" skill for this task.\n\n${pointer}${job.task ?? ""}`
|
|
129
158
|
: `${pointer}${job.task ?? ""}`;
|
|
130
159
|
const event = localEventContext(job, queueJobId, findPreviousRun);
|
|
131
|
-
|
|
160
|
+
// The snapshot only for a confirmed portfolio job, and never for a chained child or a manual run, whatever its data
|
|
161
|
+
// says (the processor's gate already holds both; this keeps a direct caller to the same rule).
|
|
162
|
+
const snapshot = portfolio === true && typeof portfolioSnapshot === "function" && event.source === "cron" ? { portfolio: () => portfolioSnapshot(job) } : {};
|
|
163
|
+
return discardOnPolicy(stampSandbox(await prepareLocal({ folder: job.folder, task, jobDir, event, ...(localPlacement ? { jobPaths: localPlacement.jobPaths, envelopeFile: localPlacement.envelopeFile ?? null } : {}), ...snapshot }), sandbox), jobDir);
|
|
132
164
|
}
|
|
133
165
|
const prepare = preparers[job.kind];
|
|
134
166
|
if (prepare) {
|
|
@@ -151,7 +183,7 @@ export function makePrepareWorkspace({
|
|
|
151
183
|
);
|
|
152
184
|
}
|
|
153
185
|
throw new Error(`unknown job kind: ${job.kind}`);
|
|
154
|
-
}
|
|
186
|
+
}
|
|
155
187
|
}
|
|
156
188
|
|
|
157
189
|
/**
|
package/src/pricing.mjs
CHANGED
|
@@ -29,8 +29,10 @@ import { getBuiltinModel, getBuiltinModels, getBuiltinProviders } from "@earendi
|
|
|
29
29
|
/**
|
|
30
30
|
* Every builtin pi-ai model, flattened to `{ provider, id, cost }`. `cost` is pi-ai's ModelCost
|
|
31
31
|
* object passed BY REFERENCE -- it is pi-ai's data, callers treat it read-only. Rates are USD per
|
|
32
|
-
* 1M tokens
|
|
33
|
-
*
|
|
32
|
+
* 1M tokens. An all-zero table is pi-ai's data, not missing data: at the 0.99.1 pin the all-zero
|
|
33
|
+
* providers include the qwen-token-plan family. It is NOT a subscription signal any more (issue #509):
|
|
34
|
+
* kimi-coding and zai-coding-cn, all-zero at 0.80.7, now carry implied API-equivalent rates on most
|
|
35
|
+
* models although their plans are prepaid, and plenty of pay-per-token providers list a free model.
|
|
34
36
|
*/
|
|
35
37
|
export function listPricedModels() {
|
|
36
38
|
const out = [];
|
|
@@ -58,9 +60,11 @@ export function getPricedModel(provider, id) {
|
|
|
58
60
|
}
|
|
59
61
|
|
|
60
62
|
/**
|
|
61
|
-
* True when all four base rates are zero
|
|
62
|
-
*
|
|
63
|
-
*
|
|
63
|
+
* True when all four base rates are zero: pi-ai rated the model at $0, which is a fact about the table
|
|
64
|
+
* and nothing more. It USED to be read as the signature of a subscription-backed provider; at the 0.99.1
|
|
65
|
+
* pin it is not (issue #509: kimi-coding and zai-coding-cn carry implied prices, and free models of
|
|
66
|
+
* metered providers are zero too), so a subscription is known only from subscriptions.json. Tiers are
|
|
67
|
+
* deliberately ignored: a zero-rate provider ships no tiers, and a priced provider
|
|
64
68
|
* with a zero base rate somewhere does not become "free" by it. Null/malformed input is false --
|
|
65
69
|
* "not zero-rated" is the safe answer for a thing that is not a model.
|
|
66
70
|
*/
|