@edgehero/pi-dispatch 2.0.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/.env.example +44 -7
  2. package/README.md +14 -6
  3. package/deploy/com.pi-dispatch.worker.plist +1 -1
  4. package/deploy/docker-compose.yml +12 -0
  5. package/deploy/egress-proxy.conf +28 -3
  6. package/deploy/pi-dispatch-egress-proxy.container +8 -2
  7. package/deploy/worker-env-wrapper.cmd +1 -1
  8. package/deploy/worker-env-wrapper.sh +3 -3
  9. package/package.json +9 -2
  10. package/src/allocation.mjs +731 -0
  11. package/src/backends.mjs +243 -0
  12. package/src/budget.mjs +40 -4
  13. package/src/cli.mjs +222 -11
  14. package/src/config.mjs +126 -5
  15. package/src/daemon-facts.mjs +3 -0
  16. package/src/deployment-venue.mjs +1 -0
  17. package/src/doctor.mjs +2316 -183
  18. package/src/dollar-budget.mjs +373 -0
  19. package/src/dollar-fingerprint.mjs +83 -0
  20. package/src/egress-cli.mjs +316 -0
  21. package/src/egress-proxy-state.mjs +35 -5
  22. package/src/egress.mjs +16 -3
  23. package/src/env-allowlist.mjs +142 -18
  24. package/src/env-file.mjs +194 -25
  25. package/src/envelope.mjs +413 -0
  26. package/src/exit-code.mjs +22 -0
  27. package/src/fleet-lease.mjs +85 -25
  28. package/src/get-token.mjs +16 -5
  29. package/src/git-dirty.mjs +67 -0
  30. package/src/github-app-setup.mjs +6 -3
  31. package/src/github-host.mjs +5 -3
  32. package/src/host-pi.mjs +19 -3
  33. package/src/identity.mjs +2 -1
  34. package/src/image-preflight.mjs +98 -24
  35. package/src/image-ref.mjs +37 -0
  36. package/src/import-pi.mjs +4 -2
  37. package/src/index.mjs +407 -62
  38. package/src/init.mjs +18 -0
  39. package/src/job-id.mjs +26 -3
  40. package/src/live-probes.mjs +24 -9
  41. package/src/model-catalog.mjs +297 -0
  42. package/src/model-endpoints.mjs +649 -0
  43. package/src/model-ref.mjs +151 -0
  44. package/src/models-json.mjs +262 -0
  45. package/src/money.mjs +144 -0
  46. package/src/octokit-log.mjs +65 -0
  47. package/src/outbox-plan.mjs +218 -0
  48. package/src/outbox.mjs +29 -9
  49. package/src/output-cap.mjs +157 -0
  50. package/src/packages.mjs +2 -2
  51. package/src/pause-windows.mjs +81 -2
  52. package/src/pi-model-loader.mjs +77 -0
  53. package/src/podman-stack.mjs +16 -3
  54. package/src/portfolio-snapshot.mjs +304 -0
  55. package/src/prepare-local.mjs +247 -12
  56. package/src/prepare.mjs +35 -3
  57. package/src/pricing.mjs +9 -5
  58. package/src/priorities.mjs +569 -0
  59. package/src/processor.mjs +603 -173
  60. package/src/project-id.mjs +17 -0
  61. package/src/projects.mjs +238 -0
  62. package/src/provider-key.mjs +32 -7
  63. package/src/provider-steering.mjs +214 -59
  64. package/src/queue.mjs +111 -6
  65. package/src/reserved-env.mjs +30 -0
  66. package/src/run-container.mjs +59 -5
  67. package/src/run-history.mjs +379 -24
  68. package/src/run-mirror.mjs +30 -0
  69. package/src/runtime-settings.mjs +104 -9
  70. package/src/schedules.mjs +33 -1
  71. package/src/scoped-limits.mjs +447 -27
  72. package/src/secrets.mjs +2 -1
  73. package/src/service.mjs +15 -4
  74. package/src/session-store.mjs +131 -6
  75. package/src/start.mjs +528 -40
  76. package/src/subscriptions.mjs +7 -3
  77. package/src/triggers-file.mjs +65 -4
  78. package/src/triggers.mjs +140 -9
  79. package/src/up.mjs +308 -34
  80. package/src/valkey-endpoint.mjs +3 -2
@@ -1,7 +1,8 @@
1
1
  import { GIT_READ_FLAGS } from "./git-hardening.mjs";
2
2
  import { execFile } from "node:child_process";
3
3
  import * as realFs from "node:fs";
4
- import { basename, join } from "node:path";
4
+ import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
5
+ import { envelopeProtectedIdentities, fileIdentity } from "./envelope.mjs";
5
6
  import { promisify } from "node:util";
6
7
  import { materializePiDir } from "./materialize.mjs";
7
8
  import { InfraRetry } from "./processor.mjs";
@@ -9,6 +10,22 @@ import { isDeterminateFsCode } from "./transient.mjs";
9
10
 
10
11
  const exec = promisify(execFile);
11
12
 
13
+ /** The run-record reason for a local job whose folder has no `.git` at its root (issue #524). */
14
+ export const LOCAL_FOLDER_NOT_A_REPO = "local-folder-not-a-repo";
15
+ /** The run-record reason for a local job whose repository has no commit at HEAD yet (issue #524). */
16
+ export const LOCAL_FOLDER_NO_COMMIT = "local-folder-no-commit";
17
+ /** The run-record reason for a local job whose repository git cannot resolve HEAD in, other than an unborn HEAD (issue #524). */
18
+ export const LOCAL_FOLDER_UNREADABLE_REPO = "local-folder-unreadable-repo";
19
+ /**
20
+ * The run-record reason for a local job whose folder is NAMED inside a job path (a `PI_DISPATCH_RUN_ROOTS` root or a cron
21
+ * trigger's `run.folder`) and RESOLVES, at prepare, to a directory outside it (issue #504 part B). A job container can
22
+ * write inside its own folder, so it can swap a link below a job path after the folder was checked; such a folder is
23
+ * refused rather than mounted wherever the link now points.
24
+ */
25
+ export const LOCAL_FOLDER_ESCAPED = "local-folder-escaped";
26
+ /** The run-record reason for a local job whose resolved folder is the envelope's folder or a directory above it (issue #504 part B). */
27
+ export const LOCAL_FOLDER_HOLDS_ENVELOPE = "local-folder-holds-envelope";
28
+
12
29
  /**
13
30
  * Prepare a LOCAL-FOLDER job. This is the zero-GitHub path: no token, no clone, no PR. The folder
14
31
  * on the operator's own machine becomes /workspace (bind-mounted read-write), edited in place.
@@ -27,11 +44,62 @@ const exec = promisify(execFile);
27
44
  * default keeps a directly-constructed call (tests, older wiring) honest: a job with no derived
28
45
  * context is a manual run.
29
46
  */
30
- export async function prepareLocalWorkspace({ folder, task, jobDir, git = defaultGit, event = { source: "manual" }, fs = realFs }) {
31
- requirePath(fs, folder, `local folder does not exist: ${folder}`, "the local folder");
32
- requirePath(fs, join(folder, ".git"), `local folder is not a git repository (v1 requires one): ${folder}`, "the local folder's .git");
47
+ export async function prepareLocalWorkspace({ folder: named, task, jobDir, git = defaultGit, event = { source: "manual" }, fs = realFs, jobPaths = null, envelopeFile = null, portfolio = null }) {
48
+ requirePath(fs, named, `local folder does not exist: ${named}`, "the local folder");
49
+ // Issue #504 part B: the folder is RESOLVED here, once, and the resolved path is the one everything below reads and the
50
+ // one the container mounts (`workspace`). The raw string was mounted before, so a link below a job path swapped after
51
+ // the folder was checked (by the admin's run-root check at enqueue, or by nothing at all for a chained child) would
52
+ // have mounted whatever it pointed to, read-write. Resolving narrows the window to the moments between this line and
53
+ // the container start, and the two checks below judge the folder that will be mounted. Residual, named: a link above
54
+ // the resolved folder swapped inside that window is followed by the runtime when it mounts.
55
+ const folder = resolveFolder(fs, named);
56
+ const placement = judgePlacement(fs, named, folder, { jobPaths: typeof jobPaths === "function" ? jobPaths() : jobPaths, envelopeFile });
57
+ if (placement) return { outcome: "policy", reason: placement };
58
+ // Issue #524: a folder that is not a repository is its own determinate refusal, RETURNED with its own reason
59
+ // (CONST-RETRY-INFRA-ONLY), not a config throw. As a throw it reached the processor's config arm, which can only
60
+ // say "this deployment is misconfigured" (`config-refused`), about a folder the operator can fix with `git init`.
61
+ // Returned here it is a prepare-stage policy refusal like `sha-gone`: before the budget reserve, so it spends
62
+ // nothing, and `prepare.mjs` removes the job dir it made. The reason is a fixed token and carries no path, so the
63
+ // worker log and the run record stay as PII-free as they are. `pi-dispatch run` refuses the same folder before
64
+ // queueing; this is the check for a folder that stopped being a repository between queue and pickup, and for a
65
+ // cron trigger's folder, which nothing checks when it fires.
66
+ if (!presentAt(fs, join(folder, ".git"), "the local folder's .git")) return { outcome: "policy", reason: LOCAL_FOLDER_NOT_A_REPO };
33
67
 
34
- const sha = (await git(folder, ["rev-parse", "HEAD"])).trim();
68
+ // Issue #524: a HEAD git cannot resolve escaped as an untagged throw, a failed job with no reason. It is now
69
+ // refused by name, RETURNED before the reserve and before anything is written, like the not-a-repo one above,
70
+ // but only once the files git needs have been read back under `presentAt`'s rule (PR #528's review, round 2).
71
+ // git's exit code says "could not", never "why": `rev-parse --verify --quiet` exits 1 for an unborn HEAD AND for
72
+ // a missing commit object, a garbage branch ref, an unreadable `refs/heads`; it exits 128 for an empty `.git`
73
+ // AND for an EACCES on `.git/HEAD` or `.git/objects`, with the same "not a git repository" either way (all
74
+ // measured, git 2.x). Its text is no help either: a locale translates it, and it names host paths.
75
+ // - `local-folder-no-commit` only for a truly unborn HEAD: the follow-up `rev-parse --verify --quiet HEAD`
76
+ // also exits 1 AND `symbolic-ref -q HEAD` succeeds (HEAD names a branch that does not exist yet). A HEAD
77
+ // that resolves to a missing object or a non-commit, or a branch ref that cannot be read, is not "commit
78
+ // once" and must not be told so.
79
+ // - `local-folder-unreadable-repo` for every other 1 or 128.
80
+ // Before either refusal, `assertGitFilesReadable` reads what git needed. An error outside the determinate
81
+ // allow-list (`transient.mjs`: EACCES and EIO are deliberately transient) throws InfraRetry, so a filesystem
82
+ // that cannot answer right now is retried rather than refused for good. Any other git failure (a missing git
83
+ // binary, a signal) still throws as it did.
84
+ let sha;
85
+ try {
86
+ sha = (await git(folder, ["rev-parse", "--verify", "--quiet", "HEAD^{commit}"])).trim();
87
+ } catch (error) {
88
+ if (error?.code !== 1 && error?.code !== 128) throw error;
89
+ assertGitFilesReadable(fs, folder);
90
+ if (error.code === 1 && (await unbornHead(git, folder))) return { outcome: "policy", reason: LOCAL_FOLDER_NO_COMMIT };
91
+ return { outcome: "policy", reason: LOCAL_FOLDER_UNREADABLE_REPO };
92
+ }
93
+
94
+ // Issue #505: a confirmed portfolio job's snapshot, built BEFORE anything is written, so a refusal
95
+ // (`portfolio-snapshot-oversize`) or an InfraRetry (Valkey) leaves nothing behind but the directory prepare.mjs removes.
96
+ // Null from the builder means the live file no longer flags the job (or this host lost its envelope): no snapshot, and
97
+ // the job runs as an ordinary cron job.
98
+ let snapshot = null;
99
+ if (typeof portfolio === "function") {
100
+ snapshot = await portfolio();
101
+ if (snapshot?.outcome === "policy") return snapshot;
102
+ }
35
103
 
36
104
  fs.mkdirSync(jobDir, { recursive: true });
37
105
  // The outbox is the container's only signal channel back to the worker (INT-OUTBOX-CONTRACT). It is
@@ -57,14 +125,108 @@ export async function prepareLocalWorkspace({ folder, task, jobDir, git = defaul
57
125
  const eventBody = {
58
126
  source: event.source,
59
127
  ...(event.trigger ? { trigger: event.trigger } : {}),
60
- folder: basename(folder),
128
+ folder: basename(named),
61
129
  sha,
62
130
  ...(event.source === "cron" ? { scheduledFor: event.scheduledFor ?? null, previousRunAt: event.previousRunAt ?? null } : {}),
63
131
  };
64
132
  fs.writeFileSync(join(jobDir, "event.json"), JSON.stringify(eventBody, null, 2), { mode: 0o444 });
133
+ // /job/portfolio.json (INT-CONTAINER-JOB-INPUTS), 0o444 beside event.json, so it reaches the container on the existing
134
+ // read-only /job mount with no new mount. Its presence is also what composes the portfolio persona in the runner.
135
+ if (typeof snapshot?.body === "string") fs.writeFileSync(join(jobDir, "portfolio.json"), snapshot.body, { mode: 0o444 });
65
136
 
66
137
  // The folder itself is /workspace (rw). No clone: local jobs edit in place.
67
- return { workspace: folder, jobDir, outboxDir, sha, materialised: written };
138
+ // `portfolio: true` only when the snapshot was written (issue #505): the plan collector requires prepare to have
139
+ // agreed, beside the pickup and the live file at collection, so a job that ran without the facts writes no plan.
140
+ return { workspace: folder, jobDir, outboxDir, sha, materialised: written, ...(typeof snapshot?.body === "string" ? { portfolio: true } : {}) };
141
+ }
142
+
143
+ /**
144
+ * The folder's real path (`realpathSync.native`). A determinate absence here (the folder vanished, or a link in it now
145
+ * dangles) is the "does not exist" config refusal `requirePath` gives; any other error is retried, `presentAt`'s rule.
146
+ */
147
+ function resolveFolder(fs, named) {
148
+ try {
149
+ return (fs.realpathSync?.native ?? fs.realpathSync)(named);
150
+ } catch (error) {
151
+ if (isDeterminateFsCode(error?.code)) {
152
+ const refusal = new Error(`local folder does not exist: ${named}`);
153
+ refusal.piDispatchConfig = true;
154
+ throw refusal;
155
+ }
156
+ throw new InfraRetry(`could not resolve the local folder (${error?.code ?? "unknown"}): ${basename(named)}`);
157
+ }
158
+ }
159
+
160
+ /** `p` is `root` or lies below it, as text. Both absolute and resolved. */
161
+ function insideOrEqual(p, root) {
162
+ return p === root || p.startsWith(root.endsWith(sep) ? root : `${root}${sep}`);
163
+ }
164
+
165
+ /** The identity of a path (`dev:ino`, links followed), or null when it cannot be read. */
166
+ function identityAt(fs, path) {
167
+ try {
168
+ return fileIdentity(fs.statSync(path, { bigint: true }));
169
+ } catch {
170
+ return null;
171
+ }
172
+ }
173
+
174
+ /**
175
+ * Where the RESOLVED folder may be mounted from (issue #504 part B), or the refusal reason:
176
+ * - `local-folder-escaped`: the folder is named inside a job path (its text, or its text against a job path's real
177
+ * path, lies inside a run root or a cron folder, or an ancestor of its text has a job path's identity) and its
178
+ * resolved path lies inside none of those job paths, judged
179
+ * by identity up the resolved path, so a firmlink or a case variant is the directory it names. A job path is a
180
+ * place a job can write; a link a job planted there must not carry another job's folder out of it. A folder named
181
+ * outside every job path (an operator's own `pi-dispatch run`) is not held to them, since nothing there is a job's;
182
+ * - `local-folder-holds-envelope`: the resolved folder is the envelope file's folder or a directory above it
183
+ * (`envelopeProtectedIdentities`), so the job could write its own bounds.
184
+ * `jobPaths` is `{ runRoots, cronFolders }` (schedules.mjs `envelopeJobPaths`), or null for none.
185
+ */
186
+ function judgePlacement(fs, named, folder, { jobPaths, envelopeFile }) {
187
+ const real = (p) => {
188
+ try {
189
+ return (fs.realpathSync?.native ?? fs.realpathSync)(p);
190
+ } catch {
191
+ return null;
192
+ }
193
+ };
194
+ const areas = [...(jobPaths?.runRoots ?? []), ...(jobPaths?.cronFolders ?? [])].filter((p) => typeof p === "string" && p.trim() !== "" && isAbsolute(p));
195
+ const text = resolve(named);
196
+ // The identities of the named path's own ancestors, walked up its TEXT (each stat follows links, as the kernel does):
197
+ // "named inside" is decided by identity too, so a case variant of a run root on a case-insensitive volume, or a
198
+ // firmlink spelling of it, is inside it, where a string comparison let it skip the check (PR #574's review).
199
+ const namedAncestors = new Set();
200
+ for (let p = text; areas.length > 0; p = dirname(p)) {
201
+ const id = identityAt(fs, p);
202
+ if (id !== null) namedAncestors.add(id);
203
+ if (dirname(p) === p) break;
204
+ }
205
+ const holding = areas.filter((area) => {
206
+ const r = real(area);
207
+ if (insideOrEqual(text, resolve(area)) || (r !== null && insideOrEqual(text, r))) return true;
208
+ const id = identityAt(fs, area);
209
+ return id !== null && namedAncestors.has(id);
210
+ });
211
+ if (holding.length > 0) {
212
+ const ids = new Set(holding.map((area) => identityAt(fs, area)).filter((id) => id !== null));
213
+ let inside = false;
214
+ for (let p = folder; ; p = dirname(p)) {
215
+ const id = identityAt(fs, p);
216
+ if (id !== null && ids.has(id)) {
217
+ inside = true;
218
+ break;
219
+ }
220
+ if (dirname(p) === p) break;
221
+ }
222
+ if (!inside) return LOCAL_FOLDER_ESCAPED;
223
+ }
224
+ if (envelopeFile !== null && envelopeFile !== undefined) {
225
+ const protectedIds = envelopeProtectedIdentities(envelopeFile, { realpathSync: fs.realpathSync?.native ?? fs.realpathSync, statSync: fs.statSync });
226
+ const id = identityAt(fs, folder);
227
+ if (id !== null && protectedIds.has(id)) return LOCAL_FOLDER_HOLDS_ENVELOPE;
228
+ }
229
+ return null;
68
230
  }
69
231
 
70
232
  /**
@@ -82,14 +244,23 @@ export async function prepareLocalWorkspace({ folder, task, jobDir, git = defaul
82
244
  * `.git` is a FILE in a worktree and in a submodule, and a symlinked project folder is ordinary.
83
245
  */
84
246
  function requirePath(fs, path, absentMessage, what) {
247
+ if (presentAt(fs, path, what)) return;
248
+ const refusal = new Error(absentMessage);
249
+ refusal.piDispatchConfig = true;
250
+ throw refusal;
251
+ }
252
+
253
+ /**
254
+ * True when something is at `path`, false when it is determinately absent, and a THROWN InfraRetry when the
255
+ * filesystem could not answer. The second half of `requirePath`, split out so the `.git` check can return its own
256
+ * refusal instead of throwing one, with the same rule about which errors mean absent (issue #316, below).
257
+ */
258
+ function presentAt(fs, path, what) {
85
259
  try {
86
260
  fs.statSync(path);
261
+ return true;
87
262
  } catch (error) {
88
- if (isDeterminateFsCode(error?.code)) {
89
- const refusal = new Error(absentMessage);
90
- refusal.piDispatchConfig = true;
91
- throw refusal;
92
- }
263
+ if (isDeterminateFsCode(error?.code)) return false;
93
264
  // Not absent, just unreachable right now. Throw the retryable class so the queue tries again
94
265
  // instead of spending the delivery on a verdict that is wrong by the time it is posted.
95
266
  // BASENAME, not the path (issue #289): an InfraRetry survives retries and its message becomes the
@@ -101,6 +272,70 @@ function requirePath(fs, path, absentMessage, what) {
101
272
  }
102
273
  }
103
274
 
275
+ /**
276
+ * True only for an unborn HEAD: one that names a branch with no commit yet (issue #524, PR #528's review). Both
277
+ * answers are git's exit codes. `rev-parse --verify --quiet HEAD` exits 0 when HEAD names SOME object, which after
278
+ * `HEAD^{commit}` failed means a missing or non-commit object; `symbolic-ref -q HEAD` fails for a detached HEAD and
279
+ * for a branch ref git cannot read (measured: a garbage ref and an EACCES `refs/heads` both exit 128 there).
280
+ */
281
+ async function unbornHead(git, folder) {
282
+ try {
283
+ await git(folder, ["rev-parse", "--verify", "--quiet", "HEAD"]);
284
+ return false;
285
+ } catch (error) {
286
+ if (error?.code !== 1) return false;
287
+ }
288
+ try {
289
+ await git(folder, ["symbolic-ref", "-q", "HEAD"]);
290
+ return true;
291
+ } catch {
292
+ return false;
293
+ }
294
+ }
295
+
296
+ /**
297
+ * Read back the files git needs to resolve HEAD, under `presentAt`'s rule: an absence the allow-list calls
298
+ * determinate is fine (the refusal stands), anything else throws InfraRetry so the job is retried (PR #528's review,
299
+ * round 2). `.git`; when it is a file (a worktree or submodule), the gitdir it names; `<gitdir>/HEAD`;
300
+ * `<gitdir>/config` when present; and the branch ref HEAD names, which is where an unreadable `refs/heads` shows.
301
+ * A ref that is absent is ordinary (an unborn branch, or one only in packed-refs). Messages carry basenames only.
302
+ */
303
+ function assertGitFilesReadable(fs, folder) {
304
+ const dotGit = join(folder, ".git");
305
+ const st = statOrRetry(fs, dotGit);
306
+ if (!st) return;
307
+ let gitDir = dotGit;
308
+ if (st.isFile?.()) {
309
+ const pointer = readOrRetry(fs, dotGit);
310
+ const named = pointer === null ? null : /^gitdir:\s*(.+)$/m.exec(pointer)?.[1]?.trim();
311
+ if (!named) return;
312
+ gitDir = resolve(folder, named);
313
+ if (!statOrRetry(fs, gitDir)) return;
314
+ }
315
+ const head = readOrRetry(fs, join(gitDir, "HEAD"));
316
+ readOrRetry(fs, join(gitDir, "config"));
317
+ const ref = head === null ? null : /^ref:\s*(refs\/\S+)\s*$/m.exec(head)?.[1];
318
+ if (ref && !ref.split("/").includes("..")) readOrRetry(fs, join(gitDir, ref));
319
+ }
320
+
321
+ function statOrRetry(fs, path) {
322
+ try {
323
+ return fs.statSync(path);
324
+ } catch (error) {
325
+ if (isDeterminateFsCode(error?.code)) return null;
326
+ throw new InfraRetry(`could not read the local folder's git files (${error?.code ?? "unknown"}): ${basename(path)}`);
327
+ }
328
+ }
329
+
330
+ function readOrRetry(fs, path) {
331
+ try {
332
+ return fs.readFileSync(path, "utf8");
333
+ } catch (error) {
334
+ if (isDeterminateFsCode(error?.code)) return null;
335
+ throw new InfraRetry(`could not read the local folder's git files (${error?.code ?? "unknown"}): ${basename(path)}`);
336
+ }
337
+ }
338
+
104
339
  async function defaultGit(gitDir, args) {
105
340
  // This was the one copy of seven missing `core.fsmonitor=false`, which is why the flags are imported
106
341
  // now rather than restated (issue #286's sweep). Nothing was exploitable -- the only command below is
package/src/prepare.mjs CHANGED
@@ -59,6 +59,14 @@ export function makePrepareWorkspace({
59
59
  // construction -- prepares exactly what it always did: no /session mount, nothing on disk.
60
60
  resolveSession = () => null,
61
61
  prepareLocal = prepareLocalWorkspace,
62
+ // Issue #504 part B: where a local job's resolved folder may lie. `{ jobPaths, envelopeFile }`: `jobPaths()` gives the
63
+ // run roots and cron folders a folder named inside must stay inside, and `envelopeFile` the file whose folder no job
64
+ // may mount. Null (a bare dispatcher) passes neither, and prepareLocal then judges nothing but the folder itself.
65
+ localPlacement = null,
66
+ // Issue #505: `(job) => null | { body } | { outcome: "policy", reason }`, the portfolio snapshot builder
67
+ // (portfolio-snapshot.mjs `makePortfolioSnapshot`). Asked only for a local job the processor confirmed as a portfolio
68
+ // job (`portfolio: true` below); null (a bare dispatcher) writes no snapshot for any job.
69
+ portfolioSnapshot = null,
62
70
  // Keyed by `job.kind`, so a new forge is one entry rather than a new `if`. A kind with no entry falls
63
71
  // through to the throw below, which is what makes an unrouted job loud instead of a silent no-op.
64
72
  preparers = { github: prepareGithubWorkspace },
@@ -70,11 +78,32 @@ export function makePrepareWorkspace({
70
78
  // not only here: a temp cleaner (systemd-tmpfiles ages /tmp) can remove an idle jobs dir, and a name under the temp
71
79
  // dir that is gone is one any account can create next.
72
80
  ensureDir = (dir) => ensureJobsDir(dir),
81
+ // Issue #524: how a thrown preparer's job dir is removed. A seam so the guard around it is testable.
82
+ removeDir = (dir) => rmSync(dir, { recursive: true, force: true }),
73
83
  }) {
74
84
  ensureDir(jobsDir);
75
- return async function prepareWorkspace(job, token, { queueJobId, piVersion = null, jobUser = null, podmanStore = null } = {}) {
85
+ return async function prepareWorkspace(job, token, { queueJobId, piVersion = null, jobUser = null, podmanStore = null, portfolio = false } = {}) {
76
86
  ensureDir(jobsDir);
77
87
  const jobDir = mkdtempSync(join(jobsDir, "job-"));
88
+ // Issue #524: a THROW out of anything below leaves `jobDir` to nobody. The processor tears down only what
89
+ // it was handed, and a throw hands it nothing, so every config refusal raised in a preparer (a local folder
90
+ // that does not exist, a forge kind with no preparer) and every infrastructure throw (a clone that failed)
91
+ // left one empty `jobs/job-*` per attempt, forever. `discardOnPolicy` covers the RETURNED refusals; this
92
+ // covers the thrown ones, in one place for every kind, rather than in each preparer that can throw.
93
+ // A retry makes a fresh directory, so nothing is lost by removing this one.
94
+ try {
95
+ return await prepareInto(job, token, jobDir, { queueJobId, piVersion, jobUser, podmanStore, portfolio });
96
+ } catch (error) {
97
+ // GUARDED: a removal that fails (a busy mount, a permission flipped mid-job) must never replace the error
98
+ // that is the job's actual outcome. The directory is then left, which is what happened before this catch.
99
+ try {
100
+ removeDir(jobDir);
101
+ } catch {}
102
+ throw error;
103
+ }
104
+ };
105
+
106
+ async function prepareInto(job, token, jobDir, { queueJobId, piVersion, jobUser, podmanStore, portfolio }) {
78
107
  // The trigger's injected skills (REQ-PER-TRIGGER-SKILLS, issue #60), COPIED here rather than
79
108
  // mounted, and copied ONCE for every job kind because this is where local and forge converge.
80
109
  //
@@ -128,7 +157,10 @@ export function makePrepareWorkspace({
128
157
  ? `Use the "${job.flow}" skill for this task.\n\n${pointer}${job.task ?? ""}`
129
158
  : `${pointer}${job.task ?? ""}`;
130
159
  const event = localEventContext(job, queueJobId, findPreviousRun);
131
- return discardOnPolicy(stampSandbox(await prepareLocal({ folder: job.folder, task, jobDir, event }), sandbox), jobDir);
160
+ // The snapshot only for a confirmed portfolio job, and never for a chained child or a manual run, whatever its data
161
+ // says (the processor's gate already holds both; this keeps a direct caller to the same rule).
162
+ const snapshot = portfolio === true && typeof portfolioSnapshot === "function" && event.source === "cron" ? { portfolio: () => portfolioSnapshot(job) } : {};
163
+ return discardOnPolicy(stampSandbox(await prepareLocal({ folder: job.folder, task, jobDir, event, ...(localPlacement ? { jobPaths: localPlacement.jobPaths, envelopeFile: localPlacement.envelopeFile ?? null } : {}), ...snapshot }), sandbox), jobDir);
132
164
  }
133
165
  const prepare = preparers[job.kind];
134
166
  if (prepare) {
@@ -151,7 +183,7 @@ export function makePrepareWorkspace({
151
183
  );
152
184
  }
153
185
  throw new Error(`unknown job kind: ${job.kind}`);
154
- };
186
+ }
155
187
  }
156
188
 
157
189
  /**
package/src/pricing.mjs CHANGED
@@ -29,8 +29,10 @@ import { getBuiltinModel, getBuiltinModels, getBuiltinProviders } from "@earendi
29
29
  /**
30
30
  * Every builtin pi-ai model, flattened to `{ provider, id, cost }`. `cost` is pi-ai's ModelCost
31
31
  * object passed BY REFERENCE -- it is pi-ai's data, callers treat it read-only. Rates are USD per
32
- * 1M tokens; an all-zero table is CORRECT data for subscription-backed providers (kimi-coding,
33
- * zai-coding-cn), not missing data -- `isZeroRated` is how callers tell the two apart.
32
+ * 1M tokens. An all-zero table is pi-ai's data, not missing data: at the 0.99.1 pin the all-zero
33
+ * providers include the qwen-token-plan family. It is NOT a subscription signal any more (issue #509):
34
+ * kimi-coding and zai-coding-cn, all-zero at 0.80.7, now carry implied API-equivalent rates on most
35
+ * models although their plans are prepaid, and plenty of pay-per-token providers list a free model.
34
36
  */
35
37
  export function listPricedModels() {
36
38
  const out = [];
@@ -58,9 +60,11 @@ export function getPricedModel(provider, id) {
58
60
  }
59
61
 
60
62
  /**
61
- * True when all four base rates are zero -- the signature of a subscription-backed provider, whose
62
- * runs meter at $0 because the plan is prepaid (see subscriptions.mjs for where the real price
63
- * lives). Tiers are deliberately ignored: a zero-rate provider ships no tiers, and a priced provider
63
+ * True when all four base rates are zero: pi-ai rated the model at $0, which is a fact about the table
64
+ * and nothing more. It USED to be read as the signature of a subscription-backed provider; at the 0.99.1
65
+ * pin it is not (issue #509: kimi-coding and zai-coding-cn carry implied prices, and free models of
66
+ * metered providers are zero too), so a subscription is known only from subscriptions.json. Tiers are
67
+ * deliberately ignored: a zero-rate provider ships no tiers, and a priced provider
64
68
  * with a zero base rate somewhere does not become "free" by it. Null/malformed input is false --
65
69
  * "not zero-rated" is the safe answer for a thing that is not a model.
66
70
  */