@edgehero/pi-dispatch 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -1
- package/deploy/com.pi-dispatch.worker.plist +16 -6
- package/deploy/docker-compose.yml +66 -0
- package/deploy/worker-env-wrapper.cmd +23 -14
- package/deploy/worker-env-wrapper.sh +22 -15
- package/package.json +1 -1
- package/src/azure-prompt.mjs +14 -8
- package/src/cli.mjs +2 -2
- package/src/copy-tree.mjs +215 -0
- package/src/doctor.mjs +283 -8
- package/src/forgejo-prompt.mjs +14 -8
- package/src/github-app-setup.mjs +7 -2
- package/src/github-prompt.mjs +97 -17
- package/src/gitlab-prompt.mjs +14 -8
- package/src/host-pi.mjs +478 -0
- package/src/import-pi.mjs +299 -139
- package/src/materialize.mjs +262 -40
- package/src/outbox.mjs +5 -0
- package/src/packages.mjs +95 -2
- package/src/prepare-github.mjs +23 -4
- package/src/prepare-local.mjs +6 -1
- package/src/prepare.mjs +73 -14
- package/src/processor.mjs +89 -10
- package/src/queue.mjs +16 -2
- package/src/run-container.mjs +7 -2
- package/src/schedules.mjs +16 -1
- package/src/service.mjs +135 -40
- package/src/session-store.mjs +19 -4
- package/src/start.mjs +54 -9
- package/src/triggers.mjs +211 -12
package/src/prepare.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { mkdirSync, mkdtempSync } from "node:fs";
|
|
1
|
+
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
|
2
2
|
import { rm } from "node:fs/promises";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { resolveJobImage } from "./image-preflight.mjs";
|
|
@@ -11,6 +11,17 @@ import { buildGitLabPrompt } from "./gitlab-prompt.mjs";
|
|
|
11
11
|
import { buildForgejoPrompt } from "./forgejo-prompt.mjs";
|
|
12
12
|
import { buildAzurePrompt } from "./azure-prompt.mjs";
|
|
13
13
|
import { prepareLocalWorkspace } from "./prepare-local.mjs";
|
|
14
|
+
import { copySkillTree } from "./copy-tree.mjs";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The subdirectory of the per-job dir a trigger's injected skills are copied into, so they reach the
|
|
18
|
+
* container at `/job/trigger-skills` on the `/job:ro` bind that already exists.
|
|
19
|
+
*
|
|
20
|
+
* The runner spells the same last segment from its own side (TRIGGER_SKILLS_DIR in
|
|
21
|
+
* image/runner/src/loader.mjs). It is not on this host and this file is not in that container, so the
|
|
22
|
+
* duplication is forced. CHANGE BOTH, IN THE SAME COMMIT.
|
|
23
|
+
*/
|
|
24
|
+
export const TRIGGER_SKILLS_SUBDIR = "trigger-skills";
|
|
14
25
|
|
|
15
26
|
/**
|
|
16
27
|
* The `prepareWorkspace` dispatcher the processor injects. Creates a per-job dir under `jobsDir`
|
|
@@ -45,10 +56,34 @@ export function makePrepareWorkspace({
|
|
|
45
56
|
// Keyed by `job.kind`, so a new forge is one entry rather than a new `if`. A kind with no entry falls
|
|
46
57
|
// through to the throw below, which is what makes an unrouted job loud instead of a silent no-op.
|
|
47
58
|
preparers = { github: prepareGithubWorkspace },
|
|
59
|
+
// REQ-PER-TRIGGER-SKILLS. Injected so the copy is testable without a real host tree, and defaulted so
|
|
60
|
+
// an unwired dispatcher behaves exactly as it did: a job with no `run.skillsDir` never calls it.
|
|
61
|
+
injectSkills = copySkillTree,
|
|
62
|
+
log = () => {},
|
|
48
63
|
}) {
|
|
49
64
|
mkdirSync(jobsDir, { recursive: true });
|
|
50
65
|
return async function prepareWorkspace(job, token, { queueJobId, piVersion = null } = {}) {
|
|
51
66
|
const jobDir = mkdtempSync(join(jobsDir, "job-"));
|
|
67
|
+
// The trigger's injected skills (REQ-PER-TRIGGER-SKILLS, issue #60), COPIED here rather than
|
|
68
|
+
// mounted, and copied ONCE for every job kind because this is where local and forge converge.
|
|
69
|
+
//
|
|
70
|
+
// Copied, not bind-mounted, and that is the decision rather than the implementation. `:ro` bounds
|
|
71
|
+
// the CONTAINER, not the host, and pi reads a skill's body on demand through the read tool, so a
|
|
72
|
+
// live bind could change under a running agent -- the copy is what pins the instruction set for the
|
|
73
|
+
// life of the job, exactly as materialising .pi/ at a fixed sha does for the repo's own. It also
|
|
74
|
+
// answers symlinks once, on the side that can (copy-tree.mjs), instead of handing pi a tree whose
|
|
75
|
+
// links it would follow. And it adds NO mount, so CONST-ISOLATION-CONTAINER-PER-JOB's enumeration
|
|
76
|
+
// is untouched -- the same trade DES-OPERATOR-GLOBAL-OVERLAY made for staged packages.
|
|
77
|
+
if (job.skillsDir) {
|
|
78
|
+
const injected = injectSkills(job.skillsDir, join(jobDir, TRIGGER_SKILLS_SUBDIR));
|
|
79
|
+
if (injected?.refused) {
|
|
80
|
+
rmSync(jobDir, { recursive: true, force: true });
|
|
81
|
+
return { outcome: "policy", reason: injected.refused };
|
|
82
|
+
}
|
|
83
|
+
// Counts only. `injected` carries no name and no path by construction, so this line cannot grow
|
|
84
|
+
// a host path by a later edit (the same shape packages.mjs's `dropped` record has).
|
|
85
|
+
log("trigger_skills_injected", { dirs: injected.dirs, files: injected.files, bytes: injected.bytes });
|
|
86
|
+
}
|
|
52
87
|
// What `cleanup` needs to retain this run's directory, stamped here because this is the only place
|
|
53
88
|
// that holds all three at once. Applied to the RESULT rather than mutated in, so a preparer's
|
|
54
89
|
// `{ outcome: "policy" }` refusal -- which carries no jobDir -- is passed through untouched.
|
|
@@ -63,23 +98,26 @@ export function makePrepareWorkspace({
|
|
|
63
98
|
? `Use the "${job.flow}" skill for this task.\n\n${pointer}${job.task ?? ""}`
|
|
64
99
|
: `${pointer}${job.task ?? ""}`;
|
|
65
100
|
const event = localEventContext(job, queueJobId, findPreviousRun);
|
|
66
|
-
return stampSandbox(await prepareLocal({ folder: job.folder, task, jobDir, event }), sandbox);
|
|
101
|
+
return discardOnPolicy(stampSandbox(await prepareLocal({ folder: job.folder, task, jobDir, event }), sandbox), jobDir);
|
|
67
102
|
}
|
|
68
103
|
const prepare = preparers[job.kind];
|
|
69
104
|
if (prepare) {
|
|
70
105
|
const host = forgeFor?.(job)?.host;
|
|
71
|
-
return
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
106
|
+
return discardOnPolicy(
|
|
107
|
+
stampSandbox(
|
|
108
|
+
await prepare(job, token, {
|
|
109
|
+
jobDir,
|
|
110
|
+
resolveDefaultBranchSha: host?.resolveDefaultBranchSha,
|
|
111
|
+
// The head ref a pull/merge-request job keys on comes from the FORGE API, never the webhook
|
|
112
|
+
// payload: an issue_comment on a PR carries no head at all, and a payload-supplied head repo
|
|
113
|
+
// is attacker-controlled data that must not decide which transcript a job is handed.
|
|
114
|
+
resolvePullRequestHead: host?.resolvePullRequestHead,
|
|
115
|
+
resolveSession,
|
|
116
|
+
piVersion,
|
|
117
|
+
}),
|
|
118
|
+
sandbox,
|
|
119
|
+
),
|
|
120
|
+
jobDir,
|
|
83
121
|
);
|
|
84
122
|
}
|
|
85
123
|
throw new Error(`unknown job kind: ${job.kind}`);
|
|
@@ -131,6 +169,27 @@ function stampSandbox(prepared, sandbox) {
|
|
|
131
169
|
return { ...prepared, sandbox };
|
|
132
170
|
}
|
|
133
171
|
|
|
172
|
+
/**
|
|
173
|
+
* Remove the mkdtemp'd job dir when the preparer REFUSED, because nothing downstream will.
|
|
174
|
+
*
|
|
175
|
+
* A determinate refusal carries no `jobDir` (see stampSandbox), and both teardown paths -- `cleanup`
|
|
176
|
+
* and `makeCleanup`'s retention branch -- guard on `prepared?.jobDir`. So the directory this function
|
|
177
|
+
* created two dozen lines up, which by then may hold a partial clone, was simply left on disk: one
|
|
178
|
+
* per refusal, forever. That has been true of `sha-gone` since it shipped and was only ever invisible
|
|
179
|
+
* because refusals are rare; issue #60 adds cap refusals that a misconfigured repo hits on EVERY
|
|
180
|
+
* delivery, which turns a slow leak into a fast one.
|
|
181
|
+
*
|
|
182
|
+
* Deliberately not folded into stampSandbox: that function's job is to decide what a RESULT carries,
|
|
183
|
+
* and a filesystem side effect hidden inside it would be the kind of thing the next reader has to
|
|
184
|
+
* discover. Deliberately `rmSync` rather than the async `rm`, so the directory is gone before the
|
|
185
|
+
* refusal is returned and no teardown ordering has to be reasoned about.
|
|
186
|
+
*/
|
|
187
|
+
function discardOnPolicy(prepared, jobDir) {
|
|
188
|
+
if (prepared?.outcome !== "policy") return prepared;
|
|
189
|
+
rmSync(jobDir, { recursive: true, force: true });
|
|
190
|
+
return prepared;
|
|
191
|
+
}
|
|
192
|
+
|
|
134
193
|
/** Remove a per-job dir after the run. The workspace (the operator's folder) is never touched here. */
|
|
135
194
|
export async function cleanup(prepared) {
|
|
136
195
|
if (prepared?.jobDir) await rm(prepared.jobDir, { recursive: true, force: true });
|
package/src/processor.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { lstatSync } from "node:fs";
|
|
1
2
|
import { checkTokenCap, recordTokenSpend, releaseBudget, reserveBudget } from "./budget.mjs";
|
|
2
3
|
import { configError } from "./config.mjs";
|
|
3
4
|
import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
|
|
@@ -9,18 +10,21 @@ import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
|
|
|
9
10
|
* The order is the contract, and every step before `runContainer` must be free of provider spend:
|
|
10
11
|
*
|
|
11
12
|
* 0. refuse a job image this host does not have -- INT-CONTAINER-RUNTIME-CONTRACT
|
|
12
|
-
* 1.
|
|
13
|
+
* 1. REFUSE an armed `run.resume` with no session store to persist into (the one fail-CLOSED case)
|
|
14
|
+
* -- REQ-RESUMABLE-SESSION
|
|
15
|
+
* 2. mint a scoped token (GitHub jobs, and local jobs opted in via `github: true`)
|
|
13
16
|
* -- CONST-TOKEN-SCOPED-PER-JOB
|
|
14
|
-
*
|
|
17
|
+
* 3. REFUSE an unprotected default branch (GitHub jobs only -- a local job has no repo)
|
|
15
18
|
* -- REQ-BRANCH-PROTECTION-PRECONDITION
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
19
|
+
* 4. resolve the default-branch SHA (fresh API), clone at it, materialise .pi/, write the prompt
|
|
20
|
+
* 5. reserve a budget slot -- CONST-BUDGET-BEFORE-TOKENS
|
|
21
|
+
* 6. ONLY NOW run the container (the only step that spends provider tokens)
|
|
22
|
+
* 7. map the container exit code to retry-vs-success
|
|
20
23
|
*
|
|
21
24
|
* Budget is reserved as late as possible but strictly before the container, so a refusal from an
|
|
22
|
-
* earlier free gate (unprotected repo, clone failure) never consumes a
|
|
23
|
-
* is the only thing that spends money, so "before tokens" means "before this
|
|
25
|
+
* earlier free gate (unprotected repo, an armed resume with no store, clone failure) never consumes a
|
|
26
|
+
* daily slot. The container is the only thing that spends money, so "before tokens" means "before this
|
|
27
|
+
* line".
|
|
24
28
|
*
|
|
25
29
|
* Returns a result object on a non-retryable outcome; THROWS on a retryable (infra) one so BullMQ
|
|
26
30
|
* retries per `attempts`. The caller (the BullMQ processor) turns the thrown/returned distinction
|
|
@@ -41,6 +45,28 @@ export async function runJob(job, deps) {
|
|
|
41
45
|
// the store, on a COMPLETED exit only. Never throws. The default is a no-op so a wiring that omits
|
|
42
46
|
// it behaves exactly as before -- no store, no promotion, no session in the record.
|
|
43
47
|
promoteSession = () => null,
|
|
48
|
+
// The session store's root, or null when the feature is unavailable (REQ-RESUMABLE-SESSION). Read
|
|
49
|
+
// ONLY to answer the fail-closed gate below -- nothing here opens it, and it never reaches the
|
|
50
|
+
// container env (docker-run.mjs mounts a per-job COPY, never the store).
|
|
51
|
+
//
|
|
52
|
+
// The default is the env read config.mjs itself performs (`env.PI_SESSIONS_DIR || null`) rather than
|
|
53
|
+
// a bare `null`, and the difference is not cosmetic. A `null` default would make the gate refuse
|
|
54
|
+
// EVERY armed job under any wiring that does not pass this key -- a false refusal that looks exactly
|
|
55
|
+
// like the true one -- whereas the env is the single source both readers derive from, so the two
|
|
56
|
+
// cannot disagree about whether a store exists. A wiring may still pass `sessionsDir` explicitly to
|
|
57
|
+
// make the seam visible; it resolves to the same value.
|
|
58
|
+
sessionsDir = process.env.PI_SESSIONS_DIR || null,
|
|
59
|
+
// REQ-PER-TRIGGER-SKILLS. Injected so the pre-spend gate is testable without a real directory, and
|
|
60
|
+
// lstat rather than stat so a symlinked skillsDir is judged on its own inode -- the habit copy-tree.mjs,
|
|
61
|
+
// outbox.mjs and sandbox-store.mjs all keep. A throw is a refusal: an unreadable path is still absent
|
|
62
|
+
// as far as this job is concerned.
|
|
63
|
+
isReadableDir = (p) => {
|
|
64
|
+
try {
|
|
65
|
+
return lstatSync(p).isDirectory();
|
|
66
|
+
} catch {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
},
|
|
44
70
|
// (job) => scoped short-lived token. Takes the JOB, not the repo: which forge mints -- and therefore
|
|
45
71
|
// which credential the container gets -- is a property of `job.kind`, and only the wiring knows the
|
|
46
72
|
// map. Called for forge-backed jobs and for local jobs opted in via `github: true`; unflagged local
|
|
@@ -141,6 +167,58 @@ export async function runJob(job, deps) {
|
|
|
141
167
|
throw new InfraRetry("docker unavailable, image preflight could not run", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
142
168
|
}
|
|
143
169
|
|
|
170
|
+
// REQ-RESUMABLE-SESSION's one fail-CLOSED case. Everything else in that feature fails OPEN and
|
|
171
|
+
// NAMES itself -- absent, expired, too-large, unparseable, locked, promote-failed -- because a cold
|
|
172
|
+
// start is a correct run. This one cannot be: with no `sessionsDir`, resolveSession returns null
|
|
173
|
+
// (session-store.mjs), so nothing is staged, no /session is mounted, the transcript dies with the
|
|
174
|
+
// container, and the NEXT job on that key cold-starts too. The job would exit 0 and look like the
|
|
175
|
+
// feature worked. That is an operator who believes a disclosure is on while it is off, with a green
|
|
176
|
+
// run to confirm the belief -- the inversion validatePackagesFlag's comment describes one flag over,
|
|
177
|
+
// arriving from the other direction.
|
|
178
|
+
//
|
|
179
|
+
// PLACEMENT IS THE POINT. Free, determinate, credential-less and I/O-less -- the answer is two
|
|
180
|
+
// values already in hand -- so it belongs among the free policy refusals and strictly before
|
|
181
|
+
// anything that spends: before the mint (no credential is needed to know the answer, so none is
|
|
182
|
+
// created only to be discarded), before the branch check's API call, before prepareWorkspace's
|
|
183
|
+
// clone, before checkTokenCap's read and before reserveBudget's INCR -- hence `budgetReserved:
|
|
184
|
+
// false`. It sits AFTER the image preflight for the reporting reason the token-cap comment below
|
|
185
|
+
// already states: a missing image blocks EVERY job of EVERY kind on this host, so it is the one an
|
|
186
|
+
// operator must fix first either way, while this blocks only the triggers that armed the flag.
|
|
187
|
+
//
|
|
188
|
+
// Strict `=== true`, the same test prepare-github.mjs uses to decide whether to resolve a session at
|
|
189
|
+
// all, so the gate and the feature cannot disagree about what "armed" means. Kind-agnostic on
|
|
190
|
+
// purpose: only forge jobs can arm the flag today (triggers.mjs refuses it on cron, and a CLI or
|
|
191
|
+
// chained job has no trigger entry that could set it), but a gate written as an enumeration of kinds
|
|
192
|
+
// is a gate the next kind skips silently.
|
|
193
|
+
if (job.resume === true && !sessionsDir) {
|
|
194
|
+
await comment(job, "Refused: this trigger set `run.resume` but PI_SESSIONS_DIR is unset, so there is nowhere to persist the transcript -- the job would run with no session and still report success. Set PI_SESSIONS_DIR to a private directory outside every repo, or drop `run.resume` from this trigger. Not run.");
|
|
195
|
+
// The variable NAME, never a value: there is no path to print here (its absence IS the refusal),
|
|
196
|
+
// and the store's path is the one setting SECURITY.md calls a PII store. `kind` is host-assigned,
|
|
197
|
+
// the same PII class as the `repo` on the branch refusal below.
|
|
198
|
+
log("refused_sessions_dir_unset", { kind: job.kind ?? null });
|
|
199
|
+
// exitCode/turns/tokens null and budgetReserved false: refused pre-container AND pre-reserve,
|
|
200
|
+
// exactly as the image refusals above. RETURNED, not thrown: an unset environment variable is
|
|
201
|
+
// determinate, and no number of retries sets it (CONST-RETRY-INFRA-ONLY).
|
|
202
|
+
return { outcome: "policy", reason: "sessions-dir-unset", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// REQ-PER-TRIGGER-SKILLS. A trigger that named a skills directory the worker cannot see would run
|
|
206
|
+
// its flow WITHOUT the skills it was written against, produce a plausible report, and exit 0. Free
|
|
207
|
+
// and determinate -- one lstat, no credential needed to know the answer -- so it belongs among the
|
|
208
|
+
// free refusals and strictly before anything that spends: before the mint (no token is created only
|
|
209
|
+
// to be discarded), before the clone, before the token-cap read and before reserveBudget
|
|
210
|
+
// (CONST-BUDGET-BEFORE-TOKENS). Last among the free gates because it is the NARROWEST: a missing
|
|
211
|
+
// image blocks every job on this host, an unset sessions dir blocks every armed trigger, a bad
|
|
212
|
+
// skillsDir blocks one trigger.
|
|
213
|
+
if (job.skillsDir && !isReadableDir(job.skillsDir)) {
|
|
214
|
+
await comment(job, "Refused: this trigger set `run.skillsDir`, and that path is absent or is not a directory on the worker host. The job would have run without the skills the flow was written against. Not run.");
|
|
215
|
+
// The FIELD name, never its value. `comment` posts publicly on the issue, so a host path here
|
|
216
|
+
// would publish the operator's filesystem layout to anyone reading the thread; the log line is
|
|
217
|
+
// the same restraint refused_sessions_dir_unset keeps.
|
|
218
|
+
log("refused_skills_dir_missing", { kind: job.kind ?? null });
|
|
219
|
+
return { outcome: "policy", reason: "skills-dir-missing", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
220
|
+
}
|
|
221
|
+
|
|
144
222
|
if (wantsForgeToken) {
|
|
145
223
|
token = await mintToken(job);
|
|
146
224
|
|
|
@@ -167,8 +245,9 @@ export async function runJob(job, deps) {
|
|
|
167
245
|
|
|
168
246
|
prepared = await prepareWorkspace(job, token, { piVersion }); // resolves SHA, clones, materialises .pi/, writes prompt
|
|
169
247
|
|
|
170
|
-
// A determinate prepare refusal
|
|
171
|
-
//
|
|
248
|
+
// A determinate prepare refusal -- sha-gone (the default branch advanced past the resolved tip),
|
|
249
|
+
// or a `pi-*` materialiser cap breach (the repo's .pi/ is too large to place in /job, issue #60)
|
|
250
|
+
// -- is POLICY: return before reserveBudget so it burns no cap slot and is never retried.
|
|
172
251
|
// Mirrors the branch-protection policy return above. Spread-plus-attribution: the prepare
|
|
173
252
|
// result keeps its own reason and fields, and the host-effective provider/model land beside
|
|
174
253
|
// them exactly as on every other terminal result.
|
package/src/queue.mjs
CHANGED
|
@@ -16,7 +16,7 @@ export function makeQueue(connection) {
|
|
|
16
16
|
* removeOnComplete keeps the dedup window ~= the retention. Unlike webhooks, local jobs are not
|
|
17
17
|
* redelivered, so a modest window is enough.
|
|
18
18
|
*/
|
|
19
|
-
export async function enqueueLocalJob(queue, { folder, flow, task, provider, model, maxTurns, image, chainDepth, parentJobId, jobId, now = new Date() }) {
|
|
19
|
+
export async function enqueueLocalJob(queue, { folder, flow, task, provider, model, maxTurns, image, skillsDir, chainDepth, parentJobId, jobId, now = new Date() }) {
|
|
20
20
|
const minute = now.toISOString().slice(0, 16); // YYYY-MM-DDTHH:MM -- the dedup window
|
|
21
21
|
// A caller-supplied jobId (the outbox collector's retry-idempotent chainedJobId) wins; otherwise the
|
|
22
22
|
// minute-windowed localJobId is the dedup key.
|
|
@@ -33,6 +33,11 @@ export async function enqueueLocalJob(queue, { folder, flow, task, provider, mod
|
|
|
33
33
|
model,
|
|
34
34
|
maxTurns,
|
|
35
35
|
...(image !== undefined && { image }),
|
|
36
|
+
// The host directory of operator-authored skills this trigger injects (REQ-PER-TRIGGER-SKILLS).
|
|
37
|
+
// Conditional like `image`, so an unflagged job's data stays byte-identical, and at JOB level rather
|
|
38
|
+
// than inside `trigger` because a worker-host path is an execution knob, not a fact about the
|
|
39
|
+
// delivery -- and `trigger` is the object copied into /job/event.json.
|
|
40
|
+
...(skillsDir !== undefined && { skillsDir }),
|
|
36
41
|
...(chainDepth !== undefined && { chainDepth }),
|
|
37
42
|
...(parentJobId !== undefined && { parentJobId }),
|
|
38
43
|
};
|
|
@@ -110,7 +115,7 @@ export async function enqueueGitLabJob(queue, fields) {
|
|
|
110
115
|
* window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
|
|
111
116
|
* has always been.
|
|
112
117
|
*/
|
|
113
|
-
export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, trigger, provider, model, maxTurns, packages, image, resume, replica, replicas }) {
|
|
118
|
+
export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, replica, replicas }) {
|
|
114
119
|
const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
|
|
115
120
|
// `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
|
|
116
121
|
// come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
|
|
@@ -133,6 +138,15 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
|
|
|
133
138
|
maxTurns,
|
|
134
139
|
...(packages !== undefined && { packages }),
|
|
135
140
|
...(image !== undefined && { image }),
|
|
141
|
+
// The host directory of operator-authored skills this trigger injects (REQ-PER-TRIGGER-SKILLS).
|
|
142
|
+
// Conditional like `image`, so an unflagged job's data stays byte-identical, and at JOB level rather
|
|
143
|
+
// than inside `trigger` because a worker-host path is an execution knob, not a fact about the
|
|
144
|
+
// delivery -- and `trigger` is the object copied into /job/event.json.
|
|
145
|
+
...(skillsDir !== undefined && { skillsDir }),
|
|
146
|
+
// The operator's standing instruction for this trigger (REQ-PER-TRIGGER-INSTRUCTION). Conditional like
|
|
147
|
+
// the rest, and at JOB level rather than inside `trigger`: it is operator config, not a fact about the
|
|
148
|
+
// delivery, and `trigger` is what /job/event.json is built from.
|
|
149
|
+
...(instructions !== undefined && { instructions }),
|
|
136
150
|
...(resume !== undefined && { resume }),
|
|
137
151
|
// Conditional for the same reason packages/image/resume are: an unflagged job's data must keep
|
|
138
152
|
// exactly the keys it has today. `replica` is this job's 1-based index and `replicas` the set size;
|
package/src/run-container.mjs
CHANGED
|
@@ -33,7 +33,10 @@ export function makeRunContainer({
|
|
|
33
33
|
spawnFn = spawn,
|
|
34
34
|
globalPiDir = null, // REQ-GLOBAL-PI-OVERLAY: operator's global pi overlay dir, mounted :ro; null = off
|
|
35
35
|
allowGlobalExtensions = true, // REQ-GLOBAL-PI-OVERLAY: the staged overlay's extensions load unless PI_GLOBAL_ALLOW_EXTENSIONS=0
|
|
36
|
-
|
|
36
|
+
// REQ-GLOBAL-PI-OVERLAY: container paths of the operator-staged packages. An array, or a RESOLVER called
|
|
37
|
+
// once per job (issue #102): the wired worker passes a resolver so a re-stage lands on the next job with
|
|
38
|
+
// no restart, while the array form stays valid for every caller that has a fixed set.
|
|
39
|
+
packagePaths = [],
|
|
37
40
|
forwardEnv = [],
|
|
38
41
|
authFromPi = false, // fall back to ~/.pi/agent/auth.json for the provider key when the env has none
|
|
39
42
|
forgeHosts = {}, // per-forge self-hosted instance URLs, so a forge CLI in the container talks to the right one
|
|
@@ -63,7 +66,9 @@ export function makeRunContainer({
|
|
|
63
66
|
// (INT-TRIGGERS-FILE-CONTRACT). The strictness that used to live in this `=== true` did not
|
|
64
67
|
// disappear, it moved: parseTriggers refuses any non-boolean run.packages fail-loud at load, so a
|
|
65
68
|
// hand-edited string "false" never becomes job data this comparison could misread as an opt-out.
|
|
66
|
-
|
|
69
|
+
// The opt-out short-circuits BEFORE the resolver runs: a trigger that withheld the staged set has no
|
|
70
|
+
// reason to make the worker read the manifest on its behalf.
|
|
71
|
+
packagePaths: job.packages === false ? [] : typeof packagePaths === "function" ? packagePaths() : packagePaths,
|
|
67
72
|
forwardEnv, // extra host var names to forward (e.g. a custom provider's key)
|
|
68
73
|
// REQ-RESUMABLE-SESSION: the fixed container path, emitted only when this job HAS a transcript.
|
|
69
74
|
// The constant is imported rather than re-typed so the mount below and this variable name one
|
package/src/schedules.mjs
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import { existsSync as fsExistsSync, readFileSync as fsReadFileSync } from "node:fs";
|
|
16
|
+
import { isAbsolute } from "node:path";
|
|
16
17
|
import { configError } from "./config.mjs";
|
|
17
18
|
import { parseTriggers } from "./triggers.mjs";
|
|
18
19
|
|
|
@@ -42,6 +43,20 @@ function normalizeCronSchedule({ on, run }, path, existsSync) {
|
|
|
42
43
|
throw configError(`cron trigger "${on.id}": run.folder does not exist: ${run.folder} (${path})`);
|
|
43
44
|
}
|
|
44
45
|
|
|
46
|
+
// `run.skillsDir` gets the same treatment, and for the same reason (REQ-PER-TRIGGER-SKILLS): the pure
|
|
47
|
+
// validator cannot check a host path, because the RECEIVER parses the same file and may run on another
|
|
48
|
+
// machine entirely. Absoluteness is checked here rather than there for a second reason -- `isAbsolute`
|
|
49
|
+
// is OS-dependent, so a shared check would let a Windows worker and a Linux receiver disagree about the
|
|
50
|
+
// same reviewed file. A broken cron trigger refuses the worker's BOOT rather than failing at 03:00.
|
|
51
|
+
if (run.skillsDir !== undefined) {
|
|
52
|
+
if (!isAbsolute(run.skillsDir)) {
|
|
53
|
+
throw configError(`cron trigger "${on.id}": run.skillsDir must be an absolute path: ${run.skillsDir} (${path})`);
|
|
54
|
+
}
|
|
55
|
+
if (!existsSync(run.skillsDir)) {
|
|
56
|
+
throw configError(`cron trigger "${on.id}": run.skillsDir does not exist: ${run.skillsDir} (${path})`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
45
60
|
// Absent provider/model/maxTurns stay absent (undefined) so the value resolves at job start against the
|
|
46
61
|
// settings overlay/env, not a default frozen here (INT-CONFIG-OVERLAY-CONTRACT). data key order matches
|
|
47
62
|
// queue.mjs -- the shape the processor's runJob consumes. The three per-trigger fields ride along the same
|
|
@@ -53,7 +68,7 @@ function normalizeCronSchedule({ on, run }, path, existsSync) {
|
|
|
53
68
|
// cron-only field: it is carried into the local `/job/event.json` (INT-CONTAINER-JOB-INPUTS) so a
|
|
54
69
|
// scheduled job can name its own trigger; the INT-TRIGGERS-FILE-CONTRACT byte-match acceptance is
|
|
55
70
|
// amended for exactly this field.
|
|
56
|
-
const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
|
|
71
|
+
const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
|
|
57
72
|
// Retention only; the deterministic repeat:<id>:<millis> jobId supplies dedup, so no jobId here, and
|
|
58
73
|
// scheduler jobs are not retried (DES-CRON-VIA-BULLMQ-SCHEDULER) so no attempts/backoff.
|
|
59
74
|
const opts = { removeOnComplete: { age: 24 * 3600 }, removeOnFail: { age: 7 * 24 * 3600 } };
|