@edgehero/pi-dispatch 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +11 -0
- package/deploy/docker-compose.yml +5 -1
- package/package.json +2 -1
- package/src/config.mjs +17 -0
- package/src/doctor.mjs +111 -4
- package/src/env-allowlist.mjs +31 -1
- package/src/forges.mjs +14 -0
- package/src/get-token.mjs +6 -1
- package/src/index.mjs +16 -0
- package/src/outbox.mjs +8 -0
- package/src/processor.mjs +141 -2
- package/src/queue.mjs +35 -3
- package/src/reserved-env.mjs +40 -0
- package/src/run-container.mjs +6 -2
- package/src/run-history.mjs +50 -31
- package/src/runtime-settings.mjs +40 -0
- package/src/schedules.mjs +1 -1
- package/src/secret-profiles.mjs +119 -0
- package/src/secrets.mjs +319 -0
- package/src/start.mjs +41 -3
- package/src/triggers-file.mjs +403 -0
- package/src/triggers.mjs +478 -21
package/src/queue.mjs
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { Queue } from "bullmq";
|
|
2
2
|
import { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId } from "./job-id.mjs";
|
|
3
3
|
import { targetSeparator } from "./forges.mjs";
|
|
4
|
+
import { PR_CLOSE_ACTIONS } from "./triggers.mjs";
|
|
5
|
+
|
|
6
|
+
// The close words in every forge's spelling, derived from the one table (never re-typed here): a
|
|
7
|
+
// matched PR action in this set marks a close job for the semantic-key discriminant below.
|
|
8
|
+
const PR_CLOSE_WORDS = new Set(Object.values(PR_CLOSE_ACTIONS));
|
|
4
9
|
|
|
5
10
|
export const QUEUE = "pi-jobs";
|
|
6
11
|
export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId };
|
|
@@ -16,7 +21,7 @@ export function makeQueue(connection) {
|
|
|
16
21
|
* removeOnComplete keeps the dedup window ~= the retention. Unlike webhooks, local jobs are not
|
|
17
22
|
* redelivered, so a modest window is enough.
|
|
18
23
|
*/
|
|
19
|
-
export async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, skillsDir, chainDepth, parentJobId, jobId, now = new Date() }) {
|
|
24
|
+
export async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, skillsDir, secrets, secretsProfile, chainDepth, parentJobId, jobId, now = new Date() }) {
|
|
20
25
|
const minute = now.toISOString().slice(0, 16); // YYYY-MM-DDTHH:MM -- the dedup window
|
|
21
26
|
// A caller-supplied jobId (the outbox collector's retry-idempotent chainedJobId) wins; otherwise the
|
|
22
27
|
// minute-windowed localJobId is the dedup key. A command job (issue #189) fills the flow slot with
|
|
@@ -47,6 +52,11 @@ export async function enqueueLocalJob(queue, { folder, flow, task, command, prov
|
|
|
47
52
|
// than inside `trigger` because a worker-host path is an execution knob, not a fact about the
|
|
48
53
|
// delivery -- and `trigger` is the object copied into /job/event.json.
|
|
49
54
|
...(skillsDir !== undefined && { skillsDir }),
|
|
55
|
+
// REQ-TRIGGER-SECRETS, on the forge path's terms: references only, resolved by the worker at job
|
|
56
|
+
// start. A cron trigger may bind secrets (a nightly deploy is the obvious user), which is where
|
|
57
|
+
// this differs from `replicas` -- that one is refused on a local job and this one is not.
|
|
58
|
+
...(secrets !== undefined && { secrets }),
|
|
59
|
+
...(secretsProfile !== undefined && { secretsProfile }),
|
|
50
60
|
...(chainDepth !== undefined && { chainDepth }),
|
|
51
61
|
...(parentJobId !== undefined && { parentJobId }),
|
|
52
62
|
};
|
|
@@ -124,7 +134,7 @@ export async function enqueueGitLabJob(queue, fields) {
|
|
|
124
134
|
* window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
|
|
125
135
|
* has always been.
|
|
126
136
|
*/
|
|
127
|
-
export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, replica, replicas }) {
|
|
137
|
+
export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, secrets, secretsProfile, replica, replicas }) {
|
|
128
138
|
const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
|
|
129
139
|
// `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
|
|
130
140
|
// come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
|
|
@@ -162,12 +172,34 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
|
|
|
162
172
|
// delivery, and `trigger` is what /job/event.json is built from.
|
|
163
173
|
...(instructions !== undefined && { instructions }),
|
|
164
174
|
...(resume !== undefined && { resume }),
|
|
175
|
+
// REQ-TRIGGER-SECRETS. The env variables this trigger binds and the profile whose resolver reads
|
|
176
|
+
// them. REFERENCES only: no value is ever enqueued, because a queued job is durable and a resolved
|
|
177
|
+
// credential in Redis would outlive the container it was scoped to. The worker resolves them at job
|
|
178
|
+
// start, pre-spend. At JOB level rather than inside `trigger` for image/skillsDir's reason: `trigger`
|
|
179
|
+
// is copied verbatim into /job/event.json, which an agent reads.
|
|
180
|
+
...(secrets !== undefined && { secrets }),
|
|
181
|
+
...(secretsProfile !== undefined && { secretsProfile }),
|
|
165
182
|
// Conditional for the same reason packages/image/resume are: an unflagged job's data must keep
|
|
166
183
|
// exactly the keys it has today. `replica` is this job's 1-based index and `replicas` the set size;
|
|
167
184
|
// both are integers, so the run record they land in stays PII-free by construction.
|
|
168
185
|
...(replica !== undefined && { replica }),
|
|
169
186
|
...(replicas !== undefined && { replicas }),
|
|
170
187
|
};
|
|
188
|
+
// A close-triggered job (issue #231) leads the semantic key's flow slot with `closed:`. Without it,
|
|
189
|
+
// a label/comment/PR job on the same target and flow inside the 10-minute window silently swallows
|
|
190
|
+
// the close job -- and because a swallowed close job writes no run record, the once trigger it was
|
|
191
|
+
// meant to spend never disarms: a permanently dead one-shot with nothing in the panel to say why.
|
|
192
|
+
// The discriminant is DERIVED from the matched rule (`issue` type, or a PR close action word) rather
|
|
193
|
+
// than carried as a job field: an execution detail of dedup is not a fact about the delivery, and
|
|
194
|
+
// `data`/`event.json` stay byte-identical. `:` is outside the skill-name charset -- enforced at load
|
|
195
|
+
// since #231 -- so no real flow can spell either prefixed form, and `closed:cmd:<name>` composes for
|
|
196
|
+
// close-dispatched commands (outermost discriminant first, then the entry-point prefix).
|
|
197
|
+
const matched = trigger?.matched;
|
|
198
|
+
// `type === "issue"` reads as "close" only while the issue vocabulary is close-only (it is; the
|
|
199
|
+
// tables say "one word each so far"). If that type ever grows a non-close action, this test must
|
|
200
|
+
// narrow to the matched action word, like the PR half already does.
|
|
201
|
+
const isCloseJob = matched?.type === "issue" || (matched?.type === "pull_request" && PR_CLOSE_WORDS.has(matched?.action));
|
|
202
|
+
const flowSlot = `${isCloseJob ? "closed:" : ""}${command !== undefined ? `cmd:${command}` : flow}`;
|
|
171
203
|
await queue.add(kind, data, {
|
|
172
204
|
jobId,
|
|
173
205
|
// A command job (issue #189) fills the semantic key's flow slot with `cmd:<command>`: a command
|
|
@@ -176,7 +208,7 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
|
|
|
176
208
|
// `cmd:` prefix keeps a command named X from coalescing against a flow named X -- `:` is outside
|
|
177
209
|
// the skill-name charset, so no real flow can spell the prefixed form -- and a flow job's key
|
|
178
210
|
// stays byte-identical to before the feature.
|
|
179
|
-
deduplication: { id: `${repo}${targetSeparator(kind, target?.type)}${target.number}:${
|
|
211
|
+
deduplication: { id: `${repo}${targetSeparator(kind, target?.type)}${target.number}:${flowSlot}${replica !== undefined ? `:r${replica}` : ""}`, ttl: SEMANTIC_WINDOW_MS }, // ttl in ms
|
|
180
212
|
attempts: 2,
|
|
181
213
|
backoff: { type: "exponential", delay: 60_000 },
|
|
182
214
|
removeOnComplete: { age: 31 * 24 * 3600 }, // age in seconds -- do not cross units with the ms ttl above
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The environment variable names `buildContainerEnv` writes itself, spelled once (issue #225).
|
|
3
|
+
*
|
|
4
|
+
* This exists because `run.secrets` lets a trigger name env variables, and a trigger that names one the
|
|
5
|
+
* closed map already owns would either be silently overwritten (the job runs without the value it asked
|
|
6
|
+
* for, on a clean exit 0) or silently WIN (a trigger redirecting `PI_OFFLINE` or `PI_MAX_TURNS`). Both
|
|
7
|
+
* are the inversion `env-allowlist.mjs`'s header exists to prevent, arriving through a new door.
|
|
8
|
+
*
|
|
9
|
+
* A SEPARATE MODULE, and it has no imports at all, on purpose. `triggers.mjs` is the shared validator:
|
|
10
|
+
* it is pure and fs-free, the receiver loads it, and `admin/build.mjs` INLINES it into the published
|
|
11
|
+
* console. Importing `env-allowlist.mjs` to reach these names would drag `node:fs`, `node:os` and pi's
|
|
12
|
+
* compat shim into all three, to read a list of strings. So the list moves down here, where both can
|
|
13
|
+
* have it for free.
|
|
14
|
+
*
|
|
15
|
+
* Only the STATIC names live here. The rest of the closed map is deployment state and cannot be known
|
|
16
|
+
* from a triggers file at all: the provider credential's variable names come from `findEnvKeys(provider,
|
|
17
|
+
* hostEnv)`, and `PI_FORWARD_ENV` is an operator env list. Those two are refused PRE-SPEND, in the
|
|
18
|
+
* processor, where the resolved provider and the host env are both in hand. `MINTED_TOKEN_VARS` and
|
|
19
|
+
* `FORGE_HOST_VARS` (forges.mjs) and `EGRESS_ENV_VARS`/`WORKER_ONLY_SECRET_VARS` (config.mjs) stay in
|
|
20
|
+
* their own modules and are imported by the validator beside this one, never copied into it.
|
|
21
|
+
*
|
|
22
|
+
* `worker/test/env-allowlist.test.mjs` pins this set against what `buildContainerEnv` actually emits, so
|
|
23
|
+
* a variable added to the closed map and not to this list fails there rather than becoming a hole here.
|
|
24
|
+
*/
|
|
25
|
+
export const CONTAINER_ENV_NAMES = new Set([
|
|
26
|
+
"PI_PROVIDER",
|
|
27
|
+
"PI_MODEL",
|
|
28
|
+
"PI_MAX_TURNS",
|
|
29
|
+
"PI_MAX_TOKENS",
|
|
30
|
+
"PI_JOB_ID",
|
|
31
|
+
"PI_GLOBAL_ALLOW_EXTENSIONS",
|
|
32
|
+
"PI_PACKAGES",
|
|
33
|
+
"PI_SESSION_FILE",
|
|
34
|
+
"PI_FLOW",
|
|
35
|
+
"PI_COMMAND",
|
|
36
|
+
"PI_OFFLINE",
|
|
37
|
+
"PLAYWRIGHT_BROWSERS_PATH",
|
|
38
|
+
"PLAYWRIGHT_MCP_BROWSER",
|
|
39
|
+
"PLAYWRIGHT_MCP_SANDBOX",
|
|
40
|
+
]);
|
package/src/run-container.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import { InfraRetry } from "./processor.mjs";
|
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* The real `runContainer` the processor injects. Launches one job container and returns
|
|
10
|
-
* `{ code, aborted, turns, tokens, session, usage }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
|
|
10
|
+
* `{ code, aborted, turns, tokens, session, usage, context }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
|
|
11
11
|
* the 30-min timeout or graceful shutdown), which the processor classifies as POLICY (no retry) per
|
|
12
12
|
* INT-RUNNER-EXIT-CODE-PROTOCOL. The numeric `code` alone cannot say this: a worker SIGKILL and a
|
|
13
13
|
* kernel OOM both surface as 137, so the abort FLAG -- not the code -- is the discriminator.
|
|
@@ -46,7 +46,7 @@ export function makeRunContainer({
|
|
|
46
46
|
}) {
|
|
47
47
|
// async so a synchronous throw (e.g. buildContainerEnv on an unconfigured provider) surfaces as
|
|
48
48
|
// a rejection, uniformly awaitable by the processor and by tests.
|
|
49
|
-
return async function runContainer({ job, token, prepared, name, signal }) {
|
|
49
|
+
return async function runContainer({ job, token, prepared, secrets = {}, name, signal }) {
|
|
50
50
|
if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null }; // killed before it could start
|
|
51
51
|
|
|
52
52
|
// Closed env allowlist: only the provider key + the declared PI_* vars. Throws (config) if
|
|
@@ -87,6 +87,10 @@ export function makeRunContainer({
|
|
|
87
87
|
// mutually exclusive with it by parse -- a job carries one or the other, never both.
|
|
88
88
|
command: typeof job.command === "string" && job.command.trim() !== "" ? job.command : undefined,
|
|
89
89
|
authFromPi, // source the provider key from pi's auth.json when the env has none
|
|
90
|
+
// REQ-TRIGGER-SECRETS: this trigger's resolved secrets, fetched by the processor BEFORE anything
|
|
91
|
+
// spent. Off the call bag rather than off `job` or the closure: it is neither a per-job fact the
|
|
92
|
+
// record may carry nor a deployment setting, it is a live credential, and `token` is its precedent.
|
|
93
|
+
secrets,
|
|
90
94
|
});
|
|
91
95
|
|
|
92
96
|
// `-net` on this container's own name (egress.mjs). null when no policy is armed, and docker-run's
|
package/src/run-history.mjs
CHANGED
|
@@ -34,13 +34,57 @@ export function sanitizeJobId(id) {
|
|
|
34
34
|
return s.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Parse one line of the buffered tail as a runner-emitted JSON line, or `null` for noise -- repairing
|
|
39
|
+
* the GLUED case (issue #224, OQ-003).
|
|
40
|
+
*
|
|
41
|
+
* The hazard: the runner's writes used to be newline-terminated but not newline-delimited, so a
|
|
42
|
+
* partial write from anything sharing the container's stdout (a subprocess that never flushed its
|
|
43
|
+
* trailing newline) lands as `<stray bytes><runner line>` in ONE line. A plain JSON.parse skips it,
|
|
44
|
+
* and because every parseExit* scan walks backwards past what does not parse, one un-newlined byte
|
|
45
|
+
* used to lose turns, tokens, usage, session and context at once -- or worse, hand the scan to a
|
|
46
|
+
* forged exit line placed earlier.
|
|
47
|
+
*
|
|
48
|
+
* The repair re-anchors on `{"event":"`, which is collision-free by construction: `event` is the
|
|
49
|
+
* first key both runner writers serialise, and JSON.stringify escapes every quote inside a string
|
|
50
|
+
* value, so these raw bytes cannot occur INSIDE a runner line -- only where one starts. Suffixes are
|
|
51
|
+
* tried left to right, so the leftmost complete object wins: a suffix beginning inside the stray
|
|
52
|
+
* bytes cannot parse to the line's end (nothing can close a JSON container after bytes the runner
|
|
53
|
+
* appended later, and the runner's own quotes terminate any string opened before them), so the first
|
|
54
|
+
* success is the glued runner object itself -- and this left-to-right scan, not just the quote
|
|
55
|
+
* escaping, is what keeps a would-be inner `{"event":"` from being chosen over the outer one.
|
|
56
|
+
* A line whose HEAD the capped tail sliced off is handled correctly either way: if the cut fell in a
|
|
57
|
+
* glued line's stray PREFIX the anchor survives and the genuine object is still repaired, and if it
|
|
58
|
+
* fell in or past the anchor no `{"event":"` survives and the line is skipped. A line truncated at the
|
|
59
|
+
* END (a mid-write death) has no complete object and is skipped too. A fragment is never MISREAD as a
|
|
60
|
+
* value: a broken one does not parse and an anchorless one is not repaired.
|
|
61
|
+
*
|
|
62
|
+
* NEVER throws, like the five scanners that call it.
|
|
63
|
+
*/
|
|
64
|
+
function parseTailLine(line) {
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse(line);
|
|
67
|
+
} catch {
|
|
68
|
+
// docker/agent noise, a truncated line, or a glued one -- try the repair before giving up.
|
|
69
|
+
}
|
|
70
|
+
let from = line.indexOf('{"event":"', 1);
|
|
71
|
+
while (from !== -1) {
|
|
72
|
+
try {
|
|
73
|
+
return JSON.parse(line.slice(from));
|
|
74
|
+
} catch {
|
|
75
|
+
from = line.indexOf('{"event":"', from + 1);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
|
|
37
81
|
/**
|
|
38
82
|
* Recover the agent's turn count from buffered container stdout, or `null` if it is not reported.
|
|
39
83
|
*
|
|
40
84
|
* The stream interleaves docker/agent noise and other JSON events (`pi_auto_retry`) with the runner's
|
|
41
85
|
* own lines. Only the success exit line carries `turns` (`image/runner/run-job.mjs:263`); the
|
|
42
86
|
* catch-path exit line (`:277`) omits it. Scan from the end and return the turns of the last `exit`
|
|
43
|
-
* event that reports an integer count.
|
|
87
|
+
* event that reports an integer count, repairing a glued line on the way (`parseTailLine`).
|
|
44
88
|
*
|
|
45
89
|
* This is read-only telemetry: it MUST NEVER throw and MUST NOT feed exit-code or retry
|
|
46
90
|
* classification -- that is the container exit code's job (INT-RUNNER-EXIT-CODE-PROTOCOL). Every parse
|
|
@@ -52,12 +96,7 @@ export function parseExitTurns(text) {
|
|
|
52
96
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
53
97
|
const line = lines[i].trim();
|
|
54
98
|
if (line === "") continue;
|
|
55
|
-
|
|
56
|
-
try {
|
|
57
|
-
parsed = JSON.parse(line);
|
|
58
|
-
} catch {
|
|
59
|
-
continue; // docker/agent noise or a truncated final line
|
|
60
|
-
}
|
|
99
|
+
const parsed = parseTailLine(line);
|
|
61
100
|
if (parsed?.event !== "exit") continue;
|
|
62
101
|
return Number.isInteger(parsed?.turns) ? parsed.turns : null;
|
|
63
102
|
}
|
|
@@ -94,12 +133,7 @@ export function parseExitSession(text) {
|
|
|
94
133
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
95
134
|
const line = lines[i].trim();
|
|
96
135
|
if (line === "") continue;
|
|
97
|
-
|
|
98
|
-
try {
|
|
99
|
-
parsed = JSON.parse(line);
|
|
100
|
-
} catch {
|
|
101
|
-
continue; // docker/agent noise or a truncated final line
|
|
102
|
-
}
|
|
136
|
+
const parsed = parseTailLine(line);
|
|
103
137
|
if (parsed?.event !== "exit") continue;
|
|
104
138
|
const sess = parsed?.session;
|
|
105
139
|
if (sess && typeof sess === "object" && !Array.isArray(sess) && typeof sess.resumed === "boolean") {
|
|
@@ -126,12 +160,7 @@ export function parseExitContext(text) {
|
|
|
126
160
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
127
161
|
const line = lines[i].trim();
|
|
128
162
|
if (line === "") continue;
|
|
129
|
-
|
|
130
|
-
try {
|
|
131
|
-
parsed = JSON.parse(line);
|
|
132
|
-
} catch {
|
|
133
|
-
continue; // docker/agent noise or a truncated final line
|
|
134
|
-
}
|
|
163
|
+
const parsed = parseTailLine(line);
|
|
135
164
|
if (parsed?.event !== "exit") continue;
|
|
136
165
|
const c = parsed?.context;
|
|
137
166
|
// A window of 0 is not a denominator, and a negative count is not a measurement. SAFE integers
|
|
@@ -153,12 +182,7 @@ export function parseExitTokens(text) {
|
|
|
153
182
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
154
183
|
const line = lines[i].trim();
|
|
155
184
|
if (line === "") continue;
|
|
156
|
-
|
|
157
|
-
try {
|
|
158
|
-
parsed = JSON.parse(line);
|
|
159
|
-
} catch {
|
|
160
|
-
continue; // docker/agent noise or a truncated final line
|
|
161
|
-
}
|
|
185
|
+
const parsed = parseTailLine(line);
|
|
162
186
|
if (parsed?.event !== "exit") continue;
|
|
163
187
|
const t = parsed?.tokens;
|
|
164
188
|
if (t && typeof t === "object" && !Array.isArray(t) && typeof t.total === "number") return t;
|
|
@@ -204,12 +228,7 @@ export function parseExitUsage(text) {
|
|
|
204
228
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
205
229
|
const line = lines[i].trim();
|
|
206
230
|
if (line === "") continue;
|
|
207
|
-
|
|
208
|
-
try {
|
|
209
|
-
parsed = JSON.parse(line);
|
|
210
|
-
} catch {
|
|
211
|
-
continue; // docker/agent noise or a truncated final line
|
|
212
|
-
}
|
|
231
|
+
const parsed = parseTailLine(line);
|
|
213
232
|
if (parsed?.event !== "exit") continue;
|
|
214
233
|
return rebuildUsage(parsed?.usage);
|
|
215
234
|
}
|
package/src/runtime-settings.mjs
CHANGED
|
@@ -27,6 +27,22 @@ import { defaultSettingsFile } from "./config.mjs";
|
|
|
27
27
|
|
|
28
28
|
export const KNOWN_KEYS = ["model", "provider", "maxTurns", "dailyCap", "weeklyCap", "monthlyCap", "maxTokens", "dailyTokenCap", "concurrency", "softHoldPct"];
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Overlay keys the OVERLAY accepts but no model-callable tool may set. `secretProfiles` maps a profile name
|
|
32
|
+
* to an absolute path to a script the worker executes, which is plainly "a capability the model would
|
|
33
|
+
* GAIN" -- the test `admin/src/index.ts` already applies to `run.image` and `run.resume`.
|
|
34
|
+
*
|
|
35
|
+
* The mechanism is the OMISSION from KNOWN_KEYS above: `dispatch_set` narrows its open string parameter
|
|
36
|
+
* with `KNOWN_KEYS.includes(...)`, and the panel's generic settings dialog is a `ui.select` over the same
|
|
37
|
+
* array, so leaving the key out closes both doors at once. This constant exists so that omission has a NAME
|
|
38
|
+
* and a reason attached to it. Without one, `KNOWN_KEYS` silently stops meaning "keys the overlay accepts"
|
|
39
|
+
* and the next contributor tidies the gap away as an oversight.
|
|
40
|
+
*
|
|
41
|
+
* A profile is declared through the operator-typed `/dispatch` command instead, which pi reaches only from
|
|
42
|
+
* its own user-input path, and every declared path is bounded by PI_SECRET_RESOLVER_ROOTS in the worker.
|
|
43
|
+
*/
|
|
44
|
+
export const OPERATOR_ONLY_KEYS = ["secretProfiles"];
|
|
45
|
+
|
|
30
46
|
function isNonEmptyString(value) {
|
|
31
47
|
return typeof value === "string" && value.trim() !== "";
|
|
32
48
|
}
|
|
@@ -85,6 +101,30 @@ function validateOverlay(candidate, log) {
|
|
|
85
101
|
if (!isIntInRange(value, 1, 10)) return { invalid: "concurrency must be an integer 1-10" };
|
|
86
102
|
overlay[key] = value;
|
|
87
103
|
break;
|
|
104
|
+
case "secretProfiles": {
|
|
105
|
+
// REQ-TRIGGER-SECRETS. `{ [name]: "/abs/path" }`, the panel-authored half of the resolver
|
|
106
|
+
// table. The shape is checked here and the PATH BOUND is not: whether a path is allowed is
|
|
107
|
+
// PI_SECRET_RESOLVER_ROOTS' question, and it is asked in the worker at resolution time rather
|
|
108
|
+
// than here, because this same validator runs inside the admin extension where a "yes" would
|
|
109
|
+
// prove nothing about the host the job will run on.
|
|
110
|
+
//
|
|
111
|
+
// An invalid value fails the WHOLE overlay, which is this function's documented rule and is
|
|
112
|
+
// deliberate here too: a half-read profile table is a deployment that believes it has a
|
|
113
|
+
// resolver it does not.
|
|
114
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
115
|
+
return { invalid: "secretProfiles must be an object mapping profile names to resolver paths" };
|
|
116
|
+
}
|
|
117
|
+
for (const name of Object.keys(value)) {
|
|
118
|
+
if (!/^[A-Za-z0-9._-]+$/.test(name)) {
|
|
119
|
+
return { invalid: "secretProfiles names may use letters, digits, dot, dash and underscore only" };
|
|
120
|
+
}
|
|
121
|
+
if (typeof value[name] !== "string" || value[name].trim() === "") {
|
|
122
|
+
return { invalid: `secretProfiles.${name} must be a non-empty absolute path` };
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
overlay[key] = { ...value };
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
88
128
|
case "softHoldPct":
|
|
89
129
|
// A percentage of each active cap; 100 would equal the hard wall (no band) and 0 has no meaning,
|
|
90
130
|
// so the enforced band is 1-99. Absence disables the soft-hold entirely.
|
package/src/schedules.mjs
CHANGED
|
@@ -74,7 +74,7 @@ function normalizeCronSchedule({ on, run }, path, existsSync) {
|
|
|
74
74
|
// key. A command trigger carries no flow/task at all (the validator enforces the XOR), so those two
|
|
75
75
|
// keys hold undefined here and drop at JSON serialization -- the command schedule's data is exactly
|
|
76
76
|
// kind/folder/command plus the shared fields.
|
|
77
|
-
const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
|
|
77
|
+
const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
|
|
78
78
|
// Retention only; the deterministic repeat:<id>:<millis> jobId supplies dedup, so no jobId here, and
|
|
79
79
|
// scheduler jobs are not retried (DES-CRON-VIA-BULLMQ-SCHEDULER) so no attempts/backoff.
|
|
80
80
|
const opts = { removeOnComplete: { age: 24 * 3600 }, removeOnFail: { age: 7 * 24 * 3600 } };
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The operator's declared secret-resolver profiles: parsing, merging, and the path bound (issue #225).
|
|
3
|
+
*
|
|
4
|
+
* A PROFILE is a name and an absolute path to a script the operator wrote. A trigger names the NAME; it
|
|
5
|
+
* can never name the path. That distinction is the whole reason `run.secretsProfile` is allowed to exist:
|
|
6
|
+
* `DES-SERVICE-ENV-SETUP-SEAM` rejected "making this reachable from configuration, which would turn a
|
|
7
|
+
* boot-time root-adjacent exec into something a trigger file could name", and selecting among execs the
|
|
8
|
+
* operator already declared is not naming one.
|
|
9
|
+
*
|
|
10
|
+
* PURE AND FS-FREE, like `triggers.mjs` and for a weaker but real version of its reason: `config.mjs`
|
|
11
|
+
* imports this module, `runtime-settings.mjs` imports `config.mjs`, and `admin/build.mjs` inlines that
|
|
12
|
+
* chain into the published console. Whether a path EXISTS is asked once, in `secrets.mjs`, on the worker
|
|
13
|
+
* that is about to spawn it.
|
|
14
|
+
*
|
|
15
|
+
* `configError` is a local copy rather than an import from `config.mjs`, which imports this module: the
|
|
16
|
+
* cycle would resolve (function declarations hoist) but would be a trap for the next reader. The same
|
|
17
|
+
* duplication, for a related reason, is in `env-allowlist.mjs`.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { isAbsolute, normalize, sep } from "node:path";
|
|
21
|
+
|
|
22
|
+
function configError(message) {
|
|
23
|
+
const error = new Error(message);
|
|
24
|
+
error.piDispatchConfig = true;
|
|
25
|
+
return error;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* A profile name. Deliberately the same charset `triggers.mjs` allows in `run.secretsProfile`, and it has
|
|
30
|
+
* to be: a name that fails here after passing there would be an operator-visible contradiction between two
|
|
31
|
+
* files that are meant to agree. Excluding `,` and `:` is load-bearing rather than tidy, since those are
|
|
32
|
+
* this variable's own separators -- a name carrying either could not round-trip through its declaration.
|
|
33
|
+
*/
|
|
34
|
+
const PROFILE_NAME = /^[A-Za-z0-9._-]+$/;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Parse `PI_SECRET_PROFILES`: `name:/abs/path,other:/abs/other`. Returns `{ [name]: path }`, empty when
|
|
38
|
+
* unset. Throws (config-tagged, so the worker refuses to boot) on anything malformed.
|
|
39
|
+
*
|
|
40
|
+
* SET-BUT-GARBLED FAILS LOUD, which is `parsePollRepos`' doctrine and matters more here: a silently
|
|
41
|
+
* dropped entry is a profile the operator believes is wired, and every trigger naming it would refuse at
|
|
42
|
+
* delivery time with the operator looking at a line that appears to declare it.
|
|
43
|
+
*
|
|
44
|
+
* Each entry splits on its FIRST colon, so `prod:C:\pi\resolve.cmd` parses on Windows. That is the same
|
|
45
|
+
* drive-letter hazard `config.mjs`'s `delimitedList` exists for, arriving from the other side: there the
|
|
46
|
+
* colon must not be read as a separator, here exactly one of them must be.
|
|
47
|
+
*/
|
|
48
|
+
export function parseSecretProfiles(raw) {
|
|
49
|
+
if (raw === undefined || raw === null || String(raw).trim() === "") return {};
|
|
50
|
+
const profiles = {};
|
|
51
|
+
for (const entry of String(raw).split(",")) {
|
|
52
|
+
const text = entry.trim();
|
|
53
|
+
if (text === "") continue; // a trailing comma is a typo, not a declaration
|
|
54
|
+
const cut = text.indexOf(":");
|
|
55
|
+
if (cut <= 0) {
|
|
56
|
+
throw configError(`PI_SECRET_PROFILES entries must be name:/absolute/path, got ${JSON.stringify(text)}`);
|
|
57
|
+
}
|
|
58
|
+
const name = text.slice(0, cut).trim();
|
|
59
|
+
const path = text.slice(cut + 1).trim();
|
|
60
|
+
if (!PROFILE_NAME.test(name)) {
|
|
61
|
+
throw configError(`PI_SECRET_PROFILES profile name ${JSON.stringify(name)} may use letters, digits, dot, dash and underscore only`);
|
|
62
|
+
}
|
|
63
|
+
if (name in profiles) {
|
|
64
|
+
throw configError(`PI_SECRET_PROFILES declares ${JSON.stringify(name)} twice -- one of the two is not the resolver you think is running`);
|
|
65
|
+
}
|
|
66
|
+
if (path === "" || !isAbsolutePath(path)) {
|
|
67
|
+
throw configError(`PI_SECRET_PROFILES profile ${JSON.stringify(name)} needs an ABSOLUTE path to its resolver -- a service manager's working directory is not your shell's, so a relative path is a different file on every host`);
|
|
68
|
+
}
|
|
69
|
+
profiles[name] = path;
|
|
70
|
+
}
|
|
71
|
+
return profiles;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Absolute on this platform, accepting a Windows drive letter or UNC root as `service.mjs` does. */
|
|
75
|
+
function isAbsolutePath(path) {
|
|
76
|
+
return isAbsolute(path) || /^([A-Za-z]:[\\/]|\\\\)/.test(path);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Union the env-declared profiles with the overlay-declared ones, refusing a name that appears in both.
|
|
81
|
+
*
|
|
82
|
+
* NEITHER SOURCE WINS, and that is a deliberate third answer. `runtime-settings.mjs` documents the
|
|
83
|
+
* overlay's precedence as `overlay > env`, so quietly inverting it for this one key would leave two rules
|
|
84
|
+
* in the codebase disagreeing about what an overlay is. But honouring it would let a settings file -- which
|
|
85
|
+
* defaults into the OS temp directory -- redirect a profile the operator wrote in `.env`. So a collision is
|
|
86
|
+
* refused instead, per delivery and naming only the profile name. This project already refuses ambiguity
|
|
87
|
+
* rather than resolving it: `PI_EGRESS` refuses any value but 0 or 1 because "a typo must never leave you
|
|
88
|
+
* believing you have a policy you do not", and two declarations of one profile is exactly that.
|
|
89
|
+
*
|
|
90
|
+
* Returns `{ profiles }` or `{ ambiguous }` naming the first colliding profile.
|
|
91
|
+
*/
|
|
92
|
+
export function mergeSecretProfiles(envProfiles = {}, overlayProfiles = {}) {
|
|
93
|
+
for (const name of Object.keys(overlayProfiles)) {
|
|
94
|
+
if (name in envProfiles) return { ambiguous: name };
|
|
95
|
+
}
|
|
96
|
+
return { profiles: { ...envProfiles, ...overlayProfiles } };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Whether `candidate` sits inside one of `roots`. Empty roots means NOTHING passes.
|
|
101
|
+
*
|
|
102
|
+
* FAIL-CLOSED BY DEFAULT is the whole design: with `PI_SECRET_RESOLVER_ROOTS` unset, an overlay-declared
|
|
103
|
+
* profile resolves to no usable path, so the panel can declare nothing and the deployment is env-only. The
|
|
104
|
+
* operator opts in to panel authoring by naming the directory their resolvers already live in. This is
|
|
105
|
+
* `PI_DISPATCH_RUN_ROOTS`' shape, and `DES-PER-TRIGGER-JOB-IMAGE` predicted it: "If a future tool ever
|
|
106
|
+
* takes an image parameter, the allowlist arrives with that tool, and this row is the reason it must."
|
|
107
|
+
*
|
|
108
|
+
* The comparison is on NORMALIZED paths with a separator boundary, so `/opt/pi-evil` does not pass for the
|
|
109
|
+
* root `/opt/pi`. Callers pass a path they have already realpath'd, because normalization alone cannot see
|
|
110
|
+
* through a symlink and this is a boundary, not a hint.
|
|
111
|
+
*/
|
|
112
|
+
export function withinRoots(candidate, roots = []) {
|
|
113
|
+
if (!Array.isArray(roots) || roots.length === 0) return false;
|
|
114
|
+
const target = normalize(candidate);
|
|
115
|
+
return roots.some((root) => {
|
|
116
|
+
const base = normalize(root).replace(new RegExp(`${sep === "\\" ? "\\\\" : sep}+$`), "");
|
|
117
|
+
return target === base || target.startsWith(base + sep);
|
|
118
|
+
});
|
|
119
|
+
}
|