@edgehero/pi-dispatch 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +11 -1
- package/deploy/worker-env-wrapper.cmd +8 -0
- package/deploy/worker-env-wrapper.sh +58 -5
- package/package.json +1 -1
- package/src/config.mjs +16 -0
- package/src/doctor.mjs +30 -1
- package/src/processor.mjs +24 -5
- package/src/run-container.mjs +10 -5
- package/src/run-history.mjs +40 -2
- package/src/session-store.mjs +294 -15
- package/src/start.mjs +3 -0
package/.env.example
CHANGED
|
@@ -56,10 +56,20 @@ PI_JOB_IMAGE=pi-job:latest # the DEFAULT job image. Any trigger may nam
|
|
|
56
56
|
# Enforced at OPEN as well as at boot: a stale transcript is a live input to a future job, not debris
|
|
57
57
|
# PI_SESSION_MAX_BYTES= # default 8388608 (8 MiB); a transcript larger than this is not resumed; 0 = no cap
|
|
58
58
|
# Not disk hygiene -- an oversized transcript is a prefill nobody sized PI_MAX_TOKENS for
|
|
59
|
+
# PI_SESSION_MAX_AGE_DAYS= # unset/0 = no bound. How old the CONVERSATION may be, read from the session header's own timestamp
|
|
60
|
+
# A DIFFERENT CLOCK from PI_SESSIONS_TTL_DAYS, not a finer setting of it: that one reads mtime, which every COMPLETED run refreshes,
|
|
61
|
+
# so a lineage that keeps finishing work never expires however old its first turn is. This one measures from the first turn
|
|
62
|
+
# A header with no readable timestamp is refused rather than assumed young (reason: conversation-too-old)
|
|
63
|
+
# PI_SESSION_MAX_RESUME_CHAIN= # unset/0 = no bound. How many times in a row one key may be resumed before the next job starts fresh
|
|
64
|
+
# The bound a long lineage actually needs: age and size grow slowly, a chain grows once per run
|
|
65
|
+
# The count is kept whether or not the bound is set, so setting it later takes effect on the next job rather than N jobs later
|
|
66
|
+
# PI_SESSION_MAX_CONTEXT_PCT= # unset = no bound; 1-100. Refuse a resume when the saved session's context is already this full, e.g. 80
|
|
67
|
+
# A SAFETY bound before an economic one: past pi's compaction threshold a resumed job replays a model-written summary of the transcript,
|
|
68
|
+
# written while that model was reading attacker-authored text (specs/open-questions.md, OQ-003). This ceiling is the host's own, and pi's threshold stays pi's
|
|
69
|
+
# The measurement comes from the job image's runner, so it is inert until you are running an image that reports it and each key has completed one run since
|
|
59
70
|
# PI_SESSIONS_ALLOW_GH_SOURCE= # unset = a run.resume job REFUSES to mint under GITHUB_AUTH_SOURCE=gh, pre-spend
|
|
60
71
|
# That source is your whole gh login: full-scope and non-expiring, and a transcript is a FILE -- any command that echoed an auth header persists it
|
|
61
72
|
# Prefer GITHUB_AUTH_SOURCE=app or a short-expiry fine-grained PAT. Set exactly 1 to accept the trade explicitly (SECURITY.md, docs/sessions.md)
|
|
62
|
-
# Not disk hygiene -- an oversized transcript is a prefill nobody sized PI_MAX_TOKENS for
|
|
63
73
|
# PI_TRIGGERS_FILE= # ABSOLUTE path to the unified triggers.json, read by BOTH worker and receiver (a relative path resolves against the service's WorkingDirectory).
|
|
64
74
|
# Unset = cron disabled for the worker; the receiver falls back to ./triggers.json in the folder it starts from (what `pi-dispatch init` scaffolds)
|
|
65
75
|
# and refuses to start when neither exists (it holds the label/comment/pull_request trigger config)
|
|
@@ -72,6 +72,14 @@ if defined ENV_SETUP (
|
|
|
72
72
|
)
|
|
73
73
|
)
|
|
74
74
|
|
|
75
|
+
REM WEAKER THAN THE .sh TWIN ON SIGNALS, deliberately and stated rather than discovered (issue #221).
|
|
76
|
+
REM cmd has no `trap`, so there is no wrapper-level handling of a stop that arrives while `.env` is being
|
|
77
|
+
REM read or while the setup script above is still running: whatever the service manager does to the tree
|
|
78
|
+
REM is what happens. nssm stops with a console event to the process group (AppStopMethodConsole), so the
|
|
79
|
+
REM worker is reached directly rather than through this file, which is why the .sh twin's forwarding
|
|
80
|
+
REM problem has no equivalent here. The asymmetry is recorded in DES-SERVICE-ENV-SETUP-SEAM and is not
|
|
81
|
+
REM closed.
|
|
82
|
+
REM
|
|
75
83
|
REM The argv runs verbatim -- absolute node, absolute script, composed by `pi-dispatch service` (see
|
|
76
84
|
REM the .sh twin for the whole contract).
|
|
77
85
|
%*
|
|
@@ -36,6 +36,34 @@ if [ "$#" -eq 0 ]; then
|
|
|
36
36
|
exit 1
|
|
37
37
|
fi
|
|
38
38
|
|
|
39
|
+
# STOP HANDLING IS ARMED HERE, above everything below that can block (issue #221). It closes two windows,
|
|
40
|
+
# both of which used to swallow a stop in silence.
|
|
41
|
+
#
|
|
42
|
+
# Until this line TERM/INT carry their DEFAULT disposition, and the sourcing below can take arbitrarily
|
|
43
|
+
# long: PI_ENV_SETUP is an operator's secrets manager, so docs/secrets.md's own worked example makes a
|
|
44
|
+
# network round trip inside it. A stop landing there killed this shell where it stood, mid-preparation,
|
|
45
|
+
# with nothing anywhere saying the environment had been half-built. That is reachable from this project's
|
|
46
|
+
# own CLI, not just from the daemon: `pi-dispatch service stop` on macOS is `launchctl kill SIGTERM` at
|
|
47
|
+
# this pid.
|
|
48
|
+
#
|
|
49
|
+
# The other window is two instructions wide, and is closed by the re-send after `child=$!` below. The
|
|
50
|
+
# handler is a FUNCTION rather than a trap string because it is installed twice -- here, and again after
|
|
51
|
+
# the sourcing -- and one behaviour spelled out in two places is one behaviour that can drift.
|
|
52
|
+
signaled=0
|
|
53
|
+
child=
|
|
54
|
+
wrapper_on_stop() {
|
|
55
|
+
signaled=1
|
|
56
|
+
# `child` is empty until the fork below has been assigned, and `kill -TERM ""` kills nothing and
|
|
57
|
+
# fails silently, so a stop arriving before then has no pid to reach. It is not lost: the re-send
|
|
58
|
+
# after `child=$!` re-delivers it, and the launch gate refuses to start at all if nothing was
|
|
59
|
+
# started yet.
|
|
60
|
+
[ -n "$child" ] && kill -TERM "$child" 2>/dev/null
|
|
61
|
+
# Never leave a nonzero status behind. `rc=$?` is read immediately after the `wait` this interrupts,
|
|
62
|
+
# and the double wait at the bottom keys on rc >= 128.
|
|
63
|
+
return 0
|
|
64
|
+
}
|
|
65
|
+
trap wrapper_on_stop TERM INT
|
|
66
|
+
|
|
39
67
|
# The env-setup seam (issue #209): `pi-dispatch service render|install --env-setup <path>` puts an
|
|
40
68
|
# operator-typed path here -- the plist's EnvironmentVariables dict on macOS, nssm's AppEnvironmentExtra
|
|
41
69
|
# on Windows -- so a secrets manager can fill this process's environment without anyone hand-editing a
|
|
@@ -74,6 +102,25 @@ if [ -n "$env_setup" ]; then
|
|
|
74
102
|
set +a
|
|
75
103
|
fi
|
|
76
104
|
|
|
105
|
+
# RE-ASSERTED after the sourcing, and this is not belt-and-braces. A sourced script runs in THIS shell,
|
|
106
|
+
# so a `trap ... TERM` inside one REPLACES the handler above and the drain silently disappears -- a
|
|
107
|
+
# manager's cleanup helper does exactly that. One line restores it. What it cannot undo is a script that
|
|
108
|
+
# IGNORES TERM (`trap '' TERM`): a signal discarded while it was ignored is already gone, and the child
|
|
109
|
+
# forked below would inherit SIG_IGN and be unable to trap TERM at all. That is why docs/secrets.md now
|
|
110
|
+
# tells operators not to touch signals in a setup script.
|
|
111
|
+
trap wrapper_on_stop TERM INT
|
|
112
|
+
|
|
113
|
+
# A stop that arrived while the environment was being prepared is honoured by NOT STARTING. Launching now
|
|
114
|
+
# would hand the service manager a worker it has already asked to go away: it would reserve a budget slot
|
|
115
|
+
# and take a job, and then need a drain nobody is waiting for. Exit 0 because 0 is the only code launchd's
|
|
116
|
+
# KeepAlive/SuccessfulExit=false leaves stopped -- the same reason the exit-2 conversion at the bottom
|
|
117
|
+
# exists. Not 2, because nothing was refused; not 1, because nothing failed; the manager's own instruction
|
|
118
|
+
# was carried out, and this says so rather than exiting mute.
|
|
119
|
+
if [ "$signaled" -eq 1 ]; then
|
|
120
|
+
echo "worker-env-wrapper: stopped before the worker started -- a stop signal arrived while the environment was being prepared, so the command was never launched; exiting 0 (nothing to restart)" >&2
|
|
121
|
+
exit 0
|
|
122
|
+
fi
|
|
123
|
+
|
|
77
124
|
# `exec` is deliberately GONE here (it used to hand this shell's pid straight to node): intercepting
|
|
78
125
|
# the exit code needs a parent still alive after node exits. launchd's KeepAlive/SuccessfulExit=false
|
|
79
126
|
# relaunches ANY nonzero exit -- including EXIT_POLICY (2, worker/src/exit-code.mjs), the determinate
|
|
@@ -81,13 +128,19 @@ fi
|
|
|
81
128
|
# deliberately never retry. A relaunch loop against a paid provider is a bill, so the conversion at
|
|
82
129
|
# the bottom turns exit 2 into the clean exit KeepAlive leaves stopped.
|
|
83
130
|
#
|
|
84
|
-
# SIGTERM still reaches node without exec: the
|
|
85
|
-
# a foreground command in sh, which blocks trap delivery) is interruptible by a trapped
|
|
86
|
-
# forwarding is immediate and node gets its full graceful drain.
|
|
87
|
-
signaled=0
|
|
88
|
-
trap 'signaled=1; kill -TERM "$child" 2>/dev/null' TERM INT
|
|
131
|
+
# SIGTERM still reaches node without exec: the handler armed at the top forwards TERM/INT to the child,
|
|
132
|
+
# and `wait` (unlike a foreground command in sh, which blocks trap delivery) is interruptible by a trapped
|
|
133
|
+
# signal, so the forwarding is immediate and node gets its full graceful drain.
|
|
89
134
|
"$@" &
|
|
90
135
|
child=$!
|
|
136
|
+
# THE FORK WINDOW (issue #221). `$!` is only readable in the parent AFTER the fork, so between the two
|
|
137
|
+
# lines above a child exists and its pid does not. A stop landing there ran the handler with nothing to
|
|
138
|
+
# forward to, set `signaled`, and was then never looked at again -- so this wrapper waited out the
|
|
139
|
+
# command's ENTIRE natural lifetime while the service manager believed it had asked it to stop. Re-sending
|
|
140
|
+
# once the pid is known costs one `[` on the healthy path and is the whole difference between a graceful
|
|
141
|
+
# drain and a hang as long as the job. Issue #207 found this same drop through the test that saw it and
|
|
142
|
+
# fixed only the test; #221 is the same window firing through a different one.
|
|
143
|
+
[ "$signaled" -eq 1 ] && kill -TERM "$child" 2>/dev/null
|
|
91
144
|
wait "$child"
|
|
92
145
|
rc=$?
|
|
93
146
|
# The double wait is load-bearing: a trapped signal interrupts the FIRST wait early (rc = 128+signum)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@edgehero/pi-dispatch",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Self-hosted job harness for the pi coding agent: a BullMQ worker that drains the queue, mints scoped forge tokens, and runs one container per job — plus the pi-dispatch CLI (init, up, doctor, service).",
|
|
6
6
|
"keywords": [
|
package/src/config.mjs
CHANGED
|
@@ -258,6 +258,22 @@ export function loadConfig(env = process.env, { fileExists = existsSync } = {})
|
|
|
258
258
|
// A bound on how large a transcript may be before it stops being resumed. Not disk hygiene: an
|
|
259
259
|
// oversized transcript is a prefill an operator never sized PI_MAX_TOKENS for.
|
|
260
260
|
sessionMaxBytes: nonNegativeInt(env, "PI_SESSION_MAX_BYTES", 8 * 1024 * 1024), // 0 = no cap
|
|
261
|
+
// How old the CONVERSATION may be, which is a different clock from sessionsTtlDays above and not a
|
|
262
|
+
// finer setting of it: the TTL reads the transcript's mtime, which the PROMOTE rename refreshes (the
|
|
263
|
+
// resolve copy does not, measured -- copyFileSync stamps the destination), so it measures time since
|
|
264
|
+
// the last COMPLETED run on this key. A key whose runs keep completing never expires however old its
|
|
265
|
+
// first turn is. This one reads the session header's own timestamp.
|
|
266
|
+
// OFF by default (0) rather than defaulted to a number: an age an operator did not choose is an
|
|
267
|
+
// opinion about their lineages that this project has no basis for.
|
|
268
|
+
sessionMaxAgeDays: nonNegativeInt(env, "PI_SESSION_MAX_AGE_DAYS", 0), // 0 = no age bound
|
|
269
|
+
// How many times in a row one key may be resumed before the next job starts fresh. The bound a long
|
|
270
|
+
// lineage actually needs: age and size both grow slowly while a chain grows once per run.
|
|
271
|
+
sessionMaxResumeChain: nonNegativeInt(env, "PI_SESSION_MAX_RESUME_CHAIN", 0), // 0 = no chain bound
|
|
272
|
+
// How full the saved context may be before a resume is refused, as a percentage of the model's own
|
|
273
|
+
// window. A PERCENTAGE, so `optionalBoundedInt` on softHoldPct's precedent rather than the 0 = off
|
|
274
|
+
// sentinel its two neighbours use: 0% would mean "never resume anything", which is a different
|
|
275
|
+
// request from "no bound", and 101 is a typo rather than a ceiling.
|
|
276
|
+
sessionMaxContextPct: optionalBoundedInt(env, "PI_SESSION_MAX_CONTEXT_PCT", 1, 100), // null = no context bound
|
|
261
277
|
chainDepthMax: nonNegativeInt(env, "PI_CHAIN_DEPTH_MAX", CHAIN_DEPTH_MAX_DEFAULT), // DES-JOB-OUTBOX-CHAINING; 0 = chaining kill-switch (fail-closed)
|
|
262
278
|
chainMaxPerJob: nonNegativeInt(env, "PI_CHAIN_MAX_PER_JOB", CHAIN_MAX_PER_JOB_DEFAULT), // INT-OUTBOX-CONTRACT: max request-<n>.json collected per parent
|
|
263
279
|
dispatchRunPerHour: nonNegativeInt(env, "PI_DISPATCH_RUN_PER_HOUR", 3), // DES-ADMIN-VIA-PI-EXTENSION; 0 = disable dispatch_run
|
package/src/doctor.mjs
CHANGED
|
@@ -1132,8 +1132,37 @@ export async function collectChecks(env, seams) {
|
|
|
1132
1132
|
ok: true,
|
|
1133
1133
|
warn: true,
|
|
1134
1134
|
label: `${resuming} trigger(s) persist agent transcripts to ${sessionsDir} -- PII-bearing, host-only, never committed`,
|
|
1135
|
-
fix: "confirm it is outside every git repo and on a disk you would put issue text on; PI_SESSIONS_TTL_DAYS
|
|
1135
|
+
fix: "confirm it is outside every git repo and on a disk you would put issue text on; PI_SESSIONS_TTL_DAYS, PI_SESSION_MAX_AGE_DAYS, PI_SESSION_MAX_RESUME_CHAIN and PI_SESSION_MAX_CONTEXT_PCT each bound a different thing about how much history one key accumulates (docs/sessions.md)",
|
|
1136
1136
|
});
|
|
1137
|
+
// Which of the four bounds are actually on, as a FACT LINE rather than a warning: how long a
|
|
1138
|
+
// lineage may run is an operator's call, not a defect, and doctor's warnings are for things that
|
|
1139
|
+
// need a decision. The line exists because these knobs are unset by default and silent when
|
|
1140
|
+
// unset, so the only way to tell a deliberate "no bound" from a forgotten one is to print it.
|
|
1141
|
+
const bounds = [
|
|
1142
|
+
["PI_SESSIONS_TTL_DAYS", env.PI_SESSIONS_TTL_DAYS, "14"],
|
|
1143
|
+
["PI_SESSION_MAX_AGE_DAYS", env.PI_SESSION_MAX_AGE_DAYS, "off"],
|
|
1144
|
+
["PI_SESSION_MAX_RESUME_CHAIN", env.PI_SESSION_MAX_RESUME_CHAIN, "off"],
|
|
1145
|
+
["PI_SESSION_MAX_CONTEXT_PCT", env.PI_SESSION_MAX_CONTEXT_PCT, "off"],
|
|
1146
|
+
];
|
|
1147
|
+
checks.push({
|
|
1148
|
+
ok: true,
|
|
1149
|
+
label: `Resume bounds: ${bounds.map(([name, value, fallback]) => `${name}=${value === undefined || value === "" ? fallback : value}`).join(", ")}`,
|
|
1150
|
+
});
|
|
1151
|
+
// The one bound that can be set and still do nothing, and the operator cannot see it from here.
|
|
1152
|
+
// Its measurement is reported by the JOB IMAGE's runner (INT-RUNNER-EXIT-CODE-PROTOCOL), so an
|
|
1153
|
+
// image older than that field reports none, the gate passes on no measurement by design, and the
|
|
1154
|
+
// bound is inert with nothing anywhere saying so. There is deliberately no image capability to
|
|
1155
|
+
// check against -- capabilities are an inclusion list for what the host DEMANDS of an image, and
|
|
1156
|
+
// telemetry is not that -- so this warning is the whole detection surface, which is exactly why
|
|
1157
|
+
// it exists rather than being left to a doc.
|
|
1158
|
+
if (env.PI_SESSION_MAX_CONTEXT_PCT) {
|
|
1159
|
+
checks.push({
|
|
1160
|
+
ok: true,
|
|
1161
|
+
warn: true,
|
|
1162
|
+
label: `PI_SESSION_MAX_CONTEXT_PCT=${env.PI_SESSION_MAX_CONTEXT_PCT} needs a job image whose runner reports context usage`,
|
|
1163
|
+
fix: `an older image reports none, and a bound with no measurement passes rather than guessing, so on such an image this bound does nothing at all. On an image that does report one the reading is kept whether or not the bound is set, so it applies from the next job. Each run's own record (${env.PI_LOGS_DIR || "the logs directory"}/<jobId>.json) carries session.reason, which names the gate that refused`,
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1137
1166
|
}
|
|
1138
1167
|
}
|
|
1139
1168
|
|
package/src/processor.mjs
CHANGED
|
@@ -44,7 +44,7 @@ export async function runJob(job, deps) {
|
|
|
44
44
|
// REQ-EGRESS-ALLOWLIST. Default admits everything, so a wiring that omits it behaves exactly as a
|
|
45
45
|
// deployment with no egress policy does -- which is also what the real factory returns when unarmed.
|
|
46
46
|
egressPreflight = async () => ({ ok: true }),
|
|
47
|
-
// (session, { piVersion }) => { promoted, reason, bytes }. Promotes this job's transcript back into
|
|
47
|
+
// (session, { piVersion, context }) => { promoted, reason, bytes }. Promotes this job's transcript back into
|
|
48
48
|
// the store, on a COMPLETED exit only. Never throws. The default is a no-op so a wiring that omits
|
|
49
49
|
// it behaves exactly as before -- no store, no promotion, no session in the record.
|
|
50
50
|
promoteSession = () => null,
|
|
@@ -351,7 +351,7 @@ export async function runJob(job, deps) {
|
|
|
351
351
|
return { outcome: "policy", reason: budget.reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: true }; // return => not retried
|
|
352
352
|
}
|
|
353
353
|
|
|
354
|
-
const { code, aborted, turns, tokens, session, usage } = await runContainer({ job, token, prepared });
|
|
354
|
+
const { code, aborted, turns, tokens, session, usage, context } = await runContainer({ job, token, prepared });
|
|
355
355
|
log("container_exit", { exitCode: code, aborted });
|
|
356
356
|
|
|
357
357
|
// Record token spend post-run (the check-AFTER half of the lagging token cap). The container ran,
|
|
@@ -388,7 +388,7 @@ export async function runJob(job, deps) {
|
|
|
388
388
|
// (CONST-RETRY-INFRA-ONLY). Same completed-only rule INT-OUTBOX-CONTRACT already uses, and
|
|
389
389
|
// it sits beside the chain collection for the same reason: both must happen before the
|
|
390
390
|
// `finally` deletes jobDir. Never throws.
|
|
391
|
-
const promoted = prepared.session ? promoteSession(prepared.session, { piVersion }) : null;
|
|
391
|
+
const promoted = prepared.session ? promoteSession(prepared.session, { piVersion, context }) : null;
|
|
392
392
|
return {
|
|
393
393
|
outcome: "completed",
|
|
394
394
|
exitCode: code,
|
|
@@ -455,7 +455,8 @@ export async function runJob(job, deps) {
|
|
|
455
455
|
* and with one number alone it is indistinguishable from an ordinary cold start.
|
|
456
456
|
*
|
|
457
457
|
* The runner's verdict WINS on `resumed`, because it is the one that observed the outcome. The host's
|
|
458
|
-
* reason is kept when the runner has none to give (a container that died before its exit line)
|
|
458
|
+
* reason is kept when the runner has none to give (a container that died before its exit line), AND when
|
|
459
|
+
* the host itself refused -- see the second precedence rule below.
|
|
459
460
|
*
|
|
460
461
|
* PII-free by construction: a boolean, a fixed enum, an integer. The key and the branch name are
|
|
461
462
|
* deliberately absent -- this record holds no attacker-chosen string, and a branch name is one.
|
|
@@ -463,12 +464,30 @@ export async function runJob(job, deps) {
|
|
|
463
464
|
function mergeSession(prepared, fromRunner, promoted = null) {
|
|
464
465
|
const host = prepared?.session;
|
|
465
466
|
if (!host && !fromRunner) return null;
|
|
467
|
+
// A HOST GATE THAT REFUSED OUTRANKS THE RUNNER'S `absent`, and without this rule it never reached a
|
|
468
|
+
// record at all. A refused read stages a 0-byte file rather than nothing (session-store.mjs, where the
|
|
469
|
+
// reasoning is pi's EEXIST race), the container is handed that file either way, and pi opens it and
|
|
470
|
+
// finds no messages -- so the runner reports `absent` on EVERY host refusal. Letting that win overwrote
|
|
471
|
+
// the answer with a restatement of the question: `expired` and `pi-version-changed` reached no
|
|
472
|
+
// completed record in the feature's whole life, and `docs/sessions.md`'s promise that every cold start
|
|
473
|
+
// is nameable in the record was false for them.
|
|
474
|
+
//
|
|
475
|
+
// Narrow on purpose, `host.resume === false` and the runner's token exactly `absent`. When the host
|
|
476
|
+
// DID stage a transcript and the runner still reports `absent`, the two genuinely disagree, and that
|
|
477
|
+
// disagreement is the event this object exists to show; the runner keeps winning there. So does its
|
|
478
|
+
// `unparseable`, which reports a degrade the host could not see.
|
|
479
|
+
const hostRefused = host?.resume === false && typeof host.reason === "string";
|
|
466
480
|
return {
|
|
467
481
|
resumed: fromRunner ? fromRunner.resumed : false,
|
|
468
482
|
// A promotion that was refused is the more useful reason to surface: "locked" or
|
|
469
483
|
// "not-a-regular-file" says why the NEXT run will cold-start, which is the thing an operator
|
|
470
484
|
// chasing "it never resumes" needs. It only ever replaces a reason on the completed path.
|
|
471
|
-
reason:
|
|
485
|
+
reason:
|
|
486
|
+
(promoted && !promoted.promoted ? promoted.reason : null) ??
|
|
487
|
+
(hostRefused && fromRunner?.reason === "absent" ? host.reason : null) ??
|
|
488
|
+
fromRunner?.reason ??
|
|
489
|
+
host?.reason ??
|
|
490
|
+
null,
|
|
472
491
|
bytes: promoted?.bytes ?? host?.bytes ?? null,
|
|
473
492
|
};
|
|
474
493
|
}
|
package/src/run-container.mjs
CHANGED
|
@@ -30,7 +30,7 @@ export function makeRunContainer({
|
|
|
30
30
|
image, // the DEPLOYMENT default (PI_JOB_IMAGE); a trigger's own run.image overrides it per job
|
|
31
31
|
hostEnv = process.env,
|
|
32
32
|
onOutput = (c) => process.stdout.write(c),
|
|
33
|
-
openJobLog = () => ({ write() {}, close: async () => ({ turns: null, tokens: null, session: null, usage: null }) }),
|
|
33
|
+
openJobLog = () => ({ write() {}, close: async () => ({ turns: null, tokens: null, session: null, usage: null, context: null }) }),
|
|
34
34
|
spawnFn = spawn,
|
|
35
35
|
globalPiDir = null, // REQ-GLOBAL-PI-OVERLAY: operator's global pi overlay dir, mounted :ro; null = off
|
|
36
36
|
allowGlobalExtensions = true, // REQ-GLOBAL-PI-OVERLAY: the staged overlay's extensions load unless PI_GLOBAL_ALLOW_EXTENSIONS=0
|
|
@@ -47,7 +47,7 @@ export function makeRunContainer({
|
|
|
47
47
|
// async so a synchronous throw (e.g. buildContainerEnv on an unconfigured provider) surfaces as
|
|
48
48
|
// a rejection, uniformly awaitable by the processor and by tests.
|
|
49
49
|
return async function runContainer({ job, token, prepared, name, signal }) {
|
|
50
|
-
if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null }; // killed before it could start
|
|
50
|
+
if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null }; // killed before it could start
|
|
51
51
|
|
|
52
52
|
// Closed env allowlist: only the provider key + the declared PI_* vars. Throws (config) if
|
|
53
53
|
// the provider is unconfigured -- the processor turns that into a pre-spend refusal.
|
|
@@ -147,20 +147,25 @@ export function makeRunContainer({
|
|
|
147
147
|
});
|
|
148
148
|
child.on("close", async (code) => {
|
|
149
149
|
const aborted = signal?.aborted === true; // capture BEFORE the await
|
|
150
|
-
// A rejecting sink.close is swallowed so a misbehaving sink cannot hang the run; turns/tokens/session/usage fall back to null.
|
|
150
|
+
// A rejecting sink.close is swallowed so a misbehaving sink cannot hang the run; turns/tokens/session/usage/context fall back to null.
|
|
151
151
|
let turns = null;
|
|
152
152
|
let tokens = null;
|
|
153
153
|
let session = null;
|
|
154
154
|
let usage = null;
|
|
155
|
+
let context = null;
|
|
155
156
|
try {
|
|
156
|
-
|
|
157
|
+
// `context = null` is a DEFAULT rather than a plain destructure: an injected sink that
|
|
158
|
+
// predates the field returns no such key, and `undefined` would then reach the record's
|
|
159
|
+
// shape where every other absence is spelled `null`.
|
|
160
|
+
({ turns, tokens, session, usage, context = null } = await sink.close());
|
|
157
161
|
} catch {
|
|
158
162
|
turns = null;
|
|
159
163
|
tokens = null;
|
|
160
164
|
session = null;
|
|
161
165
|
usage = null;
|
|
166
|
+
context = null;
|
|
162
167
|
}
|
|
163
|
-
resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage } : { code: code ?? 1, aborted: false, turns, tokens, session, usage });
|
|
168
|
+
resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage, context } : { code: code ?? 1, aborted: false, turns, tokens, session, usage, context });
|
|
164
169
|
});
|
|
165
170
|
});
|
|
166
171
|
|
package/src/run-history.mjs
CHANGED
|
@@ -110,6 +110,43 @@ export function parseExitSession(text) {
|
|
|
110
110
|
return null;
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
+
/**
|
|
114
|
+
* The container's report of how full its context was when the run ended (issue #186). Shaped exactly
|
|
115
|
+
* like `parseExitSession` above and, like it, PASSED THROUGH rather than rebuilt: both integers are
|
|
116
|
+
* host-bounded numbers, so `parseExitUsage`'s revalidating rebuild is not what this needs -- that one
|
|
117
|
+
* exists because the ledger carries id STRINGS.
|
|
118
|
+
*
|
|
119
|
+
* Absent means ABSENT, never zero. A runner predating this field, a run pi could give no context window
|
|
120
|
+
* for, and a compaction that left the count unknown all produce no key at all, and the session store
|
|
121
|
+
* reads that as "no measurement" and passes rather than inventing a denominator.
|
|
122
|
+
*/
|
|
123
|
+
export function parseExitContext(text) {
|
|
124
|
+
if (typeof text !== "string") return null;
|
|
125
|
+
const lines = text.split("\n");
|
|
126
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
127
|
+
const line = lines[i].trim();
|
|
128
|
+
if (line === "") continue;
|
|
129
|
+
let parsed;
|
|
130
|
+
try {
|
|
131
|
+
parsed = JSON.parse(line);
|
|
132
|
+
} catch {
|
|
133
|
+
continue; // docker/agent noise or a truncated final line
|
|
134
|
+
}
|
|
135
|
+
if (parsed?.event !== "exit") continue;
|
|
136
|
+
const c = parsed?.context;
|
|
137
|
+
// A window of 0 is not a denominator, and a negative count is not a measurement. SAFE integers
|
|
138
|
+
// specifically: `Number.isInteger` accepts up to ~1.8e308, and anything from 1e21 up stringifies to
|
|
139
|
+
// exponential notation, which the session store's own decimal round-trip then rejects on read --
|
|
140
|
+
// so a value in that range would be written into the store and be unreadable forever after, with
|
|
141
|
+
// the gate failing open on a measurement that said the context was full.
|
|
142
|
+
if (c && typeof c === "object" && !Array.isArray(c) && Number.isSafeInteger(c.tokens) && Number.isSafeInteger(c.window) && c.tokens >= 0 && c.window > 0) {
|
|
143
|
+
return { tokens: c.tokens, window: c.window };
|
|
144
|
+
}
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
|
|
113
150
|
export function parseExitTokens(text) {
|
|
114
151
|
if (typeof text !== "string") return null;
|
|
115
152
|
const lines = text.split("\n");
|
|
@@ -389,11 +426,12 @@ export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
|
|
|
389
426
|
}
|
|
390
427
|
|
|
391
428
|
async function close({ timeoutMs = 2000 } = {}) {
|
|
392
|
-
// Capture turns/tokens/session/usage from the tail first, so they survive even if the flush errors or times out.
|
|
429
|
+
// Capture turns/tokens/session/usage/context from the tail first, so they survive even if the flush errors or times out.
|
|
393
430
|
const turns = parseExitTurns(tail);
|
|
394
431
|
const tokens = parseExitTokens(tail);
|
|
395
432
|
const session = parseExitSession(tail);
|
|
396
433
|
const usage = parseExitUsage(tail);
|
|
434
|
+
const context = parseExitContext(tail);
|
|
397
435
|
try {
|
|
398
436
|
if (stream !== null) {
|
|
399
437
|
const s = stream;
|
|
@@ -417,7 +455,7 @@ export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
|
|
|
417
455
|
} catch (err) {
|
|
418
456
|
log("log_sink_error", { jobId, reason: err?.message });
|
|
419
457
|
}
|
|
420
|
-
return { turns, tokens, session, usage };
|
|
458
|
+
return { turns, tokens, session, usage, context };
|
|
421
459
|
}
|
|
422
460
|
|
|
423
461
|
return { write, close };
|
package/src/session-store.mjs
CHANGED
|
@@ -44,6 +44,54 @@ import { sessionKeyFor } from "./session-key.mjs";
|
|
|
44
44
|
export const SESSION_FILE_NAME = "current.jsonl";
|
|
45
45
|
const PI_VERSION_FILE = "pi-version";
|
|
46
46
|
const LOCK_FILE = "lock";
|
|
47
|
+
/**
|
|
48
|
+
* How many times in a row this key's transcript has been HANDED TO A CONTAINER. A counter rather than a
|
|
49
|
+
* derivation, because there is nothing to derive it from: the run record deliberately carries no session
|
|
50
|
+
* key (DES-SESSION-KEY-IS-DERIVED-NOT-INDEXED), so counting past runs would need the key->record index
|
|
51
|
+
* that entry refuses. One integer beside the transcript it describes is not that index; it is keyed state
|
|
52
|
+
* written where the key already is, and it answers exactly one question rather than being a query
|
|
53
|
+
* surface. Maintained even when no bound is set, deliberately -- see the write in promoteSession.
|
|
54
|
+
*
|
|
55
|
+
* IT COUNTS THE HOST'S DELIVERIES, NOT PI'S CONTINUATIONS, and that is the whole security of this bound.
|
|
56
|
+
* It counted pi's `resumed` first, on the reasoning that a transcript pi declined to continue extended
|
|
57
|
+
* nothing. That reasoning is wrong here, because the agent owns /session and therefore chooses what pi
|
|
58
|
+
* makes of the file: a transcript carrying a valid header and payload on lines pi's parser DROPS is
|
|
59
|
+
* delivered by the host every run while pi reports zero messages, so the counter reset every run and the
|
|
60
|
+
* chain bound never fired -- measured, not theorised. The host's own decision to hand the file over is
|
|
61
|
+
* the one fact in this exchange that nothing inside the container can influence.
|
|
62
|
+
*/
|
|
63
|
+
const RESUME_CHAIN_FILE = "resume-chain";
|
|
64
|
+
/**
|
|
65
|
+
* How full the context was when the run that wrote this transcript ended, as `<tokens> <window>`. Both
|
|
66
|
+
* numbers, not a precomputed percentage: the denominator is what makes the numerator readable later, and
|
|
67
|
+
* an operator looking at a refusal should be able to see what it was judged against.
|
|
68
|
+
*
|
|
69
|
+
* Reported BY THE CONTAINER, which is the only place the number exists: pi computes it from the session
|
|
70
|
+
* it is holding. That puts it at the same trust level as `turns` and `tokens`, and the residual is
|
|
71
|
+
* recorded in OQ-003 rather than papered over -- there is no host-side alternative that is not equally
|
|
72
|
+
* agent-influenced, since the transcript itself is agent-written.
|
|
73
|
+
*/
|
|
74
|
+
const CONTEXT_FILE = "context";
|
|
75
|
+
/** Both sidecar formats are a handful of bytes. Generous, and still nowhere near a job's wall clock. */
|
|
76
|
+
const SIDECAR_MAX_BYTES = 4096;
|
|
77
|
+
/**
|
|
78
|
+
* The host-effective provider and model as one token, or null when the job names neither.
|
|
79
|
+
*
|
|
80
|
+
* CONSERVATIVE BY CONSTRUCTION: the sidecar is whitespace-delimited, so a value carrying a space would
|
|
81
|
+
* split the record and be read back as a different field. Rather than escape, refuse: anything outside
|
|
82
|
+
* the charset the run record already validates model ids against is no identity, and no identity means
|
|
83
|
+
* the reading stays usable rather than being thrown away.
|
|
84
|
+
*/
|
|
85
|
+
function modelIdentity(job) {
|
|
86
|
+
const provider = typeof job?.provider === "string" ? job.provider : "";
|
|
87
|
+
const model = typeof job?.model === "string" ? job.model : "";
|
|
88
|
+
if (provider === "" || model === "") return null;
|
|
89
|
+
// Lowercased first, the same normalisation the run record's own model ids get, so a trigger written
|
|
90
|
+
// `Claude-Sonnet` and one written `claude-sonnet` are one model rather than two -- and so that a
|
|
91
|
+
// perfectly ordinary id does not fall out of the charset below and silently stop stamping.
|
|
92
|
+
const id = `${provider}/${model}`.toLowerCase();
|
|
93
|
+
return /^[a-z0-9][a-z0-9._:/-]{0,127}$/.test(id) ? id : null;
|
|
94
|
+
}
|
|
47
95
|
|
|
48
96
|
/**
|
|
49
97
|
* Read-path outcomes. Every one is a named cold start rather than a bare `false`: a feature that fails
|
|
@@ -56,6 +104,9 @@ export function makeSessionStore({
|
|
|
56
104
|
sessionsDir,
|
|
57
105
|
ttlDays,
|
|
58
106
|
maxBytes,
|
|
107
|
+
maxAgeDays = 0,
|
|
108
|
+
maxResumeChain = 0,
|
|
109
|
+
maxContextPct = null,
|
|
59
110
|
log = () => {},
|
|
60
111
|
now = () => Date.now(),
|
|
61
112
|
fs = { copyFileSync, lstatSync, mkdirSync, openSync, closeSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync },
|
|
@@ -83,11 +134,16 @@ export function makeSessionStore({
|
|
|
83
134
|
// CLI run, an unresolvable head ref), so it gets no mount and no transcript on disk.
|
|
84
135
|
if (key === null) return null;
|
|
85
136
|
|
|
137
|
+
// The model this job will actually run, for the context bound. A key is (kind, repo, ref) and
|
|
138
|
+
// carries NO model, so two triggers on one issue can name different ones, and the same token
|
|
139
|
+
// count is 78% of a 32k window and 2.5% of a 1M one. Carried on the session object rather than
|
|
140
|
+
// read again at promote time, so the reading is stamped with the model that produced it.
|
|
141
|
+
const modelId = modelIdentity(job);
|
|
86
142
|
const hostDir = join(jobDir, "session");
|
|
87
143
|
const staged = join(hostDir, SESSION_FILE_NAME);
|
|
88
144
|
fs.mkdirSync(hostDir, { recursive: true, mode: 0o700 });
|
|
89
145
|
|
|
90
|
-
const verdict = readCanonical(key, piVersion);
|
|
146
|
+
const verdict = readCanonical(key, piVersion, modelId);
|
|
91
147
|
if (verdict.resume) {
|
|
92
148
|
fs.copyFileSync(canonicalFile(key), staged);
|
|
93
149
|
} else {
|
|
@@ -98,7 +154,7 @@ export function makeSessionStore({
|
|
|
98
154
|
fs.writeFileSync(staged, "");
|
|
99
155
|
}
|
|
100
156
|
log("session_resolved", { key, resume: verdict.resume, reason: verdict.reason });
|
|
101
|
-
return { hostDir, key, ...verdict };
|
|
157
|
+
return { hostDir, key, modelId, ...verdict };
|
|
102
158
|
} catch (err) {
|
|
103
159
|
// A history fault must never fail the prepare that asked.
|
|
104
160
|
log("session_store_failed", { phase: "resolve", reason: err?.message });
|
|
@@ -114,7 +170,7 @@ export function makeSessionStore({
|
|
|
114
170
|
* one key is a real shape (REQ-QUEUE-BURST-NO-DROP), and last-write-wins there would interleave two
|
|
115
171
|
* agents' turns into one transcript.
|
|
116
172
|
*/
|
|
117
|
-
function promoteSession(session, { piVersion = null } = {}) {
|
|
173
|
+
function promoteSession(session, { piVersion = null, context = null } = {}) {
|
|
118
174
|
// The second DI-seam backstop, and unreachable for the same reason as the `!sessionsDir` return
|
|
119
175
|
// above: sessionKeyFor is total and binary (null, or 32 hex chars), so resolveSession returns null
|
|
120
176
|
// rather than a keyless session, and processor.mjs only calls this when prepare handed it one. Kept
|
|
@@ -137,16 +193,42 @@ export function makeSessionStore({
|
|
|
137
193
|
let fd;
|
|
138
194
|
try {
|
|
139
195
|
fd = fs.openSync(lock, "wx"); // exclusive create IS the lock; no daemon, no lease
|
|
140
|
-
} catch {
|
|
196
|
+
} catch (err) {
|
|
197
|
+
// EEXIST is the only failure that MEANS locked. A read-only directory, a full disk or a
|
|
198
|
+
// vanished store all failed to create the lock too, and reporting those as `locked` sends an
|
|
199
|
+
// operator looking for a stuck lock file that does not exist. Anything else falls through to
|
|
200
|
+
// the outer catch and reports `promote-failed`, which is what actually happened.
|
|
201
|
+
if (err?.code !== "EEXIST") throw err;
|
|
141
202
|
log("session_promote_skipped", { key: session.key, reason: "locked" });
|
|
142
203
|
return { promoted: false, reason: "locked" };
|
|
143
204
|
}
|
|
144
205
|
try {
|
|
145
206
|
// Atomic swap: a reader either sees the old file or the new one, never a half-written one.
|
|
146
207
|
const tmp = `${canonicalFile(session.key)}.incoming`;
|
|
208
|
+
try {
|
|
209
|
+
// `copyFileSync` follows a link at the DESTINATION, so a link planted at this name would
|
|
210
|
+
// receive the whole transcript and leave the canonical path pointing at it. The key
|
|
211
|
+
// directory's name is derived rather than random, so the path is precomputable by anyone
|
|
212
|
+
// who knows the repository and the branch; unlinking removes the link, never its target.
|
|
213
|
+
fs.unlinkSync(tmp);
|
|
214
|
+
} catch {
|
|
215
|
+
// Absent is the desired state.
|
|
216
|
+
}
|
|
147
217
|
fs.copyFileSync(staged, tmp);
|
|
148
218
|
fs.renameSync(tmp, canonicalFile(session.key));
|
|
149
219
|
fs.writeFileSync(join(dir, PI_VERSION_FILE), String(piVersion ?? ""));
|
|
220
|
+
// The two sidecars, immediately after the swap and under the same lock. NOT part of the swap
|
|
221
|
+
// itself, which is one rename and cannot be widened: what the lock buys them is that no
|
|
222
|
+
// other job can interleave, and what the ordering buys them is that they never describe a
|
|
223
|
+
// transcript older than the one now in place.
|
|
224
|
+
//
|
|
225
|
+
// EACH IS CAUGHT SEPARATELY, and that is not defensiveness for its own sake. These writes
|
|
226
|
+
// run AFTER the transcript is already promoted, so letting one throw would return
|
|
227
|
+
// `promote-failed` for a promotion that demonstrably happened -- a record that says the next
|
|
228
|
+
// run will cold start when it will in fact resume, which is worse than the bookkeeping loss
|
|
229
|
+
// it is reporting.
|
|
230
|
+
writeSidecar(dir, RESUME_CHAIN_FILE, session.key, chainValue(session));
|
|
231
|
+
writeContextSidecar(dir, session, context);
|
|
150
232
|
} finally {
|
|
151
233
|
fs.closeSync(fd);
|
|
152
234
|
try {
|
|
@@ -164,6 +246,79 @@ export function makeSessionStore({
|
|
|
164
246
|
}
|
|
165
247
|
}
|
|
166
248
|
|
|
249
|
+
/**
|
|
250
|
+
* The counter's next value. `session.resume` is the HOST's own decision to hand this key's transcript
|
|
251
|
+
* to a container, which is the only half of the exchange the container cannot influence; `resumed` (the
|
|
252
|
+
* container's verdict) is deliberately ignored for the counter and kept in the signature only because
|
|
253
|
+
* the record's own merge still wants it. A cold start resets, so a lineage always gets a fresh start
|
|
254
|
+
* from its next COMPLETED run -- a run that never completes promotes nothing and resets nothing, which
|
|
255
|
+
* is the safe direction: the key simply keeps cold-starting.
|
|
256
|
+
*/
|
|
257
|
+
function chainValue(session) {
|
|
258
|
+
return String(session.resume === true ? readResumeChain(session.key) + 1 : 0);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* One sidecar write. Two properties, both deliberate.
|
|
263
|
+
*
|
|
264
|
+
* **It cannot write THROUGH a link.** `writeFileSync` follows one, which would turn a planted symlink
|
|
265
|
+
* in a key directory into a truncating write of any worker-writable file, with the container's own
|
|
266
|
+
* integers as the payload. Writing a temp and renaming over the name replaces whatever is there --
|
|
267
|
+
* link included -- with a regular file, and never opens the link's target. The temp is unlinked first
|
|
268
|
+
* for the same reason, since a planted link at THAT name would be the same hole one step along. The
|
|
269
|
+
* read side's `lstat` guard is the other half of this; neither is sufficient alone.
|
|
270
|
+
*
|
|
271
|
+
* **It is never fatal.** This runs AFTER the transcript is already promoted, so throwing would return
|
|
272
|
+
* `promote-failed` for a promotion that demonstrably happened, telling an operator the next run will
|
|
273
|
+
* cold start when it will in fact resume. The bookkeeping loss is logged and the truth is kept.
|
|
274
|
+
*/
|
|
275
|
+
function writeSidecar(dir, name, key, value) {
|
|
276
|
+
const file = join(dir, name);
|
|
277
|
+
const tmp = `${file}.incoming`;
|
|
278
|
+
try {
|
|
279
|
+
try {
|
|
280
|
+
fs.unlinkSync(tmp);
|
|
281
|
+
} catch {
|
|
282
|
+
// Absent is the desired state.
|
|
283
|
+
}
|
|
284
|
+
fs.writeFileSync(tmp, value);
|
|
285
|
+
fs.renameSync(tmp, file);
|
|
286
|
+
} catch (err) {
|
|
287
|
+
log("session_sidecar_failed", { key, file: name, reason: err?.message });
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* The context sidecar, whose three cases are all different.
|
|
293
|
+
*
|
|
294
|
+
* A run that RESUMED and measured nothing keeps the previous reading: the transcript it promoted is
|
|
295
|
+
* the old one extended, so the last real measurement is the closest true statement available, and a
|
|
296
|
+
* zero would read as "the context emptied", which cannot have happened.
|
|
297
|
+
*
|
|
298
|
+
* A COLD START, though, promoted a transcript that shares nothing with the one the old reading
|
|
299
|
+
* described, so the reading must GO. Keeping it is what turned a single high measurement into a key
|
|
300
|
+
* that refused itself forever: the gate read a stale number, cold-started, and the cold start left the
|
|
301
|
+
* same number behind for the next run to read. That loop had no exit that did not involve deleting the
|
|
302
|
+
* store by hand.
|
|
303
|
+
*/
|
|
304
|
+
function writeContextSidecar(dir, session, context) {
|
|
305
|
+
const file = join(dir, CONTEXT_FILE);
|
|
306
|
+
if (session.resume !== true) {
|
|
307
|
+
try {
|
|
308
|
+
fs.unlinkSync(file);
|
|
309
|
+
} catch {
|
|
310
|
+
// Absent is the desired state, so failing to remove what is not there is success.
|
|
311
|
+
}
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
if (!context) return;
|
|
315
|
+
// The model rides along because the ratio is meaningless without it: a key is (kind, repo, ref) and
|
|
316
|
+
// carries no model, so two triggers on one issue can run different ones, and 25k tokens is 78% of a
|
|
317
|
+
// 32k window and 2.5% of a 1M one. A reading from another model is not a reading about this one.
|
|
318
|
+
const stamp = session.modelId ? ` ${session.modelId}` : "";
|
|
319
|
+
writeSidecar(dir, CONTEXT_FILE, session.key, `${context.tokens} ${context.window}${stamp}`);
|
|
320
|
+
}
|
|
321
|
+
|
|
167
322
|
function keyDir(key) {
|
|
168
323
|
return join(sessionsDir, key);
|
|
169
324
|
}
|
|
@@ -172,7 +327,7 @@ export function makeSessionStore({
|
|
|
172
327
|
}
|
|
173
328
|
|
|
174
329
|
/** The read path, gate by gate. The FIRST miss wins and names itself. */
|
|
175
|
-
function readCanonical(key, piVersion) {
|
|
330
|
+
function readCanonical(key, piVersion, modelId) {
|
|
176
331
|
const file = canonicalFile(key);
|
|
177
332
|
const check = inspectFile(file);
|
|
178
333
|
if (!check.ok) return COLD(check.reason);
|
|
@@ -184,26 +339,150 @@ export function makeSessionStore({
|
|
|
184
339
|
// repair that mid-run, so a version change is a cold start rather than a mid-run failure. An
|
|
185
340
|
// image that declares no version never resumes -- the safe direction, never "assume it matches".
|
|
186
341
|
if (piVersion === null) return COLD("pi-version-changed");
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
342
|
+
// Through the same guarded read as the two sidecars below it. This one predates them and was the
|
|
343
|
+
// one unguarded read left in the key directory; a symlink here would have decided a gate on the
|
|
344
|
+
// contents of some other file entirely.
|
|
345
|
+
const stamped = readSidecar(key, PI_VERSION_FILE);
|
|
346
|
+
if (stamped === null || stamped !== piVersion) return COLD("pi-version-changed");
|
|
347
|
+
|
|
348
|
+
// How many times in a row this key has already been resumed. Placed HERE, ahead of the header read,
|
|
349
|
+
// for two reasons. It is a small sidecar read exactly like the pi-version arm above it, so refusing
|
|
350
|
+
// on it skips pulling a transcript that may be megabytes; and unlike every other arm it asks about
|
|
351
|
+
// the LINEAGE rather than the file, so it needs nothing the file could tell it.
|
|
352
|
+
//
|
|
353
|
+
// The cost of that placement, stated rather than left to be discovered: a transcript that is both
|
|
354
|
+
// chain-exhausted AND corrupt reports the chain. That is the intentional refusal of the two, and the
|
|
355
|
+
// corruption is not hidden, only deferred -- this cold start's own promotion resets the counter, so
|
|
356
|
+
// the very next run reads the file and reports `unparseable`.
|
|
357
|
+
//
|
|
358
|
+
// FAILS OPEN on absence, which is the opposite of the age gate one arm down and deliberate. Every
|
|
359
|
+
// key that existed before this counter did has no file, and reading that as "already exhausted"
|
|
360
|
+
// would cold-start an operator's entire store the day they set the bound.
|
|
361
|
+
if (maxResumeChain > 0 && readResumeChain(key) >= maxResumeChain) return COLD("resume-chain-too-long");
|
|
362
|
+
|
|
363
|
+
// How full the context already is, against a ceiling the HOST owns. Not a duplicate of pi's own
|
|
364
|
+
// compaction threshold and deliberately not read from it: pi's is settable in a serviced repo's
|
|
365
|
+
// .pi/settings.json, so it is a line the repository can move, and this one cannot be. Past that
|
|
366
|
+
// threshold what a resumed job replays is not the transcript but a model-written summary of it,
|
|
367
|
+
// produced while that model was reading attacker-authored text (OQ-003), so this is a safety bound
|
|
368
|
+
// before it is an economic one.
|
|
369
|
+
//
|
|
370
|
+
// FAILS OPEN and INVENTS NO DENOMINATOR. No sidecar (every key promoted before this shipped, and
|
|
371
|
+
// every key under an image whose runner predates it), a compaction that left pi's own count
|
|
372
|
+
// unknown, or a window of zero all mean the gate has nothing to act on, and a gate with nothing to
|
|
373
|
+
// act on passes. A bytes-against-window guess was rejected rather than used as a fallback: the
|
|
374
|
+
// transcript is the whole branch INCLUDING what compaction folded away, so it over-reads exactly
|
|
375
|
+
// past the threshold this exists to catch, and there is no bytes-to-tokens calibration here to
|
|
376
|
+
// make it mean anything.
|
|
377
|
+
if (maxContextPct !== null) {
|
|
378
|
+
const seen = readContext(key);
|
|
379
|
+
// A reading STAMPED WITH ANOTHER MODEL is not a reading about this one, and using it is wrong in
|
|
380
|
+
// both directions: it refuses a job whose window is far larger than the one that was measured,
|
|
381
|
+
// and it passes one whose window is far smaller. Unknown on either side stays usable, so a
|
|
382
|
+
// deployment that names no model per trigger keeps the bound it had.
|
|
383
|
+
const foreign = seen !== null && seen.modelId !== null && modelId !== null && seen.modelId !== modelId;
|
|
384
|
+
if (seen !== null && !foreign && (seen.tokens * 100) / seen.window >= maxContextPct) return COLD("context-too-full");
|
|
192
385
|
}
|
|
193
|
-
if (stamped !== piVersion) return COLD("pi-version-changed");
|
|
194
386
|
|
|
195
|
-
// Cheapest real shape check, and the last one
|
|
196
|
-
// else the runner would throw on, so refusing here keeps
|
|
197
|
-
// surprises rather than for a file we could already tell
|
|
387
|
+
// Cheapest real shape check, and the last one before the header's own contents are used: the first
|
|
388
|
+
// line must be a pi session header. Anything else the runner would throw on, so refusing here keeps
|
|
389
|
+
// the container's degrade path for genuine surprises rather than for a file we could already tell
|
|
390
|
+
// was wrong.
|
|
391
|
+
let header = null;
|
|
198
392
|
try {
|
|
199
393
|
const head = String(fs.readFileSync(file, "utf8")).split("\n", 1)[0];
|
|
200
|
-
|
|
394
|
+
header = JSON.parse(head);
|
|
395
|
+
if (header?.type !== "session") return COLD("unparseable");
|
|
201
396
|
} catch {
|
|
202
397
|
return COLD("unparseable");
|
|
203
398
|
}
|
|
399
|
+
|
|
400
|
+
// The CONVERSATION's age, and it is a DIFFERENT CLOCK from `expired` above rather than a finer
|
|
401
|
+
// setting of it. The TTL reads the transcript's mtime, which the PROMOTE rename refreshes -- and only
|
|
402
|
+
// that: `copyFileSync` stamps its destination, never its source, so the resolve half leaves the
|
|
403
|
+
// canonical file's mtime alone (measured, because the obvious reading of the two call sites says
|
|
404
|
+
// otherwise). So `expired` is time since the last COMPLETED run on this key, and a lineage whose runs
|
|
405
|
+
// keep completing never expires however old its first turn is. pi's header carries the instant the
|
|
406
|
+
// session was created, so this
|
|
407
|
+
// costs no new persisted state -- the line is already read and parsed one gate up, and until now
|
|
408
|
+
// only its `type` was looked at.
|
|
409
|
+
//
|
|
410
|
+
// The arm is LAST because the earlier gates are cheaper and because a corrupt file is corrupt rather
|
|
411
|
+
// than old: `unparseable` must keep winning over this, or a damaged transcript would be reported as
|
|
412
|
+
// a lineage that aged out.
|
|
413
|
+
//
|
|
414
|
+
// UNREADABLE FAILS CLOSED, on the pi-version gate's precedent one arm up: a header with no usable
|
|
415
|
+
// timestamp cannot be shown to be young enough, and "assume it matches" is the direction that
|
|
416
|
+
// silently keeps resuming. Like `pi-version-changed`, one token covers all three causes (absent,
|
|
417
|
+
// wrong type, unparseable).
|
|
418
|
+
//
|
|
419
|
+
// A timestamp in the FUTURE passes, deliberately. It buys nothing to refuse one: the agent owns
|
|
420
|
+
// /session, so anything able to write a future timestamp is equally able to write the current one,
|
|
421
|
+
// and refusing would convert ordinary clock skew between a container and its host into a cold start
|
|
422
|
+
// for every key on the deployment.
|
|
423
|
+
if (maxAgeDays > 0) {
|
|
424
|
+
const started = Date.parse(typeof header.timestamp === "string" ? header.timestamp : "");
|
|
425
|
+
if (!Number.isFinite(started)) return COLD("conversation-too-old");
|
|
426
|
+
if (now() - started > maxAgeDays * 86400000) return COLD("conversation-too-old");
|
|
427
|
+
}
|
|
204
428
|
return { resume: true, reason: "resumed", bytes: check.bytes };
|
|
205
429
|
}
|
|
206
430
|
|
|
431
|
+
/**
|
|
432
|
+
* Every sidecar read goes through here, and it is the same load-bearing check `inspectFile` makes on
|
|
433
|
+
* the transcript: **`lstat`, regular files only.** The canonical store is host-only and never mounted,
|
|
434
|
+
* so nothing in a container can plant a link here -- but the directory NAME is derived rather than
|
|
435
|
+
* random (`sha256(kind, repo, ref)`), so anyone who knows the repository and the branch can compute it
|
|
436
|
+
* and pre-create the path. `readFileSync` and `writeFileSync` both follow links, which would turn a
|
|
437
|
+
* planted symlink into a read of any worker-readable file on the gate's path, and a promotion into a
|
|
438
|
+
* truncating write of any worker-writable one. The transcript has been guarded against exactly this
|
|
439
|
+
* since the feature shipped; these files inherit it rather than being the exception.
|
|
440
|
+
*
|
|
441
|
+
* SIZE-BOUNDED for the same reason the transcript is. Both formats are a handful of bytes, `maxBytes`
|
|
442
|
+
* does not cover them, and reading a 2.5 GiB file on the job's own path costs half a minute of wall
|
|
443
|
+
* clock before any container starts.
|
|
444
|
+
*/
|
|
445
|
+
function readSidecar(key, name) {
|
|
446
|
+
try {
|
|
447
|
+
const file = join(keyDir(key), name);
|
|
448
|
+
const st = fs.lstatSync(file);
|
|
449
|
+
if (!st.isFile() || st.size === 0 || st.size > SIDECAR_MAX_BYTES) return null;
|
|
450
|
+
return String(fs.readFileSync(file, "utf8")).trim();
|
|
451
|
+
} catch {
|
|
452
|
+
return null;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* The consecutive-delivery counter for a key, or 0 when there is not a readable one. Never throws and
|
|
458
|
+
* never guesses: a missing, empty, corrupt or negative counter is 0, so the only way to be refused by
|
|
459
|
+
* the chain bound is for this store to have written a number that reaches it.
|
|
460
|
+
*/
|
|
461
|
+
function readResumeChain(key) {
|
|
462
|
+
const raw = readSidecar(key, RESUME_CHAIN_FILE);
|
|
463
|
+
if (raw === null) return 0;
|
|
464
|
+
const n = Number.parseInt(raw, 10);
|
|
465
|
+
// `String(n) === raw` is the same anti-truncation guard config.mjs applies to every integer knob,
|
|
466
|
+
// and it is what keeps a corrupt "3.5" from being read as a chain of three.
|
|
467
|
+
return Number.isInteger(n) && n > 0 && String(n) === raw ? n : 0;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* The stored context occupancy for a key, or `null` when there is no measurement. Never throws, never
|
|
472
|
+
* guesses, and never returns a partial: anything it cannot read as two positive integers is no
|
|
473
|
+
* measurement at all, which the caller treats as "pass" rather than as zero.
|
|
474
|
+
*/
|
|
475
|
+
function readContext(key) {
|
|
476
|
+
const raw = readSidecar(key, CONTEXT_FILE);
|
|
477
|
+
if (raw === null) return null;
|
|
478
|
+
const [rawTokens, rawWindow, rawModel] = raw.split(/\s+/);
|
|
479
|
+
const tokens = Number.parseInt(rawTokens, 10);
|
|
480
|
+
const window = Number.parseInt(rawWindow, 10);
|
|
481
|
+
if (!Number.isInteger(tokens) || !Number.isInteger(window) || tokens < 0 || window <= 0) return null;
|
|
482
|
+
if (String(tokens) !== rawTokens || String(window) !== rawWindow) return null;
|
|
483
|
+
return { tokens, window, modelId: rawModel ?? null };
|
|
484
|
+
}
|
|
485
|
+
|
|
207
486
|
/**
|
|
208
487
|
* lstat, REGULAR FILES ONLY -- and this is the one line in the file that is load-bearing security
|
|
209
488
|
* rather than hygiene.
|
package/src/start.mjs
CHANGED
|
@@ -298,6 +298,9 @@ export async function startWorker(
|
|
|
298
298
|
sessionsDir: config.sessionsDir,
|
|
299
299
|
ttlDays: config.sessionsTtlDays,
|
|
300
300
|
maxBytes: config.sessionMaxBytes,
|
|
301
|
+
maxAgeDays: config.sessionMaxAgeDays,
|
|
302
|
+
maxResumeChain: config.sessionMaxResumeChain,
|
|
303
|
+
maxContextPct: config.sessionMaxContextPct,
|
|
301
304
|
log,
|
|
302
305
|
});
|
|
303
306
|
// Boot sweep, beside the log reaper and for the same reason it is beside rather than inside it: these
|