@edgehero/pi-dispatch 4.3.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +94 -4
- package/README.md +20 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +117 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +128 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +199 -30
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +159 -0
- package/src/console-client.mjs +848 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1663 -198
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +103 -9
- package/src/env-file.mjs +6 -4
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-auth.mjs +4 -0
- package/src/forgejo-host.mjs +127 -31
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +73 -15
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +740 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +1 -1
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +4 -1
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1554 -167
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +296 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +33 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +782 -40
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +398 -51
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
package/src/capacity-records.mjs
CHANGED
|
@@ -23,8 +23,9 @@
|
|
|
23
23
|
* written before that key, recreated by a worker that does not know it, or whose key was lost, vouches for nothing
|
|
24
24
|
* older), the deepest window any writer keeps
|
|
25
25
|
* (`mirrorWindowMs(0)`), the fleet horizon (`runs:horizon`, raised by every writer whose trim removed runs, by its
|
|
26
|
-
* OWN retention or
|
|
27
|
-
* run
|
|
26
|
+
* OWN retention or its own count cap, so a peer with a short retention cuts everyone's history and says so), the oldest
|
|
27
|
+
* run when the index is at the cap (`indexMax`, the deployment's `PI_RUN_MIRROR_MAX_RUNS`; see `readMirrorWindow`)
|
|
28
|
+
* and that run ended after the window began, and the newest run whose body has
|
|
28
29
|
* expired while its index member stayed (its writer has not trimmed since). Any of these past the window's start
|
|
29
30
|
* makes it `truncated`. The start is the one bound that survives losing the mirror's keys: a flushed or replaced
|
|
30
31
|
* Valkey, or an index that expired, loses the horizon with the runs, and the next write recreates an index whose
|
|
@@ -35,21 +36,41 @@
|
|
|
35
36
|
* only. Every Valkey call is bounded (`bounded`, host-registry.mjs' reason: this client QUEUES a command while offline
|
|
36
37
|
* rather than rejecting it, so an unbounded await is a hang, not an error).
|
|
37
38
|
*
|
|
39
|
+
* A record from a NEWER pi-dispatch (issue #761, run-history.mjs `recordReadable`) is not read by either source: a
|
|
40
|
+
* capacity report adds up slot time and sizes, fields whose meaning that release may have changed. It is skipped and
|
|
41
|
+
* COUNTED (`newerSkipped`, and a reason naming the count), so the report says it is partial rather than reading a
|
|
42
|
+
* busy host as idle. Each source skips its own before the merge, so an older attempt's readable copy of the same job
|
|
43
|
+
* (a retry that landed on a newer host) still counts its own slot. The merged count is of distinct job ids, the key
|
|
44
|
+
* `mergeRuns` keeps one record per: a host rolled back from a newer release holds its own newer records both on disk
|
|
45
|
+
* and in the mirror, and each is one record the report did not read, not two.
|
|
46
|
+
*
|
|
38
47
|
* Never throws.
|
|
39
48
|
*/
|
|
40
49
|
|
|
41
50
|
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
42
51
|
import { join } from "node:path";
|
|
43
52
|
import { READ_SCRIPT, RUNS_HORIZON, RUNS_HORIZON_MEMBER, RUNS_INDEX, RUNS_INDEX_MAX, RUNS_SINCE, mergeRuns, mirrorWindowMs, runRecordKey } from "./run-mirror.mjs";
|
|
53
|
+
import { recordReadable } from "./run-history.mjs";
|
|
44
54
|
import { SIZING_RECORD_MAX_BYTES } from "./size-records.mjs";
|
|
45
55
|
|
|
46
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* How many record bodies one MGET asks for: a full mirror at the default cap of 5,000 runs is ten round trips, never one
|
|
58
|
+
* 5,000-key command, and at the 20,000 ceiling (`RUN_MIRROR_MAX_RUNS_MAX`) forty.
|
|
59
|
+
*/
|
|
47
60
|
export const CAPACITY_MGET_CHUNK = 500;
|
|
48
61
|
/** How long one Valkey call may take before the mirror is read as unreachable. */
|
|
49
62
|
export const CAPACITY_OP_TIMEOUT_MS = 2_000;
|
|
50
63
|
|
|
51
64
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
52
65
|
|
|
66
|
+
/**
|
|
67
|
+
* A source's newer-release fields: `newerSkipped` (how many) and `newerIds` (their job ids, which the merged count
|
|
68
|
+
* de-duplicates), both absent when there are none, so a whole-result comparison of a read without one is unchanged.
|
|
69
|
+
*/
|
|
70
|
+
function newerOf(ids) {
|
|
71
|
+
return ids.length > 0 ? { newerSkipped: ids.length, newerIds: ids } : {};
|
|
72
|
+
}
|
|
73
|
+
|
|
53
74
|
/** Reject rather than wait forever (host-registry.mjs `bounded`). */
|
|
54
75
|
function bounded(promise, ms) {
|
|
55
76
|
return new Promise((resolve, reject) => {
|
|
@@ -63,9 +84,21 @@ function bounded(promise, ms) {
|
|
|
63
84
|
|
|
64
85
|
/**
|
|
65
86
|
* The mirror's records that ended after `sinceMs`: `{ records, state, fromMs, truncated, skipped }`, `state` "ok", "off"
|
|
66
|
-
* or "unreachable (<reason>)", `fromMs` where its history starts (see the header)
|
|
87
|
+
* or "unreachable (<reason>)", `fromMs` where its history starts (see the header), plus `newerSkipped`, how many
|
|
88
|
+
* records from a newer pi-dispatch were left out, and `newerIds`, their job ids, only when there were any. A newer
|
|
89
|
+
* record with no job id is not counted: `mergeRuns` keeps no record without one, so it is none the report would read.
|
|
90
|
+
* Never throws.
|
|
91
|
+
*
|
|
92
|
+
* `indexMax` is the deployment's run index cap (`PI_RUN_MIRROR_MAX_RUNS`, issue #761; the built-in 5,000 when not
|
|
93
|
+
* given). An index at or over it may have been cut by count, so its history starts no earlier than its oldest run. The
|
|
94
|
+
* cap the reader is told, not the built-in one: an index of a raised cap that holds 6,000 runs was never trimmed, and
|
|
95
|
+
* reading it against 5,000 marked a whole history as cut. A count trim since issue #599 also raises the horizon, which
|
|
96
|
+
* says the same; this clause is for a writer from before that, which trimmed at the built-in 5,000 and recorded no
|
|
97
|
+
* horizon. A reader told a cap above that writer's misses its cut (a fleet that still runs such a worker caps the
|
|
98
|
+
* index at 5,000 anyway); one told a lower cap than the fleet's reads its history as starting at the oldest run, which
|
|
99
|
+
* errs toward missing history, never toward idle time.
|
|
67
100
|
*/
|
|
68
|
-
export async function readMirrorWindow(redis, { sinceMs, nowMs, timeoutMs = CAPACITY_OP_TIMEOUT_MS, chunk = CAPACITY_MGET_CHUNK } = {}) {
|
|
101
|
+
export async function readMirrorWindow(redis, { sinceMs, nowMs, timeoutMs = CAPACITY_OP_TIMEOUT_MS, chunk = CAPACITY_MGET_CHUNK, indexMax = RUNS_INDEX_MAX } = {}) {
|
|
69
102
|
const none = (state) => ({ records: [], state, fromMs: null, truncated: false, skipped: 0 });
|
|
70
103
|
if (!redis) return none("off");
|
|
71
104
|
try {
|
|
@@ -84,6 +117,7 @@ export async function readMirrorWindow(redis, { sinceMs, nowMs, timeoutMs = CAPA
|
|
|
84
117
|
for (let i = 0; i < ids.length; i += 2) members.push({ id: ids[i], score: Number(ids[i + 1]) });
|
|
85
118
|
const records = [];
|
|
86
119
|
let skipped = 0;
|
|
120
|
+
const newerIds = [];
|
|
87
121
|
let expiredMs = NaN;
|
|
88
122
|
for (let i = 0; i < members.length; i += chunk) {
|
|
89
123
|
const part = members.slice(i, i + chunk);
|
|
@@ -102,7 +136,9 @@ export async function readMirrorWindow(redis, { sinceMs, nowMs, timeoutMs = CAPA
|
|
|
102
136
|
}
|
|
103
137
|
try {
|
|
104
138
|
const record = JSON.parse(raw);
|
|
105
|
-
if (record
|
|
139
|
+
if (record === null || typeof record !== "object" || Array.isArray(record)) return;
|
|
140
|
+
if (recordReadable(record)) records.push(record);
|
|
141
|
+
else if (typeof record.jobId === "string" && record.jobId !== "") newerIds.push(record.jobId);
|
|
106
142
|
} catch {
|
|
107
143
|
// unparseable is not showable
|
|
108
144
|
}
|
|
@@ -114,20 +150,27 @@ export async function readMirrorWindow(redis, { sinceMs, nowMs, timeoutMs = CAPA
|
|
|
114
150
|
if (Number.isFinite(startMs)) fromMs = Math.max(fromMs, startMs);
|
|
115
151
|
else if (Number.isFinite(oldestMs)) fromMs = Math.max(fromMs, oldestMs);
|
|
116
152
|
if (Number.isFinite(horizonMs)) fromMs = Math.max(fromMs, horizonMs);
|
|
117
|
-
if (size >=
|
|
153
|
+
if (size >= indexMax && Number.isFinite(oldestMs)) fromMs = Math.max(fromMs, oldestMs);
|
|
118
154
|
if (Number.isFinite(expiredMs)) fromMs = Math.max(fromMs, expiredMs);
|
|
119
155
|
fromMs = Math.min(fromMs, nowMs);
|
|
120
|
-
return { records, state: "ok", fromMs, truncated: fromMs > sinceMs, skipped };
|
|
156
|
+
return { records, state: "ok", fromMs, truncated: fromMs > sinceMs, skipped, ...newerOf(newerIds) };
|
|
121
157
|
} catch (err) {
|
|
122
158
|
return none(`unreachable (${err?.message ?? "error"})`);
|
|
123
159
|
}
|
|
124
160
|
}
|
|
125
161
|
|
|
126
162
|
/**
|
|
127
|
-
* This host's records that ended after `sinceMs`: `{ records, skipped, unreachable, absent }
|
|
163
|
+
* This host's records that ended after `sinceMs`: `{ records, skipped, unreachable, absent }`, plus `newerSkipped`, how
|
|
164
|
+
* many records of the window from a newer pi-dispatch were left out, and `newerIds`, their job ids, only when there
|
|
165
|
+
* were any. `fs` is `{ readdirSync,
|
|
128
166
|
* statSync, readFileSync }`.
|
|
167
|
+
*
|
|
168
|
+
* `deadlineAt` (issue #761, doctor's dollar window check): an instant on `clock` past which no further file is opened,
|
|
169
|
+
* the result then carrying `cutShort: true`. The loop is synchronous, so a bound raced against it from outside could
|
|
170
|
+
* not stop it; this one is checked before each file. A read cut short returns fewer records, which for every caller
|
|
171
|
+
* that passes it only lowers what it adds up. Absent, every file is read, as before.
|
|
129
172
|
*/
|
|
130
|
-
export function readLocalWindow(logsDir, { sinceMs, fs = { readdirSync, readFileSync, statSync } } = {}) {
|
|
173
|
+
export function readLocalWindow(logsDir, { sinceMs, fs = { readdirSync, readFileSync, statSync }, deadlineAt = null, clock = Date.now } = {}) {
|
|
131
174
|
let names;
|
|
132
175
|
try {
|
|
133
176
|
names = fs.readdirSync(logsDir);
|
|
@@ -137,8 +180,14 @@ export function readLocalWindow(logsDir, { sinceMs, fs = { readdirSync, readFile
|
|
|
137
180
|
}
|
|
138
181
|
const records = [];
|
|
139
182
|
let skipped = 0;
|
|
183
|
+
const newerIds = [];
|
|
184
|
+
let cutShort = false;
|
|
140
185
|
for (const name of names) {
|
|
141
186
|
if (typeof name !== "string" || !name.endsWith(".json")) continue;
|
|
187
|
+
if (Number.isFinite(deadlineAt) && clock() > deadlineAt) {
|
|
188
|
+
cutShort = true;
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
142
191
|
try {
|
|
143
192
|
const path = join(logsDir, name);
|
|
144
193
|
const st = fs.statSync(path);
|
|
@@ -156,24 +205,30 @@ export function readLocalWindow(logsDir, { sinceMs, fs = { readdirSync, readFile
|
|
|
156
205
|
}
|
|
157
206
|
const record = JSON.parse(buf.toString("utf8"));
|
|
158
207
|
if (record === null || typeof record !== "object" || Array.isArray(record) || typeof record.jobId !== "string") continue;
|
|
208
|
+
// `endedAt` is in the stable core, so a newer record is judged against the window before it is skipped: only one
|
|
209
|
+
// the report would have read is counted.
|
|
159
210
|
const end = Date.parse(record.endedAt ?? "");
|
|
160
|
-
if (Number.isFinite(end)
|
|
211
|
+
if (!Number.isFinite(end) || end <= sinceMs) continue;
|
|
212
|
+
if (recordReadable(record)) records.push(record);
|
|
213
|
+
else newerIds.push(record.jobId);
|
|
161
214
|
} catch {
|
|
162
215
|
// unparseable, or reaped between the listing and the read
|
|
163
216
|
}
|
|
164
217
|
}
|
|
165
|
-
return { records, skipped, unreachable: null, absent: false };
|
|
218
|
+
return { records, skipped, unreachable: null, absent: false, ...newerOf(newerIds), ...(cutShort ? { cutShort: true } : {}) };
|
|
166
219
|
}
|
|
167
220
|
|
|
168
221
|
/**
|
|
169
222
|
* The records of the window `[sinceMs, nowMs]` from both sources, merged, and what each covers: `{ records, mirrorState,
|
|
170
223
|
* coverage }`,
|
|
171
224
|
* `coverage` being `{ source, reason, localHost, localHosts, local: { fromMs } | null, mirror: { fromMs, truncated,
|
|
172
|
-
* hosts } | null, skipped }` as `computeCapacity` reads it
|
|
173
|
-
*
|
|
225
|
+
* hosts } | null, skipped }` as `computeCapacity` reads it, plus `newerSkipped` when records from a newer pi-dispatch
|
|
226
|
+
* were left out (the reason then names how many). `redis` may be null (no Valkey to ask); `noMirrorReason` then
|
|
227
|
+
* says why, in place of "no run mirror". `localHost` is this host's name. `indexMax` is the deployment's run index cap
|
|
228
|
+
* (`readMirrorWindow`; capacity-cli.mjs `deploymentFacts` reads it). Never throws.
|
|
174
229
|
*/
|
|
175
|
-
export async function readCapacityRecords({ redis = null, logsDir, sinceMs, nowMs, fs, retentionDays = 0, localHost = null, noMirrorReason = null, timeoutMs = CAPACITY_OP_TIMEOUT_MS, chunk = CAPACITY_MGET_CHUNK } = {}) {
|
|
176
|
-
const mirror = await readMirrorWindow(redis, { sinceMs, nowMs, timeoutMs, chunk });
|
|
230
|
+
export async function readCapacityRecords({ redis = null, logsDir, sinceMs, nowMs, fs, retentionDays = 0, localHost = null, noMirrorReason = null, timeoutMs = CAPACITY_OP_TIMEOUT_MS, chunk = CAPACITY_MGET_CHUNK, indexMax = RUNS_INDEX_MAX } = {}) {
|
|
231
|
+
const mirror = await readMirrorWindow(redis, { sinceMs, nowMs, timeoutMs, chunk, indexMax });
|
|
177
232
|
const local = readLocalWindow(logsDir, { sinceMs, ...(fs ? { fs } : {}) });
|
|
178
233
|
const mirrored = mirror.state === "ok";
|
|
179
234
|
const days = Number(retentionDays);
|
|
@@ -189,6 +244,9 @@ export async function readCapacityRecords({ redis = null, logsDir, sinceMs, nowM
|
|
|
189
244
|
if (local.unreachable) reasons.push(local.unreachable);
|
|
190
245
|
const skipped = local.skipped + mirror.skipped;
|
|
191
246
|
if (skipped > 0) reasons.push(`${skipped} record${skipped === 1 ? "" : "s"} over ${SIZING_RECORD_MAX_BYTES / 1024} KiB skipped`);
|
|
247
|
+
// Distinct job ids, as the records are merged (the header): the same newer record on disk and in the mirror is one.
|
|
248
|
+
const newerSkipped = new Set([...(local.newerIds ?? []), ...(mirror.newerIds ?? [])]).size;
|
|
249
|
+
if (newerSkipped > 0) reasons.push(`${newerSkipped} record${newerSkipped === 1 ? "" : "s"} from a newer pi-dispatch not read`);
|
|
192
250
|
const hostsOf = (records) => [...new Set(records.map((r) => r.host).filter((h) => typeof h === "string" && h !== ""))].sort();
|
|
193
251
|
return {
|
|
194
252
|
// One per job id, the later end winning (a retry can land on another host); `Infinity` keeps every run, since the
|
|
@@ -204,6 +262,7 @@ export async function readCapacityRecords({ redis = null, logsDir, sinceMs, nowM
|
|
|
204
262
|
local: localRead ? { fromMs: localFromMs } : null,
|
|
205
263
|
mirror: mirrored ? { fromMs: mirror.fromMs, truncated: mirror.truncated, hosts: hostsOf(mirror.records) } : null,
|
|
206
264
|
skipped,
|
|
265
|
+
...(newerSkipped > 0 ? { newerSkipped } : {}),
|
|
207
266
|
},
|
|
208
267
|
};
|
|
209
268
|
}
|
package/src/capacity.mjs
CHANGED
|
@@ -424,6 +424,8 @@ export function computeCapacity({ records = [], live = [], windowStartMs, nowMs,
|
|
|
424
424
|
}
|
|
425
425
|
}
|
|
426
426
|
|
|
427
|
+
const fleetSegments = [];
|
|
428
|
+
const fleetWaits = [];
|
|
427
429
|
const hosts = [];
|
|
428
430
|
const totals = zeroCounts();
|
|
429
431
|
const notShared = [];
|
|
@@ -511,6 +513,8 @@ export function computeCapacity({ records = [], live = [], windowStartMs, nowMs,
|
|
|
511
513
|
const segment = (a, b, n) => {
|
|
512
514
|
if (b <= a) return;
|
|
513
515
|
const cap = capAt(a);
|
|
516
|
+
const memoryKnown = positiveInt(cap.memMiB) && [...active].every((i) => h.runs[i].size !== null);
|
|
517
|
+
fleetSegments.push({ start: a, end: b, jobs: n, slots: cap.slots, memory: memoryKnown ? [...active].reduce((sum, i) => sum + h.runs[i].size.memMiB, 0) : null, budget: positiveInt(cap.memMiB) ? cap.memMiB : null });
|
|
514
518
|
// CPU used in this segment: each running job's measured time spread over its wall, together never more than
|
|
515
519
|
// the host's CPUs. Past it, every share is cut in proportion and the runs are counted as clamped.
|
|
516
520
|
const parts = [];
|
|
@@ -596,6 +600,7 @@ export function computeCapacity({ records = [], live = [], windowStartMs, nowMs,
|
|
|
596
600
|
const other = rest.length === 0 ? null : { count: rest.length, runMs: Number(rest.reduce((s, [, p]) => s + p.runMs, 0n)), cpuMs: Number(rest.reduce((s, [, p]) => s + p.cpuUsec, 0n) / 1000n) };
|
|
597
601
|
|
|
598
602
|
for (const [k, v] of Object.entries(h.counts)) totals[k] += v;
|
|
603
|
+
fleetWaits.push(...h.waits);
|
|
599
604
|
const waits = [...h.waits].sort((a, b) => a - b);
|
|
600
605
|
hosts.push({
|
|
601
606
|
name,
|
|
@@ -632,6 +637,7 @@ export function computeCapacity({ records = [], live = [], windowStartMs, nowMs,
|
|
|
632
637
|
const liveRowMissing = isHostName(coverage.localHost) && names.has(coverage.localHost) && !liveByName.has(coverage.localHost) ? coverage.localHost : null;
|
|
633
638
|
return {
|
|
634
639
|
v: CAPACITY_REPORT_VERSION,
|
|
640
|
+
fleet: fleetCapacity(fleetSegments, fleetWaits, { windowStartMs, nowMs, incomplete: hosts.some((h) => !h.shared || h.missingMs > 0 || h.coverage.liveUnreadable > 0) || unreadable > 0 || withoutHost > 0 || coverage.reason != null }),
|
|
635
641
|
window: { fromMs: windowStartMs, toMs: nowMs, bucketMs },
|
|
636
642
|
coverage: {
|
|
637
643
|
source: typeof coverage.source === "string" ? coverage.source : "records",
|
|
@@ -652,3 +658,62 @@ export function computeCapacity({ records = [], live = [], windowStartMs, nowMs,
|
|
|
652
658
|
hosts,
|
|
653
659
|
};
|
|
654
660
|
}
|
|
661
|
+
|
|
662
|
+
/** Aggregate the same covered segments as the host report, never an average of host percentages or percentiles. */
|
|
663
|
+
function fleetCapacity(segments, waits, { windowStartMs, nowMs, incomplete }) {
|
|
664
|
+
const hourMs = 60 * 60 * 1000;
|
|
665
|
+
const events = [];
|
|
666
|
+
let occupied = 0n;
|
|
667
|
+
let available = 0n;
|
|
668
|
+
for (const s of segments) {
|
|
669
|
+
if (s.slots === null) incomplete = true;
|
|
670
|
+
else {
|
|
671
|
+
occupied += BigInt(s.jobs) * BigInt(s.end - s.start);
|
|
672
|
+
available += BigInt(s.slots) * BigInt(s.end - s.start);
|
|
673
|
+
}
|
|
674
|
+
if (s.memory === null || s.budget === null) incomplete = true;
|
|
675
|
+
events.push({ at: s.start, delta: 1, segment: s }, { at: s.end, delta: -1, segment: s });
|
|
676
|
+
}
|
|
677
|
+
events.sort((a, b) => a.at - b.at || a.delta - b.delta);
|
|
678
|
+
const hours = new Map();
|
|
679
|
+
let jobs = 0;
|
|
680
|
+
let memory = 0;
|
|
681
|
+
let budget = 0;
|
|
682
|
+
let unknownMemory = 0;
|
|
683
|
+
let peak = 0;
|
|
684
|
+
let peakAtMs = null;
|
|
685
|
+
let at = windowStartMs;
|
|
686
|
+
const count = (end) => {
|
|
687
|
+
if (end <= at) return;
|
|
688
|
+
if (jobs > peak) { peak = jobs; peakAtMs = at; }
|
|
689
|
+
for (let t = at; t < end;) {
|
|
690
|
+
const key = Math.floor(t / hourMs) * hourMs;
|
|
691
|
+
const stop = Math.min(end, key + hourMs);
|
|
692
|
+
const len = BigInt(stop - t);
|
|
693
|
+
const h = hours.get(key) ?? { runMs: 0n, memory: 0n, budget: 0n, unknown: false };
|
|
694
|
+
h.runMs += BigInt(jobs) * len;
|
|
695
|
+
h.memory += BigInt(memory) * len;
|
|
696
|
+
h.budget += BigInt(budget) * len;
|
|
697
|
+
h.unknown ||= unknownMemory > 0;
|
|
698
|
+
hours.set(key, h);
|
|
699
|
+
t = stop;
|
|
700
|
+
}
|
|
701
|
+
};
|
|
702
|
+
for (const e of events) {
|
|
703
|
+
count(e.at);
|
|
704
|
+
at = e.at;
|
|
705
|
+
jobs += e.delta * e.segment.jobs;
|
|
706
|
+
memory += e.delta * (e.segment.memory ?? 0);
|
|
707
|
+
budget += e.delta * (e.segment.budget ?? 0);
|
|
708
|
+
if (e.segment.memory === null || e.segment.budget === null) unknownMemory += e.delta;
|
|
709
|
+
}
|
|
710
|
+
count(nowMs);
|
|
711
|
+
const busiest = [...hours].sort((a, b) => a[1].runMs === b[1].runMs ? a[0] - b[0] : a[1].runMs > b[1].runMs ? -1 : 1)[0];
|
|
712
|
+
waits.sort((a, b) => a - b);
|
|
713
|
+
return {
|
|
714
|
+
busyPerMille: perMille(occupied, available), peak, peakAtMs,
|
|
715
|
+
waits: { n: waits.length, p50Ms: waits.length ? rank(waits, 50) : null, p95Ms: waits.length ? rank(waits, 95) : null },
|
|
716
|
+
promisedMemPerMille: busiest && !busiest[1].unknown ? perMille(busiest[1].memory, busiest[1].budget) : null,
|
|
717
|
+
busiestHourMs: busiest?.[0] ?? null, incomplete,
|
|
718
|
+
};
|
|
719
|
+
}
|
package/src/chat-cli.mjs
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `pi-dispatch chat <run-id | issue-or-PR-url | session-key> [--fork] [--force] [--tools] [--pi <path>] [--console <addr>] [--yes]`
|
|
3
|
+
* (issue #647, REQ-TERMINAL-CHAT): open a run's chat in this terminal.
|
|
4
|
+
*
|
|
5
|
+
* THREE ARGUMENT CLASSES, told apart by shape alone (`classifyChatTarget`):
|
|
6
|
+
* - a session key (32 lowercase hex) is ALWAYS the fork, on this host, with no console at all: a key names a stored
|
|
7
|
+
* transcript and nothing else, so there is no run to ask about;
|
|
8
|
+
* - an http(s) URL is an issue or pull request on a forge, which the console resolves to its newest run (`?url=`);
|
|
9
|
+
* - anything else in the run-id charset is a job id (`?id=`).
|
|
10
|
+
*
|
|
11
|
+
* THE MODE IS THE CONSOLE'S (`/api/run`'s `mode`, from the one rule `chatModeOf` that the web Chat panel and
|
|
12
|
+
* `/dispatch chat` share), so the terminal can never disagree with the page about what a run is: `live` attaches,
|
|
13
|
+
* `reply` answers a parked run, `fork` forks a finished one, and `null` prints the console's sentence (`says`: queued,
|
|
14
|
+
* unknown, already answered) and exits 1. `--fork` forces the fork; with a run id whose record this host's run history
|
|
15
|
+
* holds, that needs no console either. The record a fork judges comes from the local run history when present (the
|
|
16
|
+
* record the worker wrote here), else from `/api/run`'s `run`.
|
|
17
|
+
*
|
|
18
|
+
* A live attach that ends (the run finished while watched) asks `/api/run` again and goes on in the new mode, so
|
|
19
|
+
* watching a run that parks drops straight into its reply box.
|
|
20
|
+
*
|
|
21
|
+
* A CONSOLE OFF THIS MACHINE IS A QUESTION FIRST. The console speaks plain HTTP, so signing in to an address that is not
|
|
22
|
+
* loopback or a unix socket sends the token readable to the whole path. Such an address (only `--console` or
|
|
23
|
+
* `PI_CONSOLE_URL` can name one; `consoleAddressOf` refuses it from a `.env`) gets a warning naming the host and a y/N
|
|
24
|
+
* whose default is no, before the token is read or sent; `--yes` answers it for a script (the warning still prints).
|
|
25
|
+
*/
|
|
26
|
+
import { parseArgs } from "node:util";
|
|
27
|
+
import { homedir } from "node:os";
|
|
28
|
+
import { askHidden, askYesNo, consoleAddressOf, consoleToken, dotenvTrustRefusal, makeConsoleClient } from "./console-client.mjs";
|
|
29
|
+
import { SESSION_KEY_PATTERN } from "./session-key.mjs";
|
|
30
|
+
|
|
31
|
+
/** The usage line, shared with the CLI's own usage text (a test holds both). */
|
|
32
|
+
export const CHAT_USAGE = "pi-dispatch chat <run-id | issue-or-PR-url | session-key> [--fork] [--force] [--tools] [--pi <path>] [--console <addr>] [--yes]";
|
|
33
|
+
|
|
34
|
+
/** The longest issue or pull request URL taken, the console's `?url=` limit (`/dispatch chat`'s, which a test holds to it). */
|
|
35
|
+
export const CHAT_URL_MAX = 2048;
|
|
36
|
+
|
|
37
|
+
/** A job id as the worker mints it (the console's charset). */
|
|
38
|
+
const RUN_ID_RE = /^[A-Za-z0-9._:-]{1,200}$/;
|
|
39
|
+
|
|
40
|
+
/** How many times, 2 s apart, a run that just ended is asked again before its new mode is taken as it stands. */
|
|
41
|
+
const SETTLE_TRIES = 5;
|
|
42
|
+
const SETTLE_MS = 2000;
|
|
43
|
+
|
|
44
|
+
/** The deployment keys this command reads, each only where it is needed. */
|
|
45
|
+
export const CHAT_DEPLOYMENT_KEYS = Object.freeze(["PI_CONSOLE_LISTEN", "PI_SESSIONS_DIR", "PI_LOGS_DIR"]);
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* What the argument names: `{kind: "key", key}`, `{kind: "url", url}`, `{kind: "id", id}`, or `{refusal}`. A key is
|
|
49
|
+
* tested first, so a 32-hex string is never read as a job id (job ids that are pure hex of that length are not minted).
|
|
50
|
+
*/
|
|
51
|
+
export function classifyChatTarget(arg) {
|
|
52
|
+
const s = typeof arg === "string" ? arg.trim() : "";
|
|
53
|
+
if (s === "") return { refusal: `name a run: ${CHAT_USAGE}` };
|
|
54
|
+
if (SESSION_KEY_PATTERN.test(s)) return { kind: "key", key: s };
|
|
55
|
+
if (/^https?:\/\//i.test(s)) {
|
|
56
|
+
if (s.length > CHAT_URL_MAX) return { refusal: `the URL is longer than ${CHAT_URL_MAX} characters` };
|
|
57
|
+
try {
|
|
58
|
+
new URL(s);
|
|
59
|
+
} catch {
|
|
60
|
+
return { refusal: "the URL does not parse" };
|
|
61
|
+
}
|
|
62
|
+
return { kind: "url", url: s };
|
|
63
|
+
}
|
|
64
|
+
if (RUN_ID_RE.test(s)) return { kind: "id", id: s };
|
|
65
|
+
return { refusal: "the argument is neither a run id (letters, digits and . _ : -), an issue or pull request URL, nor a 32-hex session key" };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* One line of console-supplied text, made safe for this terminal: the view's scrub (no pi-tui is loaded for it), then
|
|
70
|
+
* every line break folded to a space, so a multi-line refusal cannot draw a second, fake line of output.
|
|
71
|
+
*/
|
|
72
|
+
async function safe(text) {
|
|
73
|
+
const { scrubForTerminal } = await import("./chat-view.mjs");
|
|
74
|
+
return scrubForTerminal(text, { maxBytes: 4096 }).replace(/\s*\n\s*/g, " ");
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Run the command. Seams: `env` (this shell's), `out`/`err` (line writers), `deploymentEnv` (cli.mjs's
|
|
79
|
+
* `cliDeploymentEnv`, handed in so this module does not import the CLI's entry), `cwd`, `home`, and `deps` for tests:
|
|
80
|
+
* `makeClient`, `askToken`, `ask`, `readRecord`, `runLive`, `runReply`, `question`, `fork` (`{plan, run}`), `sleep`,
|
|
81
|
+
* `envTrust` (`dotenvTrustRefusal` of `cwd` by default).
|
|
82
|
+
* Returns the exit code.
|
|
83
|
+
*/
|
|
84
|
+
export async function runChat(argv, { env = process.env, out = (l) => process.stdout.write(`${l}\n`), err = (l) => process.stderr.write(`${l}\n`), deploymentEnv, cwd = process.cwd(), home = homedir(), deps = {} } = {}) {
|
|
85
|
+
let parsed;
|
|
86
|
+
try {
|
|
87
|
+
parsed = parseArgs({
|
|
88
|
+
args: argv,
|
|
89
|
+
allowPositionals: true,
|
|
90
|
+
options: { "fork-as-job": { type: "boolean", default: false }, entry: { type: "string" }, position: { type: "string" }, model: { type: "string" }, provider: { type: "string" }, thinking: { type: "string" }, message: { type: "string" }, fork: { type: "boolean", default: false }, force: { type: "boolean", default: false }, tools: { type: "boolean", default: false }, pi: { type: "string" }, console: { type: "string" }, yes: { type: "boolean", default: false } },
|
|
91
|
+
});
|
|
92
|
+
} catch (e) {
|
|
93
|
+
err(`error: ${e.message}\n ${CHAT_USAGE}`);
|
|
94
|
+
return 1;
|
|
95
|
+
}
|
|
96
|
+
const { values, positionals } = parsed;
|
|
97
|
+
if (positionals.length !== 1) {
|
|
98
|
+
err(`error: ${positionals.length === 0 ? "name a run" : "chat takes one run"}\n ${CHAT_USAGE}`);
|
|
99
|
+
return 1;
|
|
100
|
+
}
|
|
101
|
+
const target = classifyChatTarget(positionals[0]);
|
|
102
|
+
if (target.refusal) {
|
|
103
|
+
err(`error: ${target.refusal}`);
|
|
104
|
+
return 1;
|
|
105
|
+
}
|
|
106
|
+
if (values["fork-as-job"] && (values.fork || target.kind === "key" || values.force || values.tools || values.pi)) {
|
|
107
|
+
err("error: --fork-as-job needs a run id or issue URL and cannot use local-fork options");
|
|
108
|
+
return 1;
|
|
109
|
+
}
|
|
110
|
+
if (!values["fork-as-job"] && ["entry", "position", "model", "provider", "thinking", "message"].some((k) => values[k] !== undefined)) {
|
|
111
|
+
err("error: entry, position, model, provider, thinking and message options need --fork-as-job");
|
|
112
|
+
return 1;
|
|
113
|
+
}
|
|
114
|
+
const deployment = (keys) => (typeof deploymentEnv === "function" ? deploymentEnv(env, keys, { cwd }) : { env });
|
|
115
|
+
const forkOpts = { values, env, out, err, cwd, home, deployment, deps };
|
|
116
|
+
|
|
117
|
+
if (target.kind === "key") return forkRun({ ...forkOpts, record: null, key: target.key, newerRunOnTarget: null });
|
|
118
|
+
|
|
119
|
+
// --fork with a run id this host's history holds needs no console: the record is here.
|
|
120
|
+
if (values.fork && target.kind === "id") {
|
|
121
|
+
const local = await readLocalRecord(target.id, { deployment, deps });
|
|
122
|
+
if (local.problem) {
|
|
123
|
+
err(`error: ${local.problem}`);
|
|
124
|
+
return 1;
|
|
125
|
+
}
|
|
126
|
+
if (local.record) return forkRun({ ...forkOpts, record: local.record, key: null, newerRunOnTarget: null });
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const where = consoleAddressOf({ flag: values.console ?? null, env, deployment: () => deployment(["PI_CONSOLE_LISTEN"]), envTrust: deps.envTrust ?? (() => dotenvTrustRefusal({ dir: cwd })) });
|
|
130
|
+
if (where.refusal) {
|
|
131
|
+
err(`error: ${await safe(where.refusal)}`);
|
|
132
|
+
return 1;
|
|
133
|
+
}
|
|
134
|
+
if (!where.loopback) {
|
|
135
|
+
err(`warning: ${where.address.shown} is not on this machine, and the console speaks plain HTTP: the console token goes over the network to it unencrypted, readable to anything on the path. A tunnel to a loopback address (ssh -L 4790:127.0.0.1:4790 <host>) keeps it private.`);
|
|
136
|
+
const ok = values.yes || (await (deps.ask ?? ((q) => askYesNo(q)))(`Send the console token to ${where.address.shown} anyway? [y/N] `));
|
|
137
|
+
if (!ok) {
|
|
138
|
+
err("error: not signed in; nothing was sent");
|
|
139
|
+
return 1;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
const tok = await consoleToken({ env, askHidden: deps.askToken ?? ((q) => askHidden(q)) });
|
|
143
|
+
if (tok.refusal) {
|
|
144
|
+
err(`error: ${tok.refusal}`);
|
|
145
|
+
return 1;
|
|
146
|
+
}
|
|
147
|
+
const client = (deps.makeClient ?? makeConsoleClient)({ address: where.address, token: tok.token });
|
|
148
|
+
const lookup = target.kind === "url" ? { url: target.url } : { id: target.id };
|
|
149
|
+
try {
|
|
150
|
+
const who = await client.whoami();
|
|
151
|
+
let info = await lookupRun(client, lookup, { err });
|
|
152
|
+
if (!info) return 1;
|
|
153
|
+
if (values["fork-as-job"]) return managedFork({ client, info, who, values, deps, out, err });
|
|
154
|
+
for (let round = 0; round < 8; round += 1) {
|
|
155
|
+
const mode = values.fork ? "fork" : info.mode;
|
|
156
|
+
if (mode === "fork") {
|
|
157
|
+
const id = typeof info.run?.jobId === "string" ? info.run.jobId : null;
|
|
158
|
+
const local = id ? await readLocalRecord(id, { deployment, deps }) : { record: null };
|
|
159
|
+
if (local.problem) {
|
|
160
|
+
err(`error: ${local.problem}`);
|
|
161
|
+
return 1;
|
|
162
|
+
}
|
|
163
|
+
return forkRun({ ...forkOpts, record: local.record ?? info.run ?? null, key: null, newerRunOnTarget: typeof info.newerRunOnTarget === "string" ? info.newerRunOnTarget : null });
|
|
164
|
+
}
|
|
165
|
+
if (mode === "live") {
|
|
166
|
+
const pi = await (deps.loadPiTui ?? (async () => (await import("./chat-view.mjs")).loadPiTui()))();
|
|
167
|
+
const terminal = deps.terminal ?? new pi.ProcessTerminal();
|
|
168
|
+
const runLive = deps.runLive ?? (await import("./chat-view.mjs")).runLiveView;
|
|
169
|
+
const got = await runLive({ client, lookup, info, who, pi, terminal });
|
|
170
|
+
if (got.detached) {
|
|
171
|
+
out("detached; the run goes on");
|
|
172
|
+
return 0;
|
|
173
|
+
}
|
|
174
|
+
info = await settle(client, lookup, got.info ?? info, deps.sleep);
|
|
175
|
+
out(`the run is no longer live (${await safe(info.state ?? "?")})`);
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (mode === "reply") {
|
|
179
|
+
const view = await import("./chat-view.mjs");
|
|
180
|
+
const question = await (deps.question ?? view.questionOf)({ client, info, who });
|
|
181
|
+
const pi = await (deps.loadPiTui ?? view.loadPiTui)();
|
|
182
|
+
const terminal = deps.terminal ?? new pi.ProcessTerminal();
|
|
183
|
+
const got = await (deps.runReply ?? view.runReplyView)({ client, info, who, question, pi, terminal });
|
|
184
|
+
if (got.detached) {
|
|
185
|
+
out("left without answering");
|
|
186
|
+
return 0;
|
|
187
|
+
}
|
|
188
|
+
const jobs = Array.isArray(got.result?.jobIds) ? got.result.jobIds.map(String) : [];
|
|
189
|
+
out(`answered: the run resumes${jobs.length > 0 ? ` as ${await safe(jobs.join(", "))}` : ""}; follow it with pi-dispatch chat ${await safe(jobs[0] ?? info.run?.jobId ?? "<run-id>")}`);
|
|
190
|
+
return 0;
|
|
191
|
+
}
|
|
192
|
+
err(`error: ${await safe(typeof info.says === "string" && info.says !== "" ? info.says : "this run has no chat to open")}`);
|
|
193
|
+
return 1;
|
|
194
|
+
}
|
|
195
|
+
err("error: the run kept changing mode; open it again");
|
|
196
|
+
return 1;
|
|
197
|
+
} catch (e) {
|
|
198
|
+
err(`error: ${await safe(e?.message ?? "the console could not be asked")}`);
|
|
199
|
+
return 1;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** `/api/run` for `lookup`: its JSON, or null after printing the console's refusal (and an Azure item's candidates). */
|
|
204
|
+
async function lookupRun(client, lookup, { err }) {
|
|
205
|
+
const got = await client.run(lookup);
|
|
206
|
+
if (got.status === 200 && got.json && typeof got.json === "object") return got.json;
|
|
207
|
+
const sentence = typeof got.json?.error === "string" ? got.json.error : `the console answered ${got.status}`;
|
|
208
|
+
err(`error: ${await safe(sentence)}`);
|
|
209
|
+
const candidates = Array.isArray(got.json?.candidates) ? got.json.candidates.slice(0, 20) : [];
|
|
210
|
+
for (const c of candidates) {
|
|
211
|
+
const line = typeof c === "string" ? c : [c?.jobId ?? c?.id, c?.target, c?.url].filter((v) => typeof v === "string").join(" ");
|
|
212
|
+
if (line !== "") err(` ${await safe(line)}`);
|
|
213
|
+
}
|
|
214
|
+
if (candidates.length > 0) err(" name one of them: pi-dispatch chat <run-id>");
|
|
215
|
+
return null;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** After a live view ends, ask again until the run has left `live` (its record may land a moment after `done`). */
|
|
219
|
+
async function settle(client, lookup, info, sleep = (ms) => new Promise((r) => setTimeout(r, ms))) {
|
|
220
|
+
let current = info;
|
|
221
|
+
for (let i = 0; i < SETTLE_TRIES && current?.mode === "live"; i += 1) {
|
|
222
|
+
await sleep(SETTLE_MS);
|
|
223
|
+
const got = await client.run(lookup).catch(() => null);
|
|
224
|
+
if (got?.status === 200 && got.json) current = got.json;
|
|
225
|
+
}
|
|
226
|
+
if (current?.mode === "live") return { ...current, mode: null, says: "the run's stream ended but the console still shows it live; open it again in a moment" };
|
|
227
|
+
return current;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** This host's record of `jobId` (the run history the worker writes): `{record}` (null when absent) or `{problem}`. */
|
|
231
|
+
async function readLocalRecord(jobId, { deployment, deps }) {
|
|
232
|
+
let read = deps.readRecord;
|
|
233
|
+
if (typeof read !== "function") {
|
|
234
|
+
const d = deployment(["PI_LOGS_DIR"]);
|
|
235
|
+
if (d.problem) return { problem: `${d.problem}, so this host's run history cannot be found` };
|
|
236
|
+
const [{ makeReadRecord }, { logsDirPath }] = await Promise.all([import("./run-history.mjs"), import("./config.mjs")]);
|
|
237
|
+
read = makeReadRecord({ logsDir: logsDirPath(d.env) });
|
|
238
|
+
}
|
|
239
|
+
const got = read(jobId);
|
|
240
|
+
// An unreadable file (`UNREADABLE_RECORD` names no job), or another job's record under a sanitized name that
|
|
241
|
+
// collides, is no record: the console's copy is asked instead.
|
|
242
|
+
return { record: got !== null && typeof got === "object" && got.jobId === jobId ? got : null };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** The fork: plan, then run, printing refusals as sentences. */
|
|
246
|
+
async function forkRun({ values, env, out, err, cwd, home, deployment, deps, record, key, newerRunOnTarget }) {
|
|
247
|
+
const fork = deps.fork ?? (await import("./chat-fork.mjs"));
|
|
248
|
+
const d = deployment(["PI_SESSIONS_DIR"]);
|
|
249
|
+
if (d.problem) {
|
|
250
|
+
err(`error: ${d.problem}, so the session store cannot be found`);
|
|
251
|
+
return 1;
|
|
252
|
+
}
|
|
253
|
+
const pi = fork.resolvePiBinary({ flag: values.pi ?? null, env });
|
|
254
|
+
const piVersion = pi.path ? fork.piVersionOf(pi.path, { env }) : null;
|
|
255
|
+
const planned = fork.planFork({ record, key, env: d.env, force: values.force, newerRunOnTarget, pi, piVersion });
|
|
256
|
+
if (planned.refusal) {
|
|
257
|
+
err(`error: ${await safe(planned.refusal)}`);
|
|
258
|
+
return 1;
|
|
259
|
+
}
|
|
260
|
+
const { agentDirFrom } = await import("./host-pi.mjs");
|
|
261
|
+
return fork.runFork({ plan: planned.plan, out, ask: deps.ask ?? ((q) => askYesNo(q)), agentDir: agentDirFrom(env), cwd, home, tools: values.tools, env });
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** Managed execution uses the console's exact paid preview and confirm pipeline. */
|
|
265
|
+
async function managedFork({ client, info, who, values, deps, out, err }) {
|
|
266
|
+
if (who.role !== "operator" || who.readOnly) {
|
|
267
|
+
err("error: a managed fork needs an operator on a writable console");
|
|
268
|
+
return 1;
|
|
269
|
+
}
|
|
270
|
+
const jobId = info.run?.jobId;
|
|
271
|
+
if (!values.entry) {
|
|
272
|
+
const got = await client.getJson(`/api/fork-points?id=${encodeURIComponent(jobId ?? "")}`);
|
|
273
|
+
if (got.status !== 200) {
|
|
274
|
+
err(`error: ${await safe(got.json?.error ?? "fork points unavailable")}`);
|
|
275
|
+
return 1;
|
|
276
|
+
}
|
|
277
|
+
for (const e of got.json.entries ?? []) if (e.positions?.length) out(await safe(`${e.id} ${e.role ?? e.type} (${e.positions.join(", ")})`));
|
|
278
|
+
out("Choose a supported entry: add --entry <id> --position before|at to --fork-as-job");
|
|
279
|
+
return 0;
|
|
280
|
+
}
|
|
281
|
+
const body = { jobId, entry: values.entry, position: values.position ?? "before" };
|
|
282
|
+
for (const key of ["model", "provider", "thinking", "message"]) if (values[key] !== undefined) body[key] = values[key];
|
|
283
|
+
const got = await client.act("run-fork", body, { print: (line) => out(String(line).replace(/[\u0000-\u001f\u007f-\u009f]/g, " ")), confirm: async () => values.yes || (await (deps.ask ?? askYesNo)("Queue this paid fork? [y/N] ")) });
|
|
284
|
+
if (got.refusal) { err(`error: ${await safe(got.refusal)}`); return 1; }
|
|
285
|
+
if (got.declined) { out("left without queuing a fork"); return 0; }
|
|
286
|
+
out("fork queued; the original run and branch are preserved");
|
|
287
|
+
return 0;
|
|
288
|
+
}
|