@oakgreencc/cambium 0.0.43 → 0.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -16
- package/package.json +1 -1
- package/src/ask.mjs +90 -0
- package/src/config.mjs +6 -0
- package/src/cost.mjs +161 -0
- package/src/graphs/dispatch.mjs +29 -8
- package/src/graphs/pr.mjs +1 -0
- package/src/graphs/rescue.mjs +10 -4
- package/src/graphs/task.mjs +3 -0
- package/src/issue-state.mjs +3 -0
- package/src/labels.mjs +7 -0
- package/src/marker.mjs +8 -0
- package/src/phases/dispatch.mjs +20 -5
- package/src/phases/rescue.mjs +10 -8
- package/src/store.mjs +35 -1
- package/src/tick.mjs +49 -0
- package/src/tracing.mjs +11 -0
package/README.md
CHANGED
|
@@ -54,16 +54,17 @@ A requeue is `q:task` with attempt + 1, or, once `maxAttempts` are spent, `q:hum
|
|
|
54
54
|
|
|
55
55
|
## State on GitHub
|
|
56
56
|
|
|
57
|
-
| Label | Meaning
|
|
58
|
-
| --------------------------------------------------- |
|
|
59
|
-
| `q:intake`, `q:task`, `q:human`, `q:promote` | The queue. At most one per open issue.
|
|
60
|
-
| `q:promote` on a closed issue | Merged into `release.from`, not yet in `release.to` ([Awaiting promotion](#awaiting-promotion)).
|
|
61
|
-
| `running` | A worker holds the lease (with `q:task`).
|
|
62
|
-
| `why:budget`, `why:triage` (also `why:reject`, …) | Why it is in `q:human` (`why:triage`: sent there by triage, see below).
|
|
63
|
-
| `why:rescue` | In `q:human` because the [rescue graph](#the-rescue-graph) could not unstick it.
|
|
64
|
-
| `
|
|
65
|
-
|
|
|
66
|
-
| `
|
|
57
|
+
| Label | Meaning |
|
|
58
|
+
| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
|
|
59
|
+
| `q:intake`, `q:task`, `q:human`, `q:promote` | The queue. At most one per open issue. |
|
|
60
|
+
| `q:promote` on a closed issue | Merged into `release.from`, not yet in `release.to` ([Awaiting promotion](#awaiting-promotion)). |
|
|
61
|
+
| `running` | A worker holds the lease (with `q:task`). |
|
|
62
|
+
| `why:budget`, `why:triage` (also `why:reject`, …) | Why it is in `q:human` (`why:triage`: sent there by triage, see below). |
|
|
63
|
+
| `why:rescue` | In `q:human` because the [rescue graph](#the-rescue-graph) could not unstick it. |
|
|
64
|
+
| `ask:*` | With `q:human`: what the person is asked (`ask:decide`, `ask:verify`, `ask:chore`, `ask:approve`; plus `ask:session`). |
|
|
65
|
+
| `rescue` | A root-cause ticket the rescue graph filed. |
|
|
66
|
+
| _no `q:` label_, cambium comment, open `agent/*` PR | Submitted: in the Release Queue, which is the PR's merge-queue state. |
|
|
67
|
+
| `priority:critical` … `priority:low` | Scheduling order (configurable). Unlabelled sorts after every listed priority. |
|
|
67
68
|
|
|
68
69
|
Only cambium moves these labels; workers have `graft issue label` and `graft issue edit` denied.
|
|
69
70
|
|
|
@@ -82,6 +83,8 @@ Each issue cambium touches gets one comment it owns and edits in place. Its firs
|
|
|
82
83
|
|
|
83
84
|
Set `graft.botLogin` to the Clock App's login and only markers written by it are trusted.
|
|
84
85
|
|
|
86
|
+
**Cost.** When a task, triage or rescue worker exits, cambium reads the `result` event at the end of its stream-json log (`total_cost_usd`, `usage`) and adds it to the marker's `cost`: `{ total, triage, task, rescue, lastLog }`, each phase a tally `{ usd, input, output, cacheRead, cacheCreation, runs }` (`src/cost.mjs`). It lives on GitHub, so it survives a restart, and every marker write keeps it. `lastLog` is the last run counted, so a re-checked exit is never counted twice. A worker with no `result` (killed, truncated log) adds nothing; a marker from before `cost` counts from zero. Each run also goes into a per-repo, per-UTC-day ledger in SQLite (`costs` table) for the heartbeat's daily figures; losing the database loses only that history. The rendered comment shows `cost $<total>`.
|
|
87
|
+
|
|
85
88
|
## Workers
|
|
86
89
|
|
|
87
90
|
A worker is `claude -p --output-format stream-json --verbose --thinking-display summarized --max-turns <maxTurns> --allowedTools Read,Edit,Write,Glob,Grep,Bash --disallowedTools "Bash(graft issue label:*),Bash(graft issue edit:*),Bash(gh:*)" <worker.args>`:
|
|
@@ -148,10 +151,12 @@ One JSON file. Everything but `repos` is optional; these are the defaults:
|
|
|
148
151
|
"graft": { "command": "graft", "app": null, "botLogin": null },
|
|
149
152
|
"worker": { "command": "claude", "args": [], "graftApp": null, "fetch": true, "env": {} },
|
|
150
153
|
"update": { "auto": false, "checkMinutes": 360 },
|
|
151
|
-
"gc": { "everyMinutes": 60, "graceMinutes": 60, "staleDays": 14 }
|
|
154
|
+
"gc": { "everyMinutes": 60, "graceMinutes": 60, "staleDays": 14 },
|
|
155
|
+
"costFlagUsd": 10
|
|
152
156
|
}
|
|
153
157
|
```
|
|
154
158
|
|
|
159
|
+
- `costFlagUsd`: an open issue whose task-phase cost is over this many US dollars with no merged PR closing it is `flagged` in the heartbeat; canopy lists it in the Inbox as a session candidate. `0` turns the flag off.
|
|
155
160
|
- `worker.env` is extra environment for every worker (string values only), e.g. Claude Code's own telemetry ([Tracing](#claude-codes-own-spans-under-the-worker)).
|
|
156
161
|
|
|
157
162
|
- The daemon re-reads the file before every tick, so caps, limits, `tickSeconds`, `repos`, priority labels, `release` and `worker` take effect without a restart (logged as `config reloaded: <keys>`). `host`, `engine`, `paths`, `logs` and `graft` are baked in at start: a change to them logs `restart cambium to apply` once and is not applied. A file that does not load, or has no `repos`, keeps the running config and logs one warning until it is fixed.
|
|
@@ -310,7 +315,9 @@ Written at the end of every tick to `paths.heartbeat` (default `~/.cambium/state
|
|
|
310
315
|
}
|
|
311
316
|
```
|
|
312
317
|
|
|
313
|
-
`
|
|
318
|
+
`repos[].cost` is `{ issues, days }`. `issues`: open issues in `q:intake`, `q:task`, `q:human` or `running` whose marker has a cost, most expensive first, each `{ number, title, url, labels, cost, flagged }` (`cost` as on the marker; `flagged` per `costFlagUsd`). `days`: the last 14 UTC days of the repo's ledger with any spend, oldest first, each `{ day, triage, task, rescue, total }` in US dollars. The top-level `costFlagUsd` echoes the config.
|
|
319
|
+
|
|
320
|
+
`engine`, `graphs`, `costFlagUsd` and `repos[].cost` were added within `version: 1`; a reader of an older heartbeat finds none of them.
|
|
314
321
|
|
|
315
322
|
The mode lives in the SQLite file (`paths.db`, table `control`), so `cambium status --json` returns `{ "mode", "heartbeat" }` even when the daemon is down. The daemon's own log is `<paths.logs>/cambium.log` (one `.1` kept); launchd's stderr goes to `<paths.logs>/cambium.launchd.err.log`.
|
|
316
323
|
|
|
@@ -470,8 +477,8 @@ flowchart TD
|
|
|
470
477
|
| `blocked` | step | `q:intake` | no |
|
|
471
478
|
| `close-obsolete` | step | none | no |
|
|
472
479
|
|
|
473
|
-
- **The verdict.** The worker's final message must end with a JSON block, validated by zod (`TriageResult`): `{ verdict: 'actionable' | 'needs-human' | 'duplicate' | 'obsolete' | 'defer' | 'blocked', duplicateOf?, blockedBy?, recheckHours?, reason, title?, acceptance? }` (`duplicateOf` is required on a duplicate and cannot be the issue itself; `blockedBy`, issue numbers, likewise on blocked). cambium reads it from the worker's stdout log: the last `result` event's `result` text, its last fenced block (else the bare JSON).
|
|
474
|
-
- **Outcomes.** `actionable` → a comment with the reason, the task title and acceptance checks (when given), then `q:task` with a task-graph marker at `start-worker`, attempt 1, history kept; the checks also ride in the marker (`triage`) into the task worker's prompt. The issue body is never rewritten: cambium's write path has no issue edit, and the author's text stays as written. `needs-human` → `q:human` + `why:triage
|
|
480
|
+
- **The verdict.** The worker's final message must end with a JSON block, validated by zod (`TriageResult`): `{ verdict: 'actionable' | 'needs-human' | 'duplicate' | 'obsolete' | 'defer' | 'blocked', duplicateOf?, blockedBy?, recheckHours?, reason, title?, acceptance?, ask? }` (`duplicateOf` is required on a duplicate and cannot be the issue itself; `blockedBy`, issue numbers, likewise on blocked). cambium reads it from the worker's stdout log: the last `result` event's `result` text, its last fenced block (else the bare JSON). `ask` (`src/ask.mjs`), read only on `needs-human`: `{ kind: 'decide' | 'verify' | 'chore' | 'approve', question, options?: { label, detail }[], recommendation?, interactive }`, `options` (two or more) required for `decide`. An invalid `ask` is dropped with a note in the reason, never failing the verdict, so the issue still lands in `q:human`. The prompt asks for `blocked`, not `needs-human`, when the issue waits on another open issue that is itself a human chore or check.
|
|
481
|
+
- **Outcomes.** `actionable` → a comment with the reason, the task title and acceptance checks (when given), then `q:task` with a task-graph marker at `start-worker`, attempt 1, history kept; the checks also ride in the marker (`triage`) into the task worker's prompt. The issue body is never rewritten: cambium's write path has no issue edit, and the author's text stays as written. `needs-human` → `q:human` + `why:triage` (plus `ask:<kind>`, and `ask:session` when `interactive`), the reason as the state comment's last failure; the marker keeps the `ask`, and the state comment renders it under the status line (question, numbered options with their detail, recommendation, how to answer). `duplicate` → closed with reason `duplicate` and `Duplicate of #n (<link>). <reason>`. `obsolete` (the work no longer applies, verified: the code path is gone, already fixed, its check already passed) → closed as `not_planned` with `Obsolete: <reason>`. `defer` → stays in `q:intake` at `deferred`, `blocked` → stays in `q:intake` at `blocked` (see below). A missing or invalid verdict (no `result` event, no JSON, a schema error) or a worker killed at `wallClockMinutes` (the unlabelled `triaging → to-human` edge) → `q:human` + `why:triage` with the error.
|
|
475
482
|
- **Read-only checkout.** Triage reads code to judge feasibility and duplicates, so it gets a checkout, but a disposable one: a detached worktree at `<root>/triage-<n>` on `origin/<base>` (the repo's worktree root), no install, hydrate or `prepare`, tools `Read,Glob,Grep,Bash` with Edit/Write and every graft/git write verb denied. It is removed outright after the verdict (no `graft status`: nothing in it is worth keeping).
|
|
476
483
|
- **Retries.** A worker that cannot start, or a lost `running` lease on a `q:intake` issue (Sweep), goes back to `q:intake` with attempt + 1, then to `q:human` once `phases.dispatch.maxAttempts` are spent. A marker from another graph or a finished triage starts over at attempt 1.
|
|
477
484
|
- **Defer, not a human.** A verdict that hangs on something a machine can check later (a CD run that has not happened, a release, a merge, a deploy) is `defer`, never `needs-human`: asking a human to verify what a machine can is what the factory exists to avoid. The issue rests at `deferred` with `deferUntil` (now + `recheckHours`, else `phases.dispatch.deferHours`, default 6) and `deferReason` in its marker; Schedule skips it until then, and the next triage gets the reason in its prompt ("check it now"). A defer is not a failed attempt. A deferred issue whose GitHub "blocked by" issues are all closed, at least one after it was deferred, is triaged again at the next tick instead of waiting for `deferUntil`. After `phases.dispatch.maxDefers` (default 8) it goes to `q:human` + `why:triage` with the last reason. While it waits it carries `wait:deferred`.
|
|
@@ -575,7 +582,7 @@ flowchart TD
|
|
|
575
582
|
| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
576
583
|
| `many-queues` | More than one `q:*` label. |
|
|
577
584
|
| `running-unqueued` | `running` outside `q:task` / `q:intake` (not another host's lease). |
|
|
578
|
-
| `why-outside-human` | A `why:*` label without `q:human`.
|
|
585
|
+
| `why-outside-human` | A `why:*` or `ask:*` label without `q:human`. |
|
|
579
586
|
| `wait-outside-intake` | A `wait:*` label without `q:intake`. |
|
|
580
587
|
| `closed-labelled` | A closed issue still carrying cambium labels (`q:promote` aside). |
|
|
581
588
|
| `merged-open` | Its agent PR merged, the issue open in no queue. |
|
|
@@ -599,7 +606,7 @@ A label change counts after `rescue.graceMinutes` (default 30) since the issue's
|
|
|
599
606
|
}
|
|
600
607
|
```
|
|
601
608
|
|
|
602
|
-
**Actions**, in order: `clear-labels` (cambium labels only), `close-pr` (an open agent PR), then at most one move, last: `requeue` (to `q:task` or `q:intake` at its graph's entry, attempt 1, history kept, outcome `rescued`), `adopt-pr` (the task graph's `submit`, as the reaper adopts), `close-issue` (`completed` or `not_planned`), or `to-human` (`why:rescue`). `"stuck": false` takes no actions. Before anything is touched, the verdict is checked against the item (`checkPlan`); a missing, invalid or unworkable verdict, or a worker killed at its cap, sends the item to `q:human` + `why:rescue` with the error. When the worker exits the item is checked again: if none of the symptoms it started from still holds (someone fixed it meanwhile), the verdict is dropped.
|
|
609
|
+
**Actions**, in order: `clear-labels` (cambium labels only), `close-pr` (an open agent PR), then at most one move, last: `requeue` (to `q:task` or `q:intake` at its graph's entry, attempt 1, history kept, outcome `rescued`), `adopt-pr` (the task graph's `submit`, as the reaper adopts), `close-issue` (`completed` or `not_planned`), or `to-human` (`why:rescue`; an optional `ask`, triage's shape, adds its `ask:*` labels and is rendered in the state comment). `"stuck": false` takes no actions. Before anything is touched, the verdict is checked against the item (`checkPlan`); a missing, invalid or unworkable verdict, or a worker killed at its cap, sends the item to `q:human` + `why:rescue` with the error. When the worker exits the item is checked again: if none of the symptoms it started from still holds (someone fixed it meanwhile), the verdict is dropped.
|
|
603
610
|
|
|
604
611
|
**Root causes.** Each becomes a ticket in its `repo`: the item's own, or `rescue.causeRepo` (e.g. the repo that holds cambium itself). A ticket is labelled `q:intake` and `rescue`, so [triage](#the-dispatch-graph) takes it from there; its body is the worker's, then how the item got stuck, then a hidden `<!-- cambium:rescue-cause <fingerprint> -->`. When an open issue in that repo already carries the fingerprint, cambium comments there that the cause struck again instead of filing it twice (a repo cambium does not read is de-duplicated by what this host filed).
|
|
605
612
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@oakgreencc/cambium",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.45",
|
|
4
4
|
"description": "The scheduler daemon: reads the q:* queues from GitHub every tick, runs headless claude workers in their own worktrees up to a cap, and makes every queue transition itself.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|
package/src/ask.mjs
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* A typed ask: what a person in the Human Queue is asked to do, so `q:human` is not one undifferentiated bucket.
|
|
4
|
+
* Triage (phases/dispatch.mjs `TriageResult`) and rescue's `to-human` action (phases/rescue.mjs) may carry one.
|
|
5
|
+
* cambium projects it as `ask:<kind>` labels next to `why:*` ({@link askLabels}), and renders it in the state
|
|
6
|
+
* comment ({@link renderAsk}) so the person can answer with one comment. The labels leave with `q:human`
|
|
7
|
+
* (labels.mjs `move`).
|
|
8
|
+
*
|
|
9
|
+
* - `decide`: a choice only a person can make; `options` lists the choices (two or more).
|
|
10
|
+
* - `verify`: a check only a person can make (a physical device, an account a machine cannot reach).
|
|
11
|
+
* - `chore`: offline work (a store listing, paperwork, a console setting); a checklist, not a session.
|
|
12
|
+
* - `approve`: a yes or no on something already written.
|
|
13
|
+
* `interactive`: whether a live session with the code would help answer it.
|
|
14
|
+
*/
|
|
15
|
+
import { z } from 'zod'
|
|
16
|
+
|
|
17
|
+
export const ASK_KINDS = /** @type {const} */ (['decide', 'verify', 'chore', 'approve'])
|
|
18
|
+
/** Marks an ask whose answer a live session with the code would help (`interactive`). */
|
|
19
|
+
export const ASK_SESSION = 'ask:session'
|
|
20
|
+
|
|
21
|
+
const text = z.string().trim().min(1)
|
|
22
|
+
|
|
23
|
+
export const Ask = z
|
|
24
|
+
.object({
|
|
25
|
+
kind: z.enum(ASK_KINDS),
|
|
26
|
+
question: text,
|
|
27
|
+
options: z
|
|
28
|
+
.array(z.object({ label: text, detail: text }))
|
|
29
|
+
.min(1)
|
|
30
|
+
.optional(),
|
|
31
|
+
recommendation: text.optional(),
|
|
32
|
+
interactive: z.boolean(),
|
|
33
|
+
})
|
|
34
|
+
.refine((a) => a.kind !== 'decide' || (a.options?.length ?? 0) >= 2, {
|
|
35
|
+
message: 'a decide ask needs two or more options',
|
|
36
|
+
path: ['options'],
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
/** @typedef {z.infer<typeof Ask>} AskT */
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* An ask as a verdict field: a valid one is kept, an invalid one is dropped (undefined) without failing the verdict
|
|
43
|
+
* around it, so an issue that needs a human still gets to one.
|
|
44
|
+
*/
|
|
45
|
+
export const LenientAsk = Ask.optional().catch(undefined)
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Why `raw` is not a valid ask, or null when it is (or is absent).
|
|
49
|
+
* @param {unknown} raw @returns {string | null}
|
|
50
|
+
*/
|
|
51
|
+
export function askError(raw) {
|
|
52
|
+
if (raw === undefined) return null
|
|
53
|
+
const r = Ask.safeParse(raw)
|
|
54
|
+
return r.success
|
|
55
|
+
? null
|
|
56
|
+
: r.error.issues.map((i) => (i.path.length ? `${i.path.join('.')}: ${i.message}` : i.message)).join('; ')
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** The labels an ask adds beside `q:human`. @param {AskT | null | undefined} ask @returns {string[]} */
|
|
60
|
+
export const askLabels = (ask) => (ask ? [`ask:${ask.kind}`, ...(ask.interactive ? [ASK_SESSION] : [])] : [])
|
|
61
|
+
|
|
62
|
+
const HEAD = /** @type {const} */ ({
|
|
63
|
+
decide: 'Decide',
|
|
64
|
+
verify: 'Verify',
|
|
65
|
+
chore: 'Offline chore',
|
|
66
|
+
approve: 'Approve',
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The ask as markdown for the state comment: the question, the options, the recommendation, and how to answer.
|
|
71
|
+
* @param {AskT} ask @returns {string[]} lines
|
|
72
|
+
*/
|
|
73
|
+
export function renderAsk(ask) {
|
|
74
|
+
const one = (/** @type {string} */ s) => s.replace(/\n+/g, ' ')
|
|
75
|
+
return [
|
|
76
|
+
`**${HEAD[ask.kind]}:** ${one(ask.question)}`,
|
|
77
|
+
...(ask.options?.length
|
|
78
|
+
? ['', ...ask.options.map((o, i) => `${i + 1}. **${one(o.label)}**: ${one(o.detail)}`)]
|
|
79
|
+
: []),
|
|
80
|
+
...(ask.recommendation ? ['', `**Recommendation:** ${one(ask.recommendation)}`] : []),
|
|
81
|
+
'',
|
|
82
|
+
`_${
|
|
83
|
+
ask.kind === 'decide' && ask.options?.length
|
|
84
|
+
? 'Answer with one comment naming your choice'
|
|
85
|
+
: ask.kind === 'approve'
|
|
86
|
+
? 'Answer with one comment: yes, or what to change'
|
|
87
|
+
: 'Comment when it is done, with anything the work needs'
|
|
88
|
+
}; cambium triages the issue again on your reply.${ask.interactive ? ' A live session with the code would help here.' : ''}_`,
|
|
89
|
+
]
|
|
90
|
+
}
|
package/src/config.mjs
CHANGED
|
@@ -49,7 +49,10 @@ import { join, resolve } from 'node:path'
|
|
|
49
49
|
* worker: { command: string, args: string[], graftApp: string | null, fetch: boolean, env: Record<string, string> },
|
|
50
50
|
* update: { auto: boolean, checkMinutes: number },
|
|
51
51
|
* gc: GcConfig,
|
|
52
|
+
* costFlagUsd: number,
|
|
52
53
|
* }} Config
|
|
54
|
+
* `costFlagUsd`: an issue whose task-phase spend (cost.mjs) is over this many US dollars with no merged PR is
|
|
55
|
+
* flagged in the heartbeat, and canopy's Inbox, as a session candidate. 0 turns the flag off.
|
|
53
56
|
* @typedef {{
|
|
54
57
|
* everyMinutes: number, staleHours: number, graceMinutes: number, cooldownHours: number, maxRescues: number,
|
|
55
58
|
* causeRepo: string | null,
|
|
@@ -109,6 +112,7 @@ export function defaults(home = DEFAULT_HOME) {
|
|
|
109
112
|
worker: { command: 'claude', args: [], graftApp: null, fetch: true, env: {} },
|
|
110
113
|
update: { auto: false, checkMinutes: 360 },
|
|
111
114
|
gc: { everyMinutes: 60, graceMinutes: 60, staleDays: 14 },
|
|
115
|
+
costFlagUsd: 10,
|
|
112
116
|
}
|
|
113
117
|
}
|
|
114
118
|
|
|
@@ -185,6 +189,8 @@ export function resolveConfig(raw = {}) {
|
|
|
185
189
|
const causeRepo = config.rescue.causeRepo
|
|
186
190
|
if (causeRepo !== null && (typeof causeRepo !== 'string' || !/^[^/\s]+\/[^/\s]+$/.test(causeRepo)))
|
|
187
191
|
throw new Error(`config: rescue.causeRepo must be owner/name or null, got ${JSON.stringify(causeRepo)}`)
|
|
192
|
+
if (!(typeof config.costFlagUsd === 'number' && config.costFlagUsd >= 0))
|
|
193
|
+
throw new Error(`config: costFlagUsd must be a number, at least 0, got ${JSON.stringify(config.costFlagUsd)}`)
|
|
188
194
|
if (config.engine !== 'legacy' && config.engine !== 'graph')
|
|
189
195
|
throw new Error(`config: engine must be "legacy" or "graph", got ${JSON.stringify(config.engine)}`)
|
|
190
196
|
const merging = config.repos.filter((r) => r.merge === 'green')
|
package/src/cost.mjs
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* Per-issue cost accounting. When a worker exits, its stream-json log's `result` event (`total_cost_usd`, `usage`)
|
|
4
|
+
* is added to the issue's marker, per phase and overall, so the totals live on GitHub and survive a restart. Each
|
|
5
|
+
* run is also added to a per-repo, per-day ledger in SQLite for the heartbeat's daily figures; losing that file
|
|
6
|
+
* loses only the daily history.
|
|
7
|
+
*/
|
|
8
|
+
import { setMarker } from './issue-state.mjs'
|
|
9
|
+
import { resultEvent } from './tracing.mjs'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @typedef {{ usd: number, input: number, output: number, cacheRead: number, cacheCreation: number, runs: number }} Tally
|
|
13
|
+
* @typedef {'triage' | 'task' | 'rescue'} CostPhase
|
|
14
|
+
* @typedef {{ total: Tally, triage: Tally, task: Tally, rescue: Tally, lastLog: string | null }} Cost
|
|
15
|
+
* `lastLog`: the stdout log of the last run counted, so a run is never counted twice.
|
|
16
|
+
* @typedef {{ day: string, triage: number, task: number, rescue: number, total: number }} CostDay
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export const COST_PHASES = /** @type {const} */ (['triage', 'task', 'rescue'])
|
|
20
|
+
|
|
21
|
+
/** A worker row's `phase` → the cost phase it counts under. */
|
|
22
|
+
const PHASE_OF = /** @type {Record<string, CostPhase>} */ ({ dispatch: 'triage', task: 'task', rescue: 'rescue' })
|
|
23
|
+
|
|
24
|
+
/** @returns {Tally} */
|
|
25
|
+
export const zeroTally = () => ({ usd: 0, input: 0, output: 0, cacheRead: 0, cacheCreation: 0, runs: 0 })
|
|
26
|
+
|
|
27
|
+
const num = (/** @type {unknown} */ v) => (typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : 0)
|
|
28
|
+
|
|
29
|
+
/** @param {any} t @returns {Tally} */
|
|
30
|
+
const tallyOf = (t) => ({
|
|
31
|
+
usd: num(t?.usd),
|
|
32
|
+
input: num(t?.input),
|
|
33
|
+
output: num(t?.output),
|
|
34
|
+
cacheRead: num(t?.cacheRead),
|
|
35
|
+
cacheCreation: num(t?.cacheCreation),
|
|
36
|
+
runs: num(t?.runs),
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
/** @param {Tally} a @param {Tally} b @returns {Tally} */
|
|
40
|
+
const plus = (a, b) => ({
|
|
41
|
+
usd: Math.round((a.usd + b.usd) * 1e6) / 1e6,
|
|
42
|
+
input: a.input + b.input,
|
|
43
|
+
output: a.output + b.output,
|
|
44
|
+
cacheRead: a.cacheRead + b.cacheRead,
|
|
45
|
+
cacheCreation: a.cacheCreation + b.cacheCreation,
|
|
46
|
+
runs: a.runs + b.runs,
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* A marker's cost, zeros when it has none (a marker from before cost accounting) or it is malformed.
|
|
51
|
+
* @param {{ cost?: unknown } | null | undefined} marker @returns {Cost}
|
|
52
|
+
*/
|
|
53
|
+
export function costOf(marker) {
|
|
54
|
+
const c = /** @type {any} */ (marker?.cost)
|
|
55
|
+
return {
|
|
56
|
+
total: tallyOf(c?.total),
|
|
57
|
+
triage: tallyOf(c?.triage),
|
|
58
|
+
task: tallyOf(c?.task),
|
|
59
|
+
rescue: tallyOf(c?.rescue),
|
|
60
|
+
lastLog: typeof c?.lastLog === 'string' ? c.lastLog : null,
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* One run's cost and tokens from its stream-json log, or null when the log has no `result` event (a killed
|
|
66
|
+
* worker, a truncated log) or it carries no cost. Never throws.
|
|
67
|
+
* @param {string | Iterable<string | Record<string, any>>} stream @returns {Tally | null}
|
|
68
|
+
*/
|
|
69
|
+
export function runCost(stream) {
|
|
70
|
+
let r
|
|
71
|
+
try {
|
|
72
|
+
r = resultEvent(stream)
|
|
73
|
+
} catch {
|
|
74
|
+
return null
|
|
75
|
+
}
|
|
76
|
+
if (!r || typeof r.total_cost_usd !== 'number' || !Number.isFinite(r.total_cost_usd)) return null
|
|
77
|
+
const u = r.usage ?? {}
|
|
78
|
+
return {
|
|
79
|
+
usd: num(r.total_cost_usd),
|
|
80
|
+
input: num(u.input_tokens),
|
|
81
|
+
output: num(u.output_tokens),
|
|
82
|
+
cacheRead: num(u.cache_read_input_tokens),
|
|
83
|
+
cacheCreation: num(u.cache_creation_input_tokens),
|
|
84
|
+
runs: 1,
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* `cost` with one more run added to `phase` and the total. A run whose log was the last one counted is not added
|
|
90
|
+
* again.
|
|
91
|
+
* @param {unknown} cost a marker's `cost`, possibly missing @param {CostPhase} phase @param {Tally} run
|
|
92
|
+
* @param {string | null} log @returns {Cost}
|
|
93
|
+
*/
|
|
94
|
+
export function addCost(cost, phase, run, log) {
|
|
95
|
+
const c = costOf({ cost })
|
|
96
|
+
if (log && c.lastLog === log) return c
|
|
97
|
+
return { ...c, total: plus(c.total, run), [phase]: plus(c[phase], run), lastLog: log ?? c.lastLog }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** The UTC day of a time, `YYYY-MM-DD`. @param {number} ms */
|
|
101
|
+
export const dayOf = (ms) => new Date(ms).toISOString().slice(0, 10)
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* A finished worker's cost onto its issue's marker (and the day's ledger). Nothing happens when the worker phase
|
|
105
|
+
* is not counted, the log has no `result` (a killed worker), or this run was already counted. Never throws:
|
|
106
|
+
* accounting must not fail a tick.
|
|
107
|
+
*
|
|
108
|
+
* @param {{ config: import('./config.mjs').Config, github: import('./github.mjs').GitHub,
|
|
109
|
+
* spawner: { readLog(path: string): string },
|
|
110
|
+
* store: { addCost?(repo: string, day: string, phase: string, run: Tally): void },
|
|
111
|
+
* log: (level: 'info' | 'warn' | 'error', msg: string, where?: { repo?: string, issue?: number }) => void }} deps
|
|
112
|
+
* @param {import('./github.mjs').Issue | null | undefined} issue
|
|
113
|
+
* @param {{ repo: string, issue: number, phase: string, stdout: string }} w @param {number} now
|
|
114
|
+
* @returns {Promise<Cost | null>} the marker's new cost, when one was recorded
|
|
115
|
+
*/
|
|
116
|
+
export async function recordCost(deps, issue, w, now) {
|
|
117
|
+
const phase = PHASE_OF[w.phase]
|
|
118
|
+
if (!phase) return null
|
|
119
|
+
try {
|
|
120
|
+
const run = runCost(deps.spawner.readLog(w.stdout))
|
|
121
|
+
if (!run) return null
|
|
122
|
+
const marker = issue?.stateComment?.marker
|
|
123
|
+
if (marker && costOf(marker).lastLog === w.stdout) return null
|
|
124
|
+
let cost = null
|
|
125
|
+
if (issue && marker) {
|
|
126
|
+
cost = addCost(marker.cost, phase, run, w.stdout)
|
|
127
|
+
await setMarker(deps, w.repo, issue, { ...marker, cost })
|
|
128
|
+
}
|
|
129
|
+
deps.store.addCost?.(w.repo, dayOf(now), phase, run)
|
|
130
|
+
return cost
|
|
131
|
+
} catch (err) {
|
|
132
|
+
deps.log('warn', `cost: ${/** @type {Error} */ (err)?.message ?? err}`, { repo: w.repo, issue: w.issue })
|
|
133
|
+
return null
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Whether an issue's task spend is over `flagUsd` with no merged PR for it: a candidate for a human session.
|
|
139
|
+
* `flagUsd` 0 switches the flag off.
|
|
140
|
+
* @param {Cost} cost @param {number} flagUsd @param {boolean} merged
|
|
141
|
+
*/
|
|
142
|
+
export const overBudget = (cost, flagUsd, merged) => flagUsd > 0 && cost.task.usd > flagUsd && !merged
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Ledger rows → one entry per day (oldest first), each phase's spend and the total.
|
|
146
|
+
* @param {{ day: string, phase: string, usd: number }[]} rows @returns {CostDay[]}
|
|
147
|
+
*/
|
|
148
|
+
export function costDays(rows) {
|
|
149
|
+
/** @type {Map<string, CostDay>} */
|
|
150
|
+
const days = new Map()
|
|
151
|
+
for (const r of rows) {
|
|
152
|
+
const d = days.get(r.day) ?? { day: r.day, triage: 0, task: 0, rescue: 0, total: 0 }
|
|
153
|
+
const usd = num(r.usd)
|
|
154
|
+
if (r.phase === 'triage' || r.phase === 'task' || r.phase === 'rescue') d[r.phase] = round(d[r.phase] + usd)
|
|
155
|
+
d.total = round(d.total + usd)
|
|
156
|
+
days.set(r.day, d)
|
|
157
|
+
}
|
|
158
|
+
return [...days.values()].sort((a, b) => a.day.localeCompare(b.day))
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const round = (/** @type {number} */ n) => Math.round(n * 1e6) / 1e6
|
package/src/graphs/dispatch.mjs
CHANGED
|
@@ -11,12 +11,14 @@
|
|
|
11
11
|
import { z } from 'zod'
|
|
12
12
|
import { defineGraph, defineNode, defineWait, END } from '../graph/index.mjs'
|
|
13
13
|
import { iso, key, markerOf, MIN, setLabels, setMarker } from '../issue-state.mjs'
|
|
14
|
+
import { Ask, askLabels } from '../ask.mjs'
|
|
14
15
|
import { move, Q, WAIT, WHY } from '../labels.mjs'
|
|
15
16
|
import { newMarker, withHistory } from '../marker.mjs'
|
|
16
17
|
import { readTriage, TriageResult, triageComment, triagePrompt } from '../phases/dispatch.mjs'
|
|
17
18
|
import { humanReplies, quoted, replyContextOf } from '../human-reply.mjs'
|
|
18
19
|
import { traceWorker } from '../tracing.mjs'
|
|
19
20
|
import { fingerprintOf } from '../triage-inputs.mjs'
|
|
21
|
+
import { recordCost } from '../cost.mjs'
|
|
20
22
|
import { stopOverdue } from '../workers.mjs'
|
|
21
23
|
import { task } from './task.mjs'
|
|
22
24
|
|
|
@@ -69,6 +71,7 @@ async function land(ctx, node, marker, { extra = [], status = 'waiting' } = {})
|
|
|
69
71
|
await setMarker(ctx, slug, issue, {
|
|
70
72
|
...graphMarker(marker),
|
|
71
73
|
...(PRE_VERDICT.has(node) ? {} : { reply: null }),
|
|
74
|
+
...(node === 'to-human' ? {} : { ask: null }),
|
|
72
75
|
graph: dispatch.name,
|
|
73
76
|
graphVersion: dispatch.version,
|
|
74
77
|
phase: 'dispatch',
|
|
@@ -276,6 +279,7 @@ const triaging = declare(
|
|
|
276
279
|
)
|
|
277
280
|
return { ready: false }
|
|
278
281
|
if (!snap) return { ready: false }
|
|
282
|
+
await recordCost(ctx, issue, w, ctx.tick.now)
|
|
279
283
|
if (issue?.state !== 'OPEN' || !issue.labels.includes(Q.intake)) {
|
|
280
284
|
ctx.log('info', 'triage worker gone; issue is closed or left q:intake', { repo: w.repo, issue: w.issue })
|
|
281
285
|
return { ready: true, output: { exit: 'gone' } }
|
|
@@ -391,17 +395,19 @@ const toTask = declare(
|
|
|
391
395
|
},
|
|
392
396
|
)
|
|
393
397
|
|
|
398
|
+
const HumanIn = z.object({ ...failure, ask: Ask.optional() })
|
|
394
399
|
const toHuman = declare(
|
|
395
400
|
defineNode({
|
|
396
401
|
name: 'to-human',
|
|
397
|
-
input:
|
|
402
|
+
input: HumanIn,
|
|
398
403
|
output: Empty,
|
|
399
404
|
/**
|
|
400
|
-
* Into the Human Queue with `why:triage` and the reason in the state comment
|
|
401
|
-
*
|
|
402
|
-
*
|
|
405
|
+
* Into the Human Queue with `why:triage` and the reason in the state comment; a needs-human verdict's `ask` adds
|
|
406
|
+
* its `ask:*` labels and is rendered there too. A needs-human verdict keeps the triage inputs `start-triage`
|
|
407
|
+
* recorded (re-triage skips the issue while they hold); a spent budget drops them.
|
|
408
|
+
* @param {z.infer<typeof HumanIn>} input @param {DispatchCtx} ctx
|
|
403
409
|
*/
|
|
404
|
-
run: async (what, ctx) => {
|
|
410
|
+
run: async ({ ask, ...what }, ctx) => {
|
|
405
411
|
const issue = issueOf(ctx)
|
|
406
412
|
const m = markerOf(issue)
|
|
407
413
|
await land(
|
|
@@ -410,12 +416,13 @@ const toHuman = declare(
|
|
|
410
416
|
{
|
|
411
417
|
...withHistory(m, { at: iso(ctx.tick.now), attempt: m.attempt, ...what }),
|
|
412
418
|
lastFailure: what.reason,
|
|
419
|
+
ask: ask ?? null,
|
|
413
420
|
fingerprint: what.outcome === 'needs-human' ? (m.fingerprint ?? null) : null,
|
|
414
421
|
host: null,
|
|
415
422
|
start: null,
|
|
416
423
|
input: null,
|
|
417
424
|
},
|
|
418
|
-
{ extra: [WHY.triage] },
|
|
425
|
+
{ extra: [WHY.triage, ...askLabels(ask)] },
|
|
419
426
|
)
|
|
420
427
|
ctx.log('warn', `→ q:human (${WHY.triage}): ${what.reason.split('\n')[0]}`, {
|
|
421
428
|
repo: ctx.subject.slug,
|
|
@@ -424,7 +431,11 @@ const toHuman = declare(
|
|
|
424
431
|
return {}
|
|
425
432
|
},
|
|
426
433
|
}),
|
|
427
|
-
{
|
|
434
|
+
{
|
|
435
|
+
queue: Q.human,
|
|
436
|
+
running: false,
|
|
437
|
+
about: 'Into q:human with why:triage (plus ask:* from a needs-human ask) and the reason in the state comment.',
|
|
438
|
+
},
|
|
428
439
|
)
|
|
429
440
|
|
|
430
441
|
const DupIn = z.object({ duplicateOf: z.number().int(), reason: z.string(), log: z.string() })
|
|
@@ -666,7 +677,17 @@ export const dispatch = defineGraph({
|
|
|
666
677
|
['triaging', 'deferred', defer, (o) => ({ ...pick(o.result, ['reason', 'recheckHours']), log: o.log })],
|
|
667
678
|
['triaging', 'blocked', blocked, (o) => ({ blockedBy: o.result.blockedBy, reason: o.result.reason, log: o.log })],
|
|
668
679
|
['triaging', 'close-obsolete', obsolete, (o) => ({ reason: o.result.reason, log: o.log })],
|
|
669
|
-
[
|
|
680
|
+
[
|
|
681
|
+
'triaging',
|
|
682
|
+
'to-human',
|
|
683
|
+
needsHuman,
|
|
684
|
+
(o) => ({
|
|
685
|
+
reason: o.result.reason,
|
|
686
|
+
outcome: 'needs-human',
|
|
687
|
+
log: o.log,
|
|
688
|
+
...(o.result.ask ? { ask: o.result.ask } : {}),
|
|
689
|
+
}),
|
|
690
|
+
],
|
|
670
691
|
['triaging', 'to-human'],
|
|
671
692
|
['retry-triage', 'start-triage', attemptsLeft, () => ({})],
|
|
672
693
|
['retry-triage', 'to-human'],
|
package/src/graphs/pr.mjs
CHANGED
|
@@ -399,6 +399,7 @@ const toHuman = declare(
|
|
|
399
399
|
...withHistory(m, { ...nodeAt(what), at: iso(ctx.tick.now), attempt: m.attempt, outcome, reason, pr }),
|
|
400
400
|
phase: 'task',
|
|
401
401
|
lastFailure: reason,
|
|
402
|
+
...(m.ask ? { ask: null } : {}),
|
|
402
403
|
host: null,
|
|
403
404
|
start: null,
|
|
404
405
|
input: null,
|
package/src/graphs/rescue.mjs
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { z } from 'zod'
|
|
15
15
|
import { advance, defineGraph, defineNode, defineWait, END } from '../graph/index.mjs'
|
|
16
|
+
import { askLabels } from '../ask.mjs'
|
|
16
17
|
import { iso, key, markerOf, setLabels, setMarker } from '../issue-state.mjs'
|
|
17
18
|
import { move, Q, RESCUE_LABEL, WHY } from '../labels.mjs'
|
|
18
19
|
import { newMarker, withHistory } from '../marker.mjs'
|
|
@@ -31,6 +32,7 @@ import {
|
|
|
31
32
|
seenAgain,
|
|
32
33
|
} from '../phases/rescue.mjs'
|
|
33
34
|
import { traceWorker } from '../tracing.mjs'
|
|
35
|
+
import { recordCost } from '../cost.mjs'
|
|
34
36
|
import { stopOverdue } from '../workers.mjs'
|
|
35
37
|
import { dispatch } from './dispatch.mjs'
|
|
36
38
|
import { issueOf, task } from './task.mjs'
|
|
@@ -85,10 +87,12 @@ const rescued = (ctx, m, reason) =>
|
|
|
85
87
|
withHistory(m, { at: iso(ctx.tick.now), attempt: m.attempt, outcome: 'rescued', reason })
|
|
86
88
|
|
|
87
89
|
/**
|
|
88
|
-
* Into the Human Queue with `why:rescue`, resting at this graph's `to-human
|
|
90
|
+
* Into the Human Queue with `why:rescue`, resting at this graph's `to-human`; an `ask` adds its `ask:*` labels and is
|
|
91
|
+
* rendered in the state comment.
|
|
89
92
|
* @param {RescueCtx} ctx @param {Issue} issue @param {string} reason @param {string} outcome
|
|
93
|
+
* @param {import('../ask.mjs').AskT} [ask]
|
|
90
94
|
*/
|
|
91
|
-
async function handToHuman(ctx, issue, reason, outcome) {
|
|
95
|
+
async function handToHuman(ctx, issue, reason, outcome, ask) {
|
|
92
96
|
const m = markerOf(issue)
|
|
93
97
|
await place(
|
|
94
98
|
ctx,
|
|
@@ -102,11 +106,12 @@ async function handToHuman(ctx, issue, reason, outcome) {
|
|
|
102
106
|
status: 'waiting',
|
|
103
107
|
enteredAt: iso(ctx.tick.now),
|
|
104
108
|
lastFailure: reason,
|
|
109
|
+
ask: ask ?? null,
|
|
105
110
|
host: null,
|
|
106
111
|
start: null,
|
|
107
112
|
input: null,
|
|
108
113
|
},
|
|
109
|
-
move(issue.labels, Q.human, { extra: [WHY.rescue] }),
|
|
114
|
+
move(issue.labels, Q.human, { extra: [WHY.rescue, ...askLabels(ask)] }),
|
|
110
115
|
)
|
|
111
116
|
}
|
|
112
117
|
|
|
@@ -151,7 +156,7 @@ async function act(ctx, issue, a) {
|
|
|
151
156
|
case 'requeue':
|
|
152
157
|
return requeue(ctx, issue, a.queue, a.reason)
|
|
153
158
|
case 'to-human':
|
|
154
|
-
return handToHuman(ctx, issue, a.reason, 'rescued')
|
|
159
|
+
return handToHuman(ctx, issue, a.reason, 'rescued', a.ask)
|
|
155
160
|
case 'close-issue': {
|
|
156
161
|
const m = markerOf(issue)
|
|
157
162
|
await place(
|
|
@@ -304,6 +309,7 @@ const diagnosing = defineWait({
|
|
|
304
309
|
return { ready: false }
|
|
305
310
|
if (!snap || !issue) return { ready: false }
|
|
306
311
|
traceWorker(ctx, w)
|
|
312
|
+
await recordCost(ctx, issue, w, ctx.tick.now)
|
|
307
313
|
if (!stuck) {
|
|
308
314
|
ctx.log('info', 'rescue: no longer stuck; verdict not applied', { repo: slug, issue: number })
|
|
309
315
|
return { ready: true, output: { exit: 'resolved' } }
|
package/src/graphs/task.mjs
CHANGED
|
@@ -17,6 +17,7 @@ import { withHistory } from '../marker.mjs'
|
|
|
17
17
|
import { linkIssue, prFor, taskExit, taskPrompt } from '../phases/task.mjs'
|
|
18
18
|
import { PREPARE_FAILED_LINE } from '../prepare.mjs'
|
|
19
19
|
import { traceWorker } from '../tracing.mjs'
|
|
20
|
+
import { recordCost } from '../cost.mjs'
|
|
20
21
|
import { baseRed, isRepair } from '../base-health.mjs'
|
|
21
22
|
import { stopOverdue } from '../workers.mjs'
|
|
22
23
|
|
|
@@ -277,6 +278,7 @@ const workerRunning = declare(
|
|
|
277
278
|
)
|
|
278
279
|
return { ready: false }
|
|
279
280
|
if (!snap) return { ready: false } // a dead worker waits for a readable snapshot
|
|
281
|
+
await recordCost(ctx, issue, w, ctx.tick.now)
|
|
280
282
|
if (issue?.state !== 'OPEN') {
|
|
281
283
|
ctx.log('info', 'worker gone; issue is closed or no longer queued', { repo: w.repo, issue: w.issue })
|
|
282
284
|
return { ready: true, output: { exit: 'gone' } }
|
|
@@ -431,6 +433,7 @@ const toHuman = declare(
|
|
|
431
433
|
{
|
|
432
434
|
...withHistory(m, { ...nodeAt(what), at: iso(ctx.tick.now), attempt: m.attempt, outcome, reason, pr, log }),
|
|
433
435
|
lastFailure: reason,
|
|
436
|
+
...(m.ask ? { ask: null } : {}),
|
|
434
437
|
host: null,
|
|
435
438
|
start: null,
|
|
436
439
|
input: null,
|
package/src/issue-state.mjs
CHANGED
|
@@ -28,6 +28,9 @@ export async function setMarker(deps, slug, issue, marker) {
|
|
|
28
28
|
// takes it, so later ticks parent on the same trace. One that already has a trace keeps it.
|
|
29
29
|
const tp = /** @type {any} */ (deps).subject?.traceparent
|
|
30
30
|
if (tp && !marker.traceparent) marker = { ...marker, traceparent: tp }
|
|
31
|
+
// Cost accumulates for the life of the issue (cost.mjs): a write that starts a fresh marker keeps it.
|
|
32
|
+
const cost = issue.stateComment?.marker?.cost
|
|
33
|
+
if (cost && !marker.cost) marker = { ...marker, cost }
|
|
31
34
|
const maxAttempts = deps.config.phases[marker.phase]?.maxAttempts
|
|
32
35
|
const id = await deps.github.writeState(
|
|
33
36
|
slug,
|
package/src/labels.mjs
CHANGED
|
@@ -13,6 +13,10 @@ export const RUNNING = 'running'
|
|
|
13
13
|
* `blockedBy` issues. Any queue move drops them unless it names one again.
|
|
14
14
|
*/
|
|
15
15
|
export const WAIT = /** @type {const} */ ({ deferred: 'wait:deferred', blocked: 'wait:blocked' })
|
|
16
|
+
/**
|
|
17
|
+
* Why an issue is in q:human. Beside it, an `ask:*` label (ask.mjs) may say what the person is asked to do. Both
|
|
18
|
+
* leave with `q:human`; a new hand-off into it replaces the ask.
|
|
19
|
+
*/
|
|
16
20
|
export const WHY = /** @type {const} */ ({
|
|
17
21
|
budget: 'why:budget',
|
|
18
22
|
reject: 'why:reject',
|
|
@@ -33,11 +37,14 @@ export const queueOf = (labels) => labels.find((l) => l.startsWith('q:')) ?? nul
|
|
|
33
37
|
*/
|
|
34
38
|
export function move(labels, target, { running = false, extra = [] } = {}) {
|
|
35
39
|
const want = new Set([...(target ? [target] : []), ...extra, ...(running ? [RUNNING] : [])])
|
|
40
|
+
// A hand-off into q:human names its reason (`why:*`); its ask, if any, replaces the one there before.
|
|
41
|
+
const handOff = target === Q.human && extra.some((l) => l.startsWith('why:'))
|
|
36
42
|
const leaving = (/** @type {string} */ l) =>
|
|
37
43
|
!want.has(l) &&
|
|
38
44
|
((l.startsWith('q:') && l !== target) ||
|
|
39
45
|
l === RUNNING ||
|
|
40
46
|
(l.startsWith('why:') && target !== Q.human) ||
|
|
47
|
+
(l.startsWith('ask:') && (target !== Q.human || handOff)) ||
|
|
41
48
|
l.startsWith('wait:'))
|
|
42
49
|
return {
|
|
43
50
|
add: [...want].filter((l) => !labels.includes(l)),
|
package/src/marker.mjs
CHANGED
|
@@ -49,8 +49,12 @@
|
|
|
49
49
|
* deferReason?: string | null,
|
|
50
50
|
* blockedBy?: number[] | null,
|
|
51
51
|
* reply?: { since: string, question: string | null } | null,
|
|
52
|
+
* ask?: import('./ask.mjs').AskT | null,
|
|
52
53
|
* fingerprint?: { hash: string, at: string } | null,
|
|
54
|
+
* cost?: import('./cost.mjs').Cost | null,
|
|
53
55
|
* }} Marker
|
|
56
|
+
* `cost`: what this issue's workers have cost so far, per phase (`triage`, `task`, `rescue`) and in `total`, from
|
|
57
|
+
* each worker's `result` event (cost.mjs). Every write keeps it; a marker without one counts from zero.
|
|
54
58
|
* `mergeRefusal`: the last `graft pr merge` cambium was refused (PR graph `merge`), and on which head.
|
|
55
59
|
* `reworks`: how many times the PR graph sent this issue's PR back to a worker.
|
|
56
60
|
* `defers`, `deferUntil`, `deferReason`: the dispatch graph's `deferred` node: how many times triage deferred, until
|
|
@@ -58,11 +62,13 @@
|
|
|
58
62
|
* GitHub `blockedBy` issues it waits on (no `deferUntil`).
|
|
59
63
|
* `reply`: while triage is pending or running, what a human reply answers (human-reply.mjs): human comments after
|
|
60
64
|
* `since` are quoted in the triage prompt as the answer to `question`. Cleared when triage lands a verdict.
|
|
65
|
+
* `ask`: at a `to-human` node, what the person is asked to do (ask.mjs); the comment renders it.
|
|
61
66
|
* `fingerprint`: the triage inputs (triage-inputs.mjs) the last triage saw, taken at `start-triage` and kept by
|
|
62
67
|
* `to-human` only for a needs-human verdict; re-triage skips the issue while they are unchanged.
|
|
63
68
|
* `triage`: what the dispatch graph's triage added when it handed the issue to `q:task`; the task prompt shows it.
|
|
64
69
|
* @typedef {{ graph: string, entryNode: string, nodes?: Iterable<string> }} QueueOwner
|
|
65
70
|
*/
|
|
71
|
+
import { renderAsk } from './ask.mjs'
|
|
66
72
|
|
|
67
73
|
const PREFIX = '<!-- cambium:state '
|
|
68
74
|
const RE = /<!-- cambium:state (.*?) -->/s
|
|
@@ -188,11 +194,13 @@ export function renderMarker(input, { maxAttempts } = {}) {
|
|
|
188
194
|
(marker.pr ? ` · PR #${marker.pr}` : '') +
|
|
189
195
|
(marker.host ? ` · host \`${marker.host}\`` : '') +
|
|
190
196
|
(marker.start ? ` · started ${marker.start}` : '') +
|
|
197
|
+
(marker.cost?.total?.usd ? ` · cost $${marker.cost.total.usd.toFixed(2)}` : '') +
|
|
191
198
|
(marker.node === 'deferred' && marker.deferUntil ? ` · re-triage after ${marker.deferUntil}` : '') +
|
|
192
199
|
(marker.node === 'blocked' && marker.blockedBy?.length
|
|
193
200
|
? ` · re-triage once ${marker.blockedBy.map((n) => `#${n}`).join(', ')} close`
|
|
194
201
|
: ''),
|
|
195
202
|
]
|
|
203
|
+
if (marker.ask && marker.node === 'to-human' && marker.status !== 'done') lines.push('', ...renderAsk(marker.ask))
|
|
196
204
|
if (marker.lastFailure)
|
|
197
205
|
lines.push(
|
|
198
206
|
'',
|
package/src/phases/dispatch.mjs
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* stub below and leaves `q:intake` alone.
|
|
6
6
|
*/
|
|
7
7
|
import { z } from 'zod'
|
|
8
|
+
import { askError, LenientAsk } from '../ask.mjs'
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* @typedef {{ repo: string, intake: import('../github.mjs').Issue[], capacity: number, now: number }} DispatchInput
|
|
@@ -28,6 +29,8 @@ export const TriageResult = z
|
|
|
28
29
|
reason: z.string().trim().min(1),
|
|
29
30
|
title: z.string().trim().min(1).optional(),
|
|
30
31
|
acceptance: z.array(z.string().trim().min(1)).optional(),
|
|
32
|
+
/** For needs-human: what the person is asked to do (ask.mjs). An invalid one is dropped, not fatal. */
|
|
33
|
+
ask: LenientAsk,
|
|
31
34
|
})
|
|
32
35
|
.refine((r) => r.verdict !== 'duplicate' || r.duplicateOf !== undefined, {
|
|
33
36
|
message: 'a duplicate verdict needs duplicateOf',
|
|
@@ -116,7 +119,15 @@ export function readTriage(log, issue) {
|
|
|
116
119
|
return { ok: false, error: `invalid triage result: duplicateOf: #${issue} cannot duplicate itself` }
|
|
117
120
|
if (parsed.data.verdict === 'blocked' && parsed.data.blockedBy?.includes(issue))
|
|
118
121
|
return { ok: false, error: `invalid triage result: blockedBy: #${issue} cannot block itself` }
|
|
119
|
-
|
|
122
|
+
const { ask, ...rest } = parsed.data
|
|
123
|
+
if (rest.verdict !== 'needs-human') return { ok: true, result: rest }
|
|
124
|
+
const bad = askError(/** @type {any} */ (read.json)?.ask)
|
|
125
|
+
if (bad)
|
|
126
|
+
return {
|
|
127
|
+
ok: true,
|
|
128
|
+
result: { ...rest, reason: `${rest.reason}\n\n(triage's ask was invalid and was dropped: ${bad})` },
|
|
129
|
+
}
|
|
130
|
+
return { ok: true, result: ask ? { ...rest, ask } : rest }
|
|
120
131
|
}
|
|
121
132
|
|
|
122
133
|
/**
|
|
@@ -156,11 +167,15 @@ Your one job: decide what happens to GitHub issue #${issue.number} (below), whic
|
|
|
156
167
|
|
|
157
168
|
Verdicts:
|
|
158
169
|
- actionable: a worker could do it unattended from what is written (plus what you add), and would know when it is done. It goes to the task queue. When the issue lacks a clear task or checkable acceptance criteria, give \`title\` (an imperative task title) and \`acceptance\` (checks a reviewer can verify; CI green is always required, so do not list only that). cambium posts them on the issue and hands them to the task worker.
|
|
159
|
-
- needs-human: too foggy to ticket; a decision only a human can make (direction, priority, money, access, policy); information no one but a human has; or work that is not a code change in this repository. \`reason\` says exactly what the human must decide or supply. Never for something a machine can check, now or later.
|
|
170
|
+
- needs-human: too foggy to ticket; a decision only a human can make (direction, priority, money, access, policy); information no one but a human has; or work that is not a code change in this repository. \`reason\` says exactly what the human must decide or supply. Never for something a machine can check, now or later. Fill \`ask\` so the person can answer with one comment:
|
|
171
|
+
- \`kind\`: \`decide\` (a choice only a person can make), \`verify\` (a check only a person can make: a physical device, an account no machine can reach), \`chore\` (offline work: a store listing, paperwork, a console setting) or \`approve\` (a yes or no on something already written).
|
|
172
|
+
- \`question\`: the one question, answerable in a comment. For \`decide\`, \`options\` (two or more, each a \`label\` and the tradeoff in \`detail\`) and, where you have one, \`recommendation\`.
|
|
173
|
+
- \`interactive\`: true only when a live session with the code would help the person answer (a design choice to explore); false for chores, checks and plain choices.
|
|
174
|
+
If the issue waits on another open issue that is itself a human chore or check, it is blocked by that issue, not needs-human: do not ask again what that issue already asks.
|
|
160
175
|
- duplicate: the same work as another issue, open or closed. \`duplicateOf\` is that issue's number (never #${issue.number}); \`reason\` says why they are the same.
|
|
161
176
|
- obsolete: the work no longer applies, and you verified it: the code path is gone, the bug is already fixed on ${baseBranch}, or its verification already passed. cambium closes it as not planned. \`reason\` gives the evidence (commit, file, run).
|
|
162
177
|
- defer: the verdict hangs on something a machine can check later but not yet: a CI or CD run that has not happened, a release, a PR still merging, a deploy, an upstream fix. cambium triages it again after \`recheckHours\` (default ${deferHours}). \`reason\` says exactly what to check next time and what each answer means (e.g. "next main cd run green → obsolete; red → actionable"). Not for waiting on another issue: that is blocked.
|
|
163
|
-
- blocked: it cannot start until other issues in ${repo} are done. \`blockedBy\` lists their numbers (never #${issue.number}); cambium records each as a GitHub "blocked by" link on this issue and triages it again as soon as the last of them closes, not on a timer. \`reason\` says what each blocker must deliver.
|
|
178
|
+
- blocked: it cannot start until other issues in ${repo} are done (including an open issue waiting on a person for a chore or check). \`blockedBy\` lists their numbers (never #${issue.number}); cambium records each as a GitHub "blocked by" link on this issue and triages it again as soon as the last of them closes, not on a timer. \`reason\` says what each blocker must deliver.
|
|
164
179
|
|
|
165
180
|
How to look:
|
|
166
181
|
- Read the code here (Read, Glob, Grep) to judge what the change touches and whether it already exists.
|
|
@@ -172,9 +187,9 @@ How to look:
|
|
|
172
187
|
|
|
173
188
|
End your final message with exactly one fenced JSON block, and nothing after it:
|
|
174
189
|
\`\`\`json
|
|
175
|
-
{ "verdict": "actionable" | "needs-human" | "duplicate" | "obsolete" | "defer" | "blocked", "duplicateOf": 123, "blockedBy": [124], "recheckHours": 6, "reason": "…", "title": "…", "acceptance": ["…"] }
|
|
190
|
+
{ "verdict": "actionable" | "needs-human" | "duplicate" | "obsolete" | "defer" | "blocked", "duplicateOf": 123, "blockedBy": [124], "recheckHours": 6, "reason": "…", "title": "…", "acceptance": ["…"], "ask": { "kind": "decide" | "verify" | "chore" | "approve", "question": "…", "options": [{ "label": "…", "detail": "…" }], "recommendation": "…", "interactive": false } }
|
|
176
191
|
\`\`\`
|
|
177
|
-
\`duplicateOf\` only for duplicate; \`blockedBy\` only for blocked; \`recheckHours\` only for defer, and optional; \`title\` and \`acceptance\` only for actionable, and optional. A missing or malformed block sends the issue to a human with the error.
|
|
192
|
+
\`duplicateOf\` only for duplicate; \`blockedBy\` only for blocked; \`recheckHours\` only for defer, and optional; \`title\` and \`acceptance\` only for actionable, and optional; \`ask\` only for needs-human (\`options\` only for decide, \`recommendation\` optional). A missing or malformed block sends the issue to a human with the error.
|
|
178
193
|
${deferred ? `\nThis is re-triage ${deferred.count + 1}: an earlier triage deferred it to check this:\n${deferred.reason}\nCheck it now. If it still cannot be checked, defer again.\n` : ''}${blocked ? `\nAn earlier triage found it blocked by ${blocked.by.map((n) => `#${n}`).join(', ')}, now closed:\n${blocked.reason}\nJudge it again now that they are done.\n` : ''}${replied ? repliedNote(replied) : ''}${others ? `\nOther issues in cambium's queues right now:\n${others}\n` : ''}
|
|
179
194
|
--- Issue #${issue.number}: ${issue.title} (${issue.url})
|
|
180
195
|
${issue.body}
|
package/src/phases/rescue.mjs
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* its typed result and the checks cambium runs on it before acting, and the comments and root-cause tickets it
|
|
5
5
|
* writes.
|
|
6
6
|
*
|
|
7
|
-
* An item is stuck when its labels break the queue rules (more than one `q:*`, `running` without a queue, `why:*`
|
|
8
|
-
* outside `q:human`, `wait:*` outside `q:intake`, cambium labels left on a closed issue), when its agent PR closed
|
|
7
|
+
* An item is stuck when its labels break the queue rules (more than one `q:*`, `running` without a queue, `why:*` or
|
|
8
|
+
* `ask:*` outside `q:human`, `wait:*` outside `q:intake`, cambium labels left on a closed issue), when its agent PR closed
|
|
9
9
|
* or merged and left it open in no queue, or when it has waited in a queue, or its PR has sat untouched, far longer
|
|
10
10
|
* than cambium ever takes to move it.
|
|
11
11
|
*/
|
|
@@ -14,6 +14,7 @@ import { z } from 'zod'
|
|
|
14
14
|
import { branchIssue } from '../github.mjs'
|
|
15
15
|
import { MIN } from '../issue-state.mjs'
|
|
16
16
|
import { Q, RUNNING } from '../labels.mjs'
|
|
17
|
+
import { LenientAsk } from '../ask.mjs'
|
|
17
18
|
import { finalJson, formatZodIssue } from './dispatch.mjs'
|
|
18
19
|
|
|
19
20
|
/**
|
|
@@ -28,7 +29,8 @@ import { finalJson, formatZodIssue } from './dispatch.mjs'
|
|
|
28
29
|
const HOUR = 60 * MIN
|
|
29
30
|
|
|
30
31
|
/** The labels cambium owns, which only cambium moves. @param {string} l */
|
|
31
|
-
export const cambiumLabel = (l) =>
|
|
32
|
+
export const cambiumLabel = (l) =>
|
|
33
|
+
l.startsWith('q:') || l === RUNNING || l.startsWith('why:') || l.startsWith('ask:') || l.startsWith('wait:')
|
|
32
34
|
|
|
33
35
|
/** The issue an agent PR is for: the first one it closes that the snapshot has, else the one its branch names. @param {PullRequest} pr */
|
|
34
36
|
const issueOfPr = (pr) => pr.closes[0] ?? branchIssue(pr.branch)
|
|
@@ -94,7 +96,7 @@ export function findStuck(snap, o) {
|
|
|
94
96
|
code: 'running-unqueued',
|
|
95
97
|
detail: `labelled \`${RUNNING}\` ${queues.length ? `in ${queues.join(', ')}` : 'in no queue'}`,
|
|
96
98
|
})
|
|
97
|
-
const why = labels.filter((l) => l.startsWith('why:'))
|
|
99
|
+
const why = labels.filter((l) => l.startsWith('why:') || l.startsWith('ask:'))
|
|
98
100
|
if (why.length && !labels.includes(Q.human))
|
|
99
101
|
add(n, { code: 'why-outside-human', detail: `${why.join(', ')} without ${Q.human}` })
|
|
100
102
|
const wait = labels.filter((l) => l.startsWith('wait:'))
|
|
@@ -223,7 +225,7 @@ const text = z.string().trim().min(1)
|
|
|
223
225
|
|
|
224
226
|
export const RescueAction = z.discriminatedUnion('do', [
|
|
225
227
|
z.object({ do: z.literal('requeue'), queue: z.enum(['task', 'intake']), reason: text }),
|
|
226
|
-
z.object({ do: z.literal('to-human'), reason: text }),
|
|
228
|
+
z.object({ do: z.literal('to-human'), reason: text, ask: LenientAsk }),
|
|
227
229
|
z.object({
|
|
228
230
|
do: z.literal('close-issue'),
|
|
229
231
|
as: z.enum(['completed', 'not_planned']).default('completed'),
|
|
@@ -381,7 +383,7 @@ ${symptomList(stuck.symptoms)}${previous ? `\n\ncambium has rescued this item ${
|
|
|
381
383
|
How cambium's queues work (labels are the queue; only cambium moves them):
|
|
382
384
|
- Exactly one queue label: \`${Q.intake}\` (triage decides what it is), \`${Q.task}\` (a task worker turns it into a PR), \`${Q.human}\` (waits for a person, with a \`why:*\` label saying why), \`${Q.promote}\` (merged, awaiting the release promotion).
|
|
383
385
|
- \`${RUNNING}\`: a worker holds a lease on it right now. Without a live worker it is a lost lease.
|
|
384
|
-
- \`why:*\` only with \`${Q.human}\`; \`wait:deferred\` / \`wait:blocked\` only with \`${Q.intake}\` (triage parked it).
|
|
386
|
+
- \`why:*\` and \`ask:*\` (what the person is asked to do) only with \`${Q.human}\`; \`wait:deferred\` / \`wait:blocked\` only with \`${Q.intake}\` (triage parked it).
|
|
385
387
|
- Once a task worker publishes its PR, the issue leaves every queue and the PR graph owns it: it merges a green PR (or waits for a merge queue or a human), reworks a red or conflicting one back through \`${Q.task}\`, and closes the issue when the PR merges.
|
|
386
388
|
- cambium's state comment on the issue holds a hidden JSON marker: the graph and node the issue rests at, and its history. It is below.
|
|
387
389
|
|
|
@@ -392,13 +394,13 @@ How to look:
|
|
|
392
394
|
- Never \`gh\`. Be quick: a verdict with evidence, not an essay.
|
|
393
395
|
|
|
394
396
|
Actions, in order (cambium checks them before it does anything; an invalid list sends the issue to a human):
|
|
395
|
-
- \`{ "do": "clear-labels", "labels": ["…"], "reason": "…" }\`: remove stray cambium labels (\`q:*\`, \`${RUNNING}\`, \`why:*\`, \`wait:*\`) and nothing else.
|
|
397
|
+
- \`{ "do": "clear-labels", "labels": ["…"], "reason": "…" }\`: remove stray cambium labels (\`q:*\`, \`${RUNNING}\`, \`why:*\`, \`ask:*\`, \`wait:*\`) and nothing else.
|
|
396
398
|
- \`{ "do": "close-pr", "pr": 123, "reason": "…" }\`: close an open agent PR that is superseded or abandoned.
|
|
397
399
|
- At most one of these, last:
|
|
398
400
|
- \`{ "do": "requeue", "queue": "task" | "intake", "reason": "…" }\`: back to a queue at its start, attempt 1: \`task\` when the work is still clear, \`intake\` when it needs triage again.
|
|
399
401
|
- \`{ "do": "adopt-pr", "pr": 123 }\`: hand an open agent PR that holds the work to the PR graph.
|
|
400
402
|
- \`{ "do": "close-issue", "as": "completed" | "not_planned", "reason": "…" }\`: the work is done (say where) or no longer applies.
|
|
401
|
-
- \`{ "do": "to-human", "reason": "…" }\`: only a person can unstick it; \`reason\` says exactly what they must decide or do.
|
|
403
|
+
- \`{ "do": "to-human", "reason": "…", "ask": { … } }\`: only a person can unstick it; \`reason\` says exactly what they must decide or do. Where you can, add \`ask\` so they can answer with one comment: \`{ "kind": "decide" | "verify" | "chore" | "approve", "question": "…", "options": [{ "label": "…", "detail": "…" }], "recommendation": "…", "interactive": false }\` (\`options\`, two or more, only for decide; \`interactive\` true only when a live session with the code would help).
|
|
402
404
|
If it is not actually stuck (cambium's check was wrong), say so with \`"stuck": false\` and no actions; a wrong check is itself a root cause worth a ticket.
|
|
403
405
|
|
|
404
406
|
Root causes: only ones that need a change and that you can name with evidence (a cambium bug, a missing guard, a config gap, a process that keeps mislabelling). A one-off human relabel needs none. Each ticket's \`body\` is a task a worker could do unattended: the evidence, the change, and acceptance checks. \`fingerprint\` names the cause in a few kebab-case words, the same every time it recurs (e.g. \`sweep-skips-foreign-host-lease\`). \`repo\` is where the change belongs, one of: ${repos.join(', ')} (default ${repo}).
|
package/src/store.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
/**
|
|
3
|
-
* Local bookkeeping in SQLite: live worker processes
|
|
3
|
+
* Local bookkeeping in SQLite: live worker processes, the daemon's mode, and the per-day cost ledger (cost.mjs).
|
|
4
4
|
* Never a queue — GitHub labels are the queue. Losing this file costs at most
|
|
5
5
|
* a requeue (the sweep returns `running` issues with no tracked worker).
|
|
6
6
|
*/
|
|
@@ -28,6 +28,12 @@ CREATE TABLE IF NOT EXISTS workers (
|
|
|
28
28
|
PRIMARY KEY (repo, issue)
|
|
29
29
|
);
|
|
30
30
|
CREATE TABLE IF NOT EXISTS control (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
31
|
+
CREATE TABLE IF NOT EXISTS costs (
|
|
32
|
+
repo TEXT NOT NULL, day TEXT NOT NULL, phase TEXT NOT NULL, usd REAL NOT NULL DEFAULT 0,
|
|
33
|
+
input INTEGER NOT NULL DEFAULT 0, output INTEGER NOT NULL DEFAULT 0, cache_read INTEGER NOT NULL DEFAULT 0,
|
|
34
|
+
cache_creation INTEGER NOT NULL DEFAULT 0, runs INTEGER NOT NULL DEFAULT 0,
|
|
35
|
+
PRIMARY KEY (repo, day, phase)
|
|
36
|
+
);
|
|
31
37
|
`
|
|
32
38
|
|
|
33
39
|
/** @param {any} r @returns {Worker} */
|
|
@@ -99,6 +105,34 @@ export function openStore(path) {
|
|
|
99
105
|
removeWorker(repo, issue) {
|
|
100
106
|
db.prepare('DELETE FROM workers WHERE repo = ? AND issue = ?').run(repo, issue)
|
|
101
107
|
},
|
|
108
|
+
/**
|
|
109
|
+
* Add one worker run to a repo's ledger for `day` (`YYYY-MM-DD`, UTC) and phase.
|
|
110
|
+
* @param {string} repo @param {string} day @param {string} phase @param {import('./cost.mjs').Tally} run
|
|
111
|
+
*/
|
|
112
|
+
addCost(repo, day, phase, run) {
|
|
113
|
+
db.prepare(
|
|
114
|
+
`INSERT INTO costs (repo, day, phase, usd, input, output, cache_read, cache_creation, runs)
|
|
115
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
116
|
+
ON CONFLICT(repo, day, phase) DO UPDATE SET usd = usd + excluded.usd, input = input + excluded.input,
|
|
117
|
+
output = output + excluded.output, cache_read = cache_read + excluded.cache_read,
|
|
118
|
+
cache_creation = cache_creation + excluded.cache_creation, runs = runs + excluded.runs`,
|
|
119
|
+
).run(repo, day, phase, run.usd, run.input, run.output, run.cacheRead, run.cacheCreation, run.runs)
|
|
120
|
+
},
|
|
121
|
+
/**
|
|
122
|
+
* The ledger from `since` (`YYYY-MM-DD`) on, oldest first.
|
|
123
|
+
* @param {string} since @returns {{ repo: string, day: string, phase: string, usd: number, runs: number }[]}
|
|
124
|
+
*/
|
|
125
|
+
costs: (since) =>
|
|
126
|
+
db
|
|
127
|
+
.prepare('SELECT repo, day, phase, usd, runs FROM costs WHERE day >= ? ORDER BY day, repo, phase')
|
|
128
|
+
.all(since)
|
|
129
|
+
.map((/** @type {any} */ r) => ({
|
|
130
|
+
repo: r.repo,
|
|
131
|
+
day: r.day,
|
|
132
|
+
phase: r.phase,
|
|
133
|
+
usd: Number(r.usd),
|
|
134
|
+
runs: Number(r.runs),
|
|
135
|
+
})),
|
|
102
136
|
/** @returns {Mode} */
|
|
103
137
|
mode: () => /** @type {Mode} */ (get('mode') ?? 'running'),
|
|
104
138
|
/** @param {Mode} mode */
|
package/src/tick.mjs
CHANGED
|
@@ -20,6 +20,7 @@ import { DISPATCH_NOT_BUILT, dispatchNotBuilt } from './phases/dispatch.mjs'
|
|
|
20
20
|
import { isRepair } from './base-health.mjs'
|
|
21
21
|
import { stopOverdue } from './workers.mjs'
|
|
22
22
|
import { syncBriefs } from './briefs.mjs'
|
|
23
|
+
import { costDays, costOf, dayOf, overBudget, recordCost } from './cost.mjs'
|
|
23
24
|
|
|
24
25
|
/**
|
|
25
26
|
* @typedef {import('./github.mjs').Issue} Issue
|
|
@@ -272,6 +273,7 @@ export async function finish(deps, ctx, w, snap) {
|
|
|
272
273
|
const t = await deps.spawner.cleanup({ repo, worktree: w.worktree, branch: w.branch })
|
|
273
274
|
if (!t.removed) deps.log('warn', `kept worktree ${w.worktree}: ${t.reason}`, { repo: w.repo, issue: w.issue })
|
|
274
275
|
}
|
|
276
|
+
await recordCost(deps, issue, w, ctx.now)
|
|
275
277
|
if (issue?.state !== 'OPEN') {
|
|
276
278
|
deps.log('info', 'worker gone; issue is closed or no longer queued', { repo: w.repo, issue: w.issue })
|
|
277
279
|
return done()
|
|
@@ -650,6 +652,7 @@ async function release(deps, ctx) {
|
|
|
650
652
|
* repos: {
|
|
651
653
|
* slug: string, ok: boolean, error: string | null, counts: Counts | null,
|
|
652
654
|
* base: import('./base-health.mjs').BaseHealth | null,
|
|
655
|
+
* cost: { issues: IssueCost[], days: import('./cost.mjs').CostDay[] },
|
|
653
656
|
* }[],
|
|
654
657
|
* workers: {
|
|
655
658
|
* repo: string, issue: number, phase: string, pid: number, alive: boolean, attempt: number, branch: string,
|
|
@@ -659,8 +662,47 @@ async function release(deps, ctx) {
|
|
|
659
662
|
* events: Event[],
|
|
660
663
|
* engine: 'legacy' | 'graph',
|
|
661
664
|
* graphs: import('./graph/describe.mjs').GraphDescription[],
|
|
665
|
+
* costFlagUsd: number,
|
|
662
666
|
* }} HeartbeatState
|
|
667
|
+
* @typedef {{
|
|
668
|
+
* number: number, title: string, url: string, labels: string[], cost: import('./cost.mjs').Cost, flagged: boolean,
|
|
669
|
+
* }} IssueCost
|
|
670
|
+
* An active (`q:intake`, `q:task`, `running`) or `q:human` issue that has cost something. `flagged`: its task
|
|
671
|
+
* spend is over `costFlagUsd` and it has no merged PR (cost.mjs `overBudget`).
|
|
672
|
+
*/
|
|
673
|
+
|
|
674
|
+
/** Days of the per-repo cost ledger the heartbeat carries, today included. */
|
|
675
|
+
export const COST_DAYS = 14
|
|
676
|
+
|
|
677
|
+
/**
|
|
678
|
+
* The repo's issues worth a cost line, most expensive first.
|
|
679
|
+
* @param {Snapshot} snap @param {number} flagUsd @returns {IssueCost[]}
|
|
663
680
|
*/
|
|
681
|
+
export function issueCosts(snap, flagUsd) {
|
|
682
|
+
/** @type {IssueCost[]} */
|
|
683
|
+
const out = []
|
|
684
|
+
for (const issue of snap.issues.values()) {
|
|
685
|
+
if (issue.state !== 'OPEN') continue
|
|
686
|
+
const q = queueOf(issue.labels)
|
|
687
|
+
if (q !== Q.intake && q !== Q.task && q !== Q.human && !issue.labels.includes(RUNNING)) continue
|
|
688
|
+
const marker = issue.stateComment?.marker
|
|
689
|
+
const cost = costOf(marker)
|
|
690
|
+
if (!cost.total.runs) continue
|
|
691
|
+
const merged = snap.prs.some(
|
|
692
|
+
(p) =>
|
|
693
|
+
p.state === 'MERGED' && (p.closes.includes(issue.number) || (marker?.pr != null && p.number === marker.pr)),
|
|
694
|
+
)
|
|
695
|
+
out.push({
|
|
696
|
+
number: issue.number,
|
|
697
|
+
title: issue.title,
|
|
698
|
+
url: issue.url,
|
|
699
|
+
labels: issue.labels,
|
|
700
|
+
cost,
|
|
701
|
+
flagged: overBudget(cost, flagUsd, merged),
|
|
702
|
+
})
|
|
703
|
+
}
|
|
704
|
+
return out.sort((a, b) => b.cost.total.usd - a.cost.total.usd || a.number - b.number)
|
|
705
|
+
}
|
|
664
706
|
|
|
665
707
|
/** @param {Snapshot} snap @returns {Counts} */
|
|
666
708
|
function countsOf(snap) {
|
|
@@ -694,6 +736,8 @@ const graphDescriptions = () => {
|
|
|
694
736
|
export function heartbeatState(deps, ctx) {
|
|
695
737
|
const { config } = deps
|
|
696
738
|
const finished = deps.clock.now()
|
|
739
|
+
const since = dayOf(ctx.now - (COST_DAYS - 1) * 86_400_000)
|
|
740
|
+
const ledger = deps.store.costs(since)
|
|
697
741
|
const repos = config.repos.map((r) => {
|
|
698
742
|
const snap = ctx.snaps.get(r.slug)
|
|
699
743
|
return {
|
|
@@ -702,6 +746,10 @@ export function heartbeatState(deps, ctx) {
|
|
|
702
746
|
error: ctx.readErrors.get(r.slug) ?? null,
|
|
703
747
|
counts: snap ? countsOf(snap) : null,
|
|
704
748
|
base: snap?.base ?? null,
|
|
749
|
+
cost: {
|
|
750
|
+
issues: snap ? issueCosts(snap, config.costFlagUsd) : [],
|
|
751
|
+
days: costDays(ledger.filter((row) => row.repo === r.slug)),
|
|
752
|
+
},
|
|
705
753
|
}
|
|
706
754
|
})
|
|
707
755
|
const total = { intake: 0, task: 0, human: 0, promote: 0, running: 0, review: 0 }
|
|
@@ -746,6 +794,7 @@ export function heartbeatState(deps, ctx) {
|
|
|
746
794
|
events: deps.events?.() ?? [],
|
|
747
795
|
engine: config.engine,
|
|
748
796
|
graphs: graphDescriptions(),
|
|
797
|
+
costFlagUsd: config.costFlagUsd,
|
|
749
798
|
}
|
|
750
799
|
}
|
|
751
800
|
|
package/src/tracing.mjs
CHANGED
|
@@ -391,6 +391,17 @@ function* events(stream) {
|
|
|
391
391
|
}
|
|
392
392
|
}
|
|
393
393
|
|
|
394
|
+
/**
|
|
395
|
+
* The last `result` event in a worker's stream-json log (cost, usage, turns), or null when there is none: a killed
|
|
396
|
+
* worker or a truncated log. Never throws.
|
|
397
|
+
* @param {string | Iterable<string | Record<string, any>>} stream @returns {Record<string, any> | null}
|
|
398
|
+
*/
|
|
399
|
+
export function resultEvent(stream) {
|
|
400
|
+
let result = null
|
|
401
|
+
for (const e of events(stream)) if (e.type === 'result') result = e
|
|
402
|
+
return result
|
|
403
|
+
}
|
|
404
|
+
|
|
394
405
|
/**
|
|
395
406
|
* @typedef {{ id: string, name: string, input: any, start: number, end: number | null, error: boolean, parentToolId: string | null }} ToolUse
|
|
396
407
|
* @typedef {{ id: string, toolUseId: string | null, type?: string, description?: string, start: number, end: number | null, status?: string }} BgTask
|