@oakgreencc/cambium 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -9
- package/package.json +1 -1
- package/src/config.mjs +2 -4
- package/src/convention.mjs +141 -0
- package/src/marker.mjs +178 -21
- package/src/phases/task.mjs +0 -3
- package/src/prepare.mjs +185 -0
- package/src/tick.mjs +32 -5
- package/src/workers.mjs +79 -24
package/README.md
CHANGED
|
@@ -36,7 +36,7 @@ Every command takes `--config <file>`; the default is `$CAMBIUM_CONFIG`, then `~
|
|
|
36
36
|
Each step is bounded by `stepTimeoutSeconds`. A step that throws or times out is logged and recorded in the heartbeat, and the tick moves on. Nothing waits on a worker.
|
|
37
37
|
|
|
38
38
|
1. **Read.** One GraphQL query per repo (`graft graphql`): open issues with any `q:` label, open and recently merged PRs on `agent/*` with their check rollup, merge-queue state and the issues they close.
|
|
39
|
-
2. **Reconcile.** For each tracked worker: alive and under its wall-clock cap → nothing. Over the cap → kill its process group. Gone → exit check (a PR on its branch, or closing its issue, that is open or merged and not still on the commit CI already failed) → `submitted`; otherwise requeue with the stderr tail.
|
|
39
|
+
2. **Reconcile.** For each tracked worker: alive and under its wall-clock cap → nothing. Over the cap → kill its process group. Gone → exit check (a PR on its branch, or closing its issue, that is open or merged and not still on the commit CI already failed) → `submitted`; otherwise requeue with the stderr tail (a failed prepare step requeues as `spawn-failed`). Then the worktree is torn down if `graft status` says GitHub holds everything in it, else kept with a `warn` saying why.
|
|
40
40
|
3. **Watch** (no agent), for agent PRs whose issue is in no queue: merged → close the issue, delete the branch; red → requeue with the failing checks' summaries; left the merge queue without merging → requeue; not merged `mergeQueueMinutes` after submission → `q:human`.
|
|
41
41
|
4. **Sweep.** An issue labelled `running` with this host (or no host) in its marker and no tracked worker → requeue. This is what makes losing the SQLite file cost one requeue.
|
|
42
42
|
5. **Schedule.** Start Task workers for `q:task` issues without `running`, by priority label then age, up to `phases.task.cap` counted from live workers. A task requeued earlier in the same tick can start in that tick.
|
|
@@ -60,7 +60,7 @@ Only cambium moves these labels; workers have `graft issue label` and `graft iss
|
|
|
60
60
|
Each issue cambium touches gets one comment it owns and edits in place. Its first line is a hidden marker, the rest is a human rendering of it:
|
|
61
61
|
|
|
62
62
|
```html
|
|
63
|
-
<!-- cambium:state {"v":1,"phase":"task","stage":"queued","attempt":2,"pr":120,"branch":"agent/
|
|
63
|
+
<!-- cambium:state {"v":1,"phase":"task","stage":"queued","attempt":2,"pr":120,"branch":"agent/12-fix-the-tick","host":null,"start":null,"submittedAt":"…","queued":false,"failedSha":"deadbeef…","lastFailure":"CI red on PR #120 …","history":[{"at":"…","attempt":1,"outcome":"ci-red","reason":"…","pr":120}]} -->
|
|
64
64
|
```
|
|
65
65
|
|
|
66
66
|
- `stage`: `queued` | `running` | `submitted` | `human` | `done`.
|
|
@@ -76,12 +76,13 @@ Set `graft.botLogin` to the Clock App's login and only markers written by it are
|
|
|
76
76
|
|
|
77
77
|
A worker is `claude -p --output-format stream-json --verbose --max-turns <maxTurns> --allowedTools Read,Edit,Write,Glob,Grep,Bash --disallowedTools "Bash(graft issue label:*),Bash(graft issue edit:*),Bash(gh:*)" <worker.args>`:
|
|
78
78
|
|
|
79
|
-
- in
|
|
79
|
+
- in its own worktree, by the target repo's [worktree convention](#target-repo-convention);
|
|
80
|
+
- behind the prepare step: `node src/prepare.mjs -- claude …` runs install-fresh, hydrate and the repo's `prepare` commands in the worktree, writing to the worker's stderr log, then becomes the worker. It runs in the detached process so a long install never blocks a tick;
|
|
80
81
|
- detached, leading its own process group (a kill takes its children), stdin from the prompt file;
|
|
81
82
|
- with `CAMBIUM_REPO`, `CAMBIUM_ISSUE`, `CAMBIUM_PHASE`, `CAMBIUM_BRANCH` set, and `GRAFT_APP=<worker.graftApp>` when configured;
|
|
82
83
|
- with its prompt, stdout and stderr at `<paths.logs>/<owner>__<name>/issue-<n>/a<attempt>-<stamp>.{prompt.md,out.log,err.log}`, keeping the newest `logs.keepPerIssue` runs per issue.
|
|
83
84
|
|
|
84
|
-
The prompt states the four-point contract (task issue, `pnpm verify`, `graft publish
|
|
85
|
+
The prompt states the four-point contract (task issue, `pnpm verify`, `graft publish <branch> … --closes <n>`, `graft ci wait`), the attempt, the existing PR if any (`graft advance` onto it), the last failure, the issue's history, the issue body, and the worktree's `AGENTS.md`.
|
|
85
86
|
|
|
86
87
|
## Configuration
|
|
87
88
|
|
|
@@ -104,8 +105,7 @@ One JSON file. Everything but `repos` is optional; these are the defaults:
|
|
|
104
105
|
"home": "~/.cambium",
|
|
105
106
|
"db": "~/.cambium/cambium.db",
|
|
106
107
|
"heartbeat": "~/.cambium/state.json",
|
|
107
|
-
"logs": "~/.cambium/logs"
|
|
108
|
-
"worktrees": "~/.cambium/worktrees"
|
|
108
|
+
"logs": "~/.cambium/logs"
|
|
109
109
|
},
|
|
110
110
|
"logs": { "keepPerIssue": 20, "daemonMaxBytes": 10485760 },
|
|
111
111
|
"graft": { "command": "graft", "app": null, "botLogin": null },
|
|
@@ -118,6 +118,33 @@ One JSON file. Everything but `repos` is optional; these are the defaults:
|
|
|
118
118
|
- Setting `paths.home` moves every default path under it.
|
|
119
119
|
- `release.time` is host-local `HH:MM`.
|
|
120
120
|
|
|
121
|
+
### Target repo convention
|
|
122
|
+
|
|
123
|
+
How a repo's worktrees are made is the repo's call: a `"cambium"` key in its root `package.json`, read from the central checkout (`repos[].path`) at each start. Every field is optional; these are the defaults:
|
|
124
|
+
|
|
125
|
+
```json
|
|
126
|
+
"cambium": {
|
|
127
|
+
"worktree": { "root": ".claude/worktrees", "branch": "agent/{issue}-{slug}" },
|
|
128
|
+
"install": "pnpm install --frozen-lockfile",
|
|
129
|
+
"hydrate": { "dirs": [], "files": [] },
|
|
130
|
+
"prepare": []
|
|
131
|
+
}
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
- **Branch**: `{issue}` (required) and `{slug}`, the issue title in kebab case, at most 40 characters cut at a word. A retry keeps the branch its first attempt chose (it is in the marker), and an existing PR's branch wins over both.
|
|
135
|
+
- **Worktree**: `<repos[].path>/<root>/<topic>`, the topic being the branch less its first segment (`agent/12-fix-the-tick` → `12-fix-the-tick`), so every attempt of an issue uses the same directory. Created with `git fetch origin <baseBranch>`, then `git worktree add --no-track -B <branch> <dir> origin/<baseBranch>`: always from the fetched remote branch, never a local one. Add `<root>` to the repo's `.gitignore`.
|
|
136
|
+
- **Teardown**: once the worker is reconciled, `graft status <branch> -C <dir>` (as `graft.app`) must end with `nothing local that GitHub does not have`; then the worktree and its local branch are removed. Anything else (local commits, uncommitted paths, graft failing) keeps it and logs why.
|
|
137
|
+
- **Retry**: a leftover worktree in the topic directory is removed under the same rule and re-created fresh; one that still holds local-only work is continued in instead (logged as `continuing in the previous worktree`), so nothing is lost.
|
|
138
|
+
- **Prepare**, after the worktree is created and before the worker starts, in order:
|
|
139
|
+
1. **install-fresh**: when the worktree has a `package.json` and `node_modules/.pnpm/lock.yaml` is not a byte copy of `pnpm-lock.yaml`, run `install` (`sh -c`). `"install": false` skips it.
|
|
140
|
+
2. **hydrate**: copy the git-ignored `dirs` and `files` from the central checkout into the worktree with a copy-on-write clone (`cp -c` on APFS), falling back to a byte copy; never a link. What the worktree already has is left alone; what the checkout lacks is reported and skipped; a copy that fails is removed and fails the step. `"hydrate": false` disables it.
|
|
141
|
+
3. **prepare**: each command, `sh -c` in the worktree.
|
|
142
|
+
|
|
143
|
+
A failure ends the process with `cambium: prepare failed: …` on stderr; the reconcile requeues it as `spawn-failed` with the stderr tail.
|
|
144
|
+
- Unknown keys, wrong types, paths outside the repo and an unparseable `package.json` fail the start (requeued as `spawn-failed`) with the error.
|
|
145
|
+
|
|
146
|
+
The key decides which commands run on this host, so list `package.json` as protected (CODEOWNERS) in the target repo, next to its `"graft"` policy.
|
|
147
|
+
|
|
121
148
|
## Heartbeat (`state.json`)
|
|
122
149
|
|
|
123
150
|
Written at the end of every tick to `paths.heartbeat` (default `~/.cambium/state.json`), whole and renamed into place, so a reader never sees a partial file. canopy reads it. `version` changes only on an incompatible change. All times are ISO 8601 UTC strings; durations are milliseconds.
|
|
@@ -167,8 +194,8 @@ Written at the end of every tick to `paths.heartbeat` (default `~/.cambium/state
|
|
|
167
194
|
"pid": 90210,
|
|
168
195
|
"alive": true,
|
|
169
196
|
"attempt": 2,
|
|
170
|
-
"branch": "agent/
|
|
171
|
-
"worktree": "/Users/me/.
|
|
197
|
+
"branch": "agent/12-fix-the-tick",
|
|
198
|
+
"worktree": "/Users/me/OGCC/monorepo/.claude/worktrees/12-fix-the-tick",
|
|
172
199
|
"startedAt": "…",
|
|
173
200
|
"elapsedMs": 1260000,
|
|
174
201
|
"wallClockMs": 3600000,
|
|
@@ -181,7 +208,7 @@ Written at the end of every tick to `paths.heartbeat` (default `~/.cambium/state
|
|
|
181
208
|
{
|
|
182
209
|
"at": "…",
|
|
183
210
|
"level": "info",
|
|
184
|
-
"msg": "started task worker pid 90210 (attempt 2, agent/
|
|
211
|
+
"msg": "started task worker pid 90210 (attempt 2, agent/12-fix-the-tick)",
|
|
185
212
|
"repo": "oakgreencc/monorepo",
|
|
186
213
|
"issue": 12
|
|
187
214
|
} // level: "info" | "warn" | "error"; repo/issue optional
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@oakgreencc/cambium",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.4",
|
|
4
4
|
"description": "The scheduler daemon: reads the q:* queues from GitHub every tick, runs headless claude workers in their own worktrees up to a cap, and makes every queue transition itself.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|
package/src/config.mjs
CHANGED
|
@@ -19,7 +19,7 @@ import { join, resolve } from 'node:path'
|
|
|
19
19
|
* mergeQueueMinutes: number,
|
|
20
20
|
* priorityLabels: string[],
|
|
21
21
|
* release: { enabled: boolean, time: string },
|
|
22
|
-
* paths: { home: string, db: string, heartbeat: string, logs: string
|
|
22
|
+
* paths: { home: string, db: string, heartbeat: string, logs: string },
|
|
23
23
|
* logs: { keepPerIssue: number, daemonMaxBytes: number },
|
|
24
24
|
* graft: { command: string, app: string | null, botLogin: string | null },
|
|
25
25
|
* worker: { command: string, args: string[], graftApp: string | null, fetch: boolean },
|
|
@@ -47,7 +47,6 @@ export function defaults(home = DEFAULT_HOME) {
|
|
|
47
47
|
db: join(home, 'cambium.db'),
|
|
48
48
|
heartbeat: join(home, 'state.json'),
|
|
49
49
|
logs: join(home, 'logs'),
|
|
50
|
-
worktrees: join(home, 'worktrees'),
|
|
51
50
|
},
|
|
52
51
|
logs: { keepPerIssue: 20, daemonMaxBytes: 10 * 1024 * 1024 },
|
|
53
52
|
graft: { command: 'graft', app: null, botLogin: null },
|
|
@@ -80,8 +79,7 @@ export function resolveConfig(raw = {}) {
|
|
|
80
79
|
const home = untilde(raw?.paths?.home ?? DEFAULT_HOME)
|
|
81
80
|
/** @type {Config} */
|
|
82
81
|
const config = merge(defaults(home), raw)
|
|
83
|
-
for (const k of /** @type {const} */ (['home', 'db', 'heartbeat', 'logs'
|
|
84
|
-
config.paths[k] = untilde(config.paths[k])
|
|
82
|
+
for (const k of /** @type {const} */ (['home', 'db', 'heartbeat', 'logs'])) config.paths[k] = untilde(config.paths[k])
|
|
85
83
|
config.repos = (config.repos ?? []).map((r) => {
|
|
86
84
|
if (!r?.slug || !/^[^/\s]+\/[^/\s]+$/.test(r.slug))
|
|
87
85
|
throw new Error(`config: repo slug must be owner/name, got ${r?.slug}`)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* The target repo's worktree convention: the `"cambium"` key of its root
|
|
4
|
+
* package.json, read from the central checkout. Every field is optional;
|
|
5
|
+
* a missing one takes the default below.
|
|
6
|
+
*/
|
|
7
|
+
import { existsSync, readFileSync } from 'node:fs'
|
|
8
|
+
import { isAbsolute, join, normalize, resolve } from 'node:path'
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @typedef {{
|
|
12
|
+
* root: string, branch: string, install: string | null,
|
|
13
|
+
* hydrate: { dirs: string[], files: string[] }, prepare: string[],
|
|
14
|
+
* }} Convention
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
export const DEFAULT_CONVENTION = Object.freeze({
|
|
18
|
+
root: '.claude/worktrees',
|
|
19
|
+
branch: 'agent/{issue}-{slug}',
|
|
20
|
+
install: 'pnpm install --frozen-lockfile',
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
/** @param {string} where @param {string} msg */
|
|
24
|
+
const bad = (where, msg) => new Error(`${where} "cambium": ${msg}`)
|
|
25
|
+
|
|
26
|
+
/** @param {unknown} v @returns {v is Record<string, unknown>} */
|
|
27
|
+
const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v)
|
|
28
|
+
|
|
29
|
+
/** @param {string} where @param {Record<string, unknown>} obj @param {string} at @param {string[]} keys */
|
|
30
|
+
function onlyKeys(where, obj, at, keys) {
|
|
31
|
+
const unknown = Object.keys(obj).filter((k) => !keys.includes(k))
|
|
32
|
+
if (unknown.length)
|
|
33
|
+
throw bad(where, `unknown key ${unknown.map((k) => `${at}${k}`).join(', ')} (known: ${keys.join(', ')})`)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** A repo-relative path that stays inside the repo. @param {string} where @param {string} at @param {unknown} v */
|
|
37
|
+
function relPath(where, at, v) {
|
|
38
|
+
if (typeof v !== 'string' || !v || isAbsolute(v) || normalize(v).split(/[\\/]/).includes('..'))
|
|
39
|
+
throw bad(where, `${at} must be a relative path inside the repo, got ${JSON.stringify(v)}`)
|
|
40
|
+
return v
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** @param {string} where @param {string} at @param {unknown} v */
|
|
44
|
+
function strings(where, at, v, check = (/** @type {string} */ _a, /** @type {unknown} */ x) => x) {
|
|
45
|
+
if (v === undefined) return []
|
|
46
|
+
if (!Array.isArray(v)) throw bad(where, `${at} must be an array of strings`)
|
|
47
|
+
return v.map((x, i) => {
|
|
48
|
+
if (typeof x !== 'string' || !x) throw bad(where, `${at}[${i}] must be a non-empty string`)
|
|
49
|
+
return /** @type {string} */ (check(`${at}[${i}]`, x))
|
|
50
|
+
})
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Validate the raw `"cambium"` value and fill in the defaults.
|
|
55
|
+
*
|
|
56
|
+
* @param {unknown} raw @param {string} repoPath @param {string} [where] for error messages
|
|
57
|
+
* @returns {Convention}
|
|
58
|
+
*/
|
|
59
|
+
export function resolveConvention(raw, repoPath, where = join(repoPath, 'package.json')) {
|
|
60
|
+
const c = raw ?? {}
|
|
61
|
+
if (!isObject(c)) throw bad(where, 'must be an object')
|
|
62
|
+
onlyKeys(where, c, '', ['worktree', 'install', 'hydrate', 'prepare'])
|
|
63
|
+
|
|
64
|
+
const wt = c.worktree ?? {}
|
|
65
|
+
if (!isObject(wt)) throw bad(where, 'worktree must be an object')
|
|
66
|
+
onlyKeys(where, wt, 'worktree.', ['root', 'branch'])
|
|
67
|
+
const root = wt.root ?? DEFAULT_CONVENTION.root
|
|
68
|
+
if (typeof root !== 'string' || !root) throw bad(where, 'worktree.root must be a non-empty string')
|
|
69
|
+
const branch = wt.branch ?? DEFAULT_CONVENTION.branch
|
|
70
|
+
if (typeof branch !== 'string' || !branch.includes('{issue}'))
|
|
71
|
+
throw bad(where, 'worktree.branch must be a string containing {issue}')
|
|
72
|
+
const placeholders = branch.match(/\{[^}]*\}/g) ?? []
|
|
73
|
+
const unknownPh = placeholders.filter((p) => p !== '{issue}' && p !== '{slug}')
|
|
74
|
+
if (unknownPh.length) throw bad(where, `worktree.branch: unknown placeholder ${unknownPh.join(', ')}`)
|
|
75
|
+
if (!validBranch(branchName(branch, 1, 'x')))
|
|
76
|
+
throw bad(where, `worktree.branch is not a valid branch name: ${branch}`)
|
|
77
|
+
|
|
78
|
+
const install = c.install === undefined ? DEFAULT_CONVENTION.install : c.install
|
|
79
|
+
if (install !== false && (typeof install !== 'string' || !install))
|
|
80
|
+
throw bad(where, 'install must be a non-empty command string, or false')
|
|
81
|
+
|
|
82
|
+
/** @type {Convention['hydrate']} */
|
|
83
|
+
let hydrate = { dirs: [], files: [] }
|
|
84
|
+
if (c.hydrate !== undefined && c.hydrate !== false) {
|
|
85
|
+
if (!isObject(c.hydrate)) throw bad(where, 'hydrate must be an object or false')
|
|
86
|
+
onlyKeys(where, c.hydrate, 'hydrate.', ['dirs', 'files'])
|
|
87
|
+
hydrate = {
|
|
88
|
+
dirs: strings(where, 'hydrate.dirs', c.hydrate.dirs, (at, x) => relPath(where, at, x)),
|
|
89
|
+
files: strings(where, 'hydrate.files', c.hydrate.files, (at, x) => relPath(where, at, x)),
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
root: resolve(repoPath, root),
|
|
94
|
+
branch,
|
|
95
|
+
install: install === false ? null : install,
|
|
96
|
+
hydrate,
|
|
97
|
+
prepare: strings(where, 'prepare', c.prepare),
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The convention of the repo checked out at `repoPath`. No package.json or no
|
|
103
|
+
* `"cambium"` key → the defaults; a malformed one throws.
|
|
104
|
+
*
|
|
105
|
+
* @param {string} repoPath
|
|
106
|
+
* @returns {Convention}
|
|
107
|
+
*/
|
|
108
|
+
export function readConvention(repoPath) {
|
|
109
|
+
const file = join(repoPath, 'package.json')
|
|
110
|
+
if (!existsSync(file)) return resolveConvention(undefined, repoPath, file)
|
|
111
|
+
let pkg
|
|
112
|
+
try {
|
|
113
|
+
pkg = JSON.parse(readFileSync(file, 'utf8'))
|
|
114
|
+
} catch (err) {
|
|
115
|
+
throw new Error(`${file}: ${/** @type {Error} */ (err).message}`)
|
|
116
|
+
}
|
|
117
|
+
return resolveConvention(pkg?.cambium, repoPath, file)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** An issue title → a kebab slug of at most `max` characters, cut at a word. @param {string} title */
|
|
121
|
+
export function slugify(title, max = 40) {
|
|
122
|
+
const s = String(title)
|
|
123
|
+
.normalize('NFKD')
|
|
124
|
+
.toLowerCase()
|
|
125
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
126
|
+
.replace(/^-+|-+$/g, '')
|
|
127
|
+
if (s.length <= max) return s || 'task'
|
|
128
|
+
const cut = s.slice(0, max + 1)
|
|
129
|
+
const at = cut.lastIndexOf('-')
|
|
130
|
+
return (at > max / 2 ? cut.slice(0, at) : s.slice(0, max)).replace(/-+$/, '')
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** @param {string} template @param {number} issue @param {string} title */
|
|
134
|
+
export const branchName = (template, issue, title) =>
|
|
135
|
+
template.replaceAll('{issue}', String(issue)).replaceAll('{slug}', slugify(title))
|
|
136
|
+
|
|
137
|
+
/** `agent/12-fix-it` → `12-fix-it`: the branch less its first segment, flattened to one directory name. */
|
|
138
|
+
export const topicOf = (/** @type {string} */ branch) => branch.replace(/^[^/]+\/(?=.)/, '').replaceAll('/', '-')
|
|
139
|
+
|
|
140
|
+
/** A conservative `git check-ref-format --branch`. @param {string} b */
|
|
141
|
+
export const validBranch = (b) => /^[A-Za-z0-9._/-]+$/.test(b) && !/(^[-/.]|[/.]$|\.\.|\/\/|\/\.|\.lock$|@\{)/.test(b)
|
package/src/marker.mjs
CHANGED
|
@@ -3,16 +3,36 @@
|
|
|
3
3
|
* The cambium-owned state comment: a hidden JSON marker that is the truth,
|
|
4
4
|
* plus a human-readable rendering of it. One per issue; cambium edits it in
|
|
5
5
|
* place on every transition.
|
|
6
|
+
*
|
|
7
|
+
* Marker v2 names a position in a graph: `{ graph, graphVersion, node, status }`.
|
|
8
|
+
* v1 markers (a `stage` in the fixed task pipeline) are still read and are
|
|
9
|
+
* normalised to v2 in memory; the writer only emits v2.
|
|
10
|
+
*
|
|
11
|
+
* Until the graph runtime replaces the stage tick, v2 also carries the v1
|
|
12
|
+
* fields the tick drives (`phase`, `stage`, `pr`, `host`, …). While `stage` is
|
|
13
|
+
* present it is the tick's view and decides `node` (and `status`) on write.
|
|
6
14
|
*/
|
|
7
15
|
|
|
8
16
|
/**
|
|
9
17
|
* @typedef {'queued' | 'running' | 'submitted' | 'human' | 'done'} Stage
|
|
10
|
-
* @typedef {
|
|
18
|
+
* @typedef {'waiting' | 'entering' | 'done'} Status
|
|
11
19
|
* @typedef {{
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
20
|
+
* at: string, attempt: number, outcome: string, reason?: string, pr?: number, log?: string,
|
|
21
|
+
* node?: string | null, spanId?: string,
|
|
22
|
+
* }} HistoryEntry
|
|
23
|
+
* @typedef {{
|
|
24
|
+
* v: 2,
|
|
25
|
+
* graph: string,
|
|
26
|
+
* graphVersion: number,
|
|
27
|
+
* node: string,
|
|
28
|
+
* status: Status,
|
|
29
|
+
* input: Record<string, unknown> | null,
|
|
15
30
|
* attempt: number,
|
|
31
|
+
* traceparent: string | null,
|
|
32
|
+
* enteredAt: string | null,
|
|
33
|
+
* history: HistoryEntry[],
|
|
34
|
+
* phase: 'task' | 'dispatch',
|
|
35
|
+
* stage?: Stage,
|
|
16
36
|
* pr: number | null,
|
|
17
37
|
* branch: string | null,
|
|
18
38
|
* host: string | null,
|
|
@@ -21,28 +41,88 @@
|
|
|
21
41
|
* queued?: boolean,
|
|
22
42
|
* failedSha?: string | null,
|
|
23
43
|
* lastFailure?: string | null,
|
|
24
|
-
* history: HistoryEntry[],
|
|
25
44
|
* }} Marker
|
|
45
|
+
* @typedef {{ graph: string, entryNode: string, nodes?: Iterable<string> }} QueueOwner
|
|
26
46
|
*/
|
|
27
47
|
|
|
28
48
|
const PREFIX = '<!-- cambium:state '
|
|
29
49
|
const RE = /<!-- cambium:state (.*?) -->/s
|
|
30
50
|
const HISTORY_KEPT = 20
|
|
31
51
|
|
|
32
|
-
/**
|
|
33
|
-
export
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
52
|
+
/** The task graph's entry node: where a fresh or reset task issue starts. */
|
|
53
|
+
export const TASK_ENTRY = 'start-worker'
|
|
54
|
+
|
|
55
|
+
/** v1 `stage` → the task-graph node it corresponds to. */
|
|
56
|
+
export const STAGE_NODE = Object.freeze({
|
|
57
|
+
queued: TASK_ENTRY,
|
|
58
|
+
running: 'worker-running',
|
|
59
|
+
submitted: 'ci-running',
|
|
60
|
+
human: 'to-human',
|
|
61
|
+
done: 'close-out',
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Markers written by the stage tick (and v1 markers) predate graph versions.
|
|
66
|
+
* A real graph version is ≥ 1, so a stage-era marker always goes through the
|
|
67
|
+
* node-still-exists check the first time the graph runtime sees it.
|
|
68
|
+
*/
|
|
69
|
+
const STAGE_ERA_VERSION = 0
|
|
70
|
+
|
|
71
|
+
/** v2 defaults, without the stage tick's `stage`. @param {Partial<Marker>} from @returns {Marker} */
|
|
72
|
+
const withDefaults = (from) =>
|
|
73
|
+
syncStage({
|
|
74
|
+
graph: 'task',
|
|
75
|
+
graphVersion: STAGE_ERA_VERSION,
|
|
76
|
+
node: TASK_ENTRY,
|
|
77
|
+
status: 'waiting',
|
|
78
|
+
input: null,
|
|
38
79
|
attempt: 1,
|
|
80
|
+
traceparent: null,
|
|
81
|
+
enteredAt: null,
|
|
82
|
+
phase: 'task',
|
|
39
83
|
pr: null,
|
|
40
84
|
branch: null,
|
|
41
85
|
host: null,
|
|
42
86
|
start: null,
|
|
43
87
|
history: [],
|
|
44
88
|
...from,
|
|
45
|
-
|
|
89
|
+
v: 2,
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* A fresh marker. Unless a `node` is given it is the stage tick's: `queued`
|
|
94
|
+
* at the task graph's entry.
|
|
95
|
+
*
|
|
96
|
+
* @param {Partial<Marker>} [from] @returns {Marker}
|
|
97
|
+
*/
|
|
98
|
+
export const newMarker = (from = {}) => withDefaults(from.node ? from : { stage: 'queued', ...from })
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* While the stage tick drives the marker, `stage` decides the node: the tick
|
|
102
|
+
* only ever spreads a new `stage` into the marker.
|
|
103
|
+
*
|
|
104
|
+
* @param {Marker} m @returns {Marker}
|
|
105
|
+
*/
|
|
106
|
+
function syncStage(m) {
|
|
107
|
+
if (!m.stage || !(m.stage in STAGE_NODE)) return m
|
|
108
|
+
return { ...m, node: STAGE_NODE[m.stage], status: m.stage === 'done' ? 'done' : 'waiting' }
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* A v1 marker as v2: stage → node, phase → graph, `start` → `enteredAt`. The
|
|
113
|
+
* v1 fields stay, so the stage tick keeps working on the result.
|
|
114
|
+
*
|
|
115
|
+
* @param {Record<string, any>} v1 @returns {Marker}
|
|
116
|
+
*/
|
|
117
|
+
export function migrateV1(v1) {
|
|
118
|
+
return withDefaults({
|
|
119
|
+
stage: 'queued',
|
|
120
|
+
...v1,
|
|
121
|
+
graph: v1.phase ?? 'task',
|
|
122
|
+
graphVersion: STAGE_ERA_VERSION,
|
|
123
|
+
enteredAt: v1.start ?? null,
|
|
124
|
+
history: Array.isArray(v1.history) ? v1.history : [],
|
|
125
|
+
})
|
|
46
126
|
}
|
|
47
127
|
|
|
48
128
|
/** @param {string | null | undefined} body @returns {Marker | null} */
|
|
@@ -51,28 +131,41 @@ export function parseMarker(body) {
|
|
|
51
131
|
if (!m) return null
|
|
52
132
|
try {
|
|
53
133
|
const parsed = JSON.parse(m[1])
|
|
54
|
-
|
|
134
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null
|
|
135
|
+
if (parsed.v === 2) return withDefaults(parsed)
|
|
136
|
+
if (parsed.v === 1 || parsed.v === undefined) return migrateV1(parsed)
|
|
137
|
+
return null
|
|
55
138
|
} catch {
|
|
56
139
|
return null
|
|
57
140
|
}
|
|
58
141
|
}
|
|
59
142
|
|
|
60
|
-
/**
|
|
61
|
-
|
|
143
|
+
/**
|
|
144
|
+
* Appends to the history (newest 20 kept). An entry without a `node` records
|
|
145
|
+
* the node the marker is at, i.e. where the outcome happened.
|
|
146
|
+
*
|
|
147
|
+
* @param {Marker} marker @param {HistoryEntry} entry @returns {Marker}
|
|
148
|
+
*/
|
|
149
|
+
export const withHistory = (marker, entry) => ({
|
|
150
|
+
...marker,
|
|
151
|
+
history: [...marker.history, { node: marker.node, ...entry }].slice(-HISTORY_KEPT),
|
|
152
|
+
})
|
|
62
153
|
|
|
63
154
|
/**
|
|
64
155
|
* The comment body. `--` inside the JSON would end the HTML comment early, so
|
|
65
|
-
* it is written as a JSON escape.
|
|
156
|
+
* it is written as a JSON escape. Always emits v2.
|
|
66
157
|
*
|
|
67
|
-
* @param {Marker}
|
|
158
|
+
* @param {Marker} input
|
|
68
159
|
* @param {{ maxAttempts?: number }} [opts]
|
|
69
160
|
*/
|
|
70
|
-
export function renderMarker(
|
|
161
|
+
export function renderMarker(input, { maxAttempts } = {}) {
|
|
162
|
+
const marker = withDefaults(input)
|
|
71
163
|
const json = JSON.stringify(marker).replace(/--/g, '-\\u002d')
|
|
72
164
|
const of = maxAttempts ? `/${maxAttempts}` : ''
|
|
73
165
|
const lines = [
|
|
74
166
|
`${PREFIX}${json} -->`,
|
|
75
|
-
`**cambium** · ${marker.
|
|
167
|
+
`**cambium** · ${marker.graph} · ${marker.node}${marker.status === 'waiting' ? '' : ` (${marker.status})`}` +
|
|
168
|
+
` · attempt ${marker.attempt}${of}` +
|
|
76
169
|
(marker.pr ? ` · PR #${marker.pr}` : '') +
|
|
77
170
|
(marker.host ? ` · host \`${marker.host}\`` : '') +
|
|
78
171
|
(marker.start ? ` · started ${marker.start}` : ''),
|
|
@@ -88,10 +181,10 @@ export function renderMarker(marker, { maxAttempts } = {}) {
|
|
|
88
181
|
'</details>',
|
|
89
182
|
)
|
|
90
183
|
if (marker.history.length) {
|
|
91
|
-
lines.push('', '| at | attempt | outcome | detail |', '
|
|
184
|
+
lines.push('', '| at | node | attempt | outcome | detail |', '|---|---|---|---|---|')
|
|
92
185
|
for (const h of marker.history)
|
|
93
186
|
lines.push(
|
|
94
|
-
`| ${h.at} | ${h.attempt} | ${h.outcome} | ${[
|
|
187
|
+
`| ${h.at} | ${h.node ?? ''} | ${h.attempt} | ${h.outcome} | ${[
|
|
95
188
|
h.pr ? `#${h.pr}` : '',
|
|
96
189
|
h.reason ?? '',
|
|
97
190
|
h.log ? `log \`${h.log}\`` : '',
|
|
@@ -105,3 +198,67 @@ export function renderMarker(marker, { maxAttempts } = {}) {
|
|
|
105
198
|
lines.push('', '_Written by cambium. Only cambium moves this issue’s labels._')
|
|
106
199
|
return lines.join('\n')
|
|
107
200
|
}
|
|
201
|
+
|
|
202
|
+
// ---- Rules for the graph runtime (pure; the runtime applies them)
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Label vs node: the queue label wins. If the issue's queue label is owned by
|
|
206
|
+
* a different graph than the marker's, or (when the owner lists its `nodes`)
|
|
207
|
+
* the marker's node does not sit in that queue, or the marker's graph is done,
|
|
208
|
+
* the issue restarts at the owning graph's entry node.
|
|
209
|
+
*
|
|
210
|
+
* No queue label known to `queues`: the marker stands (nothing to compare).
|
|
211
|
+
* No marker: a fresh start at the entry node, which is not a reset.
|
|
212
|
+
*
|
|
213
|
+
* @param {string[]} labels
|
|
214
|
+
* @param {Marker | null} marker
|
|
215
|
+
* @param {Record<string, QueueOwner>} queues queue label → owning graph
|
|
216
|
+
* @returns {{ graph: string, node: string, reset: boolean } | null} null: no queue label and no marker
|
|
217
|
+
*/
|
|
218
|
+
export function effectivePosition(labels, marker, queues) {
|
|
219
|
+
const label = labels.find((l) => Object.hasOwn(queues, l))
|
|
220
|
+
if (!label) return marker ? { graph: marker.graph, node: marker.node, reset: false } : null
|
|
221
|
+
const owner = queues[label]
|
|
222
|
+
if (!marker) return { graph: owner.graph, node: owner.entryNode, reset: false }
|
|
223
|
+
const agrees =
|
|
224
|
+
marker.graph === owner.graph && marker.status !== 'done' && (!owner.nodes || new Set(owner.nodes).has(marker.node))
|
|
225
|
+
return agrees
|
|
226
|
+
? { graph: marker.graph, node: marker.node, reset: false }
|
|
227
|
+
: { graph: owner.graph, node: owner.entryNode, reset: true }
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Graph versioning: a marker written under an older version keeps running if
|
|
232
|
+
* its node still exists in the current version; otherwise it restarts at the
|
|
233
|
+
* entry node, with a history note saying why.
|
|
234
|
+
*
|
|
235
|
+
* @param {Marker} marker
|
|
236
|
+
* @param {{ version: number, nodes: Iterable<string>, entryNode: string, at: string }} current
|
|
237
|
+
* @returns {{ action: 'keep' } | { action: 'restart', node: string, note: HistoryEntry }}
|
|
238
|
+
*/
|
|
239
|
+
export function graphVersionDecision(marker, { version, nodes, entryNode, at }) {
|
|
240
|
+
if (marker.graphVersion === version || new Set(nodes).has(marker.node)) return { action: 'keep' }
|
|
241
|
+
return {
|
|
242
|
+
action: 'restart',
|
|
243
|
+
node: entryNode,
|
|
244
|
+
note: {
|
|
245
|
+
at,
|
|
246
|
+
attempt: marker.attempt,
|
|
247
|
+
node: marker.node,
|
|
248
|
+
outcome: 'graph-changed',
|
|
249
|
+
reason: `${marker.graph} v${marker.graphVersion} → v${version}: node ${marker.node} no longer exists; restarting at ${entryNode}`,
|
|
250
|
+
},
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* `status: 'entering'` is written before a step node runs. On a later tick,
|
|
256
|
+
* a marker still `entering` with no tracked process means the step died
|
|
257
|
+
* mid-flight: the node's `recover` runs (default: requeue). Anything else
|
|
258
|
+
* resumes normally.
|
|
259
|
+
*
|
|
260
|
+
* @param {Marker} marker @param {boolean} hasTrackedProcess
|
|
261
|
+
* @returns {'resume' | 'recover'}
|
|
262
|
+
*/
|
|
263
|
+
export const enteringDecision = (marker, hasTrackedProcess) =>
|
|
264
|
+
marker.status === 'entering' && !hasTrackedProcess ? 'recover' : 'resume'
|
package/src/phases/task.mjs
CHANGED
|
@@ -35,9 +35,6 @@ ${issue.body}
|
|
|
35
35
|
${agentsMd ? `\n--- AGENTS.md\n${agentsMd}` : ''}`
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
/** @param {number} issue */
|
|
39
|
-
export const taskBranch = (issue) => `agent/issue-${issue}`
|
|
40
|
-
|
|
41
38
|
/**
|
|
42
39
|
* @param {import('../github.mjs').Snapshot} snap
|
|
43
40
|
* @param {{ issue: number, branch?: string | null }} w
|
package/src/prepare.mjs
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// @ts-check
|
|
3
|
+
/**
|
|
4
|
+
* The prepare step, and the worker's entry point. The spawner starts
|
|
5
|
+
* `node prepare.mjs -- <worker command…>` detached in the new worktree with
|
|
6
|
+
* {@link PrepareSpec} in `$CAMBIUM_PREPARE`: it runs install-fresh, hydrate and
|
|
7
|
+
* the repo's own `prepare` commands, then execs the worker. Running here
|
|
8
|
+
* rather than in the tick keeps a minutes-long install from blocking it; the
|
|
9
|
+
* output lands in the worker's stderr log, and a failure exits
|
|
10
|
+
* {@link PREPARE_FAILED} with a {@link PREPARE_FAILED_LINE} line the
|
|
11
|
+
* reconcile recognises.
|
|
12
|
+
*/
|
|
13
|
+
import { spawn, spawnSync } from 'node:child_process'
|
|
14
|
+
import {
|
|
15
|
+
constants,
|
|
16
|
+
copyFileSync,
|
|
17
|
+
existsSync,
|
|
18
|
+
lstatSync,
|
|
19
|
+
mkdirSync,
|
|
20
|
+
readdirSync,
|
|
21
|
+
readFileSync,
|
|
22
|
+
rmSync,
|
|
23
|
+
statSync,
|
|
24
|
+
} from 'node:fs'
|
|
25
|
+
import { dirname, join } from 'node:path'
|
|
26
|
+
import { fileURLToPath } from 'node:url'
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @typedef {{ worktree: string, source: string } & Pick<import('./convention.mjs').Convention, 'install' | 'hydrate' | 'prepare'>} PrepareSpec
|
|
30
|
+
* @typedef {(cmd: string, cwd: string) => number} Run a shell command; its exit status
|
|
31
|
+
* @typedef {(from: string, to: string, opts: { clone: boolean }) => number} Copy a recursive copy; its exit status
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
export const PREPARE_FAILED = 78
|
|
35
|
+
export const PREPARE_FAILED_LINE = 'cambium: prepare failed'
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Is `node_modules` the install of this lockfile? pnpm keeps a byte copy of
|
|
39
|
+
* `pnpm-lock.yaml` at `node_modules/.pnpm/lock.yaml`. No package.json → there
|
|
40
|
+
* is nothing to install.
|
|
41
|
+
*
|
|
42
|
+
* @param {string} root
|
|
43
|
+
* @returns {'fresh' | 'stale' | 'none'}
|
|
44
|
+
*/
|
|
45
|
+
export function installState(root) {
|
|
46
|
+
if (!existsSync(join(root, 'package.json'))) return 'none'
|
|
47
|
+
const lock = join(root, 'pnpm-lock.yaml')
|
|
48
|
+
const installed = join(root, 'node_modules', '.pnpm', 'lock.yaml')
|
|
49
|
+
if (existsSync(lock) && existsSync(installed) && readFileSync(lock).equals(readFileSync(installed))) return 'fresh'
|
|
50
|
+
return 'stale'
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** @param {string} path */
|
|
54
|
+
const populated = (path) => {
|
|
55
|
+
try {
|
|
56
|
+
return statSync(path).isDirectory() && readdirSync(path).length > 0
|
|
57
|
+
} catch {
|
|
58
|
+
return false
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Anything at `path`, dangling symlinks included. @param {string} path */
|
|
63
|
+
const onDisk = (path) => {
|
|
64
|
+
try {
|
|
65
|
+
lstatSync(path)
|
|
66
|
+
return true
|
|
67
|
+
} catch {
|
|
68
|
+
return false
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** @type {Copy} */
|
|
73
|
+
function cpCopy(from, to, { clone }) {
|
|
74
|
+
const flags = clone ? (process.platform === 'darwin' ? ['-Rc'] : ['-R', '--reflink=auto']) : ['-R']
|
|
75
|
+
return spawnSync('cp', [...flags, from, to], { stdio: ['ignore', 2, 2] }).status ?? 1
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Copy the git-ignored `dirs` and `files` from the central checkout into the
|
|
80
|
+
* worktree: a copy-on-write clone (`cp -c`, ~free on APFS), a byte copy when
|
|
81
|
+
* the clone fails. Never a link: a write in the worktree must not reach the
|
|
82
|
+
* checkout. What the worktree already has is left alone; what the source lacks
|
|
83
|
+
* is reported and skipped. A copy that fails outright is removed and throws.
|
|
84
|
+
*
|
|
85
|
+
* @param {string} source @param {string} dest @param {{ dirs: string[], files: string[] }} lists
|
|
86
|
+
* @param {{ say?: (line: string) => void, copy?: Copy }} [io]
|
|
87
|
+
* @returns {string[]} what was copied
|
|
88
|
+
*/
|
|
89
|
+
export function hydrate(source, dest, { dirs, files }, { say = console.error, copy = cpCopy } = {}) {
|
|
90
|
+
const done = []
|
|
91
|
+
for (const rel of dirs) {
|
|
92
|
+
const from = join(source, rel)
|
|
93
|
+
const to = join(dest, rel)
|
|
94
|
+
if (populated(to)) {
|
|
95
|
+
say(`hydrate: ${rel} already present, left alone`)
|
|
96
|
+
continue
|
|
97
|
+
}
|
|
98
|
+
if (!populated(from)) {
|
|
99
|
+
say(`hydrate: ${rel} missing in ${source}; not copied`)
|
|
100
|
+
continue
|
|
101
|
+
}
|
|
102
|
+
mkdirSync(dirname(to), { recursive: true })
|
|
103
|
+
rmSync(to, { recursive: true, force: true })
|
|
104
|
+
if (copy(from, to, { clone: true }) !== 0) {
|
|
105
|
+
say(`hydrate: ${rel} clone failed, byte-copying`)
|
|
106
|
+
rmSync(to, { recursive: true, force: true })
|
|
107
|
+
if (copy(from, to, { clone: false }) !== 0) {
|
|
108
|
+
// A half-copied dir passes every presence check downstream; an absent one is honest.
|
|
109
|
+
rmSync(to, { recursive: true, force: true })
|
|
110
|
+
throw new Error(`hydrate: could not copy ${rel} from ${source}`)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
say(`hydrate: ${rel} copied`)
|
|
114
|
+
done.push(rel)
|
|
115
|
+
}
|
|
116
|
+
for (const rel of files) {
|
|
117
|
+
const from = join(source, rel)
|
|
118
|
+
const to = join(dest, rel)
|
|
119
|
+
if (onDisk(to)) {
|
|
120
|
+
say(`hydrate: ${rel} already present, left alone`)
|
|
121
|
+
continue
|
|
122
|
+
}
|
|
123
|
+
if (!existsSync(from) || !statSync(from).isFile()) {
|
|
124
|
+
say(`hydrate: ${rel} missing in ${source}; not copied`)
|
|
125
|
+
continue
|
|
126
|
+
}
|
|
127
|
+
mkdirSync(dirname(to), { recursive: true })
|
|
128
|
+
// FICLONE clones where it can and byte-copies where it cannot.
|
|
129
|
+
copyFileSync(from, to, constants.COPYFILE_EXCL | constants.COPYFILE_FICLONE)
|
|
130
|
+
say(`hydrate: ${rel} copied`)
|
|
131
|
+
done.push(rel)
|
|
132
|
+
}
|
|
133
|
+
return done
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** @type {Run} */
|
|
137
|
+
const shell = (cmd, cwd) => spawnSync('sh', ['-c', cmd], { cwd, stdio: ['ignore', 2, 2] }).status ?? 1
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* install-fresh → hydrate → the repo's `prepare` commands, in the worktree.
|
|
141
|
+
* Throws on the first failure.
|
|
142
|
+
*
|
|
143
|
+
* @param {PrepareSpec} spec
|
|
144
|
+
* @param {{ run?: Run, copy?: Copy, say?: (line: string) => void }} [io]
|
|
145
|
+
*/
|
|
146
|
+
export function prepare(spec, { run = shell, copy, say = console.error } = {}) {
|
|
147
|
+
const { worktree, source } = spec
|
|
148
|
+
if (spec.install) {
|
|
149
|
+
const state = installState(worktree)
|
|
150
|
+
if (state === 'stale') {
|
|
151
|
+
say(`install: ${spec.install}`)
|
|
152
|
+
const code = run(spec.install, worktree)
|
|
153
|
+
if (code !== 0) throw new Error(`install \`${spec.install}\` exited ${code}`)
|
|
154
|
+
} else say(`install: ${state === 'fresh' ? 'node_modules matches the lockfile' : 'no package.json'}; skipped`)
|
|
155
|
+
}
|
|
156
|
+
hydrate(source, worktree, spec.hydrate, { say, ...(copy ? { copy } : {}) })
|
|
157
|
+
for (const cmd of spec.prepare) {
|
|
158
|
+
say(`prepare: ${cmd}`)
|
|
159
|
+
const code = run(cmd, worktree)
|
|
160
|
+
if (code !== 0) throw new Error(`prepare \`${cmd}\` exited ${code}`)
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** The entry point: prepare, then become the worker. @param {string[]} argv */
|
|
165
|
+
async function main(argv) {
|
|
166
|
+
const command = argv.slice(argv.indexOf('--') + 1)
|
|
167
|
+
/** @type {PrepareSpec} */
|
|
168
|
+
const spec = JSON.parse(process.env.CAMBIUM_PREPARE ?? 'null')
|
|
169
|
+
try {
|
|
170
|
+
prepare(spec)
|
|
171
|
+
} catch (err) {
|
|
172
|
+
console.error(`${PREPARE_FAILED_LINE}: ${/** @type {Error} */ (err)?.message ?? err}`)
|
|
173
|
+
process.exit(PREPARE_FAILED)
|
|
174
|
+
}
|
|
175
|
+
const env = { ...process.env }
|
|
176
|
+
delete env.CAMBIUM_PREPARE
|
|
177
|
+
const child = spawn(command[0], command.slice(1), { cwd: spec.worktree, stdio: 'inherit', env })
|
|
178
|
+
child.on('error', (err) => {
|
|
179
|
+
console.error(`cambium: could not start ${command[0]}: ${err.message}`)
|
|
180
|
+
process.exit(127)
|
|
181
|
+
})
|
|
182
|
+
child.on('exit', (code, signal) => process.exit(code ?? (signal ? 128 : 1)))
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) await main(process.argv.slice(2))
|
package/src/tick.mjs
CHANGED
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { apply, byPriorityThenAge, move, Q, queueOf, RUNNING, WHY } from './labels.mjs'
|
|
10
10
|
import { newMarker, renderMarker, withHistory } from './marker.mjs'
|
|
11
|
-
import { prFor,
|
|
11
|
+
import { prFor, taskExit, taskPrompt } from './phases/task.mjs'
|
|
12
|
+
import { branchName } from './convention.mjs'
|
|
13
|
+
import { PREPARE_FAILED_LINE } from './prepare.mjs'
|
|
12
14
|
import { releaseDue, releaseNotBuilt } from './phases/release.mjs'
|
|
13
15
|
import { DISPATCH_NOT_BUILT, dispatchNotBuilt } from './phases/dispatch.mjs'
|
|
14
16
|
|
|
@@ -261,14 +263,21 @@ export async function finish(deps, ctx, w, snap) {
|
|
|
261
263
|
const pr = prFor(snap, w)
|
|
262
264
|
const done = async () => {
|
|
263
265
|
deps.store.removeWorker(w.repo, w.issue)
|
|
264
|
-
if (repo)
|
|
266
|
+
if (!repo) return
|
|
267
|
+
const t = await deps.spawner.cleanup({ repo, worktree: w.worktree, branch: w.branch })
|
|
268
|
+
if (!t.removed) deps.log('warn', `kept worktree ${w.worktree}: ${t.reason}`, { repo: w.repo, issue: w.issue })
|
|
265
269
|
}
|
|
266
270
|
if (issue?.state !== 'OPEN') {
|
|
267
271
|
deps.log('info', 'worker gone; issue is closed or no longer queued', { repo: w.repo, issue: w.issue })
|
|
268
272
|
return done()
|
|
269
273
|
}
|
|
270
274
|
const m = markerOf(issue)
|
|
271
|
-
const
|
|
275
|
+
const prepFailed = !w.killedReason && deps.spawner.tail(w.stderr, 15).includes(PREPARE_FAILED_LINE)
|
|
276
|
+
const verdict = w.killedReason
|
|
277
|
+
? { ok: false, reason: w.killedReason }
|
|
278
|
+
: prepFailed
|
|
279
|
+
? { ok: false, reason: 'could not start the worker: prepare failed' }
|
|
280
|
+
: taskExit(pr, m)
|
|
272
281
|
if (verdict.ok && pr) {
|
|
273
282
|
const marker = {
|
|
274
283
|
...withHistory(m, { at: iso(ctx.now), attempt: m.attempt, outcome: 'submitted', pr: pr.number, log: w.stdout }),
|
|
@@ -288,7 +297,7 @@ export async function finish(deps, ctx, w, snap) {
|
|
|
288
297
|
const reason = `${'reason' in verdict ? verdict.reason : 'worker failed'}${tail ? `\n--- stderr (last lines, ${w.stderr})\n${tail}` : ''}`
|
|
289
298
|
await requeue(deps, ctx, w.repo, issue, {
|
|
290
299
|
reason,
|
|
291
|
-
outcome: w.killedReason ? 'killed' : 'failed',
|
|
300
|
+
outcome: w.killedReason ? 'killed' : prepFailed ? 'spawn-failed' : 'failed',
|
|
292
301
|
pr: pr?.number,
|
|
293
302
|
log: w.stdout,
|
|
294
303
|
})
|
|
@@ -456,9 +465,21 @@ export async function startTask(deps, ctx, slug, issue, snap, { foreground = fal
|
|
|
456
465
|
await toHuman(deps, ctx, slug, issue, { reason: `${caps.maxAttempts} attempts spent`, outcome: 'budget' })
|
|
457
466
|
return null
|
|
458
467
|
}
|
|
468
|
+
/** @type {import('./convention.mjs').Convention} */
|
|
469
|
+
let convention
|
|
470
|
+
try {
|
|
471
|
+
convention = deps.spawner.convention(repo)
|
|
472
|
+
} catch (err) {
|
|
473
|
+
await requeue(deps, ctx, slug, issue, {
|
|
474
|
+
reason: `could not start the worker: ${/** @type {Error} */ (err)?.message ?? err}`,
|
|
475
|
+
outcome: 'spawn-failed',
|
|
476
|
+
})
|
|
477
|
+
return null
|
|
478
|
+
}
|
|
459
479
|
const known = prFor(snap, { issue: issue.number, branch: m0.branch })
|
|
460
480
|
const pr = known?.state === 'OPEN' ? known : null
|
|
461
|
-
|
|
481
|
+
// Sticky: a retry keeps the branch (and so the worktree topic) its first attempt chose.
|
|
482
|
+
const branch = pr?.branch ?? m0.branch ?? branchName(convention.branch, issue.number, issue.title)
|
|
462
483
|
const marker = {
|
|
463
484
|
...m0,
|
|
464
485
|
phase: /** @type {const} */ ('task'),
|
|
@@ -476,6 +497,7 @@ export async function startTask(deps, ctx, slug, issue, snap, { foreground = fal
|
|
|
476
497
|
try {
|
|
477
498
|
started = await deps.spawner.start({
|
|
478
499
|
repo,
|
|
500
|
+
convention,
|
|
479
501
|
issue: issue.number,
|
|
480
502
|
attempt: marker.attempt,
|
|
481
503
|
branch,
|
|
@@ -505,6 +527,11 @@ export async function startTask(deps, ctx, slug, issue, snap, { foreground = fal
|
|
|
505
527
|
stdout: started.stdout,
|
|
506
528
|
stderr: started.stderr,
|
|
507
529
|
})
|
|
530
|
+
if (started.reused)
|
|
531
|
+
deps.log('warn', `continuing in the previous worktree ${started.worktree}: ${started.reused}`, {
|
|
532
|
+
repo: slug,
|
|
533
|
+
issue: issue.number,
|
|
534
|
+
})
|
|
508
535
|
deps.log('info', `started task worker pid ${started.pid} (attempt ${marker.attempt}, ${branch})`, {
|
|
509
536
|
repo: slug,
|
|
510
537
|
issue: issue.number,
|
package/src/workers.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
/**
|
|
3
3
|
* The process seams: a {@link Spawner} that starts a `claude -p` worker in its
|
|
4
|
-
* own worktree
|
|
4
|
+
* own worktree (the target repo's convention, see convention.mjs) and process
|
|
5
|
+
* group, behind the prepare step, with its output in rotated log files on this
|
|
5
6
|
* host, and {@link Procs} to probe and kill by pid.
|
|
6
7
|
*/
|
|
7
8
|
import { spawn } from 'node:child_process'
|
|
@@ -17,18 +18,26 @@ import {
|
|
|
17
18
|
statSync,
|
|
18
19
|
writeFileSync,
|
|
19
20
|
} from 'node:fs'
|
|
20
|
-
import { join } from 'node:path'
|
|
21
|
+
import { dirname, join } from 'node:path'
|
|
22
|
+
import { fileURLToPath } from 'node:url'
|
|
23
|
+
import { readConvention, topicOf } from './convention.mjs'
|
|
21
24
|
import { exec as defaultExec } from './github.mjs'
|
|
22
25
|
|
|
23
26
|
/**
|
|
27
|
+
* @typedef {import('./convention.mjs').Convention} Convention
|
|
24
28
|
* @typedef {{
|
|
25
|
-
* repo: import('./config.mjs').Repo, issue: number, attempt: number, branch: string,
|
|
26
|
-
* maxTurns: number, buildPrompt: (agentsMd: string | null) => string, foreground?: boolean,
|
|
29
|
+
* repo: import('./config.mjs').Repo, convention: Convention, issue: number, attempt: number, branch: string,
|
|
30
|
+
* phase: string, maxTurns: number, buildPrompt: (agentsMd: string | null) => string, foreground?: boolean,
|
|
27
31
|
* }} StartSpec
|
|
28
|
-
* @typedef {{ pid: number, worktree: string, stdout: string, stderr: string, prompt: string, exited: Promise<number | null> }} Started
|
|
29
32
|
* @typedef {{
|
|
33
|
+
* pid: number, worktree: string, stdout: string, stderr: string, prompt: string, exited: Promise<number | null>,
|
|
34
|
+
* reused: string | null,
|
|
35
|
+
* }} Started `reused`: why the previous attempt's worktree was kept, when it was
|
|
36
|
+
* @typedef {{ removed: boolean, reason: string | null }} Teardown
|
|
37
|
+
* @typedef {{
|
|
38
|
+
* convention(repo: import('./config.mjs').Repo): Convention,
|
|
30
39
|
* start(spec: StartSpec): Promise<Started>,
|
|
31
|
-
* cleanup(w: { repo: import('./config.mjs').Repo, worktree: string }): Promise<
|
|
40
|
+
* cleanup(w: { repo: import('./config.mjs').Repo, worktree: string, branch: string }): Promise<Teardown>,
|
|
32
41
|
* tail(path: string, lines?: number): string,
|
|
33
42
|
* }} Spawner
|
|
34
43
|
* @typedef {{ alive(pid: number): boolean, kill(pid: number): void }} Procs
|
|
@@ -53,6 +62,11 @@ export const workerArgs = (config, maxTurns) => [
|
|
|
53
62
|
...config.worker.args,
|
|
54
63
|
]
|
|
55
64
|
|
|
65
|
+
/** The last line `graft status` prints when the worktree holds nothing GitHub lacks. */
|
|
66
|
+
export const GRAFT_CLEAN = 'nothing local that GitHub does not have'
|
|
67
|
+
|
|
68
|
+
const PREPARE_MAIN = join(dirname(fileURLToPath(import.meta.url)), 'prepare.mjs')
|
|
69
|
+
|
|
56
70
|
/** `owner/name` → a directory name. @param {string} slug */
|
|
57
71
|
export const slugDir = (slug) => slug.replace('/', '__')
|
|
58
72
|
|
|
@@ -102,16 +116,53 @@ export function processSpawner({
|
|
|
102
116
|
stamp = () => new Date().toISOString().replace(/[:.]/g, '-'),
|
|
103
117
|
}) {
|
|
104
118
|
const git = (/** @type {string} */ repoPath, /** @type {string[]} */ args) => exec('git', ['-C', repoPath, ...args])
|
|
119
|
+
const graftEnv = { ...process.env, ...(config.graft.app ? { GRAFT_APP: config.graft.app } : {}) }
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Remove a worktree only once `graft status` says GitHub holds everything in
|
|
123
|
+
* it; otherwise keep it and say why. Its local branch goes with it.
|
|
124
|
+
*
|
|
125
|
+
* @param {{ repo: import('./config.mjs').Repo, worktree: string, branch: string }} w
|
|
126
|
+
* @returns {Promise<Teardown>}
|
|
127
|
+
*/
|
|
128
|
+
async function teardown({ repo, worktree, branch }) {
|
|
129
|
+
if (!existsSync(worktree)) {
|
|
130
|
+
await git(repo.path, ['worktree', 'prune']).catch(() => {})
|
|
131
|
+
return { removed: true, reason: null }
|
|
132
|
+
}
|
|
133
|
+
let out
|
|
134
|
+
try {
|
|
135
|
+
out = await exec(config.graft.command, ['status', branch, '-C', worktree, '--repo', repo.slug], {
|
|
136
|
+
cwd: worktree,
|
|
137
|
+
env: graftEnv,
|
|
138
|
+
})
|
|
139
|
+
} catch (err) {
|
|
140
|
+
return { removed: false, reason: `graft status failed: ${/** @type {Error} */ (err)?.message ?? err}` }
|
|
141
|
+
}
|
|
142
|
+
const verdict = out.trim().split('\n').at(-1) ?? ''
|
|
143
|
+
if (verdict !== GRAFT_CLEAN) return { removed: false, reason: verdict || 'graft status printed nothing' }
|
|
144
|
+
const head = (await git(worktree, ['symbolic-ref', '--short', '-q', 'HEAD']).catch(() => '')).trim()
|
|
145
|
+
await git(repo.path, ['worktree', 'remove', '--force', worktree])
|
|
146
|
+
if (head === branch) await git(repo.path, ['branch', '-D', branch]).catch(() => {})
|
|
147
|
+
return { removed: true, reason: null }
|
|
148
|
+
}
|
|
149
|
+
|
|
105
150
|
return {
|
|
106
|
-
|
|
107
|
-
|
|
151
|
+
convention: (repo) => readConvention(repo.path),
|
|
152
|
+
async start({ repo, convention, issue, attempt, branch, phase, maxTurns, buildPrompt, foreground = false }) {
|
|
153
|
+
const worktree = join(convention.root, topicOf(branch))
|
|
108
154
|
if (config.worker.fetch) await git(repo.path, ['fetch', 'origin', repo.baseBranch])
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
155
|
+
/** @type {string | null} */
|
|
156
|
+
let reused = null
|
|
157
|
+
// The same topic every attempt: a leftover is removed if GitHub holds it all, else continued in.
|
|
158
|
+
if (existsSync(join(worktree, '.git'))) {
|
|
159
|
+
const left = await teardown({ repo, worktree, branch })
|
|
160
|
+
if (!left.removed) reused = left.reason
|
|
161
|
+
} else rmSync(worktree, { recursive: true, force: true })
|
|
162
|
+
if (!reused) {
|
|
163
|
+
mkdirSync(dirname(worktree), { recursive: true })
|
|
164
|
+
await git(repo.path, ['worktree', 'add', '--no-track', '-B', branch, worktree, `origin/${repo.baseBranch}`])
|
|
112
165
|
}
|
|
113
|
-
mkdirSync(join(worktree, '..'), { recursive: true })
|
|
114
|
-
await git(repo.path, ['worktree', 'add', '--detach', worktree, `origin/${repo.baseBranch}`])
|
|
115
166
|
const agentsPath = join(worktree, 'AGENTS.md')
|
|
116
167
|
const agentsMd = existsSync(agentsPath) ? readFileSync(agentsPath, 'utf8') : null
|
|
117
168
|
|
|
@@ -129,17 +180,23 @@ export function processSpawner({
|
|
|
129
180
|
CAMBIUM_ISSUE: String(issue),
|
|
130
181
|
CAMBIUM_PHASE: phase,
|
|
131
182
|
CAMBIUM_BRANCH: branch,
|
|
183
|
+
CAMBIUM_PREPARE: JSON.stringify({
|
|
184
|
+
worktree,
|
|
185
|
+
source: repo.path,
|
|
186
|
+
install: convention.install,
|
|
187
|
+
hydrate: convention.hydrate,
|
|
188
|
+
prepare: convention.prepare,
|
|
189
|
+
}),
|
|
132
190
|
...(config.worker.graftApp ? { GRAFT_APP: config.worker.graftApp } : {}),
|
|
133
191
|
}
|
|
134
192
|
/** @type {import('node:child_process').ChildProcess} */
|
|
135
193
|
let child
|
|
136
194
|
try {
|
|
137
|
-
child = spawnFn(
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
stdio: [fds[0], fds[1], fds[2]],
|
|
141
|
-
|
|
142
|
-
})
|
|
195
|
+
child = spawnFn(
|
|
196
|
+
process.execPath,
|
|
197
|
+
[PREPARE_MAIN, '--', config.worker.command, ...workerArgs(config, maxTurns)],
|
|
198
|
+
{ cwd: worktree, detached: !foreground, stdio: [fds[0], fds[1], fds[2]], env },
|
|
199
|
+
)
|
|
143
200
|
} finally {
|
|
144
201
|
for (const fd of fds) closeSync(fd)
|
|
145
202
|
}
|
|
@@ -147,13 +204,11 @@ export function processSpawner({
|
|
|
147
204
|
child.on('exit', (code) => res(code))
|
|
148
205
|
child.on('error', () => res(null))
|
|
149
206
|
})
|
|
150
|
-
if (!child.pid) throw new Error(`could not start ${
|
|
207
|
+
if (!child.pid) throw new Error(`could not start the worker (${process.execPath} ${PREPARE_MAIN})`)
|
|
151
208
|
if (!foreground) child.unref()
|
|
152
|
-
return { pid: child.pid, worktree, ...files, exited }
|
|
153
|
-
},
|
|
154
|
-
async cleanup({ repo, worktree }) {
|
|
155
|
-
await git(repo.path, ['worktree', 'remove', '--force', worktree]).catch(() => {})
|
|
209
|
+
return { pid: child.pid, worktree, ...files, exited, reused }
|
|
156
210
|
},
|
|
211
|
+
cleanup: teardown,
|
|
157
212
|
tail: tailFile,
|
|
158
213
|
}
|
|
159
214
|
}
|