@codedology/code-forge 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/LICENSE +21 -0
- package/README.md +127 -0
- package/bin/code-forge.mjs +145 -0
- package/docs/README.md +37 -0
- package/docs/reference/blocks.json +288 -0
- package/docs/reference/config.md +162 -0
- package/examples/first-use/README.md +11 -0
- package/examples/first-use/clamp.acceptance.yml +8 -0
- package/examples/first-use/clamp.brief.md +16 -0
- package/examples/first-use/coder/src/math.mjs +31 -0
- package/examples/first-use/coder/test/clamp.test.mjs +26 -0
- package/examples/node-lib/.env.example +3 -0
- package/examples/node-lib/.gitattributes +3 -0
- package/examples/node-lib/.github/workflows/test.yml +11 -0
- package/examples/node-lib/CHANGELOG.md +13 -0
- package/examples/node-lib/README.md +17 -0
- package/examples/node-lib/gitignore +3 -0
- package/examples/node-lib/package.json +19 -0
- package/examples/node-lib/src/list.mjs +23 -0
- package/examples/node-lib/src/math.mjs +21 -0
- package/examples/node-lib/src/strings.mjs +34 -0
- package/examples/node-lib/test/list.test.mjs +12 -0
- package/examples/node-lib/test/math.test.mjs +13 -0
- package/examples/node-lib/test/package.test.mjs +18 -0
- package/examples/node-lib/test/strings.test.mjs +28 -0
- package/package.json +52 -0
- package/schema/code-forge.schema.json +331 -0
- package/schema/facts.schema.json +28 -0
- package/scripts/first-use.mjs +517 -0
- package/scripts/gen-config-doc.mjs +241 -0
- package/skill/SKILL.md +79 -0
- package/skill/references/adapters/claude-code.md +19 -0
- package/skill/references/adapters/solo.md +34 -0
- package/skill/references/adapters/subprocess.md +15 -0
- package/skill/references/code.md +64 -0
- package/skill/references/continuity.md +31 -0
- package/skill/references/decisions.md +50 -0
- package/skill/references/degraded.md +40 -0
- package/skill/references/harden.md +26 -0
- package/skill/references/ledger.md +23 -0
- package/skill/references/plan.md +43 -0
- package/skill/references/proof.md +34 -0
- package/skill/references/review.md +42 -0
- package/skill/references/rules-core.md +72 -0
- package/skill/references/security.md +29 -0
- package/skill/scripts/forge +40 -0
- package/skill/templates/block-record.md +19 -0
- package/skill/templates/facts-sheet.md +22 -0
- package/skill/templates/plan.md +53 -0
- package/src/cli/author.mjs +87 -0
- package/src/cli/block.mjs +262 -0
- package/src/cli/doctor.mjs +74 -0
- package/src/cli/facts.mjs +124 -0
- package/src/cli/gates.mjs +280 -0
- package/src/cli/help.mjs +48 -0
- package/src/cli/init.mjs +14 -0
- package/src/cli/jev.mjs +175 -0
- package/src/cli/keys.mjs +170 -0
- package/src/cli/ledger.mjs +160 -0
- package/src/cli/list.mjs +68 -0
- package/src/cli/models.mjs +131 -0
- package/src/cli/plan.mjs +70 -0
- package/src/cli/proof.mjs +316 -0
- package/src/cli/remove.mjs +76 -0
- package/src/cli/report.mjs +73 -0
- package/src/cli/resolve.mjs +79 -0
- package/src/cli/review-file.mjs +152 -0
- package/src/cli/run.mjs +159 -0
- package/src/cli/s2.mjs +93 -0
- package/src/cli/spawn.mjs +104 -0
- package/src/cli/upgrade.mjs +103 -0
- package/src/cli/validate.mjs +108 -0
- package/src/cli/version.mjs +18 -0
- package/src/cli/worker.mjs +76 -0
- package/src/config/defaults/anthropic.mjs +18 -0
- package/src/config/defaults/index.mjs +24 -0
- package/src/config/defaults/openai.mjs +16 -0
- package/src/config/defaults/xai.mjs +18 -0
- package/src/config/known-ids.mjs +171 -0
- package/src/config/load.mjs +135 -0
- package/src/config/migrate.mjs +46 -0
- package/src/config/refresh.mjs +276 -0
- package/src/config/schema-compile.mjs +211 -0
- package/src/config/schema-paths.mjs +84 -0
- package/src/config/secret-patterns.mjs +85 -0
- package/src/config/validate.mjs +694 -0
- package/src/decide/author-loop.mjs +90 -0
- package/src/decide/escalation.mjs +262 -0
- package/src/decide/fallback-rules.mjs +172 -0
- package/src/decide/jev-client.mjs +216 -0
- package/src/decide/questions.mjs +161 -0
- package/src/decide/thresholds.mjs +199 -0
- package/src/doctor/index.mjs +105 -0
- package/src/doctor/local.mjs +162 -0
- package/src/doctor/probes.mjs +358 -0
- package/src/doctor/rows.mjs +51 -0
- package/src/doctor/worker-probe.mjs +125 -0
- package/src/engines/alias-map.mjs +78 -0
- package/src/engines/builders/claude.mjs +224 -0
- package/src/engines/builders/codex.mjs +242 -0
- package/src/engines/builders/grok.mjs +132 -0
- package/src/engines/builders/index.mjs +35 -0
- package/src/engines/builders/validate-params.mjs +34 -0
- package/src/engines/codex-home.mjs +230 -0
- package/src/engines/isolation.mjs +84 -0
- package/src/engines/probe.mjs +116 -0
- package/src/engines/provider-cli.mjs +24 -0
- package/src/engines/render-rules.mjs +41 -0
- package/src/engines/sentinel.mjs +133 -0
- package/src/engines/usage-parse.mjs +149 -0
- package/src/gates/acceptance.mjs +95 -0
- package/src/gates/detect.mjs +187 -0
- package/src/gates/invariants.mjs +43 -0
- package/src/gates/run.mjs +72 -0
- package/src/gates/safe-edit.mjs +266 -0
- package/src/gates/scope.mjs +154 -0
- package/src/gates/secret-scan.mjs +79 -0
- package/src/gates/transcript-grep.mjs +27 -0
- package/src/install/agent-env.mjs +76 -0
- package/src/install/detect.mjs +121 -0
- package/src/install/harnesses.mjs +131 -0
- package/src/install/link.mjs +463 -0
- package/src/install/wizard/answers.mjs +389 -0
- package/src/install/wizard/flags.mjs +248 -0
- package/src/install/wizard/project-file.mjs +128 -0
- package/src/install/wizard/run.mjs +366 -0
- package/src/install/wizard/steps.mjs +201 -0
- package/src/keys/backends/constants.mjs +7 -0
- package/src/keys/backends/file.mjs +99 -0
- package/src/keys/backends/keychain.mjs +81 -0
- package/src/keys/backends/security-cli.mjs +105 -0
- package/src/keys/onepassword.mjs +46 -0
- package/src/keys/store.mjs +347 -0
- package/src/ledger/calibration.mjs +109 -0
- package/src/ledger/export.mjs +52 -0
- package/src/ledger/outcome.mjs +397 -0
- package/src/ledger/paths.mjs +30 -0
- package/src/ledger/prices.mjs +49 -0
- package/src/ledger/report.mjs +250 -0
- package/src/ledger/write.mjs +279 -0
- package/src/proof/export.mjs +358 -0
- package/src/proof/lock.mjs +138 -0
- package/src/proof/red-green.mjs +519 -0
- package/src/proof/red-parse.mjs +103 -0
- package/src/proof/tiers.mjs +62 -0
- package/src/review/budget.mjs +241 -0
- package/src/review/context.mjs +156 -0
- package/src/review/engine.mjs +264 -0
- package/src/review/finding.schema.json +47 -0
- package/src/review/fixloop.mjs +467 -0
- package/src/review/gate-check.mjs +249 -0
- package/src/review/lenses/A.md +22 -0
- package/src/review/lenses/B.md +23 -0
- package/src/review/lenses/full.md +23 -0
- package/src/review/lenses/judge.md +21 -0
- package/src/review/lenses/quick.md +21 -0
- package/src/review/lenses/recheck.md +23 -0
- package/src/review/packet.mjs +355 -0
- package/src/review/triage.mjs +149 -0
- package/src/review/validate-review.mjs +70 -0
- package/src/session/author.mjs +228 -0
- package/src/session/facts.mjs +855 -0
- package/src/session/plan-check.mjs +294 -0
- package/src/session/s2.mjs +164 -0
- package/src/session/spawn.mjs +610 -0
- package/src/state/block.mjs +289 -0
- package/src/state/cli-args.mjs +67 -0
- package/src/state/paths.mjs +92 -0
- package/src/state/registry.mjs +189 -0
- package/src/state/run.mjs +266 -0
- package/src/state/signer.mjs +132 -0
- package/src/util/exec.mjs +315 -0
- package/src/util/forbidden.mjs +787 -0
- package/src/util/git.mjs +381 -0
- package/src/util/log.mjs +100 -0
- package/src/util/reaper.mjs +300 -0
- package/src/util/redact.mjs +212 -0
- package/src/util/tmp.mjs +409 -0
- package/src/worker/engine.mjs +338 -0
- package/src/worker/loop.mjs +392 -0
- package/src/worker/queue.mjs +186 -0
- package/src/worker/review-state.mjs +145 -0
- package/src/worker/ticket.mjs +195 -0
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The review worker (plan §4.8, §8.1, §8.2, §8.6; block B11).
|
|
3
|
+
*
|
|
4
|
+
* One long-lived process per run, started by the orchestrator (`forge run start`) OUTSIDE any
|
|
5
|
+
* coder sandbox. At start-up it:
|
|
6
|
+
* - reads the run record (`~/.code-forge/runs/<run>.json`) and the project config;
|
|
7
|
+
* - creates the run's temp root `<tmp.root>/<run-id>/` with ITSELF as the owner and makes it the
|
|
8
|
+
* process's run root, so every session it spawns is registered in `<run-root>/pids/` and a
|
|
9
|
+
* killed worker leaves one root that the next `run start` sweep reaps (§9.6, V12);
|
|
10
|
+
* - loads the run's HMAC key and resolves the Jev key ONCE through B2's chain (§8.1) — both stay
|
|
11
|
+
* in this process: the sessions it spawns get an env with no `CODE_FORGE_*`/`JEV_*` variable and
|
|
12
|
+
* no variable whose value is one of those secrets (§8.2);
|
|
13
|
+
* - announces itself in `<repo>/.code-forge/queue/worker.json` (what `review-file` checks).
|
|
14
|
+
* Then it polls the queue, hands each ticket to the engine hook (`./engine.mjs`, B12a), signs the
|
|
15
|
+
* outcome with the run key and writes it with its done marker and a signed ledger row.
|
|
16
|
+
*
|
|
17
|
+
* Restart: tickets live on disk and are idempotent by content hash, so a new worker simply re-reads
|
|
18
|
+
* every ticket without a done marker. A worker that is stopping abandons its in-flight ticket (no
|
|
19
|
+
* done marker) instead of recording the killed session as a failure. The new worker's pid is
|
|
20
|
+
* re-pinned in the run record by the orchestrator (`repinWorker`, = `run start --reattach`).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { spawn as nodeSpawn } from 'node:child_process';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
import { fileURLToPath } from 'node:url';
|
|
26
|
+
import { loadProjectConfig } from '../config/load.mjs';
|
|
27
|
+
import { createDefaultKeyStore, resolveKey } from '../keys/store.mjs';
|
|
28
|
+
import { appendRow, readAllRows } from '../ledger/write.mjs';
|
|
29
|
+
import { spawnSession } from '../session/spawn.mjs';
|
|
30
|
+
import { readRun, reattachWorker } from '../state/run.mjs';
|
|
31
|
+
import { loadKey, signRow } from '../state/signer.mjs';
|
|
32
|
+
import { runRoot, setRunRoot } from '../util/tmp.mjs';
|
|
33
|
+
import { reviewTicket } from './engine.mjs';
|
|
34
|
+
import { announceWorker, liveWorker, pendingTickets, readTicket, retractWorker, writeResult } from './queue.mjs';
|
|
35
|
+
import { contentHash, DELETED_HASH, repoRootOf, WorkerError } from './ticket.mjs';
|
|
36
|
+
|
|
37
|
+
/** Name of the key the worker resolves for System 1 (`keys.jev` / `system1.key` reference). */
|
|
38
|
+
export const JEV_KEY_NAME = 'jev';
|
|
39
|
+
|
|
40
|
+
export const DEFAULT_POLL_MS = 250;
|
|
41
|
+
export const DEFAULT_SESSION_TIMEOUT_S = 600;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Provider-CLI auth variables a reviewer CLI may legitimately need (the reviewer runs on the
|
|
45
|
+
* provider's own account; §8.2 bars the Jev and signer keys, not the provider login).
|
|
46
|
+
*/
|
|
47
|
+
export const PROVIDER_AUTH_VARS = Object.freeze(['ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'CODEX_API_KEY', 'XAI_API_KEY', 'GROK_API_KEY']);
|
|
48
|
+
|
|
49
|
+
/** Names that look like a secret: `*_TOKEN`, `*_KEY`, `*_SECRET`, `*_PASSWORD` (any case). */
|
|
50
|
+
export const SECRET_NAME = /(_TOKEN|_KEY|_SECRET|_PASSWORD)$/i;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The env handed to every session the worker spawns: the worker's env minus code-forge's own and
|
|
54
|
+
* Jev variables, minus every secret-shaped name except the provider-CLI auth variables, and minus
|
|
55
|
+
* any variable whose value contains one of `secrets` (§8.2).
|
|
56
|
+
* @param {NodeJS.ProcessEnv} env @param {ReadonlyArray<string | null | undefined>} secrets
|
|
57
|
+
* @returns {NodeJS.ProcessEnv}
|
|
58
|
+
*/
|
|
59
|
+
export function sessionEnv(env, secrets) {
|
|
60
|
+
const live = secrets.filter((s) => typeof s === 'string' && s.length >= 8);
|
|
61
|
+
/** @type {NodeJS.ProcessEnv} */
|
|
62
|
+
const out = {};
|
|
63
|
+
for (const [name, value] of Object.entries(env)) {
|
|
64
|
+
if (/^(CODE_FORGE_|JEV_)/i.test(name)) continue;
|
|
65
|
+
if (SECRET_NAME.test(name) && !PROVIDER_AUTH_VARS.includes(name)) continue;
|
|
66
|
+
if (typeof value === 'string' && live.some((s) => value.includes(/** @type {string} */ (s)))) continue;
|
|
67
|
+
out[name] = value;
|
|
68
|
+
}
|
|
69
|
+
return out;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Resolve the Jev key through B2's chain (env → store → 1Password). A missing key is not an error:
|
|
74
|
+
* System 1 then falls back (`system1.fallback`, B3). The value is registered for redaction by B2.
|
|
75
|
+
* @param {Record<string, any>} cfg
|
|
76
|
+
* @param {{store: import('../keys/store.mjs').KeyStore, env: NodeJS.ProcessEnv, opRead?: any}} deps
|
|
77
|
+
* @returns {Promise<string | null>}
|
|
78
|
+
*/
|
|
79
|
+
export async function resolveJevKey(cfg, { store, env, opRead }) {
|
|
80
|
+
const ref = cfg?.system1?.key ?? cfg?.keys?.[JEV_KEY_NAME];
|
|
81
|
+
const res = await resolveKey(JEV_KEY_NAME, { store, env, ref: typeof ref === 'string' ? ref : undefined, ...(opRead ? { opRead } : {}) });
|
|
82
|
+
return res.value;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** The fix-loop fields of an engine outcome the result carries (B12c, §4.11). */
|
|
86
|
+
const LOOP_FIELDS = Object.freeze(['round', 'kind', 'level', 'next', 'trigger', 'stopped', 'late']);
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* @param {Record<string, any>} outcome @returns {Record<string, any>} the fix-loop fields present.
|
|
90
|
+
*/
|
|
91
|
+
export function loopResult(outcome) {
|
|
92
|
+
/** @type {Record<string, any>} */
|
|
93
|
+
const out = {};
|
|
94
|
+
for (const name of LOOP_FIELDS) if (outcome[name] !== undefined) out[name] = outcome[name];
|
|
95
|
+
return out;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* @typedef {object} WorkerOpts
|
|
100
|
+
* @property {string} runId
|
|
101
|
+
* @property {string} repoRoot - realpath'd repository root.
|
|
102
|
+
* @property {Record<string, any>} cfg
|
|
103
|
+
* @property {string} runRootDir
|
|
104
|
+
* @property {string} slug - ledger project slug (the run record's `project`).
|
|
105
|
+
* @property {Buffer} key - the run's HMAC key.
|
|
106
|
+
* @property {number} [pollMs]
|
|
107
|
+
*/
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* @typedef {object} WorkerDeps
|
|
111
|
+
* @property {typeof reviewTicket} [review] - the engine hook (default `./engine.mjs`).
|
|
112
|
+
* @property {typeof spawnSession} [spawn]
|
|
113
|
+
* @property {import('../keys/store.mjs').KeyStore} [store] - default: B2's production store.
|
|
114
|
+
* @property {any} [opRead]
|
|
115
|
+
* @property {NodeJS.ProcessEnv} [env]
|
|
116
|
+
* @property {Partial<Record<'claude'|'codex'|'grok', string>>} [bins]
|
|
117
|
+
* @property {(row: Record<string, any>) => Promise<unknown>} [writeRow] - default: B6 `appendRow`.
|
|
118
|
+
* @property {() => Promise<Array<Record<string, any>>>} [readRows] - the ledger rows the fix-loop
|
|
119
|
+
* state is anchored in; default: B6 `readAllRows(slug)`.
|
|
120
|
+
* @property {{write: (s: string) => unknown}} [stderr]
|
|
121
|
+
* @property {import('../review/triage.mjs').JevAsk} [jev] - S1 for triage and the recheck (tests
|
|
122
|
+
* inject a mock); default: `askJev` with the resolved Jev key, none without one.
|
|
123
|
+
*/
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* @param {WorkerOpts} opts @param {WorkerDeps} [deps]
|
|
127
|
+
*/
|
|
128
|
+
export async function createWorker(opts, deps = {}) {
|
|
129
|
+
const { runId, repoRoot, cfg, runRootDir, slug, key } = opts;
|
|
130
|
+
const env = deps.env ?? process.env;
|
|
131
|
+
const review = deps.review ?? reviewTicket;
|
|
132
|
+
const spawn = deps.spawn ?? spawnSession;
|
|
133
|
+
const writeRow = deps.writeRow ?? ((/** @type {Record<string, any>} */ row) => appendRow(row, { slug }));
|
|
134
|
+
const readRows = deps.readRows ?? (() => readAllRows(slug));
|
|
135
|
+
const store = deps.store ?? (await createDefaultKeyStore(env));
|
|
136
|
+
const jevKey = await resolveJevKey(cfg, { store, env, opRead: deps.opRead });
|
|
137
|
+
const childEnv = sessionEnv(env, [jevKey, key.toString('hex')]);
|
|
138
|
+
const timeoutS = Number.isInteger(cfg?.review?.session_timeout_s) && cfg.review.session_timeout_s > 0 ? cfg.review.session_timeout_s : DEFAULT_SESSION_TIMEOUT_S;
|
|
139
|
+
const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
|
|
140
|
+
let stopping = false;
|
|
141
|
+
/** @type {(() => void) | null} */
|
|
142
|
+
let wake = null;
|
|
143
|
+
|
|
144
|
+
/** @param {Record<string, any>} row */
|
|
145
|
+
const ledger = (row) => writeRow(signRow({ run: runId, ...row }, key));
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* @param {string} id
|
|
149
|
+
* @returns {Promise<Record<string, any> | null>} the signed result, or null when abandoned.
|
|
150
|
+
*/
|
|
151
|
+
async function processTicket(id) {
|
|
152
|
+
/** @type {Record<string, any>} */
|
|
153
|
+
let result;
|
|
154
|
+
let ticket = null;
|
|
155
|
+
try {
|
|
156
|
+
ticket = readTicket(repoRoot, id);
|
|
157
|
+
} catch (err) {
|
|
158
|
+
result = { event: 'review.result', status: 'refused', reason: err instanceof WorkerError ? err.code : 'bad-ticket', approved: false };
|
|
159
|
+
}
|
|
160
|
+
if (ticket) {
|
|
161
|
+
const base = { event: 'review.result', block: ticket.block, file: ticket.file, content_hash: ticket.content_hash };
|
|
162
|
+
/** @type {string | null} */
|
|
163
|
+
let current = null;
|
|
164
|
+
try {
|
|
165
|
+
current = ticket.run === runId ? contentHash(repoRoot, ticket.file) : null;
|
|
166
|
+
} catch (err) {
|
|
167
|
+
result = { ...base, status: 'refused', reason: err instanceof WorkerError ? err.code : 'unreadable', approved: false };
|
|
168
|
+
}
|
|
169
|
+
if (result) {
|
|
170
|
+
// refused above: the path no longer passes the on-disk check, or the file cannot be read
|
|
171
|
+
} else if (ticket.run !== runId) {
|
|
172
|
+
result = { ...base, status: 'refused', reason: 'other-run', approved: false };
|
|
173
|
+
} else if (current === DELETED_HASH && ticket.content_hash !== DELETED_HASH) {
|
|
174
|
+
result = { ...base, status: 'stale', reason: 'missing-file', approved: false };
|
|
175
|
+
} else if (current !== ticket.content_hash) {
|
|
176
|
+
result = { ...base, status: 'stale', approved: false };
|
|
177
|
+
} else {
|
|
178
|
+
/** @type {import('./engine.mjs').ReviewContext} */
|
|
179
|
+
const ctx = {
|
|
180
|
+
repoRoot,
|
|
181
|
+
runId,
|
|
182
|
+
runRootDir,
|
|
183
|
+
cfg,
|
|
184
|
+
jevKey,
|
|
185
|
+
key, // signs the fix-loop state (B12c); the sessions never see it (`childEnv`)
|
|
186
|
+
readRows,
|
|
187
|
+
...(deps.jev ? { jev: deps.jev } : {}),
|
|
188
|
+
// `sessionOpts.cfg` (a per-call config, e.g. the consensus second reviewer's level) wins
|
|
189
|
+
// over the worker's; everything else the worker pins.
|
|
190
|
+
spawn: (sessionOpts) =>
|
|
191
|
+
spawn(
|
|
192
|
+
/** @type {any} */ ({ cfg, ...sessionOpts, runRoot: runRootDir, run: runId, block: ticket.block, slug, timeoutMs: timeoutS * 1000 }),
|
|
193
|
+
{ env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
|
|
194
|
+
),
|
|
195
|
+
writeRow: (row) => ledger({ ...row, block: ticket.block, file: ticket.file, content_hash: ticket.content_hash }),
|
|
196
|
+
};
|
|
197
|
+
try {
|
|
198
|
+
const outcome = await review(ticket, ctx);
|
|
199
|
+
result = {
|
|
200
|
+
...base,
|
|
201
|
+
status: outcome.status,
|
|
202
|
+
approved: outcome.approved === true,
|
|
203
|
+
engine: outcome.engine,
|
|
204
|
+
...(typeof outcome.reason === 'string' ? { reason: outcome.reason } : {}),
|
|
205
|
+
...(Array.isArray(outcome.findings) ? { findings: outcome.findings } : {}),
|
|
206
|
+
...loopResult(outcome),
|
|
207
|
+
sessions: outcome.sessions,
|
|
208
|
+
};
|
|
209
|
+
} catch {
|
|
210
|
+
result = { ...base, status: 'unavailable', reason: 'engine-error', approved: false };
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
if (stopping) return null; // the session was killed by our own stop: leave the ticket queued
|
|
215
|
+
return complete(id, result);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Sign and write the result + done marker, then the signed ledger row. A ledger failure does not
|
|
220
|
+
* undo the result: the done marker is what the queue reads.
|
|
221
|
+
* @param {string} id @param {Record<string, any>} result
|
|
222
|
+
*/
|
|
223
|
+
async function complete(id, result) {
|
|
224
|
+
const signed = writeResult({ repoRoot, runId, ticket: id, result, key });
|
|
225
|
+
try {
|
|
226
|
+
await ledger({
|
|
227
|
+
event: 'review.result',
|
|
228
|
+
ticket: id,
|
|
229
|
+
block: result.block ?? null,
|
|
230
|
+
file: result.file ?? null,
|
|
231
|
+
content_hash: result.content_hash ?? null,
|
|
232
|
+
status: result.status,
|
|
233
|
+
...(typeof result.trigger === 'string' ? { trigger: result.trigger } : {}),
|
|
234
|
+
...(typeof result.stopped === 'string' ? { stopped: result.stopped } : {}),
|
|
235
|
+
});
|
|
236
|
+
} catch {
|
|
237
|
+
// the signed result file stands; the gate reads it (§4.6)
|
|
238
|
+
}
|
|
239
|
+
return signed;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Tickets this worker could not even record a failure for: skipped until a restart. */
|
|
243
|
+
const poisoned = new Set();
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Process every pending ticket once. An unexpected throw on one ticket becomes a signed
|
|
247
|
+
* `unavailable` result (reason `worker-error`) when that can still be written, and never stops
|
|
248
|
+
* the loop; a ticket whose failure cannot be written either is skipped for this worker's life.
|
|
249
|
+
* @returns {Promise<number>} how many were completed.
|
|
250
|
+
*/
|
|
251
|
+
async function drain() {
|
|
252
|
+
let done = 0;
|
|
253
|
+
for (const id of pendingTickets(repoRoot)) {
|
|
254
|
+
if (stopping) break;
|
|
255
|
+
if (poisoned.has(id)) continue;
|
|
256
|
+
try {
|
|
257
|
+
if ((await processTicket(id)) !== null) done += 1;
|
|
258
|
+
} catch {
|
|
259
|
+
try {
|
|
260
|
+
await complete(id, { event: 'review.result', status: 'unavailable', reason: 'worker-error', approved: false });
|
|
261
|
+
done += 1;
|
|
262
|
+
} catch {
|
|
263
|
+
poisoned.add(id);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return done;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Serve the queue until `stop()`. */
|
|
271
|
+
async function run() {
|
|
272
|
+
announceWorker(repoRoot, { pid: process.pid, run: runId });
|
|
273
|
+
try {
|
|
274
|
+
while (!stopping) {
|
|
275
|
+
try {
|
|
276
|
+
await drain();
|
|
277
|
+
} catch {
|
|
278
|
+
// the queue directory itself failed to read: try again on the next poll
|
|
279
|
+
}
|
|
280
|
+
if (stopping) break;
|
|
281
|
+
await new Promise((resolve) => {
|
|
282
|
+
const timer = setTimeout(resolve, pollMs);
|
|
283
|
+
wake = () => {
|
|
284
|
+
clearTimeout(timer);
|
|
285
|
+
resolve(undefined);
|
|
286
|
+
};
|
|
287
|
+
});
|
|
288
|
+
wake = null;
|
|
289
|
+
}
|
|
290
|
+
} finally {
|
|
291
|
+
release();
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** Drain the queue once while announced, then release (`worker --once`). */
|
|
296
|
+
async function once() {
|
|
297
|
+
announceWorker(repoRoot, { pid: process.pid, run: runId });
|
|
298
|
+
try {
|
|
299
|
+
return await drain();
|
|
300
|
+
} finally {
|
|
301
|
+
stop();
|
|
302
|
+
release();
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function stop() {
|
|
307
|
+
stopping = true;
|
|
308
|
+
wake?.();
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** Withdraw this process's announcement (`worker.json`) if it is still ours. */
|
|
312
|
+
function release() {
|
|
313
|
+
retractWorker(repoRoot, process.pid);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
return { processTicket, drain, run, once, stop, release, jevKeyResolved: jevKey !== null, sessionEnv: childEnv };
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Start-up for the `worker` verb: run record, repo root, config, run temp root (owned by this
|
|
321
|
+
* process), run key; refuses when another live worker already serves this queue.
|
|
322
|
+
* @param {{runId: string, cwd?: string, pollMs?: number}} opts @param {WorkerDeps} [deps]
|
|
323
|
+
*/
|
|
324
|
+
export async function bootWorker(opts, deps = {}) {
|
|
325
|
+
const record = await readRun(opts.runId);
|
|
326
|
+
if (record.status !== 'active') throw new WorkerError('run-ended', `run ${opts.runId} has ended`);
|
|
327
|
+
const repoRoot = await repoRootOf(path.resolve(opts.cwd ?? record.workspace));
|
|
328
|
+
const loaded = await loadProjectConfig(repoRoot);
|
|
329
|
+
if (!loaded.ok) throw new WorkerError('config', loaded.message ?? 'the project config cannot be loaded');
|
|
330
|
+
const other = liveWorker(repoRoot);
|
|
331
|
+
if (other && other.pid !== process.pid) throw new WorkerError('worker-running', `worker pid ${other.pid} already serves this queue`);
|
|
332
|
+
const cfg = loaded.config;
|
|
333
|
+
const tmpRoot = typeof cfg?.tmp?.root === 'string' ? cfg.tmp.root : undefined;
|
|
334
|
+
const runRootDir = runRoot(opts.runId, { root: tmpRoot });
|
|
335
|
+
setRunRoot(runRootDir);
|
|
336
|
+
const key = await loadKey(opts.runId);
|
|
337
|
+
const worker = await createWorker({ runId: opts.runId, repoRoot, cfg, runRootDir, slug: record.project, key, pollMs: opts.pollMs }, deps);
|
|
338
|
+
return { ...worker, repoRoot, runRootDir };
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* `run start --reattach` for a restarted worker: re-pin the run record to the pid the queue
|
|
343
|
+
* announces, when that worker is alive and serves this run (the orchestrator runs this, never a
|
|
344
|
+
* coder — the record lives outside the workspace).
|
|
345
|
+
* `expectPid` (a `run start` that just launched a worker) pins ONLY that pid: an older or stale
|
|
346
|
+
* announcement naming another live pid is refused (`worker_mismatch`) instead of being pinned
|
|
347
|
+
* over the new worker.
|
|
348
|
+
* @param {{runId: string, repoRoot: string, writeRow?: (row: Record<string, any>) => Promise<unknown>, probe?: import('../state/run.mjs').StartTimeProbe, expectPid?: number}} opts
|
|
349
|
+
*/
|
|
350
|
+
export async function repinWorker({ runId, repoRoot, writeRow, probe, expectPid }) {
|
|
351
|
+
const live = liveWorker(repoRoot);
|
|
352
|
+
if (!live) throw new WorkerError('worker_down', 'no live worker serves this queue');
|
|
353
|
+
if (live.run !== runId) throw new WorkerError('other-run', `the live worker serves run ${live.run}`);
|
|
354
|
+
if (expectPid !== undefined && live.pid !== expectPid) {
|
|
355
|
+
throw new WorkerError('worker_mismatch', `the queue announces worker pid ${live.pid}, not the launched pid ${expectPid}`);
|
|
356
|
+
}
|
|
357
|
+
const { project } = await readRun(runId);
|
|
358
|
+
const write = writeRow ?? ((/** @type {Record<string, any>} */ row) => appendRow(row, { slug: project }));
|
|
359
|
+
return reattachWorker({ runId, workerPid: live.pid, writeRow: write, ...(probe ? { probe } : {}) });
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/** The CLI entry the worker is launched from. */
|
|
363
|
+
const BIN = fileURLToPath(new URL('../../bin/code-forge.mjs', import.meta.url));
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* `run start`: launch `code-forge worker --run <id>` DETACHED in the workspace (the orchestrator's
|
|
367
|
+
* context, outside any coder sandbox; argv only, no shell) and wait until it announces itself in
|
|
368
|
+
* the queue. A worker that does not announce within `timeoutMs` is killed and the start refused.
|
|
369
|
+
* @param {{runId: string, workspace: string, env?: NodeJS.ProcessEnv, timeoutMs?: number}} opts
|
|
370
|
+
* @returns {Promise<{pid: number, repoRoot: string}>}
|
|
371
|
+
*/
|
|
372
|
+
export async function launchWorker({ runId, workspace, env = process.env, timeoutMs = 15000 }) {
|
|
373
|
+
const repoRoot = await repoRootOf(workspace);
|
|
374
|
+
const child = nodeSpawn(process.execPath, [BIN, 'worker', '--run', runId], { cwd: repoRoot, env, detached: true, stdio: 'ignore', shell: false });
|
|
375
|
+
child.on('error', () => {});
|
|
376
|
+
const pid = child.pid;
|
|
377
|
+
child.unref();
|
|
378
|
+
const deadline = Date.now() + timeoutMs;
|
|
379
|
+
while (Date.now() < deadline) {
|
|
380
|
+
if (typeof pid === 'number' && liveWorker(repoRoot)?.pid === pid) return { pid, repoRoot };
|
|
381
|
+
if (child.exitCode !== null || child.signalCode !== null) break;
|
|
382
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
383
|
+
}
|
|
384
|
+
if (typeof pid === 'number') {
|
|
385
|
+
try {
|
|
386
|
+
process.kill(-pid, 'SIGKILL');
|
|
387
|
+
} catch {
|
|
388
|
+
// already gone
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
throw new WorkerError('worker_down', 'the worker did not start (run `code-forge worker --run <id>` by hand to see why)');
|
|
392
|
+
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The review queue on disk (plan §1.2, §4.8; block B11). Everything is a file inside the
|
|
3
|
+
* workspace, so a Codex `workspace-write` coder with no network can enqueue and poll:
|
|
4
|
+
*
|
|
5
|
+
* <repo>/.code-forge/queue/<ticket>.json the ticket (written by `review-file`, idempotent)
|
|
6
|
+
* <repo>/.code-forge/queue/<ticket>.done the worker's done marker `{ticket, status, result}`
|
|
7
|
+
* <repo>/.code-forge/queue/worker.json the worker's announcement `{pid, start_time, run, started_at}`
|
|
8
|
+
* <repo>/.code-forge/reviews/<run>/<ticket>.json the SIGNED result (run key HMAC, §8.6)
|
|
9
|
+
*
|
|
10
|
+
* The forbidden list (§8.4 ★) denies a coder any write under `reviews/` and to `queue/*.done`;
|
|
11
|
+
* the MAC makes an edit visible anyway (T1/T2). Writes are atomic (temp file + rename).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { randomBytes } from 'node:crypto';
|
|
15
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
import { loadKey, signRow, verifyRow } from '../state/signer.mjs';
|
|
18
|
+
import { isAlive, readStartTime, UNKNOWN_START_TIME } from '../util/reaper.mjs';
|
|
19
|
+
import { assertRowPath, assertTicketId, contentHash, ticketId, WorkerError } from './ticket.mjs';
|
|
20
|
+
|
|
21
|
+
export const WORKER_FILE = 'worker.json';
|
|
22
|
+
|
|
23
|
+
/** @param {string} repoRoot */
|
|
24
|
+
export const queueDir = (repoRoot) => path.join(repoRoot, '.code-forge', 'queue');
|
|
25
|
+
/** @param {string} repoRoot @param {string} runId */
|
|
26
|
+
export const reviewsDir = (repoRoot, runId) => path.join(repoRoot, '.code-forge', 'reviews', runId);
|
|
27
|
+
|
|
28
|
+
/** @param {string} file @param {string} text */
|
|
29
|
+
function writeAtomic(file, text) {
|
|
30
|
+
mkdirSync(path.dirname(file), { recursive: true });
|
|
31
|
+
const tmp = `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
|
|
32
|
+
try {
|
|
33
|
+
writeFileSync(tmp, text, { mode: 0o644 });
|
|
34
|
+
renameSync(tmp, file);
|
|
35
|
+
} catch (err) {
|
|
36
|
+
rmSync(tmp, { force: true });
|
|
37
|
+
throw err;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** @param {string} file @returns {any} parsed JSON, or null when missing or unparseable. */
|
|
42
|
+
function readJSON(file) {
|
|
43
|
+
try {
|
|
44
|
+
return JSON.parse(readFileSync(file, 'utf8'));
|
|
45
|
+
} catch {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* @typedef {{ticket: string, run: string, block: string, file: string, content_hash: string, enqueued_at: string}} Ticket
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Enqueue `file` (already repo-root-relative) for review. Idempotent by content hash: the same
|
|
56
|
+
* content returns the same ticket and never rewrites it.
|
|
57
|
+
* @param {{repoRoot: string, run: string, block: string, file: string, now?: Date}} opts
|
|
58
|
+
* @returns {{ticket: string, status: 'queued' | 'done', file: string, content_hash: string}}
|
|
59
|
+
*/
|
|
60
|
+
export function enqueue({ repoRoot, run, block, file, now = new Date() }) {
|
|
61
|
+
const rel = assertRowPath(file);
|
|
62
|
+
const content_hash = contentHash(repoRoot, rel);
|
|
63
|
+
const ticket = ticketId({ run, block, file: rel, content_hash });
|
|
64
|
+
const target = path.join(queueDir(repoRoot), `${ticket}.json`);
|
|
65
|
+
if (!existsSync(target)) {
|
|
66
|
+
/** @type {Ticket} */
|
|
67
|
+
const body = { ticket, run, block, file: rel, content_hash, enqueued_at: now.toISOString() };
|
|
68
|
+
writeAtomic(target, `${JSON.stringify(body)}\n`);
|
|
69
|
+
}
|
|
70
|
+
return { ticket, status: isDone(repoRoot, ticket) ? 'done' : 'queued', file: rel, content_hash };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** @param {string} repoRoot @param {string} ticket */
|
|
74
|
+
export function isDone(repoRoot, ticket) {
|
|
75
|
+
return existsSync(path.join(queueDir(repoRoot), `${assertTicketId(ticket)}.done`));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Tickets with no done marker, oldest first (enqueue time, then id). Unparseable files are
|
|
80
|
+
* skipped here and refused by the worker when it reads them.
|
|
81
|
+
* @param {string} repoRoot
|
|
82
|
+
* @returns {string[]} ticket ids
|
|
83
|
+
*/
|
|
84
|
+
export function pendingTickets(repoRoot) {
|
|
85
|
+
let names;
|
|
86
|
+
try {
|
|
87
|
+
names = readdirSync(queueDir(repoRoot));
|
|
88
|
+
} catch {
|
|
89
|
+
return [];
|
|
90
|
+
}
|
|
91
|
+
const ids = names.filter((n) => /^[0-9a-f]{24}\.json$/.test(n)).map((n) => n.slice(0, 24));
|
|
92
|
+
const pending = ids.filter((id) => !names.includes(`${id}.done`));
|
|
93
|
+
const at = (/** @type {string} */ id) => {
|
|
94
|
+
const t = readJSON(path.join(queueDir(repoRoot), `${id}.json`));
|
|
95
|
+
return typeof t?.enqueued_at === 'string' ? t.enqueued_at : '';
|
|
96
|
+
};
|
|
97
|
+
return pending.map((id) => ({ id, at: at(id) })).sort((a, b) => (a.at === b.at ? (a.id < b.id ? -1 : 1) : a.at < b.at ? -1 : 1)).map((e) => e.id);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Read and re-validate a ticket (coder-writable): its id must match its content.
|
|
102
|
+
* @param {string} repoRoot @param {string} id
|
|
103
|
+
* @returns {Ticket}
|
|
104
|
+
* @throws {WorkerError} `bad-ticket` / `bad-path`
|
|
105
|
+
*/
|
|
106
|
+
export function readTicket(repoRoot, id) {
|
|
107
|
+
const t = readJSON(path.join(queueDir(repoRoot), `${assertTicketId(id)}.json`));
|
|
108
|
+
if (!t || typeof t !== 'object') throw new WorkerError('bad-ticket', `ticket ${id} is not readable JSON`);
|
|
109
|
+
assertRowPath(t.file);
|
|
110
|
+
if (typeof t.run !== 'string' || typeof t.block !== 'string' || typeof t.content_hash !== 'string' || ticketId(t) !== id) {
|
|
111
|
+
throw new WorkerError('bad-ticket', `ticket ${id} does not match its id`);
|
|
112
|
+
}
|
|
113
|
+
return /** @type {Ticket} */ (t);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Sign `result` with the run key and write it to `reviews/<run>/<ticket>.json`, then the done
|
|
118
|
+
* marker. The result is written first, so a done marker always has its result.
|
|
119
|
+
* @param {{repoRoot: string, runId: string, ticket: string, result: Record<string, any>, key: Buffer}} opts
|
|
120
|
+
* @returns {Record<string, any>} the signed result
|
|
121
|
+
*/
|
|
122
|
+
export function writeResult({ repoRoot, runId, ticket, result, key }) {
|
|
123
|
+
const signed = signRow({ ...result, run: runId, ticket: assertTicketId(ticket) }, key);
|
|
124
|
+
const file = path.join(reviewsDir(repoRoot, runId), `${ticket}.json`);
|
|
125
|
+
writeAtomic(file, `${JSON.stringify(signed)}\n`);
|
|
126
|
+
writeAtomic(path.join(queueDir(repoRoot), `${ticket}.done`), `${JSON.stringify({ ticket, status: signed.status, result: path.relative(repoRoot, file) })}\n`);
|
|
127
|
+
return signed;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Read a result without verifying it (the coder's `--wait` cannot read the run key).
|
|
132
|
+
* @param {string} repoRoot @param {string} runId @param {string} ticket
|
|
133
|
+
* @returns {Record<string, any> | null}
|
|
134
|
+
*/
|
|
135
|
+
export function readResult(repoRoot, runId, ticket) {
|
|
136
|
+
return readJSON(path.join(reviewsDir(repoRoot, runId), `${assertTicketId(ticket)}.json`));
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Verify a result's MAC with the run key (the orchestrator and the block gate do this; §8.6).
|
|
141
|
+
* @param {string} repoRoot @param {string} runId @param {string} ticket
|
|
142
|
+
* @returns {Promise<{ok: boolean, reason?: string, result?: Record<string, any>}>}
|
|
143
|
+
*/
|
|
144
|
+
export async function verifyResult(repoRoot, runId, ticket) {
|
|
145
|
+
const file = path.join(reviewsDir(repoRoot, runId), `${assertTicketId(ticket)}.json`);
|
|
146
|
+
if (!existsSync(file)) return { ok: false, reason: 'missing' };
|
|
147
|
+
const result = readJSON(file);
|
|
148
|
+
if (!result || typeof result !== 'object' || Array.isArray(result)) return { ok: false, reason: 'unparseable' };
|
|
149
|
+
if (result.ticket !== ticket || result.run !== runId) return { ok: false, reason: 'mismatch' };
|
|
150
|
+
const check = verifyRow(result, await loadKey(runId));
|
|
151
|
+
return check.ok ? { ok: true, result } : { ok: false, reason: check.reason };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Announce the serving worker in the queue directory (what `review-file` checks for liveness).
|
|
156
|
+
* @param {string} repoRoot @param {{pid: number, run: string}} who
|
|
157
|
+
*/
|
|
158
|
+
export function announceWorker(repoRoot, { pid, run }) {
|
|
159
|
+
const body = { pid, start_time: readStartTime(pid) ?? UNKNOWN_START_TIME, run, started_at: new Date().toISOString() };
|
|
160
|
+
writeAtomic(path.join(queueDir(repoRoot), WORKER_FILE), `${JSON.stringify(body)}\n`);
|
|
161
|
+
return body;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** @param {string} repoRoot @returns {{pid: number, start_time: string, run: string} | null} */
|
|
165
|
+
export function readWorker(repoRoot) {
|
|
166
|
+
const w = readJSON(path.join(queueDir(repoRoot), WORKER_FILE));
|
|
167
|
+
return w && Number.isInteger(w.pid) && w.pid > 1 && typeof w.run === 'string' ? w : null;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Remove the announcement if it is still `pid`'s. @param {string} repoRoot @param {number} pid */
|
|
171
|
+
export function retractWorker(repoRoot, pid) {
|
|
172
|
+
if (readWorker(repoRoot)?.pid === pid) rmSync(path.join(queueDir(repoRoot), WORKER_FILE), { force: true });
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* The live worker serving this queue, or null (`worker_down`): the announced pid must be alive
|
|
177
|
+
* AND still have the announced start time (a recycled pid is not our worker).
|
|
178
|
+
* @param {string} repoRoot
|
|
179
|
+
*/
|
|
180
|
+
export function liveWorker(repoRoot) {
|
|
181
|
+
const w = readWorker(repoRoot);
|
|
182
|
+
if (!w || !isAlive(w.pid)) return null;
|
|
183
|
+
if (w.start_time !== UNKNOWN_START_TIME && readStartTime(w.pid) !== w.start_time) return null;
|
|
184
|
+
return w;
|
|
185
|
+
}
|
|
186
|
+
|