amsd-pipeline-shared 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -1
- package/bin/amsd-pipeline-shared.js +3 -0
- package/docs/gateway-files.txt +12 -0
- package/orchestrations/config/doctor-heavy-jobs.json +5 -0
- package/orchestrations/config/doctor-signatures.json +11 -0
- package/orchestrations/config/model-pricing.json +12 -4
- package/orchestrations/prompts/doctor-repair.md +34 -0
- package/orchestrations/scripts/lib/doctor/agent.js +91 -0
- package/orchestrations/scripts/lib/doctor/classify.js +56 -0
- package/orchestrations/scripts/lib/doctor/cli.js +56 -0
- package/orchestrations/scripts/lib/doctor/crash-record.js +62 -0
- package/orchestrations/scripts/lib/doctor/handle.js +121 -0
- package/orchestrations/scripts/lib/doctor/profile.js +130 -0
- package/orchestrations/scripts/lib/doctor/repair.js +166 -0
- package/orchestrations/scripts/lib/doctor/restarter.js +58 -0
- package/orchestrations/scripts/lib/doctor/state.js +65 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -12,8 +12,11 @@ workflows (`amsd-pipeline-workflow`). A fix lands here, once, and both run the s
|
|
|
12
12
|
| The Spec runner | `orchestrations/scripts/spec-mode-runner.js` | A failed mint re-proposal keeps the accepted roles and says so |
|
|
13
13
|
| The KB synthesiser | `orchestrations/scripts/lib/kb-synthesizer.js` | Runs on its own seam's model |
|
|
14
14
|
| Agent-to-agent messages | `orchestrations/scripts/lib/agent-messages.js` | send / receive / list / route on the pipeline's own bus format (`logs/messages/`, `agent-messages.jsonl`); "router" goes to the stage's router |
|
|
15
|
+
| The shared pre-launch | `orchestrations/scripts/lib/prelaunch.js`, `lib/preflight.sh`, `preflight-check.sh`, `preflight-static.sh`, `preflight-prd-integrity.sh` | Every launch of every pipeline, before any model call: the pipeline's own pre-flight unchanged, then Langfuse (a trace read back under the run's session), seed (the seeded prompts and agent files are the launched ticket's), cache (both layers; at most `--max-rebuild` prompts to rebuild, named), cost (a priced ledger call; every ladder model priced), alone (nothing else on the install but its registered services), memory, no-timeout, and the 12-item checklist. `require_preflight` runs it for every bash launcher; `amsd-pipeline-shared prelaunch` for the workflows |
|
|
16
|
+
| Codeline discovery's instructions | `orchestrations/prompts/templates/codeline-discovery.json` | A repository's `reason` describes the repository and its relation to the others, never the work it needs — the classifier reads it as the codeline's description (backlog B59) |
|
|
17
|
+
| The price table | `orchestrations/config/model-pricing.json` | Claude Sonnet 5 and Opus 5.5 priced from Anthropic's pricing page, so a call whose session never reports a cost can still be priced |
|
|
15
18
|
|
|
16
|
-
Every file sits at the path it has in a pipeline install.
|
|
19
|
+
Every file sits at the path it has in a pipeline install. Each file is first committed exactly as released
|
|
17
20
|
in amsd-pipeline 2.0.80 (`docs/gateway-baseline-2.0.80.md5`); every change since is its own commit with its tests.
|
|
18
21
|
`docs/CONSUMER-REVIEW-B56.md` reads every caller of the gateway before and after the fix.
|
|
19
22
|
|
|
@@ -34,6 +37,16 @@ with the list. The replaced copies are kept in `<install>/.amsd-pipeline-shared/
|
|
|
34
37
|
|
|
35
38
|
Needs: Node 20+, bash, jq.
|
|
36
39
|
|
|
40
|
+
## The pre-launch from the shell
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
amsd-pipeline-shared prelaunch --install <install> --project <p> --run-id <id> --out <dir> \
|
|
44
|
+
[--ticket <KEY> --runs <a recorded run's runs dir>] [--codeline <cl>] [--set claude] [--max-rebuild <n>] [--prd <file>] [--preflight no]
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Prints every check and the 12-item checklist, writes `<out>/prelaunch.json`; exit 0 all green, 1 any red. Nothing it does
|
|
48
|
+
calls a model.
|
|
49
|
+
|
|
37
50
|
## Messages from the shell
|
|
38
51
|
|
|
39
52
|
For the bash pipeline (backlog B55), one JSON line out:
|
|
@@ -56,6 +69,13 @@ The gateway, the reviewer, the mint re-proposal and the synthesiser each have a
|
|
|
56
69
|
mutation-checked (md5 before and after). The install is tested on a copy of a 2.0.80 install: exactly the package's
|
|
57
70
|
files change, an edited file is refused, an upgrade replaces only its own copies, revert gives back the release.
|
|
58
71
|
|
|
72
|
+
## The pipeline doctor
|
|
73
|
+
|
|
74
|
+
`amsd-pipeline-shared doctor validate|handle` and `orchestrations/scripts/lib/doctor/` — when a run stops on a defect in a pipeline's OWN code, the doctor reproduces it at £0, repairs it
|
|
75
|
+
test-first (RED on the base, GREEN with the fix, proven by the doctor itself), reinstalls the component through its own installer, and restarts the run within a restart quota the operator gives.
|
|
76
|
+
Off unless the pipeline's setting is `on`; Sonnet-class model only; never touches client code or the ticket; its own logs and cost ledger. Generic: every fact about a pipeline is its **profile**.
|
|
77
|
+
See `docs/DOCTOR.md`.
|
|
78
|
+
|
|
59
79
|
## Version
|
|
60
80
|
|
|
61
81
|
`amsd-pipeline-shared version`. Released with a git tag `v<version>` and the npm publish of the same version.
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
// amsd-pipeline-shared verify --into <install> every file is this package's (exit 1 and the list when not)
|
|
7
7
|
// amsd-pipeline-shared revert --into <install> put back the files this package replaced
|
|
8
8
|
// amsd-pipeline-shared messages send|receive|list --dir <bus> … the agent-to-agent bus (lib/agent-messages.js)
|
|
9
|
+
// amsd-pipeline-shared doctor validate|handle … the pipeline doctor (lib/doctor/cli.js)
|
|
9
10
|
// amsd-pipeline-shared prelaunch --install <i> --project <p> --run-id <id> --out <dir> … the shared pre-launch
|
|
10
11
|
// amsd-pipeline-shared version
|
|
11
12
|
// A file is replaced only when the install holds the released copy this package was built from (docs/gateway-baseline-
|
|
@@ -26,6 +27,8 @@ const [cmd, ...rest] = process.argv.slice(2);
|
|
|
26
27
|
if (cmd === 'version') { process.stdout.write(`${PKG.version}\n`); process.exit(0); }
|
|
27
28
|
// THE SHARED PRE-LAUNCH, for every pipeline (lib/prelaunch.js): amsd-pipeline-shared prelaunch --install <i> --project <p> …
|
|
28
29
|
if (cmd === 'prelaunch') process.exit(require(path.join(ROOT, 'orchestrations', 'scripts', 'lib', 'prelaunch.js')).prelaunchCli(rest));
|
|
30
|
+
// THE PIPELINE DOCTOR (lib/doctor/): amsd-pipeline-shared doctor validate|handle … — repairs a pipeline component's own code when a run fails on a defect in it
|
|
31
|
+
if (cmd === 'doctor') { require(path.join(ROOT, 'orchestrations', 'scripts', 'lib', 'doctor', 'cli.js')).doctorCli(rest).then((c) => process.exit(c)); return; }
|
|
29
32
|
// THE MESSAGE BUS FOR SHELL CALLERS (the bash pipeline's adoption, backlog B55): one JSON line out
|
|
30
33
|
if (cmd === 'messages') {
|
|
31
34
|
const m = require(path.join(ROOT, 'orchestrations', 'scripts', 'lib', 'agent-messages.js'));
|
package/docs/gateway-files.txt
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
config/doctor-heavy-jobs.json
|
|
2
|
+
config/doctor-signatures.json
|
|
1
3
|
config/model-pricing.json
|
|
4
|
+
prompts/doctor-repair.md
|
|
2
5
|
prompts/templates/codeline-discovery.json
|
|
3
6
|
scripts/agent-attempt-analyst.sh
|
|
4
7
|
scripts/lib/agent-contract.js
|
|
@@ -10,6 +13,15 @@ scripts/lib/constraint-sanity.js
|
|
|
10
13
|
scripts/lib/content-retry.js
|
|
11
14
|
scripts/lib/cost-emitter.js
|
|
12
15
|
scripts/lib/cost-record.sh
|
|
16
|
+
scripts/lib/doctor/agent.js
|
|
17
|
+
scripts/lib/doctor/classify.js
|
|
18
|
+
scripts/lib/doctor/cli.js
|
|
19
|
+
scripts/lib/doctor/crash-record.js
|
|
20
|
+
scripts/lib/doctor/handle.js
|
|
21
|
+
scripts/lib/doctor/profile.js
|
|
22
|
+
scripts/lib/doctor/repair.js
|
|
23
|
+
scripts/lib/doctor/restarter.js
|
|
24
|
+
scripts/lib/doctor/state.js
|
|
13
25
|
scripts/lib/engine-prompt.js
|
|
14
26
|
scripts/lib/env-file.sh
|
|
15
27
|
scripts/lib/evidence-windows.js
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "The jobs another launch must not share this host with (the operator's standing rule: check memory AND other heavy jobs before a launch) — host-wide, not any pipeline's. 'tokens' match a process's command-line entry by its exact file name; 'withFlag' matches a program started with a flag (a `claude -p` session, not an interactive `claude`).",
|
|
3
|
+
"tokens": ["vitest", "jest", "tsc", "flow.py"],
|
|
4
|
+
"withFlag": [{ "bin": "claude", "flag": "-p", "as": "claude -p" }]
|
|
5
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "What a failure LOOKS LIKE, per runtime the doctor's components are written in — data, not code, so a new runtime is a new entry and no pipeline's name or path is here. 'runtimeNames' and 'runtimeMessage' are the marks of a defect in the code itself; 'external' is what names the machine, the network, the model service or a kill instead; 'internalFrame' and 'dependencyDir' tell a stack frame that is the runtime's or a dependency's from the component's own. A pipeline's own deliberate refusals are NOT here: they are in its profile (classify.refusals).",
|
|
3
|
+
"node": {
|
|
4
|
+
"runtimeNames": ["TypeError", "ReferenceError", "SyntaxError", "RangeError"],
|
|
5
|
+
"runtimeMessage": "Unexpected token|is not valid JSON|Unexpected end of JSON|Cannot read propert|Cannot set propert|is not a function|is not defined|is not iterable|Cannot destructure|is not a constructor|Invalid array length|Maximum call stack",
|
|
6
|
+
"external": "\\bE(NOSPC|ACCES|PERM|MFILE|NOMEM|CONNRESET|CONNREFUSED|TIMEDOUT|NOTFOUND|AI_AGAIN|PIPE)\\b|bounded: cap|exited (137|143)\\b|\\b(401|403|429|500|502|503|529)\\b|overloaded|rate.?limit|credit|quota|<!DOCTYPE|<html|out of memory|killed",
|
|
7
|
+
"generalRefusal": "refused|usage:",
|
|
8
|
+
"internalFrame": "^node:",
|
|
9
|
+
"dependencyDir": "node_modules"
|
|
10
|
+
}
|
|
11
|
+
}
|
|
@@ -35,12 +35,20 @@
|
|
|
35
35
|
"outputPerMillion": 15
|
|
36
36
|
},
|
|
37
37
|
"claude-haiku-4-5": {
|
|
38
|
-
"inputPerMillion":
|
|
39
|
-
"outputPerMillion":
|
|
38
|
+
"inputPerMillion": 1,
|
|
39
|
+
"outputPerMillion": 5,
|
|
40
|
+
"cacheReadPerMillion": 0.1,
|
|
41
|
+
"cacheWrite5mPerMillion": 1.25,
|
|
42
|
+
"cacheWrite1hPerMillion": 2,
|
|
43
|
+
"$why": "Checked against the vendor: https://platform.claude.com/docs/en/about-claude/pricing (read 2026-10-19) — $1 input / $5 output per MTok; cache $1.25 5-minute write, $2 1-hour write, $0.10 hit. Was $0.80/$4 (Haiku 3.5's price)."
|
|
40
44
|
},
|
|
41
45
|
"claude-haiku-4-5-20251001": {
|
|
42
|
-
"inputPerMillion":
|
|
43
|
-
"outputPerMillion":
|
|
46
|
+
"inputPerMillion": 1,
|
|
47
|
+
"outputPerMillion": 5,
|
|
48
|
+
"cacheReadPerMillion": 0.1,
|
|
49
|
+
"cacheWrite5mPerMillion": 1.25,
|
|
50
|
+
"cacheWrite1hPerMillion": 2,
|
|
51
|
+
"$why": "Checked against the vendor: https://platform.claude.com/docs/en/about-claude/pricing (read 2026-10-19) — $1 input / $5 output per MTok; cache $1.25 5-minute write, $2 1-hour write, $0.10 hit. Was $0.80/$4 (Haiku 3.5's price)."
|
|
44
52
|
},
|
|
45
53
|
"claude-3-5-haiku-20241022": {
|
|
46
54
|
"inputPerMillion": 0.8,
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
You are the pipeline doctor. A run of {{component}} ({{description}}) stopped on a defect in that component's OWN code. You repair that code, and only that code.
|
|
2
|
+
|
|
3
|
+
THE FAILURE
|
|
4
|
+
- Run: {{run}} Attempt: {{attempt}} Step the pipeline was in: {{step}}
|
|
5
|
+
- What was thrown ({{errorName}}): {{errorMessage}}
|
|
6
|
+
- Thrown at: {{throwSite}}
|
|
7
|
+
- The crash record, the failing leg's output and the run's log tail are copies in: {{evidenceDir}} (read them)
|
|
8
|
+
- The run's preserved state, READ ONLY, never written: {{runDir}} The install it ran from: {{install}}
|
|
9
|
+
|
|
10
|
+
WHERE YOU WORK
|
|
11
|
+
- Your working directory is a git work tree of {{component}} on branch {{branch}}. It is the only place you write.
|
|
12
|
+
- You may change files under these directories only: {{repairDirs}} (the code) and {{testDirs}} (the tests). Nothing else in the repository, and nothing outside it.
|
|
13
|
+
- Never touch: {{boundaries}}.
|
|
14
|
+
|
|
15
|
+
AGENTIC FIRST — decide this before anything else
|
|
16
|
+
- Ask: did an AGENT cause this abort — its answer, its instructions, the schema it must answer in, the hand-off between two agents — or is it a clear and outright coding bug (a throw, a wrong variable, an unhandled null, a bad path) in code that involves no agent's behaviour?
|
|
17
|
+
- The agentic layer of {{component}} is: {{agenticDescription}}. It lives in: {{agenticDirs}}.
|
|
18
|
+
- If an agent caused it, repair the AGENTIC layer: sharpen the instruction, the answer schema and its field descriptions, the hand-off contract between the agents — so the agent is asked in a way that gets the right answer. Do NOT add deterministic code that tolerates, repairs or re-parses a wrong agent answer, and do not add a guard that hides an agent's behaviour.
|
|
19
|
+
- Only a clear and outright coding bug gets a deterministic fix.
|
|
20
|
+
- This abort was classified as: {{abortKind}} ({{layerHint}}). An agentic abort MUST be repaired agentically. For a crash, say which layer you found and why.
|
|
21
|
+
- An agentic repair is proven by REPLAY: replay the real reply or hand-off that failed — copy it into a fixture — against the schema, the contract or the rendered instruction in the RED test (it fails on the base, passes after), and assert on the rendered instruction or schema text that now prevents the abort.
|
|
22
|
+
|
|
23
|
+
HOW YOU WORK — in this order
|
|
24
|
+
1. Reason from the data flow to the exact input that makes the code fail. Name the file, the line and the wrong assumption BEFORE you run anything.
|
|
25
|
+
2. Reproduce it at £0 from the preserved state: copy what you need into a scratch folder; never write into the run. No model calls, no network.
|
|
26
|
+
3. Write the smallest RED test in the test directories that fails for exactly that reason: it executes the unit and asserts on the artefact it produced, and it has a negative case. Run only that file with: {{testCommand}}. Never a test file matching {{forbiddenTests}}, never a whole suite, never a whole test folder.
|
|
27
|
+
4. Fix the code until that test is GREEN. Then a mutation check: undo the fix, see the test fail, restore the file, compare md5 before and after.
|
|
28
|
+
5. Look for the same mistake elsewhere in the component (grep the pattern) and fix every instance the same way, each with a test.
|
|
29
|
+
6. `git add` only the files you changed and `git commit` on this branch. Do not push. Do not publish. Do not install or fetch anything; no network command.
|
|
30
|
+
|
|
31
|
+
YOU ARE WRONG UNTIL A TEST SAYS OTHERWISE. If you cannot reproduce the failure, or the cause is outside the component, say so and change nothing.
|
|
32
|
+
|
|
33
|
+
END with exactly one JSON object and nothing after it:
|
|
34
|
+
{"status":"repaired"|"cannot","layer":"code"|"agentic","layerWhy":"<which layer and why, in one sentence>","summary":"<what was wrong, in one sentence>","cause":"<file:line and the wrong assumption>","files":["<changed files>"],"tests":["<test files>"]}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// THE DOCTOR'S AGENT: one model family and nothing else (operator, 2026-10-20: "the agent only uses sonnet to work"). The family is the
|
|
3
|
+
// profile's; the model id is read from the pipeline's OWN model configuration (profile.model.config) — never written here — and there is no
|
|
4
|
+
// fallback to any other model: no model of that family there, no agent. The prompt is a template file (prompts/doctor-repair.md) filled from the
|
|
5
|
+
// profile and the failure, every field or refused. buildCommand never runs anything; runAgent is the only place a model is called.
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const { spawnSync } = require('child_process');
|
|
9
|
+
const { hostWrap, expand } = require('./repair.js');
|
|
10
|
+
|
|
11
|
+
/** Every string value in a JSON document, in document order. */
|
|
12
|
+
function strings(v, out = []) {
|
|
13
|
+
if (typeof v === 'string') out.push(v);
|
|
14
|
+
else if (Array.isArray(v)) v.forEach((x) => strings(x, out));
|
|
15
|
+
else if (v && typeof v === 'object') Object.values(v).forEach((x) => strings(x, out));
|
|
16
|
+
return out;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** The first model id of the profile's family in the pipeline's model configuration (an id is a token with no spaces that names the family). */
|
|
20
|
+
function modelFor(install, profile) {
|
|
21
|
+
const f = profile.model.config.replace('{install}', install);
|
|
22
|
+
let doc;
|
|
23
|
+
try { doc = JSON.parse(fs.readFileSync(f, 'utf8')); } catch (e) { throw new Error(`cannot read the model configuration ${f}: ${e.message}`); }
|
|
24
|
+
const fam = profile.model.family.toLowerCase();
|
|
25
|
+
const m = strings(doc).find((s) => /^[A-Za-z0-9._:/-]+$/.test(s) && s.toLowerCase().includes(fam));
|
|
26
|
+
if (!m) throw new Error(`no "${profile.model.family}" model in ${f}: the doctor works on that family only and does not fall back`);
|
|
27
|
+
return m;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The repair prompt with every {{field}} filled from the failure and the profile; a field left unfilled is an error, naming it. */
|
|
31
|
+
function renderPrompt(vars, profile) {
|
|
32
|
+
const t = fs.readFileSync(path.join(__dirname, '..', '..', '..', 'prompts', 'doctor-repair.md'), 'utf8');
|
|
33
|
+
const all = {
|
|
34
|
+
...vars, component: profile.component.name, description: profile.component.description,
|
|
35
|
+
repairDirs: profile.scope.repairDirs.join(', '), testDirs: profile.scope.testDirs.join(', '), boundaries: profile.boundaries.join('; '),
|
|
36
|
+
testCommand: expand(profile.test.command, { node: 'node', files: ['<test file>'] }).join(' '),
|
|
37
|
+
forbiddenTests: profile.scope.forbiddenTests.length ? profile.scope.forbiddenTests.map((r) => r.source).join(' or ') : '(none declared)',
|
|
38
|
+
agenticDescription: profile.agentic.description, agenticDirs: profile.agentic.dirs.join(', '),
|
|
39
|
+
};
|
|
40
|
+
const missing = new Set();
|
|
41
|
+
const out = t.replace(/\{\{(\w+)\}\}/g, (_, k) => { if (all[k] === undefined || all[k] === null || all[k] === '') { missing.add(k); return ''; } return String(all[k]); });
|
|
42
|
+
if (missing.size) throw new Error(`repair prompt field missing: ${[...missing].join(', ')}`);
|
|
43
|
+
return out;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The command, prepared and not started: the CLI in print mode on the profile's family, in the work tree, memory-bounded when the host has the wrapper. */
|
|
47
|
+
function buildCommand({ install, profile, prompt, cwd, claudeBin, budgetUsd }) {
|
|
48
|
+
const model = modelFor(install, profile);
|
|
49
|
+
const argv = [...hostWrap(), claudeBin || process.env.CLAUDE_BIN || 'claude', '-p', prompt, '--model', model, '--output-format', 'json',
|
|
50
|
+
'--setting-sources', 'project', '--permission-mode', 'bypassPermissions',
|
|
51
|
+
...(budgetUsd ? ['--max-budget-usd', String(budgetUsd)] : [])];
|
|
52
|
+
return { cwd, argv, model };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The answer's shape: repaired / cannot with the fields — anything else is no answer. */
|
|
56
|
+
function answerShape(v) {
|
|
57
|
+
if (!v || typeof v !== 'object') return null;
|
|
58
|
+
const strs = (a) => Array.isArray(a) && a.every((x) => typeof x === 'string');
|
|
59
|
+
if (!['repaired', 'cannot'].includes(v.status) || typeof v.summary !== 'string' || typeof v.cause !== 'string' || !strs(v.files) || !strs(v.tests)) return null;
|
|
60
|
+
// THE LAYER REPAIRED, AND WHY (operator, 2026-10-20: agentic first): a reply that does not say is not an answer
|
|
61
|
+
if (!['code', 'agentic'].includes(v.layer) || typeof v.layerWhy !== 'string' || !v.layerWhy.trim()) return null;
|
|
62
|
+
return { status: v.status, layer: v.layer, layerWhy: v.layerWhy, summary: v.summary, cause: v.cause, files: v.files, tests: v.tests };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The agent's answer: the LAST object anywhere in its reply that has the answer's shape — a bare document, a fenced block, or (as the first live
|
|
67
|
+
* session answered, 2026-10-20) a line of prose and then the object. Prose around it, a format example before it, braces inside its strings: none of
|
|
68
|
+
* that hides it; an object of the wrong shape is never taken for it.
|
|
69
|
+
*/
|
|
70
|
+
function readAnswer(text) {
|
|
71
|
+
const t = String(text || '');
|
|
72
|
+
const tryParse = (x) => { try { return answerShape(JSON.parse(x)); } catch { return null; } };
|
|
73
|
+
const opens = []; const closes = [];
|
|
74
|
+
for (let i = 0; i < t.length; i += 1) { if (t[i] === '{') opens.push(i); else if (t[i] === '}') closes.push(i); }
|
|
75
|
+
for (let a = opens.length - 1; a >= 0; a -= 1) for (let b = closes.length - 1; b >= 0 && closes[b] > opens[a]; b -= 1) { const v = tryParse(t.slice(opens[a], closes[b] + 1)); if (v) return v; }
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Runs the session (the only model call the doctor makes). Returns the answer, the session's cost record and any model of another family it used. */
|
|
80
|
+
function runAgent({ install, profile, prompt, cwd, sessionDir, env, claudeBin, budgetUsd }) {
|
|
81
|
+
const cmd = buildCommand({ install, profile, prompt, cwd, claudeBin, budgetUsd });
|
|
82
|
+
const r = spawnSync(cmd.argv[0], cmd.argv.slice(1), { cwd: cmd.cwd, stdio: ['ignore', 'pipe', 'pipe'], env: { CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS: '0', ...process.env, ...(env || {}) }, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
|
|
83
|
+
try { fs.mkdirSync(sessionDir, { recursive: true }); fs.writeFileSync(path.join(sessionDir, 'agent.stderr'), String(r.stderr || '')); fs.writeFileSync(path.join(sessionDir, 'agent.stdout'), String(r.stdout || '')); } catch { /* evidence only */ }
|
|
84
|
+
let out = null; try { out = JSON.parse(r.stdout); } catch { out = null; }
|
|
85
|
+
const session = out && out.session_id ? { sessionId: out.session_id, modelUsage: out.modelUsage || {}, totalCostUsd: out.total_cost_usd || 0 } : null;
|
|
86
|
+
const fam = profile.model.family.toLowerCase();
|
|
87
|
+
const otherFamily = session ? Object.keys(session.modelUsage).filter((m) => !m.toLowerCase().includes(fam)) : [];
|
|
88
|
+
return { exit: r.status, answer: out && typeof out.result === 'string' ? readAnswer(out.result) : null, session, otherFamily, model: cmd.model };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
module.exports = { modelFor, renderPrompt, buildCommand, readAnswer, runAgent };
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// WHEN THE DOCTOR RUNS (operator, 2026-10-20: "it has to be a REAL pipeline error"; "no false positives"). The doctor repairs one
|
|
3
|
+
// component's own code, so it is called only when the leg's own evidence shows a defect in that code — and when the evidence is anything
|
|
4
|
+
// less, the answer is "no" and the failure is reported. Precision over recall, by construction. Nothing here is about any pipeline: its
|
|
5
|
+
// repairable directories and its deliberate refusals come from its profile (profile.js); what a failure looks like in its runtime comes
|
|
6
|
+
// from the signature data (config/doctor-signatures.json).
|
|
7
|
+
// classify({ profile, portRoot, res, crash, aborted, since, signatures? }) → { call: boolean, kind, why }
|
|
8
|
+
// profile the pipeline's compiled profile portRoot the component as installed (<install>/<profile.install.dest>)
|
|
9
|
+
// res the leg's answer ({status, …}) crash the leg's crash record (crash-record.js), if any
|
|
10
|
+
// aborted the operator or a watcher ended the leg since the leg's start (ISO)
|
|
11
|
+
const fs = require('fs');
|
|
12
|
+
const path = require('path');
|
|
13
|
+
|
|
14
|
+
const no = (kind, why) => ({ call: false, kind, why });
|
|
15
|
+
const rxOf = (s) => new RegExp(s);
|
|
16
|
+
|
|
17
|
+
/** The signature set of one runtime, compiled. An unknown runtime is an error, never a guess. */
|
|
18
|
+
function signaturesFor(runtime, injected) {
|
|
19
|
+
const all = injected || JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', '..', 'config', 'doctor-signatures.json'), 'utf8'));
|
|
20
|
+
const s = all[runtime];
|
|
21
|
+
if (!s) throw new Error(`no signatures for runtime "${runtime}" (config/doctor-signatures.json knows: ${Object.keys(all).filter((k) => !k.startsWith('$')).join(', ')})`);
|
|
22
|
+
return { runtimeNames: new Set(s.runtimeNames), runtimeMessage: rxOf(s.runtimeMessage), external: rxOf(s.external), generalRefusal: rxOf(s.generalRefusal),
|
|
23
|
+
internalFrame: rxOf(s.internalFrame), dependencyDir: s.dependencyDir };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function classify({ profile, portRoot, res, crash, aborted, since, signatures }) {
|
|
27
|
+
const sig = signaturesFor(profile.classify.runtime, signatures);
|
|
28
|
+
if (!res || typeof res !== 'object') return no('no-result', 'the leg gave no result');
|
|
29
|
+
if (aborted) return no('aborted', 'the operator or a watcher ended the leg');
|
|
30
|
+
// AN AGENT-CAUSED ABORT (operator, 2026-10-20: look for the agentic solution first): a failed leg whose reason is one of the pipeline's own agent-caused
|
|
31
|
+
// messages (its profile) — an unreadable reply, a refused schema, an agent that gave no answer. It is evidenced by the message, so a failure that names the
|
|
32
|
+
// machine, the network or the model service is still external, and a lane's own verdict names no such message
|
|
33
|
+
if (res.status === 'failed') {
|
|
34
|
+
const reason = typeof res.reason === 'string' ? res.reason : '';
|
|
35
|
+
const hit = reason && profile.agentic.abortSignatures.find((r) => r.test(reason));
|
|
36
|
+
if (!hit) return no('not-a-crash', 'the leg failed on a verdict that names no agent-caused message — reported, not repaired');
|
|
37
|
+
if (sig.external.test(reason)) return no('external', 'the agent-caused message also names the machine, the network, the model service or a kill — not the pipeline');
|
|
38
|
+
return { call: true, kind: 'agentic-abort', layer: 'agentic', why: `an agent caused the abort: ${reason.slice(0, 200)}` };
|
|
39
|
+
}
|
|
40
|
+
if (res.status !== 'crashed') return no(res.status === 'no-answer' ? 'no-record' : 'not-a-crash', `the leg ended ${res.status}, not crashed — a kill, a verdict or a pause is reported, not repaired`);
|
|
41
|
+
if (!crash || typeof crash !== 'object') return no('no-record', 'no crash record: nothing proves a defect in the code');
|
|
42
|
+
if (!crash.at || !since || Date.parse(crash.at) < Date.parse(since)) return no('stale-record', 'the crash record is older than this leg');
|
|
43
|
+
const text = `${crash.name || ''}: ${crash.message || ''}`;
|
|
44
|
+
if (sig.external.test(text)) return no('external', 'the failure names the machine, the network, the model service or a kill — not the code');
|
|
45
|
+
if (sig.generalRefusal.test(text) || profile.classify.refusals.some((r) => r.test(text))) return no('refusal', 'the pipeline stopped on purpose');
|
|
46
|
+
const site = (crash.frames || []).find((f) => f && typeof f.file === 'string' && !sig.internalFrame.test(f.file)) || null;
|
|
47
|
+
if (!site) return no('outside-port', 'no frame of the stack is in any file');
|
|
48
|
+
const rel = path.relative(portRoot || '', site.file);
|
|
49
|
+
const parts = rel.split(path.sep);
|
|
50
|
+
const inPort = !!portRoot && !rel.startsWith('..') && !path.isAbsolute(rel) && profile.scope.repairDirs.includes(parts[0]) && !parts.includes(sig.dependencyDir);
|
|
51
|
+
if (!inPort) return no('outside-port', `the throw site ${site.file}:${site.line} is not the component's own code`);
|
|
52
|
+
if (!sig.runtimeNames.has(crash.name) && !sig.runtimeMessage.test(crash.message || '')) return no('ambiguous', `an Error in ${rel}:${site.line} with no sign of a code defect — reported`);
|
|
53
|
+
return { call: true, kind: 'port-defect', layer: 'code', why: `${crash.name}: ${String(crash.message).slice(0, 160)} — thrown at ${rel}:${site.line}${crash.step ? ` (step: ${crash.step})` : ''}` };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
module.exports = { classify, signaturesFor };
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// THE DOCTOR'S COMMAND LINE — for launchers that are not JavaScript (the bash pipeline): one JSON line out, and an exit code that says whether the
|
|
3
|
+
// command ran, not what the doctor decided (read `handled` / `restarted` in the line).
|
|
4
|
+
// amsd-pipeline-shared doctor validate --profile <file> exit 0 ok · 1 the profile has problems (all named in the line)
|
|
5
|
+
// amsd-pipeline-shared doctor handle --profile <file> --run-dir <d> --out-dir <d> --install <i> --run-id <id> --leg-result <json file>
|
|
6
|
+
// [--crash-file <json file>] [--aborted yes] --since <ISO> --launch-values <json file> [--protected <dir>,<dir>…]
|
|
7
|
+
// --relaunch-cmd '<JSON array: the command, to which the profile's restart arguments are appended>' --need-mb <n> --reserve-mb <n>
|
|
8
|
+
// [--heavy-rules <json file>] exit 0 it ran · 64 usage · 70 an error
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const { spawnSync } = require('child_process');
|
|
11
|
+
const { load } = require('./profile.js');
|
|
12
|
+
|
|
13
|
+
const REQUIRED = ['profile', 'runDir', 'outDir', 'install', 'runId', 'legResult', 'since', 'launchValues', 'relaunchCmd', 'needMb', 'reserveMb'];
|
|
14
|
+
const flag = (k) => `--${k.replace(/[A-Z]/g, (c) => `-${c.toLowerCase()}`)}`;
|
|
15
|
+
|
|
16
|
+
function parse(argv) {
|
|
17
|
+
const opts = {};
|
|
18
|
+
for (let i = 0; i < argv.length; i += 2) { if (!argv[i].startsWith('--')) throw new Error(`unexpected argument ${argv[i]}`); opts[argv[i].slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase())] = argv[i + 1]; }
|
|
19
|
+
return opts;
|
|
20
|
+
}
|
|
21
|
+
const readJson = (f) => JSON.parse(fs.readFileSync(f, 'utf8'));
|
|
22
|
+
const out = (o) => process.stdout.write(`${JSON.stringify(o)}\n`);
|
|
23
|
+
|
|
24
|
+
async function doctorCli(argv) {
|
|
25
|
+
const [sub, ...rest] = argv;
|
|
26
|
+
let opts;
|
|
27
|
+
try { opts = parse(rest); } catch (e) { process.stderr.write(`doctor: ${e.message}\n`); return 64; }
|
|
28
|
+
if (sub === 'validate') {
|
|
29
|
+
if (!opts.profile) { process.stderr.write('doctor validate: missing --profile\n'); return 64; }
|
|
30
|
+
const r = load(opts.profile); out({ ok: !r.problems.length, problems: r.problems });
|
|
31
|
+
return r.problems.length ? 1 : 0;
|
|
32
|
+
}
|
|
33
|
+
if (sub !== 'handle') { process.stderr.write('usage: amsd-pipeline-shared doctor validate --profile <file> | handle --profile <file> --run-dir <d> … (see lib/doctor/cli.js)\n'); return 64; }
|
|
34
|
+
const missing = REQUIRED.filter((k) => !opts[k]);
|
|
35
|
+
if (missing.length) { process.stderr.write(`doctor handle: missing ${missing.map(flag).join(', ')}\n`); return 64; }
|
|
36
|
+
try {
|
|
37
|
+
const pr = load(opts.profile);
|
|
38
|
+
if (pr.problems.length) { out({ ok: false, problems: pr.problems }); return 1; }
|
|
39
|
+
const { handle } = require('./handle.js'); const { runAgent } = require('./agent.js'); const { heavyJobs } = require('./restarter.js');
|
|
40
|
+
const lv = readJson(opts.launchValues);
|
|
41
|
+
const rules = opts.heavyRules ? readJson(opts.heavyRules) : undefined;
|
|
42
|
+
const cmd = JSON.parse(opts.relaunchCmd);
|
|
43
|
+
if (!Array.isArray(cmd) || !cmd.length || !cmd.every((x) => typeof x === 'string')) throw new Error('--relaunch-cmd must be a JSON array of strings');
|
|
44
|
+
const result = await handle({
|
|
45
|
+
profile: pr.profile, runDir: opts.runDir, outDir: opts.outDir, install: opts.install, values: lv.values || lv, runId: opts.runId,
|
|
46
|
+
res: readJson(opts.legResult), crash: opts.crashFile && fs.existsSync(opts.crashFile) ? readJson(opts.crashFile) : null, aborted: opts.aborted === 'yes', since: opts.since,
|
|
47
|
+
protectedRepos: (opts.protected || '').split(',').filter(Boolean),
|
|
48
|
+
agent: async ({ prompt, cwd, sessionDir, install, profile, budgetUsd }) => runAgent({ install, profile, prompt, cwd, sessionDir, budgetUsd }),
|
|
49
|
+
relaunch: async (args) => { const r = spawnSync(cmd[0], [...cmd.slice(1), ...args], { stdio: 'inherit' }); return { status: r.status }; },
|
|
50
|
+
needMb: Number(opts.needMb), reserveMb: Number(opts.reserveMb), heavyJobs: () => heavyJobs({ rules }),
|
|
51
|
+
});
|
|
52
|
+
out(result); return 0;
|
|
53
|
+
} catch (e) { process.stderr.write(`doctor handle: ${e.stack || e.message}\n`); return 70; }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
module.exports = { doctorCli };
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// THE CRASH RECORD (operator, 2026-10-20: "could be a crash listener"). A hard abort of a pipeline's driver — a throw nothing caught, a
|
|
3
|
+
// promise rejected with nobody awaiting it — leaves (1) a record in the file the pipeline's profile names (crash.file) in its run directory:
|
|
4
|
+
// what was thrown (name, message, stack, the stack's frames) and the step the driver was in, written whole or not at all; (2) an answer line
|
|
5
|
+
// on stdout, {status:'crashed', exit:1, reason}, where the launcher reads a leg's outcome. The launcher decides from this record
|
|
6
|
+
// (classify.js), never from a regex over stderr. A kill by a signal cannot be listened for: it leaves no record, and a leg with no
|
|
7
|
+
// record is never taken for a defect in the code. The file name is the pipeline's: nothing here names any pipeline.
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
|
|
11
|
+
let runDir = null;
|
|
12
|
+
let fileName = null;
|
|
13
|
+
let step = '';
|
|
14
|
+
|
|
15
|
+
/** The step the driver is in, kept in the record. */
|
|
16
|
+
function setStep(s) { step = String(s || ''); }
|
|
17
|
+
|
|
18
|
+
/** " at fn (/path/file.js:12:34)" and " at /path/file.js:12:34" → {fn, file, line, column}. */
|
|
19
|
+
function frames(stack) {
|
|
20
|
+
const out = [];
|
|
21
|
+
for (const l of String(stack || '').split('\n')) {
|
|
22
|
+
const m = l.match(/^\s*at (?:(.*?) \()?(.*?):(\d+):(\d+)\)?\s*$/);
|
|
23
|
+
if (m) out.push({ fn: m[1] || '', file: m[2], line: Number(m[3]), column: Number(m[4]) });
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** What was thrown, whatever it was: an Error, or anything a program may throw. */
|
|
29
|
+
function describe(e) {
|
|
30
|
+
if (e instanceof Error) return { name: e.name || 'Error', message: String(e.message), stack: String(e.stack || '') };
|
|
31
|
+
let text; try { text = typeof e === 'string' ? e : JSON.stringify(e); } catch { text = String(e); }
|
|
32
|
+
return { name: 'NonError', message: String(text), stack: '' };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Writes <dir>/<file> whole (a temp file, then a rename); returns the record. */
|
|
36
|
+
function write(dir, file, err, extra) {
|
|
37
|
+
const d = describe(err);
|
|
38
|
+
const rec = { at: new Date().toISOString(), pid: process.pid, ...d, frames: frames(d.stack), step, ...(extra || {}) };
|
|
39
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
40
|
+
const tmp = path.join(dir, `${file}.${process.pid}.tmp`);
|
|
41
|
+
fs.writeFileSync(tmp, `${JSON.stringify(rec, null, 2)}\n`);
|
|
42
|
+
fs.renameSync(tmp, path.join(dir, file));
|
|
43
|
+
return rec;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The leg's answer for a crash, and the exit. Never throws: a crash handler that crashes hides the crash. */
|
|
47
|
+
function die(e) {
|
|
48
|
+
let reason = 'crashed';
|
|
49
|
+
try { const rec = write(runDir, fileName, e); reason = `${rec.name}: ${rec.message}`.slice(0, 400); } catch (w) { reason = `crashed (the record could not be written: ${w.message})`; }
|
|
50
|
+
try { process.stdout.write(`${JSON.stringify({ status: 'crashed', exit: 1, reason })}\n`); } catch { /* nothing left to tell */ }
|
|
51
|
+
process.exit(1);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Listens for what nothing caught. */
|
|
55
|
+
function install(opts) {
|
|
56
|
+
if (!opts || !opts.runDir || !opts.file) throw new Error('crash record: install needs { runDir, file } — the file name is the pipeline\'s (profile crash.file), never a default');
|
|
57
|
+
runDir = opts.runDir; fileName = opts.file;
|
|
58
|
+
process.on('uncaughtException', die);
|
|
59
|
+
process.on('unhandledRejection', (r) => die(r));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
module.exports = { install, setStep, write, die, frames, describe };
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// THE DOCTOR, WHOLE (operator, 2026-10-20): crash → classify → repair → verify → commit → reinstall → restart, for ANY pipeline — everything
|
|
3
|
+
// about the pipeline is its profile (profile.js). Called by a launcher when a leg ends; it does nothing at all unless the pipeline's setting is
|
|
4
|
+
// on, and calls the model only for a real defect in the component's own code (classify.js). It repairs ONLY the component's code, in a work tree
|
|
5
|
+
// of the component's own repository: the repository must BE the profile's component and never one of the run's protected (client) repositories,
|
|
6
|
+
// and those are snapshotted around the agent's session — any change in any of them refuses the whole repair. Every step is recorded in the
|
|
7
|
+
// doctor's own folder (state.js) and every stop is alerted. The model session is injected (agent.js is the live one).
|
|
8
|
+
// handle({ profile, runDir, outDir, install, values, runId, res, crash, aborted, since, protectedRepos, agent, relaunch,
|
|
9
|
+
// needMb, reserveMb, availableMb?, heavyJobs? }) → { handled, restarted, stopped?, why }
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { spawnSync } = require('child_process');
|
|
13
|
+
const { classify } = require('./classify.js');
|
|
14
|
+
const { settings, open } = require('./state.js');
|
|
15
|
+
const { verifyRepair, reinstall, installedMismatches, checkComponent, snapshotRepos, repoChanges } = require('./repair.js');
|
|
16
|
+
const { renderPrompt } = require('./agent.js');
|
|
17
|
+
const { restart } = require('./restarter.js');
|
|
18
|
+
|
|
19
|
+
const git = (repo, ...a) => spawnSync('git', ['-C', repo, ...a], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
|
20
|
+
const tail = (file, n) => { try { return fs.readFileSync(file, 'utf8').split('\n').slice(-n - 1).join('\n'); } catch { return ''; } };
|
|
21
|
+
|
|
22
|
+
async function handle(a) {
|
|
23
|
+
const { profile, runDir, outDir, install, values, runId, res, crash, aborted, since, protectedRepos, agent, relaunch } = a;
|
|
24
|
+
const cfg = settings(values, profile);
|
|
25
|
+
const alert = (text) => { try { fs.mkdirSync(outDir, { recursive: true }); fs.appendFileSync(path.join(outDir, profile.logs.alerts), `${new Date().toISOString()} DOCTOR ${text}\n`); } catch { /* the doctor's own log has it */ } };
|
|
26
|
+
// the operator switched it on but it cannot run (no repository, no spending cap): said out loud, never silently off
|
|
27
|
+
if (!cfg.on) { if (cfg.problem) alert(`cannot run: ${cfg.problem}`); return { handled: false, why: cfg.problem || 'the doctor is off' }; }
|
|
28
|
+
|
|
29
|
+
const d = open(runDir, cfg, profile);
|
|
30
|
+
const portRoot = path.join(install, profile.install.dest);
|
|
31
|
+
|
|
32
|
+
const c = classify({ profile, portRoot, res, crash, aborted, since });
|
|
33
|
+
if (!c.call) { d.event('not-called', { kind: c.kind, why: c.why }); return { handled: false, kind: c.kind, why: c.why }; }
|
|
34
|
+
d.event('trigger', { kind: c.kind, layer: c.layer, why: c.why, step: (crash && crash.step) || '' });
|
|
35
|
+
|
|
36
|
+
let attempt = null;
|
|
37
|
+
const stop = (reason, why, extra) => {
|
|
38
|
+
d.event('stop', { reason, why }, attempt && attempt.n);
|
|
39
|
+
if (attempt) fs.writeFileSync(path.join(attempt.dir, 'verdict.json'), `${JSON.stringify({ status: 'stopped', reason, why }, null, 2)}\n`);
|
|
40
|
+
alert(`stopped (${reason}): ${why}`);
|
|
41
|
+
return { handled: true, restarted: false, stopped: reason, why, ...(extra || {}) };
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
// one repair per failure the restarts can follow: attempts = the quota the operator gave + 1 (the quota 0 still repairs once)
|
|
45
|
+
if (d.attemptsMade() >= cfg.restarts + 1) return stop('attempts', `${d.attemptsMade()} repair attempt(s) already made for a quota of ${cfg.restarts} restart(s)`);
|
|
46
|
+
|
|
47
|
+
const comp = checkComponent({ repo: cfg.repo, profile, protectedRepos });
|
|
48
|
+
if (!comp.ok) return stop('not-the-component', comp.why);
|
|
49
|
+
const repo = comp.root;
|
|
50
|
+
const stale = installedMismatches({ repo, branch: 'HEAD', install, profile });
|
|
51
|
+
if (stale.length) return stop('repo-not-installed', `the repository's HEAD is not what is installed (${stale.slice(0, 6).join(', ')}) — a repair on it would overwrite the install with other code`);
|
|
52
|
+
|
|
53
|
+
attempt = d.nextAttempt();
|
|
54
|
+
const A = attempt.dir;
|
|
55
|
+
fs.writeFileSync(path.join(A, 'trigger.json'), `${JSON.stringify({ res, crash, classified: c, run: runId }, null, 2)}\n`);
|
|
56
|
+
const ev = path.join(A, 'evidence'); fs.mkdirSync(ev, { recursive: true });
|
|
57
|
+
// a crash leaves its record; an abort an agent caused leaves only the leg's result — that is then the evidence
|
|
58
|
+
if (crash) fs.writeFileSync(path.join(ev, profile.crash.file), `${JSON.stringify(crash, null, 2)}\n`);
|
|
59
|
+
else fs.writeFileSync(path.join(ev, 'leg-result.json'), `${JSON.stringify(res, null, 2)}\n`);
|
|
60
|
+
for (const f of profile.logs.evidence) fs.writeFileSync(path.join(ev, path.basename(f)), tail(path.join(runDir, f), profile.logs.evidenceTailLines));
|
|
61
|
+
|
|
62
|
+
const base = git(repo, 'rev-parse', 'HEAD').stdout.trim();
|
|
63
|
+
const branch = `${profile.branch.prefix}${runId}-${attempt.n}`;
|
|
64
|
+
const wt = path.join(A, 'wt');
|
|
65
|
+
const mk = git(repo, 'worktree', 'add', '-b', branch, wt, 'HEAD');
|
|
66
|
+
if (mk.status !== 0) return stop('worktree', `cannot make the repair's work tree: ${mk.stderr.trim()}`);
|
|
67
|
+
try {
|
|
68
|
+
const top = crash && (crash.frames || []).find((f) => f && typeof f.file === 'string' && !/^node:/.test(f.file));
|
|
69
|
+
const prompt = renderPrompt({ run: runId, attempt: attempt.n, step: (crash && crash.step) || 'unknown',
|
|
70
|
+
errorName: crash ? crash.name : 'agentic-abort', errorMessage: String(crash ? crash.message : res.reason).slice(0, 400),
|
|
71
|
+
throwSite: crash ? (top ? `${top.file}:${top.line}` : 'unknown') : 'no throw — an abort an agent caused', evidenceDir: ev, runDir, install, branch,
|
|
72
|
+
abortKind: c.kind, layerHint: c.layer }, profile);
|
|
73
|
+
fs.writeFileSync(path.join(A, 'prompt.txt'), prompt);
|
|
74
|
+
|
|
75
|
+
const before = snapshotRepos(protectedRepos || []);
|
|
76
|
+
d.event('agent', { branch, base }, attempt.n);
|
|
77
|
+
const r = await agent({ prompt, cwd: wt, sessionDir: A, install, profile, budgetUsd: cfg.budget });
|
|
78
|
+
if (r.session) for (const [model, u] of Object.entries(r.session.modelUsage || {})) d.cost({ attempt: attempt.n, model, tokensIn: u.inputTokens || 0, tokensOut: u.outputTokens || 0, costUsd: u.costUSD || 0 });
|
|
79
|
+
if (r.otherFamily && r.otherFamily.length) { d.event('non-family-model', { models: r.otherFamily }, attempt.n); alert(`warning: the session used a model that is not ${profile.model.family}: ${r.otherFamily.join(', ')}`); }
|
|
80
|
+
|
|
81
|
+
// CLIENT CODE IS NEVER THE DOCTOR'S: any change in any protected repository during the session refuses the whole repair
|
|
82
|
+
const touched = repoChanges(before, snapshotRepos(protectedRepos || []));
|
|
83
|
+
if (touched.length) { d.event('client-code-touched', { repos: touched }, attempt.n); return stop('client-code-touched', `the session changed protected client code — ${touched.join('; ')} — the repair is refused and nothing is installed`); }
|
|
84
|
+
|
|
85
|
+
if (!r.answer || !['code', 'agentic'].includes(r.answer.layer)) return stop('no-answer', `the agent gave no readable answer, or one that names no layer (exit ${r.exit})`);
|
|
86
|
+
if (r.answer.status === 'cannot') return stop('cannot', `the agent could not repair it: ${r.answer.summary}`);
|
|
87
|
+
if (git(wt, 'status', '--porcelain', '--untracked-files=no').stdout.trim()) return stop('uncommitted', 'the agent left changes to tracked files uncommitted');
|
|
88
|
+
|
|
89
|
+
const layer = r.answer.layer;
|
|
90
|
+
const v = verifyRepair({ repo, base, branch, profile, layer, abortKind: c.kind });
|
|
91
|
+
fs.writeFileSync(path.join(A, 'red.txt'), v.red || ''); fs.writeFileSync(path.join(A, 'green.txt'), v.green || '');
|
|
92
|
+
fs.writeFileSync(path.join(A, 'patch.diff'), git(repo, 'diff', `${base}..${branch}`).stdout);
|
|
93
|
+
if (!v.ok) { d.event('repair-rejected', { why: v.why }, attempt.n); return stop('rejected', v.why); }
|
|
94
|
+
d.event('verified', { layer, layerWhy: r.answer.layerWhy, files: v.files, why: v.why }, attempt.n);
|
|
95
|
+
|
|
96
|
+
// AGENTIC FIRST (operator, 2026-10-20): an agentic repair changes how agents are instructed or wired; where the pipeline's profile does not let such a
|
|
97
|
+
// repair apply itself, it is verified, kept on its branch, and left for the operator — said out loud
|
|
98
|
+
if (layer === 'agentic' && !profile.agentic.autoApply) {
|
|
99
|
+
const why = `an agentic repair is verified but the pipeline's profile does not let it apply itself (agentic.autoApply is false)`;
|
|
100
|
+
fs.writeFileSync(path.join(A, 'verdict.json'), `${JSON.stringify({ status: 'verified-not-installed', layer, layerWhy: r.answer.layerWhy, summary: r.answer.summary, cause: r.answer.cause, files: v.files, branch }, null, 2)}\n`);
|
|
101
|
+
d.event('agentic-awaits-approval', { branch, layerWhy: r.answer.layerWhy }, attempt.n);
|
|
102
|
+
alert(`AGENTIC repair verified on branch ${branch} — NOT installed: ${r.answer.summary} (${r.answer.layerWhy}); the operator decides`);
|
|
103
|
+
return { handled: true, repaired: true, restarted: false, stopped: 'agentic-awaits-approval', why, branch };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const ri = reinstall({ repo, branch, install, run: runId, attempt: attempt.n, files: v.files, profile });
|
|
107
|
+
fs.writeFileSync(path.join(A, 'reinstall.txt'), `${ri.installer || ''}\n${ri.why}\n${(ri.mismatches || []).join('\n')}\n`);
|
|
108
|
+
if (!ri.ok) return stop('reinstall-failed', ri.why);
|
|
109
|
+
d.event('reinstalled', { layer, commit: ri.commit, files: v.files }, attempt.n);
|
|
110
|
+
|
|
111
|
+
fs.writeFileSync(path.join(A, 'verdict.json'), `${JSON.stringify({ status: 'repaired', layer, layerWhy: r.answer.layerWhy, summary: r.answer.summary, cause: r.answer.cause, files: v.files, branch, commit: ri.commit }, null, 2)}\n`);
|
|
112
|
+
d.event('repaired', { layer, layerWhy: r.answer.layerWhy, summary: r.answer.summary, cause: r.answer.cause, branch, commit: ri.commit }, attempt.n);
|
|
113
|
+
alert(`repaired${layer === 'agentic' ? ' (AGENTIC LAYER — how agents are instructed or wired changed, not only code)' : ''}: ${r.answer.summary} (${r.answer.cause}) — branch ${branch}, ${v.files.length} file(s), installed`);
|
|
114
|
+
|
|
115
|
+
const rs = await restart({ state: d, profile, runDir, needMb: a.needMb, reserveMb: a.reserveMb, availableMb: a.availableMb, heavyJobs: a.heavyJobs, launch: relaunch });
|
|
116
|
+
if (!rs.restarted) alert(`not restarted: ${rs.why}`);
|
|
117
|
+
return { handled: true, repaired: true, restarted: rs.restarted, why: rs.why, branch, result: rs.result };
|
|
118
|
+
} finally { git(repo, 'worktree', 'remove', '--force', wt); git(repo, 'worktree', 'prune'); }
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
module.exports = { handle };
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// THE DOCTOR'S PROFILE — what a pipeline declares so the doctor holds none of that pipeline's facts (operator, 2026-10-20: "no
|
|
3
|
+
// hardcoding … there will be multiple pipelines in play now"). The doctor repairs the code of ONE component of a pipeline when a run
|
|
4
|
+
// fails on a defect in it; everything that is about that component — its repairable directories, how its tests run, how it is
|
|
5
|
+
// installed, what it must never touch, the names of its settings and logs — is read from its profile. A pipeline's own facts are
|
|
6
|
+
// never defaulted: a profile with anything missing or wrong is refused, every problem named by its field, all at once.
|
|
7
|
+
// validate(object) → { problems: string[], profile: compiled | null } load(file) → the same, from a JSON file
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
|
|
10
|
+
const SECTIONS = ['component', 'scope', 'test', 'install', 'crash', 'classify', 'settings', 'restart', 'model', 'logs', 'branch', 'boundaries', 'agentic'];
|
|
11
|
+
const ENV_NAME = /^[A-Z][A-Z0-9_]*$/;
|
|
12
|
+
const MANIFESTS = ['package.json#files']; // how the files a component ships are listed (the only manifest the doctor can read)
|
|
13
|
+
|
|
14
|
+
function validate(input) {
|
|
15
|
+
const problems = [];
|
|
16
|
+
const p = input && typeof input === 'object' && !Array.isArray(input) ? input : {};
|
|
17
|
+
const out = {};
|
|
18
|
+
const bad = (field, why) => problems.push(`${field}: ${why}`);
|
|
19
|
+
const has = (o, k) => o && typeof o === 'object' && Object.prototype.hasOwnProperty.call(o, k);
|
|
20
|
+
|
|
21
|
+
const str = (path, v) => { if (typeof v !== 'string' || !v.trim()) { bad(path, 'a non-empty string is required'); return null; } return v; };
|
|
22
|
+
const strs = (path, v, { nonEmpty }) => {
|
|
23
|
+
if (!Array.isArray(v) || !v.every((x) => typeof x === 'string' && x.trim())) { bad(path, 'a list of non-empty strings is required'); return null; }
|
|
24
|
+
if (nonEmpty && !v.length) { bad(path, 'at least one entry is required'); return null; }
|
|
25
|
+
return v;
|
|
26
|
+
};
|
|
27
|
+
const rel = (path, v) => { // a path inside a folder: relative, no climbing out
|
|
28
|
+
const s = str(path, v); if (s === null) return null;
|
|
29
|
+
if (s.startsWith('/') || s.split('/').includes('..')) { bad(path, `"${s}" must be relative and stay inside its folder`); return null; }
|
|
30
|
+
return s;
|
|
31
|
+
};
|
|
32
|
+
const file = (path, v) => { const s = str(path, v); if (s === null) return null; if (s.includes('/') || s === '..' || s === '.') { bad(path, `"${s}" must be a plain file name`); return null; } return s; };
|
|
33
|
+
const rx = (path, v) => {
|
|
34
|
+
if (typeof v !== 'string') { bad(path, 'a pattern string is required'); return null; }
|
|
35
|
+
if (!v) { bad(path, 'a non-empty pattern is required'); return null; }
|
|
36
|
+
try { return new RegExp(v); } catch (e) { bad(path, `not a valid pattern (${e.message})`); return null; }
|
|
37
|
+
};
|
|
38
|
+
const rxs = (path, v) => { if (!Array.isArray(v)) { bad(path, 'a list of patterns is required'); return null; } return v.map((x, i) => rx(`${path}[${i}]`, x)).filter(Boolean); };
|
|
39
|
+
const section = (k) => { if (!has(p, k)) { bad(k, 'this section is required'); return null; } return p[k]; };
|
|
40
|
+
const need = (sec, o, k) => { if (!has(o, k)) { bad(`${sec}.${k}`, 'this field is required'); return false; } return true; };
|
|
41
|
+
|
|
42
|
+
for (const k of SECTIONS) section(k);
|
|
43
|
+
const c = p.component; if (c) { for (const k of ['name', 'package', 'description']) if (need('component', c, k)) str(`component.${k}`, c[k]); out.component = { ...c }; }
|
|
44
|
+
|
|
45
|
+
const s = p.scope;
|
|
46
|
+
if (s) {
|
|
47
|
+
out.scope = {};
|
|
48
|
+
if (need('scope', s, 'repairDirs')) { const v = strs('scope.repairDirs', s.repairDirs, { nonEmpty: true }); if (v) { v.forEach((d, i) => rel(`scope.repairDirs[${i}]`, d)); out.scope.repairDirs = v; } }
|
|
49
|
+
if (need('scope', s, 'testDirs')) { const v = strs('scope.testDirs', s.testDirs, { nonEmpty: true }); if (v) { v.forEach((d, i) => rel(`scope.testDirs[${i}]`, d)); out.scope.testDirs = v; } }
|
|
50
|
+
if (need('scope', s, 'testFile')) out.scope.testFile = rx('scope.testFile', s.testFile);
|
|
51
|
+
if (need('scope', s, 'forbiddenTests')) out.scope.forbiddenTests = rxs('scope.forbiddenTests', s.forbiddenTests);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const t = p.test;
|
|
55
|
+
if (t) {
|
|
56
|
+
out.test = {};
|
|
57
|
+
if (need('test', t, 'command')) { const v = strs('test.command', t.command, { nonEmpty: true }); if (v) { if (!v.includes('{files}')) bad('test.command', 'must contain {files} as an entry of its own, where the test files go'); out.test.command = v; } }
|
|
58
|
+
if (need('test', t, 'envUnset')) out.test.envUnset = strs('test.envUnset', t.envUnset, { nonEmpty: false });
|
|
59
|
+
if (need('test', t, 'greenPatterns')) { if (Array.isArray(t.greenPatterns) && !t.greenPatterns.length) bad('test.greenPatterns', 'at least one pattern is required'); out.test.greenPatterns = rxs('test.greenPatterns', t.greenPatterns); }
|
|
60
|
+
if (need('test', t, 'failedPattern')) out.test.failedPattern = rx('test.failedPattern', t.failedPattern);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const i = p.install;
|
|
64
|
+
if (i) {
|
|
65
|
+
out.install = {};
|
|
66
|
+
if (need('install', i, 'dest')) out.install.dest = rel('install.dest', i.dest);
|
|
67
|
+
if (need('install', i, 'requires')) { const v = strs('install.requires', i.requires, { nonEmpty: false }); if (v) { v.forEach((d, n) => rel(`install.requires[${n}]`, d)); out.install.requires = v; } }
|
|
68
|
+
if (need('install', i, 'command')) { const v = strs('install.command', i.command, { nonEmpty: true }); if (v) { for (const ph of ['{worktree}', '{install}']) if (!v.some((x) => x.includes(ph))) bad('install.command', `must contain ${ph}`); out.install.command = v; } }
|
|
69
|
+
if (need('install', i, 'manifest')) { if (!MANIFESTS.includes(i.manifest)) bad('install.manifest', `must be one of: ${MANIFESTS.join(', ')}`); else out.install.manifest = i.manifest; }
|
|
70
|
+
if (need('install', i, 'patchRecord')) out.install.patchRecord = file('install.patchRecord', i.patchRecord);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (p.crash) { if (need('crash', p.crash, 'file')) out.crash = { file: file('crash.file', p.crash.file) }; }
|
|
74
|
+
|
|
75
|
+
const cl = p.classify;
|
|
76
|
+
if (cl) {
|
|
77
|
+
out.classify = {};
|
|
78
|
+
if (need('classify', cl, 'runtime')) out.classify.runtime = str('classify.runtime', cl.runtime);
|
|
79
|
+
if (need('classify', cl, 'refusals')) out.classify.refusals = rxs('classify.refusals', cl.refusals);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const se = p.settings;
|
|
83
|
+
if (se) {
|
|
84
|
+
out.settings = {};
|
|
85
|
+
for (const k of ['on', 'restarts', 'repo', 'budget']) if (need('settings', se, k)) { const v = str(`settings.${k}`, se[k]); if (v !== null && !ENV_NAME.test(v)) bad(`settings.${k}`, `"${v}" must be an environment variable name (A-Z, 0-9, _)`); else out.settings[k] = v; }
|
|
86
|
+
const names = ['on', 'restarts', 'repo', 'budget'].map((k) => se[k]).filter(Boolean);
|
|
87
|
+
if (new Set(names).size !== names.length) bad('settings', 'the four setting names must be different');
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (p.restart) { if (need('restart', p.restart, 'args')) out.restart = { args: strs('restart.args', p.restart.args, { nonEmpty: true }) }; }
|
|
91
|
+
|
|
92
|
+
const m = p.model;
|
|
93
|
+
if (m) { out.model = {}; for (const k of ['family', 'config']) if (need('model', m, k)) out.model[k] = str(`model.${k}`, m[k]); }
|
|
94
|
+
|
|
95
|
+
const lg = p.logs;
|
|
96
|
+
if (lg) {
|
|
97
|
+
out.logs = {};
|
|
98
|
+
if (need('logs', lg, 'dir')) out.logs.dir = rel('logs.dir', lg.dir);
|
|
99
|
+
if (need('logs', lg, 'alerts')) out.logs.alerts = file('logs.alerts', lg.alerts);
|
|
100
|
+
if (need('logs', lg, 'evidence')) { const v = strs('logs.evidence', lg.evidence, { nonEmpty: false }); if (v) { v.forEach((d, n) => rel(`logs.evidence[${n}]`, d)); out.logs.evidence = v; } }
|
|
101
|
+
if (need('logs', lg, 'evidenceTailLines')) { if (!Number.isInteger(lg.evidenceTailLines) || lg.evidenceTailLines < 1) bad('logs.evidenceTailLines', 'a positive whole number is required'); else out.logs.evidenceTailLines = lg.evidenceTailLines; }
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// THE AGENTIC LAYER (operator, 2026-10-20: look for the agentic solution before the deterministic one; an agentic abort is repaired agentically):
|
|
105
|
+
// where the pipeline's agents are instructed and wired — prompts, answer schemas, agent definitions, hand-offs between agents — inside the repairable
|
|
106
|
+
// directories; which leg messages mean an AGENT caused the abort (patterns); whether a verified agentic repair is applied without waiting for the operator
|
|
107
|
+
const ag = p.agentic;
|
|
108
|
+
if (ag) {
|
|
109
|
+
out.agentic = {};
|
|
110
|
+
if (need('agentic', ag, 'dirs')) { const v = strs('agentic.dirs', ag.dirs, { nonEmpty: true }); if (v) { v.forEach((d, i) => { if (rel(`agentic.dirs[${i}]`, d) !== null && out.scope && out.scope.repairDirs && !out.scope.repairDirs.includes(d.split('/')[0])) bad(`agentic.dirs[${i}]`, `"${d}" must be inside one of scope.repairDirs`); }); out.agentic.dirs = v; } }
|
|
111
|
+
if (need('agentic', ag, 'abortSignatures')) out.agentic.abortSignatures = rxs('agentic.abortSignatures', ag.abortSignatures);
|
|
112
|
+
if (need('agentic', ag, 'autoApply')) { if (typeof ag.autoApply !== 'boolean') bad('agentic.autoApply', 'true or false is required'); else out.agentic.autoApply = ag.autoApply; }
|
|
113
|
+
if (need('agentic', ag, 'description')) out.agentic.description = str('agentic.description', ag.description);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (p.branch) { if (need('branch', p.branch, 'prefix')) out.branch = { prefix: str('branch.prefix', p.branch.prefix) }; }
|
|
117
|
+
if (has(p, 'boundaries')) out.boundaries = strs('boundaries', p.boundaries, { nonEmpty: true });
|
|
118
|
+
|
|
119
|
+
return problems.length ? { problems, profile: null } : { problems, profile: out };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function load(file) {
|
|
123
|
+
let text;
|
|
124
|
+
try { text = fs.readFileSync(file, 'utf8'); } catch (e) { return { problems: [`cannot read ${file}: ${e.message}`], profile: null }; }
|
|
125
|
+
let obj;
|
|
126
|
+
try { obj = JSON.parse(text); } catch (e) { return { problems: [`${file} is not JSON: ${e.message}`], profile: null }; }
|
|
127
|
+
return validate(obj);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
module.exports = { validate, load, SECTIONS, MANIFESTS };
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// THE REPAIR, ACCEPTED ONLY ON EVIDENCE, REINSTALLED THROUGH THE COMPONENT'S OWN INSTALLER, AND CLIENT CODE NEVER TOUCHED (operator,
|
|
3
|
+
// 2026-10-20: "crash - fix - test - commit - reinstall - restart"; "doctor can ONLY change pipeline related code - not client codeline
|
|
4
|
+
// code"). The agent that repairs is a model; what accepts its work is not: the doctor re-runs the test itself — RED on the base with only the
|
|
5
|
+
// test added, GREEN with the fix — and reads the diff's file list. A repair that proves nothing, touches anything outside the component's
|
|
6
|
+
// repairable directories, or leans on a monolithic test is refused, and the install is never touched. Every fact about the pipeline —
|
|
7
|
+
// its directories, how its tests run and read, its installer, where it installs, what its patch record is called — is its profile's.
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const os = require('os');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const { spawnSync } = require('child_process');
|
|
12
|
+
|
|
13
|
+
const git = (repo, ...a) => spawnSync('git', ['-C', repo, ...a], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
|
14
|
+
const bad = (why, extra) => ({ ok: false, why, ...(extra || {}) });
|
|
15
|
+
|
|
16
|
+
/** The host's memory wrapper (every process memory-bounded — the operator's standing rule), when the host has one: AMSD_BOUNDED, else ~/.claude/bin/bounded. */
|
|
17
|
+
function hostWrap() {
|
|
18
|
+
const b = process.env.AMSD_BOUNDED || path.join(os.homedir(), '.claude', 'bin', 'bounded');
|
|
19
|
+
return fs.existsSync(b) ? [b, '--want', '2048', '--'] : [];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** A profile command with its placeholders filled; an entry that is exactly {files} becomes the files. */
|
|
23
|
+
function expand(command, vars) {
|
|
24
|
+
return command.flatMap((tok) => (tok === '{files}' ? vars.files : [tok.replace(/\{(node|worktree|install)\}/g, (_, k) => vars[k])]));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** The component's tests in a work tree; the output is the evidence. */
|
|
28
|
+
function runTests(wt, files, profile) {
|
|
29
|
+
const c = [...hostWrap(), ...expand(profile.test.command, { node: process.execPath, files })];
|
|
30
|
+
// a test run started inside another test run inherits that runner's context and then reports "passed" whatever happened: the run under
|
|
31
|
+
// judgement gets the environment the profile names it needs clean, so its exit code is its own
|
|
32
|
+
const env = { ...process.env }; for (const k of profile.test.envUnset) delete env[k];
|
|
33
|
+
const r = spawnSync(c[0], c.slice(1), { cwd: wt, env, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
|
34
|
+
return { status: r.status, out: `${r.stdout || ''}${r.stderr || ''}` };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A throwaway work tree of <rev>; removed by the caller's finally. */
|
|
38
|
+
function worktree(repo, rev) {
|
|
39
|
+
const wt = fs.mkdtempSync(path.join(os.tmpdir(), 'doctor-wt-'));
|
|
40
|
+
fs.rmSync(wt, { recursive: true, force: true });
|
|
41
|
+
const r = git(repo, 'worktree', 'add', '--detach', '-q', wt, rev);
|
|
42
|
+
if (r.status !== 0) throw new Error(`worktree of ${rev}: ${r.stderr}`);
|
|
43
|
+
return wt;
|
|
44
|
+
}
|
|
45
|
+
const drop = (repo, wt) => { git(repo, 'worktree', 'remove', '--force', wt); git(repo, 'worktree', 'prune'); fs.rmSync(wt, { recursive: true, force: true }); };
|
|
46
|
+
|
|
47
|
+
function verifyRepair({ repo, base, branch, profile, layer, abortKind }) {
|
|
48
|
+
// the layer the agent says it repaired and how the abort was classified: both required, neither defaulted
|
|
49
|
+
if (!['code', 'agentic'].includes(layer)) throw new Error(`verifyRepair needs layer 'code' or 'agentic', got ${JSON.stringify(layer)}`);
|
|
50
|
+
if (typeof abortKind !== 'string' || !abortKind) throw new Error('verifyRepair needs abortKind (how the abort was classified)');
|
|
51
|
+
const d = git(repo, 'diff', '--name-only', `${base}..${branch}`);
|
|
52
|
+
if (d.status !== 0) return bad(`cannot read the repair's diff: ${d.stderr.trim()}`);
|
|
53
|
+
const files = d.stdout.split('\n').filter(Boolean);
|
|
54
|
+
if (!files.length) return bad('the repair changed nothing');
|
|
55
|
+
const { repairDirs, testDirs, testFile, forbiddenTests } = profile.scope;
|
|
56
|
+
const out = files.filter((f) => { const n = path.posix.normalize(f); return n.startsWith('..') || path.isAbsolute(n) || ![...repairDirs, ...testDirs].includes(n.split('/')[0]); });
|
|
57
|
+
if (out.length) return bad(`outside the component: ${out.join(', ')} — only ${[...repairDirs, ...testDirs].join(', ')} may change`, { files });
|
|
58
|
+
const tests = files.filter((f) => testFile.test(f));
|
|
59
|
+
// AGENTIC FIRST (operator, 2026-10-20): an abort an agent caused is repaired in the agentic layer; a repair that calls itself agentic changes only that layer
|
|
60
|
+
if (abortKind === 'agentic-abort' && layer !== 'agentic') return bad('an agentic abort is repaired agentically: a deterministic (code) change for an abort an agent caused only hides the agent\'s behaviour', { files });
|
|
61
|
+
if (layer === 'agentic') {
|
|
62
|
+
const inLayer = (f) => profile.agentic.dirs.some((d) => f === d || f.startsWith(`${d}/`));
|
|
63
|
+
const outside = files.filter((f) => !tests.includes(f) && !testDirs.includes(f.split('/')[0]) && !inLayer(f));
|
|
64
|
+
if (outside.length) return bad(`outside the agentic layer (${profile.agentic.dirs.join(', ')}): ${outside.join(', ')} — an agentic repair changes how agents are instructed and wired, nothing else`, { files });
|
|
65
|
+
}
|
|
66
|
+
const src = files.filter((f) => repairDirs.includes(f.split('/')[0]));
|
|
67
|
+
if (!src.length) return bad('no change to the component itself: only tests changed', { files });
|
|
68
|
+
if (!tests.length) return bad('no test: a repair without a test that fails first proves nothing', { files });
|
|
69
|
+
const mono = tests.filter((f) => forbiddenTests.some((r) => r.test(f)));
|
|
70
|
+
if (mono.length) return bad(`monolithic test(s): ${mono.join(', ')} — targeted tests only (operator rule)`, { files });
|
|
71
|
+
|
|
72
|
+
let wtBase = null; let wtTip = null;
|
|
73
|
+
try {
|
|
74
|
+
// RED: the base, with ONLY the repair's tests added — they must fail there (a test that cannot even load is caught by the GREEN run)
|
|
75
|
+
wtBase = worktree(repo, base);
|
|
76
|
+
const co = git(wtBase, 'checkout', branch, '--', ...tests);
|
|
77
|
+
if (co.status !== 0) return bad(`cannot place the tests on the base: ${co.stderr.trim()}`, { files });
|
|
78
|
+
const red = runTests(wtBase, tests, profile);
|
|
79
|
+
if (red.status === 0) return bad('the test passes without the fix: it is not RED, so it proves nothing', { files, red: red.out });
|
|
80
|
+
// GREEN: the branch's tip
|
|
81
|
+
wtTip = worktree(repo, branch);
|
|
82
|
+
const green = runTests(wtTip, tests, profile);
|
|
83
|
+
if (green.status !== 0 || !profile.test.greenPatterns.every((r) => r.test(green.out))) return bad('the test fails with the fix: not GREEN', { files, red: red.out, green: green.out });
|
|
84
|
+
return { ok: true, why: 'RED on the base, GREEN with the fix', files, tests, red: red.out, green: green.out };
|
|
85
|
+
} finally { if (wtBase) drop(repo, wtBase); if (wtTip) drop(repo, wtTip); }
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// ── the reinstall: the component's own installer, then every packaged file compared byte for byte ─────────────────────
|
|
89
|
+
const show = (repo, rev, rel) => spawnSync('git', ['-C', repo, 'show', `${rev}:${rel}`], { encoding: 'buffer', maxBuffer: 64 * 1024 * 1024 });
|
|
90
|
+
|
|
91
|
+
/** The files the component ships at <branch>: the manifest's entries expanded from the branch's tree, plus the manifest file itself. */
|
|
92
|
+
function packaged(repo, branch) {
|
|
93
|
+
const pj = show(repo, branch, 'package.json'); if (pj.status !== 0) return [];
|
|
94
|
+
const prefixes = (JSON.parse(pj.stdout.toString()).files || []).map((f) => f.replace(/\/$/, ''));
|
|
95
|
+
const all = git(repo, 'ls-tree', '-r', '--name-only', branch).stdout.split('\n').filter(Boolean);
|
|
96
|
+
return [...new Set([...all.filter((f) => prefixes.some((p) => f === p || f.startsWith(`${p}/`))), 'package.json'])].sort();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The packaged files that are not byte-identical under <install>/<dest> (missing counts). */
|
|
100
|
+
function installedMismatches({ repo, branch, install, profile }) {
|
|
101
|
+
const dest = path.join(install, profile.install.dest);
|
|
102
|
+
return packaged(repo, branch).filter((rel) => {
|
|
103
|
+
const want = show(repo, branch, rel); const f = path.join(dest, rel);
|
|
104
|
+
return want.status !== 0 || !fs.existsSync(f) || !want.stdout.equals(fs.readFileSync(f));
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function reinstall({ repo, branch, install, run, attempt, files, profile }) {
|
|
109
|
+
const missing = profile.install.requires.filter((r) => !fs.existsSync(path.join(install, r)));
|
|
110
|
+
if (missing.length) return bad(`${install} is not an install of this pipeline (missing: ${missing.join(', ')})`);
|
|
111
|
+
const dest = path.join(install, profile.install.dest);
|
|
112
|
+
let prev = []; try { prev = JSON.parse(fs.readFileSync(path.join(dest, profile.install.patchRecord), 'utf8')).patches || []; } catch { prev = []; }
|
|
113
|
+
const commit = git(repo, 'rev-parse', branch).stdout.trim();
|
|
114
|
+
let wt = null;
|
|
115
|
+
try {
|
|
116
|
+
wt = worktree(repo, branch);
|
|
117
|
+
const c = expand(profile.install.command, { node: process.execPath, worktree: wt, install });
|
|
118
|
+
const r = spawnSync(c[0], c.slice(1), { cwd: wt, encoding: 'utf8' });
|
|
119
|
+
if (r.status !== 0) return bad(`the component's installer failed: ${(r.stderr || r.stdout).slice(-600)}`);
|
|
120
|
+
const mismatches = installedMismatches({ repo, branch, install, profile });
|
|
121
|
+
if (mismatches.length) return bad(`installed files differ from the branch: ${mismatches.join(', ')}`, { mismatches, installer: r.stdout });
|
|
122
|
+
const patches = [...prev, { at: new Date().toISOString(), run, attempt, branch, commit, files: files || [] }];
|
|
123
|
+
fs.writeFileSync(path.join(dest, profile.install.patchRecord), `${JSON.stringify({ note: 'this install carries repairs made by the pipeline doctor that are not in a release', patches }, null, 2)}\n`);
|
|
124
|
+
return { ok: true, why: 'reinstalled through the component\'s installer; every packaged file matches the branch', mismatches: [], installer: r.stdout, commit };
|
|
125
|
+
} finally { if (wt) drop(repo, wt); }
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// ── client code is never the doctor's ───────────────────────────────────────────────────────────────────────────────────
|
|
129
|
+
const real = (p) => { try { return fs.realpathSync(p); } catch { return null; } };
|
|
130
|
+
|
|
131
|
+
/** The repository to repair must BE the profile's component, and never a protected (client) repository or one that holds or sits inside one. */
|
|
132
|
+
function checkComponent({ repo, profile, protectedRepos }) {
|
|
133
|
+
const top = git(repo, 'rev-parse', '--show-toplevel');
|
|
134
|
+
if (top.status !== 0) return bad(`${repo} is not a git repository`);
|
|
135
|
+
const root = real(top.stdout.trim());
|
|
136
|
+
let name = null; try { name = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')).name; } catch { name = null; }
|
|
137
|
+
if (name !== profile.component.package) return bad(`${root} is not the component: its package is "${name}", the profile's is "${profile.component.package}"`);
|
|
138
|
+
for (const p of (protectedRepos || [])) {
|
|
139
|
+
const q = real(p); if (!q) continue;
|
|
140
|
+
if (root === q || root.startsWith(`${q}${path.sep}`) || q.startsWith(`${root}${path.sep}`)) return bad(`${root} is protected client code (${q}): the doctor never works there`);
|
|
141
|
+
}
|
|
142
|
+
return { ok: true, why: 'the profile\'s component, not client code', root };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Everything that identifies a repository's state — its HEAD, every ref, and its status including untracked files. */
|
|
146
|
+
function snapshotRepos(repos) {
|
|
147
|
+
const out = {};
|
|
148
|
+
for (const r of repos) {
|
|
149
|
+
if (!fs.existsSync(r)) { out[r] = { missing: true }; continue; }
|
|
150
|
+
out[r] = { head: git(r, 'rev-parse', 'HEAD').stdout.trim(), refs: git(r, 'for-each-ref', '--format=%(refname) %(objectname)').stdout, status: git(r, 'status', '--porcelain', '--untracked-files=all').stdout };
|
|
151
|
+
}
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** The protected repositories whose state differs between two snapshots, each named with what changed. */
|
|
156
|
+
function repoChanges(before, after) {
|
|
157
|
+
const out = [];
|
|
158
|
+
for (const r of Object.keys(before)) {
|
|
159
|
+
const a = before[r]; const b = after[r] || { missing: true };
|
|
160
|
+
const what = ['head', 'refs', 'status', 'missing'].filter((k) => a[k] !== b[k]);
|
|
161
|
+
if (what.length) out.push(`${r}: ${what.join(', ')} changed`);
|
|
162
|
+
}
|
|
163
|
+
return out;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
module.exports = { verifyRepair, reinstall, installedMismatches, packaged, checkComponent, snapshotRepos, repoChanges, expand, hostWrap };
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// THE RESTARTER (operator, 2026-10-20: "the doctor has to be able to restart so maybe a restart agent as well"). Not an agent and not the
|
|
3
|
+
// doctor: the relaunch of a leg is a launch, and a launch obeys the rules of every launch — the quota the operator gave, the memory a
|
|
4
|
+
// launch needs on this host, no other heavy job running — so it is deterministic, and every check it makes is in its own log
|
|
5
|
+
// (<run>/<profile.logs.dir>/restarter.jsonl). A restart is counted only when it is taken; the run's recorded launch settings are replayed by the
|
|
6
|
+
// pipeline's own resume (the profile's restart.args).
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
|
|
10
|
+
/** The jobs another launch must not share the host with (operator rule) — the rules are data (config/doctor-heavy-jobs.json), not code. */
|
|
11
|
+
function heavyJobs({ exclude, rules } = {}) {
|
|
12
|
+
const R = rules || JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', '..', 'config', 'doctor-heavy-jobs.json'), 'utf8'));
|
|
13
|
+
const skip = new Set([...(exclude || []), process.pid]);
|
|
14
|
+
// this process's ancestors are the launcher's own shell and tools, never "another job"
|
|
15
|
+
for (let p = process.ppid, n = 0; p > 1 && n < 64; n += 1) { skip.add(p); let st; try { st = fs.readFileSync(`/proc/${p}/stat`, 'utf8'); } catch { break; } p = Number(st.slice(st.lastIndexOf(')') + 2).split(' ')[1]); }
|
|
16
|
+
const out = [];
|
|
17
|
+
for (const d of fs.readdirSync('/proc').filter((x) => /^\d+$/.test(x))) {
|
|
18
|
+
const pid = Number(d); if (skip.has(pid)) continue;
|
|
19
|
+
let argv; try { argv = fs.readFileSync(`/proc/${d}/cmdline`, 'utf8').split('\0').filter(Boolean); } catch { continue; }
|
|
20
|
+
const names = argv.map((t) => path.basename(t));
|
|
21
|
+
let what = names.find((t) => R.tokens.includes(t)) || null;
|
|
22
|
+
if (!what) for (const f of R.withFlag) { const i = names.findIndex((t) => t === f.bin); if (i >= 0 && argv.slice(i + 1).includes(f.flag)) { what = f.as || `${f.bin} ${f.flag}`; break; } }
|
|
23
|
+
if (what) out.push(`${what} (pid ${pid})`);
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const availableMbNow = () => { const m = fs.readFileSync('/proc/meminfo', 'utf8').match(/MemAvailable:\s+(\d+)/); return m ? Math.floor(Number(m[1]) / 1024) : 0; };
|
|
29
|
+
|
|
30
|
+
async function restart({ state, profile, runDir, needMb, reserveMb, availableMb, heavyJobs: heavy, launch }) {
|
|
31
|
+
const logf = path.join(runDir, profile.logs.dir, 'restarter.jsonl');
|
|
32
|
+
const note = (check, ok, detail) => { fs.mkdirSync(path.dirname(logf), { recursive: true }); fs.appendFileSync(logf, `${JSON.stringify({ at: new Date().toISOString(), check, ok, detail })}\n`); state.event(`restarter-${check}`, { ok, detail }); };
|
|
33
|
+
const refuse = (check, detail) => { note(check, false, detail); return { restarted: false, why: `${check}: ${detail}` }; };
|
|
34
|
+
|
|
35
|
+
const q = state.quota();
|
|
36
|
+
if (q.left < 1) return refuse('quota', `given ${q.given}, used ${q.used} — no restart left`);
|
|
37
|
+
note('quota', true, `given ${q.given}, used ${q.used}, ${q.left} left`);
|
|
38
|
+
|
|
39
|
+
const avail = (availableMb || availableMbNow)(); const room = avail - reserveMb;
|
|
40
|
+
if (room < needMb) return refuse('memory', `${avail} MB available − ${reserveMb} MB kept for the host = ${room} MB for the run; a launch needs ${needMb} MB`);
|
|
41
|
+
note('memory', true, `${avail} MB available − ${reserveMb} MB kept = ${room} MB for the run; needs ${needMb} MB`);
|
|
42
|
+
|
|
43
|
+
const jobs = (heavy || heavyJobs)();
|
|
44
|
+
if (jobs.length) return refuse('heavy-jobs', `another heavy job is running: ${jobs.join(', ')}`);
|
|
45
|
+
note('heavy-jobs', true, 'none');
|
|
46
|
+
|
|
47
|
+
state.useRestart(); // counted when it is taken, not before
|
|
48
|
+
try {
|
|
49
|
+
const result = await launch(profile.restart.args);
|
|
50
|
+
note('launch', true, JSON.stringify(result).slice(0, 300));
|
|
51
|
+
return { restarted: true, why: 'relaunched with the profile\'s restart arguments', result };
|
|
52
|
+
} catch (e) {
|
|
53
|
+
note('launch', false, e.message);
|
|
54
|
+
return { restarted: false, why: `launch: ${e.message}` };
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
module.exports = { restart, heavyJobs };
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// THE DOCTOR'S SETTINGS, QUOTA AND ITS OWN LOGS (operator, 2026-10-20). Everything the doctor does is recorded under <run>/<profile.logs.dir>/
|
|
3
|
+
// and nowhere else: the run's own logs and ledgers are never written, so the run's cost and the doctor's can never be confused.
|
|
4
|
+
// events.jsonl append-only, one line per action (seq, at, attempt, type, data) doctor.log the same, readable — the file to tail
|
|
5
|
+
// cost.jsonl every model call the doctor made, with a running total quota.json given, used, left
|
|
6
|
+
// <n>/ one folder per repair attempt (the evidence: trigger, diagnosis, red, green, patch, reinstall, verdict)
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
|
|
10
|
+
/** The doctor's launch settings, from the run's recorded launch values, under the NAMES the pipeline's profile gives them. Anything not exactly right is the safe value. */
|
|
11
|
+
function settings(values, profile) {
|
|
12
|
+
const v = values || {}; const n = profile.settings;
|
|
13
|
+
const restarts = /^\d+$/.test(String(v[n.restarts] || '')) ? Number(v[n.restarts]) : 0;
|
|
14
|
+
const repo = v[n.repo] ? String(v[n.repo]) : null;
|
|
15
|
+
// THE SPENDING CAP of one repair session (a launch setting the operator gives): without a usable one the doctor does not spend — it is not on
|
|
16
|
+
const budget = /^\d+(\.\d+)?$/.test(String(v[n.budget] || '')) && Number(v[n.budget]) > 0 ? Number(v[n.budget]) : null;
|
|
17
|
+
const wants = v[n.on] === 'on';
|
|
18
|
+
const lacks = [!repo && `${n.repo} — the checkout of ${profile.component.package} it repairs`, !budget && `${n.budget} — a spending cap in dollars for one repair session (a positive number)`].filter(Boolean);
|
|
19
|
+
const problem = wants && lacks.length ? `${n.on}=on needs ${lacks.join(' and ')}` : null;
|
|
20
|
+
return { on: wants && !!repo && !!budget, restarts, repo, budget, problem };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const readJson = (f, dflt) => { try { return JSON.parse(fs.readFileSync(f, 'utf8')); } catch { return dflt; } };
|
|
24
|
+
const jsonl = (f) => { try { return fs.readFileSync(f, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); } catch { return []; } };
|
|
25
|
+
|
|
26
|
+
/** Opens (or re-opens, on a resume) the doctor's folder in a run directory. */
|
|
27
|
+
function open(runDir, cfg, profile) {
|
|
28
|
+
const dir = path.join(runDir, profile.logs.dir);
|
|
29
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
30
|
+
const files = { events: path.join(dir, 'events.jsonl'), log: path.join(dir, 'doctor.log'), cost: path.join(dir, 'cost.jsonl'), quota: path.join(dir, 'quota.json') };
|
|
31
|
+
let seq = jsonl(files.events).reduce((m, e) => Math.max(m, e.seq || 0), 0);
|
|
32
|
+
let total = jsonl(files.cost).reduce((m, r) => Math.max(m, r.totalUsd || 0), 0);
|
|
33
|
+
let used = (readJson(files.quota, {}) || {}).used || 0;
|
|
34
|
+
const given = cfg.restarts || 0;
|
|
35
|
+
|
|
36
|
+
const writeQuota = () => fs.writeFileSync(files.quota, `${JSON.stringify({ given, used, left: Math.max(0, given - used) })}\n`);
|
|
37
|
+
writeQuota();
|
|
38
|
+
|
|
39
|
+
function event(type, data, attempt) {
|
|
40
|
+
seq += 1;
|
|
41
|
+
const rec = { seq, at: new Date().toISOString(), attempt: attempt || 0, type, data: data || {} };
|
|
42
|
+
fs.appendFileSync(files.events, `${JSON.stringify(rec)}\n`);
|
|
43
|
+
fs.appendFileSync(files.log, `${rec.at} #${seq}${attempt ? ` [attempt ${attempt}]` : ''} ${type} ${JSON.stringify(rec.data)}\n`);
|
|
44
|
+
return rec;
|
|
45
|
+
}
|
|
46
|
+
function cost(row) {
|
|
47
|
+
total += Number(row.costUsd) || 0;
|
|
48
|
+
fs.appendFileSync(files.cost, `${JSON.stringify({ at: new Date().toISOString(), ...row, totalUsd: total })}\n`);
|
|
49
|
+
return total;
|
|
50
|
+
}
|
|
51
|
+
function useRestart() {
|
|
52
|
+
if (used >= given) { event('quota-exhausted', { given, used }); return false; }
|
|
53
|
+
used += 1; writeQuota(); event('restart-counted', { given, used });
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
function nextAttempt() {
|
|
57
|
+
const n = fs.readdirSync(dir).filter((x) => /^\d+$/.test(x)).reduce((m, x) => Math.max(m, Number(x)), 0) + 1;
|
|
58
|
+
const d = path.join(dir, String(n)); fs.mkdirSync(d, { recursive: true });
|
|
59
|
+
return { n, dir: d };
|
|
60
|
+
}
|
|
61
|
+
const attemptsMade = () => fs.readdirSync(dir).filter((x) => /^\d+$/.test(x)).length;
|
|
62
|
+
return { dir, files, event, cost, spent: () => total, useRestart, nextAttempt, attemptsMade, quota: () => ({ given, used, left: Math.max(0, given - used) }) };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
module.exports = { settings, open };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "amsd-pipeline-shared",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "The AMSD pipeline's shared components, one source for the bash pipeline and its Claude workflows: the model-call gateway (llm-handler.sh — retry, ladder, self-heal, cost) and the libraries it runs.",
|
|
5
5
|
"license": "UNLICENSED",
|
|
6
6
|
"bin": {
|