opencode-harness-kit 0.5.7 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +191 -0
- package/README.md +34 -0
- package/bin/baseline.mjs +16 -1
- package/bin/oc-harness.mjs +353 -34
- package/dist/index.js +240 -51
- package/dist/rules.d.ts +6 -0
- package/lanes.json +2 -2
- package/package.json +1 -1
- package/src/config.js +9 -1
- package/src/rules.js +303 -21
package/bin/oc-harness.mjs
CHANGED
|
@@ -64,10 +64,22 @@ const has = (f) => args.includes(f)
|
|
|
64
64
|
* the second one is the conventional one. Reading only the first form made
|
|
65
65
|
* `--profile=fleet-safe` look like a flag with a missing value, which is a
|
|
66
66
|
* confusing way to be told you got the syntax wrong.
|
|
67
|
+
*
|
|
68
|
+
* A value that is itself a flag is NOT a value. `oc-harness run --role
|
|
69
|
+
* --max-rounds 5` used to set the role to the literal string "--max-rounds",
|
|
70
|
+
* and the loop then ran `opencode run --agent --max-rounds ...`, which failed
|
|
71
|
+
* on every round and marked every task blocked. The typo was silent because
|
|
72
|
+
* the value was plausible. There is no way to spell a value that starts with
|
|
73
|
+
* `--`, and there never needs to be: no flag of this CLI takes one.
|
|
67
74
|
*/
|
|
68
75
|
const opt = (f) => {
|
|
69
76
|
const i = args.indexOf(f)
|
|
70
|
-
if (i >= 0)
|
|
77
|
+
if (i >= 0) {
|
|
78
|
+
const next = args[i + 1]
|
|
79
|
+
// A missing value, or the next flag. Both mean the flag was not given
|
|
80
|
+
// what it needs - which is a typo, not an empty value.
|
|
81
|
+
return next === undefined || next.startsWith("--") ? undefined : next
|
|
82
|
+
}
|
|
71
83
|
const inline = args.find((a) => a.startsWith(`${f}=`))
|
|
72
84
|
return inline === undefined ? undefined : inline.slice(f.length + 1)
|
|
73
85
|
}
|
|
@@ -109,7 +121,11 @@ const red = (s) => `\x1b[31m${s}\x1b[0m`
|
|
|
109
121
|
// The colour out. The human report wants it; a JSON field does not, and a
|
|
110
122
|
// script that greps the message gets escape bytes it has to know to strip.
|
|
111
123
|
const plain = (s) => String(s).replace(/\x1b\[[0-9;]*m/g, "")
|
|
112
|
-
|
|
124
|
+
// A command that must not ask. `auto` sets it: it reconciles, so a question
|
|
125
|
+
// has no good answer - the one thing it can do about the user's own file is
|
|
126
|
+
// leave it alone and say so.
|
|
127
|
+
let nonInteractive = false
|
|
128
|
+
const interactive = () => process.stdin.isTTY && !flag("--yes") && !nonInteractive
|
|
113
129
|
|
|
114
130
|
/**
|
|
115
131
|
* A yes question with a yes default. Only asked on a terminal; scripts and
|
|
@@ -499,13 +515,7 @@ function removeLegacyAgentFiles() {
|
|
|
499
515
|
for (const name of readdirSync(dir)) {
|
|
500
516
|
if (!name.endsWith(".md")) continue
|
|
501
517
|
const full = join(dir, name)
|
|
502
|
-
|
|
503
|
-
try {
|
|
504
|
-
ours = readlinkSync(full).includes(join("space-harness", "agents"))
|
|
505
|
-
} catch {
|
|
506
|
-
ours = false
|
|
507
|
-
}
|
|
508
|
-
if (!ours) continue
|
|
518
|
+
if (!isLegacyAgentLink(full)) continue
|
|
509
519
|
try {
|
|
510
520
|
rmSync(full, { force: true })
|
|
511
521
|
removed.push(name)
|
|
@@ -515,6 +525,39 @@ function removeLegacyAgentFiles() {
|
|
|
515
525
|
return removed
|
|
516
526
|
}
|
|
517
527
|
|
|
528
|
+
/**
|
|
529
|
+
* The agent directory an earlier version of this kit installed from.
|
|
530
|
+
*
|
|
531
|
+
* The kit was named `space-harness` then, and it put its agent files in
|
|
532
|
+
* `<kit>/agents/`.
|
|
533
|
+
*/
|
|
534
|
+
const LEGACY_AGENT_DIR = ["space-harness", "agents"]
|
|
535
|
+
|
|
536
|
+
/**
|
|
537
|
+
* Whether a path is a symlink into an earlier version's agent directory.
|
|
538
|
+
*
|
|
539
|
+
* The match is on the parent DIRECTORY, compared segment by segment. A plain
|
|
540
|
+
* substring test - `target.includes("space-harness/agents")` - also matched
|
|
541
|
+
* `space-harness/agents-backup/`, and this function DELETES what it matches.
|
|
542
|
+
* A user who symlinked their own agent file out of a directory with a similar
|
|
543
|
+
* name lost it, because the kit thought it had installed it.
|
|
544
|
+
*
|
|
545
|
+
* @param {string} full
|
|
546
|
+
* @returns {boolean}
|
|
547
|
+
*/
|
|
548
|
+
function isLegacyAgentLink(full) {
|
|
549
|
+
let target
|
|
550
|
+
try {
|
|
551
|
+
target = readlinkSync(full)
|
|
552
|
+
} catch {
|
|
553
|
+
return false
|
|
554
|
+
}
|
|
555
|
+
const parts = dirname(target).split(/[\\/]/)
|
|
556
|
+
const n = LEGACY_AGENT_DIR.length
|
|
557
|
+
if (parts.length < n) return false
|
|
558
|
+
return LEGACY_AGENT_DIR.every((seg, i) => parts[parts.length - n + i] === seg)
|
|
559
|
+
}
|
|
560
|
+
|
|
518
561
|
/**
|
|
519
562
|
* Replace `link` with `link` plus the agents, skills and GLOBAL.md.
|
|
520
563
|
*
|
|
@@ -726,6 +769,9 @@ async function cmdInit() {
|
|
|
726
769
|
let id = opt("--profile")
|
|
727
770
|
if (!id) {
|
|
728
771
|
if (!process.stdin.isTTY) {
|
|
772
|
+
// No question can be asked without a terminal, and picking for the user
|
|
773
|
+
// is not this command's job: `init` is the front door for a person, and
|
|
774
|
+
// `auto` is the one that runs without asking. Say what to type.
|
|
729
775
|
fail("usage: oc-harness init --profile <id> [--yes] (no terminal to ask)")
|
|
730
776
|
}
|
|
731
777
|
const def = Math.max(free.findIndex((e) => e.id === index().default), 0)
|
|
@@ -742,7 +788,7 @@ async function cmdInit() {
|
|
|
742
788
|
console.log(`\n ${stamp(`profile: ${id}`)}`)
|
|
743
789
|
await cmdInstall()
|
|
744
790
|
await cmdUse(id)
|
|
745
|
-
cmdVerify()
|
|
791
|
+
if (cmdVerify().length) process.exitCode = 1
|
|
746
792
|
if (memorySpec()) {
|
|
747
793
|
console.log(
|
|
748
794
|
`\n ${stamp(
|
|
@@ -758,6 +804,143 @@ async function cmdInit() {
|
|
|
758
804
|
console.log(`\n ${green("✓")} done. ${stamp("restart opencode, then ask it something")}\n`)
|
|
759
805
|
}
|
|
760
806
|
|
|
807
|
+
/**
|
|
808
|
+
* Bring the install back to the profile, without asking anything.
|
|
809
|
+
*
|
|
810
|
+
* The four commands that keep an install alive - refresh, install, use,
|
|
811
|
+
* verify - are each a step somebody has to remember, in the right order, with
|
|
812
|
+
* the right flags. An agent that has to keep an install working has to know
|
|
813
|
+
* all of that, and it has to know which of them are allowed to destroy
|
|
814
|
+
* something. This is one command that does the whole chain.
|
|
815
|
+
*
|
|
816
|
+
* oc-harness auto [--profile <id>] [--offline] [--json]
|
|
817
|
+
*
|
|
818
|
+
* The profile is the one the installed config names, or the default. Nothing
|
|
819
|
+
* is asked: a file of the user's that the installer would have overwritten is
|
|
820
|
+
* left alone and reported, rather than being a question with no good answer.
|
|
821
|
+
*
|
|
822
|
+
* `--yes` still means consent. A profile that may train on prompts is
|
|
823
|
+
* refused without it, exactly as `use` refuses it - the automation is about
|
|
824
|
+
* the mechanics, not about the consent.
|
|
825
|
+
*
|
|
826
|
+
* `--offline` skips the refresh, for a machine that cannot reach models.dev.
|
|
827
|
+
* `--json` prints a report a script can read, and nothing else on stdout.
|
|
828
|
+
*/
|
|
829
|
+
async function cmdAuto() {
|
|
830
|
+
const wantJson = flag("--json")
|
|
831
|
+
const idx = index()
|
|
832
|
+
const steps = []
|
|
833
|
+
const notes = []
|
|
834
|
+
|
|
835
|
+
// The profile. What was asked for, else what is installed, else the default.
|
|
836
|
+
// A profile that was ASKED FOR and cannot be loaded is an error - falling
|
|
837
|
+
// back to another one would run the install against a profile the caller
|
|
838
|
+
// did not name.
|
|
839
|
+
const requested = opt("--profile")
|
|
840
|
+
let id = null
|
|
841
|
+
if (requested) {
|
|
842
|
+
try {
|
|
843
|
+
loadProfile(requested)
|
|
844
|
+
id = requested
|
|
845
|
+
} catch (err) {
|
|
846
|
+
fail(`profile "${requested}" cannot be loaded: ${err instanceof Error ? err.message : String(err)}`, wantJson)
|
|
847
|
+
}
|
|
848
|
+
} else {
|
|
849
|
+
const active = profileIdFromConfig()
|
|
850
|
+
if (active) {
|
|
851
|
+
try {
|
|
852
|
+
loadProfile(active)
|
|
853
|
+
id = active
|
|
854
|
+
} catch (err) {
|
|
855
|
+
notes.push(
|
|
856
|
+
`the installed profile ${active} cannot be loaded (${err instanceof Error ? err.message : String(err)}); using ${idx.default}`,
|
|
857
|
+
)
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
if (!id) {
|
|
861
|
+
id = idx.default
|
|
862
|
+
try {
|
|
863
|
+
loadProfile(id)
|
|
864
|
+
} catch (err) {
|
|
865
|
+
fail(`no usable profile: ${err instanceof Error ? err.message : String(err)}`, wantJson)
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
// The steps. Their human output is silenced in JSON mode, so stdout carries
|
|
871
|
+
// the report and nothing else.
|
|
872
|
+
const realLog = console.log
|
|
873
|
+
if (wantJson) console.log = () => {}
|
|
874
|
+
const problems = []
|
|
875
|
+
try {
|
|
876
|
+
// 1. The catalogue. A model that was withdrawn has to be caught before
|
|
877
|
+
// it is used, and the profile files are the only place that lives.
|
|
878
|
+
if (flag("--offline")) {
|
|
879
|
+
steps.push({ step: "refresh", status: "skipped", detail: "--offline" })
|
|
880
|
+
} else {
|
|
881
|
+
// The refresh's findings are part of the report, not decoration. In JSON
|
|
882
|
+
// mode the refresh's own output is silenced, so a withdrawn model or a
|
|
883
|
+
// profile that could not be read used to vanish from the report - and
|
|
884
|
+
// the report said "ok": true while a profile was never checked against
|
|
885
|
+
// the live catalogue.
|
|
886
|
+
const refreshed = await cmdRefreshAll()
|
|
887
|
+
for (const g of refreshed.gone) {
|
|
888
|
+
problems.push(`profile ${g} is no longer in the live catalogue`)
|
|
889
|
+
}
|
|
890
|
+
for (const s of refreshed.skipped) {
|
|
891
|
+
problems.push(
|
|
892
|
+
`profile ${s} was skipped by the refresh, so it was not checked against the live catalogue`,
|
|
893
|
+
)
|
|
894
|
+
}
|
|
895
|
+
const findings = refreshed.gone.length + refreshed.skipped.length
|
|
896
|
+
steps.push({
|
|
897
|
+
step: "refresh",
|
|
898
|
+
status: findings ? "failed" : "ok",
|
|
899
|
+
...(findings ? { detail: `${findings} finding(s)` } : {}),
|
|
900
|
+
})
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
// 2. The behavioural layer: the plugin link, the prompts, the skills, the
|
|
904
|
+
// global rules. Without them the config means nothing.
|
|
905
|
+
await cmdInstall()
|
|
906
|
+
steps.push({ step: "install", status: "ok" })
|
|
907
|
+
|
|
908
|
+
// 3. The config, recompiled from the profile.
|
|
909
|
+
await cmdUse(id)
|
|
910
|
+
steps.push({ step: "use", status: "ok", detail: id })
|
|
911
|
+
|
|
912
|
+
// 4. The check that says whether all of that worked.
|
|
913
|
+
const verifyProblems = cmdVerify()
|
|
914
|
+
problems.push(...verifyProblems)
|
|
915
|
+
steps.push({
|
|
916
|
+
step: "verify",
|
|
917
|
+
status: verifyProblems.length ? "failed" : "ok",
|
|
918
|
+
detail: verifyProblems.length ? `${verifyProblems.length} problem(s)` : "the install is consistent",
|
|
919
|
+
})
|
|
920
|
+
} finally {
|
|
921
|
+
console.log = realLog
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
const report = {
|
|
925
|
+
ok: problems.length === 0,
|
|
926
|
+
profile: id,
|
|
927
|
+
steps,
|
|
928
|
+
problems,
|
|
929
|
+
...(notes.length ? { notes } : {}),
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
if (wantJson) {
|
|
933
|
+
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
|
|
934
|
+
} else {
|
|
935
|
+
console.log(`\n${bold("auto")} ${stamp(report.ok ? "the install is in step with the profile" : `${problems.length} problem(s)`)}\n`)
|
|
936
|
+
for (const s of steps) console.log(` ${green("✓")} ${s.step}${s.detail ? ` - ${s.detail}` : ""}`)
|
|
937
|
+
for (const n of notes) console.log(` ${yellow("!")} ${n}`)
|
|
938
|
+
for (const pr of problems) console.log(` ${red("✗")} ${pr}`)
|
|
939
|
+
console.log("")
|
|
940
|
+
}
|
|
941
|
+
if (problems.length) process.exitCode = 1
|
|
942
|
+
}
|
|
943
|
+
|
|
761
944
|
async function askChoice(question) {
|
|
762
945
|
const rl = createInterface({ input: process.stdin, output: process.stdout })
|
|
763
946
|
try {
|
|
@@ -852,13 +1035,16 @@ function linkOne(src, dest, opts = {}) {
|
|
|
852
1035
|
* the profile, an instruction path that resolves to nothing, and a set of role
|
|
853
1036
|
* agents that were never installed while the config still declared them.
|
|
854
1037
|
*
|
|
855
|
-
*
|
|
1038
|
+
* Returns the problems it found rather than exiting on them, so a caller that
|
|
1039
|
+
* has more to do - `oc-harness auto` - can carry on and report the whole
|
|
1040
|
+
* picture. The exit code is the caller's to set.
|
|
1041
|
+
*
|
|
1042
|
+
* @returns {string[]} the problems, empty when the install is consistent
|
|
856
1043
|
*/
|
|
857
1044
|
function cmdVerify() {
|
|
858
1045
|
const problems = []
|
|
859
1046
|
const notes = []
|
|
860
1047
|
const ok = []
|
|
861
|
-
|
|
862
1048
|
const activeId = profileIdFromConfig() ?? index().default
|
|
863
1049
|
let p = null
|
|
864
1050
|
try {
|
|
@@ -1001,7 +1187,11 @@ function cmdVerify() {
|
|
|
1001
1187
|
for (const name of readdirSync(dir).filter((f) => f.endsWith(".md"))) {
|
|
1002
1188
|
const where = `${scope}:${dir.replace(`${homedir()}/`, "~/")}`
|
|
1003
1189
|
const lane = name.replace(/\.md$/, "")
|
|
1004
|
-
|
|
1190
|
+
// `Object.hasOwn`, not `in`. The object is the user's parsed config, and
|
|
1191
|
+
// `in` walks the prototype chain - so an agent file named `toString.md`
|
|
1192
|
+
// or `constructor.md` matched a property every object has, and `verify`
|
|
1193
|
+
// reported a clean install as shadowing a lane that does not exist.
|
|
1194
|
+
if (Object.hasOwn(/** @type {any} */ (actual?.agent) ?? {}, lane)) {
|
|
1005
1195
|
problems.push(
|
|
1006
1196
|
`agent file ${where}/${name} shadows the ${lane} lane: an agent file replaces the config entry, so the lane would not run on its own model. Run: oc-harness install`,
|
|
1007
1197
|
)
|
|
@@ -1152,9 +1342,10 @@ function cmdVerify() {
|
|
|
1152
1342
|
|
|
1153
1343
|
if (problems.length) {
|
|
1154
1344
|
console.log(`\n ${red(`${problems.length} problem(s)`)}\n`)
|
|
1155
|
-
|
|
1345
|
+
return problems
|
|
1156
1346
|
}
|
|
1157
1347
|
console.log(`\n ${green("install is consistent with the profile")}\n`)
|
|
1348
|
+
return problems
|
|
1158
1349
|
}
|
|
1159
1350
|
|
|
1160
1351
|
/**
|
|
@@ -1525,15 +1716,27 @@ async function cmdRefreshAll() {
|
|
|
1525
1716
|
const changed = []
|
|
1526
1717
|
const gone = []
|
|
1527
1718
|
const same = []
|
|
1719
|
+
// The profiles that could not be read, and the profiles whose backup could
|
|
1720
|
+
// not be written. Both used to disappear without a word: the first was a
|
|
1721
|
+
// `continue` with no output, the second was a swallowed error - and the
|
|
1722
|
+
// summary line counted every id it was given, so a broken profile was
|
|
1723
|
+
// reported as a refreshed one.
|
|
1724
|
+
const skipped = []
|
|
1725
|
+
let checked = 0
|
|
1528
1726
|
for (const id of ids) {
|
|
1529
1727
|
const path = profilePath(id)
|
|
1530
|
-
if (!path)
|
|
1728
|
+
if (!path) {
|
|
1729
|
+
skipped.push(`${id} (no file)`)
|
|
1730
|
+
continue
|
|
1731
|
+
}
|
|
1531
1732
|
let p
|
|
1532
1733
|
try {
|
|
1533
1734
|
p = JSON.parse(readFileSync(path, "utf8"))
|
|
1534
|
-
} catch {
|
|
1735
|
+
} catch (err) {
|
|
1736
|
+
skipped.push(`${id} (${err instanceof Error ? err.message : String(err)})`)
|
|
1535
1737
|
continue
|
|
1536
1738
|
}
|
|
1739
|
+
checked++
|
|
1537
1740
|
const live = catalog?.[p.provider]?.models?.[p.model]
|
|
1538
1741
|
if (!live) {
|
|
1539
1742
|
gone.push(`${id} (${p.provider}/${p.model})`)
|
|
@@ -1554,15 +1757,31 @@ async function cmdRefreshAll() {
|
|
|
1554
1757
|
}
|
|
1555
1758
|
// A refresh overwrites a file the user may have edited by hand, and for
|
|
1556
1759
|
// an npm install that file is inside node_modules, where the next
|
|
1557
|
-
// `npm install` destroys it anyway. The backup is the only copy
|
|
1558
|
-
|
|
1760
|
+
// `npm install` destroys it anyway. The backup is the only copy - so a
|
|
1761
|
+
// backup that could not be written means the refresh must not happen.
|
|
1762
|
+
const kept = keepProfileBackup(path, readFileSync(path, "utf8"))
|
|
1763
|
+
if (!kept) {
|
|
1764
|
+
skipped.push(`${id} (the backup could not be written, so the profile was left alone)`)
|
|
1765
|
+
continue
|
|
1766
|
+
}
|
|
1559
1767
|
writeFileSync(path, `${JSON.stringify(p, null, 2)}\n`)
|
|
1560
1768
|
changed.push(`${id}: ${before.join(", ")}`)
|
|
1561
1769
|
}
|
|
1562
|
-
console.log(
|
|
1770
|
+
console.log(
|
|
1771
|
+
`\n${bold("refreshed")} ${stamp(`${checked} of ${ids.length} profile(s) against the live catalogue`)}\n`,
|
|
1772
|
+
)
|
|
1563
1773
|
for (const c of changed) console.log(` ${green("~")} ${c}`)
|
|
1564
1774
|
for (const g of gone) console.log(` ${red("!")} ${g} is no longer in the catalogue`)
|
|
1565
1775
|
for (const id of same) console.log(` ${green("✓")} ${id} already matches`)
|
|
1776
|
+
for (const s of skipped) console.log(` ${red("!")} ${s} - skipped`)
|
|
1777
|
+
if (skipped.length) {
|
|
1778
|
+
console.log("")
|
|
1779
|
+
console.log(
|
|
1780
|
+
stamp(
|
|
1781
|
+
"A profile that was skipped was not checked against the live catalogue. Its model may have been withdrawn without anyone noticing.",
|
|
1782
|
+
),
|
|
1783
|
+
)
|
|
1784
|
+
}
|
|
1566
1785
|
if (gone.length) {
|
|
1567
1786
|
console.log("")
|
|
1568
1787
|
console.log(
|
|
@@ -1572,6 +1791,15 @@ async function cmdRefreshAll() {
|
|
|
1572
1791
|
)
|
|
1573
1792
|
}
|
|
1574
1793
|
console.log("")
|
|
1794
|
+
// A skipped profile is a failure, not a footnote. `refresh --all` is run by
|
|
1795
|
+
// a job that assumes every profile was checked, and a non-zero exit is the
|
|
1796
|
+
// only way that job finds out.
|
|
1797
|
+
if (skipped.length) process.exitCode = 1
|
|
1798
|
+
// The findings come back with the command. They used to be printed and
|
|
1799
|
+
// dropped, so a caller that silences the human output - `auto --json` -
|
|
1800
|
+
// never saw them: a withdrawn model or a profile that was never checked
|
|
1801
|
+
// against the catalogue went into a report that said everything was fine.
|
|
1802
|
+
return { changed, gone, same, skipped }
|
|
1575
1803
|
}
|
|
1576
1804
|
|
|
1577
1805
|
/**
|
|
@@ -1688,6 +1916,10 @@ const BACKUP_DIR = "backups"
|
|
|
1688
1916
|
*
|
|
1689
1917
|
* @param {string} path the profile file
|
|
1690
1918
|
* @param {string} text its current contents
|
|
1919
|
+
* @returns {string | null} the backup's name, or null when it could not be
|
|
1920
|
+
* written. The caller has to treat null as a refusal to go on: a profile
|
|
1921
|
+
* the user edited by hand has no other copy, and for an npm install that
|
|
1922
|
+
* file is inside node_modules, where the next `npm install` destroys it.
|
|
1691
1923
|
*/
|
|
1692
1924
|
function keepProfileBackup(path, text) {
|
|
1693
1925
|
try {
|
|
@@ -1698,9 +1930,20 @@ function keepProfileBackup(path, text) {
|
|
|
1698
1930
|
const dir = join(CONFIG_DIR, "profile-backups")
|
|
1699
1931
|
if (!existsSync(dir)) mkdirSync(dir, { recursive: true })
|
|
1700
1932
|
const stamp = new Date().toISOString().replace(/[:.]/g, "-")
|
|
1701
|
-
|
|
1933
|
+
const name = `${basename(path, ".json")}-${stamp}.json`
|
|
1934
|
+
writeFileSync(join(dir, name), text)
|
|
1935
|
+
return name
|
|
1702
1936
|
} catch {
|
|
1703
|
-
// A backup that cannot be written is not worth failing over
|
|
1937
|
+
// A backup that cannot be written is not worth failing over - unless the
|
|
1938
|
+
// caller was about to destroy the only copy of the file it holds.
|
|
1939
|
+
//
|
|
1940
|
+
// This used to swallow the error and return nothing, and both callers
|
|
1941
|
+
// then overwrote the profile anyway. A user who had edited
|
|
1942
|
+
// `profiles/<id>.json` by hand - the retention, the roles, the fallback,
|
|
1943
|
+
// the parts the code itself calls "human judgement" - lost those edits
|
|
1944
|
+
// with no backup and no message. `keepBackup` below already had the right
|
|
1945
|
+
// shape: return null, and let the caller refuse to go on.
|
|
1946
|
+
return null
|
|
1704
1947
|
}
|
|
1705
1948
|
}
|
|
1706
1949
|
|
|
@@ -2099,8 +2342,14 @@ async function cmdRefresh(id) {
|
|
|
2099
2342
|
}
|
|
2100
2343
|
// A refresh overwrites a file the user may have edited by hand, and for
|
|
2101
2344
|
// an npm install that file is inside node_modules, where the next
|
|
2102
|
-
// `npm install` destroys it anyway. The backup is the only copy
|
|
2103
|
-
|
|
2345
|
+
// `npm install` destroys it anyway. The backup is the only copy - so a
|
|
2346
|
+
// backup that could not be written means the refresh must not happen.
|
|
2347
|
+
const kept = keepProfileBackup(path, readFileSync(path, "utf8"))
|
|
2348
|
+
if (!kept) {
|
|
2349
|
+
fail(
|
|
2350
|
+
`could not write a backup of ${path}, so the profile was left alone. Fix the permissions on ${join(CONFIG_DIR, "profile-backups")} and run it again`,
|
|
2351
|
+
)
|
|
2352
|
+
}
|
|
2104
2353
|
writeFileSync(path, `${JSON.stringify(p, null, 2)}\n`)
|
|
2105
2354
|
console.log(`\n ${green("✓")} refreshed ${path}`)
|
|
2106
2355
|
for (const line of before) console.log(` ${line}`)
|
|
@@ -2163,17 +2412,26 @@ function recordWithdrawal(model, profiles) {
|
|
|
2163
2412
|
* chain has one fewer rung, and nobody is told.
|
|
2164
2413
|
*
|
|
2165
2414
|
* @param {any} catalog
|
|
2166
|
-
* @returns {Map<string, string[]
|
|
2167
|
-
* profiles that used each one
|
|
2415
|
+
* @returns {{gone: Map<string, string[]>, skipped: {id: string, why: string}[]}}
|
|
2416
|
+
* the model ids that are gone and the profiles that used each one, and the
|
|
2417
|
+
* profiles that could not be checked at all
|
|
2168
2418
|
*/
|
|
2169
2419
|
function withdrawnModels(catalog) {
|
|
2170
2420
|
/** @type {Map<string, string[]>} the model, and the profiles that use it */
|
|
2171
2421
|
const gone = new Map()
|
|
2422
|
+
// The profiles that could not be read. They used to disappear without a
|
|
2423
|
+
// word - the check was a `continue` with no output - and the summary then
|
|
2424
|
+
// said every shipped model was still in the catalogue. So a profile that
|
|
2425
|
+
// was NEVER CHECKED was reported as a profile that was checked and found
|
|
2426
|
+
// fine. A skipped profile is a failure, not a footnote.
|
|
2427
|
+
/** @type {{id: string, why: string}[]} */
|
|
2428
|
+
const skipped = []
|
|
2172
2429
|
for (const id of listProfileIds()) {
|
|
2173
2430
|
let p
|
|
2174
2431
|
try {
|
|
2175
2432
|
p = loadProfile(id)
|
|
2176
|
-
} catch {
|
|
2433
|
+
} catch (err) {
|
|
2434
|
+
skipped.push({ id, why: err instanceof Error ? err.message : String(err) })
|
|
2177
2435
|
continue
|
|
2178
2436
|
}
|
|
2179
2437
|
for (const model of new Set([
|
|
@@ -2190,7 +2448,7 @@ function withdrawnModels(catalog) {
|
|
|
2190
2448
|
gone.set(model, users)
|
|
2191
2449
|
}
|
|
2192
2450
|
}
|
|
2193
|
-
return gone
|
|
2451
|
+
return { gone, skipped }
|
|
2194
2452
|
}
|
|
2195
2453
|
|
|
2196
2454
|
/**
|
|
@@ -2211,13 +2469,20 @@ async function cmdDoctorWatch() {
|
|
|
2211
2469
|
// times a second.
|
|
2212
2470
|
const interval = Math.max(30, positiveNumber(opt("--interval"), 300))
|
|
2213
2471
|
const seen = new Set()
|
|
2472
|
+
// The profiles that could not be checked, so the watch says them once and
|
|
2473
|
+
// not every poll.
|
|
2474
|
+
const seenSkipped = new Set()
|
|
2214
2475
|
let first = true
|
|
2215
2476
|
for (;;) {
|
|
2216
2477
|
/** @type {Map<string, string[]>} */
|
|
2217
2478
|
let gone = new Map()
|
|
2479
|
+
/** @type {{id: string, why: string}[]} */
|
|
2480
|
+
let skipped = []
|
|
2218
2481
|
try {
|
|
2219
2482
|
const catalog = await fetchCatalog()
|
|
2220
|
-
|
|
2483
|
+
const found = withdrawnModels(catalog)
|
|
2484
|
+
gone = found.gone
|
|
2485
|
+
skipped = found.skipped
|
|
2221
2486
|
} catch (err) {
|
|
2222
2487
|
// A failed fetch is NOT an empty catalogue. Treating it as one
|
|
2223
2488
|
// announced every missing model as "is back", emptied the seen-set,
|
|
@@ -2236,6 +2501,14 @@ async function cmdDoctorWatch() {
|
|
|
2236
2501
|
console.log(stamp(` fix: oc-harness doctor && oc-harness refresh <profile>`))
|
|
2237
2502
|
recordWithdrawal(m, users.join(","))
|
|
2238
2503
|
}
|
|
2504
|
+
for (const s of skipped) {
|
|
2505
|
+
if (seenSkipped.has(s.id)) continue
|
|
2506
|
+
seenSkipped.add(s.id)
|
|
2507
|
+
console.error(red(` ${s.id} could not be checked against the catalogue: ${s.why}`))
|
|
2508
|
+
console.error(
|
|
2509
|
+
stamp(" a profile that was skipped was not checked. Its model may have been withdrawn without anyone noticing."),
|
|
2510
|
+
)
|
|
2511
|
+
}
|
|
2239
2512
|
if (!first) {
|
|
2240
2513
|
for (const m of seen) {
|
|
2241
2514
|
if (!current.has(m)) {
|
|
@@ -2253,6 +2526,15 @@ async function cmdDoctorWatch() {
|
|
|
2253
2526
|
console.error(stamp(` recorded in ${join(CONFIG_DIR, WITHDRAWN_LOG)}`))
|
|
2254
2527
|
process.exit(1)
|
|
2255
2528
|
}
|
|
2529
|
+
// A profile that could not be read is a profile that was not checked.
|
|
2530
|
+
// The run above says "every shipped model is still in the catalogue",
|
|
2531
|
+
// and that sentence is only true of the profiles it actually looked at.
|
|
2532
|
+
if (skipped.length) {
|
|
2533
|
+
console.error(
|
|
2534
|
+
red(` ${skipped.length} profile(s) could not be checked: ${skipped.map((s) => s.id).join(", ")}`),
|
|
2535
|
+
)
|
|
2536
|
+
process.exit(1)
|
|
2537
|
+
}
|
|
2256
2538
|
console.log(green(" every shipped model is still in the catalogue"))
|
|
2257
2539
|
return
|
|
2258
2540
|
}
|
|
@@ -2701,7 +2983,12 @@ function cmdAgents() {
|
|
|
2701
2983
|
}
|
|
2702
2984
|
const name = file.split("/").pop().replace(/\.md$/, "")
|
|
2703
2985
|
const front = stripFrontmatter(text)
|
|
2704
|
-
|
|
2986
|
+
// `Object.hasOwn`, not `in`. `in` walks the prototype chain, so an agent
|
|
2987
|
+
// file named `toString.md` or `constructor.md` matched a property every
|
|
2988
|
+
// object has - and the file was reported as shadowing a lane that does not
|
|
2989
|
+
// exist, and its missing model was never reported either. `verify` had the
|
|
2990
|
+
// same check and the same fix; this one was left behind.
|
|
2991
|
+
if (Object.hasOwn(/** @type {any} */ (config.agent) ?? {}, name)) {
|
|
2705
2992
|
console.log(
|
|
2706
2993
|
` ${yellow("!")} ${name.padEnd(14)} ${yellow("shadows a lane in the config")}`,
|
|
2707
2994
|
)
|
|
@@ -2733,7 +3020,10 @@ function cmdAgents() {
|
|
|
2733
3020
|
*/
|
|
2734
3021
|
function fail(msg, wantJson = false) {
|
|
2735
3022
|
if (wantJson) {
|
|
2736
|
-
|
|
3023
|
+
// `process.stdout.write`, not `console.log`: a JSON command silences
|
|
3024
|
+
// `console.log` so the report is the only thing on stdout, and the error
|
|
3025
|
+
// has to survive that.
|
|
3026
|
+
process.stdout.write(`${JSON.stringify({ ok: false, error: plain(msg) }, null, 2)}\n`)
|
|
2737
3027
|
console.error(`\n ${red("error")} ${msg}\n`)
|
|
2738
3028
|
process.exit(1)
|
|
2739
3029
|
}
|
|
@@ -2758,9 +3048,29 @@ async function cmdRun() {
|
|
|
2758
3048
|
|
|
2759
3049
|
const { runLoop } = await import("../src/driver/run.mjs")
|
|
2760
3050
|
|
|
3051
|
+
// A role that is not a lane is a typo, and the loop would fail every round
|
|
3052
|
+
// on it. It is refused here, while the answer is still "that is not a lane".
|
|
3053
|
+
//
|
|
3054
|
+
// A `--role` with no value is refused too, rather than falling back to the
|
|
3055
|
+
// default. `oc-harness run --role --max-rounds 5` used to set the role to
|
|
3056
|
+
// the literal string "--max-rounds" - and the loop then ran
|
|
3057
|
+
// `opencode run --agent --max-rounds ...`, which failed on every round and
|
|
3058
|
+
// marked every task blocked. A typo that quietly becomes the default is
|
|
3059
|
+
// still a typo that quietly becomes the wrong agent.
|
|
3060
|
+
const lanes = laneNames()
|
|
3061
|
+
const roleGiven = args.includes("--role") || args.some((a) => a.startsWith("--role="))
|
|
3062
|
+
const roleValue = opt("--role")
|
|
3063
|
+
if (roleGiven && roleValue === undefined) {
|
|
3064
|
+
fail(`--role needs a lane: ${lanes.join(", ")}`)
|
|
3065
|
+
}
|
|
3066
|
+
const role = roleValue ?? "build"
|
|
3067
|
+
if (!lanes.includes(role)) {
|
|
3068
|
+
fail(`--role ${role} is not one of ${lanes.join(", ")}`)
|
|
3069
|
+
}
|
|
3070
|
+
|
|
2761
3071
|
const result = runLoop({
|
|
2762
3072
|
cwd: process.cwd(),
|
|
2763
|
-
role
|
|
3073
|
+
role,
|
|
2764
3074
|
maxRounds,
|
|
2765
3075
|
budgetMinutes,
|
|
2766
3076
|
})
|
|
@@ -2811,8 +3121,13 @@ try {
|
|
|
2811
3121
|
case "init":
|
|
2812
3122
|
await cmdInit()
|
|
2813
3123
|
break
|
|
3124
|
+
case "auto":
|
|
3125
|
+
await cmdAuto()
|
|
3126
|
+
break
|
|
2814
3127
|
case "verify":
|
|
2815
|
-
|
|
3128
|
+
// The exit code is the gate. `verify` runs as the last step of the
|
|
3129
|
+
// install job in CI, and a verify that cannot fail is not a gate.
|
|
3130
|
+
if (cmdVerify().length) process.exitCode = 1
|
|
2816
3131
|
break
|
|
2817
3132
|
case "agents":
|
|
2818
3133
|
cmdAgents()
|
|
@@ -2847,6 +3162,8 @@ try {
|
|
|
2847
3162
|
|
|
2848
3163
|
list show profiles with retention and caveats
|
|
2849
3164
|
init pick a free profile, then install + use + verify in one go
|
|
3165
|
+
auto refresh + install + use + verify, asking nothing - the
|
|
3166
|
+
unattended reconcile, for an agent or a job
|
|
2850
3167
|
use <id> compile a profile into ${CONFIG_FILE}
|
|
2851
3168
|
show <id> print the resolved lane plan and derived numbers
|
|
2852
3169
|
refresh <id> update capability fields from models.dev
|
|
@@ -2874,8 +3191,10 @@ try {
|
|
|
2874
3191
|
--strict doctor: fail on perishable catalogue drift too
|
|
2875
3192
|
--watch doctor: keep watching the catalogue, report what disappeared
|
|
2876
3193
|
--check-models doctor: one run, non-zero when a shipped model is gone
|
|
2877
|
-
--json doctor, show, lanes: print the machine-readable form,
|
|
2878
|
-
nothing else
|
|
3194
|
+
--json doctor, show, lanes, auto: print the machine-readable form,
|
|
3195
|
+
and nothing else
|
|
3196
|
+
--offline auto: skip the refresh, for a machine that cannot reach
|
|
3197
|
+
models.dev
|
|
2879
3198
|
--hook install: also install the pre-commit hook into this
|
|
2880
3199
|
project. It is the project's call, so it has to be asked
|
|
2881
3200
|
for.
|