opencode-harness-kit 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GLOBAL.md +110 -0
- package/LICENSE +21 -0
- package/README.md +464 -2
- package/agents/build.md +66 -0
- package/agents/locate.md +48 -0
- package/agents/plan.md +61 -0
- package/agents/reviewer.md +62 -0
- package/agents/verify.md +54 -0
- package/bin/baseline.mjs +367 -0
- package/bin/git-hooks/pre-commit +30 -0
- package/bin/oc-harness.mjs +1025 -0
- package/dist/index.js +1038 -0
- package/docs/KNOWN-LIMITS.md +188 -0
- package/docs/RESEARCH.md +210 -0
- package/docs/demo/demo.gif +0 -0
- package/docs/demo/demo.mp4 +0 -0
- package/docs/demo/render.py +146 -0
- package/examples/muse-spark-1.3-contributor-free.json +41 -0
- package/package.json +81 -4
- package/profiles/deepseek-v4-flash-free.json +41 -0
- package/profiles/index.json +78 -0
- package/profiles/longcat-2.5-preview-free.json +41 -0
- package/profiles/muse-split-free.json +64 -0
- package/profiles/schema.json +161 -0
- package/profiles/space-bunny-free.json +42 -0
- package/profiles/x-preview-f-free.json +41 -0
- package/retention.yml +110 -0
- package/skills/space-build/SKILL.md +54 -0
- package/skills/space-finish/SKILL.md +63 -0
- package/skills/space-plan/SKILL.md +101 -0
- package/skills/space-plan/reference/task-template.md +47 -0
- package/skills/space-review/SKILL.md +53 -0
- package/src/config.js +358 -0
- package/src/effort.js +98 -0
- package/src/guard.ts +59 -0
- package/src/index.ts +80 -0
- package/src/profile.js +484 -0
- package/src/prune.ts +258 -0
- package/src/router.ts +192 -0
- package/src/rules.js +481 -0
- package/src/telemetry.ts +98 -0
package/agents/locate.md
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Finds where something lives and reports file:line evidence. Use before planning or changing anything, to close a gap in what is already known.
|
|
3
|
+
mode: subagent
|
|
4
|
+
model: opencode/space-bunny-free
|
|
5
|
+
variant: low
|
|
6
|
+
steps: 12
|
|
7
|
+
permission:
|
|
8
|
+
edit: deny
|
|
9
|
+
bash: deny
|
|
10
|
+
grep: allow
|
|
11
|
+
read: allow
|
|
12
|
+
glob: allow
|
|
13
|
+
list: allow
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
You answer one question about the code, with evidence. You do not propose fixes.
|
|
17
|
+
|
|
18
|
+
## Locate before you read
|
|
19
|
+
|
|
20
|
+
On the measured install, `read` ran 2,966 times against 135 `grep` and 114
|
|
21
|
+
`glob` calls: about twelve whole-file reads for every search. That is the
|
|
22
|
+
single most expensive habit in this setup, so the method below is not optional.
|
|
23
|
+
|
|
24
|
+
1. `grep` and `glob` first, to get candidate files and line numbers.
|
|
25
|
+
2. `read` with an `offset`/`limit` for the range you actually need.
|
|
26
|
+
3. Read a file whole only when you are going to edit it, or when the range
|
|
27
|
+
boundary matters and you need the surrounding structure.
|
|
28
|
+
|
|
29
|
+
Reading an entire file to find out whether a function exists is the mistake this
|
|
30
|
+
agent exists to stop.
|
|
31
|
+
|
|
32
|
+
## Rules
|
|
33
|
+
|
|
34
|
+
- You cannot run anything. If answering needs a command, say which one and let
|
|
35
|
+
someone else run it.
|
|
36
|
+
- Every claim needs `file:line`. A claim without a location is not a finding.
|
|
37
|
+
- If you did not read it, you did not find it. Do not guess a path.
|
|
38
|
+
- Do not design. If you notice a problem, report it as an observation with its
|
|
39
|
+
location; do not propose the fix.
|
|
40
|
+
- "This does not exist" is a real answer. Say it plainly; it saves the caller
|
|
41
|
+
more time than a plausible guess would.
|
|
42
|
+
- Stop when you can answer. Breadth is not the goal.
|
|
43
|
+
|
|
44
|
+
## Report
|
|
45
|
+
|
|
46
|
+
Under 200 words. Findings as `file:line — what it does`, ordered by relevance.
|
|
47
|
+
If you ran more than about eight searches without converging, say so and name
|
|
48
|
+
the two places you would look next.
|
package/agents/plan.md
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Plans work before code exists. Use when a task is vague, spans more than a few files, or two people could disagree about what "done" means.
|
|
3
|
+
mode: primary
|
|
4
|
+
model: opencode/space-bunny-free
|
|
5
|
+
variant: max
|
|
6
|
+
steps: 25
|
|
7
|
+
permission:
|
|
8
|
+
edit: deny
|
|
9
|
+
bash:
|
|
10
|
+
"*": deny
|
|
11
|
+
"git status *": allow
|
|
12
|
+
"git log *": allow
|
|
13
|
+
"git diff *": allow
|
|
14
|
+
"git -C * status*": allow
|
|
15
|
+
"git -C * log*": allow
|
|
16
|
+
"git -C * diff*": allow
|
|
17
|
+
"grep*": allow
|
|
18
|
+
"rg*": allow
|
|
19
|
+
"find*": allow
|
|
20
|
+
"cat*": allow
|
|
21
|
+
"ls*": allow
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
You turn a request into a plan someone else can execute without asking you
|
|
25
|
+
anything. You do not write the code.
|
|
26
|
+
|
|
27
|
+
## What to produce
|
|
28
|
+
|
|
29
|
+
1. **Goal** — one sentence, in the caller's words. If you cannot write it, the
|
|
30
|
+
request is not understood yet.
|
|
31
|
+
2. **Constraints** — what must not change, what must not be touched.
|
|
32
|
+
3. **Tasks** — each one small enough to finish in a few minutes, and each with:
|
|
33
|
+
- the exact file paths to change
|
|
34
|
+
- what the change is, concretely, not "handle errors"
|
|
35
|
+
- how to verify it: the exact command, and what passing looks like
|
|
36
|
+
4. **Risks** — the things most likely to go wrong, and the cheapest way to find
|
|
37
|
+
out early.
|
|
38
|
+
|
|
39
|
+
## Rules
|
|
40
|
+
|
|
41
|
+
- Read before you plan, but read narrowly. Grep the file names and read the
|
|
42
|
+
actual functions with an offset. A plan built on assumed file paths is worse
|
|
43
|
+
than no plan, and reading a whole file to check one function wastes context
|
|
44
|
+
the build step will need.
|
|
45
|
+
- No task may be "and then the rest". If it cannot be stated in one sentence
|
|
46
|
+
with a verify command, split it.
|
|
47
|
+
- Order tasks so each one leaves the repository working. A plan whose middle
|
|
48
|
+
step leaves tests red is a plan nobody will finish.
|
|
49
|
+
- If the request needs a decision you cannot make from the code and the
|
|
50
|
+
request, stop and ask **one** specific question, not five. Ask only when a
|
|
51
|
+
wrong guess costs real money, breaks the build irreparably, publishes
|
|
52
|
+
something, or is irreversible. Everything else is your call, and you make it.
|
|
53
|
+
- If the request is too small to need a plan, say that in one line and name the
|
|
54
|
+
single edit instead.
|
|
55
|
+
|
|
56
|
+
## Report
|
|
57
|
+
|
|
58
|
+
Markdown. No preamble, no restating the request.
|
|
59
|
+
|
|
60
|
+
See also the `space-plan` skill for the full plan → build → review → finish
|
|
61
|
+
loop and the shared task-file contract.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Reviews work in two passes without editing. Use after a build step, before a commit or a release decision.
|
|
3
|
+
mode: subagent
|
|
4
|
+
model: opencode/space-bunny-free
|
|
5
|
+
variant: max
|
|
6
|
+
steps: 15
|
|
7
|
+
permission:
|
|
8
|
+
edit: deny
|
|
9
|
+
bash:
|
|
10
|
+
"*": deny
|
|
11
|
+
"git diff *": allow
|
|
12
|
+
"git log *": allow
|
|
13
|
+
"git show*": allow
|
|
14
|
+
"git status *": allow
|
|
15
|
+
"git -C * diff*": allow
|
|
16
|
+
"git -C * log*": allow
|
|
17
|
+
"grep*": allow
|
|
18
|
+
"cat*": allow
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
You review. You never fix, because a reviewer who edits stops being a reviewer.
|
|
22
|
+
|
|
23
|
+
## Pass 1, does it do what was asked
|
|
24
|
+
|
|
25
|
+
Compare the diff against the task it came from. Ignore style entirely.
|
|
26
|
+
|
|
27
|
+
- Every requirement in the task met? Name any that is not, with `file:line`.
|
|
28
|
+
- Anything changed that the task did not ask for?
|
|
29
|
+
- Does the verify command the task named actually exist, and does it pass?
|
|
30
|
+
- Any file claimed as changed that is not in the diff?
|
|
31
|
+
|
|
32
|
+
A failure here is a blocker. Report it and stop.
|
|
33
|
+
|
|
34
|
+
## Pass 2, is it any good
|
|
35
|
+
|
|
36
|
+
Only if pass 1 is clean.
|
|
37
|
+
|
|
38
|
+
- Duplication that will bite on the next change
|
|
39
|
+
- Error paths that swallow the reason
|
|
40
|
+
- Edge cases the tests do not cover, named specifically
|
|
41
|
+
- Names that lie about what they hold
|
|
42
|
+
- Dead code, commented-out code, debug leftovers
|
|
43
|
+
|
|
44
|
+
## Rules
|
|
45
|
+
|
|
46
|
+
- Every finding needs `file:line` and a concrete failure it would cause. A
|
|
47
|
+
finding without a consequence is a preference, not a defect; label it as one
|
|
48
|
+
or drop it.
|
|
49
|
+
- Severity is real: `blocker`, `should-fix`, `nit`. Do not inflate.
|
|
50
|
+
- Do not propose a rewrite. Name the smallest change that removes the defect.
|
|
51
|
+
- Read the diff, not the whole file. Do not go hunting for issues outside the
|
|
52
|
+
change you were asked about.
|
|
53
|
+
- Silence is a valid result. If the diff is clean, say so plainly rather than
|
|
54
|
+
inventing something to report.
|
|
55
|
+
|
|
56
|
+
## Report
|
|
57
|
+
|
|
58
|
+
Two sections, pass 1 then pass 2. If pass 1 blocked, pass 2 does not run and
|
|
59
|
+
you say why.
|
|
60
|
+
|
|
61
|
+
See also the `space-review` skill for the full plan → build → review → finish
|
|
62
|
+
loop and the shared task-file contract.
|
package/agents/verify.md
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Runs the project's checks and reports results. Use before claiming a task is done.
|
|
3
|
+
mode: subagent
|
|
4
|
+
model: opencode/space-bunny-free
|
|
5
|
+
variant: low
|
|
6
|
+
steps: 20
|
|
7
|
+
permission:
|
|
8
|
+
edit: deny
|
|
9
|
+
bash:
|
|
10
|
+
"*": ask
|
|
11
|
+
"git status *": allow
|
|
12
|
+
"git diff *": allow
|
|
13
|
+
"mkdir*": allow
|
|
14
|
+
"ps*": allow
|
|
15
|
+
"python3 -m pytest*": allow
|
|
16
|
+
"pytest*": allow
|
|
17
|
+
"ruff*": allow
|
|
18
|
+
"mypy*": allow
|
|
19
|
+
"black*": allow
|
|
20
|
+
"npm test*": allow
|
|
21
|
+
"npm run *": allow
|
|
22
|
+
"npx tsc*": allow
|
|
23
|
+
"cargo test*": allow
|
|
24
|
+
"go test*": allow
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
You run checks. You do not fix them.
|
|
28
|
+
|
|
29
|
+
## Method
|
|
30
|
+
|
|
31
|
+
1. Find the project's own check commands. Look in the README, `Makefile`,
|
|
32
|
+
`package.json` scripts, or CI config. Prefer what the project already uses
|
|
33
|
+
over anything you invent.
|
|
34
|
+
2. Run them. Report what happened.
|
|
35
|
+
3. If a check fails, read the failure and reduce it to a short list: the file,
|
|
36
|
+
the line, and the message. Do not fix it.
|
|
37
|
+
|
|
38
|
+
## Rules
|
|
39
|
+
|
|
40
|
+
- Paste real output. If you did not run it, say you did not run it.
|
|
41
|
+
- One failing check is a failure. Do not report a subset as if it were the
|
|
42
|
+
whole suite.
|
|
43
|
+
- Do not add, remove, or skip a test to make a suite pass. If a test looks
|
|
44
|
+
wrong, report that as a finding and leave it failing.
|
|
45
|
+
- Exit code matters: a command that printed errors and exited 0 has not passed.
|
|
46
|
+
- If a check cannot run, say why: missing dependency, no such script. That is a
|
|
47
|
+
finding too.
|
|
48
|
+
- Truncated output is not a pass. If the runner cut the output off before the
|
|
49
|
+
summary line, say so rather than inferring success.
|
|
50
|
+
|
|
51
|
+
## Report
|
|
52
|
+
|
|
53
|
+
A table of command, exit code, and one line of the failure if any. Then the
|
|
54
|
+
short failure list. Nothing else.
|
package/bin/baseline.mjs
ADDED
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// baseline.mjs - measure a real OpenCode install so pruning thresholds are
|
|
3
|
+
// grounded in numbers instead of guesses.
|
|
4
|
+
//
|
|
5
|
+
// node bin/baseline.mjs human-readable report
|
|
6
|
+
// node bin/baseline.mjs --json > out.json machine-readable
|
|
7
|
+
// node bin/baseline.mjs --db /path/to.db override the database
|
|
8
|
+
//
|
|
9
|
+
// Node 22 has no node:sqlite, so we shell out to the sqlite3 CLI in
|
|
10
|
+
// read-only mode. Everything is aggregated in SQL, nothing is buffered twice.
|
|
11
|
+
|
|
12
|
+
import { execFileSync } from "node:child_process"
|
|
13
|
+
import { existsSync } from "node:fs"
|
|
14
|
+
import { homedir } from "node:os"
|
|
15
|
+
import { join } from "node:path"
|
|
16
|
+
|
|
17
|
+
const args = process.argv.slice(2)
|
|
18
|
+
const wantJson = args.includes("--json")
|
|
19
|
+
const dbArg = args.indexOf("--db")
|
|
20
|
+
const DB = dbArg >= 0 ? args[dbArg + 1] : join(homedir(), ".local/share/opencode/opencode.db")
|
|
21
|
+
|
|
22
|
+
if (!existsSync(DB)) {
|
|
23
|
+
console.error(`opencode database not found: ${DB}`)
|
|
24
|
+
console.error("pass --db <path> or run opencode at least once first")
|
|
25
|
+
process.exit(1)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function query(sql) {
|
|
29
|
+
const out = execFileSync("sqlite3", ["-json", `file:${DB}?mode=ro`, sql], {
|
|
30
|
+
encoding: "utf8",
|
|
31
|
+
maxBuffer: 256 * 1024 * 1024,
|
|
32
|
+
}).trim()
|
|
33
|
+
return out ? JSON.parse(out) : []
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const ASSISTANT = `json_extract(data,'$.role')='assistant'`
|
|
37
|
+
|
|
38
|
+
// Approximate percentile via row_number: pick the value at rank p*total.
|
|
39
|
+
function percentile(expr, p) {
|
|
40
|
+
const rows = query(
|
|
41
|
+
`select v from (
|
|
42
|
+
select ${expr} as v, row_number() over (order by ${expr}) as rn,
|
|
43
|
+
count(*) over () as total
|
|
44
|
+
from message where ${ASSISTANT} and ${expr} is not null
|
|
45
|
+
) where rn = cast(max(cast(total * ${p} as integer), 1) as integer)`,
|
|
46
|
+
)
|
|
47
|
+
return rows[0]?.v ?? null
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// NOTE: SQLite's OR is strictly logical and yields 0/1, not the first truthy
|
|
51
|
+
// operand like MySQL. `sum(x or 0)` therefore sums booleans. Use coalesce.
|
|
52
|
+
const IN = "coalesce(json_extract(data,'$.tokens.input'), 0)"
|
|
53
|
+
const CREAD = "coalesce(json_extract(data,'$.tokens.cache.read'), 0)"
|
|
54
|
+
const CTX = `(${IN} + ${CREAD})`
|
|
55
|
+
const OUT = "coalesce(json_extract(data,'$.tokens.output'), 0)"
|
|
56
|
+
const REA = "coalesce(json_extract(data,'$.tokens.reasoning'), 0)"
|
|
57
|
+
|
|
58
|
+
const totals = query(
|
|
59
|
+
`select
|
|
60
|
+
(select count(*) from session) as sessions,
|
|
61
|
+
(select count(distinct date(time_created/1000,'unixepoch','localtime')) from session) as days,
|
|
62
|
+
count(*) as messages,
|
|
63
|
+
sum(${IN}) as input, sum(${OUT}) as output,
|
|
64
|
+
sum(${REA}) as reasoning, sum(${CREAD}) as cache_read,
|
|
65
|
+
(select count(*) from session where time_compacting is not null) as compacted,
|
|
66
|
+
(select coalesce(sum(cost),0) from session) as cost
|
|
67
|
+
from message where ${ASSISTANT}`,
|
|
68
|
+
)[0]
|
|
69
|
+
|
|
70
|
+
const ctx = {
|
|
71
|
+
p50: percentile(CTX, 0.5),
|
|
72
|
+
p90: percentile(CTX, 0.9),
|
|
73
|
+
p99: percentile(CTX, 0.99),
|
|
74
|
+
max: query(`select max(${CTX}) as v from message where ${ASSISTANT}`)[0]?.v ?? 0,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const freshInputP50 = percentile(IN, 0.5)
|
|
78
|
+
|
|
79
|
+
const finish = query(
|
|
80
|
+
`select coalesce(json_extract(data,'$.finish'),'(none)') as finish, count(*) as n
|
|
81
|
+
from message where ${ASSISTANT} group by 1 order by n desc`,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
const byModel = query(
|
|
85
|
+
`select json_extract(data,'$.providerID') || '/' || json_extract(data,'$.modelID') as model,
|
|
86
|
+
coalesce(json_extract(data,'$.variant'),'(default)') as variant,
|
|
87
|
+
count(*) as messages,
|
|
88
|
+
sum(${OUT}) as output, sum(${REA}) as reasoning,
|
|
89
|
+
sum(${CREAD}) as cache_read, sum(${IN}) as input
|
|
90
|
+
from message where ${ASSISTANT}
|
|
91
|
+
group by model, variant order by cache_read desc`,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
// Median seconds per assistant request, per model+variant. Effort tiering only
|
|
95
|
+
// pays off if max is actually slower than the tiers below it.
|
|
96
|
+
const DURATION =
|
|
97
|
+
"(json_extract(data,'$.time.completed') - json_extract(data,'$.time.created')) / 1000.0"
|
|
98
|
+
const MODEL_ID = "json_extract(data,'$.providerID') || '/' || json_extract(data,'$.modelID')"
|
|
99
|
+
const VARIANT = "coalesce(json_extract(data,'$.variant'),'(default)')"
|
|
100
|
+
// Median via row_number. SQLite's `/` on integers truncates, so `total/2`
|
|
101
|
+
// picks rank floor(n/2): undershooting for odd n (n=5 gave rank 2 instead of
|
|
102
|
+
// 3) and returning the lower middle rank instead of the average for even n.
|
|
103
|
+
// The (total+1)/2 rank is the true median for odd n; even n averages ranks
|
|
104
|
+
// n/2 and n/2+1.
|
|
105
|
+
const medianLatency = query(
|
|
106
|
+
`select model, variant, printf('%.1f', avg(ms)) as median_ms from (
|
|
107
|
+
select model, variant, ms, rn, total from (
|
|
108
|
+
select model, variant, ms,
|
|
109
|
+
row_number() over (partition by model, variant order by ms) as rn,
|
|
110
|
+
count(*) over (partition by model, variant) as total
|
|
111
|
+
from (select ${MODEL_ID} as model, ${VARIANT} as variant, ${DURATION} as ms
|
|
112
|
+
from message
|
|
113
|
+
where ${ASSISTANT} and ${DURATION} is not null and ${DURATION} > 0)
|
|
114
|
+
)
|
|
115
|
+
) where rn = cast((total + 1) / 2 as integer)
|
|
116
|
+
or (total % 2 = 0 and rn = cast((total + 2) / 2 as integer))
|
|
117
|
+
group by model, variant`,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
// Share of stored message bytes by part type - the reason context bloats.
|
|
121
|
+
const partBytes = query(
|
|
122
|
+
`select json_extract(data,'$.type') as type, count(*) as n, sum(length(data)) as bytes
|
|
123
|
+
from part group by 1 order by bytes desc`,
|
|
124
|
+
)
|
|
125
|
+
const partTotal = partBytes.reduce((a, r) => a + (r.bytes ?? 0), 0) || 1
|
|
126
|
+
|
|
127
|
+
// Permission pressure: how many historical bash commands would prompt.
|
|
128
|
+
const BASH = `json_extract(data,'$.type')='tool' and json_extract(data,'$.tool')='bash'`
|
|
129
|
+
const bashRows = query(
|
|
130
|
+
`select json_extract(data,'$.state.input.command') as cmd from part where ${BASH}`,
|
|
131
|
+
)
|
|
132
|
+
const cmds = bashRows.map((r) => (r.cmd ?? "").trim()).filter(Boolean)
|
|
133
|
+
|
|
134
|
+
// Approximation of opencode's permission glob: `X *` means "X " followed by
|
|
135
|
+
// anything, so it does NOT match a bare `X`; the `X*` form covers both.
|
|
136
|
+
// [\s\S] rather than . because stored commands often embed newlines
|
|
137
|
+
// (heredocs, python3 -c "..."), and `.` does not cross a newline.
|
|
138
|
+
function glob(cmd, pat) {
|
|
139
|
+
const re = new RegExp(
|
|
140
|
+
"^" +
|
|
141
|
+
pat.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, "[\\s\\S]*") +
|
|
142
|
+
"$",
|
|
143
|
+
)
|
|
144
|
+
return re.test(cmd)
|
|
145
|
+
}
|
|
146
|
+
const allowed = (c) => kitAllowlist.some((p) => glob(c, p))
|
|
147
|
+
const topAskers = () => {
|
|
148
|
+
const heads = new Map()
|
|
149
|
+
for (const c of cmds) {
|
|
150
|
+
if (allowed(c)) continue
|
|
151
|
+
const p = c.split(/\s+/)
|
|
152
|
+
let h = p[0] ?? "?"
|
|
153
|
+
if (["git", "npm", "bun", "uv", "ssh", "curl", "python3", "npx"].includes(h) && p[1]) {
|
|
154
|
+
h += " " + p[1]
|
|
155
|
+
}
|
|
156
|
+
heads.set(h, (heads.get(h) ?? 0) + 1)
|
|
157
|
+
}
|
|
158
|
+
return [...heads.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20)
|
|
159
|
+
.map(([head, count]) => ({ head, count }))
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Imported, not copied. This file used to keep its own transcription of the
|
|
163
|
+
// allowlist, which meant the number it reported was whatever the list looked
|
|
164
|
+
// like when the copy was last edited. Every tuning decision in this project is
|
|
165
|
+
// supposed to rest on this measurement, so it has to measure the real list.
|
|
166
|
+
const { BASH_ALLOWLIST } = await import(new URL("../src/config.js", import.meta.url))
|
|
167
|
+
const kitAllowlist = BASH_ALLOWLIST
|
|
168
|
+
const currentAllow = [
|
|
169
|
+
"git status *", "git log *", "git diff *", "grep *",
|
|
170
|
+
"python3 src/main.py *", "python3 -m pytest *",
|
|
171
|
+
]
|
|
172
|
+
const wouldAskNow = cmds.filter((c) => !currentAllow.some((p) => glob(c, p))).length
|
|
173
|
+
const wouldAskKit = cmds.filter((c) => !allowed(c)).length
|
|
174
|
+
|
|
175
|
+
// Cost if the same work had run on a paid route (reference rates, per 1M).
|
|
176
|
+
const RATE = { input: 0.15, output: 0.6, cacheRead: 0.03 }
|
|
177
|
+
const paidEquivalent =
|
|
178
|
+
((totals.input ?? 0) / 1e6) * RATE.input +
|
|
179
|
+
((totals.output + (totals.reasoning ?? 0)) / 1e6) * RATE.output +
|
|
180
|
+
((totals.cache_read ?? 0) / 1e6) * RATE.cacheRead
|
|
181
|
+
|
|
182
|
+
// Truncation pressure: how many tool results were cut by `tool_output`, and how
|
|
183
|
+
// much of that turned into an extra round trip.
|
|
184
|
+
//
|
|
185
|
+
// This exists because the kit once shipped max_lines 200 / max_bytes 8192 on
|
|
186
|
+
// the reasoning that tight truncation "saves context", and a later install
|
|
187
|
+
// measured 415 of 2,959 reads exceeding 8 KB against 418 reads issued with an
|
|
188
|
+
// `offset`. Truncation causes the re-read roughly one for one, and at the
|
|
189
|
+
// measured ~17s median per request that round trip is more expensive than the
|
|
190
|
+
// tokens it saves. Re-run this before changing these numbers again.
|
|
191
|
+
const TOOL = `json_extract(data,'$.type')='tool'`
|
|
192
|
+
const readTruncation = query(
|
|
193
|
+
`select
|
|
194
|
+
count(*) as reads,
|
|
195
|
+
sum(case when length(coalesce(json_extract(data,'$.state.output'),'')) > 8192 then 1 else 0 end) as over_8k,
|
|
196
|
+
sum(case when length(coalesce(json_extract(data,'$.state.output'),'')) > 24576 then 1 else 0 end) as over_24k,
|
|
197
|
+
sum(case when length(coalesce(json_extract(data,'$.state.output'),'')) > 51200 then 1 else 0 end) as over_50k,
|
|
198
|
+
sum(case when json_extract(data,'$.state.metadata.display.totalLines') is not null
|
|
199
|
+
and json_extract(data,'$.state.metadata.display.lineEnd')
|
|
200
|
+
< json_extract(data,'$.state.metadata.display.totalLines')
|
|
201
|
+
then 1 else 0 end) as line_cut,
|
|
202
|
+
sum(case when json_extract(data,'$.state.input.offset') is not null then 1 else 0 end) as reread
|
|
203
|
+
from part where ${TOOL} and json_extract(data,'$.tool')='read'`,
|
|
204
|
+
)[0]
|
|
205
|
+
|
|
206
|
+
// Structured questions actually asked. A few is healthy; a stream of them means
|
|
207
|
+
// the agent is asking instead of deciding, which is what GLOBAL.md's decision
|
|
208
|
+
// rule exists to stop.
|
|
209
|
+
const questionStats = query(
|
|
210
|
+
`select count(*) as asked from part where ${TOOL} and json_extract(data,'$.tool')='question'`,
|
|
211
|
+
)[0]?.asked ?? 0
|
|
212
|
+
|
|
213
|
+
// Search-before-read discipline. The paper in docs/KNOWN-LIMITS.md found no
|
|
214
|
+
// production harness using embedding retrieval: all of them search first. A
|
|
215
|
+
// read:grep ratio in the hundreds means the agent is loading whole files to
|
|
216
|
+
// find out where something lives, which is the most expensive habit measured
|
|
217
|
+
// anywhere in this kit.
|
|
218
|
+
const toolCounts = query(
|
|
219
|
+
`select json_extract(data,'$.tool') as tool, count(*) as n from part where ${TOOL} group by 1`,
|
|
220
|
+
)
|
|
221
|
+
const toolByName = Object.fromEntries(toolCounts.map((r) => [r.tool, r.n]))
|
|
222
|
+
const searches = (toolByName.grep ?? 0) + (toolByName.glob ?? 0) + (toolByName.list ?? 0)
|
|
223
|
+
const reads = toolByName.read ?? 0
|
|
224
|
+
|
|
225
|
+
const report = {
|
|
226
|
+
database: DB,
|
|
227
|
+
generated: new Date().toISOString(),
|
|
228
|
+
volume: {
|
|
229
|
+
sessions: totals.sessions,
|
|
230
|
+
days: totals.days,
|
|
231
|
+
assistantMessages: totals.messages,
|
|
232
|
+
compactionEvents: totals.compacted,
|
|
233
|
+
},
|
|
234
|
+
tokens: {
|
|
235
|
+
input: totals.input ?? 0,
|
|
236
|
+
output: totals.output ?? 0,
|
|
237
|
+
reasoning: totals.reasoning ?? 0,
|
|
238
|
+
cacheRead: totals.cache_read ?? 0,
|
|
239
|
+
cacheToInputRatio: totals.input ? +((totals.cache_read ?? 0) / totals.input).toFixed(1) : null,
|
|
240
|
+
},
|
|
241
|
+
contextPerRequest: { ...ctx, freshInputP50: freshInputP50 },
|
|
242
|
+
contextBuckets: [
|
|
243
|
+
[0, 10_000], [10_000, 50_000], [50_000, 100_000],
|
|
244
|
+
[100_000, 200_000], [200_000, 400_000], [400_000, Number.MAX_SAFE_INTEGER],
|
|
245
|
+
].map(([lo, hi]) => ({
|
|
246
|
+
range: hi >= Number.MAX_SAFE_INTEGER ? `${lo}+` : `${lo}-${hi}`,
|
|
247
|
+
count: query(
|
|
248
|
+
`select count(*) as n from message where ${ASSISTANT} and ${CTX} >= ${lo} and ${CTX} < ${hi}`,
|
|
249
|
+
)[0].n,
|
|
250
|
+
})),
|
|
251
|
+
storedBytesByType: partBytes.map((r) => ({
|
|
252
|
+
type: r.type,
|
|
253
|
+
count: r.n,
|
|
254
|
+
megabytes: +((r.bytes ?? 0) / 1e6).toFixed(2),
|
|
255
|
+
share: +(((r.bytes ?? 0) / partTotal) * 100).toFixed(1),
|
|
256
|
+
})),
|
|
257
|
+
finishReasons: finish,
|
|
258
|
+
readPressure: {
|
|
259
|
+
reads,
|
|
260
|
+
searches,
|
|
261
|
+
readPerSearch: searches ? +((reads / searches)).toFixed(1) : null,
|
|
262
|
+
overBytes: { "8k": readTruncation.over_8k ?? 0, "24k": readTruncation.over_24k ?? 0, "50k": readTruncation.over_50k ?? 0 },
|
|
263
|
+
cutByLineCap: readTruncation.line_cut ?? 0,
|
|
264
|
+
rereadWithOffset: readTruncation.reread ?? 0,
|
|
265
|
+
},
|
|
266
|
+
questionPressure: { asked: questionStats },
|
|
267
|
+
byModel: byModel.map((r) => {
|
|
268
|
+
const lat = medianLatency.find((l) => l.model === r.model && l.variant === r.variant)
|
|
269
|
+
return {
|
|
270
|
+
model: r.model,
|
|
271
|
+
variant: r.variant,
|
|
272
|
+
messages: r.messages,
|
|
273
|
+
reasoningShare:
|
|
274
|
+
r.output + r.reasoning > 0
|
|
275
|
+
? +((r.reasoning / (r.output + r.reasoning)) * 100).toFixed(1)
|
|
276
|
+
: 0,
|
|
277
|
+
cacheRead: r.cache_read,
|
|
278
|
+
medianSeconds: lat ? Number(lat.median_ms) : null,
|
|
279
|
+
}
|
|
280
|
+
}),
|
|
281
|
+
permissionPressure: {
|
|
282
|
+
bashCommands: cmds.length,
|
|
283
|
+
wouldAskWithCurrentAllowlist: wouldAskNow,
|
|
284
|
+
wouldAskWithKitAllowlist: wouldAskKit,
|
|
285
|
+
currentPromptShare: cmds.length ? +((wouldAskNow / cmds.length) * 100).toFixed(1) : 0,
|
|
286
|
+
kitPromptShare: cmds.length ? +((wouldAskKit / cmds.length) * 100).toFixed(1) : 0,
|
|
287
|
+
topRemainingAskers: topAskers(),
|
|
288
|
+
},
|
|
289
|
+
cost: {
|
|
290
|
+
recorded: +(totals.cost ?? 0).toFixed(2),
|
|
291
|
+
paidRouteEquivalent: +paidEquivalent.toFixed(2),
|
|
292
|
+
ratePerMillion: RATE,
|
|
293
|
+
},
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
if (wantJson) {
|
|
297
|
+
console.log(JSON.stringify(report, null, 2))
|
|
298
|
+
process.exit(0)
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
const n = (x) => (x ?? 0).toLocaleString("en-US")
|
|
302
|
+
const m = (x) => `${((x ?? 0) / 1e6).toFixed(2)}M`
|
|
303
|
+
const pct = (a, b) => (b ? `${((a / b) * 100).toFixed(1)}%` : "-")
|
|
304
|
+
|
|
305
|
+
const line = (k, v) => console.log(` ${k.padEnd(26)} ${v}`)
|
|
306
|
+
|
|
307
|
+
console.log(`\nopencode baseline · ${DB}`)
|
|
308
|
+
console.log(`generated ${report.generated}\n`)
|
|
309
|
+
|
|
310
|
+
console.log("VOLUME")
|
|
311
|
+
line("sessions", n(report.volume.sessions))
|
|
312
|
+
line("days", n(report.volume.days))
|
|
313
|
+
line("assistant messages", n(report.volume.assistantMessages))
|
|
314
|
+
line("compaction events", `${n(report.volume.compactionEvents)} ${report.volume.compactionEvents === 0 ? "(never fired)" : ""}`)
|
|
315
|
+
|
|
316
|
+
console.log("\nTOKENS")
|
|
317
|
+
line("input", m(report.tokens.input))
|
|
318
|
+
line("output", m(report.tokens.output))
|
|
319
|
+
line("reasoning", m(report.tokens.reasoning))
|
|
320
|
+
line("cache read", `${m(report.tokens.cacheRead)} (${report.tokens.cacheToInputRatio}x input)`)
|
|
321
|
+
|
|
322
|
+
console.log("\nCONTEXT PER REQUEST")
|
|
323
|
+
line("p50", n(report.contextPerRequest.p50))
|
|
324
|
+
line("p90", n(report.contextPerRequest.p90))
|
|
325
|
+
line("p99", n(report.contextPerRequest.p99))
|
|
326
|
+
line("max", n(report.contextPerRequest.max))
|
|
327
|
+
line("fresh input p50", n(report.contextPerRequest.freshInputP50))
|
|
328
|
+
for (const b of report.contextBuckets) line(` ${b.range}`, n(b.count))
|
|
329
|
+
|
|
330
|
+
console.log("\nSTORED MESSAGE BYTES")
|
|
331
|
+
for (const b of report.storedBytesByType) {
|
|
332
|
+
line(b.type, `${String(b.megabytes).padStart(7)} MB ${String(b.share).padStart(5)}% (${n(b.count)} parts)`)
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
console.log("\nREAD PRESSURE")
|
|
336
|
+
line("read calls", n(report.readPressure.reads))
|
|
337
|
+
line("grep/glob/list", n(report.readPressure.searches))
|
|
338
|
+
line("reads per search", `${report.readPressure.readPerSearch ?? "-"} (whole-file reads to find code)`)
|
|
339
|
+
line("reads over 8 KB", n(report.readPressure.overBytes["8k"]))
|
|
340
|
+
line("reads over 24 KB", n(report.readPressure.overBytes["24k"]))
|
|
341
|
+
line("reads over 50 KB", n(report.readPressure.overBytes["50k"]))
|
|
342
|
+
line("cut by the 200-line cap", n(report.readPressure.cutByLineCap))
|
|
343
|
+
line("re-reads with an offset", `${n(report.readPressure.rereadWithOffset)} (this is the cost the line cap created)`)
|
|
344
|
+
|
|
345
|
+
console.log("\nQUESTIONS ASKED")
|
|
346
|
+
line("question tool calls", n(report.questionPressure.asked))
|
|
347
|
+
|
|
348
|
+
console.log("\nBY MODEL")
|
|
349
|
+
for (const r of report.byModel) {
|
|
350
|
+
const lat = r.medianSeconds != null ? ` · ${r.medianSeconds}s median` : ""
|
|
351
|
+
line(`${r.model} ${r.variant}`,
|
|
352
|
+
`${n(r.messages)} msgs · reasoning ${r.reasoningShare}% · cache ${m(r.cacheRead)}${lat}`)
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
console.log("\nPERMISSION PRESSURE (historical bash commands)")
|
|
356
|
+
line("bash commands", n(report.permissionPressure.bashCommands))
|
|
357
|
+
line("prompt with current list", `${n(report.permissionPressure.wouldAskWithCurrentAllowlist)} (${report.permissionPressure.currentPromptShare}%)`)
|
|
358
|
+
line("prompt with kit allowlist", `${n(report.permissionPressure.wouldAskWithKitAllowlist)} (${report.permissionPressure.kitPromptShare}%)`)
|
|
359
|
+
console.log("\n still prompts with kit allowlist (tune from here):")
|
|
360
|
+
for (const t of report.permissionPressure.topRemainingAskers) {
|
|
361
|
+
line(` ${t.head}`, n(t.count))
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
console.log("\nCOST")
|
|
365
|
+
line("recorded", `$${report.cost.recorded}`)
|
|
366
|
+
line("same work, paid route", `$${report.cost.paidRouteEquivalent} @ $${RATE.input}/$${RATE.output}/$${RATE.cacheRead} per 1M`)
|
|
367
|
+
console.log()
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Installed by `oc-harness install`. Remove with: rm .git/hooks/pre-commit
|
|
3
|
+
#
|
|
4
|
+
# Runs the offline gate on every commit. The live-install half is deliberately
|
|
5
|
+
# left out: measure and attribute read ~/.local/share/opencode, which is slow,
|
|
6
|
+
# machine-specific, and irrelevant to whether a commit broke the kit.
|
|
7
|
+
#
|
|
8
|
+
# On a machine that has never run OpenCode, test:install cannot run at all, so
|
|
9
|
+
# this hook uses the subset that needs nothing but the repository.
|
|
10
|
+
|
|
11
|
+
set -e
|
|
12
|
+
|
|
13
|
+
repo_root=$(git rev-parse --show-toplevel)
|
|
14
|
+
cd "$repo_root"
|
|
15
|
+
|
|
16
|
+
if [ -n "$SKIP_SPACE_HARNESS_HOOK" ]; then
|
|
17
|
+
echo "pre-commit: skipped (SKIP_SPACE_HARNESS_HOOK set)"
|
|
18
|
+
exit 0
|
|
19
|
+
fi
|
|
20
|
+
|
|
21
|
+
echo "pre-commit: npm run test:all"
|
|
22
|
+
if ! npm run test:all >/tmp/space-harness-hook.log 2>&1; then
|
|
23
|
+
echo "pre-commit: FAILED. Last lines:"
|
|
24
|
+
tail -30 /tmp/space-harness-hook.log
|
|
25
|
+
echo ""
|
|
26
|
+
echo "Fix it, or commit with --no-verify if you are certain it is not yours."
|
|
27
|
+
exit 1
|
|
28
|
+
fi
|
|
29
|
+
|
|
30
|
+
exit 0
|