opencode-harness-kit 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +544 -19
- package/CONTRIBUTING.md +242 -0
- package/GLOBAL.md +34 -5
- package/MIGRATION.md +151 -0
- package/README.md +294 -37
- package/SECURITY.md +30 -0
- package/bin/baseline.mjs +15 -7
- package/bin/git-hooks/pre-commit +41 -9
- package/bin/oc-harness.mjs +1265 -82
- package/dist/config.d.ts +127 -0
- package/dist/denylist.d.ts +33 -0
- package/dist/effort.d.ts +64 -0
- package/dist/guard.d.ts +23 -0
- package/dist/health.d.ts +43 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +532 -149
- package/dist/plugin.d.ts +15 -0
- package/dist/profile.d.ts +399 -0
- package/dist/project.d.ts +8 -0
- package/dist/prune.d.ts +6 -0
- package/dist/router.d.ts +3 -0
- package/dist/rules.d.ts +109 -0
- package/dist/safe.d.ts +7 -0
- package/dist/telemetry-store.d.ts +12 -0
- package/dist/telemetry.d.ts +3 -0
- package/docs/EXIT-CODES.md +71 -0
- package/docs/KNOWN-LIMITS.md +45 -21
- package/docs/RESEARCH.md +9 -4
- package/docs/ROUTING.md +53 -8
- package/eval/RESULTS.md +443 -0
- package/eval/ab.mjs +743 -0
- package/eval/attribute.mjs +170 -0
- package/eval/flags.mjs +88 -0
- package/eval/measure-install.mjs +323 -0
- package/eval/measure-prune.mjs +339 -0
- package/eval/metrics.js +79 -0
- package/eval/premises.mjs +200 -0
- package/eval/prepare.mjs +233 -0
- package/eval/run.mjs +376 -0
- package/eval/tasks-self.json +81 -0
- package/eval/tasks-ts.json +50 -0
- package/eval/tasks.json +83 -0
- package/eval/thresholds.mjs +134 -0
- package/eval/verify.mjs +287 -0
- package/package.json +44 -17
- package/profiles/fleet-all.json +6 -1
- package/profiles/fleet-safe.json +7 -3
- package/profiles/schema.json +182 -51
- package/prompts/build.md +4 -2
- package/skills/space-build/SKILL.md +4 -2
- package/skills/space-finish/SKILL.md +6 -2
- package/skills/space-plan/reference/task-template.md +13 -5
- package/src/config.js +182 -32
- package/src/denylist.js +95 -0
- package/src/driver/run.mjs +81 -16
- package/src/effort.js +15 -4
- package/src/guard.ts +165 -13
- package/src/health.js +80 -0
- package/src/index.ts +15 -4
- package/src/lane-usage.js +72 -18
- package/src/plugin-type-check.ts +38 -0
- package/src/plugin.ts +39 -0
- package/src/profile.js +225 -26
- package/src/project.js +47 -0
- package/src/prune.ts +110 -31
- package/src/router.ts +32 -63
- package/src/rules.js +99 -7
- package/src/safe.ts +27 -0
- package/src/sqlite.js +144 -0
- package/src/tasks.js +112 -29
- package/src/telemetry-store.ts +62 -0
- package/src/telemetry.ts +60 -57
- package/src/trust.js +70 -9
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
export type Profile = import("./profile.js").Profile;
|
|
2
|
+
/** @typedef {import("./profile.js").Profile} Profile */
|
|
3
|
+
/**
|
|
4
|
+
* Where the global rule set lives, as opencode should be told to look for it.
|
|
5
|
+
*
|
|
6
|
+
* Honours OPENCODE_CONFIG_DIR, because `~/.config/opencode` is only the default
|
|
7
|
+
* and emitting it unconditionally points at the wrong file for anyone using a
|
|
8
|
+
* custom config directory - the installer would write one place and the
|
|
9
|
+
* compiler would name another. Written with a leading `~` when it is the
|
|
10
|
+
* default, which is both correct and readable in the generated file.
|
|
11
|
+
*/
|
|
12
|
+
export declare function globalInstructionsPath(): string;
|
|
13
|
+
/**
|
|
14
|
+
* The default location, for documentation and tests. Prefer
|
|
15
|
+
* `globalInstructionsPath()` when generating a config.
|
|
16
|
+
*
|
|
17
|
+
* It is written by `oc-harness install`, which symlinks this file rather than
|
|
18
|
+
* copying it, so the repo stays the source of truth. `oc-harness verify` fails if
|
|
19
|
+
* the path it names does not resolve - the failure mode where a config points at
|
|
20
|
+
* an instruction file that was never installed is silent and makes the whole
|
|
21
|
+
* harness a no-op.
|
|
22
|
+
*/
|
|
23
|
+
export declare const GLOBAL_INSTRUCTIONS = "~/.config/opencode/GLOBAL.md";
|
|
24
|
+
/**
|
|
25
|
+
* The kit's own version, read from its package.json.
|
|
26
|
+
*
|
|
27
|
+
* @returns {string}
|
|
28
|
+
*/
|
|
29
|
+
export declare function kitVersion(): string;
|
|
30
|
+
/**
|
|
31
|
+
* The memory plugin this kit can turn on.
|
|
32
|
+
*
|
|
33
|
+
* The kit does not implement memory. A local vector store with embeddings, a
|
|
34
|
+
* web UI and auto-capture is a large thing to build badly, and there is one
|
|
35
|
+
* that is already good: `opencode-mem` (MIT). What the kit can do is make it
|
|
36
|
+
* one flag away, and keep the config honest about it.
|
|
37
|
+
*/
|
|
38
|
+
export declare const MEMORY_PLUGIN = "opencode-mem@latest";
|
|
39
|
+
/**
|
|
40
|
+
* The config with the memory plugin turned on.
|
|
41
|
+
*
|
|
42
|
+
* The plugin list is preserved. The user's own plugins are not the kit's to
|
|
43
|
+
* remove, and the entry is added rather than replaced so a config that
|
|
44
|
+
* already has it does not grow a second copy.
|
|
45
|
+
*
|
|
46
|
+
* opencode v1 calls the key `plugin` and v2 calls it `plugins`; whichever
|
|
47
|
+
* the config already uses is the one that is extended.
|
|
48
|
+
*
|
|
49
|
+
* @param {any} base the config as it is on disk
|
|
50
|
+
* @param {string} [spec] the plugin id, so a pinned version or a mirror works
|
|
51
|
+
* @returns {any}
|
|
52
|
+
*/
|
|
53
|
+
export declare function withMemoryPlugin(base: any, spec?: string): any;
|
|
54
|
+
/**
|
|
55
|
+
* Whether the memory plugin can be added to this config.
|
|
56
|
+
*
|
|
57
|
+
* A config whose plugin value is not a list is left alone: replacing a
|
|
58
|
+
* string with a list is data loss, and the promise is that the user's own
|
|
59
|
+
* plugins are never touched.
|
|
60
|
+
*
|
|
61
|
+
* @param {any} base
|
|
62
|
+
* @returns {boolean}
|
|
63
|
+
*/
|
|
64
|
+
export declare function canAddMemoryPlugin(base: any): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* The utility route a profile names, as a full model id.
|
|
67
|
+
*
|
|
68
|
+
* `small_model` is opencode's route for the short auxiliary calls. It used to
|
|
69
|
+
* be the session model, so every title and every rewritten question paid the
|
|
70
|
+
* session model's reasoning - and reasoning, not output, is what a short call
|
|
71
|
+
* costs.
|
|
72
|
+
*
|
|
73
|
+
* @param {any} p the profile
|
|
74
|
+
* @returns {string | null}
|
|
75
|
+
*/
|
|
76
|
+
export declare function utilityModel(p: any): string | null;
|
|
77
|
+
/**
|
|
78
|
+
* The instruction list a generated config carries.
|
|
79
|
+
*
|
|
80
|
+
* The kit's own file is always in it. A config that already had an
|
|
81
|
+
* `instructions` array used to replace it wholesale, which unloaded the
|
|
82
|
+
* whole behavioural layer while everything still looked correct.
|
|
83
|
+
*
|
|
84
|
+
* The user's own files keep their order and come first; the kit's is
|
|
85
|
+
* appended. Duplicates are dropped, so re-running `use` does not grow the
|
|
86
|
+
* list.
|
|
87
|
+
*
|
|
88
|
+
* @param {string[] | string | undefined} base
|
|
89
|
+
* @param {string} kitPath
|
|
90
|
+
* @returns {string[]}
|
|
91
|
+
*/
|
|
92
|
+
export declare function mergeInstructions(base: string[] | string | undefined, kitPath: string): string[];
|
|
93
|
+
/**
|
|
94
|
+
* Read-only and verification commands.
|
|
95
|
+
*
|
|
96
|
+
* Measured against 5,239 real bash calls from one install: 95.3% would prompt
|
|
97
|
+
* under a narrow allowlist, 23% under this one. The gated remainder is
|
|
98
|
+
* deliberate - ssh, curl, rm, pkill, osascript, cp, mv, kill, docker.
|
|
99
|
+
*
|
|
100
|
+
* `X*` with no space is used on purpose: opencode's `X *` pattern requires a
|
|
101
|
+
* trailing space and therefore does not match a bare `X`.
|
|
102
|
+
*
|
|
103
|
+
* The `gh` verbs below are the read half of the CLI and nothing else. `gh pr
|
|
104
|
+
* create`, `gh pr merge` and `gh api` are NOT here: they write, and `gh api` in
|
|
105
|
+
* particular can POST to any endpoint. Measured cost of gating the whole `gh`
|
|
106
|
+
* surface was 146 prompts; gating only the mutating verbs keeps those 146
|
|
107
|
+
* prompt-free without opening a publish path the guard plugin cannot police.
|
|
108
|
+
*/
|
|
109
|
+
export declare const BASH_ALLOWLIST: string[];
|
|
110
|
+
/**
|
|
111
|
+
* Hard blocks.
|
|
112
|
+
*
|
|
113
|
+
* The list lives in ./denylist.js because it is enforced in TWO places: by
|
|
114
|
+
* opencode's permission map, and by the guard plugin. The two used to be
|
|
115
|
+
* different lists, and the comment here claimed they were the same.
|
|
116
|
+
*/
|
|
117
|
+
export { BASH_DENYLIST } from "./denylist.js";
|
|
118
|
+
/**
|
|
119
|
+
* @param {Profile} p
|
|
120
|
+
* @param {{ base?: Record<string, any>, comments?: boolean, unhealthy?: Set<string> }} [opts]
|
|
121
|
+
* @returns {string}
|
|
122
|
+
*/
|
|
123
|
+
export declare function generateConfig(p: Profile, opts?: {
|
|
124
|
+
base?: Record<string, any>;
|
|
125
|
+
comments?: boolean;
|
|
126
|
+
unhealthy?: Set<string>;
|
|
127
|
+
}): string;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** @type {string[]} */
|
|
2
|
+
export declare const BASH_DENYLIST: string[];
|
|
3
|
+
/**
|
|
4
|
+
* Whether a glob pattern matches a command.
|
|
5
|
+
*
|
|
6
|
+
* `*` matches any run of characters, including none.
|
|
7
|
+
*
|
|
8
|
+
* @param {string} command
|
|
9
|
+
* @param {string} pattern
|
|
10
|
+
* @returns {boolean}
|
|
11
|
+
*/
|
|
12
|
+
export declare function matchesPattern(command: string, pattern: string): boolean;
|
|
13
|
+
/**
|
|
14
|
+
* The first denied pattern a command matches, or null.
|
|
15
|
+
*
|
|
16
|
+
* The command is matched whole, and it is matched by the CONFIG too - the
|
|
17
|
+
* permission map globs the raw command, because that is the only thing it
|
|
18
|
+
* can see. So this function is the flat, literal half.
|
|
19
|
+
*
|
|
20
|
+
* The guard does not stop here. It calls this on each SEGMENT, with the
|
|
21
|
+
* wrappers stripped and the payload of a `bash -c` followed, so
|
|
22
|
+
* `sudo bash -c "npm publish"` is caught as well. The two enforcers read the
|
|
23
|
+
* same list and one of them reads it more carefully; that is the price of
|
|
24
|
+
* having the list in two places.
|
|
25
|
+
*
|
|
26
|
+
* The two PIPE patterns are for the config's side only. `segments()` splits
|
|
27
|
+
* on `|`, so a segment never contains " | sh" and the guard catches a piped
|
|
28
|
+
* script through its own rule instead.
|
|
29
|
+
*
|
|
30
|
+
* @param {string} command
|
|
31
|
+
* @returns {string | null}
|
|
32
|
+
*/
|
|
33
|
+
export declare function deniedBy(command: string): string | null;
|
package/dist/effort.d.ts
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
export type Profile = import("./profile.js").Profile;
|
|
2
|
+
export type EffortRequest = import("./profile.js").EffortRequest;
|
|
3
|
+
export type Config = {
|
|
4
|
+
agent?: Record<string, any>;
|
|
5
|
+
};
|
|
6
|
+
export type EffortResult = {
|
|
7
|
+
profileId: string;
|
|
8
|
+
model: string;
|
|
9
|
+
defaultVariant: string | null;
|
|
10
|
+
roles: Record<string, string | null>;
|
|
11
|
+
laneModels: Record<string, string>;
|
|
12
|
+
notes: string[];
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* @typedef {import("./profile.js").Profile} Profile
|
|
16
|
+
* @typedef {import("./profile.js").EffortRequest} EffortRequest
|
|
17
|
+
* @typedef {object} Config
|
|
18
|
+
* @property {Record<string, any>} [agent]
|
|
19
|
+
* @typedef {object} EffortResult
|
|
20
|
+
* @property {string} profileId
|
|
21
|
+
* @property {string} model
|
|
22
|
+
* @property {string|null} defaultVariant
|
|
23
|
+
* @property {Record<string, string|null>} roles
|
|
24
|
+
* @property {Record<string, string>} laneModels
|
|
25
|
+
* @property {string[]} notes
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* The lanes a profile manages. A lane a profile does not mention is left alone.
|
|
29
|
+
*
|
|
30
|
+
* This used to be a hand-written list of nine lanes, and it drifted: the
|
|
31
|
+
* fleet has ten, and `longform` was missing. A second list of lane names is a
|
|
32
|
+
* second source of truth, and the lane table in `lanes.json` is the first one.
|
|
33
|
+
*
|
|
34
|
+
* @type {string[]}
|
|
35
|
+
*/
|
|
36
|
+
export declare const MANAGED_ROLES: string[];
|
|
37
|
+
/**
|
|
38
|
+
* @param {string} profileId
|
|
39
|
+
* @param {{ unhealthy?: Set<string> }} [opts]
|
|
40
|
+
* @returns {EffortResult}
|
|
41
|
+
*/
|
|
42
|
+
export declare function planEffort(profileId: string, opts?: {
|
|
43
|
+
unhealthy?: Set<string>;
|
|
44
|
+
}): EffortResult;
|
|
45
|
+
/**
|
|
46
|
+
* Mutate the resolved OpenCode config so each managed agent declares its own
|
|
47
|
+
* model and effort level.
|
|
48
|
+
*
|
|
49
|
+
* `AgentConfig.variant` only applies to the agent's own configured model, so
|
|
50
|
+
* the model is written alongside the variant instead of relying on the global
|
|
51
|
+
* default. That is also what makes per-lane effort work at all.
|
|
52
|
+
*
|
|
53
|
+
* The model is only filled in when the agent does not have one. A fleet puts
|
|
54
|
+
* ten different models behind ten agents, and overwriting them with the
|
|
55
|
+
* profile's base model here would collapse the whole fleet onto one route.
|
|
56
|
+
*
|
|
57
|
+
* @param {Config} config
|
|
58
|
+
* @param {EffortResult} result
|
|
59
|
+
* @param {{ sessionModel?: string }} [opts]
|
|
60
|
+
* @returns {Config}
|
|
61
|
+
*/
|
|
62
|
+
export declare function applyEffort(config: Config, result: EffortResult, opts?: {
|
|
63
|
+
sessionModel?: string;
|
|
64
|
+
}): Config;
|
package/dist/guard.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Plugin } from "./plugin.js";
|
|
2
|
+
export declare const MARKER = "[opencode-harness-kit] blocked";
|
|
3
|
+
/**
|
|
4
|
+
* The files a patch touches.
|
|
5
|
+
*
|
|
6
|
+
* `apply_patch` sends a patch, not a path, so the paths have to be read out
|
|
7
|
+
* of it. Two formats are in use: opencode's own `*** Update File: path`
|
|
8
|
+
* markers, and a plain unified diff with `+++ b/path` headers.
|
|
9
|
+
*
|
|
10
|
+
* @param {string} text
|
|
11
|
+
* @returns {string[]}
|
|
12
|
+
*/
|
|
13
|
+
export declare function pathsInPatch(text: string): string[];
|
|
14
|
+
/**
|
|
15
|
+
* The files a write tool is about to touch.
|
|
16
|
+
*
|
|
17
|
+
* @param {string} tool
|
|
18
|
+
* @param {any} args
|
|
19
|
+
* @returns {string[]} empty when the arguments name no file
|
|
20
|
+
*/
|
|
21
|
+
export declare function writeTargets(tool: string, args: any): string[];
|
|
22
|
+
export declare const GuardPlugin: Plugin;
|
|
23
|
+
export default GuardPlugin;
|
package/dist/health.d.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export type Health = {
|
|
2
|
+
unhealthy: Record<string, {
|
|
3
|
+
at: string;
|
|
4
|
+
reason: string;
|
|
5
|
+
}>;
|
|
6
|
+
active?: string;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* @typedef {object} Health
|
|
10
|
+
* @property {Record<string, { at: string, reason: string }>} unhealthy
|
|
11
|
+
* @property {string} [active]
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* @param {string} dir the project root
|
|
15
|
+
* @returns {string}
|
|
16
|
+
*/
|
|
17
|
+
export declare function healthPath(dir: string): string;
|
|
18
|
+
/**
|
|
19
|
+
* @param {string} dir the project root
|
|
20
|
+
* @returns {Health}
|
|
21
|
+
*/
|
|
22
|
+
export declare function readHealth(dir: string): Health;
|
|
23
|
+
/**
|
|
24
|
+
* @param {string} dir the project root
|
|
25
|
+
* @param {Health} health
|
|
26
|
+
*/
|
|
27
|
+
export declare function writeHealth(dir: string, health: Health): void;
|
|
28
|
+
/**
|
|
29
|
+
* The models a previous session found spent.
|
|
30
|
+
*
|
|
31
|
+
* Exported so the effort hook resolves a lane's variant against the same set
|
|
32
|
+
* of spent routes the router used. A lane that failed over to a model with
|
|
33
|
+
* fewer effort levels must lose its variant, or the config asks for an effort
|
|
34
|
+
* the new model does not publish.
|
|
35
|
+
*
|
|
36
|
+
* The health file is keyed by the qualified route, `opencode/<model>`. The
|
|
37
|
+
* lane chains list bare model ids, so the prefix is dropped here. Comparing
|
|
38
|
+
* the two directly would never match, and a spent route would look healthy.
|
|
39
|
+
*
|
|
40
|
+
* @param {string} dir the project root
|
|
41
|
+
* @returns {Set<string>}
|
|
42
|
+
*/
|
|
43
|
+
export declare function unhealthyModels(dir: string): Set<string>;
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Plugin } from "./plugin.js";
|
|
2
|
+
export interface HarnessOptions {
|
|
3
|
+
/** Profile id. Defaults to the one recorded in the config by `oc-harness use`. */
|
|
4
|
+
profile?: string;
|
|
5
|
+
/** Load a subset. Defaults to all four. */
|
|
6
|
+
only?: Array<"guard" | "prune" | "router" | "telemetry">;
|
|
7
|
+
}
|
|
8
|
+
export declare const HarnessPlugin: Plugin;
|
|
9
|
+
export default HarnessPlugin;
|