@spences10/pi-harness 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -175
- package/dist/assessment.d.ts +4 -12
- package/dist/assessment.js +19 -192
- package/dist/assessment.js.map +1 -1
- package/dist/enforcement/policy.d.ts +2 -0
- package/dist/enforcement/policy.js +47 -73
- package/dist/enforcement/policy.js.map +1 -1
- package/dist/extension.d.ts +1 -1
- package/dist/extension.js +178 -403
- package/dist/extension.js.map +1 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/prompt.d.ts +1 -1
- package/dist/prompt.js +1 -7
- package/dist/prompt.js.map +1 -1
- package/dist/runtime/baseline.d.ts +6 -0
- package/dist/runtime/baseline.js +145 -0
- package/dist/runtime/baseline.js.map +1 -0
- package/dist/runtime/index.d.ts +2 -1
- package/dist/runtime/index.js +72 -29
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/outcome.d.ts +1 -2
- package/dist/runtime/outcome.js +21 -35
- package/dist/runtime/outcome.js.map +1 -1
- package/dist/runtime/renderers.js +34 -70
- package/dist/runtime/renderers.js.map +1 -1
- package/dist/runtime/verification.d.ts +15 -0
- package/dist/runtime/verification.js +97 -0
- package/dist/runtime/verification.js.map +1 -0
- package/dist/schema.d.ts +12 -2
- package/dist/schema.js +6 -8
- package/dist/schema.js.map +1 -1
- package/package.json +4 -4
- package/skills/create-harness/SKILL.md +19 -62
- package/skills/execute-harness/SKILL.md +27 -66
- package/skills/review-harness/SKILL.md +21 -54
package/README.md
CHANGED
|
@@ -11,18 +11,19 @@
|
|
|
11
11
|
|
|
12
12
|
<!-- package-readme:header:end -->
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
status, and runtime enforcement. Inspired by Ornith's self-scaffolding
|
|
18
|
-
design, the runtime separates an immutable outer policy from a
|
|
19
|
-
versioned, amendable inner scaffold.
|
|
14
|
+
Prompt guidance uses Pi's named system-prompt sections. It coexists
|
|
15
|
+
with user `SYSTEM.md` and `APPEND_SYSTEM.md` files and other
|
|
16
|
+
extensions without replacing the complete prompt.
|
|
20
17
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
18
|
+
Optional task scaffolding for Pi: a contract, working-state baseline,
|
|
19
|
+
validation scripts, progress log, and reviewable outcome. Pi still
|
|
20
|
+
owns the agent loop, sessions, tools, and permission hooks.
|
|
24
21
|
|
|
25
|
-
|
|
22
|
+
Use direct work for routine changes. Use a harness when the user asks
|
|
23
|
+
for one or a concrete execution contract adds value. Assessment is
|
|
24
|
+
optional advice, not a prerequisite or a read-only security mode.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
26
27
|
|
|
27
28
|
<!-- package-readme:install:start -->
|
|
28
29
|
|
|
@@ -32,181 +33,125 @@ pi install npm:@spences10/pi-harness
|
|
|
32
33
|
|
|
33
34
|
<!-- package-readme:install:end -->
|
|
34
35
|
|
|
35
|
-
|
|
36
|
+
## Create and execute
|
|
36
37
|
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
pi install ./packages/pi-harness
|
|
40
|
-
# or for one run only
|
|
41
|
-
pi -e ./packages/pi-harness
|
|
38
|
+
```text
|
|
39
|
+
/harness create <task>
|
|
42
40
|
```
|
|
43
41
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
`
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
- `/harness` command for assess-before-create, run, review, status,
|
|
50
|
-
use, and clear
|
|
51
|
-
- `harness_assess` and `harness_assessment_submit` tools for read-only
|
|
52
|
-
orientation and structured adoption decisions
|
|
53
|
-
- guarded `harness_create` tool that cannot bypass assessment approval
|
|
54
|
-
- `harness_amend` tool for audited, versioned changes to the inner
|
|
55
|
-
scaffold
|
|
56
|
-
- `harness_update` tool for phase/status/evidence logging
|
|
57
|
-
- generated `OUTCOME.md` and `outcome.json` review artifacts
|
|
58
|
-
- dirty-baseline tracking so pre-existing repo changes are reported
|
|
59
|
-
but not treated as task drift
|
|
60
|
-
- worktree-aware validate/review scripts for team-mode executors
|
|
61
|
-
- `harness_read` tool for summaries
|
|
62
|
-
- `before_agent_start` context injection for the active harness
|
|
63
|
-
- `tool_call` enforcement for edit/write paths and forbidden commands
|
|
64
|
-
- compact TUI status for the active harness
|
|
65
|
-
- bundled `create-harness`, `execute-harness`, and `review-harness`
|
|
66
|
-
skills
|
|
67
|
-
|
|
68
|
-
## Runtime layout
|
|
42
|
+
The agent calls `harness_create` with the task and at least one
|
|
43
|
+
accepted validation command. Scope defaults to the project; provide
|
|
44
|
+
`allowed_paths` to narrow it. Normal project test development is
|
|
45
|
+
allowed by default. Set `allow_test_changes: false` only when the task
|
|
46
|
+
explicitly requires protecting those files.
|
|
69
47
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
48
|
+
Creation does not require a separate assessment or approval roundtrip.
|
|
49
|
+
It does not authorize new capabilities, commits, pushes, releases,
|
|
50
|
+
deployments, destructive operations, or peer collaboration beyond the
|
|
51
|
+
user's request.
|
|
52
|
+
|
|
53
|
+
The runtime lives under `/tmp/my-pi-harness-<id>/`:
|
|
54
|
+
|
|
55
|
+
- `harness.json`: accepted maximum scope and checks, plus the
|
|
56
|
+
execution scaffold and amendment history.
|
|
57
|
+
- `TASK.md` and `SYSTEM.md`: task and workflow guidance.
|
|
58
|
+
- `validate.sh` and `review.sh`: manual validation and drift review.
|
|
59
|
+
- `status.json` and `logs/`: progress, executed checks and output.
|
|
60
|
+
- `OUTCOME.md` and `outcome.json`: review artifacts.
|
|
61
|
+
|
|
62
|
+
### Completion means checks ran
|
|
63
|
+
|
|
64
|
+
Request `status: completed` through `harness_update`. The extension
|
|
65
|
+
runs the accepted commands, any additional scaffold checks, and the
|
|
66
|
+
drift guard through Pi's normal nested Bash tool pipeline. Permission
|
|
67
|
+
hooks, cancellation and tool-result handling still apply.
|
|
68
|
+
|
|
69
|
+
Completion requires successful checks against an unchanged contract
|
|
70
|
+
and Git-visible working state. A failure, denied tool, interruption or
|
|
71
|
+
concurrent change cannot become success merely because an agent writes
|
|
72
|
+
“passed”. Results include executed-check records and output-file
|
|
73
|
+
references. Manual evidence strings are self-reported notes, not
|
|
74
|
+
validation proof.
|
|
75
|
+
|
|
76
|
+
Failed checks can be fixed and retried; there is no one-attempt
|
|
77
|
+
escalation rule. Completed runs reject further API amendments and
|
|
78
|
+
status mutations. Their active guard clears on the next direct user
|
|
79
|
+
turn or session startup; do not create another harness just to commit
|
|
80
|
+
already-reviewed work.
|
|
81
|
+
|
|
82
|
+
### Amend strategy, not acceptance
|
|
83
|
+
|
|
84
|
+
`harness_amend` records a versioned strategy change. It cannot remove
|
|
85
|
+
or replace the accepted validation commands or relax an explicit
|
|
86
|
+
no-test-edit restriction. Additional checks are allowed. Scaffold
|
|
87
|
+
scope always intersects the accepted maximum scope, so broadening a
|
|
88
|
+
strategy does not grant access outside the contract.
|
|
89
|
+
|
|
90
|
+
A genuinely larger task needs user authorization and a new contract,
|
|
91
|
+
not an executor-authored relaxation of its own acceptance criteria.
|
|
92
|
+
|
|
93
|
+
### Baseline and worktrees
|
|
83
94
|
|
|
84
|
-
|
|
95
|
+
Dirty files are fingerprinted without changing the working tree, HEAD
|
|
96
|
+
or Git index. Later content, index, mode, symlink, deletion and rename
|
|
97
|
+
changes remain visible, including initially untracked files. Only
|
|
98
|
+
fingerprints are saved, not file contents.
|
|
99
|
+
|
|
100
|
+
A baseline belongs to the workspace where it was captured. Linked
|
|
101
|
+
worktrees use their own Git state, without another workspace's dirty
|
|
102
|
+
file exemptions. Run manual scripts from the execution worktree or set
|
|
103
|
+
`HARNESS_CWD`. Completion uses the current Pi execution directory and
|
|
104
|
+
rejects unrelated repositories.
|
|
105
|
+
|
|
106
|
+
### Optional assessment
|
|
107
|
+
|
|
108
|
+
`/assess <task>`, `harness_assess` and `harness_assessment_submit`
|
|
109
|
+
record evidence, existing primitives and a recommendation. Tools,
|
|
110
|
+
native codemode, discovery and MCP remain available. This is advisory
|
|
111
|
+
investigation: it does not enforce read-only access or add an approval
|
|
112
|
+
queue. Normal user authorization still applies.
|
|
113
|
+
|
|
114
|
+
## Commands and tools
|
|
85
115
|
|
|
86
116
|
```text
|
|
87
|
-
/assess <task>
|
|
88
|
-
/assess status
|
|
89
|
-
/assess approve [harness|direct]
|
|
90
|
-
/assess reject
|
|
91
|
-
/assess clear
|
|
92
117
|
/harness create <task>
|
|
93
|
-
/harness run
|
|
94
|
-
/harness review
|
|
118
|
+
/harness run [dir]
|
|
119
|
+
/harness review [dir]
|
|
95
120
|
/harness status [dir]
|
|
96
121
|
/harness use <dir>
|
|
97
122
|
/harness clear
|
|
123
|
+
/assess <task>
|
|
124
|
+
/assess status
|
|
125
|
+
/assess clear
|
|
98
126
|
```
|
|
99
127
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
Factory; Factory requires a separate direct user request for
|
|
109
|
-
evaluation.
|
|
110
|
-
|
|
111
|
-
An agent can enter assessment with `harness_assess`. If it calls
|
|
112
|
-
`harness_create` without approval, the extension blocks the call and
|
|
113
|
-
starts assessment. This backstop does not detect an agent that
|
|
114
|
-
misclassifies a task and never asks for harness machinery.
|
|
115
|
-
|
|
116
|
-
Use assessment when a task may benefit from an enforceable execution
|
|
117
|
-
contract because it has material risk, unresolved scope, destructive
|
|
118
|
-
effects, or complex coordination. Use the normal direct workflow for
|
|
119
|
-
bounded, low-risk changes that standard validation can verify, such as
|
|
120
|
-
documentation and copy edits, focused single-file fixes, configuration
|
|
121
|
-
or metadata updates, test expectation changes, formatting, and
|
|
122
|
-
reviewed commit or push follow-ups. A simple ambiguity may need only a
|
|
123
|
-
clarifying question. Broad uncertain refactors, migrations,
|
|
124
|
-
deployments, risky releases, external side effects, and explicit user
|
|
125
|
-
requests are good assessment candidates.
|
|
126
|
-
|
|
127
|
-
Assessment snapshots the current tool set, keeps an explicit read-only
|
|
128
|
-
research subset active, and filters Bash through a strict
|
|
129
|
-
single-command allowlist. Unknown and mutating custom tools are
|
|
130
|
-
withheld. This is a workflow guard, not an operating-system sandbox.
|
|
131
|
-
In print and JSON modes, an assessment can be submitted but remains
|
|
132
|
-
waiting because direct approval cannot be inferred. RPC clients can
|
|
133
|
-
answer Pi's extension UI request, or send `/assess approve ...` as an
|
|
134
|
-
explicit command.
|
|
135
|
-
|
|
136
|
-
`harness.json` contains two deliberately different layers:
|
|
137
|
-
|
|
138
|
-
- `policy`: runtime-owned workspace and verifier protections such as
|
|
139
|
-
cwd, forbidden paths/commands, available tools, and the dirty
|
|
140
|
-
baseline. Executors cannot weaken these through amendments.
|
|
141
|
-
- `scaffold`: the model's task interpretation and execution strategy,
|
|
142
|
-
including allowed paths, validation, test policy, and model roles.
|
|
143
|
-
It is subordinate to system, developer, and current user
|
|
144
|
-
instructions and can be revised with `harness_amend`.
|
|
145
|
-
|
|
146
|
-
Every scaffold amendment increments its version, records its reason
|
|
147
|
-
and requester, regenerates runtime prompts/scripts, and appends an
|
|
148
|
-
audit event. A harness enforcement block reports a contract mismatch;
|
|
149
|
-
it is not a platform-policy refusal. When a harness is active, the TUI
|
|
150
|
-
shows a compact footer/status indicator; use `/harness status [dir]`
|
|
151
|
-
or `harness_read` for the full contract, task, validation, and outcome
|
|
152
|
-
details.
|
|
153
|
-
|
|
154
|
-
A terminal `completed` or `failed` status seals the run and keeps
|
|
155
|
-
enforcement active for the remainder of the current turn, so an
|
|
156
|
-
executor cannot escape the outer policy by declaring its own work
|
|
157
|
-
complete. The extension automatically deactivates that terminal
|
|
158
|
-
harness on the next direct user turn or when the session starts again.
|
|
159
|
-
Extension-injected messages do not trigger cleanup. Use
|
|
160
|
-
`/harness clear` only to abandon a non-terminal run. Clearing or
|
|
161
|
-
deleting a harness affects only its `/tmp` control state; repository
|
|
162
|
-
changes remain in the working tree. Do not create a new harness merely
|
|
163
|
-
to commit or push already-reviewed work. A risky release, deployment,
|
|
164
|
-
migration, or destructive operation can still justify a dedicated
|
|
165
|
-
harness when explicitly requested or warranted by its actual risk.
|
|
166
|
-
|
|
167
|
-
Harnesses snapshot dirty files at creation time. Review and outcome
|
|
168
|
-
artifacts focus on changes after that baseline, while still recording
|
|
169
|
-
baseline files in `outcome.json`. If a team-mode executor runs in a
|
|
170
|
-
linked git worktree, run `review.sh` from that worktree or set
|
|
171
|
-
`HARNESS_CWD` so validation, guard checks, and outcome collection use
|
|
172
|
-
the executor workspace.
|
|
173
|
-
|
|
174
|
-
## Assessment and harness loops
|
|
175
|
-
|
|
176
|
-
Assessment happens before harness creation:
|
|
177
|
-
|
|
178
|
-
1. Read-only context recovery
|
|
179
|
-
2. Source-of-truth and existing-primitive capture
|
|
180
|
-
3. Assumption challenge and rejected options
|
|
181
|
-
4. Smallest vertical slice
|
|
182
|
-
5. Structured recommendation
|
|
183
|
-
6. Direct user adoption decision
|
|
184
|
-
|
|
185
|
-
An approved harness then runs:
|
|
186
|
-
|
|
187
|
-
1. Contract recovery
|
|
188
|
-
2. Surgical execution
|
|
189
|
-
3. Validation evidence
|
|
190
|
-
4. Drift review
|
|
191
|
-
5. OUTCOME.md/outcome.json review
|
|
192
|
-
6. Delta/risk report
|
|
193
|
-
|
|
194
|
-
Use `harness_amend` for bounded inner-scaffold changes. A new
|
|
195
|
-
capability or architecture decision requires assessment; an approved
|
|
196
|
-
harness recommendation can then amend the active scaffold.
|
|
197
|
-
Outer-policy expansion requires a separate execution contract instead
|
|
198
|
-
of silent scope growth.
|
|
199
|
-
|
|
200
|
-
## Using from a custom harness
|
|
201
|
-
|
|
202
|
-
```ts
|
|
203
|
-
import harness from '@spences10/pi-harness';
|
|
204
|
-
|
|
205
|
-
// pass `harness` as an ExtensionFactory to your Pi runtime
|
|
206
|
-
```
|
|
128
|
+
Tools: `harness_create`, `harness_amend`, `harness_update`,
|
|
129
|
+
`harness_read`, `harness_assess`, `harness_assessment_submit`.
|
|
130
|
+
|
|
131
|
+
Execution is direct by default. Team Mode can coordinate independently
|
|
132
|
+
opened peers only when the user authorizes collaboration. This package
|
|
133
|
+
does not spawn or supervise workers.
|
|
134
|
+
|
|
135
|
+
## Safety and compatibility
|
|
207
136
|
|
|
208
|
-
|
|
209
|
-
|
|
137
|
+
This is workflow scaffolding, **not an OS sandbox**. Direct edit/write
|
|
138
|
+
hooks and final Git drift checks enforce the accepted paths through
|
|
139
|
+
those interfaces. Shell commands and other tools retain the host
|
|
140
|
+
user's permissions and can modify control files; the runtime is not a
|
|
141
|
+
security boundary against hostile code. Use OS/container isolation for
|
|
142
|
+
that threat model. Explicit `forbidden_commands` are optional
|
|
143
|
+
substring checks, not a general shell parser.
|
|
144
|
+
|
|
145
|
+
Existing temporary runtimes keep old generated scripts: recreate them
|
|
146
|
+
before relying on these changes. Contracts without accepted checks and
|
|
147
|
+
scope cannot complete or amend through the current API.
|
|
148
|
+
|
|
149
|
+
For programmatic consumers, `create_harness_runtime` now requires
|
|
150
|
+
`validation_commands`. `update_harness_runtime` is asynchronous;
|
|
151
|
+
completion requires a trusted `HarnessVerificationOptions.run` backend
|
|
152
|
+
that executes the supplied commands in the supplied cwd and respects
|
|
153
|
+
the signal. The extension supplies this through Pi's tool pipeline.
|
|
154
|
+
Removed assessment-locking helpers are no longer exported.
|
|
210
155
|
|
|
211
156
|
## Development
|
|
212
157
|
|
|
@@ -215,7 +160,3 @@ pnpm --filter @spences10/pi-harness run check
|
|
|
215
160
|
pnpm --filter @spences10/pi-harness run test
|
|
216
161
|
pnpm --filter @spences10/pi-harness run build
|
|
217
162
|
```
|
|
218
|
-
|
|
219
|
-
## License
|
|
220
|
-
|
|
221
|
-
MIT
|
package/dist/assessment.d.ts
CHANGED
|
@@ -16,7 +16,7 @@ export declare const harness_assessment_submit_params_schema: Type.TObject<{
|
|
|
16
16
|
cwd: Type.TOptional<Type.TString>;
|
|
17
17
|
allowed_paths: Type.TOptional<Type.TArray<Type.TString>>;
|
|
18
18
|
forbidden_paths: Type.TOptional<Type.TArray<Type.TString>>;
|
|
19
|
-
validation_commands: Type.
|
|
19
|
+
validation_commands: Type.TArray<Type.TString>;
|
|
20
20
|
forbidden_commands: Type.TOptional<Type.TArray<Type.TString>>;
|
|
21
21
|
allow_test_changes: Type.TOptional<Type.TBoolean>;
|
|
22
22
|
escalation_rules: Type.TOptional<Type.TArray<Type.TString>>;
|
|
@@ -32,7 +32,7 @@ export type HarnessAssessParams = Static<typeof harness_assess_params_schema>;
|
|
|
32
32
|
export type HarnessAssessmentRecord = Static<typeof harness_assessment_submit_params_schema>;
|
|
33
33
|
export type HarnessAssessmentRecommendation = HarnessAssessmentRecord['recommendation'];
|
|
34
34
|
export type HarnessAssessmentSource = 'agent' | 'user' | 'harness-create';
|
|
35
|
-
export type HarnessAssessmentStatus = 'assessing' | 'awaiting_approval' | 'approved' | 'rejected';
|
|
35
|
+
export type HarnessAssessmentStatus = 'assessing' | 'assessed' | 'awaiting_approval' | 'approved' | 'rejected';
|
|
36
36
|
export interface HarnessAssessmentState {
|
|
37
37
|
id: string;
|
|
38
38
|
version: number;
|
|
@@ -40,20 +40,12 @@ export interface HarnessAssessmentState {
|
|
|
40
40
|
task: string;
|
|
41
41
|
source: HarnessAssessmentSource;
|
|
42
42
|
started_at: string;
|
|
43
|
-
tools_before_assessment
|
|
43
|
+
tools_before_assessment?: string[];
|
|
44
44
|
record?: HarnessAssessmentRecord;
|
|
45
45
|
decision?: HarnessAssessmentRecommendation;
|
|
46
46
|
}
|
|
47
|
-
export declare function create_assessment_state(task: string, source: HarnessAssessmentSource, tools_before_assessment
|
|
47
|
+
export declare function create_assessment_state(task: string, source: HarnessAssessmentSource, tools_before_assessment?: string[], now?: Date): HarnessAssessmentState;
|
|
48
48
|
export declare function submit_assessment_record(state: HarnessAssessmentState, record: HarnessAssessmentRecord): HarnessAssessmentState;
|
|
49
49
|
export declare function assessment_active(state: HarnessAssessmentState | undefined): boolean;
|
|
50
|
-
export declare function assessment_tool_names(active_tool_names: string[]): string[];
|
|
51
|
-
export declare function is_assessment_tool_allowed(tool_name: string): boolean;
|
|
52
|
-
export declare function check_assessment_command(command: string): {
|
|
53
|
-
ok: true;
|
|
54
|
-
} | {
|
|
55
|
-
ok: false;
|
|
56
|
-
reason: string;
|
|
57
|
-
};
|
|
58
50
|
export declare function assessment_context(state: HarnessAssessmentState): string;
|
|
59
51
|
export declare function format_assessment_record(state: HarnessAssessmentState): string;
|
package/dist/assessment.js
CHANGED
|
@@ -3,25 +3,16 @@ import { harness_create_params_schema } from './schema.js';
|
|
|
3
3
|
export const HARNESS_ASSESSMENT_CUSTOM_TYPE = 'pi-harness-assessment-state';
|
|
4
4
|
export const harness_assess_params_schema = Type.Object({
|
|
5
5
|
task: Type.String({
|
|
6
|
-
|
|
6
|
+
minLength: 1,
|
|
7
|
+
description: 'Optional investigation before choosing an execution path',
|
|
7
8
|
}),
|
|
8
9
|
});
|
|
9
10
|
export const harness_assessment_submit_params_schema = Type.Object({
|
|
10
|
-
task: Type.String(
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
})),
|
|
16
|
-
existing_primitives: Type.Array(Type.String({
|
|
17
|
-
description: 'Existing mechanisms that already satisfy part or all of the need',
|
|
18
|
-
})),
|
|
19
|
-
rejected_options: Type.Array(Type.String({
|
|
20
|
-
description: 'Rejected option and the evidence-based reason for rejection',
|
|
21
|
-
})),
|
|
22
|
-
smallest_vertical_slice: Type.String({
|
|
23
|
-
description: 'Smallest useful outcome or experiment that can test the need',
|
|
24
|
-
}),
|
|
11
|
+
task: Type.String(),
|
|
12
|
+
evidence: Type.Array(Type.String()),
|
|
13
|
+
existing_primitives: Type.Array(Type.String()),
|
|
14
|
+
rejected_options: Type.Array(Type.String()),
|
|
15
|
+
smallest_vertical_slice: Type.String(),
|
|
25
16
|
recommendation: Type.Union([
|
|
26
17
|
Type.Literal('harness'),
|
|
27
18
|
Type.Literal('direct'),
|
|
@@ -29,204 +20,40 @@ export const harness_assessment_submit_params_schema = Type.Object({
|
|
|
29
20
|
]),
|
|
30
21
|
proposed_contract: Type.Optional(harness_create_params_schema),
|
|
31
22
|
});
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
'context_get',
|
|
35
|
-
'context_list',
|
|
36
|
-
'context_search',
|
|
37
|
-
'context_stats',
|
|
38
|
-
'find',
|
|
39
|
-
'grep',
|
|
40
|
-
'harness_assess',
|
|
41
|
-
'harness_assessment_submit',
|
|
42
|
-
'harness_read',
|
|
43
|
-
'ls',
|
|
44
|
-
'lsp_definition',
|
|
45
|
-
'lsp_diagnostics',
|
|
46
|
-
'lsp_diagnostics_many',
|
|
47
|
-
'lsp_document_symbols',
|
|
48
|
-
'lsp_find_symbol',
|
|
49
|
-
'lsp_hover',
|
|
50
|
-
'lsp_references',
|
|
51
|
-
'mcp__mcp-omnisearch__ai_search',
|
|
52
|
-
'mcp__mcp-omnisearch__web_extract',
|
|
53
|
-
'mcp__mcp-omnisearch__web_search',
|
|
54
|
-
'mcp__mcp-sqlite-tools__close_database',
|
|
55
|
-
'mcp__mcp-sqlite-tools__database_info',
|
|
56
|
-
'mcp__mcp-sqlite-tools__describe_table',
|
|
57
|
-
'mcp__mcp-sqlite-tools__execute_read_query',
|
|
58
|
-
'mcp__mcp-sqlite-tools__export_schema',
|
|
59
|
-
'mcp__mcp-sqlite-tools__list_databases',
|
|
60
|
-
'mcp__mcp-sqlite-tools__list_tables',
|
|
61
|
-
'mcp__mcp-sqlite-tools__open_database',
|
|
62
|
-
'read',
|
|
63
|
-
]);
|
|
64
|
-
const SAFE_COMMAND_PATTERNS = [
|
|
65
|
-
/^(?:pwd|whoami|uname|date|uptime)(?:\s+.*)?$/,
|
|
66
|
-
/^(?:cat|head|tail|less|more|grep|rg|ls|wc|sort|uniq|diff|file|stat|du|df|tree|which|whereis|type|bat|eza)(?:\s+.*)?$/,
|
|
67
|
-
/^find(?:\s+.*)?$/,
|
|
68
|
-
/^git\s+(?:status|log|diff|show|rev-parse|ls-files|ls-tree|grep)(?:\s+.*)?$/,
|
|
69
|
-
/^git\s+branch(?:\s+(?:--list|-a|--all|-r|--remotes|--show-current|--contains|--merged|--no-merged)(?:\s+.*)?)?$/,
|
|
70
|
-
/^git\s+remote(?:\s+(?:-v|show|get-url)(?:\s+.*)?)?$/,
|
|
71
|
-
/^git\s+config\s+--get(?:\s+.*)?$/,
|
|
72
|
-
/^gh\s+(?:issue\s+(?:list|view)|pr\s+(?:list|view|diff|checks)|repo\s+view)(?:\s+.*)?$/,
|
|
73
|
-
/^gh\s+api(?:\s+.*)?$/,
|
|
74
|
-
/^(?:npm|pnpm|yarn)\s+(?:list|ls|view|info|why|outdated)(?:\s+.*)?$/,
|
|
75
|
-
/^(?:node|python|python3)\s+--version$/,
|
|
76
|
-
/^jq(?:\s+.*)?$/,
|
|
77
|
-
/^sed\s+-n(?:\s+.*)?$/,
|
|
78
|
-
];
|
|
79
|
-
const UNSAFE_READ_COMMAND_PATTERNS = [
|
|
80
|
-
/\bfind\b.*\s-(?:delete|exec|execdir|ok|okdir|fprint|fprintf|fls)\b/i,
|
|
81
|
-
/\brg\b.*\s--pre(?:-glob)?\b/i,
|
|
82
|
-
/\b(?:git\s+(?:log|diff|show)|sort)\b.*(?:\s--output(?:=|\s+)|\s-o\s+)/i,
|
|
83
|
-
/\bsed\b.*\s-i(?:\s|$|[^a-z])/i,
|
|
84
|
-
/\bgh\s+api\b.*(?:\s-X\s*|\s--method(?:=|\s+))(?!GET\b)/i,
|
|
85
|
-
/\bgh\s+api\b.*(?:\s-f\s|\s-F\s|\s--field(?:=|\s+)|\s--raw-field(?:=|\s+))/i,
|
|
86
|
-
/\bgit\s+branch\b.*\s-[dDmM]\b/i,
|
|
87
|
-
];
|
|
88
|
-
export function create_assessment_state(task, source, tools_before_assessment, now = new Date()) {
|
|
89
|
-
const normalized_task = task.trim();
|
|
90
|
-
if (!normalized_task) {
|
|
23
|
+
export function create_assessment_state(task, source, tools_before_assessment = [], now = new Date()) {
|
|
24
|
+
if (!task.trim())
|
|
91
25
|
throw new Error('Assessment task is required');
|
|
92
|
-
}
|
|
93
26
|
return {
|
|
94
27
|
id: `assessment-${now.getTime().toString(36)}`,
|
|
95
28
|
version: 1,
|
|
96
29
|
status: 'assessing',
|
|
97
|
-
task:
|
|
30
|
+
task: task.trim(),
|
|
98
31
|
source,
|
|
99
32
|
started_at: now.toISOString(),
|
|
100
|
-
tools_before_assessment
|
|
33
|
+
tools_before_assessment,
|
|
101
34
|
};
|
|
102
35
|
}
|
|
103
36
|
export function submit_assessment_record(state, record) {
|
|
104
|
-
if (record.recommendation === 'harness') {
|
|
105
|
-
if (!record.proposed_contract) {
|
|
106
|
-
throw new Error('A harness recommendation requires a proposed contract');
|
|
107
|
-
}
|
|
108
|
-
if (!record.proposed_contract.allowed_paths?.length) {
|
|
109
|
-
throw new Error('An approved harness contract requires explicit allowed paths');
|
|
110
|
-
}
|
|
111
|
-
if (!record.proposed_contract.validation_commands?.length) {
|
|
112
|
-
throw new Error('An approved harness contract requires validation commands');
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
37
|
return {
|
|
116
38
|
...state,
|
|
117
39
|
version: state.version + 1,
|
|
118
|
-
status: '
|
|
40
|
+
status: 'assessed',
|
|
119
41
|
task: record.task.trim(),
|
|
120
42
|
record,
|
|
121
|
-
decision: undefined,
|
|
122
43
|
};
|
|
123
44
|
}
|
|
124
45
|
export function assessment_active(state) {
|
|
125
|
-
return
|
|
126
|
-
state?.status === 'awaiting_approval');
|
|
127
|
-
}
|
|
128
|
-
export function assessment_tool_names(active_tool_names) {
|
|
129
|
-
const read_tools = [...new Set(active_tool_names)].filter((name) => ASSESSMENT_TOOL_NAMES.has(name));
|
|
130
|
-
return [
|
|
131
|
-
...new Set([
|
|
132
|
-
...read_tools,
|
|
133
|
-
'harness_assess',
|
|
134
|
-
'harness_assessment_submit',
|
|
135
|
-
'harness_read',
|
|
136
|
-
]),
|
|
137
|
-
];
|
|
138
|
-
}
|
|
139
|
-
export function is_assessment_tool_allowed(tool_name) {
|
|
140
|
-
return ASSESSMENT_TOOL_NAMES.has(tool_name);
|
|
141
|
-
}
|
|
142
|
-
function has_unquoted_shell_control(command) {
|
|
143
|
-
let quote;
|
|
144
|
-
for (let index = 0; index < command.length; index++) {
|
|
145
|
-
const character = command[index];
|
|
146
|
-
const next = command[index + 1];
|
|
147
|
-
if (quote === 'single') {
|
|
148
|
-
if (character === "'")
|
|
149
|
-
quote = undefined;
|
|
150
|
-
continue;
|
|
151
|
-
}
|
|
152
|
-
if (quote === 'double') {
|
|
153
|
-
if (character === '\\') {
|
|
154
|
-
index++;
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
if (character === '"') {
|
|
158
|
-
quote = undefined;
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
if (character === '`' || (character === '$' && next === '(')) {
|
|
162
|
-
return true;
|
|
163
|
-
}
|
|
164
|
-
continue;
|
|
165
|
-
}
|
|
166
|
-
if (character === '\\') {
|
|
167
|
-
index++;
|
|
168
|
-
continue;
|
|
169
|
-
}
|
|
170
|
-
if (character === "'") {
|
|
171
|
-
quote = 'single';
|
|
172
|
-
continue;
|
|
173
|
-
}
|
|
174
|
-
if (character === '"') {
|
|
175
|
-
quote = 'double';
|
|
176
|
-
continue;
|
|
177
|
-
}
|
|
178
|
-
if (character === '\n' ||
|
|
179
|
-
character === '\r' ||
|
|
180
|
-
character === ';' ||
|
|
181
|
-
character === '&' ||
|
|
182
|
-
character === '|' ||
|
|
183
|
-
character === '<' ||
|
|
184
|
-
character === '>' ||
|
|
185
|
-
character === '`' ||
|
|
186
|
-
(character === '$' && next === '(')) {
|
|
187
|
-
return true;
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
return quote !== undefined;
|
|
191
|
-
}
|
|
192
|
-
export function check_assessment_command(command) {
|
|
193
|
-
const normalized = command.trim();
|
|
194
|
-
if (!normalized) {
|
|
195
|
-
return { ok: false, reason: 'Assessment mode: empty command' };
|
|
196
|
-
}
|
|
197
|
-
if (has_unquoted_shell_control(normalized) ||
|
|
198
|
-
UNSAFE_READ_COMMAND_PATTERNS.some((pattern) => pattern.test(normalized))) {
|
|
199
|
-
return {
|
|
200
|
-
ok: false,
|
|
201
|
-
reason: 'Assessment mode: shell composition or a mutating command option is not allowed',
|
|
202
|
-
};
|
|
203
|
-
}
|
|
204
|
-
if (!SAFE_COMMAND_PATTERNS.some((pattern) => pattern.test(normalized))) {
|
|
205
|
-
return {
|
|
206
|
-
ok: false,
|
|
207
|
-
reason: 'Assessment mode: command is not in the read-only allowlist; use one read command per call',
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
return { ok: true };
|
|
46
|
+
return state?.status === 'assessing';
|
|
211
47
|
}
|
|
212
48
|
export function assessment_context(state) {
|
|
213
|
-
|
|
214
|
-
? 'An assessment record is waiting for direct user approval. Do not start implementation or call harness_create.'
|
|
215
|
-
: 'Investigate with read-only tools, then call harness_assessment_submit with one evidence-based recommendation.';
|
|
216
|
-
return `## Active harness assessment\n\nAssessment: \`${state.id}\` v${state.version}\nTask: ${state.task}\n\nRepository mutation is disabled. Treat proposed capabilities as candidates, not tasks. Establish the current source of truth, find existing primitives, reject unsupported options, and define the smallest useful vertical slice. A harness contract is optional and is valid only when the evidence shows that an enforceable execution contract adds value. Bash accepts one allowlisted read command per call; use the dedicated read, find, grep, LSP, and research tools when available.\n\n${approval}`;
|
|
49
|
+
return `## Optional harness assessment\n\nTask: ${state.task}\n\nInvestigate the need, existing primitives, and smallest useful change before recommending an execution path. This is advisory planning, not a read-only security mode: tools remain available and normal user authorization still applies. Submit findings with harness_assessment_submit. Assessment is not required to create a harness.`;
|
|
217
50
|
}
|
|
218
51
|
export function format_assessment_record(state) {
|
|
219
52
|
const record = state.record;
|
|
220
|
-
if (!record)
|
|
221
|
-
return `Assessment ${state.id}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
: `- ${empty}`;
|
|
226
|
-
const contract = record.proposed_contract;
|
|
227
|
-
const contract_section = contract
|
|
228
|
-
? `\n\n## Proposed harness contract\n\n- Task: ${contract.task}\n- Cwd: ${contract.cwd ?? 'current project'}\n- Allowed paths: ${contract.allowed_paths?.join(', ') ?? 'none'}\n- Validation: ${contract.validation_commands?.join(' && ') ?? 'none'}\n- Test changes: ${contract.allow_test_changes ? 'allowed' : 'not allowed'}\n- Forbidden paths: ${contract.forbidden_paths?.join(', ') ?? 'defaults'}\n- Forbidden commands: ${contract.forbidden_commands?.join(', ') ?? 'defaults'}\n- Escalation rules: ${contract.escalation_rules?.join('; ') ?? 'defaults'}\n- Planner: ${contract.planner_model ?? 'default'} (${contract.planner_thinking ?? 'default'})\n- Executor: ${contract.executor_model ?? 'default'} (${contract.executor_thinking ?? 'default'})\n- Reviewer: ${contract.reviewer_model ?? 'default'} (${contract.reviewer_thinking ?? 'default'})`
|
|
229
|
-
: '';
|
|
230
|
-
return `# Harness assessment: ${state.id} v${state.version}\n\n- Status: ${state.status}\n- Recommendation: ${record.recommendation}\n\n## Outcome\n\n${record.task}\n\n## Evidence\n\n${list(record.evidence, 'No evidence recorded')}\n\n## Existing primitives\n\n${list(record.existing_primitives, 'None found')}\n\n## Rejected options\n\n${list(record.rejected_options, 'None recorded')}\n\n## Smallest vertical slice\n\n${record.smallest_vertical_slice}${contract_section}`;
|
|
53
|
+
if (!record)
|
|
54
|
+
return `Assessment ${state.id}: ${state.task}`;
|
|
55
|
+
const list = (values) => values.map((value) => `- ${value}`).join('\n') ||
|
|
56
|
+
'- None recorded';
|
|
57
|
+
return `# Harness assessment\n\n${record.task}\n\nRecommendation: ${record.recommendation}\n\n## Evidence\n\n${list(record.evidence)}\n\n## Existing primitives\n\n${list(record.existing_primitives)}\n\n## Rejected options\n\n${list(record.rejected_options)}\n\n## Smallest vertical slice\n\n${record.smallest_vertical_slice}`;
|
|
231
58
|
}
|
|
232
59
|
//# sourceMappingURL=assessment.js.map
|
package/dist/assessment.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"assessment.js","sourceRoot":"","sources":["../src/assessment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAe,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAE3D,MAAM,CAAC,MAAM,8BAA8B,GAC1C,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"assessment.js","sourceRoot":"","sources":["../src/assessment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAe,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAE3D,MAAM,CAAC,MAAM,8BAA8B,GAC1C,6BAA6B,CAAC;AAC/B,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC,MAAM,CAAC;IACvD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;QACjB,SAAS,EAAE,CAAC;QACZ,WAAW,EACV,0DAA0D;KAC3D,CAAC;CACF,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,uCAAuC,GAAG,IAAI,CAAC,MAAM,CAAC;IAClE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE;IACnB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IACnC,mBAAmB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IAC9C,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IAC3C,uBAAuB,EAAE,IAAI,CAAC,MAAM,EAAE;IACtC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;QACvB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACtB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;KACtB,CAAC;IACF,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,4BAA4B,CAAC;CAC9D,CAAC,CAAC;AA+BH,MAAM,UAAU,uBAAuB,CACtC,IAAY,EACZ,MAA+B,EAC/B,uBAAuB,GAAa,EAAE,EACtC,GAAG,GAAG,IAAI,IAAI,EAAE;IAEhB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjE,OAAO;QACN,EAAE,EAAE,cAAc,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;QAC9C,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,WAAW;QACnB,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QACjB,MAAM;QACN,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE;QAC7B,uBAAuB;KACvB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,wBAAwB,CACvC,KAA6B,EAC7B,MAA+B;IAE/B,OAAO;QACN,GAAG,KAAK;QACR,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,CAAC;QAC1B,MAAM,EAAE,UAAU;QAClB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE;QACxB,MAAM;KACN,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAChC,KAAyC;IAEzC,OAAO,KAAK,EAAE,MAAM,KAAK,WAAW,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,KAA6B;IAE7B,OAAO,2CAA2C,KAAK,CAAC,IAAI,gVAAgV,CAAC;AAC9Y,CAAC;AAED,MAAM,UAAU,wBAAwB,CACvC,KAA6B;IAE7B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,IAAI,CAAC,MAAM;QAAE,OAAO,cAAc,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5D,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAE,EAAE,CACjC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9C,iBAAiB,CAAC;IACnB,OAAO,2BAA2B,MAAM,CAAC,IAAI,uBAAuB,MAAM,CAAC,cAAc,sBAAsB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,iCAAiC,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,8BAA8B,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,qCAAqC,MAAM,CAAC,uBAAuB,EAAE,CAAC;AACvU,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { HarnessContract } from '../schema.js';
|
|
2
|
+
export declare function pattern_matches(cwd: string, pattern: string, path: string): boolean;
|
|
2
3
|
export declare function check_path_allowed(contract: HarnessContract, input_path: string): {
|
|
3
4
|
ok: true;
|
|
4
5
|
} | {
|
|
@@ -11,3 +12,4 @@ export declare function check_command_allowed(contract: HarnessContract, command
|
|
|
11
12
|
ok: false;
|
|
12
13
|
reason: string;
|
|
13
14
|
};
|
|
15
|
+
export declare function build_path_helpers(): string;
|