harness-everything 0.11.7 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +53 -0
- package/README.md +1 -0
- package/behavioral-evals/README.md +37 -0
- package/behavioral-evals/cases/self-evolve-retrieval-recurrence.yaml +48 -0
- package/behavioral-evals/paired-benchmark.js +211 -5
- package/ci/contract-integrity.test.js +68 -2
- package/ci/mechanism-2ah-workflow-lifecycle.test.js +258 -2
- package/ci/mechanism-32-paired-benchmark.test.js +139 -0
- package/ci/openai-plugin-package.test.js +17 -0
- package/ci/self-evolve-loop.test.js +37 -2
- package/ci/telemetry.test.js +31 -1
- package/docs/enforcement-locks.md +114 -0
- package/grill-with-docs/ADR-FORMAT.md +22 -0
- package/grill-with-docs/SKILL.md +1 -1
- package/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/hooks/scripts/lib/telemetry.js +40 -0
- package/hooks/scripts/lib/workflow-isolation.js +10 -2
- package/hooks/scripts/lib/workflow-runtime.js +108 -5
- package/hooks/scripts/state-persist.js +9 -3
- package/hooks/scripts/workflow-disposition.js +19 -8
- package/hooks/scripts/workflow-gate.js +33 -4
- package/multi-agent-workspace/SKILL.md +1 -1
- package/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/contract-integrity/scripts/audit.js +536 -0
- package/plugins/harness-everything/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/plugins/harness-everything/hooks/scripts/lib/telemetry.js +40 -0
- package/plugins/harness-everything/hooks/scripts/lib/workflow-isolation.js +10 -2
- package/plugins/harness-everything/hooks/scripts/lib/workflow-runtime.js +108 -5
- package/plugins/harness-everything/hooks/scripts/state-persist.js +9 -3
- package/plugins/harness-everything/hooks/scripts/workflow-disposition.js +19 -8
- package/plugins/harness-everything/hooks/scripts/workflow-gate.js +33 -4
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/grill-with-docs/ADR-FORMAT.md +22 -0
- package/plugins/harness-everything/skills/grill-with-docs/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/plugins/harness-everything/skills/self-evolve/SKILL.md +1 -1
- package/plugins/harness-everything/skills/self-evolve/scripts/lesson-candidate.js +52 -0
- package/plugins/harness-everything/skills/to-spec/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-spec/references/process.md +26 -0
- package/plugins/harness-everything/skills/to-spec/templates/cli-reference.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/dev-doc.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/feature-spec.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/schema-doc.md +14 -0
- package/plugins/harness-everything/skills/to-tickets/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-tickets/references/publishing-and-templates.md +20 -1
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +3 -2
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +25 -1
- package/plugins/harness-everything/skills/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/plugins/harness-everything/skills/verification-loop/templates/verification-report.template.md +3 -0
- package/scripts/sync-openai-plugin.js +12 -0
- package/self-evolve/SKILL.md +1 -1
- package/self-evolve/scripts/lesson-candidate.js +52 -0
- package/telemetry/schemas/event.schema.json +141 -22
- package/telemetry/scripts/report.js +42 -0
- package/to-spec/SKILL.md +1 -1
- package/to-spec/references/process.md +26 -0
- package/to-spec/templates/cli-reference.md +14 -0
- package/to-spec/templates/dev-doc.md +14 -0
- package/to-spec/templates/feature-spec.md +14 -0
- package/to-spec/templates/schema-doc.md +14 -0
- package/to-tickets/SKILL.md +1 -1
- package/to-tickets/references/publishing-and-templates.md +20 -1
- package/verification-loop/SKILL.md +3 -2
- package/verification-loop/references/verification-phases.md +25 -1
- package/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/verification-loop/templates/verification-report.template.md +3 -0
|
@@ -28,4 +28,18 @@ What breaks for existing consumers, if anything, and what the mitigation is (dua
|
|
|
28
28
|
|
|
29
29
|
One realistic, complete example instance of the schema.
|
|
30
30
|
|
|
31
|
+
## Contract Requirements
|
|
32
|
+
|
|
33
|
+
For every normative schema behavior this document owns, assign a stable requirement identity:
|
|
34
|
+
|
|
35
|
+
```md
|
|
36
|
+
### REQ-001 — <short behavior name>
|
|
37
|
+
**Status:** CURRENT
|
|
38
|
+
**Revision:** 1
|
|
39
|
+
**Decision lineage:** ADR-0001 (or None)
|
|
40
|
+
<observable validation, compatibility, or migration contract>
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Do not recycle requirement IDs. A changed schema/validation contract increments the revision before implementation completion.
|
|
44
|
+
|
|
31
45
|
</spec-template>
|
package/to-tickets/SKILL.md
CHANGED
|
@@ -48,6 +48,19 @@ Give each ticket its **blocking edges** — the other tickets that must complete
|
|
|
48
48
|
|
|
49
49
|
**Wide refactors are the exception to vertical slicing.** A **wide refactor** is one mechanical change — rename a column, retype a shared symbol — whose **blast radius** fans across the whole codebase, so a single edit breaks thousands of call sites at once and no vertical slice can land green. Don't force it into a tracer bullet; sequence it as **expand–contract**. First expand: add the new form beside the old so nothing breaks. Then migrate the call sites over in batches sized by blast radius (per package, per directory), each batch its own ticket blocked by the expand, keeping CI green batch to batch because the old form still exists. Finally contract: delete the old form once no caller remains, in a ticket blocked by every migrate batch. When even the batches can't stay green alone, keep the sequence but let them share an integration branch that all block a final integrate-and-verify ticket — green is promised only there.
|
|
50
50
|
|
|
51
|
+
# Contract lineage and change impact
|
|
52
|
+
|
|
53
|
+
When a ticket comes from an authoritative spec, it must carry the exact source requirement identity and revision. A ticket may refine implementation scope, but it never silently rewrites the approved contract.
|
|
54
|
+
|
|
55
|
+
Every ticket must classify its impact:
|
|
56
|
+
|
|
57
|
+
- **none** — bookkeeping/non-behavioral work; explain why no contract changes.
|
|
58
|
+
- **implementation** — implementation changes while approved behavior and architectural rationale stay unchanged.
|
|
59
|
+
- **behavior** — caller/user-visible behavior changes. Update the living spec requirement and revision first, then obtain requirement-linked RED evidence before accepting implementation.
|
|
60
|
+
- **architecture** — a hard-to-reverse decision/rationale changes. Publish a new/superseding ADR, update the living spec requirement/revision, then obtain RED evidence.
|
|
61
|
+
|
|
62
|
+
If the impact classification changes during implementation, stop treating the original ticket as completion authority. Reconcile the source artifacts first, then update the ticket's source revision.
|
|
63
|
+
|
|
51
64
|
# Ticket Templates
|
|
52
65
|
|
|
53
66
|
<local-ticket-template>
|
|
@@ -58,9 +71,15 @@ Give each ticket its **blocking edges** — the other tickets that must complete
|
|
|
58
71
|
|
|
59
72
|
**Blocked by:** the numbers/titles of the tickets that gate this one, or "None — can start immediately".
|
|
60
73
|
|
|
74
|
+
**Source requirements:** `REQ-001@rev1` (add every requirement this ticket changes or proves).
|
|
75
|
+
|
|
76
|
+
**Change impact:** `none | implementation | behavior | architecture`
|
|
77
|
+
|
|
78
|
+
**Impact rationale:** one sentence explaining why this classification is correct.
|
|
79
|
+
|
|
61
80
|
**Status:** ready-for-agent
|
|
62
81
|
|
|
63
|
-
- [ ] Acceptance criterion 1
|
|
82
|
+
- [ ] [REQ-001] Acceptance criterion 1
|
|
64
83
|
- [ ] Acceptance criterion 2
|
|
65
84
|
|
|
66
85
|
</local-ticket-template>
|
|
@@ -4,7 +4,7 @@ description: "Run objective verification gates (build, types, lint, tests, secur
|
|
|
4
4
|
license: Apache-2.0
|
|
5
5
|
metadata:
|
|
6
6
|
author: Miya Daniel
|
|
7
|
-
version: 0.
|
|
7
|
+
version: 0.12.0
|
|
8
8
|
---
|
|
9
9
|
|
|
10
10
|
# Verification Loop
|
|
@@ -41,6 +41,7 @@ and rerun from the first affected gate.
|
|
|
41
41
|
4. Tests and coverage: `npm run test -- --coverage`.
|
|
42
42
|
5. Security scan: secrets and stray logs.
|
|
43
43
|
6. Diff review: `git diff --stat` and unintended-change checks.
|
|
44
|
-
7.
|
|
44
|
+
7. Contract reconciliation when the work cites `REQ-*` / ADR/spec/ticket lineage or a contract-integrity trace exists: run the strict `<this-skill-dir>/scripts/contract-integrity-audit.js` gate with current #58 evidence. `AUDIT`, `NOT_EVALUATED`, stale source binding, or drift is **not ready** for delivery.
|
|
45
|
+
8. Fill `<this-skill-dir>/templates/verification-report.template.md` with evidence.
|
|
45
46
|
|
|
46
47
|
Deep dive: `<this-skill-dir>/references/verification-phases.md`
|
|
@@ -10,8 +10,14 @@ flowchart TD
|
|
|
10
10
|
Lint --> Test[4. Phase 4: Test Suite Execution]
|
|
11
11
|
Test --> Security[5. Phase 5: Security & Secret Scan]
|
|
12
12
|
Security --> Diff[6. Phase 6: Diff Review]
|
|
13
|
+
Diff --> Contract{7. Contract lineage applicable?}
|
|
14
|
+
Contract -- Yes --> Reconcile[Run contract-integrity strict reconciliation]
|
|
15
|
+
Contract -- No --> CheckScript
|
|
16
|
+
Reconcile --> ContractResult{PASS?}
|
|
17
|
+
ContractResult -- No --> Fix
|
|
18
|
+
ContractResult -- Yes --> CheckScript
|
|
13
19
|
|
|
14
|
-
Diff --> CheckScript{
|
|
20
|
+
Diff --> CheckScript{8. Is verify-gate.js / Template Available?}
|
|
15
21
|
|
|
16
22
|
CheckScript -- Yes --> RunScript[Run harness-everything/scripts/verify-gate.js / Fill verification-report.template.md]
|
|
17
23
|
CheckScript -- No --> DirectReport[Output Verification Report Inline in Response]
|
|
@@ -93,6 +99,24 @@ Review each changed file for:
|
|
|
93
99
|
- Missing error handling
|
|
94
100
|
- Potential edge cases
|
|
95
101
|
|
|
102
|
+
## Phase 7: Contract Reconciliation
|
|
103
|
+
|
|
104
|
+
This phase is applicable when any of the following is true:
|
|
105
|
+
|
|
106
|
+
- the task/spec/ticket cites stable `REQ-*` identities;
|
|
107
|
+
- an ADR/spec/ticket change is classified as `behavior` or `architecture`;
|
|
108
|
+
- the project has a contract-integrity trace for the affected scope.
|
|
109
|
+
|
|
110
|
+
Use the current trace plus current #58 TDD evidence:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
node "<this-skill-dir>/scripts/contract-integrity-audit.js" <trace.json> --tdd-evidence <tdd-evidence.json> --output <report.json> --markdown <report.md>
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Delivery requires strict **PASS**. Treat `FAIL`, `AUDIT`, `NOT_EVALUATED`, source conflicts/defects, stale spec/test/implementation lineage, surviving required probes, and audit-only output as **NOT READY**. Do not downgrade the gate merely because build/tests are otherwise green.
|
|
117
|
+
|
|
118
|
+
If this phase is not applicable, record `NOT_APPLICABLE` with a one-line reason. Never use `NOT_APPLICABLE` to hide known requirement lineage.
|
|
119
|
+
|
|
96
120
|
## Output Format
|
|
97
121
|
|
|
98
122
|
After running all phases, fill in `verification-loop/templates/verification-report.template.md` with the actual results and present it.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
|
|
7
|
+
function resolveAuditRuntime() {
|
|
8
|
+
const candidates = [
|
|
9
|
+
path.resolve(__dirname, '../../contract-integrity/scripts/audit.js'),
|
|
10
|
+
path.resolve(__dirname, '../../../contract-integrity/scripts/audit.js'),
|
|
11
|
+
];
|
|
12
|
+
for (const candidate of candidates) {
|
|
13
|
+
if (fs.existsSync(candidate)) return candidate;
|
|
14
|
+
}
|
|
15
|
+
throw new Error('contract-integrity audit runtime is not packaged with this verification-loop installation');
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function main() {
|
|
19
|
+
try {
|
|
20
|
+
const target = resolveAuditRuntime();
|
|
21
|
+
const { runCli } = require(target);
|
|
22
|
+
if (typeof runCli !== 'function') throw new Error('contract-integrity audit runtime does not export runCli');
|
|
23
|
+
process.exitCode = runCli([process.execPath, target, ...process.argv.slice(2)]);
|
|
24
|
+
} catch (error) {
|
|
25
|
+
console.error('[Verification Contract Gate] ' + error.message);
|
|
26
|
+
process.exitCode = 2;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
if (require.main === module) main();
|
|
31
|
+
module.exports = { resolveAuditRuntime };
|
|
@@ -7,6 +7,9 @@ Lint: [PASS/FAIL] (X warnings)
|
|
|
7
7
|
Tests: [PASS/FAIL] (X/Y passed, Z% coverage)
|
|
8
8
|
Security: [PASS/FAIL] (X issues)
|
|
9
9
|
Diff: [X files changed]
|
|
10
|
+
Contract: [PASS/FAIL/NOT_APPLICABLE] (REQ IDs / trace report / reason)
|
|
11
|
+
Strict PASS required when contract lineage is applicable.
|
|
12
|
+
AUDIT or NOT_EVALUATED is NOT READY.
|
|
10
13
|
|
|
11
14
|
Overall: [READY/NOT READY] for PR
|
|
12
15
|
|