harness-everything 0.11.6 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +45 -0
- package/README.md +7 -3
- package/behavioral-evals/README.md +37 -0
- package/behavioral-evals/cases/self-evolve-retrieval-recurrence.yaml +48 -0
- package/behavioral-evals/paired-benchmark.js +211 -5
- package/ci/contract-integrity.test.js +68 -2
- package/ci/mechanism-2a-rule-of-3.test.js +71 -0
- package/ci/mechanism-2ah-workflow-lifecycle.test.js +276 -0
- package/ci/mechanism-2c-state-persist.test.js +45 -2
- package/ci/mechanism-2t-issue-20.test.js +0 -8
- package/ci/mechanism-30-opencode-plugin-loadability.test.js +39 -7
- package/ci/mechanism-32-paired-benchmark.test.js +139 -0
- package/ci/openai-plugin-package.test.js +17 -0
- package/ci/self-evolve-loop.test.js +37 -2
- package/ci/telemetry.test.js +31 -1
- package/docs/architecture.md +17 -0
- package/docs/audit.md +52 -5
- package/docs/release-evidence.md +3 -0
- package/docs/workflow-runtime.md +13 -0
- package/docs/workflows/environment-detection.md +47 -37
- package/docs/workflows/eval-harness.md +45 -38
- package/docs/workflows/fable-discipline.md +45 -33
- package/docs/workflows/git-commit.md +53 -45
- package/docs/workflows/grill-me.md +36 -38
- package/docs/workflows/grill-with-docs.md +36 -46
- package/docs/workflows/harness-everything.md +59 -57
- package/docs/workflows/improve-codebase-architecture.md +36 -44
- package/docs/workflows/install-cognitive-os.md +36 -29
- package/docs/workflows/repo-docs.md +29 -40
- package/docs/workflows/rewrite-commits.md +40 -36
- package/docs/workflows/security-review.md +34 -53
- package/docs/workflows/skill-creator.md +39 -56
- package/docs/workflows/skill-style.md +35 -40
- package/docs/workflows/using-git-worktrees.md +40 -44
- package/docs/workflows/verification-loop.md +43 -48
- package/docs/workflows/verify-before-claim.md +37 -40
- package/docs/workflows/zoom-out.md +39 -51
- package/grill-with-docs/ADR-FORMAT.md +22 -0
- package/grill-with-docs/SKILL.md +1 -1
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/kernel-router.js +38 -10
- package/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/hooks/scripts/lib/telemetry.js +40 -0
- package/hooks/scripts/lib/workflow-isolation.js +30 -1
- package/hooks/scripts/lib/workflow-runtime.js +18 -1
- package/hooks/scripts/rule-of-3-tracker.js +17 -9
- package/hooks/scripts/state-persist.js +19 -7
- package/hooks/scripts/workflow-disposition.js +19 -8
- package/hooks/scripts/workflow-gate.js +21 -4
- package/multi-agent-workspace/SKILL.md +1 -1
- package/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/contract-integrity/scripts/audit.js +536 -0
- package/plugins/harness-everything/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/plugins/harness-everything/hooks/scripts/lib/telemetry.js +40 -0
- package/plugins/harness-everything/hooks/scripts/lib/workflow-isolation.js +30 -1
- package/plugins/harness-everything/hooks/scripts/lib/workflow-runtime.js +18 -1
- package/plugins/harness-everything/hooks/scripts/rule-of-3-tracker.js +17 -9
- package/plugins/harness-everything/hooks/scripts/state-persist.js +19 -7
- package/plugins/harness-everything/hooks/scripts/workflow-disposition.js +19 -8
- package/plugins/harness-everything/hooks/scripts/workflow-gate.js +21 -4
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/grill-with-docs/ADR-FORMAT.md +22 -0
- package/plugins/harness-everything/skills/grill-with-docs/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +38 -10
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/plugins/harness-everything/skills/self-evolve/SKILL.md +1 -1
- package/plugins/harness-everything/skills/self-evolve/scripts/lesson-candidate.js +52 -0
- package/plugins/harness-everything/skills/to-spec/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-spec/references/process.md +26 -0
- package/plugins/harness-everything/skills/to-spec/templates/cli-reference.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/dev-doc.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/feature-spec.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/schema-doc.md +14 -0
- package/plugins/harness-everything/skills/to-tickets/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-tickets/references/publishing-and-templates.md +20 -1
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +3 -2
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +25 -1
- package/plugins/harness-everything/skills/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/plugins/harness-everything/skills/verification-loop/templates/verification-report.template.md +3 -0
- package/scripts/sync-openai-plugin.js +12 -0
- package/self-evolve/SKILL.md +1 -1
- package/self-evolve/scripts/lesson-candidate.js +52 -0
- package/telemetry/schemas/event.schema.json +141 -22
- package/telemetry/scripts/report.js +42 -0
- package/to-spec/SKILL.md +1 -1
- package/to-spec/references/process.md +26 -0
- package/to-spec/templates/cli-reference.md +14 -0
- package/to-spec/templates/dev-doc.md +14 -0
- package/to-spec/templates/feature-spec.md +14 -0
- package/to-spec/templates/schema-doc.md +14 -0
- package/to-tickets/SKILL.md +1 -1
- package/to-tickets/references/publishing-and-templates.md +20 -1
- package/verification-loop/SKILL.md +3 -2
- package/verification-loop/references/verification-phases.md +25 -1
- package/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/verification-loop/templates/verification-report.template.md +3 -0
- package/docs/issue-20-rollup.md +0 -47
- package/docs/reviews/pr-142-issue-131.md +0 -89
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
const fs = require('fs');
|
|
3
3
|
const path = require('path');
|
|
4
4
|
const { getWorkspaceRoot, getSessionDir } = require('./lib/harness-state');
|
|
5
|
-
const { loadWorkflow, saveWorkflow, peekMutationProbe, settleMutationProbe } = require('./lib/workflow-runtime');
|
|
6
|
-
const { cwdOf, shellProbeKey, workspaceFingerprint } = require('./lib/workflow-isolation');
|
|
5
|
+
const { loadWorkflow, saveWorkflow, isMajorWorkflow, peekMutationProbe, settleMutationProbe, settleMutationProbeConservative } = require('./lib/workflow-runtime');
|
|
6
|
+
const { cwdOf, shellProbeKey, workspaceFingerprint, directMutationInWorkspace } = require('./lib/workflow-isolation');
|
|
7
7
|
const { observeTool } = require('./lib/telemetry');
|
|
8
8
|
|
|
9
9
|
// Commands that count as "verification ran" for the Stop gate
|
|
@@ -42,6 +42,10 @@ function observeWorkspaceMutation(payload, root) {
|
|
|
42
42
|
try {
|
|
43
43
|
after = workspaceFingerprint(cwd);
|
|
44
44
|
} catch (error) {
|
|
45
|
+
if (!isMajorWorkflow(context.workflow)) {
|
|
46
|
+
const conservative = settleMutationProbeConservative(context, probeKey, toolName, 'unobservable-workspace');
|
|
47
|
+
return Boolean(conservative && conservative.changed);
|
|
48
|
+
}
|
|
45
49
|
context.workflow.state = 'blocked';
|
|
46
50
|
context.workflow.blockReason = 'mutation-observation-failed';
|
|
47
51
|
saveWorkflow(context);
|
|
@@ -79,9 +83,10 @@ function processState(payload) {
|
|
|
79
83
|
const exitCode = typeof rawExitCode === 'number' ? rawExitCode : undefined;
|
|
80
84
|
const stderrSignal = typeof stderr === 'string' && stderr.trim().length > 0;
|
|
81
85
|
const hookEvent = payload.hook_event_name || payload.hookEventName || '';
|
|
86
|
+
const isSuccessEvent = hookEvent === 'PostToolUse';
|
|
82
87
|
const isFailed = hookEvent === 'PostToolUseFailure' ||
|
|
83
88
|
(exitCode !== undefined && exitCode !== 0) ||
|
|
84
|
-
(exitCode === undefined && stderrSignal);
|
|
89
|
+
(!isSuccessEvent && exitCode === undefined && stderrSignal);
|
|
85
90
|
const toolName = payload.tool_name || payload.tool || 'command';
|
|
86
91
|
const observedMutation = observeWorkspaceMutation(payload, root);
|
|
87
92
|
|
|
@@ -97,8 +102,13 @@ function processState(payload) {
|
|
|
97
102
|
state.tool = toolName;
|
|
98
103
|
state.exitCode = exitCode;
|
|
99
104
|
state.errorSummary = output.trim();
|
|
100
|
-
} else if (
|
|
101
|
-
// Clear failed state or mark as idle/resolved
|
|
105
|
+
} else if (!isFailed && state.status === 'failed') {
|
|
106
|
+
// Clear failed state or mark as idle/resolved.
|
|
107
|
+
// NOTE (#166, follows #153): Claude Code Bash PostToolUse payloads
|
|
108
|
+
// carry no numeric exit code, so `exitCode === 0` never holds there.
|
|
109
|
+
// A PostToolUse event with no numeric non-zero exit status counts as
|
|
110
|
+
// success even when stderr contains diagnostics. Hosts that omit the
|
|
111
|
+
// lifecycle event name retain the conservative stderr fallback.
|
|
102
112
|
state.lastResolved = { tool: state.tool, timestamp: state.timestamp };
|
|
103
113
|
state.status = 'idle';
|
|
104
114
|
state.timestamp = new Date().toISOString();
|
|
@@ -107,8 +117,10 @@ function processState(payload) {
|
|
|
107
117
|
}
|
|
108
118
|
|
|
109
119
|
// Milestones for the Stop gate: when did the last mutation happen, and
|
|
110
|
-
// has any verification-ish command succeeded since.
|
|
111
|
-
|
|
120
|
+
// has any verification-ish command succeeded since. Direct-tool edits
|
|
121
|
+
// outside the workspace advance neither milestone (#165).
|
|
122
|
+
const isDirectTool = toolName === 'Edit' || toolName === 'Write' || toolName === 'apply_patch';
|
|
123
|
+
if ((isDirectTool && directMutationInWorkspace(payload, cwdOf(payload, root), root)) || observedMutation) {
|
|
112
124
|
state.lastEditAt = Date.now();
|
|
113
125
|
}
|
|
114
126
|
if ((toolName === 'Bash' || toolName === 'PowerShell') && !isFailed) {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
const fs = require('fs');
|
|
5
5
|
const path = require('path');
|
|
6
|
-
const { loadWorkflow, saveWorkflow, matchingRun, OPEN_STATES, ensureWorkflowBudget, recordBudgetEvent, resetWorkflowBudget, syncBudgetToRun, WORKFLOW_CONTROLLER_COMMANDS } = require('./lib/workflow-runtime');
|
|
6
|
+
const { loadWorkflow, saveWorkflow, matchingRun, OPEN_STATES, budgetLimits, ensureWorkflowBudget, recordBudgetEvent, resetWorkflowBudget, syncBudgetToRun, WORKFLOW_CONTROLLER_COMMANDS } = require('./lib/workflow-runtime');
|
|
7
7
|
const { getWorkspaceRoot, readCurrentSession } = require('./lib/harness-state');
|
|
8
8
|
const { atomicWriteJson, readJson } = require('./lib/fable-contracts');
|
|
9
9
|
|
|
@@ -32,6 +32,20 @@ function consumer() {
|
|
|
32
32
|
throw new Error('Fable consumer unavailable');
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
function activatePlan(workflow, plan) {
|
|
36
|
+
workflow.workflowPlan = plan;
|
|
37
|
+
workflow.strategy = plan.strategy;
|
|
38
|
+
workflow.strategySelection = plan.strategySelection;
|
|
39
|
+
workflow.tier = plan.tier;
|
|
40
|
+
workflow.requiredInvariants = plan.requiredInvariants || [];
|
|
41
|
+
workflow.suggestedSkills = plan.suggestedSkills || [];
|
|
42
|
+
workflow.verification = plan.verification;
|
|
43
|
+
workflow.mutationIsolation = plan.mutationIsolation || { required: false };
|
|
44
|
+
const budget = ensureWorkflowBudget(workflow);
|
|
45
|
+
budget.limits = budgetLimits(workflow);
|
|
46
|
+
budget.updatedAt = new Date().toISOString();
|
|
47
|
+
}
|
|
48
|
+
|
|
35
49
|
function main() {
|
|
36
50
|
const args = parseArgs(process.argv.slice(2));
|
|
37
51
|
if (!WORKFLOW_CONTROLLER_COMMANDS.has(args.command)) throw new Error(usage());
|
|
@@ -47,10 +61,12 @@ function main() {
|
|
|
47
61
|
if (isReplan) recordBudgetEvent(context, 'replan', { evidence: 'workflow-disposition:start' });
|
|
48
62
|
const plan = workflow.pendingPlan || workflow.workflowPlan;
|
|
49
63
|
if (!String(plan.strategy || '').startsWith('fable-')) {
|
|
64
|
+
activatePlan(workflow, plan);
|
|
50
65
|
workflow.revision++;
|
|
51
66
|
workflow.state = 'running';
|
|
67
|
+
workflow.escapes = [];
|
|
68
|
+
delete workflow.pendingPlan;
|
|
52
69
|
delete workflow.blockReason;
|
|
53
|
-
ensureWorkflowBudget(workflow);
|
|
54
70
|
saveWorkflow(context);
|
|
55
71
|
process.stdout.write(JSON.stringify({ state: workflow.state, workflowId: workflow.workflowId, revision: workflow.revision }) + '\n');
|
|
56
72
|
return;
|
|
@@ -66,18 +82,13 @@ function main() {
|
|
|
66
82
|
routerContract: { workflowPlan: plan }, stages, workspaceRoot: context.root,
|
|
67
83
|
sessionId, workflowId: workflow.workflowId,
|
|
68
84
|
});
|
|
69
|
-
workflow
|
|
70
|
-
workflow.strategy = plan.strategy;
|
|
71
|
-
workflow.tier = plan.tier;
|
|
72
|
-
workflow.verification = plan.verification;
|
|
73
|
-
workflow.mutationIsolation = plan.mutationIsolation || workflow.mutationIsolation;
|
|
85
|
+
activatePlan(workflow, plan);
|
|
74
86
|
workflow.runId = run.runId;
|
|
75
87
|
workflow.revision++;
|
|
76
88
|
workflow.state = 'running';
|
|
77
89
|
workflow.escapes = [];
|
|
78
90
|
delete workflow.pendingPlan;
|
|
79
91
|
delete workflow.blockReason;
|
|
80
|
-
ensureWorkflowBudget(workflow);
|
|
81
92
|
} else if (args.command === 'revision') {
|
|
82
93
|
recordBudgetEvent(context, 'revision', { evidence: args.evidence || 'workflow-disposition:revision' });
|
|
83
94
|
} else if (args.command === 'reset-budget') {
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
const path = require('path');
|
|
5
5
|
const fs = require('fs');
|
|
6
6
|
const { loadWorkflow, saveWorkflow, isMajorWorkflow, matchingRun, registerMutationProbe, recordBudgetEvent, readHookInput, WORKFLOW_CONTROLLER_COMMANDS } = require('./lib/workflow-runtime');
|
|
7
|
-
const { key, classifyShell, workspaceFingerprint, shellProbeKey, cwdOf, commandOf, mutationPaths, linkedWorktree, assertTargets, assertShellScope } = require('./lib/workflow-isolation');
|
|
7
|
+
const { key, classifyShell, workspaceFingerprint, shellProbeKey, cwdOf, commandOf, mutationPaths, directMutationInWorkspace, linkedWorktree, assertTargets, assertShellScope } = require('./lib/workflow-isolation');
|
|
8
8
|
|
|
9
9
|
const DIRECT = new Set(['Edit', 'Write', 'apply_patch']);
|
|
10
10
|
const SHELL = new Set(['Bash', 'PowerShell', 'exec_command']);
|
|
@@ -48,6 +48,10 @@ function decide(payload) {
|
|
|
48
48
|
throw new Error('selected Fable workflow has no correlated run. Write workflow-stages.json at the displayed session path, then use workflow-disposition.js start.');
|
|
49
49
|
}
|
|
50
50
|
if (DIRECT.has(tool)) {
|
|
51
|
+
// Bookkeeping outside the workspace (scratchpad, memory files, temp
|
|
52
|
+
// notes) is not a workspace mutation: it consumes no Tier-2 iteration
|
|
53
|
+
// and advances no mutation milestone (#165).
|
|
54
|
+
if (!directMutationInWorkspace(payload, cwd, root)) return;
|
|
51
55
|
if (workflow.strategy === 'iterative-single') {
|
|
52
56
|
recordBudgetEvent(context, 'iteration', { evidence: `${tool}:direct-mutation` });
|
|
53
57
|
}
|
|
@@ -58,9 +62,22 @@ function decide(payload) {
|
|
|
58
62
|
|
|
59
63
|
// Shell safety and shell mutation accounting are intentionally separate.
|
|
60
64
|
// `untrusted` means "not proven read-only", not "a mutation occurred".
|
|
61
|
-
// Capture an opaque pre-execution workspace fingerprint
|
|
62
|
-
//
|
|
63
|
-
|
|
65
|
+
// Capture an opaque pre-execution workspace fingerprint when possible. A
|
|
66
|
+
// non-major workflow may run outside Git, where mutation observation is an
|
|
67
|
+
// accounting aid rather than an isolation boundary; degrade conservatively
|
|
68
|
+
// instead of blocking the user's command (#162).
|
|
69
|
+
let fingerprint;
|
|
70
|
+
try {
|
|
71
|
+
fingerprint = workspaceFingerprint(cwd);
|
|
72
|
+
} catch (error) {
|
|
73
|
+
if (isMajorWorkflow(workflow)) throw error;
|
|
74
|
+
if (workflow.strategy === 'iterative-single') {
|
|
75
|
+
recordBudgetEvent(context, 'iteration', { evidence: 'shell:unobservable-workspace' });
|
|
76
|
+
}
|
|
77
|
+
workflow.lastMutationAt = Date.now();
|
|
78
|
+
saveWorkflow(context);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
64
81
|
const probeKey = shellProbeKey(payload, cwd);
|
|
65
82
|
registerMutationProbe(context, probeKey, fingerprint, workflow.strategy === 'iterative-single');
|
|
66
83
|
}
|
|
@@ -5,6 +5,18 @@ const fs = require('fs');
|
|
|
5
5
|
const path = require('path');
|
|
6
6
|
const { getWorkspaceRoot, getRuntimeRoot, resolveProjectDocs } = require('./project-docs-resolver');
|
|
7
7
|
|
|
8
|
+
let emitLessonTelemetry = () => ({ ok: false, unavailable: true });
|
|
9
|
+
for (const relative of [
|
|
10
|
+
'../../hooks/scripts/lib/telemetry.js',
|
|
11
|
+
'../../../hooks/scripts/lib/telemetry.js',
|
|
12
|
+
]) {
|
|
13
|
+
const candidate = path.resolve(__dirname, relative);
|
|
14
|
+
if (fs.existsSync(candidate)) {
|
|
15
|
+
({ emitLessonTelemetry } = require(candidate));
|
|
16
|
+
break;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
8
20
|
function option(args, ...names) {
|
|
9
21
|
for (const name of names) {
|
|
10
22
|
const index = args.indexOf(name);
|
|
@@ -166,6 +178,22 @@ function retrieveMemoryRecords({ workspace, task = '', requirement = '', role =
|
|
|
166
178
|
});
|
|
167
179
|
}
|
|
168
180
|
|
|
181
|
+
for (const record of included) {
|
|
182
|
+
if (!record.origin?.lessonCandidateId) continue;
|
|
183
|
+
emitLessonTelemetry({
|
|
184
|
+
event: 'lesson_retrieved',
|
|
185
|
+
root,
|
|
186
|
+
sessionId: record.writer?.sessionId || null,
|
|
187
|
+
candidateId: record.origin.lessonCandidateId,
|
|
188
|
+
observedAt: new Date(nowMs).toISOString(),
|
|
189
|
+
status: 'success',
|
|
190
|
+
reasonCodes: [
|
|
191
|
+
'scoped-memory-retrieval',
|
|
192
|
+
...(record.retrieval?.reasonCodes || []),
|
|
193
|
+
],
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
169
197
|
return {
|
|
170
198
|
schemaVersion: 1,
|
|
171
199
|
workspace: root,
|
package/package.json
CHANGED