harness-everything 0.11.7 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +53 -0
- package/README.md +1 -0
- package/behavioral-evals/README.md +37 -0
- package/behavioral-evals/cases/self-evolve-retrieval-recurrence.yaml +48 -0
- package/behavioral-evals/paired-benchmark.js +211 -5
- package/ci/contract-integrity.test.js +68 -2
- package/ci/mechanism-2ah-workflow-lifecycle.test.js +258 -2
- package/ci/mechanism-32-paired-benchmark.test.js +139 -0
- package/ci/openai-plugin-package.test.js +17 -0
- package/ci/self-evolve-loop.test.js +37 -2
- package/ci/telemetry.test.js +31 -1
- package/docs/enforcement-locks.md +114 -0
- package/grill-with-docs/ADR-FORMAT.md +22 -0
- package/grill-with-docs/SKILL.md +1 -1
- package/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/hooks/scripts/lib/telemetry.js +40 -0
- package/hooks/scripts/lib/workflow-isolation.js +10 -2
- package/hooks/scripts/lib/workflow-runtime.js +108 -5
- package/hooks/scripts/state-persist.js +9 -3
- package/hooks/scripts/workflow-disposition.js +19 -8
- package/hooks/scripts/workflow-gate.js +33 -4
- package/multi-agent-workspace/SKILL.md +1 -1
- package/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/contract-integrity/scripts/audit.js +536 -0
- package/plugins/harness-everything/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/plugins/harness-everything/hooks/scripts/lib/telemetry.js +40 -0
- package/plugins/harness-everything/hooks/scripts/lib/workflow-isolation.js +10 -2
- package/plugins/harness-everything/hooks/scripts/lib/workflow-runtime.js +108 -5
- package/plugins/harness-everything/hooks/scripts/state-persist.js +9 -3
- package/plugins/harness-everything/hooks/scripts/workflow-disposition.js +19 -8
- package/plugins/harness-everything/hooks/scripts/workflow-gate.js +33 -4
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/grill-with-docs/ADR-FORMAT.md +22 -0
- package/plugins/harness-everything/skills/grill-with-docs/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/plugins/harness-everything/skills/self-evolve/SKILL.md +1 -1
- package/plugins/harness-everything/skills/self-evolve/scripts/lesson-candidate.js +52 -0
- package/plugins/harness-everything/skills/to-spec/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-spec/references/process.md +26 -0
- package/plugins/harness-everything/skills/to-spec/templates/cli-reference.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/dev-doc.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/feature-spec.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/schema-doc.md +14 -0
- package/plugins/harness-everything/skills/to-tickets/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-tickets/references/publishing-and-templates.md +20 -1
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +3 -2
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +25 -1
- package/plugins/harness-everything/skills/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/plugins/harness-everything/skills/verification-loop/templates/verification-report.template.md +3 -0
- package/scripts/sync-openai-plugin.js +12 -0
- package/self-evolve/SKILL.md +1 -1
- package/self-evolve/scripts/lesson-candidate.js +52 -0
- package/telemetry/schemas/event.schema.json +141 -22
- package/telemetry/scripts/report.js +42 -0
- package/to-spec/SKILL.md +1 -1
- package/to-spec/references/process.md +26 -0
- package/to-spec/templates/cli-reference.md +14 -0
- package/to-spec/templates/dev-doc.md +14 -0
- package/to-spec/templates/feature-spec.md +14 -0
- package/to-spec/templates/schema-doc.md +14 -0
- package/to-tickets/SKILL.md +1 -1
- package/to-tickets/references/publishing-and-templates.md +20 -1
- package/verification-loop/SKILL.md +3 -2
- package/verification-loop/references/verification-phases.md +25 -1
- package/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/verification-loop/templates/verification-report.template.md +3 -0
|
@@ -76,10 +76,18 @@ function workspaceFingerprint(cwd) {
|
|
|
76
76
|
function shellProbeKey(payload, cwd) {
|
|
77
77
|
const toolUseId = String(payload?.tool_use_id || payload?.toolUseId || '').trim();
|
|
78
78
|
const tool = String(payload?.tool_name || payload?.tool || '');
|
|
79
|
+
if (toolUseId) {
|
|
80
|
+
// Host tool-use identity is stable across Pre/Post payload cwd drift.
|
|
81
|
+
return crypto.createHash('sha256')
|
|
82
|
+
.update('harness-shell-probe-v3\0')
|
|
83
|
+
.update(toolUseId).update('\0')
|
|
84
|
+
.update(tool)
|
|
85
|
+
.digest('hex');
|
|
86
|
+
}
|
|
87
|
+
// Compatibility fallback for hosts that do not expose tool_use_id.
|
|
79
88
|
const command = commandOf(payload);
|
|
80
89
|
return crypto.createHash('sha256')
|
|
81
|
-
.update('harness-shell-probe-v2\0')
|
|
82
|
-
.update(toolUseId || 'no-tool-use-id').update('\0')
|
|
90
|
+
.update('harness-shell-probe-v2-fallback\0')
|
|
83
91
|
.update(tool).update('\0')
|
|
84
92
|
.update(key(cwd)).update('\0')
|
|
85
93
|
.update(command)
|
|
@@ -4,6 +4,7 @@ const fs = require('fs');
|
|
|
4
4
|
const path = require('path');
|
|
5
5
|
const { getWorkspaceRoot, getSessionDir, getSessionId, getStateRoot } = require('./harness-state');
|
|
6
6
|
const { atomicWriteJson, readJson } = require('./fable-contracts');
|
|
7
|
+
const { workspaceFingerprint } = require('./workflow-isolation');
|
|
7
8
|
|
|
8
9
|
const OPEN_STATES = new Set(['pending', 'active', 'running', 'failed', 'blocked', 'escaped']);
|
|
9
10
|
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
@@ -45,6 +46,25 @@ function saveWorkflow(context) { atomicWriteJson(context.file, context.workflow)
|
|
|
45
46
|
|
|
46
47
|
const MUTATION_PROBE_KEY = /^[a-f0-9]{64}$/;
|
|
47
48
|
const MUTATION_PROBE_WAIT = new Int32Array(new SharedArrayBuffer(4));
|
|
49
|
+
const MUTATION_PROBE_DEFAULT_TIMEOUT_MS = 120000;
|
|
50
|
+
const MUTATION_PROBE_RECLAIM_GRACE_MS = 5000;
|
|
51
|
+
const MUTATION_PROBE_MAX_TIMEOUT_MS = 30 * 60 * 1000;
|
|
52
|
+
|
|
53
|
+
function mutationProbeLeaseMs(timeoutMs) {
|
|
54
|
+
const parsed = Number(timeoutMs);
|
|
55
|
+
const executionMs = Number.isFinite(parsed) && parsed > 0
|
|
56
|
+
? Math.min(parsed, MUTATION_PROBE_MAX_TIMEOUT_MS)
|
|
57
|
+
: MUTATION_PROBE_DEFAULT_TIMEOUT_MS;
|
|
58
|
+
return executionMs + MUTATION_PROBE_RECLAIM_GRACE_MS;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function mutationProbeExpired(probe, now = Date.now()) {
|
|
62
|
+
const reclaimAfterAt = Number(probe?.reclaimAfterAt);
|
|
63
|
+
if (Number.isFinite(reclaimAfterAt)) return now >= reclaimAfterAt;
|
|
64
|
+
const observedAt = Number(probe?.observedAt);
|
|
65
|
+
return Number.isFinite(observedAt) &&
|
|
66
|
+
now - observedAt >= mutationProbeLeaseMs(null);
|
|
67
|
+
}
|
|
48
68
|
|
|
49
69
|
function mutationProbeDir(context) {
|
|
50
70
|
return path.join(context.sessionDir, 'mutation-probes');
|
|
@@ -81,15 +101,25 @@ function withMutationProbeLock(context, callback) {
|
|
|
81
101
|
finally { fs.rmSync(lockDir, { recursive: true, force: true }); }
|
|
82
102
|
}
|
|
83
103
|
|
|
104
|
+
function probeCountsIteration(probe) {
|
|
105
|
+
return Boolean(probe && (probe.countIteration === true ||
|
|
106
|
+
(probe.countIteration === undefined && probe.reserveIteration === true)));
|
|
107
|
+
}
|
|
108
|
+
|
|
84
109
|
function mutationProbeReservations(context) {
|
|
85
110
|
const dir = mutationProbeDir(context);
|
|
86
111
|
let files = [];
|
|
87
112
|
try { files = fs.readdirSync(dir).filter(file => /^[a-f0-9]{64}\.json$/.test(file)); }
|
|
88
113
|
catch (_) { return 0; }
|
|
114
|
+
const now = Date.now();
|
|
89
115
|
let count = 0;
|
|
90
116
|
for (const file of files) {
|
|
91
117
|
const probe = readJson(path.join(dir, file));
|
|
92
|
-
if (probe?.schemaVersion
|
|
118
|
+
if (probe?.schemaVersion !== 1 || probe.reserveIteration !== true) continue;
|
|
119
|
+
// A reservation remains active for the host/tool execution timeout plus
|
|
120
|
+
// a grace period. This prevents a genuinely in-flight command from being
|
|
121
|
+
// reclaimed merely because it ran longer than an arbitrary short delay.
|
|
122
|
+
if (!mutationProbeExpired(probe, now)) count++;
|
|
93
123
|
}
|
|
94
124
|
return count;
|
|
95
125
|
}
|
|
@@ -115,22 +145,78 @@ function assertIterationCapacity(context, evidence) {
|
|
|
115
145
|
return { applicable: true, value: budget.counters.iterations, reserved, limit };
|
|
116
146
|
}
|
|
117
147
|
|
|
118
|
-
function
|
|
148
|
+
function reclaimStaleMutationProbes(context, keepKey) {
|
|
149
|
+
const dir = mutationProbeDir(context);
|
|
150
|
+
let files = [];
|
|
151
|
+
try { files = fs.readdirSync(dir).filter(file => /^[a-f0-9]{64}\.json$/.test(file)); }
|
|
152
|
+
catch (_) { return; }
|
|
153
|
+
const now = Date.now();
|
|
154
|
+
for (const name of files) {
|
|
155
|
+
const staleKey = name.slice(0, -'.json'.length);
|
|
156
|
+
if (staleKey === keepKey) continue;
|
|
157
|
+
const file = path.join(dir, name);
|
|
158
|
+
const probe = readJson(file);
|
|
159
|
+
if (!probe || probe.schemaVersion !== 1) continue;
|
|
160
|
+
if (!mutationProbeExpired(probe, now)) continue;
|
|
161
|
+
const countsIteration = probeCountsIteration(probe);
|
|
162
|
+
if (!probe.observationCwd) {
|
|
163
|
+
// Legacy probes predate stored observation roots and cannot be safely
|
|
164
|
+
// re-observed. Once outside the in-flight grace window they must not
|
|
165
|
+
// reserve capacity forever.
|
|
166
|
+
fs.unlinkSync(file);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
let currentFingerprint;
|
|
170
|
+
try {
|
|
171
|
+
currentFingerprint = workspaceFingerprint(probe.observationCwd);
|
|
172
|
+
} catch (_) {
|
|
173
|
+
fs.unlinkSync(file);
|
|
174
|
+
context.workflow.lastMutationAt = now;
|
|
175
|
+
if (countsIteration && context.workflow.strategy === 'iterative-single') {
|
|
176
|
+
recordBudgetEvent(context, 'iteration', { evidence: 'shell:reclaimed-unobservable-workspace' });
|
|
177
|
+
} else {
|
|
178
|
+
saveWorkflow(context);
|
|
179
|
+
}
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
if (currentFingerprint !== probe.fingerprint) {
|
|
183
|
+
fs.unlinkSync(file);
|
|
184
|
+
context.workflow.lastMutationAt = now;
|
|
185
|
+
if (countsIteration && context.workflow.strategy === 'iterative-single') {
|
|
186
|
+
recordBudgetEvent(context, 'iteration', { evidence: 'shell:reclaimed-stale-workspace-mutation' });
|
|
187
|
+
} else {
|
|
188
|
+
saveWorkflow(context);
|
|
189
|
+
}
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
// No visible workspace effect after the grace window: this probe was
|
|
193
|
+
// orphaned by a denial/block/missing Post event. Discard it so it cannot
|
|
194
|
+
// reserve capacity or later attribute an unrelated mutation (#161).
|
|
195
|
+
fs.unlinkSync(file);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function registerMutationProbe(context, key, fingerprint, reserveIteration, observationCwd, timeoutMs) {
|
|
119
200
|
if (!MUTATION_PROBE_KEY.test(String(key || '')) || !MUTATION_PROBE_KEY.test(String(fingerprint || ''))) {
|
|
120
201
|
throw new Error('invalid mutation probe identity');
|
|
121
202
|
}
|
|
122
203
|
return withMutationProbeLock(context, () => {
|
|
123
204
|
refreshWorkflowForProbe(context);
|
|
205
|
+
reclaimStaleMutationProbes(context, key);
|
|
124
206
|
if (reserveIteration) assertIterationCapacity(context, 'shell:untrusted');
|
|
125
207
|
const dir = mutationProbeDir(context);
|
|
126
208
|
fs.mkdirSync(dir, { recursive: true });
|
|
127
209
|
const file = mutationProbeFile(context, key);
|
|
128
210
|
if (fs.existsSync(file)) throw new Error('mutation probe is already pending for this tool call');
|
|
211
|
+
const observedAt = Date.now();
|
|
129
212
|
const probe = {
|
|
130
213
|
schemaVersion: 1,
|
|
131
214
|
fingerprint,
|
|
132
215
|
reserveIteration: Boolean(reserveIteration),
|
|
133
|
-
|
|
216
|
+
countIteration: Boolean(reserveIteration),
|
|
217
|
+
observationCwd: observationCwd ? path.resolve(observationCwd) : null,
|
|
218
|
+
observedAt,
|
|
219
|
+
reclaimAfterAt: observedAt + mutationProbeLeaseMs(timeoutMs),
|
|
134
220
|
};
|
|
135
221
|
atomicWriteJson(file, probe);
|
|
136
222
|
return probe;
|
|
@@ -152,7 +238,7 @@ function settleMutationProbe(context, key, afterFingerprint, tool) {
|
|
|
152
238
|
const changed = afterFingerprint !== probe.fingerprint;
|
|
153
239
|
if (changed) {
|
|
154
240
|
context.workflow.lastMutationAt = Date.now();
|
|
155
|
-
if (probe
|
|
241
|
+
if (probeCountsIteration(probe) && context.workflow.strategy === 'iterative-single') {
|
|
156
242
|
recordBudgetEvent(context, 'iteration', { evidence: String(tool || 'shell') + ':observed-workspace-mutation' });
|
|
157
243
|
} else {
|
|
158
244
|
saveWorkflow(context);
|
|
@@ -162,6 +248,23 @@ function settleMutationProbe(context, key, afterFingerprint, tool) {
|
|
|
162
248
|
});
|
|
163
249
|
}
|
|
164
250
|
|
|
251
|
+
function settleMutationProbeConservative(context, key, tool, evidence = 'unobservable-workspace') {
|
|
252
|
+
return withMutationProbeLock(context, () => {
|
|
253
|
+
const file = mutationProbeFile(context, key);
|
|
254
|
+
const probe = readJson(file);
|
|
255
|
+
if (!probe || probe.schemaVersion !== 1) return null;
|
|
256
|
+
refreshWorkflowForProbe(context);
|
|
257
|
+
fs.unlinkSync(file);
|
|
258
|
+
context.workflow.lastMutationAt = Date.now();
|
|
259
|
+
if (probeCountsIteration(probe) && context.workflow.strategy === 'iterative-single') {
|
|
260
|
+
recordBudgetEvent(context, 'iteration', { evidence: String(tool || 'shell') + ':' + evidence });
|
|
261
|
+
} else {
|
|
262
|
+
saveWorkflow(context);
|
|
263
|
+
}
|
|
264
|
+
return { changed: true, probe, conservative: true };
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
|
|
165
268
|
function discardMutationProbe(context, key) {
|
|
166
269
|
return withMutationProbeLock(context, () => {
|
|
167
270
|
const file = mutationProbeFile(context, key);
|
|
@@ -396,4 +499,4 @@ function readHookInput(decide) {
|
|
|
396
499
|
process.stdin.on('error', finish);
|
|
397
500
|
}
|
|
398
501
|
|
|
399
|
-
module.exports = { OPEN_STATES, SAFE_ID, WORKFLOW_CONTROLLER_COMMANDS, isMajorWorkflow, loadWorkflow, saveWorkflow, budgetLimits, ensureWorkflowBudget, mutationProbeReservations, assertIterationCapacity, registerMutationProbe, peekMutationProbe, settleMutationProbe, discardMutationProbe, clearMutationProbes, recordBudgetEvent, resetWorkflowBudget, syncBudgetToRun, matchingRun, unresolvedStages, readHookInput };
|
|
502
|
+
module.exports = { OPEN_STATES, SAFE_ID, WORKFLOW_CONTROLLER_COMMANDS, isMajorWorkflow, loadWorkflow, saveWorkflow, budgetLimits, ensureWorkflowBudget, mutationProbeReservations, assertIterationCapacity, registerMutationProbe, peekMutationProbe, settleMutationProbe, settleMutationProbeConservative, discardMutationProbe, clearMutationProbes, recordBudgetEvent, resetWorkflowBudget, syncBudgetToRun, matchingRun, unresolvedStages, readHookInput };
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
const fs = require('fs');
|
|
3
3
|
const path = require('path');
|
|
4
4
|
const { getWorkspaceRoot, getSessionDir } = require('./lib/harness-state');
|
|
5
|
-
const { loadWorkflow, saveWorkflow, peekMutationProbe, settleMutationProbe } = require('./lib/workflow-runtime');
|
|
5
|
+
const { loadWorkflow, saveWorkflow, isMajorWorkflow, peekMutationProbe, settleMutationProbe, settleMutationProbeConservative } = require('./lib/workflow-runtime');
|
|
6
6
|
const { cwdOf, shellProbeKey, workspaceFingerprint, directMutationInWorkspace } = require('./lib/workflow-isolation');
|
|
7
7
|
const { observeTool } = require('./lib/telemetry');
|
|
8
8
|
|
|
@@ -37,11 +37,17 @@ function observeWorkspaceMutation(payload, root) {
|
|
|
37
37
|
if (!context.workflow || context.workflow.state === 'deferred') return false;
|
|
38
38
|
const cwd = cwdOf(payload, root);
|
|
39
39
|
const probeKey = shellProbeKey(payload, cwd);
|
|
40
|
-
|
|
40
|
+
const probe = peekMutationProbe(context, probeKey);
|
|
41
|
+
if (!probe) return false;
|
|
42
|
+
const observationCwd = probe.observationCwd || cwd;
|
|
41
43
|
let after;
|
|
42
44
|
try {
|
|
43
|
-
after = workspaceFingerprint(
|
|
45
|
+
after = workspaceFingerprint(observationCwd);
|
|
44
46
|
} catch (error) {
|
|
47
|
+
if (!isMajorWorkflow(context.workflow)) {
|
|
48
|
+
const conservative = settleMutationProbeConservative(context, probeKey, toolName, 'unobservable-workspace');
|
|
49
|
+
return Boolean(conservative && conservative.changed);
|
|
50
|
+
}
|
|
45
51
|
context.workflow.state = 'blocked';
|
|
46
52
|
context.workflow.blockReason = 'mutation-observation-failed';
|
|
47
53
|
saveWorkflow(context);
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
const fs = require('fs');
|
|
5
5
|
const path = require('path');
|
|
6
|
-
const { loadWorkflow, saveWorkflow, matchingRun, OPEN_STATES, ensureWorkflowBudget, recordBudgetEvent, resetWorkflowBudget, syncBudgetToRun, WORKFLOW_CONTROLLER_COMMANDS } = require('./lib/workflow-runtime');
|
|
6
|
+
const { loadWorkflow, saveWorkflow, matchingRun, OPEN_STATES, budgetLimits, ensureWorkflowBudget, recordBudgetEvent, resetWorkflowBudget, syncBudgetToRun, WORKFLOW_CONTROLLER_COMMANDS } = require('./lib/workflow-runtime');
|
|
7
7
|
const { getWorkspaceRoot, readCurrentSession } = require('./lib/harness-state');
|
|
8
8
|
const { atomicWriteJson, readJson } = require('./lib/fable-contracts');
|
|
9
9
|
|
|
@@ -32,6 +32,20 @@ function consumer() {
|
|
|
32
32
|
throw new Error('Fable consumer unavailable');
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
function activatePlan(workflow, plan) {
|
|
36
|
+
workflow.workflowPlan = plan;
|
|
37
|
+
workflow.strategy = plan.strategy;
|
|
38
|
+
workflow.strategySelection = plan.strategySelection;
|
|
39
|
+
workflow.tier = plan.tier;
|
|
40
|
+
workflow.requiredInvariants = plan.requiredInvariants || [];
|
|
41
|
+
workflow.suggestedSkills = plan.suggestedSkills || [];
|
|
42
|
+
workflow.verification = plan.verification;
|
|
43
|
+
workflow.mutationIsolation = plan.mutationIsolation || { required: false };
|
|
44
|
+
const budget = ensureWorkflowBudget(workflow);
|
|
45
|
+
budget.limits = budgetLimits(workflow);
|
|
46
|
+
budget.updatedAt = new Date().toISOString();
|
|
47
|
+
}
|
|
48
|
+
|
|
35
49
|
function main() {
|
|
36
50
|
const args = parseArgs(process.argv.slice(2));
|
|
37
51
|
if (!WORKFLOW_CONTROLLER_COMMANDS.has(args.command)) throw new Error(usage());
|
|
@@ -47,10 +61,12 @@ function main() {
|
|
|
47
61
|
if (isReplan) recordBudgetEvent(context, 'replan', { evidence: 'workflow-disposition:start' });
|
|
48
62
|
const plan = workflow.pendingPlan || workflow.workflowPlan;
|
|
49
63
|
if (!String(plan.strategy || '').startsWith('fable-')) {
|
|
64
|
+
activatePlan(workflow, plan);
|
|
50
65
|
workflow.revision++;
|
|
51
66
|
workflow.state = 'running';
|
|
67
|
+
workflow.escapes = [];
|
|
68
|
+
delete workflow.pendingPlan;
|
|
52
69
|
delete workflow.blockReason;
|
|
53
|
-
ensureWorkflowBudget(workflow);
|
|
54
70
|
saveWorkflow(context);
|
|
55
71
|
process.stdout.write(JSON.stringify({ state: workflow.state, workflowId: workflow.workflowId, revision: workflow.revision }) + '\n');
|
|
56
72
|
return;
|
|
@@ -66,18 +82,13 @@ function main() {
|
|
|
66
82
|
routerContract: { workflowPlan: plan }, stages, workspaceRoot: context.root,
|
|
67
83
|
sessionId, workflowId: workflow.workflowId,
|
|
68
84
|
});
|
|
69
|
-
workflow
|
|
70
|
-
workflow.strategy = plan.strategy;
|
|
71
|
-
workflow.tier = plan.tier;
|
|
72
|
-
workflow.verification = plan.verification;
|
|
73
|
-
workflow.mutationIsolation = plan.mutationIsolation || workflow.mutationIsolation;
|
|
85
|
+
activatePlan(workflow, plan);
|
|
74
86
|
workflow.runId = run.runId;
|
|
75
87
|
workflow.revision++;
|
|
76
88
|
workflow.state = 'running';
|
|
77
89
|
workflow.escapes = [];
|
|
78
90
|
delete workflow.pendingPlan;
|
|
79
91
|
delete workflow.blockReason;
|
|
80
|
-
ensureWorkflowBudget(workflow);
|
|
81
92
|
} else if (args.command === 'revision') {
|
|
82
93
|
recordBudgetEvent(context, 'revision', { evidence: args.evidence || 'workflow-disposition:revision' });
|
|
83
94
|
} else if (args.command === 'reset-budget') {
|
|
@@ -9,6 +9,15 @@ const { key, classifyShell, workspaceFingerprint, shellProbeKey, cwdOf, commandO
|
|
|
9
9
|
const DIRECT = new Set(['Edit', 'Write', 'apply_patch']);
|
|
10
10
|
const SHELL = new Set(['Bash', 'PowerShell', 'exec_command']);
|
|
11
11
|
|
|
12
|
+
function shellTimeoutMs(payload) {
|
|
13
|
+
const input = payload?.tool_input || payload?.input || {};
|
|
14
|
+
for (const value of [input.timeout_ms, input.timeoutMs, input.timeout, payload.timeout_ms, payload.timeoutMs]) {
|
|
15
|
+
const parsed = Number(value);
|
|
16
|
+
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
|
17
|
+
}
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
|
|
12
21
|
function isController(command, cwd) {
|
|
13
22
|
// One trusted local executable, literal arguments, no shell evaluation.
|
|
14
23
|
if (/[;&|`\r\n<>$(){}]/.test(command)) return false;
|
|
@@ -62,11 +71,31 @@ function decide(payload) {
|
|
|
62
71
|
|
|
63
72
|
// Shell safety and shell mutation accounting are intentionally separate.
|
|
64
73
|
// `untrusted` means "not proven read-only", not "a mutation occurred".
|
|
65
|
-
// Capture an opaque pre-execution workspace fingerprint
|
|
66
|
-
//
|
|
67
|
-
|
|
74
|
+
// Capture an opaque pre-execution workspace fingerprint when possible. A
|
|
75
|
+
// non-major workflow may run outside Git, where mutation observation is an
|
|
76
|
+
// accounting aid rather than an isolation boundary; degrade conservatively
|
|
77
|
+
// instead of blocking the user's command (#162).
|
|
78
|
+
let fingerprint;
|
|
79
|
+
try {
|
|
80
|
+
fingerprint = workspaceFingerprint(cwd);
|
|
81
|
+
} catch (error) {
|
|
82
|
+
if (isMajorWorkflow(workflow)) throw error;
|
|
83
|
+
if (workflow.strategy === 'iterative-single') {
|
|
84
|
+
recordBudgetEvent(context, 'iteration', { evidence: 'shell:unobservable-workspace' });
|
|
85
|
+
}
|
|
86
|
+
workflow.lastMutationAt = Date.now();
|
|
87
|
+
saveWorkflow(context);
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
68
90
|
const probeKey = shellProbeKey(payload, cwd);
|
|
69
|
-
registerMutationProbe(
|
|
91
|
+
registerMutationProbe(
|
|
92
|
+
context,
|
|
93
|
+
probeKey,
|
|
94
|
+
fingerprint,
|
|
95
|
+
workflow.strategy === 'iterative-single',
|
|
96
|
+
cwd,
|
|
97
|
+
shellTimeoutMs(payload)
|
|
98
|
+
);
|
|
70
99
|
}
|
|
71
100
|
|
|
72
101
|
readHookInput(decide);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "harness-everything",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.12.1",
|
|
5
5
|
"description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Miya Daniel",
|
|
@@ -22,6 +22,28 @@ Only include these when they add genuine value. Most ADRs won't need them.
|
|
|
22
22
|
- **Considered Options** — only when the rejected alternatives are worth remembering
|
|
23
23
|
- **Consequences** — only when non-obvious downstream effects need to be called out
|
|
24
24
|
|
|
25
|
+
## Decision lifecycle and contract lineage
|
|
26
|
+
|
|
27
|
+
Once an ADR is **accepted**, keep its decision and rationale historically stable. Do not rewrite an accepted ADR to make a later design look as though it was always the decision.
|
|
28
|
+
|
|
29
|
+
Use compact metadata when the ADR participates in a behavior contract:
|
|
30
|
+
|
|
31
|
+
```md
|
|
32
|
+
ADR-ID: ADR-0001
|
|
33
|
+
Status: proposed | accepted | deprecated | superseded by ADR-0002
|
|
34
|
+
Affected requirements: REQ-001, REQ-004
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
When a later change materially changes the decision or its rationale:
|
|
38
|
+
|
|
39
|
+
1. create a new sequential ADR;
|
|
40
|
+
2. state the new decision and why the prior trade-off changed;
|
|
41
|
+
3. mark the earlier ADR `superseded by ADR-NNNN`;
|
|
42
|
+
4. update the living spec requirements affected by that decision;
|
|
43
|
+
5. let the structured contract trace carry detailed ticket/test/implementation lineage.
|
|
44
|
+
|
|
45
|
+
A ticket or implementation note never silently becomes the new architectural source of truth. If only implementation changes and the approved behavior/decision does not, keep the accepted ADR unchanged.
|
|
46
|
+
|
|
25
47
|
## Numbering
|
|
26
48
|
|
|
27
49
|
Scan `docs/adr/` for the highest existing number and increment by one.
|
|
@@ -5,6 +5,18 @@ const fs = require('fs');
|
|
|
5
5
|
const path = require('path');
|
|
6
6
|
const { getWorkspaceRoot, getRuntimeRoot, resolveProjectDocs } = require('./project-docs-resolver');
|
|
7
7
|
|
|
8
|
+
let emitLessonTelemetry = () => ({ ok: false, unavailable: true });
|
|
9
|
+
for (const relative of [
|
|
10
|
+
'../../hooks/scripts/lib/telemetry.js',
|
|
11
|
+
'../../../hooks/scripts/lib/telemetry.js',
|
|
12
|
+
]) {
|
|
13
|
+
const candidate = path.resolve(__dirname, relative);
|
|
14
|
+
if (fs.existsSync(candidate)) {
|
|
15
|
+
({ emitLessonTelemetry } = require(candidate));
|
|
16
|
+
break;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
8
20
|
function option(args, ...names) {
|
|
9
21
|
for (const name of names) {
|
|
10
22
|
const index = args.indexOf(name);
|
|
@@ -166,6 +178,22 @@ function retrieveMemoryRecords({ workspace, task = '', requirement = '', role =
|
|
|
166
178
|
});
|
|
167
179
|
}
|
|
168
180
|
|
|
181
|
+
for (const record of included) {
|
|
182
|
+
if (!record.origin?.lessonCandidateId) continue;
|
|
183
|
+
emitLessonTelemetry({
|
|
184
|
+
event: 'lesson_retrieved',
|
|
185
|
+
root,
|
|
186
|
+
sessionId: record.writer?.sessionId || null,
|
|
187
|
+
candidateId: record.origin.lessonCandidateId,
|
|
188
|
+
observedAt: new Date(nowMs).toISOString(),
|
|
189
|
+
status: 'success',
|
|
190
|
+
reasonCodes: [
|
|
191
|
+
'scoped-memory-retrieval',
|
|
192
|
+
...(record.retrieval?.reasonCodes || []),
|
|
193
|
+
],
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
169
197
|
return {
|
|
170
198
|
schemaVersion: 1,
|
|
171
199
|
workspace: root,
|
|
@@ -6,6 +6,18 @@ const path = require('path');
|
|
|
6
6
|
const { spawnSync } = require('child_process');
|
|
7
7
|
const { contentHash, scoreRuleQuality, screenMemoryForPersistence } = require('./persist-memory');
|
|
8
8
|
|
|
9
|
+
let emitLessonTelemetry = () => ({ ok: false, unavailable: true });
|
|
10
|
+
for (const relative of [
|
|
11
|
+
'../../hooks/scripts/lib/telemetry.js',
|
|
12
|
+
'../../../hooks/scripts/lib/telemetry.js',
|
|
13
|
+
]) {
|
|
14
|
+
const candidate = path.resolve(__dirname, relative);
|
|
15
|
+
if (fs.existsSync(candidate)) {
|
|
16
|
+
({ emitLessonTelemetry } = require(candidate));
|
|
17
|
+
break;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
9
21
|
let getWorkspaceRoot;
|
|
10
22
|
let getWorkspaceStateDir;
|
|
11
23
|
try {
|
|
@@ -69,6 +81,24 @@ function pushState(candidate, state, reasonCode, evidenceRef = null) {
|
|
|
69
81
|
}].slice(-32);
|
|
70
82
|
}
|
|
71
83
|
|
|
84
|
+
function emitCandidateLifecycle(input, candidate, event, state, status = 'unknown') {
|
|
85
|
+
const history = [...(candidate.history || [])].reverse().find(item => item.state === state);
|
|
86
|
+
const reasonCodes = [
|
|
87
|
+
history && history.reasonCode,
|
|
88
|
+
candidate.trigger && candidate.trigger.type ? `trigger:${candidate.trigger.type}` : null,
|
|
89
|
+
...(candidate.evaluation && Array.isArray(candidate.evaluation.reasonCodes) ? candidate.evaluation.reasonCodes : []),
|
|
90
|
+
].filter(Boolean);
|
|
91
|
+
return emitLessonTelemetry({
|
|
92
|
+
event,
|
|
93
|
+
root: input.workspace,
|
|
94
|
+
sessionId: input.sessionId || (candidate.provenance && candidate.provenance.sessionId),
|
|
95
|
+
candidateId: candidate.candidateId,
|
|
96
|
+
observedAt: history && history.observedAt,
|
|
97
|
+
status,
|
|
98
|
+
reasonCodes,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
72
102
|
function replayability(candidate) {
|
|
73
103
|
if (candidate.trigger?.type === 'verifier-fail-pass' &&
|
|
74
104
|
candidate.evidence?.previousStatus === 'fail' &&
|
|
@@ -134,6 +164,8 @@ function evaluateCandidate(input) {
|
|
|
134
164
|
};
|
|
135
165
|
pushState(candidate, 'rejected', 'candidate-screening-or-quality-failed');
|
|
136
166
|
atomicWriteJson(file, candidate);
|
|
167
|
+
emitCandidateLifecycle(input, candidate, 'lesson_screened', 'screened', screening.allowed ? 'success' : 'failure');
|
|
168
|
+
emitCandidateLifecycle(input, candidate, 'lesson_rejected', 'rejected', 'failure');
|
|
137
169
|
return { file, candidate };
|
|
138
170
|
}
|
|
139
171
|
|
|
@@ -150,6 +182,15 @@ function evaluateCandidate(input) {
|
|
|
150
182
|
pushState(candidate, 'evaluated', replay.reasonCode);
|
|
151
183
|
pushState(candidate, disposition, disposition === 'accepted' ? 'eligible-for-governed-promotion' : 'paired-or-later-evidence-required');
|
|
152
184
|
atomicWriteJson(file, candidate);
|
|
185
|
+
emitCandidateLifecycle(input, candidate, 'lesson_screened', 'screened', 'success');
|
|
186
|
+
emitCandidateLifecycle(input, candidate, 'lesson_evaluated', 'evaluated', 'success');
|
|
187
|
+
emitCandidateLifecycle(
|
|
188
|
+
input,
|
|
189
|
+
candidate,
|
|
190
|
+
disposition === 'accepted' ? 'lesson_accepted' : 'lesson_inconclusive',
|
|
191
|
+
disposition,
|
|
192
|
+
disposition === 'accepted' ? 'success' : 'unknown',
|
|
193
|
+
);
|
|
153
194
|
return { file, candidate };
|
|
154
195
|
}
|
|
155
196
|
|
|
@@ -212,6 +253,7 @@ function promoteCandidate(input) {
|
|
|
212
253
|
if (persisted) pushState(candidate, 'persisted', 'accepted-and-authorized-memory-write');
|
|
213
254
|
else pushState(candidate, 'accepted', 'memory-governance-held-as-review-candidate');
|
|
214
255
|
atomicWriteJson(file, candidate);
|
|
256
|
+
if (persisted) emitCandidateLifecycle(input, candidate, 'lesson_persisted', 'persisted', 'success');
|
|
215
257
|
return { ok: true, exitCode: 0, persisted, stdout: result.stdout, stderr: result.stderr, candidate };
|
|
216
258
|
}
|
|
217
259
|
|
|
@@ -234,6 +276,16 @@ function observeCandidate(input) {
|
|
|
234
276
|
};
|
|
235
277
|
pushState(candidate, next, `lesson-${next}`, evidenceRef);
|
|
236
278
|
atomicWriteJson(file, candidate);
|
|
279
|
+
// Retrieval telemetry is emitted by the actual scoped retrieval path so the
|
|
280
|
+
// funnel measures exposure, not an operator-authored lifecycle annotation.
|
|
281
|
+
if (next !== 'retrieved') {
|
|
282
|
+
const event = {
|
|
283
|
+
validated: 'lesson_validated',
|
|
284
|
+
regressed: 'lesson_regressed',
|
|
285
|
+
superseded: 'lesson_superseded',
|
|
286
|
+
}[next];
|
|
287
|
+
if (event) emitCandidateLifecycle(input, candidate, event, next, next === 'regressed' ? 'failure' : 'success');
|
|
288
|
+
}
|
|
237
289
|
return { file, candidate };
|
|
238
290
|
}
|
|
239
291
|
|
|
@@ -60,6 +60,32 @@ What "confirm" means depends on the shape picked:
|
|
|
60
60
|
|
|
61
61
|
Skip this step only if the conversation already pinned down the relevant surface explicitly (e.g. a prior `grill-with-docs` pass already fixed the schema).
|
|
62
62
|
|
|
63
|
+
## Stable contract identity
|
|
64
|
+
|
|
65
|
+
When the published artifact is an authoritative behavior contract, every normative requirement must have a stable identity instead of relying on prose position.
|
|
66
|
+
|
|
67
|
+
Use this compact shape in the selected template:
|
|
68
|
+
|
|
69
|
+
```md
|
|
70
|
+
## Contract Requirements
|
|
71
|
+
|
|
72
|
+
### REQ-001 — <short behavior name>
|
|
73
|
+
**Status:** CURRENT
|
|
74
|
+
**Revision:** 1
|
|
75
|
+
**Decision lineage:** ADR-0001 (or `None`)
|
|
76
|
+
<normative behavior and observable acceptance boundary>
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Rules:
|
|
80
|
+
|
|
81
|
+
- Never recycle a `REQ-*` ID for different behavior.
|
|
82
|
+
- Behavior changes increment the requirement revision and update the living spec **before** implementation is accepted.
|
|
83
|
+
- Superseded behavior remains traceable; do not erase the old requirement history merely because a ticket changed.
|
|
84
|
+
- If a ticket is classified `changeImpact: architecture`, publish the new/superseding ADR first and link it from the affected requirement.
|
|
85
|
+
- If the document is informative rather than authoritative, say so explicitly and link the authoritative requirement instead of inventing a parallel source of truth.
|
|
86
|
+
|
|
87
|
+
The detailed ADR -> spec -> ticket -> test -> implementation lineage belongs in the structured contract trace; keep the human spec concise.
|
|
88
|
+
|
|
63
89
|
## Publishing details
|
|
64
90
|
|
|
65
91
|
Write the doc using the adapted template at the resolved storage path:
|
|
@@ -28,4 +28,18 @@ Every distinct exit code or error response: what triggers it, what the caller sh
|
|
|
28
28
|
|
|
29
29
|
Other commands/endpoints this one is commonly paired with, supersedes, or is superseded by.
|
|
30
30
|
|
|
31
|
+
## Contract Requirements
|
|
32
|
+
|
|
33
|
+
For every normative behavior this document owns, assign a stable requirement identity:
|
|
34
|
+
|
|
35
|
+
```md
|
|
36
|
+
### REQ-001 — <short behavior name>
|
|
37
|
+
**Status:** CURRENT
|
|
38
|
+
**Revision:** 1
|
|
39
|
+
**Decision lineage:** ADR-0001 (or None)
|
|
40
|
+
<observable command/API behavior or error contract>
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Do not recycle requirement IDs. A changed caller-visible contract increments the revision before implementation completion.
|
|
44
|
+
|
|
31
45
|
</spec-template>
|
|
@@ -24,4 +24,18 @@ What this decision costs, even though it was the right call.
|
|
|
24
24
|
|
|
25
25
|
How this ships (flag, gradual, all-at-once) and how to undo it if it's wrong.
|
|
26
26
|
|
|
27
|
+
## Contract Requirements
|
|
28
|
+
|
|
29
|
+
If this document owns normative behavior, assign stable requirement identities:
|
|
30
|
+
|
|
31
|
+
```md
|
|
32
|
+
### REQ-001 — <short behavior name>
|
|
33
|
+
**Status:** CURRENT
|
|
34
|
+
**Revision:** 1
|
|
35
|
+
**Decision lineage:** ADR-0001 (or None)
|
|
36
|
+
<observable behavior / acceptance boundary>
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
If this document is informative only, say so and link the authoritative requirement instead. Do not create a second source of truth.
|
|
40
|
+
|
|
27
41
|
</spec-template>
|
|
@@ -56,4 +56,18 @@ A description of the things that are out of scope for this spec.
|
|
|
56
56
|
|
|
57
57
|
Any further notes about the feature.
|
|
58
58
|
|
|
59
|
+
## Contract Requirements
|
|
60
|
+
|
|
61
|
+
For every normative behavior this document owns, assign a stable requirement identity:
|
|
62
|
+
|
|
63
|
+
```md
|
|
64
|
+
### REQ-001 — <short behavior name>
|
|
65
|
+
**Status:** CURRENT
|
|
66
|
+
**Revision:** 1
|
|
67
|
+
**Decision lineage:** ADR-0001 (or None)
|
|
68
|
+
<observable behavior / acceptance boundary>
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Do not recycle requirement IDs. When approved behavior changes, increment the revision and reconcile the living spec before implementation completion.
|
|
72
|
+
|
|
59
73
|
</spec-template>
|