harness-everything 0.11.7 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +53 -0
- package/README.md +1 -0
- package/behavioral-evals/README.md +37 -0
- package/behavioral-evals/cases/self-evolve-retrieval-recurrence.yaml +48 -0
- package/behavioral-evals/paired-benchmark.js +211 -5
- package/ci/contract-integrity.test.js +68 -2
- package/ci/mechanism-2ah-workflow-lifecycle.test.js +258 -2
- package/ci/mechanism-32-paired-benchmark.test.js +139 -0
- package/ci/openai-plugin-package.test.js +17 -0
- package/ci/self-evolve-loop.test.js +37 -2
- package/ci/telemetry.test.js +31 -1
- package/docs/enforcement-locks.md +114 -0
- package/grill-with-docs/ADR-FORMAT.md +22 -0
- package/grill-with-docs/SKILL.md +1 -1
- package/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/hooks/scripts/lib/telemetry.js +40 -0
- package/hooks/scripts/lib/workflow-isolation.js +10 -2
- package/hooks/scripts/lib/workflow-runtime.js +108 -5
- package/hooks/scripts/state-persist.js +9 -3
- package/hooks/scripts/workflow-disposition.js +19 -8
- package/hooks/scripts/workflow-gate.js +33 -4
- package/multi-agent-workspace/SKILL.md +1 -1
- package/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/contract-integrity/scripts/audit.js +536 -0
- package/plugins/harness-everything/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/plugins/harness-everything/hooks/scripts/lib/telemetry.js +40 -0
- package/plugins/harness-everything/hooks/scripts/lib/workflow-isolation.js +10 -2
- package/plugins/harness-everything/hooks/scripts/lib/workflow-runtime.js +108 -5
- package/plugins/harness-everything/hooks/scripts/state-persist.js +9 -3
- package/plugins/harness-everything/hooks/scripts/workflow-disposition.js +19 -8
- package/plugins/harness-everything/hooks/scripts/workflow-gate.js +33 -4
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/grill-with-docs/ADR-FORMAT.md +22 -0
- package/plugins/harness-everything/skills/grill-with-docs/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/plugins/harness-everything/skills/self-evolve/SKILL.md +1 -1
- package/plugins/harness-everything/skills/self-evolve/scripts/lesson-candidate.js +52 -0
- package/plugins/harness-everything/skills/to-spec/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-spec/references/process.md +26 -0
- package/plugins/harness-everything/skills/to-spec/templates/cli-reference.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/dev-doc.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/feature-spec.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/schema-doc.md +14 -0
- package/plugins/harness-everything/skills/to-tickets/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-tickets/references/publishing-and-templates.md +20 -1
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +3 -2
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +25 -1
- package/plugins/harness-everything/skills/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/plugins/harness-everything/skills/verification-loop/templates/verification-report.template.md +3 -0
- package/scripts/sync-openai-plugin.js +12 -0
- package/self-evolve/SKILL.md +1 -1
- package/self-evolve/scripts/lesson-candidate.js +52 -0
- package/telemetry/schemas/event.schema.json +141 -22
- package/telemetry/scripts/report.js +42 -0
- package/to-spec/SKILL.md +1 -1
- package/to-spec/references/process.md +26 -0
- package/to-spec/templates/cli-reference.md +14 -0
- package/to-spec/templates/dev-doc.md +14 -0
- package/to-spec/templates/feature-spec.md +14 -0
- package/to-spec/templates/schema-doc.md +14 -0
- package/to-tickets/SKILL.md +1 -1
- package/to-tickets/references/publishing-and-templates.md +20 -1
- package/verification-loop/SKILL.md +3 -2
- package/verification-loop/references/verification-phases.md +25 -1
- package/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/verification-loop/templates/verification-report.template.md +3 -0
package/package.json
CHANGED
|
@@ -0,0 +1,536 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const { evaluateEvidence } = require('../../skills/tdd/scripts/quality-gate');
|
|
7
|
+
|
|
8
|
+
const VERSION = '1.0.0';
|
|
9
|
+
const MODES = new Set(['audit', 'strict']);
|
|
10
|
+
const ARTIFACT_KINDS = new Set(['ADR', 'SPEC', 'TICKET', 'TEST', 'IMPLEMENTATION']);
|
|
11
|
+
const ARTIFACT_STATUSES = new Set(['ACTIVE', 'SUPERSEDED', 'DEPRECATED']);
|
|
12
|
+
const REQUIREMENT_STATUSES = new Set(['CURRENT', 'SUPERSEDED']);
|
|
13
|
+
const SOURCE_VALIDITY = new Set(['VALID', 'SOURCE_DEFECT', 'SOURCE_CONFLICT']);
|
|
14
|
+
const PROBE_STATUSES = new Set(['KILLED', 'SURVIVED', 'INVALID', 'NOT_EVALUATED']);
|
|
15
|
+
const DRIFT = new Set([
|
|
16
|
+
'CONSISTENT',
|
|
17
|
+
'UNTRACED_CHANGE',
|
|
18
|
+
'STALE_SPEC',
|
|
19
|
+
'STALE_TEST',
|
|
20
|
+
'STALE_IMPLEMENTATION',
|
|
21
|
+
'SOURCE_DEFECT',
|
|
22
|
+
'SOURCE_CONFLICT',
|
|
23
|
+
'INTENTIONAL_SUPERSESSION',
|
|
24
|
+
'NOT_EVALUATED',
|
|
25
|
+
]);
|
|
26
|
+
|
|
27
|
+
function isObject(value) {
|
|
28
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
29
|
+
}
|
|
30
|
+
function text(value) {
|
|
31
|
+
return typeof value === 'string' && value.trim() ? value.trim() : null;
|
|
32
|
+
}
|
|
33
|
+
function round(value) {
|
|
34
|
+
return Math.round((value + Number.EPSILON) * 100) / 100;
|
|
35
|
+
}
|
|
36
|
+
function unique(values) {
|
|
37
|
+
return [...new Set(values)];
|
|
38
|
+
}
|
|
39
|
+
function safeRelativePath(value) {
|
|
40
|
+
const p = text(value);
|
|
41
|
+
if (!p || path.isAbsolute(p) || p.includes('\\0')) return false;
|
|
42
|
+
const normalized = p.replace(/\\/g, '/');
|
|
43
|
+
return normalized !== '..' && !normalized.startsWith('../') && !normalized.includes('/../');
|
|
44
|
+
}
|
|
45
|
+
function rejectUnknownKeys(value, allowed, errors, location) {
|
|
46
|
+
if (!isObject(value)) return;
|
|
47
|
+
for (const key of Object.keys(value)) {
|
|
48
|
+
if (!allowed.has(key)) errors.push(`${location} contains unknown field: ${key}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function readJson(file) {
|
|
52
|
+
return JSON.parse(fs.readFileSync(path.resolve(file), 'utf8'));
|
|
53
|
+
}
|
|
54
|
+
function stable(value) {
|
|
55
|
+
if (Array.isArray(value)) return value.map(stable);
|
|
56
|
+
if (!isObject(value)) return value;
|
|
57
|
+
return Object.fromEntries(Object.keys(value).sort().map(key => [key, stable(value[key])]));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function validateTrace(input) {
|
|
61
|
+
const errors = [];
|
|
62
|
+
if (!isObject(input)) return ['trace root must be an object'];
|
|
63
|
+
if (input.schemaVersion !== VERSION) errors.push(`schemaVersion must be ${VERSION}`);
|
|
64
|
+
if (!MODES.has(input.mode)) errors.push('mode must be audit or strict');
|
|
65
|
+
if (!Array.isArray(input.artifacts)) errors.push('artifacts must be an array');
|
|
66
|
+
if (!Array.isArray(input.requirements) || input.requirements.length === 0) errors.push('requirements must be a non-empty array');
|
|
67
|
+
if (!Array.isArray(input.probes)) errors.push('probes must be an array');
|
|
68
|
+
if (errors.length) return errors;
|
|
69
|
+
|
|
70
|
+
const artifactIds = new Set();
|
|
71
|
+
for (const [index, artifact] of input.artifacts.entries()) {
|
|
72
|
+
const p = `artifacts[${index}]`;
|
|
73
|
+
if (!isObject(artifact)) { errors.push(`${p} must be an object`); continue; }
|
|
74
|
+
rejectUnknownKeys(artifact, new Set(['id','kind','path','status','revision','requirementId','requirementRevision','changeImpact','supersedes','evidenceRef']), errors, p);
|
|
75
|
+
if (!text(artifact.id)) errors.push(`${p}.id is required`);
|
|
76
|
+
else if (artifactIds.has(artifact.id)) errors.push(`duplicate artifact id: ${artifact.id}`);
|
|
77
|
+
else artifactIds.add(artifact.id);
|
|
78
|
+
if (!ARTIFACT_KINDS.has(artifact.kind)) errors.push(`${p}.kind is invalid`);
|
|
79
|
+
if (!safeRelativePath(artifact.path)) errors.push(`${p}.path must be a repo-relative non-traversing path`);
|
|
80
|
+
if (!ARTIFACT_STATUSES.has(artifact.status)) errors.push(`${p}.status is invalid`);
|
|
81
|
+
if (!Number.isInteger(artifact.revision) || artifact.revision < 1) errors.push(`${p}.revision must be >= 1`);
|
|
82
|
+
if (artifact.requirementRevision != null && (!Number.isInteger(artifact.requirementRevision) || artifact.requirementRevision < 1)) {
|
|
83
|
+
errors.push(`${p}.requirementRevision must be null or >= 1`);
|
|
84
|
+
}
|
|
85
|
+
if (artifact.kind === 'TICKET' && !['none','implementation','behavior','architecture'].includes(artifact.changeImpact)) {
|
|
86
|
+
errors.push(`${p}.changeImpact is required for tickets`);
|
|
87
|
+
}
|
|
88
|
+
if (artifact.kind !== 'TICKET' && artifact.changeImpact !== undefined) errors.push(`${p}.changeImpact is ticket-only`);
|
|
89
|
+
if (artifact.supersedes !== undefined && (!Array.isArray(artifact.supersedes) || new Set(artifact.supersedes).size !== artifact.supersedes.length)) {
|
|
90
|
+
errors.push(`${p}.supersedes must be a unique array`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const requirementIds = new Set();
|
|
95
|
+
for (const [index, requirement] of input.requirements.entries()) {
|
|
96
|
+
const p = `requirements[${index}]`;
|
|
97
|
+
if (!isObject(requirement)) { errors.push(`${p} must be an object`); continue; }
|
|
98
|
+
rejectUnknownKeys(requirement, new Set(['requirementId','revision','status','specId','decisionIds','ticketIds','testIds','implementationIds','requiredProbeIds','sourceValidity','supersededBy']), errors, p);
|
|
99
|
+
if (!text(requirement.requirementId)) errors.push(`${p}.requirementId is required`);
|
|
100
|
+
else if (requirementIds.has(requirement.requirementId)) errors.push(`duplicate requirementId: ${requirement.requirementId}`);
|
|
101
|
+
else requirementIds.add(requirement.requirementId);
|
|
102
|
+
if (!Number.isInteger(requirement.revision) || requirement.revision < 1) errors.push(`${p}.revision must be >= 1`);
|
|
103
|
+
if (!REQUIREMENT_STATUSES.has(requirement.status)) errors.push(`${p}.status is invalid`);
|
|
104
|
+
if (!text(requirement.specId)) errors.push(`${p}.specId is required`);
|
|
105
|
+
for (const field of ['decisionIds','ticketIds','testIds','implementationIds','requiredProbeIds']) {
|
|
106
|
+
if (!Array.isArray(requirement[field])) errors.push(`${p}.${field} must be an array`);
|
|
107
|
+
else if (new Set(requirement[field]).size !== requirement[field].length) errors.push(`${p}.${field} must not contain duplicates`);
|
|
108
|
+
}
|
|
109
|
+
if (!SOURCE_VALIDITY.has(requirement.sourceValidity)) errors.push(`${p}.sourceValidity is invalid`);
|
|
110
|
+
if (requirement.status === 'SUPERSEDED' && !text(requirement.supersededBy)) errors.push(`${p}.supersededBy is required for superseded requirements`);
|
|
111
|
+
if (requirement.status === 'CURRENT' && requirement.supersededBy != null) errors.push(`${p}.supersededBy must be null/omitted for current requirements`);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const probeIds = new Set();
|
|
115
|
+
for (const [index, probe] of input.probes.entries()) {
|
|
116
|
+
const p = `probes[${index}]`;
|
|
117
|
+
if (!isObject(probe)) { errors.push(`${p} must be an object`); continue; }
|
|
118
|
+
rejectUnknownKeys(probe, new Set(['probeId','requirementId','status','sourceSection','evidenceRef','workspaceIsolation','failureClass']), errors, p);
|
|
119
|
+
if (!text(probe.probeId)) errors.push(`${p}.probeId is required`);
|
|
120
|
+
else if (probeIds.has(probe.probeId)) errors.push(`duplicate probeId: ${probe.probeId}`);
|
|
121
|
+
else probeIds.add(probe.probeId);
|
|
122
|
+
if (!text(probe.requirementId)) errors.push(`${p}.requirementId is required`);
|
|
123
|
+
if (!PROBE_STATUSES.has(probe.status)) errors.push(`${p}.status is invalid`);
|
|
124
|
+
if (!text(probe.sourceSection)) errors.push(`${p}.sourceSection is required`);
|
|
125
|
+
if (!['isolated','not-isolated','unknown'].includes(probe.workspaceIsolation)) errors.push(`${p}.workspaceIsolation is invalid`);
|
|
126
|
+
if (!['contract','build','infrastructure','flaky','none','unknown'].includes(probe.failureClass)) errors.push(`${p}.failureClass is invalid`);
|
|
127
|
+
if (['KILLED','SURVIVED'].includes(probe.status) && !text(probe.evidenceRef)) errors.push(`${p}.evidenceRef is required for evaluated probes`);
|
|
128
|
+
if (probe.status === 'KILLED' && probe.failureClass !== 'contract') {
|
|
129
|
+
errors.push(`${p}: KILLED requires failureClass=contract; compile/build/infrastructure failures are invalid protection evidence`);
|
|
130
|
+
}
|
|
131
|
+
if (['KILLED','SURVIVED'].includes(probe.status) && probe.workspaceIsolation !== 'isolated') {
|
|
132
|
+
errors.push(`${p}: evaluated probes require isolated workspace evidence`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return errors;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function artifactMap(trace) {
|
|
139
|
+
return new Map(trace.artifacts.map(artifact => [artifact.id, artifact]));
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function expectedKind(map, id, kind, errors, location) {
|
|
143
|
+
const artifact = map.get(id);
|
|
144
|
+
if (!artifact) { errors.push(`${location}: missing artifact ${id}`); return null; }
|
|
145
|
+
if (artifact.kind !== kind) { errors.push(`${location}: ${id} must be ${kind}, got ${artifact.kind}`); return null; }
|
|
146
|
+
return artifact;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function classifyLineage(requirement, map) {
|
|
150
|
+
const statuses = [];
|
|
151
|
+
const reasons = [];
|
|
152
|
+
const links = [];
|
|
153
|
+
|
|
154
|
+
const spec = map.get(requirement.specId);
|
|
155
|
+
if (!spec || spec.kind !== 'SPEC') {
|
|
156
|
+
statuses.push('UNTRACED_CHANGE');
|
|
157
|
+
reasons.push('missing-current-spec-link');
|
|
158
|
+
} else {
|
|
159
|
+
links.push({ from: requirement.requirementId, to: spec.id, relation: 'defined-by' });
|
|
160
|
+
if (spec.status !== 'ACTIVE' || spec.revision !== requirement.revision) {
|
|
161
|
+
statuses.push('STALE_SPEC');
|
|
162
|
+
reasons.push(spec.status !== 'ACTIVE' ? 'spec-not-active' : 'spec-revision-mismatch');
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (requirement.sourceValidity === 'SOURCE_DEFECT') {
|
|
167
|
+
statuses.push('SOURCE_DEFECT');
|
|
168
|
+
reasons.push('declared-source-defect');
|
|
169
|
+
}
|
|
170
|
+
if (requirement.sourceValidity === 'SOURCE_CONFLICT') {
|
|
171
|
+
statuses.push('SOURCE_CONFLICT');
|
|
172
|
+
reasons.push('declared-source-conflict');
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if (requirement.status === 'SUPERSEDED') {
|
|
176
|
+
statuses.push('INTENTIONAL_SUPERSESSION');
|
|
177
|
+
reasons.push('requirement-explicitly-superseded');
|
|
178
|
+
links.push({ from: requirement.requirementId, to: requirement.supersededBy, relation: 'superseded-by' });
|
|
179
|
+
return { statuses: unique(statuses), reasons: unique(reasons), links };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
for (const id of requirement.decisionIds || []) {
|
|
183
|
+
const adr = map.get(id);
|
|
184
|
+
if (!adr || adr.kind !== 'ADR') {
|
|
185
|
+
statuses.push('UNTRACED_CHANGE');
|
|
186
|
+
reasons.push(`missing-decision:${id}`);
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
links.push({ from: id, to: requirement.requirementId, relation: 'governs' });
|
|
190
|
+
if (adr.status !== 'ACTIVE') {
|
|
191
|
+
statuses.push('UNTRACED_CHANGE');
|
|
192
|
+
reasons.push(`decision-not-active:${id}`);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (!requirement.ticketIds?.length) {
|
|
197
|
+
statuses.push('UNTRACED_CHANGE');
|
|
198
|
+
reasons.push('no-ticket-lineage');
|
|
199
|
+
}
|
|
200
|
+
for (const id of requirement.ticketIds || []) {
|
|
201
|
+
const ticket = map.get(id);
|
|
202
|
+
if (!ticket || ticket.kind !== 'TICKET') {
|
|
203
|
+
statuses.push('UNTRACED_CHANGE');
|
|
204
|
+
reasons.push(`missing-ticket:${id}`);
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
links.push({ from: requirement.requirementId, to: id, relation: 'implemented-by-change' });
|
|
208
|
+
const targetRevision = ticket.requirementRevision;
|
|
209
|
+
if (ticket.requirementId !== requirement.requirementId) {
|
|
210
|
+
statuses.push('UNTRACED_CHANGE');
|
|
211
|
+
reasons.push(`ticket-requirement-mismatch:${id}`);
|
|
212
|
+
}
|
|
213
|
+
if (Number.isInteger(targetRevision) && targetRevision > requirement.revision && ['behavior','architecture'].includes(ticket.changeImpact)) {
|
|
214
|
+
statuses.push('STALE_SPEC');
|
|
215
|
+
statuses.push('UNTRACED_CHANGE');
|
|
216
|
+
reasons.push(`ticket-ahead-of-contract:${id}`);
|
|
217
|
+
} else if (Number.isInteger(targetRevision) && targetRevision !== requirement.revision) {
|
|
218
|
+
statuses.push('UNTRACED_CHANGE');
|
|
219
|
+
reasons.push(`ticket-revision-mismatch:${id}`);
|
|
220
|
+
}
|
|
221
|
+
if (ticket.changeImpact === 'architecture' && !(requirement.decisionIds || []).length) {
|
|
222
|
+
statuses.push('UNTRACED_CHANGE');
|
|
223
|
+
reasons.push(`architecture-change-without-decision-lineage:${id}`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
if (!requirement.testIds?.length) {
|
|
228
|
+
statuses.push('STALE_TEST');
|
|
229
|
+
reasons.push('no-test-lineage');
|
|
230
|
+
}
|
|
231
|
+
for (const id of requirement.testIds || []) {
|
|
232
|
+
const testArtifact = map.get(id);
|
|
233
|
+
if (!testArtifact || testArtifact.kind !== 'TEST') {
|
|
234
|
+
statuses.push('STALE_TEST');
|
|
235
|
+
reasons.push(`missing-test:${id}`);
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
links.push({ from: requirement.requirementId, to: id, relation: 'protected-by' });
|
|
239
|
+
if (testArtifact.requirementId !== requirement.requirementId || testArtifact.requirementRevision !== requirement.revision || testArtifact.status !== 'ACTIVE') {
|
|
240
|
+
statuses.push('STALE_TEST');
|
|
241
|
+
reasons.push(`test-lineage-mismatch:${id}`);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
if (!requirement.implementationIds?.length) {
|
|
246
|
+
statuses.push('STALE_IMPLEMENTATION');
|
|
247
|
+
reasons.push('no-implementation-lineage');
|
|
248
|
+
}
|
|
249
|
+
for (const id of requirement.implementationIds || []) {
|
|
250
|
+
const impl = map.get(id);
|
|
251
|
+
if (!impl || impl.kind !== 'IMPLEMENTATION') {
|
|
252
|
+
statuses.push('STALE_IMPLEMENTATION');
|
|
253
|
+
reasons.push(`missing-implementation:${id}`);
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
links.push({ from: requirement.requirementId, to: id, relation: 'realized-by' });
|
|
257
|
+
if (impl.requirementId !== requirement.requirementId || impl.requirementRevision !== requirement.revision || impl.status !== 'ACTIVE') {
|
|
258
|
+
statuses.push('STALE_IMPLEMENTATION');
|
|
259
|
+
reasons.push(`implementation-lineage-mismatch:${id}`);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
if (statuses.length === 0) statuses.push('CONSISTENT');
|
|
264
|
+
return { statuses: unique(statuses), reasons: unique(reasons), links };
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function protectionForRequirement(requirement, probesById, currentSpec = null) {
|
|
268
|
+
const expected = requirement.requiredProbeIds || [];
|
|
269
|
+
if (expected.length === 0) {
|
|
270
|
+
return {
|
|
271
|
+
status: 'NOT_EVALUATED',
|
|
272
|
+
score: 0,
|
|
273
|
+
required: 0,
|
|
274
|
+
killed: 0,
|
|
275
|
+
survived: 0,
|
|
276
|
+
invalid: 0,
|
|
277
|
+
missing: [],
|
|
278
|
+
reasonCodes: ['no-required-probes-declared'],
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
let killed = 0;
|
|
283
|
+
let survived = 0;
|
|
284
|
+
let invalid = 0;
|
|
285
|
+
const missing = [];
|
|
286
|
+
const reasonCodes = [];
|
|
287
|
+
|
|
288
|
+
for (const id of expected) {
|
|
289
|
+
const probe = probesById.get(id);
|
|
290
|
+
if (!probe || probe.requirementId !== requirement.requirementId) {
|
|
291
|
+
missing.push(id);
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
const sourceMatchesCurrentSpec = currentSpec && typeof probe.sourceSection === 'string' &&
|
|
295
|
+
(probe.sourceSection === currentSpec.path || probe.sourceSection.startsWith(`${currentSpec.path}#`));
|
|
296
|
+
if (!sourceMatchesCurrentSpec) {
|
|
297
|
+
invalid++;
|
|
298
|
+
reasonCodes.push('probe-source-not-current-spec');
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
if (probe.status === 'KILLED' && probe.failureClass === 'contract' && probe.workspaceIsolation === 'isolated' && text(probe.evidenceRef)) killed++;
|
|
302
|
+
else if (probe.status === 'SURVIVED') survived++;
|
|
303
|
+
else invalid++;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
if (missing.length || invalid) {
|
|
307
|
+
reasonCodes.push(...(missing.length ? ['required-probe-missing'] : []), ...(invalid ? ['invalid-probe-evidence'] : []));
|
|
308
|
+
return { status: 'NOT_EVALUATED', score: 0, required: expected.length, killed, survived, invalid, missing, reasonCodes };
|
|
309
|
+
}
|
|
310
|
+
if (survived > 0) {
|
|
311
|
+
return {
|
|
312
|
+
status: 'SURVIVED',
|
|
313
|
+
score: 0,
|
|
314
|
+
required: expected.length,
|
|
315
|
+
killed,
|
|
316
|
+
survived,
|
|
317
|
+
invalid,
|
|
318
|
+
missing,
|
|
319
|
+
reasonCodes: ['required-contract-probe-survived'],
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
if (killed === expected.length) {
|
|
323
|
+
return {
|
|
324
|
+
status: 'PROTECTED',
|
|
325
|
+
score: 100,
|
|
326
|
+
required: expected.length,
|
|
327
|
+
killed,
|
|
328
|
+
survived,
|
|
329
|
+
invalid,
|
|
330
|
+
missing,
|
|
331
|
+
reasonCodes: ['all-predeclared-contract-probes-killed'],
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
return { status: 'NOT_EVALUATED', score: 0, required: expected.length, killed, survived, invalid, missing, reasonCodes: ['probe-evidence-incomplete'] };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function completenessByRequirement(tddEvidence) {
|
|
338
|
+
if (!tddEvidence) return { report: null, byId: new Map(), score: 0, result: 'NOT_EVALUATED' };
|
|
339
|
+
const report = evaluateEvidence(tddEvidence);
|
|
340
|
+
const byId = new Map(report.requirements.map(req => [req.requirementId, { score: req.score, result: req.result }]));
|
|
341
|
+
return { report, byId, score: report.score, result: report.result };
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
function evaluateTrace(trace, options = {}) {
|
|
345
|
+
const validationErrors = validateTrace(trace);
|
|
346
|
+
const map = Array.isArray(trace?.artifacts) ? artifactMap(trace) : new Map();
|
|
347
|
+
const probesById = new Map((trace?.probes || []).map(probe => [probe.probeId, probe]));
|
|
348
|
+
const completeness = completenessByRequirement(options.tddEvidence || null);
|
|
349
|
+
const reportRequirements = [];
|
|
350
|
+
const links = [];
|
|
351
|
+
const globalErrors = [...validationErrors];
|
|
352
|
+
|
|
353
|
+
if (validationErrors.length === 0) {
|
|
354
|
+
const reqIds = new Set(trace.requirements.map(req => req.requirementId));
|
|
355
|
+
for (const artifact of trace.artifacts) {
|
|
356
|
+
for (const prior of artifact.supersedes || []) {
|
|
357
|
+
if (!map.has(prior)) globalErrors.push(`artifact ${artifact.id} supersedes missing artifact ${prior}`);
|
|
358
|
+
else links.push({ from: artifact.id, to: prior, relation: 'supersedes' });
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
for (const requirement of trace.requirements) {
|
|
362
|
+
if (requirement.status === 'SUPERSEDED' && !reqIds.has(requirement.supersededBy)) {
|
|
363
|
+
globalErrors.push(`${requirement.requirementId}: supersededBy target does not exist: ${requirement.supersededBy}`);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
for (const requirement of trace?.requirements || []) {
|
|
369
|
+
const lineage = classifyLineage(requirement, map);
|
|
370
|
+
links.push(...lineage.links);
|
|
371
|
+
const currentSpec = map.get(requirement.specId) || null;
|
|
372
|
+
const protection = protectionForRequirement(requirement, probesById, currentSpec);
|
|
373
|
+
const quality = completeness.byId.get(requirement.requirementId) || { score: 0, result: 'NOT_EVALUATED' };
|
|
374
|
+
const statuses = [...lineage.statuses];
|
|
375
|
+
const qualityDetail = completeness.report?.requirements?.find(item => item.requirementId === requirement.requirementId) || null;
|
|
376
|
+
if (qualityDetail && currentSpec && qualityDetail.source?.status === 'AUTHORITATIVE' &&
|
|
377
|
+
qualityDetail.source.path !== currentSpec.path) {
|
|
378
|
+
statuses.push('STALE_TEST');
|
|
379
|
+
lineage.reasons.push('tdd-source-not-current-spec');
|
|
380
|
+
}
|
|
381
|
+
if (requirement.status === 'CURRENT' && protection.status === 'NOT_EVALUATED') statuses.push('NOT_EVALUATED');
|
|
382
|
+
if (requirement.status === 'CURRENT' && quality.result === 'NOT_EVALUATED') statuses.push('NOT_EVALUATED');
|
|
383
|
+
const blocking = statuses.filter(status => !['CONSISTENT','INTENTIONAL_SUPERSESSION'].includes(status));
|
|
384
|
+
reportRequirements.push({
|
|
385
|
+
requirementId: requirement.requirementId,
|
|
386
|
+
revision: requirement.revision,
|
|
387
|
+
lifecycleStatus: requirement.status,
|
|
388
|
+
drift: unique(statuses),
|
|
389
|
+
reasonCodes: lineage.reasons,
|
|
390
|
+
completeness: quality,
|
|
391
|
+
protection,
|
|
392
|
+
gateEligible: requirement.status === 'SUPERSEDED' ? true : blocking.length === 0 && quality.result === 'PASS' && protection.status === 'PROTECTED',
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
const current = reportRequirements.filter(req => req.lifecycleStatus === 'CURRENT');
|
|
397
|
+
const completenessScore = current.length
|
|
398
|
+
? round(current.reduce((sum, req) => sum + req.completeness.score, 0) / current.length)
|
|
399
|
+
: 0;
|
|
400
|
+
const protectionScore = current.length
|
|
401
|
+
? round(current.reduce((sum, req) => sum + req.protection.score, 0) / current.length)
|
|
402
|
+
: 0;
|
|
403
|
+
const contractIntegrityScore = Math.min(completenessScore, protectionScore);
|
|
404
|
+
const driftCounts = Object.fromEntries([...DRIFT].map(status => [status, 0]));
|
|
405
|
+
for (const req of reportRequirements) for (const status of req.drift) driftCounts[status] = (driftCounts[status] || 0) + 1;
|
|
406
|
+
|
|
407
|
+
const strictPass = globalErrors.length === 0 &&
|
|
408
|
+
current.length > 0 &&
|
|
409
|
+
current.every(req => req.gateEligible) &&
|
|
410
|
+
contractIntegrityScore === 100;
|
|
411
|
+
|
|
412
|
+
return {
|
|
413
|
+
schemaVersion: VERSION,
|
|
414
|
+
mode: trace?.mode || null,
|
|
415
|
+
result: globalErrors.length ? 'FAIL' : (strictPass ? 'PASS' : (trace?.mode === 'audit' ? 'AUDIT' : 'FAIL')),
|
|
416
|
+
completionGate: strictPass ? 'PASS' : (current.some(req => req.drift.includes('NOT_EVALUATED')) ? 'NOT_EVALUATED' : 'FAIL'),
|
|
417
|
+
gateEligible: strictPass,
|
|
418
|
+
scores: {
|
|
419
|
+
completenessScore,
|
|
420
|
+
protectionScore,
|
|
421
|
+
contractIntegrityScore,
|
|
422
|
+
formula: 'min(completenessScore, protectionScore)',
|
|
423
|
+
},
|
|
424
|
+
counts: {
|
|
425
|
+
artifacts: trace?.artifacts?.length || 0,
|
|
426
|
+
requirements: reportRequirements.length,
|
|
427
|
+
currentRequirements: current.length,
|
|
428
|
+
probes: trace?.probes?.length || 0,
|
|
429
|
+
},
|
|
430
|
+
driftCounts,
|
|
431
|
+
requirements: reportRequirements,
|
|
432
|
+
graph: {
|
|
433
|
+
nodes: [
|
|
434
|
+
...(trace?.artifacts || []).map(artifact => ({ id: artifact.id, kind: artifact.kind, status: artifact.status, path: artifact.path })),
|
|
435
|
+
...(trace?.requirements || []).map(req => ({ id: req.requirementId, kind: 'REQUIREMENT', status: req.status, revision: req.revision })),
|
|
436
|
+
],
|
|
437
|
+
links: unique(links.map(link => JSON.stringify(stable(link)))).map(value => JSON.parse(value)),
|
|
438
|
+
},
|
|
439
|
+
tddQuality: completeness.report,
|
|
440
|
+
errors: globalErrors,
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
function markdown(report) {
|
|
445
|
+
const lines = [
|
|
446
|
+
'# Contract Integrity Report',
|
|
447
|
+
'',
|
|
448
|
+
`- Mode: **${report.mode}**`,
|
|
449
|
+
`- Result: **${report.result}**`,
|
|
450
|
+
`- Completion gate: **${report.completionGate}**`,
|
|
451
|
+
`- Completeness score: **${report.scores.completenessScore}**`,
|
|
452
|
+
`- Protection score: **${report.scores.protectionScore}**`,
|
|
453
|
+
`- Contract integrity score: **${report.scores.contractIntegrityScore}** = min(completeness, protection)`,
|
|
454
|
+
'',
|
|
455
|
+
'| Requirement | Rev | Drift | Completeness | Protection | Gate |',
|
|
456
|
+
'| --- | ---: | --- | ---: | --- | --- |',
|
|
457
|
+
];
|
|
458
|
+
for (const req of report.requirements) {
|
|
459
|
+
lines.push(`| ${req.requirementId} | ${req.revision} | ${req.drift.join(', ')} | ${req.completeness.score} | ${req.protection.status} | ${req.gateEligible ? 'PASS' : 'FAIL'} |`);
|
|
460
|
+
}
|
|
461
|
+
if (report.errors.length) {
|
|
462
|
+
lines.push('', '## Errors', '', ...report.errors.map(error => `- ${error}`));
|
|
463
|
+
}
|
|
464
|
+
lines.push('', '## Evidence boundary', '');
|
|
465
|
+
lines.push('- Completeness is derived by the existing #58 evaluator; aggregate scores supplied by agents are ignored.');
|
|
466
|
+
lines.push('- Protection counts only predeclared required probes. Extra/padded probes do not raise a requirement score.');
|
|
467
|
+
lines.push('- Build/infrastructure/flaky failures are invalid probe evidence and cannot count as killed contract probes.');
|
|
468
|
+
lines.push('- Audit mode reports legacy debt without authorizing completion; strict mode fails on any unresolved drift or NOT_EVALUATED protection.');
|
|
469
|
+
return `${lines.join('\n')}\n`;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
function parseArgs(argv) {
|
|
473
|
+
const args = { input: null, tddEvidence: null, output: null, markdown: null };
|
|
474
|
+
for (let i = 0; i < argv.length; i++) {
|
|
475
|
+
const token = argv[i];
|
|
476
|
+
if (!args.input && !token.startsWith('--')) args.input = token;
|
|
477
|
+
else if (token === '--tdd-evidence') args.tddEvidence = argv[++i];
|
|
478
|
+
else if (token === '--output') args.output = argv[++i];
|
|
479
|
+
else if (token === '--markdown') args.markdown = argv[++i];
|
|
480
|
+
else throw new Error(`unknown argument: ${token}`);
|
|
481
|
+
}
|
|
482
|
+
return args;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
function runCli(argv) {
|
|
486
|
+
let args;
|
|
487
|
+
try { args = parseArgs(argv.slice(2)); }
|
|
488
|
+
catch (error) { console.error(`Contract Integrity: FAIL\n- ${error.message}`); return 2; }
|
|
489
|
+
if (!args.input) {
|
|
490
|
+
console.error('Usage: node contract-integrity/scripts/audit.js <trace.json> [--tdd-evidence <#58-evidence.json>] [--output report.json] [--markdown report.md]');
|
|
491
|
+
return 2;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
let trace;
|
|
495
|
+
let tddEvidence = null;
|
|
496
|
+
try {
|
|
497
|
+
trace = readJson(args.input);
|
|
498
|
+
if (args.tddEvidence) tddEvidence = readJson(args.tddEvidence);
|
|
499
|
+
} catch (error) {
|
|
500
|
+
console.error(`Contract Integrity: FAIL\n- ${error.message}`);
|
|
501
|
+
return 2;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
const report = evaluateTrace(trace, { tddEvidence });
|
|
505
|
+
console.log(`Contract Integrity: ${report.result}`);
|
|
506
|
+
console.log(`Scores: completeness=${report.scores.completenessScore} protection=${report.scores.protectionScore} integrity=${report.scores.contractIntegrityScore}`);
|
|
507
|
+
console.log(`Gate: ${report.completionGate}`);
|
|
508
|
+
for (const req of report.requirements) console.log(`- ${req.requirementId}: ${req.drift.join(', ')} / protection=${req.protection.status}`);
|
|
509
|
+
for (const error of report.errors) console.error(`- error: ${error}`);
|
|
510
|
+
|
|
511
|
+
if (args.output) {
|
|
512
|
+
fs.mkdirSync(path.dirname(path.resolve(args.output)), { recursive: true });
|
|
513
|
+
fs.writeFileSync(path.resolve(args.output), `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
514
|
+
}
|
|
515
|
+
if (args.markdown) {
|
|
516
|
+
fs.mkdirSync(path.dirname(path.resolve(args.markdown)), { recursive: true });
|
|
517
|
+
fs.writeFileSync(path.resolve(args.markdown), markdown(report), 'utf8');
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
if (report.errors.length) return 2;
|
|
521
|
+
if (trace.mode === 'audit') return 0;
|
|
522
|
+
return report.result === 'PASS' ? 0 : 1;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
if (require.main === module) process.exit(runCli(process.argv));
|
|
526
|
+
|
|
527
|
+
module.exports = {
|
|
528
|
+
DRIFT,
|
|
529
|
+
VERSION,
|
|
530
|
+
classifyLineage,
|
|
531
|
+
evaluateTrace,
|
|
532
|
+
markdown,
|
|
533
|
+
protectionForRequirement,
|
|
534
|
+
runCli,
|
|
535
|
+
validateTrace,
|
|
536
|
+
};
|
|
@@ -8,6 +8,7 @@ const {
|
|
|
8
8
|
getSessionId,
|
|
9
9
|
getSessionDir,
|
|
10
10
|
} = require('./harness-state');
|
|
11
|
+
const { emitLessonTelemetry } = require('./telemetry');
|
|
11
12
|
|
|
12
13
|
const SCHEMA_VERSION = 1;
|
|
13
14
|
const TRIGGERS = new Set(['rule-of-3-recovery', 'verifier-fail-pass']);
|
|
@@ -134,6 +135,27 @@ function createLearningOpportunity(payload, input = {}) {
|
|
|
134
135
|
updatedAt: observedAt,
|
|
135
136
|
};
|
|
136
137
|
atomicWriteJson(file, candidate);
|
|
138
|
+
const lifecycleReasonCodes = [...candidate.trigger.reasonCodes, `trigger:${triggerType}`];
|
|
139
|
+
emitLessonTelemetry({
|
|
140
|
+
event: 'learning_opportunity',
|
|
141
|
+
root,
|
|
142
|
+
payload,
|
|
143
|
+
sessionId,
|
|
144
|
+
candidateId,
|
|
145
|
+
observedAt,
|
|
146
|
+
status: 'success',
|
|
147
|
+
reasonCodes: lifecycleReasonCodes,
|
|
148
|
+
});
|
|
149
|
+
emitLessonTelemetry({
|
|
150
|
+
event: 'lesson_proposed',
|
|
151
|
+
root,
|
|
152
|
+
payload,
|
|
153
|
+
sessionId,
|
|
154
|
+
candidateId,
|
|
155
|
+
observedAt,
|
|
156
|
+
status: 'success',
|
|
157
|
+
reasonCodes: lifecycleReasonCodes,
|
|
158
|
+
});
|
|
137
159
|
return { created: true, file, candidate };
|
|
138
160
|
}
|
|
139
161
|
|
|
@@ -6,11 +6,26 @@ const path = require('path');
|
|
|
6
6
|
const { getWorkspaceRoot, getStateRoot, getSessionDir } = require('./harness-state');
|
|
7
7
|
|
|
8
8
|
const TELEMETRY_SCHEMA_VERSION = 1;
|
|
9
|
+
const LESSON_EVENT_TYPES = new Set([
|
|
10
|
+
'learning_opportunity',
|
|
11
|
+
'lesson_proposed',
|
|
12
|
+
'lesson_screened',
|
|
13
|
+
'lesson_evaluated',
|
|
14
|
+
'lesson_accepted',
|
|
15
|
+
'lesson_rejected',
|
|
16
|
+
'lesson_inconclusive',
|
|
17
|
+
'lesson_persisted',
|
|
18
|
+
'lesson_retrieved',
|
|
19
|
+
'lesson_validated',
|
|
20
|
+
'lesson_regressed',
|
|
21
|
+
'lesson_superseded',
|
|
22
|
+
]);
|
|
9
23
|
const EVENT_TYPES = new Set([
|
|
10
24
|
'skill.invoke',
|
|
11
25
|
'skill.loaded',
|
|
12
26
|
'skill.complete',
|
|
13
27
|
'tool.observed',
|
|
28
|
+
...LESSON_EVENT_TYPES,
|
|
14
29
|
]);
|
|
15
30
|
const STATUSES = new Set(['unknown', 'success', 'failure', 'aborted']);
|
|
16
31
|
const FORBIDDEN_KEYS = new Set([
|
|
@@ -102,6 +117,29 @@ function telemetryFile(root, payload) {
|
|
|
102
117
|
return path.join(getStateRoot(root || getWorkspaceRoot(payload), payload), 'telemetry', 'events.jsonl');
|
|
103
118
|
}
|
|
104
119
|
|
|
120
|
+
function emitLessonTelemetry(input = {}) {
|
|
121
|
+
const event = String(input.event || '');
|
|
122
|
+
if (!LESSON_EVENT_TYPES.has(event)) {
|
|
123
|
+
return { ok: false, error: `unsupported lesson telemetry event: ${event}`, overheadMs: 0 };
|
|
124
|
+
}
|
|
125
|
+
const invocationId = localId('lesson', input.candidateId);
|
|
126
|
+
if (!invocationId) {
|
|
127
|
+
return { ok: false, error: 'lesson telemetry requires candidateId', overheadMs: 0 };
|
|
128
|
+
}
|
|
129
|
+
return emitTelemetry({
|
|
130
|
+
event,
|
|
131
|
+
host: ['claude', 'codex', 'opencode', 'unknown'].includes(input.host)
|
|
132
|
+
? input.host
|
|
133
|
+
: detectHost(input.payload || {}),
|
|
134
|
+
observedAt: input.observedAt || nowIso(),
|
|
135
|
+
sessionId: localId('session', input.sessionId),
|
|
136
|
+
invocationId,
|
|
137
|
+
skillName: 'self-evolve',
|
|
138
|
+
status: STATUSES.has(input.status) ? input.status : 'unknown',
|
|
139
|
+
reasonCodes: input.reasonCodes || [],
|
|
140
|
+
}, { root: input.root, payload: input.payload });
|
|
141
|
+
}
|
|
142
|
+
|
|
105
143
|
function emitTelemetry(input, options = {}) {
|
|
106
144
|
const started = process.hrtime.bigint();
|
|
107
145
|
try {
|
|
@@ -361,9 +399,11 @@ function readEvents(file) {
|
|
|
361
399
|
module.exports = {
|
|
362
400
|
EVENT_TYPES,
|
|
363
401
|
FORBIDDEN_KEYS,
|
|
402
|
+
LESSON_EVENT_TYPES,
|
|
364
403
|
STATUSES,
|
|
365
404
|
TELEMETRY_SCHEMA_VERSION,
|
|
366
405
|
detectHost,
|
|
406
|
+
emitLessonTelemetry,
|
|
367
407
|
emitTelemetry,
|
|
368
408
|
hostDuration,
|
|
369
409
|
localId,
|