harness-everything 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/CHANGELOG.md +52 -0
  4. package/ci/contract-integrity.test.js +203 -0
  5. package/ci/self-evolve-loop.test.js +311 -0
  6. package/ci/telemetry.test.js +193 -0
  7. package/contract-integrity/fixtures/consistent-trace.json +94 -0
  8. package/contract-integrity/fixtures/reconciled-trace.json +105 -0
  9. package/contract-integrity/fixtures/ticket-drift-trace.json +94 -0
  10. package/contract-integrity/schemas/trace.schema.json +77 -0
  11. package/contract-integrity/scripts/audit.js +536 -0
  12. package/docs/contract-integrity.md +109 -0
  13. package/docs/telemetry.md +62 -0
  14. package/hooks/hooks.json +43 -0
  15. package/hooks/scripts/contract-test.js +24 -0
  16. package/hooks/scripts/lib/learning-opportunity.js +148 -0
  17. package/hooks/scripts/lib/telemetry.js +377 -0
  18. package/hooks/scripts/rule-of-3-tracker.js +23 -0
  19. package/hooks/scripts/state-persist.js +5 -0
  20. package/hooks/scripts/telemetry-adapter.js +25 -0
  21. package/multi-agent-workspace/SKILL.md +1 -1
  22. package/multi-agent-workspace/scripts/index_memory.js +2 -0
  23. package/opencode-plugin/index.mjs +213 -1
  24. package/opencode-plugin/plugin.json +1 -1
  25. package/package.json +7 -2
  26. package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
  27. package/plugins/harness-everything/hooks/hooks.json +35 -0
  28. package/plugins/harness-everything/hooks/scripts/contract-test.js +24 -0
  29. package/plugins/harness-everything/hooks/scripts/lib/learning-opportunity.js +148 -0
  30. package/plugins/harness-everything/hooks/scripts/lib/telemetry.js +377 -0
  31. package/plugins/harness-everything/hooks/scripts/rule-of-3-tracker.js +23 -0
  32. package/plugins/harness-everything/hooks/scripts/state-persist.js +5 -0
  33. package/plugins/harness-everything/hooks/scripts/telemetry-adapter.js +25 -0
  34. package/plugins/harness-everything/plugin.json +1 -1
  35. package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +1 -1
  36. package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +2 -0
  37. package/plugins/harness-everything/skills/self-evolve/SKILL.md +14 -19
  38. package/plugins/harness-everything/skills/self-evolve/references/memory-resolution.md +32 -13
  39. package/plugins/harness-everything/skills/self-evolve/schemas/lesson-candidate.schema.json +101 -0
  40. package/plugins/harness-everything/skills/self-evolve/scripts/lesson-candidate.js +303 -0
  41. package/plugins/harness-everything/skills/self-evolve/scripts/self-regression.js +4 -1
  42. package/self-evolve/SKILL.md +14 -19
  43. package/self-evolve/references/memory-resolution.md +32 -13
  44. package/self-evolve/schemas/lesson-candidate.schema.json +101 -0
  45. package/self-evolve/scripts/lesson-candidate.js +303 -0
  46. package/self-evolve/scripts/self-regression.js +4 -1
  47. package/telemetry/schemas/event.schema.json +39 -0
  48. package/telemetry/scripts/benchmark.js +47 -0
  49. package/telemetry/scripts/report.js +133 -0
@@ -0,0 +1,377 @@
1
+ 'use strict';
2
+
3
+ const crypto = require('crypto');
4
+ const fs = require('fs');
5
+ const path = require('path');
6
+ const { getWorkspaceRoot, getStateRoot, getSessionDir } = require('./harness-state');
7
+
8
+ const TELEMETRY_SCHEMA_VERSION = 1;
9
+ const EVENT_TYPES = new Set([
10
+ 'skill.invoke',
11
+ 'skill.loaded',
12
+ 'skill.complete',
13
+ 'tool.observed',
14
+ ]);
15
+ const STATUSES = new Set(['unknown', 'success', 'failure', 'aborted']);
16
+ const FORBIDDEN_KEYS = new Set([
17
+ 'prompt', 'source', 'sourceCode', 'content', 'contents', 'toolArgs', 'toolArguments',
18
+ 'arguments', 'command', 'cwd', 'path', 'file', 'filePath', 'user', 'userId', 'email',
19
+ 'username', 'message', 'messages', 'transcript',
20
+ ]);
21
+
22
+ function telemetryEnabled() {
23
+ const value = String(process.env.HARNESS_TELEMETRY || 'local').toLowerCase();
24
+ return !['0', 'false', 'off', 'disabled', 'none'].includes(value);
25
+ }
26
+
27
+ function nowIso() {
28
+ return new Date().toISOString();
29
+ }
30
+
31
+ function localId(prefix, value) {
32
+ const text = String(value || '').trim();
33
+ if (!text) return null;
34
+ return `${prefix}-${crypto.createHash('sha256').update(text, 'utf8').digest('hex').slice(0, 16)}`;
35
+ }
36
+
37
+ function safeSkillName(value) {
38
+ const text = String(value || '').trim();
39
+ if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$/.test(text)) return null;
40
+ return text.toLowerCase();
41
+ }
42
+
43
+ function safeToolName(value) {
44
+ const text = String(value || '').trim();
45
+ if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,95}$/.test(text)) return null;
46
+ return text;
47
+ }
48
+
49
+ function finiteDuration(value) {
50
+ const n = Number(value);
51
+ return Number.isFinite(n) && n >= 0 ? Math.round(n * 1000) / 1000 : null;
52
+ }
53
+
54
+ function detectHost(payload = {}) {
55
+ const direct = String(payload.host || payload.runtime || '').toLowerCase();
56
+ if (['claude', 'codex', 'opencode'].includes(direct)) return direct;
57
+ if (process.env.CLAUDE_PLUGIN_ROOT || process.env.CLAUDECODE || process.env.CLAUDE_CODE_ENTRYPOINT) return 'claude';
58
+ if (process.env.CODEX_HOME || process.env.CODEX || process.env.PLUGIN_ROOT) return 'codex';
59
+ return 'unknown';
60
+ }
61
+
62
+ function normalizeEvent(input) {
63
+ if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('telemetry event must be an object');
64
+ for (const key of Object.keys(input)) {
65
+ if (FORBIDDEN_KEYS.has(key)) throw new Error(`telemetry field is forbidden by privacy contract: ${key}`);
66
+ }
67
+
68
+ const event = String(input.event || '');
69
+ if (!EVENT_TYPES.has(event)) throw new Error(`unsupported telemetry event: ${event}`);
70
+ const status = STATUSES.has(input.status) ? input.status : 'unknown';
71
+ const skillName = input.skillName == null ? null : safeSkillName(input.skillName);
72
+ if (input.skillName != null && !skillName) throw new Error('invalid skillName');
73
+
74
+ const timing = input.timing || {};
75
+ const normalized = {
76
+ schemaVersion: TELEMETRY_SCHEMA_VERSION,
77
+ eventId: input.eventId || crypto.randomUUID(),
78
+ event,
79
+ host: ['claude', 'codex', 'opencode', 'unknown'].includes(input.host) ? input.host : 'unknown',
80
+ observedAt: input.observedAt || nowIso(),
81
+ sessionId: input.sessionId || null,
82
+ turnId: input.turnId || null,
83
+ agentId: input.agentId || null,
84
+ invocationId: input.invocationId || null,
85
+ skillName,
86
+ skillVersion: input.skillVersion == null ? null : String(input.skillVersion).slice(0, 40),
87
+ toolName: input.toolName == null ? null : safeToolName(input.toolName),
88
+ status,
89
+ retryCount: Number.isInteger(input.retryCount) && input.retryCount >= 0 ? input.retryCount : 0,
90
+ timing: {
91
+ skillLoadDurationMs: finiteDuration(timing.skillLoadDurationMs),
92
+ activeWindowMs: finiteDuration(timing.activeWindowMs),
93
+ attributedToolDurationMs: finiteDuration(timing.attributedToolDurationMs),
94
+ },
95
+ reasonCodes: [...new Set((input.reasonCodes || []).map(v => String(v).slice(0, 80)).filter(Boolean))].slice(0, 16),
96
+ };
97
+ if (!Number.isFinite(Date.parse(normalized.observedAt))) throw new Error('observedAt must be ISO-8601');
98
+ return normalized;
99
+ }
100
+
101
+ function telemetryFile(root, payload) {
102
+ return path.join(getStateRoot(root || getWorkspaceRoot(payload), payload), 'telemetry', 'events.jsonl');
103
+ }
104
+
105
+ function emitTelemetry(input, options = {}) {
106
+ const started = process.hrtime.bigint();
107
+ try {
108
+ if (!telemetryEnabled()) return { ok: true, disabled: true, overheadMs: 0 };
109
+ const event = normalizeEvent(input);
110
+ const file = options.file || telemetryFile(options.root, options.payload);
111
+ fs.mkdirSync(path.dirname(file), { recursive: true });
112
+ fs.appendFileSync(file, `${JSON.stringify(event)}\n`, 'utf8');
113
+ const overheadMs = Number(process.hrtime.bigint() - started) / 1e6;
114
+ return { ok: true, file, event, overheadMs };
115
+ } catch (error) {
116
+ // Telemetry is observability only. It must never become an execution gate.
117
+ return {
118
+ ok: false,
119
+ error: String(error && (error.message || error)).slice(0, 240),
120
+ overheadMs: Number(process.hrtime.bigint() - started) / 1e6,
121
+ };
122
+ }
123
+ }
124
+
125
+ function sessionState(payload) {
126
+ const root = getWorkspaceRoot(payload);
127
+ const rawSession = payload && (payload.session_id || payload.sessionId || payload.sessionID);
128
+ const sessionDir = getSessionDir(root, rawSession, payload);
129
+ return {
130
+ root,
131
+ rawSession,
132
+ sessionId: localId('session', rawSession || 'default'),
133
+ file: path.join(sessionDir, 'telemetry-state.json'),
134
+ };
135
+ }
136
+
137
+ function readState(file) {
138
+ try {
139
+ const value = JSON.parse(fs.readFileSync(file, 'utf8'));
140
+ if (value && value.schemaVersion === 1 && value.pending && value.active) return value;
141
+ } catch (_) {}
142
+ return { schemaVersion: 1, pending: {}, active: {} };
143
+ }
144
+
145
+ function writeState(file, state) {
146
+ try {
147
+ fs.mkdirSync(path.dirname(file), { recursive: true });
148
+ const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
149
+ fs.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
150
+ fs.renameSync(tmp, file);
151
+ return true;
152
+ } catch (_) {
153
+ return false;
154
+ }
155
+ }
156
+
157
+ function toolUseId(payload) {
158
+ const value = payload && (
159
+ payload.tool_use_id || payload.toolUseId || payload.call_id || payload.callId ||
160
+ payload.tool_call_id || payload.toolCallId
161
+ );
162
+ return value ? localId('call', value) : null;
163
+ }
164
+
165
+ function skillNameFromPayload(payload) {
166
+ const input = (payload && (payload.tool_input || payload.toolInput || payload.input)) || {};
167
+ return safeSkillName(input.skill || input.skill_name || input.skillName || input.name);
168
+ }
169
+
170
+ function skillVersionFromRoots(skillName) {
171
+ if (!skillName) return null;
172
+ const roots = [
173
+ process.env.CLAUDE_PLUGIN_ROOT && path.join(process.env.CLAUDE_PLUGIN_ROOT, skillName, 'SKILL.md'),
174
+ process.env.PLUGIN_ROOT && path.join(process.env.PLUGIN_ROOT, 'skills', skillName, 'SKILL.md'),
175
+ ].filter(Boolean);
176
+ for (const file of roots) {
177
+ try {
178
+ const text = fs.readFileSync(file, 'utf8');
179
+ const metadata = text.match(/metadata:\s*[\s\S]{0,300}?version:\s*["']?([^\s"'\n]+)["']?/i);
180
+ if (metadata) return metadata[1].slice(0, 40);
181
+ const direct = text.match(/^version:\s*["']?([^\s"'\n]+)["']?/mi);
182
+ if (direct) return direct[1].slice(0, 40);
183
+ } catch (_) {}
184
+ }
185
+ return null;
186
+ }
187
+
188
+ function hostStatus(payload) {
189
+ const response = (payload && (payload.tool_response || payload.toolResponse || payload.tool_result)) || {};
190
+ const rawExit = response.exitCode ?? response.exit_code ?? payload?.exitCode ?? payload?.exit_code;
191
+ if (response.is_error === true || response.error || (typeof rawExit === 'number' && rawExit !== 0)) return 'failure';
192
+ if (typeof rawExit === 'number' && rawExit === 0) return 'success';
193
+ return 'unknown';
194
+ }
195
+
196
+ function hostDuration(payload) {
197
+ const response = (payload && (payload.tool_response || payload.toolResponse || payload.tool_result)) || {};
198
+ const candidates = [
199
+ response.duration_ms, response.durationMs, response.elapsed_ms, response.elapsedMs,
200
+ payload && payload.duration_ms, payload && payload.durationMs,
201
+ ];
202
+ return finiteDuration(candidates.find(v => v !== null && v !== undefined && Number.isFinite(Number(v))));
203
+ }
204
+
205
+ function recordSkillHook(payload) {
206
+ try {
207
+ const hookEvent = String(payload?.hook_event_name || payload?.hookEventName || '');
208
+ const toolName = String(payload?.tool_name || payload?.toolName || '');
209
+ const stateContext = sessionState(payload || {});
210
+ const state = readState(stateContext.file);
211
+ const host = detectHost(payload);
212
+ const callId = toolUseId(payload) || localId('call', `${Date.now()}:${toolName}`);
213
+
214
+ if (hookEvent === 'PreToolUse' && toolName === 'Skill') {
215
+ const skillName = skillNameFromPayload(payload);
216
+ if (!skillName) return { ok: true, ignored: true, reason: 'skill-name-unavailable' };
217
+ const invocationId = crypto.randomUUID();
218
+ const startedAt = Date.now();
219
+ state.pending[callId] = {
220
+ invocationId,
221
+ skillName,
222
+ skillVersion: skillVersionFromRoots(skillName),
223
+ startedAt,
224
+ };
225
+ writeState(stateContext.file, state);
226
+ return emitTelemetry({
227
+ event: 'skill.invoke',
228
+ host,
229
+ sessionId: stateContext.sessionId,
230
+ turnId: localId('turn', payload.turn_id || payload.turnId),
231
+ agentId: localId('agent', payload.agent_id || payload.agentId),
232
+ invocationId,
233
+ skillName,
234
+ skillVersion: state.pending[callId].skillVersion,
235
+ status: 'unknown',
236
+ reasonCodes: ['host-skill-tool-pre'],
237
+ }, { root: stateContext.root, payload });
238
+ }
239
+
240
+ if ((hookEvent === 'PostToolUse' || hookEvent === 'PostToolUseFailure') && toolName === 'Skill') {
241
+ const pending = state.pending[callId];
242
+ if (!pending) return { ok: true, ignored: true, reason: 'uncorrelated-skill-load' };
243
+ delete state.pending[callId];
244
+ const status = hookEvent === 'PostToolUseFailure' ? 'failure' : hostStatus(payload);
245
+ const endedAt = Date.now();
246
+ const loadDuration = Math.max(0, endedAt - pending.startedAt);
247
+ if (status === 'failure') {
248
+ writeState(stateContext.file, state);
249
+ emitTelemetry({
250
+ event: 'skill.loaded', host, sessionId: stateContext.sessionId,
251
+ invocationId: pending.invocationId, skillName: pending.skillName,
252
+ skillVersion: pending.skillVersion, status: 'failure',
253
+ timing: { skillLoadDurationMs: loadDuration },
254
+ reasonCodes: ['host-skill-tool-failed'],
255
+ }, { root: stateContext.root, payload });
256
+ return emitTelemetry({
257
+ event: 'skill.complete', host, sessionId: stateContext.sessionId,
258
+ invocationId: pending.invocationId, skillName: pending.skillName,
259
+ skillVersion: pending.skillVersion, status: 'failure',
260
+ timing: { skillLoadDurationMs: loadDuration, activeWindowMs: 0 },
261
+ reasonCodes: ['load-failure-terminal'],
262
+ }, { root: stateContext.root, payload });
263
+ }
264
+ state.active[pending.invocationId] = {
265
+ ...pending,
266
+ loadedAt: endedAt,
267
+ skillLoadDurationMs: loadDuration,
268
+ attributedToolDurationMs: null,
269
+ observedToolCount: 0,
270
+ };
271
+ writeState(stateContext.file, state);
272
+ return emitTelemetry({
273
+ event: 'skill.loaded', host, sessionId: stateContext.sessionId,
274
+ invocationId: pending.invocationId, skillName: pending.skillName,
275
+ skillVersion: pending.skillVersion, status: status === 'success' ? 'success' : 'unknown',
276
+ timing: { skillLoadDurationMs: loadDuration },
277
+ reasonCodes: ['host-skill-tool-post'],
278
+ }, { root: stateContext.root, payload });
279
+ }
280
+
281
+ if (hookEvent === 'Stop') {
282
+ const stoppedAt = Date.now();
283
+ const results = [];
284
+ for (const [invocationId, active] of Object.entries(state.active)) {
285
+ results.push(emitTelemetry({
286
+ event: 'skill.complete',
287
+ host,
288
+ sessionId: stateContext.sessionId,
289
+ invocationId,
290
+ skillName: active.skillName,
291
+ skillVersion: active.skillVersion,
292
+ status: payload?.reason === 'aborted' || payload?.stop_reason === 'aborted' ? 'aborted' : 'unknown',
293
+ timing: {
294
+ skillLoadDurationMs: active.skillLoadDurationMs,
295
+ activeWindowMs: Math.max(0, stoppedAt - active.loadedAt),
296
+ attributedToolDurationMs: active.attributedToolDurationMs,
297
+ },
298
+ reasonCodes: ['turn-boundary-completion-window'],
299
+ }, { root: stateContext.root, payload }));
300
+ delete state.active[invocationId];
301
+ }
302
+ state.pending = {};
303
+ writeState(stateContext.file, state);
304
+ return { ok: results.every(result => result.ok), results };
305
+ }
306
+
307
+ return { ok: true, ignored: true };
308
+ } catch (error) {
309
+ return { ok: false, error: String(error && (error.message || error)).slice(0, 240) };
310
+ }
311
+ }
312
+
313
+ function observeTool(payload) {
314
+ try {
315
+ if (!telemetryEnabled()) return { ok: true, disabled: true };
316
+ const stateContext = sessionState(payload || {});
317
+ const state = readState(stateContext.file);
318
+ const active = Object.entries(state.active);
319
+ if (active.length === 0) return { ok: true, ignored: true };
320
+ const toolName = safeToolName(payload?.tool_name || payload?.toolName || payload?.tool || 'unknown');
321
+ const duration = hostDuration(payload);
322
+ const host = detectHost(payload);
323
+ const status = hostStatus(payload);
324
+ const results = [];
325
+ for (const [invocationId, entry] of active) {
326
+ if (duration !== null) {
327
+ entry.attributedToolDurationMs = (entry.attributedToolDurationMs || 0) + duration;
328
+ }
329
+ entry.observedToolCount = (entry.observedToolCount || 0) + 1;
330
+ results.push(emitTelemetry({
331
+ event: 'tool.observed',
332
+ host,
333
+ sessionId: stateContext.sessionId,
334
+ invocationId,
335
+ skillName: entry.skillName,
336
+ skillVersion: entry.skillVersion,
337
+ toolName,
338
+ status,
339
+ timing: { attributedToolDurationMs: duration },
340
+ reasonCodes: [
341
+ duration === null ? 'host-tool-duration-unavailable' : 'host-tool-duration-observed',
342
+ 'skill-active-window-overlap',
343
+ ],
344
+ }, { root: stateContext.root, payload }));
345
+ }
346
+ writeState(stateContext.file, state);
347
+ return { ok: results.every(result => result.ok), results };
348
+ } catch (error) {
349
+ return { ok: false, error: String(error && (error.message || error)).slice(0, 240) };
350
+ }
351
+ }
352
+
353
+ function readEvents(file) {
354
+ if (!fs.existsSync(file)) return [];
355
+ return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter(Boolean).map((line, index) => {
356
+ try { return normalizeEvent(JSON.parse(line)); }
357
+ catch (error) { return { invalid: true, line: index + 1, error: error.message }; }
358
+ });
359
+ }
360
+
361
+ module.exports = {
362
+ EVENT_TYPES,
363
+ FORBIDDEN_KEYS,
364
+ STATUSES,
365
+ TELEMETRY_SCHEMA_VERSION,
366
+ detectHost,
367
+ emitTelemetry,
368
+ hostDuration,
369
+ localId,
370
+ normalizeEvent,
371
+ observeTool,
372
+ readEvents,
373
+ recordSkillHook,
374
+ safeSkillName,
375
+ telemetryEnabled,
376
+ telemetryFile,
377
+ };
@@ -3,6 +3,7 @@ const fs = require('fs');
3
3
  const path = require('path');
4
4
  const crypto = require('crypto');
5
5
  const { getWorkspaceRoot, getSessionDir } = require('./lib/harness-state');
6
+ const { createLearningOpportunity } = require('./lib/learning-opportunity');
6
7
 
7
8
  // PostToolUse/PostToolUseFailure hooks in Claude Code receive their payload via stdin
8
9
  let inputData = '';
@@ -114,6 +115,28 @@ process.stdin.on('end', () => {
114
115
 
115
116
  fs.writeFileSync(stateFile, JSON.stringify(state, null, 2), 'utf8');
116
117
  } else if (!isFailureEvent && explicitSuccess) {
118
+ // A successful action after an accepted zoom-out is an objective recovery
119
+ // boundary. Emit only IDs/hashes/category; never copy command/output text.
120
+ if ((state.zoomOutCycles || 0) > 0 && state.lastHash) {
121
+ const toolEventId = payload.tool_use_id || payload.toolUseId || `success-${Date.now()}`;
122
+ try {
123
+ createLearningOpportunity(payload, {
124
+ triggerType: 'rule-of-3-recovery',
125
+ sourceEventIds: [
126
+ `rule-of-3:${state.lastHash}:${state.lastFailureAt || 0}`,
127
+ `tool:${toolEventId}`,
128
+ ],
129
+ evidence: {
130
+ failureSignature: state.lastHash,
131
+ failureCategory: state.category || 'unknown',
132
+ zoomOutCycles: state.zoomOutCycles,
133
+ recovered: true,
134
+ },
135
+ });
136
+ } catch (_) {
137
+ // Learning capture is additive and must never break recovery tracking.
138
+ }
139
+ }
117
140
  // Only reset on a *confirmed* zero exit code, not merely "not a failure".
118
141
  if (state.count > 0 || state.zoomOutCycles > 0) {
119
142
  state.count = 0;
@@ -2,6 +2,7 @@
2
2
  const fs = require('fs');
3
3
  const path = require('path');
4
4
  const { getWorkspaceRoot, getSessionDir } = require('./lib/harness-state');
5
+ const { observeTool } = require('./lib/telemetry');
5
6
 
6
7
  // Commands that count as "verification ran" for the Stop gate
7
8
  // (hooks/scripts/stop-gate.js). Recall over precision: under-blocking the
@@ -88,6 +89,10 @@ function processState(payload) {
88
89
  }
89
90
 
90
91
  fs.writeFileSync(stateFile, JSON.stringify(state, null, 2), 'utf8');
92
+ // Reuse this already-running post-tool hook instead of spawning another
93
+ // process for every tool solely for telemetry. The observer is fail-open
94
+ // and stores no command/args/output content.
95
+ observeTool(payload);
91
96
  } else {
92
97
  // No standard input payload, or invalid JSON. Just maintain a heartbeat timestamp
93
98
  if (!fs.existsSync(stateFile)) {
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ const { recordSkillHook } = require('./lib/telemetry');
5
+
6
+ let input = '';
7
+ let finished = false;
8
+
9
+ function finish() {
10
+ if (finished) return;
11
+ finished = true;
12
+ try {
13
+ const payload = input.trim() ? JSON.parse(input) : {};
14
+ recordSkillHook(payload);
15
+ } catch (_) {
16
+ // Telemetry must never affect the task lifecycle.
17
+ }
18
+ process.exit(0);
19
+ }
20
+
21
+ const timer = setTimeout(finish, 300);
22
+ process.stdin.setEncoding('utf8');
23
+ process.stdin.on('data', chunk => { input += chunk; });
24
+ process.stdin.on('end', () => { clearTimeout(timer); finish(); });
25
+ process.stdin.on('error', () => { clearTimeout(timer); finish(); });
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
3
3
  "name": "harness-everything",
4
- "version": "0.9.0",
4
+ "version": "0.11.0",
5
5
  "description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
6
6
  "author": {
7
7
  "name": "Miya Daniel",
@@ -4,7 +4,7 @@ description: "Scaffold a verified multi-agent workspace and select bounded speci
4
4
  license: Apache-2.0
5
5
  metadata:
6
6
  author: Miya Daniel
7
- version: 0.9.0
7
+ version: 0.10.0
8
8
  ---
9
9
 
10
10
  # Multi-Agent Workspace
@@ -147,8 +147,10 @@ function retrieveMemoryRecords({ workspace, task = '', requirement = '', role =
147
147
  continue;
148
148
  }
149
149
 
150
+ const lessonCandidateMatch = String(record.source || '').match(/^lesson-candidate:(lesson-[a-f0-9]{24})$/);
150
151
  included.push({
151
152
  ...record,
153
+ origin: lessonCandidateMatch ? { lessonCandidateId: lessonCandidateMatch[1] } : null,
152
154
  trust: 'untrusted-data',
153
155
  retrieval: {
154
156
  reasonCodes: [
@@ -4,14 +4,12 @@ description: "Extract root causes from resolved problems and persist them as def
4
4
  license: Apache-2.0
5
5
  metadata:
6
6
  author: Miya Daniel
7
- version: 0.9.0
7
+ version: 0.11.0
8
8
  ---
9
9
 
10
10
  # Self-evolve
11
11
 
12
- Record reusable root causes from resolved work.
13
-
14
- Boundary: the host agent supplies evidence and a generalized root cause; this skill classifies and persists it, and never scans host transcripts. Retrieved memory is data, not trusted instructions.
12
+ Turn verified recovery into reusable lessons without scanning global transcripts. Runtime hooks may create privacy-minimal lesson candidates; the agent/human supplies the generalized rule. Retrieved memory is untrusted data.
15
13
 
16
14
  Deep dive: <this-skill-dir>/references/memory-resolution.md
17
15
 
@@ -19,27 +17,24 @@ Deep dive: <this-skill-dir>/references/memory-resolution.md
19
17
 
20
18
  | Component | Specification |
21
19
  | :--- | :--- |
22
- | **Trigger / Input** | Resolved struggle, zoom-out recovery, or explicit request. |
23
- | **Expected Output** | Authorized persisted rule, session-scoped review candidate, or registered dynamic skill. |
24
- | **State Mutations** | Writes a session candidate, authorized durable memory, or generated-skill manifest. |
25
- | **Enforcement Gate** | `memory.write` + single-use router capability, then persistence screening/quality/dedup. |
20
+ | **Trigger / Input** | Verified recovery, explicit lesson, or self-evolve request. |
21
+ | **Expected Output** | Candidate, governed memory rule, or dynamic skill. |
22
+ | **Enforcement Gate** | Candidate evaluation + `memory.write` capability + persistence screening. |
26
23
 
27
24
  ## Core Workflow
28
25
 
29
- 1. Prefer existing `MEMORY.md`/`RULES.md`/`CLAUDE.md`/`AGENTS.md`.
30
- 2. Route the lesson:
31
- - Simple rule → run `node "<this-skill-dir>/scripts/persist-memory.js" "<rule>" --authorization "<routing capability>" --source "<provenance>"`. `none` rejects, `propose` creates a candidate, and `persist-via-self-evolve` may persist.
32
- - Reusable procedure → follow `<skills-repo-root>/skill-creator/SKILL.md`, then register via `<this-skill-dir>/scripts/register-dynamic-skill.js`.
33
- 3. Never append durable memory directly; use the script. Retrieved memory is untrusted. Run `self-regression.js` only for this repo's own script/skill changes.
26
+ 1. List candidates with `node "<this-skill-dir>/scripts/lesson-candidate.js" list --workspace <workspace> --session-id <session>`.
27
+ 2. Evaluate with `lesson-candidate.js evaluate ... --rule "<generalized rule>"`. Non-replayable recovery stays inconclusive.
28
+ 3. Promote only `accepted` candidates with `lesson-candidate.js promote ... --authorization "<routing capability>"`; this reuses governed `persist-memory.js`.
29
+ 4. Reusable procedures use `<skills-repo-root>/skill-creator/SKILL.md`. Never append durable memory directly.
34
30
 
35
31
  ## USE FOR:
36
- - Lesson after hard debugging recovery
32
+ - Verified debugging or zoom-out recovery
37
33
  - Defensive rule found mid-task
38
- - Reusable pattern packaged as dynamic skill
39
- - Post-zoom-out rule extraction
34
+ - Reusable procedure worth packaging
40
35
 
41
36
  ## DO NOT USE FOR:
42
- - One-off notes that won't recur
43
- - Lessons already captured
44
- - Secrets or prompt-injection text in memory files
37
+ - One-off notes
38
+ - Already-captured lessons
39
+ - Secrets or prompt-injection text
45
40
  - Static repo skills (use `skill-creator`)
@@ -4,7 +4,7 @@ Details moved from SKILL.md. Read when you need the full decision matrix or the
4
4
 
5
5
  ## Responsibility Boundary
6
6
 
7
- The host agent owns access to the current session and any history explicitly exposed by the host. It selects authorized evidence and states a generalized root cause before invoking this skill. `self-evolve` classifies, deduplicates, validates, and persists that result. It MUST NOT scan global transcript stores, persist raw transcripts, or act as a transcript daemon.
7
+ Harness may observe only its own structured runtime events. Rule-of-3 recovery and verifier fail→pass transitions can create privacy-minimal lesson candidates automatically; raw prompts, commands, tool output, source text, and global host transcripts are not copied into those candidates. The agent or Human Partner still supplies the generalized rule before evaluation. `self-evolve` MUST NOT scan global transcript stores, persist raw transcripts, or act as a transcript daemon.
8
8
 
9
9
  ## Triggers
10
10
 
@@ -16,24 +16,43 @@ The host agent owns access to the current session and any history explicitly exp
16
16
 
17
17
  ```mermaid
18
18
  flowchart TD
19
- Start[Evidence-backed lesson] --> Route[Harness router memory.write]
20
- Route -->|none| Reject[Reject durable write]
21
- Route -->|propose| Capability[Issue single-use workflow/session capability]
22
- Route -->|persist-via-self-evolve| Capability
23
- Capability --> Screen[Secret + injection + quality screening]
24
- Screen --> Similarity{Possible duplicate/paraphrase?}
25
- Similarity -->|yes| Candidate[Session-scoped review candidate]
26
- Similarity -->|no| Disposition{Effective disposition}
27
- Disposition -->|propose / active Fable run| Candidate
28
- Disposition -->|persist-via-self-evolve| Durable[RULES.md + memory-index.json]
19
+ Runtime[Harness-owned recovery evidence] --> Opportunity[Learning opportunity]
20
+ Opportunity --> Candidate[Lesson candidate: proposed]
21
+ Candidate --> Screen[Generalized rule + safety/quality screening]
22
+ Screen --> Eval{Evidence class}
23
+ Eval -->|objective verifier fail→pass| Accepted[accepted for governed promotion]
24
+ Eval -->|linear recovery / insufficient replay| Hold[inconclusive]
25
+ Eval -->|unsafe/weak| Reject[rejected]
26
+ Accepted --> Auth[#134 single-use memory capability]
27
+ Auth --> Durable[RULES.md + memory-index.json]
29
28
  Durable --> Retrieve[Scoped deterministic retrieval]
30
- Retrieve --> Untrusted[Return as untrusted data/context]
29
+ Retrieve --> Outcome[retrieved → validated / regressed / superseded]
31
30
  ```
32
31
 
33
- For a simple rule, persist only the generalized constraint or tip. The router-issued capability is opaque, single-use, and bound to one workflow/session. Do not replace it with a free-form `--role` or `--session-id` claim. Active Fable runs remain candidate-only; durable promotion should happen through a later authorized coordinator/self-evolve path.
32
+ The versioned candidate contract is `<this-skill-dir>/schemas/lesson-candidate.schema.json`. Runtime creation records only structured IDs/hashes/status transitions. Candidate states are distinct from effectiveness claims: `accepted` means the evidence is grounded enough to permit governed memory promotion, while `evaluation.improvementClaim` remains false until #71/later recurrence evidence supports a behavioral claim.
33
+
34
+ For a simple rule, inspect/list the session candidate, evaluate it with a generalized constraint or tip, and promote only an `accepted` candidate. The router-issued capability is opaque, single-use, and bound to the authorizing workflow/session; that promotion session may differ from the original recovery session and both provenances remain recorded. Do not replace the capability with a free-form `--role` or `--session-id` claim. Active Fable runs remain candidate-only; durable promotion happens through a later authorized coordinator/self-evolve path.
34
35
 
35
36
  For a reusable multi-step procedure, load `skill-creator/SKILL.md`, create the draft skill, and use `register-dynamic-skill.js` to update each platform's `manifest.json` `generated[]` registry. The host agent remains responsible for deciding which evidence is relevant; these scripts do not discover session history.
36
37
 
38
+ ## Candidate Lifecycle
39
+
40
+ ```text
41
+ observed -> proposed -> screened -> evaluated
42
+ -> accepted -> persisted -> retrieved
43
+ | -> validated
44
+ | -> regressed
45
+ | -> superseded
46
+ -> rejected
47
+ -> inconclusive
48
+ ```
49
+
50
+ - `rule-of-3-recovery` is intentionally non-replayable from one linear trace and remains `inconclusive` until paired/later recurrence evidence exists.
51
+ - `verifier-fail-pass` can pass deterministic grounding checks because the retained contract records an objective fail→pass transition, but this still does not prove future behavioral improvement.
52
+ - Identical runtime evidence is idempotent and maps to one candidate ID.
53
+ - Runtime hooks can create candidates but cannot call the durable memory writer directly.
54
+ - Record later evidence explicitly with `lesson-candidate.js observe --outcome retrieved|validated|regressed|superseded --evidence "<ref>"`; outcome transitions never happen merely because a rule exists.
55
+
37
56
  ## Memory Governance Rules
38
57
 
39
58
  1. **Workflow authorization first**: `memory.write=none` rejects writes, `propose` writes only a session candidate, and `persist-via-self-evolve` is the only disposition eligible for durable rule persistence.