harness-everything 0.9.0 → 0.11.0

package/docs/telemetry.md

@@ -0,0 +1,62 @@
+# Harness Telemetry
+
+Harness telemetry is a **local operational evidence layer**, not analytics upload and not a memory store. It normalizes host-native lifecycle evidence into JSONL so benchmark/report definitions can evolve without rewriting historical events.
+
+## Privacy defaults
+
+Events contain skill/runtime metadata only:
+
+- skill name/version;
+- host;
+- locally hashed session/turn/agent IDs;
+- random invocation ID;
+- tool name (never tool arguments);
+- status/retry count;
+- separate timing fields;
+- short reason codes.
+
+The sink rejects content-bearing fields such as prompt, source/code contents, command/tool arguments, paths, transcript, messages, email/user identity. It never sends data over the network. Set `HARNESS_TELEMETRY=off` to disable local writes.
+
+Raw events live under the workspace-keyed Harness state root:
+
+```text
+~/.agents/harness-everything/workspaces/<workspace-key>/state/telemetry/events.jsonl
+```
+
+They are not project files and are not committed automatically.
+
+## Timing semantics
+
+Do not collapse these into one `duration_ms`:
+
+- `skillLoadDurationMs`: host Skill pre-event to correlated Skill post-event. This is loading/framework latency.
+- `activeWindowMs`: successful/observed skill load until the host turn/Stop boundary. This is a context exposure window, **not CPU/runtime**.
+- `attributedToolDurationMs`: tool duration observed while a skill invocation is active. It is observational/non-exclusive: overlapping active skills may each receive the same tool observation.
+
+When a host does not expose a reliable duration, the value remains `null`; Harness does not fabricate it.
+
+## Host adapters
+
+| Host | Adapter | Evidence boundary |
+| --- | --- | --- |
+| Claude Code | Narrow `Skill` Pre/Post hooks + Stop; existing `state-persist` observes host-reported tool duration | Mechanism covered in CI; live certification remains #82 |
+| OpenCode | Existing single-process plugin records skill before/after, measures tool elapsed in-process, closes on `session.idle` | Mechanism covered in CI; retained live support remains bounded by #37/#82 |
+| Codex | OpenAI package includes the same narrow Skill/Stop adapter | **Mechanism only** while plugin hook mounting/attribution is blocked by #122; no live claim |
+
+Telemetry failure is always fail-open and cannot block a task.
+
+## Reports
+
+```bash
+node telemetry/scripts/report.js --workspace <repo>
+node telemetry/scripts/report.js --file <events.jsonl> --markdown
+node telemetry/scripts/benchmark.js 250
+```
+
+The report exposes invocation counts, completion statuses, P50/P95 for each timing vocabulary, and invalid-event counts. Raw JSONL remains the source of truth; SQLite/dashboard projections can be added later without changing the event evidence contract.
+
+The overhead benchmark measures only local event normalization + append cost. Host adapter/process startup overhead should be measured separately in live host studies before using telemetry data for performance comparisons.
+
+## Benchmark use
+
+#71 may consume telemetry cost/execution evidence, but correctness pass rates stay separate. Infrastructure/telemetry failures must not become behavioral failures.

package/hooks/hooks.json

@@ -102,6 +102,17 @@
         ],
         "description": "Snapshot repo state before a subagent burst starts",
         "id": "harness:pre:subagent-scope-guard"
+      },
+      {
+        "matcher": "Skill",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/telemetry-adapter.js\""
+          }
+        ],
+        "description": "Record privacy-safe Harness skill load start telemetry",
+        "id": "harness:pre:skill-telemetry"
       }
     ],
     "Stop": [
@@ -134,6 +145,16 @@
         ],
         "description": "Pre-delivery gate - bounce the end of a turn once per edit batch when edits were never verified",
         "id": "harness:stop:stop-gate"
+      },
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/telemetry-adapter.js\""
+          }
+        ],
+        "description": "Close Harness skill active-window telemetry at the turn boundary",
+        "id": "harness:stop:skill-telemetry"
       }
     ],
     "PostToolUse": [
@@ -202,6 +223,17 @@
         ],
         "description": "Diff the whole repo after a subagent burst finishes - catches scope creep past what the task briefed",
         "id": "harness:post:subagent-scope-guard"
+      },
+      {
+        "matcher": "Skill",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/telemetry-adapter.js\""
+          }
+        ],
+        "description": "Record privacy-safe Harness skill load completion telemetry",
+        "id": "harness:post:skill-telemetry"
       }
     ],
     "PostToolUseFailure": [
@@ -226,6 +258,17 @@
         ],
         "description": "Track failed tool error signatures for Rule of 3",
         "id": "harness:post-failure:rule-of-3-tracker"
+      },
+      {
+        "matcher": "Skill",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/telemetry-adapter.js\""
+          }
+        ],
+        "description": "Record failed Harness skill loading without prompt/tool arguments",
+        "id": "harness:post-failure:skill-telemetry"
       }
     ]
   }

package/hooks/scripts/contract-test.js

@@ -23,6 +23,7 @@ const {
   getWorkerId,
 } = require('./lib/fable-contracts');
 const { loadWorkflow, matchingRun, recordBudgetEvent } = require('./lib/workflow-runtime');
+const { createLearningOpportunity } = require('./lib/learning-opportunity');
 
 function resultState(payload) {
   const toolResponse = payload.tool_response || payload.tool_result || {};
@@ -137,7 +138,30 @@ process.stdin.on('end', () => {
           }
         }
       }
+      const previousStatus = entry.contract.status;
+      const previousVerifiedAt = entry.contract.verifiedAt || entry.contract.updatedAt || null;
       const contract = updateRunContract(entry, payload, command, sessionId);
+      if (previousStatus === 'fail' && contract.status === 'pass') {
+        try {
+          createLearningOpportunity(payload, {
+            triggerType: 'verifier-fail-pass',
+            sourceEventIds: [
+              `verifier-fail:${contract.runId}:${contract.stageId}:${previousVerifiedAt || 'unknown'}`,
+              `verifier-pass:${contract.runId}:${contract.stageId}:${contract.verifiedAt}`,
+            ],
+            evidence: {
+              planId: contract.planId,
+              runId: contract.runId,
+              stageId: contract.stageId,
+              previousStatus: 'fail',
+              currentStatus: 'pass',
+              verificationEvidenceRef: contract.verificationEvidence,
+            },
+          });
+        } catch (_) {
+          // Learning capture is additive; verification disposition stays authoritative.
+        }
+      }
       console.log(`[Contract Test] ${contract.planId}/${contract.runId}/${contract.stageId}: ${contract.status.toUpperCase()} ${contract.checkCommand}`);
       if (contract.status === 'fail') {
         if (workflowContext.workflow?.workflowId) {

package/hooks/scripts/lib/learning-opportunity.js

@@ -0,0 +1,148 @@
+'use strict';
+
+const crypto = require('crypto');
+const fs = require('fs');
+const path = require('path');
+const {
+  getWorkspaceRoot,
+  getSessionId,
+  getSessionDir,
+} = require('./harness-state');
+
+const SCHEMA_VERSION = 1;
+const TRIGGERS = new Set(['rule-of-3-recovery', 'verifier-fail-pass']);
+
+function atomicWriteJson(filePath, value) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  const temporary = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+  fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+  fs.renameSync(temporary, filePath);
+}
+
+function readJson(filePath) {
+  try { return JSON.parse(fs.readFileSync(filePath, 'utf8')); } catch (_) { return null; }
+}
+
+function safeText(value, max = 160) {
+  const text = String(value || '').replace(/[\r\n\0]+/g, ' ').trim();
+  return text ? text.slice(0, max) : null;
+}
+
+function safeId(value) {
+  const text = safeText(value, 180);
+  return text && /^[A-Za-z0-9._:@/-]+$/.test(text) ? text : null;
+}
+
+function hash(value) {
+  return crypto.createHash('sha256').update(String(value), 'utf8').digest('hex');
+}
+
+function workflowIdentity(sessionDir) {
+  const workflow = readJson(path.join(sessionDir, 'workflow-run.json'));
+  return {
+    workflowId: safeId(workflow && workflow.workflowId),
+    runId: safeId(workflow && workflow.runId),
+    strategy: safeText(workflow && workflow.strategy, 80),
+  };
+}
+
+function candidateDirectory(sessionDir) {
+  return path.join(sessionDir, 'learning-candidates');
+}
+
+function sourceIds(values) {
+  return [...new Set((values || []).map(value => safeId(value)).filter(Boolean))].slice(0, 16);
+}
+
+function minimalEvidence(triggerType, evidence = {}) {
+  if (triggerType === 'rule-of-3-recovery') {
+    return {
+      failureSignature: safeId(evidence.failureSignature),
+      failureCategory: safeText(evidence.failureCategory, 40),
+      zoomOutCycles: Number.isInteger(evidence.zoomOutCycles) ? evidence.zoomOutCycles : 1,
+      recovered: evidence.recovered === true,
+    };
+  }
+  if (triggerType === 'verifier-fail-pass') {
+    return {
+      planId: safeId(evidence.planId),
+      runId: safeId(evidence.runId),
+      stageId: safeId(evidence.stageId),
+      previousStatus: evidence.previousStatus === 'fail' ? 'fail' : null,
+      currentStatus: evidence.currentStatus === 'pass' ? 'pass' : null,
+      verificationEvidenceRef: safeText(evidence.verificationEvidenceRef, 240),
+    };
+  }
+  return {};
+}
+
+function createLearningOpportunity(payload, input = {}) {
+  const triggerType = String(input.triggerType || '');
+  if (!TRIGGERS.has(triggerType)) throw new Error(`unsupported learning trigger: ${triggerType}`);
+
+  const root = getWorkspaceRoot(payload);
+  const sessionId = getSessionId(payload);
+  const sessionDir = getSessionDir(root, sessionId, payload);
+  const identity = workflowIdentity(sessionDir);
+  const ids = sourceIds(input.sourceEventIds);
+  if (ids.length === 0) throw new Error('learning opportunity requires at least one source event id');
+
+  const evidence = minimalEvidence(triggerType, input.evidence);
+  const fingerprint = hash(JSON.stringify({
+    sessionId,
+    triggerType,
+    sourceEventIds: ids,
+    evidence,
+  }));
+  const candidateId = `lesson-${fingerprint.slice(0, 24)}`;
+  const file = path.join(candidateDirectory(sessionDir), `${candidateId}.json`);
+  const existing = readJson(file);
+  if (existing) return { created: false, file, candidate: existing };
+
+  const observedAt = new Date().toISOString();
+  const candidate = {
+    schemaVersion: SCHEMA_VERSION,
+    candidateId,
+    contentHash: null,
+    state: 'proposed',
+    trigger: {
+      type: triggerType,
+      reasonCodes: triggerType === 'rule-of-3-recovery'
+        ? ['repeated-failure', 'zoom-out-completed', 'verified-subsequent-success']
+        : ['verifier-failed', 'later-verifier-pass'],
+      sourceEventIds: ids,
+      evidenceClass: triggerType === 'verifier-fail-pass' ? 'objective-transition' : 'runtime-recovery',
+    },
+    provenance: {
+      sessionId,
+      workflowId: identity.workflowId,
+      runId: identity.runId,
+      strategy: identity.strategy,
+    },
+    evidence,
+    generalizedRule: null,
+    scope: { task: null, requirement: null, role: null },
+    screening: null,
+    evaluation: null,
+    persistence: null,
+    outcome: null,
+    history: [
+      { state: 'observed', observedAt, reasonCode: 'runtime-learning-opportunity' },
+      { state: 'proposed', observedAt, reasonCode: 'objective-recovery-trigger' },
+    ],
+    createdAt: observedAt,
+    updatedAt: observedAt,
+  };
+  atomicWriteJson(file, candidate);
+  return { created: true, file, candidate };
+}
+
+module.exports = {
+  SCHEMA_VERSION,
+  TRIGGERS,
+  atomicWriteJson,
+  candidateDirectory,
+  createLearningOpportunity,
+  hash,
+  readJson,
+};