harness-everything 0.9.0 → 0.11.0

package/self-evolve/scripts/lesson-candidate.js

@@ -0,0 +1,303 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { contentHash, scoreRuleQuality, screenMemoryForPersistence } = require('./persist-memory');
+
+let getWorkspaceRoot;
+let getWorkspaceStateDir;
+try {
+  ({ getWorkspaceRoot, getWorkspaceStateDir } = require('../../scripts/lib/workspace'));
+} catch (_) {
+  getWorkspaceRoot = () => null;
+  getWorkspaceStateDir = () => null;
+}
+
+const SCHEMA_VERSION = 1;
+const TERMINAL = new Set(['rejected', 'validated', 'regressed', 'superseded']);
+const OUTCOME_STATES = new Set(['retrieved', 'validated', 'regressed', 'superseded']);
+
+function safeSessionName(value) {
+  const text = String(value || 'default').replace(/[^A-Za-z0-9._-]+/g, '_').replace(/^\.+/, '').slice(0, 128);
+  return text || 'default';
+}
+
+function atomicWriteJson(filePath, value) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  const temporary = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+  fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+  fs.renameSync(temporary, filePath);
+}
+
+function readJson(filePath) {
+  try { return JSON.parse(fs.readFileSync(filePath, 'utf8')); } catch (_) { return null; }
+}
+
+function candidatePath(workspace, sessionId, candidateId) {
+  if (!workspace || !getWorkspaceStateDir(workspace)) throw new Error('workspace state root is unavailable');
+  if (!/^lesson-[a-f0-9]{24}$/.test(String(candidateId || ''))) throw new Error('invalid candidate id');
+  return path.join(
+    getWorkspaceStateDir(workspace),
+    'state',
+    'sessions',
+    safeSessionName(sessionId),
+    'learning-candidates',
+    `${candidateId}.json`,
+  );
+}
+
+function loadCandidate({ workspace, sessionId, candidateId }) {
+  const file = candidatePath(workspace, sessionId, candidateId);
+  const candidate = readJson(file);
+  if (!candidate || candidate.schemaVersion !== SCHEMA_VERSION || candidate.candidateId !== candidateId) {
+    throw new Error('lesson candidate not found or malformed');
+  }
+  return { file, candidate };
+}
+
+function pushState(candidate, state, reasonCode, evidenceRef = null) {
+  const now = new Date().toISOString();
+  candidate.state = state;
+  candidate.updatedAt = now;
+  candidate.history = [...(candidate.history || []), {
+    state,
+    observedAt: now,
+    reasonCode,
+    evidenceRef: evidenceRef || null,
+  }].slice(-32);
+}
+
+function replayability(candidate) {
+  if (candidate.trigger?.type === 'verifier-fail-pass' &&
+      candidate.evidence?.previousStatus === 'fail' &&
+      candidate.evidence?.currentStatus === 'pass' &&
+      candidate.evidence?.stageId &&
+      candidate.evidence?.verificationEvidenceRef) {
+    return {
+      replayable: true,
+      reasonCode: 'objective-verifier-transition-retained',
+      method: 'deterministic-transition-check',
+    };
+  }
+  if (candidate.trigger?.type === 'rule-of-3-recovery') {
+    return {
+      replayable: false,
+      reasonCode: 'linear-recovery-is-not-counterfactual-replay',
+      method: 'paired-or-later-recurrence-required',
+    };
+  }
+  return {
+    replayable: false,
+    reasonCode: 'insufficient-structured-evidence',
+    method: 'live-evidence-required',
+  };
+}
+
+function evaluateCandidate(input) {
+  const { file, candidate } = loadCandidate(input);
+  if (TERMINAL.has(candidate.state) || candidate.state === 'persisted') {
+    throw new Error(`candidate in terminal state: ${candidate.state}`);
+  }
+  const rule = String(input.rule || '').trim();
+  if (!rule) throw new Error('evaluate requires --rule with a generalized reusable lesson');
+
+  const screening = screenMemoryForPersistence(rule);
+  const qualityScore = scoreRuleQuality(rule, []);
+  candidate.generalizedRule = rule;
+  candidate.contentHash = contentHash(rule);
+  candidate.scope = {
+    task: String(input.scopeTask || '').trim() || null,
+    requirement: String(input.scopeRequirement || '').trim() || null,
+    role: String(input.scopeRole || '').trim() || null,
+  };
+  candidate.screening = {
+    allowed: screening.allowed,
+    secretReasonCodes: screening.secretReasonCodes,
+    injectionReasonCodes: screening.injectionReasonCodes,
+    qualityScore,
+  };
+  pushState(candidate, 'screened', screening.allowed ? 'screening-clear' : 'screening-rejected');
+
+  if (!screening.allowed || qualityScore < 5) {
+    candidate.evaluation = {
+      disposition: 'rejected',
+      reasonCodes: [
+        ...screening.secretReasonCodes,
+        ...screening.injectionReasonCodes,
+        ...(qualityScore < 5 ? ['quality-below-threshold'] : []),
+      ],
+      replayable: false,
+      method: 'screening',
+      improvementClaim: false,
+    };
+    pushState(candidate, 'rejected', 'candidate-screening-or-quality-failed');
+    atomicWriteJson(file, candidate);
+    return { file, candidate };
+  }
+
+  const replay = replayability(candidate);
+  const disposition = replay.replayable ? 'accepted' : 'inconclusive';
+  candidate.evaluation = {
+    disposition,
+    reasonCodes: [replay.reasonCode],
+    replayable: replay.replayable,
+    method: replay.method,
+    improvementClaim: false,
+    note: 'Accepted means eligible for governed memory promotion; it is not a claim of behavioral improvement.',
+  };
+  pushState(candidate, 'evaluated', replay.reasonCode);
+  pushState(candidate, disposition, disposition === 'accepted' ? 'eligible-for-governed-promotion' : 'paired-or-later-evidence-required');
+  atomicWriteJson(file, candidate);
+  return { file, candidate };
+}
+
+function promoteCandidate(input) {
+  const { file, candidate } = loadCandidate(input);
+  if (candidate.state !== 'accepted' || candidate.evaluation?.disposition !== 'accepted') {
+    throw new Error('only accepted lesson candidates may be promoted');
+  }
+  if (!candidate.generalizedRule) throw new Error('accepted candidate has no generalized rule');
+  if (!input.authorization) throw new Error('promotion requires --authorization from the current #134 workflow contract');
+
+  const args = [
+    path.join(__dirname, 'persist-memory.js'),
+    candidate.generalizedRule,
+    '--authorization', input.authorization,
+    '--source', `lesson-candidate:${candidate.candidateId}`,
+  ];
+  if (candidate.scope?.task) args.push('--scope-task', candidate.scope.task);
+  if (candidate.scope?.requirement) args.push('--scope-requirement', candidate.scope.requirement);
+  if (candidate.scope?.role) args.push('--scope-role', candidate.scope.role);
+  if (input.retentionDays) args.push('--retention-days', String(input.retentionDays));
+
+  const result = spawnSync(process.execPath, args, {
+    cwd: input.workspace,
+    encoding: 'utf8',
+    env: process.env,
+  });
+  if (result.status !== 0) {
+    candidate.persistence = {
+      disposition: 'failed',
+      exitCode: result.status,
+      reasonCode: 'memory-governance-rejected-promotion',
+    };
+    atomicWriteJson(file, candidate);
+    return { ok: false, exitCode: result.status || 1, stdout: result.stdout, stderr: result.stderr, candidate };
+  }
+
+  const persisted = /\[Success\] Memory persisted/.test(String(result.stdout || ''));
+  let promotionWriter = null;
+  if (persisted) {
+    const index = readJson(path.join(input.workspace, 'memories', 'repo', 'memory-index.json'));
+    const record = index && Array.isArray(index.records)
+      ? index.records.find(item => item.source === `lesson-candidate:${candidate.candidateId}`)
+      : null;
+    if (record && record.writer) {
+      promotionWriter = {
+        sessionId: record.writer.sessionId || null,
+        workflowId: record.writer.workflowId || null,
+        runId: record.writer.runId || null,
+        role: record.writer.role || null,
+      };
+    }
+  }
+  candidate.persistence = {
+    disposition: persisted ? 'persisted' : 'proposed',
+    authorizedBy: 'workflow-memory-capability',
+    promotionWriter,
+    observedAt: new Date().toISOString(),
+  };
+  if (persisted) pushState(candidate, 'persisted', 'accepted-and-authorized-memory-write');
+  else pushState(candidate, 'accepted', 'memory-governance-held-as-review-candidate');
+  atomicWriteJson(file, candidate);
+  return { ok: true, exitCode: 0, persisted, stdout: result.stdout, stderr: result.stderr, candidate };
+}
+
+function observeCandidate(input) {
+  const { file, candidate } = loadCandidate(input);
+  const next = String(input.outcome || '');
+  if (!OUTCOME_STATES.has(next)) throw new Error('observe requires retrieved|validated|regressed|superseded');
+  if (next === 'retrieved' && !['persisted', 'retrieved'].includes(candidate.state)) {
+    throw new Error('only persisted lessons may be marked retrieved');
+  }
+  if (['validated', 'regressed', 'superseded'].includes(next) && !['persisted', 'retrieved', 'validated', 'regressed'].includes(candidate.state)) {
+    throw new Error(`${next} requires persisted/retrieved lesson state`);
+  }
+  const evidenceRef = String(input.evidence || '').trim();
+  if (!evidenceRef) throw new Error('outcome transition requires --evidence');
+  candidate.outcome = {
+    state: next,
+    evidenceRef,
+    observedAt: new Date().toISOString(),
+  };
+  pushState(candidate, next, `lesson-${next}`, evidenceRef);
+  atomicWriteJson(file, candidate);
+  return { file, candidate };
+}
+
+function listCandidates({ workspace, sessionId }) {
+  const base = path.join(getWorkspaceStateDir(workspace), 'state', 'sessions', safeSessionName(sessionId), 'learning-candidates');
+  let files = [];
+  try { files = fs.readdirSync(base).filter(name => /^lesson-[a-f0-9]{24}\.json$/.test(name)); } catch (_) { return []; }
+  return files.map(name => readJson(path.join(base, name))).filter(Boolean);
+}
+
+function option(args, name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : null;
+}
+
+function parseCli(argv) {
+  const command = argv[0];
+  const workspace = path.resolve(option(argv, '--workspace') || getWorkspaceRoot() || process.cwd());
+  const base = {
+    command,
+    workspace,
+    sessionId: option(argv, '--session-id') || 'default',
+    candidateId: option(argv, '--candidate-id'),
+  };
+  return {
+    ...base,
+    rule: option(argv, '--rule'),
+    authorization: option(argv, '--authorization'),
+    scopeTask: option(argv, '--scope-task'),
+    scopeRequirement: option(argv, '--scope-requirement'),
+    scopeRole: option(argv, '--scope-role'),
+    retentionDays: option(argv, '--retention-days'),
+    outcome: option(argv, '--outcome'),
+    evidence: option(argv, '--evidence'),
+  };
+}
+
+function main() {
+  try {
+    const args = parseCli(process.argv.slice(2));
+    let result;
+    if (args.command === 'list') result = listCandidates(args);
+    else if (args.command === 'show') result = loadCandidate(args).candidate;
+    else if (args.command === 'evaluate') result = evaluateCandidate(args);
+    else if (args.command === 'promote') result = promoteCandidate(args);
+    else if (args.command === 'observe') result = observeCandidate(args);
+    else throw new Error('Usage: lesson-candidate.js <list|show|evaluate|promote|observe> --workspace <root> --session-id <id> [--candidate-id <id>] ...');
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    if (result && result.ok === false) process.exitCode = result.exitCode || 1;
+  } catch (error) {
+    console.error(`[self-evolve] ${error.message}`);
+    process.exitCode = 1;
+  }
+}
+
+if (require.main === module) main();
+
+module.exports = {
+  SCHEMA_VERSION,
+  candidatePath,
+  evaluateCandidate,
+  listCandidates,
+  loadCandidate,
+  observeCandidate,
+  promoteCandidate,
+  replayability,
+};

package/self-evolve/scripts/self-regression.js

@@ -42,7 +42,7 @@ function runNode(label, script, args = [], options = {}) {
 }
 
 console.log('\n[Phase 1] Static Syntax Check...');
-const foldersToScan = ['harness-everything', 'hooks', 'environment-detection', 'self-evolve', 'ci', 'eval-harness', 'scripts', 'bin', 'to-spec', 'to-tickets', 'find-skills', 'fable-mode', 'multi-agent-workspace'];
+const foldersToScan = ['harness-everything', 'hooks', 'environment-detection', 'self-evolve', 'telemetry', 'contract-integrity', 'ci', 'eval-harness', 'scripts', 'bin', 'to-spec', 'to-tickets', 'find-skills', 'fable-mode', 'multi-agent-workspace'];
 const jsFiles = [];
 function walkDir(dir) {
   if (!fs.existsSync(dir)) return;
@@ -92,8 +92,11 @@ runNode('explicit parallel safety contract', path.join(projectRoot, 'ci', 'route
 runNode('workflow-plan consumers', path.join(projectRoot, 'ci', 'workflow-plan-consumers.test.js'));
 runNode('workflow-plan consumer edge contracts', path.join(projectRoot, 'ci', 'workflow-plan-consumers-edge.test.js'));
 runNode('phase 4 ensemble/memory evidence', path.join(projectRoot, 'ci', 'phase4-ensemble-memory.test.js'));
+runNode('self-evolve closed loop phase 1-3', path.join(projectRoot, 'ci', 'self-evolve-loop.test.js'));
 runNode('phase 5 pre-action gate', path.join(projectRoot, 'ci', 'action-gate.test.js'));
 runNode('phase 5 platform evidence boundary', path.join(projectRoot, 'ci', 'action-gate-platform-doc.test.js'));
+runNode('cross-agent telemetry', path.join(projectRoot, 'ci', 'telemetry.test.js'));
+runNode('contract integrity phase 1', path.join(projectRoot, 'ci', 'contract-integrity.test.js'));
 runNode('issue #99 runtime correctness', path.join(projectRoot, 'ci', 'issue99-runtime.test.js'));
 
 console.log('\n[Phase 3] Skill Reference and Behavioral Case Checks...');

package/telemetry/schemas/event.schema.json

@@ -0,0 +1,39 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/dyphn1/Harness-everything/telemetry/schemas/event.schema.json",
+  "title": "Harness Telemetry Event",
+  "type": "object",
+  "required": [
+    "schemaVersion", "eventId", "event", "host", "observedAt",
+    "sessionId", "turnId", "agentId", "invocationId", "skillName",
+    "skillVersion", "toolName", "status", "retryCount", "timing", "reasonCodes"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "eventId": { "type": "string", "minLength": 1 },
+    "event": { "enum": ["skill.invoke", "skill.loaded", "skill.complete", "tool.observed"] },
+    "host": { "enum": ["claude", "codex", "opencode", "unknown"] },
+    "observedAt": { "type": "string", "format": "date-time" },
+    "sessionId": { "type": ["string", "null"] },
+    "turnId": { "type": ["string", "null"] },
+    "agentId": { "type": ["string", "null"] },
+    "invocationId": { "type": ["string", "null"] },
+    "skillName": { "type": ["string", "null"] },
+    "skillVersion": { "type": ["string", "null"] },
+    "toolName": { "type": ["string", "null"] },
+    "status": { "enum": ["unknown", "success", "failure", "aborted"] },
+    "retryCount": { "type": "integer", "minimum": 0 },
+    "timing": {
+      "type": "object",
+      "required": ["skillLoadDurationMs", "activeWindowMs", "attributedToolDurationMs"],
+      "properties": {
+        "skillLoadDurationMs": { "type": ["number", "null"], "minimum": 0 },
+        "activeWindowMs": { "type": ["number", "null"], "minimum": 0 },
+        "attributedToolDurationMs": { "type": ["number", "null"], "minimum": 0 }
+      },
+      "additionalProperties": false
+    },
+    "reasonCodes": { "type": "array", "items": { "type": "string" } }
+  },
+  "additionalProperties": false
+}

package/telemetry/scripts/benchmark.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { emitTelemetry } = require('../../hooks/scripts/lib/telemetry');
+const { percentile } = require('./report');
+
+function runBenchmark(count = 250) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-telemetry-bench-'));
+  const file = path.join(root, 'events.jsonl');
+  const overhead = [];
+  try {
+    for (let i = 0; i < count; i++) {
+      const result = emitTelemetry({
+        event: 'tool.observed',
+        host: 'unknown',
+        sessionId: 'session-benchmark',
+        invocationId: 'invocation-benchmark',
+        skillName: 'benchmark-skill',
+        toolName: 'benchmark-tool',
+        status: 'success',
+        timing: { attributedToolDurationMs: 1 },
+        reasonCodes: ['overhead-benchmark'],
+      }, { file });
+      if (!result.ok) throw new Error(result.error || 'telemetry write failed');
+      overhead.push(result.overheadMs);
+    }
+    return {
+      count,
+      p50Ms: percentile(overhead, 50),
+      p95Ms: percentile(overhead, 95),
+      maxMs: Math.max(...overhead),
+      bytes: fs.statSync(file).size,
+    };
+  } finally {
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+}
+
+if (require.main === module) {
+  const count = Number(process.argv[2] || 250);
+  process.stdout.write(`${JSON.stringify(runBenchmark(Number.isInteger(count) && count > 0 ? count : 250), null, 2)}\n`);
+}
+
+module.exports = { runBenchmark };

package/telemetry/scripts/report.js

@@ -0,0 +1,133 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const {
+  readEvents,
+  telemetryFile,
+} = require('../../hooks/scripts/lib/telemetry');
+
+function percentile(values, p) {
+  const sorted = values.filter(Number.isFinite).slice().sort((a, b) => a - b);
+  if (!sorted.length) return null;
+  const index = Math.max(0, Math.ceil((p / 100) * sorted.length) - 1);
+  return sorted[index];
+}
+
+function metric(values) {
+  const clean = values.filter(Number.isFinite);
+  return {
+    count: clean.length,
+    p50: percentile(clean, 50),
+    p95: percentile(clean, 95),
+  };
+}
+
+function buildReport(events) {
+  const valid = events.filter(event => event && !event.invalid);
+  const invalid = events.filter(event => event && event.invalid);
+  const groups = new Map();
+
+  for (const event of valid) {
+    const key = `${event.host}::${event.skillName || '(unknown)'}`;
+    if (!groups.has(key)) groups.set(key, {
+      host: event.host,
+      skillName: event.skillName,
+      skillVersion: event.skillVersion,
+      invocationIds: new Set(),
+      invokes: 0,
+      loads: 0,
+      completes: 0,
+      status: { success: 0, failure: 0, aborted: 0, unknown: 0 },
+      loadDurations: [],
+      activeWindows: [],
+      attributedToolDurations: [],
+      toolEvents: 0,
+      maxRetry: 0,
+    });
+    const g = groups.get(key);
+    if (event.invocationId) g.invocationIds.add(event.invocationId);
+    if (event.skillVersion) g.skillVersion = event.skillVersion;
+    if (event.event === 'skill.invoke') g.invokes++;
+    if (event.event === 'skill.loaded') {
+      g.loads++;
+      if (Number.isFinite(event.timing.skillLoadDurationMs)) g.loadDurations.push(event.timing.skillLoadDurationMs);
+    }
+    if (event.event === 'skill.complete') {
+      g.completes++;
+      g.status[event.status] = (g.status[event.status] || 0) + 1;
+      if (Number.isFinite(event.timing.activeWindowMs)) g.activeWindows.push(event.timing.activeWindowMs);
+      if (Number.isFinite(event.timing.attributedToolDurationMs)) g.attributedToolDurations.push(event.timing.attributedToolDurationMs);
+    }
+    if (event.event === 'tool.observed') g.toolEvents++;
+    g.maxRetry = Math.max(g.maxRetry, event.retryCount || 0);
+  }
+
+  const skills = [...groups.values()].map(g => ({
+    host: g.host,
+    skillName: g.skillName,
+    skillVersion: g.skillVersion,
+    invocationCount: g.invocationIds.size || g.invokes,
+    invokeEvents: g.invokes,
+    loadEvents: g.loads,
+    completeEvents: g.completes,
+    status: g.status,
+    retryMax: g.maxRetry,
+    toolEventCount: g.toolEvents,
+    skillLoadDurationMs: metric(g.loadDurations),
+    activeWindowMs: metric(g.activeWindows),
+    attributedToolDurationMs: metric(g.attributedToolDurations),
+  })).sort((a, b) => `${a.host}/${a.skillName}`.localeCompare(`${b.host}/${b.skillName}`));
+
+  return {
+    schemaVersion: 1,
+    eventCount: valid.length,
+    invalidEventCount: invalid.length,
+    skills,
+    semantics: {
+      skillLoadDurationMs: 'PreToolUse(Skill) to correlated PostToolUse(Skill); framework/load latency only.',
+      activeWindowMs: 'Successful skill load to host Stop/turn boundary; context exposure window, not CPU time.',
+      attributedToolDurationMs: 'Sum of host-reported tool durations observed while the skill invocation is active; overlapping skills may each receive the same tool observation.',
+    },
+  };
+}
+
+function markdown(report) {
+  const lines = [
+    '# Harness Telemetry Report',
+    '',
+    '| Host | Skill | Invocations | Complete | Failure | Load P50/P95 ms | Active P50/P95 ms | Tool P50/P95 ms |',
+    '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |',
+  ];
+  for (const row of report.skills) {
+    const pair = value => value.count ? `${value.p50}/${value.p95}` : '-';
+    lines.push(`| ${row.host} | ${row.skillName || '(unknown)'} | ${row.invocationCount} | ${row.completeEvents} | ${row.status.failure} | ${pair(row.skillLoadDurationMs)} | ${pair(row.activeWindowMs)} | ${pair(row.attributedToolDurationMs)} |`);
+  }
+  lines.push('', `Events: ${report.eventCount}; invalid lines: ${report.invalidEventCount}.`);
+  lines.push('', '> Active window is not CPU/runtime. Attributed tool time is observational and non-exclusive when skills overlap.');
+  return `${lines.join('\n')}\n`;
+}
+
+function option(args, name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : null;
+}
+
+function main() {
+  try {
+    const args = process.argv.slice(2);
+    const workspace = option(args, '--workspace');
+    const file = option(args, '--file') || (workspace ? telemetryFile(path.resolve(workspace), { cwd: workspace }) : null);
+    if (!file) throw new Error('Usage: report.js --file <events.jsonl> [--markdown] or --workspace <root>');
+    const report = buildReport(readEvents(path.resolve(file)));
+    process.stdout.write(args.includes('--markdown') ? markdown(report) : `${JSON.stringify(report, null, 2)}\n`);
+  } catch (error) {
+    console.error(`[telemetry] ${error.message}`);
+    process.exitCode = 1;
+  }
+}
+
+if (require.main === module) main();
+
+module.exports = { buildReport, markdown, metric, percentile };