harness-everything 0.11.7 → 0.12.1

package/ci/mechanism-2ah-workflow-lifecycle.test.js

@@ -17,6 +17,7 @@ const env = { ...process.env, HARNESS_STATE_HOME: path.join(temp, 'state'), HARN
 process.env.HARNESS_STATE_HOME = env.HARNESS_STATE_HOME;
 const state = require(path.join(runtimeRoot, 'hooks/scripts/lib/harness-state'));
 const workflowRuntime = require(path.join(runtimeRoot, 'hooks/scripts/lib/workflow-runtime'));
+const workflowIsolation = require(path.join(runtimeRoot, 'hooks/scripts/lib/workflow-isolation'));
 const sessionId = 'workflow-lifecycle';
 const payload = { session_id: sessionId, cwd: repo };
 const sessionDir = state.getSessionDir(repo, sessionId);
@@ -44,9 +45,15 @@ const persistDirect = (tool, input, cwd = repo, response = {}) => node('hooks/sc
   tool_response: response,
 });
 let toolUseSeq = 0;
-function beginShell(command, cwd = repo) {
+function beginShell(command, cwd = repo, extraToolInput = {}) {
   const toolUseId = 'toolu_fixture_' + (++toolUseSeq);
-  const input = { ...payload, cwd, tool_name: 'Bash', tool_use_id: toolUseId, tool_input: { command } };
+  const input = {
+    ...payload,
+    cwd,
+    tool_name: 'Bash',
+    tool_use_id: toolUseId,
+    tool_input: { command, ...extraToolInput },
+  };
   return { command, cwd, toolUseId, result: node('hooks/scripts/workflow-gate.js', input) };
 }
 function persistShell(call, response = {}, hookEvent = 'PostToolUse') {
@@ -83,8 +90,37 @@ const probeFiles = () => {
   try { return fs.readdirSync(probeDir).filter(name => /^[a-f0-9]{64}\.json$/.test(name)); }
   catch (_) { return []; }
 };
+function ageProbeFiles(ms = 60000) {
+  for (const name of probeFiles()) {
+    const target = path.join(probeDir, name);
+    const probe = read(target);
+    write(target, { ...probe, observedAt: Date.now() - ms, reclaimAfterAt: Date.now() - 1 });
+  }
+}
 const stop = extra => node('hooks/scripts/workflow-stop-gate.js', { ...payload, ...extra });
 const control = (...args) => node('hooks/scripts/workflow-disposition.js', null, [args[0], '--session-id', sessionId, ...args.slice(1)]);
+function fixtureSession(fixtureSessionId) {
+  const fixturePayload = { session_id: fixtureSessionId, cwd: repo };
+  const fixtureSessionDir = state.getSessionDir(repo, fixtureSessionId);
+  const fixtureFile = path.join(fixtureSessionDir, 'workflow-run.json');
+  return {
+    payload: fixturePayload,
+    file: fixtureFile,
+    sessionDir: fixtureSessionDir,
+    route: prompt => node('harness-everything/scripts/kernel-router.js', { ...fixturePayload, prompt }),
+    gate: (tool, input, extra = {}) => node('hooks/scripts/workflow-gate.js', { ...fixturePayload, ...extra, tool_name: tool, tool_input: input }),
+    persist: (tool, input, response = {}, extra = {}) => node('hooks/scripts/state-persist.js', {
+      ...fixturePayload,
+      ...extra,
+      hook_event_name: 'PostToolUse',
+      tool_name: tool,
+      tool_input: input,
+      tool_response: response,
+    }),
+    stop: extra => node('hooks/scripts/workflow-stop-gate.js', { ...fixturePayload, ...extra }),
+    control: (...args) => node('hooks/scripts/workflow-disposition.js', null, [args[0], '--session-id', fixtureSessionId, ...args.slice(1)]),
+  };
+}
 const stages = [
   { stageId: 'build', goal: 'fix', agent: 'fable-worker', task: 'fix bounded module', dependsOn: [], writeSet: ['src'], checkCommand: 'node test-build.js', passCondition: 'exit 0' },
   { stageId: 'optional', goal: 'external scope', agent: 'fable-worker', task: 'external task', dependsOn: [], writeSet: [], checkCommand: 'node external.js', passCondition: 'exit 0' },
@@ -92,6 +128,78 @@ const stages = [
 ];
 
 try {
+  // #162: Tier-2 shell accounting must degrade conservatively when a workspace
+  // is not a Git repository. Lack of a Git fingerprint is not an isolation
+  // violation for a non-major workflow.
+  const nonGit = fixtureSession('workflow-non-git');
+  check(nonGit.route('Fix this checkout bug with a regression test').status === 0 &&
+    read(nonGit.file).strategy === 'iterative-single',
+    '#162 non-git fixture routes to Tier 2 iterative workflow');
+  check(nonGit.control('start').status === 0 && read(nonGit.file).state === 'running',
+    '#162 non-git Tier 2 workflow starts normally');
+  let nonGitIterations = read(nonGit.file).budget?.counters?.iterations || 0;
+  for (const [index, command] of ['npm --version', 'node -e "process.exit(0)"', 'cd .', 'git init -q'].entries()) {
+    const toolUseId = 'toolu_non_git_pre_' + index;
+    const admitted = nonGit.gate('Bash', { command }, { tool_use_id: toolUseId });
+    check(admitted.status === 0, '#162 non-git Tier 2 admits untrusted shell: ' + command);
+    const afterGate = read(nonGit.file);
+    check(afterGate.budget.counters.iterations === ++nonGitIterations &&
+      afterGate.lastMutationAt > 0 &&
+      afterGate.budget.events.at(-1)?.evidence === 'shell:unobservable-workspace',
+      '#162 unobservable non-git shell is conservatively counted once: ' + command);
+    check(nonGit.persist('Bash', { command }, { stdout: 'fixture success' }, { tool_use_id: toolUseId }).status === 0,
+      '#162 non-git shell PostToolUse stays fail-open without a probe: ' + command);
+  }
+
+  // A probe may have been created while Git was observable and become
+  // unobservable before PostToolUse (for example .git disappears). Tier 2
+  // must settle it conservatively instead of blocking the workflow.
+  const lostObservationId = 'toolu_non_git_post_fallback';
+  const lostObservationPayload = {
+    ...nonGit.payload,
+    tool_name: 'Bash',
+    tool_use_id: lostObservationId,
+    tool_input: { command: 'node lost-observation.js' },
+  };
+  const lostObservationContext = workflowRuntime.loadWorkflow(lostObservationPayload);
+  const lostObservationKey = workflowIsolation.shellProbeKey(lostObservationPayload, repo);
+  workflowRuntime.registerMutationProbe(lostObservationContext, lostObservationKey, 'f'.repeat(64), true);
+  const beforeLostObservation = read(nonGit.file);
+  check(nonGit.persist('Bash', { command: 'node lost-observation.js' }, { stdout: 'fixture success' },
+    { tool_use_id: lostObservationId }).status === 0,
+    '#162 Tier 2 PostToolUse degrades when workspace observation becomes unavailable');
+  const afterLostObservation = read(nonGit.file);
+  check(afterLostObservation.state === 'running' &&
+    afterLostObservation.budget.counters.iterations === beforeLostObservation.budget.counters.iterations + 1 &&
+    afterLostObservation.lastMutationAt >= (beforeLostObservation.lastMutationAt || 0),
+    '#162 lost post-observation is settled as one conservative mutation without blocking');
+
+  const nonGitTarget = path.join(repo, 'non-git-edit.txt');
+  check(nonGit.gate('Write', { file_path: nonGitTarget }).status === 0,
+    '#162 direct edit remains available in non-git Tier 2');
+  fs.writeFileSync(nonGitTarget, 'changed\n');
+  check(nonGit.persist('Write', { file_path: nonGitTarget }).status === 0,
+    '#162 direct non-git edit persists mutation milestone');
+  const verifyId = 'toolu_non_git_verify';
+  check(nonGit.gate('Bash', { command: 'node run-test.js' }, { tool_use_id: verifyId }).status === 0,
+    '#162 verification shell is admitted in non-git Tier 2');
+  check(nonGit.persist('Bash', { command: 'node run-test.js' }, { stdout: 'tests passed' },
+    { tool_use_id: verifyId }).status === 0,
+    '#162 non-git verification result is persisted');
+  check(nonGit.stop().status === 0 && read(nonGit.file).state === 'satisfied',
+    '#162 edit then verification can satisfy Stop in non-git Tier 2');
+
+  const nonGitMajor = fixtureSession('workflow-non-git-major');
+  check(nonGitMajor.route('Refactor the entire authentication architecture across all services without fable').status === 0 &&
+    read(nonGitMajor.file).tier === 'tier3',
+    '#162 non-git major fixture routes to Tier 3');
+  check(nonGitMajor.control('start').status === 0 && read(nonGitMajor.file).state === 'running',
+    '#162 non-Fable Tier 3 controller can enter running state before mutation');
+  const majorBlocked = nonGitMajor.gate('Bash', { command: 'node mutate.js' }, { tool_use_id: 'toolu_non_git_major' });
+  check(majorBlocked.status === 2 && /Git worktree isolation/i.test(majorBlocked.stderr),
+    '#162 non-git Tier 3 remains fail-closed with an isolation-specific error');
+
+  fs.unlinkSync(nonGitTarget);
   git(['init']); fs.writeFileSync(path.join(repo, 'README.md'), 'fixture\n'); git(['add', '.']);
   git(['-c', 'user.name=Harness Test', '-c', 'user.email=harness@example.invalid', 'commit', '-m', 'fixture']);
   git(['worktree', 'add', linked, '-b', 'isolated']);
@@ -225,6 +333,48 @@ try {
     promotedTier3.mutationIsolation?.required === true && !promotedTier3.pendingPlan, 'pending Tier 3 isolation becomes active atomically on start');
   check(workflowRuntime.isMajorWorkflow(promotedTier3), 'major-workflow classification flips only after explicit activation');
   check(gate('Write', { file_path: path.join(repo, 'premature-primary.js') }, repo).status === 2, 'activated Tier 3 plan still blocks primary-tree mutation');
+
+  // #163: a queued Tier-3 plan can intentionally remain single-agent when
+  // Fable is prohibited. Explicit start must still promote the whole plan,
+  // including isolation and budget metadata, before any mutation is admitted.
+  fs.unlinkSync(file);
+  check(route('Fix this checkout bug with a regression test').status === 0 && read(file).strategy === 'iterative-single',
+    '#163 fixture starts from a Tier 2 iterative workflow');
+  check(control('start').status === 0 && read(file).state === 'running',
+    '#163 fixture activates Tier 2 before a stronger route arrives');
+  check(route('Refactor the entire authentication architecture across all services and migrate the database schema without fable').status === 0,
+    '#163 stronger Tier 3 route without Fable is retained for explicit replan');
+  const queuedNonFableTier3 = read(file);
+  check(queuedNonFableTier3.state === 'blocked' &&
+    queuedNonFableTier3.pendingPlan?.tier === 'tier3' &&
+    queuedNonFableTier3.pendingPlan?.strategy === 'iterative-single' &&
+    queuedNonFableTier3.pendingPlan?.mutationIsolation?.required === true,
+    '#163 queued Tier 3 plan remains non-Fable but still requires isolation');
+  write(file, {
+    ...queuedNonFableTier3,
+    budget: {
+      ...queuedNonFableTier3.budget,
+      limits: { ...queuedNonFableTier3.budget.limits, maxIterations: 1 },
+    },
+  });
+  check(control('start').status === 0, '#163 explicit start activates queued non-Fable Tier 3 plan');
+  const promotedNonFableTier3 = read(file);
+  check(promotedNonFableTier3.workflowPlan?.tier === 'tier3' &&
+    promotedNonFableTier3.tier === 'tier3' &&
+    promotedNonFableTier3.strategy === 'iterative-single' &&
+    promotedNonFableTier3.mutationIsolation?.required === true &&
+    !promotedNonFableTier3.pendingPlan,
+    '#163 non-Fable Tier 3 plan replaces all authoritative active-plan fields');
+  check(promotedNonFableTier3.budget?.limits?.maxIterations === promotedNonFableTier3.workflowPlan?.limits?.maxIterations &&
+    promotedNonFableTier3.budget?.limits?.maxIterations !== 1,
+    '#163 plan activation re-derives budget limits from the promoted plan');
+  check(workflowRuntime.isMajorWorkflow(promotedNonFableTier3),
+    '#163 non-Fable Tier 3 promotion enables major-workflow isolation');
+  check(gate('Write', { file_path: path.join(repo, 'non-fable-primary.js') }, repo).status === 2,
+    '#163 promoted non-Fable Tier 3 blocks primary-tree mutation');
+  check(gate('Write', { file_path: path.join(linked, 'non-fable-linked.js') }, linked).status === 0,
+    '#163 promoted non-Fable Tier 3 admits linked-worktree mutation');
+
   fs.unlinkSync(file);
   check(route('Fix this checkout bug with a regression test').status === 0 && read(file).strategy === 'iterative-single', 'fresh bounded fix returns to Tier 2 iterative lifecycle');
   let tier2 = read(file);
@@ -387,6 +537,112 @@ try {
       'no-effect untrusted shell command does not advance lastMutationAt');
   }
 
+  // #161: Post payload cwd may differ from Pre payload cwd after a shell
+  // command changes directories. Stable tool_use_id identity plus the stored
+  // Pre-time observation cwd must settle the original probe.
+  const driftCwd = path.join(repo, 'hooks');
+  fs.mkdirSync(driftCwd, { recursive: true });
+  const driftNoEffectBefore = read(file);
+  const driftNoEffect = beginShell('cd hooks && node -e "process.exit(0)"', repo);
+  check(driftNoEffect.result.status === 0 && probeFiles().length === 1,
+    '#161 cwd-drift no-effect call registers one probe');
+  const driftNoEffectPost = node('hooks/scripts/state-persist.js', {
+    ...payload,
+    cwd: driftCwd,
+    hook_event_name: 'PostToolUse',
+    tool_name: 'Bash',
+    tool_use_id: driftNoEffect.toolUseId,
+    tool_input: { command: driftNoEffect.command },
+    tool_response: { stdout: 'ok' },
+  });
+  check(driftNoEffectPost.status === 0 && probeFiles().length === 0,
+    '#161 Post with drifted cwd settles the Pre-time probe');
+  check((read(file).budget?.counters?.iterations || 0) === (driftNoEffectBefore.budget?.counters?.iterations || 0),
+    '#161 cwd-drift no-effect shell consumes zero iterations');
+
+  const driftMutationBefore = read(file);
+  const driftMutation = beginShell('cd hooks && node mutate-from-subdir.js', repo);
+  check(driftMutation.result.status === 0, '#161 cwd-drift mutator is admitted under observation');
+  fs.writeFileSync(path.join(repo, 'drift-mutated.txt'), 'changed under cwd drift\n');
+  const driftMutationPost = node('hooks/scripts/state-persist.js', {
+    ...payload,
+    cwd: driftCwd,
+    hook_event_name: 'PostToolUse',
+    tool_name: 'Bash',
+    tool_use_id: driftMutation.toolUseId,
+    tool_input: { command: driftMutation.command },
+    tool_response: { stdout: 'changed' },
+  });
+  const driftMutationAfter = read(file);
+  check(driftMutationPost.status === 0 && probeFiles().length === 0 &&
+    (driftMutationAfter.budget?.counters?.iterations || 0) === (driftMutationBefore.budget?.counters?.iterations || 0) + 1 &&
+    driftMutationAfter.lastMutationAt >= (driftMutationBefore.lastMutationAt || 0),
+    '#161 cwd-drift real mutation is observed exactly once');
+
+  // A live command may legitimately run longer than the old 1s grace period.
+  // Its probe must remain reserved until the declared tool timeout expires.
+  const longRunningStateBefore = read(file);
+  const longRunningBefore = longRunningStateBefore.budget.counters.iterations;
+  const longRunningA = beginShell('node long-running-mutator.js', repo, { timeout: 60000 });
+  check(longRunningA.result.status === 0 && probeFiles().length === 1,
+    '#161 long-running call registers one timeout-bound probe');
+  {
+    const target = path.join(probeDir, probeFiles()[0]);
+    const probe = read(target);
+    write(target, { ...probe, observedAt: Date.now() - 2000 });
+  }
+  const concurrentB = beginShell('node concurrent-observer.js', repo, { timeout: 60000 });
+  check(concurrentB.result.status === 0 && probeFiles().length === 2,
+    '#161 a second Pre does not reclaim an in-flight probe merely because it is older than 1s');
+  fs.writeFileSync(path.join(repo, 'long-running-mutated.txt'), 'changed by long-running A\n');
+  check(persistShell(longRunningA).status === 0 &&
+    read(file).budget.counters.iterations === longRunningBefore + 1,
+    '#161 long-running A still settles and records its later mutation');
+  check(denyShell(concurrentB).status === 0 && probeFiles().length === 0,
+    '#161 concurrent non-mutating probe can be denied independently after A settles');
+  fs.rmSync(path.join(repo, 'long-running-mutated.txt'), { force: true });
+  write(file, longRunningStateBefore);
+  check(read(file).budget.counters.iterations === longRunningBefore,
+    '#161 long-running regression restores the shared fixture budget before later boundary tests');
+
+  // Simulate default-mode denial / sibling PreToolUse blocking: Pre fires but
+  // neither PostToolUse nor PermissionDenied follows. Force each orphan past
+  // its timeout-bound reclaim deadline so the next real Pre can reclaim it
+  // deterministically without sleeping in CI.
+  const orphanStartIterations = read(file).budget.counters.iterations;
+  for (let i = 0; i < iterationLimit; i++) {
+    const orphan = beginShell('node rejected-before-exec-' + i + '.js', repo);
+    check(orphan.result.status === 0, '#161 Pre-only orphan ' + (i + 1) + ' is initially admitted');
+    ageProbeFiles();
+  }
+  check(workflowRuntime.mutationProbeReservations(workflowRuntime.loadWorkflow(payload)) === 0,
+    '#161 stale Pre-only probes no longer reserve iteration capacity');
+  const harmlessAfterOrphans = beginShell('node harmless-after-orphans.js', repo);
+  check(harmlessAfterOrphans.result.status === 0 && probeFiles().length === 1,
+    '#161 next harmless untrusted shell is admitted after maxIterations Pre-only orphans');
+  check(persistShell(harmlessAfterOrphans).status === 0 && probeFiles().length === 0,
+    '#161 orphan reclamation leaves only the live probe and it settles normally');
+  check(read(file).budget.counters.iterations === orphanStartIterations,
+    '#161 rejected/no-effect orphan sequence consumes zero iterations');
+
+  // If an orphan really did mutate the workspace, reclaim must account for it
+  // once rather than silently dropping evidence.
+  const staleMutation = beginShell('node orphaned-mutator.js', repo);
+  check(staleMutation.result.status === 0, '#161 stale-mutator probe is registered');
+  ageProbeFiles();
+  const staleMutationBefore = read(file);
+  fs.writeFileSync(path.join(repo, 'stale-probe-mutated.txt'), 'changed before missing Post\n');
+  const reclaimTrigger = beginShell('node reclaim-trigger.js', repo);
+  const reclaimed = read(file);
+  check(reclaimTrigger.result.status === 0 &&
+    reclaimed.budget.counters.iterations === staleMutationBefore.budget.counters.iterations + 1 &&
+    reclaimed.lastMutationAt >= (staleMutationBefore.lastMutationAt || 0),
+    '#161 stale probe with workspace change is reclaimed as exactly one mutation');
+  check(persistShell(reclaimTrigger).status === 0 &&
+    read(file).budget.counters.iterations === reclaimed.budget.counters.iterations &&
+    probeFiles().length === 0,
+    '#161 reclaimed stale mutation is not double-counted by the trigger Post');
+
   const deniedCall = beginShell('node denied-by-auto-mode.js');
   check(deniedCall.result.status === 0 && probeFiles().length === 1,
     'auto-mode candidate reserves mutation capacity before permission decision');

package/ci/mechanism-32-paired-benchmark.test.js

@@ -6,8 +6,11 @@ const crypto = require('crypto');
 const helper = require('./test-helper');
 const {
   mcnemarExactP,
+  pairContract,
   prepareArmWorkspace,
+  retrieveLesson,
   summarizePairs,
+  validateLessonCase,
   validatePairRecord,
   validateRunConfig,
 } = require('../behavioral-evals/paired-benchmark');
@@ -142,6 +145,142 @@ try {
   fs.rmSync(pluginBaseline.ws, { recursive: true, force: true });
   fs.rmSync(pluginTreatment.ws, { recursive: true, force: true });
 
+  const lessonCase = {
+    id: 'lesson-retrieval-shape',
+    discipline: 'verify-before-claim',
+    loaded_skills: ['verify-before-claim'],
+    prompt: 'npm test is failing for higher retry attempts. Fix it and verify.',
+    max_turns: 8,
+    fixture: { files: [{ path: 'probe.txt', content: 'baseline\n' }] },
+    expectations: [{ type: 'file_contains', path: 'probe.txt', value: 'baseline' }],
+    lesson: {
+      candidate_id: 'lesson-0123456789abcdef01234567',
+      rule: 'When retry delay tests fail at higher attempts, inspect exponential backoff before changing caps and verify with npm test.',
+      scope_task: 'retry delay exponential backoff higher attempts npm test',
+      query_task: 'retry delay exponential backoff higher attempts npm test',
+    },
+  };
+  helper.check(
+    '32. lesson-retrieval case requires a valid accepted-lesson fixture contract',
+    validateLessonCase(lessonCase).length === 0,
+    validateLessonCase(lessonCase).join('; '),
+  );
+  const lessonConfigProblems = validateRunConfig({
+    effect_type: 'lesson-retrieval', engine: 'claude', model: 'test/model',
+    min_effect_pp: 10, repeats: 1,
+  });
+  helper.check(
+    '32. lesson-retrieval is an explicit paired effect type rather than being disguised as skill-text',
+    lessonConfigProblems.length === 0,
+    lessonConfigProblems.join('; '),
+  );
+
+  const lessonBaselinePrepared = prepareArmWorkspace(lessonCase, 'lesson-retrieval', 'baseline', 'claude');
+  const lessonTreatmentPrepared = prepareArmWorkspace(lessonCase, 'lesson-retrieval', 'treatment', 'claude');
+  const lessonStateBaseline = helper.tempDir('.paired-lesson-state-baseline');
+  const lessonStateTreatment = helper.tempDir('.paired-lesson-state-treatment');
+  const baselineLesson = retrieveLesson(
+    lessonCase, lessonBaselinePrepared.memoryStore, lessonStateBaseline, false,
+  );
+  const treatmentLesson = retrieveLesson(
+    lessonCase, lessonTreatmentPrepared.memoryStore, lessonStateTreatment, true,
+  );
+  helper.check(
+    '32. lesson memory store stays outside both agent workspaces',
+    !lessonBaselinePrepared.memoryStore.startsWith(lessonBaselinePrepared.ws + path.sep) &&
+      !lessonTreatmentPrepared.memoryStore.startsWith(lessonTreatmentPrepared.ws + path.sep) &&
+      !fs.existsSync(path.join(lessonBaselinePrepared.ws, 'memories')) &&
+      !fs.existsSync(path.join(lessonTreatmentPrepared.ws, 'memories')),
+    JSON.stringify({ baseline: lessonBaselinePrepared, treatment: lessonTreatmentPrepared }),
+  );
+  helper.check(
+    '32. both arms execute the same scoped retrieval while only treatment receives the context',
+    baselineLesson.evidence.queried === true &&
+      treatmentLesson.evidence.queried === true &&
+      baselineLesson.evidence.exposed === false &&
+      treatmentLesson.evidence.exposed === true &&
+      baselineLesson.evidence.retrieval_set_sha256 === treatmentLesson.evidence.retrieval_set_sha256 &&
+      JSON.stringify(baselineLesson.evidence.candidate_ids) === JSON.stringify(treatmentLesson.evidence.candidate_ids) &&
+      baselineLesson.prompt === lessonCase.prompt &&
+      treatmentLesson.prompt.includes('Harness scoped memory context') &&
+      treatmentLesson.prompt.includes(lessonCase.lesson.candidate_id),
+    JSON.stringify({ baseline: baselineLesson.evidence, treatment: treatmentLesson.evidence }),
+  );
+
+  const lessonContract = pairContract(lessonCase, {
+    effect_type: 'lesson-retrieval',
+    engine: 'claude',
+    model: 'test/model',
+  });
+  const lessonPair = {
+    schema_version: 1,
+    id: lessonCase.id,
+    pair_id: 'lesson-pair',
+    effect_type: 'lesson-retrieval',
+    contract: lessonContract,
+    contract_sha256: hash(lessonContract),
+    arms: {
+      baseline: {
+        ...arm('fail', true),
+        intervention: { skill_text: true, plugin_enforcement: false, lesson_retrieval: false },
+        loaded_skills: ['verify-before-claim'],
+        lesson_retrieval: baselineLesson.evidence,
+      },
+      treatment: {
+        ...arm('pass', true),
+        intervention: { skill_text: true, plugin_enforcement: false, lesson_retrieval: true },
+        loaded_skills: ['verify-before-claim'],
+        lesson_retrieval: treatmentLesson.evidence,
+      },
+    },
+  };
+  helper.check(
+    '32. attributed lesson-retrieval pair is includable only with constant skill text and identical retrieval set',
+    validatePairRecord(lessonPair).included,
+    JSON.stringify(validatePairRecord(lessonPair)),
+  );
+  const lessonSetMismatch = JSON.parse(JSON.stringify(lessonPair));
+  lessonSetMismatch.pair_id = 'lesson-set-mismatch';
+  lessonSetMismatch.arms.treatment.lesson_retrieval.retrieval_set_sha256 = 'different';
+  helper.check(
+    '32. mismatched memory retrieval set excludes the pair instead of counting it as lesson effect',
+    !validatePairRecord(lessonSetMismatch).included &&
+      validatePairRecord(lessonSetMismatch).reason === 'lesson-retrieval-set-mismatch',
+    JSON.stringify(validatePairRecord(lessonSetMismatch)),
+  );
+  const lessonTrustMissing = JSON.parse(JSON.stringify(lessonPair));
+  lessonTrustMissing.pair_id = 'lesson-trust-missing';
+  lessonTrustMissing.arms.treatment.lesson_retrieval.trust_boundary = null;
+  helper.check(
+    '32. treatment without the untrusted-memory boundary is excluded from causal evidence',
+    !validatePairRecord(lessonTrustMissing).included &&
+      validatePairRecord(lessonTrustMissing).reason === 'lesson-trust-boundary-missing',
+    JSON.stringify(validatePairRecord(lessonTrustMissing)),
+  );
+  const lessonContextTamper = JSON.parse(JSON.stringify(lessonPair));
+  lessonContextTamper.pair_id = 'lesson-context-tamper';
+  lessonContextTamper.arms.treatment.lesson_retrieval.context_sha256 = 'tampered';
+  helper.check(
+    '32. changed lesson exposure text invalidates the predeclared pair contract',
+    !validatePairRecord(lessonContextTamper).included &&
+      validatePairRecord(lessonContextTamper).reason === 'lesson-context-hash-mismatch',
+    JSON.stringify(validatePairRecord(lessonContextTamper)),
+  );
+
+  for (const target of [
+    lessonBaselinePrepared.ws, lessonTreatmentPrepared.ws,
+    lessonBaselinePrepared.memoryStore, lessonTreatmentPrepared.memoryStore,
+    lessonStateBaseline, lessonStateTreatment,
+  ]) fs.rmSync(target, { recursive: true, force: true });
+
+  const invalidLesson = JSON.parse(JSON.stringify(lessonCase));
+  invalidLesson.lesson.candidate_id = 'not-a-lesson-id';
+  helper.check(
+    '32. malformed lesson provenance is rejected before any live model call',
+    validateLessonCase(invalidLesson).some(problem => problem.includes('candidate_id')),
+    validateLessonCase(invalidLesson).join('; '),
+  );
+
   const validSkillPair = skillPair('p1', 'fail', 'pass');
   helper.check('32. a correctly isolated skill-text pair is includable', validatePairRecord(validSkillPair).included, JSON.stringify(validatePairRecord(validSkillPair)));
 

package/ci/openai-plugin-package.test.js

@@ -112,6 +112,23 @@ for (const entry of SOURCE_MANIFEST.skills) {
   assertTreesEqual(source, path.join(PLUGIN, 'skills', name), name);
 }
 
+const packagedContractAudit = path.join(PLUGIN, 'contract-integrity', 'scripts', 'audit.js');
+assert.ok(fs.existsSync(packagedContractAudit), 'OpenAI package must include shared contract-integrity runtime outside skills/');
+const packagedAuditSource = fs.readFileSync(packagedContractAudit, 'utf8');
+assert.match(packagedAuditSource, /skills\/tdd\/scripts\/quality-gate/, 'packaged contract audit must resolve #58 evaluator from packaged tdd skill');
+assert.doesNotMatch(packagedAuditSource, /\.\.\/\.\.\/tdd\/scripts\/quality-gate/, 'packaged contract audit must not retain canonical repo-only TDD import');
+const packagedAudit = require(packagedContractAudit);
+assert.strictEqual(typeof packagedAudit.runCli, 'function', 'packaged contract audit runtime must load and export runCli');
+
+const verificationContractWrapper = path.join(PLUGIN, 'skills', 'verification-loop', 'scripts', 'contract-integrity-audit.js');
+assert.ok(fs.existsSync(verificationContractWrapper), 'verification-loop must package its contract audit wrapper');
+const verificationWrapper = require(verificationContractWrapper);
+assert.strictEqual(
+  verificationWrapper.resolveAuditRuntime(),
+  packagedContractAudit,
+  'packaged verification-loop wrapper must resolve the plugin-root contract audit runtime'
+);
+
 const marketplace = JSON.parse(fs.readFileSync(path.join(ROOT, '.agents', 'plugins', 'marketplace.json'), 'utf8'));
 assert.strictEqual(marketplace.name, 'harness-everything', 'marketplace should use the publication name, not a local-only label');
 assert.strictEqual(marketplace.interface?.displayName, 'Harness Everything');

package/ci/self-evolve-loop.test.js

@@ -55,8 +55,10 @@ const {
   createLearningOpportunity,
 } = require(path.join(ROOT, 'hooks/scripts/lib/learning-opportunity.js'));
 const { retrieveMemoryRecords } = require(path.join(ROOT, 'multi-agent-workspace/scripts/index_memory.js'));
+const telemetry = require(path.join(ROOT, 'hooks/scripts/lib/telemetry.js'));
+const { buildReport } = require(path.join(ROOT, 'telemetry/scripts/report.js'));
 
-console.log('=== #141 Self-Evolve Closed Loop Phase 1-3 ===');
+console.log('=== #141 Self-Evolve Closed Loop Phase 1-4 Mechanism ===');
 
 try {
   const tracker = path.join(ROOT, 'hooks/scripts/rule-of-3-tracker.js');
@@ -248,6 +250,15 @@ try {
   }).candidate;
   check(retrievedState.state === 'retrieved' && retrievedState.outcome.evidenceRef === 'retrieval-fixture:REQ-141', 'later retrieval can advance the candidate lifecycle with evidence');
 
+  const validatedState = observeCandidate({
+    workspace,
+    sessionId: verifySession,
+    candidateId: verifyCandidate.candidateId,
+    outcome: 'validated',
+    evidence: 'paired-or-later-outcome-fixture:REQ-141',
+  }).candidate;
+  check(validatedState.state === 'validated', 'later outcome evidence can advance a retrieved lesson to validated');
+
   const duplicate = createLearningOpportunity({
     session_id: ruleSession,
     cwd: workspace,
@@ -287,6 +298,30 @@ try {
   check(sourceHook.includes('createLearningOpportunity') && verifierHook.includes('createLearningOpportunity'), 'negative-control anchors require both runtime emitters');
   check(!sourceHook.includes('persistMemory(') && !verifierHook.includes('persistMemory('), 'runtime recovery hooks cannot directly persist memory');
 
+  const telemetryEvents = telemetry.readEvents(telemetry.telemetryFile(workspace, { cwd: workspace }));
+  const telemetryReport = buildReport(telemetryEvents);
+  const lessonEvents = new Set(telemetryEvents.filter(event => !event.invalid).map(event => event.event));
+  for (const expected of [
+    'learning_opportunity', 'lesson_proposed', 'lesson_screened', 'lesson_evaluated',
+    'lesson_accepted', 'lesson_rejected', 'lesson_inconclusive', 'lesson_persisted',
+    'lesson_retrieved', 'lesson_validated',
+  ]) {
+    check(lessonEvents.has(expected), 'self-evolve lifecycle emits telemetry event: ' + expected);
+  }
+  check(telemetryReport.selfEvolve.funnel.opportunities >= 3 &&
+    telemetryReport.selfEvolve.funnel.accepted >= 1 &&
+    telemetryReport.selfEvolve.funnel.inconclusive >= 1 &&
+    telemetryReport.selfEvolve.funnel.rejected >= 1 &&
+    telemetryReport.selfEvolve.funnel.persisted >= 1 &&
+    telemetryReport.selfEvolve.funnel.retrieved >= 1 &&
+    telemetryReport.selfEvolve.funnel.validated >= 1,
+    'telemetry report exposes opportunity -> disposition -> persisted -> retrieved -> validated funnel');
+  const telemetryText = fs.readFileSync(telemetry.telemetryFile(workspace, { cwd: workspace }), 'utf8');
+  check(!telemetryText.includes('Always verify recovery.test.js') &&
+    !telemetryText.includes('retrieval-fixture:REQ-141') &&
+    !telemetryText.includes('SECRET_OUTPUT_SHOULD_NOT_BE_RETAINED'),
+    'lesson telemetry excludes generalized rules, evidence refs, and raw runtime output');
+
   const lessonSchema = JSON.parse(fs.readFileSync(path.join(ROOT, 'self-evolve/schemas/lesson-candidate.schema.json'), 'utf8'));
   check(lessonSchema.properties?.state?.enum?.includes('inconclusive') && lessonSchema.properties?.trigger?.properties?.type?.enum?.includes('verifier-fail-pass'), 'versioned lesson schema covers lifecycle and trigger vocabulary');
 
@@ -307,5 +342,5 @@ try {
   fs.rmSync(tempRoot, { recursive: true, force: true });
 }
 
-console.log(`\n${failed === 0 ? 'PASS' : 'FAIL'}: #141 self-evolve closed loop phase 1-3 (${failed} failure${failed === 1 ? '' : 's'})`);
+console.log(`\n${failed === 0 ? 'PASS' : 'FAIL'}: #141 self-evolve closed loop phase 1-4 mechanism (${failed} failure${failed === 1 ? '' : 's'})`);
 process.exit(failed === 0 ? 0 : 1);

package/ci/telemetry.test.js

@@ -102,6 +102,32 @@ async function main() {
     check(report.skills[0].invocationCount === 1 && report.skills[0].toolEventCount === 1, 'aggregation retains invocation and attributed-tool counts');
     check(report.semantics.activeWindowMs.includes('not CPU'), 'report documents active-window semantics instead of calling it runtime');
 
+    const lessonId = 'lesson-0123456789abcdef01234567';
+    for (const event of ['learning_opportunity', 'lesson_proposed', 'lesson_screened', 'lesson_evaluated', 'lesson_accepted', 'lesson_persisted', 'lesson_retrieved']) {
+      const emitted = telemetry.emitLessonTelemetry({
+        event,
+        root: workspace,
+        sessionId: session,
+        candidateId: lessonId,
+        status: 'success',
+        reasonCodes: ['fixture-lifecycle'],
+      });
+      check(emitted.ok === true, 'lesson lifecycle telemetry accepts privacy-safe event: ' + event);
+    }
+    const lifecycleEvents = telemetry.readEvents(file);
+    const lifecycleReport = buildReport(lifecycleEvents);
+    check(lifecycleReport.skills.length === 1 && lifecycleReport.skills[0].skillName === 'tdd',
+      'lesson lifecycle events do not contaminate skill execution aggregates');
+    check(lifecycleReport.selfEvolve.candidateCount === 1 &&
+      lifecycleReport.selfEvolve.funnel.opportunities === 1 &&
+      lifecycleReport.selfEvolve.funnel.accepted === 1 &&
+      lifecycleReport.selfEvolve.funnel.persisted === 1 &&
+      lifecycleReport.selfEvolve.funnel.retrieved === 1,
+      'report projects one correlated self-evolve opportunity -> accepted -> persisted -> retrieved funnel');
+    const lifecycleSerialized = fs.readFileSync(file, 'utf8');
+    check(!lifecycleSerialized.includes(lessonId) && !lifecycleSerialized.includes(session),
+      'lesson telemetry hashes candidate/session identities instead of retaining raw IDs');
+
     const forbidden = telemetry.emitTelemetry({
       event: 'skill.invoke',
       host: 'claude',
@@ -173,7 +199,11 @@ async function main() {
     }
 
     const schema = JSON.parse(fs.readFileSync(path.join(ROOT, 'telemetry/schemas/event.schema.json'), 'utf8'));
-    check(schema.properties?.timing?.properties?.activeWindowMs && schema.properties?.event?.enum?.includes('tool.observed'), 'versioned telemetry schema preserves separate timing vocabulary');
+    check(schema.properties?.timing?.properties?.activeWindowMs &&
+      schema.properties?.event?.enum?.includes('tool.observed') &&
+      schema.properties?.event?.enum?.includes('learning_opportunity') &&
+      schema.properties?.event?.enum?.includes('lesson_retrieved'),
+      'versioned telemetry schema preserves timing plus self-evolve lifecycle vocabulary');
 
   } finally {
     if (priorHome === undefined) delete process.env.HARNESS_STATE_HOME;