@aperant/framework 0.17.0 → 0.20.0

package/src/cli/produce/blind-probe.mjs

@@ -0,0 +1,245 @@
+/**
+ * @internal
+ * produce/blind-probe.mjs — the held-out cross-family blind-probe set (M2, AC4-6).
+ *
+ * A fixed 10–20 behavioral probe set the implementer/Showrunner NEVER sees
+ * (probe-blindness, US-4 / ID-06), authored + run by a provider family DISJOINT
+ * from the implementer's, re-run on a rotating sample of already-done milestones
+ * against HEAD (US-6).
+ *
+ * The probe SET is a frozen immutable artifact at
+ * `.aperant/produce/<run-id>/probes/<milestone>.json`, written ONCE via
+ * `openSync(path, 'wx')` exclusive-create — a second freeze throws EEXIST and
+ * returns the byte-identical stored set (mirrors setpoint.mjs::freezeSetpoint).
+ * Its CONTENT never enters the event log or the roadmap projection the
+ * Showrunner re-reads each tick (structural probe-blindness, ID-M2-03): the
+ * `produce.probe.frozen` event carries ONLY `{probe_set_hash, author_family}`
+ * and `produce.probe.run` carries ONLY `{result, probe_family, head_sha}`.
+ */
+
+import { createHash } from 'node:crypto'
+import {
+	closeSync,
+	existsSync,
+	fsyncSync,
+	mkdirSync,
+	openSync,
+	readFileSync,
+	writeSync,
+} from 'node:fs'
+import { join } from 'node:path'
+import {
+	appendProduceEvent,
+	readImplementerFamily,
+	readProduceRunEventsMatching,
+} from './events.mjs'
+import { normalizeFamily } from './probe-family.mjs'
+import { projectRoadmap } from './projection.mjs'
+import { resolveRunDir } from './run-id.mjs'
+
+/** sha256 of the canonical JSON of the probe content (the immutable hash stamp). */
+function computeProbeSetHash(content) {
+	return createHash('sha256').update(JSON.stringify(content)).digest('hex')
+}
+
+/** The probes/<milestone>.json path for a run + milestone. */
+export function probeSetPathFor(projectDir, runId, milestoneId) {
+	return join(resolveRunDir(projectDir, runId), 'probes', `${milestoneId}.json`)
+}
+
+/** Read + parse a frozen probe set. Returns null when absent or malformed. */
+export function readProbeSet(projectDir, runId, milestoneId) {
+	const p = probeSetPathFor(projectDir, runId, milestoneId)
+	if (!existsSync(p)) return null
+	try {
+		return JSON.parse(readFileSync(p, 'utf-8'))
+	} catch {
+		return null
+	}
+}
+
+/**
+ * Exclusive-create write of the frozen probe set on the FINAL path (NOT
+ * tmp+rename), so a SECOND write throws EEXIST — the set is immutable.
+ */
+function writeProbeSetExclusive(probePath, probeSet) {
+	mkdirSync(join(probePath, '..'), { recursive: true })
+	const payload = `${JSON.stringify(probeSet, null, 2)}\n`
+	const fd = openSync(probePath, 'wx') // throws EEXIST when already frozen
+	try {
+		writeSync(fd, payload, 0, 'utf-8')
+		fsyncSync(fd)
+	} finally {
+		closeSync(fd)
+	}
+}
+
+/**
+ * Freeze a blind-probe set for a milestone, idempotently. REFUSES fail-closed
+ * when the author family equals the implementer family (a held-out probe an
+ * implementer authored is not held out — ID-06) or when no implementer family
+ * is recorded (ID-M2-02). On a FRESH freeze appends `produce.probe.frozen`
+ * carrying ONLY the hash + author family (never the content). A second freeze
+ * returns the byte-identical stored set (`reused: true`) and never re-logs.
+ *
+ * @param {string} projectDir
+ * @param {string} runId
+ * @param {string} milestoneId
+ * @param {{ content: unknown, authorFamily: string, implementerFamily: string }} args
+ * @param {Date} [now]
+ * @returns {{ ok: true, reused: boolean, probe_set_hash: string, author_family: string }
+ *   | { ok: false, reason: string }}
+ */
+export function freezeProbeSet(
+	projectDir,
+	runId,
+	milestoneId,
+	{ content, authorFamily, implementerFamily },
+	now = new Date(),
+) {
+	if (!authorFamily) return { ok: false, reason: 'no_author_family' }
+	if (!implementerFamily) return { ok: false, reason: 'no_implementer_family' }
+	// Cross-family enforcement (normalized so claude-code === claude).
+	if (normalizeFamily(authorFamily) === normalizeFamily(implementerFamily))
+		return { ok: false, reason: 'same_family_author' }
+
+	const probePath = probeSetPathFor(projectDir, runId, milestoneId)
+	const existing = readProbeSet(projectDir, runId, milestoneId)
+	if (existing)
+		return {
+			ok: true,
+			reused: true,
+			probe_set_hash: existing.probe_set_hash,
+			author_family: existing.author_family,
+		}
+
+	const probeSetHash = computeProbeSetHash(content ?? null)
+	const probeSet = {
+		run_id: runId,
+		milestone_id: milestoneId,
+		author_family: authorFamily,
+		frozen_at: now.toISOString(),
+		probe_set_hash: probeSetHash,
+		// The probe CONTENT lives in the frozen file (blind to the projection) —
+		// it NEVER enters the event log.
+		probes: content ?? null,
+	}
+
+	try {
+		writeProbeSetExclusive(probePath, probeSet)
+	} catch (e) {
+		// Lost a concurrent race — the winner's file is authoritative; re-read it
+		// byte-identically rather than overwriting (mirrors freezeSetpoint).
+		if (e && e.code === 'EEXIST') {
+			const winner = readProbeSet(projectDir, runId, milestoneId)
+			if (winner)
+				return {
+					ok: true,
+					reused: true,
+					probe_set_hash: winner.probe_set_hash,
+					author_family: winner.author_family,
+				}
+		}
+		throw e
+	}
+
+	// Announce by hash + author family ONLY — never the content (ID-M2-03).
+	appendProduceEvent(projectDir, 'produce.probe.frozen', {
+		run_id: runId,
+		milestone_id: milestoneId,
+		probe_set_hash: probeSetHash,
+		author_family: authorFamily,
+	})
+
+	return { ok: true, reused: false, probe_set_hash: probeSetHash, author_family: authorFamily }
+}
+
+/**
+ * Record a blind-probe RUN result. REFUSES fail-closed when the probe family
+ * equals the implementer family (a probe run BY the implementer family is not
+ * cross-family — ID-08) or when no implementer family is recorded. On
+ * acceptance appends `produce.probe.run {run_id, milestone_id, result,
+ * probe_family, head_sha}` — result/family/head_sha only, never content.
+ *
+ * @param {string} projectDir
+ * @param {{ runId: string, milestoneId: string, result: string, probeFamily: string, headSha: string }} args
+ * @returns {{ ok: true, result: string, probe_family: string, head_sha: string }
+ *   | { ok: false, reason: string }}
+ */
+export function recordProbeRun(projectDir, { runId, milestoneId, result, probeFamily, headSha }) {
+	if (result !== 'pass' && result !== 'fail') return { ok: false, reason: 'invalid_result' }
+	if (!probeFamily) return { ok: false, reason: 'no_probe_family' }
+	if (!headSha) return { ok: false, reason: 'no_head_sha' }
+
+	const implementerFamily = readImplementerFamily(projectDir, runId, milestoneId)
+	if (!implementerFamily) return { ok: false, reason: 'no_recorded_implementer' }
+	if (normalizeFamily(probeFamily) === normalizeFamily(implementerFamily))
+		return { ok: false, reason: 'same_family_probe' }
+
+	appendProduceEvent(projectDir, 'produce.probe.run', {
+		run_id: runId,
+		milestone_id: milestoneId,
+		result,
+		probe_family: probeFamily,
+		head_sha: headSha,
+	})
+
+	return { ok: true, result, probe_family: probeFamily, head_sha: headSha }
+}
+
+/**
+ * Select a DETERMINISTIC rotating sample of already-`done` milestones for blind
+ * re-probing against HEAD (US-6). Rotates by fewest prior probe runs first,
+ * then oldest last-probe-run first (a never-probed done milestone sorts first),
+ * then first-seen roadmap order — so two folds of the same log select the same
+ * sample.
+ *
+ * @param {string} projectDir
+ * @param {string} runId
+ * @param {{ sample?: number }} [opts]
+ * @returns {{ run_id: string, sample_size: number, selected: Array<{ milestone_id: string, title: string, probe_run_count: number, last_probe_run_at: string | null }> }}
+ */
+export function selectSweepSample(projectDir, runId, { sample = 1 } = {}) {
+	const roadmap = projectRoadmap(projectDir, runId)
+	const done = roadmap.milestones.filter((m) => m.status === 'done')
+
+	const runEvents = readProduceRunEventsMatching(
+		projectDir,
+		runId,
+		(op) => op === 'produce.probe.run',
+	)
+	const stats = new Map()
+	for (const e of runEvents) {
+		const id = e.data && typeof e.data.milestone_id === 'string' ? e.data.milestone_id : null
+		if (!id) continue
+		const s = stats.get(id) || { count: 0, last: null }
+		s.count += 1
+		if (typeof e.t === 'string' && (s.last === null || e.t > s.last)) s.last = e.t
+		stats.set(id, s)
+	}
+
+	const withOrder = done.map((m, idx) => {
+		const s = stats.get(m.id) || { count: 0, last: null }
+		return { milestone: m, order: idx, count: s.count, last: s.last }
+	})
+
+	withOrder.sort((a, b) => {
+		if (a.count !== b.count) return a.count - b.count // fewest probe runs first
+		// oldest last-probe-run first — a never-probed done milestone (null) sorts
+		// ahead of any dated one.
+		const al = a.last ?? ''
+		const bl = b.last ?? ''
+		if (al !== bl) return al < bl ? -1 : 1
+		return a.order - b.order // first-seen roadmap order (stable tiebreak)
+	})
+
+	const n = Number.isFinite(sample) && sample > 0 ? Math.floor(sample) : 1
+	const selected = withOrder.slice(0, n).map((w) => ({
+		milestone_id: w.milestone.id,
+		title: w.milestone.title,
+		probe_run_count: w.count,
+		last_probe_run_at: w.last,
+	}))
+
+	return { run_id: runId, sample_size: selected.length, selected }
+}