@siuver/omp-debug-mode 0.1.2 → 0.1.4

package/src/evidence.ts

@@ -0,0 +1,196 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { DebugState, EvidenceArtifact, EvidenceMethod, EvidenceRequest } from "./state";
+
+/** Upper bound on accepted plan entries so one model reply cannot flood the gate. */
+export const MAX_EVIDENCE_REQUESTS = 12;
+
+const METHODS: readonly EvidenceMethod[] = ["agent_inspection", "runtime_probe", "user_report", "user_artifact"];
+
+export interface EvidencePlanParseResult {
+	found: boolean;
+	valid: boolean;
+	requests: EvidenceRequest[];
+	error?: string;
+}
+
+function isNonEmptyString(value: unknown): value is string {
+	return typeof value === "string" && value.trim().length > 0;
+}
+
+function isNonEmptyStringArray(value: unknown): value is string[] {
+	return Array.isArray(value) && value.length > 0 && value.every(isNonEmptyString);
+}
+
+function hasOnlyUniqueValues(values: readonly string[]): boolean {
+	return new Set(values).size === values.length;
+}
+
+/**
+ * Parse the `<evidence_plan>` block from a model reply. Validation is
+ * all-or-nothing: any malformed or invalid entry rejects the whole plan so a
+ * partial plan can never silently drop a hypothesis.
+ */
+export function parseEvidencePlan(text: string, round: number): EvidencePlanParseResult {
+	const match = /<evidence_plan>([\s\S]*?)<\/evidence_plan>/.exec(text);
+	if (!match) return { found: false, valid: false, requests: [] };
+	const fail = (error: string): EvidencePlanParseResult => ({ found: true, valid: false, requests: [], error });
+
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(match[1].trim());
+	} catch (error) {
+		return fail(`malformed JSON in <evidence_plan>: ${(error as Error).message}`);
+	}
+	if (!Array.isArray(parsed)) return fail("<evidence_plan> must contain a JSON array");
+	if (parsed.length === 0) return fail("<evidence_plan> array must not be empty");
+	if (parsed.length > MAX_EVIDENCE_REQUESTS) {
+		return fail(`<evidence_plan> has ${parsed.length} entries; at most ${MAX_EVIDENCE_REQUESTS} are allowed`);
+	}
+
+	const requests: EvidenceRequest[] = [];
+	const seenIds = new Set<string>();
+	for (const entry of parsed) {
+		if (typeof entry !== "object" || entry === null) return fail("every evidence plan entry must be an object");
+		const record = entry as Record<string, unknown>;
+		if (!isNonEmptyString(record.id)) return fail("every evidence plan entry needs a non-empty id");
+		if (seenIds.has(record.id)) return fail(`duplicate evidence request id ${record.id}`);
+		seenIds.add(record.id);
+		if (!isNonEmptyStringArray(record.hypothesisIds) || !hasOnlyUniqueValues(record.hypothesisIds)) {
+			return fail(`request ${record.id} needs one or more unique non-empty hypothesisIds`);
+		}
+		if (!METHODS.includes(record.method as EvidenceMethod)) {
+			return fail(`request ${record.id} has unknown method ${JSON.stringify(record.method)}`);
+		}
+		if (!isNonEmptyString(record.title)) return fail(`request ${record.id} needs a non-empty title`);
+		if (!isNonEmptyString(record.rationale)) {
+			return fail(`request ${record.id} needs a non-empty rationale explaining why this method is decisive`);
+		}
+		if (!isNonEmptyStringArray(record.instructions)) {
+			return fail(`request ${record.id} needs one or more non-empty instructions`);
+		}
+		if (record.artifactHint !== undefined && !isNonEmptyString(record.artifactHint)) {
+			return fail(`request ${record.id} has an empty artifactHint`);
+		}
+		requests.push({
+			id: record.id,
+			hypothesisIds: record.hypothesisIds,
+			method: record.method as EvidenceMethod,
+			title: record.title,
+			rationale: record.rationale,
+			instructions: record.instructions,
+			...(record.artifactHint === undefined ? {} : { artifactHint: record.artifactHint }),
+			round,
+		});
+	}
+	return { found: true, valid: true, requests };
+}
+
+export type ArtifactValidation =
+	| { ok: true; artifact: Omit<EvidenceArtifact, "id" | "addedAt" | "requestId"> }
+	| { ok: false; reason: string };
+
+/**
+ * Resolve a user-supplied path and capture filesystem metadata only. The file
+ * is referenced in place: its contents are never read, copied or deleted here.
+ */
+export function validateEvidenceArtifact(rawPath: string, cwd: string): ArtifactValidation {
+	const trimmed = rawPath.trim();
+	if (trimmed.length === 0) return { ok: false, reason: "no evidence file path provided" };
+	const absolute = path.resolve(cwd, trimmed);
+	let stats: fs.Stats;
+	try {
+		stats = fs.statSync(absolute);
+	} catch (error) {
+		return { ok: false, reason: `cannot stat ${absolute}: ${(error as Error).message}` };
+	}
+	if (!stats.isFile()) return { ok: false, reason: `${absolute} is not a regular file` };
+	try {
+		fs.accessSync(absolute, fs.constants.R_OK);
+	} catch {
+		return { ok: false, reason: `${absolute} is not readable` };
+	}
+	return {
+		ok: true,
+		artifact: { path: absolute, name: path.basename(absolute), size: stats.size, mtimeMs: stats.mtimeMs },
+	};
+}
+
+/** Record a user-supplied artifact, deduplicating by exact absolute path. */
+export function addEvidenceArtifact(
+	state: DebugState,
+	rawPath: string,
+	cwd: string,
+	requestId: string | null,
+	now = Date.now(),
+): { state: DebugState; artifact: EvidenceArtifact } | { error: string } {
+	if (requestId !== null && !state.evidenceRequests.some(request => request.id === requestId)) {
+		return { error: `unknown evidence request id ${requestId}` };
+	}
+	const validation = validateEvidenceArtifact(rawPath, cwd);
+	if (!validation.ok) return { error: validation.reason };
+	const base = validation.artifact;
+	const existing = state.evidenceArtifacts.find(artifact => artifact.path === base.path);
+	if (existing) {
+		const artifact: EvidenceArtifact = { ...existing, ...base, requestId };
+		return {
+			state: { ...state, evidenceArtifacts: state.evidenceArtifacts.map(a => (a.path === base.path ? artifact : a)) },
+			artifact,
+		};
+	}
+	const artifact: EvidenceArtifact = { id: `artifact-${now.toString(36)}`, requestId, ...base, addedAt: now };
+	return { state: { ...state, evidenceArtifacts: [...state.evidenceArtifacts, artifact] }, artifact };
+}
+
+/**
+ * Human-readable evidence description for the blackboard and the model-facing
+ * tool. Availability is re-checked on every call: a missing or unreadable file
+ * is never treated as captured evidence.
+ */
+export function describeEvidence(state: DebugState, round?: number): string {
+	const requests = round === undefined ? state.evidenceRequests : state.evidenceRequests.filter(r => r.round === round);
+	if (requests.length === 0 && state.evidenceArtifacts.length === 0 && state.evidenceObservations.length === 0) {
+		return "(none)";
+	}
+	const lines: string[] = [];
+	for (const request of requests) {
+		const linked = state.evidenceArtifacts.filter(artifact => artifact.requestId === request.id);
+		const reports = state.evidenceObservations.filter(observation => observation.requestIds.includes(request.id));
+		const hint = request.artifactHint ? `, artifactHint: ${request.artifactHint}` : "";
+		const coverage =
+			request.method === "user_artifact"
+				? linked.length > 0
+					? "artifact attached"
+					: "PENDING artifact"
+				: request.method === "user_report"
+					? reports.length > 0
+						? "report submitted"
+						: "PENDING report"
+					: "agent-collected";
+		lines.push(
+			`- ${request.id} [${request.method}] ${request.title} (${coverage})${hint}\n  hypotheses: ${request.hypothesisIds.join(", ")}\n  rationale: ${request.rationale}\n  instructions: ${request.instructions.join(" | ")}`,
+		);
+	}
+	for (const observation of state.evidenceObservations) {
+		lines.push(`- ${observation.id} [user observation, round ${observation.round}] ${observation.text}`);
+	}
+	for (const artifact of state.evidenceArtifacts) {
+		let stats: fs.Stats;
+		try {
+			stats = fs.statSync(artifact.path);
+			fs.accessSync(artifact.path, fs.constants.R_OK);
+		} catch {
+			lines.push(`- ${artifact.id} [artifact] ${artifact.path} — UNAVAILABLE (file missing or unreadable)`);
+			continue;
+		}
+		if (!stats.isFile()) {
+			lines.push(`- ${artifact.id} [artifact] ${artifact.path} — UNAVAILABLE (no longer a regular file)`);
+			continue;
+		}
+		const linked = artifact.requestId ? `, request ${artifact.requestId}` : ", unlinked";
+		lines.push(
+			`- ${artifact.id} [artifact] ${artifact.path} — available${linked}, ${stats.size} bytes, mtime ${new Date(stats.mtimeMs).toISOString()}`,
+		);
+	}
+	return lines.join("\n");
+}

package/src/gate.ts

@@ -0,0 +1,54 @@
+import { MINIMIZE_USER_INTERVENTION, PROCEED_REMINDER } from "./methodology";
+
+/** How many times a single round may be nudged to close itself properly. */
+export const MAX_GATE_NUDGES = 1;
+
+export const GATE_NUDGE =
+	"This round did not close properly. It must end with BOTH a <evidence_plan> JSON block " +
+	'({"id","hypothesisIds","method","title","rationale","instructions", optional "artifactHint"}; methods: ' +
+	"agent_inspection, runtime_probe, user_report, user_artifact) AND a <reproduction_steps> numbered list " +
+	`followed by the exact sentence "${PROCEED_REMINDER}", then stop. ` +
+	MINIMIZE_USER_INTERVENTION +
+	" Every entry needs a concrete rationale; a user_report/user_artifact rationale must name why BOTH " +
+	"autonomous inspection and model-added probes cannot answer its hypotheses. " +
+	"A user_artifact request must state the file type, path/capture instructions, how you will inspect the file, " +
+	"and why inspection/probes are inadequate. Do not start new work.";
+
+export type GateDecision =
+	/** Hand control to the user and wait for a reproduction/capture. */
+	| { kind: "gate"; missingSteps: boolean; missingEvidencePlan: boolean }
+	/** Let the agent finish the round properly before gating. */
+	| { kind: "nudge"; context: string }
+	/** Not a completed round — the agent is mid-conversation with the user. */
+	| { kind: "stay" };
+
+/**
+ * Decide what a settled agent turn means for the reproduction gate.
+ *
+ * A properly closed round carries both the `<evidence_plan>` block and the
+ * `<reproduction_steps>` block. Legacy turns that only supply reproduction
+ * steps with no probes still gate (old sessions never emit plans). A turn
+ * with work but a missing tag gets one nudge, then gates with the missing
+ * flags set so the workflow can never stall.
+ */
+export function decideGate(args: {
+	hasReproductionSteps: boolean;
+	hasEvidencePlan: boolean;
+	probesThisRound: number;
+	nudgesUsed: number;
+}): GateDecision {
+	if (args.hasReproductionSteps && args.hasEvidencePlan) {
+		return { kind: "gate", missingSteps: false, missingEvidencePlan: false };
+	}
+	// Legacy/no-probe closure: reproduction steps alone still gate the turn.
+	if (args.hasReproductionSteps && !args.hasEvidencePlan && args.probesThisRound === 0) {
+		return { kind: "gate", missingSteps: false, missingEvidencePlan: true };
+	}
+	if (args.probesThisRound === 0 && !args.hasEvidencePlan) return { kind: "stay" };
+	if (args.nudgesUsed < MAX_GATE_NUDGES) return { kind: "nudge", context: GATE_NUDGE };
+	return {
+		kind: "gate",
+		missingSteps: !args.hasReproductionSteps,
+		missingEvidencePlan: !args.hasEvidencePlan,
+	};
+}

package/src/log-files.ts

@@ -24,6 +24,75 @@ export function readJsonlLines(file: string | null): string[] {
 	}
 }
 
+interface CountEntry {
+	size: number;
+	mtimeMs: number;
+	count: number;
+}
+
+/**
+ * Line counts keyed by file identity. The reproduction gate polls the active
+ * log once a second, so re-reading every archived run each time would scale
+ * with the length of the debugging session for no new information.
+ */
+export class JsonlLineCounter {
+	#cache = new Map<string, CountEntry>();
+
+	count(file: string | null): number {
+		if (!file) return 0;
+		const stat = fs.statSync(file, { throwIfNoEntry: false });
+		if (!stat) {
+			this.#cache.delete(file);
+			return 0;
+		}
+		const hit = this.#cache.get(file);
+		if (hit && hit.size === stat.size && hit.mtimeMs === stat.mtimeMs) return hit.count;
+		const count = readJsonlLines(file).length;
+		this.#cache.set(file, { size: stat.size, mtimeMs: stat.mtimeMs, count });
+		return count;
+	}
+
+	clear(): void {
+		this.#cache.clear();
+	}
+}
+
+export interface HypothesisTally {
+	id: string;
+	count: number;
+}
+
+const UNATTRIBUTED = "(unattributed)";
+
+/**
+ * Group observations by the hypothesis each probe was meant to settle, so the
+ * user and the agent can see which hypotheses actually produced evidence
+ * before anyone reads the raw log.
+ */
+export function summarizeHypotheses(lines: readonly string[]): HypothesisTally[] {
+	const counts = new Map<string, number>();
+	for (const line of lines) {
+		let id = UNATTRIBUTED;
+		try {
+			const entry = JSON.parse(line) as { hypothesisId?: unknown };
+			if (typeof entry.hypothesisId === "string" && entry.hypothesisId.trim().length > 0) {
+				id = entry.hypothesisId.trim();
+			}
+		} catch {
+			continue;
+		}
+		counts.set(id, (counts.get(id) ?? 0) + 1);
+	}
+	return [...counts]
+		.map(([id, count]) => ({ id, count }))
+		.sort((a, b) => (b.count !== a.count ? b.count - a.count : a.id < b.id ? -1 : 1));
+}
+
+export function describeHypotheses(tallies: readonly HypothesisTally[]): string {
+	if (tallies.length === 0) return "no hypothesis-attributed observations";
+	return tallies.map(t => `${t.id}=${t.count}`).join(", ");
+}
+
 export function prepareRunLog(debugDir: string, previousRun: string | null): string {
 	const activeFile = path.join(debugDir, ACTIVE_LOG_FILE);
 	const archivedFile = previousRun ? path.join(debugDir, `${previousRun}.jsonl`) : null;