@agwab/pi-workflow 0.3.0 → 0.4.0

package/workflows/deep-research/helpers/claim-evidence-gate.mjs

@@ -1,3 +1,9 @@
+import {
+	VERIFICATION_STATUS,
+	VERIFICATION_STATUS_BUCKETS,
+	canonicalVerificationStatus,
+} from "./verification-ontology.mjs";
+
 // Deterministic claim audit for deep-research.
 //
 // Sources: plan (optional), normalize-claims (optional), verify-claims foreach
@@ -16,10 +22,6 @@ function asArray(value) {
 	if (Array.isArray(value)) return value;
 	if (value && typeof value === "object") {
 		if (Array.isArray(value.auditedClaims)) return value.auditedClaims;
-		if (Array.isArray(value.claims)) return value.claims;
-		if (Array.isArray(value.claimVerdicts)) return value.claimVerdicts;
-		if (Array.isArray(value.verdicts)) return value.verdicts;
-		if (Array.isArray(value.items)) return value.items;
 		if (
 			"status" in value ||
 			"verdict" in value ||
@@ -28,6 +30,11 @@ function asArray(value) {
 			"id" in value
 		)
 			return [value];
+		if (Array.isArray(value.results)) return value.results;
+		if (Array.isArray(value.claims)) return value.claims;
+		if (Array.isArray(value.claimVerdicts)) return value.claimVerdicts;
+		if (Array.isArray(value.verdicts)) return value.verdicts;
+		if (Array.isArray(value.items)) return value.items;
 		return Object.values(value).flatMap(asArray);
 	}
 	return [];
@@ -345,40 +352,133 @@ function compactStrings(values) {
 }
 
 function canonicalVerifierStatus(status) {
-	return status === "partiallySupported" ? "partially_supported" : status;
+	return canonicalVerificationStatus(status);
 }
 
 function conservativeVerifierStatus(statuses) {
 	const normalized = statuses.map(canonicalVerifierStatus);
 	for (const status of [
-		"conflicting",
-		"unsupported",
-		"partially_supported",
-		"unverified",
+		VERIFICATION_STATUS.CONFLICTING,
+		VERIFICATION_STATUS.UNSUPPORTED,
+		VERIFICATION_STATUS.VERIFICATION_BLOCKED,
+		VERIFICATION_STATUS.PARTIALLY_SUPPORTED,
+		VERIFICATION_STATUS.UNVERIFIED,
 	]) {
 		if (normalized.includes(status)) return status;
 	}
-	if (normalized.every((status) => status === "verified")) return "verified";
+	if (normalized.every((status) => status === VERIFICATION_STATUS.VERIFIED))
+		return VERIFICATION_STATUS.VERIFIED;
 	return (
 		normalized.find((status) => typeof status === "string" && status) ??
-		"unverified"
+		VERIFICATION_STATUS.UNVERIFIED
 	);
 }
 
-function issueForVerifierRow({ sourceId, claim, reason, claimId, index }) {
+function issueForVerifierRow({
+	sourceId,
+	claim,
+	reason,
+	claimId,
+	index,
+	...details
+}) {
 	return {
 		sourceId,
 		...(Number.isInteger(index) ? { index } : {}),
 		...(claimId ? { claimId } : {}),
+		...details,
 		reason,
 		status: verdictOf(claim),
 		nextStep:
 			reason === "unknown_claim_id"
 				? "Verify-claims output did not match any normalized verification candidate; quarantine it from claim counts."
-				: "Verifier output is missing a usable string id/claimId; rerun or repair the verifier row before counting it.",
+				: reason === "batch_result_id_not_in_source_batch"
+					? "Verifier batch output included a claim id outside the source batch; rerun or repair the batch before counting any row."
+					: reason === "unknown_verification_batch_id"
+						? "Verifier batch output came from an unknown batch id; rerun or repair the batch before counting any row."
+						: "Verifier output is missing a usable string id/claimId; rerun or repair the verifier row before counting it.",
 	};
 }
 
+function asBatchArray(value) {
+	if (Array.isArray(value?.batches)) return value.batches;
+	if (Array.isArray(value)) return value;
+	return [];
+}
+
+function buildBatchMembershipById(verificationBatches) {
+	const batches = new Map();
+	for (const batch of asBatchArray(verificationBatches)) {
+		const id = typeof batch?.id === "string" ? batch.id.trim() : "";
+		if (!id) continue;
+		const claimIds = Array.isArray(batch.claimIds)
+			? batch.claimIds
+			: Array.isArray(batch.claims)
+				? batch.claims.map(
+						(claim, index) =>
+							claimIdOf(claim).id ??
+							`candidate-${String(index + 1).padStart(3, "0")}`,
+					)
+				: [];
+		batches.set(
+			id,
+			new Set(
+				claimIds
+					.filter((claimId) => typeof claimId === "string")
+					.map((claimId) => claimId.trim())
+					.filter(Boolean),
+			),
+		);
+	}
+	return batches;
+}
+
+function verifierBatchId(sourceId) {
+	const prefix = "verify-claims.";
+	if (typeof sourceId !== "string" || !sourceId.startsWith(prefix)) return null;
+	const id = sourceId.slice(prefix.length).trim();
+	return id || null;
+}
+
+function buildBatchIdBySourceName(sourceStatuses) {
+	const bySource = new Map();
+	for (const status of Array.isArray(sourceStatuses) ? sourceStatuses : []) {
+		const source = typeof status?.source === "string" ? status.source : "";
+		const batchId = verifierBatchId(status?.specId);
+		if (source && batchId) bySource.set(source, batchId);
+	}
+	return bySource;
+}
+
+function batchMembershipIssue({
+	sourceId,
+	claimId,
+	batchMembershipById,
+	batchIdBySourceName,
+}) {
+	if (!(batchMembershipById instanceof Map) || batchMembershipById.size === 0)
+		return null;
+	const batchId =
+		verifierBatchId(sourceId) ?? batchIdBySourceName?.get(sourceId);
+	if (!batchId) return null;
+	const expectedClaimIds = batchMembershipById.get(batchId);
+	if (!expectedClaimIds) {
+		return {
+			reason: "unknown_verification_batch_id",
+			batchId,
+			expectedBatchIds: [...batchMembershipById.keys()],
+		};
+	}
+	if (!expectedClaimIds.has(claimId)) {
+		return {
+			reason: "batch_result_id_not_in_source_batch",
+			batchId,
+			expectedClaimIds: [...expectedClaimIds],
+		};
+	}
+	return null;
+}
+
 function gapForVerifierIssue(issue) {
 	return {
 		...(issue.claimId ? { claimId: issue.claimId } : {}),
@@ -437,12 +537,33 @@ function mergeVerifierRows(rows) {
 	};
 }
 
-const STATUS_BUCKETS = {
-	verified: "verified",
-	partially_supported: "partiallySupported",
-	unsupported: "unsupported",
-	conflicting: "conflicting",
-};
+function buildBatchAdoptionReadiness({ gateSummary, candidateCount }) {
+	const checks = [
+		["invalid_verifier_rows", gateSummary.invalidVerifierRows],
+		["missing_verifier_results", gateSummary.missingVerifierResults],
+		["duplicate_verifier_rows", gateSummary.duplicateVerifierRows],
+		["duplicate_status_conflicts", gateSummary.duplicateStatusConflicts],
+		["invalid_normalized_candidates", gateSummary.invalidNormalizedCandidates],
+		["source_ref_join_failures", gateSummary.sourceRefJoinFailures],
+	];
+	const blockers = checks
+		.filter(([, count]) => Number(count ?? 0) > 0)
+		.map(([reason, count]) => ({ reason, count }));
+	if (candidateCount === 0)
+		blockers.push({ reason: "no_verification_candidates", count: 0 });
+	return {
+		status: blockers.length === 0 ? "eligible_for_canary" : "blocked",
+		adopted: false,
+		canaryRequired: true,
+		reason:
+			blockers.length === 0
+				? "Verifier identity/sourceRef integrity is clean; batch adoption still requires a non-holdout canary before use."
+				: "Batch adoption is blocked until verifier identity/sourceRef integrity issues are resolved.",
+		blockers,
+	};
+}
+
+const STATUS_BUCKETS = VERIFICATION_STATUS_BUCKETS;
 
 function findSource(sources, stageId) {
 	for (const [specId, source] of Object.entries(sources ?? {})) {
@@ -451,11 +572,18 @@ function findSource(sources, stageId) {
 	return null;
 }
 
-export default async function claimEvidenceGate({ sources, options = {} }) {
+export default async function claimEvidenceGate({
+	sources,
+	options = {},
+	context = {},
+}) {
 	const plan = findSource(sources, "plan");
 	const normalizeClaims = findSource(sources, "normalize-claims");
 	const sanitizedCandidates = findSource(sources, "sanitize-claims");
 	const normalized = sanitizedCandidates ?? normalizeClaims;
+	const verificationBatches = findSource(sources, "verification-batches");
+	const batchMembershipById = buildBatchMembershipById(verificationBatches);
+	const batchIdBySourceName = buildBatchIdBySourceName(context.sourceStatuses);
 	const normalizeInputPacket = findSource(sources, "normalize-input-packet");
 	const urlToSourceRef = buildUrlSourceRefLookup(normalizeInputPacket);
 	const candidateRecords = [];
@@ -568,6 +696,25 @@ export default async function claimEvidenceGate({ sources, options = {} }) {
 			gateSummary.invalidVerifierRows += 1;
 			continue;
 		}
+		const batchIssue = batchMembershipIssue({
+			sourceId,
+			claimId: idCheck.id,
+			batchMembershipById,
+			batchIdBySourceName,
+		});
+		if (batchIssue) {
+			const issue = issueForVerifierRow({
+				sourceId,
+				claim,
+				index,
+				claimId: idCheck.id,
+				...batchIssue,
+			});
+			invalidVerifierRows.push(issue);
+			remainingGaps.push(gapForVerifierIssue(issue));
+			gateSummary.invalidVerifierRows += 1;
+			continue;
+		}
 		const row = {
 			sourceId,
 			claimId: idCheck.id,
@@ -806,6 +953,7 @@ export default async function claimEvidenceGate({ sources, options = {} }) {
 		partiallySupported: [],
 		unsupported: [],
 		conflicting: [],
+		verificationBlocked: [],
 		other: [],
 	};
 	for (const claim of auditedClaims) {
@@ -859,11 +1007,16 @@ export default async function claimEvidenceGate({ sources, options = {} }) {
 		verdictDigest: claim.verdictDigest,
 		correctionOrCounterclaim: claim.correctionOrCounterclaim,
 	}));
+	const batchAdoptionReadiness = buildBatchAdoptionReadiness({
+		gateSummary,
+		candidateCount: candidateRecords.length,
+	});
 
 	return {
 		auditedClaims,
 		claimDigests,
 		gateSummary,
+		batchAdoptionReadiness,
 		remainingGaps,
 		sourceRefJoinFailures,
 		invalidVerifierRows,

package/workflows/deep-research/helpers/normalize-input-packet.mjs

@@ -157,6 +157,60 @@ function compactGap(gap, sourceId, index) {
 	};
 }
 
+function compactBudgetLedger(source, sourceId) {
+	const ledger = asObject(source.budgetLedger);
+	if (Object.keys(ledger).length === 0) return undefined;
+	const searchBudget = Number(ledger.searchBudget);
+	const searchCallsUsed = Number(ledger.searchCallsUsed);
+	return {
+		sourceId,
+		question: stringOf(source.question)?.slice(0, 300),
+		...(Number.isFinite(searchBudget) ? { searchBudget } : {}),
+		...(Number.isFinite(searchCallsUsed) ? { searchCallsUsed } : {}),
+		searchQueriesAttempted: compactStrings(ledger.searchQueriesAttempted, 12),
+		omittedSearchQueries: compactStrings(ledger.omittedSearchQueries, 12),
+		budgetExhausted: ledger.budgetExhausted === true,
+		gapRecorded: ledger.gapRecorded === true,
+	};
+}
+
+function sourceStatusesOf(context) {
+	return asArray(context?.sourceStatuses).map(asObject);
+}
+
+function isResearchQuestionStatus(status) {
+	return [status.source, status.specId, status.displayName, status.stageId]
+		.map((value) => String(value ?? ""))
+		.some(
+			(value) =>
+				value === "research-questions" ||
+				value.startsWith("research-questions."),
+		);
+}
+
+function compactSourceStatusGap(status, index) {
+	const sourceId =
+		stringOf(status.specId) ??
+		stringOf(status.source) ??
+		stringOf(status.displayName) ??
+		`research-questions.status-${String(index + 1).padStart(3, "0")}`;
+	const detail = compactStrings(
+		[status.statusDetail, status.errorType, status.lastMessage],
+		3,
+	).join("; ");
+	return {
+		originLocator: `${sourceId}.status-gap`,
+		sourceId,
+		lead: `Research question source ${sourceId} ended with status ${String(status.status ?? "unknown")}${detail ? ` (${detail.slice(0, 300)})` : ""}.`,
+		sourceUrls: [],
+		sourceRefs: [],
+		factSlotIds: [],
+		reason: "research_question_non_completed",
+		status: String(status.status ?? "unknown"),
+		...(stringOf(status.taskId) ? { taskId: stringOf(status.taskId) } : {}),
+	};
+}
+
 function pushBounded(target, overflow, items, limit, overflowKind) {
 	for (const item of items) {
 		if (target.length < limit) target.push(item);
@@ -428,19 +482,21 @@ function buildPrecisionGuard({ claims, planSlots }) {
 	};
 }
 
-export default async function normalizeInputPacket({ sources }) {
+export default async function normalizeInputPacket({ sources, context } = {}) {
 	const plan = asObject(findSource(sources, "plan"));
 	const research = researchSources(sources);
 	const extractedFacts = [];
 	const claims = [];
 	const sourceCards = [];
 	const evidenceGaps = [];
+	const questionBudgetLedger = [];
 	const overflow = {};
 	const limits = {
 		extractedFacts: 240,
 		claims: 240,
 		sources: 160,
 		evidenceGaps: 120,
+		questionBudgetLedger: 80,
 	};
 
 	for (const { sourceId, source } of research) {
@@ -480,7 +536,29 @@ export default async function normalizeInputPacket({ sources }) {
 			limits.evidenceGaps,
 			"omittedEvidenceGaps",
 		);
+		const budgetLedger = compactBudgetLedger(source, sourceId);
+		if (budgetLedger) {
+			pushBounded(
+				questionBudgetLedger,
+				overflow,
+				[budgetLedger],
+				limits.questionBudgetLedger,
+				"omittedQuestionBudgetLedgers",
+			);
+		}
 	}
+	pushBounded(
+		evidenceGaps,
+		overflow,
+		sourceStatusesOf(context)
+			.filter(
+				(status) =>
+					isResearchQuestionStatus(status) && status.status !== "completed",
+			)
+			.map(compactSourceStatusGap),
+		limits.evidenceGaps,
+		"omittedEvidenceGaps",
+	);
 
 	const planSlots = asArray(plan.factSlots).map(compactPlanSlot);
 	const precisionGuard = buildPrecisionGuard({ claims, planSlots });
@@ -506,6 +584,7 @@ export default async function normalizeInputPacket({ sources }) {
 				claims,
 				sources: sourceCards,
 				evidenceGaps,
+				questionBudgetLedger,
 			},
 			slotPreservation,
 			precisionGuard,

package/workflows/deep-research/helpers/render-executive.mjs

@@ -8,6 +8,7 @@
 
 import { mkdir, writeFile } from "node:fs/promises";
 import { join } from "node:path";
+import { canonicalVerificationStatus } from "./verification-ontology.mjs";
 
 function findSource(sources, stageId) {
 	const entries = Object.entries(sources ?? {});
@@ -298,11 +299,24 @@ function finiteNumber(value) {
 function normalizeClaimStatus(status) {
 	const text = cleanText(status).toLowerCase();
 	if (!text) return "";
+	const canonical = canonicalVerificationStatus(text);
+	if (canonical !== "unverified") return canonical;
+	if (
+		text === "unverified" ||
+		text.includes("not verified") ||
+		text.includes("not_verified")
+	)
+		return "unverified";
+	if (
+		text.includes("verification_blocked") ||
+		text.includes("verification blocked")
+	)
+		return "verification_blocked";
 	if (text.includes("conflict")) return "conflicting";
 	if (text.includes("unsupported")) return "unsupported";
 	if (text.includes("partial")) return "partially_supported";
-	if (text.includes("verified")) return "verified";
-	return text;
+	if (/\bverified\b/.test(text)) return "verified";
+	return canonical;
 }
 
 function coverageCounts(coverage, fallback) {
@@ -316,13 +330,18 @@ function coverageCounts(coverage, fallback) {
 			fallback.partially_supported,
 		unsupported: finiteNumber(coverage.unsupported) ?? fallback.unsupported,
 		conflicting: finiteNumber(coverage.conflicting) ?? fallback.conflicting,
+		verification_blocked:
+			finiteNumber(coverage.verificationBlocked) ??
+			finiteNumber(coverage.verification_blocked) ??
+			fallback.verification_blocked,
 	};
 	if (counts.total == null) {
 		counts.total =
 			counts.verified +
 			counts.partially_supported +
 			counts.unsupported +
-			counts.conflicting;
+			counts.conflicting +
+			counts.verification_blocked;
 	}
 	return counts;
 }
@@ -338,7 +357,8 @@ function packetVerdictCounts(packet, fallback) {
 		counts.verified +
 			counts.partially_supported +
 			counts.unsupported +
-			counts.conflicting;
+			counts.conflicting +
+			counts.verification_blocked;
 	return counts;
 }
 
@@ -350,6 +370,7 @@ function claimCounts(control, packet) {
 		partially_supported: 0,
 		unsupported: 0,
 		conflicting: 0,
+		verification_blocked: 0,
 	};
 	for (const claim of claims) {
 		const status = normalizeClaimStatus(claim?.status);
@@ -372,6 +393,7 @@ function claimCounts(control, packet) {
 		"partially_supported",
 		"unsupported",
 		"conflicting",
+		"verification_blocked",
 	]) {
 		if (coverage[key] !== counts[key]) {
 			mismatches.push({
@@ -520,6 +542,8 @@ function evidenceStrength(status) {
 		case "unsupported":
 			return 1;
 		case "conflicting":
+		case "verification_blocked":
+		case "unverified":
 			return 0;
 		default:
 			return -1;
@@ -587,6 +611,7 @@ function coverageSummaryFromPacket(packet, fallback = {}) {
 		partially_supported: 0,
 		unsupported: 0,
 		conflicting: 0,
+		verification_blocked: 0,
 	});
 	if (!counts) return fallback;
 	return {
@@ -595,6 +620,7 @@ function coverageSummaryFromPacket(packet, fallback = {}) {
 		partiallySupported: counts.partially_supported,
 		unsupported: counts.unsupported,
 		conflicting: counts.conflicting,
+		verificationBlocked: counts.verification_blocked,
 		verificationCandidates: counts.total,
 		depth: packet?.researchMetadataSeed?.depth ?? fallback.depth,
 		researchQuestions:
@@ -975,7 +1001,7 @@ function renderAuditSummary(report, claimSummary, slots) {
 	return [
 		"## Audit summary",
 		"",
-		`- Claims: ${claimSummary.verified} verified, ${claimSummary.partially_supported} partially supported, ${claimSummary.unsupported} unsupported, ${claimSummary.conflicting} conflicting.`,
+		`- Claims: ${claimSummary.verified} verified, ${claimSummary.partially_supported} partially supported, ${claimSummary.unsupported} unsupported, ${claimSummary.conflicting} conflicting, ${claimSummary.verification_blocked} verification blocked.`,
 		`- Fact slots: ${slots.filled} filled, ${slots.partial} partial, ${slots.missingOrConflicting} missing/conflicting, ${slots.total} total.`,
 		...(mismatches.length > 0
 			? [
@@ -1240,6 +1266,7 @@ export default async function renderExecutive({
 				partially_supported: 0,
 				unsupported: 0,
 				conflicting: 0,
+				verification_blocked: 0,
 			},
 			factSlotSummary: {
 				total: 0,