@agwab/pi-workflow 0.3.0 → 0.4.0

package/workflows/deep-research/helpers/shadow-select-verification.mjs

@@ -0,0 +1,229 @@
+// Shadow-only selective verification reporter for deep-research.
+//
+// This helper never skips verification. It records which candidates a future
+// selector might skip, then joins those shadow decisions to actual audit output
+// so adoption can be judged with evidence before any verifier is removed.
+
+function asArray(value) {
+	if (Array.isArray(value)) return value;
+	if (value && typeof value === "object") {
+		if (Array.isArray(value.claimInventory?.verificationCandidates))
+			return value.claimInventory.verificationCandidates;
+		if (Array.isArray(value.verificationCandidates))
+			return value.verificationCandidates;
+		if (Array.isArray(value.auditedClaims)) return value.auditedClaims;
+		if (Array.isArray(value.claims)) return value.claims;
+		if (Array.isArray(value.items)) return value.items;
+	}
+	return [];
+}
+
+function findCandidates(sources) {
+	for (const [specId, source] of Object.entries(sources ?? {})) {
+		if (specId === "sanitize-claims" || specId.startsWith("sanitize-claims.")) {
+			const candidates = asArray(source);
+			if (candidates.length > 0) return candidates;
+		}
+	}
+	for (const [specId, source] of Object.entries(sources ?? {})) {
+		if (
+			specId === "normalize-claims" ||
+			specId.startsWith("normalize-claims.")
+		) {
+			const candidates = asArray(source);
+			if (candidates.length > 0) return candidates;
+		}
+	}
+	return [];
+}
+
+function findAuditedClaims(sources) {
+	for (const [specId, source] of Object.entries(sources ?? {})) {
+		if (specId === "audit-claims" || specId.startsWith("audit-claims.")) {
+			const claims = asArray(source);
+			if (claims.length > 0) return claims;
+		}
+	}
+	return [];
+}
+
+function findAuditSource(sources) {
+	for (const [specId, source] of Object.entries(sources ?? {})) {
+		if (specId === "audit-claims" || specId.startsWith("audit-claims."))
+			return source && typeof source === "object" ? source : {};
+	}
+	return {};
+}
+
+function issueCount(audit, field) {
+	const fromSummary = Number(audit?.gateSummary?.[field] ?? 0);
+	const fromArray = Array.isArray(audit?.[field]) ? audit[field].length : 0;
+	return Math.max(Number.isFinite(fromSummary) ? fromSummary : 0, fromArray);
+}
+
+function auditIntegrityBlockers(audit) {
+	return [
+		["audit_invalid_verifier_rows", issueCount(audit, "invalidVerifierRows")],
+		[
+			"audit_missing_verifier_results",
+			issueCount(audit, "missingVerifierResults"),
+		],
+		[
+			"audit_duplicate_verifier_rows",
+			issueCount(audit, "duplicateVerifierRows"),
+		],
+		[
+			"audit_duplicate_status_conflicts",
+			issueCount(audit, "duplicateStatusConflicts"),
+		],
+		[
+			"audit_invalid_normalized_candidates",
+			issueCount(audit, "invalidNormalizedCandidates"),
+		],
+		[
+			"audit_source_ref_join_failures",
+			issueCount(audit, "sourceRefJoinFailures"),
+		],
+	]
+		.filter(([, count]) => count > 0)
+		.map(([reason, count]) => ({ reason, count }));
+}
+
+function candidateId(candidate, index) {
+	const id = typeof candidate?.id === "string" ? candidate.id.trim() : "";
+	return id || `candidate-${String(index + 1).padStart(3, "0")}`;
+}
+
+function hasExactQuantitativeText(candidate) {
+	return /\b\d+(?:\.\d+)?\b|\b(v\d+(?:\.\d+)*)\b|\b\d+\s*(?:ms|s|min|hour|day|%|percent|usd|dollars?|tokens?|kb|mb|gb)\b/iu.test(
+		String(candidate?.claim ?? ""),
+	);
+}
+
+function hasCriticalSlots(candidate) {
+	const ids = Array.isArray(candidate?.factSlotIds)
+		? candidate.factSlotIds
+		: [];
+	return ids.some((id) =>
+		/(price|pricing|cost|ttl|limit|version|date|policy|security|numeric|slot-critical)/iu.test(
+			String(id),
+		),
+	);
+}
+
+function hasExactSourceHint(candidate) {
+	return (
+		Array.isArray(candidate?.sourceEvidenceHints) &&
+		candidate.sourceEvidenceHints.some(
+			(hint) =>
+				typeof hint?.quote === "string" &&
+				hint.quote.trim() &&
+				(typeof hint.sourceRef === "string" ||
+					typeof hint.sourceUrl === "string"),
+		)
+	);
+}
+
+function decide(candidate) {
+	const reasonCodes = [];
+	if (hasExactQuantitativeText(candidate))
+		reasonCodes.push("exact_quantitative");
+	if (hasCriticalSlots(candidate)) reasonCodes.push("critical_fact_slot");
+	if (!hasExactSourceHint(candidate)) reasonCodes.push("no_exact_source_hint");
+	if (reasonCodes.length > 0) {
+		return { decision: "would_verify", reasonCodes };
+	}
+	return {
+		decision: "would_skip_shadow_only",
+		reasonCodes: ["exact_source_hint_noncritical"],
+	};
+}
+
+export default async function shadowSelectVerification({ sources }) {
+	const audit = findAuditSource(sources);
+	const candidates = findCandidates(sources).map((candidate, index) => ({
+		candidate,
+		id: candidateId(candidate, index),
+	}));
+	const auditedById = new Map(
+		findAuditedClaims(sources).map((claim, index) => [
+			candidateId(claim, index),
+			claim,
+		]),
+	);
+	const decisions = candidates.map(({ candidate, id }) => {
+		const decision = decide(candidate);
+		const audited = auditedById.get(id);
+		return {
+			id,
+			decision: decision.decision,
+			reasonCodes: decision.reasonCodes,
+			actualStatus:
+				typeof audited?.status === "string" ? audited.status : undefined,
+			actualVerified: audited?.status === "verified",
+			factSlotIds: Array.isArray(candidate.factSlotIds)
+				? [...candidate.factSlotIds]
+				: [],
+		};
+	});
+	const wouldSkip = decisions.filter(
+		(decision) => decision.decision === "would_skip_shadow_only",
+	);
+	const wouldVerify = decisions.filter(
+		(decision) => decision.decision === "would_verify",
+	);
+	const skippedButVerified = wouldSkip.filter(
+		(decision) => decision.actualVerified,
+	);
+	const wouldSkipWithoutAudit = wouldSkip.filter(
+		(decision) => typeof decision.actualStatus !== "string",
+	);
+	const blockers = [];
+	if (skippedButVerified.length > 0)
+		blockers.push({
+			reason: "would_skip_verified_claims",
+			count: skippedButVerified.length,
+		});
+	if (wouldSkipWithoutAudit.length > 0)
+		blockers.push({
+			reason: "would_skip_without_audit_result",
+			count: wouldSkipWithoutAudit.length,
+		});
+	blockers.push(...auditIntegrityBlockers(audit));
+	if (wouldSkip.length === 0)
+		blockers.push({ reason: "no_shadow_skip_candidates", count: 0 });
+	const realSkipReadiness = {
+		status: blockers.length === 0 ? "eligible_for_canary" : "blocked",
+		realSkippingEnabled: false,
+		adopted: false,
+		canaryRequired: true,
+		reason:
+			blockers.length === 0
+				? "Shadow selector found skip candidates with no verified or missing-audit rows; real skipping still requires a non-holdout canary."
+				: "Real selective verification is blocked until shadow decisions prove no verified or unaudited claims would be skipped.",
+		blockers,
+	};
+	return {
+		schema: "deep-research-verification-shadow-selector-v1",
+		digest: `${wouldSkip.length} would-skip shadow candidate(s), ${wouldVerify.length} would-verify candidate(s); real skipping disabled`,
+		realSkippingEnabled: false,
+		candidateCount: decisions.length,
+		summary: {
+			wouldSkip: wouldSkip.length,
+			wouldVerify: wouldVerify.length,
+			skippedButVerified: skippedButVerified.length,
+			wouldSkipWithoutAudit: wouldSkipWithoutAudit.length,
+			skippedCritical: wouldSkip.filter((decision) =>
+				decision.reasonCodes.includes("critical_fact_slot"),
+			).length,
+			shadowOnly: true,
+		},
+		realSkipReadiness,
+		decisions,
+		w8FastProfilePrerequisite: {
+			status: "not_met",
+			reason:
+				"selective verification is shadow-only; no verified cost/speed reduction is available to package as an opt-in fast profile",
+		},
+	};
+}

package/workflows/deep-research/helpers/verification-ontology.mjs

@@ -0,0 +1,77 @@
+// Bundle-local compatibility surface for the package verification ontology.
+//
+// Workflow support helpers are bundled from the workflow spec directory, so this
+// dependency-free helper intentionally lives inside the deep-research bundle.
+// Keep it in semantic parity with src/verification-ontology.ts.
+
+export const VERIFICATION_STATUS = Object.freeze({
+	VERIFIED: "verified",
+	PARTIALLY_SUPPORTED: "partially_supported",
+	UNSUPPORTED: "unsupported",
+	CONFLICTING: "conflicting",
+	VERIFICATION_BLOCKED: "verification_blocked",
+	UNVERIFIED: "unverified",
+});
+
+export const VERIFICATION_STATUS_VALUES = Object.freeze([
+	VERIFICATION_STATUS.VERIFIED,
+	VERIFICATION_STATUS.PARTIALLY_SUPPORTED,
+	VERIFICATION_STATUS.UNSUPPORTED,
+	VERIFICATION_STATUS.CONFLICTING,
+	VERIFICATION_STATUS.VERIFICATION_BLOCKED,
+]);
+
+export const VERIFICATION_STATUS_BUCKETS = Object.freeze({
+	[VERIFICATION_STATUS.VERIFIED]: "verified",
+	[VERIFICATION_STATUS.PARTIALLY_SUPPORTED]: "partiallySupported",
+	[VERIFICATION_STATUS.UNSUPPORTED]: "unsupported",
+	[VERIFICATION_STATUS.CONFLICTING]: "conflicting",
+	[VERIFICATION_STATUS.VERIFICATION_BLOCKED]: "verificationBlocked",
+});
+
+export const VERIFICATION_STATUS_LABELS = Object.freeze({
+	[VERIFICATION_STATUS.VERIFIED]: "verified",
+	[VERIFICATION_STATUS.PARTIALLY_SUPPORTED]: "partially supported",
+	[VERIFICATION_STATUS.UNSUPPORTED]: "unsupported",
+	[VERIFICATION_STATUS.CONFLICTING]: "conflicting",
+	[VERIFICATION_STATUS.VERIFICATION_BLOCKED]: "verification blocked",
+	[VERIFICATION_STATUS.UNVERIFIED]: "unverified",
+});
+
+export function canonicalVerificationStatus(status) {
+	const text = String(status ?? "").trim();
+	if (!text) return VERIFICATION_STATUS.UNVERIFIED;
+	if (text === "partiallySupported")
+		return VERIFICATION_STATUS.PARTIALLY_SUPPORTED;
+	if (text === "verificationBlocked" || text === "blocked")
+		return VERIFICATION_STATUS.VERIFICATION_BLOCKED;
+	return Object.values(VERIFICATION_STATUS).includes(text)
+		? text
+		: VERIFICATION_STATUS.UNVERIFIED;
+}
+
+export function verificationStatusBucket(status) {
+	return (
+		VERIFICATION_STATUS_BUCKETS[canonicalVerificationStatus(status)] ?? "other"
+	);
+}
+
+export function isVerifiedStatus(status) {
+	return canonicalVerificationStatus(status) === VERIFICATION_STATUS.VERIFIED;
+}
+
+export function isVerificationBlockedStatus(status) {
+	return (
+		canonicalVerificationStatus(status) ===
+		VERIFICATION_STATUS.VERIFICATION_BLOCKED
+	);
+}
+
+export function isNonVerifiedTerminalStatus(status) {
+	return [
+		VERIFICATION_STATUS.PARTIALLY_SUPPORTED,
+		VERIFICATION_STATUS.UNSUPPORTED,
+		VERIFICATION_STATUS.CONFLICTING,
+		VERIFICATION_STATUS.VERIFICATION_BLOCKED,
+	].includes(canonicalVerificationStatus(status));
+}

package/workflows/deep-research/schemas/deep-research-executive-render-control.schema.json

@@ -24,13 +24,14 @@
     "sourceUrls": { "type": "array", "items": { "type": "string" } },
     "claimSummary": {
       "type": "object",
-      "required": ["total", "verified", "partially_supported", "unsupported", "conflicting"],
+      "required": ["total", "verified", "partially_supported", "unsupported", "conflicting", "verification_blocked"],
       "properties": {
         "total": { "type": "number" },
         "verified": { "type": "number" },
         "partially_supported": { "type": "number" },
         "unsupported": { "type": "number" },
-        "conflicting": { "type": "number" }
+        "conflicting": { "type": "number" },
+        "verification_blocked": { "type": "number" }
       },
       "additionalProperties": true
     },

package/workflows/deep-research/schemas/deep-research-research-questions-control.schema.json

@@ -38,6 +38,44 @@
         "type": "object"
       },
       "maxItems": 24
+    },
+    "additionalUnverifiedLeads": {
+      "type": "array",
+      "items": {
+        "type": "object"
+      },
+      "maxItems": 24
+    },
+    "budgetLedger": {
+      "type": "object",
+      "properties": {
+        "searchBudget": {
+          "type": "number"
+        },
+        "searchCallsUsed": {
+          "type": "number"
+        },
+        "searchQueriesAttempted": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "maxItems": 12
+        },
+        "omittedSearchQueries": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "maxItems": 12
+        },
+        "budgetExhausted": {
+          "type": "boolean"
+        },
+        "gapRecorded": {
+          "type": "boolean"
+        }
+      }
     }
   }
 }

package/workflows/deep-research/schemas/deep-research-sanitize-claims-control.schema.json

@@ -0,0 +1,63 @@
+{
+  "type": "object",
+  "required": [
+    "schema",
+    "claimInventory",
+    "factSlotCoverage"
+  ],
+  "properties": {
+    "schema": {
+      "type": "string",
+      "minLength": 1
+    },
+    "claimInventory": {
+      "type": "object",
+      "required": [
+        "verificationCandidates",
+        "preservedClaims",
+        "duplicates"
+      ],
+      "properties": {
+        "verificationCandidates": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "id": { "type": "string" },
+              "claim": { "type": "string" },
+              "sourceUrls": { "type": "array", "items": { "type": "string" } },
+              "sourceRefs": { "type": "array", "items": { "type": "string" } },
+              "sourceQuality": { "type": "string" },
+              "reasonToVerify": { "type": "string" },
+              "scopeItems": { "type": "array", "items": { "type": "string" } },
+              "factSlotIds": { "type": "array", "items": { "type": "string" } },
+              "verificationNeed": { "enum": ["core", "useful", "optional"] }
+            }
+          },
+          "maxItems": 48
+        },
+        "preservedClaims": {
+          "type": "array",
+          "items": { "type": "object" },
+          "maxItems": 24
+        },
+        "duplicates": {}
+      }
+    },
+    "factSlotCoverage": {
+      "type": "array",
+      "items": { "type": "object" },
+      "maxItems": 64
+    },
+    "coverageGaps": {
+      "type": "array",
+      "items": { "type": "object" }
+    },
+    "researchScopeCoverage": {
+      "type": "array",
+      "items": { "type": "object" }
+    },
+    "normalizationNotes": {},
+    "sanitizerDiagnostics": {}
+  }
+}

package/workflows/deep-research/schemas/deep-research-verify-claims-batch-control.schema.json

@@ -0,0 +1,47 @@
+{
+  "type": "object",
+  "required": ["schema", "digest", "results"],
+  "additionalProperties": false,
+  "properties": {
+    "schema": {
+      "type": "string",
+      "minLength": 1
+    },
+    "digest": {
+      "type": "string",
+      "minLength": 1
+    },
+    "results": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": ["id", "status", "verdictDigest", "evidence"],
+        "additionalProperties": false,
+        "properties": {
+          "id": {
+            "type": "string",
+            "minLength": 1
+          },
+          "claim": {},
+          "factSlotIds": {},
+          "status": {
+            "type": "string",
+            "enum": ["verified", "partially_supported", "unsupported", "conflicting", "verification_blocked"]
+          },
+          "confidence": {},
+          "verdictDigest": {},
+          "evidence": {
+            "type": "array",
+            "items": {
+              "type": "object"
+            },
+            "maxItems": 5
+          },
+          "caveats": {},
+          "correctionOrCounterclaim": {}
+        }
+      }
+    }
+  }
+}

package/workflows/deep-research/schemas/deep-research-verify-claims-control.schema.json

@@ -8,6 +8,7 @@
     "verdictDigest",
     "evidence"
   ],
+  "additionalProperties": false,
   "properties": {
     "schema": {
       "type": "string",
@@ -17,13 +18,17 @@
       "type": "string",
       "minLength": 1
     },
-    "id": {},
+    "id": {
+      "type": "string",
+      "minLength": 1
+    },
     "claim": {},
     "factSlotIds": {},
     "status": {
       "type": "string",
-      "enum": ["verified", "partially_supported", "unsupported", "conflicting"]
+      "enum": ["verified", "partially_supported", "unsupported", "conflicting", "verification_blocked"]
     },
+    "confidence": {},
     "verdictDigest": {},
     "evidence": {
       "type": "array",
@@ -31,6 +36,8 @@
         "type": "object"
       },
       "maxItems": 5
-    }
+    },
+    "caveats": {},
+    "correctionOrCounterclaim": {}
   }
 }