@kontourai/flow-agents 3.12.0 → 4.0.0

package/dist/pi/build/src/narrative/eval-result.js

@@ -0,0 +1,158 @@
+/**
+ * Consolidated narrative eval-result schema + ajv-free validator (#612 / D2).
+ *
+ * The grounded-narrative eval suite (evals/fixtures/narrative-evals/scorer.mjs)
+ * emits ONE `narrative-eval-result/v1` object per run: per-fixture verdicts,
+ * aggregate faithfulness metrics with an explicit uncertainty block (results are
+ * per-corpus MEASUREMENTS, not proofs — R7), the DECLARED cross-runtime
+ * capability-parity block (#620, asserted not discovered — R7/AC3), documented
+ * known_gaps (contradiction detection, #568/#425), and per-fixture raw source
+ * links for audit (AC5). The schema is versioned + self-describing so kontourai/
+ * evals#95 can ingest it unmodified (R6/AC6).
+ *
+ * Validation mirrors the repo's established ajv-free recursive walker (see
+ * envelope.ts / projection.ts / snapshot.ts) paired with a draft-07
+ * schemas/narrative-eval-result.schema.json. No new dependency; PURE aside from
+ * reading the shipped schema file. Stays inside the #619 narrative import
+ * boundary (node builtins + ./-relative narrative files only).
+ *
+ * @module
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+export const NARRATIVE_EVAL_RESULT_SCHEMA_VERSION = "narrative-eval-result/v1";
+const dateTimeRe = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
+function resolveSchemaRef(ref, root) {
+    if (!ref.startsWith("#/$defs/"))
+        return undefined;
+    return root.$defs?.[ref.slice("#/$defs/".length)];
+}
+function schemaMatches(value, schema, root) {
+    const issues = [];
+    validateSchemaValue(value, schema, "$", issues, root);
+    return issues.length === 0;
+}
+function validateSchemaValue(value, schema, loc, issues, root = schema) {
+    if (schema.$ref) {
+        const resolved = resolveSchemaRef(schema.$ref, root);
+        if (!resolved)
+            issues.push({ path: loc, message: `unsupported schema ref ${schema.$ref}` });
+        else
+            validateSchemaValue(value, resolved, loc, issues, root);
+        return;
+    }
+    if (schema.anyOf && !schema.anyOf.some((sub) => schemaMatches(value, sub, root)))
+        issues.push({ path: loc, message: "must match at least one allowed schema" });
+    if (schema.oneOf) {
+        const matches = schema.oneOf.filter((sub) => schemaMatches(value, sub, root)).length;
+        if (matches !== 1)
+            issues.push({ path: loc, message: "must match exactly one allowed schema" });
+    }
+    if (schema.not && schemaMatches(value, schema.not, root))
+        issues.push({ path: loc, message: "must not match forbidden schema" });
+    for (const sub of schema.allOf ?? [])
+        validateSchemaValue(value, sub, loc, issues, root);
+    if (schema.const !== undefined && value !== schema.const)
+        issues.push({ path: loc, message: `must equal ${String(schema.const)}` });
+    if (schema.enum && !schema.enum.includes(value))
+        issues.push({ path: loc, message: `must be one of ${schema.enum.join(", ")}` });
+    if (schema.type === "string") {
+        if (typeof value !== "string") {
+            issues.push({ path: loc, message: "must be string" });
+            return;
+        }
+        if (typeof schema.minLength === "number" && value.length < schema.minLength)
+            issues.push({ path: loc, message: "must not be empty" });
+        if (schema.pattern && !(new RegExp(schema.pattern)).test(value))
+            issues.push({ path: loc, message: "has invalid format" });
+        if (schema.format === "date-time" && (!dateTimeRe.test(value) || Number.isNaN(Date.parse(value))))
+            issues.push({ path: loc, message: "must be date-time" });
+        return;
+    }
+    if (schema.type === "number") {
+        if (typeof value !== "number" || Number.isNaN(value)) {
+            issues.push({ path: loc, message: "must be number" });
+            return;
+        }
+        if (typeof schema.minimum === "number" && value < schema.minimum)
+            issues.push({ path: loc, message: `must be at least ${schema.minimum}` });
+        if (typeof schema.maximum === "number" && value > schema.maximum)
+            issues.push({ path: loc, message: `must be at most ${schema.maximum}` });
+        return;
+    }
+    if (schema.type === "integer") {
+        if (!Number.isInteger(value)) {
+            issues.push({ path: loc, message: "must be integer" });
+            return;
+        }
+        if (typeof schema.minimum === "number" && value < schema.minimum)
+            issues.push({ path: loc, message: `must be at least ${schema.minimum}` });
+        return;
+    }
+    if (schema.type === "boolean") {
+        if (typeof value !== "boolean")
+            issues.push({ path: loc, message: "must be boolean" });
+        return;
+    }
+    if (schema.type === "array") {
+        if (!Array.isArray(value)) {
+            issues.push({ path: loc, message: "must be array" });
+            return;
+        }
+        if (typeof schema.minItems === "number" && value.length < schema.minItems)
+            issues.push({ path: loc, message: `must contain at least ${schema.minItems} item(s)` });
+        if (typeof schema.maxItems === "number" && value.length > schema.maxItems)
+            issues.push({ path: loc, message: `must contain at most ${schema.maxItems} item(s)` });
+        if (schema.uniqueItems && new Set(value.map((item) => JSON.stringify(item))).size !== value.length)
+            issues.push({ path: loc, message: "must contain unique items" });
+        if (schema.items)
+            value.forEach((item, index) => validateSchemaValue(item, schema.items, `${loc}[${index}]`, issues, root));
+        return;
+    }
+    if (schema.type === "object" || schema.required || schema.properties) {
+        if (!value || typeof value !== "object" || Array.isArray(value)) {
+            issues.push({ path: loc, message: "must be object" });
+            return;
+        }
+        const obj = value;
+        for (const key of schema.required ?? [])
+            if (!(key in obj))
+                issues.push({ path: `${loc}.${key}`, message: "is required" });
+        const properties = schema.properties ?? {};
+        if (schema.additionalProperties === false) {
+            for (const key of Object.keys(obj))
+                if (!(key in properties))
+                    issues.push({ path: `${loc}.${key}`, message: "is not allowed" });
+        }
+        else if (schema.additionalProperties && typeof schema.additionalProperties === "object") {
+            for (const key of Object.keys(obj))
+                if (!(key in properties))
+                    validateSchemaValue(obj[key], schema.additionalProperties, `${loc}.${key}`, issues, root);
+        }
+        for (const [key, child] of Object.entries(properties))
+            if (key in obj)
+                validateSchemaValue(obj[key], child, `${loc}.${key}`, issues, root);
+    }
+}
+function schemaPath() {
+    return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../schemas/narrative-eval-result.schema.json");
+}
+/**
+ * Validate a value against the shipped `narrative-eval-result/v1` schema. Returns
+ * an empty array for a conforming value; one `{path, message}` issue per defect
+ * otherwise (missing required field, wrong `schema_version`, extra property, an
+ * out-of-range rate, and so on).
+ */
+export function validateNarrativeEvalResult(value) {
+    let schema;
+    try {
+        schema = JSON.parse(fs.readFileSync(schemaPath(), "utf8"));
+    }
+    catch {
+        return [{ path: "$", message: "narrative eval result schema is unavailable" }];
+    }
+    const issues = [];
+    validateSchemaValue(value, schema, "$", issues, schema);
+    return issues;
+}

package/dist/pi/build/src/narrative/grounding-validator.d.ts

@@ -0,0 +1,104 @@
+import type { GroundedExecutionNarrative } from "./envelope.js";
+import type { UnavailableReason } from "./integrity.js";
+import { type ResolveSourceResult } from "./resolver.js";
+export type MaterialEventKind = "command_failure" | "retry_group" | "timeout" | "no_op_turn" | "file_creation";
+export type AssertionKind = "gate_status" | "observed_outcome" | "authority" | "hidden_alternative";
+export type GroundingViolation = {
+    code: "unresolved_citation";
+    statement_id: string;
+    source_ref: string;
+    reason: UnavailableReason;
+    detail: string;
+} | {
+    code: "uncovered_material_event";
+    event_kind: MaterialEventKind;
+    source_ref: string;
+    detail: string;
+} | {
+    code: "invalid_rule_binding";
+    statement_id: string;
+    detail: string;
+} | {
+    code: "prohibited_assertion";
+    statement_id: string;
+    statement_class: string;
+    assertion_kind: AssertionKind;
+    detail: string;
+} | {
+    code: "unsupported_summary";
+    statement_id: string;
+    unsupported_refs: string[];
+    detail: string;
+};
+export type GroundingViolationCode = GroundingViolation["code"];
+export interface GroundingKnownGap {
+    code: "contradiction_detection_unavailable";
+    detail: string;
+}
+export type GroundingVerdict = {
+    ok: true;
+    known_gaps: GroundingKnownGap[];
+} | {
+    ok: false;
+    violations: GroundingViolation[];
+    known_gaps: GroundingKnownGap[];
+};
+export interface EntailmentIdentity {
+    model: string;
+    provider: string;
+    config_hash: string;
+}
+export interface EntailmentVerdict {
+    statement_id: string;
+    entailed: boolean;
+    reason: string;
+}
+/**
+ * Declared provenance contract for the future model-entailment executor. When
+ * #614 ships that executor, this record will be anchored as a manifest source
+ * stream so its bytes are content-addressed and resolvable like other evidence.
+ */
+export interface EntailmentProvenance extends EntailmentIdentity {
+    prompt_version: string;
+    input_tokens: number;
+    output_tokens: number;
+    estimated_cost_usd: number;
+    wall_clock_ms: number;
+    verdicts: EntailmentVerdict[];
+    /** sha256 of the exact frozen input (statements + policy-filtered views) handed to the generator. */
+    source_hash: string;
+    /** sha256 of the exact generator output bytes (pre- or post-rejection), for tamper-evident provenance. */
+    output_hash: string;
+}
+export declare function entailmentIndependenceHolds(generator: EntailmentIdentity, validator: EntailmentIdentity): boolean;
+/** Generic (statement class, assertion kind) policy seam for current and future classes. */
+export declare function isAssertionProhibited(statementClass: string, assertionKind: AssertionKind): boolean;
+export interface ValidateNarrativeGroundingOptions {
+    /** Test seam; production always uses the frozen-manifest resolver. */
+    resolver?: (narrativeDir: string, sourceId: string) => ResolveSourceResult;
+}
+export declare class NarrativeGroundingError extends Error {
+    readonly violations: GroundingViolation[];
+    readonly name = "NarrativeGroundingError";
+    readonly code: "grounding_failed";
+    constructor(violations: GroundingViolation[]);
+}
+export interface MaterialEventClassCoverage {
+    total: number;
+    uncovered: number;
+}
+export interface MaterialEventCoverage {
+    total: number;
+    covered: number;
+    by_class: Partial<Record<MaterialEventKind, MaterialEventClassCoverage>>;
+}
+/**
+ * #612: the SAME canonical material-event extraction that
+ * validateNarrativeGrounding uses to derive `uncovered_material_event`
+ * violations, exposed as a coverage summary so the eval layer measures
+ * faithfulness against the real extraction (never a parallel one that could
+ * drift). `covered = total - sum(uncovered)`; `by_class` gives per-kind
+ * total/uncovered so the eval can report an honest omission rate per class.
+ */
+export declare function materialEventCoverage(envelope: GroundedExecutionNarrative, narrativeDir: string, opts?: ValidateNarrativeGroundingOptions): MaterialEventCoverage;
+export declare function validateNarrativeGrounding(envelope: GroundedExecutionNarrative, narrativeDir: string, opts?: ValidateNarrativeGroundingOptions): GroundingVerdict;