@schift-io/knowledge-scope 0.1.0

package/dist/types/evaluation/contracts.d.ts

@@ -0,0 +1,718 @@
+import { z } from "zod";
+export declare const retrievalDatasetSchema: z.ZodObject<{
+    datasetId: z.ZodString;
+    provenance: z.ZodEnum<["synthetic", "customer_provided"]>;
+    packDigest: z.ZodString;
+    sourceSnapshotDigest: z.ZodString;
+    questions: z.ZodEffects<z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodObject<{
+        questionId: z.ZodString;
+        inputFingerprint: z.ZodString;
+        expectedDecision: z.ZodEnum<["ready", "insufficient_evidence"]>;
+        expectedEvidence: z.ZodEffects<z.ZodArray<z.ZodObject<{
+            evidenceId: z.ZodString;
+            citation: z.ZodString;
+        }, "strict", z.ZodTypeAny, {
+            citation: string;
+            evidenceId: string;
+        }, {
+            citation: string;
+            evidenceId: string;
+        }>, "many">, {
+            citation: string;
+            evidenceId: string;
+        }[], {
+            citation: string;
+            evidenceId: string;
+        }[]>;
+        forbiddenEvidenceIds: z.ZodEffects<z.ZodArray<z.ZodString, "many">, string[], string[]>;
+    }, "strict", z.ZodTypeAny, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }>, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }>, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }>, "many">, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }[], {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }[]>;
+}, "strict", z.ZodTypeAny, {
+    datasetId: string;
+    provenance: "synthetic" | "customer_provided";
+    packDigest: string;
+    sourceSnapshotDigest: string;
+    questions: {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }[];
+}, {
+    datasetId: string;
+    provenance: "synthetic" | "customer_provided";
+    packDigest: string;
+    sourceSnapshotDigest: string;
+    questions: {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidence: {
+            citation: string;
+            evidenceId: string;
+        }[];
+        forbiddenEvidenceIds: string[];
+    }[];
+}>;
+export declare const retrievalCaptureSchema: z.ZodObject<{
+    adapter: z.ZodString;
+    datasetFingerprint: z.ZodString;
+    packDigest: z.ZodString;
+    sourceSnapshotDigest: z.ZodString;
+    results: z.ZodEffects<z.ZodArray<z.ZodObject<{
+        questionId: z.ZodString;
+        inputFingerprint: z.ZodString;
+        decision: z.ZodEnum<["ready", "insufficient_evidence"]>;
+        items: z.ZodEffects<z.ZodArray<z.ZodObject<{
+            evidenceId: z.ZodString;
+        } & {
+            citation: z.ZodNullable<z.ZodString>;
+        }, "strict", z.ZodTypeAny, {
+            citation: string | null;
+            evidenceId: string;
+        }, {
+            citation: string | null;
+            evidenceId: string;
+        }>, "many">, {
+            citation: string | null;
+            evidenceId: string;
+        }[], {
+            citation: string | null;
+            evidenceId: string;
+        }[]>;
+    }, "strict", z.ZodTypeAny, {
+        items: {
+            citation: string | null;
+            evidenceId: string;
+        }[];
+        questionId: string;
+        inputFingerprint: string;
+        decision: "ready" | "insufficient_evidence";
+    }, {
+        items: {
+            citation: string | null;
+            evidenceId: string;
+        }[];
+        questionId: string;
+        inputFingerprint: string;
+        decision: "ready" | "insufficient_evidence";
+    }>, "many">, {
+        items: {
+            citation: string | null;
+            evidenceId: string;
+        }[];
+        questionId: string;
+        inputFingerprint: string;
+        decision: "ready" | "insufficient_evidence";
+    }[], {
+        items: {
+            citation: string | null;
+            evidenceId: string;
+        }[];
+        questionId: string;
+        inputFingerprint: string;
+        decision: "ready" | "insufficient_evidence";
+    }[]>;
+}, "strict", z.ZodTypeAny, {
+    results: {
+        items: {
+            citation: string | null;
+            evidenceId: string;
+        }[];
+        questionId: string;
+        inputFingerprint: string;
+        decision: "ready" | "insufficient_evidence";
+    }[];
+    packDigest: string;
+    sourceSnapshotDigest: string;
+    adapter: string;
+    datasetFingerprint: string;
+}, {
+    results: {
+        items: {
+            citation: string | null;
+            evidenceId: string;
+        }[];
+        questionId: string;
+        inputFingerprint: string;
+        decision: "ready" | "insufficient_evidence";
+    }[];
+    packDigest: string;
+    sourceSnapshotDigest: string;
+    adapter: string;
+    datasetFingerprint: string;
+}>;
+export declare const retrievalEvaluationSchema: z.ZodObject<{
+    dataset: z.ZodObject<{
+        datasetId: z.ZodString;
+        provenance: z.ZodEnum<["synthetic", "customer_provided"]>;
+        packDigest: z.ZodString;
+        sourceSnapshotDigest: z.ZodString;
+        questions: z.ZodEffects<z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodObject<{
+            questionId: z.ZodString;
+            inputFingerprint: z.ZodString;
+            expectedDecision: z.ZodEnum<["ready", "insufficient_evidence"]>;
+            expectedEvidence: z.ZodEffects<z.ZodArray<z.ZodObject<{
+                evidenceId: z.ZodString;
+                citation: z.ZodString;
+            }, "strict", z.ZodTypeAny, {
+                citation: string;
+                evidenceId: string;
+            }, {
+                citation: string;
+                evidenceId: string;
+            }>, "many">, {
+                citation: string;
+                evidenceId: string;
+            }[], {
+                citation: string;
+                evidenceId: string;
+            }[]>;
+            forbiddenEvidenceIds: z.ZodEffects<z.ZodArray<z.ZodString, "many">, string[], string[]>;
+        }, "strict", z.ZodTypeAny, {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }, {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }>, {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }, {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }>, {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }, {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }>, "many">, {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }[], {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }[]>;
+    }, "strict", z.ZodTypeAny, {
+        datasetId: string;
+        provenance: "synthetic" | "customer_provided";
+        packDigest: string;
+        sourceSnapshotDigest: string;
+        questions: {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }[];
+    }, {
+        datasetId: string;
+        provenance: "synthetic" | "customer_provided";
+        packDigest: string;
+        sourceSnapshotDigest: string;
+        questions: {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }[];
+    }>;
+    capture: z.ZodObject<{
+        adapter: z.ZodString;
+        datasetFingerprint: z.ZodString;
+        packDigest: z.ZodString;
+        sourceSnapshotDigest: z.ZodString;
+        results: z.ZodEffects<z.ZodArray<z.ZodObject<{
+            questionId: z.ZodString;
+            inputFingerprint: z.ZodString;
+            decision: z.ZodEnum<["ready", "insufficient_evidence"]>;
+            items: z.ZodEffects<z.ZodArray<z.ZodObject<{
+                evidenceId: z.ZodString;
+            } & {
+                citation: z.ZodNullable<z.ZodString>;
+            }, "strict", z.ZodTypeAny, {
+                citation: string | null;
+                evidenceId: string;
+            }, {
+                citation: string | null;
+                evidenceId: string;
+            }>, "many">, {
+                citation: string | null;
+                evidenceId: string;
+            }[], {
+                citation: string | null;
+                evidenceId: string;
+            }[]>;
+        }, "strict", z.ZodTypeAny, {
+            items: {
+                citation: string | null;
+                evidenceId: string;
+            }[];
+            questionId: string;
+            inputFingerprint: string;
+            decision: "ready" | "insufficient_evidence";
+        }, {
+            items: {
+                citation: string | null;
+                evidenceId: string;
+            }[];
+            questionId: string;
+            inputFingerprint: string;
+            decision: "ready" | "insufficient_evidence";
+        }>, "many">, {
+            items: {
+                citation: string | null;
+                evidenceId: string;
+            }[];
+            questionId: string;
+            inputFingerprint: string;
+            decision: "ready" | "insufficient_evidence";
+        }[], {
+            items: {
+                citation: string | null;
+                evidenceId: string;
+            }[];
+            questionId: string;
+            inputFingerprint: string;
+            decision: "ready" | "insufficient_evidence";
+        }[]>;
+    }, "strict", z.ZodTypeAny, {
+        results: {
+            items: {
+                citation: string | null;
+                evidenceId: string;
+            }[];
+            questionId: string;
+            inputFingerprint: string;
+            decision: "ready" | "insufficient_evidence";
+        }[];
+        packDigest: string;
+        sourceSnapshotDigest: string;
+        adapter: string;
+        datasetFingerprint: string;
+    }, {
+        results: {
+            items: {
+                citation: string | null;
+                evidenceId: string;
+            }[];
+            questionId: string;
+            inputFingerprint: string;
+            decision: "ready" | "insufficient_evidence";
+        }[];
+        packDigest: string;
+        sourceSnapshotDigest: string;
+        adapter: string;
+        datasetFingerprint: string;
+    }>;
+    topK: z.ZodNumber;
+}, "strict", z.ZodTypeAny, {
+    dataset: {
+        datasetId: string;
+        provenance: "synthetic" | "customer_provided";
+        packDigest: string;
+        sourceSnapshotDigest: string;
+        questions: {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }[];
+    };
+    capture: {
+        results: {
+            items: {
+                citation: string | null;
+                evidenceId: string;
+            }[];
+            questionId: string;
+            inputFingerprint: string;
+            decision: "ready" | "insufficient_evidence";
+        }[];
+        packDigest: string;
+        sourceSnapshotDigest: string;
+        adapter: string;
+        datasetFingerprint: string;
+    };
+    topK: number;
+}, {
+    dataset: {
+        datasetId: string;
+        provenance: "synthetic" | "customer_provided";
+        packDigest: string;
+        sourceSnapshotDigest: string;
+        questions: {
+            questionId: string;
+            inputFingerprint: string;
+            expectedDecision: "ready" | "insufficient_evidence";
+            expectedEvidence: {
+                citation: string;
+                evidenceId: string;
+            }[];
+            forbiddenEvidenceIds: string[];
+        }[];
+    };
+    capture: {
+        results: {
+            items: {
+                citation: string | null;
+                evidenceId: string;
+            }[];
+            questionId: string;
+            inputFingerprint: string;
+            decision: "ready" | "insufficient_evidence";
+        }[];
+        packDigest: string;
+        sourceSnapshotDigest: string;
+        adapter: string;
+        datasetFingerprint: string;
+    };
+    topK: number;
+}>;
+export declare const evaluationQuestionReportSchema: z.ZodEffects<z.ZodObject<{
+    questionId: z.ZodString;
+    inputFingerprint: z.ZodString;
+    expectedEvidenceCount: z.ZodNumber;
+    retrievedAtKCount: z.ZodNumber;
+    matchedAtKCount: z.ZodNumber;
+    citationMatchCount: z.ZodNumber;
+    forbiddenEvidenceCount: z.ZodNumber;
+    expectedDecision: z.ZodEnum<["ready", "insufficient_evidence"]>;
+    actualDecision: z.ZodEnum<["ready", "insufficient_evidence"]>;
+}, "strict", z.ZodTypeAny, {
+    questionId: string;
+    inputFingerprint: string;
+    expectedDecision: "ready" | "insufficient_evidence";
+    expectedEvidenceCount: number;
+    retrievedAtKCount: number;
+    matchedAtKCount: number;
+    citationMatchCount: number;
+    forbiddenEvidenceCount: number;
+    actualDecision: "ready" | "insufficient_evidence";
+}, {
+    questionId: string;
+    inputFingerprint: string;
+    expectedDecision: "ready" | "insufficient_evidence";
+    expectedEvidenceCount: number;
+    retrievedAtKCount: number;
+    matchedAtKCount: number;
+    citationMatchCount: number;
+    forbiddenEvidenceCount: number;
+    actualDecision: "ready" | "insufficient_evidence";
+}>, {
+    questionId: string;
+    inputFingerprint: string;
+    expectedDecision: "ready" | "insufficient_evidence";
+    expectedEvidenceCount: number;
+    retrievedAtKCount: number;
+    matchedAtKCount: number;
+    citationMatchCount: number;
+    forbiddenEvidenceCount: number;
+    actualDecision: "ready" | "insufficient_evidence";
+}, {
+    questionId: string;
+    inputFingerprint: string;
+    expectedDecision: "ready" | "insufficient_evidence";
+    expectedEvidenceCount: number;
+    retrievedAtKCount: number;
+    matchedAtKCount: number;
+    citationMatchCount: number;
+    forbiddenEvidenceCount: number;
+    actualDecision: "ready" | "insufficient_evidence";
+}>;
+export declare const retrievalReportSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"schift.retrieval-evaluation.v1">;
+    datasetId: z.ZodString;
+    provenance: z.ZodEnum<["synthetic", "customer_provided"]>;
+    adapter: z.ZodString;
+    datasetFingerprint: z.ZodString;
+    inputFingerprint: z.ZodString;
+    packDigest: z.ZodString;
+    sourceSnapshotDigest: z.ZodString;
+    topK: z.ZodNumber;
+    questions: z.ZodEffects<z.ZodArray<z.ZodEffects<z.ZodObject<{
+        questionId: z.ZodString;
+        inputFingerprint: z.ZodString;
+        expectedEvidenceCount: z.ZodNumber;
+        retrievedAtKCount: z.ZodNumber;
+        matchedAtKCount: z.ZodNumber;
+        citationMatchCount: z.ZodNumber;
+        forbiddenEvidenceCount: z.ZodNumber;
+        expectedDecision: z.ZodEnum<["ready", "insufficient_evidence"]>;
+        actualDecision: z.ZodEnum<["ready", "insufficient_evidence"]>;
+    }, "strict", z.ZodTypeAny, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidenceCount: number;
+        retrievedAtKCount: number;
+        matchedAtKCount: number;
+        citationMatchCount: number;
+        forbiddenEvidenceCount: number;
+        actualDecision: "ready" | "insufficient_evidence";
+    }, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidenceCount: number;
+        retrievedAtKCount: number;
+        matchedAtKCount: number;
+        citationMatchCount: number;
+        forbiddenEvidenceCount: number;
+        actualDecision: "ready" | "insufficient_evidence";
+    }>, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidenceCount: number;
+        retrievedAtKCount: number;
+        matchedAtKCount: number;
+        citationMatchCount: number;
+        forbiddenEvidenceCount: number;
+        actualDecision: "ready" | "insufficient_evidence";
+    }, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidenceCount: number;
+        retrievedAtKCount: number;
+        matchedAtKCount: number;
+        citationMatchCount: number;
+        forbiddenEvidenceCount: number;
+        actualDecision: "ready" | "insufficient_evidence";
+    }>, "many">, {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidenceCount: number;
+        retrievedAtKCount: number;
+        matchedAtKCount: number;
+        citationMatchCount: number;
+        forbiddenEvidenceCount: number;
+        actualDecision: "ready" | "insufficient_evidence";
+    }[], {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidenceCount: number;
+        retrievedAtKCount: number;
+        matchedAtKCount: number;
+        citationMatchCount: number;
+        forbiddenEvidenceCount: number;
+        actualDecision: "ready" | "insufficient_evidence";
+    }[]>;
+    metrics: z.ZodObject<{
+        recallAtK: z.ZodNullable<z.ZodNumber>;
+        citationMatchRate: z.ZodNullable<z.ZodNumber>;
+        forbiddenEvidenceCount: z.ZodNumber;
+        decisionMatchRate: z.ZodNumber;
+    }, "strict", z.ZodTypeAny, {
+        forbiddenEvidenceCount: number;
+        recallAtK: number | null;
+        citationMatchRate: number | null;
+        decisionMatchRate: number;
+    }, {
+        forbiddenEvidenceCount: number;
+        recallAtK: number | null;
+        citationMatchRate: number | null;
+        decisionMatchRate: number;
+    }>;
+}, "strict", z.ZodTypeAny, {
+    schemaVersion: "schift.retrieval-evaluation.v1";
+    inputFingerprint: string;
+    datasetId: string;
+    provenance: "synthetic" | "customer_provided";
+    packDigest: string;
+    sourceSnapshotDigest: string;
+    questions: {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidenceCount: number;
+        retrievedAtKCount: number;
+        matchedAtKCount: number;
+        citationMatchCount: number;
+        forbiddenEvidenceCount: number;
+        actualDecision: "ready" | "insufficient_evidence";
+    }[];
+    adapter: string;
+    datasetFingerprint: string;
+    topK: number;
+    metrics: {
+        forbiddenEvidenceCount: number;
+        recallAtK: number | null;
+        citationMatchRate: number | null;
+        decisionMatchRate: number;
+    };
+}, {
+    schemaVersion: "schift.retrieval-evaluation.v1";
+    inputFingerprint: string;
+    datasetId: string;
+    provenance: "synthetic" | "customer_provided";
+    packDigest: string;
+    sourceSnapshotDigest: string;
+    questions: {
+        questionId: string;
+        inputFingerprint: string;
+        expectedDecision: "ready" | "insufficient_evidence";
+        expectedEvidenceCount: number;
+        retrievedAtKCount: number;
+        matchedAtKCount: number;
+        citationMatchCount: number;
+        forbiddenEvidenceCount: number;
+        actualDecision: "ready" | "insufficient_evidence";
+    }[];
+    adapter: string;
+    datasetFingerprint: string;
+    topK: number;
+    metrics: {
+        forbiddenEvidenceCount: number;
+        recallAtK: number | null;
+        citationMatchRate: number | null;
+        decisionMatchRate: number;
+    };
+}>;
+export type RetrievalDataset = z.infer<typeof retrievalDatasetSchema>;
+export type RetrievalCapture = z.infer<typeof retrievalCaptureSchema>;
+export type RetrievalEvaluationReport = z.infer<typeof retrievalReportSchema>;
+export type EvaluationQuestionReport = z.infer<typeof evaluationQuestionReportSchema>;
+export declare class RetrievalEvaluationError extends Error {
+    readonly code: "snapshot_mismatch" | "question_mismatch" | "input_mismatch" | "comparison_mismatch" | "report_invalid";
+    readonly name = "RetrievalEvaluationError";
+    constructor(code: "snapshot_mismatch" | "question_mismatch" | "input_mismatch" | "comparison_mismatch" | "report_invalid");
+}