@mcoda/codali 0.1.88 → 0.1.89

package/dist/eval/GatewayEvalSuite.d.ts

@@ -0,0 +1,202 @@
+import type { PercentileMetric, RateMetric } from "./MetricTypes.js";
+import type { CodaliGatewayModelTier } from "../gateway/CodaliGatewayTypes.js";
+export type CodaliGatewayEvalTaskType = "generic_question" | "code_repo_question" | "encrypted_docdex_search_question" | "product_tool_question" | "disabled_integration_question" | "image_generation_question" | "missing_evidence_question";
+export type CodaliGatewayEvalRunStatus = "passed" | "failed";
+export interface CodaliGatewayEvalCaseExpectations {
+    allowedTools?: string[];
+    deniedTools?: string[];
+    requiredTools?: string[];
+    requiredSourceTypes?: string[];
+    requiresEvidence?: boolean;
+    expectsMissingEvidence?: boolean;
+    requiresImageArtifact?: boolean;
+    requiresFinalLargeModel?: boolean;
+    minEvidencePrecision?: number;
+    maxLatencyMs?: number;
+    maxTokens?: number;
+    maxToolCalls?: number;
+    maxModelCalls?: number;
+    maxCostUsd?: number;
+}
+export interface CodaliGatewayEvalCase {
+    id: string;
+    type: CodaliGatewayEvalTaskType;
+    prompt: string;
+    expectations: CodaliGatewayEvalCaseExpectations;
+}
+export interface CodaliGatewayEvalEvidenceRecord {
+    id: string;
+    sourceType: string;
+    sourceId?: string;
+    cited?: boolean;
+    relevant?: boolean;
+}
+export interface CodaliGatewayEvalCitationRecord {
+    evidenceId?: string;
+    sourceType?: string;
+    sourceId?: string;
+}
+export interface CodaliGatewayEvalImageArtifactRecord {
+    id?: string;
+    uri?: string;
+    mimeType?: string;
+    metadata?: Record<string, unknown>;
+}
+export interface CodaliGatewayEvalRunRecord {
+    caseId: string;
+    taskType: CodaliGatewayEvalTaskType;
+    status: CodaliGatewayEvalRunStatus;
+    plannerSchemaValid: boolean;
+    selectedTaskType?: CodaliGatewayEvalTaskType;
+    calledTools: string[];
+    evidence: CodaliGatewayEvalEvidenceRecord[];
+    citations: CodaliGatewayEvalCitationRecord[];
+    finalAnswer: string;
+    finalAnswerDirect?: boolean;
+    finalModelTier?: CodaliGatewayModelTier;
+    latencyMs?: number;
+    tokensUsed?: number;
+    costUsd?: number;
+    toolCallCount?: number;
+    modelCallCount?: number;
+    imageArtifact?: CodaliGatewayEvalImageArtifactRecord;
+    missingEvidenceHandled?: boolean;
+    warnings?: string[];
+    errors?: string[];
+    metadata?: Record<string, unknown>;
+}
+export interface CodaliGatewayEvalCaseResult {
+    caseId: string;
+    taskType: CodaliGatewayEvalTaskType;
+    status: CodaliGatewayEvalRunStatus;
+    passed: boolean;
+    plannerSchemaValid: boolean;
+    evidencePrecision: number | null;
+    evidencePrecisionPassed: boolean;
+    citationSourceCorrect: boolean;
+    disabledToolLeakageDetected: boolean;
+    finalAnswerDirect: boolean;
+    finalLargeModelUsed: boolean;
+    budgetCompliant: boolean;
+    imageArtifactPresent: boolean;
+    missingEvidenceHandled: boolean;
+    calledTools: string[];
+    failures: string[];
+    warnings: string[];
+    errors: string[];
+    latencyMs?: number;
+    tokensUsed?: number;
+    costUsd?: number;
+    toolCallCount?: number;
+    modelCallCount?: number;
+}
+export interface CodaliGatewayEvalMetrics {
+    schemaVersion: 1;
+    generatedAt: string;
+    taskCount: number;
+    plannerSchemaValidityRate: RateMetric;
+    evidencePrecisionRate: RateMetric;
+    citationSourceCorrectnessRate: RateMetric;
+    disabledToolLeakageRate: RateMetric;
+    finalAnswerDirectnessRate: RateMetric;
+    finalLargeModelRate: RateMetric;
+    budgetComplianceRate: RateMetric;
+    latencyMs: PercentileMetric;
+    tokensUsed: PercentileMetric;
+    costUsd: PercentileMetric;
+    toolCallCount: PercentileMetric;
+    modelCallCount: PercentileMetric;
+}
+export interface CodaliGatewayEvalThresholds {
+    plannerSchemaValidityMin: number;
+    evidencePrecisionMin: number;
+    citationSourceCorrectnessMin: number;
+    disabledToolLeakageMax: number;
+    finalAnswerDirectnessMin: number;
+    finalLargeModelRateMin: number;
+    budgetComplianceMin: number;
+    latencyP95MsMax?: number;
+    costP95UsdMax?: number;
+    latencyRegressionRatioMax: number;
+    costRegressionRatioMax: number;
+}
+export interface CodaliGatewayEvalMetricDelta {
+    key: "latency_ms.p95" | "cost_usd.p95" | "tokens_used.p95";
+    unit: "ms" | "usd" | "tokens";
+    baseline: number | null;
+    current: number | null;
+    delta: number | null;
+    relativeDelta: number | null;
+    regression: boolean;
+}
+export interface CodaliGatewayEvalRegressionComparison {
+    schemaVersion: 1;
+    status: "baseline_missing" | "compared";
+    baselineReportId?: string;
+    baselineGeneratedAt?: string;
+    deltas: CodaliGatewayEvalMetricDelta[];
+    regressionCount: number;
+}
+export interface CodaliGatewayEvalGateFailure {
+    code: string;
+    metric: string;
+    message: string;
+    threshold: number;
+    actual: number | null;
+    baseline?: number | null;
+    delta?: number | null;
+}
+export interface CodaliGatewayEvalGateResult {
+    schemaVersion: 1;
+    passed: boolean;
+    thresholds: CodaliGatewayEvalThresholds;
+    failures: CodaliGatewayEvalGateFailure[];
+}
+export interface CodaliGatewayEvalReport {
+    schemaVersion: 1;
+    reportId: string;
+    runId: string;
+    runtime: "codali_gateway_eval";
+    mode: "gateway_smoke";
+    startedAt: string;
+    endedAt: string;
+    durationMs: number;
+    cases: CodaliGatewayEvalCaseResult[];
+    metrics: CodaliGatewayEvalMetrics;
+    regression: CodaliGatewayEvalRegressionComparison;
+    gates: CodaliGatewayEvalGateResult;
+    summary: {
+        status: CodaliGatewayEvalRunStatus;
+        total: number;
+        passed: number;
+        failed: number;
+    };
+    warnings: string[];
+    errors: string[];
+}
+export type CodaliGatewayEvalRunner = (evalCase: CodaliGatewayEvalCase) => Promise<CodaliGatewayEvalRunRecord>;
+export interface CodaliGatewayEvalSuiteOptions {
+    runId?: string;
+    reportId?: string;
+    cases?: CodaliGatewayEvalCase[];
+    runner?: CodaliGatewayEvalRunner;
+    thresholds?: Partial<CodaliGatewayEvalThresholds>;
+    baseline?: CodaliGatewayEvalReport | CodaliGatewayEvalMetrics;
+}
+export declare const DEFAULT_CODALI_GATEWAY_EVAL_THRESHOLDS: CodaliGatewayEvalThresholds;
+export declare const CODALI_GATEWAY_EVAL_CASES: CodaliGatewayEvalCase[];
+export declare const evaluateCodaliGatewayEvalCase: (evalCase: CodaliGatewayEvalCase, record: CodaliGatewayEvalRunRecord, thresholds?: CodaliGatewayEvalThresholds) => CodaliGatewayEvalCaseResult;
+export declare const aggregateCodaliGatewayEvalMetrics: (cases: CodaliGatewayEvalCaseResult[]) => CodaliGatewayEvalMetrics;
+export declare const compareCodaliGatewayEvalBaseline: (params: {
+    current: CodaliGatewayEvalMetrics;
+    baseline?: CodaliGatewayEvalReport | CodaliGatewayEvalMetrics;
+}) => CodaliGatewayEvalRegressionComparison;
+export declare const evaluateCodaliGatewayEvalGates: (params: {
+    metrics: CodaliGatewayEvalMetrics;
+    thresholds?: Partial<CodaliGatewayEvalThresholds>;
+    regression?: CodaliGatewayEvalRegressionComparison;
+}) => CodaliGatewayEvalGateResult;
+export declare const createDefaultCodaliGatewayEvalRunner: () => CodaliGatewayEvalRunner;
+export declare const runCodaliGatewayEvalSuite: (options?: CodaliGatewayEvalSuiteOptions) => Promise<CodaliGatewayEvalReport>;
+export declare const formatCodaliGatewayEvalTextReport: (report: CodaliGatewayEvalReport) => string;
+//# sourceMappingURL=GatewayEvalSuite.d.ts.map

package/dist/eval/GatewayEvalSuite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GatewayEvalSuite.d.ts","sourceRoot":"","sources":["../../src/eval/GatewayEvalSuite.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAC;AAE/E,MAAM,MAAM,yBAAyB,GACjC,kBAAkB,GAClB,oBAAoB,GACpB,kCAAkC,GAClC,uBAAuB,GACvB,+BAA+B,GAC/B,2BAA2B,GAC3B,2BAA2B,CAAC;AAEhC,MAAM,MAAM,0BAA0B,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE7D,MAAM,WAAW,iCAAiC;IAChD,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,yBAAyB,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,iCAAiC,CAAC;CACjD;AAED,MAAM,WAAW,+BAA+B;IAC9C,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,+BAA+B;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oCAAoC;IACnD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,yBAAyB,CAAC;IACpC,MAAM,EAAE,0BAA0B,CAAC;IACnC,kBAAkB,EAAE,OAAO,CAAC;IAC5B,gBAAgB,CAAC,EAAE,yBAAyB,CAAC;IAC7C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,+BAA+B,EAAE,CAAC;IAC5C,SAAS,EAAE,+BAA+B,EAAE,CAAC;IAC7C,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,cAAc,CAAC,EAAE,sBAAsB,CAAC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,oCAAoC,CAAC;IACrD,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,yBAAyB,CAAC;IACpC,MAAM,EAAE,0BAA0B,CAAC;IACnC,MAAM,EAAE,OAAO,CAAC;IAChB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,uBAAuB,EAAE,OAAO,CAAC;IACjC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,2BAA2B,EAAE,OAAO,CAAC;IACrC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,mBAAmB,EAAE,OAAO,CAAC;IAC7B,eAAe,EAAE,OAAO,CAAC;IACzB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,sBAAsB,EAAE,OAAO,CAAC;IAChC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,CAAC,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,yBAAyB,EAAE,UAAU,CAAC;IACtC,qBAAqB,EAAE,UAAU,CAAC;IAClC,6BAA6B,EAAE,UAAU,CAAC;IAC1C,uBAAuB,EAAE,UAAU,CAAC;IACpC,yBAAyB,EAAE,UAAU,CAAC;IACtC,mBAAmB,EAAE,UAAU,CAAC;IAChC,oBAAoB,EAAE,UAAU,CAAC;IACjC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,UAAU,EAAE,gBAAgB,CAAC;IAC7B,OAAO,EAAE,gBAAgB,CAAC;IAC1B,aAAa,EAAE,gBAAgB,CAAC;IAChC,cAAc,EAAE,gBAAgB,CAAC;CAClC;AAED,MAAM,WAAW,2BAA2B;IAC1C,wBAAwB,EAAE,MAAM,CAAC;IACjC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,4BAA4B,EAAE,MAAM,CAAC;IACrC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,wBAAwB,EAAE,MAAM,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,yBAAyB,EAAE,MAAM,CAAC;IAClC,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,gBAAgB,GAAG,cAAc,GAAG,iBAAiB,CAAC;IAC3D,IAAI,EAAE,IAAI,GAAG,KAAK,GAAG,QAAQ,CAAC;IAC9B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,qCAAqC;IACpD,aAAa,EAAE,CAAC,CAAC;IACjB,MAAM,EAAE,kBAAkB,GAAG,UAAU,CAAC;IACxC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,4BAA4B,EAAE,CAAC;IACvC,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,2BAA2B;IAC1C,aAAa,EAAE,CAAC,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,2BAA2B,CAAC;IACxC,QAAQ,EAAE,4BAA4B,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,CAAC,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,qBAAqB,CAAC;IAC/B,IAAI,EAAE,eAAe,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,2BAA2B,EAAE,CAAC;IACrC,OAAO,EAAE,wBAAwB,CAAC;IAClC,UAAU,EAAE,qCAAqC,CAAC;IAClD,KAAK,EAAE,2BAA2B,CAAC;IACnC,OAAO,EAAE;QACP,MAAM,EAAE,0BAA0B,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,MAAM,uBAAuB,GAAG,CACpC,QAAQ,EAAE,qBAAqB,KAC5B,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAEzC,MAAM,WAAW,6BAA6B;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,qBAAqB,EAAE,CAAC;IAChC,MAAM,CAAC,EAAE,uBAAuB,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAClD,QAAQ,CAAC,EAAE,uBAAuB,GAAG,wBAAwB,CAAC;CAC/D;AAED,eAAO,MAAM,sCAAsC,EAAE,2BAYpD,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,qBAAqB,EAiI5D,CAAC;AAmHF,eAAO,MAAM,6BAA6B,GACxC,UAAU,qBAAqB,EAC/B,QAAQ,0BAA0B,EAClC,aAAY,2BAAoE,KAC/E,2BAiEF,CAAC;AAEF,eAAO,MAAM,iCAAiC,GAC5C,OAAO,2BAA2B,EAAE,KACnC,wBAgDF,CAAC;AAkDF,eAAO,MAAM,gCAAgC,GAAI,QAAQ;IACvD,OAAO,EAAE,wBAAwB,CAAC;IAClC,QAAQ,CAAC,EAAE,uBAAuB,GAAG,wBAAwB,CAAC;CAC/D,KAAG,qCA8BH,CAAC;AAgDF,eAAO,MAAM,8BAA8B,GAAI,QAAQ;IACrD,OAAO,EAAE,wBAAwB,CAAC;IAClC,UAAU,CAAC,EAAE,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAClD,UAAU,CAAC,EAAE,qCAAqC,CAAC;CACpD,KAAG,2BAoHH,CAAC;AAgBF,eAAO,MAAM,oCAAoC,QAAO,uBAuCvD,CAAC;AAEF,eAAO,MAAM,yBAAyB,GACpC,UAAS,6BAAkC,KAC1C,OAAO,CAAC,uBAAuB,CAmFjC,CAAC;AAUF,eAAO,MAAM,iCAAiC,GAC5C,QAAQ,uBAAuB,KAC9B,MA0BF,CAAC"}