@mcoda/codali 0.1.88 → 0.1.89

package/dist/eval/CodaliGatewayLiveHarness.d.ts

@@ -0,0 +1,169 @@
+import { type AgentTierResolution, type CodaliGatewayAgentAssignment, type CodaliGatewayAgentCandidate, type CodaliGatewayAgentTierError } from "../gateway/AgentTierResolver.js";
+import type { CodaliAgentTierPolicy, CodaliGatewayModelTier } from "../gateway/CodaliGatewayTypes.js";
+export type CodaliGatewayLiveScenarioId = "generic_question" | "docdex_encrypted_repo_search" | "tool_disabled_leakage" | "multi_step_evidence" | "final_answer_large_model" | "image_generation";
+export type CodaliGatewayLiveRoleKey = "small_json" | "medium_planner" | "medium_verifier" | "large_final" | "image_worker";
+export type CodaliGatewayLiveScenarioStatus = "passed" | "failed" | "degraded" | "skipped";
+export type CodaliGatewayLiveHarnessStatus = "passed" | "failed" | "degraded";
+export interface CodaliGatewayLiveCommandResult {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+    signal?: string;
+    latencyMs: number;
+    timedOut?: boolean;
+}
+export type CodaliGatewayLiveCommandRunner = (command: string, args: string[], options: {
+    input?: string;
+    timeoutMs: number;
+    maxBuffer?: number;
+    env?: NodeJS.ProcessEnv;
+}) => Promise<CodaliGatewayLiveCommandResult>;
+export interface CodaliGatewayLiveScenarioArtifact {
+    id?: string;
+    kind?: string;
+    uri?: string;
+    mimeType?: string;
+    metadata?: Record<string, unknown>;
+}
+export interface CodaliGatewayLiveScenarioResult {
+    id: CodaliGatewayLiveScenarioId;
+    label: string;
+    status: CodaliGatewayLiveScenarioStatus;
+    role: CodaliGatewayLiveRoleKey;
+    agentSlug?: string;
+    tier?: CodaliGatewayModelTier;
+    model?: string;
+    adapter?: string;
+    latencyMs: number;
+    jsonValid?: boolean;
+    toolCallCount?: number;
+    calledTools?: string[];
+    finalAnswerStatus?: string;
+    finalModelTier?: CodaliGatewayModelTier;
+    finalModelAgentSlug?: string;
+    artifact?: CodaliGatewayLiveScenarioArtifact;
+    outputPreview?: string;
+    warnings: string[];
+    errors: string[];
+    metadata?: Record<string, unknown>;
+}
+export interface CodaliGatewayLiveScenarioDefinition {
+    id: CodaliGatewayLiveScenarioId;
+    label: string;
+    role: CodaliGatewayLiveRoleKey;
+    expectsJson?: boolean;
+    expectsArtifact?: boolean;
+    requiresGatewayToolTelemetry?: boolean;
+}
+export interface CodaliGatewayLiveScenarioRunnerInput {
+    runId: string;
+    scenario: CodaliGatewayLiveScenarioDefinition;
+    assignment?: CodaliGatewayAgentAssignment;
+    classification: CodaliGatewayLiveClassification;
+    commandRunner: CodaliGatewayLiveCommandRunner;
+    command: string;
+    timeoutMs: number;
+    forceAgentRun: boolean;
+}
+export type CodaliGatewayLiveScenarioRunner = (input: CodaliGatewayLiveScenarioRunnerInput) => Promise<CodaliGatewayLiveScenarioResult>;
+export interface CodaliGatewayLiveAgentSummary {
+    slug: string;
+    tier: CodaliGatewayModelTier;
+    source: CodaliGatewayAgentCandidate["source"];
+    healthStatus: CodaliGatewayAgentCandidate["healthStatus"];
+    latencyMs?: number;
+    model?: string;
+    adapter?: string;
+    supportsTools?: boolean;
+    supportsJsonSchema?: boolean;
+    supportsImageGeneration?: boolean;
+    contextWindow?: number;
+    costPerMillion?: number;
+    rating?: number;
+    reasoningRating?: number;
+}
+export interface CodaliGatewayLiveRoleSummary {
+    role: CodaliGatewayLiveRoleKey;
+    resolverRole: string;
+    agentSlug?: string;
+    tier?: CodaliGatewayModelTier;
+    model?: string;
+    score?: number;
+    reasons: string[];
+    status: "assigned" | "unavailable";
+    errorCodes: string[];
+}
+export interface CodaliGatewayLiveClassification {
+    resolution: AgentTierResolution;
+    assignments: Record<CodaliGatewayLiveRoleKey, CodaliGatewayAgentAssignment | undefined>;
+    agents: CodaliGatewayLiveAgentSummary[];
+    roles: Record<CodaliGatewayLiveRoleKey, CodaliGatewayLiveRoleSummary>;
+    warnings: CodaliGatewayAgentTierError[];
+    errors: CodaliGatewayAgentTierError[];
+}
+export interface CodaliGatewayLiveDiscoveryResult {
+    source: "provided" | "command";
+    command?: string;
+    args?: string[];
+    status: "succeeded" | "failed";
+    latencyMs: number;
+    inventoryCount: number;
+    errors: string[];
+}
+export interface CodaliGatewayLiveHarnessOptions {
+    runId?: string;
+    inventory?: unknown[];
+    inventoryCommand?: {
+        command?: string;
+        args?: string[];
+    };
+    command?: string;
+    commandRunner?: CodaliGatewayLiveCommandRunner;
+    scenarioRunner?: CodaliGatewayLiveScenarioRunner;
+    scenarios?: CodaliGatewayLiveScenarioId[];
+    timeoutMs?: number;
+    maxBuffer?: number;
+    forceAgentRun?: boolean;
+    allowCloudFallback?: boolean;
+    allowImageWorker?: boolean;
+    agentPolicy?: CodaliAgentTierPolicy;
+}
+export interface CodaliGatewayLiveHarnessResult {
+    schemaVersion: 1;
+    runId: string;
+    runtime: "codali_gateway_live_harness";
+    mode: "live";
+    startedAt: string;
+    endedAt: string;
+    durationMs: number;
+    discovery: CodaliGatewayLiveDiscoveryResult;
+    classification: CodaliGatewayLiveClassification;
+    scenarios: CodaliGatewayLiveScenarioResult[];
+    summary: {
+        status: CodaliGatewayLiveHarnessStatus;
+        passed: number;
+        failed: number;
+        degraded: number;
+        skipped: number;
+        jsonValidAgents: string[];
+        largeFinalSynthesizerOk: boolean;
+        imageArtifactOk: boolean;
+        missingRoles: CodaliGatewayLiveRoleKey[];
+    };
+    warnings: string[];
+    errors: string[];
+}
+export declare const CODALI_GATEWAY_LIVE_SCENARIOS: CodaliGatewayLiveScenarioDefinition[];
+export declare const defaultCodaliGatewayLiveCommandRunner: CodaliGatewayLiveCommandRunner;
+export declare const parseCodaliGatewayLiveInventory: (payload: unknown) => unknown[];
+export declare const redactCodaliGatewayLiveValue: (value: unknown) => unknown;
+export declare const classifyCodaliGatewayLiveAgents: (input: {
+    inventory: unknown[];
+    allowCloudFallback?: boolean;
+    allowImageWorker?: boolean;
+    agentPolicy?: CodaliAgentTierPolicy;
+}) => CodaliGatewayLiveClassification;
+export declare const createMcodaAgentRunScenarioRunner: () => CodaliGatewayLiveScenarioRunner;
+export declare const runCodaliGatewayLiveHarness: (options?: CodaliGatewayLiveHarnessOptions) => Promise<CodaliGatewayLiveHarnessResult>;
+export declare const formatCodaliGatewayLiveHarnessTextReport: (result: CodaliGatewayLiveHarnessResult) => string;
+//# sourceMappingURL=CodaliGatewayLiveHarness.d.ts.map

package/dist/eval/CodaliGatewayLiveHarness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CodaliGatewayLiveHarness.d.ts","sourceRoot":"","sources":["../../src/eval/CodaliGatewayLiveHarness.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EAChC,KAAK,2BAA2B,EACjC,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACvB,MAAM,kCAAkC,CAAC;AAE1C,MAAM,MAAM,2BAA2B,GACnC,kBAAkB,GAClB,8BAA8B,GAC9B,uBAAuB,GACvB,qBAAqB,GACrB,0BAA0B,GAC1B,kBAAkB,CAAC;AAEvB,MAAM,MAAM,wBAAwB,GAChC,YAAY,GACZ,gBAAgB,GAChB,iBAAiB,GACjB,aAAa,GACb,cAAc,CAAC;AAEnB,MAAM,MAAM,+BAA+B,GACvC,QAAQ,GACR,QAAQ,GACR,UAAU,GACV,SAAS,CAAC;AAEd,MAAM,MAAM,8BAA8B,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,CAAC;AAE9E,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,8BAA8B,GAAG,CAC3C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CACzB,KACE,OAAO,CAAC,8BAA8B,CAAC,CAAC;AAE7C,MAAM,WAAW,iCAAiC;IAChD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,+BAA+B;IAC9C,EAAE,EAAE,2BAA2B,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,+BAA+B,CAAC;IACxC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,sBAAsB,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,sBAAsB,CAAC;IACxC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,EAAE,iCAAiC,CAAC;IAC7C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,mCAAmC;IAClD,EAAE,EAAE,2BAA2B,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,wBAAwB,CAAC;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4BAA4B,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,MAAM,WAAW,oCAAoC;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,mCAAmC,CAAC;IAC9C,UAAU,CAAC,EAAE,4BAA4B,CAAC;IAC1C,cAAc,EAAE,+BAA+B,CAAC;IAChD,aAAa,EAAE,8BAA8B,CAAC;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,MAAM,+BAA+B,GAAG,CAC5C,KAAK,EAAE,oCAAoC,KACxC,OAAO,CAAC,+BAA+B,CAAC,CAAC;AAE9C,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,sBAAsB,CAAC;IAC7B,MAAM,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAC9C,YAAY,EAAE,2BAA2B,CAAC,cAAc,CAAC,CAAC;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,wBAAwB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,sBAAsB,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,UAAU,GAAG,aAAa,CAAC;IACnC,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,+BAA+B;IAC9C,UAAU,EAAE,mBAAmB,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,wBAAwB,EAAE,4BAA4B,GAAG,SAAS,CAAC,CAAC;IACxF,MAAM,EAAE,6BAA6B,EAAE,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC,wBAAwB,EAAE,4BAA4B,CAAC,CAAC;IACtE,QAAQ,EAAE,2BAA2B,EAAE,CAAC;IACxC,MAAM,EAAE,2BAA2B,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,gCAAgC;IAC/C,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,+BAA+B;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;IACtB,gBAAgB,CAAC,EAAE;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,8BAA8B,CAAC;IAC/C,cAAc,CAAC,EAAE,+BAA+B,CAAC;IACjD,SAAS,CAAC,EAAE,2BAA2B,EAAE,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,WAAW,CAAC,EAAE,qBAAqB,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,aAAa,EAAE,CAAC,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,6BAA6B,CAAC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,gCAAgC,CAAC;IAC5C,cAAc,EAAE,+BAA+B,CAAC;IAChD,SAAS,EAAE,+BAA+B,EAAE,CAAC;IAC7C,OAAO,EAAE;QACP,MAAM,EAAE,8BAA8B,CAAC;QACvC,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,uBAAuB,EAAE,OAAO,CAAC;QACjC,eAAe,EAAE,OAAO,CAAC;QACzB,YAAY,EAAE,wBAAwB,EAAE,CAAC;KAC1C,CAAC;IACF,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAcD,eAAO,MAAM,6BAA6B,EAAE,mCAAmC,EAwC9E,CAAC;AA0DF,eAAO,MAAM,qCAAqC,EAAE,8BAsElD,CAAC;AAEH,eAAO,MAAM,+BAA+B,GAAI,SAAS,OAAO,KAAG,OAAO,EAezE,CAAC;AAQF,eAAO,MAAM,4BAA4B,GAAI,OAAO,OAAO,KAAG,OAqB7D,CAAC;AAgFF,eAAO,MAAM,+BAA+B,GAAI,OAAO;IACrD,SAAS,EAAE,OAAO,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,WAAW,CAAC,EAAE,qBAAqB,CAAC;CACrC,KAAG,+BAyCH,CAAC;AAwNF,eAAO,MAAM,iCAAiC,QAAO,+BA+GlD,CAAC;AA2HJ,eAAO,MAAM,2BAA2B,GACtC,UAAS,+BAAoC,KAC5C,OAAO,CAAC,8BAA8B,CAwDxC,CAAC;AAEF,eAAO,MAAM,wCAAwC,GACnD,QAAQ,8BAA8B,KACrC,MA6BF,CAAC"}