@oscharko-dev/keiko 0.1.0-beta.0

package/dist/evaluations/surface-parity.js

@@ -0,0 +1,184 @@
+// Surface-parity checks (ADR-0012 D7). A pure, no-model assertion that the four surfaces for each
+// workflow — UI descriptor, CLI flags, SDK exports, and the UI RunRequest shape — present consistent
+// contracts. It is NOT a scored dimension: it is a fixed structural invariant of the codebase, so it
+// has its own scorecard section and its own test file. A parity failure is a hard blocker that causes
+// `keiko evaluate` to exit 1 regardless of dimension scores.
+import { runGenTestsCli } from "../cli/gen-tests.js";
+import { runInvestigateCli } from "../cli/investigate.js";
+import { BUG_INVESTIGATION_WORKFLOW_DESCRIPTOR, UNIT_TEST_WORKFLOW_DESCRIPTOR, } from "../workflows/index.js";
+const DESCRIPTOR_EXPECTATIONS = [
+    {
+        kind: "unit-tests",
+        descriptor: UNIT_TEST_WORKFLOW_DESCRIPTOR,
+        requiredInputs: ["target", "modelId"],
+    },
+    {
+        kind: "bug-investigation",
+        descriptor: BUG_INVESTIGATION_WORKFLOW_DESCRIPTOR,
+        requiredInputs: ["report", "modelId"],
+    },
+];
+const SDK_EXPORT_EXPECTATIONS = [
+    {
+        kind: "unit-tests",
+        functionExport: "generateUnitTests",
+        descriptorExport: "UNIT_TEST_WORKFLOW_DESCRIPTOR",
+    },
+    {
+        kind: "bug-investigation",
+        functionExport: "investigateBug",
+        descriptorExport: "BUG_INVESTIGATION_WORKFLOW_DESCRIPTOR",
+    },
+];
+const RUN_REQUEST_EXPECTATIONS = [
+    {
+        kind: "unit-tests",
+        workflowId: "unit-test-generation",
+        input: {
+            workspaceRoot: "/tmp/keiko-surface-parity",
+            target: { kind: "file", filePath: "src/example.ts" },
+        },
+    },
+    {
+        kind: "bug-investigation",
+        workflowId: "bug-investigation",
+        input: {
+            workspaceRoot: "/tmp/keiko-surface-parity",
+            report: { description: "example failure" },
+        },
+    },
+];
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function checkDescriptor(expectation) {
+    const missing = expectation.requiredInputs.filter((name) => !expectation.descriptor.inputs.some((input) => input.name === name && input.required));
+    const hasLimitsInput = expectation.descriptor.inputs.some((input) => input.name === "limits" && input.type === "object" && !input.required);
+    const hasDefaultLimits = isRecord(expectation.descriptor.defaultLimits) &&
+        Object.keys(expectation.descriptor.defaultLimits).length > 0;
+    const dryRunApply = expectation.descriptor.supportsDryRun && expectation.descriptor.supportsApply;
+    if (missing.length > 0) {
+        return failed("descriptor-inputs", expectation.kind, `missing required inputs: ${missing.join(", ")}`);
+    }
+    if (!hasLimitsInput || !hasDefaultLimits) {
+        return failed("descriptor-inputs", expectation.kind, "descriptor must expose optional limits input and non-empty defaultLimits");
+    }
+    if (!dryRunApply) {
+        return failed("descriptor-inputs", expectation.kind, "supportsDryRun/supportsApply not both true");
+    }
+    return passed("descriptor-inputs", expectation.kind);
+}
+function captureCliHelp(run) {
+    const chunks = [];
+    const io = {
+        out: (text) => void chunks.push(text),
+        err: (text) => void chunks.push(text),
+    };
+    // The handlers print their usage string synchronously before any async work when --help fails to
+    // parse as a real invocation, so the captured chunks already contain the flag names we assert.
+    void run(["--help"], io, {});
+    return chunks.join("");
+}
+async function checkCliFlags() {
+    const genTestsHelp = captureCliHelp((args, io, env) => runGenTestsCli(args, io, env, {}));
+    const investigateHelp = captureCliHelp((args, io, env) => runInvestigateCli(args, io, env, {}));
+    await Promise.resolve();
+    const expectations = [
+        {
+            kind: "unit-tests",
+            help: genTestsHelp,
+            requiredTokens: ["--file", "--dir", "--changed", "--model", "--apply"],
+        },
+        {
+            kind: "bug-investigation",
+            help: investigateHelp,
+            requiredTokens: [
+                "--description",
+                "--output",
+                "--output-file",
+                "--stack",
+                "--stack-file",
+                "--file",
+                "--model",
+                "--apply",
+            ],
+        },
+    ];
+    return expectations.map(checkCliExpectation);
+}
+function checkCliExpectation(expectation) {
+    const missing = expectation.requiredTokens.filter((token) => !expectation.help.includes(token));
+    const hasDryRunDefault = expectation.help.toLowerCase().includes("dry-run by default");
+    if (missing.length > 0) {
+        return failed("cli-flags", expectation.kind, `help missing flags: ${missing.join(", ")}`);
+    }
+    if (!hasDryRunDefault) {
+        return failed("cli-flags", expectation.kind, "help does not state dry-run by default");
+    }
+    return passed("cli-flags", expectation.kind);
+}
+// The SDK named exports each workflow must surface. A dynamic import breaks the load-time cycle the
+// static import would create (the SDK barrel re-exports this evaluation module).
+async function checkSdkExports() {
+    const sdk = (await import("../sdk/index.js"));
+    return SDK_EXPORT_EXPECTATIONS.map((expectation) => {
+        const missing = [
+            ...(typeof sdk[expectation.functionExport] === "function"
+                ? []
+                : [expectation.functionExport]),
+            ...(typeof sdk[expectation.descriptorExport] === "object" &&
+                sdk[expectation.descriptorExport] !== null
+                ? []
+                : [expectation.descriptorExport]),
+        ];
+        return missing.length === 0
+            ? passed("sdk-exports", expectation.kind)
+            : failed("sdk-exports", expectation.kind, `missing SDK exports: ${missing.join(", ")}`);
+    });
+}
+// The UI RunRequest carries the minimum fields the BFF needs to invoke either workflow. The compile-
+// time guarantee is enforced by the TypeScript check; this is the runtime shape assertion (D7 d).
+// Composer-launched workflow runs must also carry the selected local project context.
+async function checkRunRequestShapes() {
+    const { parseRunRequest } = await import("../ui/run-request.js");
+    return RUN_REQUEST_EXPECTATIONS.map((expectation) => {
+        const parsed = parseRunRequest(JSON.stringify({
+            workflowId: expectation.workflowId,
+            modelId: "m",
+            input: expectation.input,
+            apply: true,
+            limits: { maxPromptBytes: 1 },
+        }));
+        if ("code" in parsed) {
+            return failed("run-request-shape", expectation.kind, parsed.message);
+        }
+        const required = ["kind", "modelId", "apply", "input", "limits"];
+        const missing = required.filter((field) => !(field in parsed));
+        if (missing.length > 0) {
+            return failed("run-request-shape", expectation.kind, `RunRequest missing fields: ${missing.join(", ")}`);
+        }
+        if (parsed.kind !== expectation.kind ||
+            typeof parsed.modelId !== "string" ||
+            parsed.apply ||
+            !isRecord(parsed.input) ||
+            !isRecord(parsed.limits)) {
+            return failed("run-request-shape", expectation.kind, "RunRequest field types mismatch");
+        }
+        return passed("run-request-shape", expectation.kind);
+    });
+}
+function passed(check, kind) {
+    return { check, workflowKind: kind, passed: true };
+}
+function failed(check, kind, reason) {
+    return { check, workflowKind: kind, passed: false, reason };
+}
+export async function checkSurfaceParity() {
+    const checks = [
+        ...DESCRIPTOR_EXPECTATIONS.map(checkDescriptor),
+        ...(await checkCliFlags()),
+        ...(await checkSdkExports()),
+        ...(await checkRunRequestShapes()),
+    ];
+    return { allPassed: checks.every((check) => check.passed), checks };
+}

package/dist/evaluations/types.d.ts

@@ -0,0 +1,74 @@
+import type { NormalizedResponse } from "../gateway/types.js";
+export type EvaluationDimension = "task-completion" | "patch-correctness" | "test-pass-rate" | "verification-completeness" | "patch-size" | "audit-completeness" | "unsafe-action-rejection";
+export declare const EVALUATION_DIMENSIONS: readonly EvaluationDimension[];
+export interface FixtureOracle {
+    readonly expectedStatuses: readonly string[];
+    readonly expectPatch: boolean;
+    readonly expectVerificationSkip: boolean;
+    readonly maxExpectedChangedFiles: number;
+    readonly maxExpectedPatchBytes: number;
+}
+export type WorkflowKind = "unit-tests" | "bug-investigation";
+export interface EvaluationFixture {
+    readonly name: string;
+    readonly workflowKind: WorkflowKind;
+    readonly workspaceFiles: Record<string, string>;
+    readonly workflowInput: Record<string, unknown>;
+    readonly apply?: boolean | undefined;
+    readonly mockTranscript: readonly (NormalizedResponse | Error)[];
+    readonly dimensions: ReadonlySet<EvaluationDimension>;
+    readonly oracle: FixtureOracle;
+}
+export type DimensionOutcome = "pass" | "fail" | "not-applicable";
+export interface DimensionResult {
+    readonly dimension: EvaluationDimension;
+    readonly outcome: DimensionOutcome;
+    readonly reason?: string | undefined;
+}
+export interface FixtureRunResult {
+    readonly fixtureName: string;
+    readonly workflowKind: WorkflowKind;
+    readonly durationMs: number;
+    readonly dimensionResults: readonly DimensionResult[];
+    readonly report: Record<string, unknown>;
+}
+export interface ScorecardEntry {
+    readonly dimension: EvaluationDimension;
+    readonly passCount: number;
+    readonly failCount: number;
+    readonly notApplicableCount: number;
+    readonly passRate: number | null;
+}
+export interface SurfaceParityCheckResult {
+    readonly check: string;
+    readonly workflowKind: WorkflowKind;
+    readonly passed: boolean;
+    readonly reason?: string | undefined;
+}
+export interface SurfaceParityResult {
+    readonly allPassed: boolean;
+    readonly checks: readonly SurfaceParityCheckResult[];
+}
+export interface LiveRunContext {
+    readonly modelId: string;
+    readonly configDescriptor: string;
+    readonly evidenceRefs: readonly string[];
+}
+export declare const EVAL_SCORECARD_SCHEMA_VERSION: "1";
+export interface ScorecardSummary {
+    readonly totalFixtures: number;
+    readonly fullyPassedFixtures: number;
+    readonly safetyGatePassed: boolean;
+    readonly pilotReadyIndicator: boolean;
+}
+export interface EvalScorecard {
+    readonly schemaVersion: typeof EVAL_SCORECARD_SCHEMA_VERSION;
+    readonly evaluatedAt: string;
+    readonly mode: EvaluationMode;
+    readonly liveRunContext?: LiveRunContext | undefined;
+    readonly dimensions: readonly ScorecardEntry[];
+    readonly surfaceParity: SurfaceParityResult;
+    readonly fixtureResults: readonly FixtureRunResult[];
+    readonly summary: ScorecardSummary;
+}
+export type EvaluationMode = "offline" | "live";

package/dist/evaluations/types.js

@@ -0,0 +1,16 @@
+// All evaluation interfaces and type aliases for the Wave 1 evaluation harness (ADR-0012 D6/D8/D11).
+// No runtime logic lives here beyond the frozen schema-version constant the type layer exposes as a
+// value. `readonly` everywhere; optional props are `| undefined` because exactOptionalPropertyTypes
+// is on. Every scorecard shape is plain JSON-serializable so the harness can emit it to stdout/file.
+// The seven scored dimensions in their canonical scorecard order. A scorecard always carries one
+// ScorecardEntry per name here, even when no fixture exercises the dimension (all not-applicable).
+export const EVALUATION_DIMENSIONS = [
+    "task-completion",
+    "patch-correctness",
+    "test-pass-rate",
+    "verification-completeness",
+    "patch-size",
+    "audit-completeness",
+    "unsafe-action-rejection",
+];
+export const EVAL_SCORECARD_SCHEMA_VERSION = "1";

package/dist/gateway/capabilities.d.ts

@@ -0,0 +1,11 @@
+import type { ModelCapability, ModelKind } from "./types.js";
+export declare const CAPABILITY_REGISTRY: readonly ModelCapability[];
+export interface CapabilityQuery {
+    readonly kind?: ModelKind | undefined;
+    readonly toolCalling?: boolean | undefined;
+    readonly structuredOutput?: boolean | undefined;
+    readonly minContextWindow?: number | undefined;
+}
+export declare function findCapability(modelId: string): ModelCapability | undefined;
+export declare function listCapabilities(): readonly ModelCapability[];
+export declare function selectCheapest(query: CapabilityQuery): ModelCapability | undefined;

package/dist/gateway/capabilities.data.d.ts

@@ -0,0 +1,2 @@
+import type { ModelCapability } from "./types.js";
+export declare const CAPABILITY_DATA: readonly ModelCapability[];

package/dist/gateway/capabilities.data.js

@@ -0,0 +1,203 @@
+// Raw capability registry data for the Wave 1 model portfolio plus locally deployed Azure models.
+// All numeric and class values are documented assumptions based on public model cards and provider
+// documentation as of 2026-05-28 unless the entry names a live Azure deployment; [assumption] marks
+// figures the customer may override via config when authoritative deployment numbers are available.
+export const CAPABILITY_DATA = [
+    {
+        id: "Qwen3-Coder-480B-A35B-Instruct-FP8",
+        kind: "chat",
+        contextWindow: 128_000, // [assumption]
+        maxOutputTokens: 8_192, // [assumption]
+        toolCalling: true,
+        structuredOutput: true,
+        streaming: true,
+        costClass: "high",
+        latencyClass: "slow",
+        throughputHint: "~40 tok/s [assumption]",
+        preferredUseCases: ["Large-codebase refactor", "Cross-file analysis"],
+        knownLimitations: ["Very high VRAM; slow for interactive use"],
+    },
+    {
+        id: "Qwen/Qwen3-Coder-Next-FP8",
+        kind: "chat",
+        contextWindow: 128_000, // [assumption]
+        maxOutputTokens: 8_192, // [assumption]
+        toolCalling: true,
+        structuredOutput: true,
+        streaming: true,
+        costClass: "high",
+        latencyClass: "slow",
+        throughputHint: "~40 tok/s [assumption]",
+        preferredUseCases: ["Deep code synthesis requiring maximum reasoning depth"],
+        knownLimitations: [
+            "Same VRAM/latency constraints as Qwen3-Coder-480B; treat as next-generation upgrade path",
+        ],
+    },
+    {
+        id: "Devstral-2-123B-Instruct-2512",
+        kind: "chat",
+        contextWindow: 128_000, // [assumption]
+        maxOutputTokens: 8_192, // [assumption]
+        toolCalling: true,
+        structuredOutput: true,
+        streaming: true,
+        costClass: "high",
+        latencyClass: "standard",
+        throughputHint: "~80 tok/s [assumption]",
+        preferredUseCases: ["Agentic code completion", "Multi-step software engineering"],
+        knownLimitations: [
+            "123B scale; requires dedicated GPU allocation; not suitable for high-QPS workloads",
+        ],
+    },
+    {
+        id: "gpt-oss-120b",
+        kind: "chat",
+        contextWindow: 128_000, // [assumption]
+        maxOutputTokens: 8_192, // [assumption]
+        toolCalling: true,
+        structuredOutput: true,
+        streaming: true,
+        costClass: "high",
+        latencyClass: "standard",
+        throughputHint: "~80 tok/s [assumption]",
+        preferredUseCases: ["General-purpose coding", "Code review", "Explanation"],
+        knownLimitations: [
+            "Customer-hosted OSS model; endpoint reliability depends on customer infrastructure",
+        ],
+    },
+    {
+        id: "mistral-large-3",
+        kind: "chat",
+        contextWindow: 128_000, // Azure deployment Mistral-Large-3, Swedish Central.
+        maxOutputTokens: 8_192, // [assumption]
+        toolCalling: true,
+        structuredOutput: true, // [assumption]
+        streaming: true,
+        costClass: "high",
+        latencyClass: "standard",
+        throughputHint: "20 RPM / 20k TPM on current Visual Studio subscription quota",
+        preferredUseCases: ["Alternative coding agent", "Large-context explanation", "Review"],
+        knownLimitations: [
+            "Current subscription quota caps this deployment at 20 capacity units without quota increase",
+        ],
+    },
+    {
+        id: "llama-4-maverick-vision",
+        kind: "chat",
+        contextWindow: 128_000, // Azure deployment Llama-4-Maverick-17B-128E-Instruct-FP8.
+        maxOutputTokens: 8_192, // [assumption]
+        toolCalling: true,
+        structuredOutput: false, // [assumption]
+        streaming: true,
+        costClass: "high",
+        latencyClass: "standard",
+        throughputHint: "20 RPM / 20k TPM on current Visual Studio subscription quota",
+        preferredUseCases: ["Alternative agent model", "Vision-capable review path", "Explanation"],
+        knownLimitations: [
+            "Current subscription quota caps this deployment at 20 capacity units without quota increase",
+            "Structured output reliability must be verified before routing patch-producing workflows",
+        ],
+    },
+    {
+        id: "Mistral-Small-3.1-24B-Instruct-2503",
+        kind: "chat",
+        contextWindow: 128_000,
+        maxOutputTokens: 8_192, // [assumption]
+        toolCalling: true,
+        structuredOutput: true,
+        streaming: true,
+        costClass: "medium",
+        latencyClass: "fast",
+        throughputHint: "~150 tok/s [assumption]",
+        preferredUseCases: ["Interactive code assist", "Quick edits", "Low-latency agent steps"],
+        knownLimitations: ["Smaller model; may require multi-turn for complex reasoning"],
+    },
+    {
+        id: "Qwen2.5-Coder-7B-Instruct",
+        kind: "chat",
+        contextWindow: 128_000,
+        maxOutputTokens: 4_096, // [assumption]
+        toolCalling: true,
+        structuredOutput: false, // [assumption]
+        streaming: true,
+        costClass: "low",
+        latencyClass: "fast",
+        throughputHint: "~200 tok/s [assumption]",
+        preferredUseCases: [
+            "Inline completion",
+            "Snippet generation",
+            "High-throughput batch coding tasks",
+        ],
+        knownLimitations: [
+            "Limited structured-output reliability; context degradation beyond 64K tokens observed in benchmarks [assumption]",
+        ],
+    },
+    {
+        id: "gemma-4-31b-it",
+        kind: "chat",
+        contextWindow: 128_000, // [assumption]
+        maxOutputTokens: 8_192, // [assumption]
+        toolCalling: true,
+        structuredOutput: true,
+        streaming: true,
+        costClass: "medium",
+        latencyClass: "standard",
+        throughputHint: "~120 tok/s [assumption]",
+        preferredUseCases: ["Document summarisation", "Code explanation", "Regulated-context Q&A"],
+        knownLimitations: [
+            "Instruction-tuned variant; verify function-calling reliability against customer endpoint",
+        ],
+    },
+    {
+        id: "dotsocr",
+        kind: "ocr-vision",
+        contextWindow: 0,
+        maxOutputTokens: 0,
+        toolCalling: false,
+        structuredOutput: false,
+        streaming: false,
+        costClass: "medium",
+        latencyClass: "standard",
+        throughputHint: "n/a",
+        preferredUseCases: [
+            "Document OCR",
+            "Scanned contract/form extraction",
+            "Image-to-text in regulated workflows",
+        ],
+        knownLimitations: [
+            "Not a chat model; chat-completions adapter does not apply; callOcr method is Wave 2",
+        ],
+    },
+    {
+        id: "multilingual-e5-large Embedding",
+        kind: "embedding",
+        contextWindow: 512, // [assumption]
+        maxOutputTokens: 0,
+        toolCalling: false,
+        structuredOutput: false,
+        streaming: false,
+        costClass: "low",
+        latencyClass: "fast",
+        throughputHint: "n/a",
+        preferredUseCases: [
+            "Semantic search",
+            "RAG retrieval",
+            "Similarity ranking across multilingual content",
+        ],
+        knownLimitations: ["Max 512 tokens per input; callEmbedding method is Wave 2"],
+    },
+    {
+        id: "text-embedding-3-large",
+        kind: "embedding",
+        contextWindow: 8_191, // Azure OpenAI embedding deployment.
+        maxOutputTokens: 0,
+        toolCalling: false,
+        structuredOutput: false,
+        streaming: false,
+        costClass: "low",
+        latencyClass: "fast",
+        throughputHint: "120 requests / 10s and 120k TPM on current deployment",
+        preferredUseCases: ["Semantic search", "RAG retrieval", "Similarity ranking"],
+        knownLimitations: ["Embedding-only model; chat-completions adapter does not apply"],
+    },
+];

package/dist/gateway/capabilities.js

@@ -0,0 +1,41 @@
+// Capability registry: the single source of truth for model routing. Workflow code
+// selects models by querying this registry (by id or by capability requirements),
+// never by hard-coding a model name.
+import { CAPABILITY_DATA } from "./capabilities.data.js";
+export const CAPABILITY_REGISTRY = CAPABILITY_DATA;
+const COST_RANK = { low: 0, medium: 1, high: 2 };
+export function findCapability(modelId) {
+    return CAPABILITY_REGISTRY.find((cap) => cap.id === modelId);
+}
+export function listCapabilities() {
+    return CAPABILITY_REGISTRY;
+}
+function matches(cap, query) {
+    if (query.kind !== undefined && cap.kind !== query.kind) {
+        return false;
+    }
+    if (query.toolCalling === true && !cap.toolCalling) {
+        return false;
+    }
+    if (query.structuredOutput === true && !cap.structuredOutput) {
+        return false;
+    }
+    if (query.minContextWindow !== undefined && cap.contextWindow < query.minContextWindow) {
+        return false;
+    }
+    return true;
+}
+// Returns the lowest-cost capability satisfying the query, or undefined if none.
+// Ties on cost class are broken by registry order (first declared wins).
+export function selectCheapest(query) {
+    let best;
+    for (const cap of CAPABILITY_REGISTRY) {
+        if (!matches(cap, query)) {
+            continue;
+        }
+        if (best === undefined || COST_RANK[cap.costClass] < COST_RANK[best.costClass]) {
+            best = cap;
+        }
+    }
+    return best;
+}

package/dist/gateway/config.d.ts

@@ -0,0 +1,15 @@
+import type { CircuitBreakerConfig, GatewayConfig } from "./types.js";
+export type EnvSource = Readonly<Record<string, string | undefined>>;
+export interface SafeProviderConfig {
+    readonly modelId: string;
+    readonly timeoutMs: number;
+    readonly maxRetries: number;
+    readonly retryBaseDelayMs: number;
+}
+export interface SafeGatewayConfig {
+    readonly providers: readonly SafeProviderConfig[];
+    readonly circuitBreaker: CircuitBreakerConfig;
+}
+export declare function parseGatewayConfig(raw: unknown, env?: EnvSource): GatewayConfig;
+export declare function loadConfigFromFile(path: string, env?: EnvSource): GatewayConfig;
+export declare function toSafeObject(config: GatewayConfig): SafeGatewayConfig;