@kimpton-ai/evalrouter 0.1.0

package/dist/types.d.ts

@@ -0,0 +1,680 @@
+export type BenchmarkDetail = {
+    slug: string;
+    versions: Array<BenchmarkRecord>;
+};
+export type BenchmarkNotice = {
+    adaptation_notice: string;
+    evidence_sha256: string;
+    licenses: Array<LicenseNotice>;
+    manifest_sha256: string;
+    profile_id: string;
+};
+export type BenchmarkQuoteAvailability = {
+    message?: string | null;
+    reason_code?: string | null;
+    status: "ready_for_quote" | "unavailable";
+};
+export type BenchmarkRecord = {
+    category?: string | null;
+    code_license?: string | null;
+    data_license?: string | null;
+    description?: string | null;
+    display_policy?: string | null;
+    fewshot_config?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    generation_config?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    id: string;
+    license_url?: string | null;
+    manifest_sha256: string;
+    metrics?: Array<string> | null;
+    name?: string | null;
+    provenance?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    quote_availability?: BenchmarkQuoteAvailability | null;
+    required_capabilities?: Array<string> | null;
+    runner: string;
+    sampling_algorithm?: string | null;
+    slug: string;
+    source_revision?: string | null;
+    source_url?: string | null;
+    status: string;
+    task_count?: number | null;
+    version: string;
+};
+export type BillingSnapshot = {
+    as_of: string;
+    cap_microusd: string;
+    pending: boolean;
+    reserved_microusd: string;
+    spend_microusd: string;
+};
+export type CatalogBenchmarksParams = {
+    q?: string;
+    runner?: "inspect" | "lm-eval" | "evalplus" | "livecodebench" | "spreadsheetbench" | null;
+    capability?: string | null;
+    status?: "candidate" | "active" | "quarantined" | "retired" | null;
+    cursor?: string | null;
+    limit?: number;
+};
+export type CatalogEnvironmentsParams = {
+    cursor?: string | null;
+    limit?: number;
+};
+export type CatalogEvalsParams = {
+    provider?: string | null;
+    kind?: "benchmark" | "environment" | "remote" | null;
+    protocol?: "batch" | "interactive" | "agentic" | null;
+    q?: string | null;
+    behavior?: string | null;
+    availability?: string | null;
+    publisher?: string | null;
+    include_private?: boolean;
+    cursor?: string | null;
+    limit?: number;
+};
+export type CatalogModelsParams = {
+    q?: string;
+    provider?: string | null;
+    capability?: string | null;
+    cursor?: string | null;
+    limit?: number;
+};
+export type ComparisonModel = {
+    kind: "comparison";
+    model_id: string;
+    models: Array<SelectedModel>;
+};
+export type ConnectedModel = {
+    connection_id: string;
+    kind: "connection";
+};
+export type ConnectionRecord = {
+    base_url: string;
+    capabilities: Array<string>;
+    check_result: {
+        [key: string]: JsonValue_Output;
+    };
+    checked_at: string | null;
+    config: {
+        [key: string]: JsonValue_Output;
+    };
+    created_at: string;
+    disabled_at: string | null;
+    id: string;
+    model_id: string;
+    name: string;
+    protocol: "openai_chat" | "openai_completions";
+    revision: number;
+};
+export type ConnectionsListParams = {
+    cursor?: string | null;
+    limit?: number;
+};
+export type CoverageCounts = {
+    attempted: number;
+    cancelled: number;
+    errors: number;
+    graded: number;
+    planned: number;
+    skipped_budget: number;
+};
+export type CoverageRecord = {
+    mode: "sample" | "full";
+    sample_count?: number | null;
+    seed?: number | null;
+};
+export type EnvironmentFamily = {
+    family: string;
+    versions: Array<EnvironmentRecord>;
+};
+export type EnvironmentLimits = {
+    cpu_millis?: number;
+    duration_seconds?: number;
+    max_observation_bytes?: number;
+    max_output_tokens?: number;
+    max_steps?: number;
+    memory_mib?: number;
+    network?: "blocked";
+};
+export type EnvironmentRecord = {
+    adapter: string;
+    commission_basis_points?: number;
+    created_at: string;
+    description: string;
+    id: string;
+    limitations: Array<string>;
+    limits: EnvironmentLimits;
+    manifest_sha256: string;
+    name: string;
+    package_sha256: string;
+    price_version: string;
+    quote_availability?: BenchmarkQuoteAvailability | null;
+    ref: string;
+    required_capabilities: Array<string>;
+    rights: EnvironmentRights;
+    status: string;
+    supplier_fee_microusd: string;
+    task_counts: {
+        [key: string]: number;
+    };
+    validation: {
+        [key: string]: JsonValue_Output;
+    };
+    visibility: string;
+};
+export type EnvironmentRights = {
+    attribution: string;
+    commercial_execution: boolean;
+    display: boolean;
+    export: boolean;
+    license: string;
+    source_url: string;
+    training: boolean;
+    underlying_licenses?: Array<string>;
+};
+export type EvalPricing = {
+    execution_fee_microusd?: string | null;
+    price_version: string;
+    task_fee_microusd?: string | null;
+};
+export type EvalRecord = {
+    availability?: string;
+    availability_reason?: string;
+    behaviors?: Array<string>;
+    category?: string;
+    description: string;
+    detail_href?: string | null;
+    display_name: string;
+    execution_mode: "router" | "provider";
+    family?: string;
+    kind: "benchmark" | "environment" | "remote";
+    name: string;
+    pricing: EvalPricing;
+    profile_id: string | null;
+    protocol: "batch" | "interactive" | "agentic";
+    provider: string;
+    publisher?: string;
+    ref: string;
+    required_capabilities: Array<string>;
+    runner: string;
+    source?: string;
+    status: string;
+    task_count: number | null;
+    version: string;
+    visibility?: "public" | "private";
+};
+export type EvalReference = {
+    eval: string;
+    provider?: string | null;
+    split?: "canonical" | "train" | "development" | "test" | null;
+};
+export type EvalsListParams = {
+    provider?: string | null;
+    kind?: "benchmark" | "environment" | "remote" | null;
+    protocol?: "batch" | "interactive" | "agentic" | null;
+    q?: string | null;
+    behavior?: string | null;
+    availability?: string | null;
+    publisher?: string | null;
+    include_private?: boolean;
+    cursor?: string | null;
+    limit?: number;
+};
+export type EventPage = {
+    data: Array<RunEvent>;
+    has_more: boolean;
+    next_sequence: number;
+};
+export type ExecutionOptions = {
+    chat_enabled?: boolean;
+    max_models_per_run: number;
+};
+export type FullCoverage = {
+    mode: "full";
+};
+export type Interval = {
+    approximate: boolean;
+    cluster_count?: number | null;
+    confidence: number;
+    eligible_count: number;
+    gains?: number | null;
+    losses?: number | null;
+    lower: number;
+    method: string;
+    resamples?: number | null;
+    sample_count: number;
+    seed?: number | null;
+    strata_count: number;
+    target: string;
+    unrepresented_strata: number;
+    upper: number;
+    weights: string;
+};
+export type JsonValue = JsonValue_Output;
+export type JsonValue_Output = string | number | boolean | Array<JsonValue_Output> | {
+    [key: string]: JsonValue_Output;
+} | null;
+export type JudgeSelection = {
+    model: SelectedModel;
+    protocol: {
+        [key: string]: JsonValue_Output;
+    };
+};
+export type LicenseNotice = {
+    attribution: string;
+    document_sha256: string;
+    identifier: string;
+    scope: "code" | "data" | "tests" | "assets";
+    source_revision: string;
+    source_url: string;
+    text: string;
+};
+export type ManagedModel = {
+    kind: "managed";
+    route_id: string;
+};
+export type ManagedRoute = {
+    capabilities: Array<string>;
+    display_name: string;
+    generation?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    hosting_provider?: string | null;
+    id: string;
+    limits?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    model_id: string;
+    price_version: string;
+    pricing: {
+        [key: string]: JsonValue_Output;
+    };
+    provider: string;
+    revision: number;
+    status: string;
+};
+export type MetricRecord = {
+    direction: "higher_is_better" | "lower_is_better" | null;
+    key: string;
+    name: string;
+    sample_count: number | null;
+    source: string;
+    standard_error: number | null;
+    task: string;
+    uncertainty?: Uncertainty | null;
+    value: number;
+};
+export type NewConnection = {
+    api_key: string;
+    base_url: string;
+    context_window?: number;
+    max_output_tokens?: number;
+    model_id: string;
+    name: string;
+    protocol?: "openai_chat" | "openai_completions";
+};
+export type NewEvaluation = {
+    budget_usd?: number | string | null;
+    coverage?: SampleCoverage | FullCoverage | null;
+    eval: string;
+    max_charge_microusd?: string | null;
+    metadata?: {
+        [key: string]: string;
+    };
+    model: string | ManagedModel | ConnectedModel;
+    name?: string;
+    provider?: string | null;
+    split?: string | null;
+};
+export type NewQuote = {
+    selection: Profiles | EvalReference;
+    model: ManagedModel | ConnectedModel;
+    coverage?: SampleCoverage | FullCoverage | null;
+    max_charge_microusd: string;
+};
+export type NewRun = {
+    metadata?: {
+        [key: string]: string;
+    };
+    name?: string;
+    quote_id: string;
+};
+export type OKResponse = {
+    ok: boolean;
+};
+export type Page_BenchmarkRecord_ = {
+    data: Array<BenchmarkRecord>;
+    next_cursor: string | null;
+};
+export type Page_ConnectionRecord_ = {
+    data: Array<ConnectionRecord>;
+    next_cursor: string | null;
+};
+export type Page_EnvironmentRecord_ = {
+    data: Array<EnvironmentRecord>;
+    next_cursor: string | null;
+};
+export type Page_EvalRecord_ = {
+    data: Array<EvalRecord>;
+    next_cursor: string | null;
+};
+export type Page_ManagedRoute_ = {
+    data: Array<ManagedRoute>;
+    next_cursor: string | null;
+};
+export type Page_RunRecord_ = {
+    data: Array<RunRecord>;
+    next_cursor: string | null;
+};
+export type ProfileCost = {
+    charged_microusd: string;
+    model_index?: number | null;
+    pending_operations: number;
+    profile_id: string;
+    total_cost_microusd: string | null;
+};
+export type ProfileResult = {
+    cancelled: number;
+    data_license: string | null;
+    display_policy?: string | null;
+    eligible: number | null;
+    error_code: string | null;
+    errors: number;
+    graded: number;
+    headline_key: string | null;
+    license_url: string | null;
+    metrics: Array<MetricRecord>;
+    model_index?: number | null;
+    name: string;
+    native: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    planned: number;
+    profile_id: string;
+    runner: string;
+    selection: ProfileSelection;
+    skipped_budget: number;
+    source_revision: string | null;
+    source_url: string | null;
+    status: string;
+    uncertainty?: {
+        [key: string]: Uncertainty;
+    } | null;
+    uncertainty_note: string;
+    unit_id: string;
+};
+export type ProfileSelection = {
+    admission?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    attempts_per_task?: number;
+    dataset_assets_sha256: string;
+    environment?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    id: string;
+    judge?: JudgeSelection | null;
+    manifest_sha256: string;
+    population_count: number;
+    private_test?: boolean | null;
+    remote?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    sampling_algorithm: string;
+    sandbox?: SandboxPolicyView | null;
+    selected_count: number;
+    task_ids?: Array<string> | null;
+    task_ids_hash: string;
+};
+export type Profiles = {
+    profile_ids: Array<string>;
+};
+export type QuotePlan = {
+    agent?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    agents?: Array<{
+        [key: string]: JsonValue_Output;
+    }> | null;
+    coverage: CoverageRecord;
+    model: SelectedModel | ComparisonModel;
+    pricing: {
+        [key: string]: JsonValue_Output;
+    };
+    profiles: Array<ProfileSelection>;
+    recommendation?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    route?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    schema_version: 1 | 2 | 3 | 4;
+    selection_source: {
+        [key: string]: JsonValue_Output;
+    };
+    total_tasks: number;
+    warnings: Array<string>;
+};
+export type QuoteRecord = {
+    created_at: string;
+    estimated_charge_microusd: string;
+    expires_at: string;
+    id: string;
+    max_charge_microusd: string;
+    plan: QuotePlan;
+    price_version: string;
+    retention: RetentionPolicy;
+};
+export type ResultCosts = {
+    as_of: string;
+    basis: "managed_total" | "platform_fees_only" | "mixed";
+    currency: "USD";
+    note: string;
+    profiles: Array<ProfileCost>;
+};
+export type ResultRecord = {
+    benchmark_notices: Array<BenchmarkNotice>;
+    checksum: string;
+    costs: ResultCosts;
+    coverage: CoverageRecord | null;
+    envelope?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    finalized_at: string;
+    id: string;
+    manifest_hash: string;
+    model: SelectedModel | ComparisonModel;
+    profiles: Array<ProfileResult>;
+    retention: RunRetention;
+    run_id: string;
+    summary: ResultSummary;
+    version: number;
+};
+export type ResultSummary = {
+    billing_snapshot: BillingSnapshot;
+    coverage: CoverageCounts;
+    finished_at: string;
+    manifest_hash: string;
+    name: string;
+    result_version: number;
+    run_id: string;
+    schema_version: 1;
+    status: "completed" | "partial" | "failed" | "cancelled";
+    terminal_reason: string | null;
+};
+export type RetentionPolicy = {
+    raw_days: number;
+    starts_at: "run_created_at";
+    summary_days: number;
+};
+export type RunEvent = {
+    created_at: string;
+    data: {
+        [key: string]: JsonValue_Output;
+    };
+    sequence: number;
+    type: string;
+};
+export type RunReceipt = {
+    billing_state: "pending" | "settled" | "reconciliation_required";
+    cap_microusd: string;
+    created_at: string;
+    currency: "USD";
+    report_expired: boolean;
+    reserved_microusd: string;
+    run_id: string;
+    spend_microusd: string;
+    summary_expires_at: string;
+};
+export type RunRecord = {
+    agent?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    agents?: Array<{
+        [key: string]: JsonValue_Output;
+    }> | null;
+    billing_state: "pending" | "settled" | "reconciliation_required";
+    cancel_requested_at: string | null;
+    cap_microusd: string;
+    completed_count: number;
+    coverage: CoverageRecord | null;
+    created_at: string;
+    error_count: number;
+    finished_at: string | null;
+    id: string;
+    manifest_hash: string;
+    metadata: {
+        [key: string]: string;
+    };
+    model: SelectedModel | ComparisonModel;
+    name: string;
+    planned_count: number;
+    profiles: Array<ProfileSelection>;
+    reserved_microusd: string;
+    retention: RunRetention;
+    retry_at?: string | null;
+    route?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    spend_microusd: string;
+    started_at: string | null;
+    status: "queued" | "running" | "cancelling" | "finalizing" | "completed" | "partial" | "failed" | "cancelled";
+    terminal_reason: string | null;
+    waiting_reason?: string | null;
+};
+export type RunRetention = {
+    raw_evidence_expired: boolean;
+    raw_expires_at: string;
+    summary_expires_at: string;
+};
+export type RunsEventsParams = {
+    after?: number;
+    limit?: number;
+    tail?: boolean;
+};
+export type RunsExecutionsParams = {
+    sample_id?: string | null;
+    cursor?: string | null;
+    limit?: number;
+};
+export type RunsExportParams = {
+    format?: "json" | "csv" | "html";
+    version?: number | null;
+};
+export type RunsListParams = {
+    cursor?: string | null;
+    limit?: number;
+};
+export type RunsResultsParams = {
+    version?: number | null;
+};
+export type SampleCoverage = {
+    mode?: "sample";
+    sample_count?: number;
+    seed?: number;
+};
+export type SandboxExecutionPage = {
+    data: Array<SandboxExecutionRecord>;
+    next_cursor: string | null;
+    summary: SandboxExecutionSummary;
+};
+export type SandboxExecutionRecord = {
+    attempt_no: number;
+    cleanup_status: "not_requested" | "pending" | "stopped";
+    cpu_millis: number;
+    created_at: string;
+    error_code: string | null;
+    execution_started_at: string | null;
+    id: string;
+    memory_mib: number;
+    model_index: number;
+    profile_id: string;
+    sample_id: string;
+    status: "intent" | "creating" | "ready" | "executing" | "finished" | "stopping" | "stopped";
+    stopped_at: string | null;
+    task_id: string;
+    timeout_seconds: number;
+    verifier_result_recorded: boolean;
+};
+export type SandboxExecutionSummary = {
+    active_count: number;
+    allocated_count: number;
+    cleanup_pending_count: number;
+    stopped_count: number;
+};
+export type SandboxPolicyView = {
+    backend?: string;
+    cpu_millis?: number;
+    execution_class?: string;
+    execution_fee_microusd: string;
+    image_id: string;
+    image_sha256: string;
+    max_infrastructure_attempts?: number;
+    memory_mib?: number;
+    network?: string;
+    price_version: string;
+    schema_version?: 1;
+    timeout_seconds?: number;
+};
+export type SelectedModel = {
+    artifact_id?: string | null;
+    artifact_sha256?: string | null;
+    capabilities: Array<string>;
+    connection_id?: string | null;
+    generation?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    kind: string;
+    likelihood?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    limits: {
+        [key: string]: JsonValue_Output;
+    };
+    model_id: string;
+    price_version?: string | null;
+    pricing?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+    protocol: string;
+    provider_charges?: string | null;
+    revision: number;
+    route_id?: string | null;
+    routing?: {
+        [key: string]: JsonValue_Output;
+    } | null;
+};
+export type Uncertainty = {
+    interval: Interval | null;
+    reason: string | null;
+};
+export type UpdateConnection = {
+    api_key?: string | null;
+    context_window?: number | null;
+    max_output_tokens?: number | null;
+    name?: string | null;
+};

package/dist/types.js

@@ -0,0 +1,2 @@
+// Generated by scripts/generate_clients.py; OpenAPI SHA256 3cea8bc29adf3dd7ab60f48b418a77478e3e4502357e1e23de4a2f7e97088a3f.
+export {};

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@kimpton-ai/evalrouter",
+  "version": "0.1.0",
+  "license": "SEE LICENSE IN LICENSE",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kimpton-ai/kimpton-evalrouter.git"
+  },
+  "type": "module",
+  "description": "Typed Kimpton evaluation client for trusted Node.js applications",
+  "engines": {
+    "node": ">=22"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./types": {
+      "types": "./dist/types.d.ts",
+      "import": "./dist/types.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ]
+}