@imfusion/web-ui 0.6.4-dev.8.g41d03f2f → 0.7.0

package/dist/llms/evals/runner/execute.d.ts

@@ -0,0 +1,32 @@
+import { TranscriptEntry } from './types.ts';
+export type RoundOutput = {
+    session_id: string | null;
+    result?: string;
+    duration_ms?: number;
+    total_cost_usd?: number;
+    usage?: {
+        input_tokens?: number;
+        output_tokens?: number;
+    };
+    modelUsage?: Record<string, unknown>;
+    subtype?: string;
+    num_turns?: number;
+    timedOut?: boolean;
+    timeoutMs?: number;
+};
+export declare function runClaude(prompt: string, { cwd, resumeSessionId, maxTurns, model, permissionMode, timeoutMs }: {
+    cwd: string;
+    resumeSessionId?: string | null;
+    maxTurns?: number;
+    model?: string | null;
+    permissionMode?: string;
+    timeoutMs?: number;
+}): RoundOutput;
+export declare function runCodex(prompt: string, { cwd, resumeSessionId }: {
+    cwd: string;
+    resumeSessionId?: string | null;
+}): RoundOutput;
+export declare function parseJsonl(raw: string): Record<string, unknown>[];
+export declare function readTranscript(workdir: string, sessionId: string): string;
+export declare function findCodexRollout(threadId: string): string;
+export declare function codexEntries(lines: Record<string, unknown>[]): TranscriptEntry[];

package/dist/llms/evals/runner/grade.d.ts

@@ -0,0 +1,10 @@
+import { AssertionResult, Scenario, ToolUse } from './types.ts';
+export declare function gradeRouting(label: "Skill" | "Tool", config: {
+    expect?: string[];
+    forbid?: string[];
+} | undefined, actual: string[]): AssertionResult[];
+export declare function gradeTranscript(scenario: Scenario, rawTranscript: string): AssertionResult[];
+export declare function gradeWorkspace(scenario: Scenario, workdir: string): AssertionResult[];
+export declare function gradeWrites(scenario: Scenario, toolUses: ToolUse[], workdir: string): AssertionResult[];
+export declare function gradeReport(scenario: Scenario, workdir: string, initialReport: string | null): AssertionResult[];
+export declare function gradeWithLlm(scenario: Scenario, conversation: string, workspaceFiles: string, voteCount: number): AssertionResult[];

package/dist/llms/evals/runner/report-md.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env -S npx tsx
+export declare function writeReportMarkdown(): string | null;

package/dist/llms/evals/runner/report.d.ts

@@ -0,0 +1,4 @@
+import { Benchmark } from './types.ts';
+export declare function printReport(benchmark: Benchmark, { allEvidence }?: {
+    allEvidence?: boolean | undefined;
+}): void;

package/dist/llms/evals/runner/run.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env -S npx tsx
+export {};

package/dist/llms/evals/runner/scenario.d.ts

@@ -0,0 +1,3 @@
+import { Host, RunResult, Scenario } from './types.ts';
+export declare function evalIdFor(scenario: Scenario, host: Host): string;
+export declare function runScenario(scenario: Scenario, host: Host, runNumber: number, runsRoot: string, model: string | null, graderVotes: number): RunResult;

package/dist/llms/evals/runner/types.d.ts

@@ -0,0 +1,140 @@
+export declare const HOSTS: readonly ["claude", "codex"];
+export type Host = (typeof HOSTS)[number];
+export type PathRule = {
+    pattern: string;
+    text: string;
+    flags?: string;
+    /** Named paths to search instead of the `src/**` default. */
+    files?: string[];
+};
+export type Scenario = {
+    id: string;
+    description: string;
+    rounds: string[];
+    /** Skills this scenario belongs to without asserting they loaded. */
+    covers?: string[];
+    skills?: {
+        expect?: string[];
+        forbid?: string[];
+    };
+    tools?: {
+        expect?: string[];
+        forbid?: string[];
+    };
+    writes?: {
+        only?: string[];
+        expect?: string[];
+    };
+    workspace?: {
+        forbidPatterns?: PathRule[];
+        expectPatterns?: PathRule[];
+    };
+    transcript?: {
+        expectStrings?: string[];
+        forbidStrings?: string[];
+    };
+    report?: {
+        path: string;
+        headings?: string[];
+        preserveFrom?: string;
+    };
+    assertions?: string[];
+    fixture?: string;
+    setup?: string;
+    hosts?: Host[];
+    maxTurns?: number;
+    roundTimeoutMinutes?: number;
+    permissionMode?: string;
+};
+export type AssertionResult = {
+    text: string;
+    passed: boolean;
+    method: "deterministic" | "llm";
+    evidence: string;
+};
+export type ToolUse = {
+    type: "tool_use";
+    name: string;
+    input?: Record<string, unknown>;
+};
+export type TranscriptEntry = {
+    type: string;
+    message?: {
+        content?: unknown;
+    };
+};
+export type HostPlan = {
+    path: string;
+    content: string;
+};
+export type RoundMeta = {
+    prompt: string;
+    session_id: string | null;
+    duration_ms?: number;
+    total_cost_usd?: number;
+    usage?: {
+        input_tokens?: number;
+        output_tokens?: number;
+    };
+    models?: string[];
+    subtype?: string;
+    num_turns?: number;
+    timed_out?: boolean;
+};
+export type RunResult = {
+    eval_id: string;
+    executor_host: Host;
+    fixture: string;
+    description: string;
+    run_number: number;
+    models: string[];
+    result: {
+        pass_rate: number;
+        passed: number;
+        failed: number;
+        total: number;
+        time_seconds: number;
+        turns: number;
+        tokens: number;
+        cost_usd: number;
+    };
+    expectations: {
+        text: string;
+        passed: boolean;
+        evidence: string;
+    }[];
+    truncated: boolean;
+    timed_out: boolean;
+    notes: string;
+};
+export type PerEvalSummary = {
+    pass_rate_mean: number;
+    pass_rate_stddev: number;
+    time_seconds_mean: number;
+    turns_mean: number;
+    tokens_mean: number;
+    cost_usd_total: number;
+};
+export type Benchmark = {
+    metadata: BenchmarkMetadata;
+    runs: RunResult[];
+    run_summary: {
+        per_eval: Record<string, PerEvalSummary>;
+        overall_pass_rate: number;
+        total_cost_usd: number;
+    };
+    notes: string[];
+};
+export type BenchmarkMetadata = {
+    skill_family: string;
+    timestamp: string;
+    evals_run: string[];
+    runs_per_scenario: number;
+    executor_model: string;
+    executor_hosts: string[];
+    grader_model: string;
+    grader_votes: number;
+    tier: string;
+    models_observed: string[];
+    fixtures: string[];
+};

package/dist/llms/evals/runner/workspace.d.ts

@@ -0,0 +1,2 @@
+export declare function copyTemplate(dest: string): void;
+export declare function applyFixture(dest: string, fixture: string): void;

package/dist/llms/evals/runner/write-generated.d.ts

@@ -0,0 +1 @@
+export declare function writeGenerated(path: string, contents: string): string;

package/dist/provider/web-ui-provider.d.ts

@@ -1,9 +1,12 @@
 import { ReactNode } from 'react';
+import { ColorSchemePreference } from '../types/theme';
 import * as IconConfig from '../icons/icon-config';
 interface Props {
     children: ReactNode;
     /** SVG defaults for icons in this provider's subtree; omitted fields use the defaults from icon-config.tsx. */
     icons?: React.ComponentProps<typeof IconConfig.Provider>["value"];
+    /** Color scheme preference; omit to follow the OS and persist a choice in localStorage. A value pins the scheme and bypasses localStorage. */
+    colorScheme?: ColorSchemePreference;
 }
-export declare function WebUIProvider({ children, icons }: Props): import("react/jsx-runtime").JSX.Element;
+export declare function WebUIProvider({ children, icons, colorScheme }: Props): import("react/jsx-runtime").JSX.Element;
 export {};