localarena 0.1.0 → 0.2.0

package/javascript/src/index.d.ts

@@ -88,3 +88,548 @@ export class Arena {
   static fromSnapshot(snapshot: ArenaSnapshot): Arena;
   static fromJSON(value: string): Arena;
 }
+
+export const RUN_SCHEMA_VERSION: 1;
+
+export type ChatRole = "system" | "user" | "assistant";
+
+export interface ChatMessageOptions {
+  role: ChatRole;
+  content: string;
+}
+
+export class ChatMessage {
+  constructor(role: ChatRole, content: string);
+  constructor(options: ChatMessageOptions);
+  readonly role: ChatRole;
+  readonly content: string;
+}
+
+export interface GenerationRequestOptions {
+  model: string;
+  messages?: readonly (ChatMessage | ChatMessageOptions)[];
+  input?: string | readonly (ChatMessage | ChatMessageOptions)[];
+  prompt?: string | readonly (ChatMessage | ChatMessageOptions)[];
+  system?: string;
+  maxTokens?: number | null;
+  maxOutputTokens?: number | null;
+  max_tokens?: number | null;
+  temperature?: number;
+  seed?: number;
+  stop?: readonly string[];
+  extraBody?: Metadata;
+  extra_body?: Metadata;
+}
+
+export class GenerationRequest {
+  constructor(options: GenerationRequestOptions);
+  readonly model: string;
+  readonly messages: readonly ChatMessage[];
+  readonly maxTokens: number | null;
+  readonly temperature?: number;
+  readonly seed?: number;
+  readonly stop: readonly string[];
+  readonly extraBody: Readonly<Metadata>;
+}
+
+export interface TokenUsageOptions {
+  inputTokens?: number | null;
+  outputTokens?: number | null;
+  totalTokens?: number | null;
+  cachedInputTokens?: number | null;
+  reasoningTokens?: number | null;
+}
+
+export class TokenUsage {
+  constructor(options?: TokenUsageOptions);
+  readonly inputTokens?: number;
+  readonly outputTokens?: number;
+  readonly totalTokens?: number;
+  readonly cachedInputTokens?: number;
+  readonly reasoningTokens?: number;
+}
+
+export interface GenerationResultOptions {
+  text: string;
+  provider: string;
+  model: string;
+  responseModel?: string;
+  finishReason?: string;
+  usage?: TokenUsage | TokenUsageOptions;
+  latencySeconds: number;
+  attempts: number;
+  responseId?: string;
+  requestId?: string;
+  metadata?: Metadata;
+}
+
+export class GenerationResult {
+  constructor(options: GenerationResultOptions);
+  readonly text: string;
+  readonly provider: string;
+  readonly model: string;
+  readonly responseModel?: string;
+  readonly finishReason?: string;
+  readonly usage: TokenUsage;
+  readonly latencySeconds: number;
+  readonly attempts: number;
+  readonly responseId?: string;
+  readonly requestId?: string;
+  readonly metadata: Readonly<Metadata>;
+}
+
+export interface ModelInfoOptions {
+  id: string;
+  provider: string;
+  displayName?: string;
+  metadata?: Metadata;
+}
+
+export class ModelInfo {
+  constructor(options: ModelInfoOptions);
+  readonly id: string;
+  readonly provider: string;
+  readonly displayName?: string;
+  readonly metadata: Readonly<Metadata>;
+}
+
+export interface RequestPolicyOptions {
+  timeoutMs?: number;
+  maxRetries?: number;
+  retryBaseMs?: number;
+  retryMaxMs?: number;
+  maxResponseBytes?: number;
+  retryStatuses?: readonly number[];
+}
+
+export class RequestPolicy {
+  constructor(options?: RequestPolicyOptions);
+  readonly timeoutMs: number;
+  readonly maxRetries: number;
+  readonly retryBaseMs: number;
+  readonly retryMaxMs: number;
+  readonly maxResponseBytes: number;
+  readonly retryStatuses: readonly number[];
+}
+
+export interface ProviderCallOptions {
+  signal?: AbortSignal;
+  requestPolicy?: RequestPolicy | RequestPolicyOptions;
+}
+
+export interface GenerationProviderMethods {
+  generate(
+    request: GenerationRequest | GenerationRequestOptions,
+    options?: ProviderCallOptions,
+  ): Promise<GenerationResult>;
+  listModels?(options?: ProviderCallOptions): Promise<readonly ModelInfo[]>;
+}
+
+export type GenerationProvider = GenerationProviderMethods &
+  (
+    | { readonly id: string; readonly name?: string }
+    | { readonly id?: string; readonly name: string }
+  );
+
+export interface ProviderErrorDetails {
+  code?: string;
+  provider?: string;
+  status?: number;
+  retryable?: boolean;
+  requestId?: string;
+  attempts?: number;
+}
+
+export class ProviderError extends Error {
+  constructor(message: string, details?: ProviderErrorDetails);
+  readonly code: string;
+  readonly provider?: string;
+  readonly status?: number;
+  readonly retryable: boolean;
+  readonly requestId?: string;
+  readonly attempts: number;
+  toJSON(): Record<string, JsonValue | undefined>;
+}
+
+export class ProviderConfigurationError extends ProviderError {}
+export class ProviderRequestError extends ProviderError {}
+export class ProviderResponseError extends ProviderError {}
+export class ProviderTimeoutError extends ProviderError {}
+export class ProviderAbortError extends ProviderError {}
+
+export interface OpenAICompatibleProviderOptions {
+  id?: string;
+  baseURL: string;
+  apiKey?: string | null;
+  headers?: Record<string, string>;
+  defaultHeaders?: Record<string, string>;
+  requestPolicy?: RequestPolicy | RequestPolicyOptions;
+  maxTokensField?: "max_tokens" | "max_completion_tokens";
+  fetch?: (...args: any[]) => Promise<any>;
+  sleep?: (delayMs: number, signal?: AbortSignal) => Promise<void>;
+  random?: () => number;
+  now?: () => number;
+}
+
+export class OpenAICompatibleProvider {
+  constructor(options: OpenAICompatibleProviderOptions);
+  readonly id: string;
+  readonly kind: "openai-compatible";
+  readonly requestPolicy: RequestPolicy;
+  generate(
+    request: GenerationRequest | GenerationRequestOptions,
+    options?: ProviderCallOptions,
+  ): Promise<GenerationResult>;
+  listModels(options?: ProviderCallOptions): Promise<readonly ModelInfo[]>;
+  toJSON(): { id: string; kind: "openai-compatible" };
+}
+
+export type ProviderPreset =
+  | "llamacpp"
+  | "llama.cpp"
+  | "llama-cpp"
+  | "ollama"
+  | "lmstudio"
+  | "lm-studio"
+  | "LM Studio"
+  | "openrouter"
+  | "openai"
+  | "custom"
+  | "generic";
+
+export function createProvider(
+  name: ProviderPreset,
+  options?: Partial<OpenAICompatibleProviderOptions>,
+): OpenAICompatibleProvider;
+
+export interface ModelTargetOptions {
+  name: string;
+  provider: GenerationProvider;
+  model: string;
+  requestOverrides?: {
+    maxTokens?: number | null;
+    maxOutputTokens?: number | null;
+    max_tokens?: number | null;
+    temperature?: number;
+    seed?: number;
+    stop?: readonly string[];
+    extraBody?: Metadata;
+    extra_body?: Metadata;
+  };
+}
+
+export class ModelTarget {
+  constructor(options: ModelTargetOptions);
+  readonly name: string;
+  readonly provider: GenerationProvider;
+  readonly providerName: string;
+  readonly model: string;
+  readonly requestOverrides: Readonly<Metadata>;
+  generate(
+    request: Omit<GenerationRequestOptions, "model">,
+    options?: ProviderCallOptions,
+  ): Promise<GenerationResult>;
+  toJSON(): {
+    name: string;
+    provider: string;
+    model: string;
+    parameters: {
+      max_tokens: number | null;
+      temperature: number | null;
+      seed: number | null;
+      stop: string[];
+    };
+  };
+}
+
+export interface ScoreOptions {
+  value: number;
+  passed?: boolean | null;
+  reason?: string;
+  metadata?: Metadata;
+}
+
+export class Score {
+  constructor(value: number, options?: Omit<ScoreOptions, "value">);
+  constructor(options: ScoreOptions);
+  readonly value: number;
+  readonly passed: boolean | null;
+  readonly reason: string;
+  readonly metadata: Readonly<Metadata>;
+  snapshot(): ScoreSnapshot;
+  toJSON(): ScoreSnapshot;
+  static fromSnapshot(value: ScoreOptions): Score;
+}
+
+export interface ScoreSnapshot {
+  value: number;
+  passed: boolean | null;
+  reason: string | null;
+  metadata: Metadata;
+}
+
+export interface Evaluator {
+  readonly name: string;
+  evaluate(
+    task: PromptTask,
+    generation: GenerationResult,
+    options?: { signal?: AbortSignal },
+  ): Score | Promise<Score>;
+  toConfig(): Metadata & { type: string };
+}
+
+export interface PromptTaskOptions {
+  id: string;
+  messages: readonly ChatMessage[];
+  evaluator?: Evaluator | null;
+  metadata?: Metadata;
+}
+
+export class PromptTask {
+  constructor(options: PromptTaskOptions);
+  constructor(
+    id: string,
+    messages: readonly ChatMessage[],
+    options?: Omit<PromptTaskOptions, "id" | "messages">,
+  );
+  readonly id: string;
+  readonly messages: readonly ChatMessage[];
+  readonly evaluator: Evaluator | null;
+  readonly metadata: Readonly<Metadata>;
+  snapshot(): TaskSnapshot;
+  toJSON(): TaskSnapshot;
+  static fromText(
+    id: string,
+    prompt: string,
+    options?: {
+      system?: string | null;
+      evaluator?: Evaluator | null;
+      metadata?: Metadata;
+    },
+  ): PromptTask;
+}
+
+export interface TaskSnapshot {
+  id: string;
+  messages: Array<{ role: ChatRole; content: string | null }>;
+  evaluator: (Metadata & { type: string }) | null;
+  metadata: Metadata;
+}
+
+export class ExactMatch implements Evaluator {
+  constructor(expected: string, options?: {
+    strip?: boolean;
+    ignoreCase?: boolean;
+    ignore_case?: boolean;
+  });
+  readonly name: "exact";
+  readonly expected: string;
+  readonly strip: boolean;
+  readonly ignoreCase: boolean;
+  evaluate(task: PromptTask, generation: GenerationResult): Score;
+  toConfig(): Metadata & { type: "exact" };
+}
+
+export class Contains implements Evaluator {
+  constructor(expected: string | readonly string[], options?: {
+    mode?: "all" | "any";
+    ignoreCase?: boolean;
+    ignore_case?: boolean;
+  });
+  readonly name: "contains";
+  evaluate(task: PromptTask, generation: GenerationResult): Score;
+  toConfig(): Metadata & { type: "contains" };
+}
+
+export class RegexMatch implements Evaluator {
+  constructor(pattern: string, options?: {
+    ignoreCase?: boolean;
+    ignore_case?: boolean;
+    fullMatch?: boolean;
+    full_match?: boolean;
+  });
+  readonly name: "regex";
+  evaluate(task: PromptTask, generation: GenerationResult): Score;
+  toConfig(): Metadata & { type: "regex" };
+}
+
+export class JSONMatch implements Evaluator {
+  constructor(expected?: JsonValue, options?: { compare?: boolean });
+  readonly name: "json";
+  evaluate(task: PromptTask, generation: GenerationResult): Score;
+  toConfig(): Metadata & { type: "json" };
+}
+
+export const JsonMatch: typeof JSONMatch;
+
+export class NumericMatch implements Evaluator {
+  constructor(expected: number, options?: { tolerance?: number });
+  readonly name: "numeric";
+  evaluate(task: PromptTask, generation: GenerationResult): Score;
+  toConfig(): Metadata & { type: "numeric" };
+}
+
+export function evaluatorFromConfig(config?: Metadata | null): Evaluator | null;
+
+export interface ModelJudgeOptions {
+  target: ModelTarget;
+  rubric?: string;
+  referenceAnswer?: string | null;
+  reference_answer?: string | null;
+  passThreshold?: number | null;
+  pass_threshold?: number | null;
+}
+
+export class JudgeParseError extends Error {}
+
+export class ModelJudge implements Evaluator {
+  constructor(target: ModelTarget, options?: Omit<ModelJudgeOptions, "target">);
+  constructor(options: ModelJudgeOptions);
+  readonly name: "model_judge";
+  readonly target: ModelTarget;
+  readonly rubric: string;
+  readonly referenceAnswer: string | null;
+  readonly passThreshold: number | null;
+  evaluate(
+    task: PromptTask,
+    generation: GenerationResult,
+    options?: { signal?: AbortSignal },
+  ): Promise<Score>;
+  toConfig(): Metadata & { type: "model_judge" };
+}
+
+export interface ErrorMetadata {
+  status_code?: number | null;
+  retryable?: boolean;
+  attempts?: number;
+}
+
+export interface EvaluationRecordOptions {
+  id: number;
+  target: string;
+  provider: string;
+  model: string;
+  taskId: string;
+  repetition: number;
+  startedAt: string;
+  finishedAt: string;
+  durationSeconds: number;
+  generation?: GenerationResult | null;
+  score?: Score | null;
+  error?: string | null;
+  errorMetadata?: ErrorMetadata;
+}
+
+export class EvaluationRecord {
+  constructor(options: EvaluationRecordOptions);
+  readonly id: number;
+  readonly target: string;
+  readonly provider: string;
+  readonly model: string;
+  readonly taskId: string;
+  readonly repetition: number;
+  readonly startedAt: string;
+  readonly finishedAt: string;
+  readonly durationSeconds: number;
+  readonly generation: GenerationResult | null;
+  readonly score: Score | null;
+  readonly error: string | null;
+  readonly errorMetadata: Readonly<ErrorMetadata>;
+  readonly status: "ok" | "score_error" | "generation_error";
+  snapshot(options?: { includeContent?: boolean }): Metadata;
+}
+
+export interface EvaluationSummaryRow {
+  rank: number;
+  name: string;
+  provider: string;
+  model: string;
+  runs: number;
+  generated: number;
+  successful: number;
+  errors: number;
+  scored: number;
+  average_score: number | null;
+  pass_rate: number | null;
+  average_latency_seconds: number | null;
+  input_tokens: number;
+  output_tokens: number;
+  judge_input_tokens: number;
+  judge_output_tokens: number;
+  judge_latency_seconds: number;
+  elo: number;
+}
+
+export interface EvaluationRunOptions {
+  id: string;
+  name: string;
+  startedAt: string;
+  finishedAt: string;
+  models: readonly Metadata[];
+  tasks: readonly Metadata[];
+  records: readonly EvaluationRecord[];
+  includeContent?: boolean;
+}
+
+export class EvaluationRun {
+  constructor(options: EvaluationRunOptions);
+  readonly id: string;
+  readonly name: string;
+  readonly startedAt: string;
+  readonly finishedAt: string;
+  readonly models: readonly Metadata[];
+  readonly tasks: readonly Metadata[];
+  readonly records: readonly EvaluationRecord[];
+  readonly includeContent: boolean;
+  arena(): Arena;
+  summary(): EvaluationSummaryRow[];
+  snapshot(): Metadata;
+  toJSON(): Metadata;
+  toJSONString(indent?: number | null): string;
+}
+
+export interface EvaluationRunnerOptions {
+  maxConcurrency?: number;
+  repetitions?: number;
+  includeContent?: boolean;
+  now?: () => string;
+  clock?: () => number;
+}
+
+export class EvaluationRunner {
+  constructor(
+    models: readonly ModelTarget[],
+    tasks: readonly PromptTask[],
+    options?: EvaluationRunnerOptions,
+  );
+  readonly models: readonly ModelTarget[];
+  readonly tasks: readonly PromptTask[];
+  readonly maxConcurrency: number;
+  readonly repetitions: number;
+  readonly includeContent: boolean;
+  run(options?: {
+    name?: string;
+    signal?: AbortSignal;
+    progress?: (
+      completed: number,
+      total: number,
+      record: EvaluationRecord,
+    ) => void | Promise<void>;
+  }): Promise<EvaluationRun>;
+}
+
+export function runFromSnapshot(value: Metadata): EvaluationRun;
+export function runFromObject(value: Metadata): EvaluationRun;
+export function runFromJSON(value: string): EvaluationRun;
+
+export function renderHTMLReport(
+  run: EvaluationRun,
+  options?: { title?: string },
+): string;
+export const renderHtmlReport: typeof renderHTMLReport;
+export function writeHTMLReport(
+  run: EvaluationRun,
+  path: string | URL | Uint8Array,
+  options?: { title?: string },
+): Promise<string | URL | Uint8Array>;
+export const writeHtmlReport: typeof writeHTMLReport;

package/javascript/src/index.js

@@ -520,3 +520,52 @@ export class Arena {
     return Arena.fromSnapshot(snapshot);
   }
 }
+
+export {
+  ChatMessage,
+  GenerationRequest,
+  GenerationResult,
+  ModelInfo,
+  ModelTarget,
+  OpenAICompatibleProvider,
+  ProviderAbortError,
+  ProviderConfigurationError,
+  ProviderError,
+  ProviderRequestError,
+  ProviderResponseError,
+  ProviderTimeoutError,
+  RequestPolicy,
+  TokenUsage,
+  createProvider,
+} from "./providers.js";
+
+export {
+  Contains,
+  ExactMatch,
+  JSONMatch,
+  JsonMatch,
+  NumericMatch,
+  PromptTask,
+  RegexMatch,
+  Score,
+  evaluatorFromConfig,
+} from "./tasks.js";
+
+export {
+  RUN_SCHEMA_VERSION,
+  EvaluationRecord,
+  EvaluationRun,
+  EvaluationRunner,
+  JudgeParseError,
+  ModelJudge,
+  runFromJSON,
+  runFromObject,
+  runFromSnapshot,
+} from "./evaluation.js";
+
+export {
+  renderHTMLReport,
+  renderHtmlReport,
+  writeHTMLReport,
+  writeHtmlReport,
+} from "./report.js";