@tangle-network/agent-eval 0.37.0 → 0.40.1

package/dist/multishot/index.d.ts

@@ -0,0 +1,276 @@
+import { AgentProfile } from '@tangle-network/sandbox';
+import { M as MatrixResult } from '../types-DHqkLwEU.js';
+import '@tangle-network/agent-runtime/loops';
+
+interface MultishotMessage {
+    role: 'user' | 'assistant' | 'tool';
+    content: string;
+    toolCallId?: string;
+    toolCalls?: Array<{
+        id: string;
+        name: string;
+        args: Record<string, unknown>;
+    }>;
+}
+interface MultishotArtifact {
+    type: string;
+    turn: number;
+    invocation: {
+        name: string;
+        args: Record<string, unknown>;
+    };
+    content: string;
+}
+interface MultishotResult {
+    transcript: MultishotMessage[];
+    artifacts: MultishotArtifact[];
+    toolCalls: number;
+    durationMs: number;
+    costUsd: number;
+}
+interface MultishotToolDefinition {
+    type: 'function';
+    function: {
+        name: string;
+        description: string;
+        parameters: Record<string, unknown>;
+    };
+}
+type MultishotToolExecutor = (args: Record<string, unknown>, ctx: {
+    apiKey: string;
+    baseUrl: string;
+    signal?: AbortSignal;
+}) => Promise<{
+    content: string;
+    costUsd: number;
+}>;
+interface MultishotPersona {
+    /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */
+    id: string;
+    /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */
+    [k: string]: unknown;
+}
+interface MultishotShape<TPersona extends MultishotPersona> {
+    /** Opening user message (turn 0) — the persona's first ask. */
+    buildOpener: (persona: TPersona) => string;
+    /** System prompt the driver LLM uses to roleplay the persona. Should set
+     *  voice, goals, constraints, time-pressure, and the "never go silent" rule. */
+    buildDriverSystemPrompt: (persona: TPersona) => string;
+}
+declare class MultishotDriverEmptyError extends Error {
+    readonly turn: number;
+    constructor(turn: number);
+}
+
+interface RouterCompletionRequest {
+    apiKey: string;
+    baseUrl: string;
+    model: string;
+    messages: Array<Record<string, unknown>>;
+    tools?: MultishotToolDefinition[];
+    temperature?: number;
+    maxTokens?: number;
+    signal?: AbortSignal;
+}
+interface RouterToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+interface RouterCompletionResponse {
+    message: {
+        content?: string | null;
+        tool_calls?: RouterToolCall[];
+    };
+    usage?: {
+        prompt_tokens?: number;
+        completion_tokens?: number;
+    };
+}
+declare function routerCompletion(req: RouterCompletionRequest): Promise<RouterCompletionResponse>;
+declare function estimateRouterCost(model: string, usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+}): number;
+declare function defaultRouterBaseUrl(): string;
+declare function requireRouterApiKey(): string;
+
+declare const DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
+declare const DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
+interface DefaultResearcherConfig {
+    /** Replace the system prompt to bias the researcher toward a domain's
+     *  citation style. Defaults to a generic "cite sources by name" prompt. */
+    systemPrompt?: string;
+    model?: string;
+}
+interface DefaultCoderConfig {
+    /** Replace the system prompt to bias the coder toward a language /
+     *  framework / artifact style. */
+    systemPrompt?: string;
+    model?: string;
+}
+declare const DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition;
+declare const DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition;
+declare function createResearchExecutor(config?: DefaultResearcherConfig): MultishotToolExecutor;
+declare function createCodeExecutor(config?: DefaultCoderConfig): MultishotToolExecutor;
+interface DefaultToolsConfig {
+    research?: DefaultResearcherConfig;
+    code?: DefaultCoderConfig;
+    /** When true (default), each tool result is recorded as a typed artifact:
+     *  research → type='research', code → type='code'. */
+    recordArtifacts?: boolean;
+}
+interface DefaultToolsBundle {
+    tools: MultishotToolDefinition[];
+    executors: Record<string, MultishotToolExecutor>;
+    artifactTypeFor: (toolName: string) => string | undefined;
+}
+declare function defaultDelegationTools(config?: DefaultToolsConfig): DefaultToolsBundle;
+
+declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
+interface JudgeDimension {
+    /** JSON field name + score key. */
+    key: string;
+    /** Description shown in the judge's user prompt. */
+    description: string;
+}
+interface JudgeConfig<TInput> {
+    /** Display name (for trace + log). */
+    name: string;
+    /** Model used for this judge. */
+    model?: string;
+    /** 0-10 scored dimensions. */
+    dimensions: JudgeDimension[];
+    /** Judge system prompt — sets persona + JSON-only constraint. */
+    systemPrompt: string;
+    /** Build the user prompt from the typed input. Must include "Respond with
+     *  ONLY this JSON: { ... }" listing each dimension key. */
+    buildPrompt: (input: TInput) => string;
+    /** Optional model + api overrides. */
+    apiKey?: string;
+    baseUrl?: string;
+}
+interface JudgeScore {
+    /** Per-dimension 0-10 score. Missing dims default to 0. */
+    dimensions: Record<string, number>;
+    /** Mean across dimensions. */
+    composite: number;
+    /** Free-form 1-2 sentence critique from the judge (when provided). */
+    notes: string;
+}
+declare function runJudge<TInput>(judge: JudgeConfig<TInput>, input: TInput): Promise<JudgeScore>;
+/** Convenience: stringified dimension list for inclusion in a judge prompt.
+ *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
+declare function renderDimensions(dims: readonly JudgeDimension[]): string;
+/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
+declare function renderJsonFooter(dims: readonly JudgeDimension[]): string;
+
+interface ConversationJudgeInput<TPersona extends MultishotPersona> {
+    transcript: MultishotMessage[];
+    persona: TPersona;
+}
+interface ArtifactJudgeInput<TPersona extends MultishotPersona> {
+    artifact: MultishotArtifact;
+    persona: TPersona;
+}
+interface MultishotJudges<TPersona extends MultishotPersona> {
+    /** Scores the full transcript end-to-end (always runs). */
+    conversation: JudgeConfig<ConversationJudgeInput<TPersona>>;
+    /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */
+    codeReview?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+    /** Scores each non-code (research/content/template) artifact. Optional. */
+    contentQuality?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+    /** Which artifact types route to codeReview. Defaults to ['code']. */
+    codeArtifactTypes?: string[];
+    /** Which artifact types route to contentQuality. Defaults to ['research']. */
+    contentArtifactTypes?: string[];
+}
+interface CellCompositeScore {
+    composite: number;
+    conversation: JudgeScore;
+    codeReview?: {
+        perArtifact: Array<JudgeScore & {
+            turn: number;
+            type: string;
+        }>;
+        composite: number;
+    };
+    contentQuality?: {
+        perArtifact: Array<JudgeScore & {
+            turn: number;
+            type: string;
+        }>;
+        composite: number;
+    };
+}
+interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {
+    /** AgentProfile axis (matrix primary). */
+    profiles: Array<{
+        id: string;
+        value: AgentProfile;
+    }>;
+    /** Persona axis. */
+    personas: TPersona[];
+    /** Persona-shaping callbacks. */
+    shape: MultishotShape<TPersona>;
+    /** Judge configurations. */
+    judges: MultishotJudges<TPersona>;
+    /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+    tools?: MultishotToolDefinition[];
+    /** Map from tool name → inline executor. Must align with `tools`. */
+    toolExecutors?: Record<string, MultishotToolExecutor>;
+    /** Tool name → artifact type label. Defaults to research/code mapping. */
+    artifactTypeFor?: (toolName: string) => string | undefined;
+    /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */
+    runDir: string;
+    /** Replicates per (profile, persona) cell. */
+    reps?: number;
+    /** Max conversation turns per cell. */
+    maxTurns?: number;
+    /** Max concurrent cells. */
+    maxConcurrency?: number;
+    /** Total $ ceiling across the matrix; cells aborted past this. */
+    costCeiling?: number;
+    /** Agent model. */
+    agentModel?: string;
+    /** Driver model. */
+    driverModel?: string;
+    /** Pass-thru fields. */
+    apiKey?: string;
+    baseUrl?: string;
+}
+interface CellOutput {
+    turns: number;
+    toolCalls: number;
+    artifactCount: number;
+}
+interface RunMultishotMatrixResult {
+    matrix: MatrixResult<CellOutput>;
+}
+declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
+
+interface RunMultishotOptions<TPersona extends MultishotPersona> {
+    profile: AgentProfile;
+    persona: TPersona;
+    shape: MultishotShape<TPersona>;
+    /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+    tools?: MultishotToolDefinition[];
+    /** Map from tool name → executor invoked inline when the agent emits a tool_call. */
+    toolExecutors?: Record<string, MultishotToolExecutor>;
+    /** Map from tool name → artifact type label written into MultishotArtifact.type.
+     *  Tools without a mapping still execute, but their results aren't surfaced as
+     *  typed artifacts (only as tool messages in the transcript). */
+    artifactTypeFor?: (toolName: string) => string | undefined;
+    maxTurns?: number;
+    agentModel?: string;
+    driverModel?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    signal?: AbortSignal;
+}
+declare function runMultishot<TPersona extends MultishotPersona>(opts: RunMultishotOptions<TPersona>): Promise<MultishotResult>;
+
+export { type ArtifactJudgeInput, type CellCompositeScore, type ConversationJudgeInput, DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, type DefaultCoderConfig, type DefaultResearcherConfig, type DefaultToolsBundle, type DefaultToolsConfig, type JudgeConfig, type JudgeDimension, type JudgeScore, type MultishotArtifact, MultishotDriverEmptyError, type MultishotJudges, type MultishotMessage, type MultishotPersona, type MultishotResult, type MultishotShape, type MultishotToolDefinition, type MultishotToolExecutor, type RouterCompletionRequest, type RouterCompletionResponse, type RouterToolCall, type RunMultishotMatrixOptions, type RunMultishotMatrixResult, type RunMultishotOptions, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultRouterBaseUrl, estimateRouterCost, renderDimensions, renderJsonFooter, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };