@tangle-network/agent-eval 0.120.2 → 0.120.4

package/dist/multishot/index.js

@@ -0,0 +1,589 @@
+import {
+  runAgentMatrix
+} from "../chunk-JHCHEVET.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/multishot/router.ts
+async function routerCompletion(req) {
+  const body = {
+    model: req.model,
+    messages: req.messages,
+    temperature: req.temperature ?? 0.7,
+    max_tokens: req.maxTokens ?? 2e3
+  };
+  if (req.tools?.length) body.tools = req.tools;
+  const url = `${req.baseUrl.replace(/\/+$/, "")}/chat/completions`;
+  const res = await fetch(url, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${req.apiKey}`, "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: req.signal
+  });
+  if (!res.ok) {
+    const text = await res.text();
+    throw new Error(`router ${res.status}: ${text.slice(0, 300)}`);
+  }
+  const json = await res.json();
+  const choice = json.choices[0];
+  if (!choice) throw new Error(`router returned no choices: ${JSON.stringify(json).slice(0, 200)}`);
+  return { message: choice.message, usage: json.usage };
+}
+function estimateRouterCost(model, usage) {
+  if (!usage) return 0;
+  const inputTok = usage.prompt_tokens ?? 0;
+  const outputTok = usage.completion_tokens ?? 0;
+  let inPer1k = 3e-3;
+  let outPer1k = 0.015;
+  if (model.includes("gpt-4o-mini")) {
+    inPer1k = 15e-5;
+    outPer1k = 6e-4;
+  } else if (model.includes("gpt-5.4") || model.includes("claude-sonnet")) {
+    inPer1k = 3e-3;
+    outPer1k = 0.015;
+  } else if (model.includes("kimi") || model.includes("glm") || model.includes("deepseek")) {
+    inPer1k = 5e-4;
+    outPer1k = 2e-3;
+  }
+  return (inputTok * inPer1k + outputTok * outPer1k) / 1e3;
+}
+function defaultRouterBaseUrl() {
+  return (process.env.TANGLE_ROUTER_BASE_URL ?? "https://router.tangle.tools/v1").replace(
+    /\/+$/,
+    ""
+  );
+}
+function requireRouterApiKey() {
+  const key = process.env.TANGLE_API_KEY;
+  if (!key) throw new Error("multishot requires TANGLE_API_KEY (router-scoped sk-tan-* key)");
+  return key;
+}
+
+// src/multishot/default-tools.ts
+var DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
+var DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
+var GENERIC_RESEARCHER_SYSTEM = "You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.";
+var GENERIC_CODER_SYSTEM = "You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.";
+var DEFAULT_DELEGATE_RESEARCH_TOOL = {
+  type: "function",
+  function: {
+    name: "delegate_research",
+    description: "Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.",
+    parameters: {
+      type: "object",
+      properties: {
+        question: { type: "string", description: "Specific question to research" },
+        scope: {
+          type: "string",
+          description: "Optional scope: time window, geography, jurisdiction, segment"
+        }
+      },
+      required: ["question"]
+    }
+  }
+};
+var DEFAULT_DELEGATE_CODE_TOOL = {
+  type: "function",
+  function: {
+    name: "delegate_code",
+    description: "Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.",
+    parameters: {
+      type: "object",
+      properties: {
+        goal: { type: "string", description: "What the code must accomplish" },
+        language: {
+          type: "string",
+          description: "Optional language preference (default: TypeScript)"
+        }
+      },
+      required: ["goal"]
+    }
+  }
+};
+function createResearchExecutor(config = {}) {
+  const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM;
+  const model = config.model ?? DEFAULT_RESEARCHER_MODEL;
+  return async (args, ctx) => {
+    const question = String(args.question ?? "");
+    const scope = args.scope ? String(args.scope) : void 0;
+    const { message, usage } = await routerCompletion({
+      apiKey: ctx.apiKey,
+      baseUrl: ctx.baseUrl,
+      model,
+      temperature: 0.3,
+      maxTokens: 1800,
+      messages: [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: `Research: ${question}${scope ? `
+Scope: ${scope}` : ""}` }
+      ],
+      signal: ctx.signal
+    });
+    return { content: message.content ?? "", costUsd: estimateRouterCost(model, usage) };
+  };
+}
+function createCodeExecutor(config = {}) {
+  const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM;
+  const model = config.model ?? DEFAULT_CODER_MODEL;
+  return async (args, ctx) => {
+    const goal = String(args.goal ?? "");
+    const language = args.language ? String(args.language) : "TypeScript";
+    const { message, usage } = await routerCompletion({
+      apiKey: ctx.apiKey,
+      baseUrl: ctx.baseUrl,
+      model,
+      temperature: 0.2,
+      maxTokens: 2e3,
+      messages: [
+        { role: "system", content: `${systemPrompt}
+
+Language: ${language}` },
+        { role: "user", content: `Produce: ${goal}` }
+      ],
+      signal: ctx.signal
+    });
+    return { content: message.content ?? "", costUsd: estimateRouterCost(model, usage) };
+  };
+}
+function defaultDelegationTools(config = {}) {
+  return {
+    tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],
+    executors: {
+      delegate_research: createResearchExecutor(config.research),
+      delegate_code: createCodeExecutor(config.code)
+    },
+    artifactTypeFor: (name) => name === "delegate_research" ? "research" : name === "delegate_code" ? "code" : void 0
+  };
+}
+
+// src/multishot/judges.ts
+var DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
+async function runJudge(judge, input) {
+  const apiKey = judge.apiKey ?? requireRouterApiKey();
+  const baseUrl = judge.baseUrl ?? defaultRouterBaseUrl();
+  const model = judge.model ?? process.env.JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL;
+  const prompt = judge.buildPrompt(input);
+  let raw = "";
+  try {
+    const { message } = await routerCompletion({
+      apiKey,
+      baseUrl,
+      model,
+      temperature: 0,
+      maxTokens: judge.maxTokens ?? 1500,
+      messages: [
+        { role: "system", content: judge.systemPrompt },
+        { role: "user", content: prompt }
+      ]
+    });
+    raw = (message.content ?? "").trim();
+  } catch (err) {
+    return {
+      dimensions: {},
+      composite: 0,
+      failed: true,
+      notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+  let parsed = null;
+  try {
+    const cleaned = raw.replace(/^```json\s*/i, "").replace(/```\s*$/, "").trim();
+    parsed = JSON.parse(cleaned);
+  } catch {
+    return {
+      dimensions: {},
+      composite: 0,
+      failed: true,
+      notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`
+    };
+  }
+  const dimensions = {};
+  let sum = 0;
+  for (const dim of judge.dimensions) {
+    const v = Number(parsed[dim.key] ?? 0);
+    const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0;
+    dimensions[dim.key] = clamped;
+    sum += clamped;
+  }
+  return {
+    dimensions,
+    composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,
+    notes: typeof parsed.notes === "string" ? parsed.notes : ""
+  };
+}
+function renderDimensions(dims) {
+  return dims.map((d) => `- ${d.key}: ${d.description}`).join("\n");
+}
+function renderJsonFooter(dims) {
+  const fields = dims.map((d) => `"${d.key}":N`).join(",");
+  return `Respond with ONLY this JSON (no markdown, no preamble):
+{${fields},"notes":"1-2 sentence critique"}`;
+}
+
+// src/multishot/matrix.ts
+import { mkdirSync, writeFileSync } from "fs";
+import { join } from "path";
+
+// src/multishot/types.ts
+var MultishotDriverEmptyError = class extends Error {
+  constructor(turn) {
+    super(`multishot: driver returned empty content twice at turn ${turn} \u2014 failing loud`);
+    this.turn = turn;
+    this.name = "MultishotDriverEmptyError";
+  }
+  turn;
+};
+var MultishotFatalToolError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "MultishotFatalToolError";
+  }
+};
+
+// src/multishot/multishot.ts
+async function runMultishot(opts) {
+  const apiKey = opts.apiKey ?? requireRouterApiKey();
+  const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl();
+  const maxTurns = opts.maxTurns ?? 10;
+  const maxToolDispatches = opts.maxToolDispatches ?? 4;
+  const agentModel = opts.agentModel ?? "openai/gpt-5.4";
+  const driverModel = opts.driverModel ?? "openai/gpt-4o-mini";
+  const driverModels = [driverModel, ...opts.driverFallbackModels ?? []];
+  const agentMaxTokens = opts.agentMaxTokens ?? 2500;
+  const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2e3;
+  const driverMaxTokens = opts.driverMaxTokens ?? 600;
+  const bundle = opts.tools && opts.toolExecutors ? {
+    tools: opts.tools,
+    executors: opts.toolExecutors,
+    artifactTypeFor: opts.artifactTypeFor ?? (() => void 0)
+  } : defaultDelegationTools();
+  const tools = opts.tools ?? bundle.tools;
+  const executors = opts.toolExecutors ?? bundle.executors;
+  const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor;
+  const routerTransport = (req) => routerCompletion({ apiKey, baseUrl, ...req });
+  const agentTransport = opts.agentTransport ?? routerTransport;
+  const driverTransport = opts.driverTransport ?? routerTransport;
+  const start = Date.now();
+  const transcript = [];
+  const artifacts = [];
+  let toolCalls = 0;
+  let totalCostUsd = 0;
+  const opener = opts.shape.buildOpener(opts.persona);
+  transcript.push({ role: "user", content: opener });
+  const systemPrompt = opts.profile.prompt?.systemPrompt ?? "";
+  const agentMessages = [
+    { role: "system", content: systemPrompt },
+    { role: "user", content: opener }
+  ];
+  for (let turn = 0; turn < maxTurns; turn++) {
+    if (opts.signal?.aborted) throw new Error("multishot aborted");
+    let dispatchesThisTurn = 0;
+    while (true) {
+      const {
+        message: agentMsg,
+        usage: agentUsage,
+        costUsd: agentCostUsd
+      } = await agentTransport({
+        model: agentModel,
+        messages: agentMessages,
+        tools,
+        temperature: 0.7,
+        maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,
+        signal: opts.signal
+      });
+      totalCostUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage);
+      const agentText = (agentMsg.content ?? "").trim();
+      const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({
+        id: tc.id,
+        name: tc.function.name,
+        args: (() => {
+          try {
+            return JSON.parse(tc.function.arguments);
+          } catch {
+            return {};
+          }
+        })()
+      }));
+      agentMessages.push({
+        role: "assistant",
+        content: agentText || null,
+        ...agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}
+      });
+      transcript.push({
+        role: "assistant",
+        content: agentText,
+        toolCalls: agentToolCalls.length > 0 ? agentToolCalls : void 0
+      });
+      if (agentToolCalls.length === 0) break;
+      dispatchesThisTurn += agentToolCalls.length;
+      if (dispatchesThisTurn > maxToolDispatches) {
+        throw new Error(
+          `multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`
+        );
+      }
+      for (const tc of agentToolCalls) {
+        toolCalls++;
+        let toolResult = "";
+        try {
+          const executor = executors[tc.name];
+          if (!executor) {
+            toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` });
+          } else {
+            const r = await executor(tc.args, { apiKey, baseUrl, signal: opts.signal });
+            toolResult = r.content;
+            totalCostUsd += r.costUsd;
+            const artifactType = artifactTypeFor(tc.name);
+            if (artifactType) {
+              artifacts.push({
+                type: artifactType,
+                turn,
+                invocation: { name: tc.name, args: tc.args },
+                content: toolResult
+              });
+            }
+          }
+        } catch (err) {
+          if (err instanceof MultishotFatalToolError) throw err;
+          toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) });
+        }
+        agentMessages.push({ role: "tool", tool_call_id: tc.id, content: toolResult || "done" });
+        transcript.push({ role: "tool", content: toolResult || "done", toolCallId: tc.id });
+      }
+    }
+    if (turn < maxTurns - 1) {
+      const driver = await driverTurn({
+        transport: driverTransport,
+        persona: opts.persona,
+        shape: opts.shape,
+        transcript,
+        turn,
+        models: driverModels,
+        maxTokens: driverMaxTokens,
+        signal: opts.signal
+      });
+      totalCostUsd += driver.costUsd;
+      agentMessages.push({ role: "user", content: driver.content });
+      transcript.push({ role: "user", content: driver.content });
+    }
+  }
+  return { transcript, artifacts, toolCalls, durationMs: Date.now() - start, costUsd: totalCostUsd };
+}
+async function driverTurn(opts) {
+  const driverSystem = opts.shape.buildDriverSystemPrompt(opts.persona);
+  const driverMessages = [{ role: "system", content: driverSystem }];
+  for (const msg of opts.transcript) {
+    if (msg.role === "tool") continue;
+    const content = driverVisibleContent(msg);
+    if (!content) continue;
+    if (msg.role === "assistant") driverMessages.push({ role: "user", content });
+    else if (msg.role === "user") driverMessages.push({ role: "assistant", content });
+  }
+  for (const model of opts.models) {
+    for (let attempt = 0; attempt < 2; attempt++) {
+      const { message, usage, costUsd } = await opts.transport({
+        model,
+        messages: driverMessages,
+        temperature: 0.9,
+        maxTokens: opts.maxTokens,
+        signal: opts.signal
+      });
+      const content = (message.content ?? "").trim();
+      if (content.length > 0)
+        return { content, costUsd: costUsd ?? estimateRouterCost(model, usage) };
+    }
+  }
+  throw new MultishotDriverEmptyError(opts.turn);
+}
+function driverVisibleContent(msg) {
+  const text = msg.content.trim();
+  if (text.length > 0) return text;
+  if (msg.role !== "assistant" || !msg.toolCalls?.length) return null;
+  const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean);
+  if (toolNames.length === 0) return "Agent called tools.";
+  return `Agent called ${toolNames.length === 1 ? "tool" : "tools"}: ${toolNames.join(", ")}.`;
+}
+
+// src/multishot/matrix.ts
+function meanCompositeExcludingFailed(scores) {
+  if (scores.length === 0) return 0;
+  const live = scores.filter((s) => !s.failed);
+  if (live.length === 0) return null;
+  return live.reduce((sum, s) => sum + s.composite, 0) / live.length;
+}
+function computeCellComposite(input) {
+  const contributions = [];
+  if (!input.conversation.failed) contributions.push(input.conversation.composite);
+  const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : void 0;
+  if (typeof codeMean === "number") contributions.push(codeMean);
+  const contentMean = input.contentReviews ? meanCompositeExcludingFailed(input.contentReviews) : void 0;
+  if (typeof contentMean === "number") contributions.push(contentMean);
+  return {
+    composite: contributions.length === 0 ? 0 : contributions.reduce((s, v) => s + v, 0) / contributions.length,
+    codeComposite: codeMean ?? 0,
+    contentComposite: contentMean ?? 0,
+    allJudgesFailed: contributions.length === 0
+  };
+}
+async function runMultishotMatrix(opts) {
+  const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ["code"]);
+  const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ["research"]);
+  mkdirSync(opts.runDir, { recursive: true });
+  const matrix = await runAgentMatrix({
+    axes: [
+      { name: "profile", values: opts.profiles },
+      { name: "persona", values: opts.personas.map((p) => ({ id: p.id, value: p })) }
+    ],
+    reps: opts.reps ?? 1,
+    maxConcurrency: opts.maxConcurrency ?? 2,
+    costCeiling: opts.costCeiling,
+    async runCell(cell) {
+      const profile = cell.axes.profile?.value;
+      const persona = cell.axes.persona?.value;
+      const profileId = String(cell.axes.profile?.id ?? "unknown");
+      const personaId = String(cell.axes.persona?.id ?? "unknown");
+      const sim = await runMultishot({
+        profile,
+        persona,
+        shape: opts.shape,
+        tools: opts.tools,
+        toolExecutors: opts.toolExecutors,
+        artifactTypeFor: opts.artifactTypeFor,
+        maxTurns: opts.maxTurns,
+        maxToolDispatches: opts.maxToolDispatches,
+        agentModel: opts.agentModel,
+        driverModel: opts.driverModel,
+        driverFallbackModels: opts.driverFallbackModels,
+        agentMaxTokens: opts.agentMaxTokens,
+        toolFollowupMaxTokens: opts.toolFollowupMaxTokens,
+        driverMaxTokens: opts.driverMaxTokens,
+        agentTransport: opts.agentTransport,
+        driverTransport: opts.driverTransport,
+        apiKey: opts.apiKey,
+        baseUrl: opts.baseUrl
+      });
+      const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
+      const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
+      const [conversation, codeReviews, contentReviews] = await Promise.all([
+        runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {
+          transcript: sim.transcript,
+          persona
+        }),
+        opts.judges.codeReview ? Promise.all(
+          codeArtifacts.map(
+            (artifact) => runJudge(withJudgeMaxTokens(opts.judges.codeReview, opts.judgeMaxTokens), {
+              artifact,
+              persona
+            }).then((s) => ({
+              ...s,
+              turn: artifact.turn,
+              type: artifact.type
+            }))
+          )
+        ) : Promise.resolve([]),
+        opts.judges.contentQuality ? Promise.all(
+          contentArtifacts.map(
+            (artifact) => runJudge(withJudgeMaxTokens(opts.judges.contentQuality, opts.judgeMaxTokens), {
+              artifact,
+              persona
+            }).then((s) => ({
+              ...s,
+              turn: artifact.turn,
+              type: artifact.type
+            }))
+          )
+        ) : Promise.resolve([])
+      ]);
+      const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({
+        conversation,
+        codeReviews: opts.judges.codeReview ? codeReviews : void 0,
+        contentReviews: opts.judges.contentQuality ? contentReviews : void 0
+      });
+      const cellScore = { composite, conversation };
+      if (opts.judges.codeReview)
+        cellScore.codeReview = { perArtifact: codeReviews, composite: codeComposite };
+      if (opts.judges.contentQuality)
+        cellScore.contentQuality = { perArtifact: contentReviews, composite: contentComposite };
+      const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`);
+      mkdirSync(cellDir, { recursive: true });
+      writeFileSync(join(cellDir, "transcript.json"), JSON.stringify(sim.transcript, null, 2));
+      writeFileSync(join(cellDir, "artifacts.json"), JSON.stringify(sim.artifacts, null, 2));
+      writeFileSync(join(cellDir, "scores.json"), JSON.stringify(cellScore, null, 2));
+      const notes = [`convo=${conversation.composite.toFixed(1)}`];
+      if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`);
+      if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`);
+      if (allJudgesFailed) notes.push("all-judges-failed");
+      return {
+        output: {
+          turns: sim.transcript.length,
+          toolCalls: sim.toolCalls,
+          artifactCount: sim.artifacts.length
+        },
+        verdict: { valid: composite >= 5, score: composite, notes: notes.join(" ") },
+        costUsd: sim.costUsd,
+        durationMs: sim.durationMs
+      };
+    }
+  });
+  const summary = {
+    cells: matrix.summary.totalCells,
+    passRate: matrix.summary.overallPassRate,
+    meanScore: matrix.summary.overallMeanScore,
+    totalCostUsd: matrix.summary.totalCostUsd,
+    durationMs: matrix.summary.durationMs,
+    runsExecuted: matrix.summary.runsExecuted,
+    cellsSkipped: matrix.summary.cellsSkipped,
+    byProfile: matrix.byAxis.profile,
+    byPersona: matrix.byAxis.persona
+  };
+  writeFileSync(join(opts.runDir, "summary.json"), JSON.stringify(summary, null, 2));
+  const md = [
+    `# Multishot matrix`,
+    ``,
+    `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **Cost**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1e3).toFixed(0)}s`,
+    ``,
+    `## By profile`,
+    ``,
+    "| profile | pass | mean | cost |",
+    "|---|---|---|---|",
+    ...Object.entries(matrix.byAxis.profile ?? {}).map(
+      ([id, s]) => `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`
+    ),
+    ``,
+    `## By persona`,
+    ``,
+    "| persona | pass | mean | cost |",
+    "|---|---|---|---|",
+    ...Object.entries(matrix.byAxis.persona ?? {}).map(
+      ([id, s]) => `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`
+    ),
+    ``
+  ];
+  writeFileSync(join(opts.runDir, "summary.md"), md.join("\n"));
+  return { matrix };
+}
+function withJudgeMaxTokens(judge, maxTokens) {
+  if (maxTokens === void 0 || judge.maxTokens !== void 0) return judge;
+  return { ...judge, maxTokens };
+}
+export {
+  DEFAULT_CODER_MODEL,
+  DEFAULT_DELEGATE_CODE_TOOL,
+  DEFAULT_DELEGATE_RESEARCH_TOOL,
+  DEFAULT_JUDGE_MODEL,
+  DEFAULT_RESEARCHER_MODEL,
+  MultishotDriverEmptyError,
+  MultishotFatalToolError,
+  computeCellComposite,
+  createCodeExecutor,
+  createResearchExecutor,
+  defaultDelegationTools,
+  defaultRouterBaseUrl,
+  estimateRouterCost,
+  renderDimensions,
+  renderJsonFooter,
+  requireRouterApiKey,
+  routerCompletion,
+  runJudge,
+  runMultishot,
+  runMultishotMatrix
+};
+//# sourceMappingURL=index.js.map