@agentgrader/optimizer 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,193 @@
+import { z } from 'zod';
+import { AgentConfig } from '@agentgrader/core';
+
+/**
+ * Each populated array becomes one axis of the cartesian product. Axes left
+ * `undefined` are not varied - their value (if any) comes from `base`.
+ */
+declare const MatrixDimensionsSchema: z.ZodEffects<z.ZodObject<{
+    model: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    temperature: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
+    system_prompt: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    max_steps: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
+    toolkits: z.ZodOptional<z.ZodArray<z.ZodArray<z.ZodString, "many">, "many">>;
+}, "strip", z.ZodTypeAny, {
+    model?: string[] | undefined;
+    max_steps?: number[] | undefined;
+    temperature?: number[] | undefined;
+    system_prompt?: string[] | undefined;
+    toolkits?: string[][] | undefined;
+}, {
+    model?: string[] | undefined;
+    max_steps?: number[] | undefined;
+    temperature?: number[] | undefined;
+    system_prompt?: string[] | undefined;
+    toolkits?: string[][] | undefined;
+}>, {
+    model?: string[] | undefined;
+    max_steps?: number[] | undefined;
+    temperature?: number[] | undefined;
+    system_prompt?: string[] | undefined;
+    toolkits?: string[][] | undefined;
+}, {
+    model?: string[] | undefined;
+    max_steps?: number[] | undefined;
+    temperature?: number[] | undefined;
+    system_prompt?: string[] | undefined;
+    toolkits?: string[][] | undefined;
+}>;
+declare const MatrixSchema: z.ZodObject<{
+    /** used as the prefix for generated agent config ids/names */
+    name: z.ZodString;
+    base: z.ZodDefault<z.ZodObject<{
+        model: z.ZodOptional<z.ZodString>;
+        max_steps: z.ZodOptional<z.ZodNumber>;
+        temperature: z.ZodOptional<z.ZodNumber>;
+        system_prompt: z.ZodOptional<z.ZodString>;
+        tools: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        toolkits: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        model?: string | undefined;
+        max_steps?: number | undefined;
+        temperature?: number | undefined;
+        system_prompt?: string | undefined;
+        tools?: string[] | undefined;
+        toolkits?: string[] | undefined;
+    }, {
+        model?: string | undefined;
+        max_steps?: number | undefined;
+        temperature?: number | undefined;
+        system_prompt?: string | undefined;
+        tools?: string[] | undefined;
+        toolkits?: string[] | undefined;
+    }>>;
+    dimensions: z.ZodEffects<z.ZodObject<{
+        model: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        temperature: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
+        system_prompt: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        max_steps: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
+        toolkits: z.ZodOptional<z.ZodArray<z.ZodArray<z.ZodString, "many">, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        model?: string[] | undefined;
+        max_steps?: number[] | undefined;
+        temperature?: number[] | undefined;
+        system_prompt?: string[] | undefined;
+        toolkits?: string[][] | undefined;
+    }, {
+        model?: string[] | undefined;
+        max_steps?: number[] | undefined;
+        temperature?: number[] | undefined;
+        system_prompt?: string[] | undefined;
+        toolkits?: string[][] | undefined;
+    }>, {
+        model?: string[] | undefined;
+        max_steps?: number[] | undefined;
+        temperature?: number[] | undefined;
+        system_prompt?: string[] | undefined;
+        toolkits?: string[][] | undefined;
+    }, {
+        model?: string[] | undefined;
+        max_steps?: number[] | undefined;
+        temperature?: number[] | undefined;
+        system_prompt?: string[] | undefined;
+        toolkits?: string[][] | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    base: {
+        model?: string | undefined;
+        max_steps?: number | undefined;
+        temperature?: number | undefined;
+        system_prompt?: string | undefined;
+        tools?: string[] | undefined;
+        toolkits?: string[] | undefined;
+    };
+    dimensions: {
+        model?: string[] | undefined;
+        max_steps?: number[] | undefined;
+        temperature?: number[] | undefined;
+        system_prompt?: string[] | undefined;
+        toolkits?: string[][] | undefined;
+    };
+}, {
+    name: string;
+    dimensions: {
+        model?: string[] | undefined;
+        max_steps?: number[] | undefined;
+        temperature?: number[] | undefined;
+        system_prompt?: string[] | undefined;
+        toolkits?: string[][] | undefined;
+    };
+    base?: {
+        model?: string | undefined;
+        max_steps?: number | undefined;
+        temperature?: number | undefined;
+        system_prompt?: string | undefined;
+        tools?: string[] | undefined;
+        toolkits?: string[] | undefined;
+    } | undefined;
+}>;
+type Matrix = z.infer<typeof MatrixSchema>;
+/**
+ * Expands a `Matrix` into the cartesian product of its `dimensions`,
+ * producing one `AgentConfig` per combination with `base` values as
+ * defaults and a deterministic, slugified `id`/`name`.
+ */
+declare function expandMatrix(matrix: Matrix): AgentConfig[];
+
+/**
+ * Minimal shape of a persisted run row needed for aggregation. Matches both
+ * `@agentgrader/store`'s `runs` rows (`metrics` as a JSON string) and
+ * `RunSingleResult`-like objects (`metrics` as a parsed object).
+ */
+interface RunRecord {
+    agentConfigId: string;
+    passed?: boolean | null;
+    costUsd?: number | null;
+    durationMs?: number | null;
+    tokensIn?: number | null;
+    tokensOut?: number | null;
+    metrics?: string | Record<string, any> | null;
+}
+interface QualityAverages {
+    diffLines?: number;
+    filesModified?: number;
+    todosIntroduced?: number;
+    linterViolations?: number;
+    llmJudgeScore?: number;
+}
+interface AggregateResult {
+    agentConfigId: string;
+    agentConfigName: string;
+    totalRuns: number;
+    passedRuns: number;
+    solveRate: number;
+    avgCostUsd: number;
+    avgDurationMs: number;
+    avgTokensIn: number;
+    avgTokensOut: number;
+    /** only set when every run for this config has `metrics["static-quality"].quality` */
+    avgQuality?: QualityAverages;
+}
+/**
+ * Groups `runs` by `agentConfigId` and computes solve rate, average cost,
+ * duration, token usage, and (if present on every run) average quality
+ * metrics from `StaticQualityScorer`/`LlmJudgeScorer`.
+ */
+declare function aggregateResults(runs: RunRecord[], configs: AgentConfig[]): AggregateResult[];
+
+/**
+ * Filters `aggregates` down to the Pareto-optimal set across:
+ *  - `solveRate` (maximize)
+ *  - `avgCostUsd` (minimize)
+ *  - `avgQuality.linterViolations` (minimize) - only included as an
+ *    objective if every aggregate has it set, since `avgQuality` is itself
+ *    optional (only populated when a static-quality scorer ran for all runs
+ *    of a config).
+ *
+ * An aggregate is dropped if another aggregate is at least as good on every
+ * objective and strictly better on at least one (i.e. it is dominated).
+ */
+declare function paretoFront(aggregates: AggregateResult[]): AggregateResult[];
+
+export { type AggregateResult, type Matrix, MatrixDimensionsSchema, MatrixSchema, type QualityAverages, type RunRecord, aggregateResults, expandMatrix, paretoFront };

package/dist/index.js

@@ -0,0 +1,158 @@
+import { z } from 'zod';
+
+// src/matrix.ts
+var MatrixBaseSchema = z.object({
+  model: z.string().optional(),
+  max_steps: z.number().optional(),
+  temperature: z.number().optional(),
+  system_prompt: z.string().optional(),
+  tools: z.array(z.string()).optional(),
+  toolkits: z.array(z.string()).optional()
+});
+var MatrixDimensionsSchema = z.object({
+  model: z.array(z.string()).optional(),
+  temperature: z.array(z.number()).optional(),
+  system_prompt: z.array(z.string()).optional(),
+  max_steps: z.array(z.number()).optional(),
+  toolkits: z.array(z.array(z.string())).optional()
+}).refine((dims) => Object.values(dims).some((v) => Array.isArray(v) && v.length > 0), {
+  message: "Matrix must define at least one non-empty dimension"
+});
+var MatrixSchema = z.object({
+  /** used as the prefix for generated agent config ids/names */
+  name: z.string(),
+  base: MatrixBaseSchema.default({}),
+  dimensions: MatrixDimensionsSchema
+});
+var DEFAULT_MODEL = "anthropic/claude-sonnet-4-6";
+var DEFAULT_MAX_STEPS = 30;
+function expandMatrix(matrix) {
+  const { name, base, dimensions } = matrix;
+  const dimEntries = Object.entries(dimensions).filter(
+    (entry) => Array.isArray(entry[1]) && entry[1].length > 0
+  );
+  if (dimEntries.length === 0) {
+    throw new Error("Matrix must define at least one non-empty dimension");
+  }
+  let combinations = [{}];
+  for (const [key, values] of dimEntries) {
+    const next = [];
+    for (const combo of combinations) {
+      for (const value of values) {
+        next.push({ ...combo, [key]: value });
+      }
+    }
+    combinations = next;
+  }
+  return combinations.map((combo) => {
+    const id = `${name}-${dimEntries.map(([key]) => slugifyValue(combo[key])).join("-")}`;
+    const config = {
+      id,
+      name: id,
+      model: combo.model ?? base.model ?? DEFAULT_MODEL,
+      max_steps: combo.max_steps ?? base.max_steps ?? DEFAULT_MAX_STEPS
+    };
+    const temperature = combo.temperature ?? base.temperature;
+    if (temperature !== void 0) config.temperature = temperature;
+    const systemPrompt = combo.system_prompt ?? base.system_prompt;
+    if (systemPrompt !== void 0) config.system_prompt = systemPrompt;
+    const toolkits = combo.toolkits ?? base.toolkits;
+    if (toolkits !== void 0) config.toolkits = toolkits;
+    if (base.tools !== void 0) config.tools = base.tools;
+    return config;
+  });
+}
+function slugifyValue(value) {
+  if (Array.isArray(value)) {
+    return value.length > 0 ? value.map(slugifyValue).join("+") : "none";
+  }
+  return String(value).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+
+// src/aggregate.ts
+var QUALITY_KEYS = [
+  "diffLines",
+  "filesModified",
+  "todosIntroduced",
+  "linterViolations",
+  "llmJudgeScore"
+];
+function aggregateResults(runs, configs) {
+  const configsById = new Map(configs.map((c) => [c.id || c.name, c]));
+  const byConfig = /* @__PURE__ */ new Map();
+  for (const run of runs) {
+    const list = byConfig.get(run.agentConfigId);
+    if (list) list.push(run);
+    else byConfig.set(run.agentConfigId, [run]);
+  }
+  const results = [];
+  for (const [agentConfigId, configRuns] of byConfig) {
+    const total = configRuns.length;
+    const passed = configRuns.filter((r) => r.passed).length;
+    const sum = (f) => configRuns.reduce((acc, r) => acc + f(r), 0);
+    const qualities = configRuns.map((r) => extractQuality(r.metrics)).filter((q) => q !== void 0);
+    let avgQuality;
+    if (total > 0 && qualities.length === total) {
+      const avg = {};
+      for (const key of QUALITY_KEYS) {
+        const values = qualities.map((q) => q[key]).filter((v) => typeof v === "number");
+        if (values.length === qualities.length) {
+          avg[key] = values.reduce((a, b) => a + b, 0) / values.length;
+        }
+      }
+      if (Object.keys(avg).length > 0) avgQuality = avg;
+    }
+    results.push({
+      agentConfigId,
+      agentConfigName: configsById.get(agentConfigId)?.name ?? agentConfigId,
+      totalRuns: total,
+      passedRuns: passed,
+      solveRate: total > 0 ? passed / total : 0,
+      avgCostUsd: total > 0 ? sum((r) => r.costUsd ?? 0) / total : 0,
+      avgDurationMs: total > 0 ? sum((r) => r.durationMs ?? 0) / total : 0,
+      avgTokensIn: total > 0 ? sum((r) => r.tokensIn ?? 0) / total : 0,
+      avgTokensOut: total > 0 ? sum((r) => r.tokensOut ?? 0) / total : 0,
+      avgQuality
+    });
+  }
+  return results;
+}
+function extractQuality(metrics) {
+  if (!metrics) return void 0;
+  const parsed = typeof metrics === "string" ? safeParseJson(metrics) : metrics;
+  return parsed?.["static-quality"]?.quality;
+}
+function safeParseJson(value) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return void 0;
+  }
+}
+
+// src/pareto.ts
+function paretoFront(aggregates) {
+  if (aggregates.length === 0) return [];
+  const objectives = [
+    { get: (a) => a.solveRate, maximize: true },
+    { get: (a) => a.avgCostUsd, maximize: false }
+  ];
+  if (aggregates.every((a) => typeof a.avgQuality?.linterViolations === "number")) {
+    objectives.push({ get: (a) => a.avgQuality.linterViolations, maximize: false });
+  }
+  const dominates = (a, b) => {
+    let strictlyBetter = false;
+    for (const { get, maximize } of objectives) {
+      const av = get(a);
+      const bv = get(b);
+      if (maximize ? av < bv : av > bv) return false;
+      if (av !== bv) strictlyBetter = true;
+    }
+    return strictlyBetter;
+  };
+  return aggregates.filter(
+    (candidate) => !aggregates.some((other) => other !== candidate && dominates(other, candidate))
+  );
+}
+
+export { MatrixDimensionsSchema, MatrixSchema, aggregateResults, expandMatrix, paretoFront };

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@agentgrader/optimizer",
+  "version": "0.1.0",
+  "description": "Architecture optimizer for the Agentgrader framework: agent-config matrix expansion, result aggregation, and Pareto-front analysis",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts --clean --treeshake",
+    "build:watch": "tsup src/index.ts --format esm --dts --watch"
+  },
+  "dependencies": {
+    "@agentgrader/core": "^1.1.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "tsup": "^8.5.1"
+  },
+  "peerDependencies": {
+    "@agentgrader/core": "^1.1.0"
+  }
+}