@tangle-network/agent-eval 1.0.21 → 1.0.22

package/dist/index.js

@@ -1,7 +1,7 @@
 import { t as __exportAll } from "./rolldown-runtime-BJohB4LM.js";
 import { i as JudgeError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-DeEEV9Vx.js";
 import { a as hashCanonical, r as canonicalString } from "./canonical-CicJlINA.js";
-import { a as agentProfileModelId, i as agentProfileId, n as HARNESS_NATIVE_MODEL, o as expandProfileAxes, r as agentProfileHash, s as harnessAxisOf, t as CODING_HARNESSES } from "./agent-profile-Dngk3LZ0.js";
+import { a as agentProfileModelId, i as agentProfileId, n as HARNESS_NATIVE_MODEL, o as expandProfileAxes, r as agentProfileHash, s as harnessAxisOf, t as CODING_HARNESSES } from "./agent-profile-4j6f2n-x.js";
 import { a as groupRunsByAgentProfileCell, c as verifyAgentProfileCell, i as buildAgentProfileCell, l as hashJson, o as toAgentProfileJson, r as agentProfileCellKey, s as validateAgentProfileCell, t as AGENT_PROFILE_KINDS, u as manifestContentDigest } from "./agent-profile-cell-CT5JYueM.js";
 import { r as observedSplitScore, t as isRealnessGated } from "./reward-DqMfaEax.js";
 import { i as isToolSpan, n as isJudgeSpan, r as isLlmSpan, t as FAILURE_CLASSES } from "./schema-D-xu3MU_.js";

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "1.0.21",
+    "version": "1.0.22",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/wire/index.d.ts

@@ -267,10 +267,10 @@ export declare const FeedbackTrajectorySchema: z.ZodObject<{
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
   }, z.core.$strip>>;
   split: z.ZodOptional<z.ZodEnum<{
-    dev: "dev";
-    holdout: "holdout";
     train: "train";
+    dev: "dev";
     test: "test";
+    holdout: "holdout";
   }>>;
   tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
   createdAt: z.ZodString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "1.0.21",
+  "version": "1.0.22",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -244,10 +244,10 @@
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^9.1.0",
     "@hono/node-server": "^2.1.5",
-    "@tangle-network/agent-core": "^1.0.0",
-    "@tangle-network/agent-interface": "^5.10.2",
-    "@tangle-network/agent-trace-contract": "^2.2.0",
-    "@tangle-network/harness-sessions": "^1.0.8",
+    "@tangle-network/agent-core": "^1.0.3",
+    "@tangle-network/agent-interface": "^5.10.20",
+    "@tangle-network/agent-trace-contract": "^2.2.2",
+    "@tangle-network/harness-sessions": "^1.0.11",
     "hono": "^4.13.14",
     "linear-sum-assignment": "1.0.9",
     "re2js": "2.9.0",

package/dist/agent-profile-Dngk3LZ0.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"agent-profile-Dngk3LZ0.js","names":[],"sources":["../src/agent-profile.ts"],"sourcesContent":["import { createHash } from 'node:crypto'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { type HarnessType, harnessSupportsModel } from '@tangle-network/agent-interface'\nimport { ValidationError } from './errors'\nimport { canonicalString } from './ledger-core/canonical'\n\nexport type { AgentProfile, HarnessType } from '@tangle-network/agent-interface'\n\n/**\n * The agentic coding harnesses an eval sweeps by default — the ones we care about\n * ranking. This is the SINGLE source of that list; consumers import it instead of\n * re-declaring their own (a re-declared list is how the fleet drifts). Pass an\n * explicit `harnesses` (e.g. `harnessTypeSchema.options` for literally every known\n * harness) to widen beyond these.\n */\nexport const CODING_HARNESSES: readonly HarnessType[] = [\n  'opencode',\n  'claude-code',\n  'codex',\n  'kimi-code',\n]\n\nexport interface ProfileAxisSpec {\n  /** The domain profile to sweep. Its prompt/tools/skills are held fixed; only the\n   *  harness and model vary. `model.default` is the fallback model. */\n  base: AgentProfile\n  /** Harnesses to cross. Default: {@link CODING_HARNESSES}. */\n  harnesses?: readonly HarnessType[]\n  /** Models to cross. Default: `[base.model.default]` — one model, i.e. today's\n   *  single-model behaviour, so omitting this never changes an existing run. */\n  models?: readonly string[]\n  /** Force every (harness, model) pair verbatim, even ones the harness can't run —\n   *  for deliberately testing failure modes. Default (false): SNAP instead — a\n   *  vendor-locked harness runs only the swept models in its family, or its native\n   *  default when it supports none, so no harness is dropped and none gets a\n   *  guaranteed-failing foreign-model cell. */\n  keepIncompatible?: boolean\n}\n\n/** Model sentinel for a vendor-locked harness that supports none of the swept models:\n *  it carries no provider prefix, so `harnessSupportsModel` accepts it and the harness\n *  resolves it to its own native default model at runtime (e.g. kimi-code → its Kimi\n *  model). Lets `expandProfileAxes` snap-instead-of-drop without a per-harness flagship\n *  table that would rot as router catalogs change. */\nexport const HARNESS_NATIVE_MODEL = 'default'\n\n/**\n * Expand a base profile across the harness × model matrix into the `AgentProfile[]`\n * that `runProfileMatrix` / `selfImprove` score — the ONE place \"which harnesses ×\n * which models do we evaluate\" lives, so no product hand-rolls its own harness list\n * or column→profile mapping (the pattern that let those copies drift and silently\n * break the harness pivot).\n *\n * Each cell clones `base`, sets the canonical top-level `harness` and `model.default`,\n * and stamps `metadata.harness` + `metadata.harnessModel` for matrix grouping. Both\n * metadata fields are hash-bearing, so every cell gets a distinct `agentProfileId` row\n * and results join back by harness/model via {@link harnessAxisOf} with no\n * hand-recomputed key. A vendor-locked harness snaps to its family's swept models — or\n * its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — so every\n * requested harness runs; `keepIncompatible` forces every pair verbatim.\n *\n * Omit `harnesses`/`models` to sweep the full default set — the \"turn it on for\n * everything we care about\" switch, identical in shape whether one harness or all.\n */\nexport function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[] {\n  const harnesses = spec.harnesses ?? CODING_HARNESSES\n  if (harnesses.length === 0) throw new ValidationError('expandProfileAxes: no harnesses to sweep')\n  const baseModel = spec.base.model?.default\n  const models = spec.models ?? (baseModel ? [baseModel] : [])\n  if (models.length === 0) {\n    throw new ValidationError(\n      'expandProfileAxes: no models to sweep — base profile has no model.default and none were supplied',\n    )\n  }\n  const out: AgentProfile[] = []\n  const seen = new Set<string>()\n  for (const harness of harnesses) {\n    // A universal (router-backed) harness — opencode/pi/claudish, and claude-code/codex\n    // since agent-interface 5.6.0 — runs every swept model. A vendor-locked harness\n    // (kimi-code) runs only the swept models in its own family; when it supports NONE\n    // of them it snaps to its native default (the `HARNESS_NATIVE_MODEL` sentinel it\n    // resolves at runtime) rather than being dropped, so every requested harness still appears in the\n    // sweep on a model it can actually run — e.g. sweeping `deepseek/x` puts opencode\n    // on deepseek and kimi-code on its own Kimi model, a real head-to-head.\n    // `keepIncompatible` forces every (harness, model) pair verbatim (failure-mode runs).\n    const supported = spec.keepIncompatible\n      ? models\n      : models.filter((model) => harnessSupportsModel(harness, model))\n    const effective = supported.length > 0 ? supported : [HARNESS_NATIVE_MODEL]\n    for (const model of effective) {\n      const profile: AgentProfile = {\n        ...spec.base,\n        harness,\n        name: `${spec.base.name ?? 'agent'}/${harness}/${model}`,\n        model: { ...spec.base.model, default: model },\n        metadata: { ...(spec.base.metadata ?? {}), harness, harnessModel: model },\n      }\n      const id = agentProfileId(profile)\n      if (seen.has(id)) continue\n      seen.add(id)\n      out.push(profile)\n    }\n  }\n  if (out.length === 0) {\n    // Unreachable in normal use — snapping guarantees ≥1 cell per harness — but keep a\n    // fail-closed guard so a future refactor can't silently produce an empty sweep.\n    throw new ValidationError(\n      `expandProfileAxes: produced no profiles (harnesses=[${harnesses.join(', ')}], models=[${models.join(', ')}]).`,\n    )\n  }\n  return out\n}\n\n/**\n * Read the (harness, model) a matrix cell ran under, off a profile or a result row's\n * profile — the join-back for a `byHarness` pivot. Returns undefined when the profile\n * wasn't produced by {@link expandProfileAxes}. Callers group `result.byProfile` by\n * this instead of recomputing an id (recomputing the wrong key is what broke the pivot\n * in the hand-rolled copies).\n */\nexport function harnessAxisOf(\n  profile: Pick<AgentProfile, 'metadata'>,\n): { harness: HarnessType; model: string } | undefined {\n  const m = profile.metadata as Record<string, unknown> | undefined\n  const harness = m?.harness\n  const model = m?.harnessModel\n  if (typeof harness === 'string' && typeof model === 'string') {\n    return { harness: harness as HarnessType, model }\n  }\n  return undefined\n}\n\n/**\n * Collision-resistant, path-safe, human-readable profile id for eval artifacts.\n * Scorecard joins still use `agentProfileHash`; this id is for run ids, matrix\n * keys, and directory names where two profiles must not collapse onto one row.\n * The suffix is the first 64 bits of the behaviour hash, enough for ordinary\n * eval matrices while keeping filenames readable.\n */\nexport function agentProfileId(profile: AgentProfile): string {\n  const label = pathSafeProfileLabel(agentProfileDisplayLabel(profile)) ?? 'profile'\n  return `${label}-${agentProfileHash(profile).slice(0, 16)}`\n}\n\n/**\n * Model snapshot used for `RunRecord.model`. Eval surfaces require a concrete\n * model id because run records reject bare/missing model aliases.\n */\nexport function agentProfileModelId(profile: AgentProfile): string {\n  const model = profile.model?.default?.trim()\n  if (!model) {\n    const label = agentProfileDisplayLabel(profile) ?? 'unnamed profile'\n    throw new ValidationError(\n      `AgentProfile \"${label}\" has no model.default — cannot record eval run`,\n    )\n  }\n  return model\n}\n\nfunction agentProfileDisplayLabel(profile: AgentProfile): string | undefined {\n  return profile.name?.trim() || profile.version?.trim() || undefined\n}\n\nfunction pathSafeProfileLabel(label: string | undefined): string | undefined {\n  const safe = label\n    ?.trim()\n    .replace(/[^A-Za-z0-9._-]+/g, '-')\n    .replace(/-+/g, '-')\n    .replace(/^-|-$/g, '')\n  return safe || undefined\n}\n\nfunction compact<T extends Record<string, unknown>>(input: T): Partial<T> {\n  const out: Record<string, unknown> = {}\n  for (const [key, value] of Object.entries(input)) {\n    if (value !== undefined) out[key] = value\n  }\n  return out as Partial<T>\n}\n\n/**\n * Deterministic behaviour identity for the canonical\n * `@tangle-network/agent-interface` AgentProfile.\n *\n * `name` and `description` are labels and do not affect the hash. Profile\n * `version`, prompt, model hints, tools, resources, hooks, modes, permissions,\n * and extensions do affect the hash. Resource array order is hash-bearing\n * because mount order can change agent behaviour. Undefined fields are treated\n * as absent; explicit `null` fields remain hash-bearing.\n */\nexport function agentProfileHash(profile: AgentProfile): string {\n  const model = agentProfileModelId(profile)\n  const behaviour = compact({\n    ...profile,\n    name: undefined,\n    description: undefined,\n    tags: profile.tags ? [...profile.tags].sort() : undefined,\n    model: compact({ ...profile.model, default: model }),\n  })\n  return createHash('sha256').update(canonicalString(behaviour)).digest('hex')\n}\n"],"mappings":";;;;;;;;;;;;AAeA,MAAa,mBAA2C;CACtD;CACA;CACA;CACA;AACF;;;;;;AAwBA,MAAa,uBAAuB;;;;;;;;;;;;;;;;;;;AAoBpC,SAAgB,kBAAkB,MAAuC;CACvE,MAAM,YAAY,KAAK,aAAa;CACpC,IAAI,UAAU,WAAW,GAAG,MAAM,IAAI,gBAAgB,0CAA0C;CAChG,MAAM,YAAY,KAAK,KAAK,OAAO;CACnC,MAAM,SAAS,KAAK,WAAW,YAAY,CAAC,SAAS,IAAI,CAAC;CAC1D,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBACR,kGACF;CAEF,MAAM,MAAsB,CAAC;CAC7B,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,WAAW,WAAW;EAS/B,MAAM,YAAY,KAAK,mBACnB,SACA,OAAO,QAAQ,UAAU,qBAAqB,SAAS,KAAK,CAAC;EACjE,MAAM,YAAY,UAAU,SAAS,IAAI,YAAY,CAAC,oBAAoB;EAC1E,KAAK,MAAM,SAAS,WAAW;GAC7B,MAAM,UAAwB;IAC5B,GAAG,KAAK;IACR;IACA,MAAM,GAAG,KAAK,KAAK,QAAQ,QAAQ,GAAG,QAAQ,GAAG;IACjD,OAAO;KAAE,GAAG,KAAK,KAAK;KAAO,SAAS;IAAM;IAC5C,UAAU;KAAE,GAAI,KAAK,KAAK,YAAY,CAAC;KAAI;KAAS,cAAc;IAAM;GAC1E;GACA,MAAM,KAAK,eAAe,OAAO;GACjC,IAAI,KAAK,IAAI,EAAE,GAAG;GAClB,KAAK,IAAI,EAAE;GACX,IAAI,KAAK,OAAO;EAClB;CACF;CACA,IAAI,IAAI,WAAW,GAGjB,MAAM,IAAI,gBACR,uDAAuD,UAAU,KAAK,IAAI,EAAE,aAAa,OAAO,KAAK,IAAI,EAAE,IAC7G;CAEF,OAAO;AACT;;;;;;;;AASA,SAAgB,cACd,SACqD;CACrD,MAAM,IAAI,QAAQ;CAClB,MAAM,UAAU,GAAG;CACnB,MAAM,QAAQ,GAAG;CACjB,IAAI,OAAO,YAAY,YAAY,OAAO,UAAU,UAClD,OAAO;EAAW;EAAwB;CAAM;AAGpD;;;;;;;;AASA,SAAgB,eAAe,SAA+B;CAE5D,OAAO,GADO,qBAAqB,yBAAyB,OAAO,CAAC,KAAK,UACzD,GAAG,iBAAiB,OAAO,CAAC,CAAC,MAAM,GAAG,EAAE;AAC1D;;;;;AAMA,SAAgB,oBAAoB,SAA+B;CACjE,MAAM,QAAQ,QAAQ,OAAO,SAAS,KAAK;CAC3C,IAAI,CAAC,OAAO;EACV,MAAM,QAAQ,yBAAyB,OAAO,KAAK;EACnD,MAAM,IAAI,gBACR,iBAAiB,MAAM,gDACzB;CACF;CACA,OAAO;AACT;AAEA,SAAS,yBAAyB,SAA2C;CAC3E,OAAO,QAAQ,MAAM,KAAK,KAAK,QAAQ,SAAS,KAAK,KAAK,KAAA;AAC5D;AAEA,SAAS,qBAAqB,OAA+C;CAM3E,OALa,OACT,KAAK,CAAC,CACP,QAAQ,qBAAqB,GAAG,CAAC,CACjC,QAAQ,OAAO,GAAG,CAAC,CACnB,QAAQ,UAAU,EAAE,KACR,KAAA;AACjB;AAEA,SAAS,QAA2C,OAAsB;CACxE,MAAM,MAA+B,CAAC;CACtC,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,KAAK,GAC7C,IAAI,UAAU,KAAA,GAAW,IAAI,OAAO;CAEtC,OAAO;AACT;;;;;;;;;;;AAYA,SAAgB,iBAAiB,SAA+B;CAC9D,MAAM,QAAQ,oBAAoB,OAAO;CACzC,MAAM,YAAY,QAAQ;EACxB,GAAG;EACH,MAAM,KAAA;EACN,aAAa,KAAA;EACb,MAAM,QAAQ,OAAO,CAAC,GAAG,QAAQ,IAAI,CAAC,CAAC,KAAK,IAAI,KAAA;EAChD,OAAO,QAAQ;GAAE,GAAG,QAAQ;GAAO,SAAS;EAAM,CAAC;CACrD,CAAC;CACD,OAAO,WAAW,QAAQ,CAAC,CAAC,OAAO,gBAAgB,SAAS,CAAC,CAAC,CAAC,OAAO,KAAK;AAC7E"}