@tangle-network/agent-eval 0.128.2 → 0.130.0

package/dist/run-record-CnZu_gjl.d.ts

@@ -0,0 +1,357 @@
+import { c as ValidationError } from "./errors-CEk209JS.js";
+import { o as FailureClass } from "./schema-BtVldJ3T.js";
+import { AgentProfile } from "@tangle-network/agent-interface";
+//#region src/agent-profile-cell.d.ts
+type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
+type AgentProfileJsonObject = {
+  [key: string]: AgentProfileJson;
+};
+type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] | AgentProfileJsonObject;
+type AgentProfileDimensionValue = string | number | boolean | null;
+interface AgentProfileSource {
+  /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
+  kind: string;
+  /** sha256 over the canonical source profile object. */
+  hash: string;
+}
+interface AgentProfileSourceInput {
+  kind: string;
+  /** Precomputed sha256 for callers that already sign their profile artifact. */
+  hash?: string;
+  /** Full canonical runtime profile; hashed and then discarded from the cell. */
+  profile?: AgentProfileJson;
+}
+interface AgentProfileHarness {
+  id: string;
+  version?: string;
+  hash?: string;
+}
+interface AgentProfileCellInput {
+  profileId: string;
+  sourceProfile: AgentProfileSourceInput;
+  harness?: AgentProfileHarness;
+  model?: string;
+  promptHash?: string;
+  dimensions?: Record<string, AgentProfileDimensionValue>;
+}
+interface AgentProfileCell {
+  schemaVersion: AgentProfileCellSchemaVersion;
+  cellId: string;
+  profileId: string;
+  sourceProfile: AgentProfileSource;
+  harness?: AgentProfileHarness;
+  model?: string;
+  promptHash?: string;
+  dimensions?: Record<string, AgentProfileDimensionValue>;
+}
+declare class AgentProfileCellValidationError extends ValidationError {
+  readonly path: string;
+  constructor(message: string, path?: string);
+}
+declare function buildAgentProfileCell(input: AgentProfileCellInput): Promise<AgentProfileCell>;
+declare function agentProfileCellHashMaterial(cell: AgentProfileCell): Omit<AgentProfileCell, 'cellId'>;
+/**
+ * Verify an `AgentProfileCell`'s `cellId` matches the sha256 of its hash-material fields, confirming the record has not been tampered with.
+ */
+declare function verifyAgentProfileCell(cell: AgentProfileCell): Promise<boolean>;
+declare function validateAgentProfileCell(input: unknown): AgentProfileCell;
+declare function requireAgentProfileCell(record: {
+  runId: string;
+  agentProfile?: AgentProfileCell;
+}): AgentProfileCell;
+declare function agentProfileCellKey(record: {
+  runId: string;
+  agentProfile?: AgentProfileCell;
+}): string;
+declare function assertRunAgentProfileCell(record: {
+  runId: string;
+  model: string;
+  promptHash: string;
+  agentProfile?: AgentProfileCell;
+}): Promise<AgentProfileCell>;
+declare function groupRunsByAgentProfileCell<T extends {
+  runId: string;
+  agentProfile?: AgentProfileCell;
+}>(records: readonly T[]): Map<string, T[]>;
+/** Canonical `sourceProfile.kind` values. Two products fingerprinting the
+ *  same canonical profile MUST use the same kind for their cells to share
+ *  `sourceProfile.hash`. Extend rather than create new strings — adding a
+ *  new kind is a deliberate cross-product schema change. */
+declare const AGENT_PROFILE_KINDS: {
+  /** A profile declared via `defineAgentProfile(...)` from
+   *  `@tangle-network/agent-interface`. The default kind for router-backed
+   *  and sandbox-backed products. */
+  readonly AGENT_INTERFACE_PROFILE: 'agent-interface-profile';
+};
+type AgentProfileKind = (typeof AGENT_PROFILE_KINDS)[keyof typeof AGENT_PROFILE_KINDS];
+/** Canonicalize an arbitrary value into `AgentProfileJson` by JSON
+ *  round-trip. Throws when the value contains anything not representable
+ *  as JSON (functions, BigInt, cycles) — non-portable profiles fail loud
+ *  rather than silently dropping fields. */
+declare function toAgentProfileJson(value: unknown): AgentProfileJson;
+/** Canonical AgentProfile shape required when deriving a stable cell id. */
+type AgentInterfaceProfileLike = AgentProfile & {
+  name: string;
+  version: string;
+};
+/** Higher-level helper that hard-codes the canonical
+ *  `agent-interface-profile` kind plus the JSON canonicalization. Equivalent
+ *  to calling `buildAgentProfileCell` with `profileId = \`${name}@${version}\``
+ *  and `sourceProfile = { kind: AGENT_INTERFACE_PROFILE, profile: <round-tripped> }`.
+ *
+ *  Use this from any product consuming an agent-interface `AgentProfile`; the
+ *  manual `buildAgentProfileCell` call is reserved for advanced cases
+ *  (custom kinds, pre-computed source hashes, alternate profileId
+ *  conventions). */
+declare function buildAgentInterfaceProfileCell(profile: AgentInterfaceProfileLike, input: Omit<AgentProfileCellInput, 'profileId' | 'sourceProfile'>): Promise<AgentProfileCell>;
+//#endregion
+//#region src/run-record.d.ts
+/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
+ *  combined train+test pool that the optimizer is allowed to read. */
+type RunSplitTag = 'search' | 'dev' | 'holdout';
+/**
+ * Explicit execution-lifecycle result for a run.
+ *
+ * This is separate from task quality (`outcome`) and failure classification.
+ * Producers set it only from root-run or process evidence.
+ */
+type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
+interface RunTokenUsage {
+  input: number;
+  /** All generated tokens charged as output, including reasoning tokens. */
+  output: number;
+  /** Reasoning-token subset of `output`, when the provider reports it. */
+  reasoning?: number;
+  /** Prompt tokens served from a provider cache. */
+  cached?: number;
+  /** Prompt tokens written into a provider cache. */
+  cacheWrite?: number;
+}
+/**
+ * How a run's USD amount was obtained.
+ */
+type RunCostProvenance = {
+  kind: 'observed';
+  usd: number;
+} | {
+  kind: 'estimated';
+  usd: number;
+} | {
+  kind: 'uncaptured';
+  usd: null;
+};
+interface RunJudgeMetadata {
+  model: string;
+  promptVersion: string;
+  /** [0,1] confidence the judge declared. Constant judge confidence
+   *  across many runs is a fallback signal (see `canary.ts`). */
+  confidence: number;
+  /** True if the judge degraded to a fallback path (rules-only,
+   *  prior-call cache, etc.). The canary uses this to alert. */
+  fallback: boolean;
+}
+/**
+ * Per-judge / per-dimension breakdown for runs scored by an ensemble of
+ * judges over a multi-dimensional rubric.
+ *
+ * The collapsed `outcome.searchScore` / `holdoutScore` carries the
+ * composite the gate uses. The full breakdown belongs here so consumers
+ * can answer "which judge disagreed?", "which dimension dragged the
+ * composite down?", and "did half the panel fail?" without re-running.
+ *
+ * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and
+ * `composite` are convenience projections — derivable but precomputed so
+ * downstream IRR primitives (`interRaterReliability`,
+ * `corpusInterRaterAgreement`) and reporters don't pay the same
+ * aggregation twice.
+ *
+ * Fail-loud discipline: judges that errored out land in `failedJudges`
+ * by id. A missing key in `perJudge` is ambiguous (silent zero vs not
+ * run); the explicit list makes a partial-failure recorded as such.
+ */
+interface JudgeScoresRecord {
+  /** Per-judge per-dimension scores. `{ "kimi-k2.6": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */
+  perJudge: Record<string, Record<string, number>>;
+  /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
+  perDimMean: Record<string, number>;
+  /** Composite mean across successful judges. Mirrors the task score only
+   *  when `failedJudges` is empty. */
+  composite: number;
+  /** Judges that errored or returned an unparseable verdict. Recorded
+   *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
+   *  not inferred from missing keys in `perJudge`. */
+  failedJudges?: string[];
+  /** Free-form notes the judges emitted (joined across judges or
+   *  first-judge only — consumer's choice). */
+  notes?: string;
+}
+interface RunOutcome {
+  /** Score on the search/optimization split. Optional for holdout-only and
+   *  execution-only records. */
+  searchScore?: number;
+  /** Score on the held-out split. Optional for search-only and execution-only
+   *  records. When both scores are absent, the run is explicitly unlabeled. */
+  holdoutScore?: number;
+  /** Bag of any other metric the run produced — judge dimensions,
+   *  pass/fail counters, latency stats, etc. Numeric only — keeps
+   *  reporters honest. */
+  raw: Record<string, number>;
+  /** Per-judge / per-dim breakdown. Consumers writing ensemble
+   *  judgements populate this; substrate primitives like
+   *  `interRaterReliability` and `corpusInterRaterAgreement` accept
+   *  these records as input. Optional — single-judge or scalar-only
+   *  runs leave it unset. */
+  judgeScores?: JudgeScoresRecord;
+  /** Authenticity / realness verdict — did the run build the REAL thing on the
+   *  intended infra, or fake it (see `./authenticity`)? Optional: only domains
+   *  with an authenticity config populate it. Carried in the corpus so the
+   *  flywheel / off-policy learning can optimize for real completion, not gamed
+   *  pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run
+   *  must not count as a real success regardless of `score`. */
+  realness?: {
+    score: number;
+    gated: boolean;
+    reason?: string;
+  };
+}
+/**
+ * Mandatory paper-grade fields for a single evaluation run. Optional
+ * fields are extension points; mandatory fields throw if missing.
+ *
+ * Hash discipline:
+ *   - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the
+ *     model (after any steering bundle merge).
+ *   - `configHash` is the sha256 of the effective run config (model,
+ *     temperature, tools, judges, splits). The pair (promptHash,
+ *     configHash) uniquely identifies an experiment cell.
+ *
+ * Model snapshot discipline:
+ *   - `model` MUST encode a snapshot version. Bare aliases like
+ *     `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.
+ *     Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.
+ */
+interface RunRecord {
+  /** UUID for the run. */
+  runId: string;
+  /** Logical experiment grouping (a treatment vs a baseline within
+   *  the same sweep should share `experimentId`). */
+  experimentId: string;
+  /** Stable identifier for the candidate (variant) being run. The
+   *  promotion gate compares two `candidateId`s on matched items. */
+  candidateId: string;
+  /** RNG seed for the run. Always recorded — silent re-seeding is
+   *  the most common cause of non-reproducible numbers. */
+  seed: number;
+  /** Model identifier WITH snapshot version. */
+  model: string;
+  /** sha256 of the effective prompt (post-steering). */
+  promptHash: string;
+  /** sha256 of the effective config. */
+  configHash: string;
+  /** Git SHA the harness was run from. */
+  commitSha: string;
+  /** End-to-end wall-clock duration in milliseconds. */
+  wallMs: number;
+  /** Time spent queued before execution started, if known. */
+  queueMs?: number;
+  /** Total USD cost, or null when the producer could not capture one. */
+  costUsd: number | null;
+  /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
+  costProvenance: RunCostProvenance;
+  /** Token usage breakdown. */
+  tokenUsage: RunTokenUsage;
+  /** Root-run or process terminal result. Never inferred from a child span. */
+  terminalOutcome: RunTerminalOutcome;
+  /** Root-run or process failure reason. Valid only for a failed, cancelled,
+   *  or incomplete terminal result; never populated from a child span. */
+  terminalFailureReason?: string;
+  /** Judge-side metadata, if a judge was used. */
+  judgeMetadata?: RunJudgeMetadata;
+  /** Per-split scores + raw bag. */
+  outcome: RunOutcome;
+  /** Canonical task-failure class drawn from the shared
+   *  `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
+   *  evidence. Execution errors belong in
+   *  `outcome.raw.execution_error_count`. */
+  failureClass?: FailureClass;
+  /** Free-form task-failure detail scoped under a non-success
+   *  `failureClass`. It is invalid without that class. */
+  failureMode?: string;
+  /** Which split this run was drawn from. */
+  splitTag: RunSplitTag;
+  /**
+   * Stable scenario identifier the run observed or was scored against.
+   * Comparison primitives match this identity rather than input order.
+   */
+  scenarioId: string;
+  /**
+   * Canonical identity for the agent profile cell that produced this row:
+   * profile artifact hash plus optional harness/model/prompt/reporting
+   * dimensions. Use `agentProfile.cellId` to group persona sweeps and
+   * longitudinal reports by the complete source profile, not by a loose
+   * candidate label or opaque config hash.
+   */
+  agentProfile?: AgentProfileCell;
+}
+/**
+ * Canonical task-result classification.
+ *
+ * A producer may omit classification, record explicit success, or attach
+ * domain-specific detail to a non-success class. Detail can never stand alone.
+ * Execution errors belong in `outcome.raw.execution_error_count`.
+ */
+type RunTaskFailure = {
+  failureClass?: undefined;
+  failureMode?: undefined;
+} | {
+  failureClass: 'success';
+  failureMode?: undefined;
+} | {
+  failureClass: Exclude<FailureClass, 'success'>;
+  failureMode?: string;
+};
+/**
+ * Return task quality, preferring held-out evidence when both scores exist.
+ *
+ * RAW: no realness protection is applied. Built on `observedScore` rather
+ * than repeating the split derivation, so only `rollout/reward.ts` reads the
+ * raw fields. Anything that becomes training data must use `trainingScore` or
+ * `trainingReward` instead.
+ */
+declare function runTaskScore(record: RunRecord): number | undefined;
+declare class RunRecordValidationError extends ValidationError {
+  readonly path: string;
+  constructor(message: string, path?: string);
+}
+/**
+ * Strict validator. Throws `RunRecordValidationError` on the first
+ * missing or wrongly-typed field. Returns the input cast to
+ * `RunRecord` on success — the validator does not coerce.
+ */
+declare function validateRunRecord(input: unknown): RunRecord;
+/** Boolean validator — convenience for filtering arrays. */
+declare function isRunRecord(input: unknown): input is RunRecord;
+/** Non-throwing validator — returns a discriminated union. */
+declare function parseRunRecordSafe(input: unknown): {
+  ok: true;
+  value: RunRecord;
+} | {
+  ok: false;
+  error: RunRecordValidationError;
+};
+/** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */
+declare function roundTripRunRecord(record: RunRecord): RunRecord;
+/**
+ * Heuristic snapshot check. Accepts:
+ *   - `name@YYYY-MM-DD` (Anthropic style: `claude-sonnet-4-6@2025-04-15`)
+ *   - `name-YYYYMMDD`   (OpenAI style: `gpt-4o-2024-11-20`)
+ *   - `name@<arbitrary-token>` (allow opaque snapshots like `@v3`)
+ *   - explicit `:date-...` Vertex-style tags
+ *
+ * Rejects bare aliases like `claude-sonnet-4` or `gpt-4o` that remap
+ * silently as providers ship new snapshots.
+ */
+declare function modelHasSnapshot(model: string): boolean;
+//#endregion
+export { agentProfileCellHashMaterial as A, AgentProfileDimensionValue as C, AgentProfileKind as D, AgentProfileJsonObject as E, groupRunsByAgentProfileCell as F, requireAgentProfileCell as I, toAgentProfileJson as L, assertRunAgentProfileCell as M, buildAgentInterfaceProfileCell as N, AgentProfileSource as O, buildAgentProfileCell as P, validateAgentProfileCell as R, AgentProfileCellValidationError as S, AgentProfileJson as T, AGENT_PROFILE_KINDS as _, RunRecord as a, AgentProfileCellInput as b, RunTaskFailure as c, isRunRecord as d, modelHasSnapshot as f, validateRunRecord as g, runTaskScore as h, RunOutcome as i, agentProfileCellKey as j, AgentProfileSourceInput as k, RunTerminalOutcome as l, roundTripRunRecord as m, RunCostProvenance as n, RunRecordValidationError as o, parseRunRecordSafe as p, RunJudgeMetadata as r, RunSplitTag as s, JudgeScoresRecord as t, RunTokenUsage as u, AgentInterfaceProfileLike as v, AgentProfileHarness as w, AgentProfileCellSchemaVersion as x, AgentProfileCell as y, verifyAgentProfileCell as z };
+//# sourceMappingURL=run-record-CnZu_gjl.d.ts.map

package/dist/run-record-CnZu_gjl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-record-CnZu_gjl.d.ts","names":[],"sources":["../src/agent-profile-cell.ts","../src/run-record.ts"],"mappings":";;;;KAIY;KAEA;GAA4B,cAAc;;KAE1C,sDAKR,qBACA;KAEQ;UAEK;;EAEf;;EAEA;;UAGe;EACf;;EAEA;;EAEA,UAAU;;UAGK;EACf;EACA;EACA;;UAGe;EACf;EACA,eAAe;EACf,UAAU;EACV;EACA;EACA,aAAa,eAAe;;UAGb;EACf,eAAe;EACf;EACA;EACA,eAAe;EACf,UAAU;EACV;EACA;EACA,aAAa,eAAe;;cAGjB,wCAAwC;WAC1C;EACT,YAAY,iBAAiB;;iBAST,sBACpB,OAAO,wBACN,QAAQ;iBAMK,6BACd,MAAM,mBACL,KAAK;;;;iBASc,uBAAuB,MAAM,mBAAmB;iBAQtD,yBAAyB,iBAAiB;iBAqB1C,wBAAwB;EACtC;EACA,eAAe;IACb;iBAUY,oBAAoB;EAClC;EACA,eAAe;;iBAKK,0BAA0B;EAC9C;EACA;EACA;EACA,eAAe;IACb,QAAQ;iBAuBI,4BACd;EAAY;EAAe,eAAe;GAC1C,kBAAkB,MAAM,YAAY;;;;;cA0NzB;;;;WAIX;;KAGU,2BAA2B,kCAAkC;;;;;iBAMzD,mBAAmB,iBAAiB;;KAoBxC,4BAA4B;EAAiB;EAAc;;;;;;;;;;;iBAWjD,+BACpB,SAAS,2BACT,OAAO,KAAK,wDACX,QAAQ;;;;;KC5YC;;;;;;;KAQA;UAEK;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;KAMU;EACN;EAAkB;;EAClB;EAAmB;;EACnB;EAAoB;;UAET;EACf;EACA;;;EAGA;;;EAGA;;;;;;;;;;;;;;;;;;;;;UAsBe;;EAEf,UAAU,eAAe;;EAEzB,YAAY;;;EAGZ;;;;EAIA;;;EAGA;;UAGe;;;EAGf;;;EAGA;;;;EAIA,KAAK;;;;;;EAML,cAAc;;;;;;;EAOd;IAAa;IAAe;IAAgB;;;;;;;;;;;;;;;;;;;UAmB7B;;EAEf;;;EAGA;;;EAGA;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB,YAAY;;EAEZ,iBAAiB;;;EAGjB;;EAEA,gBAAgB;;EAEhB,SAAS;;;;;EAKT,eAAe;;;EAGf;;EAEA,UAAU;;;;;EAKV;;;;;;;;EAQA,eAAe;;;;;;;;;KAUL;EACN;EAA0B;;EAC1B;EAAyB;;EAEzB,cAAc,QAAQ;EACtB;;;;;;;;;;iBAWU,aAAa,QAAQ;cAmCxB,iCAAiC;WACnC;EACT,YAAY,iBAAiB;;;;;;;iBAWf,kBAAkB,iBAAiB;;iBAiOnC,YAAY,iBAAiB,SAAS;;iBAUtC,mBACd;EACG;EAAU,OAAO;;EAAgB;EAAW,OAAO;;;iBAUxC,mBAAmB,QAAQ,YAAY;;;;;;;;;;;iBA2FvC,iBAAiB"}

package/dist/run-score-iEEAWiBY.js

@@ -0,0 +1,41 @@
+//#region src/abort-signal.ts
+/** Combine active cancellation sources without wrapping a single source. */
+function combineAbortSignals(...signals) {
+	const active = [...new Set(signals.filter((signal) => signal !== void 0))];
+	if (active.length === 0) return void 0;
+	if (active.length === 1) return active[0];
+	return AbortSignal.any(active);
+}
+//#endregion
+//#region src/run-score.ts
+const DEFAULT_RUN_SCORE_WEIGHTS = {
+	success: 4,
+	goalProgress: 2,
+	repoGroundedness: 1.5,
+	driftPenalty: -1.5,
+	toolUseQuality: 1,
+	patchQuality: 1.25,
+	testReality: 1.5,
+	finalGate: 3,
+	reviewerBlockers: -2,
+	costUsd: -.2,
+	wallSeconds: -.1
+};
+function aggregateRunScore(score, weights = {}) {
+	const w = {
+		...DEFAULT_RUN_SCORE_WEIGHTS,
+		...weights
+	};
+	return w.success * clamp01(score.success) + w.goalProgress * clamp01(score.goalProgress) + w.repoGroundedness * clamp01(score.repoGroundedness) + w.driftPenalty * clamp01(score.driftPenalty) + w.toolUseQuality * clamp01(score.toolUseQuality) + w.patchQuality * clamp01(score.patchQuality) + w.testReality * clamp01(score.testReality) + w.finalGate * clamp01(score.finalGate) + w.reviewerBlockers * clamp01(score.reviewerBlockers) + w.costUsd * Math.max(0, finiteOrZero(score.costUsd)) + w.wallSeconds * Math.max(0, finiteOrZero(score.wallSeconds) / 60);
+}
+function clamp01(value) {
+	if (!Number.isFinite(value)) return 0;
+	return Math.max(0, Math.min(1, value));
+}
+function finiteOrZero(value) {
+	return Number.isFinite(value) ? value : 0;
+}
+//#endregion
+export { combineAbortSignals as i, aggregateRunScore as n, clamp01 as r, DEFAULT_RUN_SCORE_WEIGHTS as t };
+
+//# sourceMappingURL=run-score-iEEAWiBY.js.map

package/dist/run-score-iEEAWiBY.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-score-iEEAWiBY.js","names":[],"sources":["../src/abort-signal.ts","../src/run-score.ts"],"sourcesContent":["/** Combine active cancellation sources without wrapping a single source. */\nexport function combineAbortSignals(\n  ...signals: Array<AbortSignal | undefined>\n): AbortSignal | undefined {\n  const active = [\n    ...new Set(signals.filter((signal): signal is AbortSignal => signal !== undefined)),\n  ]\n  if (active.length === 0) return undefined\n  if (active.length === 1) return active[0]\n  return AbortSignal.any(active)\n}\n","export interface RunScore {\n  success: number\n  goalProgress: number\n  repoGroundedness: number\n  driftPenalty: number\n  toolUseQuality: number\n  patchQuality: number\n  testReality: number\n  finalGate: number\n  reviewerBlockers: number\n  costUsd: number\n  wallSeconds: number\n  notes?: string[]\n}\n\nexport interface RunScoreWeights {\n  success: number\n  goalProgress: number\n  repoGroundedness: number\n  driftPenalty: number\n  toolUseQuality: number\n  patchQuality: number\n  testReality: number\n  finalGate: number\n  reviewerBlockers: number\n  costUsd: number\n  wallSeconds: number\n}\n\nexport const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights = {\n  success: 4,\n  goalProgress: 2,\n  repoGroundedness: 1.5,\n  driftPenalty: -1.5,\n  toolUseQuality: 1,\n  patchQuality: 1.25,\n  testReality: 1.5,\n  finalGate: 3,\n  reviewerBlockers: -2,\n  costUsd: -0.2,\n  wallSeconds: -0.1,\n}\n\nexport function aggregateRunScore(score: RunScore, weights: Partial<RunScoreWeights> = {}): number {\n  const w = { ...DEFAULT_RUN_SCORE_WEIGHTS, ...weights }\n  return (\n    w.success * clamp01(score.success) +\n    w.goalProgress * clamp01(score.goalProgress) +\n    w.repoGroundedness * clamp01(score.repoGroundedness) +\n    w.driftPenalty * clamp01(score.driftPenalty) +\n    w.toolUseQuality * clamp01(score.toolUseQuality) +\n    w.patchQuality * clamp01(score.patchQuality) +\n    w.testReality * clamp01(score.testReality) +\n    w.finalGate * clamp01(score.finalGate) +\n    w.reviewerBlockers * clamp01(score.reviewerBlockers) +\n    w.costUsd * Math.max(0, finiteOrZero(score.costUsd)) +\n    w.wallSeconds * Math.max(0, finiteOrZero(score.wallSeconds) / 60)\n  )\n}\n\nexport function clamp01(value: number): number {\n  if (!Number.isFinite(value)) return 0\n  return Math.max(0, Math.min(1, value))\n}\n\nfunction finiteOrZero(value: number): number {\n  return Number.isFinite(value) ? value : 0\n}\n"],"mappings":";;AACA,SAAgB,oBACd,GAAG,SACsB;CACzB,MAAM,SAAS,CACb,GAAG,IAAI,IAAI,QAAQ,QAAQ,WAAkC,WAAW,KAAA,CAAS,CAAC,CACpF;CACA,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;CAChC,IAAI,OAAO,WAAW,GAAG,OAAO,OAAO;CACvC,OAAO,YAAY,IAAI,MAAM;AAC/B;;;ACmBA,MAAa,4BAA6C;CACxD,SAAS;CACT,cAAc;CACd,kBAAkB;CAClB,cAAc;CACd,gBAAgB;CAChB,cAAc;CACd,aAAa;CACb,WAAW;CACX,kBAAkB;CAClB,SAAS;CACT,aAAa;AACf;AAEA,SAAgB,kBAAkB,OAAiB,UAAoC,CAAC,GAAW;CACjG,MAAM,IAAI;EAAE,GAAG;EAA2B,GAAG;CAAQ;CACrD,OACE,EAAE,UAAU,QAAQ,MAAM,OAAO,IACjC,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,mBAAmB,QAAQ,MAAM,gBAAgB,IACnD,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,iBAAiB,QAAQ,MAAM,cAAc,IAC/C,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,cAAc,QAAQ,MAAM,WAAW,IACzC,EAAE,YAAY,QAAQ,MAAM,SAAS,IACrC,EAAE,mBAAmB,QAAQ,MAAM,gBAAgB,IACnD,EAAE,UAAU,KAAK,IAAI,GAAG,aAAa,MAAM,OAAO,CAAC,IACnD,EAAE,cAAc,KAAK,IAAI,GAAG,aAAa,MAAM,WAAW,IAAI,EAAE;AAEpE;AAEA,SAAgB,QAAQ,OAAuB;CAC7C,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO;CACpC,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAEA,SAAS,aAAa,OAAuB;CAC3C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ;AAC1C"}

package/dist/runtime-trajectory-1gyaTOoC.js

@@ -0,0 +1,93 @@
+//#region src/runtime-trajectory.ts
+const DEFAULT_SPLIT_TAG = "search";
+function projectRuntimeTrajectoryEvidence(options) {
+	const diagnostics = [];
+	const runsById = /* @__PURE__ */ new Map();
+	const events = [];
+	let recordWithRuntimeEventsCount = 0;
+	let defaultedSplitCount = 0;
+	for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {
+		const record = options.records[recordIndex];
+		const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf);
+		const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG;
+		if (record.splitTag === void 0) defaultedSplitCount += 1;
+		const rawEvents = record.runtimeEvents;
+		if (!Array.isArray(rawEvents)) {
+			diagnostics.push(`${key}: runtimeEvents is not an array; no runtime run join can be extracted`);
+			continue;
+		}
+		if (rawEvents.length === 0) {
+			diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`);
+			continue;
+		}
+		recordWithRuntimeEventsCount += 1;
+		for (let index = 0; index < rawEvents.length; index += 1) {
+			const event = parseRuntimeTrajectoryHookEvent(rawEvents[index]);
+			if (!event) {
+				diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`);
+				continue;
+			}
+			events.push(event);
+			const scenarioId = event.scenarioId ?? stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ?? stringOrUndefined(record.scenarioId);
+			const prior = runsById.get(event.runId);
+			if (!prior) {
+				runsById.set(event.runId, {
+					runId: event.runId,
+					scenarioId,
+					splitTag
+				});
+				continue;
+			}
+			if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`);
+		}
+	}
+	const runs = [...runsById.values()];
+	return {
+		runs,
+		events,
+		summary: {
+			recordCount: options.records.length,
+			recordWithRuntimeEventsCount,
+			runtimeRunCount: runs.length,
+			lifecycleEventCount: events.length,
+			defaultedSplitCount
+		},
+		diagnostics
+	};
+}
+function parseRuntimeTrajectoryHookEvent(input) {
+	if (!isRecord(input)) return null;
+	if (typeof input.id !== "string" || input.id.length === 0) return null;
+	if (typeof input.runId !== "string" || input.runId.length === 0) return null;
+	if (typeof input.target !== "string" || input.target.length === 0) return null;
+	if (typeof input.phase !== "string" || input.phase.length === 0) return null;
+	if (typeof input.timestamp !== "number" || !Number.isFinite(input.timestamp)) return null;
+	return {
+		id: input.id,
+		runId: input.runId,
+		scenarioId: stringOrUndefined(input.scenarioId),
+		target: input.target,
+		phase: input.phase,
+		timestamp: input.timestamp,
+		stepIndex: finiteNumberOrUndefined(input.stepIndex),
+		parentId: stringOrUndefined(input.parentId),
+		payload: input.payload,
+		metadata: isRecord(input.metadata) ? { ...input.metadata } : void 0
+	};
+}
+function runtimeTrajectoryRecordKey(record, index, recordIdOf) {
+	return stringOrUndefined(recordIdOf?.(record, index)) ?? stringOrUndefined(record.id) ?? `record[${index}]`;
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function stringOrUndefined(value) {
+	return typeof value === "string" && value.length > 0 ? value : void 0;
+}
+function finiteNumberOrUndefined(value) {
+	return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+//#endregion
+export { projectRuntimeTrajectoryEvidence as n, parseRuntimeTrajectoryHookEvent as t };
+
+//# sourceMappingURL=runtime-trajectory-1gyaTOoC.js.map

package/dist/runtime-trajectory-1gyaTOoC.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime-trajectory-1gyaTOoC.js","names":[],"sources":["../src/runtime-trajectory.ts"],"sourcesContent":["import type { RunSplitTag } from './run-record'\n\nexport interface RuntimeTrajectoryHookEvent {\n  id: string\n  runId: string\n  scenarioId?: string\n  target: string\n  phase: string\n  timestamp: number\n  stepIndex?: number\n  parentId?: string\n  payload?: unknown\n  metadata?: Record<string, unknown>\n}\n\nexport interface RuntimeTrajectoryRecord {\n  id?: string\n  scenarioId?: string\n  splitTag?: RunSplitTag\n  runtimeEvents?: unknown\n  [key: string]: unknown\n}\n\nexport interface RuntimeTrajectoryRunRecord {\n  runId: string\n  scenarioId?: string\n  splitTag: RunSplitTag\n}\n\nexport interface RuntimeTrajectoryEvidenceSummary {\n  recordCount: number\n  recordWithRuntimeEventsCount: number\n  runtimeRunCount: number\n  lifecycleEventCount: number\n  defaultedSplitCount: number\n}\n\nexport interface RuntimeTrajectoryEvidenceProjection {\n  runs: RuntimeTrajectoryRunRecord[]\n  events: RuntimeTrajectoryHookEvent[]\n  summary: RuntimeTrajectoryEvidenceSummary\n  diagnostics: string[]\n}\n\nexport interface ProjectRuntimeTrajectoryEvidenceOptions<\n  TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord,\n> {\n  records: TRecord[]\n  defaultSplitTag?: RunSplitTag\n  recordIdOf?: (record: TRecord, index: number) => string | undefined\n  scenarioIdOf?: (record: TRecord, index: number) => string | undefined\n}\n\nconst DEFAULT_SPLIT_TAG: RunSplitTag = 'search'\n\nexport function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(\n  options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>,\n): RuntimeTrajectoryEvidenceProjection {\n  const diagnostics: string[] = []\n  const runsById = new Map<string, RuntimeTrajectoryRunRecord>()\n  const events: RuntimeTrajectoryHookEvent[] = []\n  let recordWithRuntimeEventsCount = 0\n  let defaultedSplitCount = 0\n\n  for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {\n    const record = options.records[recordIndex]!\n    const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf)\n    const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG\n    if (record.splitTag === undefined) defaultedSplitCount += 1\n\n    const rawEvents = record.runtimeEvents\n    if (!Array.isArray(rawEvents)) {\n      diagnostics.push(\n        `${key}: runtimeEvents is not an array; no runtime run join can be extracted`,\n      )\n      continue\n    }\n    if (rawEvents.length === 0) {\n      diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`)\n      continue\n    }\n    recordWithRuntimeEventsCount += 1\n\n    for (let index = 0; index < rawEvents.length; index += 1) {\n      const event = parseRuntimeTrajectoryHookEvent(rawEvents[index])\n      if (!event) {\n        diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`)\n        continue\n      }\n      events.push(event)\n\n      const scenarioId =\n        event.scenarioId ??\n        stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ??\n        stringOrUndefined(record.scenarioId)\n      const prior = runsById.get(event.runId)\n      if (!prior) {\n        runsById.set(event.runId, { runId: event.runId, scenarioId, splitTag })\n        continue\n      }\n      if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) {\n        diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`)\n      }\n    }\n  }\n\n  const runs = [...runsById.values()]\n  return {\n    runs,\n    events,\n    summary: {\n      recordCount: options.records.length,\n      recordWithRuntimeEventsCount,\n      runtimeRunCount: runs.length,\n      lifecycleEventCount: events.length,\n      defaultedSplitCount,\n    },\n    diagnostics,\n  }\n}\n\nexport function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null {\n  if (!isRecord(input)) return null\n  if (typeof input.id !== 'string' || input.id.length === 0) return null\n  if (typeof input.runId !== 'string' || input.runId.length === 0) return null\n  if (typeof input.target !== 'string' || input.target.length === 0) return null\n  if (typeof input.phase !== 'string' || input.phase.length === 0) return null\n  if (typeof input.timestamp !== 'number' || !Number.isFinite(input.timestamp)) return null\n\n  return {\n    id: input.id,\n    runId: input.runId,\n    scenarioId: stringOrUndefined(input.scenarioId),\n    target: input.target,\n    phase: input.phase,\n    timestamp: input.timestamp,\n    stepIndex: finiteNumberOrUndefined(input.stepIndex),\n    parentId: stringOrUndefined(input.parentId),\n    payload: input.payload,\n    metadata: isRecord(input.metadata) ? { ...input.metadata } : undefined,\n  }\n}\n\nfunction runtimeTrajectoryRecordKey<TRecord extends RuntimeTrajectoryRecord>(\n  record: TRecord,\n  index: number,\n  recordIdOf?: (record: TRecord, index: number) => string | undefined,\n): string {\n  return (\n    stringOrUndefined(recordIdOf?.(record, index)) ??\n    stringOrUndefined(record.id) ??\n    `record[${index}]`\n  )\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n\nfunction stringOrUndefined(value: unknown): string | undefined {\n  return typeof value === 'string' && value.length > 0 ? value : undefined\n}\n\nfunction finiteNumberOrUndefined(value: unknown): number | undefined {\n  return typeof value === 'number' && Number.isFinite(value) ? value : undefined\n}\n"],"mappings":";AAqDA,MAAM,oBAAiC;AAEvC,SAAgB,iCACd,SACqC;CACrC,MAAM,cAAwB,CAAC;CAC/B,MAAM,2BAAW,IAAI,IAAwC;CAC7D,MAAM,SAAuC,CAAC;CAC9C,IAAI,+BAA+B;CACnC,IAAI,sBAAsB;CAE1B,KAAK,IAAI,cAAc,GAAG,cAAc,QAAQ,QAAQ,QAAQ,eAAe,GAAG;EAChF,MAAM,SAAS,QAAQ,QAAQ;EAC/B,MAAM,MAAM,2BAA2B,QAAQ,aAAa,QAAQ,UAAU;EAC9E,MAAM,WAAW,OAAO,YAAY,QAAQ,mBAAmB;EAC/D,IAAI,OAAO,aAAa,KAAA,GAAW,uBAAuB;EAE1D,MAAM,YAAY,OAAO;EACzB,IAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;GAC7B,YAAY,KACV,GAAG,IAAI,sEACT;GACA;EACF;EACA,IAAI,UAAU,WAAW,GAAG;GAC1B,YAAY,KAAK,GAAG,IAAI,yDAAyD;GACjF;EACF;EACA,gCAAgC;EAEhC,KAAK,IAAI,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,GAAG;GACxD,MAAM,QAAQ,gCAAgC,UAAU,MAAM;GAC9D,IAAI,CAAC,OAAO;IACV,YAAY,KAAK,GAAG,IAAI,kBAAkB,MAAM,4BAA4B;IAC5E;GACF;GACA,OAAO,KAAK,KAAK;GAEjB,MAAM,aACJ,MAAM,cACN,kBAAkB,QAAQ,eAAe,QAAQ,WAAW,CAAC,KAC7D,kBAAkB,OAAO,UAAU;GACrC,MAAM,QAAQ,SAAS,IAAI,MAAM,KAAK;GACtC,IAAI,CAAC,OAAO;IACV,SAAS,IAAI,MAAM,OAAO;KAAE,OAAO,MAAM;KAAO;KAAY;IAAS,CAAC;IACtE;GACF;GACA,IAAI,MAAM,eAAe,cAAc,MAAM,aAAa,UACxD,YAAY,KAAK,GAAG,IAAI,UAAU,MAAM,MAAM,yCAAyC;EAE3F;CACF;CAEA,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC;CAClC,OAAO;EACL;EACA;EACA,SAAS;GACP,aAAa,QAAQ,QAAQ;GAC7B;GACA,iBAAiB,KAAK;GACtB,qBAAqB,OAAO;GAC5B;EACF;EACA;CACF;AACF;AAEA,SAAgB,gCAAgC,OAAmD;CACjG,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;CAC7B,IAAI,OAAO,MAAM,OAAO,YAAY,MAAM,GAAG,WAAW,GAAG,OAAO;CAClE,IAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,GAAG,OAAO;CACxE,IAAI,OAAO,MAAM,WAAW,YAAY,MAAM,OAAO,WAAW,GAAG,OAAO;CAC1E,IAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,GAAG,OAAO;CACxE,IAAI,OAAO,MAAM,cAAc,YAAY,CAAC,OAAO,SAAS,MAAM,SAAS,GAAG,OAAO;CAErF,OAAO;EACL,IAAI,MAAM;EACV,OAAO,MAAM;EACb,YAAY,kBAAkB,MAAM,UAAU;EAC9C,QAAQ,MAAM;EACd,OAAO,MAAM;EACb,WAAW,MAAM;EACjB,WAAW,wBAAwB,MAAM,SAAS;EAClD,UAAU,kBAAkB,MAAM,QAAQ;EAC1C,SAAS,MAAM;EACf,UAAU,SAAS,MAAM,QAAQ,IAAI,EAAE,GAAG,MAAM,SAAS,IAAI,KAAA;CAC/D;AACF;AAEA,SAAS,2BACP,QACA,OACA,YACQ;CACR,OACE,kBAAkB,aAAa,QAAQ,KAAK,CAAC,KAC7C,kBAAkB,OAAO,EAAE,KAC3B,UAAU,MAAM;AAEpB;AAEA,SAAS,SAAS,OAAkD;CAClE,OAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,kBAAkB,OAAoC;CAC7D,OAAO,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ,KAAA;AACjE;AAEA,SAAS,wBAAwB,OAAoC;CACnE,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;AACvE"}

package/dist/runtime-trajectory-BvSZcCHD.d.ts

@@ -0,0 +1,50 @@
+import { s as RunSplitTag } from "./run-record-CnZu_gjl.js";
+//#region src/runtime-trajectory.d.ts
+interface RuntimeTrajectoryHookEvent {
+  id: string;
+  runId: string;
+  scenarioId?: string;
+  target: string;
+  phase: string;
+  timestamp: number;
+  stepIndex?: number;
+  parentId?: string;
+  payload?: unknown;
+  metadata?: Record<string, unknown>;
+}
+interface RuntimeTrajectoryRecord {
+  id?: string;
+  scenarioId?: string;
+  splitTag?: RunSplitTag;
+  runtimeEvents?: unknown;
+  [key: string]: unknown;
+}
+interface RuntimeTrajectoryRunRecord {
+  runId: string;
+  scenarioId?: string;
+  splitTag: RunSplitTag;
+}
+interface RuntimeTrajectoryEvidenceSummary {
+  recordCount: number;
+  recordWithRuntimeEventsCount: number;
+  runtimeRunCount: number;
+  lifecycleEventCount: number;
+  defaultedSplitCount: number;
+}
+interface RuntimeTrajectoryEvidenceProjection {
+  runs: RuntimeTrajectoryRunRecord[];
+  events: RuntimeTrajectoryHookEvent[];
+  summary: RuntimeTrajectoryEvidenceSummary;
+  diagnostics: string[];
+}
+interface ProjectRuntimeTrajectoryEvidenceOptions<TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord> {
+  records: TRecord[];
+  defaultSplitTag?: RunSplitTag;
+  recordIdOf?: (record: TRecord, index: number) => string | undefined;
+  scenarioIdOf?: (record: TRecord, index: number) => string | undefined;
+}
+declare function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>): RuntimeTrajectoryEvidenceProjection;
+declare function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null;
+//#endregion
+export { RuntimeTrajectoryRecord as a, projectRuntimeTrajectoryEvidence as c, RuntimeTrajectoryHookEvent as i, RuntimeTrajectoryEvidenceProjection as n, RuntimeTrajectoryRunRecord as o, RuntimeTrajectoryEvidenceSummary as r, parseRuntimeTrajectoryHookEvent as s, ProjectRuntimeTrajectoryEvidenceOptions as t };
+//# sourceMappingURL=runtime-trajectory-BvSZcCHD.d.ts.map

package/dist/runtime-trajectory-BvSZcCHD.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime-trajectory-BvSZcCHD.d.ts","names":[],"sources":["../src/runtime-trajectory.ts"],"mappings":";;UAEiB;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;EACX;GACC;;UAGc;EACf;EACA;EACA,UAAU;;UAGK;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf,MAAM;EACN,QAAQ;EACR,SAAS;EACT;;UAGe,wCACf,gBAAgB,0BAA0B;EAE1C,SAAS;EACT,kBAAkB;EAClB,cAAc,QAAQ,SAAS;EAC/B,gBAAgB,QAAQ,SAAS;;iBAKnB,iCAAiC,gBAAgB,yBAC/D,SAAS,wCAAwC,WAChD;iBAgEa,gCAAgC,iBAAiB"}