@tangle-network/agent-eval 0.133.0 → 0.133.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +1 -1
- package/dist/agent-profile-cell-OhuTee9n.js +335 -0
- package/dist/agent-profile-cell-OhuTee9n.js.map +1 -0
- package/dist/{analyze-runs-BjPn_fOS.js → analyze-runs-B-afTpCv.js} +6 -19
- package/dist/analyze-runs-B-afTpCv.js.map +1 -0
- package/dist/{analyze-runs-DZr7JW-m.d.ts → analyze-runs-BmX-h_yn.d.ts} +3 -3
- package/dist/{analyze-runs-DZr7JW-m.d.ts.map → analyze-runs-BmX-h_yn.d.ts.map} +1 -1
- package/dist/{baseline-HsBvw_dk.js → baseline-DcX5hQDv.js} +2 -70
- package/dist/{baseline-HsBvw_dk.js.map → baseline-DcX5hQDv.js.map} +1 -1
- package/dist/baseline-hG3K85h4.d.ts.map +1 -1
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-DncrVrSr.js → benchmarks-CJr1H1_a.js} +3 -3
- package/dist/{benchmarks-DncrVrSr.js.map → benchmarks-CJr1H1_a.js.map} +1 -1
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +3 -3
- package/dist/campaign/index.js +2 -2
- package/dist/{campaign-CyQMCCEU.js → campaign-BJjn1rhw.js} +19 -13
- package/dist/{campaign-CyQMCCEU.js.map → campaign-BJjn1rhw.js.map} +1 -1
- package/dist/{client-D4F9hdzR.d.ts → client-COvaLoQG.d.ts} +2 -2
- package/dist/{client-D4F9hdzR.d.ts.map → client-COvaLoQG.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +3 -3
- package/dist/contract/index.js +5 -5
- package/dist/control.js +1 -1
- package/dist/{eval-campaign-CYkUME2T.js → eval-campaign-DXhpZghy.js} +4 -5
- package/dist/{eval-campaign-CYkUME2T.js.map → eval-campaign-DXhpZghy.js.map} +1 -1
- package/dist/hosted/index.d.ts +1 -1
- package/dist/index-3cdlURSk.d.ts.map +1 -1
- package/dist/{index-CiP2DRCj.d.ts → index-BREtv3ZZ.d.ts} +3 -3
- package/dist/{index-CiP2DRCj.d.ts.map → index-BREtv3ZZ.d.ts.map} +1 -1
- package/dist/{index-B_eUhU9K.d.ts → index-C7Wue8R6.d.ts} +28 -5
- package/dist/{index-B_eUhU9K.d.ts.map → index-C7Wue8R6.d.ts.map} +1 -1
- package/dist/{index-BAvgST_9.d.ts → index-nhIYz9hn.d.ts} +67 -9
- package/dist/index-nhIYz9hn.d.ts.map +1 -0
- package/dist/index.d.ts +7 -7
- package/dist/index.js +25 -18
- package/dist/index.js.map +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +2 -2
- package/dist/ledger-core-CPZfcrC2.js +620 -0
- package/dist/ledger-core-CPZfcrC2.js.map +1 -0
- package/dist/meta-eval/index.js +2 -2
- package/dist/{mint-D5_87M5L.js → mint-BvkwcYZU.js} +2 -2
- package/dist/{mint-D5_87M5L.js.map → mint-BvkwcYZU.js.map} +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{opencode-sqlite-BGrHeDu3.js → opencode-sqlite-8r6WUfHc.js} +2 -3
- package/dist/opencode-sqlite-8r6WUfHc.js.map +1 -0
- package/dist/{paired-arms-D9D0wXj2.js → paired-arms-6XItKzd1.js} +2 -2
- package/dist/{paired-arms-D9D0wXj2.js.map → paired-arms-6XItKzd1.js.map} +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/profile-cell.js +1 -242
- package/dist/{propose-review-control-Bqb7daEJ.js → propose-review-control-SQ-n9-We.js} +2 -2
- package/dist/{propose-review-control-Bqb7daEJ.js.map → propose-review-control-SQ-n9-We.js.map} +1 -1
- package/dist/{release-report-DfmKSIEE.d.ts → release-report-CjHWa8Ia.d.ts} +2 -2
- package/dist/{release-report-DfmKSIEE.d.ts.map → release-report-CjHWa8Ia.d.ts.map} +1 -1
- package/dist/{release-report-oWt9f2k-.js → release-report-wuilQkvK.js} +3 -3
- package/dist/{release-report-oWt9f2k-.js.map → release-report-wuilQkvK.js.map} +1 -1
- package/dist/{replay-BMR4TEYY.js → replay-CJfGLdx4.js} +3 -3
- package/dist/{replay-BMR4TEYY.js.map → replay-CJfGLdx4.js.map} +1 -1
- package/dist/reporting.d.ts +2 -2
- package/dist/reporting.js +4 -4
- package/dist/{researcher-DMimgHtN.d.ts → researcher-CbSKhK8z.d.ts} +2 -2
- package/dist/{researcher-DMimgHtN.d.ts.map → researcher-CbSKhK8z.d.ts.map} +1 -1
- package/dist/{reward-hacking-BEvjdUtD.js → reward-hacking-Dl2UBzej.js} +3 -3
- package/dist/{reward-hacking-BEvjdUtD.js.map → reward-hacking-Dl2UBzej.js.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +6 -6
- package/dist/rollout/index.js +3 -3
- package/dist/{rollout-VEo41J0N.js → rollout-CreDz__7.js} +3 -3
- package/dist/{rollout-VEo41J0N.js.map → rollout-CreDz__7.js.map} +1 -1
- package/dist/{rubric-predictive-validity-B3xmbmS1.js → rubric-predictive-validity-QG7ydk0s.js} +2 -2
- package/dist/{rubric-predictive-validity-B3xmbmS1.js.map → rubric-predictive-validity-QG7ydk0s.js.map} +1 -1
- package/dist/{run-record-CN8Zd21B.js → run-record-BIwU2wdV.js} +2 -2
- package/dist/{run-record-CN8Zd21B.js.map → run-record-BIwU2wdV.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DgN7U9iR.js → skillopt-optimization-method-CF6a327Q.js} +8 -4
- package/dist/{skillopt-optimization-method-DgN7U9iR.js.map → skillopt-optimization-method-CF6a327Q.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CAASpcS3.d.ts → skillopt-optimization-method-wHF5xsUv.d.ts} +2 -2
- package/dist/{skillopt-optimization-method-CAASpcS3.d.ts.map → skillopt-optimization-method-wHF5xsUv.d.ts.map} +1 -1
- package/dist/{statistics-CnnxdpOg.js → statistics-DWM_AyLe.js} +90 -76
- package/dist/statistics-DWM_AyLe.js.map +1 -0
- package/dist/statistics-DbvkkDPa.d.ts.map +1 -1
- package/dist/{summary-report-DnUcjVpV.d.ts → summary-report-CFnQgNfg.d.ts} +2 -2
- package/dist/{summary-report-DnUcjVpV.d.ts.map → summary-report-CFnQgNfg.d.ts.map} +1 -1
- package/dist/{summary-report-BNs5nmXI.js → summary-report-Ci17nIdU.js} +4 -4
- package/dist/{summary-report-BNs5nmXI.js.map → summary-report-Ci17nIdU.js.map} +1 -1
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{supervisor-run-_lnTLM3z.js → supervisor-run-B7lUGoyZ.js} +2 -2
- package/dist/{supervisor-run-_lnTLM3z.js.map → supervisor-run-B7lUGoyZ.js.map} +1 -1
- package/dist/traces.js +1 -1
- package/package.json +1 -1
- package/dist/analyze-runs-BjPn_fOS.js.map +0 -1
- package/dist/index-BAvgST_9.d.ts.map +0 -1
- package/dist/ledger-core-eqaI3PCD.js +0 -388
- package/dist/ledger-core-eqaI3PCD.js.map +0 -1
- package/dist/opencode-sqlite-BGrHeDu3.js.map +0 -1
- package/dist/pre-registration-DakwTRXk.js +0 -96
- package/dist/pre-registration-DakwTRXk.js.map +0 -1
- package/dist/profile-cell.js.map +0 -1
- package/dist/statistics-CnnxdpOg.js.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ledger-core-CPZfcrC2.js","names":[],"sources":["../src/ledger-core/atomic-file-lock.ts","../src/ledger-core/canonical.ts","../src/ledger-core/journal-file.ts","../src/ledger-core/trusted-head.ts","../src/ledger-core/journal.ts"],"sourcesContent":["/** Crash-safe filesystem lock shared by campaign persistence and run exclusion. */\n\nimport { randomUUID } from 'node:crypto'\nimport {\n closeSync,\n constants,\n existsSync,\n fsyncSync,\n linkSync,\n openSync,\n readFileSync,\n renameSync,\n unlinkSync,\n writeFileSync,\n} from 'node:fs'\nimport { hostname } from 'node:os'\n\nexport interface AtomicFileLockOwner {\n readonly pid: number\n readonly host: string\n readonly nonce: string\n}\n\nexport interface AtomicFileLock {\n readonly owner: AtomicFileLockOwner\n release(): void\n}\n\nexport type AtomicFileLockUnavailable =\n | { readonly acquired: false; readonly reason: 'held'; readonly holder: AtomicFileLockOwner }\n | { readonly acquired: false; readonly reason: 'recovery' }\n\nexport type AtomicFileLockAcquisition =\n | { readonly acquired: true; readonly lock: AtomicFileLock }\n | AtomicFileLockUnavailable\n\nexport interface AtomicFileLockOptions {\n readonly lockPath: string\n readonly pid?: number\n}\n\nexport class AtomicFileLockError extends Error {\n override readonly name = 'AtomicFileLockError'\n}\n\ntype OwnerState =\n | { readonly state: 'missing' }\n | { readonly state: 'stale'; readonly owner: AtomicFileLockOwner }\n | { readonly state: 'held'; readonly owner: AtomicFileLockOwner }\n\n/** Report whether another lock path prevents acquisition without modifying it. */\nexport function probeAtomicFileLock(\n options: AtomicFileLockOptions,\n): AtomicFileLockUnavailable | null {\n if (existsSync(recoveryPath(options.lockPath))) return { acquired: false, reason: 'recovery' }\n const state = ownerState(options.lockPath)\n if (state.state === 'held') {\n return { acquired: false, reason: 'held', holder: state.owner }\n }\n return null\n}\n\n/**\n * Try to acquire a complete, uniquely-owned lock inode.\n *\n * A hard link publishes fully-written owner metadata atomically. Stale-owner\n * removal is serialized so one reclaimer cannot delete a new owner's lock.\n */\nexport function tryAcquireAtomicFileLock(\n options: AtomicFileLockOptions,\n): AtomicFileLockAcquisition {\n const pid = options.pid ?? process.pid\n if (!Number.isSafeInteger(pid) || pid <= 0) {\n throw new AtomicFileLockError('atomic file lock pid must be a positive integer')\n }\n const owner: AtomicFileLockOwner = { pid, host: hostname(), nonce: randomUUID() }\n for (let attempt = 0; attempt < 8; attempt += 1) {\n if (existsSync(recoveryPath(options.lockPath))) {\n return { acquired: false, reason: 'recovery' }\n }\n if (tryLinkOwner(options.lockPath, owner, `acquire.${attempt}`)) {\n return acquired(options.lockPath, owner)\n }\n\n const holder = ownerState(options.lockPath)\n if (holder.state === 'missing') continue\n if (holder.state === 'held') {\n return { acquired: false, reason: 'held', holder: holder.owner }\n }\n\n const reclaimPath = recoveryPath(options.lockPath)\n if (!tryLinkOwner(reclaimPath, owner, `reclaim.${attempt}`)) {\n return { acquired: false, reason: 'recovery' }\n }\n try {\n const current = ownerState(options.lockPath)\n if (current.state === 'held') {\n return { acquired: false, reason: 'held', holder: current.owner }\n }\n if (current.state === 'stale') {\n const tombstone = `${options.lockPath}.stale.${owner.nonce}.${attempt}`\n try {\n renameSync(options.lockPath, tombstone)\n unlinkSync(tombstone)\n } catch (error) {\n if (!isMissing(error)) throw error\n }\n }\n if (tryLinkOwner(options.lockPath, owner, `recovered.${attempt}`)) {\n return acquired(options.lockPath, owner)\n }\n } finally {\n releaseOwnedPath(reclaimPath, owner)\n }\n }\n\n throw new AtomicFileLockError(`could not acquire atomic file lock ${options.lockPath}`)\n}\n\nfunction acquired(lockPath: string, owner: AtomicFileLockOwner): AtomicFileLockAcquisition {\n return {\n acquired: true,\n lock: {\n owner,\n release: () => releaseOwnedPath(lockPath, owner),\n },\n }\n}\n\nfunction tryLinkOwner(lockPath: string, owner: AtomicFileLockOwner, suffix: string): boolean {\n const ownerPath = `${lockPath}.${owner.pid}.${owner.nonce}.${suffix}.owner`\n const descriptor = openSync(\n ownerPath,\n constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,\n 0o600,\n )\n try {\n writeFileSync(descriptor, `${canonicalOwner(owner)}\\n`, 'utf8')\n fsyncSync(descriptor)\n } catch (error) {\n closeSync(descriptor)\n unlinkIfExists(ownerPath)\n throw error\n }\n closeSync(descriptor)\n\n try {\n linkSync(ownerPath, lockPath)\n return true\n } catch (error) {\n if (!isAlreadyExists(error)) throw error\n return false\n } finally {\n unlinkIfExists(ownerPath)\n }\n}\n\nfunction ownerState(lockPath: string): OwnerState {\n const owner = readOwner(lockPath)\n if (!owner) return { state: 'missing' }\n if (owner.host !== hostname()) return { state: 'held', owner }\n try {\n process.kill(owner.pid, 0)\n return { state: 'held', owner }\n } catch (error) {\n if (isNoSuchProcess(error)) return { state: 'stale', owner }\n return { state: 'held', owner }\n }\n}\n\nfunction readOwner(lockPath: string): AtomicFileLockOwner | undefined {\n let contents: string\n try {\n contents = readFileSync(lockPath, 'utf8').trim()\n } catch (error) {\n if (isMissing(error)) return undefined\n throw error\n }\n\n let value: unknown\n try {\n value = JSON.parse(contents)\n } catch (error) {\n throw invalidOwner(lockPath, error)\n }\n if (!isOwner(value)) throw invalidOwner(lockPath)\n return value\n}\n\nfunction isOwner(value: unknown): value is AtomicFileLockOwner {\n if (typeof value !== 'object' || value === null || Array.isArray(value)) return false\n const record = value as Record<string, unknown>\n return (\n Object.keys(record).length === 3 &&\n Number.isSafeInteger(record.pid) &&\n Number(record.pid) > 0 &&\n typeof record.host === 'string' &&\n record.host.trim().length > 0 &&\n typeof record.nonce === 'string' &&\n record.nonce.trim().length > 0\n )\n}\n\nfunction invalidOwner(lockPath: string, cause?: unknown): AtomicFileLockError {\n const error = new AtomicFileLockError(\n `atomic file lock has an invalid owner (${lockPath}); refusing unsafe recovery`,\n )\n if (cause !== undefined) Object.defineProperty(error, 'cause', { value: cause })\n return error\n}\n\nfunction releaseOwnedPath(lockPath: string, owner: AtomicFileLockOwner): void {\n const current = readOwner(lockPath)\n if (!current) return\n if (canonicalOwner(current) !== canonicalOwner(owner)) {\n throw new AtomicFileLockError(`atomic file lock owner changed before release (${lockPath})`)\n }\n try {\n unlinkSync(lockPath)\n } catch (error) {\n if (!isMissing(error)) throw error\n }\n}\n\nfunction canonicalOwner(owner: AtomicFileLockOwner): string {\n return JSON.stringify({ host: owner.host, nonce: owner.nonce, pid: owner.pid })\n}\n\nfunction recoveryPath(lockPath: string): string {\n return `${lockPath}.reclaim`\n}\n\nfunction isAlreadyExists(error: unknown): boolean {\n return error instanceof Error && 'code' in error && error.code === 'EEXIST'\n}\n\nfunction isMissing(error: unknown): boolean {\n return error instanceof Error && 'code' in error && error.code === 'ENOENT'\n}\n\nfunction isNoSuchProcess(error: unknown): boolean {\n return error instanceof Error && 'code' in error && error.code === 'ESRCH'\n}\n\nfunction unlinkIfExists(path: string): void {\n try {\n unlinkSync(path)\n } catch (error) {\n if (!isMissing(error)) throw error\n }\n}\n","/**\n * Canonical bytes and digests for durable journal rows.\n *\n * Serialization is RFC 8785 through `@tangle-network/agent-interface` — the\n * stack's single identity scheme — so a value carries the same digest in every\n * package that produces or verifies one.\n *\n * Values canonical JSON cannot represent faithfully are refused, never coerced.\n * `NaN`, `Infinity`, `undefined`, class instances, cycles, and unpaired\n * surrogates each have a lossy encoding that maps distinct records onto one\n * digest, and in a hash chain the digest IS the record's identity: a coercion\n * there is a collision, not a convenience.\n */\n\nimport {\n canonicalCandidateBytes,\n canonicalCandidateJson,\n sha256Bytes,\n} from '@tangle-network/agent-interface'\nimport { ValidationError } from '../errors'\n\nexport type LedgerHash = `sha256:${string}`\n\n/** Shape of every ledger digest: `sha256:` followed by 64 lowercase hex chars. */\nexport const LEDGER_HASH_PATTERN = /^sha256:[a-f0-9]{64}$/\n\n/** A value has no faithful canonical-JSON form, so it cannot be stored or hashed. */\nexport class LedgerCanonicalizationError extends ValidationError {}\n\n/** Canonical JSON encoding (RFC 8785): object keys sorted by UTF-16 code unit,\n * recursively. This is the byte form that is hashed and stored, so equal values\n * always encode identically. */\nexport function canonicalString(value: unknown): string {\n try {\n return canonicalCandidateJson(value)\n } catch (error) {\n throw canonicalizationError(value, error)\n }\n}\n\nexport function hashCanonical(value: unknown): LedgerHash {\n try {\n return sha256Bytes(canonicalCandidateBytes(value))\n } catch (error) {\n throw canonicalizationError(value, error)\n }\n}\n\n/** Name the offending value and path so a refusal is actionable. The RFC 8785\n * encoder above is the sole authority on acceptance; this walk only explains a\n * rejection it already made. */\nfunction canonicalizationError(value: unknown, cause: unknown): LedgerCanonicalizationError {\n const defect = firstNonCanonicalDefect(value, '$', new Set())\n const reason = defect !== '' ? defect : cause instanceof Error ? cause.message : String(cause)\n return new LedgerCanonicalizationError(`value has no canonical JSON form: ${reason}`, {\n cause,\n })\n}\n\nfunction firstNonCanonicalDefect(value: unknown, path: string, ancestors: Set<object>): string {\n if (value === null) return ''\n if (value === undefined) return `${path} is undefined`\n const type = typeof value\n if (type === 'number') {\n return Number.isFinite(value) ? '' : `${path} is ${String(value)}`\n }\n if (type === 'string') {\n return isWellFormedUtf16(value as string) ? '' : `${path} contains an unpaired surrogate`\n }\n if (type === 'boolean') return ''\n if (type !== 'object') return `${path} is a ${type}`\n const object = value as object\n if (ancestors.has(object)) return `${path} closes a reference cycle`\n const prototype = Object.getPrototypeOf(object)\n if (!Array.isArray(object) && prototype !== Object.prototype && prototype !== null) {\n return `${path} is a ${object.constructor?.name ?? 'non-plain'} instance, not plain JSON data`\n }\n const nested = new Set(ancestors).add(object)\n if (Array.isArray(object)) {\n for (let index = 0; index < object.length; index += 1) {\n const defect = firstNonCanonicalDefect(object[index], `${path}[${index}]`, nested)\n if (defect !== '') return defect\n }\n return ''\n }\n for (const [key, entry] of Object.entries(object)) {\n if (!isWellFormedUtf16(key)) return `${path}.${key} has a key with an unpaired surrogate`\n const defect = firstNonCanonicalDefect(entry, `${path}.${key}`, nested)\n if (defect !== '') return defect\n }\n return ''\n}\n\nfunction isWellFormedUtf16(value: string): boolean {\n for (let index = 0; index < value.length; index += 1) {\n const code = value.charCodeAt(index)\n if (code >= 0xd800 && code <= 0xdbff) {\n const next = value.charCodeAt(index + 1)\n if (!(next >= 0xdc00 && next <= 0xdfff)) return false\n index += 1\n } else if (code >= 0xdc00 && code <= 0xdfff) {\n return false\n }\n }\n return true\n}\n","/** Filesystem durability and cross-process exclusion for append-only journal files. */\n\nimport {\n closeSync,\n constants,\n fsyncSync,\n mkdirSync,\n openSync,\n renameSync,\n writeSync,\n} from 'node:fs'\nimport { dirname } from 'node:path'\nimport { AtomicFileLockError, tryAcquireAtomicFileLock } from './atomic-file-lock'\n\n/** How a journal binding reports file-layer faults in its own error taxonomy. */\nexport interface LedgerFileContext {\n /** Noun used in error messages, e.g. 'search ledger'. */\n subject: string\n integrityError(message: string, options?: { cause?: unknown }): Error\n}\n\n/** Append one already-serialized line; the write is fsynced (file and directory)\n * before returning so an acknowledged append survives a crash. */\nexport function appendLedgerLine(path: string, line: string, context: LedgerFileContext): void {\n mkdirSync(dirname(path), { recursive: true })\n const fd = openSync(path, constants.O_CREAT | constants.O_WRONLY | constants.O_APPEND, 0o600)\n try {\n writeAll(fd, Buffer.from(line, 'utf8'), context)\n fsyncSync(fd)\n } finally {\n closeSync(fd)\n }\n fsyncDirectory(dirname(path))\n}\n\n/** Replace a small sidecar file whole: write a temporary sibling, fsync it,\n * then rename over the target. A reader therefore sees the previous contents\n * or the new ones, never a partial file. */\nexport function writeLedgerFileAtomically(\n path: string,\n contents: string,\n context: LedgerFileContext,\n): void {\n const directory = dirname(path)\n mkdirSync(directory, { recursive: true })\n const temporaryPath = `${path}.tmp`\n const fd = openSync(\n temporaryPath,\n constants.O_CREAT | constants.O_WRONLY | constants.O_TRUNC,\n 0o600,\n )\n try {\n writeAll(fd, Buffer.from(contents, 'utf8'), context)\n fsyncSync(fd)\n } finally {\n closeSync(fd)\n }\n renameSync(temporaryPath, path)\n fsyncDirectory(directory)\n}\n\nexport function withLedgerFileLock<T>(path: string, context: LedgerFileContext, run: () => T): T {\n const result = tryWithLedgerFileLock(path, context, run)\n if (!result.acquired) {\n throw context.integrityError(`${context.subject} lock is held (${path})`)\n }\n return result.value\n}\n\nexport type FileLockResult<T> = { acquired: true; value: T } | { acquired: false }\n\nexport function tryWithLedgerFileLock<T>(\n path: string,\n context: LedgerFileContext,\n run: () => T,\n): FileLockResult<T> {\n mkdirSync(dirname(path), { recursive: true })\n const lockPath = `${path}.lock`\n try {\n const acquisition = tryAcquireAtomicFileLock({ lockPath })\n if (!acquisition.acquired) return { acquired: false }\n try {\n return { acquired: true, value: run() }\n } finally {\n acquisition.lock.release()\n }\n } catch (error) {\n if (error instanceof AtomicFileLockError) {\n throw context.integrityError(error.message, { cause: error })\n }\n throw error\n }\n}\n\nfunction writeAll(fd: number, bytes: Buffer, context: LedgerFileContext): void {\n let offset = 0\n while (offset < bytes.byteLength) {\n const written = writeSync(fd, bytes, offset, bytes.byteLength - offset)\n if (written <= 0) throw context.integrityError('filesystem wrote zero bytes')\n offset += written\n }\n}\n\nfunction fsyncDirectory(path: string): void {\n const fd = openSync(path, constants.O_RDONLY)\n try {\n fsyncSync(fd)\n } finally {\n closeSync(fd)\n }\n}\n","/**\n * Trusted head — the pin that makes deletion detectable.\n *\n * ## What the chain binds\n *\n * Every entry hashes the journal header, its sequence, the previous entry's\n * hash, and its event, so editing, reordering, or removing an entry in the\n * middle of a journal invalidates every hash after it. What a hash chain\n * cannot bind is its own length. Dropping entries off the END leaves a shorter\n * journal that is internally perfect, and rebuilding a journal from sequence 0\n * with recomputed hashes produces a different, equally perfect chain. For an\n * audit log of candidate registrations and promotion decisions that is the\n * difference between a record of a search that happened and a record of one\n * that did not.\n *\n * ## What the pin binds\n *\n * `<journal>.head` holds `{sequence, entryHash}` for one entry a writer\n * actually appended, in a file separate from the journal it attests. Each\n * entry hash commits to the entire prefix before it, so checking that the\n * entry at exactly `sequence` still carries `entryHash` proves two things at\n * once: the journal is at least `sequence + 1` entries long, and those entries\n * are byte-for-byte the ones the writer saw. A journal that merely grew past\n * the pin still verifies; a truncated, rolled-back, or wholesale-rewritten one\n * cannot.\n *\n * ## What the pin does NOT defend against\n *\n * The pin is a second file, not a signature. A writer with access to BOTH the\n * journal and its pin can rewrite both consistently, and the result verifies.\n * Pinning raises the cost of undetected tampering from one file to two and\n * nothing more; defending against a writer who holds both requires an anchor\n * outside this store entirely — a signed head, or a digest published to an\n * append-only service the tamperer does not control.\n *\n * Deleting the pin file downgrades a pinned journal to an unpinned one, which\n * cannot be distinguished from a journal written before pinning existed. That\n * is what `requireTrustedHead` is for: a journal opened under it refuses to\n * read at all once a non-empty journal has no pin.\n */\n\nimport { existsSync, readFileSync } from 'node:fs'\nimport { canonicalString, LEDGER_HASH_PATTERN, type LedgerHash } from './canonical'\nimport { type LedgerFileContext, writeLedgerFileAtomically } from './journal-file'\n\n/** A `(sequence, entryHash)` pair a writer pinned outside the journal. */\nexport interface LedgerTrustedHead {\n sequence: number\n entryHash: LedgerHash\n}\n\n/** Minimum entry shape the anchor check needs. */\nexport interface LedgerAnchoredEntry {\n sequence: number\n entryHash: LedgerHash\n}\n\n/** Sibling pin file for a journal — deliberately NOT the journal, so rewriting\n * the log is not enough to rewrite the trust recorded about it. */\nexport function trustedHeadPathFor(journalPath: string): string {\n return `${journalPath}.head`\n}\n\n/** The pin, or null when this journal has never been pinned. A pin that exists\n * but does not parse is corruption or tamper of the trust record itself and\n * fails loudly: reporting it as \"no pin\" would silently downgrade what the\n * journal can prove. */\nexport function readTrustedHeadFile(\n path: string,\n context: LedgerFileContext,\n): LedgerTrustedHead | null {\n if (!existsSync(path)) return null\n const text = readFileSync(path, 'utf8')\n let raw: unknown\n try {\n raw = JSON.parse(text)\n } catch (error) {\n throw context.integrityError(`${context.subject} trusted head ${path} is not valid JSON`, {\n cause: error,\n })\n }\n if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {\n throw context.integrityError(`${context.subject} trusted head ${path} is not an object`)\n }\n const record = raw as Record<string, unknown>\n const keys = Object.keys(record).sort()\n if (keys.length !== 2 || keys[0] !== 'entryHash' || keys[1] !== 'sequence') {\n throw context.integrityError(\n `${context.subject} trusted head ${path} has keys [${keys.join(', ')}], expected [entryHash, sequence]`,\n )\n }\n const { sequence, entryHash } = record\n if (typeof sequence !== 'number' || !Number.isSafeInteger(sequence) || sequence < 0) {\n throw context.integrityError(\n `${context.subject} trusted head ${path} sequence is not a non-negative integer`,\n )\n }\n if (typeof entryHash !== 'string' || !LEDGER_HASH_PATTERN.test(entryHash)) {\n throw context.integrityError(\n `${context.subject} trusted head ${path} entryHash is not a sha256 digest`,\n )\n }\n return { sequence, entryHash: entryHash as LedgerHash }\n}\n\n/** Publish a pin by write-then-rename, so a crash leaves either the previous\n * pin or the new one and never a torn file that would lock a journal out of\n * its own trust record. */\nexport function writeTrustedHeadFile(\n path: string,\n head: LedgerTrustedHead,\n context: LedgerFileContext,\n): void {\n writeLedgerFileAtomically(path, `${canonicalString(head)}\\n`, context)\n}\n\n/** The entry at exactly `head.sequence` must still carry `head.entryHash`.\n * Entries must already have passed chain verification, which is what makes\n * index and sequence interchangeable here. */\nexport function verifyEntriesAgainstTrustedHead(\n entries: readonly LedgerAnchoredEntry[],\n head: LedgerTrustedHead,\n context: LedgerFileContext,\n subject: string,\n): void {\n const pinned = entries[head.sequence]\n if (pinned === undefined) {\n throw context.integrityError(\n `${subject} trusted head pins sequence ${head.sequence} (${head.entryHash}) but the journal has ${entries.length} entries — pinned history is missing (truncation, rollback, or rewrite)`,\n )\n }\n if (pinned.sequence !== head.sequence) {\n throw context.integrityError(\n `${subject} entry at index ${head.sequence} carries sequence ${pinned.sequence}; the chain was not verified before the trusted head`,\n )\n }\n if (pinned.entryHash !== head.entryHash) {\n throw context.integrityError(\n `${subject} entry at pinned sequence ${head.sequence} carries ${pinned.entryHash} but the trusted head recorded ${head.entryHash} — the journal was rewritten`,\n )\n }\n}\n","/**\n * Generic durable append-only journal with a SHA-256 hash chain.\n *\n * The file format is canonical JSONL: every row is the canonical JSON encoding\n * (RFC 8785, object keys sorted recursively) of one entry, and every entry\n * hashes the previous entry's hash so the log is tamper-evident. Appends are\n * serialized in-process per path and across processes via a lock file, fsynced\n * before acknowledgement, and idempotent by `eventId`. A malformed,\n * non-canonical, truncated, reordered, or conflicting log fails loudly; a bad\n * row is never skipped.\n *\n * A hash chain binds each entry to its predecessor, which cannot prove that no\n * entry was removed from the end: a valid shorter prefix is still a valid\n * chain. `pinHead` records the appended entry in a sibling `<journal>.head`\n * file and every read re-verifies the journal against a pin that is present,\n * so truncation and wholesale rewrite are refused. `trusted-head.ts` carries\n * the full threat model, including what a pin does not defend against.\n *\n * Domain vocabulary lives entirely in the consumer's codec: entry schema\n * validation, the constant header fields stamped into every entry, the error\n * taxonomy, and the state machine replayed over verified entries.\n */\n\nimport { existsSync, readFileSync } from 'node:fs'\nimport { resolve } from 'node:path'\nimport { Mutex } from '../concurrency'\nimport { canonicalString, hashCanonical, type LedgerHash } from './canonical'\nimport { appendLedgerLine, type LedgerFileContext, withLedgerFileLock } from './journal-file'\nimport {\n type LedgerTrustedHead,\n readTrustedHeadFile,\n trustedHeadPathFor,\n verifyEntriesAgainstTrustedHead,\n writeTrustedHeadFile,\n} from './trusted-head'\n\n/** Minimum shape of a journal event: the idempotency key. */\nexport interface LedgerEventBase {\n eventId: string\n}\n\n/** Chain envelope every entry carries in addition to the journal's constant\n * header fields. `entryHash` covers the header, sequence, previous hash, and\n * event, so any rewrite of history invalidates every later entry. */\nexport interface LedgerChainFields<Event extends LedgerEventBase> {\n sequence: number\n previousHash: LedgerHash | null\n event: Event\n entryHash: LedgerHash\n}\n\nexport type LedgerEntryOf<Header extends object, Event extends LedgerEventBase> = Header &\n LedgerChainFields<Event>\n\nexport interface LedgerLineContext {\n path: string\n line: number\n}\n\n/** Domain state machine replayed over chain-verified entries in order.\n * `apply` throws (via the codec's error taxonomy) on an invalid transition;\n * `finish` builds the consumer's projection once every entry was applied. */\nexport interface LedgerProjector<Entry, Projection> {\n apply(entry: Entry, index: number): void\n finish(entries: Entry[]): Projection\n}\n\nexport interface LedgerJournalCodec<\n Header extends object,\n Event extends LedgerEventBase,\n Projection,\n> extends LedgerFileContext {\n /** Constant fields stamped into every entry and covered by its hash. */\n header: Header\n conflictError(message: string): Error\n /** Validate one parsed JSON row into a typed entry. Must reject malformed\n * rows and non-canonical event encodings with the binding's own errors;\n * the journal itself verifies the stored bytes and the hash chain. */\n parseEntry(raw: unknown, context: LedgerLineContext): LedgerEntryOf<Header, Event>\n /** Reject an entry whose constant header fields do not match this journal. */\n checkEntryHeader(entry: LedgerEntryOf<Header, Event>, index: number): void\n createProjector(): LedgerProjector<LedgerEntryOf<Header, Event>, Projection>\n}\n\nexport interface LedgerReplayResult<Entry, Projection> {\n entries: Entry[]\n projection: Projection\n}\n\nexport interface LedgerAppendResult<Entry, Projection> {\n entry: Entry\n /** False when the exact event was already durably present. */\n appended: boolean\n projection: Projection\n}\n\nexport interface LedgerAppendOptions {\n /** Record the appended entry as this journal's trusted head, so a later read\n * can prove nothing was deleted from the end. Ignored on the idempotent\n * path: acknowledging an event that was already durable writes nothing, and\n * moving the pin there would jump trust forward past entries this caller\n * never saw. */\n pinHead?: boolean\n}\n\nexport interface FileLedgerJournalOptions {\n /** Refuse to read a non-empty journal that has no trusted head.\n *\n * Off by default because journals written before pinning have no pin and\n * must keep opening. Turn it on for a journal this process pins: without it,\n * deleting the sibling pin file silently downgrades the journal back to a\n * chain that cannot detect deletion. */\n requireTrustedHead?: boolean\n}\n\n// One async mutex per resolved journal path so concurrent appends from a single\n// process queue up instead of failing on the held cross-process lock file.\nconst journalMutexes = new Map<string, Mutex>()\n\nfunction mutexFor(path: string): Mutex {\n const existing = journalMutexes.get(path)\n if (existing) return existing\n const mutex = new Mutex()\n journalMutexes.set(path, mutex)\n return mutex\n}\n\n/** Durable filesystem journal. Construction performs no I/O; `append` and\n * `replay` validate the complete existing file under the locks. */\nexport class FileLedgerJournal<Header extends object, Event extends LedgerEventBase, Projection> {\n readonly path: string\n /** Sibling file holding this journal's trusted head. */\n readonly trustedHeadPath: string\n private readonly codec: LedgerJournalCodec<Header, Event, Projection>\n private readonly mutex: Mutex\n private readonly requireTrustedHead: boolean\n\n constructor(\n path: string,\n codec: LedgerJournalCodec<Header, Event, Projection>,\n options: FileLedgerJournalOptions = {},\n ) {\n this.path = resolve(path)\n this.trustedHeadPath = trustedHeadPathFor(this.path)\n this.codec = codec\n this.mutex = mutexFor(this.path)\n this.requireTrustedHead = options.requireTrustedHead === true\n }\n\n async replay(): Promise<LedgerReplayResult<LedgerEntryOf<Header, Event>, Projection>> {\n return this.mutex.runExclusive(() =>\n withLedgerFileLock(this.path, this.codec, () => this.replayLocked()),\n )\n }\n\n async append(\n event: Event,\n options: LedgerAppendOptions = {},\n ): Promise<LedgerAppendResult<LedgerEntryOf<Header, Event>, Projection>> {\n return this.mutex.runExclusive(() =>\n withLedgerFileLock(this.path, this.codec, () => {\n const before = this.replayLocked()\n const existing = before.entries.find((entry) => entry.event.eventId === event.eventId)\n if (existing) {\n if (canonicalString(existing.event) !== canonicalString(event)) {\n throw this.codec.conflictError(\n `eventId ${event.eventId} already exists with different content`,\n )\n }\n return { entry: existing, appended: false, projection: before.projection }\n }\n\n const material = {\n ...this.codec.header,\n sequence: before.entries.length,\n previousHash: before.entries.at(-1)?.entryHash ?? null,\n event,\n }\n const entry = { ...material, entryHash: hashCanonical(material) }\n\n // Apply the state transition before spending an append: an entry the\n // projector rejects must never reach the durable file.\n const projection = this.project([...before.entries, entry])\n appendLedgerLine(this.path, `${canonicalString(entry)}\\n`, this.codec)\n if (options.pinHead === true) {\n // Journal first, pin second: a crash between them leaves the pin one\n // entry behind, which still verifies and is healed by the next\n // pinning append. The reverse order would leave a pin naming an\n // entry the journal does not carry, locking the journal out.\n writeTrustedHeadFile(\n this.trustedHeadPath,\n { sequence: entry.sequence, entryHash: entry.entryHash },\n this.codec,\n )\n }\n return { entry, appended: true, projection }\n }),\n )\n }\n\n /** The pinned head, or null when this journal has never been pinned. */\n async trustedHead(): Promise<LedgerTrustedHead | null> {\n return this.mutex.runExclusive(() =>\n withLedgerFileLock(this.path, this.codec, () =>\n readTrustedHeadFile(this.trustedHeadPath, this.codec),\n ),\n )\n }\n\n /** Pin the current verified head. The chain and any existing pin are checked\n * first, so the pin only ever moves forward along a journal that still\n * carries the history it already recorded. */\n async pinTrustedHead(): Promise<LedgerTrustedHead> {\n return this.mutex.runExclusive(() =>\n withLedgerFileLock(this.path, this.codec, () => {\n const entries = this.readEntries()\n this.project(entries)\n // `requireTrustedHead` is deliberately not applied here: pinning is how\n // a journal that lacks a pin acquires one.\n this.verifyTrustedHead(entries, false)\n const last = entries.at(-1)\n if (last === undefined) {\n throw this.codec.integrityError(\n `${this.codec.subject} ${this.path} is empty; there is no head to pin`,\n )\n }\n const head: LedgerTrustedHead = { sequence: last.sequence, entryHash: last.entryHash }\n writeTrustedHeadFile(this.trustedHeadPath, head, this.codec)\n return head\n }),\n )\n }\n\n private replayLocked(): LedgerReplayResult<LedgerEntryOf<Header, Event>, Projection> {\n const entries = this.readEntries()\n const projection = this.project(entries)\n // The anchor is meaningful only over a chain that already verified, so it\n // is checked last — and on every read path, so a violated pin is refused\n // before an append can extend the journal over it.\n this.verifyTrustedHead(entries, this.requireTrustedHead)\n return { entries, projection }\n }\n\n private verifyTrustedHead(entries: LedgerEntryOf<Header, Event>[], requirePin: boolean): void {\n const pinned = readTrustedHeadFile(this.trustedHeadPath, this.codec)\n if (pinned === null) {\n if (requirePin && entries.length > 0) {\n throw this.codec.integrityError(\n `${this.codec.subject} ${this.path} has ${entries.length} entries but no trusted head at ${this.trustedHeadPath} — without its pin the journal cannot prove nothing was deleted`,\n )\n }\n return\n }\n verifyEntriesAgainstTrustedHead(\n entries,\n pinned,\n this.codec,\n `${this.codec.subject} ${this.path}`,\n )\n }\n\n private readEntries(): LedgerEntryOf<Header, Event>[] {\n if (!existsSync(this.path)) return []\n const text = readFileSync(this.path, 'utf8')\n if (text.length === 0) return []\n if (!text.endsWith('\\n')) {\n throw this.codec.integrityError(\n `${this.codec.subject} ${this.path} has a truncated final record (missing newline)`,\n )\n }\n\n const lines = text.slice(0, -1).split('\\n')\n const entries: LedgerEntryOf<Header, Event>[] = []\n for (let index = 0; index < lines.length; index += 1) {\n const line = lines[index]!\n if (line.length === 0) {\n throw this.codec.integrityError(\n `${this.codec.subject} ${this.path} has a blank row at line ${index + 1}`,\n )\n }\n let raw: unknown\n try {\n raw = JSON.parse(line)\n } catch (error) {\n throw this.codec.integrityError(\n `${this.codec.subject} ${this.path} has invalid JSON at line ${index + 1}`,\n { cause: error },\n )\n }\n const entry = this.codec.parseEntry(raw, { path: this.path, line: index + 1 })\n let canonical: string\n try {\n canonical = canonicalString(entry)\n } catch (error) {\n throw this.codec.integrityError(\n `${this.codec.subject} ${this.path} has a row with no canonical JSON form at line ${index + 1}`,\n { cause: error },\n )\n }\n if (line !== canonical) {\n throw this.codec.integrityError(\n `${this.codec.subject} ${this.path} has non-canonical bytes at line ${index + 1}`,\n )\n }\n entries.push(entry)\n }\n return entries\n }\n\n private project(entries: LedgerEntryOf<Header, Event>[]): Projection {\n const projector = this.codec.createProjector()\n const eventIds = new Set<string>()\n let expectedPrevious: LedgerHash | null = null\n for (let index = 0; index < entries.length; index += 1) {\n const entry = entries[index]!\n this.codec.checkEntryHeader(entry, index)\n if (entry.sequence !== index) {\n throw this.codec.integrityError(\n `entry ${entry.event.eventId} has sequence ${entry.sequence}, expected ${index}`,\n )\n }\n if (entry.previousHash !== expectedPrevious) {\n throw this.codec.integrityError(\n `entry ${entry.event.eventId} does not extend the previous hash`,\n )\n }\n const { entryHash: _entryHash, ...material } = entry\n const expectedHash = hashCanonical(material)\n if (entry.entryHash !== expectedHash) {\n throw this.codec.integrityError(\n `entry ${entry.event.eventId} hash mismatch: expected ${expectedHash}, got ${entry.entryHash}`,\n )\n }\n expectedPrevious = entry.entryHash\n if (eventIds.has(entry.event.eventId)) {\n throw this.codec.integrityError(\n `duplicate eventId ${entry.event.eventId} in durable ledger`,\n )\n }\n eventIds.add(entry.event.eventId)\n projector.apply(entry, index)\n }\n return projector.finish(entries)\n }\n}\n"],"mappings":";;;;;;;;;AAyCA,IAAa,sBAAb,cAAyC,MAAM;CAC7C,OAAyB;AAC3B;;AAQA,SAAgB,oBACd,SACkC;CAClC,IAAI,WAAW,aAAa,QAAQ,QAAQ,CAAC,GAAG,OAAO;EAAE,UAAU;EAAO,QAAQ;CAAW;CAC7F,MAAM,QAAQ,WAAW,QAAQ,QAAQ;CACzC,IAAI,MAAM,UAAU,QAClB,OAAO;EAAE,UAAU;EAAO,QAAQ;EAAQ,QAAQ,MAAM;CAAM;CAEhE,OAAO;AACT;;;;;;;AAQA,SAAgB,yBACd,SAC2B;CAC3B,MAAM,MAAM,QAAQ,OAAO,QAAQ;CACnC,IAAI,CAAC,OAAO,cAAc,GAAG,KAAK,OAAO,GACvC,MAAM,IAAI,oBAAoB,iDAAiD;CAEjF,MAAM,QAA6B;EAAE;EAAK,MAAM,SAAS;EAAG,OAAO,WAAW;CAAE;CAChF,KAAK,IAAI,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG;EAC/C,IAAI,WAAW,aAAa,QAAQ,QAAQ,CAAC,GAC3C,OAAO;GAAE,UAAU;GAAO,QAAQ;EAAW;EAE/C,IAAI,aAAa,QAAQ,UAAU,OAAO,WAAW,SAAS,GAC5D,OAAO,SAAS,QAAQ,UAAU,KAAK;EAGzC,MAAM,SAAS,WAAW,QAAQ,QAAQ;EAC1C,IAAI,OAAO,UAAU,WAAW;EAChC,IAAI,OAAO,UAAU,QACnB,OAAO;GAAE,UAAU;GAAO,QAAQ;GAAQ,QAAQ,OAAO;EAAM;EAGjE,MAAM,cAAc,aAAa,QAAQ,QAAQ;EACjD,IAAI,CAAC,aAAa,aAAa,OAAO,WAAW,SAAS,GACxD,OAAO;GAAE,UAAU;GAAO,QAAQ;EAAW;EAE/C,IAAI;GACF,MAAM,UAAU,WAAW,QAAQ,QAAQ;GAC3C,IAAI,QAAQ,UAAU,QACpB,OAAO;IAAE,UAAU;IAAO,QAAQ;IAAQ,QAAQ,QAAQ;GAAM;GAElE,IAAI,QAAQ,UAAU,SAAS;IAC7B,MAAM,YAAY,GAAG,QAAQ,SAAS,SAAS,MAAM,MAAM,GAAG;IAC9D,IAAI;KACF,WAAW,QAAQ,UAAU,SAAS;KACtC,WAAW,SAAS;IACtB,SAAS,OAAO;KACd,IAAI,CAAC,UAAU,KAAK,GAAG,MAAM;IAC/B;GACF;GACA,IAAI,aAAa,QAAQ,UAAU,OAAO,aAAa,SAAS,GAC9D,OAAO,SAAS,QAAQ,UAAU,KAAK;EAE3C,UAAU;GACR,iBAAiB,aAAa,KAAK;EACrC;CACF;CAEA,MAAM,IAAI,oBAAoB,sCAAsC,QAAQ,UAAU;AACxF;AAEA,SAAS,SAAS,UAAkB,OAAuD;CACzF,OAAO;EACL,UAAU;EACV,MAAM;GACJ;GACA,eAAe,iBAAiB,UAAU,KAAK;EACjD;CACF;AACF;AAEA,SAAS,aAAa,UAAkB,OAA4B,QAAyB;CAC3F,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,IAAI,GAAG,MAAM,MAAM,GAAG,OAAO;CACpE,MAAM,aAAa,SACjB,WACA,UAAU,UAAU,UAAU,SAAS,UAAU,UACjD,GACF;CACA,IAAI;EACF,cAAc,YAAY,GAAG,eAAe,KAAK,EAAE,KAAK,MAAM;EAC9D,UAAU,UAAU;CACtB,SAAS,OAAO;EACd,UAAU,UAAU;EACpB,eAAe,SAAS;EACxB,MAAM;CACR;CACA,UAAU,UAAU;CAEpB,IAAI;EACF,SAAS,WAAW,QAAQ;EAC5B,OAAO;CACT,SAAS,OAAO;EACd,IAAI,CAAC,gBAAgB,KAAK,GAAG,MAAM;EACnC,OAAO;CACT,UAAU;EACR,eAAe,SAAS;CAC1B;AACF;AAEA,SAAS,WAAW,UAA8B;CAChD,MAAM,QAAQ,UAAU,QAAQ;CAChC,IAAI,CAAC,OAAO,OAAO,EAAE,OAAO,UAAU;CACtC,IAAI,MAAM,SAAS,SAAS,GAAG,OAAO;EAAE,OAAO;EAAQ;CAAM;CAC7D,IAAI;EACF,QAAQ,KAAK,MAAM,KAAK,CAAC;EACzB,OAAO;GAAE,OAAO;GAAQ;EAAM;CAChC,SAAS,OAAO;EACd,IAAI,gBAAgB,KAAK,GAAG,OAAO;GAAE,OAAO;GAAS;EAAM;EAC3D,OAAO;GAAE,OAAO;GAAQ;EAAM;CAChC;AACF;AAEA,SAAS,UAAU,UAAmD;CACpE,IAAI;CACJ,IAAI;EACF,WAAW,aAAa,UAAU,MAAM,CAAC,CAAC,KAAK;CACjD,SAAS,OAAO;EACd,IAAI,UAAU,KAAK,GAAG,OAAO,KAAA;EAC7B,MAAM;CACR;CAEA,IAAI;CACJ,IAAI;EACF,QAAQ,KAAK,MAAM,QAAQ;CAC7B,SAAS,OAAO;EACd,MAAM,aAAa,UAAU,KAAK;CACpC;CACA,IAAI,CAAC,QAAQ,KAAK,GAAG,MAAM,aAAa,QAAQ;CAChD,OAAO;AACT;AAEA,SAAS,QAAQ,OAA8C;CAC7D,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG,OAAO;CAChF,MAAM,SAAS;CACf,OACE,OAAO,KAAK,MAAM,CAAC,CAAC,WAAW,KAC/B,OAAO,cAAc,OAAO,GAAG,KAC/B,OAAO,OAAO,GAAG,IAAI,KACrB,OAAO,OAAO,SAAS,YACvB,OAAO,KAAK,KAAK,CAAC,CAAC,SAAS,KAC5B,OAAO,OAAO,UAAU,YACxB,OAAO,MAAM,KAAK,CAAC,CAAC,SAAS;AAEjC;AAEA,SAAS,aAAa,UAAkB,OAAsC;CAC5E,MAAM,QAAQ,IAAI,oBAChB,0CAA0C,SAAS,4BACrD;CACA,IAAI,UAAU,KAAA,GAAW,OAAO,eAAe,OAAO,SAAS,EAAE,OAAO,MAAM,CAAC;CAC/E,OAAO;AACT;AAEA,SAAS,iBAAiB,UAAkB,OAAkC;CAC5E,MAAM,UAAU,UAAU,QAAQ;CAClC,IAAI,CAAC,SAAS;CACd,IAAI,eAAe,OAAO,MAAM,eAAe,KAAK,GAClD,MAAM,IAAI,oBAAoB,kDAAkD,SAAS,EAAE;CAE7F,IAAI;EACF,WAAW,QAAQ;CACrB,SAAS,OAAO;EACd,IAAI,CAAC,UAAU,KAAK,GAAG,MAAM;CAC/B;AACF;AAEA,SAAS,eAAe,OAAoC;CAC1D,OAAO,KAAK,UAAU;EAAE,MAAM,MAAM;EAAM,OAAO,MAAM;EAAO,KAAK,MAAM;CAAI,CAAC;AAChF;AAEA,SAAS,aAAa,UAA0B;CAC9C,OAAO,GAAG,SAAS;AACrB;AAEA,SAAS,gBAAgB,OAAyB;CAChD,OAAO,iBAAiB,SAAS,UAAU,SAAS,MAAM,SAAS;AACrE;AAEA,SAAS,UAAU,OAAyB;CAC1C,OAAO,iBAAiB,SAAS,UAAU,SAAS,MAAM,SAAS;AACrE;AAEA,SAAS,gBAAgB,OAAyB;CAChD,OAAO,iBAAiB,SAAS,UAAU,SAAS,MAAM,SAAS;AACrE;AAEA,SAAS,eAAe,MAAoB;CAC1C,IAAI;EACF,WAAW,IAAI;CACjB,SAAS,OAAO;EACd,IAAI,CAAC,UAAU,KAAK,GAAG,MAAM;CAC/B;AACF;;;;;;;;;;;;;;;;;AClOA,MAAa,sBAAsB;;AAGnC,IAAa,8BAAb,cAAiD,gBAAgB,CAAC;;;;AAKlE,SAAgB,gBAAgB,OAAwB;CACtD,IAAI;EACF,OAAO,uBAAuB,KAAK;CACrC,SAAS,OAAO;EACd,MAAM,sBAAsB,OAAO,KAAK;CAC1C;AACF;AAEA,SAAgB,cAAc,OAA4B;CACxD,IAAI;EACF,OAAO,YAAY,wBAAwB,KAAK,CAAC;CACnD,SAAS,OAAO;EACd,MAAM,sBAAsB,OAAO,KAAK;CAC1C;AACF;;;;AAKA,SAAS,sBAAsB,OAAgB,OAA6C;CAC1F,MAAM,SAAS,wBAAwB,OAAO,qBAAK,IAAI,IAAI,CAAC;CAE5D,OAAO,IAAI,4BAA4B,qCADxB,WAAW,KAAK,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KACP,EACpF,MACF,CAAC;AACH;AAEA,SAAS,wBAAwB,OAAgB,MAAc,WAAgC;CAC7F,IAAI,UAAU,MAAM,OAAO;CAC3B,IAAI,UAAU,KAAA,GAAW,OAAO,GAAG,KAAK;CACxC,MAAM,OAAO,OAAO;CACpB,IAAI,SAAS,UACX,OAAO,OAAO,SAAS,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,OAAO,KAAK;CAEjE,IAAI,SAAS,UACX,OAAO,kBAAkB,KAAe,IAAI,KAAK,GAAG,KAAK;CAE3D,IAAI,SAAS,WAAW,OAAO;CAC/B,IAAI,SAAS,UAAU,OAAO,GAAG,KAAK,QAAQ;CAC9C,MAAM,SAAS;CACf,IAAI,UAAU,IAAI,MAAM,GAAG,OAAO,GAAG,KAAK;CAC1C,MAAM,YAAY,OAAO,eAAe,MAAM;CAC9C,IAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,cAAc,OAAO,aAAa,cAAc,MAC5E,OAAO,GAAG,KAAK,QAAQ,OAAO,aAAa,QAAQ,YAAY;CAEjE,MAAM,SAAS,IAAI,IAAI,SAAS,CAAC,CAAC,IAAI,MAAM;CAC5C,IAAI,MAAM,QAAQ,MAAM,GAAG;EACzB,KAAK,IAAI,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,GAAG;GACrD,MAAM,SAAS,wBAAwB,OAAO,QAAQ,GAAG,KAAK,GAAG,MAAM,IAAI,MAAM;GACjF,IAAI,WAAW,IAAI,OAAO;EAC5B;EACA,OAAO;CACT;CACA,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,MAAM,GAAG;EACjD,IAAI,CAAC,kBAAkB,GAAG,GAAG,OAAO,GAAG,KAAK,GAAG,IAAI;EACnD,MAAM,SAAS,wBAAwB,OAAO,GAAG,KAAK,GAAG,OAAO,MAAM;EACtE,IAAI,WAAW,IAAI,OAAO;CAC5B;CACA,OAAO;AACT;AAEA,SAAS,kBAAkB,OAAwB;CACjD,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,GAAG;EACpD,MAAM,OAAO,MAAM,WAAW,KAAK;EACnC,IAAI,QAAQ,SAAU,QAAQ,OAAQ;GACpC,MAAM,OAAO,MAAM,WAAW,QAAQ,CAAC;GACvC,IAAI,EAAE,QAAQ,SAAU,QAAQ,QAAS,OAAO;GAChD,SAAS;EACX,OAAO,IAAI,QAAQ,SAAU,QAAQ,OACnC,OAAO;CAEX;CACA,OAAO;AACT;;;;;;AClFA,SAAgB,iBAAiB,MAAc,MAAc,SAAkC;CAC7F,UAAU,QAAQ,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;CAC5C,MAAM,KAAK,SAAS,MAAM,UAAU,UAAU,UAAU,WAAW,UAAU,UAAU,GAAK;CAC5F,IAAI;EACF,SAAS,IAAI,OAAO,KAAK,MAAM,MAAM,GAAG,OAAO;EAC/C,UAAU,EAAE;CACd,UAAU;EACR,UAAU,EAAE;CACd;CACA,eAAe,QAAQ,IAAI,CAAC;AAC9B;;;;AAKA,SAAgB,0BACd,MACA,UACA,SACM;CACN,MAAM,YAAY,QAAQ,IAAI;CAC9B,UAAU,WAAW,EAAE,WAAW,KAAK,CAAC;CACxC,MAAM,gBAAgB,GAAG,KAAK;CAC9B,MAAM,KAAK,SACT,eACA,UAAU,UAAU,UAAU,WAAW,UAAU,SACnD,GACF;CACA,IAAI;EACF,SAAS,IAAI,OAAO,KAAK,UAAU,MAAM,GAAG,OAAO;EACnD,UAAU,EAAE;CACd,UAAU;EACR,UAAU,EAAE;CACd;CACA,WAAW,eAAe,IAAI;CAC9B,eAAe,SAAS;AAC1B;AAEA,SAAgB,mBAAsB,MAAc,SAA4B,KAAiB;CAC/F,MAAM,SAAS,sBAAsB,MAAM,SAAS,GAAG;CACvD,IAAI,CAAC,OAAO,UACV,MAAM,QAAQ,eAAe,GAAG,QAAQ,QAAQ,iBAAiB,KAAK,EAAE;CAE1E,OAAO,OAAO;AAChB;AAIA,SAAgB,sBACd,MACA,SACA,KACmB;CACnB,UAAU,QAAQ,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;CAC5C,MAAM,WAAW,GAAG,KAAK;CACzB,IAAI;EACF,MAAM,cAAc,yBAAyB,EAAE,SAAS,CAAC;EACzD,IAAI,CAAC,YAAY,UAAU,OAAO,EAAE,UAAU,MAAM;EACpD,IAAI;GACF,OAAO;IAAE,UAAU;IAAM,OAAO,IAAI;GAAE;EACxC,UAAU;GACR,YAAY,KAAK,QAAQ;EAC3B;CACF,SAAS,OAAO;EACd,IAAI,iBAAiB,qBACnB,MAAM,QAAQ,eAAe,MAAM,SAAS,EAAE,OAAO,MAAM,CAAC;EAE9D,MAAM;CACR;AACF;AAEA,SAAS,SAAS,IAAY,OAAe,SAAkC;CAC7E,IAAI,SAAS;CACb,OAAO,SAAS,MAAM,YAAY;EAChC,MAAM,UAAU,UAAU,IAAI,OAAO,QAAQ,MAAM,aAAa,MAAM;EACtE,IAAI,WAAW,GAAG,MAAM,QAAQ,eAAe,6BAA6B;EAC5E,UAAU;CACZ;AACF;AAEA,SAAS,eAAe,MAAoB;CAC1C,MAAM,KAAK,SAAS,MAAM,UAAU,QAAQ;CAC5C,IAAI;EACF,UAAU,EAAE;CACd,UAAU;EACR,UAAU,EAAE;CACd;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACnDA,SAAgB,mBAAmB,aAA6B;CAC9D,OAAO,GAAG,YAAY;AACxB;;;;;AAMA,SAAgB,oBACd,MACA,SAC0B;CAC1B,IAAI,CAAC,WAAW,IAAI,GAAG,OAAO;CAC9B,MAAM,OAAO,aAAa,MAAM,MAAM;CACtC,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,IAAI;CACvB,SAAS,OAAO;EACd,MAAM,QAAQ,eAAe,GAAG,QAAQ,QAAQ,gBAAgB,KAAK,qBAAqB,EACxF,OAAO,MACT,CAAC;CACH;CACA,IAAI,OAAO,QAAQ,YAAY,QAAQ,QAAQ,MAAM,QAAQ,GAAG,GAC9D,MAAM,QAAQ,eAAe,GAAG,QAAQ,QAAQ,gBAAgB,KAAK,kBAAkB;CAEzF,MAAM,SAAS;CACf,MAAM,OAAO,OAAO,KAAK,MAAM,CAAC,CAAC,KAAK;CACtC,IAAI,KAAK,WAAW,KAAK,KAAK,OAAO,eAAe,KAAK,OAAO,YAC9D,MAAM,QAAQ,eACZ,GAAG,QAAQ,QAAQ,gBAAgB,KAAK,aAAa,KAAK,KAAK,IAAI,EAAE,kCACvE;CAEF,MAAM,EAAE,UAAU,cAAc;CAChC,IAAI,OAAO,aAAa,YAAY,CAAC,OAAO,cAAc,QAAQ,KAAK,WAAW,GAChF,MAAM,QAAQ,eACZ,GAAG,QAAQ,QAAQ,gBAAgB,KAAK,wCAC1C;CAEF,IAAI,OAAO,cAAc,YAAY,CAAC,oBAAoB,KAAK,SAAS,GACtE,MAAM,QAAQ,eACZ,GAAG,QAAQ,QAAQ,gBAAgB,KAAK,kCAC1C;CAEF,OAAO;EAAE;EAAqB;CAAwB;AACxD;;;;AAKA,SAAgB,qBACd,MACA,MACA,SACM;CACN,0BAA0B,MAAM,GAAG,gBAAgB,IAAI,EAAE,KAAK,OAAO;AACvE;;;;AAKA,SAAgB,gCACd,SACA,MACA,SACA,SACM;CACN,MAAM,SAAS,QAAQ,KAAK;CAC5B,IAAI,WAAW,KAAA,GACb,MAAM,QAAQ,eACZ,GAAG,QAAQ,8BAA8B,KAAK,SAAS,IAAI,KAAK,UAAU,wBAAwB,QAAQ,OAAO,wEACnH;CAEF,IAAI,OAAO,aAAa,KAAK,UAC3B,MAAM,QAAQ,eACZ,GAAG,QAAQ,kBAAkB,KAAK,SAAS,oBAAoB,OAAO,SAAS,qDACjF;CAEF,IAAI,OAAO,cAAc,KAAK,WAC5B,MAAM,QAAQ,eACZ,GAAG,QAAQ,4BAA4B,KAAK,SAAS,WAAW,OAAO,UAAU,iCAAiC,KAAK,UAAU,6BACnI;AAEJ;;;;;;;;;;;;;;;;;;;;;;;;;ACxBA,MAAM,iCAAiB,IAAI,IAAmB;AAE9C,SAAS,SAAS,MAAqB;CACrC,MAAM,WAAW,eAAe,IAAI,IAAI;CACxC,IAAI,UAAU,OAAO;CACrB,MAAM,QAAQ,IAAI,MAAM;CACxB,eAAe,IAAI,MAAM,KAAK;CAC9B,OAAO;AACT;;;AAIA,IAAa,oBAAb,MAAiG;CAC/F;;CAEA;CACA;CACA;CACA;CAEA,YACE,MACA,OACA,UAAoC,CAAC,GACrC;EACA,KAAK,OAAO,QAAQ,IAAI;EACxB,KAAK,kBAAkB,mBAAmB,KAAK,IAAI;EACnD,KAAK,QAAQ;EACb,KAAK,QAAQ,SAAS,KAAK,IAAI;EAC/B,KAAK,qBAAqB,QAAQ,uBAAuB;CAC3D;CAEA,MAAM,SAAgF;EACpF,OAAO,KAAK,MAAM,mBAChB,mBAAmB,KAAK,MAAM,KAAK,aAAa,KAAK,aAAa,CAAC,CACrE;CACF;CAEA,MAAM,OACJ,OACA,UAA+B,CAAC,GACuC;EACvE,OAAO,KAAK,MAAM,mBAChB,mBAAmB,KAAK,MAAM,KAAK,aAAa;GAC9C,MAAM,SAAS,KAAK,aAAa;GACjC,MAAM,WAAW,OAAO,QAAQ,MAAM,UAAU,MAAM,MAAM,YAAY,MAAM,OAAO;GACrF,IAAI,UAAU;IACZ,IAAI,gBAAgB,SAAS,KAAK,MAAM,gBAAgB,KAAK,GAC3D,MAAM,KAAK,MAAM,cACf,WAAW,MAAM,QAAQ,uCAC3B;IAEF,OAAO;KAAE,OAAO;KAAU,UAAU;KAAO,YAAY,OAAO;IAAW;GAC3E;GAEA,MAAM,WAAW;IACf,GAAG,KAAK,MAAM;IACd,UAAU,OAAO,QAAQ;IACzB,cAAc,OAAO,QAAQ,GAAG,EAAE,CAAC,EAAE,aAAa;IAClD;GACF;GACA,MAAM,QAAQ;IAAE,GAAG;IAAU,WAAW,cAAc,QAAQ;GAAE;GAIhE,MAAM,aAAa,KAAK,QAAQ,CAAC,GAAG,OAAO,SAAS,KAAK,CAAC;GAC1D,iBAAiB,KAAK,MAAM,GAAG,gBAAgB,KAAK,EAAE,KAAK,KAAK,KAAK;GACrE,IAAI,QAAQ,YAAY,MAKtB,qBACE,KAAK,iBACL;IAAE,UAAU,MAAM;IAAU,WAAW,MAAM;GAAU,GACvD,KAAK,KACP;GAEF,OAAO;IAAE;IAAO,UAAU;IAAM;GAAW;EAC7C,CAAC,CACH;CACF;;CAGA,MAAM,cAAiD;EACrD,OAAO,KAAK,MAAM,mBAChB,mBAAmB,KAAK,MAAM,KAAK,aACjC,oBAAoB,KAAK,iBAAiB,KAAK,KAAK,CACtD,CACF;CACF;;;;CAKA,MAAM,iBAA6C;EACjD,OAAO,KAAK,MAAM,mBAChB,mBAAmB,KAAK,MAAM,KAAK,aAAa;GAC9C,MAAM,UAAU,KAAK,YAAY;GACjC,KAAK,QAAQ,OAAO;GAGpB,KAAK,kBAAkB,SAAS,KAAK;GACrC,MAAM,OAAO,QAAQ,GAAG,EAAE;GAC1B,IAAI,SAAS,KAAA,GACX,MAAM,KAAK,MAAM,eACf,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,KAAK,mCACrC;GAEF,MAAM,OAA0B;IAAE,UAAU,KAAK;IAAU,WAAW,KAAK;GAAU;GACrF,qBAAqB,KAAK,iBAAiB,MAAM,KAAK,KAAK;GAC3D,OAAO;EACT,CAAC,CACH;CACF;CAEA,eAAqF;EACnF,MAAM,UAAU,KAAK,YAAY;EACjC,MAAM,aAAa,KAAK,QAAQ,OAAO;EAIvC,KAAK,kBAAkB,SAAS,KAAK,kBAAkB;EACvD,OAAO;GAAE;GAAS;EAAW;CAC/B;CAEA,kBAA0B,SAAyC,YAA2B;EAC5F,MAAM,SAAS,oBAAoB,KAAK,iBAAiB,KAAK,KAAK;EACnE,IAAI,WAAW,MAAM;GACnB,IAAI,cAAc,QAAQ,SAAS,GACjC,MAAM,KAAK,MAAM,eACf,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,KAAK,OAAO,QAAQ,OAAO,kCAAkC,KAAK,gBAAgB,gEAClH;GAEF;EACF;EACA,gCACE,SACA,QACA,KAAK,OACL,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,MAChC;CACF;CAEA,cAAsD;EACpD,IAAI,CAAC,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC;EACpC,MAAM,OAAO,aAAa,KAAK,MAAM,MAAM;EAC3C,IAAI,KAAK,WAAW,GAAG,OAAO,CAAC;EAC/B,IAAI,CAAC,KAAK,SAAS,IAAI,GACrB,MAAM,KAAK,MAAM,eACf,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,KAAK,gDACrC;EAGF,MAAM,QAAQ,KAAK,MAAM,GAAG,EAAE,CAAC,CAAC,MAAM,IAAI;EAC1C,MAAM,UAA0C,CAAC;EACjD,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,GAAG;GACpD,MAAM,OAAO,MAAM;GACnB,IAAI,KAAK,WAAW,GAClB,MAAM,KAAK,MAAM,eACf,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,KAAK,2BAA2B,QAAQ,GACxE;GAEF,IAAI;GACJ,IAAI;IACF,MAAM,KAAK,MAAM,IAAI;GACvB,SAAS,OAAO;IACd,MAAM,KAAK,MAAM,eACf,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,KAAK,4BAA4B,QAAQ,KACvE,EAAE,OAAO,MAAM,CACjB;GACF;GACA,MAAM,QAAQ,KAAK,MAAM,WAAW,KAAK;IAAE,MAAM,KAAK;IAAM,MAAM,QAAQ;GAAE,CAAC;GAC7E,IAAI;GACJ,IAAI;IACF,YAAY,gBAAgB,KAAK;GACnC,SAAS,OAAO;IACd,MAAM,KAAK,MAAM,eACf,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,KAAK,iDAAiD,QAAQ,KAC5F,EAAE,OAAO,MAAM,CACjB;GACF;GACA,IAAI,SAAS,WACX,MAAM,KAAK,MAAM,eACf,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,KAAK,mCAAmC,QAAQ,GAChF;GAEF,QAAQ,KAAK,KAAK;EACpB;EACA,OAAO;CACT;CAEA,QAAgB,SAAqD;EACnE,MAAM,YAAY,KAAK,MAAM,gBAAgB;EAC7C,MAAM,2BAAW,IAAI,IAAY;EACjC,IAAI,mBAAsC;EAC1C,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;GACtD,MAAM,QAAQ,QAAQ;GACtB,KAAK,MAAM,iBAAiB,OAAO,KAAK;GACxC,IAAI,MAAM,aAAa,OACrB,MAAM,KAAK,MAAM,eACf,SAAS,MAAM,MAAM,QAAQ,gBAAgB,MAAM,SAAS,aAAa,OAC3E;GAEF,IAAI,MAAM,iBAAiB,kBACzB,MAAM,KAAK,MAAM,eACf,SAAS,MAAM,MAAM,QAAQ,mCAC/B;GAEF,MAAM,EAAE,WAAW,YAAY,GAAG,aAAa;GAC/C,MAAM,eAAe,cAAc,QAAQ;GAC3C,IAAI,MAAM,cAAc,cACtB,MAAM,KAAK,MAAM,eACf,SAAS,MAAM,MAAM,QAAQ,2BAA2B,aAAa,QAAQ,MAAM,WACrF;GAEF,mBAAmB,MAAM;GACzB,IAAI,SAAS,IAAI,MAAM,MAAM,OAAO,GAClC,MAAM,KAAK,MAAM,eACf,qBAAqB,MAAM,MAAM,QAAQ,mBAC3C;GAEF,SAAS,IAAI,MAAM,MAAM,OAAO;GAChC,UAAU,MAAM,OAAO,KAAK;EAC9B;EACA,OAAO,UAAU,OAAO,OAAO;CACjC;AACF"}
|
package/dist/meta-eval/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { s as ValidationError } from "../errors-8YnH8WlF.js";
|
|
2
|
-
import { A as spearmanR, E as pearsonR } from "../statistics-
|
|
2
|
+
import { A as spearmanR, E as pearsonR } from "../statistics-DWM_AyLe.js";
|
|
3
3
|
import { o as llmSpans, t as aggregateLlm } from "../query-Di7eEQ79.js";
|
|
4
4
|
import { t as analyzeSeries } from "../series-convergence-CjO2QdRW.js";
|
|
5
|
-
import { t as rubricPredictiveValidity } from "../rubric-predictive-validity-
|
|
5
|
+
import { t as rubricPredictiveValidity } from "../rubric-predictive-validity-QG7ydk0s.js";
|
|
6
6
|
import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-ChBKlTd_.js";
|
|
7
7
|
//#region src/meta-eval/calibration.ts
|
|
8
8
|
async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMetric, options = {}) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { s as ValidationError } from "./errors-8YnH8WlF.js";
|
|
2
2
|
import { a as scoreOrigin, i as rolloutRewardFields } from "./reward-nw2xZGZG.js";
|
|
3
|
-
import { o as runTaskScore } from "./run-record-
|
|
3
|
+
import { o as runTaskScore } from "./run-record-BIwU2wdV.js";
|
|
4
4
|
import { i as ROLLOUT_SCHEMA, s as assertMinted } from "./schema-C6DW4ZHR.js";
|
|
5
5
|
import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
|
|
6
6
|
//#region src/rollout/mint.ts
|
|
@@ -198,4 +198,4 @@ async function mintRolloutRows(records, store, options = {}) {
|
|
|
198
198
|
//#endregion
|
|
199
199
|
export { mintRolloutRows as t };
|
|
200
200
|
|
|
201
|
-
//# sourceMappingURL=mint-
|
|
201
|
+
//# sourceMappingURL=mint-BvkwcYZU.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mint-D5_87M5L.js","names":[],"sources":["../src/rollout/mint.ts"],"sourcesContent":["/**\n * Rollout minting — `tangle.rollout.v1` lines joined from the records the\n * substrate ALREADY keeps. There is no separate rollout store: a rollout\n * is the JOIN of a RunRecord (identity, provenance, cost, outcome) with\n * its trace (spans share `runId`), projected into the canonical line.\n *\n * Composition, not duplication:\n * - identity/provenance → `RunRecord` (candidateId, splitTag, agentProfile, hashes)\n * - step structure → `buildTrajectory` over the shared TraceStore\n * - preference-pair export → `feedbackTrajectoryToOptimizerRow` (feedback-trajectory.ts)\n * - PRM / reward-model → `reward-model-export.ts`\n *\n * Anti-Goodhart invariant: a run whose `outcome.realness.gated` is true is\n * never exported with a positive reward OR with any of the numbers that reward\n * was computed from. The gate travels into the training data (`reward` forced\n * to 0, `realness_gated: true`) and the whole outcome is transformed by\n * `gateGamedOutcome` inside `assertMinted` below, which relocates `metrics` and\n * `verdict` to `provenance.gated_evidence`. Mint returns\n * `MintedRolloutLine[]`: the brand the training exporters require, which only\n * this function, `readRolloutLedger`, and an explicit `assertMinted` can mint.\n *\n * A record carrying NEITHER split score is REJECTED (`ValidationError`), never\n * minted at 0 — \"nobody graded this\" is not the same claim as \"graded a total\n * failure\", and a trainer reading 0 learns the second. Lines that already\n * carry `reward: null` (interchange imports, existing ledgers) remain valid on\n * the wire; only the RunRecord→line door refuses.\n *\n * Records without spans become labeled GAP LINES (messages: [],\n * provenance.gap) — present in the output AND surfaced in\n * `missingTraces`; a capture gap is a finding, never a silent omission.\n */\n\nimport { ValidationError } from '../errors'\nimport { type RunRecord, runTaskScore } from '../run-record'\nimport type { LlmSpan, Message, Span, ToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory } from '../trajectory'\nimport { rolloutRewardFields, scoreOrigin } from './reward'\nimport {\n assertMinted,\n type ChatMessage,\n type MintedRolloutLine,\n ROLLOUT_SCHEMA,\n type RolloutRole,\n type RolloutSplit,\n type RolloutStep,\n} from './schema'\n\n/** Redactor applied to every exported string (secrets, PII). Identity by default. */\nexport type RolloutScrubber = (text: string) => string\n\nexport interface MintRolloutOptions {\n scrub?: RolloutScrubber\n /** Cap steps per line (longest runs first drop middle steps). Default: no cap. */\n maxSteps?: number\n /** Role recorded on every minted line. Default 'agent' (a solo eval run). */\n role?: RolloutRole\n /** Task suite label. Default: the record's `experimentId`. */\n suite?: string\n /** Injected clock for deterministic output. */\n now?: () => Date\n}\n\nexport interface MintRolloutResult {\n rows: MintedRolloutLine[]\n /** runIds that had a RunRecord but no spans — emitted as gap lines AND listed here. */\n missingTraces: string[]\n}\n\nconst asText = (v: unknown, scrub: RolloutScrubber): string => {\n const s = typeof v === 'string' ? v : JSON.stringify(v)\n return scrub(s ?? '')\n}\n\nfunction projectStep(span: Span, scrub: RolloutScrubber): RolloutStep {\n const base: RolloutStep = {\n kind: span.kind,\n name: scrub(span.name),\n status: span.status,\n durationMs: span.endedAt !== undefined ? span.endedAt - span.startedAt : undefined,\n }\n if (span.kind === 'llm') {\n const llm = span as LlmSpan\n const last = llm.messages[llm.messages.length - 1]\n if (last) base.input = scrub(last.content)\n if (llm.output !== undefined) base.output = scrub(llm.output)\n } else if (span.kind === 'tool') {\n const tool = span as ToolSpan\n base.input = asText(tool.args, scrub)\n if (tool.result !== undefined) base.output = asText(tool.result, scrub)\n }\n return base\n}\n\n/** The final llm span's history + output is the completed conversation. */\nfunction finalConversation(spans: Span[], scrub: RolloutScrubber): ChatMessage[] {\n const llms = spans.filter((s): s is LlmSpan => s.kind === 'llm')\n const last = llms[llms.length - 1]\n if (!last) return []\n const messages: ChatMessage[] = last.messages.map((m: Message) => ({\n role: m.role,\n content: scrub(m.content),\n }))\n if (last.output !== undefined && last.output !== '') {\n messages.push({ role: 'assistant', content: scrub(last.output) })\n }\n return messages\n}\n\n// The reward derivations live in the leaf module `./reward` so gate and\n// reporting code can import them without dragging in the trace store; they are\n// re-exported here because the derivations shipped from this path.\nexport {\n isRealnessGated,\n observedScore,\n observedSplitScore,\n type ScoreOrigin,\n type ScorePreference,\n scoreOrigin,\n trainingReward,\n trainingScore,\n} from './reward'\n\nconst REWARD_SOURCE: Record<ReturnType<typeof scoreOrigin>, string> = {\n holdout: 'run-record/holdout-score',\n search: 'run-record/search-score',\n unscored: 'run-record/unscored',\n}\n\n/**\n * The mint door refuses an execution-only record: a missing training label is\n * not a zero reward, and not a mintable line either. Lines that already carry\n * `reward: null` — interchange imports, existing ledgers — stay valid on the\n * wire and keep their labeled gap; this guard is only about the\n * RunRecord→line door, where the producer can still be told to go score the\n * run instead of shipping an unlabeled row.\n */\nfunction requireTaskScore(record: RunRecord): void {\n if (runTaskScore(record) === undefined) {\n throw new ValidationError(`Cannot mint rollout for run ${record.runId}: task score is missing`)\n }\n}\n\nconst SPLIT_FROM_TAG: Record<RunRecord['splitTag'], RolloutSplit> = {\n search: 'search',\n dev: 'dev',\n holdout: 'holdout',\n}\n\nfunction mintLine(\n record: RunRecord,\n steps: RolloutStep[],\n messages: ChatMessage[],\n options: MintRolloutOptions,\n capturedAt: string,\n gap?: string,\n): MintedRolloutLine {\n // A missing task score is refused before anything is built: an\n // execution-only record has no training label, and a missing label is\n // neither a zero reward nor a mintable row.\n requireTaskScore(record)\n // `reward` and `realness_gated` come out of one call, so neither door into\n // the waist can write one and forget the other.\n const rewardFields = rolloutRewardFields(record)\n const uncaptured = record.costProvenance.kind === 'uncaptured'\n const terminalOutcome = record.terminalOutcome\n const isCompleted = terminalOutcome === 'succeeded' || terminalOutcome === 'failed'\n const isTruncated = terminalOutcome === 'cancelled' || terminalOutcome === 'incomplete'\n const terminalError =\n terminalOutcome === 'failed' ||\n terminalOutcome === 'cancelled' ||\n terminalOutcome === 'incomplete'\n ? (record.terminalFailureReason ?? `run ended ${terminalOutcome}`)\n : null\n // `assertMinted` rather than a cast: mint is the producer the whole gate\n // rests on, so it proves the line it just built is valid instead of asserting\n // it by fiat. The brand is unforgeable precisely because nobody casts to it.\n return assertMinted(\n {\n schema: ROLLOUT_SCHEMA,\n rollout_id: record.runId,\n parent_rollout_id: null,\n run_id: record.runId,\n experiment_id: record.experimentId,\n candidate_id: record.candidateId,\n generation: null,\n candidate_index: null,\n role: options.role ?? 'agent',\n task: {\n suite: options.suite ?? record.experimentId,\n instance_id: record.scenarioId,\n split: SPLIT_FROM_TAG[record.splitTag],\n seed: record.seed,\n rep: 0,\n },\n policy: {\n harness: null,\n harness_version: null,\n model: record.model,\n provider: null,\n profile_commit: record.commitSha,\n prompt_hash: record.promptHash,\n config_hash: record.configHash,\n agent_profile_cell_id: record.agentProfile?.cellId ?? null,\n sampling: null,\n },\n messages,\n tool_defs: [],\n ...(steps.length > 0 ? { steps } : {}),\n outcome: {\n ...rewardFields,\n reward_source: REWARD_SOURCE[scoreOrigin(record)],\n verdict: null,\n // A verbatim bulk copy, deliberately UNFILTERED here. `outcome.raw`\n // holds the per-layer verifier scores (`layer.*`) that the reward was\n // derived from, so on a gated run this dict is the reward signal in\n // component form — but filtering it at this call site is the pattern\n // that has now leaked twice, because the next producer to write a\n // reward-bearing field forgets. The gate is applied to the whole\n // outcome once, in `assertMinted` below (`gateGamedOutcome`), which\n // moves the block to `provenance.gated_evidence` when the run is gated\n // and leaves it here untouched when it is not.\n metrics: { ...record.outcome.raw },\n is_completed: isCompleted,\n is_truncated: isTruncated,\n error: terminalError,\n },\n cost: {\n usd: uncaptured ? null : record.costUsd,\n tokens_in: record.tokenUsage.input,\n tokens_out: record.tokenUsage.output,\n tokens_reasoning: record.tokenUsage.reasoning ?? null,\n cache_read: record.tokenUsage.cached ?? null,\n cache_write: record.tokenUsage.cacheWrite ?? null,\n wall_s: Math.round(record.wallMs / 1000),\n },\n artifacts: { patch_path: null, run_dir: null, transcript_ref: null },\n provenance: {\n captured_at: capturedAt,\n capture: 'mint',\n ...(gap !== undefined ? { gap } : {}),\n },\n },\n `minted rollout line for run ${record.runId}`,\n )\n}\n\n/**\n * Join RunRecords with their traces into canonical rollout lines. Records\n * without spans are emitted as labeled gap lines and reported in\n * `missingTraces`. Execution-only records without a task score are rejected\n * because a missing training label is not a zero reward.\n */\nexport async function mintRolloutRows(\n records: RunRecord[],\n store: TraceStore,\n options: MintRolloutOptions = {},\n): Promise<MintRolloutResult> {\n const scrub = options.scrub ?? ((t) => t)\n const capturedAt = (options.now?.() ?? new Date()).toISOString()\n const rows: MintedRolloutLine[] = []\n const missingTraces: string[] = []\n for (const record of records) {\n const trajectory = await buildTrajectory(store, record.runId)\n if (trajectory.steps.length === 0) {\n missingTraces.push(record.runId)\n rows.push(\n mintLine(record, [], [], options, capturedAt, 'no trace spans recorded for this runId'),\n )\n continue\n }\n let steps = trajectory.steps.map((s) => projectStep(s.span, scrub))\n if (options.maxSteps !== undefined && steps.length > options.maxSteps) {\n // Keep the head and tail — the middle of a long run is the least\n // informative for outcome attribution.\n const head = Math.ceil(options.maxSteps / 2)\n const tail = options.maxSteps - head\n steps = [...steps.slice(0, head), ...steps.slice(steps.length - tail)]\n }\n const conversation = finalConversation(\n trajectory.steps.map((s) => s.span),\n scrub,\n )\n const gap =\n conversation.length === 0 ? 'trace has no llm spans — no conversation to inline' : undefined\n rows.push(mintLine(record, steps, conversation, options, capturedAt, gap))\n }\n return { rows, missingTraces }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEA,MAAM,UAAU,GAAY,UAAmC;CAE7D,OAAO,OADG,OAAO,MAAM,WAAW,IAAI,KAAK,UAAU,CAAC,MACpC,EAAE;AACtB;AAEA,SAAS,YAAY,MAAY,OAAqC;CACpE,MAAM,OAAoB;EACxB,MAAM,KAAK;EACX,MAAM,MAAM,KAAK,IAAI;EACrB,QAAQ,KAAK;EACb,YAAY,KAAK,YAAY,KAAA,IAAY,KAAK,UAAU,KAAK,YAAY,KAAA;CAC3E;CACA,IAAI,KAAK,SAAS,OAAO;EACvB,MAAM,MAAM;EACZ,MAAM,OAAO,IAAI,SAAS,IAAI,SAAS,SAAS;EAChD,IAAI,MAAM,KAAK,QAAQ,MAAM,KAAK,OAAO;EACzC,IAAI,IAAI,WAAW,KAAA,GAAW,KAAK,SAAS,MAAM,IAAI,MAAM;CAC9D,OAAO,IAAI,KAAK,SAAS,QAAQ;EAC/B,MAAM,OAAO;EACb,KAAK,QAAQ,OAAO,KAAK,MAAM,KAAK;EACpC,IAAI,KAAK,WAAW,KAAA,GAAW,KAAK,SAAS,OAAO,KAAK,QAAQ,KAAK;CACxE;CACA,OAAO;AACT;;AAGA,SAAS,kBAAkB,OAAe,OAAuC;CAC/E,MAAM,OAAO,MAAM,QAAQ,MAAoB,EAAE,SAAS,KAAK;CAC/D,MAAM,OAAO,KAAK,KAAK,SAAS;CAChC,IAAI,CAAC,MAAM,OAAO,CAAC;CACnB,MAAM,WAA0B,KAAK,SAAS,KAAK,OAAgB;EACjE,MAAM,EAAE;EACR,SAAS,MAAM,EAAE,OAAO;CAC1B,EAAE;CACF,IAAI,KAAK,WAAW,KAAA,KAAa,KAAK,WAAW,IAC/C,SAAS,KAAK;EAAE,MAAM;EAAa,SAAS,MAAM,KAAK,MAAM;CAAE,CAAC;CAElE,OAAO;AACT;AAgBA,MAAM,gBAAgE;CACpE,SAAS;CACT,QAAQ;CACR,UAAU;AACZ;;;;;;;;;AAUA,SAAS,iBAAiB,QAAyB;CACjD,IAAI,aAAa,MAAM,MAAM,KAAA,GAC3B,MAAM,IAAI,gBAAgB,+BAA+B,OAAO,MAAM,wBAAwB;AAElG;AAEA,MAAM,iBAA8D;CAClE,QAAQ;CACR,KAAK;CACL,SAAS;AACX;AAEA,SAAS,SACP,QACA,OACA,UACA,SACA,YACA,KACmB;CAInB,iBAAiB,MAAM;CAGvB,MAAM,eAAe,oBAAoB,MAAM;CAC/C,MAAM,aAAa,OAAO,eAAe,SAAS;CAClD,MAAM,kBAAkB,OAAO;CAC/B,MAAM,cAAc,oBAAoB,eAAe,oBAAoB;CAC3E,MAAM,cAAc,oBAAoB,eAAe,oBAAoB;CAC3E,MAAM,gBACJ,oBAAoB,YACpB,oBAAoB,eACpB,oBAAoB,eACf,OAAO,yBAAyB,aAAa,oBAC9C;CAIN,OAAO,aACL;EACE,QAAQ;EACR,YAAY,OAAO;EACnB,mBAAmB;EACnB,QAAQ,OAAO;EACf,eAAe,OAAO;EACtB,cAAc,OAAO;EACrB,YAAY;EACZ,iBAAiB;EACjB,MAAM,QAAQ,QAAQ;EACtB,MAAM;GACJ,OAAO,QAAQ,SAAS,OAAO;GAC/B,aAAa,OAAO;GACpB,OAAO,eAAe,OAAO;GAC7B,MAAM,OAAO;GACb,KAAK;EACP;EACA,QAAQ;GACN,SAAS;GACT,iBAAiB;GACjB,OAAO,OAAO;GACd,UAAU;GACV,gBAAgB,OAAO;GACvB,aAAa,OAAO;GACpB,aAAa,OAAO;GACpB,uBAAuB,OAAO,cAAc,UAAU;GACtD,UAAU;EACZ;EACA;EACA,WAAW,CAAC;EACZ,GAAI,MAAM,SAAS,IAAI,EAAE,MAAM,IAAI,CAAC;EACpC,SAAS;GACP,GAAG;GACH,eAAe,cAAc,YAAY,MAAM;GAC/C,SAAS;GAUT,SAAS,EAAE,GAAG,OAAO,QAAQ,IAAI;GACjC,cAAc;GACd,cAAc;GACd,OAAO;EACT;EACA,MAAM;GACJ,KAAK,aAAa,OAAO,OAAO;GAChC,WAAW,OAAO,WAAW;GAC7B,YAAY,OAAO,WAAW;GAC9B,kBAAkB,OAAO,WAAW,aAAa;GACjD,YAAY,OAAO,WAAW,UAAU;GACxC,aAAa,OAAO,WAAW,cAAc;GAC7C,QAAQ,KAAK,MAAM,OAAO,SAAS,GAAI;EACzC;EACA,WAAW;GAAE,YAAY;GAAM,SAAS;GAAM,gBAAgB;EAAK;EACnE,YAAY;GACV,aAAa;GACb,SAAS;GACT,GAAI,QAAQ,KAAA,IAAY,EAAE,IAAI,IAAI,CAAC;EACrC;CACF,GACA,+BAA+B,OAAO,OACxC;AACF;;;;;;;AAQA,eAAsB,gBACpB,SACA,OACA,UAA8B,CAAC,GACH;CAC5B,MAAM,QAAQ,QAAQ,WAAW,MAAM;CACvC,MAAM,cAAc,QAAQ,MAAM,qBAAK,IAAI,KAAK,EAAA,CAAG,YAAY;CAC/D,MAAM,OAA4B,CAAC;CACnC,MAAM,gBAA0B,CAAC;CACjC,KAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,aAAa,MAAM,gBAAgB,OAAO,OAAO,KAAK;EAC5D,IAAI,WAAW,MAAM,WAAW,GAAG;GACjC,cAAc,KAAK,OAAO,KAAK;GAC/B,KAAK,KACH,SAAS,QAAQ,CAAC,GAAG,CAAC,GAAG,SAAS,YAAY,wCAAwC,CACxF;GACA;EACF;EACA,IAAI,QAAQ,WAAW,MAAM,KAAK,MAAM,YAAY,EAAE,MAAM,KAAK,CAAC;EAClE,IAAI,QAAQ,aAAa,KAAA,KAAa,MAAM,SAAS,QAAQ,UAAU;GAGrE,MAAM,OAAO,KAAK,KAAK,QAAQ,WAAW,CAAC;GAC3C,MAAM,OAAO,QAAQ,WAAW;GAChC,QAAQ,CAAC,GAAG,MAAM,MAAM,GAAG,IAAI,GAAG,GAAG,MAAM,MAAM,MAAM,SAAS,IAAI,CAAC;EACvE;EACA,MAAM,eAAe,kBACnB,WAAW,MAAM,KAAK,MAAM,EAAE,IAAI,GAClC,KACF;EACA,MAAM,MACJ,aAAa,WAAW,IAAI,uDAAuD,KAAA;EACrF,KAAK,KAAK,SAAS,QAAQ,OAAO,cAAc,SAAS,YAAY,GAAG,CAAC;CAC3E;CACA,OAAO;EAAE;EAAM;CAAc;AAC/B"}
|
|
1
|
+
{"version":3,"file":"mint-BvkwcYZU.js","names":[],"sources":["../src/rollout/mint.ts"],"sourcesContent":["/**\n * Rollout minting — `tangle.rollout.v1` lines joined from the records the\n * substrate ALREADY keeps. There is no separate rollout store: a rollout\n * is the JOIN of a RunRecord (identity, provenance, cost, outcome) with\n * its trace (spans share `runId`), projected into the canonical line.\n *\n * Composition, not duplication:\n * - identity/provenance → `RunRecord` (candidateId, splitTag, agentProfile, hashes)\n * - step structure → `buildTrajectory` over the shared TraceStore\n * - preference-pair export → `feedbackTrajectoryToOptimizerRow` (feedback-trajectory.ts)\n * - PRM / reward-model → `reward-model-export.ts`\n *\n * Anti-Goodhart invariant: a run whose `outcome.realness.gated` is true is\n * never exported with a positive reward OR with any of the numbers that reward\n * was computed from. The gate travels into the training data (`reward` forced\n * to 0, `realness_gated: true`) and the whole outcome is transformed by\n * `gateGamedOutcome` inside `assertMinted` below, which relocates `metrics` and\n * `verdict` to `provenance.gated_evidence`. Mint returns\n * `MintedRolloutLine[]`: the brand the training exporters require, which only\n * this function, `readRolloutLedger`, and an explicit `assertMinted` can mint.\n *\n * A record carrying NEITHER split score is REJECTED (`ValidationError`), never\n * minted at 0 — \"nobody graded this\" is not the same claim as \"graded a total\n * failure\", and a trainer reading 0 learns the second. Lines that already\n * carry `reward: null` (interchange imports, existing ledgers) remain valid on\n * the wire; only the RunRecord→line door refuses.\n *\n * Records without spans become labeled GAP LINES (messages: [],\n * provenance.gap) — present in the output AND surfaced in\n * `missingTraces`; a capture gap is a finding, never a silent omission.\n */\n\nimport { ValidationError } from '../errors'\nimport { type RunRecord, runTaskScore } from '../run-record'\nimport type { LlmSpan, Message, Span, ToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory } from '../trajectory'\nimport { rolloutRewardFields, scoreOrigin } from './reward'\nimport {\n assertMinted,\n type ChatMessage,\n type MintedRolloutLine,\n ROLLOUT_SCHEMA,\n type RolloutRole,\n type RolloutSplit,\n type RolloutStep,\n} from './schema'\n\n/** Redactor applied to every exported string (secrets, PII). Identity by default. */\nexport type RolloutScrubber = (text: string) => string\n\nexport interface MintRolloutOptions {\n scrub?: RolloutScrubber\n /** Cap steps per line (longest runs first drop middle steps). Default: no cap. */\n maxSteps?: number\n /** Role recorded on every minted line. Default 'agent' (a solo eval run). */\n role?: RolloutRole\n /** Task suite label. Default: the record's `experimentId`. */\n suite?: string\n /** Injected clock for deterministic output. */\n now?: () => Date\n}\n\nexport interface MintRolloutResult {\n rows: MintedRolloutLine[]\n /** runIds that had a RunRecord but no spans — emitted as gap lines AND listed here. */\n missingTraces: string[]\n}\n\nconst asText = (v: unknown, scrub: RolloutScrubber): string => {\n const s = typeof v === 'string' ? v : JSON.stringify(v)\n return scrub(s ?? '')\n}\n\nfunction projectStep(span: Span, scrub: RolloutScrubber): RolloutStep {\n const base: RolloutStep = {\n kind: span.kind,\n name: scrub(span.name),\n status: span.status,\n durationMs: span.endedAt !== undefined ? span.endedAt - span.startedAt : undefined,\n }\n if (span.kind === 'llm') {\n const llm = span as LlmSpan\n const last = llm.messages[llm.messages.length - 1]\n if (last) base.input = scrub(last.content)\n if (llm.output !== undefined) base.output = scrub(llm.output)\n } else if (span.kind === 'tool') {\n const tool = span as ToolSpan\n base.input = asText(tool.args, scrub)\n if (tool.result !== undefined) base.output = asText(tool.result, scrub)\n }\n return base\n}\n\n/** The final llm span's history + output is the completed conversation. */\nfunction finalConversation(spans: Span[], scrub: RolloutScrubber): ChatMessage[] {\n const llms = spans.filter((s): s is LlmSpan => s.kind === 'llm')\n const last = llms[llms.length - 1]\n if (!last) return []\n const messages: ChatMessage[] = last.messages.map((m: Message) => ({\n role: m.role,\n content: scrub(m.content),\n }))\n if (last.output !== undefined && last.output !== '') {\n messages.push({ role: 'assistant', content: scrub(last.output) })\n }\n return messages\n}\n\n// The reward derivations live in the leaf module `./reward` so gate and\n// reporting code can import them without dragging in the trace store; they are\n// re-exported here because the derivations shipped from this path.\nexport {\n isRealnessGated,\n observedScore,\n observedSplitScore,\n type ScoreOrigin,\n type ScorePreference,\n scoreOrigin,\n trainingReward,\n trainingScore,\n} from './reward'\n\nconst REWARD_SOURCE: Record<ReturnType<typeof scoreOrigin>, string> = {\n holdout: 'run-record/holdout-score',\n search: 'run-record/search-score',\n unscored: 'run-record/unscored',\n}\n\n/**\n * The mint door refuses an execution-only record: a missing training label is\n * not a zero reward, and not a mintable line either. Lines that already carry\n * `reward: null` — interchange imports, existing ledgers — stay valid on the\n * wire and keep their labeled gap; this guard is only about the\n * RunRecord→line door, where the producer can still be told to go score the\n * run instead of shipping an unlabeled row.\n */\nfunction requireTaskScore(record: RunRecord): void {\n if (runTaskScore(record) === undefined) {\n throw new ValidationError(`Cannot mint rollout for run ${record.runId}: task score is missing`)\n }\n}\n\nconst SPLIT_FROM_TAG: Record<RunRecord['splitTag'], RolloutSplit> = {\n search: 'search',\n dev: 'dev',\n holdout: 'holdout',\n}\n\nfunction mintLine(\n record: RunRecord,\n steps: RolloutStep[],\n messages: ChatMessage[],\n options: MintRolloutOptions,\n capturedAt: string,\n gap?: string,\n): MintedRolloutLine {\n // A missing task score is refused before anything is built: an\n // execution-only record has no training label, and a missing label is\n // neither a zero reward nor a mintable row.\n requireTaskScore(record)\n // `reward` and `realness_gated` come out of one call, so neither door into\n // the waist can write one and forget the other.\n const rewardFields = rolloutRewardFields(record)\n const uncaptured = record.costProvenance.kind === 'uncaptured'\n const terminalOutcome = record.terminalOutcome\n const isCompleted = terminalOutcome === 'succeeded' || terminalOutcome === 'failed'\n const isTruncated = terminalOutcome === 'cancelled' || terminalOutcome === 'incomplete'\n const terminalError =\n terminalOutcome === 'failed' ||\n terminalOutcome === 'cancelled' ||\n terminalOutcome === 'incomplete'\n ? (record.terminalFailureReason ?? `run ended ${terminalOutcome}`)\n : null\n // `assertMinted` rather than a cast: mint is the producer the whole gate\n // rests on, so it proves the line it just built is valid instead of asserting\n // it by fiat. The brand is unforgeable precisely because nobody casts to it.\n return assertMinted(\n {\n schema: ROLLOUT_SCHEMA,\n rollout_id: record.runId,\n parent_rollout_id: null,\n run_id: record.runId,\n experiment_id: record.experimentId,\n candidate_id: record.candidateId,\n generation: null,\n candidate_index: null,\n role: options.role ?? 'agent',\n task: {\n suite: options.suite ?? record.experimentId,\n instance_id: record.scenarioId,\n split: SPLIT_FROM_TAG[record.splitTag],\n seed: record.seed,\n rep: 0,\n },\n policy: {\n harness: null,\n harness_version: null,\n model: record.model,\n provider: null,\n profile_commit: record.commitSha,\n prompt_hash: record.promptHash,\n config_hash: record.configHash,\n agent_profile_cell_id: record.agentProfile?.cellId ?? null,\n sampling: null,\n },\n messages,\n tool_defs: [],\n ...(steps.length > 0 ? { steps } : {}),\n outcome: {\n ...rewardFields,\n reward_source: REWARD_SOURCE[scoreOrigin(record)],\n verdict: null,\n // A verbatim bulk copy, deliberately UNFILTERED here. `outcome.raw`\n // holds the per-layer verifier scores (`layer.*`) that the reward was\n // derived from, so on a gated run this dict is the reward signal in\n // component form — but filtering it at this call site is the pattern\n // that has now leaked twice, because the next producer to write a\n // reward-bearing field forgets. The gate is applied to the whole\n // outcome once, in `assertMinted` below (`gateGamedOutcome`), which\n // moves the block to `provenance.gated_evidence` when the run is gated\n // and leaves it here untouched when it is not.\n metrics: { ...record.outcome.raw },\n is_completed: isCompleted,\n is_truncated: isTruncated,\n error: terminalError,\n },\n cost: {\n usd: uncaptured ? null : record.costUsd,\n tokens_in: record.tokenUsage.input,\n tokens_out: record.tokenUsage.output,\n tokens_reasoning: record.tokenUsage.reasoning ?? null,\n cache_read: record.tokenUsage.cached ?? null,\n cache_write: record.tokenUsage.cacheWrite ?? null,\n wall_s: Math.round(record.wallMs / 1000),\n },\n artifacts: { patch_path: null, run_dir: null, transcript_ref: null },\n provenance: {\n captured_at: capturedAt,\n capture: 'mint',\n ...(gap !== undefined ? { gap } : {}),\n },\n },\n `minted rollout line for run ${record.runId}`,\n )\n}\n\n/**\n * Join RunRecords with their traces into canonical rollout lines. Records\n * without spans are emitted as labeled gap lines and reported in\n * `missingTraces`. Execution-only records without a task score are rejected\n * because a missing training label is not a zero reward.\n */\nexport async function mintRolloutRows(\n records: RunRecord[],\n store: TraceStore,\n options: MintRolloutOptions = {},\n): Promise<MintRolloutResult> {\n const scrub = options.scrub ?? ((t) => t)\n const capturedAt = (options.now?.() ?? new Date()).toISOString()\n const rows: MintedRolloutLine[] = []\n const missingTraces: string[] = []\n for (const record of records) {\n const trajectory = await buildTrajectory(store, record.runId)\n if (trajectory.steps.length === 0) {\n missingTraces.push(record.runId)\n rows.push(\n mintLine(record, [], [], options, capturedAt, 'no trace spans recorded for this runId'),\n )\n continue\n }\n let steps = trajectory.steps.map((s) => projectStep(s.span, scrub))\n if (options.maxSteps !== undefined && steps.length > options.maxSteps) {\n // Keep the head and tail — the middle of a long run is the least\n // informative for outcome attribution.\n const head = Math.ceil(options.maxSteps / 2)\n const tail = options.maxSteps - head\n steps = [...steps.slice(0, head), ...steps.slice(steps.length - tail)]\n }\n const conversation = finalConversation(\n trajectory.steps.map((s) => s.span),\n scrub,\n )\n const gap =\n conversation.length === 0 ? 'trace has no llm spans — no conversation to inline' : undefined\n rows.push(mintLine(record, steps, conversation, options, capturedAt, gap))\n }\n return { rows, missingTraces }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEA,MAAM,UAAU,GAAY,UAAmC;CAE7D,OAAO,OADG,OAAO,MAAM,WAAW,IAAI,KAAK,UAAU,CAAC,MACpC,EAAE;AACtB;AAEA,SAAS,YAAY,MAAY,OAAqC;CACpE,MAAM,OAAoB;EACxB,MAAM,KAAK;EACX,MAAM,MAAM,KAAK,IAAI;EACrB,QAAQ,KAAK;EACb,YAAY,KAAK,YAAY,KAAA,IAAY,KAAK,UAAU,KAAK,YAAY,KAAA;CAC3E;CACA,IAAI,KAAK,SAAS,OAAO;EACvB,MAAM,MAAM;EACZ,MAAM,OAAO,IAAI,SAAS,IAAI,SAAS,SAAS;EAChD,IAAI,MAAM,KAAK,QAAQ,MAAM,KAAK,OAAO;EACzC,IAAI,IAAI,WAAW,KAAA,GAAW,KAAK,SAAS,MAAM,IAAI,MAAM;CAC9D,OAAO,IAAI,KAAK,SAAS,QAAQ;EAC/B,MAAM,OAAO;EACb,KAAK,QAAQ,OAAO,KAAK,MAAM,KAAK;EACpC,IAAI,KAAK,WAAW,KAAA,GAAW,KAAK,SAAS,OAAO,KAAK,QAAQ,KAAK;CACxE;CACA,OAAO;AACT;;AAGA,SAAS,kBAAkB,OAAe,OAAuC;CAC/E,MAAM,OAAO,MAAM,QAAQ,MAAoB,EAAE,SAAS,KAAK;CAC/D,MAAM,OAAO,KAAK,KAAK,SAAS;CAChC,IAAI,CAAC,MAAM,OAAO,CAAC;CACnB,MAAM,WAA0B,KAAK,SAAS,KAAK,OAAgB;EACjE,MAAM,EAAE;EACR,SAAS,MAAM,EAAE,OAAO;CAC1B,EAAE;CACF,IAAI,KAAK,WAAW,KAAA,KAAa,KAAK,WAAW,IAC/C,SAAS,KAAK;EAAE,MAAM;EAAa,SAAS,MAAM,KAAK,MAAM;CAAE,CAAC;CAElE,OAAO;AACT;AAgBA,MAAM,gBAAgE;CACpE,SAAS;CACT,QAAQ;CACR,UAAU;AACZ;;;;;;;;;AAUA,SAAS,iBAAiB,QAAyB;CACjD,IAAI,aAAa,MAAM,MAAM,KAAA,GAC3B,MAAM,IAAI,gBAAgB,+BAA+B,OAAO,MAAM,wBAAwB;AAElG;AAEA,MAAM,iBAA8D;CAClE,QAAQ;CACR,KAAK;CACL,SAAS;AACX;AAEA,SAAS,SACP,QACA,OACA,UACA,SACA,YACA,KACmB;CAInB,iBAAiB,MAAM;CAGvB,MAAM,eAAe,oBAAoB,MAAM;CAC/C,MAAM,aAAa,OAAO,eAAe,SAAS;CAClD,MAAM,kBAAkB,OAAO;CAC/B,MAAM,cAAc,oBAAoB,eAAe,oBAAoB;CAC3E,MAAM,cAAc,oBAAoB,eAAe,oBAAoB;CAC3E,MAAM,gBACJ,oBAAoB,YACpB,oBAAoB,eACpB,oBAAoB,eACf,OAAO,yBAAyB,aAAa,oBAC9C;CAIN,OAAO,aACL;EACE,QAAQ;EACR,YAAY,OAAO;EACnB,mBAAmB;EACnB,QAAQ,OAAO;EACf,eAAe,OAAO;EACtB,cAAc,OAAO;EACrB,YAAY;EACZ,iBAAiB;EACjB,MAAM,QAAQ,QAAQ;EACtB,MAAM;GACJ,OAAO,QAAQ,SAAS,OAAO;GAC/B,aAAa,OAAO;GACpB,OAAO,eAAe,OAAO;GAC7B,MAAM,OAAO;GACb,KAAK;EACP;EACA,QAAQ;GACN,SAAS;GACT,iBAAiB;GACjB,OAAO,OAAO;GACd,UAAU;GACV,gBAAgB,OAAO;GACvB,aAAa,OAAO;GACpB,aAAa,OAAO;GACpB,uBAAuB,OAAO,cAAc,UAAU;GACtD,UAAU;EACZ;EACA;EACA,WAAW,CAAC;EACZ,GAAI,MAAM,SAAS,IAAI,EAAE,MAAM,IAAI,CAAC;EACpC,SAAS;GACP,GAAG;GACH,eAAe,cAAc,YAAY,MAAM;GAC/C,SAAS;GAUT,SAAS,EAAE,GAAG,OAAO,QAAQ,IAAI;GACjC,cAAc;GACd,cAAc;GACd,OAAO;EACT;EACA,MAAM;GACJ,KAAK,aAAa,OAAO,OAAO;GAChC,WAAW,OAAO,WAAW;GAC7B,YAAY,OAAO,WAAW;GAC9B,kBAAkB,OAAO,WAAW,aAAa;GACjD,YAAY,OAAO,WAAW,UAAU;GACxC,aAAa,OAAO,WAAW,cAAc;GAC7C,QAAQ,KAAK,MAAM,OAAO,SAAS,GAAI;EACzC;EACA,WAAW;GAAE,YAAY;GAAM,SAAS;GAAM,gBAAgB;EAAK;EACnE,YAAY;GACV,aAAa;GACb,SAAS;GACT,GAAI,QAAQ,KAAA,IAAY,EAAE,IAAI,IAAI,CAAC;EACrC;CACF,GACA,+BAA+B,OAAO,OACxC;AACF;;;;;;;AAQA,eAAsB,gBACpB,SACA,OACA,UAA8B,CAAC,GACH;CAC5B,MAAM,QAAQ,QAAQ,WAAW,MAAM;CACvC,MAAM,cAAc,QAAQ,MAAM,qBAAK,IAAI,KAAK,EAAA,CAAG,YAAY;CAC/D,MAAM,OAA4B,CAAC;CACnC,MAAM,gBAA0B,CAAC;CACjC,KAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,aAAa,MAAM,gBAAgB,OAAO,OAAO,KAAK;EAC5D,IAAI,WAAW,MAAM,WAAW,GAAG;GACjC,cAAc,KAAK,OAAO,KAAK;GAC/B,KAAK,KACH,SAAS,QAAQ,CAAC,GAAG,CAAC,GAAG,SAAS,YAAY,wCAAwC,CACxF;GACA;EACF;EACA,IAAI,QAAQ,WAAW,MAAM,KAAK,MAAM,YAAY,EAAE,MAAM,KAAK,CAAC;EAClE,IAAI,QAAQ,aAAa,KAAA,KAAa,MAAM,SAAS,QAAQ,UAAU;GAGrE,MAAM,OAAO,KAAK,KAAK,QAAQ,WAAW,CAAC;GAC3C,MAAM,OAAO,QAAQ,WAAW;GAChC,QAAQ,CAAC,GAAG,MAAM,MAAM,GAAG,IAAI,GAAG,GAAG,MAAM,MAAM,MAAM,SAAS,IAAI,CAAC;EACvE;EACA,MAAM,eAAe,kBACnB,WAAW,MAAM,KAAK,MAAM,EAAE,IAAI,GAClC,KACF;EACA,MAAM,MACJ,aAAa,WAAW,IAAI,uDAAuD,KAAA;EACrF,KAAK,KAAK,SAAS,QAAQ,OAAO,cAAc,SAAS,YAAY,GAAG,CAAC;CAC3E;CACA,OAAO;EAAE;EAAM;CAAc;AAC/B"}
|
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "@tangle-network/agent-eval — wire protocol",
|
|
5
|
-
"version": "0.133.
|
|
5
|
+
"version": "0.133.2",
|
|
6
6
|
"description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
|
|
7
7
|
"contact": {
|
|
8
8
|
"name": "Tangle Network",
|
|
@@ -187,11 +187,10 @@ function transcriptFromEntries(entries, options = {}) {
|
|
|
187
187
|
* throwing — callers record a gap line, never crash the backfill.
|
|
188
188
|
*/
|
|
189
189
|
const DEFAULT_OPENCODE_DB = join(homedir(), ".local", "share", "opencode", "opencode.db");
|
|
190
|
-
const nodeRequire = createRequire(import.meta.url);
|
|
191
190
|
/** Open the store read-only; null = unavailable/corrupt (caller records a gap). */
|
|
192
191
|
async function openOpencodeDb(path = DEFAULT_OPENCODE_DB) {
|
|
193
192
|
try {
|
|
194
|
-
const { DatabaseSync } =
|
|
193
|
+
const { DatabaseSync } = createRequire(import.meta.url)("node:sqlite");
|
|
195
194
|
const db = new DatabaseSync(path, { readOnly: true });
|
|
196
195
|
db.prepare("SELECT id FROM session LIMIT 1").get();
|
|
197
196
|
return db;
|
|
@@ -315,4 +314,4 @@ function readOpencodeSessionMessages(db, sessionId) {
|
|
|
315
314
|
//#endregion
|
|
316
315
|
export { readOpencodeSessionMessages as a, findClaudeTranscripts as c, transcriptFromEntries as d, openOpencodeDb as i, parseClaudeEntries as l, findOpencodeSessionById as n, DEFAULT_CLAUDE_PROJECTS_DIR as o, findOpencodeSessionsByDirectory as r, claudeProjectSlug as s, DEFAULT_OPENCODE_DB as t, readClaudeTranscript as u };
|
|
317
316
|
|
|
318
|
-
//# sourceMappingURL=opencode-sqlite-
|
|
317
|
+
//# sourceMappingURL=opencode-sqlite-8r6WUfHc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"opencode-sqlite-8r6WUfHc.js","names":["isRecord"],"sources":["../src/rollout/readers/claude-jsonl.ts","../src/rollout/readers/opencode-sqlite.ts"],"sourcesContent":["/**\n * Backfill reader over Claude Code project transcripts\n * (~/.claude/projects/<cwd-slug>/<sessionId>.jsonl) → canonical\n * chat-with-tools messages plus per-session token usage.\n *\n * Transcript lines consumed: type:\"user\" (string content or content blocks —\n * text + tool_result) and type:\"assistant\" (content blocks — thinking, text,\n * tool_use; message.usage carries tokens). Sidechain lines (isSidechain=true,\n * subagent threads) are separate invocations and are excluded from the main\n * transcript. Everything else (queue-operation, attachment, last-prompt…) is\n * transport metadata, not conversation.\n */\n\nimport { readdir, readFile } from 'node:fs/promises'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport type { ChatMessage, ChatToolCall } from '../schema'\n\nexport const DEFAULT_CLAUDE_PROJECTS_DIR = join(homedir(), '.claude', 'projects')\n\n/** Claude Code's project-directory slug for a working directory. */\nexport function claudeProjectSlug(cwd: string): string {\n return cwd.replace(/[^a-zA-Z0-9-]/g, '-')\n}\n\nexport interface ClaudeTranscriptRef {\n sessionId: string\n path: string\n}\n\n/** Transcript files recorded for sessions launched from `cwd`. */\nexport async function findClaudeTranscripts(\n cwd: string,\n projectsDir: string = DEFAULT_CLAUDE_PROJECTS_DIR,\n): Promise<ClaudeTranscriptRef[]> {\n const dir = join(projectsDir, claudeProjectSlug(cwd))\n const names = await readdir(dir).catch(() => [])\n return names\n .filter((n) => n.endsWith('.jsonl'))\n .sort()\n .map((n) => ({ sessionId: n.replace(/\\.jsonl$/, ''), path: join(dir, n) }))\n}\n\nexport interface ClaudeUsageTotals {\n tokensIn: number\n tokensOut: number\n cacheRead: number\n cacheWrite: number\n}\n\nexport interface ClaudeTranscript {\n messages: ChatMessage[]\n usage: ClaudeUsageTotals\n /** Timestamp of the first conversation line; null = empty transcript. */\n startedAt: string | null\n endedAt: string | null\n model: string | null\n}\n\nconst isRecord = (v: unknown): v is Record<string, unknown> =>\n typeof v === 'object' && v !== null && !Array.isArray(v)\n\n/**\n * One conversation line of a transcript, still in Claude Code's own shape.\n *\n * This is the single line-level parse of the format. `readClaudeTranscript`\n * projects it to canonical messages + usage; the supervision-tree reader\n * (`src/supervisor-run/claude-code-reader.ts`) projects the SAME entries to\n * spawn/settle/steer instants. Two projections, one parser — a second\n * transcript parser is how the two views silently disagree.\n */\nexport interface ClaudeEntry {\n readonly type: 'user' | 'assistant'\n /** ISO instant of the line; null when the line carried none. */\n readonly timestamp: string | null\n /** The Anthropic message body (`role`, `content`, `model`, `usage`). */\n readonly message: Record<string, unknown>\n /** Claude Code's structured tool result, when the line carries one. */\n readonly toolUseResult: unknown\n /** True on subagent threads — a separate invocation, not this transcript's turn. */\n readonly isSidechain: boolean\n /** Subagent id Claude Code stamps on sidechain lines; null on main-thread lines. */\n readonly agentId: string | null\n}\n\n/** Parse transcript jsonl text into conversation lines. Non-conversation lines are dropped. */\nexport function parseClaudeEntries(raw: string): ClaudeEntry[] {\n const out: ClaudeEntry[] = []\n for (const line of raw.split('\\n')) {\n if (!line.trim()) continue\n let entry: Record<string, unknown>\n try {\n const parsed: unknown = JSON.parse(line)\n if (!isRecord(parsed)) continue\n entry = parsed\n } catch {\n continue\n }\n if (entry.type !== 'user' && entry.type !== 'assistant') continue\n const message = entry.message\n if (!isRecord(message)) continue\n out.push({\n type: entry.type,\n timestamp: typeof entry.timestamp === 'string' ? entry.timestamp : null,\n message,\n toolUseResult: entry.toolUseResult,\n isSidechain: entry.isSidechain === true,\n agentId: typeof entry.agentId === 'string' ? entry.agentId : null,\n })\n }\n return out\n}\n\nexport interface ReadClaudeTranscriptOptions {\n /**\n * Read the sidechain (subagent) thread instead of skipping it. Subagent\n * transcripts under `<session>/subagents/agent-<id>.jsonl` are sidechain\n * lines end to end, so their usage is invisible without this.\n */\n readonly includeSidechain?: boolean\n}\n\nfunction blockText(content: unknown): string {\n if (typeof content === 'string') return content\n if (!Array.isArray(content)) return ''\n return content\n .filter(\n (b): b is Record<string, unknown> =>\n isRecord(b) && b.type === 'text' && typeof b.text === 'string',\n )\n .map((b) => b.text as string)\n .join('\\n')\n}\n\n/** Parse one transcript jsonl into canonical messages + usage totals. */\nexport async function readClaudeTranscript(\n path: string,\n options: ReadClaudeTranscriptOptions = {},\n): Promise<ClaudeTranscript> {\n return transcriptFromEntries(parseClaudeEntries(await readFile(path, 'utf8')), options)\n}\n\n/** The messages+usage projection of already-parsed entries. */\nexport function transcriptFromEntries(\n entries: readonly ClaudeEntry[],\n options: ReadClaudeTranscriptOptions = {},\n): ClaudeTranscript {\n const wantSidechain = options.includeSidechain === true\n const messages: ChatMessage[] = []\n const usage: ClaudeUsageTotals = { tokensIn: 0, tokensOut: 0, cacheRead: 0, cacheWrite: 0 }\n let startedAt: string | null = null\n let endedAt: string | null = null\n let model: string | null = null\n // Claude Code writes one jsonl line PER CONTENT BLOCK of an API message,\n // repeating message.id and usage on each — merge blocks into one canonical\n // assistant turn and count usage once per API message id.\n let lastAssistantApiId: string | null = null\n let lastAssistantIndex = -1\n\n for (const entry of entries) {\n if (entry.isSidechain !== wantSidechain) continue\n const message = entry.message\n if (entry.timestamp !== null) {\n if (startedAt === null) startedAt = entry.timestamp\n endedAt = entry.timestamp\n }\n\n if (entry.type === 'user') {\n lastAssistantApiId = null\n lastAssistantIndex = -1\n const content = message.content\n if (typeof content === 'string') {\n messages.push({ role: 'user', content })\n continue\n }\n if (!Array.isArray(content)) continue\n // A user line may interleave tool_result blocks (answers to the prior\n // assistant tool_use) with plain text; preserve order.\n let userText = ''\n for (const block of content) {\n if (!isRecord(block)) continue\n if (block.type === 'tool_result' && typeof block.tool_use_id === 'string') {\n messages.push({\n role: 'tool',\n tool_call_id: block.tool_use_id,\n content:\n blockText(block.content) || (typeof block.content === 'string' ? block.content : ''),\n })\n } else if (block.type === 'text' && typeof block.text === 'string') {\n userText += (userText.length > 0 ? '\\n' : '') + block.text\n }\n }\n if (userText.length > 0) messages.push({ role: 'user', content: userText })\n continue\n }\n\n // assistant\n if (typeof message.model === 'string') model = message.model\n const apiId = typeof message.id === 'string' ? message.id : null\n const continuesTurn = apiId !== null && apiId === lastAssistantApiId && lastAssistantIndex >= 0\n const msgUsage = message.usage\n if (isRecord(msgUsage) && !continuesTurn) {\n usage.tokensIn += typeof msgUsage.input_tokens === 'number' ? msgUsage.input_tokens : 0\n usage.tokensOut += typeof msgUsage.output_tokens === 'number' ? msgUsage.output_tokens : 0\n usage.cacheRead +=\n typeof msgUsage.cache_read_input_tokens === 'number' ? msgUsage.cache_read_input_tokens : 0\n usage.cacheWrite +=\n typeof msgUsage.cache_creation_input_tokens === 'number'\n ? msgUsage.cache_creation_input_tokens\n : 0\n }\n const content = message.content\n if (!Array.isArray(content)) continue\n let reasoning = ''\n let text = ''\n const toolCalls: ChatToolCall[] = []\n for (const block of content) {\n if (!isRecord(block)) continue\n if (\n block.type === 'thinking' &&\n typeof block.thinking === 'string' &&\n block.thinking.length > 0\n ) {\n reasoning += (reasoning.length > 0 ? '\\n' : '') + block.thinking\n } else if (block.type === 'text' && typeof block.text === 'string') {\n text += (text.length > 0 ? '\\n' : '') + block.text\n } else if (block.type === 'tool_use' && typeof block.id === 'string') {\n toolCalls.push({\n id: block.id,\n type: 'function',\n function: {\n name: typeof block.name === 'string' ? block.name : 'unknown',\n arguments: JSON.stringify(block.input ?? {}),\n },\n })\n }\n }\n if (reasoning.length === 0 && text.length === 0 && toolCalls.length === 0) continue\n if (continuesTurn) {\n const prev = messages[lastAssistantIndex]!\n if (text.length > 0) prev.content = prev.content === null ? text : `${prev.content}\\n${text}`\n if (reasoning.length > 0) {\n prev.reasoning_content =\n prev.reasoning_content === undefined\n ? reasoning\n : `${prev.reasoning_content}\\n${reasoning}`\n }\n if (toolCalls.length > 0) prev.tool_calls = [...(prev.tool_calls ?? []), ...toolCalls]\n continue\n }\n messages.push({\n role: 'assistant',\n content: text.length > 0 ? text : null,\n ...(reasoning.length > 0 ? { reasoning_content: reasoning } : {}),\n ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),\n })\n lastAssistantApiId = apiId\n lastAssistantIndex = messages.length - 1\n }\n\n return { messages, usage, startedAt, endedAt, model }\n}\n","/**\n * Read-only backfill reader over the opencode sqlite store\n * (~/.local/share/opencode/opencode.db) → canonical chat-with-tools messages.\n *\n * Schema consumed (observed, 2026-07): `session` rows carry directory /\n * parent_id / agent / model / cost / tokens_*; `message` rows carry a JSON\n * `data` blob ({role, modelID, providerID, tokens, cost, finish}); `part`\n * rows carry the actual content ({type: text|reasoning|tool|step-start|\n * step-finish|snapshot…}). Tool parts hold {callID, state:{input, output,\n * status}} — both the call and its result, which we split into an assistant\n * tool_call plus a role:\"tool\" result message.\n *\n * The store is mutable and can be corrupt (a `.corrupt-bak` sibling ships\n * next to it in the wild), so `openOpencodeDb` returns null instead of\n * throwing — callers record a gap line, never crash the backfill.\n */\n\nimport { createRequire } from 'node:module'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport type { DatabaseSync } from 'node:sqlite'\nimport type { ChatMessage, ChatToolCall } from '../schema'\n\nexport const DEFAULT_OPENCODE_DB = join(homedir(), '.local', 'share', 'opencode', 'opencode.db')\n\nexport interface OpencodeSessionRow {\n id: string\n parentId: string | null\n directory: string\n agent: string | null\n /** Raw session.model JSON: {id, providerID, variant} where present. */\n model: { id?: string; providerID?: string } | null\n costUsd: number\n tokensInput: number\n tokensOutput: number\n tokensReasoning: number\n tokensCacheRead: number\n tokensCacheWrite: number\n timeCreated: number\n timeUpdated: number\n}\n\n// `node:sqlite` is loaded through CommonJS `require`, not `import()`. esbuild\n// (bundling) and Vite (tests) both rewrite a dynamic import and strip the\n// `node:` prefix under an es20xx target, turning this builtin into a bogus\n// \"sqlite\" package lookup; composing the specifier at runtime does not reliably\n// defeat that (it still resolved through Vite's transform in some workers, so\n// the failure moved around as test files were added). A require obtained from\n// `createRequire` is not an analyzable module reference in either tool, so\n// neither can rewrite it.\n/** Open the store read-only; null = unavailable/corrupt (caller records a gap). */\nexport async function openOpencodeDb(\n path: string = DEFAULT_OPENCODE_DB,\n): Promise<DatabaseSync | null> {\n try {\n // Keep Node-only module initialization inside the Node-only operation.\n // Root imports are shared with edge consumers that do not define import.meta.url.\n const nodeRequire = createRequire(import.meta.url)\n const { DatabaseSync } = nodeRequire('node:sqlite') as typeof import('node:sqlite')\n const db = new DatabaseSync(path, { readOnly: true })\n // Probe: a corrupt store can open() fine and fail on first page read.\n db.prepare('SELECT id FROM session LIMIT 1').get()\n return db\n } catch {\n return null\n }\n}\n\nconst isRecord = (v: unknown): v is Record<string, unknown> =>\n typeof v === 'object' && v !== null && !Array.isArray(v)\n\nfunction parseSessionRow(row: Record<string, unknown>): OpencodeSessionRow {\n let model: OpencodeSessionRow['model'] = null\n if (typeof row.model === 'string' && row.model.length > 0) {\n try {\n const parsed: unknown = JSON.parse(row.model)\n if (isRecord(parsed)) model = parsed as { id?: string; providerID?: string }\n } catch {\n model = null\n }\n }\n return {\n id: String(row.id),\n parentId: row.parent_id === null || row.parent_id === undefined ? null : String(row.parent_id),\n directory: String(row.directory),\n agent: row.agent === null || row.agent === undefined ? null : String(row.agent),\n model,\n costUsd: Number(row.cost ?? 0),\n tokensInput: Number(row.tokens_input ?? 0),\n tokensOutput: Number(row.tokens_output ?? 0),\n tokensReasoning: Number(row.tokens_reasoning ?? 0),\n tokensCacheRead: Number(row.tokens_cache_read ?? 0),\n tokensCacheWrite: Number(row.tokens_cache_write ?? 0),\n timeCreated: Number(row.time_created ?? 0),\n timeUpdated: Number(row.time_updated ?? 0),\n }\n}\n\nconst SESSION_COLUMNS =\n 'id, parent_id, directory, agent, model, cost, tokens_input, tokens_output, tokens_reasoning, tokens_cache_read, tokens_cache_write, time_created, time_updated'\n\n/** Sessions whose cwd is `directory` (the worker-clone join key). */\nexport function findOpencodeSessionsByDirectory(\n db: DatabaseSync,\n directory: string,\n): OpencodeSessionRow[] {\n const rows = db\n .prepare(`SELECT ${SESSION_COLUMNS} FROM session WHERE directory = ? ORDER BY time_created`)\n .all(directory) as Array<Record<string, unknown>>\n return rows.map(parseSessionRow)\n}\n\nexport function findOpencodeSessionById(\n db: DatabaseSync,\n sessionId: string,\n): OpencodeSessionRow | null {\n const row = db.prepare(`SELECT ${SESSION_COLUMNS} FROM session WHERE id = ?`).get(sessionId) as\n | Record<string, unknown>\n | undefined\n return row === undefined ? null : parseSessionRow(row)\n}\n\ninterface OpencodePart {\n type?: string\n text?: string\n tool?: string\n callID?: string\n state?: { status?: string; input?: unknown; output?: unknown }\n}\n\nfunction toolResultContent(output: unknown): string {\n if (typeof output === 'string') return output\n if (output === null || output === undefined) return ''\n return JSON.stringify(output)\n}\n\n/**\n * Convert one session's message+part rows into canonical messages.\n * An opencode assistant message row spans several model steps; each step's\n * parts (reasoning → text → tool …) become one assistant message followed by\n * the role:\"tool\" results of its calls, preserving order.\n */\nexport function readOpencodeSessionMessages(db: DatabaseSync, sessionId: string): ChatMessage[] {\n const messageRows = db\n .prepare('SELECT id, data FROM message WHERE session_id = ? ORDER BY time_created, id')\n .all(sessionId) as Array<{ id: string; data: string }>\n const partsStmt = db.prepare('SELECT data FROM part WHERE message_id = ? ORDER BY id')\n\n const messages: ChatMessage[] = []\n for (const messageRow of messageRows) {\n let data: Record<string, unknown>\n try {\n const parsed: unknown = JSON.parse(messageRow.data)\n if (!isRecord(parsed)) continue\n data = parsed\n } catch {\n continue\n }\n const parts: OpencodePart[] = []\n for (const row of partsStmt.all(messageRow.id) as Array<{ data: string }>) {\n try {\n const parsed: unknown = JSON.parse(row.data)\n if (isRecord(parsed)) parts.push(parsed as OpencodePart)\n } catch {\n // Malformed part payload: skip the part, keep the message.\n }\n }\n\n if (data.role === 'user') {\n const text = parts\n .filter((p) => p.type === 'text' && typeof p.text === 'string')\n .map((p) => p.text as string)\n .join('\\n')\n messages.push({ role: 'user', content: text })\n continue\n }\n if (data.role !== 'assistant') continue\n\n // Split the row into steps at step-start boundaries; parts before the\n // first step-start (none observed, but tolerated) form an implicit step.\n const steps: OpencodePart[][] = []\n let current: OpencodePart[] = []\n for (const part of parts) {\n if (part.type === 'step-start') {\n if (current.length > 0) steps.push(current)\n current = []\n continue\n }\n if (part.type === 'step-finish' || part.type === 'snapshot' || part.type === 'patch') continue\n current.push(part)\n }\n if (current.length > 0) steps.push(current)\n\n for (const step of steps) {\n const reasoning = step\n .filter((p) => p.type === 'reasoning' && typeof p.text === 'string' && p.text.length > 0)\n .map((p) => p.text as string)\n .join('\\n')\n const text = step\n .filter((p) => p.type === 'text' && typeof p.text === 'string')\n .map((p) => p.text as string)\n .join('\\n')\n const toolParts = step.filter((p) => p.type === 'tool' && typeof p.callID === 'string')\n const toolCalls: ChatToolCall[] = toolParts.map((p) => ({\n id: p.callID as string,\n type: 'function',\n function: {\n name: p.tool ?? 'unknown',\n arguments: JSON.stringify(p.state?.input ?? {}),\n },\n }))\n if (reasoning.length === 0 && text.length === 0 && toolCalls.length === 0) continue\n messages.push({\n role: 'assistant',\n content: text.length > 0 ? text : null,\n ...(reasoning.length > 0 ? { reasoning_content: reasoning } : {}),\n ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),\n })\n for (const p of toolParts) {\n messages.push({\n role: 'tool',\n tool_call_id: p.callID as string,\n name: p.tool ?? 'unknown',\n content: toolResultContent(p.state?.output),\n })\n }\n }\n }\n return messages\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAkBA,MAAa,8BAA8B,KAAK,QAAQ,GAAG,WAAW,UAAU;;AAGhF,SAAgB,kBAAkB,KAAqB;CACrD,OAAO,IAAI,QAAQ,kBAAkB,GAAG;AAC1C;;AAQA,eAAsB,sBACpB,KACA,cAAsB,6BACU;CAChC,MAAM,MAAM,KAAK,aAAa,kBAAkB,GAAG,CAAC;CAEpD,QAAO,MADa,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,EAAA,CAE5C,QAAQ,MAAM,EAAE,SAAS,QAAQ,CAAC,CAAC,CACnC,KAAK,CAAC,CACN,KAAK,OAAO;EAAE,WAAW,EAAE,QAAQ,YAAY,EAAE;EAAG,MAAM,KAAK,KAAK,CAAC;CAAE,EAAE;AAC9E;AAkBA,MAAMA,cAAY,MAChB,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;;AA0BzD,SAAgB,mBAAmB,KAA4B;CAC7D,MAAM,MAAqB,CAAC;CAC5B,KAAK,MAAM,QAAQ,IAAI,MAAM,IAAI,GAAG;EAClC,IAAI,CAAC,KAAK,KAAK,GAAG;EAClB,IAAI;EACJ,IAAI;GACF,MAAM,SAAkB,KAAK,MAAM,IAAI;GACvC,IAAI,CAACA,WAAS,MAAM,GAAG;GACvB,QAAQ;EACV,QAAQ;GACN;EACF;EACA,IAAI,MAAM,SAAS,UAAU,MAAM,SAAS,aAAa;EACzD,MAAM,UAAU,MAAM;EACtB,IAAI,CAACA,WAAS,OAAO,GAAG;EACxB,IAAI,KAAK;GACP,MAAM,MAAM;GACZ,WAAW,OAAO,MAAM,cAAc,WAAW,MAAM,YAAY;GACnE;GACA,eAAe,MAAM;GACrB,aAAa,MAAM,gBAAgB;GACnC,SAAS,OAAO,MAAM,YAAY,WAAW,MAAM,UAAU;EAC/D,CAAC;CACH;CACA,OAAO;AACT;AAWA,SAAS,UAAU,SAA0B;CAC3C,IAAI,OAAO,YAAY,UAAU,OAAO;CACxC,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG,OAAO;CACpC,OAAO,QACJ,QACE,MACCA,WAAS,CAAC,KAAK,EAAE,SAAS,UAAU,OAAO,EAAE,SAAS,QAC1D,CAAC,CACA,KAAK,MAAM,EAAE,IAAc,CAAC,CAC5B,KAAK,IAAI;AACd;;AAGA,eAAsB,qBACpB,MACA,UAAuC,CAAC,GACb;CAC3B,OAAO,sBAAsB,mBAAmB,MAAM,SAAS,MAAM,MAAM,CAAC,GAAG,OAAO;AACxF;;AAGA,SAAgB,sBACd,SACA,UAAuC,CAAC,GACtB;CAClB,MAAM,gBAAgB,QAAQ,qBAAqB;CACnD,MAAM,WAA0B,CAAC;CACjC,MAAM,QAA2B;EAAE,UAAU;EAAG,WAAW;EAAG,WAAW;EAAG,YAAY;CAAE;CAC1F,IAAI,YAA2B;CAC/B,IAAI,UAAyB;CAC7B,IAAI,QAAuB;CAI3B,IAAI,qBAAoC;CACxC,IAAI,qBAAqB;CAEzB,KAAK,MAAM,SAAS,SAAS;EAC3B,IAAI,MAAM,gBAAgB,eAAe;EACzC,MAAM,UAAU,MAAM;EACtB,IAAI,MAAM,cAAc,MAAM;GAC5B,IAAI,cAAc,MAAM,YAAY,MAAM;GAC1C,UAAU,MAAM;EAClB;EAEA,IAAI,MAAM,SAAS,QAAQ;GACzB,qBAAqB;GACrB,qBAAqB;GACrB,MAAM,UAAU,QAAQ;GACxB,IAAI,OAAO,YAAY,UAAU;IAC/B,SAAS,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IACvC;GACF;GACA,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;GAG7B,IAAI,WAAW;GACf,KAAK,MAAM,SAAS,SAAS;IAC3B,IAAI,CAACA,WAAS,KAAK,GAAG;IACtB,IAAI,MAAM,SAAS,iBAAiB,OAAO,MAAM,gBAAgB,UAC/D,SAAS,KAAK;KACZ,MAAM;KACN,cAAc,MAAM;KACpB,SACE,UAAU,MAAM,OAAO,MAAM,OAAO,MAAM,YAAY,WAAW,MAAM,UAAU;IACrF,CAAC;SACI,IAAI,MAAM,SAAS,UAAU,OAAO,MAAM,SAAS,UACxD,aAAa,SAAS,SAAS,IAAI,OAAO,MAAM,MAAM;GAE1D;GACA,IAAI,SAAS,SAAS,GAAG,SAAS,KAAK;IAAE,MAAM;IAAQ,SAAS;GAAS,CAAC;GAC1E;EACF;EAGA,IAAI,OAAO,QAAQ,UAAU,UAAU,QAAQ,QAAQ;EACvD,MAAM,QAAQ,OAAO,QAAQ,OAAO,WAAW,QAAQ,KAAK;EAC5D,MAAM,gBAAgB,UAAU,QAAQ,UAAU,sBAAsB,sBAAsB;EAC9F,MAAM,WAAW,QAAQ;EACzB,IAAIA,WAAS,QAAQ,KAAK,CAAC,eAAe;GACxC,MAAM,YAAY,OAAO,SAAS,iBAAiB,WAAW,SAAS,eAAe;GACtF,MAAM,aAAa,OAAO,SAAS,kBAAkB,WAAW,SAAS,gBAAgB;GACzF,MAAM,aACJ,OAAO,SAAS,4BAA4B,WAAW,SAAS,0BAA0B;GAC5F,MAAM,cACJ,OAAO,SAAS,gCAAgC,WAC5C,SAAS,8BACT;EACR;EACA,MAAM,UAAU,QAAQ;EACxB,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;EAC7B,IAAI,YAAY;EAChB,IAAI,OAAO;EACX,MAAM,YAA4B,CAAC;EACnC,KAAK,MAAM,SAAS,SAAS;GAC3B,IAAI,CAACA,WAAS,KAAK,GAAG;GACtB,IACE,MAAM,SAAS,cACf,OAAO,MAAM,aAAa,YAC1B,MAAM,SAAS,SAAS,GAExB,cAAc,UAAU,SAAS,IAAI,OAAO,MAAM,MAAM;QACnD,IAAI,MAAM,SAAS,UAAU,OAAO,MAAM,SAAS,UACxD,SAAS,KAAK,SAAS,IAAI,OAAO,MAAM,MAAM;QACzC,IAAI,MAAM,SAAS,cAAc,OAAO,MAAM,OAAO,UAC1D,UAAU,KAAK;IACb,IAAI,MAAM;IACV,MAAM;IACN,UAAU;KACR,MAAM,OAAO,MAAM,SAAS,WAAW,MAAM,OAAO;KACpD,WAAW,KAAK,UAAU,MAAM,SAAS,CAAC,CAAC;IAC7C;GACF,CAAC;EAEL;EACA,IAAI,UAAU,WAAW,KAAK,KAAK,WAAW,KAAK,UAAU,WAAW,GAAG;EAC3E,IAAI,eAAe;GACjB,MAAM,OAAO,SAAS;GACtB,IAAI,KAAK,SAAS,GAAG,KAAK,UAAU,KAAK,YAAY,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI;GACvF,IAAI,UAAU,SAAS,GACrB,KAAK,oBACH,KAAK,sBAAsB,KAAA,IACvB,YACA,GAAG,KAAK,kBAAkB,IAAI;GAEtC,IAAI,UAAU,SAAS,GAAG,KAAK,aAAa,CAAC,GAAI,KAAK,cAAc,CAAC,GAAI,GAAG,SAAS;GACrF;EACF;EACA,SAAS,KAAK;GACZ,MAAM;GACN,SAAS,KAAK,SAAS,IAAI,OAAO;GAClC,GAAI,UAAU,SAAS,IAAI,EAAE,mBAAmB,UAAU,IAAI,CAAC;GAC/D,GAAI,UAAU,SAAS,IAAI,EAAE,YAAY,UAAU,IAAI,CAAC;EAC1D,CAAC;EACD,qBAAqB;EACrB,qBAAqB,SAAS,SAAS;CACzC;CAEA,OAAO;EAAE;EAAU;EAAO;EAAW;EAAS;CAAM;AACtD;;;;;;;;;;;;;;;;;;;AC9OA,MAAa,sBAAsB,KAAK,QAAQ,GAAG,UAAU,SAAS,YAAY,aAAa;;AA4B/F,eAAsB,eACpB,OAAe,qBACe;CAC9B,IAAI;EAIF,MAAM,EAAE,iBADY,cAAc,OAAO,KAAK,GACX,CAAC,CAAC,aAAa;EAClD,MAAM,KAAK,IAAI,aAAa,MAAM,EAAE,UAAU,KAAK,CAAC;EAEpD,GAAG,QAAQ,gCAAgC,CAAC,CAAC,IAAI;EACjD,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,YAAY,MAChB,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AAEzD,SAAS,gBAAgB,KAAkD;CACzE,IAAI,QAAqC;CACzC,IAAI,OAAO,IAAI,UAAU,YAAY,IAAI,MAAM,SAAS,GACtD,IAAI;EACF,MAAM,SAAkB,KAAK,MAAM,IAAI,KAAK;EAC5C,IAAI,SAAS,MAAM,GAAG,QAAQ;CAChC,QAAQ;EACN,QAAQ;CACV;CAEF,OAAO;EACL,IAAI,OAAO,IAAI,EAAE;EACjB,UAAU,IAAI,cAAc,QAAQ,IAAI,cAAc,KAAA,IAAY,OAAO,OAAO,IAAI,SAAS;EAC7F,WAAW,OAAO,IAAI,SAAS;EAC/B,OAAO,IAAI,UAAU,QAAQ,IAAI,UAAU,KAAA,IAAY,OAAO,OAAO,IAAI,KAAK;EAC9E;EACA,SAAS,OAAO,IAAI,QAAQ,CAAC;EAC7B,aAAa,OAAO,IAAI,gBAAgB,CAAC;EACzC,cAAc,OAAO,IAAI,iBAAiB,CAAC;EAC3C,iBAAiB,OAAO,IAAI,oBAAoB,CAAC;EACjD,iBAAiB,OAAO,IAAI,qBAAqB,CAAC;EAClD,kBAAkB,OAAO,IAAI,sBAAsB,CAAC;EACpD,aAAa,OAAO,IAAI,gBAAgB,CAAC;EACzC,aAAa,OAAO,IAAI,gBAAgB,CAAC;CAC3C;AACF;AAEA,MAAM,kBACJ;;AAGF,SAAgB,gCACd,IACA,WACsB;CAItB,OAHa,GACV,QAAQ,UAAU,gBAAgB,wDAAwD,CAAC,CAC3F,IAAI,SACG,CAAC,CAAC,IAAI,eAAe;AACjC;AAEA,SAAgB,wBACd,IACA,WAC2B;CAC3B,MAAM,MAAM,GAAG,QAAQ,UAAU,gBAAgB,2BAA2B,CAAC,CAAC,IAAI,SAAS;CAG3F,OAAO,QAAQ,KAAA,IAAY,OAAO,gBAAgB,GAAG;AACvD;AAUA,SAAS,kBAAkB,QAAyB;CAClD,IAAI,OAAO,WAAW,UAAU,OAAO;CACvC,IAAI,WAAW,QAAQ,WAAW,KAAA,GAAW,OAAO;CACpD,OAAO,KAAK,UAAU,MAAM;AAC9B;;;;;;;AAQA,SAAgB,4BAA4B,IAAkB,WAAkC;CAC9F,MAAM,cAAc,GACjB,QAAQ,6EAA6E,CAAC,CACtF,IAAI,SAAS;CAChB,MAAM,YAAY,GAAG,QAAQ,wDAAwD;CAErF,MAAM,WAA0B,CAAC;CACjC,KAAK,MAAM,cAAc,aAAa;EACpC,IAAI;EACJ,IAAI;GACF,MAAM,SAAkB,KAAK,MAAM,WAAW,IAAI;GAClD,IAAI,CAAC,SAAS,MAAM,GAAG;GACvB,OAAO;EACT,QAAQ;GACN;EACF;EACA,MAAM,QAAwB,CAAC;EAC/B,KAAK,MAAM,OAAO,UAAU,IAAI,WAAW,EAAE,GAC3C,IAAI;GACF,MAAM,SAAkB,KAAK,MAAM,IAAI,IAAI;GAC3C,IAAI,SAAS,MAAM,GAAG,MAAM,KAAK,MAAsB;EACzD,QAAQ,CAER;EAGF,IAAI,KAAK,SAAS,QAAQ;GACxB,MAAM,OAAO,MACV,QAAQ,MAAM,EAAE,SAAS,UAAU,OAAO,EAAE,SAAS,QAAQ,CAAC,CAC9D,KAAK,MAAM,EAAE,IAAc,CAAC,CAC5B,KAAK,IAAI;GACZ,SAAS,KAAK;IAAE,MAAM;IAAQ,SAAS;GAAK,CAAC;GAC7C;EACF;EACA,IAAI,KAAK,SAAS,aAAa;EAI/B,MAAM,QAA0B,CAAC;EACjC,IAAI,UAA0B,CAAC;EAC/B,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,KAAK,SAAS,cAAc;IAC9B,IAAI,QAAQ,SAAS,GAAG,MAAM,KAAK,OAAO;IAC1C,UAAU,CAAC;IACX;GACF;GACA,IAAI,KAAK,SAAS,iBAAiB,KAAK,SAAS,cAAc,KAAK,SAAS,SAAS;GACtF,QAAQ,KAAK,IAAI;EACnB;EACA,IAAI,QAAQ,SAAS,GAAG,MAAM,KAAK,OAAO;EAE1C,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,YAAY,KACf,QAAQ,MAAM,EAAE,SAAS,eAAe,OAAO,EAAE,SAAS,YAAY,EAAE,KAAK,SAAS,CAAC,CAAC,CACxF,KAAK,MAAM,EAAE,IAAc,CAAC,CAC5B,KAAK,IAAI;GACZ,MAAM,OAAO,KACV,QAAQ,MAAM,EAAE,SAAS,UAAU,OAAO,EAAE,SAAS,QAAQ,CAAC,CAC9D,KAAK,MAAM,EAAE,IAAc,CAAC,CAC5B,KAAK,IAAI;GACZ,MAAM,YAAY,KAAK,QAAQ,MAAM,EAAE,SAAS,UAAU,OAAO,EAAE,WAAW,QAAQ;GACtF,MAAM,YAA4B,UAAU,KAAK,OAAO;IACtD,IAAI,EAAE;IACN,MAAM;IACN,UAAU;KACR,MAAM,EAAE,QAAQ;KAChB,WAAW,KAAK,UAAU,EAAE,OAAO,SAAS,CAAC,CAAC;IAChD;GACF,EAAE;GACF,IAAI,UAAU,WAAW,KAAK,KAAK,WAAW,KAAK,UAAU,WAAW,GAAG;GAC3E,SAAS,KAAK;IACZ,MAAM;IACN,SAAS,KAAK,SAAS,IAAI,OAAO;IAClC,GAAI,UAAU,SAAS,IAAI,EAAE,mBAAmB,UAAU,IAAI,CAAC;IAC/D,GAAI,UAAU,SAAS,IAAI,EAAE,YAAY,UAAU,IAAI,CAAC;GAC1D,CAAC;GACD,KAAK,MAAM,KAAK,WACd,SAAS,KAAK;IACZ,MAAM;IACN,cAAc,EAAE;IAChB,MAAM,EAAE,QAAQ;IAChB,SAAS,kBAAkB,EAAE,OAAO,MAAM;GAC5C,CAAC;EAEL;CACF;CACA,OAAO;AACT"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { s as ValidationError } from "./errors-8YnH8WlF.js";
|
|
2
|
-
import { N as wilcoxonSignedRank, p as mcnemar, v as pairedBootstrap, x as pairedRiskDifference } from "./statistics-
|
|
2
|
+
import { N as wilcoxonSignedRank, p as mcnemar, v as pairedBootstrap, x as pairedRiskDifference } from "./statistics-DWM_AyLe.js";
|
|
3
3
|
//#region src/paired-arms.ts
|
|
4
4
|
/**
|
|
5
5
|
* Matched-pair arm comparison — "did the treatment arm beat the baseline arm
|
|
@@ -257,4 +257,4 @@ function metricValue(row, name) {
|
|
|
257
257
|
//#endregion
|
|
258
258
|
export { pairArms as n, pairRunRecords as r, comparePairedArms as t };
|
|
259
259
|
|
|
260
|
-
//# sourceMappingURL=paired-arms-
|
|
260
|
+
//# sourceMappingURL=paired-arms-6XItKzd1.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"paired-arms-D9D0wXj2.js","names":[],"sources":["../src/paired-arms.ts"],"sourcesContent":["/**\n * Matched-pair arm comparison — \"did the treatment arm beat the baseline arm\n * on the SAME work items?\"\n *\n * An arm A/B over run records is only trustworthy when it is PAIRED: the same\n * task/scenario/seed evaluated under both arms, compared item-by-item, so\n * inter-item difficulty variance cancels instead of masquerading as an arm\n * effect. This module owns the two error-prone steps every consumer otherwise\n * hand-rolls:\n *\n * 1. Pairing — matching rows across arms by `pairKey` (and by `repKey`\n * within multi-rep items), with leftovers REPORTED rather than silently\n * dropped (a silently unbalanced pairing biases every paired statistic\n * downstream). Pairing never keys on outcome content: matching reps by\n * their outcomes deflates discordant-pair counts and makes McNemar\n * anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.\n * 2. Composition — feeding the matched pairs to the correct paired\n * estimators that already live in `statistics`: `mcnemar` +\n * `pairedRiskDifference` for pass/fail, `pairedBootstrap` +\n * `wilcoxonSignedRank` for continuous metrics. No statistic is\n * re-implemented here.\n *\n * The row shape is deliberately structural — callers project a `RunRecord`\n * (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are\n * caller-supplied parameters; the module ships no domain literal.\n */\n\nimport { ValidationError } from './errors'\nimport type { RunRecord } from './run-record'\nimport type { McNemarResult, PairedBootstrapOptions, PairedBootstrapResult } from './statistics'\nimport {\n mcnemar,\n pairedBootstrap,\n pairedRiskDifference,\n type RiskDifferenceResult,\n wilcoxonSignedRank,\n} from './statistics'\n\n/** One arm observation of one work item. Structural on purpose: callers\n * project their own record type (e.g. a `RunRecord`) into this shape. */\nexport interface PairedArmRow {\n /** Matching key — rows sharing a `pairKey` across both arms form pairs\n * (typically the task/scenario/seed identity). */\n pairKey: string\n /** Rep identity within a `pairKey` (e.g. a seed or rep number). Required on\n * every row of a `pairKey` that has more than one rep in either arm; reps\n * then pair only on exact (`pairKey`, `repKey`) match, never on outcome\n * content. Optional when each arm has at most one rep of the item. */\n repKey?: string\n /** Arm label this row was produced under. */\n arm: string\n /** Binary outcome; omit when the comparison has no pass/fail notion. */\n pass?: boolean\n /** Named numeric measurements (score, cost, latency, …). */\n metrics?: Record<string, number>\n}\n\nexport interface PairArmsOptions {\n /** Arm treated as the control side of every pair. */\n baselineArm: string\n /** Arm treated as the treatment side of every pair. */\n treatmentArm: string\n}\n\n/** One matched (baseline, treatment) observation of the same work item. */\nexport interface MatchedPair {\n pairKey: string\n /** 0-based position of this pair within its `pairKey`, ordered by sorted\n * `repKey` (always 0 for a single-rep item). The rep identity itself is on\n * the rows (`baseline.repKey` / `treatment.repKey`). */\n repIndex: number\n baseline: PairedArmRow\n treatment: PairedArmRow\n}\n\nexport interface PairArmsResult {\n /** Matched pairs, ordered by (`pairKey`, `repIndex`). */\n pairs: MatchedPair[]\n /** Baseline rows left without a treatment counterpart — reported, never\n * silently dropped. */\n unpairedBaseline: PairedArmRow[]\n /** Treatment rows left without a baseline counterpart. */\n unpairedTreatment: PairedArmRow[]\n}\n\n/**\n * Match rows across two arms into (baseline, treatment) pairs by `pairKey`.\n *\n * A `pairKey` with at most one row per arm pairs directly, no `repKey`\n * needed. A `pairKey` with multiple reps in either arm requires `repKey` on\n * every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)\n * match — pairing is keyed purely on row identity, never on outcome content\n * (outcome-keyed matching deflates discordant counts and biases McNemar), and\n * is therefore independent of input order. Reps whose `repKey` has no\n * counterpart in the other arm, and items present in only one arm, land in\n * the unpaired lists — reported, never truncated.\n *\n * Fail-loud: throws when either named arm has zero rows (an unknown arm\n * name would otherwise read as \"everything unpaired\"), when the two arm\n * names are equal, when a multi-rep `pairKey` has a row without `repKey`, or\n * when a (`pairKey`, arm) group repeats a `repKey` (the match would be\n * ambiguous).\n */\nexport function pairArms(rows: readonly PairedArmRow[], opts: PairArmsOptions): PairArmsResult {\n const { baselineArm, treatmentArm } = opts\n if (baselineArm === treatmentArm) {\n throw new ValidationError(\n `pairArms: baselineArm and treatmentArm are both '${baselineArm}' — an arm cannot be compared to itself`,\n )\n }\n\n // arm → pairKey → rows\n const byArm = new Map<string, Map<string, PairedArmRow[]>>()\n const armsSeen = new Set<string>()\n for (const row of rows) {\n armsSeen.add(row.arm)\n if (row.arm !== baselineArm && row.arm !== treatmentArm) continue\n const byKey = byArm.get(row.arm) ?? new Map<string, PairedArmRow[]>()\n const group = byKey.get(row.pairKey) ?? []\n group.push(row)\n byKey.set(row.pairKey, group)\n byArm.set(row.arm, byKey)\n }\n\n for (const arm of [baselineArm, treatmentArm]) {\n if (!byArm.has(arm)) {\n const seen = [...armsSeen].sort().join(', ') || '<none>'\n throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`)\n }\n }\n\n const baselineByKey = byArm.get(baselineArm)!\n const treatmentByKey = byArm.get(treatmentArm)!\n\n const allKeys = [...new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort()\n const pairs: MatchedPair[] = []\n const unpairedBaseline: PairedArmRow[] = []\n const unpairedTreatment: PairedArmRow[] = []\n for (const pairKey of allKeys) {\n const b = baselineByKey.get(pairKey) ?? []\n const t = treatmentByKey.get(pairKey) ?? []\n\n if (b.length <= 1 && t.length <= 1) {\n if (b.length === 1 && t.length === 1) {\n const baseline = b[0]!\n const treatment = t[0]!\n if (baseline.repKey !== undefined || treatment.repKey !== undefined) {\n if (\n baseline.repKey === undefined ||\n treatment.repKey === undefined ||\n baseline.repKey !== treatment.repKey\n ) {\n unpairedBaseline.push(baseline)\n unpairedTreatment.push(treatment)\n continue\n }\n }\n pairs.push({ pairKey, repIndex: 0, baseline, treatment })\n } else {\n unpairedBaseline.push(...b)\n unpairedTreatment.push(...t)\n }\n continue\n }\n\n const bByRep = indexByRepKey(b, pairKey, baselineArm)\n const tByRep = indexByRepKey(t, pairKey, treatmentArm)\n const repKeys = [...new Set([...bByRep.keys(), ...tByRep.keys()])].sort()\n let repIndex = 0\n for (const repKey of repKeys) {\n const baseline = bByRep.get(repKey)\n const treatment = tByRep.get(repKey)\n if (baseline !== undefined && treatment !== undefined) {\n pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment })\n } else if (baseline !== undefined) {\n unpairedBaseline.push(baseline)\n } else if (treatment !== undefined) {\n unpairedTreatment.push(treatment)\n }\n }\n }\n\n return { pairs, unpairedBaseline, unpairedTreatment }\n}\n\n/** Index a multi-rep (pairKey, arm) group by `repKey`, enforcing that every\n * row carries one and that no repKey repeats within the group. */\nfunction indexByRepKey(\n group: readonly PairedArmRow[],\n pairKey: string,\n arm: string,\n): Map<string, PairedArmRow> {\n const byRep = new Map<string, PairedArmRow>()\n for (const row of group) {\n if (row.repKey === undefined) {\n throw new ValidationError(\n `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' ` +\n `is missing repKey — multi-rep items require an explicit repKey on every row so reps ` +\n `pair by identity (pairing reps by outcome or by index would bias the paired statistics)`,\n )\n }\n if (byRep.has(row.repKey)) {\n throw new ValidationError(\n `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' — ` +\n `(pairKey, repKey) must uniquely identify a rep within an arm`,\n )\n }\n byRep.set(row.repKey, row)\n }\n return byRep\n}\n\n/** Paired pass/fail comparison over the pairs where BOTH sides carry `pass`. */\nexport interface PairedCorrectness {\n /** Discordant pairs where the treatment passed and the baseline failed. */\n b10: number\n /** Discordant pairs where the baseline passed and the treatment failed. */\n b01: number\n /** Exact McNemar significance over the paired outcomes (`b === b10`, `c === b01`). */\n mcnemar: McNemarResult\n /** Paired effect size: p(treatment) − p(baseline) with a paired-variance CI. */\n riskDifference: RiskDifferenceResult\n}\n\n/** Paired delta summary for one named metric (delta = treatment − baseline). */\nexport interface PairedMetricDelta {\n name: string\n /** Pairs where BOTH sides carry a finite value for this metric. */\n n: number\n /** Pairs where at least one side does not carry the metric. */\n nMissing: number\n /** Median paired delta, or null when `n === 0`. */\n medianDelta: number | null\n /** Mean paired delta, or null when `n === 0`. */\n meanDelta: number | null\n /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when\n * `n === 0` — a zero-width [0, 0] interval on no data would read as a\n * measured tight null. */\n bootstrapCi: PairedBootstrapResult | null\n /** Wilcoxon signed-rank test on the paired deltas; null when `n === 0`. */\n wilcoxon: { w: number; p: number } | null\n}\n\nexport interface ComparePairedArmsOptions extends PairArmsOptions {\n /** Metrics to compare. Default: every metric name observed on any matched\n * pair, sorted. A name that appears on no pair is still reported (with\n * `n = 0`) so a misspelled metric is visible instead of vanishing. */\n metricNames?: string[]\n /** Passed through to `pairedBootstrap` — set `seed` for reproducible CIs. */\n bootstrap?: PairedBootstrapOptions\n}\n\nexport interface PairedArmsComparison {\n nPairs: number\n nUnpairedBaseline: number\n nUnpairedTreatment: number\n /** null when no matched pair carries `pass` on both sides — a pass/fail\n * verdict over rows that never measured pass/fail would be fabricated. */\n correctness: PairedCorrectness | null\n metricDeltas: PairedMetricDelta[]\n}\n\n/**\n * Full matched-pair arm comparison: pair via {@link pairArms}, then compose\n * the paired estimators from `statistics` over the matched pairs.\n *\n * Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`\n * is that subset's size); each metric uses only the pairs where both sides\n * carry a finite value for it, with the remainder counted in `nMissing`.\n * Deltas are treatment − baseline throughout.\n *\n * Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a\n * non-finite metric value — silently treating corrupt telemetry as \"metric\n * absent\" would misreport it as missing coverage.\n */\nexport function comparePairedArms(\n rows: readonly PairedArmRow[],\n opts: ComparePairedArmsOptions,\n): PairedArmsComparison {\n const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts)\n\n let correctness: PairedCorrectness | null = null\n const baselinePass: number[] = []\n const treatmentPass: number[] = []\n for (const pair of pairs) {\n if (pair.baseline.pass === undefined || pair.treatment.pass === undefined) continue\n baselinePass.push(pair.baseline.pass ? 1 : 0)\n treatmentPass.push(pair.treatment.pass ? 1 : 0)\n }\n if (baselinePass.length > 0) {\n const mc = mcnemar(baselinePass, treatmentPass)\n correctness = {\n b10: mc.b,\n b01: mc.c,\n mcnemar: mc,\n riskDifference: pairedRiskDifference(baselinePass, treatmentPass),\n }\n }\n\n const metricNames =\n opts.metricNames ??\n [\n ...new Set(\n pairs.flatMap((p) => [\n ...Object.keys(p.baseline.metrics ?? {}),\n ...Object.keys(p.treatment.metrics ?? {}),\n ]),\n ),\n ].sort()\n\n const metricDeltas: PairedMetricDelta[] = metricNames.map((name) => {\n const before: number[] = []\n const after: number[] = []\n let nMissing = 0\n for (const pair of pairs) {\n const b = metricValue(pair.baseline, name)\n const t = metricValue(pair.treatment, name)\n if (b === undefined || t === undefined) {\n nMissing++\n continue\n }\n before.push(b)\n after.push(t)\n }\n const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap)\n return {\n name,\n n: before.length,\n nMissing,\n medianDelta: bootstrapCi?.median ?? null,\n meanDelta: bootstrapCi?.mean ?? null,\n bootstrapCi,\n wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after),\n }\n })\n\n return {\n nPairs: pairs.length,\n nUnpairedBaseline: unpairedBaseline.length,\n nUnpairedTreatment: unpairedTreatment.length,\n correctness,\n metricDeltas,\n }\n}\n\nexport interface MatchedRunRecordPair {\n pairKey: string\n repKey: string\n baseline: RunRecord\n treatment: RunRecord\n}\n\nexport interface PairRunRecordsResult {\n pairs: MatchedRunRecordPair[]\n unpairedBaseline: RunRecord[]\n unpairedTreatment: RunRecord[]\n}\n\ninterface RunRecordArmRow extends PairedArmRow {\n run: RunRecord\n repKey: string\n}\n\n/**\n * Pair two RunRecord arms by the identity of the evaluated work:\n * `(experimentId, scenarioId, seed)`.\n *\n * Falling back to array order, candidate id, or experiment id can compare\n * different tasks and fabricate lift. Duplicate identities throw.\n */\nexport function pairRunRecords(\n baselineRuns: readonly RunRecord[],\n treatmentRuns: readonly RunRecord[],\n): PairRunRecordsResult {\n const baselineRows = runRecordArmRows(baselineRuns, 'baseline')\n const treatmentRows = runRecordArmRows(treatmentRuns, 'treatment')\n validateRunRecordArmRows(baselineRows, 'baseline')\n validateRunRecordArmRows(treatmentRows, 'treatment')\n if (baselineRows.length === 0 || treatmentRows.length === 0) {\n return {\n pairs: [],\n unpairedBaseline: baselineRows.map((row) => row.run),\n unpairedTreatment: treatmentRows.map((row) => row.run),\n }\n }\n\n const result = pairArms([...baselineRows, ...treatmentRows], {\n baselineArm: 'baseline',\n treatmentArm: 'treatment',\n })\n return {\n pairs: result.pairs.map((pair) => {\n const baseline = pair.baseline as RunRecordArmRow\n const treatment = pair.treatment as RunRecordArmRow\n return {\n pairKey: pair.pairKey,\n repKey: baseline.repKey,\n baseline: baseline.run,\n treatment: treatment.run,\n }\n }),\n unpairedBaseline: result.unpairedBaseline.map((row) => (row as RunRecordArmRow).run),\n unpairedTreatment: result.unpairedTreatment.map((row) => (row as RunRecordArmRow).run),\n }\n}\n\nfunction runRecordArmRows(runs: readonly RunRecord[], arm: string): RunRecordArmRow[] {\n return runs.map((run) => {\n const scenarioId = run.scenarioId.trim()\n if (!scenarioId) {\n throw new ValidationError(\n `pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`,\n )\n }\n return {\n pairKey: JSON.stringify([run.experimentId, scenarioId]),\n repKey: String(run.seed),\n arm,\n run,\n }\n })\n}\n\nfunction validateRunRecordArmRows(rows: readonly RunRecordArmRow[], arm: string): void {\n const byPairKey = new Map<string, RunRecordArmRow[]>()\n for (const row of rows) {\n const group = byPairKey.get(row.pairKey) ?? []\n group.push(row)\n byPairKey.set(row.pairKey, group)\n }\n for (const [pairKey, group] of byPairKey) {\n if (group.length > 1) indexByRepKey(group, pairKey, arm)\n }\n}\n\nfunction metricValue(row: PairedArmRow, name: string): number | undefined {\n const v = row.metrics?.[name]\n if (v === undefined) return undefined\n if (!Number.isFinite(v)) {\n throw new ValidationError(\n `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`,\n )\n }\n return v\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuGA,SAAgB,SAAS,MAA+B,MAAuC;CAC7F,MAAM,EAAE,aAAa,iBAAiB;CACtC,IAAI,gBAAgB,cAClB,MAAM,IAAI,gBACR,oDAAoD,YAAY,wCAClE;CAIF,MAAM,wBAAQ,IAAI,IAAyC;CAC3D,MAAM,2BAAW,IAAI,IAAY;CACjC,KAAK,MAAM,OAAO,MAAM;EACtB,SAAS,IAAI,IAAI,GAAG;EACpB,IAAI,IAAI,QAAQ,eAAe,IAAI,QAAQ,cAAc;EACzD,MAAM,QAAQ,MAAM,IAAI,IAAI,GAAG,qBAAK,IAAI,IAA4B;EACpE,MAAM,QAAQ,MAAM,IAAI,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,KAAK,GAAG;EACd,MAAM,IAAI,IAAI,SAAS,KAAK;EAC5B,MAAM,IAAI,IAAI,KAAK,KAAK;CAC1B;CAEA,KAAK,MAAM,OAAO,CAAC,aAAa,YAAY,GAC1C,IAAI,CAAC,MAAM,IAAI,GAAG,GAEhB,MAAM,IAAI,gBAAgB,8BAA8B,IAAI,mBAD/C,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,KAAK,SACoC,EAAE;CAI1F,MAAM,gBAAgB,MAAM,IAAI,WAAW;CAC3C,MAAM,iBAAiB,MAAM,IAAI,YAAY;CAE7C,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;CACvF,MAAM,QAAuB,CAAC;CAC9B,MAAM,mBAAmC,CAAC;CAC1C,MAAM,oBAAoC,CAAC;CAC3C,KAAK,MAAM,WAAW,SAAS;EAC7B,MAAM,IAAI,cAAc,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,IAAI,eAAe,IAAI,OAAO,KAAK,CAAC;EAE1C,IAAI,EAAE,UAAU,KAAK,EAAE,UAAU,GAAG;GAClC,IAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;IACpC,MAAM,WAAW,EAAE;IACnB,MAAM,YAAY,EAAE;IACpB,IAAI,SAAS,WAAW,KAAA,KAAa,UAAU,WAAW,KAAA,GAEtD;SAAA,SAAS,WAAW,KAAA,KACpB,UAAU,WAAW,KAAA,KACrB,SAAS,WAAW,UAAU,QAC9B;MACA,iBAAiB,KAAK,QAAQ;MAC9B,kBAAkB,KAAK,SAAS;MAChC;KACF;;IAEF,MAAM,KAAK;KAAE;KAAS,UAAU;KAAG;KAAU;IAAU,CAAC;GAC1D,OAAO;IACL,iBAAiB,KAAK,GAAG,CAAC;IAC1B,kBAAkB,KAAK,GAAG,CAAC;GAC7B;GACA;EACF;EAEA,MAAM,SAAS,cAAc,GAAG,SAAS,WAAW;EACpD,MAAM,SAAS,cAAc,GAAG,SAAS,YAAY;EACrD,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,OAAO,KAAK,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;EACxE,IAAI,WAAW;EACf,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,WAAW,OAAO,IAAI,MAAM;GAClC,MAAM,YAAY,OAAO,IAAI,MAAM;GACnC,IAAI,aAAa,KAAA,KAAa,cAAc,KAAA,GAC1C,MAAM,KAAK;IAAE;IAAS,UAAU;IAAY;IAAU;GAAU,CAAC;QAC5D,IAAI,aAAa,KAAA,GACtB,iBAAiB,KAAK,QAAQ;QACzB,IAAI,cAAc,KAAA,GACvB,kBAAkB,KAAK,SAAS;EAEpC;CACF;CAEA,OAAO;EAAE;EAAO;EAAkB;CAAkB;AACtD;;;AAIA,SAAS,cACP,OACA,SACA,KAC2B;CAC3B,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,OAAO,OAAO;EACvB,IAAI,IAAI,WAAW,KAAA,GACjB,MAAM,IAAI,gBACR,sBAAsB,QAAQ,mDAAmD,IAAI,8KAGvF;EAEF,IAAI,MAAM,IAAI,IAAI,MAAM,GACtB,MAAM,IAAI,gBACR,+BAA+B,IAAI,OAAO,iBAAiB,QAAQ,YAAY,IAAI,iEAErF;EAEF,MAAM,IAAI,IAAI,QAAQ,GAAG;CAC3B;CACA,OAAO;AACT;;;;;;;;;;;;;;AAiEA,SAAgB,kBACd,MACA,MACsB;CACtB,MAAM,EAAE,OAAO,kBAAkB,sBAAsB,SAAS,MAAM,IAAI;CAE1E,IAAI,cAAwC;CAC5C,MAAM,eAAyB,CAAC;CAChC,MAAM,gBAA0B,CAAC;CACjC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,KAAK,SAAS,SAAS,KAAA,KAAa,KAAK,UAAU,SAAS,KAAA,GAAW;EAC3E,aAAa,KAAK,KAAK,SAAS,OAAO,IAAI,CAAC;EAC5C,cAAc,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;CAChD;CACA,IAAI,aAAa,SAAS,GAAG;EAC3B,MAAM,KAAK,QAAQ,cAAc,aAAa;EAC9C,cAAc;GACZ,KAAK,GAAG;GACR,KAAK,GAAG;GACR,SAAS;GACT,gBAAgB,qBAAqB,cAAc,aAAa;EAClE;CACF;CAaA,MAAM,gBAVJ,KAAK,eACL,CACE,GAAG,IAAI,IACL,MAAM,SAAS,MAAM,CACnB,GAAG,OAAO,KAAK,EAAE,SAAS,WAAW,CAAC,CAAC,GACvC,GAAG,OAAO,KAAK,EAAE,UAAU,WAAW,CAAC,CAAC,CAC1C,CAAC,CACH,CACF,CAAC,CAAC,KAAK,EAAA,CAE6C,KAAK,SAAS;EAClE,MAAM,SAAmB,CAAC;EAC1B,MAAM,QAAkB,CAAC;EACzB,IAAI,WAAW;EACf,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,IAAI,YAAY,KAAK,UAAU,IAAI;GACzC,MAAM,IAAI,YAAY,KAAK,WAAW,IAAI;GAC1C,IAAI,MAAM,KAAA,KAAa,MAAM,KAAA,GAAW;IACtC;IACA;GACF;GACA,OAAO,KAAK,CAAC;GACb,MAAM,KAAK,CAAC;EACd;EACA,MAAM,cAAc,OAAO,WAAW,IAAI,OAAO,gBAAgB,QAAQ,OAAO,KAAK,SAAS;EAC9F,OAAO;GACL;GACA,GAAG,OAAO;GACV;GACA,aAAa,aAAa,UAAU;GACpC,WAAW,aAAa,QAAQ;GAChC;GACA,UAAU,OAAO,WAAW,IAAI,OAAO,mBAAmB,QAAQ,KAAK;EACzE;CACF,CAAC;CAED,OAAO;EACL,QAAQ,MAAM;EACd,mBAAmB,iBAAiB;EACpC,oBAAoB,kBAAkB;EACtC;EACA;CACF;AACF;;;;;;;;AA2BA,SAAgB,eACd,cACA,eACsB;CACtB,MAAM,eAAe,iBAAiB,cAAc,UAAU;CAC9D,MAAM,gBAAgB,iBAAiB,eAAe,WAAW;CACjE,yBAAyB,cAAc,UAAU;CACjD,yBAAyB,eAAe,WAAW;CACnD,IAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GACxD,OAAO;EACL,OAAO,CAAC;EACR,kBAAkB,aAAa,KAAK,QAAQ,IAAI,GAAG;EACnD,mBAAmB,cAAc,KAAK,QAAQ,IAAI,GAAG;CACvD;CAGF,MAAM,SAAS,SAAS,CAAC,GAAG,cAAc,GAAG,aAAa,GAAG;EAC3D,aAAa;EACb,cAAc;CAChB,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,KAAK,SAAS;GAChC,MAAM,WAAW,KAAK;GACtB,MAAM,YAAY,KAAK;GACvB,OAAO;IACL,SAAS,KAAK;IACd,QAAQ,SAAS;IACjB,UAAU,SAAS;IACnB,WAAW,UAAU;GACvB;EACF,CAAC;EACD,kBAAkB,OAAO,iBAAiB,KAAK,QAAS,IAAwB,GAAG;EACnF,mBAAmB,OAAO,kBAAkB,KAAK,QAAS,IAAwB,GAAG;CACvF;AACF;AAEA,SAAS,iBAAiB,MAA4B,KAAgC;CACpF,OAAO,KAAK,KAAK,QAAQ;EACvB,MAAM,aAAa,IAAI,WAAW,KAAK;EACvC,IAAI,CAAC,YACH,MAAM,IAAI,gBACR,wBAAwB,IAAI,MAAM,+EACpC;EAEF,OAAO;GACL,SAAS,KAAK,UAAU,CAAC,IAAI,cAAc,UAAU,CAAC;GACtD,QAAQ,OAAO,IAAI,IAAI;GACvB;GACA;EACF;CACF,CAAC;AACH;AAEA,SAAS,yBAAyB,MAAkC,KAAmB;CACrF,MAAM,4BAAY,IAAI,IAA+B;CACrD,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,UAAU,IAAI,IAAI,OAAO,KAAK,CAAC;EAC7C,MAAM,KAAK,GAAG;EACd,UAAU,IAAI,IAAI,SAAS,KAAK;CAClC;CACA,KAAK,MAAM,CAAC,SAAS,UAAU,WAC7B,IAAI,MAAM,SAAS,GAAG,cAAc,OAAO,SAAS,GAAG;AAE3D;AAEA,SAAS,YAAY,KAAmB,MAAkC;CACxE,MAAM,IAAI,IAAI,UAAU;CACxB,IAAI,MAAM,KAAA,GAAW,OAAO,KAAA;CAC5B,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,gBACR,mDAAmD,KAAK,gBAAgB,IAAI,QAAQ,UAAU,IAAI,IAAI,MAAM,GAC9G;CAEF,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"paired-arms-6XItKzd1.js","names":[],"sources":["../src/paired-arms.ts"],"sourcesContent":["/**\n * Matched-pair arm comparison — \"did the treatment arm beat the baseline arm\n * on the SAME work items?\"\n *\n * An arm A/B over run records is only trustworthy when it is PAIRED: the same\n * task/scenario/seed evaluated under both arms, compared item-by-item, so\n * inter-item difficulty variance cancels instead of masquerading as an arm\n * effect. This module owns the two error-prone steps every consumer otherwise\n * hand-rolls:\n *\n * 1. Pairing — matching rows across arms by `pairKey` (and by `repKey`\n * within multi-rep items), with leftovers REPORTED rather than silently\n * dropped (a silently unbalanced pairing biases every paired statistic\n * downstream). Pairing never keys on outcome content: matching reps by\n * their outcomes deflates discordant-pair counts and makes McNemar\n * anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.\n * 2. Composition — feeding the matched pairs to the correct paired\n * estimators that already live in `statistics`: `mcnemar` +\n * `pairedRiskDifference` for pass/fail, `pairedBootstrap` +\n * `wilcoxonSignedRank` for continuous metrics. No statistic is\n * re-implemented here.\n *\n * The row shape is deliberately structural — callers project a `RunRecord`\n * (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are\n * caller-supplied parameters; the module ships no domain literal.\n */\n\nimport { ValidationError } from './errors'\nimport type { RunRecord } from './run-record'\nimport type { McNemarResult, PairedBootstrapOptions, PairedBootstrapResult } from './statistics'\nimport {\n mcnemar,\n pairedBootstrap,\n pairedRiskDifference,\n type RiskDifferenceResult,\n wilcoxonSignedRank,\n} from './statistics'\n\n/** One arm observation of one work item. Structural on purpose: callers\n * project their own record type (e.g. a `RunRecord`) into this shape. */\nexport interface PairedArmRow {\n /** Matching key — rows sharing a `pairKey` across both arms form pairs\n * (typically the task/scenario/seed identity). */\n pairKey: string\n /** Rep identity within a `pairKey` (e.g. a seed or rep number). Required on\n * every row of a `pairKey` that has more than one rep in either arm; reps\n * then pair only on exact (`pairKey`, `repKey`) match, never on outcome\n * content. Optional when each arm has at most one rep of the item. */\n repKey?: string\n /** Arm label this row was produced under. */\n arm: string\n /** Binary outcome; omit when the comparison has no pass/fail notion. */\n pass?: boolean\n /** Named numeric measurements (score, cost, latency, …). */\n metrics?: Record<string, number>\n}\n\nexport interface PairArmsOptions {\n /** Arm treated as the control side of every pair. */\n baselineArm: string\n /** Arm treated as the treatment side of every pair. */\n treatmentArm: string\n}\n\n/** One matched (baseline, treatment) observation of the same work item. */\nexport interface MatchedPair {\n pairKey: string\n /** 0-based position of this pair within its `pairKey`, ordered by sorted\n * `repKey` (always 0 for a single-rep item). The rep identity itself is on\n * the rows (`baseline.repKey` / `treatment.repKey`). */\n repIndex: number\n baseline: PairedArmRow\n treatment: PairedArmRow\n}\n\nexport interface PairArmsResult {\n /** Matched pairs, ordered by (`pairKey`, `repIndex`). */\n pairs: MatchedPair[]\n /** Baseline rows left without a treatment counterpart — reported, never\n * silently dropped. */\n unpairedBaseline: PairedArmRow[]\n /** Treatment rows left without a baseline counterpart. */\n unpairedTreatment: PairedArmRow[]\n}\n\n/**\n * Match rows across two arms into (baseline, treatment) pairs by `pairKey`.\n *\n * A `pairKey` with at most one row per arm pairs directly, no `repKey`\n * needed. A `pairKey` with multiple reps in either arm requires `repKey` on\n * every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)\n * match — pairing is keyed purely on row identity, never on outcome content\n * (outcome-keyed matching deflates discordant counts and biases McNemar), and\n * is therefore independent of input order. Reps whose `repKey` has no\n * counterpart in the other arm, and items present in only one arm, land in\n * the unpaired lists — reported, never truncated.\n *\n * Fail-loud: throws when either named arm has zero rows (an unknown arm\n * name would otherwise read as \"everything unpaired\"), when the two arm\n * names are equal, when a multi-rep `pairKey` has a row without `repKey`, or\n * when a (`pairKey`, arm) group repeats a `repKey` (the match would be\n * ambiguous).\n */\nexport function pairArms(rows: readonly PairedArmRow[], opts: PairArmsOptions): PairArmsResult {\n const { baselineArm, treatmentArm } = opts\n if (baselineArm === treatmentArm) {\n throw new ValidationError(\n `pairArms: baselineArm and treatmentArm are both '${baselineArm}' — an arm cannot be compared to itself`,\n )\n }\n\n // arm → pairKey → rows\n const byArm = new Map<string, Map<string, PairedArmRow[]>>()\n const armsSeen = new Set<string>()\n for (const row of rows) {\n armsSeen.add(row.arm)\n if (row.arm !== baselineArm && row.arm !== treatmentArm) continue\n const byKey = byArm.get(row.arm) ?? new Map<string, PairedArmRow[]>()\n const group = byKey.get(row.pairKey) ?? []\n group.push(row)\n byKey.set(row.pairKey, group)\n byArm.set(row.arm, byKey)\n }\n\n for (const arm of [baselineArm, treatmentArm]) {\n if (!byArm.has(arm)) {\n const seen = [...armsSeen].sort().join(', ') || '<none>'\n throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`)\n }\n }\n\n const baselineByKey = byArm.get(baselineArm)!\n const treatmentByKey = byArm.get(treatmentArm)!\n\n const allKeys = [...new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort()\n const pairs: MatchedPair[] = []\n const unpairedBaseline: PairedArmRow[] = []\n const unpairedTreatment: PairedArmRow[] = []\n for (const pairKey of allKeys) {\n const b = baselineByKey.get(pairKey) ?? []\n const t = treatmentByKey.get(pairKey) ?? []\n\n if (b.length <= 1 && t.length <= 1) {\n if (b.length === 1 && t.length === 1) {\n const baseline = b[0]!\n const treatment = t[0]!\n if (baseline.repKey !== undefined || treatment.repKey !== undefined) {\n if (\n baseline.repKey === undefined ||\n treatment.repKey === undefined ||\n baseline.repKey !== treatment.repKey\n ) {\n unpairedBaseline.push(baseline)\n unpairedTreatment.push(treatment)\n continue\n }\n }\n pairs.push({ pairKey, repIndex: 0, baseline, treatment })\n } else {\n unpairedBaseline.push(...b)\n unpairedTreatment.push(...t)\n }\n continue\n }\n\n const bByRep = indexByRepKey(b, pairKey, baselineArm)\n const tByRep = indexByRepKey(t, pairKey, treatmentArm)\n const repKeys = [...new Set([...bByRep.keys(), ...tByRep.keys()])].sort()\n let repIndex = 0\n for (const repKey of repKeys) {\n const baseline = bByRep.get(repKey)\n const treatment = tByRep.get(repKey)\n if (baseline !== undefined && treatment !== undefined) {\n pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment })\n } else if (baseline !== undefined) {\n unpairedBaseline.push(baseline)\n } else if (treatment !== undefined) {\n unpairedTreatment.push(treatment)\n }\n }\n }\n\n return { pairs, unpairedBaseline, unpairedTreatment }\n}\n\n/** Index a multi-rep (pairKey, arm) group by `repKey`, enforcing that every\n * row carries one and that no repKey repeats within the group. */\nfunction indexByRepKey(\n group: readonly PairedArmRow[],\n pairKey: string,\n arm: string,\n): Map<string, PairedArmRow> {\n const byRep = new Map<string, PairedArmRow>()\n for (const row of group) {\n if (row.repKey === undefined) {\n throw new ValidationError(\n `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' ` +\n `is missing repKey — multi-rep items require an explicit repKey on every row so reps ` +\n `pair by identity (pairing reps by outcome or by index would bias the paired statistics)`,\n )\n }\n if (byRep.has(row.repKey)) {\n throw new ValidationError(\n `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' — ` +\n `(pairKey, repKey) must uniquely identify a rep within an arm`,\n )\n }\n byRep.set(row.repKey, row)\n }\n return byRep\n}\n\n/** Paired pass/fail comparison over the pairs where BOTH sides carry `pass`. */\nexport interface PairedCorrectness {\n /** Discordant pairs where the treatment passed and the baseline failed. */\n b10: number\n /** Discordant pairs where the baseline passed and the treatment failed. */\n b01: number\n /** Exact McNemar significance over the paired outcomes (`b === b10`, `c === b01`). */\n mcnemar: McNemarResult\n /** Paired effect size: p(treatment) − p(baseline) with a paired-variance CI. */\n riskDifference: RiskDifferenceResult\n}\n\n/** Paired delta summary for one named metric (delta = treatment − baseline). */\nexport interface PairedMetricDelta {\n name: string\n /** Pairs where BOTH sides carry a finite value for this metric. */\n n: number\n /** Pairs where at least one side does not carry the metric. */\n nMissing: number\n /** Median paired delta, or null when `n === 0`. */\n medianDelta: number | null\n /** Mean paired delta, or null when `n === 0`. */\n meanDelta: number | null\n /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when\n * `n === 0` — a zero-width [0, 0] interval on no data would read as a\n * measured tight null. */\n bootstrapCi: PairedBootstrapResult | null\n /** Wilcoxon signed-rank test on the paired deltas; null when `n === 0`. */\n wilcoxon: { w: number; p: number } | null\n}\n\nexport interface ComparePairedArmsOptions extends PairArmsOptions {\n /** Metrics to compare. Default: every metric name observed on any matched\n * pair, sorted. A name that appears on no pair is still reported (with\n * `n = 0`) so a misspelled metric is visible instead of vanishing. */\n metricNames?: string[]\n /** Passed through to `pairedBootstrap` — set `seed` for reproducible CIs. */\n bootstrap?: PairedBootstrapOptions\n}\n\nexport interface PairedArmsComparison {\n nPairs: number\n nUnpairedBaseline: number\n nUnpairedTreatment: number\n /** null when no matched pair carries `pass` on both sides — a pass/fail\n * verdict over rows that never measured pass/fail would be fabricated. */\n correctness: PairedCorrectness | null\n metricDeltas: PairedMetricDelta[]\n}\n\n/**\n * Full matched-pair arm comparison: pair via {@link pairArms}, then compose\n * the paired estimators from `statistics` over the matched pairs.\n *\n * Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`\n * is that subset's size); each metric uses only the pairs where both sides\n * carry a finite value for it, with the remainder counted in `nMissing`.\n * Deltas are treatment − baseline throughout.\n *\n * Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a\n * non-finite metric value — silently treating corrupt telemetry as \"metric\n * absent\" would misreport it as missing coverage.\n */\nexport function comparePairedArms(\n rows: readonly PairedArmRow[],\n opts: ComparePairedArmsOptions,\n): PairedArmsComparison {\n const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts)\n\n let correctness: PairedCorrectness | null = null\n const baselinePass: number[] = []\n const treatmentPass: number[] = []\n for (const pair of pairs) {\n if (pair.baseline.pass === undefined || pair.treatment.pass === undefined) continue\n baselinePass.push(pair.baseline.pass ? 1 : 0)\n treatmentPass.push(pair.treatment.pass ? 1 : 0)\n }\n if (baselinePass.length > 0) {\n const mc = mcnemar(baselinePass, treatmentPass)\n correctness = {\n b10: mc.b,\n b01: mc.c,\n mcnemar: mc,\n riskDifference: pairedRiskDifference(baselinePass, treatmentPass),\n }\n }\n\n const metricNames =\n opts.metricNames ??\n [\n ...new Set(\n pairs.flatMap((p) => [\n ...Object.keys(p.baseline.metrics ?? {}),\n ...Object.keys(p.treatment.metrics ?? {}),\n ]),\n ),\n ].sort()\n\n const metricDeltas: PairedMetricDelta[] = metricNames.map((name) => {\n const before: number[] = []\n const after: number[] = []\n let nMissing = 0\n for (const pair of pairs) {\n const b = metricValue(pair.baseline, name)\n const t = metricValue(pair.treatment, name)\n if (b === undefined || t === undefined) {\n nMissing++\n continue\n }\n before.push(b)\n after.push(t)\n }\n const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap)\n return {\n name,\n n: before.length,\n nMissing,\n medianDelta: bootstrapCi?.median ?? null,\n meanDelta: bootstrapCi?.mean ?? null,\n bootstrapCi,\n wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after),\n }\n })\n\n return {\n nPairs: pairs.length,\n nUnpairedBaseline: unpairedBaseline.length,\n nUnpairedTreatment: unpairedTreatment.length,\n correctness,\n metricDeltas,\n }\n}\n\nexport interface MatchedRunRecordPair {\n pairKey: string\n repKey: string\n baseline: RunRecord\n treatment: RunRecord\n}\n\nexport interface PairRunRecordsResult {\n pairs: MatchedRunRecordPair[]\n unpairedBaseline: RunRecord[]\n unpairedTreatment: RunRecord[]\n}\n\ninterface RunRecordArmRow extends PairedArmRow {\n run: RunRecord\n repKey: string\n}\n\n/**\n * Pair two RunRecord arms by the identity of the evaluated work:\n * `(experimentId, scenarioId, seed)`.\n *\n * Falling back to array order, candidate id, or experiment id can compare\n * different tasks and fabricate lift. Duplicate identities throw.\n */\nexport function pairRunRecords(\n baselineRuns: readonly RunRecord[],\n treatmentRuns: readonly RunRecord[],\n): PairRunRecordsResult {\n const baselineRows = runRecordArmRows(baselineRuns, 'baseline')\n const treatmentRows = runRecordArmRows(treatmentRuns, 'treatment')\n validateRunRecordArmRows(baselineRows, 'baseline')\n validateRunRecordArmRows(treatmentRows, 'treatment')\n if (baselineRows.length === 0 || treatmentRows.length === 0) {\n return {\n pairs: [],\n unpairedBaseline: baselineRows.map((row) => row.run),\n unpairedTreatment: treatmentRows.map((row) => row.run),\n }\n }\n\n const result = pairArms([...baselineRows, ...treatmentRows], {\n baselineArm: 'baseline',\n treatmentArm: 'treatment',\n })\n return {\n pairs: result.pairs.map((pair) => {\n const baseline = pair.baseline as RunRecordArmRow\n const treatment = pair.treatment as RunRecordArmRow\n return {\n pairKey: pair.pairKey,\n repKey: baseline.repKey,\n baseline: baseline.run,\n treatment: treatment.run,\n }\n }),\n unpairedBaseline: result.unpairedBaseline.map((row) => (row as RunRecordArmRow).run),\n unpairedTreatment: result.unpairedTreatment.map((row) => (row as RunRecordArmRow).run),\n }\n}\n\nfunction runRecordArmRows(runs: readonly RunRecord[], arm: string): RunRecordArmRow[] {\n return runs.map((run) => {\n const scenarioId = run.scenarioId.trim()\n if (!scenarioId) {\n throw new ValidationError(\n `pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`,\n )\n }\n return {\n pairKey: JSON.stringify([run.experimentId, scenarioId]),\n repKey: String(run.seed),\n arm,\n run,\n }\n })\n}\n\nfunction validateRunRecordArmRows(rows: readonly RunRecordArmRow[], arm: string): void {\n const byPairKey = new Map<string, RunRecordArmRow[]>()\n for (const row of rows) {\n const group = byPairKey.get(row.pairKey) ?? []\n group.push(row)\n byPairKey.set(row.pairKey, group)\n }\n for (const [pairKey, group] of byPairKey) {\n if (group.length > 1) indexByRepKey(group, pairKey, arm)\n }\n}\n\nfunction metricValue(row: PairedArmRow, name: string): number | undefined {\n const v = row.metrics?.[name]\n if (v === undefined) return undefined\n if (!Number.isFinite(v)) {\n throw new ValidationError(\n `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`,\n )\n }\n return v\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuGA,SAAgB,SAAS,MAA+B,MAAuC;CAC7F,MAAM,EAAE,aAAa,iBAAiB;CACtC,IAAI,gBAAgB,cAClB,MAAM,IAAI,gBACR,oDAAoD,YAAY,wCAClE;CAIF,MAAM,wBAAQ,IAAI,IAAyC;CAC3D,MAAM,2BAAW,IAAI,IAAY;CACjC,KAAK,MAAM,OAAO,MAAM;EACtB,SAAS,IAAI,IAAI,GAAG;EACpB,IAAI,IAAI,QAAQ,eAAe,IAAI,QAAQ,cAAc;EACzD,MAAM,QAAQ,MAAM,IAAI,IAAI,GAAG,qBAAK,IAAI,IAA4B;EACpE,MAAM,QAAQ,MAAM,IAAI,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,KAAK,GAAG;EACd,MAAM,IAAI,IAAI,SAAS,KAAK;EAC5B,MAAM,IAAI,IAAI,KAAK,KAAK;CAC1B;CAEA,KAAK,MAAM,OAAO,CAAC,aAAa,YAAY,GAC1C,IAAI,CAAC,MAAM,IAAI,GAAG,GAEhB,MAAM,IAAI,gBAAgB,8BAA8B,IAAI,mBAD/C,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,KAAK,SACoC,EAAE;CAI1F,MAAM,gBAAgB,MAAM,IAAI,WAAW;CAC3C,MAAM,iBAAiB,MAAM,IAAI,YAAY;CAE7C,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;CACvF,MAAM,QAAuB,CAAC;CAC9B,MAAM,mBAAmC,CAAC;CAC1C,MAAM,oBAAoC,CAAC;CAC3C,KAAK,MAAM,WAAW,SAAS;EAC7B,MAAM,IAAI,cAAc,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,IAAI,eAAe,IAAI,OAAO,KAAK,CAAC;EAE1C,IAAI,EAAE,UAAU,KAAK,EAAE,UAAU,GAAG;GAClC,IAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;IACpC,MAAM,WAAW,EAAE;IACnB,MAAM,YAAY,EAAE;IACpB,IAAI,SAAS,WAAW,KAAA,KAAa,UAAU,WAAW,KAAA,GAEtD;SAAA,SAAS,WAAW,KAAA,KACpB,UAAU,WAAW,KAAA,KACrB,SAAS,WAAW,UAAU,QAC9B;MACA,iBAAiB,KAAK,QAAQ;MAC9B,kBAAkB,KAAK,SAAS;MAChC;KACF;;IAEF,MAAM,KAAK;KAAE;KAAS,UAAU;KAAG;KAAU;IAAU,CAAC;GAC1D,OAAO;IACL,iBAAiB,KAAK,GAAG,CAAC;IAC1B,kBAAkB,KAAK,GAAG,CAAC;GAC7B;GACA;EACF;EAEA,MAAM,SAAS,cAAc,GAAG,SAAS,WAAW;EACpD,MAAM,SAAS,cAAc,GAAG,SAAS,YAAY;EACrD,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,OAAO,KAAK,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;EACxE,IAAI,WAAW;EACf,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,WAAW,OAAO,IAAI,MAAM;GAClC,MAAM,YAAY,OAAO,IAAI,MAAM;GACnC,IAAI,aAAa,KAAA,KAAa,cAAc,KAAA,GAC1C,MAAM,KAAK;IAAE;IAAS,UAAU;IAAY;IAAU;GAAU,CAAC;QAC5D,IAAI,aAAa,KAAA,GACtB,iBAAiB,KAAK,QAAQ;QACzB,IAAI,cAAc,KAAA,GACvB,kBAAkB,KAAK,SAAS;EAEpC;CACF;CAEA,OAAO;EAAE;EAAO;EAAkB;CAAkB;AACtD;;;AAIA,SAAS,cACP,OACA,SACA,KAC2B;CAC3B,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,OAAO,OAAO;EACvB,IAAI,IAAI,WAAW,KAAA,GACjB,MAAM,IAAI,gBACR,sBAAsB,QAAQ,mDAAmD,IAAI,8KAGvF;EAEF,IAAI,MAAM,IAAI,IAAI,MAAM,GACtB,MAAM,IAAI,gBACR,+BAA+B,IAAI,OAAO,iBAAiB,QAAQ,YAAY,IAAI,iEAErF;EAEF,MAAM,IAAI,IAAI,QAAQ,GAAG;CAC3B;CACA,OAAO;AACT;;;;;;;;;;;;;;AAiEA,SAAgB,kBACd,MACA,MACsB;CACtB,MAAM,EAAE,OAAO,kBAAkB,sBAAsB,SAAS,MAAM,IAAI;CAE1E,IAAI,cAAwC;CAC5C,MAAM,eAAyB,CAAC;CAChC,MAAM,gBAA0B,CAAC;CACjC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,KAAK,SAAS,SAAS,KAAA,KAAa,KAAK,UAAU,SAAS,KAAA,GAAW;EAC3E,aAAa,KAAK,KAAK,SAAS,OAAO,IAAI,CAAC;EAC5C,cAAc,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;CAChD;CACA,IAAI,aAAa,SAAS,GAAG;EAC3B,MAAM,KAAK,QAAQ,cAAc,aAAa;EAC9C,cAAc;GACZ,KAAK,GAAG;GACR,KAAK,GAAG;GACR,SAAS;GACT,gBAAgB,qBAAqB,cAAc,aAAa;EAClE;CACF;CAaA,MAAM,gBAVJ,KAAK,eACL,CACE,GAAG,IAAI,IACL,MAAM,SAAS,MAAM,CACnB,GAAG,OAAO,KAAK,EAAE,SAAS,WAAW,CAAC,CAAC,GACvC,GAAG,OAAO,KAAK,EAAE,UAAU,WAAW,CAAC,CAAC,CAC1C,CAAC,CACH,CACF,CAAC,CAAC,KAAK,EAAA,CAE6C,KAAK,SAAS;EAClE,MAAM,SAAmB,CAAC;EAC1B,MAAM,QAAkB,CAAC;EACzB,IAAI,WAAW;EACf,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,IAAI,YAAY,KAAK,UAAU,IAAI;GACzC,MAAM,IAAI,YAAY,KAAK,WAAW,IAAI;GAC1C,IAAI,MAAM,KAAA,KAAa,MAAM,KAAA,GAAW;IACtC;IACA;GACF;GACA,OAAO,KAAK,CAAC;GACb,MAAM,KAAK,CAAC;EACd;EACA,MAAM,cAAc,OAAO,WAAW,IAAI,OAAO,gBAAgB,QAAQ,OAAO,KAAK,SAAS;EAC9F,OAAO;GACL;GACA,GAAG,OAAO;GACV;GACA,aAAa,aAAa,UAAU;GACpC,WAAW,aAAa,QAAQ;GAChC;GACA,UAAU,OAAO,WAAW,IAAI,OAAO,mBAAmB,QAAQ,KAAK;EACzE;CACF,CAAC;CAED,OAAO;EACL,QAAQ,MAAM;EACd,mBAAmB,iBAAiB;EACpC,oBAAoB,kBAAkB;EACtC;EACA;CACF;AACF;;;;;;;;AA2BA,SAAgB,eACd,cACA,eACsB;CACtB,MAAM,eAAe,iBAAiB,cAAc,UAAU;CAC9D,MAAM,gBAAgB,iBAAiB,eAAe,WAAW;CACjE,yBAAyB,cAAc,UAAU;CACjD,yBAAyB,eAAe,WAAW;CACnD,IAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GACxD,OAAO;EACL,OAAO,CAAC;EACR,kBAAkB,aAAa,KAAK,QAAQ,IAAI,GAAG;EACnD,mBAAmB,cAAc,KAAK,QAAQ,IAAI,GAAG;CACvD;CAGF,MAAM,SAAS,SAAS,CAAC,GAAG,cAAc,GAAG,aAAa,GAAG;EAC3D,aAAa;EACb,cAAc;CAChB,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,KAAK,SAAS;GAChC,MAAM,WAAW,KAAK;GACtB,MAAM,YAAY,KAAK;GACvB,OAAO;IACL,SAAS,KAAK;IACd,QAAQ,SAAS;IACjB,UAAU,SAAS;IACnB,WAAW,UAAU;GACvB;EACF,CAAC;EACD,kBAAkB,OAAO,iBAAiB,KAAK,QAAS,IAAwB,GAAG;EACnF,mBAAmB,OAAO,kBAAkB,KAAK,QAAS,IAAwB,GAAG;CACvF;AACF;AAEA,SAAS,iBAAiB,MAA4B,KAAgC;CACpF,OAAO,KAAK,KAAK,QAAQ;EACvB,MAAM,aAAa,IAAI,WAAW,KAAK;EACvC,IAAI,CAAC,YACH,MAAM,IAAI,gBACR,wBAAwB,IAAI,MAAM,+EACpC;EAEF,OAAO;GACL,SAAS,KAAK,UAAU,CAAC,IAAI,cAAc,UAAU,CAAC;GACtD,QAAQ,OAAO,IAAI,IAAI;GACvB;GACA;EACF;CACF,CAAC;AACH;AAEA,SAAS,yBAAyB,MAAkC,KAAmB;CACrF,MAAM,4BAAY,IAAI,IAA+B;CACrD,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,UAAU,IAAI,IAAI,OAAO,KAAK,CAAC;EAC7C,MAAM,KAAK,GAAG;EACd,UAAU,IAAI,IAAI,SAAS,KAAK;CAClC;CACA,KAAK,MAAM,CAAC,SAAS,UAAU,WAC7B,IAAI,MAAM,SAAS,GAAG,cAAc,OAAO,SAAS,GAAG;AAE3D;AAEA,SAAS,YAAY,KAAmB,MAAkC;CACxE,MAAM,IAAI,IAAI,UAAU;CACxB,IAAI,MAAM,KAAA,GAAW,OAAO,KAAA;CAC5B,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,gBACR,mDAAmD,KAAK,gBAAgB,IAAI,QAAQ,UAAU,IAAI,IAAI,MAAM,GAC9G;CAEF,OAAO;AACT"}
|
package/dist/pipelines/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { t as executionTrackByLane } from "../execution-tracks-CpgFPpS5.js";
|
|
2
|
-
import { E as pearsonR, u as interRaterReliability } from "../statistics-
|
|
2
|
+
import { E as pearsonR, u as interRaterReliability } from "../statistics-DWM_AyLe.js";
|
|
3
3
|
import { s as isToolSpan } from "../schema-CRhEY1SO.js";
|
|
4
4
|
import { t as buildTrajectory } from "../trajectory-D_7rLrvE.js";
|
|
5
5
|
import { i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, s as runFailureClass, t as aggregateLlm } from "../query-Di7eEQ79.js";
|
|
6
|
-
import { a as DEFAULT_RULES, i as computeToolUseMetrics, o as classifyFailure, t as compareToBaseline } from "../baseline-
|
|
6
|
+
import { a as DEFAULT_RULES, i as computeToolUseMetrics, o as classifyFailure, t as compareToBaseline } from "../baseline-DcX5hQDv.js";
|
|
7
7
|
//#region src/pipelines/budget-breach.ts
|
|
8
8
|
async function budgetBreachView(store, options = {}) {
|
|
9
9
|
const runs = await store.listRuns({
|