@tangle-network/agent-eval 0.41.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/campaign/index.d.ts +48 -368
- package/dist/campaign/index.js +67 -1
- package/dist/campaign/index.js.map +1 -1
- package/dist/{chunk-AU2JLNSZ.js → chunk-H4TOS272.js} +1 -65
- package/dist/chunk-H4TOS272.js.map +1 -0
- package/dist/{chunk-NKLGKF2Q.js → chunk-KQ26DYTQ.js} +2 -18
- package/dist/chunk-KQ26DYTQ.js.map +1 -0
- package/dist/{chunk-EGIPWXHL.js → chunk-MNL6LXGQ.js} +98 -2
- package/dist/chunk-MNL6LXGQ.js.map +1 -0
- package/dist/{chunk-5U2DOJU4.js → chunk-N4SBKEPJ.js} +199 -2
- package/dist/chunk-N4SBKEPJ.js.map +1 -0
- package/dist/{chunk-LCIDRYGP.js → chunk-PD3MH6WU.js} +8 -8
- package/dist/{control-CmLJk3IG.d.ts → control-ojEWkMfJ.d.ts} +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/{feedback-trajectory-Dvy-bt7x.d.ts → feedback-trajectory-BSxqEpu7.d.ts} +1 -1
- package/dist/index.d.ts +227 -687
- package/dist/index.js +753 -1237
- package/dist/index.js.map +1 -1
- package/dist/integrity-CTDhR1Sg.d.ts +81 -0
- package/dist/llm-client-BXVRUZyX.d.ts +234 -0
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +67 -3
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{integrity-DYR5gWlb.d.ts → raw-provider-sink-C46HDghv.d.ts} +1 -80
- package/dist/{release-report-Di84bXD7.d.ts → release-report-BtpgWRI0.d.ts} +21 -3
- package/dist/reporting.d.ts +2 -3
- package/dist/reporting.js +4 -8
- package/dist/{researcher-DeZ_EArp.d.ts → researcher-CoJMs2Iz.d.ts} +116 -205
- package/dist/rl.d.ts +103 -221
- package/dist/rl.js +44 -199
- package/dist/rl.js.map +1 -1
- package/dist/sequential-DdV5ShjT.d.ts +561 -0
- package/dist/traces.d.ts +3 -2
- package/dist/traces.js +5 -5
- package/dist/types-BLbRTxoc.d.ts +367 -0
- package/dist/wire/index.d.ts +1 -1
- package/package.json +1 -6
- package/dist/chunk-5U2DOJU4.js.map +0 -1
- package/dist/chunk-AU2JLNSZ.js.map +0 -1
- package/dist/chunk-DMW5VENN.js +0 -1412
- package/dist/chunk-DMW5VENN.js.map +0 -1
- package/dist/chunk-EGIPWXHL.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-NKLGKF2Q.js.map +0 -1
- package/dist/multi-layer-verifier-BNi4-8lR.d.ts +0 -141
- package/dist/optimization.d.ts +0 -11
- package/dist/optimization.js +0 -71
- package/dist/optimization.js.map +0 -1
- package/dist/sequential-5iSVfzl2.d.ts +0 -139
- package/dist/summary-report-DuZXOk7K.d.ts +0 -917
- /package/dist/{chunk-LCIDRYGP.js.map → chunk-PD3MH6WU.js.map} +0 -0
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/feedback-trajectory.ts","../src/held-out-gate.ts","../src/pareto.ts","../src/prompt-evolution.ts","../src/multi-shot-optimization.ts","../src/reflective-mutation.ts","../src/researcher.ts"],"sourcesContent":["import type { ControlEvalResult, ControlRunResult, ControlStep } from './control-runtime'\nimport type { DatasetScenario, DatasetSplit } from './dataset'\n\nexport type FeedbackArtifactType =\n | 'text'\n | 'code'\n | 'plan'\n | 'research'\n | 'action'\n | 'ui'\n | 'decision'\n | 'data'\n | 'other'\n\nexport type FeedbackLabelSource = 'user' | 'judge' | 'environment' | 'metric' | 'policy' | 'system'\n\nexport type FeedbackLabelKind =\n | 'approve'\n | 'reject'\n | 'select'\n | 'edit'\n | 'rank'\n | 'rate'\n | 'comment'\n | 'metric_outcome'\n | 'policy_block'\n | 'revision_request'\n\nexport type FeedbackSeverity = 'info' | 'warning' | 'error' | 'critical'\n\nexport interface FeedbackTask {\n intent: string\n context?: unknown\n}\n\nexport interface ProposedSideEffect {\n type: string\n risk?: 'low' | 'medium' | 'high'\n costUsd?: number\n externalSideEffect?: boolean\n requiresApproval?: boolean\n metadata?: Record<string, unknown>\n}\n\nexport interface FeedbackLabel {\n id?: string\n source: FeedbackLabelSource\n kind: FeedbackLabelKind\n value: unknown\n reason?: string\n severity?: FeedbackSeverity\n createdAt: string\n metadata?: Record<string, unknown>\n}\n\nexport interface FeedbackAttempt {\n id: string\n stepIndex: number\n artifactType: FeedbackArtifactType\n artifact: unknown\n options?: unknown[]\n proposedAction?: ProposedSideEffect\n evals?: ControlEvalResult[]\n feedback?: FeedbackLabel[]\n createdAt: string\n metadata?: Record<string, unknown>\n}\n\nexport interface FeedbackOutcome {\n success?: boolean\n score?: number\n metrics?: Record<string, number>\n costUsd?: number\n detail?: string\n observedAt?: string\n metadata?: Record<string, unknown>\n}\n\nexport interface FeedbackTrajectory {\n id: string\n projectId?: string\n scenarioId?: string\n task: FeedbackTask\n attempts: FeedbackAttempt[]\n labels: FeedbackLabel[]\n outcome?: FeedbackOutcome\n split?: DatasetSplit\n tags?: Record<string, string>\n createdAt: string\n updatedAt?: string\n metadata?: Record<string, unknown>\n}\n\nexport interface FeedbackTrajectoryStore {\n save(trajectory: FeedbackTrajectory): Promise<void>\n get(id: string): Promise<FeedbackTrajectory | null>\n list(filter?: FeedbackTrajectoryFilter): Promise<FeedbackTrajectory[]>\n appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory>\n appendLabel(id: string, label: FeedbackLabel, attemptId?: string): Promise<FeedbackTrajectory>\n}\n\nexport interface FeedbackTrajectoryFilter {\n projectId?: string\n scenarioId?: string\n split?: DatasetSplit\n tag?: [string, string]\n}\n\nexport interface FeedbackSplitPolicy {\n trainPct?: number\n devPct?: number\n testPct?: number\n holdoutPct?: number\n}\n\nexport interface PreferenceMemoryEntry {\n instruction: string\n rationale: string\n weight: number\n sourceTrajectoryId: string\n sourceLabelId?: string\n category?: string\n}\n\nexport interface FeedbackOptimizerRow {\n scenarioId: string\n trajectoryId: string\n labelKinds: FeedbackLabelKind[]\n score?: number\n metadata?: Record<string, unknown>\n}\n\nexport interface FeedbackReplayResult {\n trajectoryId: string\n pass: boolean\n score?: number\n labels: FeedbackLabel[]\n outcome?: FeedbackOutcome\n metadata?: Record<string, unknown>\n}\n\nexport interface FeedbackReplayAdapter {\n replay(\n trajectory: FeedbackTrajectory,\n ):\n | Promise<Omit<FeedbackReplayResult, 'trajectoryId'>>\n | Omit<FeedbackReplayResult, 'trajectoryId'>\n}\n\nconst DEFAULT_SPLIT_POLICY: Required<FeedbackSplitPolicy> = {\n trainPct: 70,\n devPct: 15,\n testPct: 10,\n holdoutPct: 5,\n}\n\nexport class InMemoryFeedbackTrajectoryStore implements FeedbackTrajectoryStore {\n private readonly trajectories = new Map<string, FeedbackTrajectory>()\n\n async save(trajectory: FeedbackTrajectory): Promise<void> {\n this.trajectories.set(trajectory.id, cloneTrajectory(trajectory))\n }\n\n async get(id: string): Promise<FeedbackTrajectory | null> {\n const trajectory = this.trajectories.get(id)\n return trajectory ? cloneTrajectory(trajectory) : null\n }\n\n async list(filter: FeedbackTrajectoryFilter = {}): Promise<FeedbackTrajectory[]> {\n return [...this.trajectories.values()]\n .filter((trajectory) => matchesFilter(trajectory, filter))\n .map(cloneTrajectory)\n }\n\n async appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory> {\n const trajectory = this.trajectories.get(id)\n if (!trajectory)\n throw new Error(`FeedbackTrajectoryStore.appendAttempt: unknown trajectory \"${id}\"`)\n const next = cloneTrajectory({\n ...trajectory,\n attempts: [...trajectory.attempts, attempt],\n updatedAt: attempt.createdAt,\n })\n this.trajectories.set(id, next)\n return cloneTrajectory(next)\n }\n\n async appendLabel(\n id: string,\n label: FeedbackLabel,\n attemptId?: string,\n ): Promise<FeedbackTrajectory> {\n const trajectory = this.trajectories.get(id)\n if (!trajectory)\n throw new Error(`FeedbackTrajectoryStore.appendLabel: unknown trajectory \"${id}\"`)\n const attempts = attemptId\n ? trajectory.attempts.map((attempt) =>\n attempt.id === attemptId\n ? { ...attempt, feedback: [...(attempt.feedback ?? []), label] }\n : attempt,\n )\n : trajectory.attempts\n const next = cloneTrajectory({\n ...trajectory,\n attempts,\n labels: attemptId ? trajectory.labels : [...trajectory.labels, label],\n updatedAt: label.createdAt,\n })\n this.trajectories.set(id, next)\n return cloneTrajectory(next)\n }\n}\n\nexport class FileSystemFeedbackTrajectoryStore implements FeedbackTrajectoryStore {\n private readonly dir: string\n private readonly memory = new InMemoryFeedbackTrajectoryStore()\n private loaded = false\n\n constructor(options: { dir: string }) {\n this.dir = options.dir\n }\n\n async save(trajectory: FeedbackTrajectory): Promise<void> {\n await this.load()\n await this.memory.save(trajectory)\n await this.append({ op: 'save', trajectory })\n }\n\n async get(id: string): Promise<FeedbackTrajectory | null> {\n await this.load()\n return this.memory.get(id)\n }\n\n async list(filter: FeedbackTrajectoryFilter = {}): Promise<FeedbackTrajectory[]> {\n await this.load()\n return this.memory.list(filter)\n }\n\n async appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory> {\n await this.load()\n const next = await this.memory.appendAttempt(id, attempt)\n await this.append({ op: 'appendAttempt', id, attempt })\n return next\n }\n\n async appendLabel(\n id: string,\n label: FeedbackLabel,\n attemptId?: string,\n ): Promise<FeedbackTrajectory> {\n await this.load()\n const next = await this.memory.appendLabel(id, label, attemptId)\n await this.append({ op: 'appendLabel', id, label, attemptId })\n return next\n }\n\n private async append(record: unknown): Promise<void> {\n const { appendFile, mkdir } = await import('node:fs/promises')\n const { join } = await import('node:path')\n await mkdir(this.dir, { recursive: true })\n await appendFile(\n join(this.dir, 'feedback-trajectories.ndjson'),\n `${JSON.stringify(record)}\\n`,\n 'utf8',\n )\n }\n\n private async load(): Promise<void> {\n if (this.loaded) return\n const { readFile } = await import('node:fs/promises')\n const { join } = await import('node:path')\n const file = join(this.dir, 'feedback-trajectories.ndjson')\n try {\n const raw = await readFile(file, 'utf8')\n for (const line of raw.split('\\n')) {\n if (!line.trim()) continue\n try {\n const record = JSON.parse(line) as\n | { op: 'save'; trajectory: FeedbackTrajectory }\n | { op: 'appendAttempt'; id: string; attempt: FeedbackAttempt }\n | { op: 'appendLabel'; id: string; label: FeedbackLabel; attemptId?: string }\n if (record.op === 'save') await this.memory.save(record.trajectory)\n if (record.op === 'appendAttempt')\n await this.memory.appendAttempt(record.id, record.attempt)\n if (record.op === 'appendLabel')\n await this.memory.appendLabel(record.id, record.label, record.attemptId)\n } catch {\n /* corrupt records are skipped so one bad line does not discard the corpus */\n }\n }\n } catch {\n /* first run */\n }\n this.loaded = true\n }\n}\n\nexport function createFeedbackTrajectory(input: {\n id?: string\n projectId?: string\n scenarioId?: string\n task: FeedbackTask\n attempts?: FeedbackAttempt[]\n labels?: FeedbackLabel[]\n outcome?: FeedbackOutcome\n split?: DatasetSplit\n tags?: Record<string, string>\n createdAt?: string\n metadata?: Record<string, unknown>\n}): FeedbackTrajectory {\n const createdAt = input.createdAt ?? new Date().toISOString()\n const id =\n input.id ??\n `ft_${stableHash(`${input.projectId ?? ''}|${input.scenarioId ?? ''}|${input.task.intent}|${createdAt}`).toString(16)}`\n return {\n id,\n projectId: input.projectId,\n scenarioId: input.scenarioId,\n task: input.task,\n attempts: input.attempts ?? [],\n labels: input.labels ?? [],\n outcome: input.outcome,\n split: input.split,\n tags: input.tags,\n createdAt,\n metadata: input.metadata,\n }\n}\n\nexport function assignFeedbackSplit(\n trajectory: Pick<FeedbackTrajectory, 'id' | 'projectId' | 'scenarioId' | 'task'>,\n policy: FeedbackSplitPolicy = {},\n): DatasetSplit {\n const split = { ...DEFAULT_SPLIT_POLICY, ...policy }\n const total = split.trainPct + split.devPct + split.testPct + split.holdoutPct\n if (total <= 0) throw new Error('assignFeedbackSplit: split percentages must sum above zero')\n const bucket =\n stableHash(\n `${trajectory.projectId ?? ''}|${trajectory.scenarioId ?? ''}|${trajectory.id}|${trajectory.task.intent}`,\n ) % total\n if (bucket < split.trainPct) return 'train'\n if (bucket < split.trainPct + split.devPct) return 'dev'\n if (bucket < split.trainPct + split.devPct + split.testPct) return 'test'\n return 'holdout'\n}\n\nexport function withAssignedFeedbackSplit(\n trajectory: FeedbackTrajectory,\n policy?: FeedbackSplitPolicy,\n): FeedbackTrajectory {\n return {\n ...trajectory,\n split: trajectory.split ?? assignFeedbackSplit(trajectory, policy),\n }\n}\n\nexport function feedbackTrajectoryToDatasetScenario(\n trajectory: FeedbackTrajectory,\n): DatasetScenario {\n const withSplit = withAssignedFeedbackSplit(trajectory)\n return {\n id: withSplit.scenarioId ?? withSplit.id,\n split: withSplit.split,\n payload: withSplit,\n tags: {\n ...(withSplit.projectId ? { projectId: withSplit.projectId } : {}),\n ...(withSplit.tags ?? {}),\n source: 'feedback-trajectory',\n },\n }\n}\n\nexport function feedbackTrajectoriesToDatasetScenarios(\n trajectories: FeedbackTrajectory[],\n): DatasetScenario[] {\n return trajectories.map(feedbackTrajectoryToDatasetScenario)\n}\n\nexport function feedbackTrajectoryToOptimizerRow(\n trajectory: FeedbackTrajectory,\n): FeedbackOptimizerRow {\n const labels = allLabels(trajectory)\n return {\n scenarioId: trajectory.scenarioId ?? trajectory.id,\n trajectoryId: trajectory.id,\n labelKinds: [...new Set(labels.map((label) => label.kind))],\n score: trajectory.outcome?.score ?? scoreFromLabels(labels),\n metadata: {\n projectId: trajectory.projectId,\n split: trajectory.split,\n intent: trajectory.task.intent,\n attempts: trajectory.attempts.length,\n outcome: trajectory.outcome,\n labels,\n },\n }\n}\n\nexport function feedbackTrajectoriesToOptimizerRows(\n trajectories: FeedbackTrajectory[],\n): FeedbackOptimizerRow[] {\n return trajectories.map(feedbackTrajectoryToOptimizerRow)\n}\n\nexport async function replayFeedbackTrajectory(\n trajectory: FeedbackTrajectory,\n adapter: FeedbackReplayAdapter,\n): Promise<FeedbackReplayResult> {\n try {\n const result = await adapter.replay(trajectory)\n return {\n trajectoryId: trajectory.id,\n ...result,\n }\n } catch (err) {\n const createdAt = new Date().toISOString()\n const message = err instanceof Error ? err.message : String(err)\n return {\n trajectoryId: trajectory.id,\n pass: false,\n labels: [\n {\n source: 'system',\n kind: 'reject',\n value: false,\n reason: message,\n severity: 'error',\n createdAt,\n },\n ],\n outcome: {\n success: false,\n score: 0,\n detail: message,\n observedAt: createdAt,\n },\n metadata: { replayError: true },\n }\n }\n}\n\nexport async function replayFeedbackTrajectories(\n trajectories: FeedbackTrajectory[],\n adapter: FeedbackReplayAdapter,\n): Promise<FeedbackReplayResult[]> {\n const results: FeedbackReplayResult[] = []\n for (const trajectory of trajectories) {\n results.push(await replayFeedbackTrajectory(trajectory, adapter))\n }\n return results\n}\n\nexport function summarizePreferenceMemory(\n trajectories: FeedbackTrajectory[],\n options: { maxEntries?: number } = {},\n): PreferenceMemoryEntry[] {\n const maxEntries = options.maxEntries ?? 20\n const entries: PreferenceMemoryEntry[] = []\n for (const trajectory of trajectories) {\n for (const label of allLabels(trajectory)) {\n const instruction = instructionFromLabel(trajectory, label)\n if (!instruction) continue\n entries.push({\n instruction,\n rationale: label.reason ?? `${label.kind} label from ${label.source}`,\n weight: weightForLabel(label),\n sourceTrajectoryId: trajectory.id,\n sourceLabelId: label.id,\n category: label.kind,\n })\n }\n }\n\n const byInstruction = new Map<string, PreferenceMemoryEntry>()\n for (const entry of entries) {\n const key = entry.instruction.toLowerCase().replace(/\\s+/g, ' ').trim()\n const existing = byInstruction.get(key)\n if (!existing || entry.weight > existing.weight) byInstruction.set(key, entry)\n }\n return [...byInstruction.values()].sort((a, b) => b.weight - a.weight).slice(0, maxEntries)\n}\n\nexport function renderPreferenceMemoryMarkdown(entries: PreferenceMemoryEntry[]): string {\n const lines = ['# Preference Memory', '']\n for (const entry of entries) {\n lines.push(`- ${entry.instruction}`)\n lines.push(` Rationale: ${entry.rationale}`)\n lines.push(` Source: ${entry.sourceTrajectoryId}`)\n lines.push('')\n }\n return `${lines.join('\\n').trim()}\\n`\n}\n\nexport function serializeFeedbackTrajectoriesJsonl(trajectories: FeedbackTrajectory[]): string {\n return `${trajectories\n .slice()\n .sort((a, b) => a.id.localeCompare(b.id))\n .map((trajectory) => JSON.stringify(canonicalize(trajectory)))\n .join('\\n')}\\n`\n}\n\nexport function parseFeedbackTrajectoriesJsonl(jsonl: string): FeedbackTrajectory[] {\n const trajectories: FeedbackTrajectory[] = []\n for (const line of jsonl.split('\\n')) {\n if (!line.trim()) continue\n trajectories.push(JSON.parse(line) as FeedbackTrajectory)\n }\n return trajectories\n}\n\nexport function controlRunToFeedbackTrajectory<TState, TAction, TActionResult>(\n run: ControlRunResult<TState, TAction, TActionResult>,\n options: {\n projectId?: string\n scenarioId?: string\n artifactType?: FeedbackArtifactType\n artifactFromStep?: (step: ControlStep<TState, TAction, TActionResult>) => unknown\n proposedActionFromStep?: (\n step: ControlStep<TState, TAction, TActionResult>,\n ) => ProposedSideEffect | undefined\n createdAt?: string\n } = {},\n): FeedbackTrajectory {\n const createdAt = options.createdAt ?? new Date().toISOString()\n const trajectoryId =\n run.runId ?? `ft_control_${stableHash(`${run.intent}|${createdAt}`).toString(16)}`\n return createFeedbackTrajectory({\n id: trajectoryId,\n projectId: options.projectId,\n scenarioId: options.scenarioId,\n task: { intent: run.intent },\n createdAt,\n attempts: run.steps.map((step) => ({\n id: `${trajectoryId}_step_${step.index}`,\n stepIndex: step.index,\n artifactType: options.artifactType ?? 'action',\n artifact: options.artifactFromStep?.(step) ?? step.actionOutcome?.result ?? step.decision,\n proposedAction: options.proposedActionFromStep?.(step),\n evals: step.evalsAfter,\n createdAt: step.startedAt,\n metadata: {\n decision: step.decision,\n actionOutcome: step.actionOutcome,\n },\n })),\n labels: [\n {\n source: 'system',\n kind: run.pass ? 'approve' : 'reject',\n value: run.pass,\n reason: run.reason,\n severity: run.pass ? 'info' : 'error',\n createdAt,\n },\n ],\n outcome: {\n success: run.pass,\n score: run.score,\n costUsd: run.spentCostUsd,\n detail: run.reason,\n observedAt: createdAt,\n metadata: {\n stoppedBy: run.stoppedBy,\n failureClass: run.failureClass,\n },\n },\n })\n}\n\nfunction allLabels(trajectory: FeedbackTrajectory): FeedbackLabel[] {\n const labels = [\n ...trajectory.labels,\n ...trajectory.attempts.flatMap((attempt) => attempt.feedback ?? []),\n ]\n const seen = new Set<string>()\n return labels.filter((label) => {\n const key =\n label.id ?? `${label.source}|${label.kind}|${label.createdAt}|${JSON.stringify(label.value)}`\n if (seen.has(key)) return false\n seen.add(key)\n return true\n })\n}\n\nfunction scoreFromLabels(labels: FeedbackLabel[]): number | undefined {\n if (!labels.length) return undefined\n const scored = labels\n .map((label) => {\n if (label.kind === 'approve' || label.kind === 'select') return 1\n if (label.kind === 'reject' || label.kind === 'policy_block') return 0\n if (label.kind === 'rate' && typeof label.value === 'number')\n return Math.max(0, Math.min(1, label.value))\n return undefined\n })\n .filter((value): value is number => typeof value === 'number')\n if (!scored.length) return undefined\n return Math.round((scored.reduce((sum, value) => sum + value, 0) / scored.length) * 1000) / 1000\n}\n\nfunction instructionFromLabel(\n trajectory: FeedbackTrajectory,\n label: FeedbackLabel,\n): string | undefined {\n if (label.kind === 'reject' && label.reason)\n return `Avoid outputs like \"${compact(trajectory.task.intent, 80)}\" when: ${label.reason}`\n if (label.kind === 'revision_request' && label.reason)\n return `Revise similar work by applying: ${label.reason}`\n if (label.kind === 'select' && label.reason)\n return `Prefer selected options for \"${compact(trajectory.task.intent, 80)}\" because: ${label.reason}`\n if (label.kind === 'approve' && label.reason)\n return `Repeat the pattern approved for \"${compact(trajectory.task.intent, 80)}\": ${label.reason}`\n if (label.kind === 'comment' && label.reason) return label.reason\n return undefined\n}\n\nfunction weightForLabel(label: FeedbackLabel): number {\n const severity =\n label.severity === 'critical'\n ? 4\n : label.severity === 'error'\n ? 3\n : label.severity === 'warning'\n ? 2\n : 1\n const source =\n label.source === 'user'\n ? 3\n : label.source === 'metric' || label.source === 'environment'\n ? 2\n : 1\n return severity * source\n}\n\nfunction matchesFilter(trajectory: FeedbackTrajectory, filter: FeedbackTrajectoryFilter): boolean {\n if (filter.projectId && trajectory.projectId !== filter.projectId) return false\n if (filter.scenarioId && trajectory.scenarioId !== filter.scenarioId) return false\n if (filter.split && trajectory.split !== filter.split) return false\n if (filter.tag) {\n const [key, value] = filter.tag\n if (trajectory.tags?.[key] !== value) return false\n }\n return true\n}\n\nfunction cloneTrajectory(trajectory: FeedbackTrajectory): FeedbackTrajectory {\n return JSON.parse(JSON.stringify(trajectory)) as FeedbackTrajectory\n}\n\nfunction compact(value: string, max: number): string {\n const normalized = value.replace(/\\s+/g, ' ').trim()\n return normalized.length > max ? `${normalized.slice(0, max).trim()}...` : normalized\n}\n\nfunction stableHash(input: string): number {\n let hash = 2166136261\n for (let i = 0; i < input.length; i += 1) {\n hash ^= input.charCodeAt(i)\n hash = Math.imul(hash, 16777619)\n }\n return hash >>> 0\n}\n\nfunction canonicalize(value: unknown): unknown {\n if (value === null || typeof value !== 'object') return value\n if (Array.isArray(value)) return value.map(canonicalize)\n const out: Record<string, unknown> = {}\n for (const key of Object.keys(value as Record<string, unknown>).sort()) {\n out[key] = canonicalize((value as Record<string, unknown>)[key])\n }\n return out\n}\n","/**\n * HeldOutGate — first-class held-out paired-delta promotion gate.\n *\n * Encodes the \"honesty override\" pattern that lived inline in\n * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.\n * The optimizer's best-guess is one thing; what we should actually\n * ship is another. The gate is the line between them.\n *\n * A candidate is promoted iff ALL three pass:\n *\n * 1. **Productive runs**: the candidate has at least\n * `minProductiveRuns` paired observations on items where BOTH\n * candidate and baseline produced a real (non-silent) score.\n * 2. **Paired delta**: the lower bound of the bootstrap CI on the\n * median per-item delta (candidate − baseline) on the HOLDOUT\n * split is strictly greater than `pairedDeltaThreshold`.\n * 3. **Overfit gap**: the candidate's gap between search-split\n * score and holdout-split score is no worse (more positive)\n * than the baseline's gap by more than `overfitGapThreshold`.\n * \"Better on search, worse on holdout\" is the canonical\n * overfit pattern; this catches it.\n *\n * The decision carries a machine-readable `rejectionCode` plus an\n * `evidence` block with every number the gate looked at, so the\n * downstream researcher / paper / dashboard can re-derive the\n * verdict without re-running.\n *\n * See also:\n * - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`\n * - `src/run-record.ts` for the input row schema\n * - `src/reference-replay.ts` for the older, reference-replay-\n * specific promotion path (still useful for replay-style evals).\n */\n\nimport type { RunRecord } from './run-record'\nimport { pairedBootstrap, wilcoxonSignedRank } from './statistics'\n\nexport type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap'\n\nexport interface HeldOutGateConfig {\n /** Minimum number of paired (candidate, baseline) holdout observations\n * required before the gate will even consider promoting. Default 3. */\n minProductiveRuns?: number\n /** The bootstrap-CI lower bound on the median paired holdout delta\n * must exceed this to promote. Default 0. */\n pairedDeltaThreshold?: number\n /** Maximum allowed worsening of (search − holdout) gap relative to\n * baseline. Default 0.15 (i.e. candidate may overfit by up to 15\n * absolute score points more than baseline before rejection). */\n overfitGapThreshold?: number\n /** Stable label of the baseline candidate. Required — paper-grade\n * evaluation never compares two unlabelled candidates. */\n baselineKey: string\n /** Confidence level for the bootstrap CI. Default 0.95. */\n confidence?: number\n /** Bootstrap resamples. Default 2000. */\n bootstrapResamples?: number\n /** Optional deterministic seed for the bootstrap. Default undefined\n * (Math.random). */\n seed?: number\n}\n\nexport interface GateEvidence {\n /** Number of paired (candidate, baseline) holdout observations used. */\n productiveRuns: number\n /** Median of (candidate − baseline) paired holdout deltas. */\n medianPairedDelta: number\n /** Bootstrap CI on the median paired holdout delta. */\n pairedCI: { low: number; high: number }\n /** Wilcoxon signed-rank p-value on the paired holdout deltas. */\n pairedPValue: number\n /** Mean candidate score on the search split (NaN if none). */\n searchScore: number\n /** Mean candidate score on the holdout split (NaN if none). */\n holdoutScore: number\n /** Candidate (search − holdout) gap. */\n overfitGap: number\n /** Baseline (search − holdout) gap. */\n baselineOverfitGap: number\n}\n\nexport interface GateDecision {\n /** Final promote/no-promote verdict. */\n promote: boolean\n /** The candidate that was evaluated. */\n candidateId: string\n /** The baseline it was compared against. */\n baselineId: string\n /** Every number the gate looked at, for audit + paper export. */\n evidence: GateEvidence\n /** Human-readable reason. */\n reason: string\n /** Machine-readable rejection code, or null on promote. */\n rejectionCode: HeldOutGateRejectionCode | null\n}\n\n/**\n * Held-out paired-delta promotion gate. Construct once with config,\n * call `evaluate(candidateRuns, baselineRuns)` per (candidate,\n * baseline) pair. Stateless across calls.\n */\nexport class HeldOutGate {\n private readonly minProductiveRuns: number\n private readonly pairedDeltaThreshold: number\n private readonly overfitGapThreshold: number\n private readonly baselineKey: string\n private readonly confidence: number\n private readonly resamples: number\n private readonly seed?: number\n\n constructor(config: HeldOutGateConfig) {\n if (!config.baselineKey) {\n throw new Error('HeldOutGate: baselineKey is required')\n }\n this.minProductiveRuns = config.minProductiveRuns ?? 3\n this.pairedDeltaThreshold = config.pairedDeltaThreshold ?? 0\n this.overfitGapThreshold = config.overfitGapThreshold ?? 0.15\n this.baselineKey = config.baselineKey\n this.confidence = config.confidence ?? 0.95\n this.resamples = config.bootstrapResamples ?? 2000\n this.seed = config.seed\n }\n\n /** Decide whether `candidate` should replace `baseline`. Pairing\n * is by (experimentId, seed) — identical experiment + seed pairs\n * the candidate run with the matching baseline run. Pairs without\n * a holdout score on both sides are dropped. */\n evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision {\n const candidateId = inferCandidateId(candidate, this.baselineKey)\n const baselineId = this.baselineKey\n\n // Pair holdout runs by (experimentId, seed).\n const baselineHoldoutByKey = indexHoldoutByKey(baseline)\n const beforeHoldout: number[] = []\n const afterHoldout: number[] = []\n for (const run of candidate) {\n if (run.splitTag !== 'holdout') continue\n if (run.outcome.holdoutScore === undefined) continue\n const key = pairKey(run)\n const counterpart = baselineHoldoutByKey.get(key)\n if (counterpart === undefined) continue\n beforeHoldout.push(counterpart)\n afterHoldout.push(run.outcome.holdoutScore)\n }\n\n const productiveRuns = beforeHoldout.length\n\n // Always compute the gap numbers — useful even when we reject on\n // few_runs (you want to see why).\n const candidateSearchMean = mean(scores(candidate, 'searchScore', 'search'))\n const candidateHoldoutMean = mean(scores(candidate, 'holdoutScore', 'holdout'))\n const baselineSearchMean = mean(scores(baseline, 'searchScore', 'search'))\n const baselineHoldoutMean = mean(scores(baseline, 'holdoutScore', 'holdout'))\n\n const overfitGap = safeDiff(candidateSearchMean, candidateHoldoutMean)\n const baselineOverfitGap = safeDiff(baselineSearchMean, baselineHoldoutMean)\n\n // Few-runs gate.\n if (productiveRuns < this.minProductiveRuns) {\n return {\n promote: false,\n candidateId,\n baselineId,\n evidence: {\n productiveRuns,\n medianPairedDelta: productiveRuns > 0 ? medianDelta(beforeHoldout, afterHoldout) : 0,\n pairedCI: { low: 0, high: 0 },\n pairedPValue: 1,\n searchScore: candidateSearchMean,\n holdoutScore: candidateHoldoutMean,\n overfitGap,\n baselineOverfitGap,\n },\n reason: `few_runs: ${productiveRuns} paired holdout observation(s) < min ${this.minProductiveRuns}`,\n rejectionCode: 'few_runs',\n }\n }\n\n // Paired bootstrap on holdout deltas.\n const ci = pairedBootstrap(beforeHoldout, afterHoldout, {\n confidence: this.confidence,\n resamples: this.resamples,\n statistic: 'median',\n seed: this.seed,\n })\n const wilcoxon = wilcoxonSignedRank(beforeHoldout, afterHoldout)\n\n const evidence: GateEvidence = {\n productiveRuns,\n medianPairedDelta: ci.median,\n pairedCI: { low: ci.low, high: ci.high },\n pairedPValue: wilcoxon.p,\n searchScore: candidateSearchMean,\n holdoutScore: candidateHoldoutMean,\n overfitGap,\n baselineOverfitGap,\n }\n\n // Negative-delta gate (CI lower bound must clear the threshold).\n if (!(ci.low > this.pairedDeltaThreshold)) {\n return {\n promote: false,\n candidateId,\n baselineId,\n evidence,\n reason:\n `negative_delta: paired holdout median Δ=${fmt(ci.median)} ` +\n `CI=[${fmt(ci.low)}, ${fmt(ci.high)}] does not clear threshold ${fmt(this.pairedDeltaThreshold)}`,\n rejectionCode: 'negative_delta',\n }\n }\n\n // Overfit-gap gate. We allow some absolute slack —\n // candidate.gap ≤ baseline.gap + overfitGapThreshold.\n if (\n Number.isFinite(overfitGap) &&\n Number.isFinite(baselineOverfitGap) &&\n overfitGap > baselineOverfitGap + this.overfitGapThreshold\n ) {\n return {\n promote: false,\n candidateId,\n baselineId,\n evidence,\n reason:\n `overfit_gap: candidate gap=${fmt(overfitGap)} exceeds baseline gap=${fmt(baselineOverfitGap)} ` +\n `by more than ${fmt(this.overfitGapThreshold)}`,\n rejectionCode: 'overfit_gap',\n }\n }\n\n return {\n promote: true,\n candidateId,\n baselineId,\n evidence,\n reason:\n `promote: paired holdout median Δ=${fmt(ci.median)} ` +\n `CI=[${fmt(ci.low)}, ${fmt(ci.high)}] over ${productiveRuns} pairs; ` +\n `overfit gap candidate=${fmt(overfitGap)} vs baseline=${fmt(baselineOverfitGap)}`,\n rejectionCode: null,\n }\n }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nfunction inferCandidateId(candidate: RunRecord[], baselineKey: string): string {\n for (const run of candidate) {\n if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId\n }\n // All candidate rows match the baseline key — caller mistake, but\n // surface the symptom rather than throwing inside the gate.\n return candidate[0]?.candidateId ?? '(unknown candidate)'\n}\n\nfunction indexHoldoutByKey(runs: RunRecord[]): Map<string, number> {\n const out = new Map<string, number>()\n for (const r of runs) {\n if (r.splitTag !== 'holdout') continue\n if (r.outcome.holdoutScore === undefined) continue\n out.set(pairKey(r), r.outcome.holdoutScore)\n }\n return out\n}\n\nfunction pairKey(r: RunRecord): string {\n return `${r.experimentId}::${r.seed}`\n}\n\nfunction scores(\n runs: RunRecord[],\n field: 'searchScore' | 'holdoutScore',\n splitFilter: 'search' | 'holdout',\n): number[] {\n const out: number[] = []\n for (const r of runs) {\n if (r.splitTag !== splitFilter) continue\n const v = r.outcome[field]\n if (typeof v === 'number' && Number.isFinite(v)) out.push(v)\n }\n return out\n}\n\nfunction mean(xs: number[]): number {\n if (xs.length === 0) return Number.NaN\n return xs.reduce((s, x) => s + x, 0) / xs.length\n}\n\nfunction safeDiff(a: number, b: number): number {\n if (!Number.isFinite(a) || !Number.isFinite(b)) return Number.NaN\n return a - b\n}\n\nfunction medianDelta(before: number[], after: number[]): number {\n const ds = before.map((b, i) => after[i]! - b).sort((x, y) => x - y)\n if (ds.length === 0) return 0\n const mid = Math.floor(ds.length / 2)\n return ds.length % 2 === 0 ? (ds[mid - 1]! + ds[mid]!) / 2 : ds[mid]!\n}\n\nfunction fmt(x: number): string {\n if (!Number.isFinite(x)) return String(x)\n return x.toFixed(4)\n}\n","/**\n * Pareto frontier — multi-objective optimization over candidate runs.\n *\n * Lifted from ADC pareto.ts and blueprint-agent frontier.ts. When you're\n * trading off (cost, latency, quality) or (passRate, tokenBudget,\n * ttfb), you rarely have a single \"winner\" — you have a set of\n * non-dominated candidates. This module exposes:\n *\n * - `paretoFrontier`: filter a set of candidates to the non-dominated ones\n * - `dominates`: does A dominate B across all objectives?\n *\n * Each objective is declared with a direction: 'maximize' (higher=better)\n * or 'minimize' (lower=better). Candidates are any object; pass an\n * `objective(candidate)` accessor.\n */\n\nexport type Direction = 'maximize' | 'minimize'\n\nexport interface Objective<T> {\n /** Stable label used in reports. */\n name: string\n direction: Direction\n value: (candidate: T) => number\n}\n\nexport interface ParetoResult<T> {\n frontier: T[]\n dominated: T[]\n /** Index map: frontier[i] dominates each of dominatedBy[i]. */\n dominanceMap: Array<{ dominator: T; dominated: T[] }>\n}\n\n/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */\nexport function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean {\n let strictlyBetter = false\n for (const obj of objectives) {\n const av = obj.value(a)\n const bv = obj.value(b)\n if (!Number.isFinite(av) || !Number.isFinite(bv)) return false\n const aIsBetter = obj.direction === 'maximize' ? av > bv : av < bv\n const aIsWorse = obj.direction === 'maximize' ? av < bv : av > bv\n if (aIsWorse) return false\n if (aIsBetter) strictlyBetter = true\n }\n return strictlyBetter\n}\n\n/**\n * Compute the non-dominated frontier. Candidates with NaN/Infinity on any\n * objective are excluded (can't rank them). A candidate enters the frontier\n * iff no other candidate dominates it.\n */\nexport function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T> {\n if (objectives.length === 0) {\n throw new Error('paretoFrontier: at least 1 objective required')\n }\n const valid = candidates.filter((c) => objectives.every((o) => Number.isFinite(o.value(c))))\n const frontier: T[] = []\n const dominated: T[] = []\n for (const c of valid) {\n const isDominated = valid.some((other) => other !== c && dominates(other, c, objectives))\n if (isDominated) dominated.push(c)\n else frontier.push(c)\n }\n const dominanceMap = frontier.map((d) => ({\n dominator: d,\n dominated: dominated.filter((x) => dominates(d, x, objectives)),\n }))\n return { frontier, dominated, dominanceMap }\n}\n\n/**\n * Weighted-sum scalarisation. Use as a tie-break / single-winner selector\n * when callers don't want to consume a frontier. Each objective contributes\n * its normalised value (0..1 via min-max across the candidate pool) times\n * its weight; missing weights default to 1/N.\n *\n * Direction is honoured automatically — `minimize` axes have their values\n * inverted before scaling so \"higher scalar = better\" always holds.\n */\nexport function scalarScore<T>(\n candidates: T[],\n objectives: Objective<T>[],\n options: { weights?: Partial<Record<string, number>> } = {},\n): Array<{ candidate: T; score: number }> {\n if (candidates.length === 0) return []\n const weights = options.weights ?? {}\n const totalWeight = objectives.reduce((s, o) => s + (weights[o.name] ?? 1), 0)\n\n // Pre-compute min/max per objective for normalisation.\n const ranges = objectives.map((obj) => {\n const values = candidates.map((c) => obj.value(c)).filter((v) => Number.isFinite(v))\n if (values.length === 0) return { min: 0, max: 1 }\n const min = Math.min(...values)\n const max = Math.max(...values)\n return { min, max: max === min ? min + 1 : max }\n })\n\n return candidates.map((c) => {\n let score = 0\n objectives.forEach((obj, i) => {\n const v = obj.value(c)\n if (!Number.isFinite(v)) return\n const { min, max } = ranges[i]!\n const normalised = (v - min) / (max - min)\n const directional = obj.direction === 'maximize' ? normalised : 1 - normalised\n const weight = (weights[obj.name] ?? 1) / totalWeight\n score += directional * weight\n })\n return { candidate: c, score }\n })\n}\n\n/**\n * NSGA-II crowding distance — secondary sort for ties on the frontier.\n *\n * When the Pareto front collapses to a single point (or many candidates tie\n * on dominance), naive selection picks arbitrarily and the population\n * degenerates over generations. NSGA-II preserves diversity by preferring\n * candidates with more empty space around them on the frontier.\n *\n * Returns an array of `{ candidate, distance }` in the SAME order as the\n * input. Higher distance = more isolated = should be preferred when\n * preserving diversity.\n */\nexport function crowdingDistance<T>(\n candidates: T[],\n objectives: Objective<T>[],\n): Array<{ candidate: T; distance: number }> {\n const distances = new Map<T, number>(candidates.map((c) => [c, 0]))\n\n for (const obj of objectives) {\n const sorted = [...candidates].sort((a, b) => obj.value(a) - obj.value(b))\n const min = obj.value(sorted[0]!)\n const max = obj.value(sorted[sorted.length - 1]!)\n const range = max - min || 1\n\n // Boundary points get infinity (always preferred for diversity).\n distances.set(sorted[0]!, Infinity)\n distances.set(sorted[sorted.length - 1]!, Infinity)\n for (let i = 1; i < sorted.length - 1; i++) {\n const prev = obj.value(sorted[i - 1]!)\n const next = obj.value(sorted[i + 1]!)\n const current = distances.get(sorted[i]!)!\n if (current === Infinity) continue\n distances.set(sorted[i]!, current + (next - prev) / range)\n }\n }\n\n return candidates.map((c) => ({ candidate: c, distance: distances.get(c) ?? 0 }))\n}\n\n/**\n * Pareto frontier with tie-break by crowding distance — the canonical\n * NSGA-II selection step. Returns the frontier sorted by descending crowding\n * distance so callers can `.slice(0, k)` to pick K diverse winners.\n */\nexport function paretoFrontierWithCrowding<T>(\n candidates: T[],\n objectives: Objective<T>[],\n): Array<{ candidate: T; distance: number }> {\n const { frontier } = paretoFrontier(candidates, objectives)\n if (frontier.length === 0) return []\n const distances = crowdingDistance(frontier, objectives)\n return distances.sort((a, b) => b.distance - a.distance)\n}\n","/**\n * PromptEvolutionLoop — population-based reflective-mutation evolution.\n *\n * Above the existing `AxGepaSteeringOptimizer` (which RANKS variants),\n * this loop GENERATES variants. Each generation:\n * 1. Score the population across (variant × scenario × rep).\n * 2. Pick survivors from the Pareto frontier (with crowding-distance tie-break).\n * 3. Ask the mutator for replacements until population size is restored.\n * 4. Repeat for N generations OR until convergence.\n *\n * Domain-agnostic. Consumers supply:\n * - A seed population of `EvolvableVariant`s.\n * - A `ScoreAdapter` that runs (variant, scenario, rep) → `TrialResult`.\n * - A `MutateAdapter` that produces children given trace evidence.\n * - Pareto `Objective<TrialAggregate>[]` defining the multi-objective vector.\n *\n * The loop owns: population management, parallel scheduling (concurrency-\n * limited), Pareto selection with crowding distance, generation reporting.\n *\n * It does NOT own: rendering trials to a model, executing prompts, choosing\n * mutation primitives, persisting to disk. Those are the consumer's call.\n */\n\nimport { type Objective, paretoFrontierWithCrowding, scalarScore } from './pareto'\n\nexport interface EvolvableVariant<P = unknown> {\n /** Stable id for the variant — surfaces in reports and trial results. */\n id: string\n /** Variant payload — interpretation is the consumer's responsibility. */\n payload: P\n /** Generation index (0 = seed, then 1, 2, ...). */\n generation: number\n /** Parent variant id when produced via mutation; absent for seeds. */\n parentId?: string\n /** Human label for reports. */\n label: string\n /** What the mutator was trying to fix. */\n rationale?: string\n}\n\nexport interface TrialResult {\n variantId: string\n scenarioId: string\n rep: number\n ok: boolean\n /** Primary scalar score the consumer cares about (e.g., recall, accuracy). */\n score: number\n /** Token cost (or any cost-like dimension). */\n cost?: number\n /** Wall time in ms. */\n durationMs?: number\n /** Free-form metric bag for objective accessors. */\n metrics?: Record<string, number>\n error?: string\n /**\n * Whether the judge LLM call(s) that produced this trial's score actually\n * completed. `undefined` means \"consumer didn't report\"; `false` means\n * the judge aborted/failed and the score is synthetic (typically 0 or\n * partial). `aggregateTrials({mode: 'exclude-failed'})` skips these\n * trials so a silent-zero judge can't pollute the composite.\n */\n judgeSucceeded?: boolean\n /** Number of judge attempts (informational, populated by `withJudgeRetry`). */\n judgeAttempts?: number\n /** Last judge error message when `judgeSucceeded === false`. */\n judgeError?: string\n}\n\n/** Aggregated trial summary for one (variant, scenario) pair across reps. */\nexport interface ScenarioAggregate {\n variantId: string\n scenarioId: string\n meanScore: number\n meanCost: number\n meanDurationMs: number\n okRate: number\n trials: number\n /** Mean of every numeric metric across reps. */\n metrics: Record<string, number>\n}\n\n/** Aggregated trial summary for one variant across all scenarios. */\nexport interface VariantAggregate {\n variantId: string\n meanScore: number\n meanCost: number\n meanDurationMs: number\n okRate: number\n scenarios: ScenarioAggregate[]\n /** Mean of every numeric metric, averaged across scenarios. */\n metrics: Record<string, number>\n}\n\nexport interface ScoreAdapter<P = unknown> {\n score(args: {\n variant: EvolvableVariant<P>\n scenarioId: string\n rep: number\n }): Promise<TrialResult>\n}\n\nexport interface MutateAdapter<P = unknown> {\n mutate(args: {\n parent: EvolvableVariant<P>\n parentAggregate: VariantAggregate\n topTrials: TrialResult[]\n bottomTrials: TrialResult[]\n childCount: number\n generation: number\n }): Promise<EvolvableVariant<P>[]>\n}\n\nexport interface PromptEvolutionConfig<P = unknown> {\n runId: string\n /** What component is being mutated — surfaces in reports + reflection prompts. */\n target: string\n seedVariants: EvolvableVariant<P>[]\n scenarioIds: string[]\n reps: number\n generations: number\n populationSize: number\n /** Maximum concurrent score() calls. */\n scoreConcurrency: number\n scoreAdapter: ScoreAdapter<P>\n mutateAdapter: MutateAdapter<P>\n /** Pareto objectives over `VariantAggregate`. Ordered by importance. */\n objectives: Objective<VariantAggregate>[]\n /** Optional weights for the scalar tie-break selector (by objective name). */\n scalarWeights?: Record<string, number>\n /** Stop early if a generation produces no Pareto improvement. Default true. */\n earlyStopOnNoImprovement?: boolean\n onProgress?: (event: PromptEvolutionEvent) => void\n /**\n * Optional cache key for memoising scored (variantId, scenarioId, rep)\n * tuples. When provided AND a cache instance is passed, repeated trials\n * skip re-scoring. Cache keys are stable across runs.\n */\n cache?: TrialCache\n}\n\nexport interface TrialCache {\n get(key: string): TrialResult | undefined\n set(key: string, value: TrialResult): void\n}\n\nexport class InMemoryTrialCache implements TrialCache {\n private store = new Map<string, TrialResult>()\n get(key: string): TrialResult | undefined {\n return this.store.get(key)\n }\n set(key: string, value: TrialResult): void {\n this.store.set(key, value)\n }\n size(): number {\n return this.store.size\n }\n clear(): void {\n this.store.clear()\n }\n}\n\nexport type PromptEvolutionEvent =\n | { type: 'generation-start'; generation: number; populationSize: number }\n | {\n type: 'trial-complete'\n generation: number\n variantId: string\n scenarioId: string\n rep: number\n ok: boolean\n score: number\n cached: boolean\n }\n | { type: 'generation-complete'; report: GenerationReport<unknown> }\n | { type: 'converged'; generation: number; reason: string }\n\nexport interface GenerationReport<P = unknown> {\n runId: string\n target: string\n generation: number\n variants: EvolvableVariant<P>[]\n aggregates: VariantAggregate[]\n /** Frontier candidates, sorted by descending crowding distance. */\n paretoFrontIds: string[]\n /** Scalar-best variant id — used for the single \"winner\" if callers want one. */\n winnerId: string\n /** Trials that fed this generation (kept for downstream reporting). */\n trials: TrialResult[]\n}\n\nexport interface PromptEvolutionResult<P = unknown> {\n runId: string\n target: string\n generations: GenerationReport<P>[]\n /** Best variant by scalar score in the final generation. */\n bestVariant: EvolvableVariant<P>\n /** Best aggregate (matches bestVariant). */\n bestAggregate: VariantAggregate\n}\n\nexport async function runPromptEvolution<P>(\n config: PromptEvolutionConfig<P>,\n): Promise<PromptEvolutionResult<P>> {\n const generations: GenerationReport<P>[] = []\n let population = [...config.seedVariants]\n let bestVariant: EvolvableVariant<P> = population[0]!\n let bestAggregate: VariantAggregate | null = null\n\n for (let generation = 0; generation < config.generations; generation++) {\n config.onProgress?.({ type: 'generation-start', generation, populationSize: population.length })\n\n const trials = await scorePopulation(population, config, generation)\n const aggregates = aggregateTrials(population, config.scenarioIds, trials)\n\n const front = paretoFrontierWithCrowding(aggregates, config.objectives)\n const frontIds = new Set(front.map((c) => c.candidate.variantId))\n\n const scored = scalarScore(aggregates, config.objectives, { weights: config.scalarWeights })\n scored.sort((a, b) => b.score - a.score)\n const winnerId = scored[0]?.candidate.variantId ?? aggregates[0]?.variantId ?? population[0]!.id\n\n const report: GenerationReport<P> = {\n runId: config.runId,\n target: config.target,\n generation,\n variants: population,\n aggregates,\n paretoFrontIds: front.map((c) => c.candidate.variantId),\n winnerId,\n trials,\n }\n generations.push(report)\n config.onProgress?.({ type: 'generation-complete', report })\n\n const winnerAgg = aggregates.find((a) => a.variantId === winnerId)\n if (winnerAgg) {\n const winner = population.find((v) => v.id === winnerId)\n if (winner) bestVariant = winner\n bestAggregate = winnerAgg\n }\n\n // Convergence: no Pareto-or-scalar improvement vs previous generation.\n if (config.earlyStopOnNoImprovement !== false && generations.length >= 2) {\n const prev = generations[generations.length - 2]!\n const noChange =\n prev.winnerId === winnerId && samePopulation(prev.paretoFrontIds, [...frontIds])\n if (noChange) {\n config.onProgress?.({\n type: 'converged',\n generation,\n reason: 'no improvement vs previous generation',\n })\n break\n }\n }\n\n if (generation === config.generations - 1) break\n\n population = await nextPopulation(population, aggregates, trials, front, config, generation + 1)\n }\n\n return {\n runId: config.runId,\n target: config.target,\n generations,\n bestVariant,\n bestAggregate:\n bestAggregate ??\n aggregateTrials(population, config.scenarioIds, []).find(\n (a) => a.variantId === bestVariant.id,\n )!,\n }\n}\n\nasync function scorePopulation<P>(\n population: EvolvableVariant<P>[],\n config: PromptEvolutionConfig<P>,\n generation: number,\n): Promise<TrialResult[]> {\n const jobs: Array<() => Promise<TrialResult>> = []\n for (const variant of population) {\n for (const scenarioId of config.scenarioIds) {\n for (let rep = 0; rep < config.reps; rep++) {\n jobs.push(async () => {\n const cacheKey = `${variant.id}|${scenarioId}|${rep}`\n const cached = config.cache?.get(cacheKey)\n if (cached) {\n config.onProgress?.({\n type: 'trial-complete',\n generation,\n variantId: variant.id,\n scenarioId,\n rep,\n ok: cached.ok,\n score: cached.score,\n cached: true,\n })\n return cached\n }\n const result = await config.scoreAdapter.score({ variant, scenarioId, rep })\n config.cache?.set(cacheKey, result)\n config.onProgress?.({\n type: 'trial-complete',\n generation,\n variantId: variant.id,\n scenarioId,\n rep,\n ok: result.ok,\n score: result.score,\n cached: false,\n })\n return result\n })\n }\n }\n }\n return runWithConcurrency(jobs, config.scoreConcurrency)\n}\n\nasync function runWithConcurrency<T>(\n jobs: Array<() => Promise<T>>,\n concurrency: number,\n): Promise<T[]> {\n const results: T[] = new Array(jobs.length)\n const limit = Math.max(1, concurrency)\n let next = 0\n async function worker(): Promise<void> {\n while (true) {\n const i = next++\n if (i >= jobs.length) return\n results[i] = await jobs[i]!()\n }\n }\n await Promise.all(Array.from({ length: limit }, () => worker()))\n return results\n}\n\nfunction aggregateTrials<P>(\n population: EvolvableVariant<P>[],\n scenarioIds: string[],\n trials: TrialResult[],\n): VariantAggregate[] {\n return population.map((variant) => {\n const variantTrials = trials.filter((t) => t.variantId === variant.id)\n const scenarios = scenarioIds.map<ScenarioAggregate>((sid) => {\n const scenarioTrials = variantTrials.filter((t) => t.scenarioId === sid)\n const okTrials = scenarioTrials.filter((t) => t.ok)\n // Mean score must include every successfully-graded trial — a trial\n // with score=0.6 and ok=false (below quality_bar) is real signal, not\n // noise. Only `error` trials (agent crash, judge crash) carry a\n // synthetic score and are excluded. okRate continues to reflect the\n // pass/fail rate against the configured quality_bar.\n const gradedTrials = scenarioTrials.filter((t) => !t.error)\n const metrics = aggregateMetrics(gradedTrials.map((t) => t.metrics ?? {}))\n return {\n variantId: variant.id,\n scenarioId: sid,\n meanScore: mean(gradedTrials.map((t) => t.score)),\n meanCost: mean(gradedTrials.map((t) => t.cost ?? 0)),\n meanDurationMs: mean(gradedTrials.map((t) => t.durationMs ?? 0)),\n okRate: scenarioTrials.length === 0 ? 0 : okTrials.length / scenarioTrials.length,\n trials: scenarioTrials.length,\n metrics,\n }\n })\n return {\n variantId: variant.id,\n meanScore: mean(scenarios.map((s) => s.meanScore)),\n meanCost: mean(scenarios.map((s) => s.meanCost)),\n meanDurationMs: mean(scenarios.map((s) => s.meanDurationMs)),\n okRate: mean(scenarios.map((s) => s.okRate)),\n scenarios,\n metrics: aggregateMetrics(scenarios.map((s) => s.metrics)),\n }\n })\n}\n\nfunction aggregateMetrics(rows: Array<Record<string, number>>): Record<string, number> {\n const buckets = new Map<string, number[]>()\n for (const row of rows) {\n for (const [k, v] of Object.entries(row)) {\n if (!Number.isFinite(v)) continue\n const list = buckets.get(k) ?? []\n list.push(v)\n buckets.set(k, list)\n }\n }\n const out: Record<string, number> = {}\n for (const [k, list] of buckets) out[k] = mean(list)\n return out\n}\n\nfunction mean(xs: number[]): number {\n if (xs.length === 0) return 0\n return xs.reduce((a, b) => a + b, 0) / xs.length\n}\n\nasync function nextPopulation<P>(\n current: EvolvableVariant<P>[],\n aggregates: VariantAggregate[],\n trials: TrialResult[],\n front: Array<{ candidate: VariantAggregate; distance: number }>,\n config: PromptEvolutionConfig<P>,\n nextGeneration: number,\n): Promise<EvolvableVariant<P>[]> {\n const survivorIds = new Set(front.map((c) => c.candidate.variantId))\n const survivors = current.filter((v) => survivorIds.has(v.id))\n\n // Pick the best survivor (by scalar) as the mutation parent.\n const ranked = scalarScore(aggregates, config.objectives, { weights: config.scalarWeights }).sort(\n (a, b) => b.score - a.score,\n )\n const parentId = ranked[0]?.candidate.variantId ?? current[0]!.id\n const parent = current.find((v) => v.id === parentId) ?? current[0]!\n const parentAggregate = aggregates.find((a) => a.variantId === parent.id) ?? aggregates[0]!\n\n const topTrials = topKTrialsByScore(trials, parent.id, 3)\n const bottomTrials = bottomKTrialsByScore(trials, parent.id, 3)\n const childCount = Math.max(0, config.populationSize - survivors.length)\n let children: EvolvableVariant<P>[] = []\n if (childCount > 0) {\n children = await config.mutateAdapter.mutate({\n parent,\n parentAggregate,\n topTrials,\n bottomTrials,\n childCount,\n generation: nextGeneration,\n })\n children = children\n .slice(0, childCount)\n .map((c) => ({ ...c, generation: nextGeneration, parentId: parent.id }))\n }\n return [...survivors, ...children]\n}\n\nfunction topKTrialsByScore(trials: TrialResult[], variantId: string, k: number): TrialResult[] {\n return trials\n .filter((t) => t.variantId === variantId && t.ok)\n .sort((a, b) => b.score - a.score)\n .slice(0, k)\n}\n\nfunction bottomKTrialsByScore(trials: TrialResult[], variantId: string, k: number): TrialResult[] {\n return trials\n .filter((t) => t.variantId === variantId && t.ok)\n .sort((a, b) => a.score - b.score)\n .slice(0, k)\n}\n\nfunction samePopulation(a: string[], b: string[]): boolean {\n if (a.length !== b.length) return false\n const setA = new Set(a)\n return b.every((id) => setA.has(id))\n}\n","/**\n * Multi-shot optimization adapter.\n *\n * This is the canonical bridge between variable-length agent trajectories\n * and `runPromptEvolution`. Apps provide four things:\n *\n * - variants: prompt/config/tool-policy candidates\n * - runner: executes one full task trajectory for a variant\n * - scorer: turns that trajectory into score + actionable side information\n * - mutator: proposes new variants from top/bottom scored trials\n *\n * The adapter owns the boring but easy-to-get-wrong glue: stable seeds,\n * score/cost objectives, error-to-trial conversion, ASI metric projection,\n * and optional paired holdout gating via `HeldOutGate`.\n */\n\nimport { type GateDecision, HeldOutGate, type HeldOutGateConfig } from './held-out-gate'\nimport type { Objective } from './pareto'\nimport {\n type EvolvableVariant,\n type PromptEvolutionEvent,\n type PromptEvolutionResult,\n runPromptEvolution,\n type ScoreAdapter,\n type TrialCache,\n type TrialResult,\n type VariantAggregate,\n} from './prompt-evolution'\nimport type { TrialTrace } from './reflective-mutation'\nimport { type RunRecord, type RunSplitTag, validateRunRecord } from './run-record'\n\nexport type MultiShotSplit = 'search' | 'dev' | 'holdout'\n\nexport type AsiSeverity = 'info' | 'warning' | 'error' | 'critical'\n\nexport type MultiShotVariant<P = unknown> = EvolvableVariant<P>\n\nexport interface ActionableSideInfo {\n /** Stable expectation/check id when available. */\n expectationId?: string\n /** Human-readable diagnosis of what happened. */\n message: string\n severity?: AsiSeverity\n /** Concrete trace excerpt, file path, tool call, screenshot id, etc. */\n evidence?: string\n /** Prompt/tool/context surface likely responsible. */\n responsibleSurface?: string\n /** Suggested fix in natural language. */\n suggestion?: string\n /** Whether this expectation was satisfied. Defaults to false for ASI rows. */\n matched?: boolean\n metadata?: Record<string, unknown>\n}\n\nexport interface MultiShotTrace {\n scenarioId: string\n /** Full turn/tool trace. Shape is intentionally app-owned. */\n turns?: unknown[]\n toolCalls?: unknown[]\n artifacts?: unknown[]\n /** Compact final output or summary used by reflection prompts. */\n transcript?: string\n output?: unknown\n metadata?: Record<string, unknown>\n}\n\nexport interface MultiShotRun {\n trace: MultiShotTrace\n costUsd?: number\n durationMs?: number\n tokenUsage?: { input?: number; output?: number; cached?: number }\n metadata?: Record<string, unknown>\n}\n\nexport interface MultiShotRunInput<P = unknown> {\n variant: EvolvableVariant<P>\n scenarioId: string\n rep: number\n split: MultiShotSplit\n /** Stable paired seed for baseline/candidate comparisons. */\n seed: number\n}\n\nexport interface MultiShotRunner<P = unknown> {\n run(input: MultiShotRunInput<P>): Promise<MultiShotRun> | MultiShotRun\n}\n\nexport interface MultiShotScore {\n /** Primary score in [0,1]. The adapter clamps for safety. */\n score: number\n /** Pass/fail for top/bottom trial selection. Defaults to true. */\n ok?: boolean\n costUsd?: number\n durationMs?: number\n metrics?: Record<string, number>\n asi?: ActionableSideInfo[]\n /** Optional rich output shown to reflection mutators. */\n emitted?: string\n metadata?: Record<string, unknown>\n}\n\nexport interface MultiShotScorer<P = unknown> {\n score(\n input: MultiShotRunInput<P> & { run: MultiShotRun },\n ): Promise<MultiShotScore> | MultiShotScore\n}\n\nexport interface MultiShotTrialResult extends TrialResult {\n split: MultiShotSplit\n seed: number\n trace?: MultiShotTrace\n asi?: ActionableSideInfo[]\n emitted?: string\n metadata?: Record<string, unknown>\n}\n\nexport interface MultiShotMutateAdapter<P = unknown> {\n mutate(args: {\n parent: EvolvableVariant<P>\n parentAggregate: VariantAggregate\n topTrials: MultiShotTrialResult[]\n bottomTrials: MultiShotTrialResult[]\n childCount: number\n generation: number\n }): Promise<EvolvableVariant<P>[]>\n}\n\nexport interface MultiShotGateConfig<P = unknown> {\n /** Search rows are optional, but enable HeldOutGate's overfit-gap check. */\n searchScenarioIds?: string[]\n holdoutScenarioIds: string[]\n reps?: number\n gate: HeldOutGateConfig\n /** Convert scored trajectory runs into paper-grade RunRecords. */\n toRunRecord(input: {\n variant: EvolvableVariant<P>\n scenarioId: string\n rep: number\n split: RunSplitTag\n seed: number\n trial: MultiShotTrialResult\n }): RunRecord\n}\n\nexport interface MultiShotOptimizationConfig<P = unknown> {\n runId: string\n target: string\n seedVariants: EvolvableVariant<P>[]\n searchScenarioIds: string[]\n reps: number\n generations: number\n populationSize: number\n scoreConcurrency?: number\n runner: MultiShotRunner<P>\n scorer: MultiShotScorer<P>\n mutateAdapter: MultiShotMutateAdapter<P>\n objectives?: Objective<VariantAggregate>[]\n scalarWeights?: Record<string, number>\n cache?: TrialCache\n earlyStopOnNoImprovement?: boolean\n seedBase?: number\n onProgress?: (event: PromptEvolutionEvent) => void\n gate?: MultiShotGateConfig<P>\n}\n\nexport interface MultiShotGateResult {\n decision: GateDecision\n candidateRuns: RunRecord[]\n baselineRuns: RunRecord[]\n}\n\nexport interface MultiShotOptimizationResult<P = unknown> {\n evolution: PromptEvolutionResult<P>\n /** Best candidate on the optimizer-visible search split. */\n searchBestVariant: EvolvableVariant<P>\n searchBestAggregate: VariantAggregate\n /** Variant callers should actually ship after optional holdout gating. */\n promotedVariant: EvolvableVariant<P>\n promotedAggregate: VariantAggregate\n /** Null when no gate was configured or the search-best candidate was the baseline. */\n gate: MultiShotGateResult | null\n}\n\nexport async function runMultiShotOptimization<P>(\n config: MultiShotOptimizationConfig<P>,\n): Promise<MultiShotOptimizationResult<P>> {\n validateConfig(config)\n\n const scoreAdapter: ScoreAdapter<P> = {\n score: (args) => scoreOne(config, args.variant, args.scenarioId, args.rep, 'search'),\n }\n\n const evolution = await runPromptEvolution<P>({\n runId: config.runId,\n target: config.target,\n seedVariants: config.seedVariants,\n scenarioIds: config.searchScenarioIds,\n reps: config.reps,\n generations: config.generations,\n populationSize: config.populationSize,\n scoreConcurrency: config.scoreConcurrency ?? 1,\n scoreAdapter,\n mutateAdapter: {\n mutate: (args) =>\n config.mutateAdapter.mutate({\n ...args,\n topTrials: args.topTrials as MultiShotTrialResult[],\n bottomTrials: args.bottomTrials as MultiShotTrialResult[],\n }),\n },\n objectives: config.objectives ?? defaultMultiShotObjectives(),\n scalarWeights: config.scalarWeights,\n earlyStopOnNoImprovement: config.earlyStopOnNoImprovement,\n cache: config.cache,\n onProgress: config.onProgress,\n })\n\n let gate: MultiShotGateResult | null = null\n const baseline = config.seedVariants[0]!\n let promotedVariant = evolution.bestVariant\n let promotedAggregate = evolution.bestAggregate\n if (config.gate && evolution.bestVariant.id !== baseline.id) {\n gate = await evaluateMultiShotGate(config, baseline, evolution.bestVariant)\n if (!gate.decision.promote) {\n promotedVariant = baseline\n promotedAggregate = aggregateFor(evolution, baseline.id)\n }\n }\n\n return {\n evolution,\n searchBestVariant: evolution.bestVariant,\n searchBestAggregate: evolution.bestAggregate,\n promotedVariant,\n promotedAggregate,\n gate,\n }\n}\n\nexport function defaultMultiShotObjectives(): Objective<VariantAggregate>[] {\n return [\n { name: 'score', direction: 'maximize', value: (a) => a.meanScore },\n { name: 'cost', direction: 'minimize', value: (a) => a.meanCost },\n ]\n}\n\nexport function trialTraceFromMultiShotTrial(trial: MultiShotTrialResult): TrialTrace {\n return {\n id: `${trial.variantId}/${trial.scenarioId}/r${trial.rep}`,\n score: trial.score,\n inputName: trial.scenarioId,\n expectations: (trial.asi ?? []).map((item, i) => ({\n id: item.expectationId ?? `asi-${i}`,\n phrase: item.message,\n matched: item.matched ?? false,\n })),\n emitted: trial.emitted ?? traceExcerpt(trial.trace),\n metrics: trial.metrics,\n }\n}\n\nasync function evaluateMultiShotGate<P>(\n config: MultiShotOptimizationConfig<P>,\n baseline: EvolvableVariant<P>,\n candidate: EvolvableVariant<P>,\n): Promise<MultiShotGateResult> {\n const gateConfig = config.gate!\n const reps = gateConfig.reps ?? config.reps\n const candidateRuns: RunRecord[] = []\n const baselineRuns: RunRecord[] = []\n\n const searchIds = gateConfig.searchScenarioIds ?? config.searchScenarioIds\n for (const scenarioId of searchIds) {\n for (let rep = 0; rep < reps; rep++) {\n const seed = seedFor(config, scenarioId, rep)\n const baseTrial = await scoreOne(config, baseline, scenarioId, rep, 'search')\n const candTrial = await scoreOne(config, candidate, scenarioId, rep, 'search')\n baselineRuns.push(\n toValidatedRecord(config, baseline, scenarioId, rep, 'search', seed, baseTrial),\n )\n candidateRuns.push(\n toValidatedRecord(config, candidate, scenarioId, rep, 'search', seed, candTrial),\n )\n }\n }\n\n for (const scenarioId of gateConfig.holdoutScenarioIds) {\n for (let rep = 0; rep < reps; rep++) {\n const seed = seedFor(config, scenarioId, rep)\n const baseTrial = await scoreOne(config, baseline, scenarioId, rep, 'holdout')\n const candTrial = await scoreOne(config, candidate, scenarioId, rep, 'holdout')\n baselineRuns.push(\n toValidatedRecord(config, baseline, scenarioId, rep, 'holdout', seed, baseTrial),\n )\n candidateRuns.push(\n toValidatedRecord(config, candidate, scenarioId, rep, 'holdout', seed, candTrial),\n )\n }\n }\n\n const decision = new HeldOutGate(gateConfig.gate).evaluate(candidateRuns, baselineRuns)\n return { decision, candidateRuns, baselineRuns }\n}\n\nasync function scoreOne<P>(\n config: MultiShotOptimizationConfig<P>,\n variant: EvolvableVariant<P>,\n scenarioId: string,\n rep: number,\n split: MultiShotSplit,\n): Promise<MultiShotTrialResult> {\n const seed = seedFor(config, scenarioId, rep)\n const input: MultiShotRunInput<P> = { variant, scenarioId, rep, split, seed }\n try {\n const run = await config.runner.run(input)\n const scored = await config.scorer.score({ ...input, run })\n const asi = scored.asi ?? []\n return {\n variantId: variant.id,\n scenarioId,\n rep,\n ok: scored.ok ?? true,\n score: clamp01(scored.score),\n cost: scored.costUsd ?? run.costUsd ?? 0,\n durationMs: scored.durationMs ?? run.durationMs ?? 0,\n metrics: {\n ...numericMetrics(scored.metrics),\n ...asiMetrics(asi),\n },\n split,\n seed,\n trace: run.trace,\n asi,\n emitted: scored.emitted ?? traceExcerpt(run.trace),\n metadata: scored.metadata,\n }\n } catch (err) {\n return {\n variantId: variant.id,\n scenarioId,\n rep,\n ok: false,\n score: 0,\n cost: 0,\n durationMs: 0,\n metrics: { error: 1 },\n error: err instanceof Error ? err.message : String(err),\n split,\n seed,\n asi: [\n {\n severity: 'critical',\n message: err instanceof Error ? err.message : String(err),\n responsibleSurface: config.target,\n },\n ],\n emitted: '',\n }\n }\n}\n\nfunction toValidatedRecord<P>(\n config: MultiShotOptimizationConfig<P>,\n variant: EvolvableVariant<P>,\n scenarioId: string,\n rep: number,\n split: RunSplitTag,\n seed: number,\n trial: MultiShotTrialResult,\n): RunRecord {\n const record = config.gate!.toRunRecord({ variant, scenarioId, rep, split, seed, trial })\n return validateRunRecord(record)\n}\n\nfunction validateConfig<P>(config: MultiShotOptimizationConfig<P>): void {\n if (!config.runId.trim()) throw new Error('runMultiShotOptimization: runId must not be empty')\n if (!config.target.trim()) throw new Error('runMultiShotOptimization: target must not be empty')\n if (config.seedVariants.length === 0) {\n throw new Error('runMultiShotOptimization: seedVariants must not be empty')\n }\n if (config.searchScenarioIds.length === 0) {\n throw new Error('runMultiShotOptimization: searchScenarioIds must not be empty')\n }\n requirePositiveInteger(config.reps, 'reps')\n requirePositiveInteger(config.generations, 'generations')\n requirePositiveInteger(config.populationSize, 'populationSize')\n if (config.scoreConcurrency !== undefined)\n requirePositiveInteger(config.scoreConcurrency, 'scoreConcurrency')\n if (config.populationSize < config.seedVariants.length) {\n throw new Error('runMultiShotOptimization: populationSize must be >= seedVariants.length')\n }\n assertUnique(\n config.seedVariants.map((v) => v.id),\n 'seedVariants.id',\n )\n assertUnique(config.searchScenarioIds, 'searchScenarioIds')\n\n if (config.gate) {\n if (config.gate.holdoutScenarioIds.length === 0) {\n throw new Error('runMultiShotOptimization: gate.holdoutScenarioIds must not be empty')\n }\n if (config.gate.reps !== undefined) requirePositiveInteger(config.gate.reps, 'gate.reps')\n assertUnique(config.gate.holdoutScenarioIds, 'gate.holdoutScenarioIds')\n if (config.gate.searchScenarioIds)\n assertUnique(config.gate.searchScenarioIds, 'gate.searchScenarioIds')\n const searchIds = new Set(config.searchScenarioIds)\n for (const id of config.gate.holdoutScenarioIds) {\n if (searchIds.has(id)) {\n throw new Error(\n `runMultiShotOptimization: holdout scenario \"${id}\" also appears in searchScenarioIds`,\n )\n }\n }\n const baselineId = config.seedVariants[0]!.id\n if (config.gate.gate.baselineKey !== baselineId) {\n throw new Error(\n `runMultiShotOptimization: gate.gate.baselineKey must match first seed variant id \"${baselineId}\"`,\n )\n }\n }\n}\n\nfunction requirePositiveInteger(value: number, name: string): void {\n if (!Number.isInteger(value) || value <= 0) {\n throw new Error(`runMultiShotOptimization: ${name} must be a positive integer`)\n }\n}\n\nfunction assertUnique(values: string[], name: string): void {\n const seen = new Set<string>()\n for (const value of values) {\n if (!value.trim())\n throw new Error(`runMultiShotOptimization: ${name} must not contain empty values`)\n if (seen.has(value)) throw new Error(`runMultiShotOptimization: duplicate ${name} \"${value}\"`)\n seen.add(value)\n }\n}\n\nfunction aggregateFor<P>(evolution: PromptEvolutionResult<P>, variantId: string): VariantAggregate {\n const final = evolution.generations[evolution.generations.length - 1]\n const aggregate = final?.aggregates.find((a) => a.variantId === variantId)\n if (!aggregate) {\n throw new Error(`runMultiShotOptimization: missing aggregate for variant \"${variantId}\"`)\n }\n return aggregate\n}\n\nfunction seedFor<P>(\n config: MultiShotOptimizationConfig<P>,\n scenarioId: string,\n rep: number,\n): number {\n const base = config.seedBase ?? 0\n return (base + stableHash(`${scenarioId}\\x1f${rep}`)) % Number.MAX_SAFE_INTEGER\n}\n\nfunction stableHash(input: string): number {\n let h = 2166136261\n for (let i = 0; i < input.length; i++) {\n h ^= input.charCodeAt(i)\n h = Math.imul(h, 16777619)\n }\n return h >>> 0\n}\n\nfunction clamp01(n: number): number {\n if (!Number.isFinite(n)) return 0\n return Math.max(0, Math.min(1, n))\n}\n\nfunction numericMetrics(metrics: Record<string, number> | undefined): Record<string, number> {\n const out: Record<string, number> = {}\n for (const [k, v] of Object.entries(metrics ?? {})) {\n if (Number.isFinite(v)) out[k] = v\n }\n return out\n}\n\nfunction asiMetrics(asi: ActionableSideInfo[]): Record<string, number> {\n const out: Record<string, number> = { asi: asi.length }\n for (const item of asi.slice(0, 1000)) {\n const sev = normalizeSeverity(item.severity)\n out[`asi.${sev}`] = (out[`asi.${sev}`] ?? 0) + 1\n if (item.responsibleSurface) {\n const key = `surface.${metricKeySegment(item.responsibleSurface)}`\n out[key] = (out[key] ?? 0) + 1\n }\n }\n return out\n}\n\nfunction normalizeSeverity(severity: AsiSeverity | undefined): AsiSeverity {\n if (\n severity === 'info' ||\n severity === 'warning' ||\n severity === 'error' ||\n severity === 'critical'\n ) {\n return severity\n }\n return 'error'\n}\n\nfunction metricKeySegment(raw: string): string {\n return (\n raw\n .trim()\n .replace(/[^a-zA-Z0-9._-]+/g, '_')\n .slice(0, 80) || 'unknown'\n )\n}\n\nfunction traceExcerpt(trace: MultiShotTrace | undefined): string | undefined {\n if (!trace) return undefined\n if (typeof trace.output === 'string') return trace.output\n if (trace.transcript) return trace.transcript\n if (trace.turns) {\n try {\n const clipped = trace.turns.slice(0, 20)\n const suffix =\n trace.turns.length > clipped.length\n ? ` ... ${trace.turns.length - clipped.length} more turn(s)`\n : ''\n return `${JSON.stringify(clipped).slice(0, 2000)}${suffix}`\n } catch {\n return '[unserializable trace turns]'\n }\n }\n return undefined\n}\n","/**\n * Reflective mutation — primitives for trace-conditioned prompt rewriting.\n *\n * Used by `prompt-evolution.ts` (and any consumer running iterative\n * improvement). Given a parent prompt + concrete trace evidence (top trials,\n * bottom trials, missed expectations), produce an LLM-ready prompt that\n * proposes targeted mutations — not blind rephrasings.\n *\n * Why this lives outside `prompt-evolution.ts`: any consumer that wants to\n * run reflective rewriting WITHOUT the population/Pareto machinery can\n * import these primitives directly.\n *\n * Quality bar (vs. naive \"mutate this prompt\"):\n * - Show parent ↔ children diff, not just one variant\n * - Quote specific missed goldens with their match phrases\n * - Surface the model's actual emitted output side-by-side with what was expected\n * - Quote concrete mutation primitives so the model has a vocabulary\n */\n\nexport interface TrialTrace {\n /** Stable id for the trial — surfaces in the prompt for grounding. */\n id: string\n /** Score the trial received on its primary metric. */\n score: number\n /** Candidate inputs the agent was given (e.g., the fixture or scenario). */\n inputName?: string\n /**\n * Goldens / expectations this trial was tested against, with whether each\n * was matched. The reflection prompt quotes the missed ones specifically.\n */\n expectations?: Array<{ id: string; phrase: string; matched: boolean }>\n /** Free-form text — what the agent actually emitted (e.g., findings, plan). */\n emitted?: string\n /** Optional structured metrics (recall, precision, cost, latency). */\n metrics?: Record<string, number>\n}\n\nexport interface ReflectionContext {\n /** What is being mutated — appears in the system prompt for orientation. */\n target: string\n /** Current variant's payload — JSON-serialised for the prompt. */\n parentPayload: unknown\n /** Best-performing trials this generation. */\n topTrials: TrialTrace[]\n /** Worst-performing trials this generation — the missed-golden source. */\n bottomTrials: TrialTrace[]\n /** How many children the mutator should propose. */\n childCount: number\n /** Optional: domain-specific mutation primitives the model can pick from. */\n mutationPrimitives?: string[]\n}\n\nexport const DEFAULT_MUTATION_PRIMITIVES: string[] = [\n 'Strengthen an imperative (\"should\" → \"must\")',\n 'Add a concrete example pulled from a missed-golden phrase',\n 'Remove a redundant rule that did not improve recall',\n 'Add a counterfactual (\"if X is missing, the score is capped at Y\")',\n 'Reorder sections so the highest-impact rule is first',\n 'Replace abstract language with a domain-specific noun the trial misses',\n]\n\n/**\n * Build the LLM-ready reflection prompt. Output is plain text — pass it as\n * the user message. The system message should be small and stable (e.g.\n * \"Output ONLY a JSON object matching the schema below.\").\n */\nexport function buildReflectionPrompt(ctx: ReflectionContext): string {\n const primitives = ctx.mutationPrimitives ?? DEFAULT_MUTATION_PRIMITIVES\n const sections: string[] = []\n\n sections.push(`# Mutation target: ${ctx.target}`)\n sections.push('')\n sections.push(\n `You are tuning the prompt component named \\`${ctx.target}\\`. The current variant is shown below; you have ${ctx.topTrials.length} top trials and ${ctx.bottomTrials.length} bottom trials as evidence. Propose ${ctx.childCount} mutation${ctx.childCount === 1 ? '' : 's'} that fix specific weaknesses visible in the bottom trials. Avoid blank rephrasings.`,\n )\n sections.push('')\n\n sections.push('## Current variant')\n sections.push('```json')\n sections.push(JSON.stringify(ctx.parentPayload, null, 2))\n sections.push('```')\n sections.push('')\n\n if (ctx.bottomTrials.length > 0) {\n sections.push('## Failures (bottom trials) — what went wrong')\n sections.push('')\n for (const trial of ctx.bottomTrials) {\n sections.push(\n `### Trial \\`${trial.id}\\` — score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ''}`,\n )\n const missed = (trial.expectations ?? []).filter((e) => !e.matched)\n if (missed.length > 0) {\n sections.push('')\n sections.push('**Missed expectations:**')\n for (const m of missed) {\n sections.push(`- \\`${m.id}\\`: should match phrase \\`${quote(m.phrase)}\\``)\n }\n }\n if (trial.emitted) {\n sections.push('')\n sections.push('**What the agent emitted:**')\n sections.push('```')\n sections.push(truncate(trial.emitted, 600))\n sections.push('```')\n }\n sections.push('')\n }\n }\n\n if (ctx.topTrials.length > 0) {\n sections.push('## Successes (top trials) — what to preserve')\n sections.push('')\n for (const trial of ctx.topTrials) {\n sections.push(\n `- \\`${trial.id}\\`: score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ''}`,\n )\n }\n sections.push('')\n }\n\n sections.push('## Allowed mutation primitives')\n sections.push('')\n for (const p of primitives) sections.push(`- ${p}`)\n sections.push('')\n\n sections.push('## Output schema')\n sections.push('')\n sections.push('Respond with a JSON object — no prose, no markdown fences:')\n sections.push('```json')\n sections.push(\n JSON.stringify(\n {\n proposals: [\n {\n label: '<short label, ≤ 40 chars>',\n rationale: '<which failure this targets and which primitive you used>',\n payload: '<full payload of the new variant — same shape as the current variant>',\n },\n ],\n },\n null,\n 2,\n ),\n )\n sections.push('```')\n\n return sections.join('\\n')\n}\n\nfunction truncate(s: string, max: number): string {\n if (s.length <= max) return s\n return `${s.slice(0, max)}… [truncated]`\n}\n\nfunction quote(s: string): string {\n return s.replace(/`/g, '\\\\`')\n}\n\nexport interface ReflectionProposal {\n label: string\n rationale: string\n payload: unknown\n}\n\n/**\n * Parse the model's JSON response back into proposals. Tolerates markdown\n * fences and surrounding prose. Returns at most `maxProposals`.\n */\n/**\n * Walk the input as JSON-aware (string vs not, escape-aware) and close\n * unclosed `{` / `[` in LIFO order at the tail. If the input was already\n * balanced returns it unchanged. If a string was open at end-of-input we\n * also close it with `\"` first, since a truncated string-mid-value is the\n * most common LLM cap-hit failure mode and JSON.parse cannot proceed\n * without one.\n *\n * Returns null when the structure is unrecoverable (e.g. depth would go\n * negative — that's an *over*-closed prefix, not a truncation).\n */\nfunction autoCloseTruncatedJson(raw: string): string | null {\n const stack: Array<'{' | '['> = []\n let inString = false\n let escaped = false\n for (const c of raw) {\n if (escaped) {\n escaped = false\n continue\n }\n if (inString) {\n if (c === '\\\\') {\n escaped = true\n continue\n }\n if (c === '\"') {\n inString = false\n continue\n }\n continue\n }\n if (c === '\"') {\n inString = true\n continue\n }\n if (c === '{' || c === '[') stack.push(c)\n else if (c === '}') {\n if (stack.pop() !== '{') return null\n } else if (c === ']') {\n if (stack.pop() !== '[') return null\n }\n }\n if (stack.length === 0 && !inString) return raw\n let suffix = ''\n if (inString) suffix += '\"'\n while (stack.length > 0) {\n const opener = stack.pop()!\n suffix += opener === '{' ? '}' : ']'\n }\n return raw + suffix\n}\n\nexport function parseReflectionResponse(raw: string, maxProposals?: number): ReflectionProposal[] {\n let text = raw.trim()\n if (text.startsWith('```')) text = text.replace(/^```(?:json)?\\n?/, '').replace(/\\n?```$/, '')\n\n // Try to parse as either a JSON object `{proposals: [...]}` or a bare\n // array `[...]`. LLMs frequently emit one or the other depending on how\n // they read the schema example; accept both.\n let parsed: unknown = null\n const objectStart = text.indexOf('{')\n const objectEnd = text.lastIndexOf('}')\n const arrayStart = text.indexOf('[')\n const arrayEnd = text.lastIndexOf(']')\n // Prefer whichever delimiter comes first (the model committed to that shape).\n const tryObjectFirst = objectStart >= 0 && (arrayStart < 0 || objectStart < arrayStart)\n const candidates: string[] = []\n if (tryObjectFirst) {\n if (objectStart >= 0 && objectEnd > objectStart)\n candidates.push(text.slice(objectStart, objectEnd + 1))\n if (arrayStart >= 0 && arrayEnd > arrayStart)\n candidates.push(text.slice(arrayStart, arrayEnd + 1))\n } else {\n if (arrayStart >= 0 && arrayEnd > arrayStart)\n candidates.push(text.slice(arrayStart, arrayEnd + 1))\n if (objectStart >= 0 && objectEnd > objectStart)\n candidates.push(text.slice(objectStart, objectEnd + 1))\n }\n for (const slice of candidates) {\n try {\n parsed = JSON.parse(slice)\n break\n } catch {\n // try next\n }\n }\n\n // Truncation-tolerant fallback: LLMs frequently hit a max_tokens cap\n // mid-emission, leaving N unclosed `}` / `]` at the tail. Close them in\n // order from the deepest unclosed structure outward, by walking the\n // candidate slice and tracking depth, then retrying JSON.parse. This\n // recovers any complete proposals before the cutoff and drops the rest.\n if (parsed == null) {\n for (const slice of candidates) {\n const closed = autoCloseTruncatedJson(slice)\n if (closed != null && closed !== slice) {\n try {\n parsed = JSON.parse(closed)\n break\n } catch {\n // give up on this candidate\n }\n }\n }\n }\n\n if (parsed == null) return []\n\n // Normalize: accept `{proposals: [...]}` or a bare array.\n let proposalsRaw: unknown\n if (Array.isArray(parsed)) {\n proposalsRaw = parsed\n } else if (parsed && typeof parsed === 'object') {\n proposalsRaw = (parsed as { proposals?: unknown }).proposals\n }\n if (!Array.isArray(proposalsRaw)) return []\n\n const out: ReflectionProposal[] = []\n for (const p of proposalsRaw) {\n if (!p || typeof p !== 'object') continue\n const obj = p as { label?: unknown; rationale?: unknown; payload?: unknown }\n if (!('payload' in obj)) continue\n out.push({\n label: typeof obj.label === 'string' ? obj.label : 'mutation',\n rationale: typeof obj.rationale === 'string' ? obj.rationale : '',\n payload: obj.payload,\n })\n if (maxProposals !== undefined && out.length >= maxProposals) break\n }\n return out\n}\n","/**\n * Researcher interface — stable hook for an external autonomous-research\n * agent to drive the meta-loop.\n *\n * Implementations live downstream (typically in a private repo that\n * runs the actual LLM). This package ships only the contract + a\n * `NoopResearcher` so consumers can wire the surface without being\n * forced to implement every method up front.\n *\n * The four methods mirror the four stages of the paper \"Two Loops,\n * Three Roles\":\n *\n * inspectFailures — given the observed runs, what failure modes\n * are present? (data → diagnosis)\n * proposeChange — given diagnosed failure modes, what\n * structural changes should we try?\n * (diagnosis → plan delta)\n * applyChange — fold the proposed deltas into a concrete\n * experiment plan against an existing baseline.\n * (plan delta → executable plan)\n * evaluateChange — run the plan, return runs + the gate verdict.\n * (executable plan → verdict)\n *\n * Composition is the discipline: a Researcher implementation MUST\n * keep these four steps separate and inspectable. Conflating\n * \"diagnose + propose + run\" into a single LLM call defeats the\n * point of the framework — you can't audit which step lied.\n *\n * THIS INTERFACE IS STABLE. Breaking changes require a new module\n * (e.g. `Researcher2`) so existing implementations keep working.\n */\n\nimport type { GateDecision } from './held-out-gate'\nimport type { RunRecord, RunSplitTag } from './run-record'\n\n/** A diagnosed failure mode with the run-IDs that exhibit it. */\nexport interface FailureMode {\n /** Short machine-readable code. Must be stable across runs of the\n * same researcher to enable longitudinal tracking. */\n code: string\n /** Human-readable description for the paper / dashboard. */\n description: string\n evidence: {\n /** Run IDs (from `RunRecord.runId`) where this failure mode was\n * observed. */\n runIds: string[]\n /** Number of run samples that informed the diagnosis. */\n samples: number\n }\n}\n\n/** A single steering change the researcher wants to try. */\nexport interface SteeringChange {\n kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget'\n /** Implementation-specific payload. Researcher implementations\n * define the schema — keep this `unknown` here to avoid coupling\n * the public interface to any one researcher's internal model. */\n payload: unknown\n /** Why the researcher proposed this change. Goes into the audit\n * trail next to the failure-mode evidence. */\n rationale: string\n /** Optional self-reported expected delta on the headline metric. */\n expectedDelta?: number\n}\n\n/** A single experiment plan, mapped onto the search/holdout splits. */\nexport interface ExperimentPlan {\n baselineCandidateId: string\n proposedCandidateId: string\n changes: SteeringChange[]\n /** USD ceiling for the entire experiment. The runner must stop\n * before exceeding this and report a partial result. */\n evaluationBudgetUsd: number\n /** Item IDs (your dataset keys) for the search vs holdout splits. */\n splits: { search: string[]; holdout: string[] }\n}\n\n/** Result of running a plan: every run, plus the gate verdict. */\nexport interface ExperimentResult {\n plan: ExperimentPlan\n runs: RunRecord[]\n gateDecision: GateDecision\n}\n\n/**\n * The researcher loop. Stable, four-step, inspectable.\n *\n * ┌──────────┐ inspectFailures ┌──────────┐ proposeChange ┌──────────┐\n * │ runs │ ─────────────────▶│ failures │ ──────────────▶│ changes │\n * └──────────┘ └──────────┘ └────┬─────┘\n * │\n * ▼\n * ┌────────────────┐ applyChange ┌────────┐\n * │ ExperimentPlan │ ◀────────────│ base │\n * └────────┬───────┘ └────────┘\n * │\n * evaluateChange ▼\n * ┌────────────────┐\n * │ ExperimentResult│\n * └────────────────┘\n */\nexport interface Researcher {\n inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>\n proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>\n applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>\n evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>\n}\n\nexport interface CallbackResearcherOptions {\n inspectFailures: Researcher['inspectFailures']\n proposeChange: Researcher['proposeChange']\n applyChange: Researcher['applyChange']\n evaluateChange: Researcher['evaluateChange']\n}\n\n/**\n * Minimal concrete researcher for tests, scripts, and small integrations.\n * Larger autonomous researchers can still implement `Researcher` directly.\n */\nexport class CallbackResearcher implements Researcher {\n constructor(private readonly callbacks: CallbackResearcherOptions) {}\n\n inspectFailures(runs: RunRecord[]): Promise<FailureMode[]> {\n return this.callbacks.inspectFailures(runs)\n }\n\n proposeChange(failures: FailureMode[]): Promise<SteeringChange[]> {\n return this.callbacks.proposeChange(failures)\n }\n\n applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan> {\n return this.callbacks.applyChange(changes, baseline)\n }\n\n evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult> {\n return this.callbacks.evaluateChange(plan)\n }\n}\n\n/**\n * No-op researcher — fails loud on every method. Use as a placeholder\n * in code paths that wire the interface but don't have an implementation\n * yet. Importantly, this does NOT silently succeed: a no-op researcher\n * that returned empty arrays would muffle the loop's signal that\n * nobody implemented the brain.\n */\nexport class NoopResearcher implements Researcher {\n private readonly hint: string\n\n constructor(hint = 'NoopResearcher: no implementation wired') {\n this.hint = hint\n }\n\n async inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]> {\n throw new Error(`${this.hint} (inspectFailures not implemented)`)\n }\n\n async proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]> {\n throw new Error(`${this.hint} (proposeChange not implemented)`)\n }\n\n async applyChange(\n _changes: SteeringChange[],\n _baseline: ExperimentPlan,\n ): Promise<ExperimentPlan> {\n throw new Error(`${this.hint} (applyChange not implemented)`)\n }\n\n async evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult> {\n throw new Error(`${this.hint} (evaluateChange not implemented)`)\n }\n}\n\n/** Re-export the split alias so callers don't have to import twice. */\nexport type { RunSplitTag }\n"],"mappings":";;;;;;;;;AAqJA,IAAM,uBAAsD;AAAA,EAC1D,UAAU;AAAA,EACV,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,YAAY;AACd;AAEO,IAAM,kCAAN,MAAyE;AAAA,EAC7D,eAAe,oBAAI,IAAgC;AAAA,EAEpE,MAAM,KAAK,YAA+C;AACxD,SAAK,aAAa,IAAI,WAAW,IAAI,gBAAgB,UAAU,CAAC;AAAA,EAClE;AAAA,EAEA,MAAM,IAAI,IAAgD;AACxD,UAAM,aAAa,KAAK,aAAa,IAAI,EAAE;AAC3C,WAAO,aAAa,gBAAgB,UAAU,IAAI;AAAA,EACpD;AAAA,EAEA,MAAM,KAAK,SAAmC,CAAC,GAAkC;AAC/E,WAAO,CAAC,GAAG,KAAK,aAAa,OAAO,CAAC,EAClC,OAAO,CAAC,eAAe,cAAc,YAAY,MAAM,CAAC,EACxD,IAAI,eAAe;AAAA,EACxB;AAAA,EAEA,MAAM,cAAc,IAAY,SAAuD;AACrF,UAAM,aAAa,KAAK,aAAa,IAAI,EAAE;AAC3C,QAAI,CAAC;AACH,YAAM,IAAI,MAAM,8DAA8D,EAAE,GAAG;AACrF,UAAM,OAAO,gBAAgB;AAAA,MAC3B,GAAG;AAAA,MACH,UAAU,CAAC,GAAG,WAAW,UAAU,OAAO;AAAA,MAC1C,WAAW,QAAQ;AAAA,IACrB,CAAC;AACD,SAAK,aAAa,IAAI,IAAI,IAAI;AAC9B,WAAO,gBAAgB,IAAI;AAAA,EAC7B;AAAA,EAEA,MAAM,YACJ,IACA,OACA,WAC6B;AAC7B,UAAM,aAAa,KAAK,aAAa,IAAI,EAAE;AAC3C,QAAI,CAAC;AACH,YAAM,IAAI,MAAM,4DAA4D,EAAE,GAAG;AACnF,UAAM,WAAW,YACb,WAAW,SAAS;AAAA,MAAI,CAAC,YACvB,QAAQ,OAAO,YACX,EAAE,GAAG,SAAS,UAAU,CAAC,GAAI,QAAQ,YAAY,CAAC,GAAI,KAAK,EAAE,IAC7D;AAAA,IACN,IACA,WAAW;AACf,UAAM,OAAO,gBAAgB;AAAA,MAC3B,GAAG;AAAA,MACH;AAAA,MACA,QAAQ,YAAY,WAAW,SAAS,CAAC,GAAG,WAAW,QAAQ,KAAK;AAAA,MACpE,WAAW,MAAM;AAAA,IACnB,CAAC;AACD,SAAK,aAAa,IAAI,IAAI,IAAI;AAC9B,WAAO,gBAAgB,IAAI;AAAA,EAC7B;AACF;AAEO,IAAM,oCAAN,MAA2E;AAAA,EAC/D;AAAA,EACA,SAAS,IAAI,gCAAgC;AAAA,EACtD,SAAS;AAAA,EAEjB,YAAY,SAA0B;AACpC,SAAK,MAAM,QAAQ;AAAA,EACrB;AAAA,EAEA,MAAM,KAAK,YAA+C;AACxD,UAAM,KAAK,KAAK;AAChB,UAAM,KAAK,OAAO,KAAK,UAAU;AACjC,UAAM,KAAK,OAAO,EAAE,IAAI,QAAQ,WAAW,CAAC;AAAA,EAC9C;AAAA,EAEA,MAAM,IAAI,IAAgD;AACxD,UAAM,KAAK,KAAK;AAChB,WAAO,KAAK,OAAO,IAAI,EAAE;AAAA,EAC3B;AAAA,EAEA,MAAM,KAAK,SAAmC,CAAC,GAAkC;AAC/E,UAAM,KAAK,KAAK;AAChB,WAAO,KAAK,OAAO,KAAK,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,cAAc,IAAY,SAAuD;AACrF,UAAM,KAAK,KAAK;AAChB,UAAM,OAAO,MAAM,KAAK,OAAO,cAAc,IAAI,OAAO;AACxD,UAAM,KAAK,OAAO,EAAE,IAAI,iBAAiB,IAAI,QAAQ,CAAC;AACtD,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,YACJ,IACA,OACA,WAC6B;AAC7B,UAAM,KAAK,KAAK;AAChB,UAAM,OAAO,MAAM,KAAK,OAAO,YAAY,IAAI,OAAO,SAAS;AAC/D,UAAM,KAAK,OAAO,EAAE,IAAI,eAAe,IAAI,OAAO,UAAU,CAAC;AAC7D,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,OAAO,QAAgC;AACnD,UAAM,EAAE,YAAY,MAAM,IAAI,MAAM,OAAO,aAAkB;AAC7D,UAAM,EAAE,KAAK,IAAI,MAAM,OAAO,MAAW;AACzC,UAAM,MAAM,KAAK,KAAK,EAAE,WAAW,KAAK,CAAC;AACzC,UAAM;AAAA,MACJ,KAAK,KAAK,KAAK,8BAA8B;AAAA,MAC7C,GAAG,KAAK,UAAU,MAAM,CAAC;AAAA;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,OAAsB;AAClC,QAAI,KAAK,OAAQ;AACjB,UAAM,EAAE,SAAS,IAAI,MAAM,OAAO,aAAkB;AACpD,UAAM,EAAE,KAAK,IAAI,MAAM,OAAO,MAAW;AACzC,UAAM,OAAO,KAAK,KAAK,KAAK,8BAA8B;AAC1D,QAAI;AACF,YAAM,MAAM,MAAM,SAAS,MAAM,MAAM;AACvC,iBAAW,QAAQ,IAAI,MAAM,IAAI,GAAG;AAClC,YAAI,CAAC,KAAK,KAAK,EAAG;AAClB,YAAI;AACF,gBAAM,SAAS,KAAK,MAAM,IAAI;AAI9B,cAAI,OAAO,OAAO,OAAQ,OAAM,KAAK,OAAO,KAAK,OAAO,UAAU;AAClE,cAAI,OAAO,OAAO;AAChB,kBAAM,KAAK,OAAO,cAAc,OAAO,IAAI,OAAO,OAAO;AAC3D,cAAI,OAAO,OAAO;AAChB,kBAAM,KAAK,OAAO,YAAY,OAAO,IAAI,OAAO,OAAO,OAAO,SAAS;AAAA,QAC3E,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF,QAAQ;AAAA,IAER;AACA,SAAK,SAAS;AAAA,EAChB;AACF;AAEO,SAAS,yBAAyB,OAYlB;AACrB,QAAM,YAAY,MAAM,cAAa,oBAAI,KAAK,GAAE,YAAY;AAC5D,QAAM,KACJ,MAAM,MACN,MAAM,WAAW,GAAG,MAAM,aAAa,EAAE,IAAI,MAAM,cAAc,EAAE,IAAI,MAAM,KAAK,MAAM,IAAI,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC;AACvH,SAAO;AAAA,IACL;AAAA,IACA,WAAW,MAAM;AAAA,IACjB,YAAY,MAAM;AAAA,IAClB,MAAM,MAAM;AAAA,IACZ,UAAU,MAAM,YAAY,CAAC;AAAA,IAC7B,QAAQ,MAAM,UAAU,CAAC;AAAA,IACzB,SAAS,MAAM;AAAA,IACf,OAAO,MAAM;AAAA,IACb,MAAM,MAAM;AAAA,IACZ;AAAA,IACA,UAAU,MAAM;AAAA,EAClB;AACF;AAEO,SAAS,oBACd,YACA,SAA8B,CAAC,GACjB;AACd,QAAM,QAAQ,EAAE,GAAG,sBAAsB,GAAG,OAAO;AACnD,QAAM,QAAQ,MAAM,WAAW,MAAM,SAAS,MAAM,UAAU,MAAM;AACpE,MAAI,SAAS,EAAG,OAAM,IAAI,MAAM,4DAA4D;AAC5F,QAAM,SACJ;AAAA,IACE,GAAG,WAAW,aAAa,EAAE,IAAI,WAAW,cAAc,EAAE,IAAI,WAAW,EAAE,IAAI,WAAW,KAAK,MAAM;AAAA,EACzG,IAAI;AACN,MAAI,SAAS,MAAM,SAAU,QAAO;AACpC,MAAI,SAAS,MAAM,WAAW,MAAM,OAAQ,QAAO;AACnD,MAAI,SAAS,MAAM,WAAW,MAAM,SAAS,MAAM,QAAS,QAAO;AACnE,SAAO;AACT;AAEO,SAAS,0BACd,YACA,QACoB;AACpB,SAAO;AAAA,IACL,GAAG;AAAA,IACH,OAAO,WAAW,SAAS,oBAAoB,YAAY,MAAM;AAAA,EACnE;AACF;AAEO,SAAS,oCACd,YACiB;AACjB,QAAM,YAAY,0BAA0B,UAAU;AACtD,SAAO;AAAA,IACL,IAAI,UAAU,cAAc,UAAU;AAAA,IACtC,OAAO,UAAU;AAAA,IACjB,SAAS;AAAA,IACT,MAAM;AAAA,MACJ,GAAI,UAAU,YAAY,EAAE,WAAW,UAAU,UAAU,IAAI,CAAC;AAAA,MAChE,GAAI,UAAU,QAAQ,CAAC;AAAA,MACvB,QAAQ;AAAA,IACV;AAAA,EACF;AACF;AAEO,SAAS,uCACd,cACmB;AACnB,SAAO,aAAa,IAAI,mCAAmC;AAC7D;AAEO,SAAS,iCACd,YACsB;AACtB,QAAM,SAAS,UAAU,UAAU;AACnC,SAAO;AAAA,IACL,YAAY,WAAW,cAAc,WAAW;AAAA,IAChD,cAAc,WAAW;AAAA,IACzB,YAAY,CAAC,GAAG,IAAI,IAAI,OAAO,IAAI,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC;AAAA,IAC1D,OAAO,WAAW,SAAS,SAAS,gBAAgB,MAAM;AAAA,IAC1D,UAAU;AAAA,MACR,WAAW,WAAW;AAAA,MACtB,OAAO,WAAW;AAAA,MAClB,QAAQ,WAAW,KAAK;AAAA,MACxB,UAAU,WAAW,SAAS;AAAA,MAC9B,SAAS,WAAW;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,oCACd,cACwB;AACxB,SAAO,aAAa,IAAI,gCAAgC;AAC1D;AAEA,eAAsB,yBACpB,YACA,SAC+B;AAC/B,MAAI;AACF,UAAM,SAAS,MAAM,QAAQ,OAAO,UAAU;AAC9C,WAAO;AAAA,MACL,cAAc,WAAW;AAAA,MACzB,GAAG;AAAA,IACL;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,aAAY,oBAAI,KAAK,GAAE,YAAY;AACzC,UAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,WAAO;AAAA,MACL,cAAc,WAAW;AAAA,MACzB,MAAM;AAAA,MACN,QAAQ;AAAA,QACN;AAAA,UACE,QAAQ;AAAA,UACR,MAAM;AAAA,UACN,OAAO;AAAA,UACP,QAAQ;AAAA,UACR,UAAU;AAAA,UACV;AAAA,QACF;AAAA,MACF;AAAA,MACA,SAAS;AAAA,QACP,SAAS;AAAA,QACT,OAAO;AAAA,QACP,QAAQ;AAAA,QACR,YAAY;AAAA,MACd;AAAA,MACA,UAAU,EAAE,aAAa,KAAK;AAAA,IAChC;AAAA,EACF;AACF;AAEA,eAAsB,2BACpB,cACA,SACiC;AACjC,QAAM,UAAkC,CAAC;AACzC,aAAW,cAAc,cAAc;AACrC,YAAQ,KAAK,MAAM,yBAAyB,YAAY,OAAO,CAAC;AAAA,EAClE;AACA,SAAO;AACT;AAEO,SAAS,0BACd,cACA,UAAmC,CAAC,GACX;AACzB,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,UAAmC,CAAC;AAC1C,aAAW,cAAc,cAAc;AACrC,eAAW,SAAS,UAAU,UAAU,GAAG;AACzC,YAAM,cAAc,qBAAqB,YAAY,KAAK;AAC1D,UAAI,CAAC,YAAa;AAClB,cAAQ,KAAK;AAAA,QACX;AAAA,QACA,WAAW,MAAM,UAAU,GAAG,MAAM,IAAI,eAAe,MAAM,MAAM;AAAA,QACnE,QAAQ,eAAe,KAAK;AAAA,QAC5B,oBAAoB,WAAW;AAAA,QAC/B,eAAe,MAAM;AAAA,QACrB,UAAU,MAAM;AAAA,MAClB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,gBAAgB,oBAAI,IAAmC;AAC7D,aAAW,SAAS,SAAS;AAC3B,UAAM,MAAM,MAAM,YAAY,YAAY,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACtE,UAAM,WAAW,cAAc,IAAI,GAAG;AACtC,QAAI,CAAC,YAAY,MAAM,SAAS,SAAS,OAAQ,eAAc,IAAI,KAAK,KAAK;AAAA,EAC/E;AACA,SAAO,CAAC,GAAG,cAAc,OAAO,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU;AAC5F;AAEO,SAAS,+BAA+B,SAA0C;AACvF,QAAM,QAAQ,CAAC,uBAAuB,EAAE;AACxC,aAAW,SAAS,SAAS;AAC3B,UAAM,KAAK,KAAK,MAAM,WAAW,EAAE;AACnC,UAAM,KAAK,gBAAgB,MAAM,SAAS,EAAE;AAC5C,UAAM,KAAK,aAAa,MAAM,kBAAkB,EAAE;AAClD,UAAM,KAAK,EAAE;AAAA,EACf;AACA,SAAO,GAAG,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC;AAAA;AACnC;AAEO,SAAS,mCAAmC,cAA4C;AAC7F,SAAO,GAAG,aACP,MAAM,EACN,KAAK,CAAC,GAAG,MAAM,EAAE,GAAG,cAAc,EAAE,EAAE,CAAC,EACvC,IAAI,CAAC,eAAe,KAAK,UAAU,aAAa,UAAU,CAAC,CAAC,EAC5D,KAAK,IAAI,CAAC;AAAA;AACf;AAEO,SAAS,+BAA+B,OAAqC;AAClF,QAAM,eAAqC,CAAC;AAC5C,aAAW,QAAQ,MAAM,MAAM,IAAI,GAAG;AACpC,QAAI,CAAC,KAAK,KAAK,EAAG;AAClB,iBAAa,KAAK,KAAK,MAAM,IAAI,CAAuB;AAAA,EAC1D;AACA,SAAO;AACT;AAEO,SAAS,+BACd,KACA,UASI,CAAC,GACe;AACpB,QAAM,YAAY,QAAQ,cAAa,oBAAI,KAAK,GAAE,YAAY;AAC9D,QAAM,eACJ,IAAI,SAAS,cAAc,WAAW,GAAG,IAAI,MAAM,IAAI,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC;AAClF,SAAO,yBAAyB;AAAA,IAC9B,IAAI;AAAA,IACJ,WAAW,QAAQ;AAAA,IACnB,YAAY,QAAQ;AAAA,IACpB,MAAM,EAAE,QAAQ,IAAI,OAAO;AAAA,IAC3B;AAAA,IACA,UAAU,IAAI,MAAM,IAAI,CAAC,UAAU;AAAA,MACjC,IAAI,GAAG,YAAY,SAAS,KAAK,KAAK;AAAA,MACtC,WAAW,KAAK;AAAA,MAChB,cAAc,QAAQ,gBAAgB;AAAA,MACtC,UAAU,QAAQ,mBAAmB,IAAI,KAAK,KAAK,eAAe,UAAU,KAAK;AAAA,MACjF,gBAAgB,QAAQ,yBAAyB,IAAI;AAAA,MACrD,OAAO,KAAK;AAAA,MACZ,WAAW,KAAK;AAAA,MAChB,UAAU;AAAA,QACR,UAAU,KAAK;AAAA,QACf,eAAe,KAAK;AAAA,MACtB;AAAA,IACF,EAAE;AAAA,IACF,QAAQ;AAAA,MACN;AAAA,QACE,QAAQ;AAAA,QACR,MAAM,IAAI,OAAO,YAAY;AAAA,QAC7B,OAAO,IAAI;AAAA,QACX,QAAQ,IAAI;AAAA,QACZ,UAAU,IAAI,OAAO,SAAS;AAAA,QAC9B;AAAA,MACF;AAAA,IACF;AAAA,IACA,SAAS;AAAA,MACP,SAAS,IAAI;AAAA,MACb,OAAO,IAAI;AAAA,MACX,SAAS,IAAI;AAAA,MACb,QAAQ,IAAI;AAAA,MACZ,YAAY;AAAA,MACZ,UAAU;AAAA,QACR,WAAW,IAAI;AAAA,QACf,cAAc,IAAI;AAAA,MACpB;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,UAAU,YAAiD;AAClE,QAAM,SAAS;AAAA,IACb,GAAG,WAAW;AAAA,IACd,GAAG,WAAW,SAAS,QAAQ,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;AAAA,EACpE;AACA,QAAM,OAAO,oBAAI,IAAY;AAC7B,SAAO,OAAO,OAAO,CAAC,UAAU;AAC9B,UAAM,MACJ,MAAM,MAAM,GAAG,MAAM,MAAM,IAAI,MAAM,IAAI,IAAI,MAAM,SAAS,IAAI,KAAK,UAAU,MAAM,KAAK,CAAC;AAC7F,QAAI,KAAK,IAAI,GAAG,EAAG,QAAO;AAC1B,SAAK,IAAI,GAAG;AACZ,WAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,gBAAgB,QAA6C;AACpE,MAAI,CAAC,OAAO,OAAQ,QAAO;AAC3B,QAAM,SAAS,OACZ,IAAI,CAAC,UAAU;AACd,QAAI,MAAM,SAAS,aAAa,MAAM,SAAS,SAAU,QAAO;AAChE,QAAI,MAAM,SAAS,YAAY,MAAM,SAAS,eAAgB,QAAO;AACrE,QAAI,MAAM,SAAS,UAAU,OAAO,MAAM,UAAU;AAClD,aAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,KAAK,CAAC;AAC7C,WAAO;AAAA,EACT,CAAC,EACA,OAAO,CAAC,UAA2B,OAAO,UAAU,QAAQ;AAC/D,MAAI,CAAC,OAAO,OAAQ,QAAO;AAC3B,SAAO,KAAK,MAAO,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO,SAAU,GAAI,IAAI;AAC9F;AAEA,SAAS,qBACP,YACA,OACoB;AACpB,MAAI,MAAM,SAAS,YAAY,MAAM;AACnC,WAAO,uBAAuB,QAAQ,WAAW,KAAK,QAAQ,EAAE,CAAC,WAAW,MAAM,MAAM;AAC1F,MAAI,MAAM,SAAS,sBAAsB,MAAM;AAC7C,WAAO,oCAAoC,MAAM,MAAM;AACzD,MAAI,MAAM,SAAS,YAAY,MAAM;AACnC,WAAO,gCAAgC,QAAQ,WAAW,KAAK,QAAQ,EAAE,CAAC,cAAc,MAAM,MAAM;AACtG,MAAI,MAAM,SAAS,aAAa,MAAM;AACpC,WAAO,oCAAoC,QAAQ,WAAW,KAAK,QAAQ,EAAE,CAAC,MAAM,MAAM,MAAM;AAClG,MAAI,MAAM,SAAS,aAAa,MAAM,OAAQ,QAAO,MAAM;AAC3D,SAAO;AACT;AAEA,SAAS,eAAe,OAA8B;AACpD,QAAM,WACJ,MAAM,aAAa,aACf,IACA,MAAM,aAAa,UACjB,IACA,MAAM,aAAa,YACjB,IACA;AACV,QAAM,SACJ,MAAM,WAAW,SACb,IACA,MAAM,WAAW,YAAY,MAAM,WAAW,gBAC5C,IACA;AACR,SAAO,WAAW;AACpB;AAEA,SAAS,cAAc,YAAgC,QAA2C;AAChG,MAAI,OAAO,aAAa,WAAW,cAAc,OAAO,UAAW,QAAO;AAC1E,MAAI,OAAO,cAAc,WAAW,eAAe,OAAO,WAAY,QAAO;AAC7E,MAAI,OAAO,SAAS,WAAW,UAAU,OAAO,MAAO,QAAO;AAC9D,MAAI,OAAO,KAAK;AACd,UAAM,CAAC,KAAK,KAAK,IAAI,OAAO;AAC5B,QAAI,WAAW,OAAO,GAAG,MAAM,MAAO,QAAO;AAAA,EAC/C;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,YAAoD;AAC3E,SAAO,KAAK,MAAM,KAAK,UAAU,UAAU,CAAC;AAC9C;AAEA,SAAS,QAAQ,OAAe,KAAqB;AACnD,QAAM,aAAa,MAAM,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACnD,SAAO,WAAW,SAAS,MAAM,GAAG,WAAW,MAAM,GAAG,GAAG,EAAE,KAAK,CAAC,QAAQ;AAC7E;AAEA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO;AACX,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;AACxC,YAAQ,MAAM,WAAW,CAAC;AAC1B,WAAO,KAAK,KAAK,MAAM,QAAQ;AAAA,EACjC;AACA,SAAO,SAAS;AAClB;AAEA,SAAS,aAAa,OAAyB;AAC7C,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO;AACxD,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,IAAI,YAAY;AACvD,QAAM,MAA+B,CAAC;AACtC,aAAW,OAAO,OAAO,KAAK,KAAgC,EAAE,KAAK,GAAG;AACtE,QAAI,GAAG,IAAI,aAAc,MAAkC,GAAG,CAAC;AAAA,EACjE;AACA,SAAO;AACT;;;ACzjBO,IAAM,cAAN,MAAkB;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,QAA2B;AACrC,QAAI,CAAC,OAAO,aAAa;AACvB,YAAM,IAAI,MAAM,sCAAsC;AAAA,IACxD;AACA,SAAK,oBAAoB,OAAO,qBAAqB;AACrD,SAAK,uBAAuB,OAAO,wBAAwB;AAC3D,SAAK,sBAAsB,OAAO,uBAAuB;AACzD,SAAK,cAAc,OAAO;AAC1B,SAAK,aAAa,OAAO,cAAc;AACvC,SAAK,YAAY,OAAO,sBAAsB;AAC9C,SAAK,OAAO,OAAO;AAAA,EACrB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,WAAwB,UAAqC;AACpE,UAAM,cAAc,iBAAiB,WAAW,KAAK,WAAW;AAChE,UAAM,aAAa,KAAK;AAGxB,UAAM,uBAAuB,kBAAkB,QAAQ;AACvD,UAAM,gBAA0B,CAAC;AACjC,UAAM,eAAyB,CAAC;AAChC,eAAW,OAAO,WAAW;AAC3B,UAAI,IAAI,aAAa,UAAW;AAChC,UAAI,IAAI,QAAQ,iBAAiB,OAAW;AAC5C,YAAM,MAAM,QAAQ,GAAG;AACvB,YAAM,cAAc,qBAAqB,IAAI,GAAG;AAChD,UAAI,gBAAgB,OAAW;AAC/B,oBAAc,KAAK,WAAW;AAC9B,mBAAa,KAAK,IAAI,QAAQ,YAAY;AAAA,IAC5C;AAEA,UAAM,iBAAiB,cAAc;AAIrC,UAAM,sBAAsB,KAAK,OAAO,WAAW,eAAe,QAAQ,CAAC;AAC3E,UAAM,uBAAuB,KAAK,OAAO,WAAW,gBAAgB,SAAS,CAAC;AAC9E,UAAM,qBAAqB,KAAK,OAAO,UAAU,eAAe,QAAQ,CAAC;AACzE,UAAM,sBAAsB,KAAK,OAAO,UAAU,gBAAgB,SAAS,CAAC;AAE5E,UAAM,aAAa,SAAS,qBAAqB,oBAAoB;AACrE,UAAM,qBAAqB,SAAS,oBAAoB,mBAAmB;AAG3E,QAAI,iBAAiB,KAAK,mBAAmB;AAC3C,aAAO;AAAA,QACL,SAAS;AAAA,QACT;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR;AAAA,UACA,mBAAmB,iBAAiB,IAAI,YAAY,eAAe,YAAY,IAAI;AAAA,UACnF,UAAU,EAAE,KAAK,GAAG,MAAM,EAAE;AAAA,UAC5B,cAAc;AAAA,UACd,aAAa;AAAA,UACb,cAAc;AAAA,UACd;AAAA,UACA;AAAA,QACF;AAAA,QACA,QAAQ,aAAa,cAAc,wCAAwC,KAAK,iBAAiB;AAAA,QACjG,eAAe;AAAA,MACjB;AAAA,IACF;AAGA,UAAM,KAAK,gBAAgB,eAAe,cAAc;AAAA,MACtD,YAAY,KAAK;AAAA,MACjB,WAAW,KAAK;AAAA,MAChB,WAAW;AAAA,MACX,MAAM,KAAK;AAAA,IACb,CAAC;AACD,UAAM,WAAW,mBAAmB,eAAe,YAAY;AAE/D,UAAM,WAAyB;AAAA,MAC7B;AAAA,MACA,mBAAmB,GAAG;AAAA,MACtB,UAAU,EAAE,KAAK,GAAG,KAAK,MAAM,GAAG,KAAK;AAAA,MACvC,cAAc,SAAS;AAAA,MACvB,aAAa;AAAA,MACb,cAAc;AAAA,MACd;AAAA,MACA;AAAA,IACF;AAGA,QAAI,EAAE,GAAG,MAAM,KAAK,uBAAuB;AACzC,aAAO;AAAA,QACL,SAAS;AAAA,QACT;AAAA,QACA;AAAA,QACA;AAAA,QACA,QACE,gDAA2C,IAAI,GAAG,MAAM,CAAC,QAClD,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC,8BAA8B,IAAI,KAAK,oBAAoB,CAAC;AAAA,QACjG,eAAe;AAAA,MACjB;AAAA,IACF;AAIA,QACE,OAAO,SAAS,UAAU,KAC1B,OAAO,SAAS,kBAAkB,KAClC,aAAa,qBAAqB,KAAK,qBACvC;AACA,aAAO;AAAA,QACL,SAAS;AAAA,QACT;AAAA,QACA;AAAA,QACA;AAAA,QACA,QACE,8BAA8B,IAAI,UAAU,CAAC,yBAAyB,IAAI,kBAAkB,CAAC,iBAC7E,IAAI,KAAK,mBAAmB,CAAC;AAAA,QAC/C,eAAe;AAAA,MACjB;AAAA,IACF;AAEA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,MACA;AAAA,MACA;AAAA,MACA,QACE,yCAAoC,IAAI,GAAG,MAAM,CAAC,QAC3C,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC,UAAU,cAAc,iCAClC,IAAI,UAAU,CAAC,gBAAgB,IAAI,kBAAkB,CAAC;AAAA,MACjF,eAAe;AAAA,IACjB;AAAA,EACF;AACF;AAIA,SAAS,iBAAiB,WAAwB,aAA6B;AAC7E,aAAW,OAAO,WAAW;AAC3B,QAAI,IAAI,eAAe,IAAI,gBAAgB,YAAa,QAAO,IAAI;AAAA,EACrE;AAGA,SAAO,UAAU,CAAC,GAAG,eAAe;AACtC;AAEA,SAAS,kBAAkB,MAAwC;AACjE,QAAM,MAAM,oBAAI,IAAoB;AACpC,aAAW,KAAK,MAAM;AACpB,QAAI,EAAE,aAAa,UAAW;AAC9B,QAAI,EAAE,QAAQ,iBAAiB,OAAW;AAC1C,QAAI,IAAI,QAAQ,CAAC,GAAG,EAAE,QAAQ,YAAY;AAAA,EAC5C;AACA,SAAO;AACT;AAEA,SAAS,QAAQ,GAAsB;AACrC,SAAO,GAAG,EAAE,YAAY,KAAK,EAAE,IAAI;AACrC;AAEA,SAAS,OACP,MACA,OACA,aACU;AACV,QAAM,MAAgB,CAAC;AACvB,aAAW,KAAK,MAAM;AACpB,QAAI,EAAE,aAAa,YAAa;AAChC,UAAM,IAAI,EAAE,QAAQ,KAAK;AACzB,QAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,EAAG,KAAI,KAAK,CAAC;AAAA,EAC7D;AACA,SAAO;AACT;AAEA,SAAS,KAAK,IAAsB;AAClC,MAAI,GAAG,WAAW,EAAG,QAAO,OAAO;AACnC,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAEA,SAAS,SAAS,GAAW,GAAmB;AAC9C,MAAI,CAAC,OAAO,SAAS,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO,OAAO;AAC9D,SAAO,IAAI;AACb;AAEA,SAAS,YAAY,QAAkB,OAAyB;AAC9D,QAAM,KAAK,OAAO,IAAI,CAAC,GAAG,MAAM,MAAM,CAAC,IAAK,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACnE,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,QAAM,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;AACpC,SAAO,GAAG,SAAS,MAAM,KAAK,GAAG,MAAM,CAAC,IAAK,GAAG,GAAG,KAAM,IAAI,GAAG,GAAG;AACrE;AAEA,SAAS,IAAI,GAAmB;AAC9B,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO,OAAO,CAAC;AACxC,SAAO,EAAE,QAAQ,CAAC;AACpB;;;AC/QO,SAAS,UAAa,GAAM,GAAM,YAAqC;AAC5E,MAAI,iBAAiB;AACrB,aAAW,OAAO,YAAY;AAC5B,UAAM,KAAK,IAAI,MAAM,CAAC;AACtB,UAAM,KAAK,IAAI,MAAM,CAAC;AACtB,QAAI,CAAC,OAAO,SAAS,EAAE,KAAK,CAAC,OAAO,SAAS,EAAE,EAAG,QAAO;AACzD,UAAM,YAAY,IAAI,cAAc,aAAa,KAAK,KAAK,KAAK;AAChE,UAAM,WAAW,IAAI,cAAc,aAAa,KAAK,KAAK,KAAK;AAC/D,QAAI,SAAU,QAAO;AACrB,QAAI,UAAW,kBAAiB;AAAA,EAClC;AACA,SAAO;AACT;AAOO,SAAS,eAAkB,YAAiB,YAA6C;AAC9F,MAAI,WAAW,WAAW,GAAG;AAC3B,UAAM,IAAI,MAAM,+CAA+C;AAAA,EACjE;AACA,QAAM,QAAQ,WAAW,OAAO,CAAC,MAAM,WAAW,MAAM,CAAC,MAAM,OAAO,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAC3F,QAAM,WAAgB,CAAC;AACvB,QAAM,YAAiB,CAAC;AACxB,aAAW,KAAK,OAAO;AACrB,UAAM,cAAc,MAAM,KAAK,CAAC,UAAU,UAAU,KAAK,UAAU,OAAO,GAAG,UAAU,CAAC;AACxF,QAAI,YAAa,WAAU,KAAK,CAAC;AAAA,QAC5B,UAAS,KAAK,CAAC;AAAA,EACtB;AACA,QAAM,eAAe,SAAS,IAAI,CAAC,OAAO;AAAA,IACxC,WAAW;AAAA,IACX,WAAW,UAAU,OAAO,CAAC,MAAM,UAAU,GAAG,GAAG,UAAU,CAAC;AAAA,EAChE,EAAE;AACF,SAAO,EAAE,UAAU,WAAW,aAAa;AAC7C;AAWO,SAAS,YACd,YACA,YACA,UAAyD,CAAC,GAClB;AACxC,MAAI,WAAW,WAAW,EAAG,QAAO,CAAC;AACrC,QAAM,UAAU,QAAQ,WAAW,CAAC;AACpC,QAAM,cAAc,WAAW,OAAO,CAAC,GAAG,MAAM,KAAK,QAAQ,EAAE,IAAI,KAAK,IAAI,CAAC;AAG7E,QAAM,SAAS,WAAW,IAAI,CAAC,QAAQ;AACrC,UAAM,SAAS,WAAW,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,OAAO,SAAS,CAAC,CAAC;AACnF,QAAI,OAAO,WAAW,EAAG,QAAO,EAAE,KAAK,GAAG,KAAK,EAAE;AACjD,UAAM,MAAM,KAAK,IAAI,GAAG,MAAM;AAC9B,UAAM,MAAM,KAAK,IAAI,GAAG,MAAM;AAC9B,WAAO,EAAE,KAAK,KAAK,QAAQ,MAAM,MAAM,IAAI,IAAI;AAAA,EACjD,CAAC;AAED,SAAO,WAAW,IAAI,CAAC,MAAM;AAC3B,QAAI,QAAQ;AACZ,eAAW,QAAQ,CAAC,KAAK,MAAM;AAC7B,YAAM,IAAI,IAAI,MAAM,CAAC;AACrB,UAAI,CAAC,OAAO,SAAS,CAAC,EAAG;AACzB,YAAM,EAAE,KAAK,IAAI,IAAI,OAAO,CAAC;AAC7B,YAAM,cAAc,IAAI,QAAQ,MAAM;AACtC,YAAM,cAAc,IAAI,cAAc,aAAa,aAAa,IAAI;AACpE,YAAM,UAAU,QAAQ,IAAI,IAAI,KAAK,KAAK;AAC1C,eAAS,cAAc;AAAA,IACzB,CAAC;AACD,WAAO,EAAE,WAAW,GAAG,MAAM;AAAA,EAC/B,CAAC;AACH;AAcO,SAAS,iBACd,YACA,YAC2C;AAC3C,QAAM,YAAY,IAAI,IAAe,WAAW,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;AAElE,aAAW,OAAO,YAAY;AAC5B,UAAM,SAAS,CAAC,GAAG,UAAU,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC;AACzE,UAAM,MAAM,IAAI,MAAM,OAAO,CAAC,CAAE;AAChC,UAAM,MAAM,IAAI,MAAM,OAAO,OAAO,SAAS,CAAC,CAAE;AAChD,UAAM,QAAQ,MAAM,OAAO;AAG3B,cAAU,IAAI,OAAO,CAAC,GAAI,QAAQ;AAClC,cAAU,IAAI,OAAO,OAAO,SAAS,CAAC,GAAI,QAAQ;AAClD,aAAS,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK;AAC1C,YAAM,OAAO,IAAI,MAAM,OAAO,IAAI,CAAC,CAAE;AACrC,YAAM,OAAO,IAAI,MAAM,OAAO,IAAI,CAAC,CAAE;AACrC,YAAM,UAAU,UAAU,IAAI,OAAO,CAAC,CAAE;AACxC,UAAI,YAAY,SAAU;AAC1B,gBAAU,IAAI,OAAO,CAAC,GAAI,WAAW,OAAO,QAAQ,KAAK;AAAA,IAC3D;AAAA,EACF;AAEA,SAAO,WAAW,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,UAAU,UAAU,IAAI,CAAC,KAAK,EAAE,EAAE;AAClF;AAOO,SAAS,2BACd,YACA,YAC2C;AAC3C,QAAM,EAAE,SAAS,IAAI,eAAe,YAAY,UAAU;AAC1D,MAAI,SAAS,WAAW,EAAG,QAAO,CAAC;AACnC,QAAM,YAAY,iBAAiB,UAAU,UAAU;AACvD,SAAO,UAAU,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AACzD;;;ACpBO,IAAM,qBAAN,MAA+C;AAAA,EAC5C,QAAQ,oBAAI,IAAyB;AAAA,EAC7C,IAAI,KAAsC;AACxC,WAAO,KAAK,MAAM,IAAI,GAAG;AAAA,EAC3B;AAAA,EACA,IAAI,KAAa,OAA0B;AACzC,SAAK,MAAM,IAAI,KAAK,KAAK;AAAA,EAC3B;AAAA,EACA,OAAe;AACb,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EACA,QAAc;AACZ,SAAK,MAAM,MAAM;AAAA,EACnB;AACF;AAyCA,eAAsB,mBACpB,QACmC;AACnC,QAAM,cAAqC,CAAC;AAC5C,MAAI,aAAa,CAAC,GAAG,OAAO,YAAY;AACxC,MAAI,cAAmC,WAAW,CAAC;AACnD,MAAI,gBAAyC;AAE7C,WAAS,aAAa,GAAG,aAAa,OAAO,aAAa,cAAc;AACtE,WAAO,aAAa,EAAE,MAAM,oBAAoB,YAAY,gBAAgB,WAAW,OAAO,CAAC;AAE/F,UAAM,SAAS,MAAM,gBAAgB,YAAY,QAAQ,UAAU;AACnE,UAAM,aAAa,gBAAgB,YAAY,OAAO,aAAa,MAAM;AAEzE,UAAM,QAAQ,2BAA2B,YAAY,OAAO,UAAU;AACtE,UAAM,WAAW,IAAI,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,SAAS,CAAC;AAEhE,UAAM,SAAS,YAAY,YAAY,OAAO,YAAY,EAAE,SAAS,OAAO,cAAc,CAAC;AAC3F,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,UAAM,WAAW,OAAO,CAAC,GAAG,UAAU,aAAa,WAAW,CAAC,GAAG,aAAa,WAAW,CAAC,EAAG;AAE9F,UAAM,SAA8B;AAAA,MAClC,OAAO,OAAO;AAAA,MACd,QAAQ,OAAO;AAAA,MACf;AAAA,MACA,UAAU;AAAA,MACV;AAAA,MACA,gBAAgB,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,SAAS;AAAA,MACtD;AAAA,MACA;AAAA,IACF;AACA,gBAAY,KAAK,MAAM;AACvB,WAAO,aAAa,EAAE,MAAM,uBAAuB,OAAO,CAAC;AAE3D,UAAM,YAAY,WAAW,KAAK,CAAC,MAAM,EAAE,cAAc,QAAQ;AACjE,QAAI,WAAW;AACb,YAAM,SAAS,WAAW,KAAK,CAAC,MAAM,EAAE,OAAO,QAAQ;AACvD,UAAI,OAAQ,eAAc;AAC1B,sBAAgB;AAAA,IAClB;AAGA,QAAI,OAAO,6BAA6B,SAAS,YAAY,UAAU,GAAG;AACxE,YAAM,OAAO,YAAY,YAAY,SAAS,CAAC;AAC/C,YAAM,WACJ,KAAK,aAAa,YAAY,eAAe,KAAK,gBAAgB,CAAC,GAAG,QAAQ,CAAC;AACjF,UAAI,UAAU;AACZ,eAAO,aAAa;AAAA,UAClB,MAAM;AAAA,UACN;AAAA,UACA,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAAA,IACF;AAEA,QAAI,eAAe,OAAO,cAAc,EAAG;AAE3C,iBAAa,MAAM,eAAe,YAAY,YAAY,QAAQ,OAAO,QAAQ,aAAa,CAAC;AAAA,EACjG;AAEA,SAAO;AAAA,IACL,OAAO,OAAO;AAAA,IACd,QAAQ,OAAO;AAAA,IACf;AAAA,IACA;AAAA,IACA,eACE,iBACA,gBAAgB,YAAY,OAAO,aAAa,CAAC,CAAC,EAAE;AAAA,MAClD,CAAC,MAAM,EAAE,cAAc,YAAY;AAAA,IACrC;AAAA,EACJ;AACF;AAEA,eAAe,gBACb,YACA,QACA,YACwB;AACxB,QAAM,OAA0C,CAAC;AACjD,aAAW,WAAW,YAAY;AAChC,eAAW,cAAc,OAAO,aAAa;AAC3C,eAAS,MAAM,GAAG,MAAM,OAAO,MAAM,OAAO;AAC1C,aAAK,KAAK,YAAY;AACpB,gBAAM,WAAW,GAAG,QAAQ,EAAE,IAAI,UAAU,IAAI,GAAG;AACnD,gBAAM,SAAS,OAAO,OAAO,IAAI,QAAQ;AACzC,cAAI,QAAQ;AACV,mBAAO,aAAa;AAAA,cAClB,MAAM;AAAA,cACN;AAAA,cACA,WAAW,QAAQ;AAAA,cACnB;AAAA,cACA;AAAA,cACA,IAAI,OAAO;AAAA,cACX,OAAO,OAAO;AAAA,cACd,QAAQ;AAAA,YACV,CAAC;AACD,mBAAO;AAAA,UACT;AACA,gBAAM,SAAS,MAAM,OAAO,aAAa,MAAM,EAAE,SAAS,YAAY,IAAI,CAAC;AAC3E,iBAAO,OAAO,IAAI,UAAU,MAAM;AAClC,iBAAO,aAAa;AAAA,YAClB,MAAM;AAAA,YACN;AAAA,YACA,WAAW,QAAQ;AAAA,YACnB;AAAA,YACA;AAAA,YACA,IAAI,OAAO;AAAA,YACX,OAAO,OAAO;AAAA,YACd,QAAQ;AAAA,UACV,CAAC;AACD,iBAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO,mBAAmB,MAAM,OAAO,gBAAgB;AACzD;AAEA,eAAe,mBACb,MACA,aACc;AACd,QAAM,UAAe,IAAI,MAAM,KAAK,MAAM;AAC1C,QAAM,QAAQ,KAAK,IAAI,GAAG,WAAW;AACrC,MAAI,OAAO;AACX,iBAAe,SAAwB;AACrC,WAAO,MAAM;AACX,YAAM,IAAI;AACV,UAAI,KAAK,KAAK,OAAQ;AACtB,cAAQ,CAAC,IAAI,MAAM,KAAK,CAAC,EAAG;AAAA,IAC9B;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,MAAM,GAAG,MAAM,OAAO,CAAC,CAAC;AAC/D,SAAO;AACT;AAEA,SAAS,gBACP,YACA,aACA,QACoB;AACpB,SAAO,WAAW,IAAI,CAAC,YAAY;AACjC,UAAM,gBAAgB,OAAO,OAAO,CAAC,MAAM,EAAE,cAAc,QAAQ,EAAE;AACrE,UAAM,YAAY,YAAY,IAAuB,CAAC,QAAQ;AAC5D,YAAM,iBAAiB,cAAc,OAAO,CAAC,MAAM,EAAE,eAAe,GAAG;AACvE,YAAM,WAAW,eAAe,OAAO,CAAC,MAAM,EAAE,EAAE;AAMlD,YAAM,eAAe,eAAe,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK;AAC1D,YAAM,UAAU,iBAAiB,aAAa,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;AACzE,aAAO;AAAA,QACL,WAAW,QAAQ;AAAA,QACnB,YAAY;AAAA,QACZ,WAAWA,MAAK,aAAa,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;AAAA,QAChD,UAAUA,MAAK,aAAa,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;AAAA,QACnD,gBAAgBA,MAAK,aAAa,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;AAAA,QAC/D,QAAQ,eAAe,WAAW,IAAI,IAAI,SAAS,SAAS,eAAe;AAAA,QAC3E,QAAQ,eAAe;AAAA,QACvB;AAAA,MACF;AAAA,IACF,CAAC;AACD,WAAO;AAAA,MACL,WAAW,QAAQ;AAAA,MACnB,WAAWA,MAAK,UAAU,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC;AAAA,MACjD,UAAUA,MAAK,UAAU,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC;AAAA,MAC/C,gBAAgBA,MAAK,UAAU,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC;AAAA,MAC3D,QAAQA,MAAK,UAAU,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;AAAA,MAC3C;AAAA,MACA,SAAS,iBAAiB,UAAU,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;AAAA,IAC3D;AAAA,EACF,CAAC;AACH;AAEA,SAAS,iBAAiB,MAA6D;AACrF,QAAM,UAAU,oBAAI,IAAsB;AAC1C,aAAW,OAAO,MAAM;AACtB,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,GAAG,GAAG;AACxC,UAAI,CAAC,OAAO,SAAS,CAAC,EAAG;AACzB,YAAM,OAAO,QAAQ,IAAI,CAAC,KAAK,CAAC;AAChC,WAAK,KAAK,CAAC;AACX,cAAQ,IAAI,GAAG,IAAI;AAAA,IACrB;AAAA,EACF;AACA,QAAM,MAA8B,CAAC;AACrC,aAAW,CAAC,GAAG,IAAI,KAAK,QAAS,KAAI,CAAC,IAAIA,MAAK,IAAI;AACnD,SAAO;AACT;AAEA,SAASA,MAAK,IAAsB;AAClC,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAEA,eAAe,eACb,SACA,YACA,QACA,OACA,QACA,gBACgC;AAChC,QAAM,cAAc,IAAI,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,SAAS,CAAC;AACnE,QAAM,YAAY,QAAQ,OAAO,CAAC,MAAM,YAAY,IAAI,EAAE,EAAE,CAAC;AAG7D,QAAM,SAAS,YAAY,YAAY,OAAO,YAAY,EAAE,SAAS,OAAO,cAAc,CAAC,EAAE;AAAA,IAC3F,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE;AAAA,EACxB;AACA,QAAM,WAAW,OAAO,CAAC,GAAG,UAAU,aAAa,QAAQ,CAAC,EAAG;AAC/D,QAAM,SAAS,QAAQ,KAAK,CAAC,MAAM,EAAE,OAAO,QAAQ,KAAK,QAAQ,CAAC;AAClE,QAAM,kBAAkB,WAAW,KAAK,CAAC,MAAM,EAAE,cAAc,OAAO,EAAE,KAAK,WAAW,CAAC;AAEzF,QAAM,YAAY,kBAAkB,QAAQ,OAAO,IAAI,CAAC;AACxD,QAAM,eAAe,qBAAqB,QAAQ,OAAO,IAAI,CAAC;AAC9D,QAAM,aAAa,KAAK,IAAI,GAAG,OAAO,iBAAiB,UAAU,MAAM;AACvE,MAAI,WAAkC,CAAC;AACvC,MAAI,aAAa,GAAG;AAClB,eAAW,MAAM,OAAO,cAAc,OAAO;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AAAA,IACd,CAAC;AACD,eAAW,SACR,MAAM,GAAG,UAAU,EACnB,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,YAAY,gBAAgB,UAAU,OAAO,GAAG,EAAE;AAAA,EAC3E;AACA,SAAO,CAAC,GAAG,WAAW,GAAG,QAAQ;AACnC;AAEA,SAAS,kBAAkB,QAAuB,WAAmB,GAA0B;AAC7F,SAAO,OACJ,OAAO,CAAC,MAAM,EAAE,cAAc,aAAa,EAAE,EAAE,EAC/C,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,MAAM,GAAG,CAAC;AACf;AAEA,SAAS,qBAAqB,QAAuB,WAAmB,GAA0B;AAChG,SAAO,OACJ,OAAO,CAAC,MAAM,EAAE,cAAc,aAAa,EAAE,EAAE,EAC/C,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,MAAM,GAAG,CAAC;AACf;AAEA,SAAS,eAAe,GAAa,GAAsB;AACzD,MAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,QAAM,OAAO,IAAI,IAAI,CAAC;AACtB,SAAO,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;AACrC;;;AC/QA,eAAsB,yBACpB,QACyC;AACzC,iBAAe,MAAM;AAErB,QAAM,eAAgC;AAAA,IACpC,OAAO,CAAC,SAAS,SAAS,QAAQ,KAAK,SAAS,KAAK,YAAY,KAAK,KAAK,QAAQ;AAAA,EACrF;AAEA,QAAM,YAAY,MAAM,mBAAsB;AAAA,IAC5C,OAAO,OAAO;AAAA,IACd,QAAQ,OAAO;AAAA,IACf,cAAc,OAAO;AAAA,IACrB,aAAa,OAAO;AAAA,IACpB,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,gBAAgB,OAAO;AAAA,IACvB,kBAAkB,OAAO,oBAAoB;AAAA,IAC7C;AAAA,IACA,eAAe;AAAA,MACb,QAAQ,CAAC,SACP,OAAO,cAAc,OAAO;AAAA,QAC1B,GAAG;AAAA,QACH,WAAW,KAAK;AAAA,QAChB,cAAc,KAAK;AAAA,MACrB,CAAC;AAAA,IACL;AAAA,IACA,YAAY,OAAO,cAAc,2BAA2B;AAAA,IAC5D,eAAe,OAAO;AAAA,IACtB,0BAA0B,OAAO;AAAA,IACjC,OAAO,OAAO;AAAA,IACd,YAAY,OAAO;AAAA,EACrB,CAAC;AAED,MAAI,OAAmC;AACvC,QAAM,WAAW,OAAO,aAAa,CAAC;AACtC,MAAI,kBAAkB,UAAU;AAChC,MAAI,oBAAoB,UAAU;AAClC,MAAI,OAAO,QAAQ,UAAU,YAAY,OAAO,SAAS,IAAI;AAC3D,WAAO,MAAM,sBAAsB,QAAQ,UAAU,UAAU,WAAW;AAC1E,QAAI,CAAC,KAAK,SAAS,SAAS;AAC1B,wBAAkB;AAClB,0BAAoB,aAAa,WAAW,SAAS,EAAE;AAAA,IACzD;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,mBAAmB,UAAU;AAAA,IAC7B,qBAAqB,UAAU;AAAA,IAC/B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,6BAA4D;AAC1E,SAAO;AAAA,IACL,EAAE,MAAM,SAAS,WAAW,YAAY,OAAO,CAAC,MAAM,EAAE,UAAU;AAAA,IAClE,EAAE,MAAM,QAAQ,WAAW,YAAY,OAAO,CAAC,MAAM,EAAE,SAAS;AAAA,EAClE;AACF;AAEO,SAAS,6BAA6B,OAAyC;AACpF,SAAO;AAAA,IACL,IAAI,GAAG,MAAM,SAAS,IAAI,MAAM,UAAU,KAAK,MAAM,GAAG;AAAA,IACxD,OAAO,MAAM;AAAA,IACb,WAAW,MAAM;AAAA,IACjB,eAAe,MAAM,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,OAAO;AAAA,MAChD,IAAI,KAAK,iBAAiB,OAAO,CAAC;AAAA,MAClC,QAAQ,KAAK;AAAA,MACb,SAAS,KAAK,WAAW;AAAA,IAC3B,EAAE;AAAA,IACF,SAAS,MAAM,WAAW,aAAa,MAAM,KAAK;AAAA,IAClD,SAAS,MAAM;AAAA,EACjB;AACF;AAEA,eAAe,sBACb,QACA,UACA,WAC8B;AAC9B,QAAM,aAAa,OAAO;AAC1B,QAAM,OAAO,WAAW,QAAQ,OAAO;AACvC,QAAM,gBAA6B,CAAC;AACpC,QAAM,eAA4B,CAAC;AAEnC,QAAM,YAAY,WAAW,qBAAqB,OAAO;AACzD,aAAW,cAAc,WAAW;AAClC,aAAS,MAAM,GAAG,MAAM,MAAM,OAAO;AACnC,YAAM,OAAO,QAAQ,QAAQ,YAAY,GAAG;AAC5C,YAAM,YAAY,MAAM,SAAS,QAAQ,UAAU,YAAY,KAAK,QAAQ;AAC5E,YAAM,YAAY,MAAM,SAAS,QAAQ,WAAW,YAAY,KAAK,QAAQ;AAC7E,mBAAa;AAAA,QACX,kBAAkB,QAAQ,UAAU,YAAY,KAAK,UAAU,MAAM,SAAS;AAAA,MAChF;AACA,oBAAc;AAAA,QACZ,kBAAkB,QAAQ,WAAW,YAAY,KAAK,UAAU,MAAM,SAAS;AAAA,MACjF;AAAA,IACF;AAAA,EACF;AAEA,aAAW,cAAc,WAAW,oBAAoB;AACtD,aAAS,MAAM,GAAG,MAAM,MAAM,OAAO;AACnC,YAAM,OAAO,QAAQ,QAAQ,YAAY,GAAG;AAC5C,YAAM,YAAY,MAAM,SAAS,QAAQ,UAAU,YAAY,KAAK,SAAS;AAC7E,YAAM,YAAY,MAAM,SAAS,QAAQ,WAAW,YAAY,KAAK,SAAS;AAC9E,mBAAa;AAAA,QACX,kBAAkB,QAAQ,UAAU,YAAY,KAAK,WAAW,MAAM,SAAS;AAAA,MACjF;AACA,oBAAc;AAAA,QACZ,kBAAkB,QAAQ,WAAW,YAAY,KAAK,WAAW,MAAM,SAAS;AAAA,MAClF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,WAAW,IAAI,YAAY,WAAW,IAAI,EAAE,SAAS,eAAe,YAAY;AACtF,SAAO,EAAE,UAAU,eAAe,aAAa;AACjD;AAEA,eAAe,SACb,QACA,SACA,YACA,KACA,OAC+B;AAC/B,QAAM,OAAO,QAAQ,QAAQ,YAAY,GAAG;AAC5C,QAAM,QAA8B,EAAE,SAAS,YAAY,KAAK,OAAO,KAAK;AAC5E,MAAI;AACF,UAAM,MAAM,MAAM,OAAO,OAAO,IAAI,KAAK;AACzC,UAAM,SAAS,MAAM,OAAO,OAAO,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC;AAC1D,UAAM,MAAM,OAAO,OAAO,CAAC;AAC3B,WAAO;AAAA,MACL,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA,IAAI,OAAO,MAAM;AAAA,MACjB,OAAO,QAAQ,OAAO,KAAK;AAAA,MAC3B,MAAM,OAAO,WAAW,IAAI,WAAW;AAAA,MACvC,YAAY,OAAO,cAAc,IAAI,cAAc;AAAA,MACnD,SAAS;AAAA,QACP,GAAG,eAAe,OAAO,OAAO;AAAA,QAChC,GAAG,WAAW,GAAG;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,IAAI;AAAA,MACX;AAAA,MACA,SAAS,OAAO,WAAW,aAAa,IAAI,KAAK;AAAA,MACjD,UAAU,OAAO;AAAA,IACnB;AAAA,EACF,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,MAAM;AAAA,MACN,YAAY;AAAA,MACZ,SAAS,EAAE,OAAO,EAAE;AAAA,MACpB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACtD;AAAA,MACA;AAAA,MACA,KAAK;AAAA,QACH;AAAA,UACE,UAAU;AAAA,UACV,SAAS,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,UACxD,oBAAoB,OAAO;AAAA,QAC7B;AAAA,MACF;AAAA,MACA,SAAS;AAAA,IACX;AAAA,EACF;AACF;AAEA,SAAS,kBACP,QACA,SACA,YACA,KACA,OACA,MACA,OACW;AACX,QAAM,SAAS,OAAO,KAAM,YAAY,EAAE,SAAS,YAAY,KAAK,OAAO,MAAM,MAAM,CAAC;AACxF,SAAO,kBAAkB,MAAM;AACjC;AAEA,SAAS,eAAkB,QAA8C;AACvE,MAAI,CAAC,OAAO,MAAM,KAAK,EAAG,OAAM,IAAI,MAAM,mDAAmD;AAC7F,MAAI,CAAC,OAAO,OAAO,KAAK,EAAG,OAAM,IAAI,MAAM,oDAAoD;AAC/F,MAAI,OAAO,aAAa,WAAW,GAAG;AACpC,UAAM,IAAI,MAAM,0DAA0D;AAAA,EAC5E;AACA,MAAI,OAAO,kBAAkB,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,+DAA+D;AAAA,EACjF;AACA,yBAAuB,OAAO,MAAM,MAAM;AAC1C,yBAAuB,OAAO,aAAa,aAAa;AACxD,yBAAuB,OAAO,gBAAgB,gBAAgB;AAC9D,MAAI,OAAO,qBAAqB;AAC9B,2BAAuB,OAAO,kBAAkB,kBAAkB;AACpE,MAAI,OAAO,iBAAiB,OAAO,aAAa,QAAQ;AACtD,UAAM,IAAI,MAAM,yEAAyE;AAAA,EAC3F;AACA;AAAA,IACE,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,EAAE;AAAA,IACnC;AAAA,EACF;AACA,eAAa,OAAO,mBAAmB,mBAAmB;AAE1D,MAAI,OAAO,MAAM;AACf,QAAI,OAAO,KAAK,mBAAmB,WAAW,GAAG;AAC/C,YAAM,IAAI,MAAM,qEAAqE;AAAA,IACvF;AACA,QAAI,OAAO,KAAK,SAAS,OAAW,wBAAuB,OAAO,KAAK,MAAM,WAAW;AACxF,iBAAa,OAAO,KAAK,oBAAoB,yBAAyB;AACtE,QAAI,OAAO,KAAK;AACd,mBAAa,OAAO,KAAK,mBAAmB,wBAAwB;AACtE,UAAM,YAAY,IAAI,IAAI,OAAO,iBAAiB;AAClD,eAAW,MAAM,OAAO,KAAK,oBAAoB;AAC/C,UAAI,UAAU,IAAI,EAAE,GAAG;AACrB,cAAM,IAAI;AAAA,UACR,+CAA+C,EAAE;AAAA,QACnD;AAAA,MACF;AAAA,IACF;AACA,UAAM,aAAa,OAAO,aAAa,CAAC,EAAG;AAC3C,QAAI,OAAO,KAAK,KAAK,gBAAgB,YAAY;AAC/C,YAAM,IAAI;AAAA,QACR,qFAAqF,UAAU;AAAA,MACjG;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,uBAAuB,OAAe,MAAoB;AACjE,MAAI,CAAC,OAAO,UAAU,KAAK,KAAK,SAAS,GAAG;AAC1C,UAAM,IAAI,MAAM,6BAA6B,IAAI,6BAA6B;AAAA,EAChF;AACF;AAEA,SAAS,aAAa,QAAkB,MAAoB;AAC1D,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,MAAM,KAAK;AACd,YAAM,IAAI,MAAM,6BAA6B,IAAI,gCAAgC;AACnF,QAAI,KAAK,IAAI,KAAK,EAAG,OAAM,IAAI,MAAM,uCAAuC,IAAI,KAAK,KAAK,GAAG;AAC7F,SAAK,IAAI,KAAK;AAAA,EAChB;AACF;AAEA,SAAS,aAAgB,WAAqC,WAAqC;AACjG,QAAM,QAAQ,UAAU,YAAY,UAAU,YAAY,SAAS,CAAC;AACpE,QAAM,YAAY,OAAO,WAAW,KAAK,CAAC,MAAM,EAAE,cAAc,SAAS;AACzE,MAAI,CAAC,WAAW;AACd,UAAM,IAAI,MAAM,4DAA4D,SAAS,GAAG;AAAA,EAC1F;AACA,SAAO;AACT;AAEA,SAAS,QACP,QACA,YACA,KACQ;AACR,QAAM,OAAO,OAAO,YAAY;AAChC,UAAQ,OAAOC,YAAW,GAAG,UAAU,IAAO,GAAG,EAAE,KAAK,OAAO;AACjE;AAEA,SAASA,YAAW,OAAuB;AACzC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,SAAK,MAAM,WAAW,CAAC;AACvB,QAAI,KAAK,KAAK,GAAG,QAAQ;AAAA,EAC3B;AACA,SAAO,MAAM;AACf;AAEA,SAAS,QAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,eAAe,SAAqE;AAC3F,QAAM,MAA8B,CAAC;AACrC,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,WAAW,CAAC,CAAC,GAAG;AAClD,QAAI,OAAO,SAAS,CAAC,EAAG,KAAI,CAAC,IAAI;AAAA,EACnC;AACA,SAAO;AACT;AAEA,SAAS,WAAW,KAAmD;AACrE,QAAM,MAA8B,EAAE,KAAK,IAAI,OAAO;AACtD,aAAW,QAAQ,IAAI,MAAM,GAAG,GAAI,GAAG;AACrC,UAAM,MAAM,kBAAkB,KAAK,QAAQ;AAC3C,QAAI,OAAO,GAAG,EAAE,KAAK,IAAI,OAAO,GAAG,EAAE,KAAK,KAAK;AAC/C,QAAI,KAAK,oBAAoB;AAC3B,YAAM,MAAM,WAAW,iBAAiB,KAAK,kBAAkB,CAAC;AAChE,UAAI,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK;AAAA,IAC/B;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,UAAgD;AACzE,MACE,aAAa,UACb,aAAa,aACb,aAAa,WACb,aAAa,YACb;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,SAAS,iBAAiB,KAAqB;AAC7C,SACE,IACG,KAAK,EACL,QAAQ,qBAAqB,GAAG,EAChC,MAAM,GAAG,EAAE,KAAK;AAEvB;AAEA,SAAS,aAAa,OAAuD;AAC3E,MAAI,CAAC,MAAO,QAAO;AACnB,MAAI,OAAO,MAAM,WAAW,SAAU,QAAO,MAAM;AACnD,MAAI,MAAM,WAAY,QAAO,MAAM;AACnC,MAAI,MAAM,OAAO;AACf,QAAI;AACF,YAAM,UAAU,MAAM,MAAM,MAAM,GAAG,EAAE;AACvC,YAAM,SACJ,MAAM,MAAM,SAAS,QAAQ,SACzB,QAAQ,MAAM,MAAM,SAAS,QAAQ,MAAM,kBAC3C;AACN,aAAO,GAAG,KAAK,UAAU,OAAO,EAAE,MAAM,GAAG,GAAI,CAAC,GAAG,MAAM;AAAA,IAC3D,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;;;AC7dO,IAAM,8BAAwC;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAOO,SAAS,sBAAsB,KAAgC;AACpE,QAAM,aAAa,IAAI,sBAAsB;AAC7C,QAAM,WAAqB,CAAC;AAE5B,WAAS,KAAK,sBAAsB,IAAI,MAAM,EAAE;AAChD,WAAS,KAAK,EAAE;AAChB,WAAS;AAAA,IACP,+CAA+C,IAAI,MAAM,oDAAoD,IAAI,UAAU,MAAM,mBAAmB,IAAI,aAAa,MAAM,uCAAuC,IAAI,UAAU,YAAY,IAAI,eAAe,IAAI,KAAK,GAAG;AAAA,EAC7Q;AACA,WAAS,KAAK,EAAE;AAEhB,WAAS,KAAK,oBAAoB;AAClC,WAAS,KAAK,SAAS;AACvB,WAAS,KAAK,KAAK,UAAU,IAAI,eAAe,MAAM,CAAC,CAAC;AACxD,WAAS,KAAK,KAAK;AACnB,WAAS,KAAK,EAAE;AAEhB,MAAI,IAAI,aAAa,SAAS,GAAG;AAC/B,aAAS,KAAK,oDAA+C;AAC7D,aAAS,KAAK,EAAE;AAChB,eAAW,SAAS,IAAI,cAAc;AACpC,eAAS;AAAA,QACP,eAAe,MAAM,EAAE,mBAAc,MAAM,MAAM,QAAQ,CAAC,CAAC,GAAG,MAAM,YAAY,KAAK,MAAM,SAAS,MAAM,EAAE;AAAA,MAC9G;AACA,YAAM,UAAU,MAAM,gBAAgB,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO;AAClE,UAAI,OAAO,SAAS,GAAG;AACrB,iBAAS,KAAK,EAAE;AAChB,iBAAS,KAAK,0BAA0B;AACxC,mBAAW,KAAK,QAAQ;AACtB,mBAAS,KAAK,OAAO,EAAE,EAAE,6BAA6B,MAAM,EAAE,MAAM,CAAC,IAAI;AAAA,QAC3E;AAAA,MACF;AACA,UAAI,MAAM,SAAS;AACjB,iBAAS,KAAK,EAAE;AAChB,iBAAS,KAAK,6BAA6B;AAC3C,iBAAS,KAAK,KAAK;AACnB,iBAAS,KAAK,SAAS,MAAM,SAAS,GAAG,CAAC;AAC1C,iBAAS,KAAK,KAAK;AAAA,MACrB;AACA,eAAS,KAAK,EAAE;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,IAAI,UAAU,SAAS,GAAG;AAC5B,aAAS,KAAK,mDAA8C;AAC5D,aAAS,KAAK,EAAE;AAChB,eAAW,SAAS,IAAI,WAAW;AACjC,eAAS;AAAA,QACP,OAAO,MAAM,EAAE,aAAa,MAAM,MAAM,QAAQ,CAAC,CAAC,GAAG,MAAM,YAAY,KAAK,MAAM,SAAS,MAAM,EAAE;AAAA,MACrG;AAAA,IACF;AACA,aAAS,KAAK,EAAE;AAAA,EAClB;AAEA,WAAS,KAAK,gCAAgC;AAC9C,WAAS,KAAK,EAAE;AAChB,aAAW,KAAK,WAAY,UAAS,KAAK,KAAK,CAAC,EAAE;AAClD,WAAS,KAAK,EAAE;AAEhB,WAAS,KAAK,kBAAkB;AAChC,WAAS,KAAK,EAAE;AAChB,WAAS,KAAK,iEAA4D;AAC1E,WAAS,KAAK,SAAS;AACvB,WAAS;AAAA,IACP,KAAK;AAAA,MACH;AAAA,QACE,WAAW;AAAA,UACT;AAAA,YACE,OAAO;AAAA,YACP,WAAW;AAAA,YACX,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,WAAS,KAAK,KAAK;AAEnB,SAAO,SAAS,KAAK,IAAI;AAC3B;AAEA,SAAS,SAAS,GAAW,KAAqB;AAChD,MAAI,EAAE,UAAU,IAAK,QAAO;AAC5B,SAAO,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;AAC3B;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,EAAE,QAAQ,MAAM,KAAK;AAC9B;AAuBA,SAAS,uBAAuB,KAA4B;AAC1D,QAAM,QAA0B,CAAC;AACjC,MAAI,WAAW;AACf,MAAI,UAAU;AACd,aAAW,KAAK,KAAK;AACnB,QAAI,SAAS;AACX,gBAAU;AACV;AAAA,IACF;AACA,QAAI,UAAU;AACZ,UAAI,MAAM,MAAM;AACd,kBAAU;AACV;AAAA,MACF;AACA,UAAI,MAAM,KAAK;AACb,mBAAW;AACX;AAAA,MACF;AACA;AAAA,IACF;AACA,QAAI,MAAM,KAAK;AACb,iBAAW;AACX;AAAA,IACF;AACA,QAAI,MAAM,OAAO,MAAM,IAAK,OAAM,KAAK,CAAC;AAAA,aAC/B,MAAM,KAAK;AAClB,UAAI,MAAM,IAAI,MAAM,IAAK,QAAO;AAAA,IAClC,WAAW,MAAM,KAAK;AACpB,UAAI,MAAM,IAAI,MAAM,IAAK,QAAO;AAAA,IAClC;AAAA,EACF;AACA,MAAI,MAAM,WAAW,KAAK,CAAC,SAAU,QAAO;AAC5C,MAAI,SAAS;AACb,MAAI,SAAU,WAAU;AACxB,SAAO,MAAM,SAAS,GAAG;AACvB,UAAM,SAAS,MAAM,IAAI;AACzB,cAAU,WAAW,MAAM,MAAM;AAAA,EACnC;AACA,SAAO,MAAM;AACf;AAEO,SAAS,wBAAwB,KAAa,cAA6C;AAChG,MAAI,OAAO,IAAI,KAAK;AACpB,MAAI,KAAK,WAAW,KAAK,EAAG,QAAO,KAAK,QAAQ,oBAAoB,EAAE,EAAE,QAAQ,WAAW,EAAE;AAK7F,MAAI,SAAkB;AACtB,QAAM,cAAc,KAAK,QAAQ,GAAG;AACpC,QAAM,YAAY,KAAK,YAAY,GAAG;AACtC,QAAM,aAAa,KAAK,QAAQ,GAAG;AACnC,QAAM,WAAW,KAAK,YAAY,GAAG;AAErC,QAAM,iBAAiB,eAAe,MAAM,aAAa,KAAK,cAAc;AAC5E,QAAM,aAAuB,CAAC;AAC9B,MAAI,gBAAgB;AAClB,QAAI,eAAe,KAAK,YAAY;AAClC,iBAAW,KAAK,KAAK,MAAM,aAAa,YAAY,CAAC,CAAC;AACxD,QAAI,cAAc,KAAK,WAAW;AAChC,iBAAW,KAAK,KAAK,MAAM,YAAY,WAAW,CAAC,CAAC;AAAA,EACxD,OAAO;AACL,QAAI,cAAc,KAAK,WAAW;AAChC,iBAAW,KAAK,KAAK,MAAM,YAAY,WAAW,CAAC,CAAC;AACtD,QAAI,eAAe,KAAK,YAAY;AAClC,iBAAW,KAAK,KAAK,MAAM,aAAa,YAAY,CAAC,CAAC;AAAA,EAC1D;AACA,aAAW,SAAS,YAAY;AAC9B,QAAI;AACF,eAAS,KAAK,MAAM,KAAK;AACzB;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF;AAOA,MAAI,UAAU,MAAM;AAClB,eAAW,SAAS,YAAY;AAC9B,YAAM,SAAS,uBAAuB,KAAK;AAC3C,UAAI,UAAU,QAAQ,WAAW,OAAO;AACtC,YAAI;AACF,mBAAS,KAAK,MAAM,MAAM;AAC1B;AAAA,QACF,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,UAAU,KAAM,QAAO,CAAC;AAG5B,MAAI;AACJ,MAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,mBAAe;AAAA,EACjB,WAAW,UAAU,OAAO,WAAW,UAAU;AAC/C,mBAAgB,OAAmC;AAAA,EACrD;AACA,MAAI,CAAC,MAAM,QAAQ,YAAY,EAAG,QAAO,CAAC;AAE1C,QAAM,MAA4B,CAAC;AACnC,aAAW,KAAK,cAAc;AAC5B,QAAI,CAAC,KAAK,OAAO,MAAM,SAAU;AACjC,UAAM,MAAM;AACZ,QAAI,EAAE,aAAa,KAAM;AACzB,QAAI,KAAK;AAAA,MACP,OAAO,OAAO,IAAI,UAAU,WAAW,IAAI,QAAQ;AAAA,MACnD,WAAW,OAAO,IAAI,cAAc,WAAW,IAAI,YAAY;AAAA,MAC/D,SAAS,IAAI;AAAA,IACf,CAAC;AACD,QAAI,iBAAiB,UAAa,IAAI,UAAU,aAAc;AAAA,EAChE;AACA,SAAO;AACT;;;ACnLO,IAAM,qBAAN,MAA+C;AAAA,EACpD,YAA6B,WAAsC;AAAtC;AAAA,EAAuC;AAAA,EAAvC;AAAA,EAE7B,gBAAgB,MAA2C;AACzD,WAAO,KAAK,UAAU,gBAAgB,IAAI;AAAA,EAC5C;AAAA,EAEA,cAAc,UAAoD;AAChE,WAAO,KAAK,UAAU,cAAc,QAAQ;AAAA,EAC9C;AAAA,EAEA,YAAY,SAA2B,UAAmD;AACxF,WAAO,KAAK,UAAU,YAAY,SAAS,QAAQ;AAAA,EACrD;AAAA,EAEA,eAAe,MAAiD;AAC9D,WAAO,KAAK,UAAU,eAAe,IAAI;AAAA,EAC3C;AACF;AASO,IAAM,iBAAN,MAA2C;AAAA,EAC/B;AAAA,EAEjB,YAAY,OAAO,2CAA2C;AAC5D,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,MAAM,gBAAgB,OAA4C;AAChE,UAAM,IAAI,MAAM,GAAG,KAAK,IAAI,oCAAoC;AAAA,EAClE;AAAA,EAEA,MAAM,cAAc,WAAqD;AACvE,UAAM,IAAI,MAAM,GAAG,KAAK,IAAI,kCAAkC;AAAA,EAChE;AAAA,EAEA,MAAM,YACJ,UACA,WACyB;AACzB,UAAM,IAAI,MAAM,GAAG,KAAK,IAAI,gCAAgC;AAAA,EAC9D;AAAA,EAEA,MAAM,eAAe,OAAkD;AACrE,UAAM,IAAI,MAAM,GAAG,KAAK,IAAI,mCAAmC;AAAA,EACjE;AACF;","names":["mean","stableHash"]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/summary-report.ts"],"sourcesContent":["/**\n * Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather\n * than replacing it.\n *\n * Three artefacts:\n *\n * - `summaryTable` Markdown table of per-candidate means,\n * 95% bootstrap CIs, BH-adjusted Wilcoxon\n * p-values, and Cohen's d versus a\n * comparator candidate.\n * - `paretoChart` Abstract spec for a cost vs quality\n * scatter, with gate decisions overlaid.\n * Returns numbers + labels — caller\n * chooses the plotting library.\n * - `gainHistogram`\n * Per-item paired holdout deltas as a\n * histogram spec (bins + counts + median +\n * CI). Same \"data, not images\" contract.\n *\n * The figure types are PlotSpecs — JSON-friendly, library-agnostic.\n * They aren't React components and they aren't PNGs; they are\n * what you'd hand to vega-lite, plotly, matplotlib, or your own\n * Canvas renderer to draw the actual figure.\n */\n\nimport type { GateDecision } from './held-out-gate'\nimport type { FailureClusterReport } from './pipelines/failure-cluster'\nimport { canonicalize, hashJson } from './pre-registration'\nimport type { RunRecord } from './run-record'\nimport {\n benjaminiHochberg,\n cohensD,\n confidenceInterval,\n pairedBootstrap,\n pairedMde,\n wilcoxonSignedRank,\n} from './statistics'\n\n// ── summaryTable ───────────────────────────────────────────────────────\n\nexport interface SummaryTableOptions {\n /** Comparator candidate id. Wilcoxon + Cohen's d are computed\n * versus this candidate. Required for paired stats columns. */\n comparator?: string\n /** Which split to read scores from. Default 'holdout'. */\n split?: 'search' | 'holdout'\n /** Confidence level for the bootstrap CI on the mean. Default 0.95. */\n confidence?: number\n /** FDR for BH adjustment of the comparison p-values. Default 0.05. */\n fdr?: number\n}\n\nexport interface SummaryTableRow {\n candidateId: string\n n: number\n mean: number\n ciLow: number\n ciHigh: number\n /** BH-adjusted q-value vs comparator. NaN if no comparator. */\n qValue: number\n /** Cohen's d vs comparator. NaN if no comparator. */\n cohensD: number\n}\n\nexport interface SummaryTable {\n rows: SummaryTableRow[]\n comparator: string | null\n split: 'search' | 'holdout'\n /** Pre-rendered markdown — drop into a paper or PR. */\n markdown: string\n}\n\n/**\n * Table 1 helper. Buckets runs by `candidateId`, computes mean +\n * bootstrap CI on the chosen split, and (when a comparator is given)\n * BH-adjusted Wilcoxon p + Cohen's d versus that comparator.\n */\nexport function summaryTable(runs: RunRecord[], opts: SummaryTableOptions = {}): SummaryTable {\n const split = opts.split ?? 'holdout'\n const confidence = opts.confidence ?? 0.95\n const fdr = opts.fdr ?? 0.05\n const comparator = opts.comparator ?? null\n const scoreField = split === 'holdout' ? 'holdoutScore' : 'searchScore'\n\n const byCandidate = new Map<string, { runs: RunRecord[]; scores: number[] }>()\n for (const r of runs) {\n if (r.splitTag !== split) continue\n const v = r.outcome[scoreField]\n if (typeof v !== 'number' || !Number.isFinite(v)) continue\n const bucket = byCandidate.get(r.candidateId) ?? { runs: [], scores: [] }\n bucket.runs.push(r)\n bucket.scores.push(v)\n byCandidate.set(r.candidateId, bucket)\n }\n\n const candidateIds = [...byCandidate.keys()].sort()\n const compRuns = comparator ? byCandidate.get(comparator) : undefined\n\n // First pass: per-candidate means + CIs + raw p-values.\n const tentative: Array<SummaryTableRow & { rawP: number }> = []\n for (const id of candidateIds) {\n const bucket = byCandidate.get(id)!\n const ci = confidenceInterval(bucket.scores, confidence)\n let rawP = Number.NaN\n let d = Number.NaN\n if (comparator && compRuns && id !== comparator) {\n const paired = pairScoresByKey(bucket.runs, compRuns.runs, scoreField)\n if (paired.before.length >= 6) {\n rawP = wilcoxonSignedRank(paired.before, paired.after).p\n }\n d = cohensD(compRuns.scores, bucket.scores)\n }\n tentative.push({\n candidateId: id,\n n: bucket.scores.length,\n mean: ci.mean,\n ciLow: ci.lower,\n ciHigh: ci.upper,\n qValue: rawP,\n cohensD: d,\n rawP,\n })\n }\n\n // BH-adjust across the comparison set (skip NaN rows / the\n // comparator itself). Adjustment is a no-op when there are 0 or 1\n // comparators.\n if (comparator) {\n const idxs: number[] = []\n const ps: number[] = []\n for (let i = 0; i < tentative.length; i++) {\n const r = tentative[i]!\n if (r.candidateId === comparator) continue\n if (!Number.isFinite(r.rawP)) continue\n idxs.push(i)\n ps.push(r.rawP)\n }\n if (ps.length > 0) {\n const { qValues } = benjaminiHochberg(ps, fdr)\n for (let k = 0; k < idxs.length; k++) {\n tentative[idxs[k]!]!.qValue = qValues[k]!\n }\n }\n }\n\n const rows = tentative.map(({ rawP: _rawP, ...rest }) => rest)\n const markdown = renderSummaryTableMarkdown(rows, comparator, split)\n return { rows, comparator, split, markdown }\n}\n\nfunction pairScoresByKey(\n candidate: RunRecord[],\n baseline: RunRecord[],\n scoreField: 'searchScore' | 'holdoutScore',\n): { before: number[]; after: number[] } {\n const baseIdx = new Map<string, number>()\n for (const r of baseline) {\n const v = r.outcome[scoreField]\n if (typeof v === 'number' && Number.isFinite(v)) {\n baseIdx.set(`${r.experimentId}::${r.seed}`, v)\n }\n }\n const before: number[] = []\n const after: number[] = []\n for (const r of candidate) {\n const v = r.outcome[scoreField]\n if (typeof v !== 'number' || !Number.isFinite(v)) continue\n const key = `${r.experimentId}::${r.seed}`\n const b = baseIdx.get(key)\n if (b === undefined) continue\n before.push(b)\n after.push(v)\n }\n return { before, after }\n}\n\nfunction renderSummaryTableMarkdown(\n rows: SummaryTableRow[],\n comparator: string | null,\n split: 'search' | 'holdout',\n): string {\n const lines: string[] = []\n const cmpLabel = comparator ? ` (vs ${comparator})` : ''\n lines.push(`Summary Table — ${split} split${cmpLabel}`)\n lines.push('')\n lines.push(\"| Candidate | N | Mean | 95% CI | q (BH) | Cohen's d |\")\n lines.push('|---|---:|---:|---|---:|---:|')\n for (const r of rows) {\n const ci = `[${fmt(r.ciLow)}, ${fmt(r.ciHigh)}]`\n const q = Number.isFinite(r.qValue) ? r.qValue.toFixed(4) : '—'\n const d = Number.isFinite(r.cohensD) ? r.cohensD.toFixed(3) : '—'\n lines.push(`| ${r.candidateId} | ${r.n} | ${fmt(r.mean)} | ${ci} | ${q} | ${d} |`)\n }\n return lines.join('\\n')\n}\n\n// ── paretoChart ─────────────────────────────────────────────────────\n\nexport interface ParetoPoint {\n candidateId: string\n /** Mean USD cost per run on the chosen split. */\n cost: number\n /** Mean score on the chosen split. */\n quality: number\n /** Number of runs that informed this point. */\n n: number\n /** Whether this candidate is on the Pareto frontier — high\n * quality, low cost, no dominator. */\n onFrontier: boolean\n /** Optional gate verdict for this candidate, if a `GateDecision`\n * for it was passed in. */\n gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null\n}\n\nexport interface ParetoFigureSpec {\n kind: 'pareto-cost-quality'\n split: 'search' | 'holdout'\n points: ParetoPoint[]\n axes: { x: 'costUsd'; y: 'score' }\n}\n\n/**\n * Cost vs quality scatter spec. `gateDecisions` is keyed by\n * candidate id; if present, every point picks up the gate verdict\n * for overlay.\n */\nexport function paretoChart(\n runs: RunRecord[],\n opts: {\n split?: 'search' | 'holdout'\n gateDecisions?: Record<string, GateDecision>\n } = {},\n): ParetoFigureSpec {\n const split = opts.split ?? 'holdout'\n const scoreField = split === 'holdout' ? 'holdoutScore' : 'searchScore'\n\n const buckets = new Map<string, { cost: number[]; quality: number[] }>()\n for (const r of runs) {\n if (r.splitTag !== split) continue\n const v = r.outcome[scoreField]\n if (typeof v !== 'number' || !Number.isFinite(v)) continue\n const bucket = buckets.get(r.candidateId) ?? { cost: [], quality: [] }\n bucket.cost.push(r.costUsd)\n bucket.quality.push(v)\n buckets.set(r.candidateId, bucket)\n }\n\n const points: ParetoPoint[] = []\n for (const [candidateId, bucket] of buckets.entries()) {\n points.push({\n candidateId,\n cost: avg(bucket.cost),\n quality: avg(bucket.quality),\n n: bucket.cost.length,\n onFrontier: false,\n gate: opts.gateDecisions?.[candidateId]\n ? gateLabel(opts.gateDecisions[candidateId]!)\n : undefined,\n })\n }\n\n // Pareto: minimize cost, maximize quality. A point is dominated if\n // some other point has lower-or-equal cost AND higher-or-equal\n // quality with strict inequality somewhere.\n for (const p of points) {\n p.onFrontier = !points.some((q) => q !== p && dominates(q, p))\n }\n\n return {\n kind: 'pareto-cost-quality',\n split,\n axes: { x: 'costUsd', y: 'score' },\n points,\n }\n}\n\nfunction dominates(a: ParetoPoint, b: ParetoPoint): boolean {\n return a.cost <= b.cost && a.quality >= b.quality && (a.cost < b.cost || a.quality > b.quality)\n}\n\nfunction gateLabel(d: GateDecision): ParetoPoint['gate'] {\n if (d.promote) return 'promote'\n if (d.rejectionCode === 'few_runs') return 'reject_few_runs'\n if (d.rejectionCode === 'negative_delta') return 'reject_negative_delta'\n if (d.rejectionCode === 'overfit_gap') return 'reject_overfit_gap'\n return null\n}\n\n// ── gainHistogram ───────────────────────────────────────────\n\nexport interface GainDistributionBin {\n /** Inclusive lower edge. */\n lo: number\n /** Exclusive upper edge (or inclusive if it's the last bin). */\n hi: number\n /** Number of pairs whose delta lands in this bin. */\n count: number\n}\n\nexport interface GainDistributionFigureSpec {\n kind: 'gain-distribution'\n candidateId: string\n comparator: string\n split: 'search' | 'holdout'\n /** Number of pairs used. */\n n: number\n bins: GainDistributionBin[]\n median: number\n ci: { low: number; high: number }\n}\n\nexport interface GainDistributionOptions {\n /** Number of histogram bins. Default 11 (so the centre is exact at 0). */\n bins?: number\n /** Which split to use. Default 'holdout'. */\n split?: 'search' | 'holdout'\n /** Confidence level for the CI. Default 0.95. */\n confidence?: number\n /** Bootstrap resamples. Default 2000. */\n resamples?: number\n /** Deterministic seed. */\n seed?: number\n}\n\n/**\n * Held-out improvement distribution: per-pair delta (candidate −\n * comparator), histogrammed. Includes the bootstrap CI on the median\n * delta — same primitive the promotion gate uses.\n */\nexport function gainHistogram(\n runs: RunRecord[],\n candidateId: string,\n comparator: string,\n opts: GainDistributionOptions = {},\n): GainDistributionFigureSpec {\n const split = opts.split ?? 'holdout'\n const scoreField = split === 'holdout' ? 'holdoutScore' : 'searchScore'\n const binCount = opts.bins ?? 11\n if (binCount < 1) throw new Error('gainHistogram: bins must be ≥ 1')\n\n const candidate = runs.filter((r) => r.candidateId === candidateId && r.splitTag === split)\n const baseline = runs.filter((r) => r.candidateId === comparator && r.splitTag === split)\n // pairScoresByKey returns before=baseline-score, after=candidate-score\n // for each (experimentId, seed) pair where both sides recorded a\n // valid score on this split. delta = after - before = candidate - baseline.\n const { before, after } = pairScoresByKey(candidate, baseline, scoreField)\n const n = before.length\n\n if (n === 0) {\n return {\n kind: 'gain-distribution',\n candidateId,\n comparator,\n split,\n n: 0,\n bins: [],\n median: 0,\n ci: { low: 0, high: 0 },\n }\n }\n\n const deltas = before.map((b, i) => after[i]! - b)\n const sortedDeltas = [...deltas].sort((a, b) => a - b)\n const median = medianOfSorted(sortedDeltas)\n const min = sortedDeltas[0]!\n const max = sortedDeltas[sortedDeltas.length - 1]!\n\n // Symmetric bins around the wider of (|min|, |max|) so the chart\n // visually centres on zero without dropping outliers.\n const bound = Math.max(Math.abs(min), Math.abs(max), 1e-6)\n const lo = -bound\n const hi = bound\n const width = (hi - lo) / binCount\n const bins: GainDistributionBin[] = []\n for (let i = 0; i < binCount; i++) {\n bins.push({ lo: lo + i * width, hi: lo + (i + 1) * width, count: 0 })\n }\n for (const d of deltas) {\n let idx = Math.floor((d - lo) / width)\n if (idx < 0) idx = 0\n if (idx >= binCount) idx = binCount - 1\n bins[idx]!.count += 1\n }\n\n const ci = pairedBootstrap(before, after, {\n confidence: opts.confidence ?? 0.95,\n resamples: opts.resamples ?? 2000,\n statistic: 'median',\n seed: opts.seed,\n })\n\n return {\n kind: 'gain-distribution',\n candidateId,\n comparator,\n split,\n n,\n bins,\n median,\n ci: { low: ci.low, high: ci.high },\n }\n}\n\n// ── researchReport ───────────────────────────────────────────────────\n\nexport type ResearchReportDecision =\n | 'promote'\n | 'hold'\n | 'reject'\n | 'equivalent'\n | 'needs_more_data'\n\n/**\n * Hard floor below which a paired comparison is treated as uninformative\n * regardless of `minPairs`. Mirrors the lower limit on Wilcoxon signed-rank\n * exact tables; below this the test has no power to separate effect sizes.\n */\nexport const RESEARCH_REPORT_HARD_PAIR_FLOOR = 6\n\nexport interface ResearchReportOptions {\n /** Human-readable report title. */\n title?: string\n /** Comparator candidate id. Required for statistical decision guidance. */\n comparator?: string\n /** Which split to use for the primary decision. Default 'holdout'. */\n split?: 'search' | 'holdout'\n /** Confidence level used by lower-level report helpers. Default 0.95. */\n confidence?: number\n /** FDR threshold for q-values. Default 0.05. */\n fdr?: number\n /**\n * Soft floor on paired observations before issuing a directional\n * promote / reject. Below this we report `needs_more_data` and surface the\n * minimum detectable effect at the current N. Default 20 — chosen so the\n * Wilcoxon signed-rank approximation is reasonable and so the paired\n * bootstrap CI has non-degenerate coverage. Hard floor is enforced at\n * `RESEARCH_REPORT_HARD_PAIR_FLOOR` (6) regardless of this value.\n */\n minPairs?: number\n /**\n * Region of Practical Equivalence on the paired delta. When a candidate's\n * paired-delta CI is fully contained in `[low, high]`, the decision is\n * `equivalent` rather than `hold`. Sourced from the domain owner — there is\n * no statistically-defensible default.\n */\n rope?: { low: number; high: number }\n /**\n * Power for the minimum detectable effect (MDE) reported on each candidate.\n * Default 0.8.\n */\n mdePower?: number\n /**\n * Two-sided alpha for the MDE. Default matches `fdr` so the reported MDE\n * lines up with the test the report actually runs.\n */\n mdeAlpha?: number\n /** Optional held-out gate decisions keyed by candidate id. */\n gateDecisions?: Record<string, GateDecision>\n /** Optional failure clusters from failureClusterView. */\n failureClusters?: FailureClusterReport\n /** Build gain histograms for these candidates. Defaults to all non-comparator candidates. */\n candidateIds?: string[]\n /** Deterministic bootstrap seed passed to gainHistogram and the posterior helper. */\n seed?: number\n /** Report timestamp. Defaults to current time. */\n generatedAt?: string\n /**\n * Hash of a preregistered protocol (e.g. `signManifest({...}).contentHash`).\n * Embedded verbatim in the report so the analysis can be cited as the\n * preregistered one rather than a post-hoc fishing expedition.\n */\n preregistrationHash?: string\n}\n\nexport interface ResearchReportRecommendation {\n decision: ResearchReportDecision\n candidateId: string | null\n rationale: string[]\n risks: string[]\n nextActions: string[]\n}\n\nexport interface ResearchReportCandidate {\n candidateId: string\n n: number\n mean: number\n ciLow: number\n ciHigh: number\n qValue: number\n cohensD: number\n meanDeltaVsComparator: number | null\n pairedN: number\n medianGain: number | null\n meanGain: number | null\n gainCi: { low: number; high: number } | null\n /**\n * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed\n * from the same resamples that produce the gain CI; interpretable as\n * \"fraction of resamples in which the candidate beats the comparator on\n * matched pairs.\"\n */\n prGreaterThanZero: number | null\n prInRope: number | null\n /**\n * Minimum detectable effect (in score units) at the candidate's paired N,\n * the configured power, and the configured alpha. Standardised by the\n * observed paired-delta SD and inverted via `requiredSampleSize`. Reported\n * for every candidate so a `needs_more_data` verdict is actionable.\n */\n mde: number | null\n onParetoFrontier: boolean\n gate?: ParetoPoint['gate']\n decision: ResearchReportDecision\n decisionReason: string\n}\n\nexport interface ResearchReportMethodology {\n /**\n * Plain-language assumptions the report depends on. Read these first when\n * deciding whether the verdict is load-bearing for a launch decision.\n */\n assumptions: string[]\n /** Tests and estimators the verdict was computed from. */\n methods: string[]\n /** Alternatives the author considered and why this report didn't take them. */\n alternatives: string[]\n /** Failure modes — when this report should NOT drive a decision. */\n whenNotToApply: string[]\n /** Citations for the methodological choices above. */\n citations: string[]\n}\n\nexport interface ResearchReport {\n kind: 'agent-eval-research-report'\n title: string\n generatedAt: string\n split: 'search' | 'holdout'\n comparator: string | null\n /**\n * SHA-256 over the canonicalised set of `(runId, candidateId, split)` triples\n * the report was computed from, plus the comparator and split. Stable across\n * key insertion order; recomputable by the reader to verify provenance.\n */\n runFingerprint: string\n preregistrationHash: string | null\n rope: { low: number; high: number } | null\n executiveSummary: string[]\n recommendation: ResearchReportRecommendation\n candidates: ResearchReportCandidate[]\n summary: SummaryTable\n charts: {\n pareto: ParetoFigureSpec\n gains: GainDistributionFigureSpec[]\n }\n methodology: ResearchReportMethodology\n failureClusters?: FailureClusterReport\n markdown: string\n html: string\n}\n\n/**\n * Internal: paired posterior summary on (candidate − comparator) deltas.\n *\n * Returns the bootstrap CI on the median (matching `gainHistogram`) plus\n * Bayesian-flavoured posterior summaries Pr(Δ>0) and Pr(Δ∈ROPE) computed\n * from a Bayesian-bootstrap-flavoured resample distribution on the mean\n * (Rubin 1981 — non-informative bootstrap-prior duality), and the\n * minimum detectable paired effect at the configured power and α.\n *\n * `null` is returned when no paired observations exist; callers must\n * gate on `n` before consuming the bootstrap statistics.\n */\nfunction pairedPosterior(\n runs: RunRecord[],\n candidateId: string,\n comparator: string,\n opts: {\n split: 'search' | 'holdout'\n confidence: number\n seed?: number\n rope: { low: number; high: number } | null\n mdePower: number\n mdeAlpha: number\n },\n): {\n n: number\n meanDelta: number\n medianDelta: number\n sdDelta: number\n ci: { low: number; high: number }\n prGreaterThanZero: number\n prInRope: number | null\n mde: number\n} | null {\n const scoreField = opts.split === 'holdout' ? 'holdoutScore' : 'searchScore'\n const candidate = runs.filter((r) => r.candidateId === candidateId && r.splitTag === opts.split)\n const baseline = runs.filter((r) => r.candidateId === comparator && r.splitTag === opts.split)\n const { before, after } = pairScoresByKey(candidate, baseline, scoreField)\n const n = before.length\n if (n === 0) return null\n\n const deltas = before.map((b, i) => after[i]! - b)\n const meanDelta = deltas.reduce((s, x) => s + x, 0) / n\n const sortedDeltas = [...deltas].sort((a, b) => a - b)\n const medianDelta = medianOfSorted(sortedDeltas)\n const sdDelta = stdev(deltas, meanDelta)\n\n const ci = pairedBootstrap(before, after, {\n confidence: opts.confidence,\n resamples: 2000,\n statistic: 'median',\n seed: opts.seed,\n })\n\n // Enumerate bootstrap-mean samples to derive posterior summaries on the\n // mean delta. Same RNG family as `pairedBootstrap` but kept local so we can\n // examine the full sample distribution rather than just quantiles.\n const meanSamples = bootstrapMeanSamples(deltas, 2000, opts.seed)\n const prGreaterThanZero =\n meanSamples.length === 0 ? 0 : meanSamples.filter((s) => s > 0).length / meanSamples.length\n const prInRope =\n opts.rope === null || meanSamples.length === 0\n ? null\n : meanSamples.filter((s) => s >= opts.rope!.low && s <= opts.rope!.high).length /\n meanSamples.length\n\n const dStandardised = pairedMde({ nPaired: n, alpha: opts.mdeAlpha, power: opts.mdePower })\n const mde = sdDelta === 0 ? 0 : dStandardised * sdDelta\n\n return {\n n,\n meanDelta,\n medianDelta,\n sdDelta,\n ci: { low: ci.low, high: ci.high },\n prGreaterThanZero,\n prInRope,\n mde,\n }\n}\n\nfunction bootstrapMeanSamples(deltas: number[], resamples: number, seed?: number): number[] {\n const n = deltas.length\n if (n === 0) return []\n if (n === 1) return new Array<number>(resamples).fill(deltas[0]!)\n const rng = seedRng(seed)\n const samples = new Array<number>(resamples)\n for (let b = 0; b < resamples; b++) {\n let sum = 0\n for (let k = 0; k < n; k++) sum += deltas[Math.floor(rng() * n)]!\n samples[b] = sum / n\n }\n return samples\n}\n\nfunction seedRng(seed?: number): () => number {\n if (seed === undefined) return Math.random\n let s = seed >>> 0\n return () => {\n s = (s + 0x6d2b79f5) >>> 0\n let t = s\n t = Math.imul(t ^ (t >>> 15), t | 1)\n t ^= t + Math.imul(t ^ (t >>> 7), t | 61)\n return ((t ^ (t >>> 14)) >>> 0) / 4294967296\n }\n}\n\nfunction stdev(xs: number[], mean: number): number {\n if (xs.length < 2) return 0\n let sse = 0\n for (const x of xs) sse += (x - mean) ** 2\n return Math.sqrt(sse / (xs.length - 1))\n}\n\n/**\n * Executive research report for CPO / AI-lead / launch-review consumption.\n *\n * Composes:\n * - `summaryTable` marginal stats with BH-FDR-adjusted q-values\n * - `paretoChart` cost-vs-quality frontier with gate overlay\n * - `gainHistogram` per-candidate paired-delta distribution\n * - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),\n * Pr(Δ∈ROPE), MDE at the configured power\n *\n * Decisions are made on paired evidence — never on marginal means alone —\n * and respect any held-out gate decision the caller passes through. The\n * report embeds a SHA-256 fingerprint of the input run set and, optionally,\n * the hash of a preregistered protocol so a downstream reader can verify\n * provenance and that the analysis was the preregistered one.\n *\n * Async because the fingerprint uses Web Crypto via `hashJson`; deterministic\n * for any fixed `runs`, `seed`, and ROPE.\n */\nexport async function researchReport(\n runs: RunRecord[],\n opts: ResearchReportOptions = {},\n): Promise<ResearchReport> {\n const split = opts.split ?? 'holdout'\n const comparator = opts.comparator ?? null\n const confidence = opts.confidence ?? 0.95\n const fdr = opts.fdr ?? 0.05\n const minPairs = Math.max(opts.minPairs ?? 20, RESEARCH_REPORT_HARD_PAIR_FLOOR)\n const rope = opts.rope ?? null\n const mdePower = opts.mdePower ?? 0.8\n const mdeAlpha = opts.mdeAlpha ?? fdr\n const title = opts.title ?? 'Agent Evaluation Research Report'\n const generatedAt = opts.generatedAt ?? new Date().toISOString()\n const preregistrationHash = opts.preregistrationHash ?? null\n\n if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) {\n throw new Error(\n `researchReport: rope must satisfy low ≤ high with finite bounds, got ${JSON.stringify(rope)}`,\n )\n }\n\n const summary = summaryTable(runs, {\n comparator: comparator ?? undefined,\n split,\n confidence,\n fdr,\n })\n const pareto = paretoChart(runs, { split, gateDecisions: opts.gateDecisions })\n const candidateIds =\n opts.candidateIds ?? summary.rows.map((r) => r.candidateId).filter((id) => id !== comparator)\n const gains = comparator\n ? candidateIds.map((id) =>\n gainHistogram(runs, id, comparator, {\n split,\n confidence,\n seed: opts.seed,\n }),\n )\n : []\n\n const gainByCandidate = new Map(gains.map((g) => [g.candidateId, g]))\n const paretoByCandidate = new Map(pareto.points.map((p) => [p.candidateId, p]))\n const posteriorByCandidate = new Map<string, ReturnType<typeof pairedPosterior>>()\n if (comparator) {\n for (const id of candidateIds) {\n posteriorByCandidate.set(\n id,\n pairedPosterior(runs, id, comparator, {\n split,\n confidence,\n seed: opts.seed,\n rope,\n mdePower,\n mdeAlpha,\n }),\n )\n }\n }\n\n const candidates = summary.rows\n .map((row) => {\n const gain = gainByCandidate.get(row.candidateId)\n const point = paretoByCandidate.get(row.candidateId)\n const posterior = posteriorByCandidate.get(row.candidateId) ?? null\n const classified = classifyCandidate(row, {\n comparator,\n posterior,\n point,\n fdr,\n minPairs,\n rope,\n })\n return {\n candidateId: row.candidateId,\n n: row.n,\n mean: row.mean,\n ciLow: row.ciLow,\n ciHigh: row.ciHigh,\n qValue: row.qValue,\n cohensD: row.cohensD,\n meanDeltaVsComparator: posterior ? posterior.meanDelta : null,\n pairedN: posterior?.n ?? gain?.n ?? 0,\n medianGain: posterior ? posterior.medianDelta : gain ? gain.median : null,\n meanGain: posterior ? posterior.meanDelta : null,\n gainCi: posterior ? posterior.ci : gain ? gain.ci : null,\n prGreaterThanZero: posterior ? posterior.prGreaterThanZero : null,\n prInRope: posterior ? posterior.prInRope : null,\n mde: posterior ? posterior.mde : null,\n onParetoFrontier: point?.onFrontier ?? false,\n gate: point?.gate,\n decision: classified.decision,\n decisionReason: classified.reason,\n } satisfies ResearchReportCandidate\n })\n .sort((a, b) => {\n const decisionRank = decisionWeight(b.decision) - decisionWeight(a.decision)\n if (decisionRank !== 0) return decisionRank\n return b.mean - a.mean\n })\n\n const recommendation = buildRecommendation(candidates, {\n comparator,\n failureClusters: opts.failureClusters,\n rope,\n minPairs,\n preregistrationHash,\n })\n const executiveSummary = buildExecutiveSummary(candidates, recommendation, {\n comparator,\n split,\n failureClusters: opts.failureClusters,\n preregistrationHash,\n })\n const methodology = buildMethodology({\n split,\n comparator,\n fdr,\n minPairs,\n rope,\n confidence,\n mdePower,\n mdeAlpha,\n })\n\n const runFingerprint = await hashJson(\n canonicalize({\n triples: runs\n .filter((r) => r.splitTag === split)\n .map((r) => ({ runId: r.runId, candidateId: r.candidateId, splitTag: r.splitTag }))\n .sort((a, b) => a.runId.localeCompare(b.runId)),\n comparator,\n split,\n }),\n )\n\n const markdown = renderResearchMarkdown({\n title,\n generatedAt,\n split,\n comparator,\n rope,\n runFingerprint,\n preregistrationHash,\n executiveSummary,\n recommendation,\n candidates,\n summary,\n pareto,\n gains,\n methodology,\n failureClusters: opts.failureClusters,\n })\n const html = renderResearchHtml(markdown, title)\n\n return {\n kind: 'agent-eval-research-report',\n title,\n generatedAt,\n split,\n comparator,\n runFingerprint,\n preregistrationHash,\n rope,\n executiveSummary,\n recommendation,\n candidates,\n summary,\n charts: { pareto, gains },\n methodology,\n failureClusters: opts.failureClusters,\n markdown,\n html,\n }\n}\n\nfunction buildMethodology(ctx: {\n split: 'search' | 'holdout'\n comparator: string | null\n fdr: number\n minPairs: number\n rope: { low: number; high: number } | null\n confidence: number\n mdePower: number\n mdeAlpha: number\n}): ResearchReportMethodology {\n const assumptions: string[] = [\n 'Pairs are matched by (experimentId, seed); the candidate and comparator see the same scenarios in the same order.',\n 'Paired deltas are exchangeable conditional on the matched scenario — no mid-run distribution shift.',\n `Decisions are pre-specified at fdr=${ctx.fdr}, minPairs=${ctx.minPairs}, confidence=${ctx.confidence}; deviating from these post-hoc invalidates the false-discovery control.`,\n ]\n if (ctx.rope) {\n assumptions.push(\n `The Region of Practical Equivalence ${formatRope(ctx.rope)} is supplied by the domain owner; equivalent verdicts are only meaningful if that range is treated as the standing definition of \"no material difference.\"`,\n )\n }\n if (ctx.comparator === null) {\n assumptions.push('No comparator was configured; this run is descriptive, not causal.')\n }\n const methods: string[] = [\n \"Marginal scores summarised with BH-FDR-adjusted Wilcoxon signed-rank q-values and Cohen's d via summaryTable.\",\n 'Paired evidence summarised with bootstrap CI on the median delta and Bayesian-bootstrap-style Pr(Δ>0) and Pr(Δ∈ROPE) on the mean delta.',\n `Minimum detectable effect reported per candidate at α=${ctx.mdeAlpha} (two-sided), power=${ctx.mdePower}, standardised by the observed paired-delta SD.`,\n 'Pareto frontier flagged as a separate axis (cost vs quality); a candidate can be on-frontier without winning the paired test.',\n 'Held-out gate decisions, when supplied, override the statistical verdict in the reject direction.',\n ]\n const alternatives: string[] = [\n 'Paired t-test rejected: not robust to the heavy-tailed score distributions common in agent benchmarks.',\n 'Unpaired Mann–Whitney rejected: matched scenarios make pairing free; unpaired throws away that variance reduction.',\n 'Sequential / always-valid inference (e-values, mSPRT) is the right tool for iterative sweeps and is out of scope for this single-look report — preregister and run once, or wrap this report in an alpha-spending schedule.',\n 'Hierarchical Bayesian shrinkage across many candidates is future work; the current ranking uses raw paired statistics.',\n ]\n const whenNotToApply: string[] = [\n `Paired N below ${RESEARCH_REPORT_HARD_PAIR_FLOOR} on any candidate — the bootstrap CI is degenerate.`,\n 'Comparator chosen post-hoc by inspecting the same data; q-values are no longer false-discovery-controlled.',\n 'Scenarios not drawn under a stable preregistered protocol; the report can describe the data but cannot anchor a launch decision.',\n 'Score distributions with mid-run shift (judge model swap, rubric change, infra outage) — pair exchangeability is violated.',\n ]\n const citations: string[] = [\n 'Benjamini, Y. & Hochberg, Y. (1995). Controlling the false discovery rate: a practical and powerful approach to multiple testing. JRSS B, 57(1), 289–300.',\n 'Wilcoxon, F. (1945). Individual comparisons by ranking methods. Biometrics Bulletin, 1(6), 80–83.',\n 'Efron, B. (1979). Bootstrap methods: another look at the jackknife. Annals of Statistics, 7(1), 1–26.',\n 'Rubin, D. B. (1981). The Bayesian bootstrap. Annals of Statistics, 9(1), 130–134.',\n 'Kruschke, J. K. (2018). Rejecting or accepting parameter values in Bayesian estimation. Advances in Methods and Practices in Psychological Science, 1(2), 270–280. (ROPE.)',\n ]\n return { assumptions, methods, alternatives, whenNotToApply, citations }\n}\n\nfunction formatRope(rope: { low: number; high: number }): string {\n return `[${fmt(rope.low)}, ${fmt(rope.high)}]`\n}\n\nfunction classifyCandidate(\n row: SummaryTableRow,\n ctx: {\n comparator: string | null\n posterior: ReturnType<typeof pairedPosterior> | null\n point?: ParetoPoint\n fdr: number\n minPairs: number\n rope: { low: number; high: number } | null\n },\n): { decision: ResearchReportDecision; reason: string } {\n if (ctx.comparator && row.candidateId === ctx.comparator) {\n return { decision: 'hold', reason: 'Comparator baseline.' }\n }\n if (!ctx.comparator) {\n return {\n decision: ctx.point?.onFrontier ? 'hold' : 'needs_more_data',\n reason:\n 'No comparator configured; report ranks candidates but cannot anchor a promotion call.',\n }\n }\n // Held-out gate is authoritative against — promote requires statistical\n // evidence even if the gate said `promote` (gate is necessary, not sufficient).\n if (ctx.point?.gate && ctx.point.gate !== 'promote') {\n return { decision: 'reject', reason: `Held-out gate returned ${ctx.point.gate}.` }\n }\n if (!ctx.posterior || ctx.posterior.n < RESEARCH_REPORT_HARD_PAIR_FLOOR) {\n return {\n decision: 'needs_more_data',\n reason: `Only ${ctx.posterior?.n ?? 0} paired observations; below hard floor of ${RESEARCH_REPORT_HARD_PAIR_FLOOR} for any paired inference.`,\n }\n }\n const ci = ctx.posterior.ci\n if (ctx.rope && ci.low >= ctx.rope.low && ci.high <= ctx.rope.high) {\n return {\n decision: 'equivalent',\n reason: `Paired-delta CI [${fmt(ci.low)}, ${fmt(ci.high)}] is fully inside ROPE ${formatRope(ctx.rope)}; candidate is practically equivalent to comparator.`,\n }\n }\n const significant = Number.isFinite(row.qValue) && row.qValue <= ctx.fdr\n const gainPositive = ci.low > 0\n const gainNegative = ci.high < 0\n if (gainNegative) {\n return {\n decision: 'reject',\n reason: `Paired-delta CI [${fmt(ci.low)}, ${fmt(ci.high)}] lies entirely below zero.`,\n }\n }\n if (ctx.posterior.n < ctx.minPairs) {\n return {\n decision: 'needs_more_data',\n reason: `Only ${ctx.posterior.n} paired observations; minimum detectable effect at this N is ${fmt(ctx.posterior.mde)} score units (need ≥ ${ctx.minPairs} pairs to issue a directional verdict).`,\n }\n }\n if (significant && gainPositive) {\n return {\n decision: 'promote',\n reason: `BH-adjusted q=${fmt(row.qValue)} ≤ ${ctx.fdr} and paired-delta CI [${fmt(ci.low)}, ${fmt(ci.high)}] excludes zero; Pr(Δ>0)=${fmt(ctx.posterior.prGreaterThanZero)}.`,\n }\n }\n return {\n decision: 'hold',\n reason: `Pr(Δ>0)=${fmt(ctx.posterior.prGreaterThanZero)} but CI [${fmt(ci.low)}, ${fmt(ci.high)}] crosses zero; effect not decisive at fdr=${ctx.fdr}.`,\n }\n}\n\nfunction buildRecommendation(\n candidates: ResearchReportCandidate[],\n ctx: {\n comparator: string | null\n failureClusters?: FailureClusterReport\n rope: { low: number; high: number } | null\n minPairs: number\n preregistrationHash: string | null\n },\n): ResearchReportRecommendation {\n const nonComparator = candidates.filter((c) => c.candidateId !== ctx.comparator)\n const bestPromote = nonComparator.find((c) => c.decision === 'promote')\n const bestEquivalent = nonComparator.find((c) => c.decision === 'equivalent')\n const chosen = bestPromote ?? bestEquivalent ?? nonComparator[0] ?? null\n const decision: ResearchReportDecision = bestPromote\n ? 'promote'\n : nonComparator.some((c) => c.decision === 'needs_more_data')\n ? 'needs_more_data'\n : bestEquivalent\n ? 'equivalent'\n : nonComparator.some((c) => c.decision === 'hold')\n ? 'hold'\n : 'reject'\n\n const rationale: string[] = []\n const risks: string[] = []\n const nextActions: string[] = []\n\n if (chosen) {\n rationale.push(`${chosen.candidateId}: ${chosen.decisionReason}`)\n if (chosen.gainCi) {\n const probSummary =\n chosen.prGreaterThanZero !== null ? `, Pr(Δ>0)=${fmt(chosen.prGreaterThanZero)}` : ''\n rationale.push(\n `Median paired gain CI: [${fmt(chosen.gainCi.low)}, ${fmt(chosen.gainCi.high)}]${probSummary}.`,\n )\n }\n if (chosen.mde !== null && Number.isFinite(chosen.mde)) {\n rationale.push(`MDE at current paired N=${chosen.pairedN}: ${fmt(chosen.mde)} score units.`)\n }\n }\n if (!ctx.comparator) {\n risks.push('No comparator was configured; verdict is descriptive, not causal.')\n nextActions.push('Re-run with a stable comparator candidate for paired inference.')\n }\n if (!ctx.preregistrationHash) {\n risks.push(\n 'No preregistration hash supplied; readers cannot verify the analysis was specified before data inspection.',\n )\n nextActions.push(\n 'Sign a HypothesisManifest before the next sweep and pass `preregistrationHash` so the report cites it.',\n )\n }\n if (ctx.rope === null && nonComparator.length > 0) {\n risks.push(\n 'No ROPE configured; the report cannot distinguish \"equivalent\" from \"inconclusive\".',\n )\n nextActions.push(\n 'Define a domain-specific Region of Practical Equivalence and pass it to lock in the equivalence threshold.',\n )\n }\n const inconclusive = nonComparator.filter((c) => c.decision === 'needs_more_data')\n if (inconclusive.length > 0) {\n const worst = inconclusive.reduce((a, b) => (b.pairedN < a.pairedN ? b : a))\n risks.push(\n `${inconclusive.length} candidate(s) below soft floor (${ctx.minPairs} pairs); thinnest is ${worst.candidateId} with ${worst.pairedN}.`,\n )\n nextActions.push(\n `Collect at least ${ctx.minPairs - worst.pairedN} more matched holdout runs for ${worst.candidateId}.`,\n )\n }\n const rejected = nonComparator.filter((c) => c.decision === 'reject')\n if (rejected.length > 0) {\n risks.push(\n `${rejected.length} candidate(s) failed the paired test or held-out gate; do not ship those variants.`,\n )\n }\n if (ctx.failureClusters && ctx.failureClusters.clusters.length > 0) {\n const top = ctx.failureClusters.clusters[0]!\n risks.push(`Top failure cluster: ${top.failureClass} across ${top.runCount} run(s).`)\n nextActions.push('Prioritize the largest failure cluster before broad rollout.')\n }\n if (decision === 'promote') {\n nextActions.push('Ship behind the existing promotion gate and monitor canaries.')\n } else if (decision === 'hold') {\n nextActions.push('Keep current production candidate while expanding holdout evidence.')\n } else if (decision === 'equivalent') {\n nextActions.push(\n 'Either keep the comparator (no quality regression) or promote on cost/latency grounds — equivalence does not justify either; the choice is a product decision, not a stats one.',\n )\n } else if (decision === 'reject') {\n nextActions.push(\n 'Do not promote this sweep; inspect failures and generate a revised candidate.',\n )\n }\n\n return {\n decision,\n candidateId: chosen?.candidateId ?? null,\n rationale,\n risks,\n nextActions,\n }\n}\n\nfunction buildExecutiveSummary(\n candidates: ResearchReportCandidate[],\n recommendation: ResearchReportRecommendation,\n ctx: {\n comparator: string | null\n split: 'search' | 'holdout'\n failureClusters?: FailureClusterReport\n preregistrationHash: string | null\n },\n): string[] {\n const lines: string[] = []\n const nonComparator = candidates.filter((c) => c.candidateId !== ctx.comparator)\n lines.push(\n `Evaluated ${nonComparator.length} candidate(s) on the ${ctx.split} split${ctx.comparator ? ` against ${ctx.comparator}` : ''}.`,\n )\n lines.push(\n `Recommendation: ${recommendation.decision}${recommendation.candidateId ? ` ${recommendation.candidateId}` : ''}.`,\n )\n const promoted = nonComparator.filter((c) => c.decision === 'promote').length\n const held = nonComparator.filter((c) => c.decision === 'hold').length\n const equivalent = nonComparator.filter((c) => c.decision === 'equivalent').length\n const rejected = nonComparator.filter((c) => c.decision === 'reject').length\n const more = nonComparator.filter((c) => c.decision === 'needs_more_data').length\n lines.push(\n `Decision mix: ${promoted} promote, ${equivalent} equivalent, ${held} hold, ${rejected} reject, ${more} need more data.`,\n )\n const frontier = nonComparator.filter((c) => c.onParetoFrontier).map((c) => c.candidateId)\n if (frontier.length > 0) lines.push(`Pareto-frontier candidates: ${frontier.join(', ')}.`)\n if (ctx.failureClusters) {\n lines.push(\n `Failure clustering found ${ctx.failureClusters.totalFailures}/${ctx.failureClusters.totalRuns} failed runs across ${ctx.failureClusters.clusters.length} reportable cluster(s).`,\n )\n }\n lines.push(\n ctx.preregistrationHash\n ? `Preregistered analysis: ${ctx.preregistrationHash.slice(0, 12)}…`\n : 'Analysis is post-hoc — no preregistration hash supplied.',\n )\n return lines\n}\n\nfunction renderResearchMarkdown(report: {\n title: string\n generatedAt: string\n split: 'search' | 'holdout'\n comparator: string | null\n executiveSummary: string[]\n recommendation: ResearchReportRecommendation\n candidates: ResearchReportCandidate[]\n summary: SummaryTable\n pareto: ParetoFigureSpec\n gains: GainDistributionFigureSpec[]\n rope: { low: number; high: number } | null\n runFingerprint: string\n preregistrationHash: string | null\n methodology: ResearchReportMethodology\n failureClusters?: FailureClusterReport\n}): string {\n const lines: string[] = []\n lines.push(`# ${report.title}`)\n lines.push('')\n lines.push(`**Generated:** ${report.generatedAt}`)\n lines.push(`**Primary split:** ${report.split}`)\n lines.push(`**Comparator:** ${report.comparator ?? 'not configured'}`)\n lines.push(`**ROPE:** ${report.rope ? formatRope(report.rope) : 'not configured'}`)\n lines.push(`**Run fingerprint:** \\`${report.runFingerprint}\\``)\n lines.push(\n `**Preregistration:** ${report.preregistrationHash ? `\\`${report.preregistrationHash}\\`` : 'none'}`,\n )\n lines.push('')\n lines.push('## Executive Summary')\n lines.push('')\n for (const item of report.executiveSummary) lines.push(`- ${item}`)\n lines.push('')\n lines.push('## Recommendation')\n lines.push('')\n lines.push(`**Decision:** ${report.recommendation.decision}`)\n lines.push(`**Candidate:** ${report.recommendation.candidateId ?? 'N/A'}`)\n lines.push('')\n lines.push('### Rationale')\n lines.push('')\n for (const item of report.recommendation.rationale) lines.push(`- ${item}`)\n lines.push('')\n lines.push('### Risks')\n lines.push('')\n for (const item of report.recommendation.risks.length\n ? report.recommendation.risks\n : ['No material report-level risks detected.']) {\n lines.push(`- ${item}`)\n }\n lines.push('')\n lines.push('### Next Actions')\n lines.push('')\n for (const item of report.recommendation.nextActions) lines.push(`- ${item}`)\n lines.push('')\n lines.push('## Candidate Decision Table')\n lines.push('')\n lines.push(\n '| Candidate | Decision | Mean | Δ̄ | Pr(Δ>0) | q | d | Paired N | Median Gain CI | MDE | Pareto | Gate |',\n )\n lines.push('|---|---|---:|---:|---:|---:|---:|---:|---|---:|---|---|')\n for (const c of report.candidates) {\n const delta = c.meanDeltaVsComparator === null ? '-' : signed(c.meanDeltaVsComparator)\n const prGt = c.prGreaterThanZero === null ? '-' : c.prGreaterThanZero.toFixed(3)\n const q = Number.isFinite(c.qValue) ? c.qValue.toFixed(4) : '-'\n const d = Number.isFinite(c.cohensD) ? c.cohensD.toFixed(3) : '-'\n const gain = c.gainCi ? `[${fmt(c.gainCi.low)}, ${fmt(c.gainCi.high)}]` : '-'\n const mde = c.mde === null || !Number.isFinite(c.mde) ? '-' : fmt(c.mde)\n lines.push(\n `| ${c.candidateId} | ${c.decision} | ${fmt(c.mean)} | ${delta} | ${prGt} | ${q} | ${d} | ${c.pairedN} | ${gain} | ${mde} | ${c.onParetoFrontier ? 'yes' : 'no'} | ${c.gate ?? '-'} |`,\n )\n }\n lines.push('')\n lines.push('## Statistical Summary')\n lines.push('')\n lines.push(report.summary.markdown)\n lines.push('')\n lines.push('## Methodology')\n lines.push('')\n lines.push('### Assumptions')\n lines.push('')\n for (const item of report.methodology.assumptions) lines.push(`- ${item}`)\n lines.push('')\n lines.push('### Methods')\n lines.push('')\n for (const item of report.methodology.methods) lines.push(`- ${item}`)\n lines.push('')\n lines.push('### Alternatives Considered')\n lines.push('')\n for (const item of report.methodology.alternatives) lines.push(`- ${item}`)\n lines.push('')\n lines.push('### When NOT To Apply')\n lines.push('')\n for (const item of report.methodology.whenNotToApply) lines.push(`- ${item}`)\n lines.push('')\n lines.push('### Citations')\n lines.push('')\n for (const item of report.methodology.citations) lines.push(`- ${item}`)\n lines.push('')\n lines.push('## Chart Specs')\n lines.push('')\n lines.push(\n 'The report carries JSON chart specs for Pareto cost/quality and paired gain histograms.',\n )\n lines.push('')\n lines.push('```json')\n lines.push(JSON.stringify({ pareto: report.pareto, gains: report.gains }, null, 2))\n lines.push('```')\n if (report.failureClusters) {\n lines.push('')\n lines.push('## Failure Clusters')\n lines.push('')\n lines.push('| Failure Class | Runs | Scenarios | Tool | Example |')\n lines.push('|---|---:|---:|---|---|')\n for (const c of report.failureClusters.clusters.slice(0, 10)) {\n lines.push(\n `| ${c.failureClass} | ${c.runCount} | ${c.scenarioIds.length} | ${c.toolName ?? '-'} | ${escapePipes(c.exampleError ?? c.exampleRunId)} |`,\n )\n }\n }\n return lines.join('\\n')\n}\n\nfunction renderResearchHtml(markdown: string, title: string): string {\n const body = markdownToHtml(markdown)\n return [\n '<!doctype html>',\n '<html lang=\"en\">',\n '<head>',\n '<meta charset=\"utf-8\">',\n '<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">',\n `<title>${escapeHtml(title)}</title>`,\n '<style>',\n 'body{font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,\"Segoe UI\",sans-serif;margin:0;color:#172026;background:#f7f8f8;}',\n 'main{max-width:1080px;margin:0 auto;padding:40px 24px 64px;background:#fff;min-height:100vh;}',\n 'h1{font-size:34px;line-height:1.15;margin:0 0 20px;}h2{margin-top:34px;border-top:1px solid #d9dfdf;padding-top:22px;}h3{margin-top:22px;}',\n 'p,li{line-height:1.55;}table{border-collapse:collapse;width:100%;margin:16px 0;font-size:14px;}th,td{border:1px solid #d9dfdf;padding:8px;text-align:left;}th{background:#eef2f2;}',\n 'code,pre{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;}pre{overflow:auto;background:#111827;color:#f9fafb;padding:16px;border-radius:6px;}',\n '</style>',\n '</head>',\n '<body><main>',\n body,\n '</main></body></html>',\n ].join('\\n')\n}\n\nfunction markdownToHtml(markdown: string): string {\n const lines = markdown.split('\\n')\n const html: string[] = []\n let inList = false\n let inCode = false\n let code: string[] = []\n let table: string[] = []\n\n const flushList = () => {\n if (inList) {\n html.push('</ul>')\n inList = false\n }\n }\n const flushTable = () => {\n if (table.length === 0) return\n html.push(renderMarkdownTable(table))\n table = []\n }\n\n for (const line of lines) {\n if (line.startsWith('```')) {\n if (inCode) {\n html.push(`<pre><code>${escapeHtml(code.join('\\n'))}</code></pre>`)\n code = []\n inCode = false\n } else {\n flushList()\n flushTable()\n inCode = true\n }\n continue\n }\n if (inCode) {\n code.push(line)\n continue\n }\n if (line.startsWith('|')) {\n flushList()\n table.push(line)\n continue\n }\n flushTable()\n if (line.startsWith('- ')) {\n if (!inList) {\n html.push('<ul>')\n inList = true\n }\n html.push(`<li>${inlineMarkdown(line.slice(2))}</li>`)\n continue\n }\n flushList()\n if (line.startsWith('# ')) html.push(`<h1>${inlineMarkdown(line.slice(2))}</h1>`)\n else if (line.startsWith('## ')) html.push(`<h2>${inlineMarkdown(line.slice(3))}</h2>`)\n else if (line.startsWith('### ')) html.push(`<h3>${inlineMarkdown(line.slice(4))}</h3>`)\n else if (line.trim() === '') html.push('')\n else html.push(`<p>${inlineMarkdown(line)}</p>`)\n }\n flushList()\n flushTable()\n return html.join('\\n')\n}\n\nfunction renderMarkdownTable(lines: string[]): string {\n const rows = lines\n .filter((line) => !/^\\|[-:\\s|]+\\|$/.test(line))\n .map((line) =>\n line\n .slice(1, -1)\n .split('|')\n .map((cell) => inlineMarkdown(cell.trim())),\n )\n if (rows.length === 0) return ''\n const [head, ...body] = rows\n const th = head!.map((cell) => `<th>${cell}</th>`).join('')\n const trs = body\n .map((row) => `<tr>${row.map((cell) => `<td>${cell}</td>`).join('')}</tr>`)\n .join('\\n')\n return `<table><thead><tr>${th}</tr></thead><tbody>${trs}</tbody></table>`\n}\n\nfunction inlineMarkdown(s: string): string {\n return escapeHtml(s).replace(/\\*\\*([^*]+)\\*\\*/g, '<strong>$1</strong>')\n}\n\nfunction escapeHtml(s: string): string {\n return s\n .replace(/&/g, '&')\n .replace(/</g, '<')\n .replace(/>/g, '>')\n .replace(/\"/g, '"')\n}\n\nfunction escapePipes(s: string): string {\n return s.replace(/\\|/g, '\\\\|')\n}\n\nfunction decisionWeight(decision: ResearchReportDecision): number {\n if (decision === 'promote') return 5\n if (decision === 'equivalent') return 4\n if (decision === 'hold') return 3\n if (decision === 'needs_more_data') return 2\n return 1\n}\n\nfunction signed(x: number): string {\n return `${x >= 0 ? '+' : ''}${fmt(x)}`\n}\n\n// ── tiny helpers ─────────────────────────────────────────────────────\n\nfunction avg(xs: number[]): number {\n if (xs.length === 0) return Number.NaN\n return xs.reduce((s, x) => s + x, 0) / xs.length\n}\n\nfunction medianOfSorted(sorted: number[]): number {\n if (sorted.length === 0) return 0\n const mid = Math.floor(sorted.length / 2)\n return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]!\n}\n\nfunction fmt(x: number): string {\n if (!Number.isFinite(x)) return String(x)\n return x.toFixed(4)\n}\n"],"mappings":";;;;;;;;;;;;;;AA6EO,SAAS,aAAa,MAAmB,OAA4B,CAAC,GAAiB;AAC5F,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,MAAM,KAAK,OAAO;AACxB,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,aAAa,UAAU,YAAY,iBAAiB;AAE1D,QAAM,cAAc,oBAAI,IAAqD;AAC7E,aAAW,KAAK,MAAM;AACpB,QAAI,EAAE,aAAa,MAAO;AAC1B,UAAM,IAAI,EAAE,QAAQ,UAAU;AAC9B,QAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,EAAG;AAClD,UAAM,SAAS,YAAY,IAAI,EAAE,WAAW,KAAK,EAAE,MAAM,CAAC,GAAG,QAAQ,CAAC,EAAE;AACxE,WAAO,KAAK,KAAK,CAAC;AAClB,WAAO,OAAO,KAAK,CAAC;AACpB,gBAAY,IAAI,EAAE,aAAa,MAAM;AAAA,EACvC;AAEA,QAAM,eAAe,CAAC,GAAG,YAAY,KAAK,CAAC,EAAE,KAAK;AAClD,QAAM,WAAW,aAAa,YAAY,IAAI,UAAU,IAAI;AAG5D,QAAM,YAAuD,CAAC;AAC9D,aAAW,MAAM,cAAc;AAC7B,UAAM,SAAS,YAAY,IAAI,EAAE;AACjC,UAAM,KAAK,mBAAmB,OAAO,QAAQ,UAAU;AACvD,QAAI,OAAO,OAAO;AAClB,QAAI,IAAI,OAAO;AACf,QAAI,cAAc,YAAY,OAAO,YAAY;AAC/C,YAAM,SAAS,gBAAgB,OAAO,MAAM,SAAS,MAAM,UAAU;AACrE,UAAI,OAAO,OAAO,UAAU,GAAG;AAC7B,eAAO,mBAAmB,OAAO,QAAQ,OAAO,KAAK,EAAE;AAAA,MACzD;AACA,UAAI,QAAQ,SAAS,QAAQ,OAAO,MAAM;AAAA,IAC5C;AACA,cAAU,KAAK;AAAA,MACb,aAAa;AAAA,MACb,GAAG,OAAO,OAAO;AAAA,MACjB,MAAM,GAAG;AAAA,MACT,OAAO,GAAG;AAAA,MACV,QAAQ,GAAG;AAAA,MACX,QAAQ;AAAA,MACR,SAAS;AAAA,MACT;AAAA,IACF,CAAC;AAAA,EACH;AAKA,MAAI,YAAY;AACd,UAAM,OAAiB,CAAC;AACxB,UAAM,KAAe,CAAC;AACtB,aAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,YAAM,IAAI,UAAU,CAAC;AACrB,UAAI,EAAE,gBAAgB,WAAY;AAClC,UAAI,CAAC,OAAO,SAAS,EAAE,IAAI,EAAG;AAC9B,WAAK,KAAK,CAAC;AACX,SAAG,KAAK,EAAE,IAAI;AAAA,IAChB;AACA,QAAI,GAAG,SAAS,GAAG;AACjB,YAAM,EAAE,QAAQ,IAAI,kBAAkB,IAAI,GAAG;AAC7C,eAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,kBAAU,KAAK,CAAC,CAAE,EAAG,SAAS,QAAQ,CAAC;AAAA,MACzC;AAAA,IACF;AAAA,EACF;AAEA,QAAM,OAAO,UAAU,IAAI,CAAC,EAAE,MAAM,OAAO,GAAG,KAAK,MAAM,IAAI;AAC7D,QAAM,WAAW,2BAA2B,MAAM,YAAY,KAAK;AACnE,SAAO,EAAE,MAAM,YAAY,OAAO,SAAS;AAC7C;AAEA,SAAS,gBACP,WACA,UACA,YACuC;AACvC,QAAM,UAAU,oBAAI,IAAoB;AACxC,aAAW,KAAK,UAAU;AACxB,UAAM,IAAI,EAAE,QAAQ,UAAU;AAC9B,QAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,GAAG;AAC/C,cAAQ,IAAI,GAAG,EAAE,YAAY,KAAK,EAAE,IAAI,IAAI,CAAC;AAAA,IAC/C;AAAA,EACF;AACA,QAAM,SAAmB,CAAC;AAC1B,QAAM,QAAkB,CAAC;AACzB,aAAW,KAAK,WAAW;AACzB,UAAM,IAAI,EAAE,QAAQ,UAAU;AAC9B,QAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,EAAG;AAClD,UAAM,MAAM,GAAG,EAAE,YAAY,KAAK,EAAE,IAAI;AACxC,UAAM,IAAI,QAAQ,IAAI,GAAG;AACzB,QAAI,MAAM,OAAW;AACrB,WAAO,KAAK,CAAC;AACb,UAAM,KAAK,CAAC;AAAA,EACd;AACA,SAAO,EAAE,QAAQ,MAAM;AACzB;AAEA,SAAS,2BACP,MACA,YACA,OACQ;AACR,QAAM,QAAkB,CAAC;AACzB,QAAM,WAAW,aAAa,QAAQ,UAAU,MAAM;AACtD,QAAM,KAAK,wBAAmB,KAAK,SAAS,QAAQ,EAAE;AACtD,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,wDAAwD;AACnE,QAAM,KAAK,+BAA+B;AAC1C,aAAW,KAAK,MAAM;AACpB,UAAM,KAAK,IAAI,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,MAAM,CAAC;AAC7C,UAAM,IAAI,OAAO,SAAS,EAAE,MAAM,IAAI,EAAE,OAAO,QAAQ,CAAC,IAAI;AAC5D,UAAM,IAAI,OAAO,SAAS,EAAE,OAAO,IAAI,EAAE,QAAQ,QAAQ,CAAC,IAAI;AAC9D,UAAM,KAAK,KAAK,EAAE,WAAW,MAAM,EAAE,CAAC,MAAM,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;AAAA,EACnF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;AAgCO,SAAS,YACd,MACA,OAGI,CAAC,GACa;AAClB,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,aAAa,UAAU,YAAY,iBAAiB;AAE1D,QAAM,UAAU,oBAAI,IAAmD;AACvE,aAAW,KAAK,MAAM;AACpB,QAAI,EAAE,aAAa,MAAO;AAC1B,UAAM,IAAI,EAAE,QAAQ,UAAU;AAC9B,QAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,EAAG;AAClD,UAAM,SAAS,QAAQ,IAAI,EAAE,WAAW,KAAK,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,EAAE;AACrE,WAAO,KAAK,KAAK,EAAE,OAAO;AAC1B,WAAO,QAAQ,KAAK,CAAC;AACrB,YAAQ,IAAI,EAAE,aAAa,MAAM;AAAA,EACnC;AAEA,QAAM,SAAwB,CAAC;AAC/B,aAAW,CAAC,aAAa,MAAM,KAAK,QAAQ,QAAQ,GAAG;AACrD,WAAO,KAAK;AAAA,MACV;AAAA,MACA,MAAM,IAAI,OAAO,IAAI;AAAA,MACrB,SAAS,IAAI,OAAO,OAAO;AAAA,MAC3B,GAAG,OAAO,KAAK;AAAA,MACf,YAAY;AAAA,MACZ,MAAM,KAAK,gBAAgB,WAAW,IAClC,UAAU,KAAK,cAAc,WAAW,CAAE,IAC1C;AAAA,IACN,CAAC;AAAA,EACH;AAKA,aAAW,KAAK,QAAQ;AACtB,MAAE,aAAa,CAAC,OAAO,KAAK,CAAC,MAAM,MAAM,KAAK,UAAU,GAAG,CAAC,CAAC;AAAA,EAC/D;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA,MAAM,EAAE,GAAG,WAAW,GAAG,QAAQ;AAAA,IACjC;AAAA,EACF;AACF;AAEA,SAAS,UAAU,GAAgB,GAAyB;AAC1D,SAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE;AACzF;AAEA,SAAS,UAAU,GAAsC;AACvD,MAAI,EAAE,QAAS,QAAO;AACtB,MAAI,EAAE,kBAAkB,WAAY,QAAO;AAC3C,MAAI,EAAE,kBAAkB,iBAAkB,QAAO;AACjD,MAAI,EAAE,kBAAkB,cAAe,QAAO;AAC9C,SAAO;AACT;AA2CO,SAAS,cACd,MACA,aACA,YACA,OAAgC,CAAC,GACL;AAC5B,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,aAAa,UAAU,YAAY,iBAAiB;AAC1D,QAAM,WAAW,KAAK,QAAQ;AAC9B,MAAI,WAAW,EAAG,OAAM,IAAI,MAAM,sCAAiC;AAEnE,QAAM,YAAY,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,eAAe,EAAE,aAAa,KAAK;AAC1F,QAAM,WAAW,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,cAAc,EAAE,aAAa,KAAK;AAIxF,QAAM,EAAE,QAAQ,MAAM,IAAI,gBAAgB,WAAW,UAAU,UAAU;AACzE,QAAM,IAAI,OAAO;AAEjB,MAAI,MAAM,GAAG;AACX,WAAO;AAAA,MACL,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAG;AAAA,MACH,MAAM,CAAC;AAAA,MACP,QAAQ;AAAA,MACR,IAAI,EAAE,KAAK,GAAG,MAAM,EAAE;AAAA,IACxB;AAAA,EACF;AAEA,QAAM,SAAS,OAAO,IAAI,CAAC,GAAG,MAAM,MAAM,CAAC,IAAK,CAAC;AACjD,QAAM,eAAe,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACrD,QAAM,SAAS,eAAe,YAAY;AAC1C,QAAM,MAAM,aAAa,CAAC;AAC1B,QAAM,MAAM,aAAa,aAAa,SAAS,CAAC;AAIhD,QAAM,QAAQ,KAAK,IAAI,KAAK,IAAI,GAAG,GAAG,KAAK,IAAI,GAAG,GAAG,IAAI;AACzD,QAAM,KAAK,CAAC;AACZ,QAAM,KAAK;AACX,QAAM,SAAS,KAAK,MAAM;AAC1B,QAAM,OAA8B,CAAC;AACrC,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,SAAK,KAAK,EAAE,IAAI,KAAK,IAAI,OAAO,IAAI,MAAM,IAAI,KAAK,OAAO,OAAO,EAAE,CAAC;AAAA,EACtE;AACA,aAAW,KAAK,QAAQ;AACtB,QAAI,MAAM,KAAK,OAAO,IAAI,MAAM,KAAK;AACrC,QAAI,MAAM,EAAG,OAAM;AACnB,QAAI,OAAO,SAAU,OAAM,WAAW;AACtC,SAAK,GAAG,EAAG,SAAS;AAAA,EACtB;AAEA,QAAM,KAAK,gBAAgB,QAAQ,OAAO;AAAA,IACxC,YAAY,KAAK,cAAc;AAAA,IAC/B,WAAW,KAAK,aAAa;AAAA,IAC7B,WAAW;AAAA,IACX,MAAM,KAAK;AAAA,EACb,CAAC;AAED,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,IAAI,EAAE,KAAK,GAAG,KAAK,MAAM,GAAG,KAAK;AAAA,EACnC;AACF;AAgBO,IAAM,kCAAkC;AA2J/C,SAAS,gBACP,MACA,aACA,YACA,MAiBO;AACP,QAAM,aAAa,KAAK,UAAU,YAAY,iBAAiB;AAC/D,QAAM,YAAY,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,eAAe,EAAE,aAAa,KAAK,KAAK;AAC/F,QAAM,WAAW,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,cAAc,EAAE,aAAa,KAAK,KAAK;AAC7F,QAAM,EAAE,QAAQ,MAAM,IAAI,gBAAgB,WAAW,UAAU,UAAU;AACzE,QAAM,IAAI,OAAO;AACjB,MAAI,MAAM,EAAG,QAAO;AAEpB,QAAM,SAAS,OAAO,IAAI,CAAC,GAAG,MAAM,MAAM,CAAC,IAAK,CAAC;AACjD,QAAM,YAAY,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AACtD,QAAM,eAAe,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACrD,QAAM,cAAc,eAAe,YAAY;AAC/C,QAAM,UAAU,MAAM,QAAQ,SAAS;AAEvC,QAAM,KAAK,gBAAgB,QAAQ,OAAO;AAAA,IACxC,YAAY,KAAK;AAAA,IACjB,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,KAAK;AAAA,EACb,CAAC;AAKD,QAAM,cAAc,qBAAqB,QAAQ,KAAM,KAAK,IAAI;AAChE,QAAM,oBACJ,YAAY,WAAW,IAAI,IAAI,YAAY,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,SAAS,YAAY;AACvF,QAAM,WACJ,KAAK,SAAS,QAAQ,YAAY,WAAW,IACzC,OACA,YAAY,OAAO,CAAC,MAAM,KAAK,KAAK,KAAM,OAAO,KAAK,KAAK,KAAM,IAAI,EAAE,SACvE,YAAY;AAElB,QAAM,gBAAgB,UAAU,EAAE,SAAS,GAAG,OAAO,KAAK,UAAU,OAAO,KAAK,SAAS,CAAC;AAC1F,QAAM,MAAM,YAAY,IAAI,IAAI,gBAAgB;AAEhD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,IAAI,EAAE,KAAK,GAAG,KAAK,MAAM,GAAG,KAAK;AAAA,IACjC;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,qBAAqB,QAAkB,WAAmB,MAAyB;AAC1F,QAAM,IAAI,OAAO;AACjB,MAAI,MAAM,EAAG,QAAO,CAAC;AACrB,MAAI,MAAM,EAAG,QAAO,IAAI,MAAc,SAAS,EAAE,KAAK,OAAO,CAAC,CAAE;AAChE,QAAM,MAAM,QAAQ,IAAI;AACxB,QAAM,UAAU,IAAI,MAAc,SAAS;AAC3C,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,IAAK,QAAO,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAC/D,YAAQ,CAAC,IAAI,MAAM;AAAA,EACrB;AACA,SAAO;AACT;AAEA,SAAS,QAAQ,MAA6B;AAC5C,MAAI,SAAS,OAAW,QAAO,KAAK;AACpC,MAAI,IAAI,SAAS;AACjB,SAAO,MAAM;AACX,QAAK,IAAI,eAAgB;AACzB,QAAI,IAAI;AACR,QAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;AACnC,SAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;AACxC,aAAS,IAAK,MAAM,QAAS,KAAK;AAAA,EACpC;AACF;AAEA,SAAS,MAAM,IAAc,MAAsB;AACjD,MAAI,GAAG,SAAS,EAAG,QAAO;AAC1B,MAAI,MAAM;AACV,aAAW,KAAK,GAAI,SAAQ,IAAI,SAAS;AACzC,SAAO,KAAK,KAAK,OAAO,GAAG,SAAS,EAAE;AACxC;AAqBA,eAAsB,eACpB,MACA,OAA8B,CAAC,GACN;AACzB,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,MAAM,KAAK,OAAO;AACxB,QAAM,WAAW,KAAK,IAAI,KAAK,YAAY,IAAI,+BAA+B;AAC9E,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,WAAW,KAAK,YAAY;AAClC,QAAM,WAAW,KAAK,YAAY;AAClC,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,cAAc,KAAK,gBAAe,oBAAI,KAAK,GAAE,YAAY;AAC/D,QAAM,sBAAsB,KAAK,uBAAuB;AAExD,MAAI,QAAQ,EAAE,OAAO,SAAS,KAAK,GAAG,KAAK,OAAO,SAAS,KAAK,IAAI,KAAK,KAAK,OAAO,KAAK,OAAO;AAC/F,UAAM,IAAI;AAAA,MACR,6EAAwE,KAAK,UAAU,IAAI,CAAC;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,UAAU,aAAa,MAAM;AAAA,IACjC,YAAY,cAAc;AAAA,IAC1B;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,QAAM,SAAS,YAAY,MAAM,EAAE,OAAO,eAAe,KAAK,cAAc,CAAC;AAC7E,QAAM,eACJ,KAAK,gBAAgB,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,OAAO,UAAU;AAC9F,QAAM,QAAQ,aACV,aAAa;AAAA,IAAI,CAAC,OAChB,cAAc,MAAM,IAAI,YAAY;AAAA,MAClC;AAAA,MACA;AAAA,MACA,MAAM,KAAK;AAAA,IACb,CAAC;AAAA,EACH,IACA,CAAC;AAEL,QAAM,kBAAkB,IAAI,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;AACpE,QAAM,oBAAoB,IAAI,IAAI,OAAO,OAAO,IAAI,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;AAC9E,QAAM,uBAAuB,oBAAI,IAAgD;AACjF,MAAI,YAAY;AACd,eAAW,MAAM,cAAc;AAC7B,2BAAqB;AAAA,QACnB;AAAA,QACA,gBAAgB,MAAM,IAAI,YAAY;AAAA,UACpC;AAAA,UACA;AAAA,UACA,MAAM,KAAK;AAAA,UACX;AAAA,UACA;AAAA,UACA;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,QAAM,aAAa,QAAQ,KACxB,IAAI,CAAC,QAAQ;AACZ,UAAM,OAAO,gBAAgB,IAAI,IAAI,WAAW;AAChD,UAAM,QAAQ,kBAAkB,IAAI,IAAI,WAAW;AACnD,UAAM,YAAY,qBAAqB,IAAI,IAAI,WAAW,KAAK;AAC/D,UAAM,aAAa,kBAAkB,KAAK;AAAA,MACxC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,WAAO;AAAA,MACL,aAAa,IAAI;AAAA,MACjB,GAAG,IAAI;AAAA,MACP,MAAM,IAAI;AAAA,MACV,OAAO,IAAI;AAAA,MACX,QAAQ,IAAI;AAAA,MACZ,QAAQ,IAAI;AAAA,MACZ,SAAS,IAAI;AAAA,MACb,uBAAuB,YAAY,UAAU,YAAY;AAAA,MACzD,SAAS,WAAW,KAAK,MAAM,KAAK;AAAA,MACpC,YAAY,YAAY,UAAU,cAAc,OAAO,KAAK,SAAS;AAAA,MACrE,UAAU,YAAY,UAAU,YAAY;AAAA,MAC5C,QAAQ,YAAY,UAAU,KAAK,OAAO,KAAK,KAAK;AAAA,MACpD,mBAAmB,YAAY,UAAU,oBAAoB;AAAA,MAC7D,UAAU,YAAY,UAAU,WAAW;AAAA,MAC3C,KAAK,YAAY,UAAU,MAAM;AAAA,MACjC,kBAAkB,OAAO,cAAc;AAAA,MACvC,MAAM,OAAO;AAAA,MACb,UAAU,WAAW;AAAA,MACrB,gBAAgB,WAAW;AAAA,IAC7B;AAAA,EACF,CAAC,EACA,KAAK,CAAC,GAAG,MAAM;AACd,UAAM,eAAe,eAAe,EAAE,QAAQ,IAAI,eAAe,EAAE,QAAQ;AAC3E,QAAI,iBAAiB,EAAG,QAAO;AAC/B,WAAO,EAAE,OAAO,EAAE;AAAA,EACpB,CAAC;AAEH,QAAM,iBAAiB,oBAAoB,YAAY;AAAA,IACrD;AAAA,IACA,iBAAiB,KAAK;AAAA,IACtB;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,QAAM,mBAAmB,sBAAsB,YAAY,gBAAgB;AAAA,IACzE;AAAA,IACA;AAAA,IACA,iBAAiB,KAAK;AAAA,IACtB;AAAA,EACF,CAAC;AACD,QAAM,cAAc,iBAAiB;AAAA,IACnC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AAED,QAAM,iBAAiB,MAAM;AAAA,IAC3B,aAAa;AAAA,MACX,SAAS,KACN,OAAO,CAAC,MAAM,EAAE,aAAa,KAAK,EAClC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,aAAa,EAAE,aAAa,UAAU,EAAE,SAAS,EAAE,EACjF,KAAK,CAAC,GAAG,MAAM,EAAE,MAAM,cAAc,EAAE,KAAK,CAAC;AAAA,MAChD;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAEA,QAAM,WAAW,uBAAuB;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,iBAAiB,KAAK;AAAA,EACxB,CAAC;AACD,QAAM,OAAO,mBAAmB,UAAU,KAAK;AAE/C,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,EAAE,QAAQ,MAAM;AAAA,IACxB;AAAA,IACA,iBAAiB,KAAK;AAAA,IACtB;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,iBAAiB,KASI;AAC5B,QAAM,cAAwB;AAAA,IAC5B;AAAA,IACA;AAAA,IACA,sCAAsC,IAAI,GAAG,cAAc,IAAI,QAAQ,gBAAgB,IAAI,UAAU;AAAA,EACvG;AACA,MAAI,IAAI,MAAM;AACZ,gBAAY;AAAA,MACV,uCAAuC,WAAW,IAAI,IAAI,CAAC;AAAA,IAC7D;AAAA,EACF;AACA,MAAI,IAAI,eAAe,MAAM;AAC3B,gBAAY,KAAK,oEAAoE;AAAA,EACvF;AACA,QAAM,UAAoB;AAAA,IACxB;AAAA,IACA;AAAA,IACA,8DAAyD,IAAI,QAAQ,uBAAuB,IAAI,QAAQ;AAAA,IACxG;AAAA,IACA;AAAA,EACF;AACA,QAAM,eAAyB;AAAA,IAC7B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,QAAM,iBAA2B;AAAA,IAC/B,kBAAkB,+BAA+B;AAAA,IACjD;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,QAAM,YAAsB;AAAA,IAC1B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,EAAE,aAAa,SAAS,cAAc,gBAAgB,UAAU;AACzE;AAEA,SAAS,WAAW,MAA6C;AAC/D,SAAO,IAAI,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC;AAC7C;AAEA,SAAS,kBACP,KACA,KAQsD;AACtD,MAAI,IAAI,cAAc,IAAI,gBAAgB,IAAI,YAAY;AACxD,WAAO,EAAE,UAAU,QAAQ,QAAQ,uBAAuB;AAAA,EAC5D;AACA,MAAI,CAAC,IAAI,YAAY;AACnB,WAAO;AAAA,MACL,UAAU,IAAI,OAAO,aAAa,SAAS;AAAA,MAC3C,QACE;AAAA,IACJ;AAAA,EACF;AAGA,MAAI,IAAI,OAAO,QAAQ,IAAI,MAAM,SAAS,WAAW;AACnD,WAAO,EAAE,UAAU,UAAU,QAAQ,0BAA0B,IAAI,MAAM,IAAI,IAAI;AAAA,EACnF;AACA,MAAI,CAAC,IAAI,aAAa,IAAI,UAAU,IAAI,iCAAiC;AACvE,WAAO;AAAA,MACL,UAAU;AAAA,MACV,QAAQ,QAAQ,IAAI,WAAW,KAAK,CAAC,6CAA6C,+BAA+B;AAAA,IACnH;AAAA,EACF;AACA,QAAM,KAAK,IAAI,UAAU;AACzB,MAAI,IAAI,QAAQ,GAAG,OAAO,IAAI,KAAK,OAAO,GAAG,QAAQ,IAAI,KAAK,MAAM;AAClE,WAAO;AAAA,MACL,UAAU;AAAA,MACV,QAAQ,oBAAoB,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC,0BAA0B,WAAW,IAAI,IAAI,CAAC;AAAA,IACxG;AAAA,EACF;AACA,QAAM,cAAc,OAAO,SAAS,IAAI,MAAM,KAAK,IAAI,UAAU,IAAI;AACrE,QAAM,eAAe,GAAG,MAAM;AAC9B,QAAM,eAAe,GAAG,OAAO;AAC/B,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,UAAU;AAAA,MACV,QAAQ,oBAAoB,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;AAAA,IAC1D;AAAA,EACF;AACA,MAAI,IAAI,UAAU,IAAI,IAAI,UAAU;AAClC,WAAO;AAAA,MACL,UAAU;AAAA,MACV,QAAQ,QAAQ,IAAI,UAAU,CAAC,gEAAgE,IAAI,IAAI,UAAU,GAAG,CAAC,6BAAwB,IAAI,QAAQ;AAAA,IAC3J;AAAA,EACF;AACA,MAAI,eAAe,cAAc;AAC/B,WAAO;AAAA,MACL,UAAU;AAAA,MACV,QAAQ,iBAAiB,IAAI,IAAI,MAAM,CAAC,WAAM,IAAI,GAAG,yBAAyB,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC,iCAA4B,IAAI,IAAI,UAAU,iBAAiB,CAAC;AAAA,IAC5K;AAAA,EACF;AACA,SAAO;AAAA,IACL,UAAU;AAAA,IACV,QAAQ,gBAAW,IAAI,IAAI,UAAU,iBAAiB,CAAC,YAAY,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC,8CAA8C,IAAI,GAAG;AAAA,EACtJ;AACF;AAEA,SAAS,oBACP,YACA,KAO8B;AAC9B,QAAM,gBAAgB,WAAW,OAAO,CAAC,MAAM,EAAE,gBAAgB,IAAI,UAAU;AAC/E,QAAM,cAAc,cAAc,KAAK,CAAC,MAAM,EAAE,aAAa,SAAS;AACtE,QAAM,iBAAiB,cAAc,KAAK,CAAC,MAAM,EAAE,aAAa,YAAY;AAC5E,QAAM,SAAS,eAAe,kBAAkB,cAAc,CAAC,KAAK;AACpE,QAAM,WAAmC,cACrC,YACA,cAAc,KAAK,CAAC,MAAM,EAAE,aAAa,iBAAiB,IACxD,oBACA,iBACE,eACA,cAAc,KAAK,CAAC,MAAM,EAAE,aAAa,MAAM,IAC7C,SACA;AAEV,QAAM,YAAsB,CAAC;AAC7B,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAwB,CAAC;AAE/B,MAAI,QAAQ;AACV,cAAU,KAAK,GAAG,OAAO,WAAW,KAAK,OAAO,cAAc,EAAE;AAChE,QAAI,OAAO,QAAQ;AACjB,YAAM,cACJ,OAAO,sBAAsB,OAAO,kBAAa,IAAI,OAAO,iBAAiB,CAAC,KAAK;AACrF,gBAAU;AAAA,QACR,2BAA2B,IAAI,OAAO,OAAO,GAAG,CAAC,KAAK,IAAI,OAAO,OAAO,IAAI,CAAC,IAAI,WAAW;AAAA,MAC9F;AAAA,IACF;AACA,QAAI,OAAO,QAAQ,QAAQ,OAAO,SAAS,OAAO,GAAG,GAAG;AACtD,gBAAU,KAAK,2BAA2B,OAAO,OAAO,KAAK,IAAI,OAAO,GAAG,CAAC,eAAe;AAAA,IAC7F;AAAA,EACF;AACA,MAAI,CAAC,IAAI,YAAY;AACnB,UAAM,KAAK,mEAAmE;AAC9E,gBAAY,KAAK,iEAAiE;AAAA,EACpF;AACA,MAAI,CAAC,IAAI,qBAAqB;AAC5B,UAAM;AAAA,MACJ;AAAA,IACF;AACA,gBAAY;AAAA,MACV;AAAA,IACF;AAAA,EACF;AACA,MAAI,IAAI,SAAS,QAAQ,cAAc,SAAS,GAAG;AACjD,UAAM;AAAA,MACJ;AAAA,IACF;AACA,gBAAY;AAAA,MACV;AAAA,IACF;AAAA,EACF;AACA,QAAM,eAAe,cAAc,OAAO,CAAC,MAAM,EAAE,aAAa,iBAAiB;AACjF,MAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,QAAQ,aAAa,OAAO,CAAC,GAAG,MAAO,EAAE,UAAU,EAAE,UAAU,IAAI,CAAE;AAC3E,UAAM;AAAA,MACJ,GAAG,aAAa,MAAM,mCAAmC,IAAI,QAAQ,wBAAwB,MAAM,WAAW,SAAS,MAAM,OAAO;AAAA,IACtI;AACA,gBAAY;AAAA,MACV,oBAAoB,IAAI,WAAW,MAAM,OAAO,kCAAkC,MAAM,WAAW;AAAA,IACrG;AAAA,EACF;AACA,QAAM,WAAW,cAAc,OAAO,CAAC,MAAM,EAAE,aAAa,QAAQ;AACpE,MAAI,SAAS,SAAS,GAAG;AACvB,UAAM;AAAA,MACJ,GAAG,SAAS,MAAM;AAAA,IACpB;AAAA,EACF;AACA,MAAI,IAAI,mBAAmB,IAAI,gBAAgB,SAAS,SAAS,GAAG;AAClE,UAAM,MAAM,IAAI,gBAAgB,SAAS,CAAC;AAC1C,UAAM,KAAK,wBAAwB,IAAI,YAAY,WAAW,IAAI,QAAQ,UAAU;AACpF,gBAAY,KAAK,8DAA8D;AAAA,EACjF;AACA,MAAI,aAAa,WAAW;AAC1B,gBAAY,KAAK,+DAA+D;AAAA,EAClF,WAAW,aAAa,QAAQ;AAC9B,gBAAY,KAAK,qEAAqE;AAAA,EACxF,WAAW,aAAa,cAAc;AACpC,gBAAY;AAAA,MACV;AAAA,IACF;AAAA,EACF,WAAW,aAAa,UAAU;AAChC,gBAAY;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,aAAa,QAAQ,eAAe;AAAA,IACpC;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,sBACP,YACA,gBACA,KAMU;AACV,QAAM,QAAkB,CAAC;AACzB,QAAM,gBAAgB,WAAW,OAAO,CAAC,MAAM,EAAE,gBAAgB,IAAI,UAAU;AAC/E,QAAM;AAAA,IACJ,aAAa,cAAc,MAAM,wBAAwB,IAAI,KAAK,SAAS,IAAI,aAAa,YAAY,IAAI,UAAU,KAAK,EAAE;AAAA,EAC/H;AACA,QAAM;AAAA,IACJ,mBAAmB,eAAe,QAAQ,GAAG,eAAe,cAAc,IAAI,eAAe,WAAW,KAAK,EAAE;AAAA,EACjH;AACA,QAAM,WAAW,cAAc,OAAO,CAAC,MAAM,EAAE,aAAa,SAAS,EAAE;AACvE,QAAM,OAAO,cAAc,OAAO,CAAC,MAAM,EAAE,aAAa,MAAM,EAAE;AAChE,QAAM,aAAa,cAAc,OAAO,CAAC,MAAM,EAAE,aAAa,YAAY,EAAE;AAC5E,QAAM,WAAW,cAAc,OAAO,CAAC,MAAM,EAAE,aAAa,QAAQ,EAAE;AACtE,QAAM,OAAO,cAAc,OAAO,CAAC,MAAM,EAAE,aAAa,iBAAiB,EAAE;AAC3E,QAAM;AAAA,IACJ,iBAAiB,QAAQ,aAAa,UAAU,gBAAgB,IAAI,UAAU,QAAQ,YAAY,IAAI;AAAA,EACxG;AACA,QAAM,WAAW,cAAc,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW;AACzF,MAAI,SAAS,SAAS,EAAG,OAAM,KAAK,+BAA+B,SAAS,KAAK,IAAI,CAAC,GAAG;AACzF,MAAI,IAAI,iBAAiB;AACvB,UAAM;AAAA,MACJ,4BAA4B,IAAI,gBAAgB,aAAa,IAAI,IAAI,gBAAgB,SAAS,uBAAuB,IAAI,gBAAgB,SAAS,MAAM;AAAA,IAC1J;AAAA,EACF;AACA,QAAM;AAAA,IACJ,IAAI,sBACA,2BAA2B,IAAI,oBAAoB,MAAM,GAAG,EAAE,CAAC,WAC/D;AAAA,EACN;AACA,SAAO;AACT;AAEA,SAAS,uBAAuB,QAgBrB;AACT,QAAM,QAAkB,CAAC;AACzB,QAAM,KAAK,KAAK,OAAO,KAAK,EAAE;AAC9B,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,kBAAkB,OAAO,WAAW,EAAE;AACjD,QAAM,KAAK,sBAAsB,OAAO,KAAK,EAAE;AAC/C,QAAM,KAAK,mBAAmB,OAAO,cAAc,gBAAgB,EAAE;AACrE,QAAM,KAAK,aAAa,OAAO,OAAO,WAAW,OAAO,IAAI,IAAI,gBAAgB,EAAE;AAClF,QAAM,KAAK,0BAA0B,OAAO,cAAc,IAAI;AAC9D,QAAM;AAAA,IACJ,wBAAwB,OAAO,sBAAsB,KAAK,OAAO,mBAAmB,OAAO,MAAM;AAAA,EACnG;AACA,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,sBAAsB;AACjC,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,iBAAkB,OAAM,KAAK,KAAK,IAAI,EAAE;AAClE,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,mBAAmB;AAC9B,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,iBAAiB,OAAO,eAAe,QAAQ,EAAE;AAC5D,QAAM,KAAK,kBAAkB,OAAO,eAAe,eAAe,KAAK,EAAE;AACzE,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,eAAe;AAC1B,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,eAAe,UAAW,OAAM,KAAK,KAAK,IAAI,EAAE;AAC1E,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,WAAW;AACtB,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,eAAe,MAAM,SAC3C,OAAO,eAAe,QACtB,CAAC,0CAA0C,GAAG;AAChD,UAAM,KAAK,KAAK,IAAI,EAAE;AAAA,EACxB;AACA,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,kBAAkB;AAC7B,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,eAAe,YAAa,OAAM,KAAK,KAAK,IAAI,EAAE;AAC5E,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,6BAA6B;AACxC,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ;AAAA,EACF;AACA,QAAM,KAAK,0DAA0D;AACrE,aAAW,KAAK,OAAO,YAAY;AACjC,UAAM,QAAQ,EAAE,0BAA0B,OAAO,MAAM,OAAO,EAAE,qBAAqB;AACrF,UAAM,OAAO,EAAE,sBAAsB,OAAO,MAAM,EAAE,kBAAkB,QAAQ,CAAC;AAC/E,UAAM,IAAI,OAAO,SAAS,EAAE,MAAM,IAAI,EAAE,OAAO,QAAQ,CAAC,IAAI;AAC5D,UAAM,IAAI,OAAO,SAAS,EAAE,OAAO,IAAI,EAAE,QAAQ,QAAQ,CAAC,IAAI;AAC9D,UAAM,OAAO,EAAE,SAAS,IAAI,IAAI,EAAE,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,OAAO,IAAI,CAAC,MAAM;AAC1E,UAAM,MAAM,EAAE,QAAQ,QAAQ,CAAC,OAAO,SAAS,EAAE,GAAG,IAAI,MAAM,IAAI,EAAE,GAAG;AACvE,UAAM;AAAA,MACJ,KAAK,EAAE,WAAW,MAAM,EAAE,QAAQ,MAAM,IAAI,EAAE,IAAI,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,MAAM,IAAI,MAAM,GAAG,MAAM,EAAE,mBAAmB,QAAQ,IAAI,MAAM,EAAE,QAAQ,GAAG;AAAA,IACpL;AAAA,EACF;AACA,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,wBAAwB;AACnC,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,OAAO,QAAQ,QAAQ;AAClC,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,gBAAgB;AAC3B,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,iBAAiB;AAC5B,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,YAAY,YAAa,OAAM,KAAK,KAAK,IAAI,EAAE;AACzE,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,aAAa;AACxB,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,YAAY,QAAS,OAAM,KAAK,KAAK,IAAI,EAAE;AACrE,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,6BAA6B;AACxC,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,YAAY,aAAc,OAAM,KAAK,KAAK,IAAI,EAAE;AAC1E,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,uBAAuB;AAClC,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,YAAY,eAAgB,OAAM,KAAK,KAAK,IAAI,EAAE;AAC5E,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,eAAe;AAC1B,QAAM,KAAK,EAAE;AACb,aAAW,QAAQ,OAAO,YAAY,UAAW,OAAM,KAAK,KAAK,IAAI,EAAE;AACvE,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,gBAAgB;AAC3B,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ;AAAA,EACF;AACA,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,SAAS;AACpB,QAAM,KAAK,KAAK,UAAU,EAAE,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,GAAG,MAAM,CAAC,CAAC;AAClF,QAAM,KAAK,KAAK;AAChB,MAAI,OAAO,iBAAiB;AAC1B,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,qBAAqB;AAChC,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,uDAAuD;AAClE,UAAM,KAAK,yBAAyB;AACpC,eAAW,KAAK,OAAO,gBAAgB,SAAS,MAAM,GAAG,EAAE,GAAG;AAC5D,YAAM;AAAA,QACJ,KAAK,EAAE,YAAY,MAAM,EAAE,QAAQ,MAAM,EAAE,YAAY,MAAM,MAAM,EAAE,YAAY,GAAG,MAAM,YAAY,EAAE,gBAAgB,EAAE,YAAY,CAAC;AAAA,MACzI;AAAA,IACF;AAAA,EACF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,mBAAmB,UAAkB,OAAuB;AACnE,QAAM,OAAO,eAAe,QAAQ;AACpC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,WAAW,KAAK,CAAC;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,eAAe,UAA0B;AAChD,QAAM,QAAQ,SAAS,MAAM,IAAI;AACjC,QAAM,OAAiB,CAAC;AACxB,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,OAAiB,CAAC;AACtB,MAAI,QAAkB,CAAC;AAEvB,QAAM,YAAY,MAAM;AACtB,QAAI,QAAQ;AACV,WAAK,KAAK,OAAO;AACjB,eAAS;AAAA,IACX;AAAA,EACF;AACA,QAAM,aAAa,MAAM;AACvB,QAAI,MAAM,WAAW,EAAG;AACxB,SAAK,KAAK,oBAAoB,KAAK,CAAC;AACpC,YAAQ,CAAC;AAAA,EACX;AAEA,aAAW,QAAQ,OAAO;AACxB,QAAI,KAAK,WAAW,KAAK,GAAG;AAC1B,UAAI,QAAQ;AACV,aAAK,KAAK,cAAc,WAAW,KAAK,KAAK,IAAI,CAAC,CAAC,eAAe;AAClE,eAAO,CAAC;AACR,iBAAS;AAAA,MACX,OAAO;AACL,kBAAU;AACV,mBAAW;AACX,iBAAS;AAAA,MACX;AACA;AAAA,IACF;AACA,QAAI,QAAQ;AACV,WAAK,KAAK,IAAI;AACd;AAAA,IACF;AACA,QAAI,KAAK,WAAW,GAAG,GAAG;AACxB,gBAAU;AACV,YAAM,KAAK,IAAI;AACf;AAAA,IACF;AACA,eAAW;AACX,QAAI,KAAK,WAAW,IAAI,GAAG;AACzB,UAAI,CAAC,QAAQ;AACX,aAAK,KAAK,MAAM;AAChB,iBAAS;AAAA,MACX;AACA,WAAK,KAAK,OAAO,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO;AACrD;AAAA,IACF;AACA,cAAU;AACV,QAAI,KAAK,WAAW,IAAI,EAAG,MAAK,KAAK,OAAO,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO;AAAA,aACvE,KAAK,WAAW,KAAK,EAAG,MAAK,KAAK,OAAO,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO;AAAA,aAC7E,KAAK,WAAW,MAAM,EAAG,MAAK,KAAK,OAAO,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO;AAAA,aAC9E,KAAK,KAAK,MAAM,GAAI,MAAK,KAAK,EAAE;AAAA,QACpC,MAAK,KAAK,MAAM,eAAe,IAAI,CAAC,MAAM;AAAA,EACjD;AACA,YAAU;AACV,aAAW;AACX,SAAO,KAAK,KAAK,IAAI;AACvB;AAEA,SAAS,oBAAoB,OAAyB;AACpD,QAAM,OAAO,MACV,OAAO,CAAC,SAAS,CAAC,iBAAiB,KAAK,IAAI,CAAC,EAC7C;AAAA,IAAI,CAAC,SACJ,KACG,MAAM,GAAG,EAAE,EACX,MAAM,GAAG,EACT,IAAI,CAAC,SAAS,eAAe,KAAK,KAAK,CAAC,CAAC;AAAA,EAC9C;AACF,MAAI,KAAK,WAAW,EAAG,QAAO;AAC9B,QAAM,CAAC,MAAM,GAAG,IAAI,IAAI;AACxB,QAAM,KAAK,KAAM,IAAI,CAAC,SAAS,OAAO,IAAI,OAAO,EAAE,KAAK,EAAE;AAC1D,QAAM,MAAM,KACT,IAAI,CAAC,QAAQ,OAAO,IAAI,IAAI,CAAC,SAAS,OAAO,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC,OAAO,EACzE,KAAK,IAAI;AACZ,SAAO,qBAAqB,EAAE,uBAAuB,GAAG;AAC1D;AAEA,SAAS,eAAe,GAAmB;AACzC,SAAO,WAAW,CAAC,EAAE,QAAQ,oBAAoB,qBAAqB;AACxE;AAEA,SAAS,WAAW,GAAmB;AACrC,SAAO,EACJ,QAAQ,MAAM,OAAO,EACrB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,QAAQ;AAC3B;AAEA,SAAS,YAAY,GAAmB;AACtC,SAAO,EAAE,QAAQ,OAAO,KAAK;AAC/B;AAEA,SAAS,eAAe,UAA0C;AAChE,MAAI,aAAa,UAAW,QAAO;AACnC,MAAI,aAAa,aAAc,QAAO;AACtC,MAAI,aAAa,OAAQ,QAAO;AAChC,MAAI,aAAa,kBAAmB,QAAO;AAC3C,SAAO;AACT;AAEA,SAAS,OAAO,GAAmB;AACjC,SAAO,GAAG,KAAK,IAAI,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;AACtC;AAIA,SAAS,IAAI,IAAsB;AACjC,MAAI,GAAG,WAAW,EAAG,QAAO,OAAO;AACnC,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAEA,SAAS,eAAe,QAA0B;AAChD,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,QAAM,MAAM,KAAK,MAAM,OAAO,SAAS,CAAC;AACxC,SAAO,OAAO,SAAS,MAAM,KAAK,OAAO,MAAM,CAAC,IAAK,OAAO,GAAG,KAAM,IAAI,OAAO,GAAG;AACrF;AAEA,SAAS,IAAI,GAAmB;AAC9B,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO,OAAO,CAAC;AACxC,SAAO,EAAE,QAAQ,CAAC;AACpB;","names":[]}
|
package/dist/chunk-MAZ26DC7.js
DELETED
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
// src/sequential.ts
|
|
2
|
-
function pairedEvalueSequence(deltas, opts = {}) {
|
|
3
|
-
const c = opts.bound ?? 1;
|
|
4
|
-
const alpha = opts.alpha ?? 0.05;
|
|
5
|
-
const initialShrink = opts.initialBetShrinkage ?? 0.5;
|
|
6
|
-
const rope = opts.rope ?? null;
|
|
7
|
-
if (c <= 0) throw new Error("pairedEvalueSequence: bound must be > 0");
|
|
8
|
-
if (alpha <= 0 || alpha >= 1) throw new Error("pairedEvalueSequence: alpha must be in (0,1)");
|
|
9
|
-
if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) {
|
|
10
|
-
throw new Error("pairedEvalueSequence: rope must satisfy low \u2264 high");
|
|
11
|
-
}
|
|
12
|
-
const steps = [];
|
|
13
|
-
let clipped = false;
|
|
14
|
-
let evalue = 1;
|
|
15
|
-
let decisionFiredAt = null;
|
|
16
|
-
let sum = 0;
|
|
17
|
-
let sumSq = 0;
|
|
18
|
-
let count = 0;
|
|
19
|
-
for (let i = 0; i < deltas.length; i++) {
|
|
20
|
-
let d = deltas[i];
|
|
21
|
-
if (d < -c || d > c) {
|
|
22
|
-
d = Math.max(-c, Math.min(c, d));
|
|
23
|
-
clipped = true;
|
|
24
|
-
}
|
|
25
|
-
const muHat = count === 0 ? 0 : sum / count;
|
|
26
|
-
const varHat = count === 0 ? c * c : Math.max(1e-12, sumSq / count - muHat * muHat);
|
|
27
|
-
const t = i + 1;
|
|
28
|
-
const shrink = initialShrink * Math.min(1, count / 32);
|
|
29
|
-
let lambda = muHat / (varHat + c * c) * shrink;
|
|
30
|
-
const lambdaMax = 0.99 / c;
|
|
31
|
-
if (lambda > lambdaMax) lambda = lambdaMax;
|
|
32
|
-
if (lambda < -lambdaMax) lambda = -lambdaMax;
|
|
33
|
-
evalue = evalue * (1 + lambda * d);
|
|
34
|
-
if (!Number.isFinite(evalue) || evalue < 0) evalue = 0;
|
|
35
|
-
sum += d;
|
|
36
|
-
sumSq += d * d;
|
|
37
|
-
count += 1;
|
|
38
|
-
const pValue = Math.min(1, 1 / Math.max(evalue, 1e-300));
|
|
39
|
-
const cs = empiricalBernsteinCs(sum, sumSq, count, c, alpha);
|
|
40
|
-
let decision = "continue";
|
|
41
|
-
if (rope && cs.low >= rope.low && cs.high <= rope.high) decision = "equivalent";
|
|
42
|
-
else if (evalue >= 2 / alpha && muHat > 0) decision = "promote_now";
|
|
43
|
-
else if (evalue >= 2 / alpha && muHat < 0) decision = "reject_now";
|
|
44
|
-
else if (rope && cs.high < rope.low) decision = "reject_now";
|
|
45
|
-
if (decision !== "continue" && decisionFiredAt === null) decisionFiredAt = t;
|
|
46
|
-
steps.push({ t, delta: d, evalue, pValue, csLow: cs.low, csHigh: cs.high, decision });
|
|
47
|
-
}
|
|
48
|
-
const finalDecision = steps.length === 0 ? "continue" : steps[steps.length - 1].decision;
|
|
49
|
-
return { steps, finalDecision, decisionFiredAt, clipped };
|
|
50
|
-
}
|
|
51
|
-
function evaluateInterimReleaseConfidence(input) {
|
|
52
|
-
const candidates = input.deltaSeries.map((s) => {
|
|
53
|
-
const seq = pairedEvalueSequence(s.deltas, {
|
|
54
|
-
alpha: input.alpha,
|
|
55
|
-
bound: input.bound,
|
|
56
|
-
rope: input.rope
|
|
57
|
-
});
|
|
58
|
-
const last = seq.steps[seq.steps.length - 1];
|
|
59
|
-
return {
|
|
60
|
-
candidateId: s.candidateId,
|
|
61
|
-
decision: seq.finalDecision,
|
|
62
|
-
decisionFiredAt: seq.decisionFiredAt,
|
|
63
|
-
finalEvalue: last?.evalue ?? 1,
|
|
64
|
-
finalPValue: last?.pValue ?? 1,
|
|
65
|
-
pairs: seq.steps.length,
|
|
66
|
-
csLow: last?.csLow ?? Number.NEGATIVE_INFINITY,
|
|
67
|
-
csHigh: last?.csHigh ?? Number.POSITIVE_INFINITY
|
|
68
|
-
};
|
|
69
|
-
});
|
|
70
|
-
const promote = candidates.find((c) => c.decision === "promote_now");
|
|
71
|
-
if (promote)
|
|
72
|
-
return {
|
|
73
|
-
candidates,
|
|
74
|
-
recommendation: { decision: "promote_now", candidateId: promote.candidateId }
|
|
75
|
-
};
|
|
76
|
-
const live = candidates.find((c) => c.decision === "continue");
|
|
77
|
-
if (live) return { candidates, recommendation: { decision: "continue", candidateId: null } };
|
|
78
|
-
const equiv = candidates.find((c) => c.decision === "equivalent");
|
|
79
|
-
if (equiv)
|
|
80
|
-
return {
|
|
81
|
-
candidates,
|
|
82
|
-
recommendation: { decision: "equivalent", candidateId: equiv.candidateId }
|
|
83
|
-
};
|
|
84
|
-
return { candidates, recommendation: { decision: "reject_now", candidateId: null } };
|
|
85
|
-
}
|
|
86
|
-
function empiricalBernsteinCs(sum, sumSq, n, bound, alpha) {
|
|
87
|
-
if (n === 0) return { low: -bound, high: bound };
|
|
88
|
-
const mean = sum / n;
|
|
89
|
-
const variance = Math.max(0, sumSq / n - mean * mean);
|
|
90
|
-
const psi = Math.log(2 / alpha) + 1.7 * Math.log(Math.log(Math.max(Math.E, n)) + 1);
|
|
91
|
-
const radius = Math.sqrt(2 * variance * psi / n) + 3 * bound * psi / n;
|
|
92
|
-
return { low: mean - radius, high: mean + radius };
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
export {
|
|
96
|
-
pairedEvalueSequence,
|
|
97
|
-
evaluateInterimReleaseConfidence
|
|
98
|
-
};
|
|
99
|
-
//# sourceMappingURL=chunk-MAZ26DC7.js.map
|