@rulemetric/cli 0.6.9 → 0.7.1

package/dist/commands/evals/agent.js

@@ -1,29 +1,29 @@
 import {
   runAgentLoop
-} from "../../chunk-BBLTN35F.js";
-import "../../chunk-JEJ3J5J6.js";
-import "../../chunk-RCMXTLQF.js";
-import "../../chunk-N5A2IQAK.js";
-import "../../chunk-OTOWLUOJ.js";
-import "../../chunk-5ADHO5QF.js";
-import "../../chunk-Y4BJXXYV.js";
+} from "../../chunk-7DULPSFU.js";
+import "../../chunk-GIUBARWS.js";
+import "../../chunk-6NBS5XFS.js";
 import "../../chunk-W53GKIZQ.js";
-import "../../chunk-3QVMIJUL.js";
-import "../../chunk-XZXS2W24.js";
+import "../../chunk-YOBA3NAM.js";
 import "../../chunk-RQ2TMLKG.js";
-import "../../chunk-VBGUNFKY.js";
-import "../../chunk-FZKLLNDS.js";
-import "../../chunk-AHDRN6NE.js";
-import "../../chunk-DGHWRQXL.js";
+import "../../chunk-PMI56ED5.js";
+import "../../chunk-OO7JDFS4.js";
 import "../../chunk-E3BIT53W.js";
 import "../../chunk-QSN77T7C.js";
 import {
   closeDb
-} from "../../chunk-6EMZI4Y3.js";
-import "../../chunk-3POYC5NA.js";
-import "../../chunk-DR5WLIOU.js";
+} from "../../chunk-EI5BHWXA.js";
+import "../../chunk-XK42ZBDE.js";
+import "../../chunk-RMTT4KDY.js";
+import "../../chunk-OK3I555A.js";
+import "../../chunk-XZXS2W24.js";
+import "../../chunk-FZKLLNDS.js";
+import "../../chunk-JULOLTZS.js";
+import "../../chunk-Y4BJXXYV.js";
+import "../../chunk-DGHWRQXL.js";
 import "../../chunk-J7N3DLH6.js";
 import "../../chunk-52WIYRZH.js";
+import "../../chunk-KJKBDQ4D.js";
 import "../../chunk-OQSQC7VB.js";
 import "../../chunk-EKP32DLN.js";
 import "../../chunk-F5N6CLJJ.js";
@@ -33,11 +33,12 @@ import "../../chunk-YNIH3KMU.js";
 import "../../chunk-QN34TGD3.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
-import "../../chunk-ZRLYHBPN.js";
+import "../../chunk-ZUJBM6HE.js";
 import {
   getDefaultConfigDir,
   loadToken
@@ -270,12 +271,32 @@ var EvalAgentCommand = class _EvalAgentCommand extends BaseCommand {
     process.on("SIGTERM", () => {
       void shutdown("SIGTERM");
     });
+    let restartForUpgrade = false;
     try {
       await runAgentLoop({
         signal: controller.signal,
         log: (msg) => this.log(msg),
-        version: this.config.version
+        version: this.config.version,
+        readInstalledVersion: () => {
+          try {
+            const pkg = JSON.parse(
+              readFileSync(join(this.config.root, "package.json"), "utf-8")
+            );
+            return pkg.version ?? null;
+          } catch {
+            return null;
+          }
+        },
+        onVersionDrift: () => {
+          restartForUpgrade = true;
+        }
       });
+      if (restartForUpgrade) {
+        this.log("[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code.");
+        await Promise.allSettled([flushSentry(2e3), shutdownTelemetry(), closeDb()]);
+        releaseAgentLock();
+        process.exit(1);
+      }
     } finally {
       releaseAgentLock();
       await closeDb();

package/dist/commands/evals/agent.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../../../packages/telemetry/src/init.ts", "../../../../../packages/telemetry/src/otlp-headers.ts", "../../../../../packages/telemetry/src/hono-middleware.ts", "../../../../../packages/telemetry/src/span-attributes.ts", "../../../../../packages/telemetry/src/sentry.ts", "../../../src/commands/evals/agent.ts"],
-  "sourcesContent": ["import { trace, context, type Tracer } from '@opentelemetry/api';\nimport { NodeSDK } from '@opentelemetry/sdk-node';\nimport { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';\nimport {\n  BatchSpanProcessor,\n  ConsoleSpanExporter,\n  SimpleSpanProcessor,\n} from '@opentelemetry/sdk-trace-node';\nimport { Resource } from '@opentelemetry/resources';\nimport { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';\nimport { parseOtlpHeaders } from './otlp-headers.js';\n\nlet sdk: NodeSDK | null = null;\n\nexport interface TelemetryConfig {\n  serviceName: string;\n  serviceVersion?: string;\n}\n\n/**\n * Boot states for `initTelemetry`. Returned discriminator lets callers\n * emit a precise boot log:\n *   - 'otlp'    \u2014 spans ship over OTLP to a real backend (Honeycomb etc.)\n *   - 'console' \u2014 spans print to stdout as JSON (Fly logs ingest them);\n *                 the \"open + zero third-party\" default\n *   - 'off'     \u2014 telemetry globally disabled (TELEMETRY_ENABLED=false)\n *                 OR a prior `initTelemetry()` already booted the SDK\n *                 in this process (idempotent skip)\n */\nexport type TelemetryBootState = 'otlp' | 'console' | 'silent' | 'off';\n\nexport function isTelemetryEnabled(): boolean {\n  return process.env.TELEMETRY_ENABLED !== 'false';\n}\n\n/**\n * Initialize OpenTelemetry tracing. Two transport modes, both built on\n * the same OTel SDK \u2014 only the exporter changes:\n *\n *   - OTLP backend  \u2192 set `OTEL_EXPORTER_OTLP_ENDPOINT` (+ optional\n *                     `OTEL_EXPORTER_OTLP_HEADERS` for vendor auth)\n *   - Console (stdout) \u2192 the default when no endpoint is set\n *\n * The console path is intentionally on-by-default. Without a real backend\n * configured, spans are written to stdout as JSON and end up in Fly's log\n * stream alongside everything else \u2014 searchable, structured, and you\n * stay inside the open OpenTelemetry spec without paying a third party.\n * Swap to OTLP later by setting one env var; no code change needed.\n *\n * Returns `'off'` when:\n *   - TELEMETRY_ENABLED=false (operator kill switch \u2014 beats any config)\n *   - The SDK was already started in this process (idempotent skip)\n *\n * See packages/telemetry/src/init.test.ts for the full state table.\n */\nexport function initTelemetry(config: TelemetryConfig): TelemetryBootState {\n  if (!isTelemetryEnabled()) {\n    return 'off';\n  }\n  if (sdk !== null) {\n    // OTel's global tracer provider gets set on first start(); calling\n    // start() twice triggers an SDK warning. Treat second-call as a no-op.\n    return 'off';\n  }\n\n  const resource = new Resource({\n    [ATTR_SERVICE_NAME]: config.serviceName,\n    [ATTR_SERVICE_VERSION]: config.serviceVersion ?? '0.1.0',\n  });\n\n  const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;\n  let mode: TelemetryBootState;\n  let spanProcessor: BatchSpanProcessor | SimpleSpanProcessor | undefined;\n\n  if (endpoint) {\n    // OTLP path. Headers via OTEL_EXPORTER_OTLP_HEADERS (W3C convention,\n    // comma-separated key=value pairs). Parsed explicitly rather than\n    // relying on SDK-version-specific auto-discovery \u2014 predictable boot\n    // matters more than tracking whatever the current default is.\n    //   Honeycomb:     x-honeycomb-team=hcaik_xxx (plus optional dataset)\n    //   Grafana Cloud: authorization=Basic <base64creds>\n    const headers = parseOtlpHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS);\n    const exporter = new OTLPTraceExporter({\n      url: `${endpoint}/v1/traces`,\n      headers,\n    });\n    spanProcessor = new BatchSpanProcessor(exporter);\n    mode = 'otlp';\n  } else if (process.env.OTEL_SPAN_EXPORT === 'none') {\n    // Silent path: trace context still propagates (so trace IDs are available\n    // for correlation and an OTLP endpoint can be added later with zero code\n    // change), but NO span processor is registered \u2014 nothing is written to\n    // stdout. This exists because the ConsoleSpanExporter below writes one\n    // JSON span per ended span, and on the local launchd API (no OTLP\n    // endpoint, fixed log file, no Fly-managed rotation) that grew api-stdout\n    // to 370MB / ~400k spans. Set OTEL_SPAN_EXPORT=none on that process; Fly\n    // leaves it unset so console spans stay grep-able in its managed log stream.\n    spanProcessor = undefined;\n    mode = 'silent';\n  } else {\n    // Console (stdout) path. SimpleSpanProcessor \u2014 not batched \u2014 so spans\n    // appear in chronological order in the log stream. Acceptable cost for\n    // tracing: a span is written per ended span, vs. a batched POST for OTLP.\n    // Logs that would otherwise need a UI to make sense (trace tree\n    // visualization) are still queryable via `flyctl logs | grep traceId=...`\n    // or any log-aggregator that ingests Fly's stream.\n    const exporter = new ConsoleSpanExporter();\n    spanProcessor = new SimpleSpanProcessor(exporter);\n    mode = 'console';\n  }\n\n  sdk = new NodeSDK({\n    resource,\n    spanProcessors: spanProcessor ? [spanProcessor] : [],\n  });\n\n  sdk.start();\n  return mode;\n}\n\nexport async function shutdownTelemetry(): Promise<void> {\n  if (sdk) {\n    await sdk.shutdown();\n    sdk = null;\n  }\n}\n\nexport function getTracer(name?: string): Tracer {\n  return trace.getTracer(name ?? 'rulemetric');\n}\n\nexport function getTraceId(): string | undefined {\n  const span = trace.getSpan(context.active());\n  return span?.spanContext().traceId;\n}\n", "// Parse the `OTEL_EXPORTER_OTLP_HEADERS` env var into a plain headers object\n// that the OTLP exporter constructor accepts.\n//\n// OpenTelemetry SDK env-vars spec (\u00A710) defines this as a comma-separated list\n// of `key=value` pairs. We parse it ourselves rather than relying on the\n// `@opentelemetry/exporter-trace-otlp-http` package's auto-discovery, because\n// that auto-discovery has shifted between SDK versions and we want predictable\n// behaviour at boot.\n//\n// Real-world usage:\n//   - Honeycomb:   `x-honeycomb-team=hcaik_xxx` (and optionally `,x-honeycomb-dataset=...`)\n//   - Grafana Cloud: `authorization=Basic <base64>` (note: base64 padding can be `==`)\n//\n// Edge cases (covered by `otlp-headers.test.ts`):\n//   - undefined or empty \u2192 {} (caller will get a no-headers exporter)\n//   - whitespace around `=` and `,` is trimmed (operator-friendly)\n//   - only the first `=` is the separator, so base64 padding (==) survives in\n//     values\n//   - malformed entries (no `=`) are skipped silently\n//   - empty keys are dropped (never inject a \"\" header)\n//   - duplicate keys: last value wins, matching URL search-params semantics\n\nexport function parseOtlpHeaders(raw: string | undefined): Record<string, string> {\n  if (!raw) return {};\n  const out: Record<string, string> = {};\n  for (const entry of raw.split(',')) {\n    const eq = entry.indexOf('=');\n    if (eq === -1) continue;\n    const key = entry.slice(0, eq).trim();\n    const value = entry.slice(eq + 1).trim();\n    if (!key) continue;\n    out[key] = value;\n  }\n  return out;\n}\n", "import { SpanKind, SpanStatusCode } from '@opentelemetry/api';\nimport type { MiddlewareHandler } from 'hono';\nimport { getTracer, isTelemetryEnabled } from './init.js';\n\nexport function telemetryMiddleware(): MiddlewareHandler {\n  return async (c, next) => {\n    if (!isTelemetryEnabled()) {\n      await next();\n      return;\n    }\n\n    const tracer = getTracer();\n    const method = c.req.method;\n    const path = c.req.path;\n\n    await tracer.startActiveSpan(\n      `${method} ${path}`,\n      {\n        kind: SpanKind.SERVER,\n        attributes: {\n          'http.method': method,\n          'http.url': c.req.url,\n          'http.route': path,\n        },\n      },\n      async (span) => {\n        try {\n          await next();\n\n          span.setAttribute('http.status_code', c.res.status);\n\n          const userId = c.get('userId' as never);\n          if (userId) {\n            span.setAttribute('rulemetric.user_id', userId as string);\n          }\n\n          if (c.res.status >= 400) {\n            span.setStatus({ code: SpanStatusCode.ERROR });\n          }\n        } catch (err) {\n          span.setStatus({ code: SpanStatusCode.ERROR });\n          span.recordException(err as Error);\n          throw err;\n        } finally {\n          span.end();\n        }\n      },\n    );\n  };\n}\n", "import { trace, context as otelContext } from '@opentelemetry/api';\n\n/**\n * Set attributes on the currently-active span, if one exists. Safe no-op\n * when no span is active (e.g. running outside of a Hono request handler,\n * or with telemetry disabled).\n *\n * Use this from business-logic code that wants to expose a discriminator\n * for telemetry queries \u2014 e.g. \"which branch did `resolveCallerOrgId` take\"\n * in `apps/api/src/routes/sessions.ts`. Span attributes are searchable in\n * any OpenTelemetry backend (Honeycomb, Grafana Cloud, ...) and in\n * `flyctl logs` when the console exporter is active.\n *\n * Keeping this thin wrapper in the telemetry package means application\n * code never imports `@opentelemetry/api` directly \u2014 the package boundary\n * stays the only place that knows about the OTel SDK shape.\n */\nexport function setSpanAttributes(\n  attributes: Record<string, string | number | boolean>,\n): void {\n  const span = trace.getSpan(otelContext.active());\n  if (!span) return;\n  for (const [key, value] of Object.entries(attributes)) {\n    span.setAttribute(key, value);\n  }\n}\n", "// Thin wrapper around @sentry/node so callers don't have to guard every\n// captureException with `if (process.env.SENTRY_DSN)`. The wrapper handles\n// three concerns:\n//\n//   1. Init only when SENTRY_DSN is set. Otherwise every entry point becomes\n//      a silent no-op \u2014 never throws, never logs noise, never opens a\n//      network transport.\n//   2. Idempotent init. Double-init is a Sentry SDK foot-gun (the second\n//      call wins, but the warning lands at runtime). The module-level\n//      `enabled` flag short-circuits the second call.\n//   3. tracesSampleRate = 0 by default. We already have OpenTelemetry\n//      shipping spans to a tracing backend; paying Sentry's quota for the\n//      same data isn't useful. Sentry's job here is errors-only.\n//\n// Wiring:\n//   - `apps/api/src/index.ts`           \u2192 initSentry({ serviceName: 'rulemetric-api' })\n//   - `apps/api/src/middleware/auth.ts` \u2192 setSentryUser(userId) after JWT verify\n//   - `apps/cli/src/commands/evals/agent.ts` \u2192 initSentry({ serviceName: 'rulemetric-agent' })\n//     (the thin-agent worker \u2014 Phase 4c moved the init here from the old\n//     graphile `listen.ts`; that path no longer inits Sentry)\n//\n// All four hooks are no-op-safe when DSN is unset, so the wiring doesn't\n// require feature-flag plumbing.\n\nimport * as Sentry from '@sentry/node';\n\nlet enabled = false;\n\nexport interface InitSentryConfig {\n  serviceName: string;\n}\n\n/**\n * Initialize Sentry if SENTRY_DSN is set. Presence of the DSN is now the\n * single switch: set it (incl. as a Fly secret in prod) to enable error\n * capture, leave it unset for a silent no-op.\n *\n * Previously gated to non-production by policy; that gate was dropped so\n * production errors actually get reported. Errors-only (`tracesSampleRate:\n * 0`) keeps Sentry quota bounded \u2014 OpenTelemetry still owns tracing.\n *\n * Returns true when init actually ran, false when it was skipped (DSN unset\n * or a prior call already initialized). Idempotent.\n */\nexport function initSentry(config: InitSentryConfig): boolean {\n  if (enabled) return false;\n  const dsn = process.env.SENTRY_DSN;\n  if (!dsn) return false;\n\n  Sentry.init({\n    dsn,\n    serverName: config.serviceName,\n    environment: process.env.NODE_ENV ?? 'development',\n    // We rely on OpenTelemetry for tracing \u2014 Sentry stays errors-only here.\n    // Override at the env-var level if you ever want Sentry performance too:\n    // `SENTRY_TRACES_SAMPLE_RATE=0.05` (Sentry SDK reads it natively).\n    tracesSampleRate: 0,\n  });\n\n  enabled = true;\n  return true;\n}\n\n/**\n * Report an exception. Coerces non-Error throws to Error so Sentry's\n * grouping/stack-trace logic has something to work with. Silent no-op when\n * SENTRY_DSN is unset.\n */\nexport function captureException(\n  err: unknown,\n  context?: { extra?: Record<string, unknown>; tags?: Record<string, string> },\n): void {\n  if (!enabled) return;\n  const e = err instanceof Error ? err : new Error(typeof err === 'string' ? err : JSON.stringify(err));\n  Sentry.captureException(e, context);\n}\n\n/**\n * Attach (or clear, with null) the active user on the Sentry scope. Called\n * from the API auth middleware so any captured exception during the request\n * carries `user.id`. Silent no-op when SENTRY_DSN is unset.\n */\nexport function setSentryUser(userId: string | null): void {\n  if (!enabled) return;\n  if (userId === null) {\n    Sentry.setUser(null);\n    return;\n  }\n  Sentry.setUser({ id: userId });\n}\n\n/**\n * Flush any queued events with a timeout. Critical to call before\n * `process.exit()` in worker shutdown \u2014 Sentry batches events and the SIGTERM\n * path was previously dropping the final exception (the one we'd most want\n * to see). Resolves immediately when SENTRY_DSN is unset.\n */\nexport async function flushSentry(timeoutMs: number): Promise<void> {\n  if (!enabled) return;\n  await Sentry.flush(timeoutMs);\n}\n\n/**\n * Reset the module-level state. Test-only \u2014 exported so test suites can\n * unwind the singleton between cases without `vi.resetModules` plumbing in\n * every block.\n */\nexport function _resetSentryForTests(): void {\n  enabled = false;\n}\n", "import { initSentry, flushSentry, initTelemetry, shutdownTelemetry } from '@rulemetric/telemetry';\nimport { BaseCommand } from '../../base-command.js';\nimport { loadToken, getDefaultConfigDir } from '../../lib/auth.js';\nimport { runAgentLoop } from '../../lib/agent-loop.js';\nimport { closeDb } from '@rulemetric/db';\nimport { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { execFileSync } from 'node:child_process';\n\n// Anchored under the config dir (honors RULEMETRIC_CONFIG_DIR; default\n// ~/.config/rulemetric, like gateway.pid), NOT os.tmpdir(), to avoid\n// launch-context TMPDIR divergence (launchd vs SSH/sudo/CI). Config-dir-scoped\n// so an isolated config (e.g. a cold-start test harness) can run its own agent\n// without colliding with the machine's real launchd-worker lock; two agents\n// sharing one config dir \u2014 the orphaned-fleet case this guard exists for \u2014 are\n// still refused.\nconst AGENT_LOCK_PATH = join(getDefaultConfigDir(), 'evals-agent.lock');\n\nfunction pidAlive(pid: number): boolean {\n  try {\n    process.kill(pid, 0);\n    return true;\n  } catch (e) {\n    // ESRCH = no such process (stale); EPERM = exists but owned by another user\n    return (e as { code?: string }).code === 'EPERM';\n  }\n}\n\n// Defends against PID reuse: a SIGKILL/OOM leaves the lock behind and the OS may\n// later assign that PID to an unrelated process \u2014 `pidAlive` alone would then\n// wedge `evals agent` (refuse to start forever). Confirm the holder is actually\n// an evals-agent before treating the lock as held; otherwise it's reclaimable.\nfunction isLiveAgent(pid: number): boolean {\n  if (!pidAlive(pid)) return false;\n  try {\n    const cmd = execFileSync('ps', ['-p', String(pid), '-o', 'command='], {\n      encoding: 'utf-8',\n      stdio: ['ignore', 'pipe', 'ignore'],\n    });\n    return /run\\.js\\s+evals\\s+agent|evals\\s+agent/.test(cmd);\n  } catch {\n    // ps unavailable/failed \u2014 fall back to liveness only (conservative: held)\n    return true;\n  }\n}\n\n// Single-instance guard. Each `evals agent` daemon holds its own postgres-js\n// pool; an orphaned, unsupervised fleet (4 stale agents found alive 3+ days on\n// 2026-06-21) was a confirmed driver of the recurring Supavisor 200-client\n// saturation. ATOMIC acquire via O_EXCL (flag 'wx') closes the TOCTOU window\n// where two simultaneously-starting agents could both acquire. On EEXIST the\n// holder is liveness-checked; a stale lock (holder gone) is reclaimed once.\n// See docs/incidents/2026-06-21-pool-exhaustion.md.\nfunction acquireAgentLock(): { ok: true } | { ok: false; holder: number } {\n  try {\n    mkdirSync(dirname(AGENT_LOCK_PATH), { recursive: true });\n  } catch {\n    /* ignore \u2014 writeFileSync will surface a real problem */\n  }\n  for (let attempt = 0; attempt < 2; attempt++) {\n    try {\n      // 'wx' = O_CREAT | O_EXCL: fails atomically with EEXIST if it exists.\n      writeFileSync(AGENT_LOCK_PATH, String(process.pid), { flag: 'wx' });\n      return { ok: true };\n    } catch (e) {\n      if ((e as { code?: string }).code !== 'EEXIST') throw e;\n      let holder = 0;\n      try {\n        holder = Number(readFileSync(AGENT_LOCK_PATH, 'utf-8').trim());\n      } catch {\n        /* unreadable \u2014 treat as stale */\n      }\n      if (holder && holder !== process.pid && isLiveAgent(holder)) {\n        return { ok: false, holder };\n      }\n      // stale (holder gone) / ours / unreadable \u2014 drop it and retry the atomic create once\n      try { unlinkSync(AGENT_LOCK_PATH); } catch { /* raced with another reclaimer */ }\n    }\n  }\n  // Last resort after a reclaim race \u2014 best-effort non-exclusive write.\n  try { writeFileSync(AGENT_LOCK_PATH, String(process.pid)); } catch { /* ignore */ }\n  return { ok: true };\n}\n\nfunction releaseAgentLock(): void {\n  try {\n    if (\n      existsSync(AGENT_LOCK_PATH) &&\n      Number(readFileSync(AGENT_LOCK_PATH, 'utf-8').trim()) === process.pid\n    ) {\n      unlinkSync(AGENT_LOCK_PATH);\n    }\n  } catch {\n    /* best-effort */\n  }\n}\n\n/**\n * `rulemetric evals agent` \u2014 Phase 4b parallel-run thin agent.\n *\n * Polls the /api/work endpoints (Phase 4a) instead of binding a Postgres\n * LISTEN/NOTIFY connection. This is the scaffold for the cutover described\n * in docs/plans/2026-05-25-thin-agent-migration.md; the default worker is\n * the sole worker mode since Phase 4c deleted the graphile path.\n *\n * Both commands can run side-by-side on the same user during the cutover \u2014\n * the API's atomic SKIP-LOCKED claim guarantees a job is handed to exactly\n * one agent. Picking which one runs is a launchd / shell choice.\n */\nexport default class EvalAgentCommand extends BaseCommand {\n  static override description =\n    'The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.';\n\n  static override examples = [\n    '<%= config.bin %> evals agent',\n  ];\n\n  async run(): Promise<void> {\n    await this.parse(EvalAgentCommand);\n    this.requireAuth();\n\n    // Single-instance guard \u2014 refuse to start a second agent. Each extra agent\n    // holds its own DB pool; an orphaned fleet was a confirmed cause of pooler\n    // saturation (pool-exhaustion RCA 2026-06-21).\n    const lock = acquireAgentLock();\n    if (!lock.ok) {\n      this.log(\n        `Another rulemetric evals agent is already running (pid ${lock.holder}). ` +\n          'Refusing to start a second instance.',\n      );\n      return;\n    }\n\n    process.env.PG_APPLICATION_NAME ||= 'rulemetric-agent';\n\n    const sentryEnabled = initSentry({ serviceName: 'rulemetric-agent' });\n    if (sentryEnabled) this.log('[sentry] error capture enabled');\n\n    const otelMode = initTelemetry({ serviceName: 'rulemetric-agent' });\n    if (otelMode === 'otlp') {\n      this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`);\n    } else if (otelMode === 'console') {\n      this.log('[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)');\n    } else {\n      this.log('[otel] disabled (TELEMETRY_ENABLED=false)');\n    }\n\n    // Early auth sanity check \u2014 the agent loop also reads the token on every\n    // request via api-client, but failing here gives a friendlier error than\n    // a 401 thirty seconds into the first long-poll.\n    const token = loadToken();\n    if (!token && !process.env.RULEMETRIC_API_KEY) {\n      this.error('No auth token found. Run `rulemetric auth login` first.');\n    }\n\n    this.log('Starting thin agent (Phase 4b parallel-run)...');\n    this.log('Long-polling /api/work/next. Press Ctrl+C to stop.\\n');\n\n    const controller = new AbortController();\n\n    const shutdown = async (signal: string) => {\n      this.log(`\\n[${signal}] shutting down agent...`);\n      controller.abort();\n      // Flush telemetry + drain the pool on the way out. All members run\n      // concurrently under Promise.allSettled (flushSentry 2s, closeDb 5s), so\n      // the total stays well under launchd's ~20s shutdown grace.\n      await Promise.allSettled([\n        flushSentry(2_000),\n        shutdownTelemetry(),\n        // Drain the postgres-js pool so this agent's Supavisor client sessions\n        // are released (Terminate) instead of orphaned as ghosts on every\n        // launchd bounce. Pool-exhaustion RCA 2026-06-21.\n        closeDb(),\n      ]);\n      releaseAgentLock();\n      process.exit(0);\n    };\n    process.on('SIGINT', () => { void shutdown('SIGINT'); });\n    process.on('SIGTERM', () => { void shutdown('SIGTERM'); });\n\n    try {\n      await runAgentLoop({\n        signal: controller.signal,\n        log: (msg) => this.log(msg),\n        version: this.config.version,\n      });\n    } finally {\n      releaseAgentLock();\n      // Drain on the throw path too (defense-in-depth). The signal-handler drain\n      // covers normal shutdown; a thrown runAgentLoop must not skip it. closeDb\n      // is idempotent, so the happy-path SIGTERM drain stays a no-op.\n      await closeDb();\n    }\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,OAAO,eAA4B;AAC5C,SAAS,eAAe;AACxB,SAAS,yBAAyB;AAClC,SACE,oBACA,qBACA,2BACK;AACP,SAAS,gBAAgB;AACzB,SAAS,mBAAmB,4BAA4B;;;ACalD,SAAU,iBAAiB,KAAuB;AACtD,MAAI,CAAC;AAAK,WAAO,CAAA;AACjB,QAAM,MAA8B,CAAA;AACpC,aAAW,SAAS,IAAI,MAAM,GAAG,GAAG;AAClC,UAAM,KAAK,MAAM,QAAQ,GAAG;AAC5B,QAAI,OAAO;AAAI;AACf,UAAM,MAAM,MAAM,MAAM,GAAG,EAAE,EAAE,KAAI;AACnC,UAAM,QAAQ,MAAM,MAAM,KAAK,CAAC,EAAE,KAAI;AACtC,QAAI,CAAC;AAAK;AACV,QAAI,GAAG,IAAI;EACb;AACA,SAAO;AACT;;;ADtBA,IAAI,MAAsB;AAmBpB,SAAU,qBAAkB;AAChC,SAAO,QAAQ,IAAI,sBAAsB;AAC3C;AAsBM,SAAU,cAAc,QAAuB;AACnD,MAAI,CAAC,mBAAkB,GAAI;AACzB,WAAO;EACT;AACA,MAAI,QAAQ,MAAM;AAGhB,WAAO;EACT;AAEA,QAAM,WAAW,IAAI,SAAS;IAC5B,CAAC,iBAAiB,GAAG,OAAO;IAC5B,CAAC,oBAAoB,GAAG,OAAO,kBAAkB;GAClD;AAED,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI;AACJ,MAAI;AAEJ,MAAI,UAAU;AAOZ,UAAM,UAAU,iBAAiB,QAAQ,IAAI,0BAA0B;AACvE,UAAM,WAAW,IAAI,kBAAkB;MACrC,KAAK,GAAG,QAAQ;MAChB;KACD;AACD,oBAAgB,IAAI,mBAAmB,QAAQ;AAC/C,WAAO;EACT,WAAW,QAAQ,IAAI,qBAAqB,QAAQ;AASlD,oBAAgB;AAChB,WAAO;EACT,OAAO;AAOL,UAAM,WAAW,IAAI,oBAAmB;AACxC,oBAAgB,IAAI,oBAAoB,QAAQ;AAChD,WAAO;EACT;AAEA,QAAM,IAAI,QAAQ;IAChB;IACA,gBAAgB,gBAAgB,CAAC,aAAa,IAAI,CAAA;GACnD;AAED,MAAI,MAAK;AACT,SAAO;AACT;AAEA,eAAsB,oBAAiB;AACrC,MAAI,KAAK;AACP,UAAM,IAAI,SAAQ;AAClB,UAAM;EACR;AACF;;;AE7HA,SAAS,UAAU,sBAAsB;;;ACAzC,SAAS,SAAAA,QAAO,WAAW,mBAAmB;;;ACwB9C,YAAY,YAAY;AAExB,IAAI,UAAU;AAkBR,SAAU,WAAW,QAAwB;AACjD,MAAI;AAAS,WAAO;AACpB,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC;AAAK,WAAO;AAEjB,EAAO,YAAK;IACV;IACA,YAAY,OAAO;IACnB,aAAa,QAAQ,IAAI,YAAY;;;;IAIrC,kBAAkB;GACnB;AAED,YAAU;AACV,SAAO;AACT;AAoCA,eAAsB,YAAY,WAAiB;AACjD,MAAI,CAAC;AAAS;AACd,QAAa,aAAM,SAAS;AAC9B;;;AC/FA,SAAS,YAAY,WAAW,cAAc,YAAY,qBAAqB;AAC/E,SAAS,SAAS,YAAY;AAC9B,SAAS,oBAAoB;AAS7B,IAAM,kBAAkB,KAAK,oBAAoB,GAAG,kBAAkB;AAEtE,SAAS,SAAS,KAAsB;AACtC,MAAI;AACF,YAAQ,KAAK,KAAK,CAAC;AACnB,WAAO;AAAA,EACT,SAAS,GAAG;AAEV,WAAQ,EAAwB,SAAS;AAAA,EAC3C;AACF;AAMA,SAAS,YAAY,KAAsB;AACzC,MAAI,CAAC,SAAS,GAAG,EAAG,QAAO;AAC3B,MAAI;AACF,UAAM,MAAM,aAAa,MAAM,CAAC,MAAM,OAAO,GAAG,GAAG,MAAM,UAAU,GAAG;AAAA,MACpE,UAAU;AAAA,MACV,OAAO,CAAC,UAAU,QAAQ,QAAQ;AAAA,IACpC,CAAC;AACD,WAAO,wCAAwC,KAAK,GAAG;AAAA,EACzD,QAAQ;AAEN,WAAO;AAAA,EACT;AACF;AASA,SAAS,mBAAiE;AACxE,MAAI;AACF,cAAU,QAAQ,eAAe,GAAG,EAAE,WAAW,KAAK,CAAC;AAAA,EACzD,QAAQ;AAAA,EAER;AACA,WAAS,UAAU,GAAG,UAAU,GAAG,WAAW;AAC5C,QAAI;AAEF,oBAAc,iBAAiB,OAAO,QAAQ,GAAG,GAAG,EAAE,MAAM,KAAK,CAAC;AAClE,aAAO,EAAE,IAAI,KAAK;AAAA,IACpB,SAAS,GAAG;AACV,UAAK,EAAwB,SAAS,SAAU,OAAM;AACtD,UAAI,SAAS;AACb,UAAI;AACF,iBAAS,OAAO,aAAa,iBAAiB,OAAO,EAAE,KAAK,CAAC;AAAA,MAC/D,QAAQ;AAAA,MAER;AACA,UAAI,UAAU,WAAW,QAAQ,OAAO,YAAY,MAAM,GAAG;AAC3D,eAAO,EAAE,IAAI,OAAO,OAAO;AAAA,MAC7B;AAEA,UAAI;AAAE,mBAAW,eAAe;AAAA,MAAG,QAAQ;AAAA,MAAqC;AAAA,IAClF;AAAA,EACF;AAEA,MAAI;AAAE,kBAAc,iBAAiB,OAAO,QAAQ,GAAG,CAAC;AAAA,EAAG,QAAQ;AAAA,EAAe;AAClF,SAAO,EAAE,IAAI,KAAK;AACpB;AAEA,SAAS,mBAAyB;AAChC,MAAI;AACF,QACE,WAAW,eAAe,KAC1B,OAAO,aAAa,iBAAiB,OAAO,EAAE,KAAK,CAAC,MAAM,QAAQ,KAClE;AACA,iBAAW,eAAe;AAAA,IAC5B;AAAA,EACF,QAAQ;AAAA,EAER;AACF;AAcA,IAAqB,mBAArB,MAAqB,0BAAyB,YAAY;AAAA,EACxD,OAAgB,cACd;AAAA,EAEF,OAAgB,WAAW;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,KAAK,MAAM,iBAAgB;AACjC,SAAK,YAAY;AAKjB,UAAM,OAAO,iBAAiB;AAC9B,QAAI,CAAC,KAAK,IAAI;AACZ,WAAK;AAAA,QACH,0DAA0D,KAAK,MAAM;AAAA,MAEvE;AACA;AAAA,IACF;AAEA,YAAQ,IAAI,wBAAwB;AAEpC,UAAM,gBAAgB,WAAW,EAAE,aAAa,mBAAmB,CAAC;AACpE,QAAI,cAAe,MAAK,IAAI,gCAAgC;AAE5D,UAAM,WAAW,cAAc,EAAE,aAAa,mBAAmB,CAAC;AAClE,QAAI,aAAa,QAAQ;AACvB,WAAK,IAAI,8BAA8B,QAAQ,IAAI,2BAA2B,EAAE;AAAA,IAClF,WAAW,aAAa,WAAW;AACjC,WAAK,IAAI,6FAA6F;AAAA,IACxG,OAAO;AACL,WAAK,IAAI,2CAA2C;AAAA,IACtD;AAKA,UAAM,QAAQ,UAAU;AACxB,QAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,oBAAoB;AAC7C,WAAK,MAAM,yDAAyD;AAAA,IACtE;AAEA,SAAK,IAAI,gDAAgD;AACzD,SAAK,IAAI,sDAAsD;AAE/D,UAAM,aAAa,IAAI,gBAAgB;AAEvC,UAAM,WAAW,OAAO,WAAmB;AACzC,WAAK,IAAI;AAAA,GAAM,MAAM,0BAA0B;AAC/C,iBAAW,MAAM;AAIjB,YAAM,QAAQ,WAAW;AAAA,QACvB,YAAY,GAAK;AAAA,QACjB,kBAAkB;AAAA;AAAA;AAAA;AAAA,QAIlB,QAAQ;AAAA,MACV,CAAC;AACD,uBAAiB;AACjB,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,GAAG,UAAU,MAAM;AAAE,WAAK,SAAS,QAAQ;AAAA,IAAG,CAAC;AACvD,YAAQ,GAAG,WAAW,MAAM;AAAE,WAAK,SAAS,SAAS;AAAA,IAAG,CAAC;AAEzD,QAAI;AACF,YAAM,aAAa;AAAA,QACjB,QAAQ,WAAW;AAAA,QACnB,KAAK,CAAC,QAAQ,KAAK,IAAI,GAAG;AAAA,QAC1B,SAAS,KAAK,OAAO;AAAA,MACvB,CAAC;AAAA,IACH,UAAE;AACA,uBAAiB;AAIjB,YAAM,QAAQ;AAAA,IAChB;AAAA,EACF;AACF;",
+  "sourcesContent": ["import { trace, context, type Tracer } from '@opentelemetry/api';\nimport { NodeSDK } from '@opentelemetry/sdk-node';\nimport { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';\nimport {\n  BatchSpanProcessor,\n  ConsoleSpanExporter,\n  SimpleSpanProcessor,\n} from '@opentelemetry/sdk-trace-node';\nimport { Resource } from '@opentelemetry/resources';\nimport { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';\nimport { parseOtlpHeaders } from './otlp-headers.js';\n\nlet sdk: NodeSDK | null = null;\n\nexport interface TelemetryConfig {\n  serviceName: string;\n  serviceVersion?: string;\n}\n\n/**\n * Boot states for `initTelemetry`. Returned discriminator lets callers\n * emit a precise boot log:\n *   - 'otlp'    \u2014 spans ship over OTLP to a real backend (Honeycomb etc.)\n *   - 'console' \u2014 spans print to stdout as JSON (Fly logs ingest them);\n *                 the \"open + zero third-party\" default\n *   - 'off'     \u2014 telemetry globally disabled (TELEMETRY_ENABLED=false)\n *                 OR a prior `initTelemetry()` already booted the SDK\n *                 in this process (idempotent skip)\n */\nexport type TelemetryBootState = 'otlp' | 'console' | 'silent' | 'off';\n\nexport function isTelemetryEnabled(): boolean {\n  return process.env.TELEMETRY_ENABLED !== 'false';\n}\n\n/**\n * Initialize OpenTelemetry tracing. Two transport modes, both built on\n * the same OTel SDK \u2014 only the exporter changes:\n *\n *   - OTLP backend  \u2192 set `OTEL_EXPORTER_OTLP_ENDPOINT` (+ optional\n *                     `OTEL_EXPORTER_OTLP_HEADERS` for vendor auth)\n *   - Console (stdout) \u2192 the default when no endpoint is set\n *\n * The console path is intentionally on-by-default. Without a real backend\n * configured, spans are written to stdout as JSON and end up in Fly's log\n * stream alongside everything else \u2014 searchable, structured, and you\n * stay inside the open OpenTelemetry spec without paying a third party.\n * Swap to OTLP later by setting one env var; no code change needed.\n *\n * Returns `'off'` when:\n *   - TELEMETRY_ENABLED=false (operator kill switch \u2014 beats any config)\n *   - The SDK was already started in this process (idempotent skip)\n *\n * See packages/telemetry/src/init.test.ts for the full state table.\n */\nexport function initTelemetry(config: TelemetryConfig): TelemetryBootState {\n  if (!isTelemetryEnabled()) {\n    return 'off';\n  }\n  if (sdk !== null) {\n    // OTel's global tracer provider gets set on first start(); calling\n    // start() twice triggers an SDK warning. Treat second-call as a no-op.\n    return 'off';\n  }\n\n  const resource = new Resource({\n    [ATTR_SERVICE_NAME]: config.serviceName,\n    [ATTR_SERVICE_VERSION]: config.serviceVersion ?? '0.1.0',\n  });\n\n  const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;\n  let mode: TelemetryBootState;\n  let spanProcessor: BatchSpanProcessor | SimpleSpanProcessor | undefined;\n\n  if (endpoint) {\n    // OTLP path. Headers via OTEL_EXPORTER_OTLP_HEADERS (W3C convention,\n    // comma-separated key=value pairs). Parsed explicitly rather than\n    // relying on SDK-version-specific auto-discovery \u2014 predictable boot\n    // matters more than tracking whatever the current default is.\n    //   Honeycomb:     x-honeycomb-team=hcaik_xxx (plus optional dataset)\n    //   Grafana Cloud: authorization=Basic <base64creds>\n    const headers = parseOtlpHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS);\n    const exporter = new OTLPTraceExporter({\n      url: `${endpoint}/v1/traces`,\n      headers,\n    });\n    spanProcessor = new BatchSpanProcessor(exporter);\n    mode = 'otlp';\n  } else if (process.env.OTEL_SPAN_EXPORT === 'none') {\n    // Silent path: trace context still propagates (so trace IDs are available\n    // for correlation and an OTLP endpoint can be added later with zero code\n    // change), but NO span processor is registered \u2014 nothing is written to\n    // stdout. This exists because the ConsoleSpanExporter below writes one\n    // JSON span per ended span, and on the local launchd API (no OTLP\n    // endpoint, fixed log file, no Fly-managed rotation) that grew api-stdout\n    // to 370MB / ~400k spans. Set OTEL_SPAN_EXPORT=none on that process; Fly\n    // leaves it unset so console spans stay grep-able in its managed log stream.\n    spanProcessor = undefined;\n    mode = 'silent';\n  } else {\n    // Console (stdout) path. SimpleSpanProcessor \u2014 not batched \u2014 so spans\n    // appear in chronological order in the log stream. Acceptable cost for\n    // tracing: a span is written per ended span, vs. a batched POST for OTLP.\n    // Logs that would otherwise need a UI to make sense (trace tree\n    // visualization) are still queryable via `flyctl logs | grep traceId=...`\n    // or any log-aggregator that ingests Fly's stream.\n    const exporter = new ConsoleSpanExporter();\n    spanProcessor = new SimpleSpanProcessor(exporter);\n    mode = 'console';\n  }\n\n  sdk = new NodeSDK({\n    resource,\n    spanProcessors: spanProcessor ? [spanProcessor] : [],\n  });\n\n  sdk.start();\n  return mode;\n}\n\nexport async function shutdownTelemetry(): Promise<void> {\n  if (sdk) {\n    await sdk.shutdown();\n    sdk = null;\n  }\n}\n\nexport function getTracer(name?: string): Tracer {\n  return trace.getTracer(name ?? 'rulemetric');\n}\n\nexport function getTraceId(): string | undefined {\n  const span = trace.getSpan(context.active());\n  return span?.spanContext().traceId;\n}\n", "// Parse the `OTEL_EXPORTER_OTLP_HEADERS` env var into a plain headers object\n// that the OTLP exporter constructor accepts.\n//\n// OpenTelemetry SDK env-vars spec (\u00A710) defines this as a comma-separated list\n// of `key=value` pairs. We parse it ourselves rather than relying on the\n// `@opentelemetry/exporter-trace-otlp-http` package's auto-discovery, because\n// that auto-discovery has shifted between SDK versions and we want predictable\n// behaviour at boot.\n//\n// Real-world usage:\n//   - Honeycomb:   `x-honeycomb-team=hcaik_xxx` (and optionally `,x-honeycomb-dataset=...`)\n//   - Grafana Cloud: `authorization=Basic <base64>` (note: base64 padding can be `==`)\n//\n// Edge cases (covered by `otlp-headers.test.ts`):\n//   - undefined or empty \u2192 {} (caller will get a no-headers exporter)\n//   - whitespace around `=` and `,` is trimmed (operator-friendly)\n//   - only the first `=` is the separator, so base64 padding (==) survives in\n//     values\n//   - malformed entries (no `=`) are skipped silently\n//   - empty keys are dropped (never inject a \"\" header)\n//   - duplicate keys: last value wins, matching URL search-params semantics\n\nexport function parseOtlpHeaders(raw: string | undefined): Record<string, string> {\n  if (!raw) return {};\n  const out: Record<string, string> = {};\n  for (const entry of raw.split(',')) {\n    const eq = entry.indexOf('=');\n    if (eq === -1) continue;\n    const key = entry.slice(0, eq).trim();\n    const value = entry.slice(eq + 1).trim();\n    if (!key) continue;\n    out[key] = value;\n  }\n  return out;\n}\n", "import { SpanKind, SpanStatusCode } from '@opentelemetry/api';\nimport type { MiddlewareHandler } from 'hono';\nimport { getTracer, isTelemetryEnabled } from './init.js';\n\nexport function telemetryMiddleware(): MiddlewareHandler {\n  return async (c, next) => {\n    if (!isTelemetryEnabled()) {\n      await next();\n      return;\n    }\n\n    const tracer = getTracer();\n    const method = c.req.method;\n    const path = c.req.path;\n\n    await tracer.startActiveSpan(\n      `${method} ${path}`,\n      {\n        kind: SpanKind.SERVER,\n        attributes: {\n          'http.method': method,\n          'http.url': c.req.url,\n          'http.route': path,\n        },\n      },\n      async (span) => {\n        try {\n          await next();\n\n          span.setAttribute('http.status_code', c.res.status);\n\n          const userId = c.get('userId' as never);\n          if (userId) {\n            span.setAttribute('rulemetric.user_id', userId as string);\n          }\n\n          if (c.res.status >= 400) {\n            span.setStatus({ code: SpanStatusCode.ERROR });\n          }\n        } catch (err) {\n          span.setStatus({ code: SpanStatusCode.ERROR });\n          span.recordException(err as Error);\n          throw err;\n        } finally {\n          span.end();\n        }\n      },\n    );\n  };\n}\n", "import { trace, context as otelContext } from '@opentelemetry/api';\n\n/**\n * Set attributes on the currently-active span, if one exists. Safe no-op\n * when no span is active (e.g. running outside of a Hono request handler,\n * or with telemetry disabled).\n *\n * Use this from business-logic code that wants to expose a discriminator\n * for telemetry queries \u2014 e.g. \"which branch did `resolveCallerOrgId` take\"\n * in `apps/api/src/routes/sessions.ts`. Span attributes are searchable in\n * any OpenTelemetry backend (Honeycomb, Grafana Cloud, ...) and in\n * `flyctl logs` when the console exporter is active.\n *\n * Keeping this thin wrapper in the telemetry package means application\n * code never imports `@opentelemetry/api` directly \u2014 the package boundary\n * stays the only place that knows about the OTel SDK shape.\n */\nexport function setSpanAttributes(\n  attributes: Record<string, string | number | boolean>,\n): void {\n  const span = trace.getSpan(otelContext.active());\n  if (!span) return;\n  for (const [key, value] of Object.entries(attributes)) {\n    span.setAttribute(key, value);\n  }\n}\n", "// Thin wrapper around @sentry/node so callers don't have to guard every\n// captureException with `if (process.env.SENTRY_DSN)`. The wrapper handles\n// three concerns:\n//\n//   1. Init only when SENTRY_DSN is set. Otherwise every entry point becomes\n//      a silent no-op \u2014 never throws, never logs noise, never opens a\n//      network transport.\n//   2. Idempotent init. Double-init is a Sentry SDK foot-gun (the second\n//      call wins, but the warning lands at runtime). The module-level\n//      `enabled` flag short-circuits the second call.\n//   3. tracesSampleRate = 0 by default. We already have OpenTelemetry\n//      shipping spans to a tracing backend; paying Sentry's quota for the\n//      same data isn't useful. Sentry's job here is errors-only.\n//\n// Wiring:\n//   - `apps/api/src/index.ts`           \u2192 initSentry({ serviceName: 'rulemetric-api' })\n//   - `apps/api/src/middleware/auth.ts` \u2192 setSentryUser(userId) after JWT verify\n//   - `apps/cli/src/commands/evals/agent.ts` \u2192 initSentry({ serviceName: 'rulemetric-agent' })\n//     (the thin-agent worker \u2014 Phase 4c moved the init here from the old\n//     graphile `listen.ts`; that path no longer inits Sentry)\n//\n// All four hooks are no-op-safe when DSN is unset, so the wiring doesn't\n// require feature-flag plumbing.\n\nimport * as Sentry from '@sentry/node';\n\nlet enabled = false;\n\nexport interface InitSentryConfig {\n  serviceName: string;\n}\n\n/**\n * Initialize Sentry if SENTRY_DSN is set. Presence of the DSN is now the\n * single switch: set it (incl. as a Fly secret in prod) to enable error\n * capture, leave it unset for a silent no-op.\n *\n * Previously gated to non-production by policy; that gate was dropped so\n * production errors actually get reported. Errors-only (`tracesSampleRate:\n * 0`) keeps Sentry quota bounded \u2014 OpenTelemetry still owns tracing.\n *\n * Returns true when init actually ran, false when it was skipped (DSN unset\n * or a prior call already initialized). Idempotent.\n */\nexport function initSentry(config: InitSentryConfig): boolean {\n  if (enabled) return false;\n  const dsn = process.env.SENTRY_DSN;\n  if (!dsn) return false;\n\n  Sentry.init({\n    dsn,\n    serverName: config.serviceName,\n    environment: process.env.NODE_ENV ?? 'development',\n    // We rely on OpenTelemetry for tracing \u2014 Sentry stays errors-only here.\n    // Override at the env-var level if you ever want Sentry performance too:\n    // `SENTRY_TRACES_SAMPLE_RATE=0.05` (Sentry SDK reads it natively).\n    tracesSampleRate: 0,\n  });\n\n  enabled = true;\n  return true;\n}\n\n/**\n * Report an exception. Coerces non-Error throws to Error so Sentry's\n * grouping/stack-trace logic has something to work with. Silent no-op when\n * SENTRY_DSN is unset.\n */\nexport function captureException(\n  err: unknown,\n  context?: { extra?: Record<string, unknown>; tags?: Record<string, string> },\n): void {\n  if (!enabled) return;\n  const e = err instanceof Error ? err : new Error(typeof err === 'string' ? err : JSON.stringify(err));\n  Sentry.captureException(e, context);\n}\n\n/**\n * Attach (or clear, with null) the active user on the Sentry scope. Called\n * from the API auth middleware so any captured exception during the request\n * carries `user.id`. Silent no-op when SENTRY_DSN is unset.\n */\nexport function setSentryUser(userId: string | null): void {\n  if (!enabled) return;\n  if (userId === null) {\n    Sentry.setUser(null);\n    return;\n  }\n  Sentry.setUser({ id: userId });\n}\n\n/**\n * Flush any queued events with a timeout. Critical to call before\n * `process.exit()` in worker shutdown \u2014 Sentry batches events and the SIGTERM\n * path was previously dropping the final exception (the one we'd most want\n * to see). Resolves immediately when SENTRY_DSN is unset.\n */\nexport async function flushSentry(timeoutMs: number): Promise<void> {\n  if (!enabled) return;\n  await Sentry.flush(timeoutMs);\n}\n\n/**\n * Reset the module-level state. Test-only \u2014 exported so test suites can\n * unwind the singleton between cases without `vi.resetModules` plumbing in\n * every block.\n */\nexport function _resetSentryForTests(): void {\n  enabled = false;\n}\n", "import { initSentry, flushSentry, initTelemetry, shutdownTelemetry } from '@rulemetric/telemetry';\nimport { BaseCommand } from '../../base-command.js';\nimport { loadToken, getDefaultConfigDir } from '../../lib/auth.js';\nimport { runAgentLoop } from '../../lib/agent-loop.js';\nimport { closeDb } from '@rulemetric/db';\nimport { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { execFileSync } from 'node:child_process';\n\n// Anchored under the config dir (honors RULEMETRIC_CONFIG_DIR; default\n// ~/.config/rulemetric, like gateway.pid), NOT os.tmpdir(), to avoid\n// launch-context TMPDIR divergence (launchd vs SSH/sudo/CI). Config-dir-scoped\n// so an isolated config (e.g. a cold-start test harness) can run its own agent\n// without colliding with the machine's real launchd-worker lock; two agents\n// sharing one config dir \u2014 the orphaned-fleet case this guard exists for \u2014 are\n// still refused.\nconst AGENT_LOCK_PATH = join(getDefaultConfigDir(), 'evals-agent.lock');\n\nfunction pidAlive(pid: number): boolean {\n  try {\n    process.kill(pid, 0);\n    return true;\n  } catch (e) {\n    // ESRCH = no such process (stale); EPERM = exists but owned by another user\n    return (e as { code?: string }).code === 'EPERM';\n  }\n}\n\n// Defends against PID reuse: a SIGKILL/OOM leaves the lock behind and the OS may\n// later assign that PID to an unrelated process \u2014 `pidAlive` alone would then\n// wedge `evals agent` (refuse to start forever). Confirm the holder is actually\n// an evals-agent before treating the lock as held; otherwise it's reclaimable.\nfunction isLiveAgent(pid: number): boolean {\n  if (!pidAlive(pid)) return false;\n  try {\n    const cmd = execFileSync('ps', ['-p', String(pid), '-o', 'command='], {\n      encoding: 'utf-8',\n      stdio: ['ignore', 'pipe', 'ignore'],\n    });\n    return /run\\.js\\s+evals\\s+agent|evals\\s+agent/.test(cmd);\n  } catch {\n    // ps unavailable/failed \u2014 fall back to liveness only (conservative: held)\n    return true;\n  }\n}\n\n// Single-instance guard. Each `evals agent` daemon holds its own postgres-js\n// pool; an orphaned, unsupervised fleet (4 stale agents found alive 3+ days on\n// 2026-06-21) was a confirmed driver of the recurring Supavisor 200-client\n// saturation. ATOMIC acquire via O_EXCL (flag 'wx') closes the TOCTOU window\n// where two simultaneously-starting agents could both acquire. On EEXIST the\n// holder is liveness-checked; a stale lock (holder gone) is reclaimed once.\n// See docs/incidents/2026-06-21-pool-exhaustion.md.\nfunction acquireAgentLock(): { ok: true } | { ok: false; holder: number } {\n  try {\n    mkdirSync(dirname(AGENT_LOCK_PATH), { recursive: true });\n  } catch {\n    /* ignore \u2014 writeFileSync will surface a real problem */\n  }\n  for (let attempt = 0; attempt < 2; attempt++) {\n    try {\n      // 'wx' = O_CREAT | O_EXCL: fails atomically with EEXIST if it exists.\n      writeFileSync(AGENT_LOCK_PATH, String(process.pid), { flag: 'wx' });\n      return { ok: true };\n    } catch (e) {\n      if ((e as { code?: string }).code !== 'EEXIST') throw e;\n      let holder = 0;\n      try {\n        holder = Number(readFileSync(AGENT_LOCK_PATH, 'utf-8').trim());\n      } catch {\n        /* unreadable \u2014 treat as stale */\n      }\n      if (holder && holder !== process.pid && isLiveAgent(holder)) {\n        return { ok: false, holder };\n      }\n      // stale (holder gone) / ours / unreadable \u2014 drop it and retry the atomic create once\n      try { unlinkSync(AGENT_LOCK_PATH); } catch { /* raced with another reclaimer */ }\n    }\n  }\n  // Last resort after a reclaim race \u2014 best-effort non-exclusive write.\n  try { writeFileSync(AGENT_LOCK_PATH, String(process.pid)); } catch { /* ignore */ }\n  return { ok: true };\n}\n\nfunction releaseAgentLock(): void {\n  try {\n    if (\n      existsSync(AGENT_LOCK_PATH) &&\n      Number(readFileSync(AGENT_LOCK_PATH, 'utf-8').trim()) === process.pid\n    ) {\n      unlinkSync(AGENT_LOCK_PATH);\n    }\n  } catch {\n    /* best-effort */\n  }\n}\n\n/**\n * `rulemetric evals agent` \u2014 Phase 4b parallel-run thin agent.\n *\n * Polls the /api/work endpoints (Phase 4a) instead of binding a Postgres\n * LISTEN/NOTIFY connection. This is the scaffold for the cutover described\n * in docs/plans/2026-05-25-thin-agent-migration.md; the default worker is\n * the sole worker mode since Phase 4c deleted the graphile path.\n *\n * Both commands can run side-by-side on the same user during the cutover \u2014\n * the API's atomic SKIP-LOCKED claim guarantees a job is handed to exactly\n * one agent. Picking which one runs is a launchd / shell choice.\n */\nexport default class EvalAgentCommand extends BaseCommand {\n  static override description =\n    'The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.';\n\n  static override examples = [\n    '<%= config.bin %> evals agent',\n  ];\n\n  async run(): Promise<void> {\n    await this.parse(EvalAgentCommand);\n    this.requireAuth();\n\n    // Single-instance guard \u2014 refuse to start a second agent. Each extra agent\n    // holds its own DB pool; an orphaned fleet was a confirmed cause of pooler\n    // saturation (pool-exhaustion RCA 2026-06-21).\n    const lock = acquireAgentLock();\n    if (!lock.ok) {\n      this.log(\n        `Another rulemetric evals agent is already running (pid ${lock.holder}). ` +\n          'Refusing to start a second instance.',\n      );\n      return;\n    }\n\n    process.env.PG_APPLICATION_NAME ||= 'rulemetric-agent';\n\n    const sentryEnabled = initSentry({ serviceName: 'rulemetric-agent' });\n    if (sentryEnabled) this.log('[sentry] error capture enabled');\n\n    const otelMode = initTelemetry({ serviceName: 'rulemetric-agent' });\n    if (otelMode === 'otlp') {\n      this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`);\n    } else if (otelMode === 'console') {\n      this.log('[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)');\n    } else {\n      this.log('[otel] disabled (TELEMETRY_ENABLED=false)');\n    }\n\n    // Early auth sanity check \u2014 the agent loop also reads the token on every\n    // request via api-client, but failing here gives a friendlier error than\n    // a 401 thirty seconds into the first long-poll.\n    const token = loadToken();\n    if (!token && !process.env.RULEMETRIC_API_KEY) {\n      this.error('No auth token found. Run `rulemetric auth login` first.');\n    }\n\n    this.log('Starting thin agent (Phase 4b parallel-run)...');\n    this.log('Long-polling /api/work/next. Press Ctrl+C to stop.\\n');\n\n    const controller = new AbortController();\n\n    const shutdown = async (signal: string) => {\n      this.log(`\\n[${signal}] shutting down agent...`);\n      controller.abort();\n      // Flush telemetry + drain the pool on the way out. All members run\n      // concurrently under Promise.allSettled (flushSentry 2s, closeDb 5s), so\n      // the total stays well under launchd's ~20s shutdown grace.\n      await Promise.allSettled([\n        flushSentry(2_000),\n        shutdownTelemetry(),\n        // Drain the postgres-js pool so this agent's Supavisor client sessions\n        // are released (Terminate) instead of orphaned as ghosts on every\n        // launchd bounce. Pool-exhaustion RCA 2026-06-21.\n        closeDb(),\n      ]);\n      releaseAgentLock();\n      process.exit(0);\n    };\n    process.on('SIGINT', () => { void shutdown('SIGINT'); });\n    process.on('SIGTERM', () => { void shutdown('SIGTERM'); });\n\n    // Set when the loop stops because `npm i -g` replaced the CLI under this\n    // running process \u2014 exit NON-ZERO after cleanup so the supervisor respawns\n    // onto the new code (systemd is Restart=on-failure; launchd KeepAlive\n    // restarts on any exit). Without this, a remote worker keeps executing the\n    // old code until someone manually restarts the unit.\n    let restartForUpgrade = false;\n    try {\n      await runAgentLoop({\n        signal: controller.signal,\n        log: (msg) => this.log(msg),\n        version: this.config.version,\n        readInstalledVersion: () => {\n          try {\n            const pkg = JSON.parse(\n              readFileSync(join(this.config.root, 'package.json'), 'utf-8'),\n            ) as { version?: string };\n            return pkg.version ?? null;\n          } catch {\n            return null;\n          }\n        },\n        onVersionDrift: () => {\n          restartForUpgrade = true;\n        },\n      });\n      if (restartForUpgrade) {\n        this.log('[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code.');\n        await Promise.allSettled([flushSentry(2_000), shutdownTelemetry(), closeDb()]);\n        releaseAgentLock();\n        process.exit(1);\n      }\n    } finally {\n      releaseAgentLock();\n      // Drain on the throw path too (defense-in-depth). The signal-handler drain\n      // covers normal shutdown; a thrown runAgentLoop must not skip it. closeDb\n      // is idempotent, so the happy-path SIGTERM drain stays a no-op.\n      await closeDb();\n    }\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,OAAO,eAA4B;AAC5C,SAAS,eAAe;AACxB,SAAS,yBAAyB;AAClC,SACE,oBACA,qBACA,2BACK;AACP,SAAS,gBAAgB;AACzB,SAAS,mBAAmB,4BAA4B;;;ACalD,SAAU,iBAAiB,KAAuB;AACtD,MAAI,CAAC;AAAK,WAAO,CAAA;AACjB,QAAM,MAA8B,CAAA;AACpC,aAAW,SAAS,IAAI,MAAM,GAAG,GAAG;AAClC,UAAM,KAAK,MAAM,QAAQ,GAAG;AAC5B,QAAI,OAAO;AAAI;AACf,UAAM,MAAM,MAAM,MAAM,GAAG,EAAE,EAAE,KAAI;AACnC,UAAM,QAAQ,MAAM,MAAM,KAAK,CAAC,EAAE,KAAI;AACtC,QAAI,CAAC;AAAK;AACV,QAAI,GAAG,IAAI;EACb;AACA,SAAO;AACT;;;ADtBA,IAAI,MAAsB;AAmBpB,SAAU,qBAAkB;AAChC,SAAO,QAAQ,IAAI,sBAAsB;AAC3C;AAsBM,SAAU,cAAc,QAAuB;AACnD,MAAI,CAAC,mBAAkB,GAAI;AACzB,WAAO;EACT;AACA,MAAI,QAAQ,MAAM;AAGhB,WAAO;EACT;AAEA,QAAM,WAAW,IAAI,SAAS;IAC5B,CAAC,iBAAiB,GAAG,OAAO;IAC5B,CAAC,oBAAoB,GAAG,OAAO,kBAAkB;GAClD;AAED,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI;AACJ,MAAI;AAEJ,MAAI,UAAU;AAOZ,UAAM,UAAU,iBAAiB,QAAQ,IAAI,0BAA0B;AACvE,UAAM,WAAW,IAAI,kBAAkB;MACrC,KAAK,GAAG,QAAQ;MAChB;KACD;AACD,oBAAgB,IAAI,mBAAmB,QAAQ;AAC/C,WAAO;EACT,WAAW,QAAQ,IAAI,qBAAqB,QAAQ;AASlD,oBAAgB;AAChB,WAAO;EACT,OAAO;AAOL,UAAM,WAAW,IAAI,oBAAmB;AACxC,oBAAgB,IAAI,oBAAoB,QAAQ;AAChD,WAAO;EACT;AAEA,QAAM,IAAI,QAAQ;IAChB;IACA,gBAAgB,gBAAgB,CAAC,aAAa,IAAI,CAAA;GACnD;AAED,MAAI,MAAK;AACT,SAAO;AACT;AAEA,eAAsB,oBAAiB;AACrC,MAAI,KAAK;AACP,UAAM,IAAI,SAAQ;AAClB,UAAM;EACR;AACF;;;AE7HA,SAAS,UAAU,sBAAsB;;;ACAzC,SAAS,SAAAA,QAAO,WAAW,mBAAmB;;;ACwB9C,YAAY,YAAY;AAExB,IAAI,UAAU;AAkBR,SAAU,WAAW,QAAwB;AACjD,MAAI;AAAS,WAAO;AACpB,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC;AAAK,WAAO;AAEjB,EAAO,YAAK;IACV;IACA,YAAY,OAAO;IACnB,aAAa,QAAQ,IAAI,YAAY;;;;IAIrC,kBAAkB;GACnB;AAED,YAAU;AACV,SAAO;AACT;AAoCA,eAAsB,YAAY,WAAiB;AACjD,MAAI,CAAC;AAAS;AACd,QAAa,aAAM,SAAS;AAC9B;;;AC/FA,SAAS,YAAY,WAAW,cAAc,YAAY,qBAAqB;AAC/E,SAAS,SAAS,YAAY;AAC9B,SAAS,oBAAoB;AAS7B,IAAM,kBAAkB,KAAK,oBAAoB,GAAG,kBAAkB;AAEtE,SAAS,SAAS,KAAsB;AACtC,MAAI;AACF,YAAQ,KAAK,KAAK,CAAC;AACnB,WAAO;AAAA,EACT,SAAS,GAAG;AAEV,WAAQ,EAAwB,SAAS;AAAA,EAC3C;AACF;AAMA,SAAS,YAAY,KAAsB;AACzC,MAAI,CAAC,SAAS,GAAG,EAAG,QAAO;AAC3B,MAAI;AACF,UAAM,MAAM,aAAa,MAAM,CAAC,MAAM,OAAO,GAAG,GAAG,MAAM,UAAU,GAAG;AAAA,MACpE,UAAU;AAAA,MACV,OAAO,CAAC,UAAU,QAAQ,QAAQ;AAAA,IACpC,CAAC;AACD,WAAO,wCAAwC,KAAK,GAAG;AAAA,EACzD,QAAQ;AAEN,WAAO;AAAA,EACT;AACF;AASA,SAAS,mBAAiE;AACxE,MAAI;AACF,cAAU,QAAQ,eAAe,GAAG,EAAE,WAAW,KAAK,CAAC;AAAA,EACzD,QAAQ;AAAA,EAER;AACA,WAAS,UAAU,GAAG,UAAU,GAAG,WAAW;AAC5C,QAAI;AAEF,oBAAc,iBAAiB,OAAO,QAAQ,GAAG,GAAG,EAAE,MAAM,KAAK,CAAC;AAClE,aAAO,EAAE,IAAI,KAAK;AAAA,IACpB,SAAS,GAAG;AACV,UAAK,EAAwB,SAAS,SAAU,OAAM;AACtD,UAAI,SAAS;AACb,UAAI;AACF,iBAAS,OAAO,aAAa,iBAAiB,OAAO,EAAE,KAAK,CAAC;AAAA,MAC/D,QAAQ;AAAA,MAER;AACA,UAAI,UAAU,WAAW,QAAQ,OAAO,YAAY,MAAM,GAAG;AAC3D,eAAO,EAAE,IAAI,OAAO,OAAO;AAAA,MAC7B;AAEA,UAAI;AAAE,mBAAW,eAAe;AAAA,MAAG,QAAQ;AAAA,MAAqC;AAAA,IAClF;AAAA,EACF;AAEA,MAAI;AAAE,kBAAc,iBAAiB,OAAO,QAAQ,GAAG,CAAC;AAAA,EAAG,QAAQ;AAAA,EAAe;AAClF,SAAO,EAAE,IAAI,KAAK;AACpB;AAEA,SAAS,mBAAyB;AAChC,MAAI;AACF,QACE,WAAW,eAAe,KAC1B,OAAO,aAAa,iBAAiB,OAAO,EAAE,KAAK,CAAC,MAAM,QAAQ,KAClE;AACA,iBAAW,eAAe;AAAA,IAC5B;AAAA,EACF,QAAQ;AAAA,EAER;AACF;AAcA,IAAqB,mBAArB,MAAqB,0BAAyB,YAAY;AAAA,EACxD,OAAgB,cACd;AAAA,EAEF,OAAgB,WAAW;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,KAAK,MAAM,iBAAgB;AACjC,SAAK,YAAY;AAKjB,UAAM,OAAO,iBAAiB;AAC9B,QAAI,CAAC,KAAK,IAAI;AACZ,WAAK;AAAA,QACH,0DAA0D,KAAK,MAAM;AAAA,MAEvE;AACA;AAAA,IACF;AAEA,YAAQ,IAAI,wBAAwB;AAEpC,UAAM,gBAAgB,WAAW,EAAE,aAAa,mBAAmB,CAAC;AACpE,QAAI,cAAe,MAAK,IAAI,gCAAgC;AAE5D,UAAM,WAAW,cAAc,EAAE,aAAa,mBAAmB,CAAC;AAClE,QAAI,aAAa,QAAQ;AACvB,WAAK,IAAI,8BAA8B,QAAQ,IAAI,2BAA2B,EAAE;AAAA,IAClF,WAAW,aAAa,WAAW;AACjC,WAAK,IAAI,6FAA6F;AAAA,IACxG,OAAO;AACL,WAAK,IAAI,2CAA2C;AAAA,IACtD;AAKA,UAAM,QAAQ,UAAU;AACxB,QAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,oBAAoB;AAC7C,WAAK,MAAM,yDAAyD;AAAA,IACtE;AAEA,SAAK,IAAI,gDAAgD;AACzD,SAAK,IAAI,sDAAsD;AAE/D,UAAM,aAAa,IAAI,gBAAgB;AAEvC,UAAM,WAAW,OAAO,WAAmB;AACzC,WAAK,IAAI;AAAA,GAAM,MAAM,0BAA0B;AAC/C,iBAAW,MAAM;AAIjB,YAAM,QAAQ,WAAW;AAAA,QACvB,YAAY,GAAK;AAAA,QACjB,kBAAkB;AAAA;AAAA;AAAA;AAAA,QAIlB,QAAQ;AAAA,MACV,CAAC;AACD,uBAAiB;AACjB,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,GAAG,UAAU,MAAM;AAAE,WAAK,SAAS,QAAQ;AAAA,IAAG,CAAC;AACvD,YAAQ,GAAG,WAAW,MAAM;AAAE,WAAK,SAAS,SAAS;AAAA,IAAG,CAAC;AAOzD,QAAI,oBAAoB;AACxB,QAAI;AACF,YAAM,aAAa;AAAA,QACjB,QAAQ,WAAW;AAAA,QACnB,KAAK,CAAC,QAAQ,KAAK,IAAI,GAAG;AAAA,QAC1B,SAAS,KAAK,OAAO;AAAA,QACrB,sBAAsB,MAAM;AAC1B,cAAI;AACF,kBAAM,MAAM,KAAK;AAAA,cACf,aAAa,KAAK,KAAK,OAAO,MAAM,cAAc,GAAG,OAAO;AAAA,YAC9D;AACA,mBAAO,IAAI,WAAW;AAAA,UACxB,QAAQ;AACN,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,QACA,gBAAgB,MAAM;AACpB,8BAAoB;AAAA,QACtB;AAAA,MACF,CAAC;AACD,UAAI,mBAAmB;AACrB,aAAK,IAAI,qFAAgF;AACzF,cAAM,QAAQ,WAAW,CAAC,YAAY,GAAK,GAAG,kBAAkB,GAAG,QAAQ,CAAC,CAAC;AAC7E,yBAAiB;AACjB,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,UAAE;AACA,uBAAiB;AAIjB,YAAM,QAAQ;AAAA,IAChB;AAAA,EACF;AACF;",
   "names": ["trace"]
 }

package/dist/commands/evals/benchmark.js

@@ -1,16 +1,17 @@
 import {
   aggregateBenchmark,
   postBenchmark
-} from "../../chunk-3QH73QOQ.js";
+} from "../../chunk-PED7L5GX.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiGet
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/benchmark.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/benchmark.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet } from '../../lib/api-client.js';\nimport { aggregateBenchmark, postBenchmark } from '../../lib/eval-benchmark.js';\n\nexport default class EvalBenchmarkCommand extends BaseCommand {\n  static override description = 'Aggregate eval runs into a benchmark for a target';\n\n  static override examples = [\n    '<%= config.bin %> evals benchmark <target-id>',\n    '<%= config.bin %> evals benchmark <target-id> --iteration 2',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    iteration: Flags.integer({ description: 'Filter runs by iteration', default: 1 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalBenchmarkCommand);\n    this.requireAuth();\n\n    const target = await apiGet<{ name: string; version: number }>(`/api/eval-targets/${args.targetId}`);\n    this.log(`Aggregating benchmark for \"${target.name}\" v${target.version}, iteration ${flags.iteration}`);\n\n    const result = await aggregateBenchmark(args.targetId, flags.iteration);\n\n    if (result.totalRuns === 0) {\n      this.error('No completed runs found. Run evals first with `rulemetric evals run`.');\n    }\n\n    this.log(`\\nAggregated ${result.totalRuns} runs\\n`);\n\n    // Display summary\n    for (const [config, stats] of Object.entries(result.runSummary)) {\n      if (config === 'delta') {\n        this.log(`\\nDelta (with_target - without_target):`);\n        for (const [metric, value] of Object.entries(stats as unknown as Record<string, string>)) {\n          this.log(`  ${metric}: ${value}`);\n        }\n        continue;\n      }\n      this.log(`${config}:`);\n      for (const [metric, stat] of Object.entries(stats)) {\n        const s = stat as { mean: number; stddev: number; min: number; max: number };\n        this.log(`  ${metric}: mean=${s.mean} stddev=${s.stddev} [${s.min}, ${s.max}]`);\n      }\n    }\n\n    if (result.notes.length > 0) {\n      this.log('\\nNotes:');\n      for (const note of result.notes) {\n        this.log(`  - ${note}`);\n      }\n    }\n\n    // Post to API\n    const benchmark = await postBenchmark(args.targetId, flags.iteration, result);\n    this.log(`\\nBenchmark saved: ${benchmark.id}`);\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAK5B,IAAqB,uBAArB,MAAqB,8BAA6B,YAAY;AAAA,EAC5D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,WAAW,MAAM,QAAQ,EAAE,aAAa,4BAA4B,SAAS,EAAE,CAAC;AAAA,EAClF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,qBAAoB;AAC7D,SAAK,YAAY;AAEjB,UAAM,SAAS,MAAM,OAA0C,qBAAqB,KAAK,QAAQ,EAAE;AACnG,SAAK,IAAI,8BAA8B,OAAO,IAAI,MAAM,OAAO,OAAO,eAAe,MAAM,SAAS,EAAE;AAEtG,UAAM,SAAS,MAAM,mBAAmB,KAAK,UAAU,MAAM,SAAS;AAEtE,QAAI,OAAO,cAAc,GAAG;AAC1B,WAAK,MAAM,uEAAuE;AAAA,IACpF;AAEA,SAAK,IAAI;AAAA,aAAgB,OAAO,SAAS;AAAA,CAAS;AAGlD,eAAW,CAAC,QAAQ,KAAK,KAAK,OAAO,QAAQ,OAAO,UAAU,GAAG;AAC/D,UAAI,WAAW,SAAS;AACtB,aAAK,IAAI;AAAA,sCAAyC;AAClD,mBAAW,CAAC,QAAQ,KAAK,KAAK,OAAO,QAAQ,KAA0C,GAAG;AACxF,eAAK,IAAI,KAAK,MAAM,KAAK,KAAK,EAAE;AAAA,QAClC;AACA;AAAA,MACF;AACA,WAAK,IAAI,GAAG,MAAM,GAAG;AACrB,iBAAW,CAAC,QAAQ,IAAI,KAAK,OAAO,QAAQ,KAAK,GAAG;AAClD,cAAM,IAAI;AACV,aAAK,IAAI,KAAK,MAAM,UAAU,EAAE,IAAI,WAAW,EAAE,MAAM,KAAK,EAAE,GAAG,KAAK,EAAE,GAAG,GAAG;AAAA,MAChF;AAAA,IACF;AAEA,QAAI,OAAO,MAAM,SAAS,GAAG;AAC3B,WAAK,IAAI,UAAU;AACnB,iBAAW,QAAQ,OAAO,OAAO;AAC/B,aAAK,IAAI,OAAO,IAAI,EAAE;AAAA,MACxB;AAAA,IACF;AAGA,UAAM,YAAY,MAAM,cAAc,KAAK,UAAU,MAAM,WAAW,MAAM;AAC5E,SAAK,IAAI;AAAA,mBAAsB,UAAU,EAAE,EAAE;AAAA,EAC/C;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAK5B,IAAqB,uBAArB,MAAqB,8BAA6B,YAAY;AAAA,EAC5D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,WAAW,MAAM,QAAQ,EAAE,aAAa,4BAA4B,SAAS,EAAE,CAAC;AAAA,EAClF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,qBAAoB;AAC7D,SAAK,YAAY;AAEjB,UAAM,SAAS,MAAM,OAA0C,qBAAqB,KAAK,QAAQ,EAAE;AACnG,SAAK,IAAI,8BAA8B,OAAO,IAAI,MAAM,OAAO,OAAO,eAAe,MAAM,SAAS,EAAE;AAEtG,UAAM,SAAS,MAAM,mBAAmB,KAAK,UAAU,MAAM,SAAS;AAEtE,QAAI,OAAO,cAAc,GAAG;AAC1B,WAAK,MAAM,uEAAuE;AAAA,IACpF;AAEA,SAAK,IAAI;AAAA,aAAgB,OAAO,SAAS;AAAA,CAAS;AAGlD,eAAW,CAAC,QAAQ,KAAK,KAAK,OAAO,QAAQ,OAAO,UAAU,GAAG;AAC/D,UAAI,WAAW,SAAS;AACtB,aAAK,IAAI;AAAA,sCAAyC;AAClD,mBAAW,CAAC,QAAQ,KAAK,KAAK,OAAO,QAAQ,KAA0C,GAAG;AACxF,eAAK,IAAI,KAAK,MAAM,KAAK,KAAK,EAAE;AAAA,QAClC;AACA;AAAA,MACF;AACA,WAAK,IAAI,GAAG,MAAM,GAAG;AACrB,iBAAW,CAAC,QAAQ,IAAI,KAAK,OAAO,QAAQ,KAAK,GAAG;AAClD,cAAM,IAAI;AACV,aAAK,IAAI,KAAK,MAAM,UAAU,EAAE,IAAI,WAAW,EAAE,MAAM,KAAK,EAAE,GAAG,KAAK,EAAE,GAAG,GAAG;AAAA,MAChF;AAAA,IACF;AAEA,QAAI,OAAO,MAAM,SAAS,GAAG;AAC3B,WAAK,IAAI,UAAU;AACnB,iBAAW,QAAQ,OAAO,OAAO;AAC/B,aAAK,IAAI,OAAO,IAAI,EAAE;AAAA,MACxB;AAAA,IACF;AAGA,UAAM,YAAY,MAAM,cAAc,KAAK,UAAU,MAAM,WAAW,MAAM;AAC5E,SAAK,IAAI;AAAA,mBAAsB,UAAU,EAAE,EAAE;AAAA,EAC/C;AACF;",
   "names": []
 }

package/dist/commands/evals/compare.js

@@ -3,13 +3,14 @@ import {
 } from "../../chunk-QN34TGD3.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiGet
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/compare.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/compare.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  content: string;\n  version: number;\n}\n\ninterface Eval {\n  id: string;\n  name: string;\n  prompt: string;\n  expectations: string[];\n}\n\nexport default class EvalCompareCommand extends BaseCommand {\n  static override description = 'Blind A/B comparison between two eval target versions';\n\n  static override examples = [\n    '<%= config.bin %> evals compare <target-a> <target-b>',\n  ];\n\n  static override args = {\n    targetA: Args.string({ description: 'First eval target ID', required: true }),\n    targetB: Args.string({ description: 'Second eval target ID', required: true }),\n  };\n\n  static override flags = {\n    model: Flags.string({ description: 'Model for execution and comparison' }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalCompareCommand);\n    this.requireAuth();\n\n    const [targetA, targetB] = await Promise.all([\n      apiGet<EvalTarget>(`/api/eval-targets/${args.targetA}`),\n      apiGet<EvalTarget>(`/api/eval-targets/${args.targetB}`),\n    ]);\n\n    // Use evals from target A (they should be testing the same thing)\n    const evals = await apiGet<Eval[]>(`/api/eval-targets/${args.targetA}/evals`);\n    if (evals.length === 0) {\n      this.error('No evals found for target A. Create some first.');\n    }\n\n    this.log(`Comparing \"${targetA.name}\" v${targetA.version} vs \"${targetB.name}\" v${targetB.version}`);\n    this.log(`Running ${evals.length} eval(s)...\\n`);\n\n    // Randomize A/B assignment for blindness\n    const isSwapped = Math.random() > 0.5;\n    const contentA = isSwapped ? targetB.content : targetA.content;\n    const contentB = isSwapped ? targetA.content : targetB.content;\n    const labelA = isSwapped ? targetB.name : targetA.name;\n    const labelB = isSwapped ? targetA.name : targetB.name;\n\n    const comparisons: Array<{\n      evalName: string;\n      outputA: string;\n      outputB: string;\n    }> = [];\n\n    for (const eval_ of evals) {\n      this.log(`[${eval_.name}] Running A...`);\n      const resultA = await executeEval({\n        prompt: eval_.prompt,\n        targetContent: contentA,\n        timeout: flags.timeout,\n        model: flags.model,\n      });\n\n      this.log(`[${eval_.name}] Running B...`);\n      const resultB = await executeEval({\n        prompt: eval_.prompt,\n        targetContent: contentB,\n        timeout: flags.timeout,\n        model: flags.model,\n      });\n\n      comparisons.push({\n        evalName: eval_.name,\n        outputA: resultA.output,\n        outputB: resultB.output,\n      });\n    }\n\n    // Run blind comparison via LLM\n    this.log('\\nRunning blind comparison...');\n\n    const comparisonPrompt = comparisons.map((c) => `\n## Eval: ${c.evalName}\n\n### Output A\n${c.outputA}\n\n### Output B\n${c.outputB}\n`).join('\\n---\\n');\n\n    const comparatorResult = await executeEval({\n      prompt: `Compare these two sets of outputs. For each eval, determine which output (A or B) is better.\nThen provide an overall winner with reasoning.\n\n${comparisonPrompt}\n\nRespond with JSON:\n{\n  \"per_eval\": [\n    { \"eval_name\": \"...\", \"winner\": \"A|B|TIE\", \"reasoning\": \"...\" }\n  ],\n  \"overall\": {\n    \"winner\": \"A|B|TIE\",\n    \"reasoning\": \"...\",\n    \"strengths_winner\": [\"...\"],\n    \"weaknesses_loser\": [\"...\"]\n  }\n}`,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    // Unblind and display results\n    this.log('\\n=== Comparison Results ===\\n');\n    this.log(`A = \"${labelA}\" | B = \"${labelB}\"`);\n    if (isSwapped) this.log('(Labels were swapped for blind comparison)');\n    this.log('');\n    this.log(comparatorResult.output);\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmB5B,IAAqB,qBAArB,MAAqB,4BAA2B,YAAY;AAAA,EAC1D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,SAAS,KAAK,OAAO,EAAE,aAAa,wBAAwB,UAAU,KAAK,CAAC;AAAA,IAC5E,SAAS,KAAK,OAAO,EAAE,aAAa,yBAAyB,UAAU,KAAK,CAAC;AAAA,EAC/E;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,OAAO,MAAM,OAAO,EAAE,aAAa,qCAAqC,CAAC;AAAA,IACzE,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,mBAAkB;AAC3D,SAAK,YAAY;AAEjB,UAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI;AAAA,MAC3C,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,MACtD,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,IACxD,CAAC;AAGD,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,OAAO,QAAQ;AAC5E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,iDAAiD;AAAA,IAC9D;AAEA,SAAK,IAAI,cAAc,QAAQ,IAAI,MAAM,QAAQ,OAAO,QAAQ,QAAQ,IAAI,MAAM,QAAQ,OAAO,EAAE;AACnG,SAAK,IAAI,WAAW,MAAM,MAAM;AAAA,CAAe;AAG/C,UAAM,YAAY,KAAK,OAAO,IAAI;AAClC,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAClD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAElD,UAAM,cAID,CAAC;AAEN,eAAW,SAAS,OAAO;AACzB,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,kBAAY,KAAK;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB,SAAS,QAAQ;AAAA,MACnB,CAAC;AAAA,IACH;AAGA,SAAK,IAAI,+BAA+B;AAExC,UAAM,mBAAmB,YAAY,IAAI,CAAC,MAAM;AAAA,WACzC,EAAE,QAAQ;AAAA;AAAA;AAAA,EAGnB,EAAE,OAAO;AAAA;AAAA;AAAA,EAGT,EAAE,OAAO;AAAA,CACV,EAAE,KAAK,SAAS;AAEb,UAAM,mBAAmB,MAAM,YAAY;AAAA,MACzC,QAAQ;AAAA;AAAA;AAAA,EAGZ,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAcZ,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAGD,SAAK,IAAI,gCAAgC;AACzC,SAAK,IAAI,QAAQ,MAAM,YAAY,MAAM,GAAG;AAC5C,QAAI,UAAW,MAAK,IAAI,4CAA4C;AACpE,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,iBAAiB,MAAM;AAAA,EAClC;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmB5B,IAAqB,qBAArB,MAAqB,4BAA2B,YAAY;AAAA,EAC1D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,SAAS,KAAK,OAAO,EAAE,aAAa,wBAAwB,UAAU,KAAK,CAAC;AAAA,IAC5E,SAAS,KAAK,OAAO,EAAE,aAAa,yBAAyB,UAAU,KAAK,CAAC;AAAA,EAC/E;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,OAAO,MAAM,OAAO,EAAE,aAAa,qCAAqC,CAAC;AAAA,IACzE,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,mBAAkB;AAC3D,SAAK,YAAY;AAEjB,UAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI;AAAA,MAC3C,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,MACtD,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,IACxD,CAAC;AAGD,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,OAAO,QAAQ;AAC5E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,iDAAiD;AAAA,IAC9D;AAEA,SAAK,IAAI,cAAc,QAAQ,IAAI,MAAM,QAAQ,OAAO,QAAQ,QAAQ,IAAI,MAAM,QAAQ,OAAO,EAAE;AACnG,SAAK,IAAI,WAAW,MAAM,MAAM;AAAA,CAAe;AAG/C,UAAM,YAAY,KAAK,OAAO,IAAI;AAClC,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAClD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAElD,UAAM,cAID,CAAC;AAEN,eAAW,SAAS,OAAO;AACzB,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,kBAAY,KAAK;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB,SAAS,QAAQ;AAAA,MACnB,CAAC;AAAA,IACH;AAGA,SAAK,IAAI,+BAA+B;AAExC,UAAM,mBAAmB,YAAY,IAAI,CAAC,MAAM;AAAA,WACzC,EAAE,QAAQ;AAAA;AAAA;AAAA,EAGnB,EAAE,OAAO;AAAA;AAAA;AAAA,EAGT,EAAE,OAAO;AAAA,CACV,EAAE,KAAK,SAAS;AAEb,UAAM,mBAAmB,MAAM,YAAY;AAAA,MACzC,QAAQ;AAAA;AAAA;AAAA,EAGZ,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAcZ,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAGD,SAAK,IAAI,gCAAgC;AACzC,SAAK,IAAI,QAAQ,MAAM,YAAY,MAAM,GAAG;AAC5C,QAAI,UAAW,MAAK,IAAI,4CAA4C;AACpE,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,iBAAiB,MAAM;AAAA,EAClC;AACF;",
   "names": []
 }

package/dist/commands/evals/create.js

@@ -1,12 +1,13 @@
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiPost
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/create.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/create.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { readFileSync } from 'node:fs';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiPost } from '../../lib/api-client.js';\n\ninterface EvalInput {\n  name: string;\n  prompt: string;\n  expectedOutput?: string;\n  expectations?: string[];\n  files?: unknown[];\n}\n\nexport default class EvalCreate extends BaseCommand {\n  static override description = 'Create test cases (evals) for an eval target';\n\n  static override examples = [\n    '<%= config.bin %> evals create <target-id> --name \"ts-check\" --prompt \"Write a function to add two numbers\"',\n    '<%= config.bin %> evals create <target-id> --from-file evals.json',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    name: Flags.string({ char: 'n', description: 'Eval name' }),\n    prompt: Flags.string({ char: 'p', description: 'Test prompt' }),\n    'expected-output': Flags.string({ description: 'Expected output description' }),\n    expectations: Flags.string({ char: 'e', description: 'Assertion (repeatable)', multiple: true }),\n    'from-file': Flags.string({ description: 'Bulk create from JSON file' }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalCreate);\n    this.requireAuth();\n\n    if (flags['from-file']) {\n      const data = JSON.parse(readFileSync(flags['from-file'], 'utf-8'));\n      const evalsArray: EvalInput[] = Array.isArray(data) ? data : data.evals;\n\n      for (const evalItem of evalsArray) {\n        const result = await apiPost<{ id: string; name: string }>(`/api/eval-targets/${args.targetId}/evals`, {\n          name: evalItem.name,\n          prompt: evalItem.prompt,\n          expectedOutput: evalItem.expectedOutput ?? '',\n          expectations: evalItem.expectations ?? [],\n          files: evalItem.files ?? [],\n        });\n        this.log(`Created eval: ${result.name} (${result.id})`);\n      }\n\n      this.log(`Created ${evalsArray.length} evals.`);\n      return;\n    }\n\n    if (!flags.name || !flags.prompt) {\n      this.error('Provide --name and --prompt, or use --from-file for bulk creation');\n    }\n\n    const result = await apiPost<{ id: string; name: string }>(`/api/eval-targets/${args.targetId}/evals`, {\n      name: flags.name,\n      prompt: flags.prompt,\n      expectedOutput: flags['expected-output'] ?? '',\n      expectations: flags.expectations ?? [],\n    });\n\n    this.log(`Created eval: ${result.name} (${result.id})`);\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAC5B,SAAS,oBAAoB;AAY7B,IAAqB,aAArB,MAAqB,oBAAmB,YAAY;AAAA,EAClD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,YAAY,CAAC;AAAA,IAC1D,QAAQ,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,cAAc,CAAC;AAAA,IAC9D,mBAAmB,MAAM,OAAO,EAAE,aAAa,8BAA8B,CAAC;AAAA,IAC9E,cAAc,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,0BAA0B,UAAU,KAAK,CAAC;AAAA,IAC/F,aAAa,MAAM,OAAO,EAAE,aAAa,6BAA6B,CAAC;AAAA,EACzE;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,WAAU;AACnD,SAAK,YAAY;AAEjB,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,OAAO,KAAK,MAAM,aAAa,MAAM,WAAW,GAAG,OAAO,CAAC;AACjE,YAAM,aAA0B,MAAM,QAAQ,IAAI,IAAI,OAAO,KAAK;AAElE,iBAAW,YAAY,YAAY;AACjC,cAAMA,UAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,UACrG,MAAM,SAAS;AAAA,UACf,QAAQ,SAAS;AAAA,UACjB,gBAAgB,SAAS,kBAAkB;AAAA,UAC3C,cAAc,SAAS,gBAAgB,CAAC;AAAA,UACxC,OAAO,SAAS,SAAS,CAAC;AAAA,QAC5B,CAAC;AACD,aAAK,IAAI,iBAAiBA,QAAO,IAAI,KAAKA,QAAO,EAAE,GAAG;AAAA,MACxD;AAEA,WAAK,IAAI,WAAW,WAAW,MAAM,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,QAAQ;AAChC,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAEA,UAAM,SAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,MACrG,MAAM,MAAM;AAAA,MACZ,QAAQ,MAAM;AAAA,MACd,gBAAgB,MAAM,iBAAiB,KAAK;AAAA,MAC5C,cAAc,MAAM,gBAAgB,CAAC;AAAA,IACvC,CAAC;AAED,SAAK,IAAI,iBAAiB,OAAO,IAAI,KAAK,OAAO,EAAE,GAAG;AAAA,EACxD;AACF;",
+  "mappings": ";;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAC5B,SAAS,oBAAoB;AAY7B,IAAqB,aAArB,MAAqB,oBAAmB,YAAY;AAAA,EAClD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,YAAY,CAAC;AAAA,IAC1D,QAAQ,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,cAAc,CAAC;AAAA,IAC9D,mBAAmB,MAAM,OAAO,EAAE,aAAa,8BAA8B,CAAC;AAAA,IAC9E,cAAc,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,0BAA0B,UAAU,KAAK,CAAC;AAAA,IAC/F,aAAa,MAAM,OAAO,EAAE,aAAa,6BAA6B,CAAC;AAAA,EACzE;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,WAAU;AACnD,SAAK,YAAY;AAEjB,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,OAAO,KAAK,MAAM,aAAa,MAAM,WAAW,GAAG,OAAO,CAAC;AACjE,YAAM,aAA0B,MAAM,QAAQ,IAAI,IAAI,OAAO,KAAK;AAElE,iBAAW,YAAY,YAAY;AACjC,cAAMA,UAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,UACrG,MAAM,SAAS;AAAA,UACf,QAAQ,SAAS;AAAA,UACjB,gBAAgB,SAAS,kBAAkB;AAAA,UAC3C,cAAc,SAAS,gBAAgB,CAAC;AAAA,UACxC,OAAO,SAAS,SAAS,CAAC;AAAA,QAC5B,CAAC;AACD,aAAK,IAAI,iBAAiBA,QAAO,IAAI,KAAKA,QAAO,EAAE,GAAG;AAAA,MACxD;AAEA,WAAK,IAAI,WAAW,WAAW,MAAM,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,QAAQ;AAChC,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAEA,UAAM,SAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,MACrG,MAAM,MAAM;AAAA,MACZ,QAAQ,MAAM;AAAA,MACd,gBAAgB,MAAM,iBAAiB,KAAK;AAAA,MAC5C,cAAc,MAAM,gBAAgB,CAAC;AAAA,IACvC,CAAC;AAED,SAAK,IAAI,iBAAiB,OAAO,IAAI,KAAK,OAAO,EAAE,GAAG;AAAA,EACxD;AACF;",
   "names": ["result"]
 }

package/dist/commands/evals/optimize-description.js

@@ -7,15 +7,16 @@ import {
 } from "../../chunk-QN34TGD3.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiGet,
   apiPost,
   apiPut
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/optimize-description.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/optimize-description.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet, apiPost, apiPut } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\nimport { testTrigger, computeTriggerScore } from '../../lib/eval-trigger-tester.js';\nimport type { TriggerTestResult } from '../../lib/eval-trigger-tester.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  type: string;\n  content: string;\n  description: string;\n  version: number;\n}\n\ninterface TriggerQuery {\n  query: string;\n  should_trigger: boolean;\n}\n\ninterface TriggerEvalSet {\n  id: string;\n  queries: TriggerQuery[];\n  status: string;\n  trainIndices: number[];\n  testIndices: number[];\n}\n\nexport default class EvalOptimizeDescriptionCommand extends BaseCommand {\n  static override description = 'Optimize eval target description for better triggering accuracy';\n\n  static override examples = [\n    '<%= config.bin %> evals optimize-description <target-id>',\n    '<%= config.bin %> evals optimize-description <target-id> --queries 10',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    'max-iterations': Flags.integer({ description: 'Maximum optimization iterations', default: 5 }),\n    model: Flags.string({ description: 'Model for trigger testing and improvement' }),\n    queries: Flags.integer({ description: 'Number of trigger queries to generate', default: 20 }),\n    'runs-per-query': Flags.integer({ description: 'Runs per query for reliability', default: 3 }),\n    'poll-interval': Flags.integer({ description: 'Ms between approval polls', default: 5000 }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalOptimizeDescriptionCommand);\n    this.requireAuth();\n\n    const target = await apiGet<EvalTarget>(`/api/eval-targets/${args.targetId}`);\n\n    this.log(`Optimizing description for \"${target.name}\" v${target.version}`);\n    this.log(`Current description: \"${target.description}\"`);\n    this.log('');\n\n    // Step 1: Generate trigger queries\n    this.log('Generating trigger queries...');\n    const queries = await this.generateTriggerQueries(target, flags.queries, flags);\n\n    // Step 2: Post to API for review\n    const triggerEval = await apiPost<TriggerEvalSet>(`/api/eval-targets/${args.targetId}/trigger-evals`, {\n      queries,\n    });\n\n    this.log(`\\nGenerated ${queries.length} trigger queries.`);\n    this.log(`Review and approve them at:`);\n    this.log(`  http://localhost:5174/eval-targets/${args.targetId}`);\n    this.log(`  (Triggers tab \u2192 click to review)`);\n    this.log('\\nWaiting for approval...');\n\n    // Step 3: Poll for approval\n    let approved = false;\n    while (!approved) {\n      await new Promise((resolve) => setTimeout(resolve, flags['poll-interval']));\n      const sets = await apiGet<TriggerEvalSet[]>(\n        `/api/eval-targets/${args.targetId}/trigger-evals?status=approved`,\n      );\n      const current = sets.find((s) => s.id === triggerEval.id);\n      if (current) {\n        approved = true;\n        // Use potentially edited queries\n        queries.length = 0;\n        queries.push(...current.queries);\n        this.log('Trigger queries approved!');\n      }\n    }\n\n    // Step 4: Split train/test (60/40)\n    const indices = queries.map((_, i) => i).sort(() => Math.random() - 0.5);\n    const splitIdx = Math.ceil(indices.length * 0.6);\n    const trainIndices = indices.slice(0, splitIdx);\n    const testIndices = indices.slice(splitIdx);\n\n    // Save split to API\n    await apiPut(`/api/trigger-evals/${triggerEval.id}`, {\n      trainIndices,\n      testIndices,\n    });\n\n    const trainQueries = trainIndices.map((i) => queries[i]);\n    const testQueries = testIndices.map((i) => queries[i]);\n\n    this.log(`\\nSplit: ${trainQueries.length} train, ${testQueries.length} test`);\n\n    // Step 5: Optimization loop\n    let currentDescription = target.description;\n    let bestDescription = currentDescription;\n    let bestTestScore = 0;\n\n    for (let iteration = 1; iteration <= flags['max-iterations']; iteration++) {\n      this.log(`\\n=== Iteration ${iteration} ===`);\n      this.log(`Description: \"${currentDescription}\"`);\n\n      // Test on train set (multiple runs per query)\n      this.log('Testing train set...');\n      const trainResults = await this.testQuerySet(trainQueries, target, currentDescription, flags);\n      const trainScore = computeTriggerScore(trainResults);\n      this.log(`Train accuracy: ${(trainScore * 100).toFixed(1)}%`);\n\n      this.printTriggerResults(trainResults);\n\n      // Test on test set\n      this.log('Testing test set...');\n      const testResults = await this.testQuerySet(testQueries, target, currentDescription, flags);\n      const testScore = computeTriggerScore(testResults);\n      this.log(`Test accuracy: ${(testScore * 100).toFixed(1)}%`);\n\n      // Record optimization run\n      await apiPost('/api/optimization-runs', {\n        evalTargetId: target.id,\n        type: 'description',\n        iteration,\n        inputVersion: target.version,\n        trainScore,\n        testScore,\n        analysis: {\n          description: currentDescription,\n          trainResults: trainResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),\n          testResults: testResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),\n        },\n        status: 'completed',\n      });\n\n      // Track best by test score\n      if (testScore > bestTestScore) {\n        bestTestScore = testScore;\n        bestDescription = currentDescription;\n      }\n\n      // Stop if perfect\n      if (trainScore >= 1.0) {\n        this.log('Perfect train score. Stopping.');\n        break;\n      }\n\n      // Propose improved description\n      this.log('Generating improved description...');\n      const failures = trainResults.filter((r) => !r.correct);\n      currentDescription = await this.improveDescription(target, currentDescription, failures, flags);\n    }\n\n    // Step 6: Apply best description\n    this.log(`\\n=== Optimization Complete ===`);\n    this.log(`Best test accuracy: ${(bestTestScore * 100).toFixed(1)}%`);\n    this.log(`\\nBefore: \"${target.description}\"`);\n    this.log(`After:  \"${bestDescription}\"`);\n\n    if (bestDescription !== target.description) {\n      await apiPut(`/api/eval-targets/${target.id}`, {\n        description: bestDescription,\n      });\n      this.log('\\nDescription updated.');\n    } else {\n      this.log('\\nNo improvement found. Description unchanged.');\n    }\n  }\n\n  private async generateTriggerQueries(\n    target: EvalTarget,\n    count: number,\n    flags: { model?: string; timeout: number },\n  ): Promise<TriggerQuery[]> {\n    // Try to seed with real session data\n    let sessionSeed = '';\n    try {\n      const sessions = await apiGet<Array<{ id: string }>>('/api/sessions?limit=10');\n      if (sessions.length > 0) {\n        sessionSeed = '\\n\\nReal user prompts from recent sessions (use as inspiration for realistic queries):\\n' +\n          sessions.slice(0, 5).map((s) => `- Session ${s.id}`).join('\\n');\n      }\n    } catch {\n      // No sessions available, that's fine\n    }\n\n    const shouldTriggerCount = Math.ceil(count / 2);\n    const shouldNotTriggerCount = count - shouldTriggerCount;\n\n    const prompt = `Generate ${count} trigger eval queries for a skill/instruction.\n\n## Skill Name\n${target.name}\n\n## Skill Description\n${target.description}\n\n## Skill Content (first 1000 chars)\n${target.content.slice(0, 1000)}\n${sessionSeed}\n\nGenerate ${shouldTriggerCount} should-trigger queries and ${shouldNotTriggerCount} should-not-trigger queries.\n\nRules:\n- Queries must be realistic \u2014 what a real user would actually type\n- Include concrete details (file paths, project names, specific problems)\n- Vary length and formality\n- Should-not-trigger queries should be near-misses (share keywords but need something different)\n- Do NOT make should-not-trigger queries obviously irrelevant\n\nRespond with ONLY a valid JSON array:\n[\n  {\"query\": \"realistic user prompt...\", \"should_trigger\": true},\n  {\"query\": \"another prompt...\", \"should_trigger\": false}\n]`;\n\n    const result = await executeEval({\n      prompt,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    try {\n      let jsonStr = result.output.trim();\n      const jsonMatch = jsonStr.match(/```(?:json)?\\s*([\\s\\S]*?)```/);\n      if (jsonMatch) jsonStr = jsonMatch[1].trim();\n      const arrMatch = jsonStr.match(/\\[[\\s\\S]*\\]/);\n      if (arrMatch) jsonStr = arrMatch[0];\n\n      return JSON.parse(jsonStr) as TriggerQuery[];\n    } catch {\n      // Fallback: generate minimal set\n      return [\n        { query: `Help me use ${target.name}`, should_trigger: true },\n        { query: 'Write a hello world program', should_trigger: false },\n      ];\n    }\n  }\n\n  private async testQuerySet(\n    queries: TriggerQuery[],\n    target: EvalTarget,\n    description: string,\n    flags: { model?: string; timeout: number; 'runs-per-query': number },\n  ): Promise<TriggerTestResult[]> {\n    const results: TriggerTestResult[] = [];\n\n    for (const q of queries) {\n      // Run multiple times and take majority vote\n      const runs: TriggerTestResult[] = [];\n      for (let run = 0; run < flags['runs-per-query']; run++) {\n        const result = await testTrigger({\n          query: q.query,\n          shouldTrigger: q.should_trigger,\n          skillName: target.name,\n          skillDescription: description,\n          skillContent: target.content,\n          model: flags.model,\n          timeout: flags.timeout,\n        });\n        runs.push(result);\n      }\n\n      // Majority vote\n      const triggerCount = runs.filter((r) => r.didTrigger).length;\n      const didTrigger = triggerCount > flags['runs-per-query'] / 2;\n\n      results.push({\n        query: q.query,\n        shouldTrigger: q.should_trigger,\n        didTrigger,\n        correct: didTrigger === q.should_trigger,\n        confidence: runs.reduce((sum, r) => sum + r.confidence, 0) / runs.length,\n        evidence: runs[0].evidence,\n      });\n    }\n\n    return results;\n  }\n\n  private printTriggerResults(results: TriggerTestResult[]): void {\n    for (const r of results) {\n      const status = r.correct ? 'OK' : 'FAIL';\n      const expected = r.shouldTrigger ? 'should trigger' : 'should NOT trigger';\n      const actual = r.didTrigger ? 'triggered' : 'did NOT trigger';\n      this.log(`  [${status}] ${expected} / ${actual}: \"${r.query.slice(0, 60)}...\"`);\n    }\n  }\n\n  private async improveDescription(\n    target: EvalTarget,\n    currentDescription: string,\n    failures: TriggerTestResult[],\n    flags: { model?: string; timeout: number },\n  ): Promise<string> {\n    const failureText = failures.map((f) => {\n      const expected = f.shouldTrigger ? 'SHOULD trigger but DID NOT' : 'SHOULD NOT trigger but DID';\n      return `- ${expected}: \"${f.query}\"`;\n    }).join('\\n');\n\n    const prompt = `You are optimizing a skill description for better triggering accuracy.\n\n## Skill Name\n${target.name}\n\n## Current Description\n${currentDescription}\n\n## Skill Content (first 500 chars)\n${target.content.slice(0, 500)}\n\n## Failures\n${failureText}\n\nPropose an improved description that:\n- Fixes the trigger failures above\n- Keeps the description concise but comprehensive\n- Uses specific keywords and contexts for when to trigger\n- Includes \"Use PROACTIVELY\" phrasing for important trigger contexts\n\nRespond with ONLY the new description text (no JSON, no explanation, just the description string).`;\n\n    const result = await executeEval({\n      prompt,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    return result.output.trim().replace(/^[\"']|[\"']$/g, '');\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AA6B5B,IAAqB,iCAArB,MAAqB,wCAAuC,YAAY;AAAA,EACtE,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,OAAO,MAAM,OAAO,EAAE,aAAa,4CAA4C,CAAC;AAAA,IAChF,SAAS,MAAM,QAAQ,EAAE,aAAa,yCAAyC,SAAS,GAAG,CAAC;AAAA,IAC5F,kBAAkB,MAAM,QAAQ,EAAE,aAAa,kCAAkC,SAAS,EAAE,CAAC;AAAA,IAC7F,iBAAiB,MAAM,QAAQ,EAAE,aAAa,6BAA6B,SAAS,IAAK,CAAC;AAAA,IAC1F,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,+BAA8B;AACvE,SAAK,YAAY;AAEjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAE5E,SAAK,IAAI,+BAA+B,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzE,SAAK,IAAI,yBAAyB,OAAO,WAAW,GAAG;AACvD,SAAK,IAAI,EAAE;AAGX,SAAK,IAAI,+BAA+B;AACxC,UAAM,UAAU,MAAM,KAAK,uBAAuB,QAAQ,MAAM,SAAS,KAAK;AAG9E,UAAM,cAAc,MAAM,QAAwB,qBAAqB,KAAK,QAAQ,kBAAkB;AAAA,MACpG;AAAA,IACF,CAAC;AAED,SAAK,IAAI;AAAA,YAAe,QAAQ,MAAM,mBAAmB;AACzD,SAAK,IAAI,6BAA6B;AACtC,SAAK,IAAI,wCAAwC,KAAK,QAAQ,EAAE;AAChE,SAAK,IAAI,yCAAoC;AAC7C,SAAK,IAAI,2BAA2B;AAGpC,QAAI,WAAW;AACf,WAAO,CAAC,UAAU;AAChB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,MAAM,eAAe,CAAC,CAAC;AAC1E,YAAM,OAAO,MAAM;AAAA,QACjB,qBAAqB,KAAK,QAAQ;AAAA,MACpC;AACA,YAAM,UAAU,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE;AACxD,UAAI,SAAS;AACX,mBAAW;AAEX,gBAAQ,SAAS;AACjB,gBAAQ,KAAK,GAAG,QAAQ,OAAO;AAC/B,aAAK,IAAI,2BAA2B;AAAA,MACtC;AAAA,IACF;AAGA,UAAM,UAAU,QAAQ,IAAI,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AACvE,UAAM,WAAW,KAAK,KAAK,QAAQ,SAAS,GAAG;AAC/C,UAAM,eAAe,QAAQ,MAAM,GAAG,QAAQ;AAC9C,UAAM,cAAc,QAAQ,MAAM,QAAQ;AAG1C,UAAM,OAAO,sBAAsB,YAAY,EAAE,IAAI;AAAA,MACnD;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,eAAe,aAAa,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AACvD,UAAM,cAAc,YAAY,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AAErD,SAAK,IAAI;AAAA,SAAY,aAAa,MAAM,WAAW,YAAY,MAAM,OAAO;AAG5E,QAAI,qBAAqB,OAAO;AAChC,QAAI,kBAAkB;AACtB,QAAI,gBAAgB;AAEpB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAC3C,WAAK,IAAI,iBAAiB,kBAAkB,GAAG;AAG/C,WAAK,IAAI,sBAAsB;AAC/B,YAAM,eAAe,MAAM,KAAK,aAAa,cAAc,QAAQ,oBAAoB,KAAK;AAC5F,YAAM,aAAa,oBAAoB,YAAY;AACnD,WAAK,IAAI,oBAAoB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE5D,WAAK,oBAAoB,YAAY;AAGrC,WAAK,IAAI,qBAAqB;AAC9B,YAAM,cAAc,MAAM,KAAK,aAAa,aAAa,QAAQ,oBAAoB,KAAK;AAC1F,YAAM,YAAY,oBAAoB,WAAW;AACjD,WAAK,IAAI,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG1D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa;AAAA,UACb,cAAc,aAAa,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,UACxI,aAAa,YAAY,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,QACxI;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,0BAAkB;AAAA,MACpB;AAGA,UAAI,cAAc,GAAK;AACrB,aAAK,IAAI,gCAAgC;AACzC;AAAA,MACF;AAGA,WAAK,IAAI,oCAAoC;AAC7C,YAAM,WAAW,aAAa,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO;AACtD,2BAAqB,MAAM,KAAK,mBAAmB,QAAQ,oBAAoB,UAAU,KAAK;AAAA,IAChG;AAGA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,wBAAwB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,GAAG;AACnE,SAAK,IAAI;AAAA,WAAc,OAAO,WAAW,GAAG;AAC5C,SAAK,IAAI,YAAY,eAAe,GAAG;AAEvC,QAAI,oBAAoB,OAAO,aAAa;AAC1C,YAAM,OAAO,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC7C,aAAa;AAAA,MACf,CAAC;AACD,WAAK,IAAI,wBAAwB;AAAA,IACnC,OAAO;AACL,WAAK,IAAI,gDAAgD;AAAA,IAC3D;AAAA,EACF;AAAA,EAEA,MAAc,uBACZ,QACA,OACA,OACyB;AAEzB,QAAI,cAAc;AAClB,QAAI;AACF,YAAM,WAAW,MAAM,OAA8B,wBAAwB;AAC7E,UAAI,SAAS,SAAS,GAAG;AACvB,sBAAc,6FACZ,SAAS,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,aAAa,EAAE,EAAE,EAAE,EAAE,KAAK,IAAI;AAAA,MAClE;AAAA,IACF,QAAQ;AAAA,IAER;AAEA,UAAM,qBAAqB,KAAK,KAAK,QAAQ,CAAC;AAC9C,UAAM,wBAAwB,QAAQ;AAEtC,UAAM,SAAS,YAAY,KAAK;AAAA;AAAA;AAAA,EAGlC,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,OAAO,WAAW;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAI,CAAC;AAAA,EAC7B,WAAW;AAAA;AAAA,WAEF,kBAAkB,+BAA+B,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe7E,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI;AACF,UAAI,UAAU,OAAO,OAAO,KAAK;AACjC,YAAM,YAAY,QAAQ,MAAM,8BAA8B;AAC9D,UAAI,UAAW,WAAU,UAAU,CAAC,EAAE,KAAK;AAC3C,YAAM,WAAW,QAAQ,MAAM,aAAa;AAC5C,UAAI,SAAU,WAAU,SAAS,CAAC;AAElC,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AAEN,aAAO;AAAA,QACL,EAAE,OAAO,eAAe,OAAO,IAAI,IAAI,gBAAgB,KAAK;AAAA,QAC5D,EAAE,OAAO,+BAA+B,gBAAgB,MAAM;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,aACZ,SACA,QACA,aACA,OAC8B;AAC9B,UAAM,UAA+B,CAAC;AAEtC,eAAW,KAAK,SAAS;AAEvB,YAAM,OAA4B,CAAC;AACnC,eAAS,MAAM,GAAG,MAAM,MAAM,gBAAgB,GAAG,OAAO;AACtD,cAAM,SAAS,MAAM,YAAY;AAAA,UAC/B,OAAO,EAAE;AAAA,UACT,eAAe,EAAE;AAAA,UACjB,WAAW,OAAO;AAAA,UAClB,kBAAkB;AAAA,UAClB,cAAc,OAAO;AAAA,UACrB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,aAAK,KAAK,MAAM;AAAA,MAClB;AAGA,YAAM,eAAe,KAAK,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE;AACtD,YAAM,aAAa,eAAe,MAAM,gBAAgB,IAAI;AAE5D,cAAQ,KAAK;AAAA,QACX,OAAO,EAAE;AAAA,QACT,eAAe,EAAE;AAAA,QACjB;AAAA,QACA,SAAS,eAAe,EAAE;AAAA,QAC1B,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC,IAAI,KAAK;AAAA,QAClE,UAAU,KAAK,CAAC,EAAE;AAAA,MACpB,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,SAAoC;AAC9D,eAAW,KAAK,SAAS;AACvB,YAAM,SAAS,EAAE,UAAU,OAAO;AAClC,YAAM,WAAW,EAAE,gBAAgB,mBAAmB;AACtD,YAAM,SAAS,EAAE,aAAa,cAAc;AAC5C,WAAK,IAAI,MAAM,MAAM,KAAK,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM;AAAA,IAChF;AAAA,EACF;AAAA,EAEA,MAAc,mBACZ,QACA,oBACA,UACA,OACiB;AACjB,UAAM,cAAc,SAAS,IAAI,CAAC,MAAM;AACtC,YAAM,WAAW,EAAE,gBAAgB,+BAA+B;AAClE,aAAO,KAAK,QAAQ,MAAM,EAAE,KAAK;AAAA,IACnC,CAAC,EAAE,KAAK,IAAI;AAEZ,UAAM,SAAS;AAAA;AAAA;AAAA,EAGjB,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,kBAAkB;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA;AAAA,EAG5B,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAUT,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,WAAO,OAAO,OAAO,KAAK,EAAE,QAAQ,gBAAgB,EAAE;AAAA,EACxD;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AA6B5B,IAAqB,iCAArB,MAAqB,wCAAuC,YAAY;AAAA,EACtE,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,OAAO,MAAM,OAAO,EAAE,aAAa,4CAA4C,CAAC;AAAA,IAChF,SAAS,MAAM,QAAQ,EAAE,aAAa,yCAAyC,SAAS,GAAG,CAAC;AAAA,IAC5F,kBAAkB,MAAM,QAAQ,EAAE,aAAa,kCAAkC,SAAS,EAAE,CAAC;AAAA,IAC7F,iBAAiB,MAAM,QAAQ,EAAE,aAAa,6BAA6B,SAAS,IAAK,CAAC;AAAA,IAC1F,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,+BAA8B;AACvE,SAAK,YAAY;AAEjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAE5E,SAAK,IAAI,+BAA+B,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzE,SAAK,IAAI,yBAAyB,OAAO,WAAW,GAAG;AACvD,SAAK,IAAI,EAAE;AAGX,SAAK,IAAI,+BAA+B;AACxC,UAAM,UAAU,MAAM,KAAK,uBAAuB,QAAQ,MAAM,SAAS,KAAK;AAG9E,UAAM,cAAc,MAAM,QAAwB,qBAAqB,KAAK,QAAQ,kBAAkB;AAAA,MACpG;AAAA,IACF,CAAC;AAED,SAAK,IAAI;AAAA,YAAe,QAAQ,MAAM,mBAAmB;AACzD,SAAK,IAAI,6BAA6B;AACtC,SAAK,IAAI,wCAAwC,KAAK,QAAQ,EAAE;AAChE,SAAK,IAAI,yCAAoC;AAC7C,SAAK,IAAI,2BAA2B;AAGpC,QAAI,WAAW;AACf,WAAO,CAAC,UAAU;AAChB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,MAAM,eAAe,CAAC,CAAC;AAC1E,YAAM,OAAO,MAAM;AAAA,QACjB,qBAAqB,KAAK,QAAQ;AAAA,MACpC;AACA,YAAM,UAAU,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE;AACxD,UAAI,SAAS;AACX,mBAAW;AAEX,gBAAQ,SAAS;AACjB,gBAAQ,KAAK,GAAG,QAAQ,OAAO;AAC/B,aAAK,IAAI,2BAA2B;AAAA,MACtC;AAAA,IACF;AAGA,UAAM,UAAU,QAAQ,IAAI,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AACvE,UAAM,WAAW,KAAK,KAAK,QAAQ,SAAS,GAAG;AAC/C,UAAM,eAAe,QAAQ,MAAM,GAAG,QAAQ;AAC9C,UAAM,cAAc,QAAQ,MAAM,QAAQ;AAG1C,UAAM,OAAO,sBAAsB,YAAY,EAAE,IAAI;AAAA,MACnD;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,eAAe,aAAa,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AACvD,UAAM,cAAc,YAAY,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AAErD,SAAK,IAAI;AAAA,SAAY,aAAa,MAAM,WAAW,YAAY,MAAM,OAAO;AAG5E,QAAI,qBAAqB,OAAO;AAChC,QAAI,kBAAkB;AACtB,QAAI,gBAAgB;AAEpB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAC3C,WAAK,IAAI,iBAAiB,kBAAkB,GAAG;AAG/C,WAAK,IAAI,sBAAsB;AAC/B,YAAM,eAAe,MAAM,KAAK,aAAa,cAAc,QAAQ,oBAAoB,KAAK;AAC5F,YAAM,aAAa,oBAAoB,YAAY;AACnD,WAAK,IAAI,oBAAoB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE5D,WAAK,oBAAoB,YAAY;AAGrC,WAAK,IAAI,qBAAqB;AAC9B,YAAM,cAAc,MAAM,KAAK,aAAa,aAAa,QAAQ,oBAAoB,KAAK;AAC1F,YAAM,YAAY,oBAAoB,WAAW;AACjD,WAAK,IAAI,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG1D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa;AAAA,UACb,cAAc,aAAa,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,UACxI,aAAa,YAAY,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,QACxI;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,0BAAkB;AAAA,MACpB;AAGA,UAAI,cAAc,GAAK;AACrB,aAAK,IAAI,gCAAgC;AACzC;AAAA,MACF;AAGA,WAAK,IAAI,oCAAoC;AAC7C,YAAM,WAAW,aAAa,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO;AACtD,2BAAqB,MAAM,KAAK,mBAAmB,QAAQ,oBAAoB,UAAU,KAAK;AAAA,IAChG;AAGA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,wBAAwB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,GAAG;AACnE,SAAK,IAAI;AAAA,WAAc,OAAO,WAAW,GAAG;AAC5C,SAAK,IAAI,YAAY,eAAe,GAAG;AAEvC,QAAI,oBAAoB,OAAO,aAAa;AAC1C,YAAM,OAAO,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC7C,aAAa;AAAA,MACf,CAAC;AACD,WAAK,IAAI,wBAAwB;AAAA,IACnC,OAAO;AACL,WAAK,IAAI,gDAAgD;AAAA,IAC3D;AAAA,EACF;AAAA,EAEA,MAAc,uBACZ,QACA,OACA,OACyB;AAEzB,QAAI,cAAc;AAClB,QAAI;AACF,YAAM,WAAW,MAAM,OAA8B,wBAAwB;AAC7E,UAAI,SAAS,SAAS,GAAG;AACvB,sBAAc,6FACZ,SAAS,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,aAAa,EAAE,EAAE,EAAE,EAAE,KAAK,IAAI;AAAA,MAClE;AAAA,IACF,QAAQ;AAAA,IAER;AAEA,UAAM,qBAAqB,KAAK,KAAK,QAAQ,CAAC;AAC9C,UAAM,wBAAwB,QAAQ;AAEtC,UAAM,SAAS,YAAY,KAAK;AAAA;AAAA;AAAA,EAGlC,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,OAAO,WAAW;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAI,CAAC;AAAA,EAC7B,WAAW;AAAA;AAAA,WAEF,kBAAkB,+BAA+B,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe7E,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI;AACF,UAAI,UAAU,OAAO,OAAO,KAAK;AACjC,YAAM,YAAY,QAAQ,MAAM,8BAA8B;AAC9D,UAAI,UAAW,WAAU,UAAU,CAAC,EAAE,KAAK;AAC3C,YAAM,WAAW,QAAQ,MAAM,aAAa;AAC5C,UAAI,SAAU,WAAU,SAAS,CAAC;AAElC,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AAEN,aAAO;AAAA,QACL,EAAE,OAAO,eAAe,OAAO,IAAI,IAAI,gBAAgB,KAAK;AAAA,QAC5D,EAAE,OAAO,+BAA+B,gBAAgB,MAAM;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,aACZ,SACA,QACA,aACA,OAC8B;AAC9B,UAAM,UAA+B,CAAC;AAEtC,eAAW,KAAK,SAAS;AAEvB,YAAM,OAA4B,CAAC;AACnC,eAAS,MAAM,GAAG,MAAM,MAAM,gBAAgB,GAAG,OAAO;AACtD,cAAM,SAAS,MAAM,YAAY;AAAA,UAC/B,OAAO,EAAE;AAAA,UACT,eAAe,EAAE;AAAA,UACjB,WAAW,OAAO;AAAA,UAClB,kBAAkB;AAAA,UAClB,cAAc,OAAO;AAAA,UACrB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,aAAK,KAAK,MAAM;AAAA,MAClB;AAGA,YAAM,eAAe,KAAK,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE;AACtD,YAAM,aAAa,eAAe,MAAM,gBAAgB,IAAI;AAE5D,cAAQ,KAAK;AAAA,QACX,OAAO,EAAE;AAAA,QACT,eAAe,EAAE;AAAA,QACjB;AAAA,QACA,SAAS,eAAe,EAAE;AAAA,QAC1B,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC,IAAI,KAAK;AAAA,QAClE,UAAU,KAAK,CAAC,EAAE;AAAA,MACpB,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,SAAoC;AAC9D,eAAW,KAAK,SAAS;AACvB,YAAM,SAAS,EAAE,UAAU,OAAO;AAClC,YAAM,WAAW,EAAE,gBAAgB,mBAAmB;AACtD,YAAM,SAAS,EAAE,aAAa,cAAc;AAC5C,WAAK,IAAI,MAAM,MAAM,KAAK,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM;AAAA,IAChF;AAAA,EACF;AAAA,EAEA,MAAc,mBACZ,QACA,oBACA,UACA,OACiB;AACjB,UAAM,cAAc,SAAS,IAAI,CAAC,MAAM;AACtC,YAAM,WAAW,EAAE,gBAAgB,+BAA+B;AAClE,aAAO,KAAK,QAAQ,MAAM,EAAE,KAAK;AAAA,IACnC,CAAC,EAAE,KAAK,IAAI;AAEZ,UAAM,SAAS;AAAA;AAAA;AAAA,EAGjB,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,kBAAkB;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA;AAAA,EAG5B,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAUT,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,WAAO,OAAO,OAAO,KAAK,EAAE,QAAQ,gBAAgB,EAAE;AAAA,EACxD;AACF;",
   "names": []
 }

package/dist/commands/evals/optimize.js

@@ -9,15 +9,16 @@ import {
 } from "../../chunk-QN34TGD3.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiGet,
   apiPost,
   apiPut
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/optimize.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/optimize.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet, apiPost, apiPut } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\nimport { gradeEvalOutput } from '../../lib/eval-grader.js';\nimport { analyzeEvalResults } from '../../lib/eval-analyzer.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  type: string;\n  content: string;\n  description: string;\n  version: number;\n}\n\ninterface Eval {\n  id: string;\n  name: string;\n  prompt: string;\n  expectedOutput: string;\n  expectations: string[];\n}\n\nexport default class EvalOptimizeCommand extends BaseCommand {\n  static override description = 'Iteratively optimize eval target content using eval results as feedback';\n\n  static override examples = [\n    '<%= config.bin %> evals optimize <target-id>',\n    '<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    auto: Flags.boolean({ description: 'Skip interactive review, apply suggestions automatically', default: false }),\n    'max-iterations': Flags.integer({ description: 'Maximum optimization iterations', default: 5 }),\n    threshold: Flags.string({ description: 'Stop if pass rate reaches this threshold (0-1)', default: '0.95' }),\n    model: Flags.string({ description: 'Model for eval execution' }),\n    'analyzer-model': Flags.string({ description: 'Model for analysis/suggestions' }),\n    'runs-per-eval': Flags.integer({ description: 'Runs per eval per iteration', default: 1 }),\n    parallel: Flags.integer({ description: 'Number of parallel eval executions', default: 1 }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalOptimizeCommand);\n    this.requireAuth();\n\n    let target = await apiGet<EvalTarget>(`/api/eval-targets/${args.targetId}`);\n    const allEvals = await apiGet<Eval[]>(`/api/eval-targets/${args.targetId}/evals`);\n\n    if (allEvals.length === 0) {\n      this.error('No evals found. Create some first with `rulemetric evals create`.');\n    }\n\n    // Split evals into train/test (60/40)\n    const shuffled = [...allEvals].sort(() => Math.random() - 0.5);\n    const splitIdx = Math.ceil(shuffled.length * 0.6);\n    const trainEvals = shuffled.slice(0, splitIdx);\n    const testEvals = shuffled.slice(splitIdx);\n\n    this.log(`Optimizing \"${target.name}\" v${target.version}`);\n    this.log(`Evals: ${trainEvals.length} train, ${testEvals.length} test`);\n    const threshold = parseFloat(flags.threshold);\n    this.log(`Config: max ${flags['max-iterations']} iterations, threshold ${(threshold * 100).toFixed(0)}%`);\n    if (flags.auto) this.log('Mode: auto (no interactive review)');\n    this.log('');\n\n    let bestVersion = target.version;\n    let bestTestScore = 0;\n    let noImprovementCount = 0;\n\n    for (let iteration = 1; iteration <= flags['max-iterations']; iteration++) {\n      this.log(`\\n=== Iteration ${iteration} ===`);\n\n      // Record optimization run as running\n      const optRun = await apiPost<{ id: string }>('/api/optimization-runs', {\n        evalTargetId: target.id,\n        type: 'content',\n        iteration,\n        inputVersion: target.version,\n        status: 'running',\n      });\n\n      // Run train evals\n      this.log('Running train evals...');\n      const trainResults = await this.runEvalsOnTarget(target, trainEvals, flags);\n      const trainScore = this.computePassRate(trainResults);\n      this.log(`Train pass rate: ${(trainScore * 100).toFixed(1)}%`);\n\n      // Check if we've hit the threshold\n      if (trainScore >= threshold) {\n        this.log(`Train score ${(trainScore * 100).toFixed(1)}% >= threshold ${(threshold * 100).toFixed(0)}%. Stopping.`);\n        // Run test evals for final score\n        const testResults = await this.runEvalsOnTarget(target, testEvals, flags);\n        const testScore = this.computePassRate(testResults);\n        this.log(`Final test score: ${(testScore * 100).toFixed(1)}%`);\n\n        await apiPost('/api/optimization-runs', {\n          evalTargetId: target.id,\n          type: 'content',\n          iteration,\n          inputVersion: target.version,\n          trainScore,\n          testScore,\n          analysis: { reason: 'threshold_reached' },\n          status: 'completed',\n        });\n        break;\n      }\n\n      // Analyze and propose improvements\n      this.log('Analyzing results...');\n      const evalResults = trainResults.map((r) => ({\n        evalName: r.evalName,\n        prompt: r.prompt,\n        expectations: r.expectations,\n        grading: r.grading,\n      }));\n\n      const analysis = await analyzeEvalResults({\n        content: target.content,\n        evalResults,\n        model: flags['analyzer-model'] ?? flags.model,\n        timeout: flags.timeout,\n      });\n\n      this.log(`\\nAssessment: ${analysis.overallAssessment}`);\n      this.log(`Suggestions: ${analysis.suggestions.length}`);\n      for (const s of analysis.suggestions) {\n        this.log(`  - [${s.section}] ${s.reasoning}`);\n      }\n\n      // In interactive mode, ask for approval\n      if (!flags.auto) {\n        this.log('\\nProposed changes above. Applying in auto mode would proceed automatically.');\n        this.log('(Interactive approval not yet implemented \u2014 use --auto for now)');\n      }\n\n      if (analysis.proposedContent === target.content) {\n        this.log('No changes proposed. Stopping.');\n        await apiPost('/api/optimization-runs', {\n          evalTargetId: target.id,\n          type: 'content',\n          iteration,\n          inputVersion: target.version,\n          trainScore,\n          analysis: { reason: 'no_changes' },\n          status: 'completed',\n        });\n        break;\n      }\n\n      // Create new version\n      const newTarget = await apiPut<EvalTarget>(`/api/eval-targets/${target.id}`, {\n        content: analysis.proposedContent,\n      });\n      this.log(`Created v${newTarget.version}`);\n\n      // Run test evals on new version\n      this.log('Running test evals...');\n      const testResults = await this.runEvalsOnTarget(newTarget, testEvals, flags);\n      const testScore = this.computePassRate(testResults);\n      this.log(`Test pass rate: ${(testScore * 100).toFixed(1)}%`);\n\n      // Update optimization run\n      await apiPost('/api/optimization-runs', {\n        evalTargetId: newTarget.id,\n        type: 'content',\n        iteration,\n        inputVersion: target.version,\n        outputVersion: newTarget.version,\n        trainScore,\n        testScore,\n        analysis: {\n          suggestions: analysis.suggestions,\n          overallAssessment: analysis.overallAssessment,\n        },\n        status: 'completed',\n      });\n\n      // Track best\n      if (testScore > bestTestScore) {\n        bestTestScore = testScore;\n        bestVersion = newTarget.version;\n        noImprovementCount = 0;\n      } else {\n        noImprovementCount++;\n        if (noImprovementCount >= 2) {\n          this.log('No improvement for 2 consecutive iterations. Stopping.');\n          break;\n        }\n      }\n\n      target = newTarget;\n    }\n\n    this.log(`\\n=== Optimization Complete ===`);\n    this.log(`Best version: v${bestVersion} (test score: ${(bestTestScore * 100).toFixed(1)}%)`);\n  }\n\n  private async runEvalsOnTarget(\n    target: EvalTarget,\n    evalSet: Eval[],\n    flags: { model?: string; timeout: number; 'runs-per-eval': number; parallel: number },\n  ): Promise<Array<{ evalName: string; prompt: string; expectations: string[]; grading: Record<string, unknown> }>> {\n    const results: Array<{ evalName: string; prompt: string; expectations: string[]; grading: Record<string, unknown> }> = [];\n\n    for (const eval_ of evalSet) {\n      const execResult = await executeEval({\n        prompt: eval_.prompt,\n        targetContent: target.content,\n        timeout: flags.timeout,\n        model: flags.model,\n      });\n\n      if (execResult.error) {\n        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });\n        continue;\n      }\n\n      if (eval_.expectations.length > 0) {\n        const grading = await gradeEvalOutput({\n          prompt: eval_.prompt,\n          output: execResult.output,\n          expectations: eval_.expectations,\n          model: flags.model,\n          timeout: flags.timeout,\n        });\n        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: grading as unknown as Record<string, unknown> });\n      } else {\n        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });\n      }\n    }\n\n    return results;\n  }\n\n  private computePassRate(results: Array<{ grading: Record<string, unknown> }>): number {\n    const graded = results.filter((r) => Object.keys(r.grading).length > 0);\n    if (graded.length === 0) return 0;\n    const rates = graded.map((r) => (r.grading as any).summary?.pass_rate ?? 0);\n    return rates.reduce((a: number, b: number) => a + b, 0) / rates.length;\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAwB5B,IAAqB,sBAArB,MAAqB,6BAA4B,YAAY;AAAA,EAC3D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,QAAQ,EAAE,aAAa,4DAA4D,SAAS,MAAM,CAAC;AAAA,IAC/G,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,WAAW,MAAM,OAAO,EAAE,aAAa,kDAAkD,SAAS,OAAO,CAAC;AAAA,IAC1G,OAAO,MAAM,OAAO,EAAE,aAAa,2BAA2B,CAAC;AAAA,IAC/D,kBAAkB,MAAM,OAAO,EAAE,aAAa,iCAAiC,CAAC;AAAA,IAChF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,+BAA+B,SAAS,EAAE,CAAC;AAAA,IACzF,UAAU,MAAM,QAAQ,EAAE,aAAa,sCAAsC,SAAS,EAAE,CAAC;AAAA,IACzF,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,oBAAmB;AAC5D,SAAK,YAAY;AAEjB,QAAI,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAC1E,UAAM,WAAW,MAAM,OAAe,qBAAqB,KAAK,QAAQ,QAAQ;AAEhF,QAAI,SAAS,WAAW,GAAG;AACzB,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAGA,UAAM,WAAW,CAAC,GAAG,QAAQ,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AAC7D,UAAM,WAAW,KAAK,KAAK,SAAS,SAAS,GAAG;AAChD,UAAM,aAAa,SAAS,MAAM,GAAG,QAAQ;AAC7C,UAAM,YAAY,SAAS,MAAM,QAAQ;AAEzC,SAAK,IAAI,eAAe,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzD,SAAK,IAAI,UAAU,WAAW,MAAM,WAAW,UAAU,MAAM,OAAO;AACtE,UAAM,YAAY,WAAW,MAAM,SAAS;AAC5C,SAAK,IAAI,eAAe,MAAM,gBAAgB,CAAC,2BAA2B,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AACxG,QAAI,MAAM,KAAM,MAAK,IAAI,oCAAoC;AAC7D,SAAK,IAAI,EAAE;AAEX,QAAI,cAAc,OAAO;AACzB,QAAI,gBAAgB;AACpB,QAAI,qBAAqB;AAEzB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAG3C,YAAM,SAAS,MAAM,QAAwB,0BAA0B;AAAA,QACrE,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,QAAQ;AAAA,MACV,CAAC;AAGD,WAAK,IAAI,wBAAwB;AACjC,YAAM,eAAe,MAAM,KAAK,iBAAiB,QAAQ,YAAY,KAAK;AAC1E,YAAM,aAAa,KAAK,gBAAgB,YAAY;AACpD,WAAK,IAAI,qBAAqB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG7D,UAAI,cAAc,WAAW;AAC3B,aAAK,IAAI,gBAAgB,aAAa,KAAK,QAAQ,CAAC,CAAC,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,cAAc;AAEjH,cAAMA,eAAc,MAAM,KAAK,iBAAiB,QAAQ,WAAW,KAAK;AACxE,cAAMC,aAAY,KAAK,gBAAgBD,YAAW;AAClD,aAAK,IAAI,sBAAsBC,aAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE7D,cAAM,QAAQ,0BAA0B;AAAA,UACtC,cAAc,OAAO;AAAA,UACrB,MAAM;AAAA,UACN;AAAA,UACA,cAAc,OAAO;AAAA,UACrB;AAAA,UACA,WAAAA;AAAA,UACA,UAAU,EAAE,QAAQ,oBAAoB;AAAA,UACxC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAGA,WAAK,IAAI,sBAAsB;AAC/B,YAAM,cAAc,aAAa,IAAI,CAAC,OAAO;AAAA,QAC3C,UAAU,EAAE;AAAA,QACZ,QAAQ,EAAE;AAAA,QACV,cAAc,EAAE;AAAA,QAChB,SAAS,EAAE;AAAA,MACb,EAAE;AAEF,YAAM,WAAW,MAAM,mBAAmB;AAAA,QACxC,SAAS,OAAO;AAAA,QAChB;AAAA,QACA,OAAO,MAAM,gBAAgB,KAAK,MAAM;AAAA,QACxC,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,WAAK,IAAI;AAAA,cAAiB,SAAS,iBAAiB,EAAE;AACtD,WAAK,IAAI,gBAAgB,SAAS,YAAY,MAAM,EAAE;AACtD,iBAAW,KAAK,SAAS,aAAa;AACpC,aAAK,IAAI,QAAQ,EAAE,OAAO,KAAK,EAAE,SAAS,EAAE;AAAA,MAC9C;AAGA,UAAI,CAAC,MAAM,MAAM;AACf,aAAK,IAAI,8EAA8E;AACvF,aAAK,IAAI,sEAAiE;AAAA,MAC5E;AAEA,UAAI,SAAS,oBAAoB,OAAO,SAAS;AAC/C,aAAK,IAAI,gCAAgC;AACzC,cAAM,QAAQ,0BAA0B;AAAA,UACtC,cAAc,OAAO;AAAA,UACrB,MAAM;AAAA,UACN;AAAA,UACA,cAAc,OAAO;AAAA,UACrB;AAAA,UACA,UAAU,EAAE,QAAQ,aAAa;AAAA,UACjC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAGA,YAAM,YAAY,MAAM,OAAmB,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC3E,SAAS,SAAS;AAAA,MACpB,CAAC;AACD,WAAK,IAAI,YAAY,UAAU,OAAO,EAAE;AAGxC,WAAK,IAAI,uBAAuB;AAChC,YAAM,cAAc,MAAM,KAAK,iBAAiB,WAAW,WAAW,KAAK;AAC3E,YAAM,YAAY,KAAK,gBAAgB,WAAW;AAClD,WAAK,IAAI,oBAAoB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG3D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,UAAU;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,eAAe,UAAU;AAAA,QACzB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,mBAAmB,SAAS;AAAA,QAC9B;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,sBAAc,UAAU;AACxB,6BAAqB;AAAA,MACvB,OAAO;AACL;AACA,YAAI,sBAAsB,GAAG;AAC3B,eAAK,IAAI,wDAAwD;AACjE;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAAA,IACX;AAEA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,kBAAkB,WAAW,kBAAkB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,IAAI;AAAA,EAC7F;AAAA,EAEA,MAAc,iBACZ,QACA,SACA,OACgH;AAChH,UAAM,UAAiH,CAAC;AAExH,eAAW,SAAS,SAAS;AAC3B,YAAM,aAAa,MAAM,YAAY;AAAA,QACnC,QAAQ,MAAM;AAAA,QACd,eAAe,OAAO;AAAA,QACtB,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,UAAI,WAAW,OAAO;AACpB,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,SAAS,CAAC,EAAE,CAAC;AAC1G;AAAA,MACF;AAEA,UAAI,MAAM,aAAa,SAAS,GAAG;AACjC,cAAM,UAAU,MAAM,gBAAgB;AAAA,UACpC,QAAQ,MAAM;AAAA,UACd,QAAQ,WAAW;AAAA,UACnB,cAAc,MAAM;AAAA,UACpB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,QAAuD,CAAC;AAAA,MACvJ,OAAO;AACL,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,SAAS,CAAC,EAAE,CAAC;AAAA,MAC5G;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,gBAAgB,SAA8D;AACpF,UAAM,SAAS,QAAQ,OAAO,CAAC,MAAM,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AACtE,QAAI,OAAO,WAAW,EAAG,QAAO;AAChC,UAAM,QAAQ,OAAO,IAAI,CAAC,MAAO,EAAE,QAAgB,SAAS,aAAa,CAAC;AAC1E,WAAO,MAAM,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,MAAM;AAAA,EAClE;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAwB5B,IAAqB,sBAArB,MAAqB,6BAA4B,YAAY;AAAA,EAC3D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,QAAQ,EAAE,aAAa,4DAA4D,SAAS,MAAM,CAAC;AAAA,IAC/G,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,WAAW,MAAM,OAAO,EAAE,aAAa,kDAAkD,SAAS,OAAO,CAAC;AAAA,IAC1G,OAAO,MAAM,OAAO,EAAE,aAAa,2BAA2B,CAAC;AAAA,IAC/D,kBAAkB,MAAM,OAAO,EAAE,aAAa,iCAAiC,CAAC;AAAA,IAChF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,+BAA+B,SAAS,EAAE,CAAC;AAAA,IACzF,UAAU,MAAM,QAAQ,EAAE,aAAa,sCAAsC,SAAS,EAAE,CAAC;AAAA,IACzF,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,oBAAmB;AAC5D,SAAK,YAAY;AAEjB,QAAI,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAC1E,UAAM,WAAW,MAAM,OAAe,qBAAqB,KAAK,QAAQ,QAAQ;AAEhF,QAAI,SAAS,WAAW,GAAG;AACzB,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAGA,UAAM,WAAW,CAAC,GAAG,QAAQ,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AAC7D,UAAM,WAAW,KAAK,KAAK,SAAS,SAAS,GAAG;AAChD,UAAM,aAAa,SAAS,MAAM,GAAG,QAAQ;AAC7C,UAAM,YAAY,SAAS,MAAM,QAAQ;AAEzC,SAAK,IAAI,eAAe,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzD,SAAK,IAAI,UAAU,WAAW,MAAM,WAAW,UAAU,MAAM,OAAO;AACtE,UAAM,YAAY,WAAW,MAAM,SAAS;AAC5C,SAAK,IAAI,eAAe,MAAM,gBAAgB,CAAC,2BAA2B,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AACxG,QAAI,MAAM,KAAM,MAAK,IAAI,oCAAoC;AAC7D,SAAK,IAAI,EAAE;AAEX,QAAI,cAAc,OAAO;AACzB,QAAI,gBAAgB;AACpB,QAAI,qBAAqB;AAEzB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAG3C,YAAM,SAAS,MAAM,QAAwB,0BAA0B;AAAA,QACrE,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,QAAQ;AAAA,MACV,CAAC;AAGD,WAAK,IAAI,wBAAwB;AACjC,YAAM,eAAe,MAAM,KAAK,iBAAiB,QAAQ,YAAY,KAAK;AAC1E,YAAM,aAAa,KAAK,gBAAgB,YAAY;AACpD,WAAK,IAAI,qBAAqB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG7D,UAAI,cAAc,WAAW;AAC3B,aAAK,IAAI,gBAAgB,aAAa,KAAK,QAAQ,CAAC,CAAC,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,cAAc;AAEjH,cAAMA,eAAc,MAAM,KAAK,iBAAiB,QAAQ,WAAW,KAAK;AACxE,cAAMC,aAAY,KAAK,gBAAgBD,YAAW;AAClD,aAAK,IAAI,sBAAsBC,aAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE7D,cAAM,QAAQ,0BAA0B;AAAA,UACtC,cAAc,OAAO;AAAA,UACrB,MAAM;AAAA,UACN;AAAA,UACA,cAAc,OAAO;AAAA,UACrB;AAAA,UACA,WAAAA;AAAA,UACA,UAAU,EAAE,QAAQ,oBAAoB;AAAA,UACxC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAGA,WAAK,IAAI,sBAAsB;AAC/B,YAAM,cAAc,aAAa,IAAI,CAAC,OAAO;AAAA,QAC3C,UAAU,EAAE;AAAA,QACZ,QAAQ,EAAE;AAAA,QACV,cAAc,EAAE;AAAA,QAChB,SAAS,EAAE;AAAA,MACb,EAAE;AAEF,YAAM,WAAW,MAAM,mBAAmB;AAAA,QACxC,SAAS,OAAO;AAAA,QAChB;AAAA,QACA,OAAO,MAAM,gBAAgB,KAAK,MAAM;AAAA,QACxC,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,WAAK,IAAI;AAAA,cAAiB,SAAS,iBAAiB,EAAE;AACtD,WAAK,IAAI,gBAAgB,SAAS,YAAY,MAAM,EAAE;AACtD,iBAAW,KAAK,SAAS,aAAa;AACpC,aAAK,IAAI,QAAQ,EAAE,OAAO,KAAK,EAAE,SAAS,EAAE;AAAA,MAC9C;AAGA,UAAI,CAAC,MAAM,MAAM;AACf,aAAK,IAAI,8EAA8E;AACvF,aAAK,IAAI,sEAAiE;AAAA,MAC5E;AAEA,UAAI,SAAS,oBAAoB,OAAO,SAAS;AAC/C,aAAK,IAAI,gCAAgC;AACzC,cAAM,QAAQ,0BAA0B;AAAA,UACtC,cAAc,OAAO;AAAA,UACrB,MAAM;AAAA,UACN;AAAA,UACA,cAAc,OAAO;AAAA,UACrB;AAAA,UACA,UAAU,EAAE,QAAQ,aAAa;AAAA,UACjC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAGA,YAAM,YAAY,MAAM,OAAmB,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC3E,SAAS,SAAS;AAAA,MACpB,CAAC;AACD,WAAK,IAAI,YAAY,UAAU,OAAO,EAAE;AAGxC,WAAK,IAAI,uBAAuB;AAChC,YAAM,cAAc,MAAM,KAAK,iBAAiB,WAAW,WAAW,KAAK;AAC3E,YAAM,YAAY,KAAK,gBAAgB,WAAW;AAClD,WAAK,IAAI,oBAAoB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG3D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,UAAU;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,eAAe,UAAU;AAAA,QACzB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,mBAAmB,SAAS;AAAA,QAC9B;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,sBAAc,UAAU;AACxB,6BAAqB;AAAA,MACvB,OAAO;AACL;AACA,YAAI,sBAAsB,GAAG;AAC3B,eAAK,IAAI,wDAAwD;AACjE;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAAA,IACX;AAEA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,kBAAkB,WAAW,kBAAkB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,IAAI;AAAA,EAC7F;AAAA,EAEA,MAAc,iBACZ,QACA,SACA,OACgH;AAChH,UAAM,UAAiH,CAAC;AAExH,eAAW,SAAS,SAAS;AAC3B,YAAM,aAAa,MAAM,YAAY;AAAA,QACnC,QAAQ,MAAM;AAAA,QACd,eAAe,OAAO;AAAA,QACtB,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,UAAI,WAAW,OAAO;AACpB,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,SAAS,CAAC,EAAE,CAAC;AAC1G;AAAA,MACF;AAEA,UAAI,MAAM,aAAa,SAAS,GAAG;AACjC,cAAM,UAAU,MAAM,gBAAgB;AAAA,UACpC,QAAQ,MAAM;AAAA,UACd,QAAQ,WAAW;AAAA,UACnB,cAAc,MAAM;AAAA,UACpB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,QAAuD,CAAC;AAAA,MACvJ,OAAO;AACL,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,SAAS,CAAC,EAAE,CAAC;AAAA,MAC5G;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,gBAAgB,SAA8D;AACpF,UAAM,SAAS,QAAQ,OAAO,CAAC,MAAM,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AACtE,QAAI,OAAO,WAAW,EAAG,QAAO;AAChC,UAAM,QAAQ,OAAO,IAAI,CAAC,MAAO,EAAE,QAAgB,SAAS,aAAa,CAAC;AAC1E,WAAO,MAAM,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,MAAM;AAAA,EAClE;AACF;",
   "names": ["testResults", "testScore"]
 }

package/dist/commands/evals/run.js

@@ -6,14 +6,15 @@ import {
 } from "../../chunk-QN34TGD3.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiGet,
   apiPost
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/run.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/run.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet, apiPost } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\nimport { gradeEvalOutput } from '../../lib/eval-grader.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  type: string;\n  content: string;\n  version: number;\n}\n\ninterface Eval {\n  id: string;\n  name: string;\n  prompt: string;\n  expectedOutput: string;\n  expectations: string[];\n}\n\ninterface EvalRunResult {\n  evalId: string;\n  evalTargetId: string;\n  evalTargetVersion: number;\n  configuration: string;\n  runNumber: number;\n  iteration: number;\n  status: string;\n  outputs: Record<string, unknown>;\n  timing: Record<string, unknown>;\n  grading: Record<string, unknown>;\n}\n\nexport default class EvalRunCommand extends BaseCommand {\n  static override description = 'Run evals against a target, optionally grade results, and post to API';\n\n  static override examples = [\n    '<%= config.bin %> evals run <target-id>',\n    '<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2',\n    '<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    parallel: Flags.integer({ description: 'Number of parallel eval executions', default: 1 }),\n    'runs-per-eval': Flags.integer({ description: 'Number of runs per eval for variance analysis', default: 1 }),\n    iteration: Flags.integer({ description: 'Iteration number for tracking', default: 1 }),\n    'skip-baseline': Flags.boolean({ description: 'Skip baseline (without_target) runs', default: false }),\n    'skip-grading': Flags.boolean({ description: 'Skip LLM grading step', default: false }),\n    model: Flags.string({ description: 'Model for eval execution (default: user configured)' }),\n    'grader-model': Flags.string({ description: 'Model for grading (default: same as execution model)' }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalRunCommand);\n    this.requireAuth();\n\n    // Fetch target and its evals\n    const target = await apiGet<EvalTarget>(`/api/eval-targets/${args.targetId}`);\n    const evals = await apiGet<Eval[]>(`/api/eval-targets/${args.targetId}/evals`);\n\n    if (evals.length === 0) {\n      this.error(`No evals found for target \"${target.name}\". Create some with \\`rulemetric evals create\\`.`);\n    }\n\n    this.log(`Running ${evals.length} eval(s) for \"${target.name}\" v${target.version}`);\n    this.log(`Config: ${flags['runs-per-eval']} run(s) per eval, parallel=${flags.parallel}, iteration=${flags.iteration}`);\n    if (flags['skip-baseline']) this.log('Skipping baseline runs');\n    if (flags['skip-grading']) this.log('Skipping grading');\n    this.log('');\n\n    const configurations: Array<{ name: string; content?: string }> = [\n      { name: 'with_target', content: target.content },\n    ];\n    if (!flags['skip-baseline']) {\n      configurations.push({ name: 'without_target' });\n    }\n\n    // Build the work queue\n    interface WorkItem {\n      eval_: Eval;\n      configuration: string;\n      targetContent?: string;\n      runNumber: number;\n    }\n    const workQueue: WorkItem[] = [];\n\n    for (const eval_ of evals) {\n      for (const config of configurations) {\n        for (let run = 1; run <= flags['runs-per-eval']; run++) {\n          workQueue.push({\n            eval_: eval_,\n            configuration: config.name,\n            targetContent: config.content,\n            runNumber: run,\n          });\n        }\n      }\n    }\n\n    // Execute with concurrency control\n    const results: EvalRunResult[] = [];\n    const concurrency = flags.parallel;\n\n    for (let i = 0; i < workQueue.length; i += concurrency) {\n      const batch = workQueue.slice(i, i + concurrency);\n      const batchResults = await Promise.all(\n        batch.map((item) => this.executeAndGrade(item, target, flags)),\n      );\n      results.push(...batchResults);\n    }\n\n    // Post results to API\n    this.log('');\n    this.log('Posting results to API...');\n\n    for (const result of results) {\n      await apiPost('/api/eval-runs', result);\n    }\n\n    // Summary\n    this.log('');\n    this.log('=== Results Summary ===');\n\n    for (const config of configurations) {\n      const configResults = results.filter((r) => r.configuration === config.name);\n      const completedRuns = configResults.filter((r) => r.status === 'completed');\n      const gradedRuns = completedRuns.filter((r) => Object.keys(r.grading).length > 0);\n\n      this.log(`\\n${config.name}:`);\n      this.log(`  Runs: ${completedRuns.length}/${configResults.length} completed`);\n\n      if (gradedRuns.length > 0) {\n        const passRates = gradedRuns\n          .map((r) => (r.grading as Record<string, any>).summary?.pass_rate ?? 0);\n        const avgPassRate = passRates.reduce((a: number, b: number) => a + b, 0) / passRates.length;\n        this.log(`  Avg pass_rate: ${(avgPassRate * 100).toFixed(1)}%`);\n      }\n\n      for (const r of configResults) {\n        const evalName = evals.find((e) => e.id === r.evalId)?.name ?? r.evalId;\n        const passRate = (r.grading as Record<string, any>).summary?.pass_rate;\n        const passRateStr = passRate !== undefined ? ` pass_rate=${(passRate * 100).toFixed(0)}%` : '';\n        const timeStr = (r.timing as Record<string, any>).duration_ms ? ` ${((r.timing as Record<string, any>).duration_ms / 1000).toFixed(1)}s` : '';\n        this.log(`  [${r.status}] ${evalName} run#${r.runNumber}${passRateStr}${timeStr}`);\n      }\n    }\n\n    if (!flags['skip-baseline'] && results.length > 0) {\n      const withTarget = results.filter((r) => r.configuration === 'with_target' && Object.keys(r.grading).length > 0);\n      const withoutTarget = results.filter((r) => r.configuration === 'without_target' && Object.keys(r.grading).length > 0);\n\n      if (withTarget.length > 0 && withoutTarget.length > 0) {\n        const avgWith = withTarget.map((r) => (r.grading as Record<string, any>).summary?.pass_rate ?? 0).reduce((a: number, b: number) => a + b, 0) / withTarget.length;\n        const avgWithout = withoutTarget.map((r) => (r.grading as Record<string, any>).summary?.pass_rate ?? 0).reduce((a: number, b: number) => a + b, 0) / withoutTarget.length;\n        const delta = avgWith - avgWithout;\n        this.log(`\\nDelta: ${delta > 0 ? '+' : ''}${(delta * 100).toFixed(1)}% (with_target vs without_target)`);\n      }\n    }\n  }\n\n  private async executeAndGrade(\n    item: { eval_: Eval; configuration: string; targetContent?: string; runNumber: number },\n    target: EvalTarget,\n    flags: { 'skip-grading': boolean; model?: string; 'grader-model'?: string; timeout: number; iteration: number },\n  ): Promise<EvalRunResult> {\n    const { eval_, configuration, targetContent, runNumber } = item;\n\n    this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 executing...`);\n\n    // Execute the eval\n    const execResult = await executeEval({\n      prompt: eval_.prompt,\n      targetContent,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    if (execResult.error) {\n      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 FAILED: ${execResult.error}`);\n      return {\n        evalId: eval_.id,\n        evalTargetId: target.id,\n        evalTargetVersion: target.version,\n        configuration,\n        runNumber,\n        iteration: flags.iteration,\n        status: 'failed',\n        outputs: { error: execResult.error },\n        timing: { duration_ms: execResult.durationMs, total_tokens: execResult.totalTokens },\n        grading: {},\n      };\n    }\n\n    this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 completed in ${(execResult.durationMs / 1000).toFixed(1)}s`);\n\n    // Grade if not skipped and expectations exist\n    let grading: Record<string, unknown> = {};\n\n    if (!flags['skip-grading'] && eval_.expectations.length > 0) {\n      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 grading...`);\n\n      const gradingResult = await gradeEvalOutput({\n        prompt: eval_.prompt,\n        output: execResult.output,\n        expectations: eval_.expectations,\n        model: flags['grader-model'] ?? flags.model,\n        timeout: flags.timeout,\n      });\n\n      grading = gradingResult as unknown as Record<string, unknown>;\n      const passRate = gradingResult.summary.pass_rate;\n      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 grading done: pass_rate=${(passRate * 100).toFixed(0)}%`);\n    }\n\n    return {\n      evalId: eval_.id,\n      evalTargetId: target.id,\n      evalTargetVersion: target.version,\n      configuration,\n      runNumber,\n      iteration: flags.iteration,\n      status: 'completed',\n      outputs: { transcript: execResult.output },\n      timing: {\n        duration_ms: execResult.durationMs,\n        total_tokens: execResult.totalTokens,\n        total_duration_seconds: execResult.durationMs / 1000,\n      },\n      grading,\n    };\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmC5B,IAAqB,iBAArB,MAAqB,wBAAuB,YAAY;AAAA,EACtD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,UAAU,MAAM,QAAQ,EAAE,aAAa,sCAAsC,SAAS,EAAE,CAAC;AAAA,IACzF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,iDAAiD,SAAS,EAAE,CAAC;AAAA,IAC3G,WAAW,MAAM,QAAQ,EAAE,aAAa,iCAAiC,SAAS,EAAE,CAAC;AAAA,IACrF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,uCAAuC,SAAS,MAAM,CAAC;AAAA,IACrG,gBAAgB,MAAM,QAAQ,EAAE,aAAa,yBAAyB,SAAS,MAAM,CAAC;AAAA,IACtF,OAAO,MAAM,OAAO,EAAE,aAAa,sDAAsD,CAAC;AAAA,IAC1F,gBAAgB,MAAM,OAAO,EAAE,aAAa,uDAAuD,CAAC;AAAA,IACpG,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,eAAc;AACvD,SAAK,YAAY;AAGjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAC5E,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,QAAQ,QAAQ;AAE7E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,8BAA8B,OAAO,IAAI,kDAAkD;AAAA,IACxG;AAEA,SAAK,IAAI,WAAW,MAAM,MAAM,iBAAiB,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AAClF,SAAK,IAAI,WAAW,MAAM,eAAe,CAAC,8BAA8B,MAAM,QAAQ,eAAe,MAAM,SAAS,EAAE;AACtH,QAAI,MAAM,eAAe,EAAG,MAAK,IAAI,wBAAwB;AAC7D,QAAI,MAAM,cAAc,EAAG,MAAK,IAAI,kBAAkB;AACtD,SAAK,IAAI,EAAE;AAEX,UAAM,iBAA4D;AAAA,MAChE,EAAE,MAAM,eAAe,SAAS,OAAO,QAAQ;AAAA,IACjD;AACA,QAAI,CAAC,MAAM,eAAe,GAAG;AAC3B,qBAAe,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAAA,IAChD;AASA,UAAM,YAAwB,CAAC;AAE/B,eAAW,SAAS,OAAO;AACzB,iBAAW,UAAU,gBAAgB;AACnC,iBAAS,MAAM,GAAG,OAAO,MAAM,eAAe,GAAG,OAAO;AACtD,oBAAU,KAAK;AAAA,YACb;AAAA,YACA,eAAe,OAAO;AAAA,YACtB,eAAe,OAAO;AAAA,YACtB,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,UAAM,UAA2B,CAAC;AAClC,UAAM,cAAc,MAAM;AAE1B,aAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,aAAa;AACtD,YAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,WAAW;AAChD,YAAM,eAAe,MAAM,QAAQ;AAAA,QACjC,MAAM,IAAI,CAAC,SAAS,KAAK,gBAAgB,MAAM,QAAQ,KAAK,CAAC;AAAA,MAC/D;AACA,cAAQ,KAAK,GAAG,YAAY;AAAA,IAC9B;AAGA,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,2BAA2B;AAEpC,eAAW,UAAU,SAAS;AAC5B,YAAM,QAAQ,kBAAkB,MAAM;AAAA,IACxC;AAGA,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,yBAAyB;AAElC,eAAW,UAAU,gBAAgB;AACnC,YAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,OAAO,IAAI;AAC3E,YAAM,gBAAgB,cAAc,OAAO,CAAC,MAAM,EAAE,WAAW,WAAW;AAC1E,YAAM,aAAa,cAAc,OAAO,CAAC,MAAM,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAEhF,WAAK,IAAI;AAAA,EAAK,OAAO,IAAI,GAAG;AAC5B,WAAK,IAAI,WAAW,cAAc,MAAM,IAAI,cAAc,MAAM,YAAY;AAE5E,UAAI,WAAW,SAAS,GAAG;AACzB,cAAM,YAAY,WACf,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC;AACxE,cAAM,cAAc,UAAU,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,UAAU;AACrF,aAAK,IAAI,qBAAqB,cAAc,KAAK,QAAQ,CAAC,CAAC,GAAG;AAAA,MAChE;AAEA,iBAAW,KAAK,eAAe;AAC7B,cAAM,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE;AACjE,cAAM,WAAY,EAAE,QAAgC,SAAS;AAC7D,cAAM,cAAc,aAAa,SAAY,eAAe,WAAW,KAAK,QAAQ,CAAC,CAAC,MAAM;AAC5F,cAAM,UAAW,EAAE,OAA+B,cAAc,KAAM,EAAE,OAA+B,cAAc,KAAM,QAAQ,CAAC,CAAC,MAAM;AAC3I,aAAK,IAAI,MAAM,EAAE,MAAM,KAAK,QAAQ,QAAQ,EAAE,SAAS,GAAG,WAAW,GAAG,OAAO,EAAE;AAAA,MACnF;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,eAAe,KAAK,QAAQ,SAAS,GAAG;AACjD,YAAM,aAAa,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,iBAAiB,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAC/G,YAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,oBAAoB,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAErH,UAAI,WAAW,SAAS,KAAK,cAAc,SAAS,GAAG;AACrD,cAAM,UAAU,WAAW,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC,EAAE,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,WAAW;AAC1J,cAAM,aAAa,cAAc,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC,EAAE,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,cAAc;AACnK,cAAM,QAAQ,UAAU;AACxB,aAAK,IAAI;AAAA,SAAY,QAAQ,IAAI,MAAM,EAAE,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC,mCAAmC;AAAA,MACzG;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,gBACZ,MACA,QACA,OACwB;AACxB,UAAM,EAAE,OAAO,eAAe,eAAe,UAAU,IAAI;AAE3D,SAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,sBAAiB;AAG3E,UAAM,aAAa,MAAM,YAAY;AAAA,MACnC,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI,WAAW,OAAO;AACpB,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,mBAAc,WAAW,KAAK,EAAE;AAC1F,aAAO;AAAA,QACL,QAAQ,MAAM;AAAA,QACd,cAAc,OAAO;AAAA,QACrB,mBAAmB,OAAO;AAAA,QAC1B;AAAA,QACA;AAAA,QACA,WAAW,MAAM;AAAA,QACjB,QAAQ;AAAA,QACR,SAAS,EAAE,OAAO,WAAW,MAAM;AAAA,QACnC,QAAQ,EAAE,aAAa,WAAW,YAAY,cAAc,WAAW,YAAY;AAAA,QACnF,SAAS,CAAC;AAAA,MACZ;AAAA,IACF;AAEA,SAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,yBAAoB,WAAW,aAAa,KAAM,QAAQ,CAAC,CAAC,GAAG;AAGzH,QAAI,UAAmC,CAAC;AAExC,QAAI,CAAC,MAAM,cAAc,KAAK,MAAM,aAAa,SAAS,GAAG;AAC3D,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,oBAAe;AAEzE,YAAM,gBAAgB,MAAM,gBAAgB;AAAA,QAC1C,QAAQ,MAAM;AAAA,QACd,QAAQ,WAAW;AAAA,QACnB,cAAc,MAAM;AAAA,QACpB,OAAO,MAAM,cAAc,KAAK,MAAM;AAAA,QACtC,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,gBAAU;AACV,YAAM,WAAW,cAAc,QAAQ;AACvC,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,oCAA+B,WAAW,KAAK,QAAQ,CAAC,CAAC,GAAG;AAAA,IACxH;AAEA,WAAO;AAAA,MACL,QAAQ,MAAM;AAAA,MACd,cAAc,OAAO;AAAA,MACrB,mBAAmB,OAAO;AAAA,MAC1B;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,QAAQ;AAAA,MACR,SAAS,EAAE,YAAY,WAAW,OAAO;AAAA,MACzC,QAAQ;AAAA,QACN,aAAa,WAAW;AAAA,QACxB,cAAc,WAAW;AAAA,QACzB,wBAAwB,WAAW,aAAa;AAAA,MAClD;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmC5B,IAAqB,iBAArB,MAAqB,wBAAuB,YAAY;AAAA,EACtD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,UAAU,MAAM,QAAQ,EAAE,aAAa,sCAAsC,SAAS,EAAE,CAAC;AAAA,IACzF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,iDAAiD,SAAS,EAAE,CAAC;AAAA,IAC3G,WAAW,MAAM,QAAQ,EAAE,aAAa,iCAAiC,SAAS,EAAE,CAAC;AAAA,IACrF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,uCAAuC,SAAS,MAAM,CAAC;AAAA,IACrG,gBAAgB,MAAM,QAAQ,EAAE,aAAa,yBAAyB,SAAS,MAAM,CAAC;AAAA,IACtF,OAAO,MAAM,OAAO,EAAE,aAAa,sDAAsD,CAAC;AAAA,IAC1F,gBAAgB,MAAM,OAAO,EAAE,aAAa,uDAAuD,CAAC;AAAA,IACpG,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,eAAc;AACvD,SAAK,YAAY;AAGjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAC5E,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,QAAQ,QAAQ;AAE7E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,8BAA8B,OAAO,IAAI,kDAAkD;AAAA,IACxG;AAEA,SAAK,IAAI,WAAW,MAAM,MAAM,iBAAiB,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AAClF,SAAK,IAAI,WAAW,MAAM,eAAe,CAAC,8BAA8B,MAAM,QAAQ,eAAe,MAAM,SAAS,EAAE;AACtH,QAAI,MAAM,eAAe,EAAG,MAAK,IAAI,wBAAwB;AAC7D,QAAI,MAAM,cAAc,EAAG,MAAK,IAAI,kBAAkB;AACtD,SAAK,IAAI,EAAE;AAEX,UAAM,iBAA4D;AAAA,MAChE,EAAE,MAAM,eAAe,SAAS,OAAO,QAAQ;AAAA,IACjD;AACA,QAAI,CAAC,MAAM,eAAe,GAAG;AAC3B,qBAAe,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAAA,IAChD;AASA,UAAM,YAAwB,CAAC;AAE/B,eAAW,SAAS,OAAO;AACzB,iBAAW,UAAU,gBAAgB;AACnC,iBAAS,MAAM,GAAG,OAAO,MAAM,eAAe,GAAG,OAAO;AACtD,oBAAU,KAAK;AAAA,YACb;AAAA,YACA,eAAe,OAAO;AAAA,YACtB,eAAe,OAAO;AAAA,YACtB,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,UAAM,UAA2B,CAAC;AAClC,UAAM,cAAc,MAAM;AAE1B,aAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,aAAa;AACtD,YAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,WAAW;AAChD,YAAM,eAAe,MAAM,QAAQ;AAAA,QACjC,MAAM,IAAI,CAAC,SAAS,KAAK,gBAAgB,MAAM,QAAQ,KAAK,CAAC;AAAA,MAC/D;AACA,cAAQ,KAAK,GAAG,YAAY;AAAA,IAC9B;AAGA,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,2BAA2B;AAEpC,eAAW,UAAU,SAAS;AAC5B,YAAM,QAAQ,kBAAkB,MAAM;AAAA,IACxC;AAGA,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,yBAAyB;AAElC,eAAW,UAAU,gBAAgB;AACnC,YAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,OAAO,IAAI;AAC3E,YAAM,gBAAgB,cAAc,OAAO,CAAC,MAAM,EAAE,WAAW,WAAW;AAC1E,YAAM,aAAa,cAAc,OAAO,CAAC,MAAM,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAEhF,WAAK,IAAI;AAAA,EAAK,OAAO,IAAI,GAAG;AAC5B,WAAK,IAAI,WAAW,cAAc,MAAM,IAAI,cAAc,MAAM,YAAY;AAE5E,UAAI,WAAW,SAAS,GAAG;AACzB,cAAM,YAAY,WACf,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC;AACxE,cAAM,cAAc,UAAU,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,UAAU;AACrF,aAAK,IAAI,qBAAqB,cAAc,KAAK,QAAQ,CAAC,CAAC,GAAG;AAAA,MAChE;AAEA,iBAAW,KAAK,eAAe;AAC7B,cAAM,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE;AACjE,cAAM,WAAY,EAAE,QAAgC,SAAS;AAC7D,cAAM,cAAc,aAAa,SAAY,eAAe,WAAW,KAAK,QAAQ,CAAC,CAAC,MAAM;AAC5F,cAAM,UAAW,EAAE,OAA+B,cAAc,KAAM,EAAE,OAA+B,cAAc,KAAM,QAAQ,CAAC,CAAC,MAAM;AAC3I,aAAK,IAAI,MAAM,EAAE,MAAM,KAAK,QAAQ,QAAQ,EAAE,SAAS,GAAG,WAAW,GAAG,OAAO,EAAE;AAAA,MACnF;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,eAAe,KAAK,QAAQ,SAAS,GAAG;AACjD,YAAM,aAAa,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,iBAAiB,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAC/G,YAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,oBAAoB,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAErH,UAAI,WAAW,SAAS,KAAK,cAAc,SAAS,GAAG;AACrD,cAAM,UAAU,WAAW,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC,EAAE,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,WAAW;AAC1J,cAAM,aAAa,cAAc,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC,EAAE,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,cAAc;AACnK,cAAM,QAAQ,UAAU;AACxB,aAAK,IAAI;AAAA,SAAY,QAAQ,IAAI,MAAM,EAAE,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC,mCAAmC;AAAA,MACzG;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,gBACZ,MACA,QACA,OACwB;AACxB,UAAM,EAAE,OAAO,eAAe,eAAe,UAAU,IAAI;AAE3D,SAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,sBAAiB;AAG3E,UAAM,aAAa,MAAM,YAAY;AAAA,MACnC,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI,WAAW,OAAO;AACpB,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,mBAAc,WAAW,KAAK,EAAE;AAC1F,aAAO;AAAA,QACL,QAAQ,MAAM;AAAA,QACd,cAAc,OAAO;AAAA,QACrB,mBAAmB,OAAO;AAAA,QAC1B;AAAA,QACA;AAAA,QACA,WAAW,MAAM;AAAA,QACjB,QAAQ;AAAA,QACR,SAAS,EAAE,OAAO,WAAW,MAAM;AAAA,QACnC,QAAQ,EAAE,aAAa,WAAW,YAAY,cAAc,WAAW,YAAY;AAAA,QACnF,SAAS,CAAC;AAAA,MACZ;AAAA,IACF;AAEA,SAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,yBAAoB,WAAW,aAAa,KAAM,QAAQ,CAAC,CAAC,GAAG;AAGzH,QAAI,UAAmC,CAAC;AAExC,QAAI,CAAC,MAAM,cAAc,KAAK,MAAM,aAAa,SAAS,GAAG;AAC3D,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,oBAAe;AAEzE,YAAM,gBAAgB,MAAM,gBAAgB;AAAA,QAC1C,QAAQ,MAAM;AAAA,QACd,QAAQ,WAAW;AAAA,QACnB,cAAc,MAAM;AAAA,QACpB,OAAO,MAAM,cAAc,KAAK,MAAM;AAAA,QACtC,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,gBAAU;AACV,YAAM,WAAW,cAAc,QAAQ;AACvC,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,oCAA+B,WAAW,KAAK,QAAQ,CAAC,CAAC,GAAG;AAAA,IACxH;AAEA,WAAO;AAAA,MACL,QAAQ,MAAM;AAAA,MACd,cAAc,OAAO;AAAA,MACrB,mBAAmB,OAAO;AAAA,MAC1B;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,QAAQ;AAAA,MACR,SAAS,EAAE,YAAY,WAAW,OAAO;AAAA,MACzC,QAAQ;AAAA,QACN,aAAa,WAAW;AAAA,QACxB,cAAc,WAAW;AAAA,QACzB,wBAAwB,WAAW,aAAa;AAAA,MAClD;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;",
   "names": []
 }