@graphorin/observability 0.6.0 → 0.7.0

package/src/eval/index.ts

@@ -0,0 +1,16 @@
+/**
+ * Minimal inline eval runner.
+ *
+ * @packageDocumentation
+ */
+
+export { runEval } from './runner.js';
+export type {
+  Case,
+  Dataset,
+  EvalCaseResult,
+  EvalReport,
+  RunEvalOptions,
+  ScoreResult,
+  Scorer,
+} from './types.js';

package/src/eval/runner.ts

@@ -0,0 +1,146 @@
+/**
+ * `runEval(...)` - minimal inline eval runner. Walks every case in
+ * the dataset, calls `agent.run(case.input)`, applies every supplied
+ * scorer, and returns an aggregated {@link EvalReport}.
+ *
+ * The runner is deliberately tiny - it has no parallelism, no
+ * retries, no reporters, no dataset loaders. Production scenarios
+ * that need any of those should `pnpm add @graphorin/evals` (post-MVP).
+ *
+ * @packageDocumentation
+ */
+
+import type {
+  Case,
+  EvalCaseResult,
+  EvalReport,
+  RunEvalOptions,
+  ScoreResult,
+  Scorer,
+} from './types.js';
+
+/**
+ * @stable
+ */
+export async function runEval<I, O>(opts: RunEvalOptions<I, O>): Promise<EvalReport<I, O>> {
+  const iterations = Math.max(1, opts.iterations ?? 1);
+  const signal = opts.signal;
+  const results: EvalCaseResult<I, O>[] = [];
+
+  for (let iter = 0; iter < iterations; iter++) {
+    for (let idx = 0; idx < opts.dataset.cases.length; idx++) {
+      throwIfAborted(signal);
+      const sampleCase = opts.dataset.cases[idx];
+      if (sampleCase === undefined) continue;
+      // EB-6: disambiguate a caller-provided id per iteration too, not just the
+      // synthetic fallback - otherwise iterations>1 emits duplicate caseIds.
+      const baseId = sampleCase.id ?? `case-${idx}`;
+      const caseId = iterations === 1 ? baseId : `${baseId}-iter-${iter}`;
+
+      const startedAt = Date.now();
+      const output = await opts.agent.run(sampleCase.input);
+      const durationMs = Date.now() - startedAt;
+
+      const scores: EvalCaseResult<I, O>['scores'][number][] = [];
+      for (const scorer of opts.scorers) {
+        throwIfAborted(signal);
+        const result = await safeScore(scorer, sampleCase, output, durationMs);
+        scores.push({ scorer: scorer.name, result });
+      }
+
+      results.push({
+        caseId,
+        input: sampleCase.input,
+        output,
+        durationMs,
+        scores,
+      });
+    }
+  }
+
+  return summarize(results, opts.scorers);
+}
+
+async function safeScore<I, O>(
+  scorer: Scorer<I, O>,
+  c: Case<I, O>,
+  output: O,
+  durationMs: number,
+): Promise<ScoreResult> {
+  try {
+    return await scorer.score({ case: c, output, durationMs });
+  } catch (err) {
+    return {
+      pass: false,
+      reason: `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`,
+    };
+  }
+}
+
+function summarize<I, O>(
+  results: ReadonlyArray<EvalCaseResult<I, O>>,
+  scorers: ReadonlyArray<Scorer<I, O>>,
+): EvalReport<I, O> {
+  const total = results.length;
+  let passed = 0;
+  let failed = 0;
+  let durationSum = 0;
+  const byScorer: Record<
+    string,
+    { passed: number; failed: number; scoreSum: number; scoreCount: number }
+  > = {};
+
+  for (const scorer of scorers) {
+    byScorer[scorer.name] = { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };
+  }
+
+  for (const r of results) {
+    durationSum += r.durationMs;
+    let passEntire = true;
+    for (const { scorer, result } of r.scores) {
+      const bucket = byScorer[scorer] ?? { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };
+      if (result.pass) bucket.passed += 1;
+      else {
+        bucket.failed += 1;
+        passEntire = false;
+      }
+      if (typeof result.score === 'number' && Number.isFinite(result.score)) {
+        bucket.scoreSum += result.score;
+        bucket.scoreCount += 1;
+      }
+      byScorer[scorer] = bucket;
+    }
+    if (passEntire) passed += 1;
+    else failed += 1;
+  }
+
+  const summaryByScorer: Record<
+    string,
+    { passed: number; failed: number; avgScore: number | null }
+  > = {};
+  for (const [scorer, b] of Object.entries(byScorer)) {
+    summaryByScorer[scorer] = {
+      passed: b.passed,
+      failed: b.failed,
+      avgScore: b.scoreCount === 0 ? null : b.scoreSum / b.scoreCount,
+    };
+  }
+
+  return {
+    results,
+    summary: {
+      total,
+      passed,
+      failed,
+      avgDurationMs: total === 0 ? 0 : durationSum / total,
+      byScorer: Object.freeze(summaryByScorer),
+    },
+  };
+}
+
+function throwIfAborted(signal: AbortSignal | undefined): void {
+  if (signal?.aborted === true) {
+    const reason = signal.reason;
+    throw reason instanceof Error ? reason : new Error('Eval run aborted');
+  }
+}

package/src/eval/types.ts

@@ -0,0 +1,122 @@
+/**
+ * Public types for the minimal inline eval runner. Full orchestrator,
+ * scorer libraries, dataset loaders, and reporters live in the
+ * separate `@graphorin/evals` package (post-MVP).
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * One sample from an eval dataset.
+ *
+ * @stable
+ */
+export interface Case<I, O = unknown, M = Readonly<Record<string, unknown>>> {
+  readonly id?: string;
+  readonly input: I;
+  readonly expected?: O;
+  readonly metadata?: M;
+}
+
+/**
+ * @stable
+ */
+export interface Dataset<I, O = unknown, M = Readonly<Record<string, unknown>>> {
+  readonly cases: ReadonlyArray<Case<I, O, M>>;
+  readonly metadata?: {
+    readonly name?: string;
+    readonly description?: string;
+    readonly createdAt?: Date;
+  };
+}
+
+/**
+ * Output of {@link Scorer.score}.
+ *
+ * @stable
+ */
+export interface ScoreResult {
+  readonly pass: boolean;
+  /** Optional normalized score in `[0, 1]`. */
+  readonly score?: number;
+  readonly reason?: string;
+  readonly metadata?: Readonly<Record<string, unknown>>;
+}
+
+/**
+ * @stable
+ */
+export interface Scorer<I, O = unknown> {
+  readonly name: string;
+  score(args: {
+    readonly case: Case<I, O>;
+    readonly output: O;
+    readonly durationMs: number;
+  }): Promise<ScoreResult>;
+}
+
+/**
+ * Per-case result.
+ *
+ * @stable
+ */
+export interface EvalCaseResult<I, O> {
+  readonly caseId: string;
+  readonly input: I;
+  readonly output: O;
+  readonly durationMs: number;
+  readonly scores: ReadonlyArray<{ readonly scorer: string; readonly result: ScoreResult }>;
+}
+
+/**
+ * Final report shape.
+ *
+ * @stable
+ */
+export interface EvalReport<I, O> {
+  readonly results: ReadonlyArray<EvalCaseResult<I, O>>;
+  readonly summary: {
+    readonly total: number;
+    readonly passed: number;
+    readonly failed: number;
+    readonly avgDurationMs: number;
+    readonly byScorer: Readonly<
+      Record<
+        string,
+        { readonly passed: number; readonly failed: number; readonly avgScore: number | null }
+      >
+    >;
+    /**
+     * 95% Wilson score interval on the overall pass rate (E8 / evals-05).
+     * Always present on reports produced by `runEvals`; optional so older
+     * persisted reports keep parsing.
+     */
+    readonly passRateCi?: { readonly lo: number; readonly hi: number };
+    /**
+     * pass^k stability metric - fraction of base cases whose EVERY repeat
+     * iteration passed. Present only when the run used `iterations > 1`.
+     */
+    readonly passHatK?: {
+      readonly k: number;
+      readonly baseCases: number;
+      readonly value: number;
+    };
+  };
+  /**
+   * `true` when the run was cut short by an aborted signal - `results` and
+   * `summary` then cover only the cases that finished before the abort (a
+   * partial report). Absent on a normal full run. See `runEvals`.
+   */
+  readonly aborted?: boolean;
+}
+
+/**
+ * @stable
+ */
+export interface RunEvalOptions<I, O> {
+  readonly agent: { readonly run: (input: I) => Promise<O> };
+  readonly dataset: Dataset<I, O>;
+  readonly scorers: ReadonlyArray<Scorer<I, O>>;
+  readonly iterations?: number;
+  readonly signal?: AbortSignal;
+}

package/src/exporters/console.ts

@@ -0,0 +1,75 @@
+/**
+ * `ConsoleExporter` - pretty-prints finished spans to `console.log`.
+ *
+ * Useful for development and unit testing. Production deployments
+ * should use {@link JSONLExporter} (for replay) or
+ * {@link OTLPHttpExporter} (for remote OTLP collectors).
+ *
+ * @packageDocumentation
+ */
+
+import type { SpanRecord, TraceExporter } from './types.js';
+
+/**
+ * Configuration shape for {@link createConsoleExporter}.
+ *
+ * @stable
+ */
+export interface ConsoleExporterOptions {
+  /** Identifier reported via `exporter.id`. Defaults to `'console'`. */
+  readonly id?: string;
+  /** When `true`, emit JSON pretty-printed across multiple lines. */
+  readonly pretty?: boolean;
+  /** Custom sink. Defaults to `console.log`. */
+  readonly sink?: (line: string) => void;
+}
+
+/**
+ * Build a console-based trace exporter. Call `withValidation(exporter)`
+ * before passing the result to `createTracer({ exporters })`.
+ *
+ * @stable
+ */
+export function createConsoleExporter(opts: ConsoleExporterOptions = {}): TraceExporter {
+  const id = opts.id ?? 'console';
+  const pretty = opts.pretty === true;
+  const sink = opts.sink ?? ((line: string) => console.log(line));
+
+  let closed = false;
+  return {
+    id,
+    async export(record: SpanRecord): Promise<void> {
+      if (closed) return;
+      const payload = pretty
+        ? JSON.stringify(serializableRecord(record), null, 2)
+        : JSON.stringify(serializableRecord(record));
+      sink(payload);
+    },
+    async flush(): Promise<void> {
+      // Console.log is synchronous on Node; nothing to flush.
+    },
+    async shutdown(): Promise<void> {
+      closed = true;
+    },
+  };
+}
+
+/**
+ * @internal
+ */
+export function serializableRecord(record: SpanRecord): Record<string, unknown> {
+  return {
+    type: record.type,
+    id: record.id,
+    traceId: record.traceId,
+    ...(record.parentId === undefined ? {} : { parentId: record.parentId }),
+    name: record.name,
+    startUnixNano: record.startUnixNano,
+    endUnixNano: record.endUnixNano,
+    status: record.status,
+    ...(record.statusMessage === undefined ? {} : { statusMessage: record.statusMessage }),
+    attributes: record.attributes,
+    events: record.events,
+    ...(record.droppedReason === undefined ? {} : { droppedReason: record.droppedReason }),
+  };
+}

package/src/exporters/index.ts

@@ -0,0 +1,36 @@
+/**
+ * Exporter surface for `@graphorin/observability`.
+ *
+ * Every exporter MUST be wrapped via {@link withValidation} before it
+ * is registered with the tracer. Un-wrapped exporters cause
+ * `createTracer(...)` to throw at startup unless the operator opts
+ * out explicitly with `validation: 'off'` (in which case a startup
+ * WARN is logged).
+ *
+ * @packageDocumentation
+ */
+
+export {
+  type ConsoleExporterOptions,
+  createConsoleExporter,
+} from './console.js';
+export {
+  createJSONLExporter,
+  type JSONLExporterOptions,
+} from './jsonl.js';
+export {
+  createOTLPHttpExporter,
+  type OTLPHttpExporterOptions,
+} from './otlp-http.js';
+export type {
+  SpanRecord,
+  SpanRecordEvent,
+  TraceExporter,
+} from './types.js';
+export { VALIDATED_EXPORTER_BRAND } from './types.js';
+export {
+  isValidatedExporter,
+  tryGetValidatorCounters,
+  type WithValidationOptions,
+  withValidation,
+} from './with-validation.js';

package/src/exporters/jsonl.ts

@@ -0,0 +1,167 @@
+/**
+ * `JSONLExporter` - append-only newline-delimited JSON output. Used as
+ * the default trace replay log; encryption-at-rest is opt-in via the
+ * encryption hook (Phase 16).
+ *
+ * The exporter:
+ * - creates the target directory with mode `0700`,
+ * - opens the per-session file with mode `0600`,
+ * - rotates per UTC month + per session id (`YYYY-MM/<sessionId>.jsonl`),
+ * - flushes every write (no internal buffer) so the file is suitable
+ *   for tail-style replay even after a crash.
+ *
+ * The output is deterministic - see {@link serializableRecord}.
+ *
+ * @packageDocumentation
+ */
+
+import { mkdir, open } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { serializableRecord } from './console.js';
+import type { SpanRecord, TraceExporter } from './types.js';
+
+/** @internal - used by tests to override the date provider. */
+export type DateProvider = () => Date;
+
+/**
+ * Configuration shape for {@link createJSONLExporter}.
+ *
+ * @stable
+ */
+export interface JSONLExporterOptions {
+  /** Identifier reported via `exporter.id`. Defaults to `'jsonl'`. */
+  readonly id?: string;
+  /** Root directory for the trace files. Created if missing. */
+  readonly path: string;
+  /**
+   * Resolver that picks the file path for a given span record. Defaults
+   * to `<rootPath>/<YYYY-MM>/<sessionId>.jsonl` keyed off the
+   * `graphorin.session.id` attribute (or `unsessioned.jsonl` when
+   * absent).
+   */
+  readonly resolveFilePath?: (record: SpanRecord, root: string) => string;
+  /**
+   * Override the date used when picking the rotation directory. Defaults
+   * to `() => new Date()`.
+   *
+   * @internal
+   */
+  readonly now?: DateProvider;
+  /**
+   * Upper bound on simultaneously-open file handles (RP-20). The pool keys
+   * by `(session, UTC-month)`, so a long-living server would otherwise hold
+   * one fd per pair forever; the least-recently-used handle is closed once
+   * the cap is reached. Defaults to `64`.
+   */
+  readonly maxOpenHandles?: number;
+}
+
+/**
+ * A JSONL exporter, plus an introspection seam for the bounded handle pool.
+ *
+ * @stable
+ */
+export interface JSONLExporter extends TraceExporter {
+  /** Number of file handles currently open in the pool (RP-20). */
+  openHandleCount(): number;
+}
+
+/**
+ * Build a JSONL trace exporter. Call `withValidation(exporter)` before
+ * passing the result to `createTracer({ exporters })`.
+ *
+ * @stable
+ */
+export function createJSONLExporter(opts: JSONLExporterOptions): JSONLExporter {
+  const id = opts.id ?? 'jsonl';
+  const root = opts.path;
+  const now = opts.now ?? (() => new Date());
+  const resolveFilePath = opts.resolveFilePath ?? defaultPathResolver(now);
+  const maxOpenHandles = Math.max(1, opts.maxOpenHandles ?? 64);
+
+  // Pool of file handles, keyed by absolute path. Each handle is opened
+  // in append mode the first time we touch the path; subsequent exports
+  // re-use the existing handle. The pool is LRU-bounded (RP-20) - the Map's
+  // insertion order is the recency order, so the first entry is the LRU.
+  const handles = new Map<string, FileHandleEntry>();
+  let closed = false;
+
+  return {
+    id,
+    async export(record: SpanRecord): Promise<void> {
+      if (closed) return;
+      const filePath = resolveFilePath(record, root);
+      const entry = await openOrReuse(handles, filePath, maxOpenHandles);
+      const line = `${JSON.stringify(serializableRecord(record))}\n`;
+      await entry.handle.write(line, null, 'utf8');
+    },
+    async flush(): Promise<void> {
+      for (const entry of handles.values()) {
+        await entry.handle.datasync();
+      }
+    },
+    async shutdown(): Promise<void> {
+      closed = true;
+      for (const entry of handles.values()) {
+        await entry.handle.close();
+      }
+      handles.clear();
+    },
+    openHandleCount(): number {
+      return handles.size;
+    },
+  };
+}
+
+interface FileHandleEntry {
+  readonly handle: import('node:fs/promises').FileHandle;
+}
+
+async function openOrReuse(
+  pool: Map<string, FileHandleEntry>,
+  filePath: string,
+  maxOpenHandles: number,
+): Promise<FileHandleEntry> {
+  const cached = pool.get(filePath);
+  if (cached !== undefined) {
+    // Mark most-recently-used by re-inserting at the tail of the Map.
+    pool.delete(filePath);
+    pool.set(filePath, cached);
+    return cached;
+  }
+
+  // RP-20: evict least-recently-used handles before exceeding the cap so the
+  // open-fd count stays bounded. An evicted path simply re-opens (append mode)
+  // on its next export - no data is lost.
+  while (pool.size >= maxOpenHandles) {
+    const lruKey = pool.keys().next().value;
+    if (lruKey === undefined) break;
+    const lru = pool.get(lruKey);
+    pool.delete(lruKey);
+    if (lru !== undefined) {
+      try {
+        await lru.handle.close();
+      } catch {
+        // Best-effort - a failed close must not block the new open.
+      }
+    }
+  }
+
+  await mkdir(dirname(filePath), { recursive: true, mode: 0o700 });
+  const handle = await open(filePath, 'a', 0o600);
+  const entry: FileHandleEntry = { handle };
+  pool.set(filePath, entry);
+  return entry;
+}
+
+function defaultPathResolver(now: DateProvider) {
+  return (record: SpanRecord, root: string): string => {
+    const sessionAttr = record.attributes['graphorin.session.id'];
+    const sessionId = typeof sessionAttr === 'string' ? sessionAttr : 'unsessioned';
+    const date = now();
+    const yyyy = String(date.getUTCFullYear());
+    const mm = String(date.getUTCMonth() + 1).padStart(2, '0');
+    const safeSession = sessionId.replace(/[^A-Za-z0-9._-]/g, '_');
+    return join(root, `${yyyy}-${mm}`, `${safeSession}.jsonl`);
+  };
+}