baselineos 0.2.0-beta.1

package/dist/llm-tracer-CIIujuO-.d.ts

@@ -0,0 +1,493 @@
+import { AgentBus } from './core/agent-bus.js';
+
+/**
+ * ModelVersionRegistry — SIGNAL-048
+ *
+ * Tracks which model version handled each task execution. Provides a
+ * queryable registry of deployed model versions, their metadata, and
+ * per-task attribution for audit and regression analysis.
+ *
+ * Capabilities:
+ *   - Register model versions with metadata (provider, family, deployment date)
+ *   - Set the active version per provider
+ *   - Record per-task model attribution
+ *   - List all versions / versions by provider
+ *   - Emit governance:model-version-changed on active version change
+ *
+ * Usage:
+ *   const registry = new ModelVersionRegistry({ bus });
+ *   registry.register('claude-sonnet-4-6', { provider: 'anthropic', family: 'claude-4' });
+ *   registry.setActive('anthropic', 'claude-sonnet-4-6');
+ *   registry.recordTaskModel('task-123', 'claude-sonnet-4-6');
+ *   registry.getForTask('task-123'); // → 'claude-sonnet-4-6'
+ *
+ * @license Apache-2.0
+ */
+
+interface ModelVersionEntry {
+    modelId: string;
+    provider: string;
+    family?: string;
+    /** ISO date string when this version was first registered */
+    registeredAt: string;
+    /** ISO date string when this version was last set as active */
+    activatedAt?: string;
+    /** Arbitrary metadata: context window, pricing tier, capabilities, etc. */
+    metadata?: Record<string, unknown>;
+}
+interface ModelVersionRegistryConfig {
+    bus?: AgentBus;
+}
+declare class ModelVersionRegistry {
+    private readonly versions;
+    /** provider → active modelId */
+    private readonly activeVersions;
+    /** taskId → modelId */
+    private readonly taskModels;
+    private readonly bus?;
+    constructor(config?: ModelVersionRegistryConfig);
+    /**
+     * Register a model version. Idempotent — re-registering updates metadata.
+     */
+    register(modelId: string, meta: Omit<ModelVersionEntry, 'modelId' | 'registeredAt'>): void;
+    /**
+     * Set the active model version for a provider.
+     * Publishes governance:model-version-changed on the bus.
+     */
+    setActive(provider: string, modelId: string): void;
+    /**
+     * Get the currently active model version for a provider.
+     * Returns undefined if none has been set.
+     */
+    getActive(provider: string): string | undefined;
+    /**
+     * Record which model version was used for a given task.
+     */
+    recordTaskModel(taskId: string, modelId: string): void;
+    /**
+     * Return the model version used for a given task.
+     * Returns undefined if no attribution has been recorded.
+     */
+    getForTask(taskId: string): string | undefined;
+    /**
+     * Return entry for a specific model version, or undefined if unknown.
+     */
+    get(modelId: string): ModelVersionEntry | undefined;
+    /**
+     * List all registered model versions, optionally filtered by provider.
+     */
+    listVersions(provider?: string): ModelVersionEntry[];
+    /**
+     * List all providers that have an active model version set.
+     */
+    listActiveProviders(): Array<{
+        provider: string;
+        modelId: string;
+    }>;
+    /**
+     * Return a summary of per-model task counts (how many tasks used each model).
+     */
+    getTaskCounts(): Map<string, number>;
+    /**
+     * Remove all task attribution records (e.g., on rotation period boundary).
+     */
+    clearTaskRecords(): void;
+}
+
+interface GroundTruthCase {
+    /** Unique identifier for this eval case */
+    id: string;
+    /** Human-readable description */
+    description: string;
+    /** Simulated input text / task description */
+    input: string;
+    /** Expected model output (verbatim or representative) */
+    expectedOutput: string;
+    /** Acceptance criteria labels for this case */
+    criteria: string[];
+    /** Expected token range [min, max] for cost SLO check */
+    expectedTokenRange?: [number, number];
+    /** Whether this is a benign (non-refusal) request */
+    benign?: boolean;
+}
+interface ProdEvalCheckContext {
+    case: GroundTruthCase;
+    /** Simulated output — either expectedOutput or a probe value */
+    output: string;
+}
+interface ProdEvalCheckResult {
+    checkId: string;
+    caseId: string;
+    passed: boolean;
+    reason?: string;
+}
+type ProdEvalCheck = (ctx: ProdEvalCheckContext) => Promise<ProdEvalCheckResult[]>;
+interface ProdEvalReport {
+    runId: string;
+    timestamp: string;
+    totalCases: number;
+    totalChecks: number;
+    passed: number;
+    failed: number;
+    score: number;
+    results: ProdEvalCheckResult[];
+    durationMs: number;
+}
+/**
+ * Per-model evaluation result stored in prod-eval-history.json (SIGNAL-053).
+ */
+interface VersionEvalResult {
+    /** Unique run identifier */
+    runId: string;
+    /** Model ID that was evaluated (e.g. 'claude-sonnet-4-6') */
+    modelId: string;
+    /** ISO timestamp of the eval run */
+    timestamp: string;
+    /** Overall score 0–100 */
+    score: number;
+    /** Fraction of checks that passed (0–1) */
+    passRate: number;
+    /** Check IDs that failed */
+    failedChecks: string[];
+    /** Total number of cases evaluated */
+    totalCases: number;
+}
+/**
+ * Result of comparing two VersionEvalResults (SIGNAL-053).
+ */
+interface VersionComparison {
+    /** Score delta: b.score − a.score (positive = b is better) */
+    deltaScore: number;
+    /** PassRate delta: b.passRate − a.passRate */
+    deltaPassRate: number;
+    /** Checks that failed in b but not in a (new regressions) */
+    newRegressions: string[];
+    /** Checks that failed in a but not in b (improvements) */
+    improvements: string[];
+    /** True if b is strictly better than a */
+    improved: boolean;
+    /**
+     * Promotion recommendation:
+     *   'promote'  — b is measurably better with no new regressions
+     *   'hold'     — marginal improvement or insufficient delta
+     *   'rollback' — b is worse than a
+     */
+    recommend: 'promote' | 'hold' | 'rollback';
+    /**
+     * 'high' when ≥ 10 cases; 'low' when fewer cases may produce noisy results.
+     */
+    confidence: 'high' | 'low';
+}
+interface ProductionEvalPipelineConfig {
+    bus?: AgentBus;
+    /** Path to ground-truth JSON dataset. Default: src/data/prod-eval-dataset.json */
+    datasetPath?: string;
+    /** Path to write the report JSON. Default: packages/baselineos/prod-eval-report.json */
+    reportPath?: string;
+    /** Path to the eval history JSON file (SIGNAL-053). Default: src/data/prod-eval-history.json */
+    historyPath?: string;
+    /**
+     * Minimum score delta required to recommend promotion (default: 3).
+     * deltaScore must exceed this threshold for 'promote' recommendation.
+     */
+    promotionThreshold?: number;
+    /**
+     * ModelVersionRegistry for auto model attribution (SIGNAL-056).
+     * When set, every run() call resolves the active model and appends a
+     * VersionEvalResult to history automatically — no need to call runForModel().
+     * Provider defaults to 'anthropic' unless modelAttributionProvider is set.
+     */
+    modelVersionRegistry?: ModelVersionRegistry;
+    /** Provider key used to look up the active model. Default: 'anthropic' */
+    modelAttributionProvider?: string;
+}
+declare class ProductionEvalPipeline {
+    private readonly checks;
+    private readonly config;
+    private readonly bus?;
+    private readonly promotionThreshold;
+    constructor(config?: ProductionEvalPipelineConfig);
+    /** Register a custom check function. */
+    addCheck(check: ProdEvalCheck): this;
+    /**
+     * Register all five built-in production checks.
+     */
+    addBuiltinChecks(): this;
+    /**
+     * Load the ground-truth dataset from disk.
+     */
+    loadDataset(): GroundTruthCase[];
+    /**
+     * Run all checks against all ground-truth cases.
+     * Returns a ProdEvalReport.
+     *
+     * When modelVersionRegistry is configured (SIGNAL-056), automatically
+     * resolves the active model and appends a VersionEvalResult to history.
+     */
+    run(cases?: GroundTruthCase[]): Promise<ProdEvalReport>;
+    /**
+     * Run the eval suite and record the result under a specific modelId.
+     * Appends to prod-eval-history.json for trend tracking.
+     *
+     * Uses _runCore() directly to avoid double-appending when modelVersionRegistry
+     * is also configured.
+     */
+    runForModel(modelId: string, cases?: GroundTruthCase[]): Promise<VersionEvalResult>;
+    /**
+     * Core eval execution — runs checks, persists report, publishes bus event.
+     * Called by both run() and runForModel() to avoid code duplication.
+     */
+    private _runCore;
+    /**
+     * Compare two VersionEvalResults and return a promotion recommendation.
+     * `a` is the baseline (current production); `b` is the candidate.
+     */
+    compareVersions(a: VersionEvalResult, b: VersionEvalResult): VersionComparison;
+    /**
+     * Append a VersionEvalResult to the eval history file.
+     */
+    appendHistory(result: VersionEvalResult): void;
+    /**
+     * Return all VersionEvalResults, optionally filtered by modelId.
+     */
+    getHistory(modelId?: string): VersionEvalResult[];
+    /**
+     * Return the rolling average eval score and delta from windowSize entries ago.
+     * Returns undefined when insufficient history exists.
+     */
+    getQualityTrend(modelId: string, windowSize?: number): {
+        current: number;
+        rollingAvg: number;
+        delta: number;
+    } | undefined;
+    private _historyPath;
+}
+
+/**
+ * TraceCurator — SIGNAL-047
+ *
+ * Converts sampled production LLM traces into GroundTruthCase entries
+ * that feed the ProductionEvalPipeline dataset. Enables continuous
+ * ground-truth expansion from real inference without manual labelling.
+ *
+ * Architecture:
+ *   1. Accept trace records (from Langfuse callbacks or a local trace buffer)
+ *   2. Apply sampling strategy (reservoir / recency / confidence-weighted)
+ *   3. Score and filter traces using configurable quality criteria
+ *   4. Append accepted cases to the ground-truth dataset on disk
+ *   5. Publish governance:trace-curated on AgentBus
+ *
+ * Sampling strategies:
+ *   reservoir   — uniform random sample, bounded by maxDatasetSize
+ *   recency     — keeps the N most recent accepted traces
+ *   confidence  — preferentially retains high-confidence traces
+ *
+ * Usage:
+ *   const curator = new TraceCurator({ datasetPath, bus });
+ *   curator.ingest(traceRecord);          // real-time ingestion
+ *   const result = await curator.flush(); // write accepted cases to disk
+ *
+ * @license Apache-2.0
+ */
+
+type SamplingStrategy = 'reservoir' | 'recency' | 'confidence';
+interface TraceRecord {
+    /** Unique trace identifier (task ID in Langfuse) */
+    traceId: string;
+    /** Human-readable task title */
+    title: string;
+    /** The input prompt / task description sent to the model */
+    input: string;
+    /** The model's final output for this trace */
+    output: string;
+    /** Model confidence signal (0–1). Use self-verify score when available. */
+    confidence: number;
+    /** Whether this was a benign (non-refusal) request */
+    benign?: boolean;
+    /** Approximate token count for cost SLO */
+    tokens?: number;
+    /** ISO timestamp of the trace */
+    timestamp: string;
+    /** Whether the trace passed self-verification */
+    verified?: boolean;
+}
+interface CurationResult {
+    ingested: number;
+    accepted: number;
+    rejected: number;
+    datasetSize: number;
+    newCases: GroundTruthCase[];
+}
+interface TraceCuratorConfig {
+    bus?: AgentBus;
+    /** Path to ground-truth JSON dataset. Default: src/data/prod-eval-dataset.json */
+    datasetPath?: string;
+    /** Sampling strategy. Default: reservoir */
+    strategy?: SamplingStrategy;
+    /** Maximum dataset size (oldest entries pruned when exceeded). Default: 500 */
+    maxDatasetSize?: number;
+    /** Minimum confidence to accept a trace. Default: 0.7 */
+    minConfidence?: number;
+    /** Only accept traces where self-verify passed. Default: false */
+    requireVerified?: boolean;
+    /** Minimum input length in characters. Default: 20 */
+    minInputLength?: number;
+}
+declare class TraceCurator {
+    private readonly bus?;
+    private readonly datasetPath;
+    private readonly strategy;
+    private readonly maxDatasetSize;
+    private readonly minConfidence;
+    private readonly requireVerified;
+    private readonly minInputLength;
+    /** In-memory buffer of traces ingested since last flush */
+    private readonly buffer;
+    constructor(config?: TraceCuratorConfig);
+    /**
+     * Ingest a trace record into the buffer.
+     * Call flush() to persist accepted cases to disk.
+     */
+    ingest(trace: TraceRecord): void;
+    /**
+     * Ingest multiple traces at once.
+     */
+    ingestBatch(traces: TraceRecord[]): void;
+    /**
+     * Apply quality filters and accept/reject buffered traces.
+     * Returns accepted TraceRecords without writing to disk.
+     */
+    filter(traces: TraceRecord[]): TraceRecord[];
+    /**
+     * Flush buffered traces to the ground-truth dataset.
+     * Applies sampling strategy, deduplicates by traceId, and respects maxDatasetSize.
+     */
+    flush(): Promise<CurationResult>;
+    /** Return the current buffer without flushing. */
+    peekBuffer(): TraceRecord[];
+    /** Clear the buffer without writing to disk. */
+    clearBuffer(): void;
+    /** Return the number of cases currently in the on-disk dataset. */
+    getDatasetSize(): number;
+    /**
+     * Export the curated dataset to a fine-tuning JSONL file.
+     *
+     * Each line is a JSON object in the messages format expected by the
+     * target API. PII-free by construction — upstream LlmTracer masking
+     * ensures no PII reaches the dataset.
+     *
+     * Formats:
+     *   'openai'     — { "messages": [system?, user, assistant] }
+     *   'anthropic'  — { "system": "...", "messages": [user, assistant] }
+     *
+     * @param format      Target fine-tuning API format
+     * @param outputPath  Absolute path to write the .jsonl file
+     * @param options.systemPrompt  Optional system prompt to include per example
+     * @param options.minExamples   Minimum dataset size before warning (default 10)
+     */
+    exportFineTuneDataset(format: 'openai' | 'anthropic', outputPath: string, options?: {
+        systemPrompt?: string;
+        minExamples?: number;
+    }): {
+        exported: number;
+        skipped: number;
+        path: string;
+    };
+    private toCuratedCase;
+    private loadDataset;
+    private applyStrategy;
+}
+
+/**
+ * LLM Tracer — SIGNAL-014
+ *
+ * Logs every LLM call made by AnthropicEngine to Langfuse for prompt-level
+ * observability. Captures model, token usage, latency, input/output, and
+ * task metadata. Traces are keyed by task.id, so a multi-step task (execute
+ * → self-verify → review) produces one Langfuse trace with multiple
+ * generation spans.
+ *
+ * Self-hosted Langfuse:
+ *   docker compose -f docker/docker-compose.monitoring.yml up -d
+ *   → http://localhost:3001  (admin@baselineos.dev / baseline)
+ *
+ * Configuration (env or constructor):
+ *   LANGFUSE_PUBLIC_KEY   — project public key (default: baseline-public-key)
+ *   LANGFUSE_SECRET_KEY   — project secret key (default: baseline-secret-key)
+ *   LANGFUSE_BASE_URL     — Langfuse server URL (default: http://localhost:3001)
+ *
+ * The tracer is fail-safe: any Langfuse error is swallowed silently so it
+ * can never interrupt task execution.
+ *
+ * @license Apache-2.0
+ */
+
+interface LlmTracerConfig {
+    /** Langfuse project public key. Default: LANGFUSE_PUBLIC_KEY env or 'baseline-public-key' */
+    publicKey?: string;
+    /** Langfuse project secret key. Default: LANGFUSE_SECRET_KEY env or 'baseline-secret-key' */
+    secretKey?: string;
+    /** Langfuse server base URL. Default: LANGFUSE_BASE_URL env or http://localhost:3001 */
+    baseUrl?: string;
+    /** Flush batch size (default: 15) */
+    flushAt?: number;
+    /** Flush interval in ms (default: 30_000) */
+    flushInterval?: number;
+    /**
+     * Scan and redact PII from system prompts, input messages, and output
+     * before sending to Langfuse. Default: true (SIGNAL-037).
+     */
+    enablePiiMasking?: boolean;
+    /**
+     * TraceCurator to ingest sampled traces into the ground-truth dataset.
+     * When provided, each logGeneration() call feeds a TraceRecord into the
+     * curator buffer for downstream curation and eval dataset expansion (SIGNAL-049).
+     */
+    curator?: TraceCurator;
+}
+interface LlmGenerationOptions {
+    /** Span name within the trace, e.g. 'execute', 'self-verify', 'review', 'agent-loop:3' */
+    name: string;
+    /** Model identifier, e.g. 'claude-sonnet-4-6' */
+    model: string;
+    /** System prompt sent to the model */
+    systemPrompt?: string;
+    /** User + assistant turns passed to the model */
+    inputMessages: Array<{
+        role: string;
+        content: unknown;
+    }>;
+    /** Raw text output from the model */
+    output: string;
+    inputTokens: number;
+    outputTokens: number;
+    startTime: Date;
+    endTime: Date;
+    /** Arbitrary metadata attached to the generation (taskId, agentId, etc.) */
+    metadata?: Record<string, unknown>;
+}
+declare class LlmTracer {
+    private readonly langfuse;
+    private readonly piiScanner;
+    private readonly curator;
+    constructor(config?: LlmTracerConfig);
+    /** Redact PII from a string if masking is enabled. No-op when scanner is null. */
+    private maskPii;
+    /**
+     * Log a single LLM generation to Langfuse.
+     *
+     * Multiple calls with the same `traceId` accumulate under one trace,
+     * so all generations for a task are grouped together automatically.
+     *
+     * @param traceId   task.id — the top-level trace identifier
+     * @param traceName task.title — human-readable trace label in the UI
+     * @param options   generation details
+     */
+    logGeneration(traceId: string, traceName: string, options: LlmGenerationOptions): void;
+    /** Flush pending spans. Call on graceful shutdown. */
+    flush(): Promise<void>;
+    /** Shutdown Langfuse client (flushes pending spans). */
+    shutdown(): void;
+}
+
+export { type CurationResult as C, type GroundTruthCase as G, LlmTracer as L, ModelVersionRegistry as M, ProductionEvalPipeline as P, type SamplingStrategy as S, TraceCurator as T, type VersionEvalResult as V, type VersionComparison as a, type LlmGenerationOptions as b, type LlmTracerConfig as c, type ModelVersionEntry as d, type ModelVersionRegistryConfig as e, type ProdEvalCheck as f, type ProdEvalCheckContext as g, type ProdEvalCheckResult as h, type ProdEvalReport as i, type ProductionEvalPipelineConfig as j, type TraceCuratorConfig as k, type TraceRecord as l };