@seanhogg/builderforce-memory 2026.7.14 → 2026.9.27

package/src/telemetry/exporters.ts

@@ -0,0 +1,187 @@
+/**
+ * telemetry/exporters.ts — where finished spans go.
+ *
+ * Three sinks cover the deployment spectrum without a vendor dependency:
+ *   • {@link InMemorySpanExporter} — tests and in-process dashboards.
+ *   • {@link ConsoleSpanExporter}  — local development.
+ *   • {@link OtlpHttpSpanExporter} — any OTLP/HTTP collector, which includes
+ *     Google Cloud Trace, Datadog, Honeycomb, Grafana Tempo and the OpenTelemetry
+ *     Collector. It speaks the protocol over `fetch` rather than importing the
+ *     OpenTelemetry SDK, so the package stays zero-dependency and still runs in a
+ *     browser or a Cloudflare Worker where the Node SDK cannot.
+ */
+
+import type { AttributeValue, SpanData, SpanExporter } from './types.js';
+
+export class InMemorySpanExporter implements SpanExporter {
+    private readonly _spans: SpanData[] = [];
+
+    export(spans: readonly SpanData[]): void {
+        this._spans.push(...spans);
+    }
+
+    get spans(): readonly SpanData[] { return this._spans; }
+
+    /** Spans belonging to one trace, oldest first. */
+    trace(traceId: string): SpanData[] {
+        return this._spans.filter((s) => s.traceId === traceId);
+    }
+
+    reset(): void { this._spans.length = 0; }
+}
+
+export interface ConsoleSpanExporterOptions {
+    /** Sink for the rendered lines. Default `console.log`. */
+    write?: (line: string) => void;
+    /** Include span attributes. Default false — one line per span. */
+    verbose?: boolean;
+}
+
+export class ConsoleSpanExporter implements SpanExporter {
+    private readonly _write: (line: string) => void;
+    private readonly _verbose: boolean;
+
+    constructor(opts: ConsoleSpanExporterOptions = {}) {
+        this._write = opts.write ?? ((line) => console.log(line));
+        this._verbose = opts.verbose ?? false;
+    }
+
+    export(spans: readonly SpanData[]): void {
+        for (const span of spans) {
+            const cost = span.costUsd !== undefined ? ` $${span.costUsd.toFixed(6)}` : '';
+            const tokens = span.usage ? ` ${span.usage.inputTokens}in/${span.usage.outputTokens}out` : '';
+            const status = span.status === 'error' ? ` ERROR: ${span.error ?? ''}` : '';
+            this._write(
+                `[${span.traceId.slice(0, 8)}] ${span.kind.padEnd(9)} ${span.name} ` +
+                `${Math.round(span.durationMs ?? 0)}ms${tokens}${cost}${status}`,
+            );
+            if (this._verbose && Object.keys(span.attributes).length > 0) {
+                this._write(`         ${JSON.stringify(span.attributes)}`);
+            }
+        }
+    }
+}
+
+export interface OtlpHttpSpanExporterOptions {
+    /** Collector endpoint, e.g. `https://otlp.example.com/v1/traces`. */
+    url: string;
+    /** Extra headers (auth token, GCP project header, …). */
+    headers?: Record<string, string>;
+    /** `service.name` reported on the resource. Default `evermind`. */
+    serviceName?: string;
+    /** Injected for test. Default global `fetch`. */
+    fetchImpl?: typeof fetch;
+    /**
+     * Called when an export request fails. Default: swallow. Telemetry must never
+     * take down the request it is measuring, so failures are reported, not thrown.
+     */
+    onError?: (err: unknown) => void;
+}
+
+/** OTLP span kind codes: UNSPECIFIED 0, INTERNAL 1, SERVER 2, CLIENT 3. */
+const OTLP_CLIENT = 3;
+const OTLP_INTERNAL = 1;
+
+export class OtlpHttpSpanExporter implements SpanExporter {
+    private readonly _url: string;
+    private readonly _headers: Record<string, string>;
+    private readonly _serviceName: string;
+    private readonly _fetch: typeof fetch;
+    private readonly _onError: (err: unknown) => void;
+
+    constructor(opts: OtlpHttpSpanExporterOptions) {
+        this._url = opts.url;
+        this._headers = opts.headers ?? {};
+        this._serviceName = opts.serviceName ?? 'evermind';
+        this._fetch = opts.fetchImpl ?? ((...args) => fetch(...args));
+        this._onError = opts.onError ?? (() => {});
+    }
+
+    async export(spans: readonly SpanData[]): Promise<void> {
+        if (spans.length === 0) return;
+        try {
+            const res = await this._fetch(this._url, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json', ...this._headers },
+                body: JSON.stringify(toOtlpPayload(spans, this._serviceName)),
+            });
+            if (!res.ok) this._onError(new Error(`OTLP export failed: ${res.status}`));
+        } catch (err) {
+            this._onError(err);
+        }
+    }
+}
+
+/** Wraps spans in the OTLP/HTTP JSON envelope. */
+export function toOtlpPayload(spans: readonly SpanData[], serviceName = 'evermind'): unknown {
+    return {
+        resourceSpans: [{
+            resource: { attributes: [otlpAttr('service.name', serviceName)] },
+            scopeSpans: [{
+                scope: { name: '@seanhogg/builderforce-memory' },
+                spans: spans.map(toOtlpSpan),
+            }],
+        }],
+    };
+}
+
+/**
+ * Converts one span to the OTLP wire shape. The mapping is a rename, not a
+ * re-model, because {@link SpanData} was designed against this schema.
+ *
+ * LLM attributes follow the OpenTelemetry GenAI semantic conventions
+ * (`gen_ai.*`) so a stock trace viewer renders token counts without config.
+ */
+export function toOtlpSpan(span: SpanData): unknown {
+    const attributes = Object.entries(span.attributes).map(([k, v]) => otlpAttr(k, v));
+
+    if (span.usage) {
+        attributes.push(
+            otlpAttr('gen_ai.request.model', span.usage.model),
+            otlpAttr('gen_ai.usage.input_tokens', span.usage.inputTokens),
+            otlpAttr('gen_ai.usage.output_tokens', span.usage.outputTokens),
+        );
+        if (span.usage.cachedInputTokens !== undefined) {
+            attributes.push(otlpAttr('gen_ai.usage.cached_input_tokens', span.usage.cachedInputTokens));
+        }
+    }
+    if (span.costUsd !== undefined) attributes.push(otlpAttr('gen_ai.usage.cost_usd', span.costUsd));
+    if (span.timeToFirstTokenMs !== undefined) {
+        attributes.push(otlpAttr('gen_ai.server.time_to_first_token_ms', span.timeToFirstTokenMs));
+    }
+
+    const out: Record<string, unknown> = {
+        traceId: span.traceId,
+        spanId: span.spanId,
+        name: span.name,
+        kind: span.kind === 'llm' || span.kind === 'embedding' ? OTLP_CLIENT : OTLP_INTERNAL,
+        startTimeUnixNano: msToNano(span.startedAt),
+        endTimeUnixNano: msToNano(span.endedAt ?? span.startedAt),
+        attributes,
+        events: span.events.map((e) => ({
+            name: e.name,
+            timeUnixNano: msToNano(e.at),
+            attributes: Object.entries(e.attributes ?? {}).map(([k, v]) => otlpAttr(k, v)),
+        })),
+        status: span.status === 'error'
+            ? { code: 2, message: span.error ?? '' }
+            : { code: span.status === 'ok' ? 1 : 0 },
+    };
+    if (span.parentSpanId) out['parentSpanId'] = span.parentSpanId;
+    return out;
+}
+
+function otlpAttr(key: string, value: AttributeValue): unknown {
+    if (typeof value === 'number') {
+        return Number.isInteger(value)
+            ? { key, value: { intValue: value } }
+            : { key, value: { doubleValue: value } };
+    }
+    if (typeof value === 'boolean') return { key, value: { boolValue: value } };
+    return { key, value: { stringValue: value } };
+}
+
+/** OTLP timestamps are nanoseconds since the epoch, as a decimal string. */
+function msToNano(ms: number): string {
+    return `${Math.round(ms)}000000`;
+}

package/src/telemetry/index.ts

@@ -0,0 +1,56 @@
+/**
+ * telemetry — granular tracing plus LLM-native metrics (tokens/sec, TTFT,
+ * cost-per-request, cache-hit rate), built on ONE span stream so the trace view
+ * and the metric view can never disagree.
+ */
+
+export { Tracer, Span } from './Tracer.js';
+export type { TracerOptions, SpanOptions } from './Tracer.js';
+
+export { parseTraceparent, formatTraceparent } from './traceparent.js';
+export type { TraceContext } from './traceparent.js';
+
+export { MetricsRegistry, percentile } from './MetricsRegistry.js';
+export type {
+    LlmMetricSnapshot,
+    KindMetricSnapshot,
+    MetricsRegistryOptions,
+} from './MetricsRegistry.js';
+
+export { InstrumentedBridge } from './InstrumentedBridge.js';
+export type { InstrumentedBridgeOptions } from './InstrumentedBridge.js';
+
+export {
+    InMemorySpanExporter,
+    ConsoleSpanExporter,
+    OtlpHttpSpanExporter,
+    toOtlpPayload,
+    toOtlpSpan,
+} from './exporters.js';
+export type {
+    ConsoleSpanExporterOptions,
+    OtlpHttpSpanExporterOptions,
+} from './exporters.js';
+
+export {
+    estimateCostUsd,
+    estimateTokens,
+    resolveRate,
+    DEFAULT_PRICE_BOOK,
+    ANTHROPIC_PRICE_BOOK,
+    EVERMIND_PRICE_BOOK,
+    DEFAULT_CACHE_READ_MULTIPLIER,
+    DEFAULT_CACHE_WRITE_MULTIPLIER,
+} from './pricing.js';
+export type { ModelRate, PriceBook } from './pricing.js';
+
+export type {
+    AttributeValue,
+    SpanKind,
+    SpanStatus,
+    SpanEvent,
+    SpanData,
+    SpanExporter,
+    LlmUsage,
+    BridgeCallInfo,
+} from './types.js';

package/src/telemetry/pricing.ts

@@ -0,0 +1,129 @@
+/**
+ * telemetry/pricing.ts — cost-per-request, computed from real token counts.
+ *
+ * "Cost per request" is an LLM-native metric that cannot be derived from latency
+ * or call count: it depends on the model, the input/output split, AND how much of
+ * the input was a cache read (cheap) versus a cache write (dear). This module is
+ * the ONE place that arithmetic lives, so the bridges, the metrics registry, and
+ * the eval harness cannot disagree about what a run cost.
+ *
+ * Rates are DATA, not code: an enterprise on a negotiated rate card, a partner
+ * platform (Bedrock/Vertex bill separately from the first-party API), or a model
+ * released after this file was written is handled by passing a {@link PriceBook},
+ * never by editing a branch. Unknown models cost `undefined` — reported as an
+ * `unpricedCalls` count rather than silently billed at zero, because a silent
+ * zero is how cost dashboards lie.
+ */
+
+import type { LlmUsage } from './types.js';
+
+export interface ModelRate {
+    /** USD per 1M input tokens. */
+    inputPerMTok: number;
+    /** USD per 1M output tokens. */
+    outputPerMTok: number;
+    /**
+     * Multiplier applied to `inputPerMTok` for tokens READ from the provider's
+     * prompt cache. Anthropic bills cache reads at 10% of input. Default 0.1.
+     */
+    cacheReadMultiplier?: number;
+    /**
+     * Multiplier applied to `inputPerMTok` for tokens WRITTEN to the prompt
+     * cache. Anthropic bills a 5-minute cache write at 125% of input. Default 1.25.
+     */
+    cacheWriteMultiplier?: number;
+}
+
+/** Model id → rate. Keys match exactly, or by longest prefix (see {@link resolveRate}). */
+export type PriceBook = Record<string, ModelRate>;
+
+export const DEFAULT_CACHE_READ_MULTIPLIER = 0.1;
+export const DEFAULT_CACHE_WRITE_MULTIPLIER = 1.25;
+
+/**
+ * First-party Anthropic API rates (USD / 1M tokens), current as of 2026-08-20.
+ *
+ * Bedrock and Vertex are partner-operated and priced separately — supply those
+ * rates via a custom {@link PriceBook} rather than assuming these apply.
+ */
+export const ANTHROPIC_PRICE_BOOK: PriceBook = {
+    'claude-fable-5':    { inputPerMTok: 10, outputPerMTok: 50 },
+    'claude-mythos-5':   { inputPerMTok: 10, outputPerMTok: 50 },
+    'claude-opus-5':     { inputPerMTok: 5,  outputPerMTok: 25 },
+    'claude-opus-4-8':   { inputPerMTok: 5,  outputPerMTok: 25 },
+    'claude-opus-4-7':   { inputPerMTok: 5,  outputPerMTok: 25 },
+    'claude-opus-4-6':   { inputPerMTok: 5,  outputPerMTok: 25 },
+    'claude-sonnet-5':   { inputPerMTok: 3,  outputPerMTok: 15 },
+    'claude-sonnet-4-6': { inputPerMTok: 3,  outputPerMTok: 15 },
+    'claude-haiku-4-5':  { inputPerMTok: 1,  outputPerMTok: 5  },
+};
+
+/**
+ * Evermind serves `evermind/<ref>` traffic on-device, so it has no per-token
+ * vendor bill. It is priced explicitly at zero rather than left unknown — the
+ * whole point of routing to Evermind is that the saving is *measurable*.
+ */
+export const EVERMIND_PRICE_BOOK: PriceBook = {
+    'evermind': { inputPerMTok: 0, outputPerMTok: 0 },
+    'ssm':      { inputPerMTok: 0, outputPerMTok: 0 },
+};
+
+export const DEFAULT_PRICE_BOOK: PriceBook = { ...ANTHROPIC_PRICE_BOOK, ...EVERMIND_PRICE_BOOK };
+
+/**
+ * Resolves a model id to a rate: exact match first, then the longest key the
+ * model id starts with. Prefix matching is what makes dated snapshots
+ * (`claude-opus-4-5@20251101`) and platform-prefixed ids (`anthropic.claude-opus-5`)
+ * resolve without a per-variant entry.
+ */
+export function resolveRate(model: string, book: PriceBook = DEFAULT_PRICE_BOOK): ModelRate | undefined {
+    const exact = book[model];
+    if (exact) return exact;
+
+    let best: ModelRate | undefined;
+    let bestLen = 0;
+    for (const [key, rate] of Object.entries(book)) {
+        if (key.length > bestLen && (model.startsWith(key) || model.includes(`/${key}`))) {
+            best = rate;
+            bestLen = key.length;
+        }
+    }
+    return best;
+}
+
+/**
+ * Cost in USD for one call, or `undefined` when the model is not in the book.
+ *
+ * A locally-served call (`localCacheHit`) is free by construction — it never
+ * reached a provider — so it costs 0 even for an unpriced model.
+ */
+export function estimateCostUsd(usage: LlmUsage, book: PriceBook = DEFAULT_PRICE_BOOK): number | undefined {
+    if (usage.localCacheHit) return 0;
+
+    const rate = resolveRate(usage.model, book);
+    if (!rate) return undefined;
+
+    const cacheRead  = rate.cacheReadMultiplier  ?? DEFAULT_CACHE_READ_MULTIPLIER;
+    const cacheWrite = rate.cacheWriteMultiplier ?? DEFAULT_CACHE_WRITE_MULTIPLIER;
+
+    const perToken = (perMTok: number) => perMTok / 1_000_000;
+
+    return (
+        usage.inputTokens              * perToken(rate.inputPerMTok) +
+        usage.outputTokens             * perToken(rate.outputPerMTok) +
+        (usage.cachedInputTokens ?? 0) * perToken(rate.inputPerMTok * cacheRead) +
+        (usage.cacheWriteTokens  ?? 0) * perToken(rate.inputPerMTok * cacheWrite)
+    );
+}
+
+/**
+ * Character-based token estimate — the fallback when a provider reports no usage.
+ *
+ * ~4 characters per token is the standard English approximation. Estimates are
+ * flagged (`estimated: true`) all the way through to the metric snapshot so a
+ * cost figure never silently mixes measured and guessed numbers.
+ */
+export function estimateTokens(text: string): number {
+    if (!text) return 0;
+    return Math.max(1, Math.ceil(text.length / 4));
+}

package/src/telemetry/traceparent.ts

@@ -0,0 +1,89 @@
+/**
+ * telemetry/traceparent.ts — W3C Trace Context propagation.
+ *
+ * A {@link Tracer} on its own produces one span tree per PROCESS. A cloud run is not
+ * one process: portal → api → relay → runtime is four, and without propagation each
+ * hop opens its own root trace, so the operator gets four unrelated trees and has to
+ * correlate them by timestamp. Head-based propagation is what makes them one tree —
+ * the caller serialises `{ traceId, spanId, sampled }` into a `traceparent` header,
+ * the callee parses it and starts its root span as a CHILD of the remote span.
+ *
+ * The sampling flag travels with it on purpose: a sampling decision taken per-hop
+ * would record hop 1 and drop hop 3, which is a tree with holes in it — the same
+ * failure the in-process `_sampled` map exists to prevent, one network hop out.
+ *
+ * Pure string work, zero dependencies, no OpenTelemetry SDK — the wire format is a
+ * 55-character ASCII string and re-implementing it costs less than the dependency.
+ *
+ * @see https://www.w3.org/TR/trace-context/
+ */
+
+/** A trace position received from (or sent to) another process. */
+export interface TraceContext {
+    /** 32 lowercase hex chars, never all-zero. */
+    traceId: string;
+    /** 16 lowercase hex chars, never all-zero — the REMOTE span, parent of ours. */
+    spanId: string;
+    /** The `sampled` flag (bit 0x01). False means the caller decided to drop the trace. */
+    sampled: boolean;
+}
+
+/** The only trace-flag bit W3C defines today. */
+const FLAG_SAMPLED = 0x01;
+
+const TRACE_ID_LENGTH = 32;
+const SPAN_ID_LENGTH = 16;
+
+const HEX = /^[0-9a-f]+$/;
+
+/** True when `value` is `length` lowercase hex chars and not entirely zeros. */
+function isUsableId(value: string, length: number): boolean {
+    return value.length === length && HEX.test(value) && /[1-9a-f]/.test(value);
+}
+
+/**
+ * Parse a `traceparent` header, or `null` when it is absent or unusable.
+ *
+ * Rejecting rather than repairing is deliberate: a malformed header means the caller
+ * is not actually in the trace we would be joining, and inventing a link to it
+ * produces a tree that misattributes latency. `null` makes the callee start a fresh
+ * root, which is at least true.
+ *
+ * Unknown future versions are accepted by reading the first four fields — that is
+ * what the spec requires, so a later spec revision does not silently break every
+ * hop. Version `ff` is invalid by definition and is rejected.
+ */
+export function parseTraceparent(header: string | null | undefined): TraceContext | null {
+    if (typeof header !== 'string') return null;
+    const parts = header.trim().toLowerCase().split('-');
+    if (parts.length < 4) return null;
+
+    const [version, traceId, spanId, flags] = parts as [string, string, string, string];
+    if (version.length !== 2 || !HEX.test(version) || version === 'ff') return null;
+    // Version 00 is exactly four fields; a longer header at version 00 is malformed.
+    if (version === '00' && parts.length !== 4) return null;
+    if (!isUsableId(traceId, TRACE_ID_LENGTH) || !isUsableId(spanId, SPAN_ID_LENGTH)) return null;
+    if (flags.length !== 2 || !HEX.test(flags)) return null;
+
+    return { traceId, spanId, sampled: (parseInt(flags, 16) & FLAG_SAMPLED) !== 0 };
+}
+
+/**
+ * Serialise a trace position as a version-`00` `traceparent` header.
+ *
+ * Ids are normalised to the wire widths — left-padded when short, right-trimmed when
+ * long — so a tracer configured with a non-standard id factory still emits a legal
+ * header rather than one every downstream collector drops.
+ */
+export function formatTraceparent(ctx: TraceContext): string {
+    const traceId = normaliseId(ctx.traceId, TRACE_ID_LENGTH);
+    const spanId = normaliseId(ctx.spanId, SPAN_ID_LENGTH);
+    const flags = (ctx.sampled ? FLAG_SAMPLED : 0).toString(16).padStart(2, '0');
+    return `00-${traceId}-${spanId}-${flags}`;
+}
+
+/** Pad/trim an id to the wire width, dropping any non-hex byte to `0`. */
+function normaliseId(value: string, length: number): string {
+    const hex = (value ?? '').toLowerCase().replace(/[^0-9a-f]/g, '');
+    return hex.length >= length ? hex.slice(0, length) : hex.padStart(length, '0');
+}

package/src/telemetry/types.ts

@@ -0,0 +1,105 @@
+/**
+ * telemetry/types.ts — the observability vocabulary shared by every enterprise
+ * subsystem (ingest, retrieval, orchestration, eval, bridges).
+ *
+ * Enterprise buyers do not ask "is it fast", they ask "show me the span tree for
+ * this request, the token count per model, and what it cost". That needs one
+ * canonical record, not per-subsystem logging. A {@link SpanData} is that record:
+ * a tracing span that ALSO carries LLM-native usage and cost, so a single
+ * exporter pipeline feeds both a trace viewer and the metric aggregates.
+ *
+ * The shape is deliberately OTLP-compatible (trace/span ids, parent linkage,
+ * attributes, events, status) so `toOtlpSpan()` is a pure rename rather than a
+ * re-model — but nothing here depends on the OpenTelemetry SDK, which keeps the
+ * package zero-dependency and browser-safe.
+ */
+
+/** Attribute values are kept primitive so a span serializes to JSON losslessly. */
+export type AttributeValue = string | number | boolean;
+
+export type SpanKind =
+    | 'llm'
+    | 'embedding'
+    | 'retrieval'
+    | 'ingest'
+    | 'agent'
+    | 'tool'
+    | 'graph'
+    | 'internal';
+
+export type SpanStatus = 'unset' | 'ok' | 'error';
+
+export interface SpanEvent {
+    name: string;
+    /** Epoch millis. */
+    at: number;
+    attributes?: Record<string, AttributeValue>;
+}
+
+/**
+ * Token accounting for one model call.
+ *
+ * `cachedInputTokens` and `cacheWriteTokens` are split out because they are
+ * billed at different multipliers than plain input (see {@link PriceBook}); a
+ * cost model that folds them into `inputTokens` overstates spend by ~10x on a
+ * cache-heavy workload, which is exactly the workload enterprises run.
+ */
+export interface LlmUsage {
+    model: string;
+    inputTokens: number;
+    outputTokens: number;
+    /** Input tokens served from the provider's prompt cache (billed at a discount). */
+    cachedInputTokens?: number;
+    /** Input tokens written INTO the provider's prompt cache (billed at a premium). */
+    cacheWriteTokens?: number;
+    /** True when the whole call was served locally (exact/semantic cache) — cost 0. */
+    localCacheHit?: boolean;
+    /** True when token counts are inferred rather than reported by the provider. */
+    estimated?: boolean;
+}
+
+/** A completed (or in-flight) span, ready to export. */
+export interface SpanData {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    kind: SpanKind;
+    /** Epoch millis. */
+    startedAt: number;
+    /** Epoch millis; absent while the span is still open. */
+    endedAt?: number;
+    durationMs?: number;
+    attributes: Record<string, AttributeValue>;
+    events: SpanEvent[];
+    status: SpanStatus;
+    error?: string;
+    /** Present on `llm` spans. */
+    usage?: LlmUsage;
+    /** Derived from `usage` + the tracer's price book, in USD. */
+    costUsd?: number;
+    /** Millis from request start to the first streamed token, when streaming. */
+    timeToFirstTokenMs?: number;
+}
+
+/** Sink for finished spans. Implementations must not throw into the caller. */
+export interface SpanExporter {
+    export(spans: readonly SpanData[]): void | Promise<void>;
+    /** Optional flush hook for buffered/remote exporters. */
+    shutdown?(): void | Promise<void>;
+}
+
+/**
+ * A bridge that reports what its last call actually consumed.
+ *
+ * Optional on {@link TransformerBridge} so third-party bridges stay valid; when
+ * present, {@link InstrumentedBridge} prefers these provider-reported numbers
+ * over its own character-based estimate.
+ */
+export interface BridgeCallInfo {
+    usage?: LlmUsage;
+    /** True when the call never reached the provider (exact or semantic cache hit). */
+    cacheHit?: boolean;
+    /** Which cache tier served it, when `cacheHit`. */
+    cacheTier?: string;
+}