@theoremjs/agents 0.0.0-stage → 0.3.0

package/esm/src/observability/otlp.js

@@ -0,0 +1,98 @@
+// OTLP has no record-level slot for `TraceRecord.metadata` (the resource is the service's
+// identity), so each key lands on the record's top spans as `theorem.metadata.<key>`.
+import { inlineContent } from './trace-record.js';
+const SCOPE_NAME = '@theoremjs/agents';
+const SPAN_KIND = { INTERNAL: 1, CLIENT: 3 };
+const STATUS_CODE = { UNSET: 0, OK: 1, ERROR: 2 };
+function anyValue(value) {
+    if (typeof value === 'string') {
+        return { stringValue: value };
+    }
+    if (typeof value === 'boolean') {
+        return { boolValue: value };
+    }
+    if (typeof value === 'number') {
+        // int64 is a decimal string in OTLP/JSON.
+        return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
+    }
+    if (Array.isArray(value)) {
+        return { arrayValue: { values: value.flatMap((item) => anyValue(item) ?? []) } };
+    }
+    if (value && typeof value === 'object') {
+        return { kvlistValue: { values: keyValues(value) } };
+    }
+    // null / undefined: OTLP has no empty value, so the key is left out.
+    return undefined;
+}
+function keyValues(fields) {
+    return Object.entries(fields).flatMap(([key, field]) => {
+        const value = anyValue(field);
+        return value ? [{ key, value }] : [];
+    });
+}
+function inlined(record, attributes) {
+    return keyValues(inlineContent(record, attributes));
+}
+const METADATA_PREFIX = 'theorem.metadata.';
+function metadataAttributes(record) {
+    return keyValues(record.metadata ?? {}).map(({ key, value }) => ({
+        key: `${METADATA_PREFIX}${key}`,
+        value,
+    }));
+}
+/** Includes the root, which may hang under a host `traceparent` span that lives elsewhere. */
+function topSpanIds(record) {
+    const ids = new Set(record.spans.map((span) => span.spanId));
+    return new Set(record.spans
+        .filter((span) => !(span.parentSpanId && ids.has(span.parentSpanId)))
+        .map((span) => span.spanId));
+}
+function otlpSpan(record, span, metadata) {
+    return {
+        traceId: span.traceId,
+        spanId: span.spanId,
+        ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}),
+        name: span.name,
+        kind: SPAN_KIND[span.kind],
+        startTimeUnixNano: span.startTimeUnixNano,
+        endTimeUnixNano: span.endTimeUnixNano,
+        attributes: [...inlined(record, span.attributes), ...metadata],
+        events: span.events.map((event) => ({
+            timeUnixNano: event.timeUnixNano,
+            name: event.name,
+            attributes: inlined(record, event.attributes),
+        })),
+        links: span.links.map((link) => ({
+            traceId: link.traceId,
+            spanId: link.spanId,
+            attributes: inlined(record, link.attributes),
+        })),
+        status: {
+            code: STATUS_CODE[span.status.code],
+            ...(span.status.message ? { message: span.status.message } : {}),
+        },
+    };
+}
+/**
+ * One resource per record, one scope, with every stored reference inlined. For a backend that
+ * takes protobuf only, send this through an OpenTelemetry Collector.
+ */
+function toOtlpJson(records) {
+    return {
+        resourceSpans: records.map((record) => {
+            const top = topSpanIds(record);
+            const metadata = metadataAttributes(record);
+            return {
+                resource: { attributes: keyValues(record.resource) },
+                scopeSpans: [
+                    {
+                        scope: { name: SCOPE_NAME },
+                        schemaUrl: record.schemaUrl,
+                        spans: record.spans.map((span) => otlpSpan(record, span, top.has(span.spanId) ? metadata : [])),
+                    },
+                ],
+            };
+        }),
+    };
+}
+export { toOtlpJson };

package/esm/src/observability/phoenix.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Eval results as Phoenix span annotations, which Phoenix reads over REST
+ * (`POST /v1/span_annotations`), not OTLP. Pure: it builds the body, the host posts it.
+ *
+ * @module
+ */
+import "../../_dnt.polyfills.js";
+import { type TraceRecord } from './trace-record.js';
+interface PhoenixSpanAnnotation {
+    /** The judged root span, as OTLP hex. */
+    span_id: string;
+    name: string;
+    /** `LLM` for a judge's result, `CODE` for a code grader's. */
+    annotator_kind: 'LLM' | 'CODE';
+    result: {
+        label?: string;
+        score?: number;
+        explanation?: string;
+    };
+    metadata: Record<string, string | number | boolean>;
+}
+/**
+ * Each annotation lands on the root the trial judged. Send as
+ * `{ data: phoenixAnnotations(records) }`; Phoenix keeps one annotation per span and name, so
+ * grading a record again replaces them.
+ */
+declare function phoenixAnnotations(records: readonly TraceRecord[]): PhoenixSpanAnnotation[];
+export type { PhoenixSpanAnnotation };
+export { phoenixAnnotations };

package/esm/src/observability/phoenix.js

@@ -0,0 +1,71 @@
+/**
+ * Eval results as Phoenix span annotations, which Phoenix reads over REST
+ * (`POST /v1/span_annotations`), not OTLP. Pure: it builds the body, the host posts it.
+ *
+ * @module
+ */
+import "../../_dnt.polyfills.js";
+import { inlineContent } from './trace-record.js';
+const TRIAL_SPAN = 'theorem.eval.trial';
+const RESULT_EVENT = 'gen_ai.evaluation.result';
+function scalar(value) {
+    return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'
+        ? value
+        : undefined;
+}
+function present(entries) {
+    const kept = {};
+    for (const [key, value] of Object.entries(entries))
+        if (value !== undefined)
+            kept[key] = value;
+    return kept;
+}
+/** `undefined` when the policy did not keep the explanation. */
+function explanationOf(record, value) {
+    if (value === undefined)
+        return undefined;
+    const text = inlineContent(record, value);
+    return typeof text === 'string' ? text : undefined;
+}
+function annotation(record, span, attributes) {
+    const name = attributes['gen_ai.evaluation.name'];
+    if (typeof name !== 'string')
+        return undefined;
+    const label = attributes['gen_ai.evaluation.score.label'];
+    const score = attributes['gen_ai.evaluation.score.value'];
+    const explanation = explanationOf(record, attributes['gen_ai.evaluation.explanation']);
+    return {
+        span_id: span.parentSpanId,
+        name,
+        annotator_kind: attributes['theorem.evaluation.source'] === 'model' ? 'LLM' : 'CODE',
+        result: {
+            ...(typeof label === 'string' ? { label } : {}),
+            ...(typeof score === 'number' ? { score } : {}),
+            ...(explanation === undefined ? {} : { explanation }),
+        },
+        metadata: present({
+            suite: scalar(span.attributes['theorem.evaluation.suite']),
+            case: scalar(span.attributes['theorem.evaluation.case']),
+            trial: scalar(span.attributes['theorem.evaluation.trial']),
+            passed: scalar(attributes['theorem.evaluation.passed']),
+            error_type: scalar(attributes['error.type']),
+            grader_version: scalar(attributes['theorem.evaluation.grader.version']),
+        }),
+    };
+}
+/**
+ * Each annotation lands on the root the trial judged. Send as
+ * `{ data: phoenixAnnotations(records) }`; Phoenix keeps one annotation per span and name, so
+ * grading a record again replaces them.
+ */
+function phoenixAnnotations(records) {
+    return records.flatMap((record) => record.spans.flatMap((span) => {
+        if (span.name !== TRIAL_SPAN || span.parentSpanId === undefined)
+            return [];
+        const trial = { ...span, parentSpanId: span.parentSpanId };
+        return span.events
+            .filter((event) => event.name === RESULT_EVENT)
+            .flatMap((event) => annotation(record, trial, event.attributes) ?? []);
+    }));
+}
+export { phoenixAnnotations };

package/esm/src/observability/policy.d.ts

@@ -0,0 +1,14 @@
+import type { TraceSink } from './trace-sink.js';
+import type { ProfileObservabilitySpec, ResolvedObservabilityPolicy } from './types.js';
+/**
+ * Precedence: explicit `override` (runTurn's third arg), then profile `writeTo`, then noop. An
+ * override always records, ignoring `sampleRate`, so tests and one-off capture are deterministic.
+ */
+declare function resolveTraceWriter(args: {
+    override?: TraceSink;
+    observability?: ProfileObservabilitySpec;
+}): {
+    sink: TraceSink;
+    policy: ResolvedObservabilityPolicy;
+};
+export { resolveTraceWriter };

package/esm/src/observability/policy.js

@@ -0,0 +1,69 @@
+import { requireTraceDestination } from './destinations.js';
+import { resolveObservabilityPolicy } from './resolve-policy.js';
+import { noopSink } from './trace.js';
+function bindOnWriteError(sink, onWriteError) {
+    if (!onWriteError && !sink.onError) {
+        return sink;
+    }
+    return {
+        write: (record, context) => sink.write(record, context),
+        onError: sink.onError ?? onWriteError,
+    };
+}
+/** Trace-id bits the sampling decision reads: the low 32 (8 hex digits). */
+const SAMPLE_HEX_DIGITS = 8;
+const SAMPLE_SPACE = 2 ** 32;
+/**
+ * Decided by trace id (OpenTelemetry `TraceIdRatioBased`), so every record of one trace (a turn,
+ * its specialists, a Live session's responses, a host's cutout) is kept or dropped together.
+ */
+function traceSampled(record, sampleRate) {
+    const traceId = record.spans[0]?.traceId ?? '';
+    return Number.parseInt(traceId.slice(-SAMPLE_HEX_DIGITS), 16) / SAMPLE_SPACE < sampleRate;
+}
+function withSampleRate(sink, sampleRate) {
+    if (sampleRate >= 1) {
+        return sink;
+    }
+    if (sampleRate <= 0) {
+        return noopSink();
+    }
+    return {
+        write: async (record, context) => {
+            if (traceSampled(record, sampleRate)) {
+                await sink.write(record, context);
+            }
+        },
+        onError: sink.onError,
+    };
+}
+function sinkFromWriteTo(writeTo, policy) {
+    if (writeTo === undefined || writeTo === false) {
+        return noopSink();
+    }
+    if (typeof writeTo !== 'string') {
+        return bindOnWriteError(writeTo, policy.onWriteError);
+    }
+    return bindOnWriteError(requireTraceDestination(writeTo), policy.onWriteError);
+}
+/**
+ * Precedence: explicit `override` (runTurn's third arg), then profile `writeTo`, then noop. An
+ * override always records, ignoring `sampleRate`, so tests and one-off capture are deterministic.
+ */
+function resolveTraceWriter(args) {
+    const policy = resolveObservabilityPolicy(args.observability);
+    if (args.override) {
+        return {
+            policy,
+            sink: bindOnWriteError(args.override, policy.onWriteError),
+        };
+    }
+    if (!policy.record) {
+        return { policy, sink: noopSink() };
+    }
+    return {
+        policy,
+        sink: withSampleRate(sinkFromWriteTo(policy.writeTo, policy), policy.sampleRate),
+    };
+}
+export { resolveTraceWriter };

package/esm/src/observability/resolve-policy.d.ts

@@ -0,0 +1,8 @@
+import type { ProfileObservabilitySpec, ResolvedObservabilityPolicy } from './types.js';
+/** Default record retention in days; `<= 0` keeps records forever. */
+declare const DEFAULT_RETAIN_DAYS = 14;
+/** Default JSONL file size, in MiB, before the writer rotates to a new file. */
+declare const DEFAULT_ROTATE_MIB = 32;
+/** An omitted block or `writeTo: false` resolves `record: false`; `sampleRate` applies at write. */
+declare function resolveObservabilityPolicy(spec: ProfileObservabilitySpec | undefined): ResolvedObservabilityPolicy;
+export { DEFAULT_RETAIN_DAYS, DEFAULT_ROTATE_MIB, resolveObservabilityPolicy };

package/esm/src/observability/resolve-policy.js

@@ -0,0 +1,58 @@
+// Pure, with no sinks or file system, so kernel type consumers never pull the JSONL sink into their
+// graph.
+import { TheoremError } from '../guardrails/error.js';
+/** Default record retention in days; `<= 0` keeps records forever. */
+const DEFAULT_RETAIN_DAYS = 14;
+/** Default JSONL file size, in MiB, before the writer rotates to a new file. */
+const DEFAULT_ROTATE_MIB = 32;
+function resolveInclude(spec) {
+    // No authored block records only through an explicit capture sink (tests,
+    // `runTurn(..., sink)`), which keeps wire and raw evidence. Authored blocks
+    // default those two off.
+    const authored = spec !== undefined;
+    return {
+        upstreamLog: spec?.include?.upstreamLog ?? true,
+        outboundWire: spec?.include?.outboundWire ?? !authored,
+        evidenceRaw: spec?.include?.evidenceRaw ?? !authored,
+        usage: spec?.include?.usage ?? true,
+        guardrailDecisions: spec?.include?.guardrailDecisions ?? true,
+        guardrailMatchPreview: spec?.include?.guardrailMatchPreview ?? false,
+    };
+}
+function resolveScrub(spec) {
+    return {
+        sensitive: spec?.scrub?.sensitive ?? true,
+        injection: spec?.scrub?.injection ?? true,
+        canary: spec?.scrub?.canary ?? true,
+    };
+}
+function clampSampleRate(value) {
+    if (value === undefined) {
+        return 1;
+    }
+    if (!Number.isFinite(value)) {
+        throw new TheoremError('config', 'observability.sampleRate must be a finite number');
+    }
+    if (value < 0 || value > 1) {
+        throw new TheoremError('config', 'observability.sampleRate must be between 0 and 1 inclusive');
+    }
+    return value;
+}
+/** An omitted block or `writeTo: false` resolves `record: false`; `sampleRate` applies at write. */
+function resolveObservabilityPolicy(spec) {
+    const writeTo = spec?.writeTo;
+    const sampleRate = clampSampleRate(spec?.sampleRate);
+    const record = spec !== undefined && writeTo !== false && writeTo !== undefined;
+    return {
+        record,
+        writeTo,
+        sampleRate,
+        include: resolveInclude(spec),
+        scrub: resolveScrub(spec),
+        resource: { ...(spec?.resource ?? {}) },
+        retainForDays: spec?.retainForDays ?? DEFAULT_RETAIN_DAYS,
+        rotateAfterMiB: spec?.rotateAfterMiB ?? DEFAULT_ROTATE_MIB,
+        onWriteError: spec?.onWriteError,
+    };
+}
+export { DEFAULT_RETAIN_DAYS, DEFAULT_ROTATE_MIB, resolveObservabilityPolicy };

package/esm/src/observability/spans.d.ts

@@ -0,0 +1,16 @@
+declare const OMIT_INJECTION = "[omitted - injection]";
+declare const OMIT_SENSITIVE = "[omitted -sensitive]";
+type RedactKind = 'injection' | 'sensitive';
+interface RedactSpan {
+    start: number;
+    end: number;
+    kind: RedactKind;
+}
+declare function blobAt(match: RegExpMatchArray): {
+    blob: string;
+    index: number;
+} | undefined;
+declare function applySpans(text: string, spans: RedactSpan[]): string;
+declare function spansFromPatterns(text: string, patterns: RegExp[], kind: RedactKind): RedactSpan[];
+export type { RedactKind, RedactSpan };
+export { applySpans, blobAt, OMIT_INJECTION, OMIT_SENSITIVE, spansFromPatterns };

package/esm/src/observability/spans.js

@@ -0,0 +1,56 @@
+const OMIT_INJECTION = '[omitted - injection]';
+const OMIT_SENSITIVE = '[omitted -sensitive]';
+function tokenFor(kind) {
+    if (kind === 'injection') {
+        return OMIT_INJECTION;
+    }
+    return OMIT_SENSITIVE;
+}
+function blobAt(match) {
+    const [blob] = match;
+    const { index } = match;
+    if (blob && index !== undefined) {
+        return { blob, index };
+    }
+    return undefined;
+}
+function mergeSpans(spans) {
+    const sorted = [...spans].sort((left, right) => left.start - right.start || right.end - left.end);
+    const out = [];
+    for (const span of sorted) {
+        if (span.end > span.start) {
+            const last = out.at(-1);
+            if (!last || span.start >= last.end) {
+                out.push(span);
+            }
+            else if (span.end > last.end) {
+                last.end = span.end;
+            }
+        }
+    }
+    return out;
+}
+function applySpans(text, spans) {
+    let out = text;
+    const merged = mergeSpans(spans);
+    for (let i = merged.length - 1; i >= 0; i -= 1) {
+        const span = merged[i];
+        if (span) {
+            out = out.slice(0, span.start) + tokenFor(span.kind) + out.slice(span.end);
+        }
+    }
+    return out;
+}
+function spansFromPatterns(text, patterns, kind) {
+    const spans = [];
+    for (const pattern of patterns) {
+        for (const match of text.matchAll(pattern)) {
+            const found = blobAt(match);
+            if (found) {
+                spans.push({ start: found.index, end: found.index + found.blob.length, kind });
+            }
+        }
+    }
+    return spans;
+}
+export { applySpans, blobAt, OMIT_INJECTION, OMIT_SENSITIVE, spansFromPatterns };

package/esm/src/observability/trace-catalog.d.ts

@@ -0,0 +1,111 @@
+import type { TraceSpan } from './trace-span.js';
+/** How a value reads: the unit or shape a viewer formats it by. */
+type TraceValueFormat = 'text'
+/** An identifier: shown as is, in monospace. */
+ | 'id' | 'number' | 'tokens' | 'seconds' | 'milliseconds' | 'usd' | 'boolean'
+/** A list of strings. */
+ | 'list' | 'object'
+/** A `{ content_sha256 }` reference to text in the record's `content`. */
+ | 'content'
+/** A `{ json_sha256 }` reference to JSON in the record's `content`. */
+ | 'json'
+/** Semconv chat messages whose parts reference `content`. */
+ | 'messages'
+/** Semconv message parts whose text references `content`. */
+ | 'parts'
+/** A unix-nanosecond timestamp string. */
+ | 'time';
+type TraceAttributeGroup = 'agent' | 'request' | 'response' | 'messages' | 'usage' | 'tool' | 'http' | 'error' | 'record' | 'part' | 'decision' | 'evaluation';
+interface TraceOptionMeta {
+    label: string;
+    doc: string;
+}
+interface TraceAttributeMeta {
+    label: string;
+    doc: string;
+    format: TraceValueFormat;
+    group: TraceAttributeGroup;
+    options?: Readonly<Record<string, TraceOptionMeta>>;
+    /** The options name the common values only: another value is real and shows as is. */
+    open?: true;
+    /** For an `object`: what its keys mean (for a list of objects, each item's keys). */
+    fields?: Readonly<Record<string, TraceAttributeMeta>>;
+}
+interface TraceEventMeta {
+    label: string;
+    doc: string;
+    /** Keys this event carries. A key missing here reads from the span attribute catalog. */
+    attributes: Readonly<Record<string, TraceAttributeMeta>>;
+}
+/** The kinds of span THEOREM writes, plus `host` for a span the host recorded itself. */
+type TraceSpanType = 'turn' | 'session' | 'call' | 'response' | 'tool' | 'http' | 'cutout' | 'decision' | 'evaluation' | 'host';
+/** `subject` is the thing the span acted on (the model, tool or agent), when it names one. */
+interface TraceSpanMeta {
+    type: TraceSpanType;
+    label: string;
+    doc: string;
+    subject?: string;
+}
+declare const TRACE_ATTRIBUTE_GROUPS: Readonly<Record<TraceAttributeGroup, TraceOptionMeta>>;
+declare const TRACE_STATUS: Readonly<Record<TraceSpan['status']['code'], TraceOptionMeta>>;
+/** A record's and a span's own fields, beside their attributes. */
+declare const TRACE_FIELDS: {
+    resource: {
+        label: string;
+        doc: string;
+    };
+    metadata: {
+        label: string;
+        doc: string;
+    };
+    span: {
+        label: string;
+        doc: string;
+    };
+    type: {
+        label: string;
+        doc: string;
+    };
+    children: {
+        label: string;
+        doc: string;
+    };
+    traceId: {
+        label: string;
+        doc: string;
+    };
+    spanId: {
+        label: string;
+        doc: string;
+    };
+    status: {
+        label: string;
+        doc: string;
+    };
+    start: {
+        label: string;
+        doc: string;
+    };
+    duration: {
+        label: string;
+        doc: string;
+    };
+    events: {
+        label: string;
+        doc: string;
+    };
+    links: {
+        label: string;
+        doc: string;
+    };
+};
+/** What an attribute key means; `undefined` for a key THEOREM does not write. */
+declare function traceAttributeMeta(key: string): TraceAttributeMeta | undefined;
+/** What an event records; `undefined` for an event THEOREM does not write. */
+declare function traceEventMeta(name: string): TraceEventMeta | undefined;
+/** The event's own entry, else the span attribute of that key. */
+declare function traceEventAttributeMeta(event: string, key: string): TraceAttributeMeta | undefined;
+declare const TRACE_SPAN_TYPES: Readonly<Record<TraceSpanType, TraceOptionMeta>>;
+declare function traceSpanMeta(span: TraceSpan): TraceSpanMeta;
+export type { TraceAttributeGroup, TraceAttributeMeta, TraceEventMeta, TraceOptionMeta, TraceSpanMeta, TraceSpanType, TraceValueFormat, };
+export { TRACE_ATTRIBUTE_GROUPS, TRACE_FIELDS, TRACE_SPAN_TYPES, TRACE_STATUS, traceAttributeMeta, traceEventAttributeMeta, traceEventMeta, traceSpanMeta, };