@theoremjs/agents 0.0.0-stage → 0.3.1

package/esm/src/kernel/engine/tool-trace.d.ts

@@ -0,0 +1,58 @@
+import type { GuardrailEvent, ToolOrigin } from '../../guardrails/types.js';
+import { type SpanHandle, type TraceAttributes } from '../../observability/trace-span.js';
+import type { ToolPermission } from '../schema.js';
+import type { ToolFailure } from '../tools/types.js';
+import type { InteractionPart, TurnEvent } from '../types.js';
+import { type GuardrailCheck, type SentToolCall } from './turn-trace.js';
+/** A check at the tool boundary: the call's arguments, the remote-content gate, its destinations, the result, or the network target. */
+type ToolCheck = Extract<GuardrailCheck, 'tool_arguments' | 'taint' | 'destination' | 'tool_result' | 'tool_failure' | 'network' | 'network_request'>;
+/**
+ * Records one tool-boundary check on the tool's span with the time it took:
+ * its decision, or a pass (`allow`) when it let the call or result through.
+ * Trace only; the host still hears about decisions alone. An untraced call has no span.
+ */
+declare function recordToolCheck(span: SpanHandle | undefined, check: ToolCheck, durationMs: number, guardrail: GuardrailEvent | undefined): void;
+/** How a tool call ended. Only `error` is a failure; the rest were stopped or finished. */
+type ToolOutcome = 'ok' | 'error' | 'denied' | 'gated' | 'paused' | 'cancelled';
+interface ToolCallStart {
+    name: string;
+    callId: string;
+    call: SentToolCall;
+    /** Absent when the tool is not registered. */
+    origin?: ToolOrigin;
+    permission?: ToolPermission;
+    /** The host's answer to an earlier gate, when this call resumes one. */
+    approved?: boolean;
+    /** The turn step whose model call asked for this tool. */
+    step?: number;
+}
+interface ToolCallEnd {
+    outcome: ToolOutcome;
+    /** What the model reads back: its text and any media. Absent when nothing is read back. */
+    result?: {
+        text: string;
+        parts?: readonly InteractionPart[];
+    };
+    /** The tool's raw output, present only when its body completed. */
+    data?: {
+        value: unknown;
+    };
+    /** The failure, on `error`: its kind (`error.type`) and the tool's code for it. */
+    failure?: Pick<ToolFailure, 'code' | 'kind'>;
+    /** Thrown out of the call (not an abort). */
+    thrown?: unknown;
+}
+interface ToolCallTrace {
+    span: SpanHandle;
+    observe: (event: TurnEvent) => void;
+    end: (end: ToolCallEnd) => void;
+}
+declare function toolSpanName(name: string): string;
+/**
+ * Spans from before `pre_tool` to settlement, so it covers the hooks, the gates, and the body.
+ * Provider-run tools (search, maps, code execution) get no span: Theorem did not run them.
+ * `open` places it: a child of the turn's root, or the root of a host-invoked tool's own record.
+ */
+declare function startToolTrace(open: (name: string, attributes: TraceAttributes) => SpanHandle, start: ToolCallStart): ToolCallTrace;
+export type { ToolCallEnd, ToolCallStart, ToolCallTrace, ToolCheck, ToolOutcome };
+export { recordToolCheck, startToolTrace, toolSpanName };

package/esm/src/kernel/engine/tool-trace.js

@@ -0,0 +1,132 @@
+import { errorKind } from '../../guardrails/error.js';
+import { traceContent, traceJson, } from '../../observability/trace-span.js';
+import { authScopeRefusedSchema } from '../auth/scope-refusal.js';
+import { guardrailAttributes, guardrailCheckAttributes, optional, recordException, toolArgumentsText, tracePart, } from './turn-trace.js';
+const TOOL_CHECK_STAGE = {
+    tool_arguments: 'tool_call',
+    taint: 'tool_call',
+    destination: 'tool_call',
+    tool_result: 'tool_result',
+    tool_failure: 'tool_result',
+    network: 'network',
+    network_request: 'network',
+};
+/** Decisions already recorded with their check's time, so `observe` does not record them twice. */
+const timedDecisions = new WeakSet();
+/**
+ * Records one tool-boundary check on the tool's span with the time it took:
+ * its decision, or a pass (`allow`) when it let the call or result through.
+ * Trace only; the host still hears about decisions alone. An untraced call has no span.
+ */
+function recordToolCheck(span, check, durationMs, guardrail) {
+    if (!span)
+        return;
+    if (guardrail)
+        timedDecisions.add(guardrail);
+    span.event('theorem.guardrail', guardrailCheckAttributes(check, durationMs, guardrail, {
+        stage: TOOL_CHECK_STAGE[check],
+        trust: 'untrusted',
+    }));
+}
+function toolSpanName(name) {
+    return name ? `execute_tool ${name}` : 'execute_tool';
+}
+function toolSpanAttributes(start) {
+    return {
+        'gen_ai.operation.name': 'execute_tool',
+        'gen_ai.tool.name': start.name,
+        'gen_ai.tool.call.id': start.callId,
+        'gen_ai.tool.type': 'function',
+        'gen_ai.tool.call.arguments': traceContent(toolArgumentsText(start.call)),
+        ...optional('theorem.tool.origin', start.origin),
+        ...optional('theorem.tool.permission', start.permission),
+        ...optional('theorem.tool.approved', start.approved),
+        ...optional('theorem.step', start.step),
+    };
+}
+/** A gate as `theorem.gate` event attributes. The auth `state` is a secret and never recorded. */
+function gateAttributes(gate) {
+    const auth = gate.kind === 'auth' ? gate.authChallenge : undefined;
+    return {
+        kind: gate.kind,
+        ...optional('permission', gate.permission),
+        ...(gate.summary ? { summary: traceContent(gate.summary) } : {}),
+        ...(auth
+            ? {
+                auth: {
+                    slot: auth.slot,
+                    type: auth.authType,
+                    service: auth.service,
+                    ...optional('issuer', auth.issuer),
+                    ...optional('resource', auth.resource),
+                    ...optional('required_scopes', auth.requiredScopes),
+                },
+            }
+            : {}),
+    };
+}
+const OUTCOME_STATUS = {
+    ok: 'OK',
+    error: 'ERROR',
+    denied: 'UNSET',
+    gated: 'UNSET',
+    paused: 'UNSET',
+    cancelled: 'UNSET',
+};
+/**
+ * Spans from before `pre_tool` to settlement, so it covers the hooks, the gates, and the body.
+ * Provider-run tools (search, maps, code execution) get no span: Theorem did not run them.
+ * `open` places it: a child of the turn's root, or the root of a host-invoked tool's own record.
+ */
+function startToolTrace(open, start) {
+    const span = open(toolSpanName(start.name), toolSpanAttributes(start));
+    return {
+        span,
+        observe: (event) => {
+            if (event.type === 'guardrail' && !timedDecisions.has(event.guardrail)) {
+                span.event('theorem.guardrail', guardrailAttributes(event.guardrail));
+            }
+            if (event.type === 'tool' && event.tool.phase === 'gate') {
+                span.event('theorem.gate', gateAttributes(event.tool.gate));
+            }
+            if (event.type === 'tool' && event.tool.phase === 'progress') {
+                const refused = authScopeRefusedSchema.safeParse(event.tool.data);
+                if (refused.success) {
+                    const { slot, requested, declared } = refused.data;
+                    span.event('theorem.auth.scope_refused', { slot, requested, declared });
+                }
+            }
+            if (event.type === 'tool' && event.tool.phase === 'warning') {
+                const { code, message, severity } = event.tool.warning;
+                span.event('theorem.tool.warning', {
+                    code,
+                    message: traceContent(message),
+                    ...optional('severity', severity),
+                });
+            }
+            if (event.type === 'citation') {
+                span.event('theorem.grounding', { sources: traceJson(event.sources) });
+            }
+        },
+        end: (end) => {
+            if (end.thrown !== undefined)
+                recordException(span, end.thrown);
+            const errorType = end.outcome === 'error'
+                ? (end.failure?.kind ?? (end.thrown === undefined ? undefined : errorKind(end.thrown)))
+                : undefined;
+            span.set({
+                ...(end.result ? { 'gen_ai.tool.call.result': traceContent(end.result.text) } : {}),
+                ...(end.result?.parts?.length
+                    ? { 'theorem.tool.call.result.parts': end.result.parts.map(tracePart) }
+                    : {}),
+                ...(end.data ? { 'theorem.tool.data': traceJson(end.data.value) } : {}),
+                'theorem.tool.outcome': end.outcome,
+                ...optional('theorem.tool.failure.code', end.failure?.code),
+                ...optional('error.type', errorType),
+            });
+            const code = OUTCOME_STATUS[end.outcome];
+            span.end(code === 'ERROR' && errorType ? { code, message: errorType } : { code });
+        },
+    };
+}
+export { recordToolCheck, startToolTrace, toolSpanName };

package/esm/src/kernel/engine/tree.d.ts

@@ -0,0 +1,2 @@
+declare function mapStrings(value: unknown, map: (text: string) => string): unknown;
+export { mapStrings };

package/esm/src/kernel/engine/tree.js

@@ -0,0 +1,17 @@
+function mapStrings(value, map) {
+    if (typeof value === 'string') {
+        return map(value);
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => mapStrings(item, map));
+    }
+    if (value && typeof value === 'object') {
+        const out = {};
+        for (const [key, nested] of Object.entries(value)) {
+            out[key] = mapStrings(nested, map);
+        }
+        return out;
+    }
+    return value;
+}
+export { mapStrings };

package/esm/src/kernel/engine/turn-trace.d.ts

@@ -0,0 +1,119 @@
+import type { GuardrailEvent, GuardrailStage, TrustLevel } from '../../guardrails/types.js';
+import { type SpanHandle, type SpanLinkInput, type SpanOptions, type TraceAttributes, type TraceAttributeValue } from '../../observability/trace-span.js';
+import type { ToolFailure } from '../tools/types.js';
+import type { InteractionPart, ModelBinding, ModelEvent, ModelRequest, ProviderTransport, TurnEvent, TurnRequest, TurnStop, TurnTokens, TurnTraceLink } from '../types.js';
+import type { CallUsage } from './runner/usage.js';
+type TracePart = {
+    [key: string]: TraceAttributeValue;
+};
+declare function tracePart(part: InteractionPart): TracePart;
+/** Output parts folded from provider events: adjacent deltas of one kind merge. */
+declare class OutputFold {
+    readonly parts: TracePart[];
+    private open?;
+    /** Each call's `tool_call` part by `callId`, for the failure that follows a malformed one. */
+    private readonly calls;
+    add(event: ModelEvent, nowUnixNano: string): void;
+    private appendText;
+    /**
+     * The model's call as a `tool_call` part. A malformed call's failure follows
+     * it from the provider; the part then records the text the model sent.
+     */
+    private addTool;
+    private push;
+}
+interface SentToolCall {
+    arguments: Record<string, unknown>;
+    failure?: ToolFailure;
+}
+/**
+ * A tool call's arguments as the model sent them. Malformed arguments keep the
+ * provider's raw text; parsed ones are the same JSON the kernel sends back in
+ * history, so both hash alike.
+ */
+declare function toolArgumentsText(call: SentToolCall): string;
+/**
+ * A guardrail decision as `theorem.guardrail` event attributes. The matched
+ * substring stays exact (not scrubbed: scrubbing it would erase what it shows);
+ * the record drops it unless the profile includes `guardrailMatchPreview`.
+ */
+declare function guardrailAttributes(guardrail: GuardrailEvent): TraceAttributes;
+/**
+ * A guardrail check whose time the trace records: the turn's input and output,
+ * each tool boundary, the checks that run on streamed output, and a live
+ * session's own gates.
+ */
+type GuardrailCheck = 'input' | 'egress' | 'tool_arguments' | 'taint' | 'destination' | 'tool_result' | 'tool_failure' | 'network' | 'network_request' | StreamCheck | 'live_input';
+/**
+ * A check that runs many times on one model call's streamed output: the
+ * progressive gate on each piece of reply text, the canary scan of every other
+ * streamed event, the gate on each piece of a tool call the model is drafting,
+ * and a live session's gate on each batch it sends.
+ */
+type StreamCheck = 'output_stream' | 'stream_canary' | 'tool_draft' | 'live_output';
+/**
+ * One timed guardrail check as `theorem.guardrail` attributes: its decision,
+ * or a pass (`allow`, no hits) when it let the text through. Trace only; the
+ * host still hears about hits alone.
+ */
+declare function guardrailCheckAttributes(check: GuardrailCheck, durationMs: number, guardrail: GuardrailEvent | undefined, passed: {
+    stage: GuardrailStage;
+    trust: TrustLevel;
+}): TraceAttributes;
+declare function optional(key: string, value: TraceAttributeValue | undefined): TraceAttributes;
+declare function usageAttributes(tokens: TurnTokens): TraceAttributes;
+declare function urlAttributes(url: unknown): TraceAttributes;
+declare function recordException(span: SpanHandle, err: unknown): void;
+/** Close a root that a throw ended: the exception, its kind as `error.type`, and ERROR. */
+declare function endThrownSpan(span: SpanHandle, err: unknown): void;
+interface CallEnd {
+    /** The call's one usage report (estimated sides filled), when there is one. */
+    tokens?: TurnTokens;
+    stop?: TurnStop;
+    /** Thrown out of the stream (not an abort). */
+    thrown?: unknown;
+}
+interface CallTrace {
+    span: SpanHandle;
+    /** Pass as the adapter's `tapUpstream`. */
+    tap: (row: Record<string, unknown>) => void;
+    /** Every provider event, before guardrails. */
+    observe: (event: ModelEvent) => void;
+    /** A stream check's decision; it carries the time that check has spent on the call so far. */
+    guardrail: (event: GuardrailEvent) => void;
+    /** Add one run of a stream check to this call. */
+    guardTime: (check: StreamCheck, ms: number) => void;
+    end: (end: CallEnd) => void;
+}
+/**
+ * `open` places the span: a child of the turn's root, or the root of a Live response's own record.
+ * `usage.conversation` is read again at the end, since a Live response sets it only when it closes.
+ */
+declare function startCallTrace(open: (name: string, options: SpanOptions) => SpanHandle, args: {
+    req: ModelRequest;
+    usage: CallUsage;
+    binding: ModelBinding | undefined;
+    transport: ProviderTransport;
+    step: number;
+    /** The runner's validation attempt; Live has none. */
+    attempt?: number;
+}): CallTrace;
+declare function traceLinks(links: readonly TurnTraceLink[] | undefined): SpanLinkInput[];
+declare function turnSpanOptions(req: TurnRequest): SpanOptions;
+interface TurnEnd {
+    seen: readonly TurnEvent[];
+    /** Those events as output parts, folded as they were delivered. */
+    delivered: OutputFold;
+    /** Validation / egress attempts that made a model call. */
+    attempts: number;
+    calls: number;
+    /** Thrown out of the turn (not an abort). */
+    thrown?: unknown;
+}
+/**
+ * Close a turn's `invoke_agent` span. Usage is the sum of this turn's own
+ * calls; a nested turn (compaction) carries its own.
+ */
+declare function endTurnSpan(root: SpanHandle, end: TurnEnd): void;
+export type { CallEnd, CallTrace, GuardrailCheck, SentToolCall, StreamCheck, TracePart, TurnEnd };
+export { endThrownSpan, endTurnSpan, guardrailAttributes, guardrailCheckAttributes, OutputFold, optional, recordException, startCallTrace, toolArgumentsText, traceLinks, tracePart, turnSpanOptions, urlAttributes, usageAttributes, };