@loompad/cli 0.1.0 → 1.1.1

package/dist/observability/index.js

@@ -0,0 +1,348 @@
+/**
+ * The Loom observability layer: folds LoomEvents into OpenTelemetry signals
+ * and ships them over OTLP. One process-wide config, three exporters.
+ *
+ * Mapping (LoomEvent → span, GenAI semantic conventions where they apply):
+ *   run_complete            → gen_ai.agent.turn   (duration, cost, tokens, model)
+ *   tool_call               → gen_ai.tool.call
+ *   handoff                 → loom.baton.handoff
+ *   route_*                 → loom.route.<phase>
+ *   memory_add/update/forget→ loom.memory.<op>
+ *   error                   → span with ERROR status
+ *
+ * Metrics (LoomEvent → datapoint, see ./metrics.ts for names and temporality):
+ *   run_complete            → gen_ai.client.token.usage, gen_ai.client.operation.duration,
+ *                             loom.turns, loom.cost.usd (only if reported)
+ *   status turn_started     → loom.agents.active
+ *   handoff                 → loom.handoffs
+ *
+ * Logs (LoomEvent → log record, see ./logs.ts): the event stream itself,
+ * correlated to the turn's trace so a span links to what the agent said.
+ *
+ * One event can produce all three. Callers still only call recordAgentEvent().
+ */
+import { LoomTelemetry, newTraceId, resolveTelemetryConfig } from "./otlp.js";
+import { LoomMetrics } from "./metrics.js";
+import { LoomLogs, eventToLogRecord } from "./logs.js";
+let singleton = null;
+let metricsSingleton = null;
+let logsSingleton = null;
+let sharedConfig = null;
+// One trace per agent turn: minted when a turn starts, reused by every span in
+// that turn (tool calls, the completion), cleared when it ends — so the backend
+// shows a real span tree per turn instead of one orphan span per event.
+const turnTrace = new Map();
+/**
+ * The trace of the turn that most recently CLOSED, per agent. The decision
+ * miner runs off `run_complete` — i.e. after the turn's trace has been retired
+ * — and still needs the id to link each decision to the spans it came out of.
+ * One entry per agent, so this is bounded by the roster, not by history.
+ */
+const lastTurnTrace = new Map();
+/**
+ * The trace id a span belongs to, advancing the per-agent turn as it goes.
+ *
+ * Also reports the turn boundary it just crossed, because the trace map is
+ * already the one place that knows where an agent's turn begins and ends.
+ * `opened` is true on the event that started a turn, `closed` on the one that
+ * ended it — which is what the active-agents gauge counts.
+ */
+function traceIdFor(event) {
+    const agent = event.agentId ?? "";
+    const isTurnStart = event.kind === "status" && event.payload?.state === "turn_started";
+    const opened = isTurnStart || !turnTrace.has(agent);
+    if (opened)
+        turnTrace.set(agent, newTraceId());
+    const traceId = turnTrace.get(agent);
+    // A turn ends when it completes — or when it dies. An adapter that throws
+    // emits `error` and may never reach run_complete; treating only the happy
+    // path as an ending would leave the trace (and the gauge) open forever.
+    const closed = event.kind === "run_complete" || event.kind === "error";
+    if (closed) {
+        lastTurnTrace.set(agent, traceId);
+        turnTrace.delete(agent); // this span closes the turn
+    }
+    return { traceId, opened, closed };
+}
+/**
+ * The trace id of an agent's in-flight turn, or the one that just finished.
+ *
+ * Undefined when telemetry is off, because then no trace was ever minted and
+ * there is nothing in the backend to link to. Callers must leave the field out
+ * rather than substituting "" — an empty trace id renders as a link that opens
+ * a search for nothing, which reads as a broken feature rather than an absent
+ * one.
+ */
+export function turnTraceId(agentId) {
+    return turnTrace.get(agentId) ?? lastTurnTrace.get(agentId);
+}
+/**
+ * One resolved config for all three signals.
+ *
+ * Resolved once and shared so traces, metrics and logs can never disagree about
+ * the endpoint or the service name — a metric filed under a different
+ * service.name than its trace is invisible in the backend's service view, and
+ * that failure is silent.
+ */
+function config() {
+    if (!sharedConfig)
+        sharedConfig = resolveTelemetryConfig();
+    return sharedConfig;
+}
+export function telemetry() {
+    if (!singleton)
+        singleton = new LoomTelemetry(config());
+    return singleton;
+}
+export function metrics() {
+    if (!metricsSingleton)
+        metricsSingleton = new LoomMetrics(config());
+    return metricsSingleton;
+}
+export function logs() {
+    if (!logsSingleton)
+        logsSingleton = new LoomLogs(config());
+    return logsSingleton;
+}
+/** Push every buffered signal now — for shutdown, and for tests that assert. */
+export function flushTelemetry() {
+    singleton?.flush();
+    metricsSingleton?.flush();
+    logsSingleton?.flush();
+}
+const MS_TO_NS = 1000000n;
+function nsWindow(endMs, durationMs) {
+    const end = BigInt(Math.max(0, Math.round(endMs)));
+    const dur = BigInt(Math.max(0, Math.round(durationMs)));
+    return { startNs: (end - dur) * MS_TO_NS, endNs: end * MS_TO_NS };
+}
+function num(v) {
+    const n = Number(v);
+    return Number.isFinite(n) ? n : undefined;
+}
+/**
+ * Pure mapper: a LoomEvent → the span it should emit, or null for kinds that
+ * aren't telemetry-worthy. Exported so the mapping is unit-testable without a
+ * live exporter or collector.
+ */
+export function eventToSpan(event, ctx = {}) {
+    const p = event.payload ?? {};
+    const base = {
+        "loom.project": ctx.project,
+        "loom.chat": event.chat,
+        "loom.event.kind": event.kind,
+        "gen_ai.agent.id": event.agentId,
+    };
+    switch (event.kind) {
+        case "run_complete": {
+            const durationMs = num(p.durationMs) ?? 0;
+            return {
+                name: "gen_ai.agent.turn",
+                ...nsWindow(event.ts, durationMs),
+                attributes: {
+                    ...base,
+                    "gen_ai.system": str(p.adapter) || str(p.system) || "loom",
+                    "gen_ai.operation.name": "chat",
+                    "gen_ai.request.model": str(p.model) || undefined,
+                    "gen_ai.usage.input_tokens": num(p.tokensIn) ?? num(p.inputTokens),
+                    "gen_ai.usage.output_tokens": num(p.tokensOut) ?? num(p.outputTokens),
+                    "gen_ai.usage.cost_usd": num(p.costUsd),
+                    "loom.turn.duration_ms": durationMs,
+                },
+                error: p.error ? str(p.error) : undefined,
+            };
+        }
+        case "tool_call":
+            return {
+                name: "gen_ai.tool.call",
+                ...nsWindow(event.ts, num(p.durationMs) ?? 0),
+                attributes: { ...base, "gen_ai.tool.name": str(p.tool) || str(p.name) || "tool" },
+            };
+        case "handoff":
+            return {
+                name: "loom.baton.handoff",
+                ...nsWindow(event.ts, 0),
+                attributes: {
+                    ...base,
+                    "loom.handoff.to": str(p.to) || str(p.agent),
+                    "loom.handoff.from": str(p.from) || event.agentId,
+                },
+            };
+        case "route_started":
+        case "route_step":
+        case "route_paused":
+        case "route_resumed":
+        case "route_completed":
+        case "route_failed":
+            return {
+                name: `loom.route.${event.kind.slice("route_".length)}`,
+                ...nsWindow(event.ts, num(p.durationMs) ?? 0),
+                attributes: { ...base, "loom.route.id": str(p.routeId) || str(p.id) },
+                error: event.kind === "route_failed" ? str(p.error) || "route failed" : undefined,
+            };
+        case "memory_add":
+        case "memory_update":
+        case "memory_forget":
+            return {
+                name: `loom.memory.${event.kind.slice("memory_".length)}`,
+                ...nsWindow(event.ts, 0),
+                attributes: {
+                    ...base,
+                    "loom.memory.kind": str(p.kind) || str(p.type),
+                    "loom.memory.scope": str(p.scope),
+                },
+            };
+        case "error":
+            return { name: "loom.error", ...nsWindow(event.ts, 0), attributes: base, error: str(p.message) || "error" };
+        default:
+            return null;
+    }
+}
+/**
+ * Pure mapper: a LoomEvent → the datapoints it should produce.
+ *
+ * The rule that governs every branch: a number goes out only if the event
+ * carried it. An adapter that reports tokens but no cost produces token
+ * datapoints and no cost datapoint — not a cost datapoint of 0. Codex is
+ * exactly this case, and the reasoning is written out at the top of
+ * src/adapters/codex.ts: a fake $0.001 presented as fact is worse than a blank.
+ * On a counter it is worse still, because zeros are indistinguishable from real
+ * cheap turns once summed, and the total then reads as authoritative.
+ */
+export function eventToMetrics(event) {
+    const p = event.payload ?? {};
+    const ops = [];
+    if (event.kind === "run_complete") {
+        const agent = event.agentId;
+        const model = str(p.model) || undefined;
+        const system = str(p.adapter) || str(p.system) || undefined;
+        // The GenAI convention's dimensions. `gen_ai.operation.name` is "chat"
+        // because that is what an agent turn is under the convention's vocabulary.
+        const genai = {
+            "gen_ai.agent.id": agent,
+            "gen_ai.request.model": model,
+            "gen_ai.system": system,
+            "gen_ai.operation.name": "chat",
+        };
+        const tokensIn = num(p.tokensIn) ?? num(p.inputTokens);
+        const tokensOut = num(p.tokensOut) ?? num(p.outputTokens);
+        // > 0 rather than != null: adapters that don't track usage write a literal
+        // 0 into these fields (see runtime.trackCost), and a 0-token turn is not a
+        // thing that happens. Counting those would report a busy agent as free.
+        if (tokensIn !== undefined && tokensIn > 0) {
+            ops.push({ op: "count", name: "gen_ai.client.token.usage", value: tokensIn, isInt: true, attributes: { ...genai, "gen_ai.token.type": "input" } });
+        }
+        if (tokensOut !== undefined && tokensOut > 0) {
+            ops.push({ op: "count", name: "gen_ai.client.token.usage", value: tokensOut, isInt: true, attributes: { ...genai, "gen_ai.token.type": "output" } });
+        }
+        const durationMs = num(p.durationMs);
+        if (durationMs !== undefined && durationMs >= 0) {
+            // The convention specifies seconds, not milliseconds. Loom measures in
+            // ms everywhere else; convert here rather than shipping a metric that
+            // says "s" and holds ms.
+            ops.push({ op: "histogram", name: "gen_ai.client.operation.duration", value: durationMs / 1000, attributes: genai });
+        }
+        ops.push({
+            op: "count",
+            name: "loom.turns",
+            value: 1,
+            isInt: true,
+            attributes: { "gen_ai.agent.id": agent, "gen_ai.request.model": model, "gen_ai.system": system, status: p.error ? "error" : "ok" },
+        });
+        const costUsd = num(p.costUsd);
+        // Present, finite and positive — the three things that make it a real
+        // reported spend rather than a placeholder. Anything else emits nothing.
+        if (costUsd !== undefined && costUsd > 0) {
+            ops.push({ op: "count", name: "loom.cost.usd", value: costUsd, isInt: false, attributes: { "gen_ai.agent.id": agent, "gen_ai.request.model": model, "gen_ai.system": system } });
+        }
+        return ops;
+    }
+    if (event.kind === "handoff") {
+        ops.push({
+            op: "count",
+            name: "loom.handoffs",
+            value: 1,
+            isInt: true,
+            attributes: {
+                "loom.handoff.from": str(p.from) || event.agentId || undefined,
+                "loom.handoff.to": str(p.to) || str(p.agent) || undefined,
+            },
+        });
+    }
+    return ops;
+}
+/**
+ * Agents with a turn in flight.
+ *
+ * Driven by the turn boundaries `traceIdFor` already computes, NOT by the
+ * `turn_started` status event. Only some adapters emit that status — codex,
+ * grok, claude-code and the antigravity bridge do; echo, opencode and the
+ * antigravity CLI never do — so a gauge keyed on it would read zero for half
+ * the roster while those agents were demonstrably working. A turn's trace
+ * opening is the one signal that holds for every adapter, because it is
+ * derived from the events rather than announced by them.
+ *
+ * Held here rather than asked of the daemon because this module must not reach
+ * into the runtime, and because the answer has to be derived from events for
+ * the same reason every other number is.
+ */
+const activeAgents = new Set();
+/** Maintain the active set, returning true when it actually moved. */
+function trackActive(agent, opened, closed) {
+    if (!agent)
+        return false; // human and system events are nobody's turn
+    if (closed)
+        return activeAgents.delete(agent);
+    if (opened && !activeAgents.has(agent)) {
+        activeAgents.add(agent);
+        return true;
+    }
+    return false;
+}
+/**
+ * Record one live agent event, fanning it out to every enabled signal.
+ *
+ * Best-effort and non-throwing: observability must never break the agent loop.
+ * The three signals are independent — logs off still means traces and metrics —
+ * so each checks its own switch, and the shared trace id is minted once so a
+ * log record can point at the span covering the same event.
+ */
+export function recordAgentEvent(event, ctx = {}) {
+    try {
+        const t = telemetry();
+        const m = metrics();
+        const l = logs();
+        if (!t.enabled && !m.enabled && !l.enabled)
+            return;
+        // Advance the per-agent turn trace for every event, whatever is enabled —
+        // it is the id logs correlate on, not just a trace-exporter detail.
+        const { traceId, opened, closed } = traceIdFor(event);
+        const span = eventToSpan(event, ctx);
+        const spanId = span ? t.span({ ...span, traceId }) : undefined;
+        if (l.enabled) {
+            const record = eventToLogRecord(event, ctx);
+            // traceId always; spanId only when this event really produced that span.
+            // Pointing a log at a span id we invented would be a link to nothing.
+            if (record)
+                l.log({ ...record, traceId, ...(spanId ? { spanId } : {}) });
+        }
+        if (m.enabled) {
+            for (const op of eventToMetrics(event)) {
+                if (op.op === "count")
+                    m.addCount(op.name, op.value, { ...op.attributes, "loom.project": ctx.project }, op.isInt);
+                else
+                    m.record(op.name, op.value, { ...op.attributes, "loom.project": ctx.project });
+            }
+            if (trackActive(event.agentId, opened, closed)) {
+                m.setGauge("loom.agents.active", activeAgents.size, { "loom.project": ctx.project });
+            }
+        }
+    }
+    catch {
+        /* never let telemetry throw into the agent loop */
+    }
+}
+function str(v) {
+    return typeof v === "string" ? v : v == null ? "" : String(v);
+}
+//# sourceMappingURL=index.js.map

package/dist/observability/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/observability/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,sBAAsB,EAAwC,MAAM,WAAW,CAAC;AACpH,OAAO,EAAE,WAAW,EAAyB,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAEvD,IAAI,SAAS,GAAyB,IAAI,CAAC;AAC3C,IAAI,gBAAgB,GAAuB,IAAI,CAAC;AAChD,IAAI,aAAa,GAAoB,IAAI,CAAC;AAC1C,IAAI,YAAY,GAA2B,IAAI,CAAC;AAEhD,+EAA+E;AAC/E,gFAAgF;AAChF,wEAAwE;AACxE,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;AAC5C;;;;;GAKG;AACH,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;AAEhD;;;;;;;GAOG;AACH,SAAS,UAAU,CAAC,KAAgB;IAClC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IAClC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAK,KAAK,CAAC,OAAmC,EAAE,KAAK,KAAK,cAAc,CAAC;IACpH,MAAM,MAAM,GAAG,WAAW,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpD,IAAI,MAAM;QAAE,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;IACtC,0EAA0E;IAC1E,0EAA0E;IAC1E,wEAAwE;IACxE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;IACvE,IAAI,MAAM,EAAE,CAAC;QACX,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,4BAA4B;IACvD,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,MAAM;IACb,IAAI,CAAC,YAAY;QAAE,YAAY,GAAG,sBAAsB,EAAE,CAAC;IAC3D,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,IAAI,CAAC,SAAS;QAAE,SAAS,GAAG,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;IACxD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,OAAO;IACrB,IAAI,CAAC,gBAAgB;QAAE,gBAAgB,GAAG,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IACpE,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,IAAI;IAClB,IAAI,CAAC,aAAa;QAAE,aAAa,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3D,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,cAAc;IAC5B,SAAS,EAAE,KAAK,EAAE,CAAC;IACnB,gBAAgB,EAAE,KAAK,EAAE,CAAC;IAC1B,aAAa,EAAE,KAAK,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,QAAQ,GAAG,QAAU,CAAC;AAE5B,SAAS,QAAQ,CAAC,KAAa,EAAE,UAAkB;IACjD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,EAAE,OAAO,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAG,QAAQ,EAAE,CAAC;AACpE,CAAC;AAED,SAAS,GAAG,CAAC,CAAU;IACrB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACpB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC5C,CAAC;AAID;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgB,EAAE,MAAoB,EAAE;IAClE,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG;QACX,cAAc,EAAE,GAAG,CAAC,OAAO;QAC3B,WAAW,EAAE,KAAK,CAAC,IAAI;QACvB,iBAAiB,EAAE,KAAK,CAAC,IAAI;QAC7B,iBAAiB,EAAE,KAAK,CAAC,OAAO;KACjC,CAAC;IACF,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YAC1C,OAAO;gBACL,IAAI,EAAE,mBAAmB;gBACzB,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,CAAC;gBACjC,UAAU,EAAE;oBACV,GAAG,IAAI;oBACP,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,MAAM;oBAC1D,uBAAuB,EAAE,MAAM;oBAC/B,sBAAsB,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS;oBACjD,2BAA2B,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;oBAClE,4BAA4B,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC;oBACrE,uBAAuB,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;oBACvC,uBAAuB,EAAE,UAAU;iBACpC;gBACD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;aAC1C,CAAC;QACJ,CAAC;QACD,KAAK,WAAW;YACd,OAAO;gBACL,IAAI,EAAE,kBAAkB;gBACxB,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC7C,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE;aAClF,CAAC;QACJ,KAAK,SAAS;YACZ,OAAO;gBACL,IAAI,EAAE,oBAAoB;gBAC1B,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxB,UAAU,EAAE;oBACV,GAAG,IAAI;oBACP,iBAAiB,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;oBAC5C,mBAAmB,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO;iBAClD;aACF,CAAC;QACJ,KAAK,eAAe,CAAC;QACrB,KAAK,YAAY,CAAC;QAClB,KAAK,cAAc,CAAC;QACpB,KAAK,eAAe,CAAC;QACrB,KAAK,iBAAiB,CAAC;QACvB,KAAK,cAAc;YACjB,OAAO;gBACL,IAAI,EAAE,cAAc,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACvD,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC7C,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;gBACrE,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,SAAS;aAClF,CAAC;QACJ,KAAK,YAAY,CAAC;QAClB,KAAK,eAAe,CAAC;QACrB,KAAK,eAAe;YAClB,OAAO;gBACL,IAAI,EAAE,eAAe,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;gBACzD,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxB,UAAU,EAAE;oBACV,GAAG,IAAI;oBACP,kBAAkB,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC9C,mBAAmB,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;iBAClC;aACF,CAAC;QACJ,KAAK,OAAO;YACV,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9G;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAWD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAAC,KAAgB;IAC7C,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAe,EAAE,CAAC;IAE3B,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC;QAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC;QACxC,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;QAC5D,uEAAuE;QACvE,2EAA2E;QAC3E,MAAM,KAAK,GAAqB;YAC9B,iBAAiB,EAAE,KAAK;YACxB,sBAAsB,EAAE,KAAK;YAC7B,eAAe,EAAE,MAAM;YACvB,uBAAuB,EAAE,MAAM;SAChC,CAAC;QAEF,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAC1D,2EAA2E;QAC3E,2EAA2E;QAC3E,wEAAwE;QACxE,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,GAAG,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QACrJ,CAAC;QACD,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,GAAG,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;QACvJ,CAAC;QAED,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;YAChD,uEAAuE;YACvE,sEAAsE;YACtE,yBAAyB;YACzB,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,kCAAkC,EAAE,KAAK,EAAE,UAAU,GAAG,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QACvH,CAAC;QAED,GAAG,CAAC,IAAI,CAAC;YACP,EAAE,EAAE,OAAO;YACX,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,IAAI;YACX,UAAU,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,sBAAsB,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE;SACnI,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/B,sEAAsE;QACtE,yEAAyE;QACzE,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,sBAAsB,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QACnL,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,GAAG,CAAC,IAAI,CAAC;YACP,EAAE,EAAE,OAAO;YACX,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,IAAI;YACX,UAAU,EAAE;gBACV,mBAAmB,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,IAAI,SAAS;gBAC9D,iBAAiB,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS;aAC1D;SACF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;AAEvC,sEAAsE;AACtE,SAAS,WAAW,CAAC,KAAyB,EAAE,MAAe,EAAE,MAAe;IAC9E,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC,CAAC,4CAA4C;IACtE,IAAI,MAAM;QAAE,OAAO,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAgB,EAAE,MAAoB,EAAE;IACvE,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;QACpB,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;QACjB,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,OAAO;YAAE,OAAO;QAEnD,0EAA0E;QAC1E,oEAAoE;QACpE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAE/D,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC5C,yEAAyE;YACzE,sEAAsE;YACtE,IAAI,MAAM;gBAAE,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,KAAK,MAAM,EAAE,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvC,IAAI,EAAE,CAAC,EAAE,KAAK,OAAO;oBAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;;oBAC7G,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACtF,CAAC;YACD,IAAI,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;gBAC/C,CAAC,CAAC,QAAQ,CAAC,oBAAoB,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,mDAAmD;IACrD,CAAC;AACH,CAAC;AAED,SAAS,GAAG,CAAC,CAAU;IACrB,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC"}

package/dist/observability/insights.d.ts

@@ -0,0 +1,151 @@
+/**
+ * Observatory insights, read back from the backend's ClickHouse store: the spans
+ * behind the Span Replay viewer and the Trace Waterfall, the per-agent burn-rate
+ * series, and a deterministic 0–100 Agent Health Score. All of it is derived from
+ * the same real gen_ai spans Loom exports — nothing here is synthetic.
+ *
+ * The scoring (`healthScore`) is a pure function so it is unit-tested without a
+ * live ClickHouse, and it is the same math the UI badges render.
+ */
+import type { LoomEvent } from "../types.js";
+export type InsightSpan = {
+    traceId: string;
+    spanId: string;
+    ts: number;
+    name: string;
+    ms: number;
+    code: number;
+    msg: string;
+    agent: string;
+    ade: string;
+    model: string;
+    tin: number;
+    tout: number;
+    cost: number;
+    handoffFrom: string;
+    handoffTo: string;
+};
+/** Recent spans for a project (optionally one agent), newest first. */
+export declare function fetchSpans(project: string, opts?: {
+    agent?: string;
+    limit?: number;
+}): Promise<InsightSpan[]>;
+/** How many error spans an agent has emitted since `sinceMs` — the self-heal recheck signal. */
+export declare function recentAgentErrors(project: string, agent: string, sinceMs: number): Promise<number>;
+/** Every span in one trace, oldest first — the waterfall's rows. */
+export declare function traceSpans(traceId: string): Promise<InsightSpan[]>;
+/**
+ * Fallback: derive InsightSpans from the local event log — the same data the
+ * backend gets — so the Observatory's replay/health/waterfall still work when
+ * ClickHouse is down or the agent hasn't exported anything yet.
+ */
+export declare function insightSpansFromLog(events: LoomEvent[], agent?: string): InsightSpan[];
+export type BurnBucket = {
+    t: number;
+    byAgent: Record<string, number>;
+    total: number;
+};
+export type BurnSeries = {
+    hours: number;
+    buckets: BurnBucket[];
+    totalUsd: number;
+    /** USD/hour over the covered window, and the linear projection for 24h. */
+    ratePerHour: number;
+    projected24h: number;
+};
+/** Per-agent cost over time, bucketed — the burn-rate sparkline's data + a linear projection. */
+export declare function burnSeries(project: string, opts?: {
+    hours?: number;
+    buckets?: number;
+}): Promise<BurnSeries>;
+/**
+ * The metric tables the collector writes into. Defaults are that layout;
+ * both are overridable for a backend that stores its metrics elsewhere.
+ */
+export declare const METRIC_SAMPLES_TABLE: string;
+export declare const METRIC_SERIES_TABLE: string;
+/** Expand each requested name to itself plus its histogram parts, deduped. */
+export declare function expandMetricNames(names: string[]): string[];
+export type MetricPoint = {
+    /** Bucket start, epoch ms. */
+    t: number;
+    /**
+     * Sum of the samples in the bucket. Loom's counters arrive as small
+     * per-export deltas (verified on the live stack: loom.turns reports 1/2/3,
+     * never a running total), so summing is the honest "how much happened in this
+     * window" number for a Sum-typed metric.
+     */
+    sum: number;
+    /** Mean of the samples in the bucket — the right read for a Gauge like loom.agents.active. */
+    avg: number;
+    max: number;
+    /** How many raw samples went into the bucket, so the UI can tell thin data from zero. */
+    n: number;
+};
+export type MetricSeries = {
+    metric: string;
+    /** The backend's type column: "Sum" | "Gauge" | "Histogram" | "". */
+    type: string;
+    unit: string;
+    temporality: string;
+    /** The series' label set (agent id, model, project, …), internal `__*` keys stripped. */
+    labels: Record<string, string>;
+    /** ClickHouse fingerprint as a string — UInt64 overflows a JS number. */
+    fingerprint: string;
+    /** Which of sum/avg the UI should plot by default, chosen from `type`. */
+    prefer: "sum" | "avg";
+    points: MetricPoint[];
+};
+export type MetricQueryOpts = {
+    project: string;
+    sinceMs?: number;
+    stepMs?: number;
+};
+/**
+ * Build the SQL for a metric series query — split out so the join, the project
+ * filter and the bucketing are testable without a live ClickHouse.
+ *
+ * Note what the inner subquery does *not* do: filter time_series_v4 by
+ * unix_milli. That column is floored to the hour there, so a naive
+ * `>= sinceMs` would drop the metadata row for a series whose hour started
+ * before the window and take the whole series with it. The metric_name +
+ * project filters already narrow it to a handful of fingerprints, so the time
+ * filter belongs on the samples side only, where it is exact.
+ *
+ * The GROUP BY inside the subquery collapses the one-row-per-hour duplication;
+ * without it the join would multiply every sample by the number of hours the
+ * series has existed.
+ */
+export declare function buildMetricQuery(names: string[], opts: MetricQueryOpts): string;
+/** Fold the flat (series, bucket) rows into one entry per series, points ordered. */
+export declare function rowsToSeries(rows: Record<string, unknown>[]): MetricSeries[];
+/**
+ * Read named metrics back for one project, one entry per distinct label set.
+ * Throws if ClickHouse is unreachable — like logs, there is no local substitute
+ * for a metric the backend never received, and the route reports that honestly.
+ */
+export declare function fetchMetricSeries(names: string[], opts: MetricQueryOpts): Promise<MetricSeries[]>;
+/** The metrics Loom actually emits — the default set when a caller names none. */
+export declare const LOOM_METRIC_NAMES: string[];
+export type Health = {
+    score: number;
+    grade: "healthy" | "degraded" | "unhealthy";
+    turns: number;
+    errorCount: number;
+    buckets: {
+        errorRate: number;
+        latency: number;
+        tokenBloat: number;
+        recency: number;
+    };
+};
+/**
+ * A deterministic 0–100 health score from an agent's own spans. Four penalty
+ * buckets subtract from 100:
+ *   errorRate (≤40) — share of spans that errored
+ *   latency   (≤25) — share of turns slower than 30s
+ *   tokenBloat(≤20) — average input tokens above an 80k comfort line
+ *   recency   (≤15) — how recently the last error fired (fades over 30 min)
+ * Pure: same input, same score — this is what the UI badge shows.
+ */
+export declare function healthScore(spans: InsightSpan[], now?: number): Health;