@reticlehq/server 2.2.0 → 2.3.0

package/dist/telemetry/session-metrics.js

@@ -0,0 +1,329 @@
+/**
+ * The in-process session accumulator — the reason Reticle's telemetry bill does not scale with how
+ * hard people use it.
+ *
+ * The old design emitted one event per MCP tool call. That reads as harmless until you multiply it
+ * out: a single agent verification loop is 50–200 tool calls, so a thousand daily users would push
+ * millions of events a day, and PostHog bills per ingested event. Worse, it was millions of events
+ * answering a question that ONE property answers better — `{reticle_act: 40, reticle_assert: 12}`
+ * tells you the session was a real verification loop, where 52 separate rows tell you only that 52
+ * things happened, with the shape scattered across them.
+ *
+ * So counts live here, in memory, and leave as a single `daemon_stopped` event.
+ *
+ * Everything recorded is a COUNT or a fingerprint. No tool arguments, no results, no selectors, no
+ * URLs, no error messages — the accumulator has no way to hold them, which is a stronger guarantee
+ * than a rule saying it shouldn't.
+ */
+import { BrowserLaunchKind, } from '@reticlehq/core';
+import { errorSkeleton, fingerprintError } from './error-fingerprint.js';
+import { describeToolParams } from './argument-shape.js';
+import { machineSnapshot } from './machine-snapshot.js';
+/** Cap the distinct error shapes held in memory — a pathological loop must not grow unbounded. */
+const MAX_ERROR_KINDS = 40;
+/** Cap distinct MCP clients recorded; more than a handful on one daemon is already the story. */
+const MAX_CLIENTS = 8;
+/** Bounds on the parameter map. Our own schemas are already finite; these are a belt-and-braces cap. */
+const MAX_PARAM_TOOLS = 60;
+const MAX_PARAMS_PER_TOOL = 24;
+/**
+ * How many recent tool calls to remember for the crash breadcrumb.
+ *
+ * Twelve is roughly one verification loop — enough to see the APPROACH (`snapshot → act → act →
+ * assert`) rather than just the last step, which is what turns "it crashed in act-tools" into "it
+ * crashed acting on a page it had just navigated away from". Names only, so this is a dozen short
+ * strings and costs nothing to hold.
+ */
+const BREADCRUMB_LENGTH = 12;
+/**
+ * Mutable counters for one daemon lifetime. A plain class, not a store: it is process-local, it never
+ * persists, and it dies with the daemon.
+ */
+export class SessionMetrics {
+    #toolCalls = 0;
+    #toolErrors = 0;
+    #verifications = 0;
+    #toolCounts = new Map();
+    #toolParams = new Map();
+    /** Ring of the most recent tool NAMES — the agent's approach run, for a crash report. */
+    #breadcrumb = [];
+    /** The tool currently in flight, so a crash can name its trigger point. */
+    #inFlight;
+    #sdkFailures = 0;
+    #sdkFailureKinds = new Map();
+    #errorKinds = new Map();
+    #connections = new Map();
+    #toolTiming = new Map();
+    #busyMs = 0;
+    #concurrent = 0;
+    #peakConcurrent = 0;
+    #unknownToolCalls = 0;
+    #bugsFound = 0;
+    #browserMs = 0;
+    #browserCommands = 0;
+    #bugKinds = new Map();
+    /**
+     * Kinds already reported this session — the dedup key, deliberately SEPARATE from `#bugKinds`.
+     *
+     * `#bugKinds` is capped at MAX_ERROR_KINDS, and once full a genuinely new kind is never inserted,
+     * so asking it "have I seen this?" answers no forever and marks every later occurrence as first —
+     * inflating the distinct-defect count precisely where a session got interesting. This set is not
+     * capped: the kind vocabulary is a bounded enum plus oracle names, not user data.
+     */
+    #seenBugKinds = new Set();
+    #clients = new Set();
+    #startedAt;
+    #now;
+    /** Clock injected — this file must stay testable without a real one, per the repo's clock rule. */
+    constructor(now) {
+        this.#now = now;
+        this.#startedAt = now();
+    }
+    /**
+     * Record a call starting. Returns a function to call when it finishes — that closure is what makes
+     * timing correct under CONCURRENCY: several agents can be inside `runTool` at once, so a single
+     * "last start time" field would attribute one tool's duration to another. Each call carries its own.
+     */
+    startToolCall(tool, args) {
+        this.recordToolCall(tool, args);
+        this.#concurrent += 1;
+        if (this.#concurrent > this.#peakConcurrent)
+            this.#peakConcurrent = this.#concurrent;
+        let settled = false;
+        return (durationMs) => {
+            if (settled)
+                return; // a double-settle would corrupt the concurrency count
+            settled = true;
+            this.#concurrent = Math.max(0, this.#concurrent - 1);
+            this.#busyMs += durationMs;
+            const timing = this.#toolTiming.get(tool) ?? { totalMs: 0, maxMs: 0 };
+            timing.totalMs += durationMs;
+            timing.maxMs = Math.max(timing.maxMs, durationMs);
+            this.#toolTiming.set(tool, timing);
+        };
+    }
+    /**
+     * Time spent waiting on the BROWSER, across all commands.
+     *
+     * Compared against `busyMs` this answers the question a single total cannot: of the time an agent
+     * spends inside Reticle, how much is us and how much is the app under test? Without the split a
+     * 4-second `reticle_act` is unattributable — it could be our overhead or the app taking 4 seconds
+     * to settle, and those have opposite fixes. A high ratio here is a finding about their app; a low
+     * one alongside a high `busyMs` is a performance bug of ours.
+     */
+    recordBrowserLatency(ms) {
+        this.#browserMs += ms;
+        this.#browserCommands += 1;
+    }
+    /** A call for a tool that does not exist. Non-zero means our surface is confusing the agent. */
+    recordUnknownTool() {
+        this.#unknownToolCalls += 1;
+    }
+    recordToolCall(tool, args) {
+        this.#toolCalls += 1;
+        bump(this.#toolCounts, tool);
+        this.#inFlight = tool;
+        this.#breadcrumb.push(tool);
+        if (this.#breadcrumb.length > BREADCRUMB_LENGTH)
+            this.#breadcrumb.shift();
+        if (args === undefined)
+            return;
+        // Which PARAMETERS get used, per tool — names only (plus a short allowlist of our own enums).
+        // Bounded by our own schemas: a tool has a fixed handful of declared parameters, so this map
+        // cannot grow with traffic the way a value-carrying one would.
+        let perTool = this.#toolParams.get(tool);
+        if (perTool === undefined) {
+            if (this.#toolParams.size >= MAX_PARAM_TOOLS)
+                return;
+            perTool = new Map();
+            this.#toolParams.set(tool, perTool);
+        }
+        for (const param of describeToolParams(args)) {
+            if (perTool.size < MAX_PARAMS_PER_TOOL || perTool.has(param))
+                bump(perTool, param);
+        }
+    }
+    /**
+     * One failed tool call. Grouped by fingerprint, but stored WITH its skeleton message and the tool
+     * that produced it — a bare fingerprint could be ranked and never diagnosed, which made the top
+     * error in the dashboard a number nobody could act on.
+     */
+    recordToolError(message, tool) {
+        this.#toolErrors += 1;
+        const fingerprint = fingerprintError(message);
+        const existing = this.#errorKinds.get(fingerprint);
+        if (existing !== undefined) {
+            existing.count += 1;
+            return;
+        }
+        // Stop growing once the cap is hit; the tail is noise and the head is what gets fixed.
+        if (this.#errorKinds.size >= MAX_ERROR_KINDS)
+            return;
+        this.#errorKinds.set(fingerprint, {
+            fingerprint,
+            count: 1,
+            message: errorSkeleton(message),
+            ...(tool !== undefined ? { tool } : {}),
+        });
+    }
+    /**
+     * A failure reported by the IN-PAGE half of Reticle, over the bridge.
+     *
+     * Kept in its own bucket rather than mixed into tool errors: a broken observer and a failing tool
+     * call are different defects with different owners, and merging them would hide the browser-side
+     * one inside a much larger pile. Same treatment otherwise — fingerprinted, variables stripped.
+     */
+    recordSdkFailure(site, message) {
+        this.#sdkFailures += 1;
+        const fingerprint = fingerprintError(`${site}|${message}`);
+        const existing = this.#sdkFailureKinds.get(fingerprint);
+        if (existing !== undefined) {
+            existing.count += 1;
+            return;
+        }
+        if (this.#sdkFailureKinds.size >= MAX_ERROR_KINDS)
+            return;
+        this.#sdkFailureKinds.set(fingerprint, {
+            fingerprint,
+            count: 1,
+            message: errorSkeleton(message),
+            tool: site,
+        });
+    }
+    /** The agent's recent approach run + the call in flight — context for a crash report. */
+    get trail() {
+        return { breadcrumb: [...this.#breadcrumb], inFlight: this.#inFlight };
+    }
+    recordVerification() {
+        this.#verifications += 1;
+    }
+    /**
+     * One defect found in the app under test — the outcome number, kept broken down by kind.
+     *
+     * Returns TRUE when this kind is new to the session. Every occurrence is still counted; the flag
+     * only lets a reader separate "how many distinct defects" from "how often they were hit", which
+     * are different claims and were previously indistinguishable in the data.
+     */
+    recordBug(kind) {
+        this.#bugsFound += 1;
+        if (this.#bugKinds.size < MAX_ERROR_KINDS || this.#bugKinds.has(kind))
+            bump(this.#bugKinds, kind);
+        const first = !this.#seenBugKinds.has(kind);
+        this.#seenBugKinds.add(kind);
+        return first;
+    }
+    /**
+     * One connection ATTEMPT. Call before the attempt; settle it with the outcome.
+     *
+     * The previous version counted only successes, and counted them inconsistently — incrementing
+     * before the await on the CDP path and after it on the launch path — so one number meant attempts,
+     * the others meant successes, and nothing in the data said which. A connection metric that cannot
+     * express failure misses the only question worth asking: how often can people not get a browser?
+     */
+    recordConnectAttempt(kind) {
+        const stats = this.#connections.get(kind) ?? { attempts: 0, successes: 0 };
+        stats.attempts += 1;
+        this.#connections.set(kind, stats);
+        let settled = false;
+        return (failure) => {
+            if (settled)
+                return;
+            settled = true;
+            if (failure === undefined) {
+                stats.successes += 1;
+                return;
+            }
+            stats.failures ??= {};
+            stats.failures[failure] = (stats.failures[failure] ?? 0) + 1;
+        };
+    }
+    recordClient(name) {
+        if (this.#clients.size < MAX_CLIENTS)
+            this.#clients.add(name.slice(0, 64));
+    }
+    /**
+     * Roll the counters into the event payload. `final` marks a clean shutdown; a periodic flush passes
+     * false so a session killed before it can exit is not lost entirely (see the flush note in cli.ts).
+     */
+    summarize(final) {
+        const machine = machineSnapshot();
+        return {
+            durationMs: Math.max(0, this.#now() - this.#startedAt),
+            toolCalls: this.#toolCalls,
+            toolCounts: Object.fromEntries(this.#toolCounts),
+            toolErrors: this.#toolErrors,
+            ...(this.#errorKinds.size > 0 ? { errors: [...this.#errorKinds.values()] } : {}),
+            sdkFailures: this.#sdkFailures,
+            ...(this.#sdkFailureKinds.size > 0 ? { sdkErrors: [...this.#sdkFailureKinds.values()] } : {}),
+            verifications: this.#verifications,
+            bugsFound: this.#bugsFound,
+            ...(this.#bugKinds.size > 0 ? { bugKinds: Object.fromEntries(this.#bugKinds) } : {}),
+            ...(this.#toolParams.size > 0
+                ? {
+                    toolParams: Object.fromEntries([...this.#toolParams].map(([tool, params]) => [tool, Object.fromEntries(params)])),
+                }
+                : {}),
+            ...(this.#connections.size > 0 ? { connections: Object.fromEntries(this.#connections) } : {}),
+            ...(this.#toolTiming.size > 0 ? { toolTiming: Object.fromEntries(this.#toolTiming) } : {}),
+            busyMs: this.#busyMs,
+            browserMs: this.#browserMs,
+            browserCommands: this.#browserCommands,
+            peakConcurrentTools: this.#peakConcurrent,
+            unknownToolCalls: this.#unknownToolCalls,
+            ...(machine !== undefined ? { machine } : {}),
+            ...(this.#clients.size > 0 ? { clients: [...this.#clients] } : {}),
+            final,
+        };
+    }
+    /** True when nothing at all happened — a flush of an idle daemon is not worth an event. */
+    get empty() {
+        return (this.#toolCalls === 0 &&
+            this.#verifications === 0 &&
+            this.#toolErrors === 0 &&
+            this.#bugsFound === 0 &&
+            this.#sdkFailures === 0);
+    }
+    /** Zero the counters after a non-final flush so the next flush reports the NEXT window, not a total. */
+    reset() {
+        this.#toolCalls = 0;
+        this.#toolErrors = 0;
+        this.#verifications = 0;
+        this.#toolCounts.clear();
+        this.#toolParams.clear();
+        this.#errorKinds.clear();
+        this.#sdkFailures = 0;
+        this.#sdkFailureKinds.clear();
+        this.#connections.clear();
+        this.#toolTiming.clear();
+        this.#busyMs = 0;
+        this.#browserMs = 0;
+        this.#browserCommands = 0;
+        this.#peakConcurrent = 0;
+        this.#unknownToolCalls = 0;
+        this.#bugsFound = 0;
+        this.#bugKinds.clear();
+        this.#seenBugKinds.clear();
+    }
+}
+function bump(map, key) {
+    map.set(key, (map.get(key) ?? 0) + 1);
+}
+/**
+ * The process-wide accumulator. A module singleton because the two call sites that feed it — the tool
+ * chokepoint and the MCP error boundary — are far apart in the graph and threading a metrics object
+ * through every ToolDeps would be a large, purely mechanical change for no added safety.
+ */
+let current;
+export const getSessionMetrics = (now = () => Date.now()) => (current ??= new SessionMetrics(now));
+/** Record one browser round-trip. Free-function form so the session hot path is a single call. */
+export const recordBrowserLatency = (ms) => {
+    getSessionMetrics().recordBrowserLatency(ms);
+};
+/** Narrow-and-record one in-page failure. Keeps the session hot path to a single call. */
+export const recordSdkFailure = (failure) => {
+    getSessionMetrics().recordSdkFailure(failure.site, failure.message);
+};
+/** Tests only — drop the singleton so each case starts from zero. */
+export const resetSessionMetrics = () => {
+    current = undefined;
+};

package/dist/telemetry/telemetry.d.ts

@@ -0,0 +1,68 @@
+import { TelemetryEventKind, type Crash, type Feedback, type Identity, type BugFound, type InitOutcome, type McpConnection, type ProjectProfile, type SessionSummary, type TelemetryActor, type Verification, type VersionChange } from '@reticlehq/core';
+/** The current telemetry state and which control set it — what `reticle telemetry status` prints. */
+export declare const describeTelemetry: (env?: NodeJS.ProcessEnv, dir?: string, cwd?: string) => {
+    enabled: boolean;
+    reason: string;
+    policyUrl: string;
+};
+/** Persist (or lift) the machine-wide opt-out — `reticle telemetry disable` / `enable`. */
+export declare const setTelemetryEnabled: (enabled: boolean, dir?: string) => void;
+/**
+ * The per-event extras — one optional block per event kind, so a payload can only ever be attached
+ * to the event it belongs to. `feedback` is the only one carrying author-written text, and it only
+ * ever arrives from `submitFeedback`, which redacts and validates it first. Nothing else may set it.
+ */
+export interface TelemetryExtra {
+    detach?: boolean;
+    actor?: TelemetryActor;
+    /** `cli_command_run`: which subcommand a human ran. */
+    command?: string;
+    /** `cli_command_run`: flag NAMES present on the invocation. Never values. */
+    flags?: string[];
+    /** `daemon_stopped`: the whole session rolled up. Replaces the old per-tool-call event. */
+    session?: SessionSummary;
+    /** `project_profiled`: the project's shape and depth-of-use. */
+    project?: ProjectProfile;
+    /** `verification_completed`: one verdict. */
+    verification?: Verification;
+    /** `version_changed`: update or rollback. */
+    versionChange?: VersionChange;
+    /** `runtime_crashed`: a fingerprinted crash. */
+    crash?: Crash;
+    feedback?: Feedback;
+    /** `identified`: a self-declared identity. Opt-in, typed by a human, never inferred. */
+    identity?: Identity;
+    /** `mcp_client_connected`: a client attached, and whether it had attached before. */
+    connection?: McpConnection;
+    /** `init_completed`: how `reticle init` went. */
+    init?: InitOutcome;
+    /** `bug_found`: one defect Reticle found in the app under test. */
+    bug?: BugFound;
+}
+export interface Telemetry {
+    /**
+     * Fire one event, non-blocking. `detach: true` hands the send to a disowned child so a short-lived
+     * CLI process can exit immediately instead of waiting out the POST (use it from command entry
+     * points; daemon-side events send in-process).
+     */
+    emit(kind: TelemetryEventKind, extra?: TelemetryExtra): Promise<void>;
+    readonly enabled: boolean;
+    /** True the very first run on this machine — the CLI emits INSTALL alongside the first INVOKE. */
+    readonly firstRun: boolean;
+}
+/**
+ * Build the telemetry emitter for this process. Resolves identity + opt-out ONCE; each `emit` builds a
+ * fully-validated core event, maps it into PostHog capture shape, and fires a best-effort POST.
+ * `version` is the running reticle version (for the `version` dimension); `now`/`fetchImpl` are
+ * injected so this is testable without a clock or network.
+ */
+export declare const createTelemetry: (opts: {
+    version: string;
+    env?: NodeJS.ProcessEnv;
+    cwd?: string;
+    now?: () => number;
+    fetchImpl?: typeof fetch;
+    spawnImpl?: (command: string, args: string[]) => void;
+}) => Telemetry;
+export declare const getTelemetry: () => Telemetry;
+export { TelemetryEventKind };