@reticlehq/server 2.2.0 → 2.3.0

package/dist/telemetry/project-profile.d.ts

@@ -0,0 +1,44 @@
+import { ProjectSize, type ProjectProfile } from '@reticlehq/core';
+/**
+ * The feature FAMILIES, and the on-disk evidence that a project has adopted each. Named rather than
+ * counted so the vocabulary stays stable as tools are added, renamed, or merged into action-dispatched
+ * families — a count would silently change meaning the next time the tool surface is reorganized.
+ */
+export declare const FeatureFamily: {
+    readonly DETERMINISTIC_REPLAY: "deterministic_replay";
+    readonly VISUAL_BASELINE: "visual_baseline";
+    readonly TEXT_BASELINE: "text_baseline";
+    readonly CAPABILITY_CONTRACT: "capability_contract";
+    readonly RUN_ARTIFACTS: "run_artifacts";
+    readonly BUG_CAPSULES: "bug_capsules";
+    readonly CROSS_RUN_HISTORY: "cross_run_history";
+};
+export type FeatureFamily = (typeof FeatureFamily)[keyof typeof FeatureFamily];
+/** Count source files, stopping at the cap (the cap itself lands in the largest bucket, correctly). */
+export declare function countSourceFiles(root: string, readDir?: DirReader): number;
+export declare function sizeBucket(sourceFiles: number): ProjectSize;
+/**
+ * Project age in whole WEEKS, from the repo's first commit.
+ *
+ * Weeks rather than a date, deliberately: an exact creation timestamp combined with a framework and a
+ * size bucket starts to narrow toward a specific repository, and the question being asked — "is
+ * Reticle being adopted on greenfield spikes or on mature codebases?" — does not need that precision.
+ *
+ * Read from git's own reflog tail rather than by shelling out, for the same reason the origin lookup
+ * does: this runs on a startup path and must not cost a subprocess or require a `git` binary.
+ */
+export declare function projectAgeWeeks(root: string, now: number, read?: FileReader): number | undefined;
+/** True when package.json declares workspaces — monorepos exercise very different code paths. */
+export declare function isMonorepo(root: string, read?: FileReader): boolean;
+interface DirEntry {
+    name: string;
+    isDirectory: boolean;
+}
+type DirReader = (dir: string) => DirEntry[];
+type FileReader = (path: string) => string;
+/**
+ * Build the full profile. Best-effort throughout: an unreadable project yields a profile of zeroes
+ * rather than an exception, because a telemetry snapshot must never be able to fail a daemon start.
+ */
+export declare function profileProject(cwd: string, now: number): ProjectProfile;
+export {};

package/dist/telemetry/project-profile.js

@@ -0,0 +1,205 @@
+/**
+ * What kind of project is this, and how much of Reticle does it actually use?
+ *
+ * A DAU chart cannot tell apart someone running forty saved flows with visual baselines and a
+ * checked-in contract from someone who called `reticle_snapshot` twice and never came back. Both are
+ * one active user. Only one of them is retained, and only one of them is a company. `featureDepth`
+ * is the number that separates them, and it is the closest thing here to an activation metric.
+ *
+ * Everything is derived from files already on disk in the project, and everything is reduced before
+ * it leaves: sizes become buckets, ages become whole weeks, feature use becomes a fixed vocabulary of
+ * family names we define ourselves. No file names, no paths, no flow names, no dependency inventory.
+ */
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { ProjectSize, ReticleDir } from '@reticlehq/core';
+import { gitFacts } from './git-facts.js';
+import { detectStack } from './feedback-context.js';
+/**
+ * The feature FAMILIES, and the on-disk evidence that a project has adopted each. Named rather than
+ * counted so the vocabulary stays stable as tools are added, renamed, or merged into action-dispatched
+ * families — a count would silently change meaning the next time the tool surface is reorganized.
+ */
+export const FeatureFamily = {
+    DETERMINISTIC_REPLAY: 'deterministic_replay',
+    VISUAL_BASELINE: 'visual_baseline',
+    TEXT_BASELINE: 'text_baseline',
+    CAPABILITY_CONTRACT: 'capability_contract',
+    RUN_ARTIFACTS: 'run_artifacts',
+    BUG_CAPSULES: 'bug_capsules',
+    CROSS_RUN_HISTORY: 'cross_run_history',
+};
+const ALL_FAMILIES = Object.values(FeatureFamily);
+/** Source-file thresholds for each bucket. A bucket answers "toy or real codebase"; a count fingerprints. */
+const SIZE_THRESHOLDS = [
+    [50, ProjectSize.TINY],
+    [250, ProjectSize.SMALL],
+    [1000, ProjectSize.MEDIUM],
+    [5000, ProjectSize.LARGE],
+];
+const SOURCE_EXTENSIONS = /\.(?:tsx?|jsx?|mjs|cjs|vue|svelte|astro)$/;
+/** Never walked: cost, and nothing in them describes the project the team actually wrote. */
+const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'build', '.next', '.turbo', 'coverage']);
+/** Bound the walk — a huge monorepo must not turn a telemetry snapshot into a filesystem crawl. */
+const MAX_FILES_SCANNED = 6000;
+const MAX_WALK_DEPTH = 8;
+const MS_PER_WEEK = 7 * 24 * 60 * 60 * 1000;
+/** Count source files, stopping at the cap (the cap itself lands in the largest bucket, correctly). */
+export function countSourceFiles(root, readDir = defaultReadDir) {
+    let count = 0;
+    const walk = (dir, depth) => {
+        if (depth > MAX_WALK_DEPTH || count >= MAX_FILES_SCANNED)
+            return;
+        for (const entry of readDir(dir)) {
+            if (count >= MAX_FILES_SCANNED)
+                return;
+            if (entry.isDirectory) {
+                if (!SKIP_DIRS.has(entry.name) && !entry.name.startsWith('.')) {
+                    walk(join(dir, entry.name), depth + 1);
+                }
+            }
+            else if (SOURCE_EXTENSIONS.test(entry.name)) {
+                count += 1;
+            }
+        }
+    };
+    walk(root, 0);
+    return count;
+}
+export function sizeBucket(sourceFiles) {
+    for (const [threshold, bucket] of SIZE_THRESHOLDS) {
+        if (sourceFiles < threshold)
+            return bucket;
+    }
+    return ProjectSize.HUGE;
+}
+/**
+ * Project age in whole WEEKS, from the repo's first commit.
+ *
+ * Weeks rather than a date, deliberately: an exact creation timestamp combined with a framework and a
+ * size bucket starts to narrow toward a specific repository, and the question being asked — "is
+ * Reticle being adopted on greenfield spikes or on mature codebases?" — does not need that precision.
+ *
+ * Read from git's own reflog tail rather than by shelling out, for the same reason the origin lookup
+ * does: this runs on a startup path and must not cost a subprocess or require a `git` binary.
+ */
+export function projectAgeWeeks(root, now, read = defaultRead) {
+    for (const path of [join(root, '.git', 'logs', 'HEAD')]) {
+        try {
+            const first = read(path).split('\n')[0];
+            // reflog line: `<old> <new> <name> <email> <unix-ts> <tz>\t<message>`
+            const seconds = first?.match(/\s(\d{9,11})\s[+-]\d{4}/)?.[1];
+            if (seconds === undefined)
+                continue;
+            const ageMs = now - parseInt(seconds, 10) * 1000;
+            if (ageMs < 0)
+                continue;
+            return Math.floor(ageMs / MS_PER_WEEK);
+        }
+        catch {
+            continue;
+        }
+    }
+    return undefined;
+}
+/** True when package.json declares workspaces — monorepos exercise very different code paths. */
+export function isMonorepo(root, read = defaultRead) {
+    try {
+        const pkg = JSON.parse(read(join(root, 'package.json')));
+        if (pkg.workspaces !== undefined)
+            return true;
+    }
+    catch {
+        /* no package.json — fall through to the pnpm check */
+    }
+    try {
+        read(join(root, 'pnpm-workspace.yaml'));
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+const defaultReadDir = (dir) => {
+    try {
+        return readdirSync(dir, { withFileTypes: true }).map((e) => ({
+            name: e.name,
+            isDirectory: e.isDirectory(),
+        }));
+    }
+    catch {
+        return [];
+    }
+};
+const defaultRead = (path) => readFileSync(path, 'utf8');
+/** How many entries a `.reticle` subdirectory holds, matching an optional extension. 0 when absent. */
+function countIn(reticleRoot, subdir, extension) {
+    try {
+        const names = readdirSync(join(reticleRoot, subdir));
+        return extension === undefined ? names.length : names.filter((n) => extension.test(n)).length;
+    }
+    catch {
+        return 0;
+    }
+}
+function exists(path) {
+    try {
+        statSync(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Build the full profile. Best-effort throughout: an unreadable project yields a profile of zeroes
+ * rather than an exception, because a telemetry snapshot must never be able to fail a daemon start.
+ */
+export function profileProject(cwd, now) {
+    const reticleRoot = join(cwd, ReticleDir.ROOT);
+    const flowCount = countIn(reticleRoot, ReticleDir.FLOWS_SUBDIR, /\.json$/);
+    const baselineCount = countIn(reticleRoot, ReticleDir.BASELINES_SUBDIR);
+    // `.diff.png` files are diff OUTPUT, not baselines — counting them would double every visual user.
+    const visualBaselineCount = countIn(reticleRoot, ReticleDir.VISUAL_SUBDIR, /(?<!\.diff)\.png$/);
+    const runCount = countIn(reticleRoot, ReticleDir.RUNS_SUBDIR, /\.json$/);
+    const capsuleCount = countIn(reticleRoot, ReticleDir.CAPSULES_SUBDIR, /\.json$/);
+    const hasContract = exists(join(reticleRoot, ReticleDir.CONTRACT_FILE));
+    const hasHistory = exists(join(reticleRoot, ReticleDir.PROJECT_FILE));
+    const featuresUsed = [];
+    if (flowCount > 0)
+        featuresUsed.push(FeatureFamily.DETERMINISTIC_REPLAY);
+    if (visualBaselineCount > 0)
+        featuresUsed.push(FeatureFamily.VISUAL_BASELINE);
+    if (baselineCount > 0)
+        featuresUsed.push(FeatureFamily.TEXT_BASELINE);
+    if (hasContract)
+        featuresUsed.push(FeatureFamily.CAPABILITY_CONTRACT);
+    if (runCount > 0)
+        featuresUsed.push(FeatureFamily.RUN_ARTIFACTS);
+    if (capsuleCount > 0)
+        featuresUsed.push(FeatureFamily.BUG_CAPSULES);
+    if (hasHistory)
+        featuresUsed.push(FeatureFamily.CROSS_RUN_HISTORY);
+    const { stack, stackMajor } = detectStack(cwd);
+    const ageWeeks = projectAgeWeeks(cwd, now);
+    const git = gitFacts(cwd);
+    return {
+        git: git.state,
+        ...(git.forge !== undefined ? { forge: git.forge } : {}),
+        ...(stack !== undefined ? { stack } : {}),
+        ...(stackMajor !== undefined ? { stackMajor } : {}),
+        size: sizeBucket(countSourceFiles(cwd)),
+        monorepo: isMonorepo(cwd),
+        ...(ageWeeks !== undefined ? { ageWeeks } : {}),
+        flowCount,
+        baselineCount,
+        visualBaselineCount,
+        runCount,
+        hasContract,
+        capsuleCount,
+        featuresUsed,
+        // Rounded to 2dp: the exact ratio is a function of the counts above, and an unrounded float would
+        // just be a higher-cardinality restatement of them.
+        featureDepth: Math.round((featuresUsed.length / ALL_FAMILIES.length) * 100) / 100,
+    };
+}

package/dist/telemetry/run-telemetry.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Telemetry for the verification RUNNER — the path `reticle verify` and the HTTP verify surface take.
+ *
+ * This path does not go through `runTool`, so for a while it produced nothing at all: no timing, no
+ * `verification_completed`, and — worst — no `bug_found`. Which meant every defect Reticle caught in
+ * CI was invisible to the one number we intend to publish, on precisely the surface most likely to
+ * run at scale and unattended. A metric that is blind to CI is not measuring the product.
+ *
+ * Reported from the assembled run rather than from inside the replay loop: the run artifact is the
+ * finished, already-classified verdict, so this cannot double-count and does not need to understand
+ * how a replay works.
+ */
+import { type ReticleVerificationRun } from '@reticlehq/core';
+/** The tool name reported for this path, so CI runs are distinguishable from agent-driven ones. */
+export declare const VERIFY_SURFACE = "reticle_verify";
+/**
+ * Report one completed verification run: the verdict, and one bug per failing flow.
+ *
+ * Best-effort and wrapped — a verification's exit code must never depend on whether a metric sent.
+ */
+export declare function reportRunTelemetry(run: ReticleVerificationRun): void;

package/dist/telemetry/run-telemetry.js

@@ -0,0 +1,62 @@
+/**
+ * Telemetry for the verification RUNNER — the path `reticle verify` and the HTTP verify surface take.
+ *
+ * This path does not go through `runTool`, so for a while it produced nothing at all: no timing, no
+ * `verification_completed`, and — worst — no `bug_found`. Which meant every defect Reticle caught in
+ * CI was invisible to the one number we intend to publish, on precisely the surface most likely to
+ * run at scale and unattended. A metric that is blind to CI is not measuring the product.
+ *
+ * Reported from the assembled run rather than from inside the replay loop: the run artifact is the
+ * finished, already-classified verdict, so this cannot double-count and does not need to understand
+ * how a replay works.
+ */
+import { BugSource, RunFlowStatus, TelemetryActor, TelemetryEventKind, VerdictStatus, } from '@reticlehq/core';
+import { getSessionMetrics } from './session-metrics.js';
+import { getTelemetry } from './telemetry.js';
+/** The tool name reported for this path, so CI runs are distinguishable from agent-driven ones. */
+export const VERIFY_SURFACE = 'reticle_verify';
+/**
+ * Report one completed verification run: the verdict, and one bug per failing flow.
+ *
+ * Best-effort and wrapped — a verification's exit code must never depend on whether a metric sent.
+ */
+export function reportRunTelemetry(run) {
+    try {
+        const failed = run.flows.filter((flow) => flow.status === RunFlowStatus.FAIL);
+        const metrics = getSessionMetrics();
+        metrics.recordVerification();
+        void getTelemetry().emit(TelemetryEventKind.VERIFICATION_COMPLETED, {
+            // A `reticle verify` was typed by a person or scheduled by their CI — either way it is not the
+            // agent's own loop, which is the distinction `actor` exists to draw.
+            actor: TelemetryActor.HUMAN,
+            verification: {
+                via: VERIFY_SURFACE,
+                verified: run.verdict.status === VerdictStatus.PASS ? 'yes' : 'no',
+                passed: run.verdict.status === VerdictStatus.PASS,
+                // The runner replays saved flows; it does not evaluate a fresh assertion that could pass over
+                // a contradiction, so this path cannot produce a false green and must not claim one.
+                falseGreenCaught: false,
+                durationMs: run.durationMs,
+            },
+        });
+        for (const _flow of failed) {
+            // Every failing flow in one run is a separate regression instance, but they share a kind — so
+            // only the first is a distinct defect. A 40-flow suite failing wholesale is one regression to
+            // fix far more often than it is forty.
+            const first = metrics.recordBug('flow-regression');
+            void getTelemetry().emit(TelemetryEventKind.BUG_FOUND, {
+                actor: TelemetryActor.HUMAN,
+                bug: {
+                    source: BugSource.REPLAY,
+                    kind: 'flow-regression',
+                    falseGreen: false,
+                    tool: VERIFY_SURFACE,
+                    repeat: !first,
+                },
+            });
+        }
+    }
+    catch {
+        /* a verdict must never depend on a metric */
+    }
+}

package/dist/telemetry/session-metrics.d.ts

@@ -0,0 +1,103 @@
+/**
+ * The in-process session accumulator — the reason Reticle's telemetry bill does not scale with how
+ * hard people use it.
+ *
+ * The old design emitted one event per MCP tool call. That reads as harmless until you multiply it
+ * out: a single agent verification loop is 50–200 tool calls, so a thousand daily users would push
+ * millions of events a day, and PostHog bills per ingested event. Worse, it was millions of events
+ * answering a question that ONE property answers better — `{reticle_act: 40, reticle_assert: 12}`
+ * tells you the session was a real verification loop, where 52 separate rows tell you only that 52
+ * things happened, with the shape scattered across them.
+ *
+ * So counts live here, in memory, and leave as a single `daemon_stopped` event.
+ *
+ * Everything recorded is a COUNT or a fingerprint. No tool arguments, no results, no selectors, no
+ * URLs, no error messages — the accumulator has no way to hold them, which is a stronger guarantee
+ * than a rule saying it shouldn't.
+ */
+import { BrowserLaunchKind, type ConnectFailure, type SessionSummary } from '@reticlehq/core';
+/**
+ * Mutable counters for one daemon lifetime. A plain class, not a store: it is process-local, it never
+ * persists, and it dies with the daemon.
+ */
+export declare class SessionMetrics {
+    #private;
+    /** Clock injected — this file must stay testable without a real one, per the repo's clock rule. */
+    constructor(now: () => number);
+    /**
+     * Record a call starting. Returns a function to call when it finishes — that closure is what makes
+     * timing correct under CONCURRENCY: several agents can be inside `runTool` at once, so a single
+     * "last start time" field would attribute one tool's duration to another. Each call carries its own.
+     */
+    startToolCall(tool: string, args?: Record<string, unknown>): (durationMs: number) => void;
+    /**
+     * Time spent waiting on the BROWSER, across all commands.
+     *
+     * Compared against `busyMs` this answers the question a single total cannot: of the time an agent
+     * spends inside Reticle, how much is us and how much is the app under test? Without the split a
+     * 4-second `reticle_act` is unattributable — it could be our overhead or the app taking 4 seconds
+     * to settle, and those have opposite fixes. A high ratio here is a finding about their app; a low
+     * one alongside a high `busyMs` is a performance bug of ours.
+     */
+    recordBrowserLatency(ms: number): void;
+    /** A call for a tool that does not exist. Non-zero means our surface is confusing the agent. */
+    recordUnknownTool(): void;
+    recordToolCall(tool: string, args?: Record<string, unknown>): void;
+    /**
+     * One failed tool call. Grouped by fingerprint, but stored WITH its skeleton message and the tool
+     * that produced it — a bare fingerprint could be ranked and never diagnosed, which made the top
+     * error in the dashboard a number nobody could act on.
+     */
+    recordToolError(message: string, tool?: string): void;
+    /**
+     * A failure reported by the IN-PAGE half of Reticle, over the bridge.
+     *
+     * Kept in its own bucket rather than mixed into tool errors: a broken observer and a failing tool
+     * call are different defects with different owners, and merging them would hide the browser-side
+     * one inside a much larger pile. Same treatment otherwise — fingerprinted, variables stripped.
+     */
+    recordSdkFailure(site: string, message: string): void;
+    /** The agent's recent approach run + the call in flight — context for a crash report. */
+    get trail(): {
+        breadcrumb: string[];
+        inFlight: string | undefined;
+    };
+    recordVerification(): void;
+    /**
+     * One defect found in the app under test — the outcome number, kept broken down by kind.
+     *
+     * Returns TRUE when this kind is new to the session. Every occurrence is still counted; the flag
+     * only lets a reader separate "how many distinct defects" from "how often they were hit", which
+     * are different claims and were previously indistinguishable in the data.
+     */
+    recordBug(kind: string): boolean;
+    /**
+     * One connection ATTEMPT. Call before the attempt; settle it with the outcome.
+     *
+     * The previous version counted only successes, and counted them inconsistently — incrementing
+     * before the await on the CDP path and after it on the launch path — so one number meant attempts,
+     * the others meant successes, and nothing in the data said which. A connection metric that cannot
+     * express failure misses the only question worth asking: how often can people not get a browser?
+     */
+    recordConnectAttempt(kind: BrowserLaunchKind): (failure?: ConnectFailure) => void;
+    recordClient(name: string): void;
+    /**
+     * Roll the counters into the event payload. `final` marks a clean shutdown; a periodic flush passes
+     * false so a session killed before it can exit is not lost entirely (see the flush note in cli.ts).
+     */
+    summarize(final: boolean): SessionSummary;
+    /** True when nothing at all happened — a flush of an idle daemon is not worth an event. */
+    get empty(): boolean;
+    /** Zero the counters after a non-final flush so the next flush reports the NEXT window, not a total. */
+    reset(): void;
+}
+export declare const getSessionMetrics: (now?: () => number) => SessionMetrics;
+/** Record one browser round-trip. Free-function form so the session hot path is a single call. */
+export declare const recordBrowserLatency: (ms: number) => void;
+/** Narrow-and-record one in-page failure. Keeps the session hot path to a single call. */
+export declare const recordSdkFailure: (failure: {
+    site: string;
+    message: string;
+}) => void;
+/** Tests only — drop the singleton so each case starts from zero. */
+export declare const resetSessionMetrics: () => void;