@beryl-so/cli 0.27.0 → 0.29.0

package/README.md

@@ -245,6 +245,17 @@ Trigger a run of a project's tests (e.g. in CI), then watch, inspect, and downlo
 | `beryl runs download <run-id>` | Download a run's results, with its artifacts, to disk | `runs_download` |
 | `beryl runs explain <result-id>` | Explain, with AI, why a test result failed | `runs_explain` |
 
+### health
+
+Site Health — how your site reads to search engines and visitors: content, speed, mobile, links and security, graded from a real check of your live pages.
+
+`beryl health` with no subcommand runs `health get`.
+
+| Command | Summary | MCP tool |
+| --- | --- | --- |
+| `beryl health get` | Show the latest Site Health report for a project environment | `health_get` |
+| `beryl health run` | Run a fresh Site Health check for a project environment | `health_run` |
+
 ### explorations
 
 Inspect the agent's exploration runs — how it crawled a site and authored its tests.

package/dist/adapters/cli.js

@@ -4,6 +4,7 @@ import { CliError, EXIT_OK, EXIT_USAGE, UsageError } from "../errors.js";
 import { ApiClient } from "../http.js";
 import { autoFormat, bold, cyan, dim } from "../output.js";
 import { commandGroups, findCommand, groupSummary } from "../registry/index.js";
+import { withTool } from "../telemetry.js";
 import { cliVersion } from "../version-check.js";
 import { mcpToolFor } from "./mcp.js";
 export const GLOBAL_FLAGS = [
@@ -324,7 +325,7 @@ export async function runCli(argv) {
     const client = new ApiClient(config.apiUrl, config.token);
     const ctx = createContext({ client, config, json: parsed.json, mcp: false });
     try {
-        const result = (await spec.run(ctx, parsed.input)) ?? {};
+        const result = (await withTool(spec.name, () => spec.run(ctx, parsed.input))) ?? {};
         if (parsed.json) {
             if (result.data !== undefined)
                 process.stdout.write(JSON.stringify(result.data) + "\n");

package/dist/adapters/mcp.js

@@ -1,13 +1,15 @@
 import { Server } from "@modelcontextprotocol/sdk/server/index.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
+import { instrument } from "@posthog/mcp";
 import fs from "node:fs";
 import { loadConfig } from "../config.js";
 import { createContext } from "../context.js";
-import { CliError } from "../errors.js";
-import { ApiClient } from "../http.js";
+import { AuthError, CliError } from "../errors.js";
+import { ApiClient, ApiError } from "../http.js";
 import { parseArgv } from "./cli.js";
 import { commands } from "../registry/index.js";
+import { analyticsOptions, createPostHogClient, setClient, setSurface, shutdownTelemetry, withTool, } from "../telemetry.js";
 import { cliVersion, warnIfStale } from "../version-check.js";
 export function toolName(spec) {
     return spec.name.replace(/ /g, "_").replace(/-/g, "_");
@@ -190,6 +192,48 @@ export function currentAuth(fallback) {
 }
 export function __resetAuthCacheForTests() {
     authCache = undefined;
+    identifyCache = undefined;
+}
+let identifyCache;
+/** Resolves the signed-in account once per token so MCP events land on the same PostHog
+ *  person as that user's webapp sessions. Anonymous (null) when unauthenticated: the
+ *  session is still counted, just not attributed. */
+export function identifyUser(baseCtx) {
+    return async () => {
+        const auth = currentAuth(baseCtx);
+        const cached = identifyCache;
+        if (cached && cached.token === auth.config.token)
+            return cached.identity;
+        let identity = null;
+        if (auth.config.token) {
+            try {
+                const me = (await auth.client.get("/account/"));
+                if (me?.id) {
+                    identity = {
+                        distinctId: me.id,
+                        // Internal dogfooding is stamped rather than dropped: unlike the webapp we
+                        // want our own MCP sessions visible when debugging, so dashboards exclude
+                        // them by cohort instead.
+                        properties: {
+                            email: me.email,
+                            name: me.name,
+                            is_internal: Boolean(me.is_vibemonitor),
+                        },
+                    };
+                }
+            }
+            catch (err) {
+                // A rejected token is a settled answer worth caching; a network blip or a 5xx is
+                // not. This server outlives the blip, so caching one would leave every later tool
+                // call in the session anonymous with no way back.
+                const settled = err instanceof AuthError || (err instanceof ApiError && err.status < 500);
+                if (!settled)
+                    return null;
+            }
+        }
+        identifyCache = { token: auth.config.token, identity };
+        return identity;
+    };
 }
 // The running version is stated up front because this server is long-lived and never
 // hot-reloads: a session can sit on a days-old build while `@latest` has moved, and
@@ -210,6 +254,7 @@ export function mcpInstructions() {
         "call `guide` before authoring your first test plan.");
 }
 export async function serveMcp(baseCtx) {
+    setSurface("mcp");
     // Fire-and-forget staleness warning: a stale MCP server silently exposes fewer
     // tools, and stderr is the one channel a stdio MCP server can safely log to.
     void warnIfStale(cliVersion(), (msg) => console.error(msg));
@@ -250,7 +295,8 @@ export async function serveMcp(baseCtx) {
             err: push,
         });
         try {
-            const result = (await spec.run(ctx, toInput(spec, request.params.arguments ?? {}))) ?? {};
+            const input = toInput(spec, request.params.arguments ?? {});
+            const result = (await withTool(request.params.name, () => spec.run(ctx, input))) ?? {};
             return toolResult(result, lines);
         }
         catch (err) {
@@ -261,8 +307,25 @@ export async function serveMcp(baseCtx) {
             };
         }
     });
+    server.oninitialized = () => setClient(server.getClientVersion());
+    instrument(server, createPostHogClient(), analyticsOptions(baseCtx.config.apiUrl, identifyUser(baseCtx)));
+    // A coding agent ends an MCP server either by closing the pipe or by signalling it, and
+    // only the first path reaches onclose — without these the last batch of events dies with
+    // the process. Bounded inside shutdownTelemetry, so a wedged flush can't hold up exit.
+    const flushAndExit = (signal) => {
+        void shutdownTelemetry().then(() => {
+            process.kill(process.pid, signal);
+        });
+    };
+    for (const signal of ["SIGINT", "SIGTERM"]) {
+        process.once(signal, () => {
+            process.removeAllListeners(signal);
+            flushAndExit(signal);
+        });
+    }
     await server.connect(new StdioServerTransport());
     await new Promise((resolve) => {
         server.onclose = resolve;
     });
+    await shutdownTelemetry();
 }

package/dist/commands/health.js

@@ -0,0 +1,43 @@
+import { flagStr, projectPath } from "./util.js";
+const ENV_FLAG = {
+    name: "env",
+    type: "string",
+    description: "Environment id (defaults to the project's default environment)",
+};
+export const healthCommands = [
+    {
+        name: "health get",
+        groupDefault: true,
+        groupSummary: "Site Health — how your site reads to search engines and visitors: content, " +
+            "speed, mobile, links and security, graded from a real check of your live pages.",
+        summary: "Show the latest Site Health report for a project environment",
+        description: "A check runs automatically when a project or environment gets its URL. While one " +
+            "is in flight the report comes back with status queued/running and no grades yet; " +
+            "call again to pick up the finished result.",
+        scope: "project",
+        flags: [ENV_FLAG],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            return {
+                data: await ctx.client.get(`${projectPath(workspaceId, projectId)}/seo-report`, {
+                    environment_id: flagStr(input, "env"),
+                }),
+            };
+        },
+    },
+    {
+        name: "health run",
+        summary: "Run a fresh Site Health check for a project environment",
+        description: "Queues a new check and returns immediately. Returns the in-flight report instead " +
+            "of stacking a second one when a check is already running.",
+        scope: "project",
+        flags: [ENV_FLAG],
+        examples: ["beryl health run", "beryl health run --env 4f…"],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            return {
+                data: await ctx.client.post(`${projectPath(workspaceId, projectId)}/seo-report`, undefined, { environment_id: flagStr(input, "env") }),
+            };
+        },
+    },
+];

package/dist/http.js

@@ -1,4 +1,5 @@
 import { AuthError, CliError } from "./errors.js";
+import { attributionHeaders } from "./telemetry.js";
 export const API_PREFIX = "/api/v1";
 export class ApiError extends CliError {
     status;
@@ -29,7 +30,11 @@ export class ApiClient {
         return this.token ? { Authorization: `Bearer ${this.token}` } : {};
     }
     async request(method, path, opts = {}) {
-        const headers = { ...this.authHeaders(), ...opts.headers };
+        const headers = {
+            ...attributionHeaders(),
+            ...this.authHeaders(),
+            ...opts.headers,
+        };
         let body;
         if (opts.form) {
             body = opts.form;

package/dist/registry/index.js

@@ -4,6 +4,7 @@ import { authCommands } from "../commands/auth.js";
 import { configCommands } from "../commands/config-vars.js";
 import { environmentCommands } from "../commands/environments.js";
 import { explorationCommands } from "../commands/explorations.js";
+import { healthCommands } from "../commands/health.js";
 import { mailboxCommands } from "../commands/mailboxes.js";
 import { initCommands } from "../commands/init.js";
 import { mcpCommands } from "../commands/mcp.js";
@@ -57,6 +58,7 @@ export const commands = [
     ...environmentCommands,
     ...testCommands,
     ...runCommands,
+    ...healthCommands,
     ...explorationCommands,
     ...configCommands,
     ...mailboxCommands,

package/dist/telemetry.js

@@ -0,0 +1,272 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { randomUUID } from "node:crypto";
+import { PostHog } from "posthog-node";
+import { cliVersion } from "./version-check.js";
+// The webapp already ships this public (write-only) project key in its JS bundle, and
+// dev/prod deploys share it — environments are separated by the `environment` property
+// on every event, not by project. Nothing here is configurable on a customer machine.
+const POSTHOG_KEY = "phc_JRs3q08uFEuLKGR6llGAEBpwAuVHtPoEXAj9IcFFhRH";
+const POSTHOG_HOST = "https://us.i.posthog.com";
+// One process = one session, for both surfaces: a CLI invocation is a one-shot session,
+// an MCP server lives as long as the coding agent keeps it spawned.
+export const sessionId = randomUUID();
+let surface = "cli";
+let client;
+export function setSurface(value) {
+    surface = value;
+}
+export function setClient(value) {
+    client = value;
+}
+export function currentSurface() {
+    return surface;
+}
+export function currentClient() {
+    return client;
+}
+// The MCP SDK dispatches tool calls concurrently, so "which tool is running" has to be
+// per-async-context, not a module global.
+const toolStore = new AsyncLocalStorage();
+export function withTool(tool, fn) {
+    return toolStore.run(tool, fn);
+}
+export function currentTool() {
+    return toolStore.getStore();
+}
+export function userAgent() {
+    const parts = [surface];
+    if (client)
+        parts.push(client.version ? `${client.name}/${client.version}` : client.name);
+    parts.push(`${process.platform} ${process.arch}`, `node/${process.versions.node}`);
+    return `beryl-cli/${cliVersion()} (${parts.join("; ")})`;
+}
+export function attributionHeaders() {
+    const headers = {
+        "User-Agent": userAgent(),
+        "X-Beryl-Session": sessionId,
+    };
+    const tool = currentTool();
+    if (tool)
+        headers["X-Beryl-Tool"] = tool;
+    return headers;
+}
+export function analyticsEnvironment(apiUrl) {
+    let host;
+    try {
+        host = new URL(apiUrl).hostname;
+    }
+    catch {
+        return "local";
+    }
+    if (host === "api.beryl.so")
+        return "prod";
+    if (host === "dev.beryl.so")
+        return "dev";
+    return "local";
+}
+// Values kept verbatim: ids, references, enum choices and other low-cardinality
+// selectors that make the Sessions view readable. Everything else is reduced to a
+// shape, so free-text arguments (plans, credentials, mail bodies) never leave the
+// machine. Keys are the MCP tool's parameter names, hyphens and all.
+const KEEP_VALUE_KEYS = new Set([
+    "account",
+    "auth",
+    "env",
+    "env-id",
+    "environment",
+    "format",
+    "frequency",
+    "id",
+    "limit",
+    "login-method",
+    "mailbox",
+    "name",
+    "project",
+    "role",
+    "run",
+    "scope",
+    "status",
+    "test",
+    "type",
+    "url",
+    "workspace",
+]);
+// Catch-all for keys we never enumerated. Deliberately a loose substring match, which
+// makes it fire on innocent names too (`login-method`, `--force-new-login`, `--otp`) —
+// so the allowlist above and booleans, both settled decisions, are checked first, and
+// this only decides the keys nobody classified. It still guards a real `--otp 123456`,
+// because numbers are kept only after it has had its say.
+const SECRET_KEY = /pass|secret|token|key|otp|code|credential|login|value|blob|cookie|email/i;
+const PAT_PREFIX = "beryl_pat_";
+const REDACTED = "<redacted>";
+function shapeOf(value) {
+    if (typeof value === "string")
+        return `<string:${value.length}>`;
+    if (Array.isArray(value))
+        return `<array:${value.length}>`;
+    if (typeof value === "object")
+        return "<object>";
+    return `<${typeof value}>`;
+}
+function containsSecret(value) {
+    if (typeof value === "string")
+        return value.includes(PAT_PREFIX);
+    if (Array.isArray(value))
+        return value.some(containsSecret);
+    if (value && typeof value === "object")
+        return Object.values(value).some(containsSecret);
+    return false;
+}
+function isRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+export function redactArguments(args) {
+    if (!isRecord(args))
+        return shapeOf(args);
+    const out = {};
+    for (const [key, value] of Object.entries(args)) {
+        if (value === null || value === undefined)
+            continue;
+        if (containsSecret(value)) {
+            out[key] = REDACTED;
+        }
+        else if (KEEP_VALUE_KEYS.has(key) && typeof value === "string") {
+            out[key] = value;
+        }
+        else if (typeof value === "boolean") {
+            // true/false cannot carry a credential, whatever the key is called.
+            out[key] = value;
+        }
+        else if (SECRET_KEY.test(key)) {
+            out[key] = shapeOf(value);
+        }
+        else if (typeof value === "number") {
+            out[key] = value;
+        }
+        else {
+            out[key] = shapeOf(value);
+        }
+    }
+    return out;
+}
+// The SDK reports parameters as the JSON-RPC envelope — {request:{id,jsonrpc,method,
+// params:{name,arguments,_meta}}} — so the caller's values live at request.params.arguments.
+// An envelope we don't recognise is treated as arguments wholesale rather than passed
+// through: an unfamiliar shape must fail closed, not leak.
+//
+// `_meta` gets the same treatment as arguments. It is the protocol's open extension point,
+// so its contents are whatever the connected client decided to put there (today, Claude
+// Code's tool-use and progress ids) — unknown by definition, and not ours to forward.
+export function redactCapturedParameters(captured) {
+    if (!isRecord(captured))
+        return shapeOf(captured);
+    const request = captured.request;
+    const params = isRecord(request) ? request.params : undefined;
+    if (!isRecord(request) || !isRecord(params))
+        return redactArguments(captured);
+    const redacted = { ...params };
+    for (const key of ["arguments", "_meta"]) {
+        if (params[key] !== undefined)
+            redacted[key] = redactArguments(params[key]);
+    }
+    return { ...captured, request: { ...request, params: redacted } };
+}
+// How much a tool answered with, never what. Reads lengths off strings the SDK already
+// built — constant time whatever the payload — so this must not serialize the result.
+export function responseShape(response) {
+    if (!isRecord(response))
+        return undefined;
+    const content = response.content;
+    if (!Array.isArray(content))
+        return undefined;
+    let bytes = 0;
+    for (const part of content) {
+        if (!isRecord(part))
+            continue;
+        // text / image+audio / embedded resource, whose payload hangs one level deeper.
+        const resource = isRecord(part.resource) ? part.resource : undefined;
+        const payload = part.text ?? part.data ?? resource?.text ?? resource?.blob;
+        if (typeof payload === "string")
+            bytes += payload.length;
+    }
+    return { response_parts: content.length, response_bytes: bytes };
+}
+// Agent-written free text: it is told not to include secrets, but that is a prompt, not a
+// guarantee — so it gets the same PAT check as any argument, and a hard length cap.
+const MAX_INTENT = 500;
+function redactIntent(intent) {
+    if (typeof intent !== "string")
+        return undefined;
+    if (intent.includes(PAT_PREFIX))
+        return REDACTED;
+    return intent.length > MAX_INTENT ? intent.slice(0, MAX_INTENT) : intent;
+}
+// Tool responses carry plans, mailbox contents and run artifacts wholesale — there is no
+// key-level policy that makes them safe, so only their size survives.
+export const beforeSend = (event) => {
+    const props = event.properties;
+    Object.assign(props, responseShape(props.$mcp_response));
+    delete props.$mcp_response;
+    if (props.$mcp_parameters !== undefined) {
+        props.$mcp_parameters = redactCapturedParameters(props.$mcp_parameters);
+    }
+    if (props.$mcp_intent !== undefined) {
+        props.$mcp_intent = redactIntent(props.$mcp_intent);
+    }
+    return event;
+};
+export function baseEventProperties(apiUrl) {
+    return {
+        environment: analyticsEnvironment(apiUrl),
+        cli_version: cliVersion(),
+        os: process.platform,
+        arch: process.arch,
+        node_version: process.versions.node,
+        api_url: apiUrl,
+        // Same uuid as the X-Beryl-Session header, so a PostHog session and the Sentry tags
+        // on the API requests it caused describe the same thing.
+        beryl_session_id: sessionId,
+    };
+}
+let posthog;
+export function createPostHogClient() {
+    posthog ??= new PostHog(POSTHOG_KEY, { host: POSTHOG_HOST, disableGeoip: false });
+    return posthog;
+}
+export function analyticsOptions(apiUrl, identify) {
+    const properties = baseEventProperties(apiUrl);
+    return {
+        identify,
+        beforeSend,
+        eventProperties: () => properties,
+        // Worth its cost: without the agent's own reason for a call, a session reads as a list
+        // of tool names and you cannot tell deliberate work from a loop. The description is
+        // ours and deliberately terse — the SDK's default one is ~4x longer, and every word is
+        // paid for on all 86 advertised tools on every request (measured: +12.8k tokens vs +3k).
+        context: {
+            description: "Why this call is being made and how it serves the user's goal, in under 15 words. " +
+                "Third person. Never include credentials, tokens or personal data.",
+        },
+        // Would inject a second parameter into every tool to survive reconnects. A stdio
+        // process is already one session, and a user's work is stitched by distinct_id
+        // anyway — not worth another schema rewrite for tidier grouping.
+        enableConversationId: false,
+        reportMissing: false,
+        // Tool failures are already returned to the agent as isError and reported by the
+        // API's own Sentry; a second $exception stream would double-count them.
+        enableExceptionAutocapture: false,
+        logger: (message) => process.stderr.write(`[beryl telemetry] ${message}\n`),
+    };
+}
+export async function shutdownTelemetry() {
+    if (!posthog)
+        return;
+    const client = posthog;
+    posthog = undefined;
+    try {
+        await client.shutdown(2000);
+    }
+    catch {
+        // Telemetry must never delay or fail the process it is observing.
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beryl-so/cli",
-  "version": "0.27.0",
+  "version": "0.29.0",
   "description": "Beryl on the command line — projects, runs, the exploring agent, and an MCP server over the same commands.",
   "license": "MIT",
   "type": "module",
@@ -31,7 +31,9 @@
     "docs": "tsx scripts/gen-docs.ts"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.29.0"
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "@posthog/mcp": "^0.11.6",
+    "posthog-node": "^5.49.1"
   },
   "devDependencies": {
     "@playwright/test": "^1.61.1",