libretto 0.6.16 → 0.6.18

package/src/cli/core/session-telemetry.ts

@@ -1,4 +1,7 @@
 import type { BrowserContext, Page } from "playwright";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { gzipSync } from "node:zlib";
 import {
   filterSemanticClasses,
   INTERACTIVE_ROLE_NAMES,
@@ -16,18 +19,69 @@ type InstallSessionTelemetryOptions = {
   logAction: (entry: TelemetryEntry) => void;
   logNetwork: (entry: TelemetryEntry) => void;
   includeUserDomActions?: boolean;
+  rawNetworkDir?: string;
 };
 
+const BODY_PREVIEW_CHARS = 4096;
+const MAX_SAVED_BODY_BYTES = 10 * 1024 * 1024;
+const LOG_RESOURCE_TYPES = new Set(["document", "xhr", "fetch"]);
+const SKIP_RESOURCE_TYPES = new Set(["image", "font", "media", "stylesheet"]);
+const NOISE_URL_RE =
+  /(google-analytics|googletagmanager|googleadservices|googlesyndication|doubleclick|facebook\.com\/tr|pinterest|criteo|snapchat|2mdn\.net|adtrafficquality|safeframe|recaptcha|analytics|beacon|pixel|\/ads?\/|\/collect|\/event|\/pagead\/|\/gmp\/conversion|\/ccm\/|\/rmkt\/|favicon|\.map(?:\?|$))/i;
+const TEXT_CONTENT_TYPE_RE =
+  /json|html|text|xml|graphql|javascript|x-www-form-urlencoded/i;
+
+function shouldLogNetworkEntry(
+  method: string,
+  url: string,
+  resourceType: string,
+): boolean {
+  if (url.startsWith("chrome-extension://")) return false;
+  if (NOISE_URL_RE.test(url)) return false;
+  if (resourceType === "ping") return false;
+  if (LOG_RESOURCE_TYPES.has(resourceType)) return true;
+  if (["POST", "PUT", "PATCH", "DELETE"].includes(method)) return true;
+  if (SKIP_RESOURCE_TYPES.has(resourceType)) return false;
+  return false;
+}
+
+function isTextLikeContentType(contentType: string | null): boolean {
+  return contentType !== null && TEXT_CONTENT_TYPE_RE.test(contentType);
+}
+
+function bodyPreview(value: string): string {
+  return value.slice(0, BODY_PREVIEW_CHARS);
+}
+
+function saveBodySidecar(
+  rawNetworkDir: string | undefined,
+  id: number,
+  kind: "request" | "response",
+  contentType: string | null,
+  body: string,
+): string | null {
+  if (!rawNetworkDir) return null;
+  mkdirSync(rawNetworkDir, { recursive: true });
+  const ext = contentType?.includes("json")
+    ? "json"
+    : contentType?.includes("html")
+      ? "html"
+      : "txt";
+  const filename = `${String(id).padStart(6, "0")}.${kind}.${ext}.gz`;
+  writeFileSync(join(rawNetworkDir, filename), gzipSync(body));
+  return `raw-network/${filename}`;
+}
+
 export async function installSessionTelemetry(
   options: InstallSessionTelemetryOptions,
 ): Promise<void> {
-  const STATIC_EXT_RE =
-    /\.(css|js|png|jpg|jpeg|gif|woff|woff2|ttf|ico|svg)(\?|$)/i;
-  const { context, initialPage, logAction, logNetwork } = options;
+  const { context, initialPage, logAction, logNetwork, rawNetworkDir } =
+    options;
   const includeUserDomActions = options.includeUserDomActions ?? false;
   const pageIdCache = new WeakMap<Page, string>();
   const wrappedPages = new WeakSet<Page>();
   const exposedPages = new WeakSet<Page>();
+  let networkId = 0;
 
   const resolvePageId = async (page: Page): Promise<string> => {
     if (pageIdCache.has(page)) return pageIdCache.get(page)!;
@@ -748,20 +802,129 @@ export async function installSessionTelemetry(
     page.on("response", async (response) => {
       const request = response.request();
       const url = request.url();
-      if (STATIC_EXT_RE.test(url) || url.startsWith("chrome-extension://"))
-        return;
+      const method = request.method();
+      const resourceType = request.resourceType();
+      if (!shouldLogNetworkEntry(method, url, resourceType)) return;
+
+      const id = ++networkId;
+      const requestHeaders = request.headers();
+      const responseHeaders = response.headers();
+      const contentType = responseHeaders["content-type"] ?? null;
+      const requestContentType = requestHeaders["content-type"] ?? null;
+      const requestBody = request.postData();
+      const requestBodyBytes =
+        requestBody === null ? null : Buffer.byteLength(requestBody);
+      let requestBodyPath: string | null = null;
+      let requestBodyOmittedReason: string | null = null;
+      let responseBodyPreview: string | null = null;
+      let responseBodyPath: string | null = null;
+      let responseBodyBytes: number | null = null;
+      let responseBodyTruncated = false;
+      let responseBodyOmittedReason: string | null = null;
+      let errorText: string | null = null;
+
+      if (requestBody === null) {
+        requestBodyOmittedReason = "no-request-body";
+      } else if (!isTextLikeContentType(requestContentType)) {
+        requestBodyOmittedReason = "binary-content-type";
+      } else if (requestBodyBytes !== null && requestBodyBytes > MAX_SAVED_BODY_BYTES) {
+        requestBodyOmittedReason = "body-too-large";
+      } else {
+        requestBodyPath = saveBodySidecar(
+          rawNetworkDir,
+          id,
+          "request",
+          requestContentType,
+          requestBody,
+        );
+      }
+
+      if (!isTextLikeContentType(contentType) || !LOG_RESOURCE_TYPES.has(resourceType)) {
+        responseBodyOmittedReason = "binary-content-type";
+      } else {
+        try {
+          const responseBody = await response.text();
+          responseBodyBytes = Buffer.byteLength(responseBody);
+          responseBodyPreview = bodyPreview(responseBody);
+          if (responseBodyBytes > MAX_SAVED_BODY_BYTES) {
+            responseBodyTruncated = true;
+            responseBodyOmittedReason = "body-too-large";
+          } else {
+            responseBodyPath = saveBodySidecar(
+              rawNetworkDir,
+              id,
+              "response",
+              contentType,
+              responseBody,
+            );
+          }
+        } catch (error: any) {
+          responseBodyOmittedReason = "read-error";
+          errorText = error?.message ?? String(error);
+        }
+      }
+
       emitNetwork({
+        id,
         pageId,
-        method: request.method(),
+        method,
         url,
+        resourceType,
         status: response.status(),
-        contentType: response.headers()["content-type"] ?? null,
-        postData:
-          request.method() === "POST" ||
-          request.method() === "PUT" ||
-          request.method() === "PATCH"
-            ? (request.postData() ?? "").substring(0, 2000)
-            : undefined,
+        statusText: response.statusText(),
+        contentType,
+        requestHeaders,
+        responseHeaders,
+        requestBodyPreview: requestBody ? bodyPreview(requestBody) : null,
+        requestBodyPath,
+        requestBodyBytes,
+        requestBodyTruncated:
+          requestBody !== null &&
+          requestBodyBytes !== null &&
+          requestBodyBytes > MAX_SAVED_BODY_BYTES,
+        requestBodyOmittedReason,
+        responseBodyPreview,
+        responseBodyPath,
+        responseBodyBytes,
+        responseBodyTruncated,
+        responseBodyOmittedReason,
+        errorText,
+        postData: requestBody ? bodyPreview(requestBody) : undefined,
+        responseBody: null,
+        size: null,
+        durationMs: null,
+      });
+    });
+
+    page.on("requestfailed", async (request) => {
+      const url = request.url();
+      const method = request.method();
+      const resourceType = request.resourceType();
+      if (!shouldLogNetworkEntry(method, url, resourceType)) return;
+
+      const id = ++networkId;
+      emitNetwork({
+        id,
+        pageId,
+        method,
+        url,
+        resourceType,
+        status: null,
+        statusText: null,
+        contentType: null,
+        requestHeaders: request.headers(),
+        responseHeaders: null,
+        requestBodyPreview: null,
+        requestBodyPath: null,
+        requestBodyBytes: null,
+        requestBodyTruncated: false,
+        requestBodyOmittedReason: null,
+        responseBodyPreview: null,
+        responseBodyPath: null,
+        responseBodyBytes: null,
+        responseBodyTruncated: false,
+        responseBodyOmittedReason: "request-failed",
+        errorText: request.failure()?.errorText ?? null,
         responseBody: null,
         size: null,
         durationMs: null,

package/src/cli/core/skill-version.ts

@@ -14,7 +14,7 @@ const INSTALLED_SKILL_PATHS = [
 
 let cachedCliVersion: string | null = null;
 
-function readCurrentCliVersion(): string {
+export function readCurrentCliVersion(): string {
   if (cachedCliVersion) {
     return cachedCliVersion;
   }

package/src/cli/core/telemetry.ts

@@ -11,16 +11,32 @@ import {
 import { assertSessionStateExistsOrThrow } from "./session.js";
 
 export type NetworkLogEntry = {
+  id?: number;
   ts: string;
   pageId?: string;
   method: string;
   url: string;
-  status: number;
+  resourceType?: string;
+  status: number | null;
+  statusText?: string | null;
   contentType: string | null;
+  requestHeaders?: Record<string, string> | null;
+  responseHeaders?: Record<string, string> | null;
+  requestBodyPreview?: string | null;
+  requestBodyPath?: string | null;
+  requestBodyBytes?: number | null;
+  requestBodyTruncated?: boolean;
+  requestBodyOmittedReason?: string | null;
+  responseBodyPreview?: string | null;
+  responseBodyPath?: string | null;
+  responseBodyBytes?: number | null;
+  responseBodyTruncated?: boolean;
+  responseBodyOmittedReason?: string | null;
+  errorText?: string | null;
   postData?: string;
   responseBody?: string | null;
-  size: number | null;
-  durationMs: number | null;
+  size?: number | null;
+  durationMs?: number | null;
 };
 
 export function readNetworkLog(

package/src/cli/router.ts

@@ -7,12 +7,14 @@ import { experimentsCommand } from "./commands/experiments.js";
 import { setupCommand } from "./commands/setup.js";
 import { statusCommand } from "./commands/status.js";
 import { snapshotCommand } from "./commands/snapshot.js";
+import { searchCommand } from "./commands/search.js";
+import { updateCommand } from "./commands/update.js";
 import { SimpleCLI } from "affordance";
 
 export const cliRoutes = {
   ...browserCommands,
   cloud: SimpleCLI.group({
-    description: "Libretto Cloud commands",
+    description: "Deploy workflows and manage hosted Libretto",
     routes: {
       deploy: deployCommand,
       auth: authCommands,
@@ -21,11 +23,20 @@ export const cliRoutes = {
   }),
   experiments: experimentsCommand,
   ...executionCommands,
+  search: searchCommand,
   setup: setupCommand,
   status: statusCommand,
   snapshot: snapshotCommand,
+  update: updateCommand,
 };
 
 export function createCLIApp() {
-  return SimpleCLI.define("libretto", cliRoutes);
+  return SimpleCLI.define("libretto", cliRoutes, {
+    appendHelpText: [
+      "Options:",
+      "  --session <name>  Required for session-scoped commands",
+      "  -h, --help",
+      "  -v, --version",
+    ].join("\n"),
+  });
 }

package/src/shared/html-search/search-html.spec.ts

@@ -0,0 +1,65 @@
+import { describe, expect, test } from "vitest";
+import { formatHtmlForSearch, searchFormattedHtml } from "./search-html.js";
+
+describe("HTML search", () => {
+  test("formats condensed HTML before searching", () => {
+    const formatted = formatHtmlForSearch(
+      '<!doctype html><html><body><main><p data-testid="target">Needle</p></main></body></html>',
+    );
+
+    expect(formatted).toContain('<p data-testid="target">');
+    expect(formatted).toContain("Needle");
+  });
+
+  test("returns merged matching regions with context", () => {
+    const formatted = [
+      "<html>",
+      "<body>",
+      "<main>",
+      "<section>",
+      "<h1>Heading</h1>",
+      '<p class="target">Needle</p>',
+      "<p>More content</p>",
+      "</section>",
+      "</main>",
+      "</body>",
+      "</html>",
+    ].join("\n");
+
+    const matches = searchFormattedHtml(formatted, "Needle", 2);
+
+    expect(matches).toEqual([
+      {
+        startLine: 4,
+        endLine: 8,
+        lines: [
+          "<section>",
+          "<h1>Heading</h1>",
+          '<p class="target">Needle</p>',
+          "<p>More content</p>",
+          "</section>",
+        ],
+      },
+    ]);
+  });
+
+  test("limits matching regions before adding context", () => {
+    const formatted = Array.from(
+      { length: 12 },
+      (_value, index) => `<p>Needle ${index}</p>`,
+    ).join("\n");
+
+    const matches = searchFormattedHtml(formatted, "Needle", 0, 8);
+
+    expect(matches).toEqual([
+      {
+        startLine: 1,
+        endLine: 8,
+        lines: Array.from(
+          { length: 8 },
+          (_value, index) => `<p>Needle ${index}</p>`,
+        ),
+      },
+    ]);
+  });
+});

package/src/shared/html-search/search-html.ts

@@ -0,0 +1,75 @@
+import { condenseDom } from "../condense-dom/condense-dom.js";
+
+const DEFAULT_CONTEXT_LINES = 4;
+const DEFAULT_MATCH_LIMIT = 8;
+
+export type SearchHtmlMatch = {
+  startLine: number;
+  endLine: number;
+  lines: string[];
+};
+
+export function formatHtmlForSearch(html: string): string {
+  const condensed = condenseDom(html).html;
+  const separated = condensed
+    .replace(/>\s+</g, ">\n<")
+    .replace(/(<[^/!][^>]*>)([^<\n][\s\S]*?)(<\/[^>]+>)/g, "$1\n$2\n$3");
+
+  const lines = separated
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+
+  let indent = 0;
+  return lines
+    .map((line) => {
+      if (/^<\//.test(line)) indent = Math.max(0, indent - 1);
+      const formatted = `${"  ".repeat(indent)}${line}`;
+      if (isOpeningTag(line)) indent += 1;
+      return formatted;
+    })
+    .join("\n");
+}
+
+export function searchFormattedHtml(
+  formattedHtml: string,
+  pattern: string,
+  contextLines = DEFAULT_CONTEXT_LINES,
+  matchLimit = DEFAULT_MATCH_LIMIT,
+): SearchHtmlMatch[] {
+  const regex = new RegExp(pattern);
+  const lines = formattedHtml.split("\n");
+  const matchingIndexes = lines
+    .map((line, index) => (regex.test(line) ? index : -1))
+    .filter((index) => index >= 0)
+    .slice(0, matchLimit);
+
+  const matches: SearchHtmlMatch[] = [];
+  for (const matchingIndex of matchingIndexes) {
+    const startLine = Math.max(0, matchingIndex - contextLines);
+    const endLine = Math.min(lines.length - 1, matchingIndex + contextLines);
+    const previous = matches.at(-1);
+    if (previous && startLine <= previous.endLine) {
+      previous.endLine = Math.max(previous.endLine, endLine + 1);
+      previous.lines = lines.slice(previous.startLine - 1, previous.endLine);
+      continue;
+    }
+    matches.push({
+      startLine: startLine + 1,
+      endLine: endLine + 1,
+      lines: lines.slice(startLine, endLine + 1),
+    });
+  }
+  return matches;
+}
+
+function isOpeningTag(line: string): boolean {
+  return (
+    /^<[^/!?][^>]*>$/.test(line) &&
+    !/\/>$/.test(line) &&
+    !/^<(area|base|br|col|embed|hr|img|input|link|meta|param|source|track|wbr)\b/i.test(
+      line,
+    ) &&
+    !/^<[^>]+>.*<\/[^>]+>$/.test(line)
+  );
+}