nolo-cli 0.1.23 → 0.1.29

package/integrations/x-reader/backends/rawCdp.ts

@@ -0,0 +1,218 @@
+import {
+  createXReadFailure,
+  type XPost,
+  type XReadResult,
+  type XThread,
+} from "../types";
+import { parseVisibleXPostText } from "../visibleTextParser";
+import type { LocalBrowserReader } from "./localBrowser";
+
+type CdpResponse = {
+  id?: number;
+  method?: string;
+  params?: Record<string, unknown>;
+  result?: Record<string, unknown>;
+  error?: { message?: string };
+};
+
+export type RawCdpXReaderOptions = {
+  endpoint: string;
+  timeoutMs?: number;
+};
+
+export async function getBrowserWebSocketUrl(
+  endpoint: string,
+  fetchImpl: typeof fetch = fetch,
+): Promise<string> {
+  const response = await fetchImpl(`${endpoint.replace(/\/$/, "")}/json/version`);
+  if (!response.ok) {
+    throw new Error(`CDP version endpoint returned ${response.status}`);
+  }
+
+  const data = (await response.json()) as { webSocketDebuggerUrl?: string };
+  if (!data.webSocketDebuggerUrl) {
+    throw new Error("CDP version endpoint did not include webSocketDebuggerUrl");
+  }
+
+  return data.webSocketDebuggerUrl;
+}
+
+async function connectRawCdp(wsUrl: string, timeoutMs: number): Promise<WebSocket> {
+  return await new Promise((resolve, reject) => {
+    const ws = new WebSocket(wsUrl);
+    const timeout = setTimeout(
+      () => reject(new Error(`CDP WebSocket open timed out after ${timeoutMs}ms`)),
+      timeoutMs,
+    );
+
+    ws.onopen = () => {
+      clearTimeout(timeout);
+      resolve(ws);
+    };
+    ws.onerror = () => {
+      clearTimeout(timeout);
+      reject(new Error("CDP WebSocket failed to open"));
+    };
+  });
+}
+
+export async function readVisibleTextOverRawCdp(args: {
+  endpoint: string;
+  url: string;
+  timeoutMs?: number;
+}): Promise<string> {
+  const timeoutMs = args.timeoutMs ?? 15000;
+  const ws = await connectRawCdp(
+    await getBrowserWebSocketUrl(args.endpoint),
+    timeoutMs,
+  );
+  let nextId = 1;
+  const pending = new Map<
+    number,
+    { resolve(value: CdpResponse): void; reject(error: Error): void }
+  >();
+
+  ws.onmessage = (event) => {
+    const message = JSON.parse(String(event.data)) as CdpResponse;
+    if (typeof message.id !== "number") {
+      return;
+    }
+
+    const waiter = pending.get(message.id);
+    if (!waiter) {
+      return;
+    }
+    pending.delete(message.id);
+
+    if (message.error) {
+      waiter.reject(new Error(message.error.message ?? "CDP command failed"));
+    } else {
+      waiter.resolve(message);
+    }
+  };
+
+  function send(
+    method: string,
+    params: Record<string, unknown> = {},
+    sessionId?: string,
+  ): Promise<CdpResponse> {
+    const id = nextId++;
+    ws.send(JSON.stringify({ id, method, params, sessionId }));
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        pending.delete(id);
+        reject(new Error(`${method} timed out after ${timeoutMs}ms`));
+      }, timeoutMs);
+
+      pending.set(id, {
+        resolve(value) {
+          clearTimeout(timeout);
+          resolve(value);
+        },
+        reject(error) {
+          clearTimeout(timeout);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  try {
+    const target = await send("Target.createTarget", { url: "about:blank" });
+    const targetId = String(target.result?.targetId ?? "");
+    const attached = await send("Target.attachToTarget", {
+      targetId,
+      flatten: true,
+    });
+    const sessionId = String(attached.result?.sessionId ?? "");
+
+    await send("Page.enable", {}, sessionId);
+    await send("Runtime.enable", {}, sessionId);
+    await send("Page.navigate", { url: args.url }, sessionId);
+
+    const deadline = Date.now() + timeoutMs;
+    let lastText = "";
+    let stableReads = 0;
+    const targetHandle = /x\.com\/([^/]+)\/status\//.exec(args.url)?.[1];
+    while (Date.now() <= deadline) {
+      const evaluated = await send(
+        "Runtime.evaluate",
+        {
+          expression: "document.body ? document.body.innerText : ''",
+          returnByValue: true,
+        },
+        sessionId,
+      );
+      const result = evaluated.result?.result as { value?: unknown } | undefined;
+      const text = typeof result?.value === "string" ? result.value : "";
+      if (
+        text.includes(`@${targetHandle}`) &&
+        text.length === lastText.length &&
+        text.length > 0
+      ) {
+        stableReads += 1;
+      } else {
+        stableReads = 0;
+      }
+      lastText = text;
+      if ((!targetHandle && lastText) || stableReads >= 2) {
+        return lastText;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 500));
+    }
+
+    return lastText;
+  } finally {
+    ws.close();
+  }
+}
+
+export function createRawCdpXReader(
+  options: RawCdpXReaderOptions,
+): LocalBrowserReader {
+  const timeoutMs = options.timeoutMs ?? 15000;
+
+  return {
+    async readVisiblePost(url: string): Promise<XReadResult<XPost>> {
+      const fetchedAt = new Date().toISOString();
+      try {
+        const visibleText = await readVisibleTextOverRawCdp({
+          endpoint: options.endpoint,
+          url,
+          timeoutMs,
+        });
+        return parseVisibleXPostText(visibleText, {
+          url,
+          fetchedAt,
+        });
+      } catch (error) {
+        return createXReadFailure({
+          code: "network_error",
+          message: error instanceof Error ? error.message : "Unknown raw CDP failure.",
+          nextStep:
+            "Start Chrome with --remote-debugging-port or check the raw CDP endpoint.",
+          backend: "desktop_local_browser",
+          fetchedAt,
+        });
+      }
+    },
+    async readVisibleThread(url: string): Promise<XReadResult<XThread>> {
+      const postResult = await this.readVisiblePost(url);
+      if (!postResult.ok) {
+        return postResult;
+      }
+
+      return {
+        ok: true,
+        backend: "desktop_local_browser",
+        fetchedAt: postResult.fetchedAt,
+        data: {
+          root: postResult.data,
+          posts: [postResult.data],
+          completeness: "single_post",
+          missingReason: "unsupported_content",
+        },
+      };
+    },
+  };
+}

package/integrations/x-reader/bridge/chromeBridgeManager.ts

@@ -0,0 +1,235 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { mkdir } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+export type ChromeBridgeSession = {
+  endpoint: string;
+  port: number;
+  profileDir: string;
+  webSocketDebuggerUrl: string;
+};
+
+type ChromeBridgeProcess = Pick<ChildProcess, "kill" | "killed" | "exitCode" | "once">;
+
+export type ChromeBridgeManagerOptions = {
+  chromePath?: string | (() => string | Promise<string>);
+  profileDir?: string;
+  profileRoot?: string;
+  host?: string;
+  headless?: boolean;
+  disableSandbox?: boolean;
+  startupTimeoutMs?: number;
+  pollIntervalMs?: number;
+  portAllocator?: () => Promise<number>;
+  spawnProcess?: (
+    command: string,
+    args: string[],
+    options: { stdio: "ignore"; windowsHide: boolean },
+  ) => ChromeBridgeProcess;
+  ensureDir?: (path: string) => Promise<void>;
+  fetchVersion?: (
+    versionEndpoint: string,
+  ) => Promise<{ webSocketDebuggerUrl?: string }>;
+};
+
+export type ChromeBridgeManager = {
+  start(): Promise<ChromeBridgeSession>;
+  stop(): Promise<void>;
+  getSession(): ChromeBridgeSession | null;
+};
+
+const DEFAULT_HOST = "127.0.0.1";
+
+function defaultChromePath() {
+  return process.platform === "win32"
+    ? "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
+    : "google-chrome";
+}
+
+async function resolveChromePath(
+  chromePath: NonNullable<ChromeBridgeManagerOptions["chromePath"]>,
+) {
+  return typeof chromePath === "function" ? await chromePath() : chromePath;
+}
+
+function defaultProfileRoot() {
+  return join(tmpdir(), "nolo-x-reader-bridge");
+}
+
+async function defaultFetchVersion(versionEndpoint: string) {
+  const response = await fetch(versionEndpoint);
+  if (!response.ok) {
+    throw new Error(`CDP version endpoint returned ${response.status}`);
+  }
+
+  return (await response.json()) as { webSocketDebuggerUrl?: string };
+}
+
+async function defaultPortAllocator() {
+  return 9222 + Math.floor(Math.random() * 1000);
+}
+
+function buildChromeArgs(args: {
+  host: string;
+  port: number;
+  profileDir: string;
+  headless: boolean;
+  disableSandbox: boolean;
+}) {
+  const chromeArgs = [
+    "--disable-gpu",
+    `--remote-debugging-address=${args.host}`,
+    `--remote-debugging-port=${args.port}`,
+    `--user-data-dir=${args.profileDir}`,
+    "--no-first-run",
+    "--no-default-browser-check",
+    "about:blank",
+  ];
+
+  if (args.headless) {
+    chromeArgs.unshift("--headless=new");
+  }
+
+  if (args.disableSandbox) {
+    chromeArgs.unshift("--disable-dev-shm-usage");
+    chromeArgs.unshift("--no-sandbox");
+  }
+
+  return chromeArgs;
+}
+
+async function waitForVersion(args: {
+  endpoint: string;
+  fetchVersion: NonNullable<ChromeBridgeManagerOptions["fetchVersion"]>;
+  startupTimeoutMs: number;
+  pollIntervalMs: number;
+}) {
+  const deadline = Date.now() + args.startupTimeoutMs;
+  let lastError: unknown;
+
+  while (Date.now() <= deadline) {
+    try {
+      const version = await args.fetchVersion(`${args.endpoint}/json/version`);
+      if (version.webSocketDebuggerUrl) {
+        return version.webSocketDebuggerUrl;
+      }
+      lastError = new Error("CDP version response did not include webSocketDebuggerUrl");
+    } catch (error) {
+      lastError = error;
+    }
+    await Bun.sleep(args.pollIntervalMs);
+  }
+
+  const message = lastError instanceof Error ? lastError.message : String(lastError);
+  throw new Error(`Chrome bridge did not become ready: ${message}`);
+}
+
+export function createChromeBridgeManager(
+  options: ChromeBridgeManagerOptions = {},
+): ChromeBridgeManager {
+  const host = options.host ?? DEFAULT_HOST;
+  const chromePath = options.chromePath ?? defaultChromePath;
+  const fixedProfileDir = options.profileDir;
+  const profileRoot = options.profileRoot ?? defaultProfileRoot();
+  const headless = options.headless ?? true;
+  const disableSandbox = options.disableSandbox ?? process.platform !== "win32";
+  const startupTimeoutMs = options.startupTimeoutMs ?? 15000;
+  const pollIntervalMs = options.pollIntervalMs ?? 250;
+  const portAllocator = options.portAllocator ?? defaultPortAllocator;
+  const spawnProcess =
+    options.spawnProcess ??
+    ((command, args, spawnOptions) =>
+      spawn(command, args, {
+        stdio: spawnOptions.stdio,
+        windowsHide: spawnOptions.windowsHide,
+      }));
+  const ensureDir = options.ensureDir ?? ((path: string) => mkdir(path, { recursive: true }));
+  const fetchVersion = options.fetchVersion ?? defaultFetchVersion;
+
+  let activeSession: ChromeBridgeSession | null = null;
+  let activeProcess: ChromeBridgeProcess | null = null;
+  let startupPromise: Promise<ChromeBridgeSession> | null = null;
+
+  async function start() {
+    if (activeSession && activeProcess && !activeProcess.killed && activeProcess.exitCode === null) {
+      return activeSession;
+    }
+
+    if (startupPromise) {
+      return startupPromise;
+    }
+
+    startupPromise = (async () => {
+      const port = await portAllocator();
+      const endpoint = `http://${host}:${port}`;
+      const profileDir = fixedProfileDir ?? join(profileRoot, `profile-${randomUUID()}`);
+      await ensureDir(profileDir);
+      const executablePath = await resolveChromePath(chromePath);
+
+      let child: ChromeBridgeProcess;
+      try {
+        child = spawnProcess(
+          executablePath,
+          buildChromeArgs({ host, port, profileDir, headless, disableSandbox }),
+          {
+            stdio: "ignore",
+            windowsHide: true,
+          },
+        );
+      } catch (error: any) {
+        throw Object.assign(
+          new Error(
+            `Chrome bridge browser is unavailable at ${executablePath}: ${error?.message ?? String(error)}`,
+          ),
+          { code: "READ_X_POST_BROWSER_UNAVAILABLE" },
+        );
+      }
+      activeProcess = child;
+      child.once("exit", () => {
+        if (activeProcess === child) {
+          activeProcess = null;
+          activeSession = null;
+        }
+      });
+
+      const webSocketDebuggerUrl = await waitForVersion({
+        endpoint,
+        fetchVersion,
+        startupTimeoutMs,
+        pollIntervalMs,
+      });
+
+      activeSession = {
+        endpoint,
+        port,
+        profileDir,
+        webSocketDebuggerUrl,
+      };
+      return activeSession;
+    })().finally(() => {
+      startupPromise = null;
+    });
+
+    return startupPromise;
+  }
+
+  async function stop() {
+    const child = activeProcess;
+    activeProcess = null;
+    activeSession = null;
+
+    if (child && !child.killed && child.exitCode === null) {
+      child.kill();
+    }
+  }
+
+  return {
+    start,
+    stop,
+    getSession() {
+      return activeSession;
+    },
+  };
+}

package/integrations/x-reader/bridge/readXPostWithBridge.ts

@@ -0,0 +1,73 @@
+import type { XPost, XReadResult } from "../types";
+import {
+  createLocalBrowserBackend,
+  type LocalBrowserReader,
+} from "../backends/localBrowser";
+import { createRawCdpXReader } from "../backends/rawCdp";
+import {
+  createChromeBridgeManager,
+  type ChromeBridgeManager,
+} from "./chromeBridgeManager";
+
+export type ReadXPostWithBridgeOptions = {
+  bridge?: ChromeBridgeManager;
+  readerFactory?: (endpoint: string) => LocalBrowserReader;
+  keepOpen?: boolean;
+  chromePath?: string;
+  profileDir?: string;
+  profileRoot?: string;
+  headless?: boolean;
+};
+
+function envFlagEnabled(value: string | undefined) {
+  if (value == null) return undefined;
+  return !["0", "false", "no", "off"].includes(value.trim().toLowerCase());
+}
+
+async function resolveDefaultChromePath() {
+  const envPath = process.env.NOLO_X_READER_CHROME_PATH?.trim();
+  if (envPath) {
+    return envPath;
+  }
+
+  if (process.platform !== "win32") {
+    try {
+      const { chromium } = await import("playwright");
+      return chromium.executablePath();
+    } catch {
+      // Fall back to PATH-based Chrome lookup below.
+    }
+  }
+
+  return process.platform === "win32"
+    ? "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
+    : "google-chrome";
+}
+
+export async function readXPostWithBridge(
+  url: string,
+  options: ReadXPostWithBridgeOptions = {},
+): Promise<XReadResult<XPost>> {
+  const bridge =
+    options.bridge ??
+    createChromeBridgeManager({
+      chromePath: options.chromePath ?? resolveDefaultChromePath,
+      profileDir: options.profileDir ?? process.env.NOLO_X_READER_PROFILE_DIR,
+      profileRoot: options.profileRoot ?? process.env.NOLO_X_READER_PROFILE_ROOT,
+      headless:
+        options.headless ??
+        envFlagEnabled(process.env.NOLO_X_READER_HEADLESS) ??
+        true,
+    });
+  const session = await bridge.start();
+  try {
+    const reader =
+      options.readerFactory?.(session.endpoint) ??
+      createRawCdpXReader({ endpoint: session.endpoint });
+    return await createLocalBrowserBackend(reader).readPost(url);
+  } finally {
+    if (!options.keepOpen) {
+      await bridge.stop();
+    }
+  }
+}

package/integrations/x-reader/fixtureParser.ts

@@ -0,0 +1,101 @@
+import {
+  createXReadFailure,
+  type XBackendName,
+  type XMedia,
+  type XPost,
+  type XReadResult,
+} from "./types";
+
+type ParseOptions = {
+  backend: XBackendName;
+  fetchedAt?: string;
+};
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value : undefined;
+}
+
+function asNumber(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function parseMedia(value: unknown): XMedia[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  return value.map((item) => {
+    const media = asRecord(item) ?? {};
+    const type = asString(media.type);
+    return {
+      type:
+        type === "image" || type === "video" || type === "gif"
+          ? type
+          : "unknown",
+      url: asString(media.url),
+      previewUrl: asString(media.previewUrl),
+      altText: asString(media.altText),
+    };
+  });
+}
+
+export function parseXPostFixture(
+  value: unknown,
+  options: ParseOptions,
+): XReadResult<XPost> {
+  const fetchedAt = options.fetchedAt ?? new Date().toISOString();
+  const raw = asRecord(value);
+  const author = asRecord(raw?.author);
+  const id = asString(raw?.id);
+  const url = asString(raw?.url);
+  const handle = asString(author?.handle);
+  const text = asString(raw?.text);
+
+  if (!raw || !id || !url || !handle || !text) {
+    return createXReadFailure({
+      code: "parse_error",
+      message: "Fixture is missing id, url, author.handle, or text.",
+      nextStep: "Check the sanitized X fixture shape before parsing it.",
+      backend: options.backend,
+      fetchedAt,
+    });
+  }
+
+  const metrics = asRecord(raw.metrics);
+  return {
+    ok: true,
+    backend: options.backend,
+    fetchedAt,
+    data: {
+      id,
+      url,
+      author: {
+        handle,
+        displayName: asString(author?.displayName),
+        id: asString(author?.id),
+        verified:
+          typeof author?.verified === "boolean" ? author.verified : undefined,
+        profileUrl: asString(author?.profileUrl),
+      },
+      text,
+      createdAt: asString(raw.createdAt),
+      media: parseMedia(raw.media),
+      metrics: metrics
+        ? {
+            replies: asNumber(metrics.replies),
+            reposts: asNumber(metrics.reposts),
+            likes: asNumber(metrics.likes),
+            views: asNumber(metrics.views),
+          }
+        : undefined,
+      sourceBackend: options.backend,
+      fetchedAt,
+    },
+  };
+}

package/integrations/x-reader/index.ts

@@ -0,0 +1,9 @@
+export * from "./types";
+export * from "./redaction";
+export * from "./fixtureParser";
+export * from "./visibleTextParser";
+export * from "./backends/publicFallback";
+export * from "./backends/localBrowser";
+export * from "./backends/rawCdp";
+export * from "./bridge/chromeBridgeManager";
+export * from "./bridge/readXPostWithBridge";

package/integrations/x-reader/redaction.ts

@@ -0,0 +1,27 @@
+const SENSITIVE_KEY_PATTERN =
+  /^(authorization|cookie|set-cookie|x-csrf-token|csrf|ct0|auth_token|token|access_token|refresh_token|secret)$/i;
+
+const SENSITIVE_STRING_PATTERN =
+  /\b(Bearer\s+[A-Za-z0-9._~+/=-]+|auth_token=[^;\s]+|ct0=[^;\s]+)\b/i;
+
+export function redactXReaderValue(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map((item) => redactXReaderValue(item));
+  }
+
+  if (value && typeof value === "object") {
+    const redacted: Record<string, unknown> = {};
+    for (const [key, child] of Object.entries(value)) {
+      redacted[key] = SENSITIVE_KEY_PATTERN.test(key)
+        ? "[REDACTED]"
+        : redactXReaderValue(child);
+    }
+    return redacted;
+  }
+
+  if (typeof value === "string" && SENSITIVE_STRING_PATTERN.test(value)) {
+    return "[REDACTED]";
+  }
+
+  return value;
+}