nolo-cli 0.1.30 → 0.1.32

package/integrations/x-reader/backends/rawCdp.ts

@@ -1,218 +0,0 @@
-import {
-  createXReadFailure,
-  type XPost,
-  type XReadResult,
-  type XThread,
-} from "../types";
-import { parseVisibleXPostText } from "../visibleTextParser";
-import type { LocalBrowserReader } from "./localBrowser";
-
-type CdpResponse = {
-  id?: number;
-  method?: string;
-  params?: Record<string, unknown>;
-  result?: Record<string, unknown>;
-  error?: { message?: string };
-};
-
-export type RawCdpXReaderOptions = {
-  endpoint: string;
-  timeoutMs?: number;
-};
-
-export async function getBrowserWebSocketUrl(
-  endpoint: string,
-  fetchImpl: typeof fetch = fetch,
-): Promise<string> {
-  const response = await fetchImpl(`${endpoint.replace(/\/$/, "")}/json/version`);
-  if (!response.ok) {
-    throw new Error(`CDP version endpoint returned ${response.status}`);
-  }
-
-  const data = (await response.json()) as { webSocketDebuggerUrl?: string };
-  if (!data.webSocketDebuggerUrl) {
-    throw new Error("CDP version endpoint did not include webSocketDebuggerUrl");
-  }
-
-  return data.webSocketDebuggerUrl;
-}
-
-async function connectRawCdp(wsUrl: string, timeoutMs: number): Promise<WebSocket> {
-  return await new Promise((resolve, reject) => {
-    const ws = new WebSocket(wsUrl);
-    const timeout = setTimeout(
-      () => reject(new Error(`CDP WebSocket open timed out after ${timeoutMs}ms`)),
-      timeoutMs,
-    );
-
-    ws.onopen = () => {
-      clearTimeout(timeout);
-      resolve(ws);
-    };
-    ws.onerror = () => {
-      clearTimeout(timeout);
-      reject(new Error("CDP WebSocket failed to open"));
-    };
-  });
-}
-
-export async function readVisibleTextOverRawCdp(args: {
-  endpoint: string;
-  url: string;
-  timeoutMs?: number;
-}): Promise<string> {
-  const timeoutMs = args.timeoutMs ?? 15000;
-  const ws = await connectRawCdp(
-    await getBrowserWebSocketUrl(args.endpoint),
-    timeoutMs,
-  );
-  let nextId = 1;
-  const pending = new Map<
-    number,
-    { resolve(value: CdpResponse): void; reject(error: Error): void }
-  >();
-
-  ws.onmessage = (event) => {
-    const message = JSON.parse(String(event.data)) as CdpResponse;
-    if (typeof message.id !== "number") {
-      return;
-    }
-
-    const waiter = pending.get(message.id);
-    if (!waiter) {
-      return;
-    }
-    pending.delete(message.id);
-
-    if (message.error) {
-      waiter.reject(new Error(message.error.message ?? "CDP command failed"));
-    } else {
-      waiter.resolve(message);
-    }
-  };
-
-  function send(
-    method: string,
-    params: Record<string, unknown> = {},
-    sessionId?: string,
-  ): Promise<CdpResponse> {
-    const id = nextId++;
-    ws.send(JSON.stringify({ id, method, params, sessionId }));
-    return new Promise((resolve, reject) => {
-      const timeout = setTimeout(() => {
-        pending.delete(id);
-        reject(new Error(`${method} timed out after ${timeoutMs}ms`));
-      }, timeoutMs);
-
-      pending.set(id, {
-        resolve(value) {
-          clearTimeout(timeout);
-          resolve(value);
-        },
-        reject(error) {
-          clearTimeout(timeout);
-          reject(error);
-        },
-      });
-    });
-  }
-
-  try {
-    const target = await send("Target.createTarget", { url: "about:blank" });
-    const targetId = String(target.result?.targetId ?? "");
-    const attached = await send("Target.attachToTarget", {
-      targetId,
-      flatten: true,
-    });
-    const sessionId = String(attached.result?.sessionId ?? "");
-
-    await send("Page.enable", {}, sessionId);
-    await send("Runtime.enable", {}, sessionId);
-    await send("Page.navigate", { url: args.url }, sessionId);
-
-    const deadline = Date.now() + timeoutMs;
-    let lastText = "";
-    let stableReads = 0;
-    const targetHandle = /x\.com\/([^/]+)\/status\//.exec(args.url)?.[1];
-    while (Date.now() <= deadline) {
-      const evaluated = await send(
-        "Runtime.evaluate",
-        {
-          expression: "document.body ? document.body.innerText : ''",
-          returnByValue: true,
-        },
-        sessionId,
-      );
-      const result = evaluated.result?.result as { value?: unknown } | undefined;
-      const text = typeof result?.value === "string" ? result.value : "";
-      if (
-        text.includes(`@${targetHandle}`) &&
-        text.length === lastText.length &&
-        text.length > 0
-      ) {
-        stableReads += 1;
-      } else {
-        stableReads = 0;
-      }
-      lastText = text;
-      if ((!targetHandle && lastText) || stableReads >= 2) {
-        return lastText;
-      }
-      await new Promise((resolve) => setTimeout(resolve, 500));
-    }
-
-    return lastText;
-  } finally {
-    ws.close();
-  }
-}
-
-export function createRawCdpXReader(
-  options: RawCdpXReaderOptions,
-): LocalBrowserReader {
-  const timeoutMs = options.timeoutMs ?? 15000;
-
-  return {
-    async readVisiblePost(url: string): Promise<XReadResult<XPost>> {
-      const fetchedAt = new Date().toISOString();
-      try {
-        const visibleText = await readVisibleTextOverRawCdp({
-          endpoint: options.endpoint,
-          url,
-          timeoutMs,
-        });
-        return parseVisibleXPostText(visibleText, {
-          url,
-          fetchedAt,
-        });
-      } catch (error) {
-        return createXReadFailure({
-          code: "network_error",
-          message: error instanceof Error ? error.message : "Unknown raw CDP failure.",
-          nextStep:
-            "Start Chrome with --remote-debugging-port or check the raw CDP endpoint.",
-          backend: "desktop_local_browser",
-          fetchedAt,
-        });
-      }
-    },
-    async readVisibleThread(url: string): Promise<XReadResult<XThread>> {
-      const postResult = await this.readVisiblePost(url);
-      if (!postResult.ok) {
-        return postResult;
-      }
-
-      return {
-        ok: true,
-        backend: "desktop_local_browser",
-        fetchedAt: postResult.fetchedAt,
-        data: {
-          root: postResult.data,
-          posts: [postResult.data],
-          completeness: "single_post",
-          missingReason: "unsupported_content",
-        },
-      };
-    },
-  };
-}

package/integrations/x-reader/bridge/chromeBridgeManager.ts

@@ -1,235 +0,0 @@
-import { spawn, type ChildProcess } from "node:child_process";
-import { randomUUID } from "node:crypto";
-import { mkdir } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-export type ChromeBridgeSession = {
-  endpoint: string;
-  port: number;
-  profileDir: string;
-  webSocketDebuggerUrl: string;
-};
-
-type ChromeBridgeProcess = Pick<ChildProcess, "kill" | "killed" | "exitCode" | "once">;
-
-export type ChromeBridgeManagerOptions = {
-  chromePath?: string | (() => string | Promise<string>);
-  profileDir?: string;
-  profileRoot?: string;
-  host?: string;
-  headless?: boolean;
-  disableSandbox?: boolean;
-  startupTimeoutMs?: number;
-  pollIntervalMs?: number;
-  portAllocator?: () => Promise<number>;
-  spawnProcess?: (
-    command: string,
-    args: string[],
-    options: { stdio: "ignore"; windowsHide: boolean },
-  ) => ChromeBridgeProcess;
-  ensureDir?: (path: string) => Promise<void>;
-  fetchVersion?: (
-    versionEndpoint: string,
-  ) => Promise<{ webSocketDebuggerUrl?: string }>;
-};
-
-export type ChromeBridgeManager = {
-  start(): Promise<ChromeBridgeSession>;
-  stop(): Promise<void>;
-  getSession(): ChromeBridgeSession | null;
-};
-
-const DEFAULT_HOST = "127.0.0.1";
-
-function defaultChromePath() {
-  return process.platform === "win32"
-    ? "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
-    : "google-chrome";
-}
-
-async function resolveChromePath(
-  chromePath: NonNullable<ChromeBridgeManagerOptions["chromePath"]>,
-) {
-  return typeof chromePath === "function" ? await chromePath() : chromePath;
-}
-
-function defaultProfileRoot() {
-  return join(tmpdir(), "nolo-x-reader-bridge");
-}
-
-async function defaultFetchVersion(versionEndpoint: string) {
-  const response = await fetch(versionEndpoint);
-  if (!response.ok) {
-    throw new Error(`CDP version endpoint returned ${response.status}`);
-  }
-
-  return (await response.json()) as { webSocketDebuggerUrl?: string };
-}
-
-async function defaultPortAllocator() {
-  return 9222 + Math.floor(Math.random() * 1000);
-}
-
-function buildChromeArgs(args: {
-  host: string;
-  port: number;
-  profileDir: string;
-  headless: boolean;
-  disableSandbox: boolean;
-}) {
-  const chromeArgs = [
-    "--disable-gpu",
-    `--remote-debugging-address=${args.host}`,
-    `--remote-debugging-port=${args.port}`,
-    `--user-data-dir=${args.profileDir}`,
-    "--no-first-run",
-    "--no-default-browser-check",
-    "about:blank",
-  ];
-
-  if (args.headless) {
-    chromeArgs.unshift("--headless=new");
-  }
-
-  if (args.disableSandbox) {
-    chromeArgs.unshift("--disable-dev-shm-usage");
-    chromeArgs.unshift("--no-sandbox");
-  }
-
-  return chromeArgs;
-}
-
-async function waitForVersion(args: {
-  endpoint: string;
-  fetchVersion: NonNullable<ChromeBridgeManagerOptions["fetchVersion"]>;
-  startupTimeoutMs: number;
-  pollIntervalMs: number;
-}) {
-  const deadline = Date.now() + args.startupTimeoutMs;
-  let lastError: unknown;
-
-  while (Date.now() <= deadline) {
-    try {
-      const version = await args.fetchVersion(`${args.endpoint}/json/version`);
-      if (version.webSocketDebuggerUrl) {
-        return version.webSocketDebuggerUrl;
-      }
-      lastError = new Error("CDP version response did not include webSocketDebuggerUrl");
-    } catch (error) {
-      lastError = error;
-    }
-    await Bun.sleep(args.pollIntervalMs);
-  }
-
-  const message = lastError instanceof Error ? lastError.message : String(lastError);
-  throw new Error(`Chrome bridge did not become ready: ${message}`);
-}
-
-export function createChromeBridgeManager(
-  options: ChromeBridgeManagerOptions = {},
-): ChromeBridgeManager {
-  const host = options.host ?? DEFAULT_HOST;
-  const chromePath = options.chromePath ?? defaultChromePath;
-  const fixedProfileDir = options.profileDir;
-  const profileRoot = options.profileRoot ?? defaultProfileRoot();
-  const headless = options.headless ?? true;
-  const disableSandbox = options.disableSandbox ?? process.platform !== "win32";
-  const startupTimeoutMs = options.startupTimeoutMs ?? 15000;
-  const pollIntervalMs = options.pollIntervalMs ?? 250;
-  const portAllocator = options.portAllocator ?? defaultPortAllocator;
-  const spawnProcess =
-    options.spawnProcess ??
-    ((command, args, spawnOptions) =>
-      spawn(command, args, {
-        stdio: spawnOptions.stdio,
-        windowsHide: spawnOptions.windowsHide,
-      }));
-  const ensureDir = options.ensureDir ?? ((path: string) => mkdir(path, { recursive: true }));
-  const fetchVersion = options.fetchVersion ?? defaultFetchVersion;
-
-  let activeSession: ChromeBridgeSession | null = null;
-  let activeProcess: ChromeBridgeProcess | null = null;
-  let startupPromise: Promise<ChromeBridgeSession> | null = null;
-
-  async function start() {
-    if (activeSession && activeProcess && !activeProcess.killed && activeProcess.exitCode === null) {
-      return activeSession;
-    }
-
-    if (startupPromise) {
-      return startupPromise;
-    }
-
-    startupPromise = (async () => {
-      const port = await portAllocator();
-      const endpoint = `http://${host}:${port}`;
-      const profileDir = fixedProfileDir ?? join(profileRoot, `profile-${randomUUID()}`);
-      await ensureDir(profileDir);
-      const executablePath = await resolveChromePath(chromePath);
-
-      let child: ChromeBridgeProcess;
-      try {
-        child = spawnProcess(
-          executablePath,
-          buildChromeArgs({ host, port, profileDir, headless, disableSandbox }),
-          {
-            stdio: "ignore",
-            windowsHide: true,
-          },
-        );
-      } catch (error: any) {
-        throw Object.assign(
-          new Error(
-            `Chrome bridge browser is unavailable at ${executablePath}: ${error?.message ?? String(error)}`,
-          ),
-          { code: "READ_X_POST_BROWSER_UNAVAILABLE" },
-        );
-      }
-      activeProcess = child;
-      child.once("exit", () => {
-        if (activeProcess === child) {
-          activeProcess = null;
-          activeSession = null;
-        }
-      });
-
-      const webSocketDebuggerUrl = await waitForVersion({
-        endpoint,
-        fetchVersion,
-        startupTimeoutMs,
-        pollIntervalMs,
-      });
-
-      activeSession = {
-        endpoint,
-        port,
-        profileDir,
-        webSocketDebuggerUrl,
-      };
-      return activeSession;
-    })().finally(() => {
-      startupPromise = null;
-    });
-
-    return startupPromise;
-  }
-
-  async function stop() {
-    const child = activeProcess;
-    activeProcess = null;
-    activeSession = null;
-
-    if (child && !child.killed && child.exitCode === null) {
-      child.kill();
-    }
-  }
-
-  return {
-    start,
-    stop,
-    getSession() {
-      return activeSession;
-    },
-  };
-}

package/integrations/x-reader/bridge/readXPostWithBridge.ts

@@ -1,73 +0,0 @@
-import type { XPost, XReadResult } from "../types";
-import {
-  createLocalBrowserBackend,
-  type LocalBrowserReader,
-} from "../backends/localBrowser";
-import { createRawCdpXReader } from "../backends/rawCdp";
-import {
-  createChromeBridgeManager,
-  type ChromeBridgeManager,
-} from "./chromeBridgeManager";
-
-export type ReadXPostWithBridgeOptions = {
-  bridge?: ChromeBridgeManager;
-  readerFactory?: (endpoint: string) => LocalBrowserReader;
-  keepOpen?: boolean;
-  chromePath?: string;
-  profileDir?: string;
-  profileRoot?: string;
-  headless?: boolean;
-};
-
-function envFlagEnabled(value: string | undefined) {
-  if (value == null) return undefined;
-  return !["0", "false", "no", "off"].includes(value.trim().toLowerCase());
-}
-
-async function resolveDefaultChromePath() {
-  const envPath = process.env.NOLO_X_READER_CHROME_PATH?.trim();
-  if (envPath) {
-    return envPath;
-  }
-
-  if (process.platform !== "win32") {
-    try {
-      const { chromium } = await import("playwright");
-      return chromium.executablePath();
-    } catch {
-      // Fall back to PATH-based Chrome lookup below.
-    }
-  }
-
-  return process.platform === "win32"
-    ? "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
-    : "google-chrome";
-}
-
-export async function readXPostWithBridge(
-  url: string,
-  options: ReadXPostWithBridgeOptions = {},
-): Promise<XReadResult<XPost>> {
-  const bridge =
-    options.bridge ??
-    createChromeBridgeManager({
-      chromePath: options.chromePath ?? resolveDefaultChromePath,
-      profileDir: options.profileDir ?? process.env.NOLO_X_READER_PROFILE_DIR,
-      profileRoot: options.profileRoot ?? process.env.NOLO_X_READER_PROFILE_ROOT,
-      headless:
-        options.headless ??
-        envFlagEnabled(process.env.NOLO_X_READER_HEADLESS) ??
-        true,
-    });
-  const session = await bridge.start();
-  try {
-    const reader =
-      options.readerFactory?.(session.endpoint) ??
-      createRawCdpXReader({ endpoint: session.endpoint });
-    return await createLocalBrowserBackend(reader).readPost(url);
-  } finally {
-    if (!options.keepOpen) {
-      await bridge.stop();
-    }
-  }
-}

package/integrations/x-reader/fixtureParser.ts

@@ -1,101 +0,0 @@
-import {
-  createXReadFailure,
-  type XBackendName,
-  type XMedia,
-  type XPost,
-  type XReadResult,
-} from "./types";
-
-type ParseOptions = {
-  backend: XBackendName;
-  fetchedAt?: string;
-};
-
-function asRecord(value: unknown): Record<string, unknown> | undefined {
-  return value && typeof value === "object" && !Array.isArray(value)
-    ? (value as Record<string, unknown>)
-    : undefined;
-}
-
-function asString(value: unknown): string | undefined {
-  return typeof value === "string" && value.trim() ? value : undefined;
-}
-
-function asNumber(value: unknown): number | undefined {
-  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
-}
-
-function parseMedia(value: unknown): XMedia[] {
-  if (!Array.isArray(value)) {
-    return [];
-  }
-
-  return value.map((item) => {
-    const media = asRecord(item) ?? {};
-    const type = asString(media.type);
-    return {
-      type:
-        type === "image" || type === "video" || type === "gif"
-          ? type
-          : "unknown",
-      url: asString(media.url),
-      previewUrl: asString(media.previewUrl),
-      altText: asString(media.altText),
-    };
-  });
-}
-
-export function parseXPostFixture(
-  value: unknown,
-  options: ParseOptions,
-): XReadResult<XPost> {
-  const fetchedAt = options.fetchedAt ?? new Date().toISOString();
-  const raw = asRecord(value);
-  const author = asRecord(raw?.author);
-  const id = asString(raw?.id);
-  const url = asString(raw?.url);
-  const handle = asString(author?.handle);
-  const text = asString(raw?.text);
-
-  if (!raw || !id || !url || !handle || !text) {
-    return createXReadFailure({
-      code: "parse_error",
-      message: "Fixture is missing id, url, author.handle, or text.",
-      nextStep: "Check the sanitized X fixture shape before parsing it.",
-      backend: options.backend,
-      fetchedAt,
-    });
-  }
-
-  const metrics = asRecord(raw.metrics);
-  return {
-    ok: true,
-    backend: options.backend,
-    fetchedAt,
-    data: {
-      id,
-      url,
-      author: {
-        handle,
-        displayName: asString(author?.displayName),
-        id: asString(author?.id),
-        verified:
-          typeof author?.verified === "boolean" ? author.verified : undefined,
-        profileUrl: asString(author?.profileUrl),
-      },
-      text,
-      createdAt: asString(raw.createdAt),
-      media: parseMedia(raw.media),
-      metrics: metrics
-        ? {
-            replies: asNumber(metrics.replies),
-            reposts: asNumber(metrics.reposts),
-            likes: asNumber(metrics.likes),
-            views: asNumber(metrics.views),
-          }
-        : undefined,
-      sourceBackend: options.backend,
-      fetchedAt,
-    },
-  };
-}

package/integrations/x-reader/index.ts

@@ -1,9 +0,0 @@
-export * from "./types";
-export * from "./redaction";
-export * from "./fixtureParser";
-export * from "./visibleTextParser";
-export * from "./backends/publicFallback";
-export * from "./backends/localBrowser";
-export * from "./backends/rawCdp";
-export * from "./bridge/chromeBridgeManager";
-export * from "./bridge/readXPostWithBridge";

package/integrations/x-reader/redaction.ts

@@ -1,27 +0,0 @@
-const SENSITIVE_KEY_PATTERN =
-  /^(authorization|cookie|set-cookie|x-csrf-token|csrf|ct0|auth_token|token|access_token|refresh_token|secret)$/i;
-
-const SENSITIVE_STRING_PATTERN =
-  /\b(Bearer\s+[A-Za-z0-9._~+/=-]+|auth_token=[^;\s]+|ct0=[^;\s]+)\b/i;
-
-export function redactXReaderValue(value: unknown): unknown {
-  if (Array.isArray(value)) {
-    return value.map((item) => redactXReaderValue(item));
-  }
-
-  if (value && typeof value === "object") {
-    const redacted: Record<string, unknown> = {};
-    for (const [key, child] of Object.entries(value)) {
-      redacted[key] = SENSITIVE_KEY_PATTERN.test(key)
-        ? "[REDACTED]"
-        : redactXReaderValue(child);
-    }
-    return redacted;
-  }
-
-  if (typeof value === "string" && SENSITIVE_STRING_PATTERN.test(value)) {
-    return "[REDACTED]";
-  }
-
-  return value;
-}