@canivel/ralph 0.2.0

package/skills/dev-browser/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "dev-browser",
+  "version": "0.0.1",
+  "type": "module",
+  "imports": {
+    "@/*": "./src/*"
+  },
+  "scripts": {
+    "start-server": "npx tsx scripts/start-server.ts",
+    "start-extension": "npx tsx scripts/start-relay.ts",
+    "dev": "npx tsx --watch src/index.ts",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@hono/node-server": "^1.19.7",
+    "@hono/node-ws": "^1.2.0",
+    "express": "^4.21.0",
+    "hono": "^4.11.1",
+    "playwright": "^1.49.0"
+  },
+  "devDependencies": {
+    "@types/express": "^5.0.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.0.0",
+    "vitest": "^2.1.0"
+  },
+  "optionalDependencies": {
+    "@rollup/rollup-linux-x64-gnu": "^4.0.0"
+  }
+}

package/skills/dev-browser/references/scraping.md

@@ -0,0 +1,155 @@
+# Data Scraping Guide
+
+For large datasets (followers, posts, search results), **intercept and replay network requests** rather than scrolling and parsing the DOM. This is faster, more reliable, and handles pagination automatically.
+
+## Why Not Scroll?
+
+Scrolling is slow, unreliable, and wastes time. APIs return structured data with pagination built in. Always prefer API replay.
+
+## Start Small, Then Scale
+
+**Don't try to automate everything at once.** Work incrementally:
+
+1. **Capture one request** - verify you're intercepting the right endpoint
+2. **Inspect one response** - understand the schema before writing extraction code
+3. **Extract a few items** - make sure your parsing logic works
+4. **Then scale up** - add pagination loop only after the basics work
+
+This prevents wasting time debugging a complex script when the issue is a simple path like `data.user.timeline` vs `data.user.result.timeline`.
+
+## Step-by-Step Workflow
+
+### 1. Capture Request Details
+
+First, intercept a request to understand URL structure and required headers:
+
+```typescript
+import { connect, waitForPageLoad } from "@/client.js";
+import * as fs from "node:fs";
+
+const client = await connect();
+const page = await client.page("site");
+
+let capturedRequest = null;
+page.on("request", (request) => {
+  const url = request.url();
+  // Look for API endpoints (adjust pattern for your target site)
+  if (url.includes("/api/") || url.includes("/graphql/")) {
+    capturedRequest = {
+      url: url,
+      headers: request.headers(),
+      method: request.method(),
+    };
+    fs.writeFileSync("tmp/request-details.json", JSON.stringify(capturedRequest, null, 2));
+    console.log("Captured request:", url.substring(0, 80) + "...");
+  }
+});
+
+await page.goto("https://example.com/profile");
+await waitForPageLoad(page);
+await page.waitForTimeout(3000);
+
+await client.disconnect();
+```
+
+### 2. Capture Response to Understand Schema
+
+Save a raw response to inspect the data structure:
+
+```typescript
+page.on("response", async (response) => {
+  const url = response.url();
+  if (url.includes("UserTweets") || url.includes("/api/data")) {
+    const json = await response.json();
+    fs.writeFileSync("tmp/api-response.json", JSON.stringify(json, null, 2));
+    console.log("Captured response");
+  }
+});
+```
+
+Then analyze the structure to find:
+
+- Where the data array lives (e.g., `data.user.result.timeline.instructions[].entries`)
+- Where pagination cursors are (e.g., `cursor-bottom` entries)
+- What fields you need to extract
+
+### 3. Replay API with Pagination
+
+Once you understand the schema, replay requests directly:
+
+```typescript
+import { connect } from "@/client.js";
+import * as fs from "node:fs";
+
+const client = await connect();
+const page = await client.page("site");
+
+const results = new Map(); // Use Map for deduplication
+const headers = JSON.parse(fs.readFileSync("tmp/request-details.json", "utf8")).headers;
+const baseUrl = "https://example.com/api/data";
+
+let cursor = null;
+let hasMore = true;
+
+while (hasMore) {
+  // Build URL with pagination cursor
+  const params = { count: 20 };
+  if (cursor) params.cursor = cursor;
+  const url = `${baseUrl}?params=${encodeURIComponent(JSON.stringify(params))}`;
+
+  // Execute fetch in browser context (has auth cookies/headers)
+  const response = await page.evaluate(
+    async ({ url, headers }) => {
+      const res = await fetch(url, { headers });
+      return res.json();
+    },
+    { url, headers }
+  );
+
+  // Extract data and cursor (adjust paths for your API)
+  const entries = response?.data?.entries || [];
+  for (const entry of entries) {
+    if (entry.type === "cursor-bottom") {
+      cursor = entry.value;
+    } else if (entry.id && !results.has(entry.id)) {
+      results.set(entry.id, {
+        id: entry.id,
+        text: entry.content,
+        timestamp: entry.created_at,
+      });
+    }
+  }
+
+  console.log(`Fetched page, total: ${results.size}`);
+
+  // Check stop conditions
+  if (!cursor || entries.length === 0) hasMore = false;
+
+  // Rate limiting - be respectful
+  await new Promise((r) => setTimeout(r, 500));
+}
+
+// Export results
+const data = Array.from(results.values());
+fs.writeFileSync("tmp/results.json", JSON.stringify(data, null, 2));
+console.log(`Saved ${data.length} items`);
+
+await client.disconnect();
+```
+
+## Key Patterns
+
+| Pattern                 | Description                                            |
+| ----------------------- | ------------------------------------------------------ |
+| `page.on('request')`    | Capture outgoing request URL + headers                 |
+| `page.on('response')`   | Capture response data to understand schema             |
+| `page.evaluate(fetch)`  | Replay requests in browser context (inherits auth)     |
+| `Map` for deduplication | APIs often return overlapping data across pages        |
+| Cursor-based pagination | Look for `cursor`, `next_token`, `offset` in responses |
+
+## Tips
+
+- **Extension mode**: `page.context().cookies()` doesn't work - capture auth headers from intercepted requests instead
+- **Rate limiting**: Add 500ms+ delays between requests to avoid blocks
+- **Stop conditions**: Check for empty results, missing cursor, or reaching a date/ID threshold
+- **GraphQL APIs**: URL params often include `variables` and `features` JSON objects - capture and reuse them

package/skills/dev-browser/scripts/start-relay.ts

@@ -0,0 +1,32 @@
+/**
+ * Start the CDP relay server for Chrome extension mode
+ *
+ * Usage: npm run start-extension
+ */
+
+import { serveRelay } from "@/relay.js";
+
+const PORT = parseInt(process.env.PORT || "9222", 10);
+const HOST = process.env.HOST || "127.0.0.1";
+
+async function main() {
+  const server = await serveRelay({
+    port: PORT,
+    host: HOST,
+  });
+
+  // Handle shutdown
+  const shutdown = async () => {
+    console.log("\nShutting down relay server...");
+    await server.stop();
+    process.exit(0);
+  };
+
+  process.on("SIGINT", shutdown);
+  process.on("SIGTERM", shutdown);
+}
+
+main().catch((err) => {
+  console.error("Failed to start relay server:", err);
+  process.exit(1);
+});

package/skills/dev-browser/scripts/start-server.ts

@@ -0,0 +1,117 @@
+import { serve } from "@/index.js";
+import { execSync } from "child_process";
+import { mkdirSync, existsSync, readdirSync } from "fs";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const tmpDir = join(__dirname, "..", "tmp");
+const profileDir = join(__dirname, "..", "profiles");
+
+// Create tmp and profile directories if they don't exist
+console.log("Creating tmp directory...");
+mkdirSync(tmpDir, { recursive: true });
+console.log("Creating profiles directory...");
+mkdirSync(profileDir, { recursive: true });
+
+// Install Playwright browsers if not already installed
+console.log("Checking Playwright browser installation...");
+
+function findPackageManager(): { name: string; command: string } | null {
+  const managers = [
+    { name: "bun", command: "bunx playwright install chromium" },
+    { name: "pnpm", command: "pnpm exec playwright install chromium" },
+    { name: "npm", command: "npx playwright install chromium" },
+  ];
+
+  for (const manager of managers) {
+    try {
+      execSync(`which ${manager.name}`, { stdio: "ignore" });
+      return manager;
+    } catch {
+      // Package manager not found, try next
+    }
+  }
+  return null;
+}
+
+function isChromiumInstalled(): boolean {
+  const homeDir = process.env.HOME || process.env.USERPROFILE || "";
+  const playwrightCacheDir = join(homeDir, ".cache", "ms-playwright");
+
+  if (!existsSync(playwrightCacheDir)) {
+    return false;
+  }
+
+  // Check for chromium directories (e.g., chromium-1148, chromium_headless_shell-1148)
+  try {
+    const entries = readdirSync(playwrightCacheDir);
+    return entries.some((entry) => entry.startsWith("chromium"));
+  } catch {
+    return false;
+  }
+}
+
+try {
+  if (!isChromiumInstalled()) {
+    console.log("Playwright Chromium not found. Installing (this may take a minute)...");
+
+    const pm = findPackageManager();
+    if (!pm) {
+      throw new Error("No package manager found (tried bun, pnpm, npm)");
+    }
+
+    console.log(`Using ${pm.name} to install Playwright...`);
+    execSync(pm.command, { stdio: "inherit" });
+    console.log("Chromium installed successfully.");
+  } else {
+    console.log("Playwright Chromium already installed.");
+  }
+} catch (error) {
+  console.error("Failed to install Playwright browsers:", error);
+  console.log("You may need to run: npx playwright install chromium");
+}
+
+// Check if server is already running
+console.log("Checking for existing servers...");
+try {
+  const res = await fetch("http://localhost:9222", {
+    signal: AbortSignal.timeout(1000),
+  });
+  if (res.ok) {
+    console.log("Server already running on port 9222");
+    process.exit(0);
+  }
+} catch {
+  // Server not running, continue to start
+}
+
+// Clean up stale CDP port if HTTP server isn't running (crash recovery)
+// This handles the case where Node crashed but Chrome is still running on 9223
+try {
+  const pid = execSync("lsof -ti:9223", { encoding: "utf-8" }).trim();
+  if (pid) {
+    console.log(`Cleaning up stale Chrome process on CDP port 9223 (PID: ${pid})`);
+    execSync(`kill -9 ${pid}`);
+  }
+} catch {
+  // No process on CDP port, which is expected
+}
+
+console.log("Starting dev browser server...");
+const headless = process.env.HEADLESS === "true";
+const server = await serve({
+  port: 9222,
+  headless,
+  profileDir,
+});
+
+console.log(`Dev browser server started`);
+console.log(`  WebSocket: ${server.wsEndpoint}`);
+console.log(`  Tmp directory: ${tmpDir}`);
+console.log(`  Profile directory: ${profileDir}`);
+console.log(`\nReady`);
+console.log(`\nPress Ctrl+C to stop`);
+
+// Keep the process running
+await new Promise(() => {});

package/skills/dev-browser/server.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Get the directory where this script is located
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# Change to the script directory
+cd "$SCRIPT_DIR"
+
+# Parse command line arguments
+HEADLESS=false
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --headless) HEADLESS=true ;;
+        *) echo "Unknown parameter: $1"; exit 1 ;;
+    esac
+    shift
+done
+
+echo "Installing dependencies..."
+npm install
+
+echo "Starting dev-browser server..."
+export HEADLESS=$HEADLESS
+npx tsx scripts/start-server.ts