orangeslice 1.4.2 → 1.5.0

package/README.md

@@ -14,7 +14,8 @@ This copies documentation to `./orangeslice-docs/` and installs the package. Poi
 |----------|------------|
 | `b2b` | Query 1B+ LinkedIn profiles, companies, funding, jobs |
 | `serp` | Google search for news, articles, reviews |
-| `firecrawl` | Scrape websites, extract social URLs |
+| `firecrawl` | Scrape static websites, extract social URLs |
+| `browser` | Playwright automation for dynamic/JS sites |
 
 ## Quick Example
 

package/dist/browser.d.ts

@@ -0,0 +1,68 @@
+export interface BrowserResponse {
+    success: boolean;
+    result?: any;
+    error?: string;
+    browser_live_view_url?: string;
+}
+export interface BrowserOptions {
+    /** Browser pool ID (default: pre-warmed pool) */
+    pool?: string;
+    /** Execution timeout in seconds */
+    timeout_sec?: number;
+    /** Timeout for acquiring browser from pool */
+    acquire_timeout_seconds?: number;
+}
+/**
+ * Execute Playwright code with `page` in scope.
+ * Browser is automatically acquired from a pre-warmed pool and released when done.
+ *
+ * @param code - Playwright code to execute (has `page` in scope)
+ * @param options - Optional settings for timeout and pool
+ *
+ * @example
+ * // Get page snapshot for analysis
+ * const response = await browser.execute(`
+ *   await page.goto(url, { waitUntil: 'domcontentloaded' });
+ *   return await page._snapshotForAI();
+ * `);
+ *
+ * @example
+ * // Extract data from page
+ * const response = await browser.execute(`
+ *   await page.goto("https://example.com", { waitUntil: 'domcontentloaded' });
+ *   return await page.evaluate(() => {
+ *     return [...document.querySelectorAll('.item')].map(el => ({
+ *       title: el.querySelector('h2')?.textContent?.trim(),
+ *       url: el.querySelector('a')?.href
+ *     }));
+ *   });
+ * `);
+ * // response = { success: true, result: [...] }
+ */
+export declare function execute(code: string, options?: BrowserOptions): Promise<BrowserResponse>;
+/**
+ * Get a page snapshot for AI analysis.
+ * Useful for discovering selectors before extraction.
+ *
+ * @param url - URL to navigate to
+ *
+ * @example
+ * const snapshot = await browser.snapshot("https://example.com/products");
+ * // Returns page HTML structure for selector discovery
+ */
+export declare function snapshot(url: string): Promise<BrowserResponse>;
+/**
+ * Extract text content from a URL.
+ *
+ * @param url - URL to navigate to
+ *
+ * @example
+ * const response = await browser.text("https://example.com");
+ * // response.result = page text content
+ */
+export declare function text(url: string): Promise<BrowserResponse>;
+export declare const browser: {
+    execute: typeof execute;
+    snapshot: typeof snapshot;
+    text: typeof text;
+};

package/dist/browser.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.browser = void 0;
+exports.execute = execute;
+exports.snapshot = snapshot;
+exports.text = text;
+const queue_1 = require("./queue");
+const API_URL = process.env.ORANGESLICE_API_URL || "https://orangeslice.ai/api/function?functionId=browser";
+// Shared queue for browser requests (limit concurrent browser sessions)
+const queue = (0, queue_1.createQueue)(2);
+const rateLimiter = (0, queue_1.createRateLimiter)(500); // 500ms between requests
+/**
+ * Helper to make POST request, handling redirects manually
+ * (Node.js fetch has issues with POST body on redirects)
+ */
+async function fetchWithRedirect(url, body) {
+    let response = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body,
+        redirect: "manual",
+    });
+    // Handle redirect manually - re-POST to the new location
+    if (response.status >= 300 && response.status < 400) {
+        const location = response.headers.get("location");
+        if (location) {
+            response = await fetch(location, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body,
+            });
+        }
+    }
+    return response;
+}
+/**
+ * Execute Playwright code with `page` in scope.
+ * Browser is automatically acquired from a pre-warmed pool and released when done.
+ *
+ * @param code - Playwright code to execute (has `page` in scope)
+ * @param options - Optional settings for timeout and pool
+ *
+ * @example
+ * // Get page snapshot for analysis
+ * const response = await browser.execute(`
+ *   await page.goto(url, { waitUntil: 'domcontentloaded' });
+ *   return await page._snapshotForAI();
+ * `);
+ *
+ * @example
+ * // Extract data from page
+ * const response = await browser.execute(`
+ *   await page.goto("https://example.com", { waitUntil: 'domcontentloaded' });
+ *   return await page.evaluate(() => {
+ *     return [...document.querySelectorAll('.item')].map(el => ({
+ *       title: el.querySelector('h2')?.textContent?.trim(),
+ *       url: el.querySelector('a')?.href
+ *     }));
+ *   });
+ * `);
+ * // response = { success: true, result: [...] }
+ */
+async function execute(code, options = {}) {
+    return queue(async () => {
+        return rateLimiter(async () => {
+            const body = JSON.stringify({ code, ...options });
+            const response = await fetchWithRedirect(API_URL, body);
+            if (!response.ok) {
+                throw new Error(`Browser request failed: ${response.status} ${response.statusText}`);
+            }
+            const data = (await response.json());
+            return data;
+        });
+    });
+}
+/**
+ * Get a page snapshot for AI analysis.
+ * Useful for discovering selectors before extraction.
+ *
+ * @param url - URL to navigate to
+ *
+ * @example
+ * const snapshot = await browser.snapshot("https://example.com/products");
+ * // Returns page HTML structure for selector discovery
+ */
+async function snapshot(url) {
+    const code = `
+      await page.goto(${JSON.stringify(url)}, { waitUntil: 'domcontentloaded' });
+      return await page._snapshotForAI();
+   `;
+    return execute(code);
+}
+/**
+ * Extract text content from a URL.
+ *
+ * @param url - URL to navigate to
+ *
+ * @example
+ * const response = await browser.text("https://example.com");
+ * // response.result = page text content
+ */
+async function text(url) {
+    const code = `
+      await page.goto(${JSON.stringify(url)}, { waitUntil: 'domcontentloaded' });
+      return await page.evaluate(() => document.body.innerText);
+   `;
+    return execute(code);
+}
+// Export as namespace
+exports.browser = {
+    execute,
+    snapshot,
+    text,
+};

package/dist/index.d.ts

@@ -1,7 +1,8 @@
 import { b2b } from "./b2b";
 import { serp } from "./serp";
 import { firecrawl } from "./firecrawl";
-export { b2b, serp, firecrawl };
+import { browser } from "./browser";
+export { b2b, serp, firecrawl, browser };
 /**
  * Main orangeslice namespace - AI sales agent toolkit
  *
@@ -14,9 +15,15 @@ export { b2b, serp, firecrawl };
  * // Google Search
  * const results = await orangeslice.serp.search("best CRM software 2024");
  *
- * // Website Scraping
+ * // Website Scraping (simple)
  * const page = await orangeslice.firecrawl.scrape("https://stripe.com/about");
  *
+ * // Browser Automation (Playwright)
+ * const data = await orangeslice.browser.execute(`
+ *   await page.goto("https://example.com", { waitUntil: 'domcontentloaded' });
+ *   return await page.evaluate(() => document.title);
+ * `);
+ *
  * // All calls are automatically rate-limited and queued
  */
 export declare const orangeslice: {
@@ -34,5 +41,10 @@ export declare const orangeslice: {
         markdown: typeof import("./firecrawl").markdown;
         socials: typeof import("./firecrawl").socials;
     };
+    browser: {
+        execute: typeof import("./browser").execute;
+        snapshot: typeof import("./browser").snapshot;
+        text: typeof import("./browser").text;
+    };
 };
 export default orangeslice;

package/dist/index.js

@@ -1,12 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.orangeslice = exports.firecrawl = exports.serp = exports.b2b = void 0;
+exports.orangeslice = exports.browser = exports.firecrawl = exports.serp = exports.b2b = void 0;
 const b2b_1 = require("./b2b");
 Object.defineProperty(exports, "b2b", { enumerable: true, get: function () { return b2b_1.b2b; } });
 const serp_1 = require("./serp");
 Object.defineProperty(exports, "serp", { enumerable: true, get: function () { return serp_1.serp; } });
 const firecrawl_1 = require("./firecrawl");
 Object.defineProperty(exports, "firecrawl", { enumerable: true, get: function () { return firecrawl_1.firecrawl; } });
+const browser_1 = require("./browser");
+Object.defineProperty(exports, "browser", { enumerable: true, get: function () { return browser_1.browser; } });
 /**
  * Main orangeslice namespace - AI sales agent toolkit
  *
@@ -19,14 +21,21 @@ Object.defineProperty(exports, "firecrawl", { enumerable: true, get: function ()
  * // Google Search
  * const results = await orangeslice.serp.search("best CRM software 2024");
  *
- * // Website Scraping
+ * // Website Scraping (simple)
  * const page = await orangeslice.firecrawl.scrape("https://stripe.com/about");
  *
+ * // Browser Automation (Playwright)
+ * const data = await orangeslice.browser.execute(`
+ *   await page.goto("https://example.com", { waitUntil: 'domcontentloaded' });
+ *   return await page.evaluate(() => document.title);
+ * `);
+ *
  * // All calls are automatically rate-limited and queued
  */
 exports.orangeslice = {
     b2b: b2b_1.b2b,
     serp: serp_1.serp,
     firecrawl: firecrawl_1.firecrawl,
+    browser: browser_1.browser,
 };
 exports.default = exports.orangeslice;

package/docs/AGENTS.md

@@ -3,7 +3,7 @@
 You are a B2B sales research agent with access to:
 - **1.15 billion LinkedIn profiles** and millions of companies
 - **Google Search** (SERP API)
-- **Website scraping** (Firecrawl)
+- **Website scraping** (Firecrawl + Browser automation)
 
 ## What You Can Do
 
@@ -14,8 +14,8 @@ You are a B2B sales research agent with access to:
 | **Employee lookup** | `b2b` | Search employees by title, role, or department |
 | **Funding intelligence** | `b2b` | Find recently funded companies and their investors |
 | **Google search** | `serp` | Search for company news, press releases, reviews |
-| **Website scraping** | `firecrawl` | Extract content from company websites |
-
+| **Website scraping** | `firecrawl` | Extract content from static websites |
+| **Browser automation** | `browser` | Scrape dynamic/JS sites with Playwright |
 
 ## Quick Start
 
@@ -31,8 +31,14 @@ const company = await orangeslice.b2b.sql(`
 // 2. Google Search - Find news, articles, reviews
 const news = await orangeslice.serp.search("Stripe funding 2024");
 
-// 3. Website Scraping - Get company page content
+// 3. Website Scraping (simple) - Static pages
 const about = await orangeslice.firecrawl.scrape("https://stripe.com/about");
+
+// 4. Browser Automation (advanced) - Dynamic/JS pages
+const data = await orangeslice.browser.execute(`
+  await page.goto("https://example.com", { waitUntil: 'domcontentloaded' });
+  return await page.evaluate(() => document.title);
+`);
 ```
 
 All calls are automatically rate-limited. Fire away freely.
@@ -184,6 +190,102 @@ const site = await orangeslice.firecrawl.scrape("https://company.com", 5);
 
 ---
 
+## Browser Automation (Playwright)
+
+Execute Playwright code with `page` in scope. Use for dynamic/JS-rendered pages that Firecrawl can't handle.
+
+**When to use Browser vs Firecrawl:**
+- `firecrawl` - Static pages, simple content extraction
+- `browser` - Dynamic pages, JS rendering, complex interactions, bot-protected sites
+
+```typescript
+// Execute Playwright code - page is already available
+const response = await orangeslice.browser.execute(`
+  await page.goto("https://example.com", { waitUntil: 'domcontentloaded' });
+  return await page.evaluate(() => {
+    return [...document.querySelectorAll('.item')].map(el => ({
+      title: el.querySelector('h2')?.textContent?.trim(),
+      url: el.querySelector('a')?.href
+    }));
+  });
+`);
+// response = { success: true, result: [...] } or { success: false, error: "..." }
+
+// Get page snapshot for selector discovery
+const snapshot = await orangeslice.browser.snapshot("https://example.com");
+
+// Just get text content
+const text = await orangeslice.browser.text("https://example.com");
+```
+
+### Workflow: Analyze → Extract
+
+**Step 1: Discover selectors first**
+```typescript
+const response = await orangeslice.browser.execute(`
+  await page.goto(url, { waitUntil: 'domcontentloaded' });
+  return await page._snapshotForAI();  // Get page structure
+`);
+// Analyze snapshot to find selectors
+```
+
+**Step 2: Extract with discovered selectors**
+```typescript
+const response = await orangeslice.browser.execute(`
+  await page.goto(url, { waitUntil: 'domcontentloaded' });
+  return await page.evaluate(() => {
+    return [...document.querySelectorAll('.discovered-selector')].map(e => ({
+      name: e.querySelector('h2')?.textContent?.trim(),
+      price: e.querySelector('.price')?.textContent?.trim()
+    }));
+  });
+`);
+```
+
+### Bot Protection
+
+For bot-protected sites, use single-session navigation:
+
+```typescript
+const response = await orangeslice.browser.execute(`
+  // Navigate to entry page (passes bot check once)
+  await page.goto(entryUrl, { waitUntil: 'domcontentloaded' });
+  
+  // Get all URLs to visit
+  const urls = await page.evaluate(() => 
+    [...document.querySelectorAll('a.link')].map(a => a.href)
+  );
+  
+  // Visit each IN THE SAME SESSION
+  const results = [];
+  for (const url of urls.slice(0, 10)) {
+    await page.goto(url, { waitUntil: 'domcontentloaded' });
+    const data = await page.evaluate(() => ({
+      title: document.querySelector('h1')?.textContent?.trim()
+    }));
+    results.push(data);
+  }
+  return results;
+`);
+```
+
+### Options
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `timeout_sec` | number | Execution timeout (default 60, max 180) |
+| `acquire_timeout_seconds` | number | Browser pool acquire timeout |
+
+### Rules
+
+1. **Always use `{ waitUntil: 'domcontentloaded' }`** - Prevents hanging
+2. **Check `response.success`** - Don't just destructure `result`
+3. **Analyze before extracting** - Use `_snapshotForAI()` to find selectors
+4. **Return objects, not HTML** - Use `page.evaluate()` to extract structured data
+5. **3 minute hard limit** - Plan multi-page scrapes accordingly
+
+---
+
 ## Key Tables
 
 | Table | Records | Use For |
@@ -271,6 +373,7 @@ The `orangeslice` package automatically handles rate limiting:
 | `b2b` | 2 concurrent | 100ms |
 | `serp` | 2 concurrent | 200ms |
 | `firecrawl` | 2 concurrent | 500ms |
+| `browser` | 2 concurrent | 500ms |
 
 You can fire off many calls - they'll be queued automatically.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orangeslice",
-  "version": "1.4.2",
+  "version": "1.5.0",
   "description": "Turn any AI agent into a B2B sales research assistant with 1B+ LinkedIn profiles",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",