@lobu/cli 6.0.1 → 6.1.1

package/dist/connectors/__tests__/browser-scraper-utils.test.ts

@@ -0,0 +1,186 @@
+import { describe, expect, mock, test } from 'bun:test';
+
+// browser-scraper-utils.ts pulls runtime symbols (acquireBrowser,
+// captureErrorArtifacts) from @lobu/connector-sdk, which itself pulls in
+// playwright. Stub the SDK so the pure helpers (validateUrlDomain,
+// getBrowserCookies, validateCookieNotExpired, filterByCheckpoint) can be
+// imported without spinning up a real browser stack.
+mock.module('@lobu/connector-sdk', () => ({
+  acquireBrowser: () => {
+    throw new Error('not used in unit tests');
+  },
+  captureErrorArtifacts: () => {
+    throw new Error('not used in unit tests');
+  },
+}));
+
+const {
+  filterByCheckpoint,
+  getBrowserCookies,
+  validateCookieNotExpired,
+  validateUrlDomain,
+} = await import('../browser-scraper-utils.ts');
+
+describe('validateUrlDomain', () => {
+  test('accepts a well-formed https URL on the expected domain', () => {
+    expect(() =>
+      validateUrlDomain('https://www.trustpilot.com/review/foo', 'trustpilot.com')
+    ).not.toThrow();
+  });
+
+  test('accepts a subdomain of the expected domain', () => {
+    expect(() => validateUrlDomain('https://api.example.com/x', 'example.com')).not.toThrow();
+  });
+
+  test('rejects a malformed URL', () => {
+    expect(() => validateUrlDomain('not a url', 'example.com')).toThrow(/Invalid example.com URL/);
+  });
+
+  test('rejects http (non-https) URLs', () => {
+    expect(() => validateUrlDomain('http://www.example.com/', 'example.com')).toThrow(
+      /must use https: protocol/
+    );
+  });
+
+  test('rejects a URL on a different domain', () => {
+    expect(() => validateUrlDomain('https://evil.com/foo', 'example.com')).toThrow(
+      /must be on example.com/
+    );
+  });
+
+  test('rejects a substring-match hostname (security: notexample.com is NOT example.com)', () => {
+    expect(() => validateUrlDomain('https://notexample.com/x', 'example.com')).toThrow(
+      /must be on example.com/
+    );
+    expect(() => validateUrlDomain('https://eviltrustpilot.com/x', 'trustpilot.com')).toThrow(
+      /must be on trustpilot.com/
+    );
+  });
+
+  test('accepts the apex domain itself (no subdomain)', () => {
+    expect(() => validateUrlDomain('https://example.com/x', 'example.com')).not.toThrow();
+  });
+});
+
+describe('getBrowserCookies', () => {
+  test('prefers checkpoint cookies over session-state cookies', () => {
+    const cookies = getBrowserCookies(
+      { cookies: [{ name: 'checkpoint-cookie' }] },
+      { cookies: [{ name: 'session-cookie' }] },
+      'connector.x'
+    );
+    expect(cookies).toEqual([{ name: 'checkpoint-cookie' }]);
+  });
+
+  test('falls back to session-state cookies when checkpoint has none', () => {
+    const cookies = getBrowserCookies(
+      null,
+      { cookies: [{ name: 'session-cookie' }] },
+      'connector.x'
+    );
+    expect(cookies).toEqual([{ name: 'session-cookie' }]);
+  });
+
+  test('throws a descriptive error when no cookies are present anywhere', () => {
+    expect(() => getBrowserCookies(null, null, 'connector.x')).toThrow(
+      /No browser cookies found/
+    );
+    expect(() => getBrowserCookies(null, null, 'connector.x')).toThrow(/connector\.x/);
+  });
+
+  test('throws when checkpoint cookies array is empty and no session', () => {
+    expect(() => getBrowserCookies({ cookies: [] }, undefined, 'connector.x')).toThrow(
+      /No browser cookies found/
+    );
+  });
+
+  test('handles undefined sessionState explicitly', () => {
+    expect(() => getBrowserCookies(null, undefined, 'connector.x')).toThrow(
+      /No browser cookies found/
+    );
+  });
+});
+
+describe('validateCookieNotExpired', () => {
+  test('does nothing when the cookie is missing entirely', () => {
+    expect(() =>
+      validateCookieNotExpired([{ name: 'other' }], 'session', 'connector.x')
+    ).not.toThrow();
+  });
+
+  test('does nothing when the cookie has no expires field', () => {
+    expect(() =>
+      validateCookieNotExpired([{ name: 'session' }], 'session', 'connector.x')
+    ).not.toThrow();
+  });
+
+  test('does nothing when expires is 0 (session cookie)', () => {
+    expect(() =>
+      validateCookieNotExpired([{ name: 'session', expires: 0 }], 'session', 'connector.x')
+    ).not.toThrow();
+  });
+
+  test('does nothing when the cookie expires in the future', () => {
+    const futureUnix = Math.floor(Date.now() / 1000) + 60 * 60;
+    expect(() =>
+      validateCookieNotExpired(
+        [{ name: 'session', expires: futureUnix }],
+        'session',
+        'connector.x'
+      )
+    ).not.toThrow();
+  });
+
+  test('throws when the cookie has expired', () => {
+    const pastUnix = Math.floor(Date.now() / 1000) - 60 * 60 * 24;
+    expect(() =>
+      validateCookieNotExpired(
+        [{ name: 'session', expires: pastUnix }],
+        'session',
+        'connector.x'
+      )
+    ).toThrow(/session expired on/);
+  });
+
+  test('error message includes the connector slug for the re-auth hint', () => {
+    const pastUnix = Math.floor(Date.now() / 1000) - 60 * 60 * 24;
+    expect(() =>
+      validateCookieNotExpired(
+        [{ name: 'session', expires: pastUnix }],
+        'session',
+        'connector.x'
+      )
+    ).toThrow(/--connector connector\.x/);
+  });
+});
+
+describe('filterByCheckpoint', () => {
+  const events = [
+    { occurred_at: new Date('2024-01-01T00:00:00Z') } as any,
+    { occurred_at: new Date('2024-06-01T00:00:00Z') } as any,
+    { occurred_at: new Date('2024-12-31T00:00:00Z') } as any,
+  ];
+
+  test('returns every event when no checkpoint is set', () => {
+    expect(filterByCheckpoint(events, null)).toEqual(events);
+  });
+
+  test('returns every event when checkpoint has no last_timestamp', () => {
+    expect(filterByCheckpoint(events, {})).toEqual(events);
+  });
+
+  test('keeps only events strictly newer than last_timestamp', () => {
+    const filtered = filterByCheckpoint(events, {
+      last_timestamp: '2024-06-01T00:00:00Z',
+    });
+    // strict `>` — event at exactly the cutoff is filtered out
+    expect(filtered).toHaveLength(1);
+    expect(filtered[0]).toBe(events[2]);
+  });
+
+  test('returns empty array when cutoff is past every event', () => {
+    expect(
+      filterByCheckpoint(events, { last_timestamp: '2099-01-01T00:00:00Z' })
+    ).toEqual([]);
+  });
+});

package/dist/connectors/browser-scraper-utils.ts

@@ -0,0 +1,214 @@
+/**
+ * Shared utilities for browser-based scraper connectors.
+ *
+ * Provides common patterns used across Trustpilot, G2, Glassdoor, Capterra,
+ * and similar connectors that launch a stealth browser and scrape review pages.
+ */
+
+import {
+  acquireBrowser,
+  type CdpPage,
+  captureErrorArtifacts,
+  type EventEnvelope,
+} from '@lobu/connector-sdk';
+import type { Browser, Cookie, Page } from 'playwright';
+
+// -----------------------------------------------------------------------------
+// Browser auth cookie helpers
+// -----------------------------------------------------------------------------
+
+export function getBrowserCookies(
+  checkpoint: Record<string, unknown> | null,
+  sessionState: Record<string, unknown> | null | undefined,
+  connectorKey: string
+): any[] {
+  const sessionCookies = (sessionState?.cookies as any[]) ?? [];
+  const cookies = (checkpoint as any)?.cookies ?? sessionCookies;
+  if (!cookies || cookies.length === 0) {
+    throw new Error(
+      `No browser cookies found. Run: lobu memory browser-auth --connector ${connectorKey} --auth-profile-slug <SLUG>`
+    );
+  }
+  return cookies;
+}
+
+export function validateCookieNotExpired(
+  cookies: any[],
+  cookieName: string,
+  connectorKey: string
+): void {
+  const cookie = cookies.find((c: any) => c.name === cookieName);
+  if (cookie?.expires && cookie.expires > 0) {
+    const expiresAt = new Date(cookie.expires * 1000);
+    if (expiresAt < new Date()) {
+      throw new Error(
+        `${cookieName} expired on ${expiresAt.toISOString()}. Re-run: lobu memory browser-auth --connector ${connectorKey} --auth-profile-slug <SLUG>`
+      );
+    }
+  }
+}
+
+// -----------------------------------------------------------------------------
+// URL validation
+// -----------------------------------------------------------------------------
+
+/**
+ * Validate that a URL is well-formed, uses HTTPS, and belongs to the expected
+ * domain (hostname ends with `expectedDomain`).
+ *
+ * @throws If the URL is invalid, not HTTPS, or on the wrong domain.
+ */
+export function validateUrlDomain(url: string, expectedDomain: string): void {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`Invalid ${expectedDomain} URL: ${url}`);
+  }
+  if (parsed.protocol !== 'https:') {
+    throw new Error(`${expectedDomain} URL must use https: protocol, got ${parsed.protocol}`);
+  }
+  if (
+    parsed.hostname !== expectedDomain &&
+    !parsed.hostname.endsWith(`.${expectedDomain}`)
+  ) {
+    throw new Error(`URL must be on ${expectedDomain}, got ${parsed.hostname}`);
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Browser lifecycle
+// -----------------------------------------------------------------------------
+
+export interface BrowserSession {
+  /** Playwright Browser (null when using raw CDP). */
+  browser: Browser | null;
+  /** Page handle — Playwright Page or CdpPage. Both support goto/evaluate/waitForSelector. */
+  page: Page | CdpPage;
+  screenshotDir: string;
+  /** Which backend was used ('cdp' or 'playwright'). */
+  backend: 'cdp' | 'playwright';
+  /** If false, don't close the browser (CDP — it's the user's Chrome). */
+  ownsBrowser: boolean;
+}
+
+/**
+ * Acquire a stealth browser session.
+ *
+ * By default launches a fresh Playwright browser (safe for DOM scraping).
+ * Pass `cdpUrl: 'auto'` to try CDP first — uses raw CDP protocol to avoid
+ * Playwright's connectOverCDP crash on browsers with many tabs.
+ */
+export async function openStealthBrowser(opts?: {
+  cdpUrl?: string | 'auto' | null;
+  cookies?: Cookie[];
+  authDomains?: string[];
+}): Promise<BrowserSession> {
+  const acquired = await acquireBrowser({
+    cdpUrl: opts?.cdpUrl ?? null,
+    cookies: opts?.cookies ?? [],
+    authDomains: opts?.authDomains ?? [],
+    stealth: true,
+  });
+
+  const page = acquired.cdpPage ?? acquired.page;
+  if (!page) throw new Error('No page available from browser acquisition');
+
+  return {
+    browser: acquired.browser,
+    page,
+    screenshotDir: acquired.screenshotDir,
+    backend: acquired.backend,
+    ownsBrowser: acquired.ownsBrowser,
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Cookie consent
+// -----------------------------------------------------------------------------
+
+/**
+ * Attempt to dismiss a cookie consent banner by clicking an accept button.
+ *
+ * @param page    - Playwright page instance
+ * @param selector - CSS selector for the accept/dismiss button
+ * @param timeout  - How long to wait for the button to appear (ms, default 2000)
+ */
+export async function handleCookieConsent(
+  page: Page | CdpPage,
+  selector: string,
+  timeout = 2000
+): Promise<void> {
+  try {
+    const found = await page.waitForSelector(selector, { timeout });
+    if (found) {
+      // CdpPage.waitForSelector returns boolean, Playwright returns ElementHandle
+      if (typeof found === 'boolean') {
+        await (page as CdpPage).click(selector);
+      } else {
+        await found.click();
+      }
+    }
+  } catch {
+    // No cookie banner found or already dismissed — continue
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Checkpoint filtering
+// -----------------------------------------------------------------------------
+
+/**
+ * Filter events that are newer than the checkpoint's `last_timestamp`.
+ * If no checkpoint is set, all events are returned.
+ */
+export function filterByCheckpoint(
+  events: EventEnvelope[],
+  checkpoint: Record<string, unknown> | null
+): EventEnvelope[] {
+  const lastTimestamp = checkpoint?.last_timestamp as string | undefined;
+  if (!lastTimestamp) return events;
+
+  const cutoff = new Date(lastTimestamp);
+  return events.filter((e) => e.occurred_at > cutoff);
+}
+
+// -----------------------------------------------------------------------------
+// Error handling with browser cleanup
+// -----------------------------------------------------------------------------
+
+/**
+ * Run a scraper function inside a try/catch that captures error artifacts
+ * (screenshot + HTML snapshot) and ensures the browser is always closed.
+ *
+ * @param session       - The browser session from `openStealthBrowser()`
+ * @param connectorName - Short name used for artifact filenames (e.g. "trustpilot-sync")
+ * @param fn            - The async scraper logic receiving the page
+ * @returns             - Whatever `fn` returns
+ */
+export async function withBrowserErrorCapture<T>(
+  session: BrowserSession,
+  connectorName: string,
+  fn: (page: Page | CdpPage) => Promise<T>
+): Promise<T> {
+  try {
+    return await fn(session.page);
+  } catch (error: any) {
+    // captureErrorArtifacts only works with Playwright pages
+    if (session.backend === 'playwright' && session.page) {
+      await captureErrorArtifacts(
+        session.page as Page,
+        error,
+        connectorName,
+        session.screenshotDir
+      );
+    }
+    throw error;
+  } finally {
+    if (session.backend === 'cdp') {
+      await (session.page as CdpPage).close();
+    } else if (session.ownsBrowser && session.browser) {
+      await session.browser.close();
+    }
+  }
+}

package/dist/connectors/capterra.ts

@@ -0,0 +1,273 @@
+/**
+ * Capterra Connector
+ * Scrapes software reviews from Capterra using browser rendering with stealth mode.
+ */
+
+import {
+  type ActionContext,
+  type ActionResult,
+  type ConnectorDefinition,
+  ConnectorRuntime,
+  calculateEngagementScore,
+  type EventEnvelope,
+  type SyncContext,
+  type SyncResult,
+} from '@lobu/connector-sdk';
+import {
+  handleCookieConsent,
+  openStealthBrowser,
+  validateUrlDomain,
+  withBrowserErrorCapture,
+} from './browser-scraper-utils.ts';
+
+interface CapterraReview {
+  id: string;
+  rating: number;
+  title: string;
+  text: string;
+  date: string;
+  author: string;
+  helpfulCount: number;
+}
+
+export default class CapterraConnector extends ConnectorRuntime {
+  readonly definition: ConnectorDefinition = {
+    key: 'capterra',
+    name: 'Capterra',
+    version: '1.0.0',
+    faviconDomain: 'capterra.com',
+    description: 'Scrapes software reviews from Capterra.',
+    authSchema: {
+      methods: [{ type: 'none' }],
+    },
+    feeds: {
+      reviews: {
+        key: 'reviews',
+        name: 'Reviews',
+        description: 'Capterra software reviews',
+        configSchema: {
+          type: 'object',
+          required: ['product_id'],
+          properties: {
+            product_id: {
+              type: 'string',
+              description: 'Capterra product ID (e.g., "12345")',
+              minLength: 1,
+            },
+            product_name: {
+              type: 'string',
+              description:
+                'Product name slug for URL (e.g., "spotify"). Optional - Capterra will redirect without it.',
+              minLength: 1,
+            },
+            vendor_name: {
+              type: 'string',
+              description:
+                'Vendor/company name (e.g., "Spotify AB"). Optional but recommended for disambiguation.',
+              minLength: 1,
+            },
+            lookback_days: {
+              type: 'integer',
+              description:
+                'Number of days to look back for historical data. Default: 365 (1 year). Maximum: 730 (2 years).',
+              minimum: 1,
+              maximum: 730,
+              default: 365,
+            },
+          },
+        },
+        eventKinds: {
+          review: {
+            description: 'A Capterra software review',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                rating: { type: 'number', description: 'Star rating (0-5)' },
+                helpful_count: { type: 'number', description: 'Number of helpful votes' },
+              },
+            },
+          },
+        },
+      },
+    },
+    optionsSchema: {
+      type: 'object',
+      required: ['product_id'],
+      properties: {
+        product_id: {
+          type: 'string',
+          description: 'Capterra product ID (e.g., "12345")',
+          minLength: 1,
+        },
+        product_name: {
+          type: 'string',
+          description:
+            'Product name slug for URL (e.g., "spotify"). Optional - Capterra will redirect without it.',
+          minLength: 1,
+        },
+        vendor_name: {
+          type: 'string',
+          description:
+            'Vendor/company name (e.g., "Spotify AB"). Optional but recommended for disambiguation.',
+          minLength: 1,
+        },
+        lookback_days: {
+          type: 'integer',
+          description:
+            'Number of days to look back for historical data. Default: 365 (1 year). Maximum: 730 (2 years).',
+          minimum: 1,
+          maximum: 730,
+          default: 365,
+        },
+      },
+    },
+  };
+
+  async sync(ctx: SyncContext): Promise<SyncResult> {
+    const productId = ctx.config.product_id as string;
+    const productName = ctx.config.product_name as string | undefined;
+
+    const baseUrl = productName
+      ? `https://www.capterra.com/p/${productId}/${productName}/reviews`
+      : `https://www.capterra.com/p/${productId}/reviews`;
+    validateUrlDomain(baseUrl, 'capterra.com');
+
+    const session = await openStealthBrowser({ cdpUrl: 'auto' });
+
+    return withBrowserErrorCapture(session, 'capterra-sync', async (page) => {
+      await page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
+
+      await handleCookieConsent(page, '[data-test="cookie-accept"], #onetrust-accept-btn-handler');
+
+      // Wait for review cards
+      try {
+        await page.waitForSelector('[data-test="review-card"], .review-card', { timeout: 10000 });
+      } catch {
+        // Review selectors not found - page structure may have changed
+      }
+
+      // Extract reviews using page.evaluate
+      const rawReviews: CapterraReview[] = await page.evaluate(() => {
+        const reviewElements = Array.from(
+          document.querySelectorAll('[data-test="review-card"], .review-card')
+        );
+
+        return reviewElements.map((el: Element, index: number) => {
+          // Extract rating (usually shown as stars)
+          const ratingElement = el.querySelector(
+            '[data-test="rating"], .rating, [aria-label*="star"]'
+          );
+          let rating = 0;
+          if (ratingElement) {
+            const ariaLabel = ratingElement.getAttribute('aria-label');
+            const ratingMatch = ariaLabel?.match(/(\d+(?:\.\d+)?)/);
+            rating = ratingMatch ? parseFloat(ratingMatch[1]) : 0;
+          }
+
+          // Extract review title
+          const titleElement = el.querySelector(
+            '[data-test="review-title"], .review-title, h3, h4'
+          );
+          const title = titleElement?.textContent?.trim() || '';
+
+          // Extract review text
+          const textElement = el.querySelector(
+            '[data-test="review-body"], .review-body, .review-content, .review-text'
+          );
+          const text = textElement?.textContent?.trim() || '';
+
+          // Extract date
+          const dateElement = el.querySelector('[data-test="review-date"], .review-date, time');
+          const dateText =
+            dateElement?.textContent?.trim() || dateElement?.getAttribute('datetime') || '';
+
+          // Parse relative dates like "2 weeks ago"
+          let date = new Date();
+          if (dateText) {
+            const weeksMatch = dateText.match(/(\d+)\s+weeks?\s+ago/i);
+            const monthsMatch = dateText.match(/(\d+)\s+months?\s+ago/i);
+            const daysMatch = dateText.match(/(\d+)\s+days?\s+ago/i);
+
+            if (weeksMatch) {
+              date = new Date(Date.now() - parseInt(weeksMatch[1], 10) * 7 * 24 * 60 * 60 * 1000);
+            } else if (monthsMatch) {
+              date = new Date(Date.now() - parseInt(monthsMatch[1], 10) * 30 * 24 * 60 * 60 * 1000);
+            } else if (daysMatch) {
+              date = new Date(Date.now() - parseInt(daysMatch[1], 10) * 24 * 60 * 60 * 1000);
+            } else {
+              // Try parsing as date
+              const parsed = new Date(dateText);
+              if (!Number.isNaN(parsed.getTime())) {
+                date = parsed;
+              }
+            }
+          }
+
+          // Extract author
+          const authorElement = el.querySelector(
+            '[data-test="reviewer-name"], .reviewer-name, .author'
+          );
+          const author = authorElement?.textContent?.trim() || 'Anonymous';
+
+          // Extract review ID from data attributes or generate
+          const reviewId =
+            (el as HTMLElement).getAttribute('data-review-id') ||
+            (el as HTMLElement).id ||
+            `${date.toISOString()}_${index}`.replace(/[^a-zA-Z0-9]/g, '_');
+
+          // Extract helpful count
+          const helpfulElement = el.querySelector('[data-test="helpful-count"], .helpful-count');
+          const helpfulCount = helpfulElement
+            ? parseInt(helpfulElement.textContent?.replace(/\D/g, '') || '0', 10)
+            : 0;
+
+          return {
+            id: reviewId,
+            rating,
+            title,
+            text,
+            date: date.toISOString(),
+            author,
+            helpfulCount,
+          };
+        });
+      });
+
+      // Filter reviews with content
+      const reviews = rawReviews.filter((r) => r.text.length > 0);
+
+      // Transform to EventEnvelope
+      const events: EventEnvelope[] = reviews.map((review) => {
+        const engagementData = {
+          rating: review.rating,
+          helpful_count: review.helpfulCount,
+        };
+
+        return {
+          origin_id: review.id,
+          title: review.title,
+          payload_text: review.text,
+          author_name: review.author,
+          occurred_at: new Date(review.date),
+          origin_type: 'review',
+          score: calculateEngagementScore('capterra', engagementData),
+          source_url: baseUrl,
+          metadata: engagementData,
+        };
+      });
+
+      return {
+        events,
+        checkpoint: { last_sync_at: new Date().toISOString() },
+        metadata: {
+          items_found: rawReviews.length,
+          items_skipped: rawReviews.length - reviews.length,
+        },
+      };
+    });
+  }
+
+  async execute(_ctx: ActionContext): Promise<ActionResult> {
+    return { success: false, error: 'Actions not supported' };
+  }
+}