@lobu/cli 6.0.1 → 7.0.0

package/dist/connectors/__tests__/browser-scraper-utils.test.ts

@@ -0,0 +1,186 @@
+import { describe, expect, mock, test } from 'bun:test';
+
+// browser-scraper-utils.ts pulls runtime symbols (acquireBrowser,
+// captureErrorArtifacts) from @lobu/connector-sdk, which itself pulls in
+// playwright. Stub the SDK so the pure helpers (validateUrlDomain,
+// getBrowserCookies, validateCookieNotExpired, filterByCheckpoint) can be
+// imported without spinning up a real browser stack.
+mock.module('@lobu/connector-sdk', () => ({
+  acquireBrowser: () => {
+    throw new Error('not used in unit tests');
+  },
+  captureErrorArtifacts: () => {
+    throw new Error('not used in unit tests');
+  },
+}));
+
+const {
+  filterByCheckpoint,
+  getBrowserCookies,
+  validateCookieNotExpired,
+  validateUrlDomain,
+} = await import('../browser-scraper-utils.ts');
+
+describe('validateUrlDomain', () => {
+  test('accepts a well-formed https URL on the expected domain', () => {
+    expect(() =>
+      validateUrlDomain('https://www.trustpilot.com/review/foo', 'trustpilot.com')
+    ).not.toThrow();
+  });
+
+  test('accepts a subdomain of the expected domain', () => {
+    expect(() => validateUrlDomain('https://api.example.com/x', 'example.com')).not.toThrow();
+  });
+
+  test('rejects a malformed URL', () => {
+    expect(() => validateUrlDomain('not a url', 'example.com')).toThrow(/Invalid example.com URL/);
+  });
+
+  test('rejects http (non-https) URLs', () => {
+    expect(() => validateUrlDomain('http://www.example.com/', 'example.com')).toThrow(
+      /must use https: protocol/
+    );
+  });
+
+  test('rejects a URL on a different domain', () => {
+    expect(() => validateUrlDomain('https://evil.com/foo', 'example.com')).toThrow(
+      /must be on example.com/
+    );
+  });
+
+  test('rejects a substring-match hostname (security: notexample.com is NOT example.com)', () => {
+    expect(() => validateUrlDomain('https://notexample.com/x', 'example.com')).toThrow(
+      /must be on example.com/
+    );
+    expect(() => validateUrlDomain('https://eviltrustpilot.com/x', 'trustpilot.com')).toThrow(
+      /must be on trustpilot.com/
+    );
+  });
+
+  test('accepts the apex domain itself (no subdomain)', () => {
+    expect(() => validateUrlDomain('https://example.com/x', 'example.com')).not.toThrow();
+  });
+});
+
+describe('getBrowserCookies', () => {
+  test('prefers checkpoint cookies over session-state cookies', () => {
+    const cookies = getBrowserCookies(
+      { cookies: [{ name: 'checkpoint-cookie' }] },
+      { cookies: [{ name: 'session-cookie' }] },
+      'connector.x'
+    );
+    expect(cookies).toEqual([{ name: 'checkpoint-cookie' }]);
+  });
+
+  test('falls back to session-state cookies when checkpoint has none', () => {
+    const cookies = getBrowserCookies(
+      null,
+      { cookies: [{ name: 'session-cookie' }] },
+      'connector.x'
+    );
+    expect(cookies).toEqual([{ name: 'session-cookie' }]);
+  });
+
+  test('throws a descriptive error when no cookies are present anywhere', () => {
+    expect(() => getBrowserCookies(null, null, 'connector.x')).toThrow(
+      /No browser cookies found/
+    );
+    expect(() => getBrowserCookies(null, null, 'connector.x')).toThrow(/connector\.x/);
+  });
+
+  test('throws when checkpoint cookies array is empty and no session', () => {
+    expect(() => getBrowserCookies({ cookies: [] }, undefined, 'connector.x')).toThrow(
+      /No browser cookies found/
+    );
+  });
+
+  test('handles undefined sessionState explicitly', () => {
+    expect(() => getBrowserCookies(null, undefined, 'connector.x')).toThrow(
+      /No browser cookies found/
+    );
+  });
+});
+
+describe('validateCookieNotExpired', () => {
+  test('does nothing when the cookie is missing entirely', () => {
+    expect(() =>
+      validateCookieNotExpired([{ name: 'other' }], 'session', 'connector.x')
+    ).not.toThrow();
+  });
+
+  test('does nothing when the cookie has no expires field', () => {
+    expect(() =>
+      validateCookieNotExpired([{ name: 'session' }], 'session', 'connector.x')
+    ).not.toThrow();
+  });
+
+  test('does nothing when expires is 0 (session cookie)', () => {
+    expect(() =>
+      validateCookieNotExpired([{ name: 'session', expires: 0 }], 'session', 'connector.x')
+    ).not.toThrow();
+  });
+
+  test('does nothing when the cookie expires in the future', () => {
+    const futureUnix = Math.floor(Date.now() / 1000) + 60 * 60;
+    expect(() =>
+      validateCookieNotExpired(
+        [{ name: 'session', expires: futureUnix }],
+        'session',
+        'connector.x'
+      )
+    ).not.toThrow();
+  });
+
+  test('throws when the cookie has expired', () => {
+    const pastUnix = Math.floor(Date.now() / 1000) - 60 * 60 * 24;
+    expect(() =>
+      validateCookieNotExpired(
+        [{ name: 'session', expires: pastUnix }],
+        'session',
+        'connector.x'
+      )
+    ).toThrow(/session expired on/);
+  });
+
+  test('error message includes the connector slug for the re-auth hint', () => {
+    const pastUnix = Math.floor(Date.now() / 1000) - 60 * 60 * 24;
+    expect(() =>
+      validateCookieNotExpired(
+        [{ name: 'session', expires: pastUnix }],
+        'session',
+        'connector.x'
+      )
+    ).toThrow(/--connector connector\.x/);
+  });
+});
+
+describe('filterByCheckpoint', () => {
+  const events = [
+    { occurred_at: new Date('2024-01-01T00:00:00Z') } as any,
+    { occurred_at: new Date('2024-06-01T00:00:00Z') } as any,
+    { occurred_at: new Date('2024-12-31T00:00:00Z') } as any,
+  ];
+
+  test('returns every event when no checkpoint is set', () => {
+    expect(filterByCheckpoint(events, null)).toEqual(events);
+  });
+
+  test('returns every event when checkpoint has no last_timestamp', () => {
+    expect(filterByCheckpoint(events, {})).toEqual(events);
+  });
+
+  test('keeps only events strictly newer than last_timestamp', () => {
+    const filtered = filterByCheckpoint(events, {
+      last_timestamp: '2024-06-01T00:00:00Z',
+    });
+    // strict `>` — event at exactly the cutoff is filtered out
+    expect(filtered).toHaveLength(1);
+    expect(filtered[0]).toBe(events[2]);
+  });
+
+  test('returns empty array when cutoff is past every event', () => {
+    expect(
+      filterByCheckpoint(events, { last_timestamp: '2099-01-01T00:00:00Z' })
+    ).toEqual([]);
+  });
+});

package/dist/connectors/apple_health.ts

@@ -0,0 +1,138 @@
+/**
+ * Apple Health Connector (V1 runtime) — Lobu device app.
+ *
+ * Runs on a device bridge with HealthKit access. On iOS this reads the phone's
+ * Health store directly; on supported macOS installs HealthKit exposes the
+ * per-user store synced from iPhone and Apple Watch via iCloud Health. The
+ * Lobu device bridge holds the HealthKit entitlement, requests read permission
+ * once via a system sheet, and queries:
+ *
+ * - `daily_summaries`: daily totals for steps, distance, active energy,
+ *   exercise minutes, and resting heart rate.
+ * - `workouts`: individual workout sessions (type, duration, energy, distance).
+ *
+ * The connector DEFINITION (feeds, event kinds, options) is the source of truth
+ * for what data ends up in Lobu and how it's shaped. The EXECUTION lives in the
+ * Lobu device app, which polls /api/workers/* as a user-scoped worker
+ * advertising the `healthkit` capability and streams events back through the
+ * standard worker protocol — same `runs` lifecycle as every other connector.
+ *
+ * The TS sync()/execute() here are safety stubs: if a server-side worker
+ * somehow bypassed the capability gate (`required_capability='healthkit'`),
+ * the run would throw immediately instead of silently producing no events.
+ */
+
+import {
+  type ActionResult,
+  type ConnectorDefinition,
+  ConnectorRuntime,
+  type SyncContext,
+  type SyncResult,
+} from '@lobu/connector-sdk';
+
+const BRIDGE_ONLY_MESSAGE =
+  'apple.health runs only on a worker advertising capability "healthkit" (Lobu with Apple Health permission). ' +
+  'This run was claimed by a worker without that capability — check connector_definitions.required_capability and the poll-time capability filter.';
+
+export default class AppleHealthConnector extends ConnectorRuntime {
+  readonly definition: ConnectorDefinition = {
+    key: 'apple.health',
+    name: 'Apple Health',
+    description:
+      'Sync Apple Health daily activity summaries and workouts from Lobu on your device. ' +
+      'macOS reads HealthKit data synced from the user\'s iPhone (and Apple Watch) via iCloud Health.',
+    version: '0.1.0',
+    faviconDomain: 'apple.com',
+    requiredCapability: 'healthkit',
+    runtime: {
+      platforms: ['ios', 'macos'],
+      scopes: ['steps', 'distance', 'active-calories', 'exercise-minutes', 'workouts', 'resting-heart-rate'],
+    },
+    authSchema: {
+      methods: [{ type: 'none' }],
+    },
+    feeds: {
+      daily_summaries: {
+        key: 'daily_summaries',
+        name: 'Daily summaries',
+        description:
+          'Daily Apple Health activity summaries: steps, distance, active energy, ' +
+          'exercise minutes, and resting heart rate.',
+        configSchema: {
+          type: 'object',
+          properties: {
+            backfill_days: {
+              type: 'integer',
+              minimum: 1,
+              maximum: 3650,
+              default: 30,
+              description: 'How many days the bridge backfills on a fresh sync (incremental syncs only re-query changed days).',
+            },
+          },
+        },
+        eventKinds: {
+          health_daily_summary: {
+            description: 'A daily summary of Apple Health activity data.',
+            metadataSchema: {
+              type: 'object',
+              required: ['source', 'origin_id', 'date'],
+              properties: {
+                source: { type: 'string', const: 'apple_health' },
+                origin_id: { type: 'string' },
+                date: { type: 'string', format: 'date' },
+                steps: { type: 'number' },
+                distance_m: { type: 'number' },
+                active_energy_kcal: { type: 'number' },
+                exercise_minutes: { type: 'number' },
+                resting_heart_rate_bpm: { type: ['number', 'null'] },
+              },
+            },
+          },
+        },
+      },
+      workouts: {
+        key: 'workouts',
+        name: 'Workouts',
+        description: 'Workout sessions recorded in Apple Health.',
+        configSchema: {
+          type: 'object',
+          properties: {
+            backfill_days: {
+              type: 'integer',
+              minimum: 1,
+              maximum: 3650,
+              default: 30,
+              description: 'How many days the bridge backfills on a fresh sync.',
+            },
+          },
+        },
+        eventKinds: {
+          health_workout: {
+            description: 'A workout recorded in Apple Health.',
+            metadataSchema: {
+              type: 'object',
+              required: ['source', 'origin_id', 'workout_type'],
+              properties: {
+                source: { type: 'string', const: 'apple_health' },
+                origin_id: { type: 'string' },
+                workout_type: { type: 'string' },
+                started_at: { type: 'string' },
+                duration_s: { type: 'number' },
+                active_energy_kcal: { type: ['number', 'null'] },
+                distance_m: { type: ['number', 'null'] },
+              },
+            },
+          },
+        },
+      },
+    },
+  };
+
+  async sync(_ctx: SyncContext): Promise<SyncResult> {
+    throw new Error(BRIDGE_ONLY_MESSAGE);
+  }
+
+  async execute(): Promise<ActionResult> {
+    throw new Error(BRIDGE_ONLY_MESSAGE);
+  }
+}

package/dist/connectors/apple_screen_time.ts

@@ -0,0 +1,82 @@
+/**
+ * Apple Screen Time Connector (V1 runtime) — Lobu for Mac only.
+ *
+ * Runs on Lobu for Mac, which reads `~/Library/Application Support/
+ * Knowledge/knowledgeC.db` (the on-device Knowledge store backing Apple's
+ * Settings → Screen Time UI). With Full Disk Access granted, the Mac can
+ * pull per-app foreground time by day for both Mac usage and (if the user
+ * enables Screen Time iCloud sync) iOS usage.
+ *
+ * iOS does NOT advertise the `screentime` capability — Apple's
+ * FamilyControls + DeviceActivityReport design prevents per-app data from
+ * leaving the device on iOS. The Mac path is the workable one.
+ */
+
+import {
+  type ActionResult,
+  type ConnectorDefinition,
+  ConnectorRuntime,
+  type SyncContext,
+  type SyncResult,
+} from '@lobu/connector-sdk';
+
+const BRIDGE_ONLY =
+  'Apple Screen Time runs only on a worker advertising capability "screentime" (Lobu for Mac with Full Disk Access).';
+
+export default class AppleScreenTimeConnector extends ConnectorRuntime {
+  readonly definition: ConnectorDefinition = {
+    key: 'apple.screen_time',
+    name: 'Apple Screen Time',
+    description:
+      'Daily per-app usage totals from Lobu for Mac, sourced from the Apple Knowledge store. Captures both Mac usage and (if Screen Time iCloud sync is on) the user\'s iOS device usage.',
+    version: '0.1.0',
+    faviconDomain: 'apple.com',
+    requiredCapability: 'screentime',
+    runtime: { platforms: ['macos'] },
+    authSchema: { methods: [{ type: 'none' }] },
+    feeds: {
+      daily_app_usage: {
+        key: 'daily_app_usage',
+        name: 'Daily app usage',
+        description:
+          'Per-day total foreground time for each application (identified by bundle id).',
+        configSchema: {
+          type: 'object',
+          properties: {
+            backfill_days: {
+              type: 'integer',
+              minimum: 1,
+              maximum: 90,
+              default: 14,
+              description: 'How many days the bridge should backfill on each sync.',
+            },
+          },
+        },
+        eventKinds: {
+          screen_time_daily_app: {
+            description: 'Total time the user spent in one application on a given day.',
+            metadataSchema: {
+              type: 'object',
+              required: ['source', 'origin_id', 'date', 'bundle_id', 'seconds'],
+              properties: {
+                source: { type: 'string', const: 'apple_screen_time' },
+                origin_id: { type: 'string' },
+                date: { type: 'string', format: 'date' },
+                bundle_id: { type: 'string' },
+                seconds: { type: 'number', minimum: 0 },
+              },
+            },
+          },
+        },
+      },
+    },
+  };
+
+  async sync(_ctx: SyncContext): Promise<SyncResult> {
+    throw new Error(BRIDGE_ONLY);
+  }
+
+  async execute(): Promise<ActionResult> {
+    throw new Error(BRIDGE_ONLY);
+  }
+}

package/dist/connectors/browser-scraper-utils.ts

@@ -0,0 +1,246 @@
+/**
+ * Shared utilities for browser-based scraper connectors.
+ *
+ * Provides common patterns used across Trustpilot, G2, Glassdoor, Capterra,
+ * and similar connectors that launch a stealth browser and scrape review pages.
+ */
+
+import {
+  acquireBrowser,
+  type CdpPage,
+  captureErrorArtifacts,
+  type EventEnvelope,
+} from '@lobu/connector-sdk';
+import type { Browser, Cookie, Page } from 'playwright';
+
+// -----------------------------------------------------------------------------
+// Browser auth cookie helpers
+// -----------------------------------------------------------------------------
+
+export function getBrowserCookies(
+  checkpoint: Record<string, unknown> | null,
+  sessionState: Record<string, unknown> | null | undefined,
+  connectorKey: string
+): any[] {
+  const sessionCookies = (sessionState?.cookies as any[]) ?? [];
+  const cookies = (checkpoint as any)?.cookies ?? sessionCookies;
+  // Device-bound browser profiles ship cookies via --user-data-dir on disk
+  // rather than this jsonb blob; the persistent context loads them itself.
+  if ((!cookies || cookies.length === 0) && !sessionState?.user_data_dir) {
+    throw new Error(
+      `No browser cookies found. Run: lobu memory browser-auth --connector ${connectorKey} --auth-profile-slug <SLUG>`
+    );
+  }
+  return cookies ?? [];
+}
+
+/**
+ * Pull the device-bound managed --user-data-dir from session_state, if the
+ * connection's auth profile is owned by a device worker. When set, callers
+ * should pass it to openStealthBrowser instead of relying on the cookies/CDP
+ * cascade — Chrome reads cookies from that profile dir directly.
+ */
+export function getBrowserUserDataDir(
+  sessionState: Record<string, unknown> | null | undefined
+): string | undefined {
+  const value = sessionState?.user_data_dir;
+  return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+/**
+ * Pull the device-bound CDP endpoint URL from session_state (set when the
+ * user picked "Attach via CDP" mode on their browser profile). When set,
+ * callers should pass it through as `cdpUrl` so the connector attaches to
+ * the exact running Chrome the user chose — instead of `'auto'`, which can
+ * land on the wrong browser when several debuggable Chromiums are running
+ * or a non-default port was configured.
+ */
+export function getBrowserCdpUrl(
+  sessionState: Record<string, unknown> | null | undefined
+): string | undefined {
+  const value = sessionState?.cdp_url;
+  return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+export function validateCookieNotExpired(
+  cookies: any[],
+  cookieName: string,
+  connectorKey: string
+): void {
+  const cookie = cookies.find((c: any) => c.name === cookieName);
+  if (cookie?.expires && cookie.expires > 0) {
+    const expiresAt = new Date(cookie.expires * 1000);
+    if (expiresAt < new Date()) {
+      throw new Error(
+        `${cookieName} expired on ${expiresAt.toISOString()}. Re-run: lobu memory browser-auth --connector ${connectorKey} --auth-profile-slug <SLUG>`
+      );
+    }
+  }
+}
+
+// -----------------------------------------------------------------------------
+// URL validation
+// -----------------------------------------------------------------------------
+
+/**
+ * Validate that a URL is well-formed, uses HTTPS, and belongs to the expected
+ * domain (hostname ends with `expectedDomain`).
+ *
+ * @throws If the URL is invalid, not HTTPS, or on the wrong domain.
+ */
+export function validateUrlDomain(url: string, expectedDomain: string): void {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`Invalid ${expectedDomain} URL: ${url}`);
+  }
+  if (parsed.protocol !== 'https:') {
+    throw new Error(`${expectedDomain} URL must use https: protocol, got ${parsed.protocol}`);
+  }
+  if (
+    parsed.hostname !== expectedDomain &&
+    !parsed.hostname.endsWith(`.${expectedDomain}`)
+  ) {
+    throw new Error(`URL must be on ${expectedDomain}, got ${parsed.hostname}`);
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Browser lifecycle
+// -----------------------------------------------------------------------------
+
+export interface BrowserSession {
+  /** Playwright Browser (null when using raw CDP). */
+  browser: Browser | null;
+  /** Page handle — Playwright Page or CdpPage. Both support goto/evaluate/waitForSelector. */
+  page: Page | CdpPage;
+  screenshotDir: string;
+  /** Which backend was used ('cdp' or 'playwright'). */
+  backend: 'cdp' | 'playwright';
+  /** If false, don't close the browser (CDP — it's the user's Chrome). */
+  ownsBrowser: boolean;
+}
+
+/**
+ * Acquire a stealth browser session.
+ *
+ * By default launches a fresh Playwright browser (safe for DOM scraping).
+ * Pass `cdpUrl: 'auto'` to try CDP first — uses raw CDP protocol to avoid
+ * Playwright's connectOverCDP crash on browsers with many tabs.
+ */
+export async function openStealthBrowser(opts?: {
+  cdpUrl?: string | 'auto' | null;
+  cookies?: Cookie[];
+  authDomains?: string[];
+  userDataDir?: string;
+}): Promise<BrowserSession> {
+  const acquired = await acquireBrowser({
+    cdpUrl: opts?.userDataDir ? null : (opts?.cdpUrl ?? null),
+    cookies: opts?.cookies ?? [],
+    authDomains: opts?.authDomains ?? [],
+    stealth: true,
+    userDataDir: opts?.userDataDir,
+  });
+
+  const page = acquired.cdpPage ?? acquired.page;
+  if (!page) throw new Error('No page available from browser acquisition');
+
+  return {
+    browser: acquired.browser,
+    page,
+    screenshotDir: acquired.screenshotDir,
+    backend: acquired.backend,
+    ownsBrowser: acquired.ownsBrowser,
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Cookie consent
+// -----------------------------------------------------------------------------
+
+/**
+ * Attempt to dismiss a cookie consent banner by clicking an accept button.
+ *
+ * @param page    - Playwright page instance
+ * @param selector - CSS selector for the accept/dismiss button
+ * @param timeout  - How long to wait for the button to appear (ms, default 2000)
+ */
+export async function handleCookieConsent(
+  page: Page | CdpPage,
+  selector: string,
+  timeout = 2000
+): Promise<void> {
+  try {
+    const found = await page.waitForSelector(selector, { timeout });
+    if (found) {
+      // CdpPage.waitForSelector returns boolean, Playwright returns ElementHandle
+      if (typeof found === 'boolean') {
+        await (page as CdpPage).click(selector);
+      } else {
+        await found.click();
+      }
+    }
+  } catch {
+    // No cookie banner found or already dismissed — continue
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Checkpoint filtering
+// -----------------------------------------------------------------------------
+
+/**
+ * Filter events that are newer than the checkpoint's `last_timestamp`.
+ * If no checkpoint is set, all events are returned.
+ */
+export function filterByCheckpoint(
+  events: EventEnvelope[],
+  checkpoint: Record<string, unknown> | null
+): EventEnvelope[] {
+  const lastTimestamp = checkpoint?.last_timestamp as string | undefined;
+  if (!lastTimestamp) return events;
+
+  const cutoff = new Date(lastTimestamp);
+  return events.filter((e) => e.occurred_at > cutoff);
+}
+
+// -----------------------------------------------------------------------------
+// Error handling with browser cleanup
+// -----------------------------------------------------------------------------
+
+/**
+ * Run a scraper function inside a try/catch that captures error artifacts
+ * (screenshot + HTML snapshot) and ensures the browser is always closed.
+ *
+ * @param session       - The browser session from `openStealthBrowser()`
+ * @param connectorName - Short name used for artifact filenames (e.g. "trustpilot-sync")
+ * @param fn            - The async scraper logic receiving the page
+ * @returns             - Whatever `fn` returns
+ */
+export async function withBrowserErrorCapture<T>(
+  session: BrowserSession,
+  connectorName: string,
+  fn: (page: Page | CdpPage) => Promise<T>
+): Promise<T> {
+  try {
+    return await fn(session.page);
+  } catch (error: any) {
+    // captureErrorArtifacts only works with Playwright pages
+    if (session.backend === 'playwright' && session.page) {
+      await captureErrorArtifacts(
+        session.page as Page,
+        error,
+        connectorName,
+        session.screenshotDir
+      );
+    }
+    throw error;
+  } finally {
+    if (session.backend === 'cdp') {
+      await (session.page as CdpPage).close();
+    } else if (session.ownsBrowser && session.browser) {
+      await session.browser.close();
+    }
+  }
+}