@lobu/cli 6.0.0 → 6.1.1

package/dist/connectors/glassdoor.ts

@@ -0,0 +1,291 @@
+/**
+ * Glassdoor Connector (V1 runtime)
+ *
+ * Scrapes employee reviews from Glassdoor using Playwright.
+ */
+
+import { createHash } from 'node:crypto';
+import {
+  type ActionContext,
+  type ActionResult,
+  type ConnectorDefinition,
+  ConnectorRuntime,
+  calculateEngagementScore,
+  type EventEnvelope,
+  type SyncContext,
+  type SyncResult,
+} from '@lobu/connector-sdk';
+import {
+  handleCookieConsent,
+  openStealthBrowser,
+  validateUrlDomain,
+  withBrowserErrorCapture,
+} from './browser-scraper-utils.ts';
+
+/**
+ * Generates a deterministic external ID for a Glassdoor review.
+ * Uses the native review ID from the DOM when available, otherwise
+ * derives a stable hash from review content to avoid duplicates.
+ */
+function deriveReviewExternalId(companyName: string, review: GlassdoorReview): string {
+  if (review.id) return review.id;
+
+  const contentKey = [
+    review.date,
+    review.author,
+    (review.title || review.pros || review.cons).slice(0, 80),
+  ]
+    .filter(Boolean)
+    .join('|');
+
+  const hash = createHash('sha256').update(contentKey).digest('hex').slice(0, 12);
+  const slug = companyName.toLowerCase().replace(/[^a-z0-9]+/g, '-');
+  return `glassdoor-${slug}-${hash}`;
+}
+
+/**
+ * Raw review data extracted from a Glassdoor page
+ */
+interface GlassdoorReview {
+  id: string;
+  rating: number;
+  title: string;
+  pros: string;
+  cons: string;
+  date: string;
+  author: string;
+}
+
+interface GlassdoorConfig {
+  company_name: string;
+  company_id?: string;
+  lookback_days?: number;
+}
+
+export default class GlassdoorConnector extends ConnectorRuntime {
+  readonly definition: ConnectorDefinition = {
+    key: 'glassdoor',
+    name: 'Glassdoor',
+    description: 'Scrapes employee reviews from Glassdoor.',
+    version: '1.0.0',
+    faviconDomain: 'glassdoor.com',
+    authSchema: {
+      methods: [{ type: 'none' }],
+    },
+    feeds: {
+      reviews: {
+        key: 'reviews',
+        name: 'Employee Reviews',
+        description: 'Scrapes employee reviews for a given company.',
+        configSchema: {
+          type: 'object',
+          required: ['company_name'],
+          properties: {
+            company_name: {
+              type: 'string',
+              minLength: 1,
+              description: 'Company name for search-based lookup',
+            },
+            company_id: {
+              type: 'string',
+              description: 'Glassdoor company ID if known',
+            },
+            lookback_days: {
+              type: 'integer',
+              minimum: 1,
+              maximum: 730,
+              default: 365,
+              description:
+                'Number of days to look back for historical data. Default: 365 (1 year). Maximum: 730 (2 years).',
+            },
+          },
+        },
+        eventKinds: {
+          review: {
+            description: 'A Glassdoor employee review',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                rating: { type: 'number', description: 'Overall rating (0-5)' },
+                title: { type: 'string', description: 'Review headline' },
+                pros: { type: 'string' },
+                cons: { type: 'string' },
+              },
+            },
+          },
+        },
+      },
+    },
+    optionsSchema: {
+      type: 'object',
+      required: ['company_name'],
+      properties: {
+        company_name: {
+          type: 'string',
+          minLength: 1,
+          description: 'Company name for search-based lookup',
+        },
+        company_id: {
+          type: 'string',
+          description: 'Glassdoor company ID if known',
+        },
+        lookback_days: {
+          type: 'integer',
+          minimum: 1,
+          maximum: 730,
+          default: 365,
+          description:
+            'Number of days to look back for historical data. Default: 365 (1 year). Maximum: 730 (2 years).',
+        },
+      },
+    },
+  };
+
+  async sync(ctx: SyncContext): Promise<SyncResult> {
+    const config = ctx.config as GlassdoorConfig;
+    const { company_name, company_id } = config;
+
+    if (!company_name) {
+      return {
+        events: [],
+        checkpoint: ctx.checkpoint,
+        metadata: { items_found: 0, error: 'company_name is required' },
+      };
+    }
+
+    const baseUrl = company_id
+      ? `https://www.glassdoor.com/Reviews/company-reviews-${company_id}.htm`
+      : `https://www.glassdoor.com/Reviews/${company_name}-reviews-SRCH_KE0.htm`;
+    validateUrlDomain(baseUrl, 'glassdoor.com');
+
+    const session = await openStealthBrowser({ cdpUrl: 'auto' });
+
+    return withBrowserErrorCapture(session, 'glassdoor-sync', async (page) => {
+      // Configure viewport and user-agent to mimic a real browser
+      await page.setViewportSize({ width: 1920, height: 1080 });
+      await page.setExtraHTTPHeaders({
+        'User-Agent':
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      });
+
+      // Navigate to the reviews page
+      await page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
+
+      await handleCookieConsent(page, '#onetrust-accept-btn-handler');
+
+      // Human-like delay before interacting with the page
+      await page.waitForTimeout(2000);
+
+      // Wait for review elements to render
+      try {
+        await page.waitForSelector(
+          '[data-test="review-list-item"], .empReview, [data-test="employerReview"]',
+          { timeout: 10000 }
+        );
+      } catch {
+        // Reviews may not be present (auth wall, empty page, etc.)
+      }
+
+      // Extract raw reviews from the page DOM
+      const rawReviews = await page.evaluate((): GlassdoorReview[] => {
+        // Try multiple selector strategies as Glassdoor frequently changes their HTML
+        const reviewElements =
+          Array.from(document.querySelectorAll('[data-test="review-list-item"]')).length > 0
+            ? Array.from(document.querySelectorAll('[data-test="review-list-item"]'))
+            : Array.from(document.querySelectorAll('.empReview')).length > 0
+              ? Array.from(document.querySelectorAll('.empReview'))
+              : Array.from(document.querySelectorAll('[data-test="employerReview"]'));
+
+        return reviewElements.map((el: Element) => {
+          // Try multiple selector patterns for each field
+          const ratingEl =
+            el.querySelector('[data-test="overall-rating"]') ||
+            el.querySelector('.rating') ||
+            el.querySelector('[class*="rating"]');
+
+          const titleEl =
+            el.querySelector('[data-test="review-title"]') ||
+            el.querySelector('.reviewLink') ||
+            el.querySelector('[class*="title"]');
+
+          const prosEl =
+            el.querySelector('[data-test="pros"]') ||
+            el.querySelector('[data-pros]') ||
+            el.querySelector('.pros');
+
+          const consEl =
+            el.querySelector('[data-test="cons"]') ||
+            el.querySelector('[data-cons]') ||
+            el.querySelector('.cons');
+
+          const dateEl =
+            el.querySelector('[data-test="review-date"]') ||
+            el.querySelector('.date') ||
+            el.querySelector('time');
+
+          const authorEl =
+            el.querySelector('[data-test="employee-info"]') ||
+            el.querySelector('.authorInfo') ||
+            el.querySelector('[class*="author"]');
+
+          // Try to get review ID from various attributes
+          const reviewId =
+            (el as HTMLElement).getAttribute('data-review-id') ||
+            (el as HTMLElement).getAttribute('id') ||
+            (el as HTMLElement).getAttribute('data-id') ||
+            '';
+
+          return {
+            id: reviewId,
+            rating: parseFloat(ratingEl?.textContent?.trim() || '0'),
+            title: titleEl?.textContent?.trim() || '',
+            pros: prosEl?.textContent?.trim() || '',
+            cons: consEl?.textContent?.trim() || '',
+            date: dateEl?.getAttribute('datetime') || dateEl?.textContent?.trim() || '',
+            author: authorEl?.textContent?.trim() || '',
+          };
+        });
+      });
+
+      // Filter reviews that have at least pros or cons
+      const validReviews = rawReviews.filter((r) => Boolean(r.pros || r.cons));
+
+      // Transform to EventEnvelope format
+      const events: EventEnvelope[] = validReviews.map((review) => {
+        const externalId = deriveReviewExternalId(company_name, review);
+        const content = `${review.title}\n\nPros: ${review.pros}\n\nCons: ${review.cons}`;
+
+        return {
+          origin_id: externalId,
+          payload_text: content,
+          author_name: review.author || undefined,
+          occurred_at: review.date ? new Date(review.date) : new Date(),
+          origin_type: 'review',
+          score: calculateEngagementScore('glassdoor', { rating: review.rating }),
+          source_url: `${baseUrl}#review_${review.id}`,
+          metadata: {
+            rating: review.rating,
+            title: review.title,
+            pros: review.pros,
+            cons: review.cons,
+          },
+        };
+      });
+
+      return {
+        events,
+        checkpoint: {
+          last_sync_at: new Date().toISOString(),
+        } as Record<string, unknown>,
+        metadata: {
+          items_found: events.length,
+          items_skipped: rawReviews.length - validReviews.length,
+        },
+      };
+    });
+  }
+
+  async execute(_ctx: ActionContext): Promise<ActionResult> {
+    return { success: false, error: 'Actions not supported' };
+  }
+}

package/dist/connectors/gmaps.ts

@@ -0,0 +1,197 @@
+/**
+ * Google Maps Connector (V1 runtime)
+ *
+ * Fetches business reviews using Google Places API.
+ */
+
+import {
+  type ActionContext,
+  type ActionResult,
+  type ConnectorDefinition,
+  ConnectorRuntime,
+  calculateEngagementScore,
+  type EventEnvelope,
+  type SyncContext,
+  type SyncResult,
+} from '@lobu/connector-sdk';
+import { filterByCheckpoint } from './browser-scraper-utils.ts';
+
+interface GMapsReview {
+  author_name: string;
+  author_url?: string;
+  profile_photo_url?: string;
+  rating: number;
+  relative_time_description?: string;
+  text: string;
+  time: number;
+}
+
+interface PlaceDetailsResponse {
+  status: string;
+  result?: {
+    name?: string;
+    reviews?: GMapsReview[];
+    url?: string;
+  };
+}
+
+interface FindPlaceResponse {
+  candidates?: Array<{ place_id: string }>;
+}
+
+interface GMapsCheckpoint {
+  last_timestamp?: string;
+}
+
+const configSchema = {
+  type: 'object',
+  properties: {
+    place_id: {
+      type: 'string',
+      description: 'Google Place ID',
+    },
+    business_name: {
+      type: 'string',
+      description: 'Business name for search-based fallback',
+    },
+  },
+};
+
+export default class GoogleMapsConnector extends ConnectorRuntime {
+  readonly definition: ConnectorDefinition = {
+    key: 'gmaps',
+    name: 'Google Maps',
+    description: 'Fetches business reviews using Google Places API.',
+    version: '1.0.0',
+    faviconDomain: 'maps.google.com',
+    authSchema: {
+      methods: [
+        {
+          type: 'env_keys',
+          required: true,
+          fields: [
+            {
+              key: 'GOOGLE_MAPS_API_KEY',
+              label: 'Google Maps API Key',
+              secret: true,
+            },
+          ],
+        },
+      ],
+    },
+    feeds: {
+      reviews: {
+        key: 'reviews',
+        name: 'Business Reviews',
+        description: 'Fetch reviews for a business on Google Maps.',
+        configSchema,
+        eventKinds: {
+          review: {
+            description: 'A Google Maps business review',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                rating: { type: 'number', description: 'Star rating (1-5)' },
+                author_url: { type: 'string', format: 'uri' },
+                profile_photo_url: { type: 'string', format: 'uri' },
+                relative_time_description: { type: 'string' },
+              },
+            },
+          },
+        },
+      },
+    },
+    optionsSchema: configSchema,
+  };
+
+  async sync(ctx: SyncContext): Promise<SyncResult> {
+    const apiKey = ctx.config.GOOGLE_MAPS_API_KEY as string | undefined;
+    if (!apiKey) {
+      throw new Error('GOOGLE_MAPS_API_KEY is required');
+    }
+
+    let placeId = ctx.config.place_id as string | undefined;
+    const businessName = ctx.config.business_name as string | undefined;
+
+    if (!placeId && !businessName) {
+      throw new Error('Either place_id or business_name is required');
+    }
+
+    // If no place_id, search by business name
+    if (!placeId && businessName) {
+      const searchUrl = `https://maps.googleapis.com/maps/api/place/findplacefromtext/json?input=${encodeURIComponent(businessName)}&inputtype=textquery&fields=place_id&key=${apiKey}`;
+      const searchResponse = await fetch(searchUrl);
+      if (!searchResponse.ok) {
+        throw new Error(
+          `Google Places search failed (${searchResponse.status}): ${await searchResponse.text()}`
+        );
+      }
+      const searchData = (await searchResponse.json()) as FindPlaceResponse;
+      if (!searchData.candidates || searchData.candidates.length === 0) {
+        throw new Error(`Business not found: ${businessName}`);
+      }
+      placeId = searchData.candidates[0].place_id;
+    }
+
+    // Fetch place details with reviews
+    const detailsUrl = `https://maps.googleapis.com/maps/api/place/details/json?place_id=${placeId}&fields=name,reviews,url&key=${apiKey}`;
+    const detailsResponse = await fetch(detailsUrl);
+    if (!detailsResponse.ok) {
+      throw new Error(
+        `Google Places details failed (${detailsResponse.status}): ${await detailsResponse.text()}`
+      );
+    }
+    const data = (await detailsResponse.json()) as PlaceDetailsResponse;
+
+    if (data.status !== 'OK') {
+      throw new Error(`Google Places API error: ${data.status}`);
+    }
+
+    const place = data.result;
+    const reviews = place?.reviews ?? [];
+    const placeUrl = place?.url ?? `https://maps.google.com/?q=place_id:${placeId}`;
+
+    // Transform reviews to EventEnvelope[] — skip reviews without text
+    let events: EventEnvelope[] = reviews
+      .filter((review) => review.text)
+      .map((review) => ({
+        origin_id: `${placeId}_${review.time}`,
+        payload_text: review.text,
+        author_name: review.author_name || undefined,
+        occurred_at: new Date(review.time * 1000),
+        origin_type: 'review',
+        source_url: placeUrl,
+        score: calculateEngagementScore('gmaps', { rating: review.rating }),
+        metadata: {
+          rating: review.rating,
+          author_url: review.author_url,
+          profile_photo_url: review.profile_photo_url,
+          relative_time_description: review.relative_time_description,
+        },
+      }));
+
+    // Filter by checkpoint
+    const checkpoint = ctx.checkpoint as GMapsCheckpoint | null;
+    events = filterByCheckpoint(events, checkpoint);
+
+    // Sort descending by occurred_at
+    events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
+
+    const newCheckpoint: Record<string, unknown> =
+      events.length > 0
+        ? { last_timestamp: events[0].occurred_at.toISOString() }
+        : { last_timestamp: checkpoint?.last_timestamp ?? null };
+
+    return {
+      events,
+      checkpoint: newCheckpoint,
+      metadata: {
+        items_found: reviews.length,
+      },
+    };
+  }
+
+  async execute(_ctx: ActionContext): Promise<ActionResult> {
+    return { success: false, error: 'Actions not supported' };
+  }
+}