@lobu/cli 6.0.1 → 7.0.0

package/dist/connectors/capterra.ts

@@ -0,0 +1,277 @@
+/**
+ * Capterra Connector
+ * Scrapes software reviews from Capterra using browser rendering with stealth mode.
+ */
+
+import {
+  type ActionContext,
+  type ActionResult,
+  type ConnectorDefinition,
+  ConnectorRuntime,
+  calculateEngagementScore,
+  type EventEnvelope,
+  type SyncContext,
+  type SyncResult,
+} from '@lobu/connector-sdk';
+import {
+  getBrowserCdpUrl,
+  getBrowserUserDataDir,
+  handleCookieConsent,
+  openStealthBrowser,
+  validateUrlDomain,
+  withBrowserErrorCapture,
+} from './browser-scraper-utils.ts';
+
+interface CapterraReview {
+  id: string;
+  rating: number;
+  title: string;
+  text: string;
+  date: string;
+  author: string;
+  helpfulCount: number;
+}
+
+export default class CapterraConnector extends ConnectorRuntime {
+  readonly definition: ConnectorDefinition = {
+    key: 'capterra',
+    name: 'Capterra',
+    version: '1.0.0',
+    faviconDomain: 'capterra.com',
+    description: 'Scrapes software reviews from Capterra.',
+    authSchema: {
+      methods: [{ type: 'none' }],
+    },
+    feeds: {
+      reviews: {
+        key: 'reviews',
+        name: 'Reviews',
+        description: 'Capterra software reviews',
+        configSchema: {
+          type: 'object',
+          required: ['product_id'],
+          properties: {
+            product_id: {
+              type: 'string',
+              description: 'Capterra product ID (e.g., "12345")',
+              minLength: 1,
+            },
+            product_name: {
+              type: 'string',
+              description:
+                'Product name slug for URL (e.g., "spotify"). Optional - Capterra will redirect without it.',
+              minLength: 1,
+            },
+            vendor_name: {
+              type: 'string',
+              description:
+                'Vendor/company name (e.g., "Spotify AB"). Optional but recommended for disambiguation.',
+              minLength: 1,
+            },
+            lookback_days: {
+              type: 'integer',
+              description:
+                'Number of days to look back for historical data. Default: 365 (1 year). Maximum: 730 (2 years).',
+              minimum: 1,
+              maximum: 730,
+              default: 365,
+            },
+          },
+        },
+        eventKinds: {
+          review: {
+            description: 'A Capterra software review',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                rating: { type: 'number', description: 'Star rating (0-5)' },
+                helpful_count: { type: 'number', description: 'Number of helpful votes' },
+              },
+            },
+          },
+        },
+      },
+    },
+    optionsSchema: {
+      type: 'object',
+      required: ['product_id'],
+      properties: {
+        product_id: {
+          type: 'string',
+          description: 'Capterra product ID (e.g., "12345")',
+          minLength: 1,
+        },
+        product_name: {
+          type: 'string',
+          description:
+            'Product name slug for URL (e.g., "spotify"). Optional - Capterra will redirect without it.',
+          minLength: 1,
+        },
+        vendor_name: {
+          type: 'string',
+          description:
+            'Vendor/company name (e.g., "Spotify AB"). Optional but recommended for disambiguation.',
+          minLength: 1,
+        },
+        lookback_days: {
+          type: 'integer',
+          description:
+            'Number of days to look back for historical data. Default: 365 (1 year). Maximum: 730 (2 years).',
+          minimum: 1,
+          maximum: 730,
+          default: 365,
+        },
+      },
+    },
+  };
+
+  async sync(ctx: SyncContext): Promise<SyncResult> {
+    const productId = ctx.config.product_id as string;
+    const productName = ctx.config.product_name as string | undefined;
+
+    const baseUrl = productName
+      ? `https://www.capterra.com/p/${productId}/${productName}/reviews`
+      : `https://www.capterra.com/p/${productId}/reviews`;
+    validateUrlDomain(baseUrl, 'capterra.com');
+
+    const userDataDir = getBrowserUserDataDir(ctx.sessionState);
+    const cdpUrl = getBrowserCdpUrl(ctx.sessionState) ?? 'auto';
+    const session = await openStealthBrowser({ cdpUrl, userDataDir });
+
+    return withBrowserErrorCapture(session, 'capterra-sync', async (page) => {
+      await page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
+
+      await handleCookieConsent(page, '[data-test="cookie-accept"], #onetrust-accept-btn-handler');
+
+      // Wait for review cards
+      try {
+        await page.waitForSelector('[data-test="review-card"], .review-card', { timeout: 10000 });
+      } catch {
+        // Review selectors not found - page structure may have changed
+      }
+
+      // Extract reviews using page.evaluate
+      const rawReviews: CapterraReview[] = await page.evaluate(() => {
+        const reviewElements = Array.from(
+          document.querySelectorAll('[data-test="review-card"], .review-card')
+        );
+
+        return reviewElements.map((el: Element, index: number) => {
+          // Extract rating (usually shown as stars)
+          const ratingElement = el.querySelector(
+            '[data-test="rating"], .rating, [aria-label*="star"]'
+          );
+          let rating = 0;
+          if (ratingElement) {
+            const ariaLabel = ratingElement.getAttribute('aria-label');
+            const ratingMatch = ariaLabel?.match(/(\d+(?:\.\d+)?)/);
+            rating = ratingMatch ? parseFloat(ratingMatch[1]) : 0;
+          }
+
+          // Extract review title
+          const titleElement = el.querySelector(
+            '[data-test="review-title"], .review-title, h3, h4'
+          );
+          const title = titleElement?.textContent?.trim() || '';
+
+          // Extract review text
+          const textElement = el.querySelector(
+            '[data-test="review-body"], .review-body, .review-content, .review-text'
+          );
+          const text = textElement?.textContent?.trim() || '';
+
+          // Extract date
+          const dateElement = el.querySelector('[data-test="review-date"], .review-date, time');
+          const dateText =
+            dateElement?.textContent?.trim() || dateElement?.getAttribute('datetime') || '';
+
+          // Parse relative dates like "2 weeks ago"
+          let date = new Date();
+          if (dateText) {
+            const weeksMatch = dateText.match(/(\d+)\s+weeks?\s+ago/i);
+            const monthsMatch = dateText.match(/(\d+)\s+months?\s+ago/i);
+            const daysMatch = dateText.match(/(\d+)\s+days?\s+ago/i);
+
+            if (weeksMatch) {
+              date = new Date(Date.now() - parseInt(weeksMatch[1], 10) * 7 * 24 * 60 * 60 * 1000);
+            } else if (monthsMatch) {
+              date = new Date(Date.now() - parseInt(monthsMatch[1], 10) * 30 * 24 * 60 * 60 * 1000);
+            } else if (daysMatch) {
+              date = new Date(Date.now() - parseInt(daysMatch[1], 10) * 24 * 60 * 60 * 1000);
+            } else {
+              // Try parsing as date
+              const parsed = new Date(dateText);
+              if (!Number.isNaN(parsed.getTime())) {
+                date = parsed;
+              }
+            }
+          }
+
+          // Extract author
+          const authorElement = el.querySelector(
+            '[data-test="reviewer-name"], .reviewer-name, .author'
+          );
+          const author = authorElement?.textContent?.trim() || 'Anonymous';
+
+          // Extract review ID from data attributes or generate
+          const reviewId =
+            (el as HTMLElement).getAttribute('data-review-id') ||
+            (el as HTMLElement).id ||
+            `${date.toISOString()}_${index}`.replace(/[^a-zA-Z0-9]/g, '_');
+
+          // Extract helpful count
+          const helpfulElement = el.querySelector('[data-test="helpful-count"], .helpful-count');
+          const helpfulCount = helpfulElement
+            ? parseInt(helpfulElement.textContent?.replace(/\D/g, '') || '0', 10)
+            : 0;
+
+          return {
+            id: reviewId,
+            rating,
+            title,
+            text,
+            date: date.toISOString(),
+            author,
+            helpfulCount,
+          };
+        });
+      });
+
+      // Filter reviews with content
+      const reviews = rawReviews.filter((r) => r.text.length > 0);
+
+      // Transform to EventEnvelope
+      const events: EventEnvelope[] = reviews.map((review) => {
+        const engagementData = {
+          rating: review.rating,
+          helpful_count: review.helpfulCount,
+        };
+
+        return {
+          origin_id: review.id,
+          title: review.title,
+          payload_text: review.text,
+          author_name: review.author,
+          occurred_at: new Date(review.date),
+          origin_type: 'review',
+          score: calculateEngagementScore('capterra', engagementData),
+          source_url: baseUrl,
+          metadata: engagementData,
+        };
+      });
+
+      return {
+        events,
+        checkpoint: { last_sync_at: new Date().toISOString() },
+        metadata: {
+          items_found: rawReviews.length,
+          items_skipped: rawReviews.length - reviews.length,
+        },
+      };
+    });
+  }
+
+  async execute(_ctx: ActionContext): Promise<ActionResult> {
+    return { success: false, error: 'Actions not supported' };
+  }
+}

package/dist/connectors/g2.ts

@@ -0,0 +1,290 @@
+/**
+ * G2 Connector (V1 runtime)
+ *
+ * Scrapes B2B software reviews from G2.com using browser rendering with stealth mode.
+ */
+
+import {
+  type ActionContext,
+  type ActionResult,
+  type ConnectorDefinition,
+  ConnectorRuntime,
+  calculateEngagementScore,
+  type EventEnvelope,
+  type SyncContext,
+  type SyncResult,
+} from '@lobu/connector-sdk';
+import {
+  getBrowserCdpUrl,
+  getBrowserUserDataDir,
+  handleCookieConsent,
+  openStealthBrowser,
+  validateUrlDomain,
+  withBrowserErrorCapture,
+} from './browser-scraper-utils.ts';
+
+interface G2Review {
+  rating: number;
+  title: string;
+  text: string;
+  author: string;
+  jobTitle: string;
+  industry: string;
+  companySize: string;
+  date: string;
+  badges: string[];
+  reviewUrl: string;
+  helpfulCount: number;
+}
+
+interface G2Checkpoint {
+  last_sync_at?: string;
+  pages_crawled?: number;
+}
+
+const configSchema = {
+  type: 'object',
+  required: ['product_url'],
+  properties: {
+    product_url: {
+      type: 'string',
+      description: 'Full G2 product review URL e.g. https://www.g2.com/products/confluence/reviews',
+    },
+    lookback_days: {
+      type: 'integer',
+      minimum: 1,
+      maximum: 730,
+      default: 365,
+      description: 'Number of days to look back for reviews (default 365)',
+    },
+  },
+};
+
+export default class G2Connector extends ConnectorRuntime {
+  readonly definition: ConnectorDefinition = {
+    key: 'g2',
+    name: 'G2',
+    description: 'Scrapes B2B software reviews from G2.com.',
+    version: '1.0.0',
+    faviconDomain: 'g2.com',
+    authSchema: {
+      methods: [{ type: 'none' }],
+    },
+    feeds: {
+      reviews: {
+        key: 'reviews',
+        name: 'Product Reviews',
+        description: 'Scrape reviews for a G2 product listing.',
+        configSchema,
+        eventKinds: {
+          review: {
+            description: 'A G2 B2B software review',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                rating: { type: 'number', description: 'Star rating (0-5)' },
+                helpful_count: { type: 'number' },
+                job_title: { type: 'string', description: 'Reviewer job title' },
+                industry: { type: 'string', description: 'Reviewer industry' },
+                company_size: { type: 'string', description: 'Reviewer company size' },
+                badges: { type: 'array', items: { type: 'string' }, description: 'Review badges' },
+              },
+            },
+          },
+        },
+      },
+    },
+    optionsSchema: configSchema,
+  };
+
+  async sync(ctx: SyncContext): Promise<SyncResult> {
+    const productUrl = ctx.config.product_url as string;
+
+    if (!productUrl?.match(/^https:\/\/www\.g2\.com\/products\/[^/]+\/reviews/)) {
+      return {
+        events: [],
+        checkpoint: ctx.checkpoint,
+        metadata: { items_found: 0, error: 'Invalid product_url' },
+      };
+    }
+    validateUrlDomain(productUrl, 'g2.com');
+
+    // Extract product key from URL for origin_id generation
+    const productMatch = productUrl.match(/\/products\/([^/]+)/);
+    const productKey = productMatch ? productMatch[1] : 'unknown';
+
+    const baseUrl = productUrl;
+    const allEvents: EventEnvelope[] = [];
+
+    const userDataDir = getBrowserUserDataDir(ctx.sessionState);
+    const cdpUrl = getBrowserCdpUrl(ctx.sessionState) ?? 'auto';
+    const session = await openStealthBrowser({ cdpUrl, userDataDir });
+
+    return withBrowserErrorCapture(session, 'g2-sync', async (page) => {
+      const maxPages = 5;
+
+      for (let pageNum = 1; pageNum <= maxPages; pageNum++) {
+        const pageUrl = pageNum === 1 ? baseUrl : `${baseUrl}?page=${pageNum}`;
+
+        await page.goto(pageUrl, { waitUntil: 'domcontentloaded' });
+
+        if (pageNum === 1) {
+          await handleCookieConsent(page, '#onetrust-accept-btn-handler');
+        }
+
+        // Wait for review cards to load
+        try {
+          await page.waitForSelector('[itemprop="review"]', { timeout: 10000 });
+        } catch {
+          // No reviews found on this page — stop paginating
+          break;
+        }
+
+        // Extract reviews from the page
+        const reviews: G2Review[] = await page.evaluate(() => {
+          const results: G2Review[] = [];
+          const reviewCards = document.querySelectorAll('[itemprop="review"]');
+
+          reviewCards.forEach((card) => {
+            try {
+              // Extract author name from meta tag
+              const authorMeta = card.querySelector('[itemprop="author"] meta[itemprop="name"]');
+              const author = authorMeta?.getAttribute('content') || 'Anonymous';
+
+              // Extract author details from sibling divs with elv-text-subtle class
+              const authorContainer = card.querySelector('[itemprop="author"]');
+              const parentDiv = authorContainer?.closest('.elv-gap-2')?.parentElement;
+              const detailDivs = parentDiv
+                ? Array.from(parentDiv.querySelectorAll('.elv-text-subtle'))
+                : [];
+
+              // Parse author details (job title, industry, company size)
+              let jobTitle = '';
+              let industry = '';
+              let companySize = '';
+
+              if (detailDivs.length >= 3) {
+                jobTitle = detailDivs[0]?.textContent?.trim() || '';
+                industry = detailDivs[1]?.textContent?.trim() || '';
+                companySize = detailDivs[2]?.textContent?.trim() || '';
+              } else if (detailDivs.length === 2) {
+                jobTitle = detailDivs[0]?.textContent?.trim() || '';
+                companySize = detailDivs[1]?.textContent?.trim() || '';
+              } else if (detailDivs.length === 1) {
+                companySize = detailDivs[0]?.textContent?.trim() || '';
+              }
+
+              // Extract date from meta tag
+              const dateMeta = card.querySelector('meta[itemprop="datePublished"]');
+              const dateStr = dateMeta?.getAttribute('content') || '';
+
+              // Extract rating
+              const ratingMeta = card.querySelector('[itemprop="ratingValue"]');
+              const rating = ratingMeta ? parseFloat(ratingMeta.getAttribute('content') || '0') : 0;
+
+              // Extract review title
+              const titleDiv = card.querySelector('[itemprop="name"] .elv-font-bold');
+              const title = titleDiv?.textContent?.trim().replace(/^"|"$/g, '') || '';
+
+              // Extract review body - use innerText to preserve visual spacing/newlines
+              const reviewBodyEl = card.querySelector('[itemprop="reviewBody"]');
+              const reviewBody = (reviewBodyEl as HTMLElement)?.innerText?.trim() || '';
+
+              // Extract badges
+              const badgeEls = card.querySelectorAll(
+                '[class*="badge"], [class*="tag"], .elv-rounded-sm.elv-border'
+              );
+              const badges = Array.from(badgeEls)
+                .map((el) => el.textContent?.trim())
+                .filter((text): text is string => !!text && text.length < 50 && text.length > 3);
+
+              // Extract review URL
+              const linkEl = card.querySelector('a[href*="survey_responses"]');
+              const href = linkEl?.getAttribute('href') || '';
+              const reviewUrl = href
+                ? href.startsWith('http')
+                  ? href
+                  : `https://www.g2.com${href}`
+                : '';
+
+              // Skip reviews with minimal content
+              if ((reviewBody || '').length < 50) return;
+
+              results.push({
+                rating,
+                title,
+                text: reviewBody,
+                author,
+                jobTitle,
+                industry,
+                companySize,
+                date: dateStr,
+                badges: badges.slice(0, 10),
+                reviewUrl,
+                helpfulCount: 0,
+              });
+            } catch (e) {
+              console.error('[G2Connector] Error parsing review card:', e);
+            }
+          });
+
+          return results;
+        });
+
+        // Transform reviews to EventEnvelope format
+        for (const review of reviews) {
+          const event: EventEnvelope = {
+            origin_id: `g2-${productKey}-${review.date || 'nodate'}-${review.author.replace(/\s+/g, '-')}`,
+            title: review.title,
+            payload_text: review.text,
+            author_name: review.author,
+            occurred_at: review.date ? new Date(review.date) : new Date(),
+            origin_type: 'review',
+            score: calculateEngagementScore('g2', {
+              rating: review.rating,
+              helpful_count: 0,
+            }),
+            source_url: review.reviewUrl || baseUrl,
+            metadata: {
+              rating: review.rating,
+              helpful_count: review.helpfulCount,
+              job_title: review.jobTitle,
+              industry: review.industry,
+              company_size: review.companySize,
+              badges: review.badges,
+            },
+          };
+          allEvents.push(event);
+        }
+
+        // If this page had no reviews, stop paginating
+        if (reviews.length === 0) break;
+
+        // Delay between pages
+        if (pageNum < maxPages) {
+          await new Promise((resolve) => setTimeout(resolve, 2000));
+        }
+
+        // Rate limit
+        await new Promise((resolve) => setTimeout(resolve, 6000));
+      }
+
+      const newCheckpoint: G2Checkpoint = {
+        last_sync_at: new Date().toISOString(),
+        pages_crawled: Math.min(5, allEvents.length > 0 ? Math.ceil(allEvents.length / 10) : 0),
+      };
+
+      return {
+        events: allEvents,
+        checkpoint: newCheckpoint as Record<string, unknown>,
+        metadata: {
+          items_found: allEvents.length,
+        },
+      };
+    });
+  }
+
+  async execute(_ctx: ActionContext): Promise<ActionResult> {
+    return { success: false, error: 'Actions not supported' };
+  }
+}