@gonzih/of-scraper 1.0.0

package/src/index.ts

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+import { connectRedis, xadd, isSeen, markSeen } from './redis';
+import { launchBrowser, ensureLoggedIn, takeErrorScreenshot } from './browser';
+import { scrapeNewMessages } from './messages';
+import { scrapeProfile } from './profiles';
+import { scrapeFinancials } from './financials';
+import type { Browser, Page } from 'puppeteer';
+
+const POLL_INTERVAL_MS = parseInt(process.env.POLL_INTERVAL_MS ?? '30000', 10);
+const RATE_LIMIT_MS = 2000;
+
+async function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function processMessage(
+  page: Page,
+  messageId: string,
+  fromUsername: string,
+  text: string,
+  timestamp: string,
+  threadId: string
+): Promise<void> {
+  const ts = new Date().toISOString();
+  console.log(`[${ts}] Processing message ${messageId} from @${fromUsername}`);
+
+  await xadd('of:messages', {
+    messageId,
+    fromUsername,
+    text,
+    timestamp,
+    threadId,
+  });
+
+  await sleep(RATE_LIMIT_MS);
+
+  const profile = await scrapeProfile(page, fromUsername);
+  if (profile) {
+    await xadd('of:profiles', {
+      username: profile.username,
+      displayName: profile.displayName,
+      isSubscribed: String(profile.isSubscribed),
+      profilePic: profile.profilePic,
+      bio: profile.bio,
+      fetchedAt: profile.fetchedAt,
+    });
+  }
+
+  await sleep(RATE_LIMIT_MS);
+
+  const financials = await scrapeFinancials(page, fromUsername, threadId);
+  if (financials) {
+    await xadd('of:financials', {
+      username: financials.username,
+      totalTips: financials.totalTips,
+      totalPurchases: financials.totalPurchases,
+      subscriptionStatus: financials.subscriptionStatus,
+      lastPaymentDate: financials.lastPaymentDate,
+      lifetimeValue: financials.lifetimeValue,
+      updatedAt: financials.updatedAt,
+    });
+  }
+}
+
+async function pollLoop(browser: Browser, page: Page): Promise<void> {
+  while (true) {
+    const loopStart = new Date().toISOString();
+    console.log(`[${loopStart}] Starting poll cycle`);
+
+    try {
+      const messages = await scrapeNewMessages(page);
+
+      for (const msg of messages) {
+        const seen = await isSeen(msg.messageId);
+        if (seen) {
+          continue;
+        }
+
+        await markSeen(msg.messageId);
+        await processMessage(
+          page,
+          msg.messageId,
+          msg.fromUsername,
+          msg.text,
+          msg.timestamp,
+          msg.threadId
+        );
+      }
+    } catch (err) {
+      console.error(`[${new Date().toISOString()}] Poll cycle error:`, err);
+      try {
+        await takeErrorScreenshot(page);
+      } catch {
+        // ignore screenshot errors
+      }
+    }
+
+    console.log(`[${new Date().toISOString()}] Poll cycle complete — sleeping ${POLL_INTERVAL_MS}ms`);
+    await sleep(POLL_INTERVAL_MS);
+  }
+}
+
+async function main(): Promise<void> {
+  console.log(`[${new Date().toISOString()}] of-scraper starting up`);
+  console.log(`[${new Date().toISOString()}] POLL_INTERVAL_MS=${POLL_INTERVAL_MS}`);
+
+  await connectRedis();
+
+  let browser: Browser | null = null;
+  let page: Page | null = null;
+
+  try {
+    browser = await launchBrowser();
+    page = await ensureLoggedIn(browser);
+  } catch (err) {
+    console.error(`[${new Date().toISOString()}] Failed to launch browser or log in:`, err);
+    if (browser) await browser.close().catch(() => {});
+    process.exit(1);
+  }
+
+  const shutdown = async (): Promise<void> => {
+    console.log(`[${new Date().toISOString()}] Shutting down...`);
+    if (browser) await browser.close().catch(() => {});
+    process.exit(0);
+  };
+
+  process.on('SIGINT', () => { shutdown().catch(console.error); });
+  process.on('SIGTERM', () => { shutdown().catch(console.error); });
+
+  await pollLoop(browser, page);
+}
+
+main().catch((err) => {
+  console.error(`[${new Date().toISOString()}] Fatal error:`, err);
+  process.exit(1);
+});

package/src/messages.ts

@@ -0,0 +1,163 @@
+import type { Page } from 'puppeteer';
+
+export interface Message {
+  messageId: string;
+  fromUsername: string;
+  text: string;
+  timestamp: string;
+  threadId: string;
+}
+
+export async function scrapeNewMessages(page: Page): Promise<Message[]> {
+  const ts = new Date().toISOString();
+  console.log(`[${ts}] Polling https://onlyfans.com/my/chats for new messages`);
+
+  try {
+    await page.goto('https://onlyfans.com/my/chats', {
+      waitUntil: 'networkidle2',
+      timeout: 30000,
+    });
+  } catch (err) {
+    console.error(`[${new Date().toISOString()}] Failed to navigate to chats:`, err);
+    return [];
+  }
+
+  try {
+    await page.waitForSelector('.b-chats__list, .b-chat-item, [class*="chat-list"], [class*="chatItem"]', {
+      timeout: 15000,
+    });
+  } catch {
+    console.warn(`[${new Date().toISOString()}] Chat list selector not found — page may not be loaded`);
+    return [];
+  }
+
+  const messages = await page.evaluate((): Message[] => {
+    const results: Message[] = [];
+
+    const chatItemSelectors = [
+      '.b-chat-item',
+      '[class*="chatItem"]',
+      '[data-type="chat-item"]',
+      '.b-chats__item',
+    ];
+
+    let chatItems: Element[] = [];
+    for (const sel of chatItemSelectors) {
+      const found = Array.from(document.querySelectorAll(sel));
+      if (found.length > 0) {
+        chatItems = found;
+        break;
+      }
+    }
+
+    for (const item of chatItems) {
+      try {
+        const link = item.querySelector('a[href*="/my/chats/chat/"]');
+        const href = link?.getAttribute('href') ?? '';
+        const threadMatch = href.match(/\/my\/chats\/chat\/(\d+)/);
+        const threadId = threadMatch ? threadMatch[1] : '';
+
+        const nameEl =
+          item.querySelector('.b-chat-item__name') ??
+          item.querySelector('[class*="userName"]') ??
+          item.querySelector('[class*="name"]');
+        const fromUsername = nameEl?.textContent?.trim() ?? '';
+
+        const textEl =
+          item.querySelector('.b-chat-item__message') ??
+          item.querySelector('[class*="lastMessage"]') ??
+          item.querySelector('[class*="message"]');
+        const text = textEl?.textContent?.trim() ?? '';
+
+        const timeEl =
+          item.querySelector('time') ??
+          item.querySelector('[class*="time"]') ??
+          item.querySelector('[class*="date"]');
+        const timestamp =
+          timeEl?.getAttribute('datetime') ?? timeEl?.textContent?.trim() ?? new Date().toISOString();
+
+        const messageId = `${threadId}_${btoa(encodeURIComponent(text)).slice(0, 16)}`;
+
+        if (threadId && fromUsername) {
+          results.push({ messageId, fromUsername, text, timestamp, threadId });
+        }
+      } catch {
+        // Skip malformed items
+      }
+    }
+
+    return results;
+  });
+
+  console.log(`[${new Date().toISOString()}] Found ${messages.length} chat threads`);
+  return messages;
+}
+
+export async function scrapeThreadMessages(page: Page, threadId: string): Promise<Message[]> {
+  const url = `https://onlyfans.com/my/chats/chat/${threadId}`;
+  try {
+    await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+  } catch (err) {
+    console.error(`[${new Date().toISOString()}] Failed to navigate to thread ${threadId}:`, err);
+    return [];
+  }
+
+  try {
+    await page.waitForSelector('.b-chat__messages, [class*="messages"]', { timeout: 15000 });
+  } catch {
+    return [];
+  }
+
+  const messages = await page.evaluate((tId: string): Message[] => {
+    const results: Message[] = [];
+    const messageSelectors = [
+      '.b-message__wrapper',
+      '[class*="messageItem"]',
+      '[class*="message-item"]',
+    ];
+
+    let items: Element[] = [];
+    for (const sel of messageSelectors) {
+      const found = Array.from(document.querySelectorAll(sel));
+      if (found.length > 0) {
+        items = found;
+        break;
+      }
+    }
+
+    for (const item of items) {
+      try {
+        const isSelf =
+          item.classList.contains('m-my') ||
+          item.querySelector('[class*="my-message"]') !== null ||
+          item.getAttribute('data-type') === 'outgoing';
+        if (isSelf) continue;
+
+        const textEl = item.querySelector('.b-message__text, [class*="messageText"]');
+        const text = textEl?.textContent?.trim() ?? '';
+
+        const timeEl = item.querySelector('time, [class*="time"]');
+        const timestamp =
+          timeEl?.getAttribute('datetime') ?? timeEl?.textContent?.trim() ?? new Date().toISOString();
+
+        const messageId =
+          item.getAttribute('data-id') ??
+          item.getAttribute('id') ??
+          `${tId}_${btoa(encodeURIComponent(timestamp + text)).slice(0, 16)}`;
+
+        const usernameEl = item.querySelector('[class*="userName"], [class*="name"]');
+        const fromUsername = usernameEl?.textContent?.trim() ?? '';
+
+        if (text) {
+          results.push({ messageId, fromUsername, text, timestamp, threadId: tId });
+        }
+      } catch {
+        // skip
+      }
+    }
+
+    return results;
+  }, threadId);
+
+  return messages;
+}

package/src/profiles.ts

@@ -0,0 +1,105 @@
+import type { Page } from 'puppeteer';
+
+export interface Profile {
+  username: string;
+  displayName: string;
+  isSubscribed: boolean;
+  profilePic: string;
+  bio: string;
+  fetchedAt: string;
+}
+
+export async function scrapeProfile(page: Page, username: string): Promise<Profile | null> {
+  const url = `https://onlyfans.com/${username}`;
+  console.log(`[${new Date().toISOString()}] Fetching profile for @${username}`);
+
+  try {
+    await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+  } catch (err) {
+    console.error(`[${new Date().toISOString()}] Failed to navigate to profile ${username}:`, err);
+    return null;
+  }
+
+  try {
+    await page.waitForSelector(
+      '.b-profile__main, [class*="profile"], .b-user-card, [class*="userCard"]',
+      { timeout: 15000 }
+    );
+  } catch {
+    console.warn(`[${new Date().toISOString()}] Profile selectors not found for @${username}`);
+  }
+
+  const profile = await page.evaluate((uname: string): Profile => {
+    const nameSelectors = [
+      '.b-username__name',
+      '[class*="userName"]',
+      'h1[class*="name"]',
+      '.g-user-name',
+    ];
+    let displayName = uname;
+    for (const sel of nameSelectors) {
+      const el = document.querySelector(sel);
+      if (el?.textContent?.trim()) {
+        displayName = el.textContent.trim();
+        break;
+      }
+    }
+
+    const picSelectors = [
+      '.b-user-avatar img',
+      '[class*="avatar"] img',
+      '.g-avatar img',
+      'img[class*="avatar"]',
+    ];
+    let profilePic = '';
+    for (const sel of picSelectors) {
+      const el = document.querySelector<HTMLImageElement>(sel);
+      if (el?.src) {
+        profilePic = el.src;
+        break;
+      }
+    }
+
+    const bioSelectors = [
+      '.b-profile__about-text',
+      '[class*="bio"]',
+      '[class*="about"]',
+      '.g-desc',
+    ];
+    let bio = '';
+    for (const sel of bioSelectors) {
+      const el = document.querySelector(sel);
+      if (el?.textContent?.trim()) {
+        bio = el.textContent.trim();
+        break;
+      }
+    }
+
+    const subSelectors = [
+      '[class*="subscribeButton"]',
+      'button[class*="subscribe"]',
+      '.b-btn-subscibe',
+    ];
+    let isSubscribed = false;
+    for (const sel of subSelectors) {
+      const el = document.querySelector(sel);
+      if (el) {
+        const btnText = el.textContent?.toLowerCase() ?? '';
+        isSubscribed = btnText.includes('subscribed') || btnText.includes('unsubscribe');
+        break;
+      }
+    }
+
+    return {
+      username: uname,
+      displayName,
+      isSubscribed,
+      profilePic,
+      bio,
+      fetchedAt: new Date().toISOString(),
+    };
+  }, username);
+
+  console.log(`[${new Date().toISOString()}] Profile fetched for @${username}: ${profile.displayName}`);
+  return profile;
+}

package/src/redis.ts

@@ -0,0 +1,34 @@
+import Redis from 'ioredis';
+
+const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379';
+
+export const redis = new Redis(REDIS_URL, {
+  lazyConnect: true,
+  maxRetriesPerRequest: 3,
+});
+
+redis.on('error', (err: Error) => {
+  console.error(`[${new Date().toISOString()}] Redis error:`, err.message);
+});
+
+export async function connectRedis(): Promise<void> {
+  await redis.connect();
+  console.log(`[${new Date().toISOString()}] Redis connected to ${REDIS_URL}`);
+}
+
+export async function xadd(stream: string, fields: Record<string, string>): Promise<string | null> {
+  const args: string[] = [];
+  for (const [k, v] of Object.entries(fields)) {
+    args.push(k, v);
+  }
+  return redis.xadd(stream, '*', ...args);
+}
+
+export async function isSeen(messageId: string): Promise<boolean> {
+  const result = await redis.sismember('of:seen_messages', messageId);
+  return result === 1;
+}
+
+export async function markSeen(messageId: string): Promise<void> {
+  await redis.sadd('of:seen_messages', messageId);
+}

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020", "DOM"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}