@gonzih/of-scraper 1.0.0

package/dist/messages.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrapeThreadMessages = exports.scrapeNewMessages = void 0;
+async function scrapeNewMessages(page) {
+    const ts = new Date().toISOString();
+    console.log(`[${ts}] Polling https://onlyfans.com/my/chats for new messages`);
+    try {
+        await page.goto('https://onlyfans.com/my/chats', {
+            waitUntil: 'networkidle2',
+            timeout: 30000,
+        });
+    }
+    catch (err) {
+        console.error(`[${new Date().toISOString()}] Failed to navigate to chats:`, err);
+        return [];
+    }
+    try {
+        await page.waitForSelector('.b-chats__list, .b-chat-item, [class*="chat-list"], [class*="chatItem"]', {
+            timeout: 15000,
+        });
+    }
+    catch {
+        console.warn(`[${new Date().toISOString()}] Chat list selector not found — page may not be loaded`);
+        return [];
+    }
+    const messages = await page.evaluate(() => {
+        const results = [];
+        const chatItemSelectors = [
+            '.b-chat-item',
+            '[class*="chatItem"]',
+            '[data-type="chat-item"]',
+            '.b-chats__item',
+        ];
+        let chatItems = [];
+        for (const sel of chatItemSelectors) {
+            const found = Array.from(document.querySelectorAll(sel));
+            if (found.length > 0) {
+                chatItems = found;
+                break;
+            }
+        }
+        for (const item of chatItems) {
+            try {
+                const link = item.querySelector('a[href*="/my/chats/chat/"]');
+                const href = link?.getAttribute('href') ?? '';
+                const threadMatch = href.match(/\/my\/chats\/chat\/(\d+)/);
+                const threadId = threadMatch ? threadMatch[1] : '';
+                const nameEl = item.querySelector('.b-chat-item__name') ??
+                    item.querySelector('[class*="userName"]') ??
+                    item.querySelector('[class*="name"]');
+                const fromUsername = nameEl?.textContent?.trim() ?? '';
+                const textEl = item.querySelector('.b-chat-item__message') ??
+                    item.querySelector('[class*="lastMessage"]') ??
+                    item.querySelector('[class*="message"]');
+                const text = textEl?.textContent?.trim() ?? '';
+                const timeEl = item.querySelector('time') ??
+                    item.querySelector('[class*="time"]') ??
+                    item.querySelector('[class*="date"]');
+                const timestamp = timeEl?.getAttribute('datetime') ?? timeEl?.textContent?.trim() ?? new Date().toISOString();
+                const messageId = `${threadId}_${btoa(encodeURIComponent(text)).slice(0, 16)}`;
+                if (threadId && fromUsername) {
+                    results.push({ messageId, fromUsername, text, timestamp, threadId });
+                }
+            }
+            catch {
+                // Skip malformed items
+            }
+        }
+        return results;
+    });
+    console.log(`[${new Date().toISOString()}] Found ${messages.length} chat threads`);
+    return messages;
+}
+exports.scrapeNewMessages = scrapeNewMessages;
+async function scrapeThreadMessages(page, threadId) {
+    const url = `https://onlyfans.com/my/chats/chat/${threadId}`;
+    try {
+        await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+    }
+    catch (err) {
+        console.error(`[${new Date().toISOString()}] Failed to navigate to thread ${threadId}:`, err);
+        return [];
+    }
+    try {
+        await page.waitForSelector('.b-chat__messages, [class*="messages"]', { timeout: 15000 });
+    }
+    catch {
+        return [];
+    }
+    const messages = await page.evaluate((tId) => {
+        const results = [];
+        const messageSelectors = [
+            '.b-message__wrapper',
+            '[class*="messageItem"]',
+            '[class*="message-item"]',
+        ];
+        let items = [];
+        for (const sel of messageSelectors) {
+            const found = Array.from(document.querySelectorAll(sel));
+            if (found.length > 0) {
+                items = found;
+                break;
+            }
+        }
+        for (const item of items) {
+            try {
+                const isSelf = item.classList.contains('m-my') ||
+                    item.querySelector('[class*="my-message"]') !== null ||
+                    item.getAttribute('data-type') === 'outgoing';
+                if (isSelf)
+                    continue;
+                const textEl = item.querySelector('.b-message__text, [class*="messageText"]');
+                const text = textEl?.textContent?.trim() ?? '';
+                const timeEl = item.querySelector('time, [class*="time"]');
+                const timestamp = timeEl?.getAttribute('datetime') ?? timeEl?.textContent?.trim() ?? new Date().toISOString();
+                const messageId = item.getAttribute('data-id') ??
+                    item.getAttribute('id') ??
+                    `${tId}_${btoa(encodeURIComponent(timestamp + text)).slice(0, 16)}`;
+                const usernameEl = item.querySelector('[class*="userName"], [class*="name"]');
+                const fromUsername = usernameEl?.textContent?.trim() ?? '';
+                if (text) {
+                    results.push({ messageId, fromUsername, text, timestamp, threadId: tId });
+                }
+            }
+            catch {
+                // skip
+            }
+        }
+        return results;
+    }, threadId);
+    return messages;
+}
+exports.scrapeThreadMessages = scrapeThreadMessages;
+//# sourceMappingURL=messages.js.map

package/dist/messages.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"messages.js","sourceRoot":"","sources":["../src/messages.ts"],"names":[],"mappings":";;;AAUO,KAAK,UAAU,iBAAiB,CAAC,IAAU;IAChD,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,0DAA0D,CAAC,CAAC;IAE9E,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE;YAC/C,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACjF,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,eAAe,CAAC,yEAAyE,EAAE;YACpG,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,yDAAyD,CAAC,CAAC;QACpG,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAc,EAAE;QACnD,MAAM,OAAO,GAAc,EAAE,CAAC;QAE9B,MAAM,iBAAiB,GAAG;YACxB,cAAc;YACd,qBAAqB;YACrB,yBAAyB;YACzB,gBAAgB;SACjB,CAAC;QAEF,IAAI,SAAS,GAAc,EAAE,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;YACzD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,SAAS,GAAG,KAAK,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC;gBAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC3D,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAEnD,MAAM,MAAM,GACV,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC;oBACxC,IAAI,CAAC,aAAa,CAAC,qBAAqB,CAAC;oBACzC,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;gBACxC,MAAM,YAAY,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAEvD,MAAM,MAAM,GACV,IAAI,CAAC,aAAa,CAAC,uBAAuB,CAAC;oBAC3C,IAAI,CAAC,aAAa,CAAC,wBAAwB,CAAC;oBAC5C,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;gBAC3C,MAAM,IAAI,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAE/C,MAAM,MAAM,GACV,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC;oBAC1B,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC;oBACrC,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;gBACxC,MAAM,SAAS,GACb,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAE9F,MAAM,SAAS,GAAG,GAAG,QAAQ,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;gBAE/E,IAAI,QAAQ,IAAI,YAAY,EAAE,CAAC;oBAC7B,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,WAAW,QAAQ,CAAC,MAAM,eAAe,CAAC,CAAC;IACnF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAnFD,8CAmFC;AAEM,KAAK,UAAU,oBAAoB,CAAC,IAAU,EAAE,QAAgB;IACrE,MAAM,GAAG,GAAG,sCAAsC,QAAQ,EAAE,CAAC;IAC7D,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,kCAAkC,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9F,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,eAAe,CAAC,wCAAwC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAW,EAAa,EAAE;QAC9D,MAAM,OAAO,GAAc,EAAE,CAAC;QAC9B,MAAM,gBAAgB,GAAG;YACvB,qBAAqB;YACrB,wBAAwB;YACxB,yBAAyB;SAC1B,CAAC;QAEF,IAAI,KAAK,GAAc,EAAE,CAAC;QAC1B,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;YACzD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,KAAK,GAAG,KAAK,CAAC;gBACd,MAAM;YACR,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,MAAM,GACV,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC/B,IAAI,CAAC,aAAa,CAAC,uBAAuB,CAAC,KAAK,IAAI;oBACpD,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,UAAU,CAAC;gBAChD,IAAI,MAAM;oBAAE,SAAS;gBAErB,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,0CAA0C,CAAC,CAAC;gBAC9E,MAAM,IAAI,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAE/C,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,uBAAuB,CAAC,CAAC;gBAC3D,MAAM,SAAS,GACb,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAE9F,MAAM,SAAS,GACb,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC;oBAC5B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;oBACvB,GAAG,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;gBAEtE,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,sCAAsC,CAAC,CAAC;gBAC9E,MAAM,YAAY,GAAG,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAE3D,IAAI,IAAI,EAAE,CAAC;oBACT,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;YACT,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC,EAAE,QAAQ,CAAC,CAAC;IAEb,OAAO,QAAQ,CAAC;AAClB,CAAC;AAnED,oDAmEC"}

package/dist/profiles.d.ts

@@ -0,0 +1,11 @@
+import type { Page } from 'puppeteer';
+export interface Profile {
+    username: string;
+    displayName: string;
+    isSubscribed: boolean;
+    profilePic: string;
+    bio: string;
+    fetchedAt: string;
+}
+export declare function scrapeProfile(page: Page, username: string): Promise<Profile | null>;
+//# sourceMappingURL=profiles.d.ts.map

package/dist/profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../src/profiles.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEtC,MAAM,WAAW,OAAO;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CA6FzF"}

package/dist/profiles.js

@@ -0,0 +1,90 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrapeProfile = void 0;
+async function scrapeProfile(page, username) {
+    const url = `https://onlyfans.com/${username}`;
+    console.log(`[${new Date().toISOString()}] Fetching profile for @${username}`);
+    try {
+        await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+    }
+    catch (err) {
+        console.error(`[${new Date().toISOString()}] Failed to navigate to profile ${username}:`, err);
+        return null;
+    }
+    try {
+        await page.waitForSelector('.b-profile__main, [class*="profile"], .b-user-card, [class*="userCard"]', { timeout: 15000 });
+    }
+    catch {
+        console.warn(`[${new Date().toISOString()}] Profile selectors not found for @${username}`);
+    }
+    const profile = await page.evaluate((uname) => {
+        const nameSelectors = [
+            '.b-username__name',
+            '[class*="userName"]',
+            'h1[class*="name"]',
+            '.g-user-name',
+        ];
+        let displayName = uname;
+        for (const sel of nameSelectors) {
+            const el = document.querySelector(sel);
+            if (el?.textContent?.trim()) {
+                displayName = el.textContent.trim();
+                break;
+            }
+        }
+        const picSelectors = [
+            '.b-user-avatar img',
+            '[class*="avatar"] img',
+            '.g-avatar img',
+            'img[class*="avatar"]',
+        ];
+        let profilePic = '';
+        for (const sel of picSelectors) {
+            const el = document.querySelector(sel);
+            if (el?.src) {
+                profilePic = el.src;
+                break;
+            }
+        }
+        const bioSelectors = [
+            '.b-profile__about-text',
+            '[class*="bio"]',
+            '[class*="about"]',
+            '.g-desc',
+        ];
+        let bio = '';
+        for (const sel of bioSelectors) {
+            const el = document.querySelector(sel);
+            if (el?.textContent?.trim()) {
+                bio = el.textContent.trim();
+                break;
+            }
+        }
+        const subSelectors = [
+            '[class*="subscribeButton"]',
+            'button[class*="subscribe"]',
+            '.b-btn-subscibe',
+        ];
+        let isSubscribed = false;
+        for (const sel of subSelectors) {
+            const el = document.querySelector(sel);
+            if (el) {
+                const btnText = el.textContent?.toLowerCase() ?? '';
+                isSubscribed = btnText.includes('subscribed') || btnText.includes('unsubscribe');
+                break;
+            }
+        }
+        return {
+            username: uname,
+            displayName,
+            isSubscribed,
+            profilePic,
+            bio,
+            fetchedAt: new Date().toISOString(),
+        };
+    }, username);
+    console.log(`[${new Date().toISOString()}] Profile fetched for @${username}: ${profile.displayName}`);
+    return profile;
+}
+exports.scrapeProfile = scrapeProfile;
+//# sourceMappingURL=profiles.js.map

package/dist/profiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.js","sourceRoot":"","sources":["../src/profiles.ts"],"names":[],"mappings":";;;AAWO,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,QAAgB;IAC9D,MAAM,GAAG,GAAG,wBAAwB,QAAQ,EAAE,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,2BAA2B,QAAQ,EAAE,CAAC,CAAC;IAE/E,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,mCAAmC,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,eAAe,CACxB,yEAAyE,EACzE,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,sCAAsC,QAAQ,EAAE,CAAC,CAAC;IAC7F,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAa,EAAW,EAAE;QAC7D,MAAM,aAAa,GAAG;YACpB,mBAAmB;YACnB,qBAAqB;YACrB,mBAAmB;YACnB,cAAc;SACf,CAAC;QACF,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC5B,WAAW,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG;YACnB,oBAAoB;YACpB,uBAAuB;YACvB,eAAe;YACf,sBAAsB;SACvB,CAAC;QACF,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YAC/B,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAmB,GAAG,CAAC,CAAC;YACzD,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC;gBACZ,UAAU,GAAG,EAAE,CAAC,GAAG,CAAC;gBACpB,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG;YACnB,wBAAwB;YACxB,gBAAgB;YAChB,kBAAkB;YAClB,SAAS;SACV,CAAC;QACF,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YAC/B,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC5B,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;gBAC5B,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG;YACnB,4BAA4B;YAC5B,4BAA4B;YAC5B,iBAAiB;SAClB,CAAC;QACF,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YAC/B,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,EAAE,EAAE,CAAC;gBACP,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;gBACpD,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;gBACjF,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,KAAK;YACf,WAAW;YACX,YAAY;YACZ,UAAU;YACV,GAAG;YACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;IACJ,CAAC,EAAE,QAAQ,CAAC,CAAC;IAEb,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,0BAA0B,QAAQ,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACtG,OAAO,OAAO,CAAC;AACjB,CAAC;AA7FD,sCA6FC"}

package/dist/redis.d.ts

@@ -0,0 +1,7 @@
+import Redis from 'ioredis';
+export declare const redis: Redis;
+export declare function connectRedis(): Promise<void>;
+export declare function xadd(stream: string, fields: Record<string, string>): Promise<string | null>;
+export declare function isSeen(messageId: string): Promise<boolean>;
+export declare function markSeen(messageId: string): Promise<void>;
+//# sourceMappingURL=redis.d.ts.map

package/dist/redis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis.d.ts","sourceRoot":"","sources":["../src/redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,SAAS,CAAC;AAI5B,eAAO,MAAM,KAAK,OAGhB,CAAC;AAMH,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAGlD;AAED,wBAAsB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAMjG;AAED,wBAAsB,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGhE;AAED,wBAAsB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/D"}

package/dist/redis.js

@@ -0,0 +1,38 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.markSeen = exports.isSeen = exports.xadd = exports.connectRedis = exports.redis = void 0;
+const ioredis_1 = __importDefault(require("ioredis"));
+const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379';
+exports.redis = new ioredis_1.default(REDIS_URL, {
+    lazyConnect: true,
+    maxRetriesPerRequest: 3,
+});
+exports.redis.on('error', (err) => {
+    console.error(`[${new Date().toISOString()}] Redis error:`, err.message);
+});
+async function connectRedis() {
+    await exports.redis.connect();
+    console.log(`[${new Date().toISOString()}] Redis connected to ${REDIS_URL}`);
+}
+exports.connectRedis = connectRedis;
+async function xadd(stream, fields) {
+    const args = [];
+    for (const [k, v] of Object.entries(fields)) {
+        args.push(k, v);
+    }
+    return exports.redis.xadd(stream, '*', ...args);
+}
+exports.xadd = xadd;
+async function isSeen(messageId) {
+    const result = await exports.redis.sismember('of:seen_messages', messageId);
+    return result === 1;
+}
+exports.isSeen = isSeen;
+async function markSeen(messageId) {
+    await exports.redis.sadd('of:seen_messages', messageId);
+}
+exports.markSeen = markSeen;
+//# sourceMappingURL=redis.js.map

package/dist/redis.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis.js","sourceRoot":"","sources":["../src/redis.ts"],"names":[],"mappings":";;;;;;AAAA,sDAA4B;AAE5B,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,wBAAwB,CAAC;AAEvD,QAAA,KAAK,GAAG,IAAI,iBAAK,CAAC,SAAS,EAAE;IACxC,WAAW,EAAE,IAAI;IACjB,oBAAoB,EAAE,CAAC;CACxB,CAAC,CAAC;AAEH,aAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;IAC/B,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,gBAAgB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;AAC3E,CAAC,CAAC,CAAC;AAEI,KAAK,UAAU,YAAY;IAChC,MAAM,aAAK,CAAC,OAAO,EAAE,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,wBAAwB,SAAS,EAAE,CAAC,CAAC;AAC/E,CAAC;AAHD,oCAGC;AAEM,KAAK,UAAU,IAAI,CAAC,MAAc,EAAE,MAA8B;IACvE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,aAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;AAC1C,CAAC;AAND,oBAMC;AAEM,KAAK,UAAU,MAAM,CAAC,SAAiB;IAC5C,MAAM,MAAM,GAAG,MAAM,aAAK,CAAC,SAAS,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;IACpE,OAAO,MAAM,KAAK,CAAC,CAAC;AACtB,CAAC;AAHD,wBAGC;AAEM,KAAK,UAAU,QAAQ,CAAC,SAAiB;IAC9C,MAAM,aAAK,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;AAClD,CAAC;AAFD,4BAEC"}

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@gonzih/of-scraper",
+  "version": "1.0.0",
+  "description": "OnlyFans scraper that feeds messages, profiles, and financials into Redis Streams",
+  "main": "dist/index.js",
+  "bin": {
+    "@gonzih/of-scraper": "dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "dev": "ts-node src/index.ts"
+  },
+  "keywords": [],
+  "author": "gonzih",
+  "license": "MIT",
+  "dependencies": {
+    "ioredis": "5.3.2",
+    "puppeteer": "25.0.4",
+    "puppeteer-extra": "3.3.6",
+    "puppeteer-extra-plugin-stealth": "2.11.2"
+  },
+  "devDependencies": {
+    "@types/node": "20.11.5",
+    "typescript": "5.3.3"
+  }
+}

package/src/browser.ts

@@ -0,0 +1,134 @@
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import type { Browser, Page } from 'puppeteer';
+import type { CookieParam } from 'puppeteer-core';
+import * as fs from 'fs';
+import * as path from 'path';
+
+puppeteer.use(StealthPlugin());
+
+const SESSION_FILE = process.env.SESSION_FILE ?? './session.json';
+const HEADLESS = process.env.HEADLESS !== 'false';
+
+interface SessionData {
+  cookies: CookieParam[];
+  localStorage: Record<string, string>;
+}
+
+export async function launchBrowser(): Promise<Browser> {
+  const browser = await puppeteer.launch({
+    headless: HEADLESS,
+    args: [
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      '--disable-blink-features=AutomationControlled',
+    ],
+  });
+  console.log(`[${new Date().toISOString()}] Browser launched (headless=${HEADLESS})`);
+  return browser;
+}
+
+export async function saveSession(page: Page): Promise<void> {
+  const cookies = await page.cookies();
+  const localStorage: Record<string, string> = await page.evaluate(() => {
+    const data: Record<string, string> = {};
+    for (let i = 0; i < window.localStorage.length; i++) {
+      const key = window.localStorage.key(i);
+      if (key !== null) {
+        data[key] = window.localStorage.getItem(key) ?? '';
+      }
+    }
+    return data;
+  });
+  const session: SessionData = { cookies, localStorage };
+  const sessionPath = path.resolve(SESSION_FILE);
+  fs.writeFileSync(sessionPath, JSON.stringify(session, null, 2));
+  console.log(`[${new Date().toISOString()}] Session saved to ${sessionPath}`);
+}
+
+export async function restoreSession(page: Page): Promise<void> {
+  const sessionPath = path.resolve(SESSION_FILE);
+  if (!fs.existsSync(sessionPath)) {
+    console.log(`[${new Date().toISOString()}] No session file found at ${sessionPath}`);
+    return;
+  }
+  const raw = fs.readFileSync(sessionPath, 'utf-8');
+  const session: SessionData = JSON.parse(raw);
+
+  await page.goto('https://onlyfans.com', { waitUntil: 'domcontentloaded' });
+  await page.setCookie(...session.cookies);
+
+  await page.evaluate((lsData: Record<string, string>) => {
+    for (const [key, value] of Object.entries(lsData)) {
+      window.localStorage.setItem(key, value);
+    }
+  }, session.localStorage);
+
+  console.log(`[${new Date().toISOString()}] Session restored from ${sessionPath}`);
+}
+
+export async function sessionFileExists(): Promise<boolean> {
+  return fs.existsSync(path.resolve(SESSION_FILE));
+}
+
+export async function ensureLoggedIn(browser: Browser): Promise<Page> {
+  const page = await browser.newPage();
+  await page.setViewport({ width: 1280, height: 900 });
+  await page.setUserAgent(
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+  );
+
+  if (await sessionFileExists()) {
+    console.log(`[${new Date().toISOString()}] Restoring existing session...`);
+    await restoreSession(page);
+    await page.goto('https://onlyfans.com/my/chats', { waitUntil: 'networkidle2', timeout: 30000 });
+
+    const isLoggedIn = await checkLoggedIn(page);
+    if (isLoggedIn) {
+      console.log(`[${new Date().toISOString()}] Session valid — logged in`);
+      return page;
+    }
+    console.log(`[${new Date().toISOString()}] Session invalid or expired`);
+  }
+
+  // Need to log in manually
+  await page.goto('https://onlyfans.com/login', { waitUntil: 'networkidle2', timeout: 30000 });
+  console.log(`[${new Date().toISOString()}] Waiting for manual login at https://onlyfans.com/login ...`);
+
+  // Wait for navigation away from /login (indicates successful login)
+  await page.waitForFunction(
+    () => !window.location.href.includes('/login'),
+    { timeout: 300000 } // 5 minute timeout for human to log in
+  );
+
+  console.log(`[${new Date().toISOString()}] Login detected — saving session`);
+  await saveSession(page);
+
+  return page;
+}
+
+async function checkLoggedIn(page: Page): Promise<boolean> {
+  try {
+    const result = await page.evaluate(() => {
+      return (
+        document.querySelector('.b-header__user') !== null ||
+        document.querySelector('[data-type="messages"]') !== null ||
+        document.querySelector('.g-header__left .b-ddmenu') !== null ||
+        document.querySelector('.b-chats') !== null ||
+        document.querySelector('.l-sidebar__user') !== null
+      );
+    });
+    return result;
+  } catch {
+    return false;
+  }
+}
+
+export async function takeErrorScreenshot(page: Page): Promise<void> {
+  try {
+    await page.screenshot({ path: '/tmp/of_login_required.png', fullPage: true });
+    console.log(`[${new Date().toISOString()}] Screenshot saved to /tmp/of_login_required.png`);
+  } catch (err) {
+    console.error(`[${new Date().toISOString()}] Failed to take screenshot:`, err);
+  }
+}

package/src/financials.ts

@@ -0,0 +1,104 @@
+import type { Page } from 'puppeteer';
+
+export interface Financials {
+  username: string;
+  totalTips: string;
+  totalPurchases: string;
+  subscriptionStatus: string;
+  lastPaymentDate: string;
+  lifetimeValue: string;
+  updatedAt: string;
+}
+
+export async function scrapeFinancials(page: Page, username: string, threadId: string): Promise<Financials | null> {
+  console.log(`[${new Date().toISOString()}] Fetching financials for @${username} (thread ${threadId})`);
+
+  const chatUrl = `https://onlyfans.com/my/chats/chat/${threadId}`;
+  try {
+    await page.goto(chatUrl, { waitUntil: 'networkidle2', timeout: 30000 });
+  } catch (err) {
+    console.error(`[${new Date().toISOString()}] Failed to navigate to thread for financials:`, err);
+    return null;
+  }
+
+  // Try to open the statistics/info panel
+  try {
+    const statsBtnSelectors = [
+      'button[class*="statistic"]',
+      'button[class*="stat"]',
+      '[data-type="stats"]',
+      'button[title*="Statistic"]',
+      'button[aria-label*="statistic"]',
+      '.b-chat__info-btn',
+      '[class*="infoBtn"]',
+    ];
+    for (const sel of statsBtnSelectors) {
+      const btn = await page.$(sel);
+      if (btn) {
+        await btn.click();
+        await new Promise((r) => setTimeout(r, 1500));
+        break;
+      }
+    }
+  } catch {
+    // Stats panel may already be visible or not available
+  }
+
+  const financials = await page.evaluate((uname: string): Financials => {
+    const getText = (selectors: string[]): string => {
+      for (const sel of selectors) {
+        const el = document.querySelector(sel);
+        if (el?.textContent?.trim()) return el.textContent.trim();
+      }
+      return '0';
+    };
+
+    const totalTips = getText([
+      '[class*="tips"] [class*="amount"]',
+      '[class*="tip"] [class*="total"]',
+      '[data-type="tips"]',
+      '.b-stat__tips',
+    ]);
+
+    const totalPurchases = getText([
+      '[class*="purchase"] [class*="amount"]',
+      '[class*="purchase"] [class*="total"]',
+      '[data-type="purchase"]',
+      '.b-stat__purchases',
+    ]);
+
+    const subscriptionStatus = getText([
+      '[class*="subscription"] [class*="status"]',
+      '[class*="subscriptionStatus"]',
+      '[data-type="subscriptionStatus"]',
+    ]) || 'unknown';
+
+    const lastPaymentDate = getText([
+      '[class*="lastPayment"]',
+      '[class*="last-payment"]',
+      '[data-type="lastPayment"]',
+    ]);
+
+    const lifetimeValue = getText([
+      '[class*="lifetimeValue"]',
+      '[class*="lifetime"]',
+      '[class*="totalSpent"]',
+      '[data-type="lifetimeValue"]',
+    ]);
+
+    return {
+      username: uname,
+      totalTips,
+      totalPurchases,
+      subscriptionStatus,
+      lastPaymentDate,
+      lifetimeValue,
+      updatedAt: new Date().toISOString(),
+    };
+  }, username);
+
+  console.log(
+    `[${new Date().toISOString()}] Financials for @${username}: tips=${financials.totalTips}, purchases=${financials.totalPurchases}`
+  );
+  return financials;
+}