commentscraper 1.0.0

package/core/scrapers/notion.js

@@ -0,0 +1,256 @@
+import { notionRichTextToPlain } from '../utils.js';
+
+/**
+ * Extract the Notion page ID from a URL.
+ * @param {string} url
+ * @returns {string|null} Page ID in UUID format or null
+ */
+export function extractNotionPageId(url) {
+  try {
+    const urlObj = new URL(url);
+    const path = urlObj.pathname;
+    const match = path.match(/([a-f0-9]{32})/i) || path.match(/([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})/i);
+    if (!match) return null;
+    const raw = match[1].replace(/-/g, '');
+    return `${raw.slice(0, 8)}-${raw.slice(8, 12)}-${raw.slice(12, 16)}-${raw.slice(16, 20)}-${raw.slice(20)}`;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Get the API base URL from a Notion page URL.
+ */
+export function getNotionApiBase(url) {
+  try {
+    const urlObj = new URL(url);
+    return `${urlObj.protocol}//${urlObj.host}`;
+  } catch {
+    return 'https://www.notion.so';
+  }
+}
+
+/**
+ * Fetch Notion page data using the unofficial loadPageChunk API.
+ * @param {string} url - Notion page URL
+ * @param {{ progress: { send: function } }} opts
+ */
+export async function fetchNotionData(url, { progress }) {
+  try {
+    const pageId = extractNotionPageId(url);
+    if (!pageId) return { success: false, error: 'Could not extract Notion page ID from URL' };
+
+    const apiBase = getNotionApiBase(url);
+    progress.send('Fetching Notion page data...', 10);
+
+    const chunkRes = await fetch(`${apiBase}/api/v3/loadPageChunk`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        pageId,
+        limit: 100,
+        cursor: { stack: [] },
+        chunkNumber: 0,
+        verticalColumns: false,
+      }),
+    });
+
+    if (!chunkRes.ok) {
+      return { success: false, error: `Notion API returned ${chunkRes.status}` };
+    }
+
+    const chunk = await chunkRes.json();
+    const recordMap = chunk.recordMap || {};
+    const blocks = recordMap.block || {};
+
+    const rootBlock = blocks[pageId]?.value;
+    if (!rootBlock) {
+      return { success: false, error: 'Page not found in Notion response' };
+    }
+
+    progress.send('Parsing Notion data...', 40);
+
+    const isCollection = rootBlock.type === 'collection_view' || rootBlock.type === 'collection_view_page';
+
+    if (isCollection) {
+      return await fetchNotionCollection(apiBase, rootBlock, recordMap, url, { progress });
+    }
+
+    return extractNotionPageBlocks(rootBlock, blocks, url, { progress });
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+}
+
+/**
+ * Fetch and parse a Notion collection (database view).
+ */
+export async function fetchNotionCollection(apiBase, rootBlock, recordMap, url, { progress }) {
+  try {
+    const collectionId = rootBlock.collection_id;
+    const viewIds = rootBlock.view_ids || [];
+
+    if (!collectionId || viewIds.length === 0) {
+      return { success: false, error: 'No collection or view found on this page' };
+    }
+
+    progress.send('Fetching Notion database...', 50);
+
+    const queryRes = await fetch(`${apiBase}/api/v3/queryCollection`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        collection: { id: collectionId },
+        collectionView: { id: viewIds[0] },
+        loader: {
+          type: 'reducer',
+          reducers: {
+            collection_group_results: {
+              type: 'results',
+              limit: 999,
+            },
+          },
+          searchQuery: '',
+          userTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+        },
+      }),
+    });
+
+    if (!queryRes.ok) {
+      return { success: false, error: `Notion queryCollection returned ${queryRes.status}` };
+    }
+
+    const queryData = await queryRes.json();
+    const collectionBlocks = queryData.recordMap?.block || {};
+    const collectionData = queryData.recordMap?.collection || {};
+
+    const collection = collectionData[collectionId]?.value;
+    const schema = collection?.schema || {};
+    const collectionName = collection?.name?.[0]?.[0] || 'Untitled Database';
+
+    const resultIds = queryData.result?.reducerResults?.collection_group_results?.blockIds || [];
+
+    progress.send(`Found ${resultIds.length} rows, parsing...`, 70);
+
+    const comments = [];
+    for (const rowId of resultIds) {
+      const rowBlock = collectionBlocks[rowId]?.value;
+      if (!rowBlock || !rowBlock.properties) continue;
+
+      const parts = [];
+      let rowTitle = '';
+      for (const [propId, propDef] of Object.entries(schema)) {
+        const rawValue = rowBlock.properties[propId];
+        if (!rawValue) continue;
+        const textValue = notionRichTextToPlain(rawValue);
+        if (!textValue) continue;
+
+        if (propDef.type === 'title') {
+          rowTitle = textValue;
+          parts.unshift(`${textValue}`);
+        } else {
+          parts.push(`${propDef.name}: ${textValue}`);
+        }
+      }
+
+      if (parts.length === 0) continue;
+
+      const lastEdited = rowBlock.last_edited_time
+        ? new Date(rowBlock.last_edited_time).toISOString()
+        : '';
+
+      const rowPageId = rowId.replace(/-/g, '');
+      comments.push({
+        text: parts.join(' | '),
+        author: '',
+        timestamp: lastEdited,
+        permalink: `${url.split('?')[0].replace(/\/$/, '')}/${rowTitle ? encodeURIComponent(rowTitle.replace(/\s+/g, '-')) + '-' : ''}${rowPageId}`,
+        links: [],
+        score: '',
+        depth: 0,
+      });
+    }
+
+    progress.send(`Loaded ${comments.length} rows`, 95);
+
+    return {
+      success: true,
+      comments,
+      post: {
+        title: collectionName,
+        body: '',
+        url,
+        subreddit: 'Notion',
+      },
+      method: 'json',
+    };
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+}
+
+/**
+ * Extract text blocks from a regular Notion page.
+ */
+export function extractNotionPageBlocks(rootBlock, blocks, url, { progress }) {
+  const pageTitle = notionRichTextToPlain(rootBlock.properties?.title) || 'Untitled';
+  const comments = [];
+
+  const contentIds = rootBlock.content || [];
+  processNotionBlocks(contentIds, blocks, comments, 0, url);
+
+  progress.send(`Loaded ${comments.length} blocks`, 95);
+
+  return {
+    success: true,
+    comments,
+    post: {
+      title: pageTitle,
+      body: '',
+      url,
+      subreddit: 'Notion',
+    },
+    method: 'json',
+  };
+}
+
+/**
+ * Recursively process Notion blocks into flat comment array.
+ */
+export function processNotionBlocks(blockIds, allBlocks, out, depth, pageUrl) {
+  for (const blockId of blockIds) {
+    const block = allBlocks[blockId]?.value;
+    if (!block) continue;
+
+    const type = block.type;
+
+    if (['page', 'collection_view', 'collection_view_page', 'divider', 'table_of_contents'].includes(type)) continue;
+
+    const text = notionRichTextToPlain(block.properties?.title);
+    if (text) {
+      const prefix = type === 'header' ? '# '
+        : type === 'sub_header' ? '## '
+        : type === 'sub_sub_header' ? '### '
+        : type === 'bulleted_list' ? '- '
+        : type === 'numbered_list' ? '\u2022 '
+        : type === 'to_do' ? (block.properties?.checked?.[0]?.[0] === 'Yes' ? '\u2611 ' : '\u2610 ')
+        : type === 'quote' ? '> '
+        : type === 'callout' ? '\ud83d\udca1 '
+        : '';
+
+      out.push({
+        text: prefix + text,
+        author: '',
+        timestamp: '',
+        permalink: pageUrl,
+        links: [],
+        score: '',
+        depth,
+      });
+    }
+
+    if (block.content && block.content.length > 0) {
+      processNotionBlocks(block.content, allBlocks, out, depth + 1, pageUrl);
+    }
+  }
+}

package/core/scrapers/producthunt.js

@@ -0,0 +1,46 @@
+import { SUPABASE_URL } from '../config.js';
+
+/**
+ * Fetch Product Hunt post comments via Supabase Edge Function proxy.
+ * @param {string} url - PH post URL
+ * @param {{ progress: { send: function } }} opts
+ */
+export async function fetchProductHuntComments(url, { progress }) {
+  try {
+    const match = url.match(/producthunt\.com\/(?:posts|products)\/([^/?#]+)/);
+    if (!match) return { success: false, error: 'Could not extract product slug from URL.' };
+
+    const slug = match[1];
+    progress.send('Fetching Product Hunt comments...', 15);
+
+    const response = await fetch(`${SUPABASE_URL}/functions/v1/fetch-ph-comments`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ slug }),
+    });
+
+    const data = await response.json();
+
+    if (!response.ok || !data.success) {
+      return { success: false, error: data.error || `Edge Function error (${response.status})` };
+    }
+
+    const comments = (data.comments || []).map(c => ({
+      text: c.text || '',
+      author: c.author || '',
+      timestamp: c.timestamp || '',
+      permalink: c.commentId
+        ? `https://www.producthunt.com/posts/${slug}#comment-${c.commentId}`
+        : '',
+      links: [],
+      score: 'Hidden',
+      depth: c.depth || 0,
+    }));
+
+    progress.send(`Loaded ${comments.length} comments`, 95);
+
+    return { success: true, comments, post: data.post || null, method: 'json' };
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+}

package/core/scrapers/reddit-profile.js

@@ -0,0 +1,146 @@
+import { extractLinksFromMarkdown } from '../utils.js';
+
+const USER_AGENT = 'RedditCommentScraper/3.7.0';
+
+/**
+ * Fetch a Reddit user's profile activity (comments + posts).
+ * @param {string} url - Reddit profile URL
+ * @param {{ progress: { send: function } }} opts
+ */
+export async function fetchRedditProfile(url, { progress }) {
+  try {
+    const match = url.match(/\/u(?:ser)?\/([^/?#/]+)/i);
+    if (!match) return { success: false, error: 'Could not extract username from URL.' };
+
+    const username = match[1];
+    progress.send(`Fetching profile for u/${username}...`, 10);
+
+    const [commentsResult, postsResult] = await Promise.all([
+      fetchRedditProfilePages(username, 'comments', 10, 300, { progress }),
+      fetchRedditProfilePages(username, 'submitted', 10, 300, { progress }),
+    ]);
+
+    if (!commentsResult.success && !postsResult.success) {
+      return { success: false, error: commentsResult.error || postsResult.error };
+    }
+
+    const allItems = [
+      ...(commentsResult.items || []),
+      ...(postsResult.items || []),
+    ];
+
+    allItems.sort((a, b) => {
+      const ta = a.timestamp ? new Date(a.timestamp).getTime() : 0;
+      const tb = b.timestamp ? new Date(b.timestamp).getTime() : 0;
+      return tb - ta;
+    });
+
+    progress.send(`Loaded ${allItems.length} items from u/${username}`, 95);
+
+    const post = {
+      title: `u/${username}'s Profile`,
+      body: `${commentsResult.items?.length || 0} comments, ${postsResult.items?.length || 0} posts`,
+      url: `https://www.reddit.com/user/${username}`,
+      subreddit: 'Reddit Profile',
+      author: username,
+    };
+
+    return { success: true, comments: allItems, post, method: 'json' };
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+}
+
+/**
+ * Paginate one Reddit user endpoint (comments or submitted).
+ */
+export async function fetchRedditProfilePages(username, endpoint, maxPages = 10, delayMs = 300, { progress }) {
+  const items = [];
+  let after = null;
+
+  for (let page = 0; page < maxPages; page++) {
+    try {
+      if (page > 0) {
+        await new Promise(r => setTimeout(r, delayMs));
+      }
+
+      let fetchUrl = `https://www.reddit.com/user/${username}/${endpoint}.json?limit=100&raw_json=1&include_over_18=on`;
+      if (after) fetchUrl += `&after=${after}`;
+
+      const response = await fetch(fetchUrl, {
+        headers: { 'User-Agent': USER_AGENT }
+      });
+
+      if (!response.ok) {
+        if (response.status === 403) {
+          return { success: false, items: [], error: `u/${username}'s profile is private or suspended.` };
+        }
+        if (response.status === 404) {
+          return { success: false, items: [], error: `User u/${username} not found.` };
+        }
+        if (response.status === 429) {
+          return { success: false, items: [], error: 'Rate limited by Reddit. Please try again in a moment.' };
+        }
+        return { success: false, items: [], error: `Reddit returned status ${response.status}` };
+      }
+
+      const data = await response.json();
+      const children = data?.data?.children || [];
+
+      if (children.length === 0) break;
+
+      for (const child of children) {
+        const d = child.data;
+        if (!d) continue;
+
+        if (d.author === '[deleted]' || d.author === '[removed]') continue;
+
+        if (child.kind === 't1') {
+          if (d.body === '[deleted]' || d.body === '[removed]') continue;
+
+          const links = extractLinksFromMarkdown(d.body || '');
+          items.push({
+            text: d.body || '',
+            author: d.author || '',
+            timestamp: d.created_utc ? new Date(d.created_utc * 1000).toISOString() : '',
+            permalink: d.permalink ? `https://www.reddit.com${d.permalink}` : '',
+            links,
+            score: d.score ?? 'Hidden',
+            depth: 0,
+            subreddit: d.subreddit || '',
+            linkTitle: d.link_title || '',
+            type: 'comment',
+          });
+        } else if (child.kind === 't3') {
+          const postText = d.title || '';
+          const bodyText = d.selftext || '';
+          const fullText = bodyText ? `${postText}\n\n${bodyText}` : postText;
+          const links = extractLinksFromMarkdown(bodyText);
+
+          items.push({
+            text: fullText,
+            author: d.author || '',
+            timestamp: d.created_utc ? new Date(d.created_utc * 1000).toISOString() : '',
+            permalink: d.permalink ? `https://www.reddit.com${d.permalink}` : '',
+            links,
+            score: d.score ?? 'Hidden',
+            depth: 0,
+            subreddit: d.subreddit || '',
+            linkTitle: '',
+            type: 'post',
+          });
+        }
+      }
+
+      after = data?.data?.after;
+      if (!after) break;
+
+      const pct = 10 + Math.round(((page + 1) / maxPages) * 40) + (endpoint === 'submitted' ? 40 : 0);
+      progress.send(`Loading ${endpoint}... ${items.length} found`, Math.min(pct, 90));
+    } catch (error) {
+      break;
+    }
+  }
+
+  return { success: true, items };
+}

package/core/scrapers/reddit.js

@@ -0,0 +1,227 @@
+import { extractLinksFromMarkdown } from '../utils.js';
+
+const USER_AGENT = 'RedditCommentScraper/3.7.0';
+
+/**
+ * Fetch and parse Reddit thread data via JSON endpoint.
+ * @param {string} url - Reddit thread URL
+ * @param {{ progress: { send: function } }} opts
+ * @returns {Promise<Object>}
+ */
+export async function fetchRedditJSON(url, { progress }) {
+  try {
+    const urlObj = new URL(url);
+    let basePath = urlObj.pathname.replace(/\/comment\/\w+/, '').replace(/\/$/, '');
+    urlObj.pathname = basePath + '.json';
+    urlObj.searchParams.delete('context');
+    urlObj.searchParams.set('limit', '500');
+    const jsonUrl = urlObj.toString();
+
+    progress.send('Fetching comments...', 15);
+
+    const response = await fetch(jsonUrl, {
+      headers: { 'User-Agent': USER_AGENT }
+    });
+
+    if (!response.ok) {
+      if (response.status === 429) {
+        return { success: false, error: 'Rate limited by Reddit. Please try again in a moment.', rateLimited: true };
+      }
+      return { success: false, error: `Reddit returned status ${response.status}` };
+    }
+
+    const data = await response.json();
+    const postData = parsePostData(data[0]);
+    const linkId = data[0]?.data?.children?.[0]?.data?.name || '';
+    const { comments, moreObjects } = parseCommentsWithMore(data[1]?.data?.children || []);
+
+    if (moreObjects.length > 0 && linkId) {
+      progress.send(`Found ${comments.length} comments, loading more...`, 30);
+      const additionalComments = await fetchAllMoreComments(linkId, moreObjects, comments.length, { progress });
+      comments.push(...additionalComments);
+    }
+
+    progress.send(`Loaded ${comments.length} comments`, 95);
+
+    return {
+      success: true,
+      comments,
+      post: postData,
+      method: 'json'
+    };
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+}
+
+/**
+ * Fetch additional comments from "more" objects.
+ */
+export async function fetchAllMoreComments(linkId, moreObjects, initialCount = 0, { progress }) {
+  const allComments = [];
+
+  let allChildrenIds = [];
+  for (const more of moreObjects) {
+    if (more.children && more.children.length > 0) {
+      allChildrenIds.push(...more.children);
+    }
+  }
+
+  const BATCH_SIZE = 100;
+  const batches = [];
+  for (let i = 0; i < allChildrenIds.length; i += BATCH_SIZE) {
+    batches.push(allChildrenIds.slice(i, i + BATCH_SIZE));
+  }
+
+  for (let i = 0; i < batches.length; i++) {
+    const batch = batches[i];
+
+    try {
+      if (i > 0) {
+        await new Promise(resolve => setTimeout(resolve, 200));
+      }
+
+      const moreComments = await fetchMoreChildren(linkId, batch);
+      allComments.push(...moreComments);
+
+      const pct = 30 + Math.round((i + 1) / batches.length * 60);
+      const totalSoFar = initialCount + allComments.length;
+      progress.send(`Loading comments... ${totalSoFar} found`, pct);
+    } catch (error) {
+      // Continue with other batches even if one fails
+    }
+  }
+
+  return allComments;
+}
+
+/**
+ * Fetch a batch of comments using Reddit's /api/morechildren endpoint.
+ */
+export async function fetchMoreChildren(linkId, childrenIds) {
+  const url = 'https://www.reddit.com/api/morechildren.json';
+
+  const params = new URLSearchParams({
+    api_type: 'json',
+    link_id: linkId,
+    children: childrenIds.join(',')
+  });
+
+  const response = await fetch(`${url}?${params.toString()}`, {
+    headers: { 'User-Agent': USER_AGENT }
+  });
+
+  if (!response.ok) {
+    if (response.status === 429) {
+      throw new Error('Rate limited by Reddit');
+    }
+    throw new Error(`Reddit returned status ${response.status}`);
+  }
+
+  const data = await response.json();
+  const things = data?.json?.data?.things || [];
+
+  const comments = [];
+  for (const thing of things) {
+    if (thing.kind !== 't1') continue;
+
+    const comment = thing.data;
+    if (comment.author === '[deleted]' && comment.body === '[deleted]') continue;
+    if (comment.author === '[removed]' && comment.body === '[removed]') continue;
+
+    const permalink = comment.permalink
+      ? `https://www.reddit.com${comment.permalink}`
+      : '';
+
+    const links = extractLinksFromMarkdown(comment.body || '');
+
+    const timestamp = comment.created_utc
+      ? new Date(comment.created_utc * 1000).toISOString()
+      : '';
+
+    comments.push({
+      text: comment.body || '',
+      author: comment.author || '',
+      timestamp,
+      permalink,
+      links,
+      score: comment.score ?? 'Hidden',
+      depth: comment.depth || 0
+    });
+  }
+
+  return comments;
+}
+
+/**
+ * Parse comments and collect "more" objects for later fetching.
+ */
+export function parseCommentsWithMore(children, depth = 0) {
+  const comments = [];
+  const moreObjects = [];
+
+  for (const child of children) {
+    if (child.kind === 'more') {
+      if (child.data && child.data.children && child.data.children.length > 0) {
+        moreObjects.push({
+          children: child.data.children,
+          count: child.data.count || 0,
+          depth
+        });
+      }
+      continue;
+    }
+
+    if (child.kind !== 't1') continue;
+
+    const comment = child.data;
+    if (comment.author === '[deleted]' && comment.body === '[deleted]') continue;
+    if (comment.author === '[removed]' && comment.body === '[removed]') continue;
+
+    const permalink = comment.permalink
+      ? `https://www.reddit.com${comment.permalink}`
+      : '';
+
+    const links = extractLinksFromMarkdown(comment.body || '');
+
+    const timestamp = comment.created_utc
+      ? new Date(comment.created_utc * 1000).toISOString()
+      : '';
+
+    comments.push({
+      text: comment.body || '',
+      author: comment.author || '',
+      timestamp,
+      permalink,
+      links,
+      score: comment.score ?? 'Hidden',
+      depth
+    });
+
+    if (comment.replies && comment.replies.data && comment.replies.data.children) {
+      const nested = parseCommentsWithMore(comment.replies.data.children, depth + 1);
+      comments.push(...nested.comments);
+      moreObjects.push(...nested.moreObjects);
+    }
+  }
+
+  return { comments, moreObjects };
+}
+
+/**
+ * Parse post metadata from Reddit JSON.
+ */
+export function parsePostData(listing) {
+  const post = listing?.data?.children?.[0]?.data;
+  if (!post) return { title: '', body: '', url: '', subreddit: '' };
+
+  return {
+    title: post.title || '',
+    body: post.selftext || '',
+    url: `https://www.reddit.com${post.permalink}`,
+    subreddit: post.subreddit || '',
+    author: post.author || '',
+    score: post.score || 0,
+    created_utc: post.created_utc || 0
+  };
+}