commentscraper 1.0.0

package/SKILL.md

@@ -0,0 +1,112 @@
+---
+name: commentscraper
+description: |
+  Scrape comments, reviews, and discussions from Reddit threads, Hacker News
+  posts, Steam game reviews, Product Hunt comments, Notion pages, and Reddit
+  user profiles. Use when the user wants to extract comments from a URL, pull
+  Reddit thread data, grab HN discussion, get Steam reviews, scrape user
+  activity, fetch post comments, do market research from Reddit, analyze
+  customer sentiment, research what people say about a product, extract
+  pain points from discussions, or gather competitive intelligence from
+  forums. Outputs structured JSON with comment text, authors, scores,
+  timestamps, and thread depth. Do NOT trigger for general web scraping,
+  crawling documentation, or fetching arbitrary web pages.
+license: proprietary
+compatibility:
+  - Claude Code
+  - OpenClaw
+  - Cursor
+  - Codex
+  - Gemini CLI
+metadata:
+  author: DDTechSolution
+  version: "1.0"
+allowed-tools: Bash(commentscraper*)
+---
+
+# CommentScraper CLI
+
+Extracts comments and reviews from 6 platforms via their public APIs. No browser needed.
+
+## When to Use
+
+- User shares a Reddit, HN, Steam, Product Hunt, or Notion URL and wants comments
+- User asks to research what people say about a topic, product, or company
+- User wants pain points, sentiment, or themes from online discussions
+- User needs a Reddit user's posting history
+
+## When NOT to Use
+
+- General web scraping or crawling (use Firecrawl instead)
+- Fetching page content that isn't comments/reviews
+- Platforms not listed below
+
+## Setup
+
+```bash
+npm install -g commentscraper
+commentscraper login
+```
+
+## Commands
+
+```bash
+# Scrape comments from any supported URL
+commentscraper scrape "<url>" --quiet
+
+# With format and output options
+commentscraper scrape "<url>" --format json|csv|text --output file.json --quiet
+
+# Check auth status
+commentscraper whoami
+```
+
+Always use `--quiet` when piping output to avoid progress text in stdout.
+
+## Supported Platforms
+
+| Platform | Example URL |
+|----------|------------|
+| Reddit | `https://reddit.com/r/startups/comments/abc123/post/` |
+| Reddit Profile | `https://reddit.com/user/spez` |
+| Hacker News | `https://news.ycombinator.com/item?id=12345` |
+| Steam | `https://store.steampowered.com/app/730/` |
+| Product Hunt | `https://producthunt.com/posts/some-product` |
+| Notion | `https://example.notion.site/page-id` |
+
+## Output Schema
+
+JSON to stdout:
+```json
+{
+  "scrapedAt": "ISO 8601",
+  "source": "reddit|hackernews|steam|producthunt|notion|redditprofile",
+  "url": "original URL",
+  "count": 342,
+  "post": { "title": "...", "body": "...", "url": "...", "subreddit": "..." },
+  "comments": [{
+    "text": "Comment body",
+    "author": "Username",
+    "timestamp": "ISO 8601",
+    "permalink": "Direct link",
+    "score": 42,
+    "depth": 0
+  }]
+}
+```
+
+## Examples
+
+```bash
+# Scrape Reddit thread and analyze themes
+commentscraper scrape "https://reddit.com/r/startups/comments/abc123/post/" --quiet
+
+# Export HN discussion to file
+commentscraper scrape "https://news.ycombinator.com/item?id=41967900" --output hn.json --quiet
+
+# Steam reviews as CSV
+commentscraper scrape "https://store.steampowered.com/app/730/" --format csv --quiet
+
+# Research a Reddit user
+commentscraper scrape "https://reddit.com/user/spez" --quiet
+```

package/bin/commentscraper.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+
+import { parseArgs } from 'node:util';
+
+const args = process.argv.slice(2);
+const command = args[0];
+
+if (!command || command === '--help' || command === '-h') {
+  console.log(`
+commentscraper - Scrape comments and reviews for AI agents
+
+USAGE
+  commentscraper <command> [options]
+
+COMMANDS
+  scrape <url>    Scrape comments from a URL
+  login           Authenticate with your account
+  logout          Remove stored credentials
+  whoami          Show current user and plan
+  platforms       List supported platforms
+
+OPTIONS (scrape)
+  --format <json|csv|text>   Output format (default: json)
+  --output <file>            Write to file instead of stdout
+  --quiet                    Suppress progress output
+
+EXAMPLES
+  commentscraper scrape "https://reddit.com/r/programming/comments/abc123/post/"
+  commentscraper scrape "https://news.ycombinator.com/item?id=12345" --format csv
+  commentscraper scrape "https://store.steampowered.com/app/730/" --output reviews.json
+
+SUBSCRIPTION
+  CLI requires a paid subscription (Pro or All Access).
+  Pro ($4.99/mo): Reddit only
+  All Access ($19.99/mo): All platforms
+  https://redditcommentscraper.com/pricing
+`);
+  process.exit(0);
+}
+
+if (command === 'platforms') {
+  console.log(`
+Supported platforms:
+  Reddit           reddit.com/r/.../comments/...       Pro+
+  Reddit Profile   reddit.com/user/...                 Pro+
+  Hacker News      news.ycombinator.com/item?id=...    All Access
+  Steam            store.steampowered.com/app/...      All Access
+  Product Hunt     producthunt.com/posts/...           All Access
+  Notion           *.notion.site/...                   All Access
+`);
+  process.exit(0);
+}
+
+if (command === 'login') {
+  const { loginCommand } = await import('../commands/login.js');
+  await loginCommand();
+} else if (command === 'logout') {
+  const { logoutCommand } = await import('../commands/logout.js');
+  logoutCommand();
+} else if (command === 'whoami') {
+  const { whoamiCommand } = await import('../commands/whoami.js');
+  await whoamiCommand();
+} else if (command === 'scrape') {
+  const url = args[1];
+  if (!url) {
+    console.error('Usage: commentscraper scrape <url> [--format json|csv|text] [--output file] [--quiet]');
+    process.exit(1);
+  }
+
+  const { values } = parseArgs({
+    args: args.slice(2),
+    options: {
+      format: { type: 'string', default: 'json' },
+      output: { type: 'string', short: 'o' },
+      quiet: { type: 'boolean', default: false, short: 'q' },
+    },
+    strict: false,
+  });
+
+  const { scrapeCommand } = await import('../commands/scrape.js');
+  await scrapeCommand(url, values);
+} else {
+  console.error(`Unknown command: ${command}`);
+  console.error('Run: commentscraper --help');
+  process.exit(1);
+}

package/commands/login.js

@@ -0,0 +1,47 @@
+import { createDeviceCode, pollDeviceCode } from '../lib/auth.js';
+import { writeConfig } from '../lib/config.js';
+
+export async function loginCommand() {
+  try {
+    console.log('Requesting device code...');
+    const { device_code, user_code, verify_url } = await createDeviceCode();
+
+    const url = verify_url || 'https://redditcommentscraper.com/cli-auth';
+    console.log();
+    console.log(`  Open this URL:  ${url}`);
+    console.log(`  Enter code:     ${user_code}`);
+    console.log();
+    console.log('Waiting for authorization...');
+
+    // Poll every 5 seconds for up to 10 minutes
+    const MAX_ATTEMPTS = 120;
+    for (let i = 0; i < MAX_ATTEMPTS; i++) {
+      await new Promise(r => setTimeout(r, 5000));
+
+      const result = await pollDeviceCode(device_code);
+
+      if (result.status === 'approved') {
+        writeConfig({
+          cli_token: result.cli_token,
+          email: result.email,
+          plan: result.plan,
+        });
+        console.log(`Logged in as ${result.email} (${result.plan} plan)`);
+        return;
+      }
+
+      if (result.status === 'expired') {
+        console.error('Device code expired. Please try again.');
+        process.exit(1);
+      }
+
+      // Still pending, keep polling
+    }
+
+    console.error('Timed out waiting for authorization.');
+    process.exit(1);
+  } catch (error) {
+    console.error(`Login failed: ${error.message}`);
+    process.exit(1);
+  }
+}

package/commands/logout.js

@@ -0,0 +1,12 @@
+import { deleteConfig, readConfig } from '../lib/config.js';
+
+export function logoutCommand() {
+  const config = readConfig();
+  if (!config) {
+    console.log('Not logged in.');
+    return;
+  }
+
+  deleteConfig();
+  console.log(`Logged out (was ${config.email}).`);
+}

package/commands/scrape.js

@@ -0,0 +1,108 @@
+import { writeFileSync } from 'node:fs';
+import { scrapeUrl } from '../core/scrapers/index.js';
+import { getPlatformFromUrl, platformRequiresAllAccess, CLI_PLATFORMS } from '../core/platform-detect.js';
+import { verifyToken } from '../lib/auth.js';
+import { createConsoleProgress } from '../lib/cli-progress.js';
+
+/**
+ * @param {string} url
+ * @param {{ format: string, output: string, quiet: boolean }} opts
+ */
+export async function scrapeCommand(url, opts = {}) {
+  const { format = 'json', output, quiet = false } = opts;
+
+  // 1. Verify auth + plan
+  const auth = await verifyToken();
+  if (!auth.valid) {
+    console.error(`Error: ${auth.error}`);
+    process.exit(1);
+  }
+
+  // 2. Check platform support
+  const platform = getPlatformFromUrl(url);
+  if (!platform) {
+    const supported = Object.values(CLI_PLATFORMS).map(p => p.name).join(', ');
+    console.error(`Unsupported URL. Supported platforms: ${supported}`);
+    process.exit(1);
+  }
+
+  // 3. Check plan access
+  if (auth.plan === 'free') {
+    console.error('CLI requires a Pro or All Access subscription.');
+    console.error('Upgrade at https://redditcommentscraper.com/pricing');
+    process.exit(1);
+  }
+
+  if (platformRequiresAllAccess(platform) && auth.plan === 'pro') {
+    console.error(`${CLI_PLATFORMS[platform].name} requires an All Access subscription.`);
+    console.error('Pro plan supports Reddit only. Upgrade at https://redditcommentscraper.com/pricing');
+    process.exit(1);
+  }
+
+  // 4. Scrape
+  const progress = createConsoleProgress(quiet);
+  const result = await scrapeUrl(url, { progress });
+
+  if (!result.success) {
+    console.error(`Scrape failed: ${result.error}`);
+    process.exit(1);
+  }
+
+  // 5. Format output
+  let formatted;
+  if (format === 'csv') {
+    formatted = toCSV(result);
+  } else if (format === 'text') {
+    formatted = toText(result);
+  } else {
+    formatted = JSON.stringify({
+      scrapedAt: new Date().toISOString(),
+      source: result.platform,
+      url,
+      count: result.comments.length,
+      post: result.post,
+      comments: result.comments,
+    }, null, 2);
+  }
+
+  // 6. Output
+  if (output) {
+    writeFileSync(output, formatted, 'utf8');
+    if (!quiet) console.error(`Written to ${output}`);
+  } else {
+    process.stdout.write(formatted + '\n');
+  }
+}
+
+function toCSV(result) {
+  const escape = (s) => {
+    if (typeof s !== 'string') s = String(s ?? '');
+    if (s.includes('"') || s.includes(',') || s.includes('\n')) {
+      return '"' + s.replace(/"/g, '""') + '"';
+    }
+    return s;
+  };
+
+  const headers = ['text', 'author', 'timestamp', 'permalink', 'score', 'depth'];
+  const lines = [headers.join(',')];
+
+  for (const c of result.comments) {
+    lines.push(headers.map(h => escape(c[h])).join(','));
+  }
+
+  return lines.join('\n');
+}
+
+function toText(result) {
+  const lines = [];
+  if (result.post?.title) lines.push(`# ${result.post.title}`, '');
+
+  for (const c of result.comments) {
+    const indent = '  '.repeat(c.depth || 0);
+    const meta = [c.author, c.score !== 'Hidden' ? `${c.score} pts` : null].filter(Boolean).join(' | ');
+    if (meta) lines.push(`${indent}[${meta}]`);
+    lines.push(`${indent}${c.text}`, '');
+  }
+
+  return lines.join('\n');
+}

package/commands/whoami.js

@@ -0,0 +1,21 @@
+import { verifyToken } from '../lib/auth.js';
+import { readConfig } from '../lib/config.js';
+
+export async function whoamiCommand() {
+  const config = readConfig();
+  if (!config) {
+    console.log('Not logged in. Run: commentscraper login');
+    process.exit(1);
+  }
+
+  const result = await verifyToken();
+
+  if (!result.valid) {
+    console.error(`Token invalid: ${result.error}`);
+    console.error('Run: commentscraper login');
+    process.exit(1);
+  }
+
+  console.log(`Email: ${result.email}`);
+  console.log(`Plan:  ${result.plan}`);
+}

package/core/config.js

@@ -0,0 +1,2 @@
+export const SUPABASE_URL = 'https://qaottociftsaoepttyhm.supabase.co';
+export const SUPABASE_ANON_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InFhb3R0b2NpZnRzYW9lcHR0eWhtIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NjM1MDc2MjIsImV4cCI6MjA3OTA4MzYyMn0.2YQ8YWXvuLHejlAU1WeMJTvMivrTLifYmAGtOOx53WM';

package/core/package.json

@@ -0,0 +1 @@
+{ "type": "module" }

package/core/platform-detect.js

@@ -0,0 +1,37 @@
+/**
+ * Platforms supported by the CLI (API-based, no browser needed).
+ */
+export const CLI_PLATFORMS = {
+  reddit: { name: 'Reddit', paidOnly: false },
+  redditprofile: { name: 'Reddit Profile', paidOnly: false },
+  hackernews: { name: 'Hacker News', paidOnly: true },
+  steam: { name: 'Steam', paidOnly: true },
+  producthunt: { name: 'Product Hunt', paidOnly: true },
+  notion: { name: 'Notion', paidOnly: true },
+};
+
+/**
+ * Determine which platform a URL belongs to (API-capable subset).
+ * @param {string} url
+ * @returns {string|null} Platform key or null
+ */
+export function getPlatformFromUrl(url) {
+  if (!url) return null;
+  if (url.includes('reddit.com')
+    && /\/u(?:ser)?\/[^/?#/]+(?:\/(overview|comments|submitted))?\/?(\?|#|$)/i.test(url)
+    && !/\/comments\/[a-z0-9]{5,}/i.test(url)) return 'redditprofile';
+  if (url.includes('reddit.com')) return 'reddit';
+  if (url.includes('news.ycombinator.com')) return 'hackernews';
+  if (url.includes('store.steampowered.com')) return 'steam';
+  if (url.includes('producthunt.com')) return 'producthunt';
+  if (url.includes('notion.site') || url.includes('notion.so')) return 'notion';
+  return null;
+}
+
+/**
+ * Check if a platform requires All Access plan.
+ * Reddit and Reddit Profile work with Pro. Everything else needs All Access.
+ */
+export function platformRequiresAllAccess(platform) {
+  return CLI_PLATFORMS[platform]?.paidOnly === true;
+}

package/core/progress.js

@@ -0,0 +1,11 @@
+/**
+ * Progress reporter factory.
+ * @param {function(string, number): void} callback - receives (message, percent)
+ * @returns {{ send: function(string, number): void }}
+ */
+export function createProgressReporter(callback) {
+  return { send: (msg, pct) => callback(msg, pct) };
+}
+
+/** No-op progress reporter for silent usage */
+export const silentProgress = { send: () => {} };

package/core/scrapers/hackernews.js

@@ -0,0 +1,99 @@
+import { hnHtmlToText } from '../utils.js';
+
+const HN_API = 'https://hacker-news.firebaseio.com/v0';
+
+/**
+ * Fetch and parse a Hacker News thread via the public Firebase API.
+ * @param {string} url - HN thread URL
+ * @param {{ progress: { send: function } }} opts
+ */
+export async function fetchHackerNewsJSON(url, { progress }) {
+  try {
+    const match = url.match(/[?&]id=(\d+)/);
+    if (!match) return { success: false, error: 'Invalid Hacker News URL' };
+
+    const storyId = match[1];
+    progress.send('Fetching story data...', 10);
+
+    const story = await fetchHNItem(storyId);
+    if (!story) return { success: false, error: 'Story not found' };
+
+    const post = {
+      title: story.title || '',
+      body: hnHtmlToText(story.text || ''),
+      url: story.url || `https://news.ycombinator.com/item?id=${storyId}`,
+      subreddit: 'Hacker News',
+    };
+
+    if (!story.kids || story.kids.length === 0) {
+      return { success: true, comments: [], post, method: 'json' };
+    }
+
+    progress.send(`Loading ${story.descendants || '?'} comments...`, 20);
+
+    const comments = [];
+    await fetchHNCommentTree(story.kids, 0, comments, story.descendants || 0, { progress });
+
+    progress.send(`Loaded ${comments.length} comments`, 95);
+
+    return { success: true, comments, post, method: 'json' };
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+}
+
+/**
+ * Fetch a single HN item by ID.
+ */
+export async function fetchHNItem(id) {
+  const res = await fetch(`${HN_API}/item/${id}.json`);
+  if (!res.ok) return null;
+  return res.json();
+}
+
+/**
+ * Recursively fetch a comment tree in batches.
+ */
+export async function fetchHNCommentTree(kidIds, depth, out, totalExpected, { progress }) {
+  const BATCH = 25;
+
+  for (let i = 0; i < kidIds.length; i += BATCH) {
+    const batch = kidIds.slice(i, i + BATCH);
+
+    if (i > 0) await new Promise(r => setTimeout(r, 100));
+
+    const items = await Promise.all(batch.map(id => fetchHNItem(id)));
+
+    for (const item of items) {
+      if (!item || item.dead || item.deleted) continue;
+      if (item.type !== 'comment') continue;
+
+      const text = hnHtmlToText(item.text || '');
+      if (!text) continue;
+
+      const links = [];
+      const linkRx = /href="([^"]+)"/g;
+      let m;
+      while ((m = linkRx.exec(item.text || '')) !== null) {
+        if (!m[1].startsWith('javascript:')) links.push(m[1]);
+      }
+
+      out.push({
+        text,
+        author: item.by || '',
+        timestamp: item.time ? new Date(item.time * 1000).toISOString() : '',
+        permalink: `https://news.ycombinator.com/item?id=${item.id}`,
+        links,
+        score: 'Hidden',
+        depth,
+      });
+
+      if (item.kids && item.kids.length > 0) {
+        await fetchHNCommentTree(item.kids, depth + 1, out, totalExpected, { progress });
+      }
+    }
+
+    const pct = 20 + Math.round((out.length / Math.max(totalExpected, 1)) * 70);
+    progress.send(`Loading comments... ${out.length} found`, Math.min(pct, 90));
+  }
+}

package/core/scrapers/index.js

@@ -0,0 +1,44 @@
+import { getPlatformFromUrl } from '../platform-detect.js';
+import { fetchRedditJSON } from './reddit.js';
+import { fetchHackerNewsJSON } from './hackernews.js';
+import { fetchSteamReviews } from './steam.js';
+import { fetchProductHuntComments } from './producthunt.js';
+import { fetchRedditProfile } from './reddit-profile.js';
+import { fetchNotionData } from './notion.js';
+
+const SCRAPER_MAP = {
+  reddit: fetchRedditJSON,
+  redditprofile: fetchRedditProfile,
+  hackernews: fetchHackerNewsJSON,
+  steam: fetchSteamReviews,
+  producthunt: fetchProductHuntComments,
+  notion: fetchNotionData,
+};
+
+/**
+ * Scrape comments from a URL by detecting the platform and dispatching
+ * to the appropriate scraper.
+ * @param {string} url
+ * @param {{ progress: { send: function } }} opts
+ * @returns {Promise<Object>} - { success, comments, post, method, platform }
+ */
+export async function scrapeUrl(url, { progress }) {
+  const platform = getPlatformFromUrl(url);
+
+  if (!platform) {
+    return {
+      success: false,
+      error: `Unsupported URL. Supported platforms: Reddit, Hacker News, Steam, Product Hunt, Notion, Reddit Profiles.`,
+    };
+  }
+
+  const scraper = SCRAPER_MAP[platform];
+  if (!scraper) {
+    return { success: false, error: `No scraper available for platform: ${platform}` };
+  }
+
+  const result = await scraper(url, { progress });
+  return { ...result, platform };
+}
+
+export { getPlatformFromUrl } from '../platform-detect.js';