openalmanac 0.2.34 → 0.2.36

package/skills/reddit-wiki/scripts/ingest.js

@@ -0,0 +1,663 @@
+#!/usr/bin/env node
+
+/**
+ * Reddit Subreddit Ingest — Download + Filter + Convert
+ *
+ * Two-step pipeline:
+ *   1. download — fetch raw posts/comments from Arctic Shift API, save as JSONL
+ *   2. filter   — score posts by quality, convert qualifying ones to markdown
+ *
+ * Usage:
+ *   node ingest.js <subreddit> download [options]
+ *   node ingest.js <subreddit> filter [options]
+ *   node ingest.js <subreddit> count              (just show stats)
+ *
+ * Download options:
+ *   --since <year>       Only download from this year onward
+ *   --posts-only         Skip comments
+ *
+ * Filter options:
+ *   --quality <level>    high (top 10%), medium (top 30%), low (top 60%), all
+ *   --stats-only         Show quality distribution without writing files
+ *
+ * Common:
+ *   --output <dir>       Base directory (default: ~/.openalmanac/corpus/<subreddit>/)
+ */
+
+import { writeFileSync, readFileSync, mkdirSync, existsSync, createWriteStream } from "node:fs";
+import { createReadStream } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { createInterface } from "node:readline";
+
+const ARCTIC_SHIFT_BASE = "https://arctic-shift.photon-reddit.com";
+const KB_PER_POST = 3.4;
+const KB_PER_COMMENT = 1.4;
+
+/* ── CLI parsing ───────────────────────────────────────────────── */
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+  if (args.length < 2 || args[0].startsWith("-")) {
+    console.error("Usage:");
+    console.error("  node ingest.js <subreddit> download [--since <year>] [--posts-only]");
+    console.error("  node ingest.js <subreddit> filter [--quality high|medium|low|all] [--stats-only]");
+    console.error("  node ingest.js <subreddit> count");
+    console.error("");
+    console.error("Options:");
+    console.error("  --output <dir>       Base directory (default: ~/.openalmanac/corpus/<subreddit>/)");
+    console.error("  --since <year>       Only download from this year onward");
+    console.error("  --posts-only         Skip comments during download");
+    console.error("  --quality <level>    high (top 10%), medium (top 30%), low (top 60%), all");
+    console.error("  --stats-only         Show quality stats without writing files");
+    process.exit(1);
+  }
+
+  const subreddit = args[0].replace(/^r\//, "");
+  const command = args[1]; // download, filter, or count
+
+  const opts = {
+    subreddit,
+    command,
+    output: join(homedir(), ".openalmanac", "corpus", subreddit),
+    since: null,
+    postsOnly: false,
+    quality: "medium",
+    statsOnly: false,
+  };
+
+  for (let i = 2; i < args.length; i++) {
+    switch (args[i]) {
+      case "--output":
+        opts.output = args[++i];
+        break;
+      case "--since":
+        opts.since = parseInt(args[++i], 10);
+        break;
+      case "--posts-only":
+        opts.postsOnly = true;
+        break;
+      case "--quality":
+        opts.quality = args[++i];
+        break;
+      case "--stats-only":
+        opts.statsOnly = true;
+        break;
+    }
+  }
+
+  return opts;
+}
+
+/* ── Arctic Shift API helpers ──────────────────────────────────── */
+
+async function fetchJson(url) {
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`HTTP ${res.status}: ${url}`);
+  return res.json();
+}
+
+async function getSubredditCounts(subreddit, since) {
+  const afterParam = since ? `&after=${since}-01-01` : "";
+  const [posts, comments] = await Promise.all([
+    fetchJson(
+      `${ARCTIC_SHIFT_BASE}/api/time_series?key=r/${subreddit}/posts/count&precision=year${afterParam}`
+    ),
+    fetchJson(
+      `${ARCTIC_SHIFT_BASE}/api/time_series?key=r/${subreddit}/comments/count&precision=year${afterParam}`
+    ),
+  ]);
+
+  const totalPosts = (posts.data || []).reduce((sum, d) => sum + (d.value || 0), 0);
+  const totalComments = (comments.data || []).reduce((sum, d) => sum + (d.value || 0), 0);
+
+  return { totalPosts, totalComments };
+}
+
+async function* paginateSearch(subreddit, type, since, limit = 100) {
+  let after = since ? `${since}-01-01` : "2005-01-01";
+
+  while (true) {
+    const url =
+      `${ARCTIC_SHIFT_BASE}/api/${type}/search?subreddit=${subreddit}` +
+      `&after=${after}&sort=asc&sort_type=created_utc&limit=${limit}`;
+
+    let data;
+    try {
+      data = await fetchJson(url);
+    } catch (err) {
+      await new Promise((r) => setTimeout(r, 2000));
+      try {
+        data = await fetchJson(url);
+      } catch {
+        console.error(`\n  Failed to fetch page after retry: ${err.message}`);
+        break;
+      }
+    }
+
+    const items = data.data || [];
+    if (items.length === 0) break;
+
+    yield items;
+
+    const lastCreated = items[items.length - 1].created_utc;
+    if (!lastCreated) break;
+    after = new Date(lastCreated * 1000).toISOString();
+  }
+}
+
+/* ── Quality scoring ───────────────────────────────────────────── */
+
+// Flair patterns that signal educational/knowledge content
+const KNOWLEDGE_FLAIRS = /question|how.?to|guide|tutorial|tip|advice|discussion|help|info/i;
+
+function computeQualityScore(post, commentsByPost) {
+  const score = post.score || 0;
+  const text = (post.selftext || "").trim();
+  const textLen = (text === "[deleted]" || text === "[removed]") ? 0 : text.length;
+  const commentCount = post.num_comments || 0;
+  const flair = post.link_flair_text || "";
+  const isSelf = post.is_self !== false;
+
+  // Total comment text length for this post
+  const comments = commentsByPost.get(post.id) || [];
+  const totalCommentText = comments.reduce((sum, c) => {
+    const body = (c.body || "").trim();
+    return sum + (body === "[deleted]" || body === "[removed]" ? 0 : body.length);
+  }, 0);
+
+  // Normalize each signal to 0-1 range using log scale for heavy-tailed distributions
+  const normScore = Math.min(1, Math.log1p(score) / Math.log1p(500));
+  const normText = Math.min(1, Math.log1p(textLen) / Math.log1p(5000));
+  const normComments = Math.min(1, Math.log1p(commentCount) / Math.log1p(200));
+  const normCommentText = Math.min(1, Math.log1p(totalCommentText) / Math.log1p(50000));
+
+  // Weighted combination
+  let quality =
+    normScore * 0.3 +
+    normText * 0.25 +
+    normComments * 0.25 +
+    normCommentText * 0.2;
+
+  // Bonuses
+  if (KNOWLEDGE_FLAIRS.test(flair)) quality += 0.1;
+  if (isSelf && textLen > 0) quality += 0.05;
+  // Discussion exceeded the post — top comment has 2x+ the post score
+  if (comments.length > 0) {
+    const topCommentScore = Math.max(...comments.map((c) => c.score || 0));
+    if (topCommentScore > score * 2 && score > 0) quality += 0.05;
+  }
+
+  return Math.min(1, quality);
+}
+
+function getQualityThreshold(quality, scores) {
+  if (quality === "all") return 0;
+  const sorted = [...scores].sort((a, b) => b - a);
+  const percentiles = {
+    high: 0.1,   // top 10%
+    medium: 0.3, // top 30%
+    low: 0.6,    // top 60%
+  };
+  const pct = percentiles[quality] || 0.3;
+  const idx = Math.floor(sorted.length * pct);
+  return sorted[Math.min(idx, sorted.length - 1)] || 0;
+}
+
+/* ── Markdown conversion ───────────────────────────────────────── */
+
+function slugify(text, maxLen = 60) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, "")
+    .replace(/[\s-]+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, maxLen)
+    .replace(/-$/, "");
+}
+
+function formatDate(utcTimestamp) {
+  return new Date(utcTimestamp * 1000).toISOString().slice(0, 10);
+}
+
+function escapeYaml(text) {
+  if (/[:#{}[\]&*?|>!%@`"'\\,\n]/.test(text)) {
+    return JSON.stringify(text);
+  }
+  return text;
+}
+
+function buildCommentTree(comments) {
+  const byId = new Map();
+  const roots = [];
+
+  for (const c of comments) {
+    c.children = [];
+    byId.set(`t1_${c.id}`, c);
+  }
+
+  for (const c of comments) {
+    const parent = c.parent_id || "";
+    if (parent.startsWith("t1_") && byId.has(parent)) {
+      byId.get(parent).children.push(c);
+    } else {
+      roots.push(c);
+    }
+  }
+
+  function sortTree(nodes) {
+    nodes.sort((a, b) => (b.score || 0) - (a.score || 0));
+    for (const n of nodes) sortTree(n.children);
+    return nodes;
+  }
+
+  return sortTree(roots);
+}
+
+function renderComment(comment, depth = 0, minScore = 3) {
+  if ((comment.score || 0) < minScore) return "";
+
+  const author = comment.author || "[deleted]";
+  const score = comment.score || 0;
+  const body = (comment.body || "").trim();
+  const created = comment.created_utc;
+
+  if (!body || body === "[deleted]" || body === "[removed]") {
+    return (comment.children || [])
+      .map((c) => renderComment(c, depth + 1, minScore))
+      .join("");
+  }
+
+  const prefix = depth > 0 ? ">".repeat(depth) + " " : "";
+  const dateStr = created ? ` \u00b7 ${formatDate(created)}` : "";
+  const lines = [`${prefix}**${author}** (score: ${score}${dateStr})`];
+
+  for (const line of body.split("\n")) {
+    lines.push(prefix ? `${prefix}${line}` : line);
+  }
+  lines.push("");
+
+  for (const child of comment.children || []) {
+    const rendered = renderComment(child, depth + 1, minScore);
+    if (rendered) lines.push(rendered);
+  }
+
+  return lines.join("\n");
+}
+
+function renderPost(post, comments, subreddit) {
+  const postId = post.id;
+  const title = post.title || "(untitled)";
+  const author = post.author || "[deleted]";
+  const score = post.score || 0;
+  const created = post.created_utc || 0;
+  const permalink = post.permalink || "";
+  const url = post.url || "";
+  const domain = post.domain || "";
+  const flair = post.link_flair_text || "";
+  const isSelf = post.is_self !== false;
+  const selftext = (post.selftext || "").trim();
+  const numComments = post.num_comments || 0;
+
+  const sourceUrl = permalink
+    ? `https://www.reddit.com${permalink}`
+    : `https://www.reddit.com/r/${subreddit}/comments/${postId}/`;
+
+  const fm = [
+    "---",
+    `source: ${escapeYaml(sourceUrl)}`,
+    `subreddit: ${subreddit}`,
+    `post_id: ${postId}`,
+    `author: ${escapeYaml(author)}`,
+    `score: ${score}`,
+    `created: ${formatDate(created)}`,
+    `comment_count: ${numComments}`,
+    `citation_key: reddit-${subreddit}-${postId}`,
+  ];
+  if (flair) fm.push(`flair: ${escapeYaml(flair)}`);
+  if (!isSelf && url) {
+    fm.push(`external_url: ${escapeYaml(url)}`);
+    fm.push(`domain: ${domain}`);
+  }
+  fm.push("---");
+
+  const body = [fm.join("\n"), "", `# ${title}`, ""];
+
+  if (!isSelf && url) {
+    body.push(`**Link:** [${domain}](${url})`, "");
+  }
+
+  if (selftext && selftext !== "[deleted]" && selftext !== "[removed]") {
+    body.push(selftext, "");
+  }
+
+  const tree = buildCommentTree(comments);
+  const rendered = tree
+    .map((c) => renderComment(c, 0, 3))
+    .filter((r) => r.trim());
+
+  if (rendered.length > 0) {
+    body.push("---", "", "## Comments", "", ...rendered);
+  }
+
+  return body.join("\n");
+}
+
+/* ── JSONL helpers ─────────────────────────────────────────────── */
+
+async function* readJsonl(filepath) {
+  const rl = createInterface({
+    input: createReadStream(filepath),
+    crlfDelay: Infinity,
+  });
+  for await (const line of rl) {
+    if (line.trim()) {
+      try {
+        yield JSON.parse(line);
+      } catch {
+        // skip malformed lines
+      }
+    }
+  }
+}
+
+/* ── Progress display ──────────────────────────────────────────── */
+
+function printProgress(label, current, total) {
+  if (total > 0) {
+    const pct = Math.round((current / total) * 100);
+    const filled = Math.round(pct / 5);
+    const bar = "\u2588".repeat(filled) + "\u2591".repeat(20 - filled);
+    process.stderr.write(`\r  ${label}: ${current.toLocaleString()} / ~${total.toLocaleString()} ${bar} ${pct}%`);
+  } else {
+    process.stderr.write(`\r  ${label}: ${current.toLocaleString()} downloaded...`);
+  }
+}
+
+/* ── Commands ──────────────────────────────────────────────────── */
+
+async function runCount(opts) {
+  const { subreddit } = opts;
+
+  console.error("\nFetching subreddit stats...");
+  const totalCounts = await getSubredditCounts(subreddit, null);
+  const filteredCounts = opts.since ? await getSubredditCounts(subreddit, opts.since) : totalCounts;
+
+  const estPosts = filteredCounts.totalPosts > 0 ? filteredCounts.totalPosts : totalCounts.totalPosts;
+  const estComments = filteredCounts.totalComments > 0 ? filteredCounts.totalComments : totalCounts.totalComments;
+  const estSizeMB = Math.round(
+    (estPosts * KB_PER_POST + estComments * KB_PER_COMMENT) / 1024
+  );
+
+  console.error(`  Total posts:    ~${totalCounts.totalPosts.toLocaleString()}`);
+  console.error(`  Total comments: ~${totalCounts.totalComments.toLocaleString()}`);
+  if (opts.since) {
+    console.error(`  Since ${opts.since}:    ~${estPosts.toLocaleString()} posts, ~${estComments.toLocaleString()} comments`);
+  }
+  console.error(`  Est. download: ~${estSizeMB >= 1024 ? (estSizeMB / 1024).toFixed(1) + " GB" : estSizeMB + " MB"}`);
+
+  console.log(
+    JSON.stringify({
+      subreddit,
+      total_posts: totalCounts.totalPosts,
+      total_comments: totalCounts.totalComments,
+      filtered_posts: estPosts,
+      filtered_comments: estComments,
+      estimated_size_mb: estSizeMB,
+      since: opts.since,
+    })
+  );
+}
+
+async function runDownload(opts) {
+  const { subreddit } = opts;
+  const rawDir = join(opts.output, "raw");
+  mkdirSync(rawDir, { recursive: true });
+
+  // Get estimated counts for progress display
+  console.error("\nFetching subreddit stats...");
+  const totalCounts = await getSubredditCounts(subreddit, null);
+  const filteredCounts = opts.since ? await getSubredditCounts(subreddit, opts.since) : totalCounts;
+  const estPosts = filteredCounts.totalPosts > 0 ? filteredCounts.totalPosts : totalCounts.totalPosts;
+  const estComments = filteredCounts.totalComments > 0 ? filteredCounts.totalComments : totalCounts.totalComments;
+
+  console.error(`  ~${estPosts.toLocaleString()} posts, ~${estComments.toLocaleString()} comments to download`);
+
+  // Download posts → raw/posts.jsonl
+  const postsPath = join(rawDir, "posts.jsonl");
+  const postsStream = createWriteStream(postsPath);
+  let postCount = 0;
+
+  console.error("\nDownloading posts...");
+  for await (const batch of paginateSearch(subreddit, "posts", opts.since)) {
+    for (const post of batch) {
+      postsStream.write(JSON.stringify(post) + "\n");
+    }
+    postCount += batch.length;
+    printProgress("Posts", postCount, estPosts);
+  }
+  postsStream.end();
+  console.error("");
+
+  // Download comments → raw/comments.jsonl
+  let commentCount = 0;
+  if (!opts.postsOnly) {
+    const commentsPath = join(rawDir, "comments.jsonl");
+    const commentsStream = createWriteStream(commentsPath);
+
+    console.error("Downloading comments...");
+    for await (const batch of paginateSearch(subreddit, "comments", opts.since)) {
+      for (const comment of batch) {
+        commentsStream.write(JSON.stringify(comment) + "\n");
+      }
+      commentCount += batch.length;
+      printProgress("Comments", commentCount, estComments);
+    }
+    commentsStream.end();
+    console.error("");
+  }
+
+  // Write download metadata
+  const metaPath = join(rawDir, "meta.json");
+  const meta = {
+    subreddit,
+    downloaded_at: new Date().toISOString(),
+    since: opts.since,
+    posts_downloaded: postCount,
+    comments_downloaded: commentCount,
+    posts_only: opts.postsOnly,
+  };
+  writeFileSync(metaPath, JSON.stringify(meta, null, 2));
+
+  console.error(`\n${"─".repeat(40)}`);
+  console.error(`Done. ${postCount.toLocaleString()} posts, ${commentCount.toLocaleString()} comments saved to ${rawDir}`);
+
+  console.log(JSON.stringify(meta));
+}
+
+async function runFilter(opts) {
+  const { subreddit } = opts;
+  const rawDir = join(opts.output, "raw");
+  const entriesDir = join(opts.output, "entries");
+
+  // Check raw data exists
+  const postsPath = join(rawDir, "posts.jsonl");
+  if (!existsSync(postsPath)) {
+    console.error(`Error: No raw data found at ${rawDir}`);
+    console.error(`Run 'node ingest.js ${subreddit} download' first.`);
+    process.exit(1);
+  }
+
+  // Load all posts
+  console.error("\nLoading posts...");
+  const allPosts = new Map();
+  for await (const post of readJsonl(postsPath)) {
+    allPosts.set(post.id, post);
+  }
+  console.error(`  ${allPosts.size.toLocaleString()} posts loaded`);
+
+  // Load comments grouped by post
+  const commentsByPost = new Map();
+  const commentsPath = join(rawDir, "comments.jsonl");
+  if (existsSync(commentsPath)) {
+    console.error("Loading comments...");
+    let commentCount = 0;
+    for await (const comment of readJsonl(commentsPath)) {
+      const linkId = comment.link_id || "";
+      if (!linkId.startsWith("t3_")) continue;
+      const postId = linkId.slice(3);
+      if (!commentsByPost.has(postId)) commentsByPost.set(postId, []);
+      commentsByPost.get(postId).push(comment);
+      commentCount++;
+    }
+    console.error(`  ${commentCount.toLocaleString()} comments loaded`);
+  }
+
+  // Score every post
+  console.error("\nScoring posts...");
+  const scored = [];
+  for (const [postId, post] of allPosts) {
+    const text = (post.selftext || "").trim();
+    if (text === "[deleted]" || text === "[removed]") continue;
+    if ((post.author || "") === "[deleted]") continue;
+
+    const quality = computeQualityScore(post, commentsByPost);
+    scored.push({ post, quality });
+  }
+  scored.sort((a, b) => b.quality - a.quality);
+
+  const allScores = scored.map((s) => s.quality);
+
+  // Compute stats for each quality level
+  const levels = ["high", "medium", "low", "all"];
+  const levelCounts = {};
+  for (const level of levels) {
+    const threshold = getQualityThreshold(level, allScores);
+    levelCounts[level] = scored.filter((s) => s.quality >= threshold).length;
+  }
+
+  // Show distribution
+  console.error(`\n  Quality distribution (${scored.length.toLocaleString()} scorable posts):`);
+  console.error(`    high   (top 10%):  ${levelCounts.high.toLocaleString()} posts`);
+  console.error(`    medium (top 30%):  ${levelCounts.medium.toLocaleString()} posts`);
+  console.error(`    low    (top 60%):  ${levelCounts.low.toLocaleString()} posts`);
+  console.error(`    all:               ${levelCounts.all.toLocaleString()} posts`);
+
+  // Show sample posts at each level
+  const showSample = (label, idx) => {
+    if (idx < scored.length) {
+      const s = scored[idx];
+      const title = (s.post.title || "").slice(0, 60);
+      const score = s.post.score || 0;
+      const comments = s.post.num_comments || 0;
+      console.error(`    ${label}: "${title}" (score: ${score}, comments: ${comments}, quality: ${s.quality.toFixed(2)})`);
+    }
+  };
+
+  console.error(`\n  Samples:`);
+  showSample("top     ", 0);
+  showSample("10%     ", Math.floor(scored.length * 0.1));
+  showSample("30%     ", Math.floor(scored.length * 0.3));
+  showSample("60%     ", Math.floor(scored.length * 0.6));
+  showSample("bottom  ", scored.length - 1);
+
+  if (opts.statsOnly) {
+    console.log(
+      JSON.stringify({
+        subreddit,
+        total_posts: allPosts.size,
+        scorable_posts: scored.length,
+        levels: levelCounts,
+        top_post: scored[0] ? { title: scored[0].post.title, score: scored[0].post.score, quality: scored[0].quality } : null,
+      })
+    );
+    return;
+  }
+
+  // Apply quality filter
+  const threshold = getQualityThreshold(opts.quality, allScores);
+  const qualifying = scored.filter((s) => s.quality >= threshold);
+
+  console.error(`\n  Filtering at "${opts.quality}" → ${qualifying.length.toLocaleString()} posts`);
+
+  // Write markdown entries
+  console.error("Writing entries...");
+  mkdirSync(entriesDir, { recursive: true });
+
+  let written = 0;
+  for (const { post } of qualifying) {
+    const title = post.title || "untitled";
+    const slug = slugify(title);
+    const filename = slug ? `${post.id}-${slug}.md` : `${post.id}.md`;
+    const filepath = join(entriesDir, filename);
+
+    const comments = commentsByPost.get(post.id) || [];
+    const md = renderPost(post, comments, subreddit);
+    writeFileSync(filepath, md);
+    written++;
+  }
+
+  // Initialize absorb log
+  const absorbLogPath = join(opts.output, "absorb_log.json");
+  if (!existsSync(absorbLogPath)) {
+    writeFileSync(
+      absorbLogPath,
+      JSON.stringify(
+        { entries: {}, stats: { total_entries: written, absorbed: 0, remaining: written } },
+        null,
+        2
+      )
+    );
+  } else {
+    // Update total count in existing log
+    const log = JSON.parse(readFileSync(absorbLogPath, "utf-8"));
+    log.stats.total_entries = written;
+    log.stats.remaining = written - log.stats.absorbed;
+    writeFileSync(absorbLogPath, JSON.stringify(log, null, 2));
+  }
+
+  console.error(`\n${"─".repeat(40)}`);
+  console.error(`Done. ${written.toLocaleString()} entries written to ${entriesDir}`);
+
+  console.log(
+    JSON.stringify({
+      subreddit,
+      total_posts: allPosts.size,
+      quality_level: opts.quality,
+      qualifying_posts: qualifying.length,
+      entries_written: written,
+      entries_dir: entriesDir,
+      absorb_log: absorbLogPath,
+    })
+  );
+}
+
+/* ── Main ──────────────────────────────────────────────────────── */
+
+async function main() {
+  const opts = parseArgs();
+
+  console.error(`\nReddit Ingest — r/${opts.subreddit}`);
+  console.error("─".repeat(40));
+
+  switch (opts.command) {
+    case "count":
+      await runCount(opts);
+      break;
+    case "download":
+      await runDownload(opts);
+      break;
+    case "filter":
+      await runFilter(opts);
+      break;
+    default:
+      console.error(`Unknown command: ${opts.command}`);
+      console.error("Use: download, filter, or count");
+      process.exit(1);
+  }
+}
+
+main().catch((err) => {
+  console.error(`\nError: ${err.message}`);
+  process.exit(1);
+});