openalmanac 0.2.35 → 0.2.36

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { createServer } from "./server.js";
 import { runLogin, runLogout } from "./login.js";
-import { runSetup } from "./setup.js";
+import { runSetup, runRedditSetup } from "./setup.js";
 const command = process.argv[2];
 if (command === "setup") {
     runSetup().catch((e) => {
@@ -9,6 +9,12 @@ if (command === "setup") {
         process.exit(1);
     });
 }
+else if (command === "reddit") {
+    runRedditSetup().catch((e) => {
+        console.error(e instanceof Error ? e.message : e);
+        process.exit(1);
+    });
+}
 else if (command === "login") {
     runLogin().catch((e) => {
         console.error(e instanceof Error ? e.message : e);

package/dist/setup.d.ts

@@ -1 +1,2 @@
 export declare function runSetup(): Promise<void>;
+export declare function runRedditSetup(): Promise<void>;

package/dist/setup.js

@@ -1,6 +1,7 @@
-import { readFileSync, writeFileSync, mkdirSync, existsSync } from "fs";
+import { readFileSync, writeFileSync, mkdirSync, existsSync, cpSync } from "fs";
 import { homedir } from "os";
-import { join } from "path";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
 import { performLogin } from "./login-core.js";
 import { getAuthStatus } from "./auth.js";
 const TOOL_GROUPS = [
@@ -560,3 +561,109 @@ export async function runSetup() {
     printResult(agent, loginResult, mcpChanged, count);
     process.exit(0);
 }
+/* ── Skill installation ────────────────────────────────────────── */
+function getPackageSkillsDir() {
+    const thisFile = fileURLToPath(import.meta.url);
+    // dist/setup.js → package root → skills/
+    return join(dirname(thisFile), "..", "skills");
+}
+function installSkill(skillName) {
+    const src = join(getPackageSkillsDir(), skillName);
+    if (!existsSync(src)) {
+        throw new Error(`Skill "${skillName}" not found in package at ${src}`);
+    }
+    const dest = join(homedir(), ".claude", "skills", skillName);
+    // Always overwrite to ensure latest version
+    mkdirSync(dirname(dest), { recursive: true });
+    cpSync(src, dest, { recursive: true, force: true });
+    return true;
+}
+/* ── Reddit-specific tool groups ───────────────────────────────── */
+const REDDIT_EXTRA_TOOLS = [
+    "Bash(node *)",
+    "Bash(curl *)",
+];
+/* ── Reddit setup banner ───────────────────────────────────────── */
+function printRedditBanner() {
+    process.stdout.write("\n");
+    for (let i = 0; i < LOGO_LINES.length; i++) {
+        process.stdout.write(`${GRADIENT[i]}${LOGO_LINES[i]}${RST}\n`);
+    }
+    process.stdout.write(`\n${DIM}  Turn any subreddit into a wiki${RST}\n`);
+}
+/* ── Reddit result screen ──────────────────────────────────────── */
+function printRedditResult(agent, loginResult, mcpChanged, toolCount) {
+    process.stdout.write("\x1b[2J\x1b[H");
+    printRedditBanner();
+    printBadge();
+    w("");
+    stepDone(`Agent \u2192 ${WHITE_BOLD}${agent}${RST}`);
+    w(BAR);
+    stepDone(`MCP server ${mcpChanged ? "configured" : `${DIM}already configured${RST}`}`);
+    w(BAR);
+    stepDone(`${BLUE}${toolCount}${RST} tool${toolCount !== 1 ? "s" : ""} allowed`);
+    w(BAR);
+    stepDone(loginLabel(loginResult));
+    w(BAR);
+    stepDone(`${BLUE}/reddit-wiki${RST} skill installed`);
+    w(BAR);
+    stepDone(`${BLUE}Setup complete${RST}`);
+    w("");
+    // Next steps box
+    const innerW = 62;
+    const row = (content) => {
+        const padding = Math.max(0, innerW - vis(content));
+        return `  ${BLUE_DIM}\u2502${RST}${content}${" ".repeat(padding)}${BLUE_DIM}\u2502${RST}`;
+    };
+    const empty = row("");
+    w(`  ${BLUE_DIM}\u256d${"─".repeat(innerW)}\u256e${RST}`);
+    w(empty);
+    w(row(`  ${WHITE_BOLD}Next steps${RST}`));
+    w(empty);
+    w(row(`  ${BLUE}1.${RST}  Type ${WHITE_BOLD}claude${RST} to start Claude Code`));
+    w(row(`  ${BLUE}2.${RST}  Run ${BLUE}/reddit-wiki r/<subreddit>${RST}`));
+    w(empty);
+    w(`  ${BLUE_DIM}\u2570${"─".repeat(innerW)}\u256f${RST}`);
+    w("");
+}
+/* ── Reddit entry point ────────────────────────────────────────── */
+export async function runRedditSetup() {
+    const skipTui = process.argv.includes("--yes") || process.argv.includes("-y");
+    const interactive = process.stdin.isTTY && !skipTui;
+    let agent = "Claude Code";
+    if (interactive) {
+        agent = await runAgentSelect();
+    }
+    const mcpChanged = configureMcp();
+    let tools;
+    if (interactive) {
+        tools = await runToolSelect(agent, mcpChanged);
+    }
+    else {
+        tools = TOOL_GROUPS.flatMap((g) => g.tools);
+    }
+    // Add reddit-specific tool permissions
+    tools = [...tools, ...REDDIT_EXTRA_TOOLS];
+    const count = configurePermissions(tools);
+    // Login step
+    let loginResult;
+    if (interactive) {
+        loginResult = await runLoginStep(agent, mcpChanged, count);
+    }
+    else {
+        try {
+            const result = await performLogin();
+            loginResult =
+                result.status === "already_logged_in"
+                    ? { status: "already", name: result.name }
+                    : { status: "done" };
+        }
+        catch {
+            loginResult = { status: "skipped" };
+        }
+    }
+    // Install the reddit-wiki skill
+    installSkill("reddit-wiki");
+    printRedditResult(agent, loginResult, mcpChanged, count);
+    process.exit(0);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openalmanac",
-  "version": "0.2.35",
+  "version": "0.2.36",
   "description": "OpenAlmanac — pull, edit, and push articles to the open knowledge base",
   "type": "module",
   "bin": {
@@ -32,6 +32,7 @@
     "node": ">=18.0.0"
   },
   "files": [
-    "dist"
+    "dist",
+    "skills"
   ]
 }

package/skills/reddit-wiki/SKILL.md

@@ -0,0 +1,335 @@
+---
+name: reddit-wiki
+description: Turn any subreddit into a published wiki on Almanac
+allowed-tools: Bash(node *), Bash(curl *), mcp__almanac__search_articles, mcp__almanac__search_communities, mcp__almanac__list_articles, mcp__almanac__read, mcp__almanac__download, mcp__almanac__new, mcp__almanac__publish, mcp__almanac__search_web, mcp__almanac__read_webpage, mcp__almanac__search_images, mcp__almanac__view_images, mcp__almanac__register_sources, mcp__almanac__login, mcp__almanac__create_community, Read(~/.openalmanac/**), Write(~/.openalmanac/**), Edit(~/.openalmanac/**)
+argument-hint: r/<subreddit>
+---
+
+# Reddit Wiki
+
+Turn a subreddit into a published wiki on Almanac. You are an enthusiastic researcher who genuinely finds this stuff interesting — share what you discover, don't just report status.
+
+## Your personality
+
+You're building a wiki WITH the user, not FOR them. Share interesting things you find in the data. Get excited about surprising discoveries. But never be fake — if something isn't interesting, don't pretend it is. No small talk. Everything you say should be real information.
+
+Never estimate how long things will take. Do show data sizes so the user knows what they're getting.
+
+## Flow overview
+
+Two phases:
+1. **Foundation** — Plan and write 15-20 core articles with images, citations, and wikilinks
+2. **Deep Absorb** — Process the corpus batch by batch, discovering niche topics and enriching existing articles
+
+## Naming convention
+
+- **User-facing**: Always say `r/lockpicking` (with `r/` prefix)
+- **File paths**: Bare name — `~/.openalmanac/corpus/lockpicking/`
+- **API calls / community slugs**: Bare name — `subreddit=lockpicking`
+- **Accept both** as input: `r/lockpicking` or `lockpicking`
+
+## Step 1: Scout
+
+Extract the subreddit name from the argument (strip `r/` prefix if present). Use the bare name for all API calls and file paths. Use `r/<name>` when talking to the user.
+
+Run these three things in parallel (silently — don't narrate the tool calls):
+1. `search_communities("<subreddit_name>")`
+2. `search_articles` with 5-10 key topic terms you'd expect in this community
+3. Get subreddit stats from Arctic Shift:
+
+```bash
+node ${CLAUDE_SKILL_DIR}/scripts/ingest.js $1 count
+```
+
+This returns JSON with `total_posts`, `total_comments`, and `estimated_size_mb`.
+
+Now greet the user. Tell them:
+- What already exists on Almanac for this community (articles, stubs, community)
+- Share something genuinely interesting about it if you know anything
+- Subreddit stats (posts, comments)
+- The two-phase plan (brief — one line each)
+- Download depth options with size estimates
+
+Present the download options with a recommendation. For small subreddits (< 50k posts), recommend full history. For large ones (> 500k posts), recommend last 3 years.
+
+```
+How deep should I go?
+
+  › Full history (recommended)
+    ~X GB download. Everything since YYYY.
+
+    Last 3 years
+    ~X MB download.
+
+    Last year
+    ~X MB. Quick start.
+```
+
+Wait for the user to choose.
+
+## Step 2: Download + Conversation
+
+Download is a two-step process: first download raw data, then filter by quality.
+
+Start the download in the background:
+
+```bash
+node ${CLAUDE_SKILL_DIR}/scripts/ingest.js <subreddit> download --since <year>
+```
+
+This saves raw JSONL to `~/.openalmanac/corpus/<subreddit>/raw/`. The raw data is kept so you can re-filter later with different quality thresholds without re-downloading.
+
+While it downloads, share interesting context about the community. Use your knowledge and do a quick `search_web` if helpful. Share REAL information — facts, history, notable members, what makes this community unique. Not questions, not small talk.
+
+When the download finishes, run the filter step:
+
+```bash
+node ${CLAUDE_SKILL_DIR}/scripts/ingest.js <subreddit> filter --stats-only
+```
+
+This shows the quality distribution — how many posts at each quality level, with sample posts. Use this to decide the right quality threshold. Then present it to the user:
+
+```
+Download complete. X posts, Y comments from r/<subreddit>.
+
+Here's what the data looks like:
+
+  high quality (top 10%):  ~300 posts — best discussions, guides, educational content
+  medium (top 30%):        ~900 posts — solid community knowledge
+  low (top 60%):           ~1,800 posts — includes questions and casual posts
+  all:                     ~3,000 posts
+
+I'd recommend medium for the foundation — good balance of quality and coverage.
+We can dip into the rest during Phase 2.
+```
+
+Wait for the user to pick (or confirm your recommendation), then run:
+
+```bash
+node ${CLAUDE_SKILL_DIR}/scripts/ingest.js <subreddit> filter --quality medium
+```
+
+This writes markdown entries to `~/.openalmanac/corpus/<subreddit>/entries/`. Each entry has citation-ready frontmatter with `citation_key` and `source` (Reddit permalink).
+
+Report the results:
+- How many entries were created
+- Where they're stored
+
+## Step 3: Phase 1 — Foundation
+
+### Plan topics
+
+Read 20-30 corpus entries (prioritize high-score posts) to understand the landscape. Also check what already exists:
+
+```
+list_articles(community_slug: "<subreddit>", sort: "most_referenced")
+```
+
+Identify 15-20 core topics grouped by theme. These should be the foundational articles every reader of this wiki would expect. Present them to the user grouped by theme:
+
+```
+Here's what I'd build for the foundation:
+
+  Lock Anatomy
+    › Cylinder, Warding, Master Keying
+
+  Techniques
+    › Bumping, Comb Picking, Impressioning
+
+  [etc.]
+
+Want to add or change anything?
+```
+
+Include your recommendation. Wait for the user to confirm or adjust.
+
+### Scaffold entities
+
+Before any writing, scaffold all planned articles as local files:
+
+1. **Check what exists online:** `search_articles` with ALL planned entity names in one batch call
+2. **Check local folder:** Read `~/.openalmanac/articles/<subreddit>/` to see what's already scaffolded
+3. **Create missing:** `new(articles: [{title, community_slug}, ...])` for everything not found
+
+This creates the entity map. Writing agents will check the local folder to know what slugs exist.
+
+### Write articles
+
+Tell the user what's happening:
+
+```
+Kicking off the writing agents:
+
+  • Agent 1: Lock Anatomy — Cylinder, Warding, Master Keying
+  • Agent 2: Techniques — Bumping, Comb Picking, Impressioning
+  • Agent 3: Famous Locks — American 1100, Abus 55/40
+  • Agent 4: Community — LockPickingLawyer, Belt System
+```
+
+Spin up 4-5 parallel writing agents, ~3-4 articles each. Group by theme so related articles are written by the same agent (better cross-referencing).
+
+**Each writing agent's brief must include:**
+
+1. **Which articles to write** (the scaffolded .md files to fill in)
+2. **Corpus entries to read** — point to specific files in `~/.openalmanac/corpus/<subreddit>/` relevant to its topics
+3. **The entity map** — list all scaffolded slugs so the agent uses correct wikilinks
+4. **These citation rules:**
+   - Every source MUST have a public URL
+   - Corpus entries have `citation_key` and `source` (Reddit permalink) in their frontmatter — use them as `[@citation_key]` markers and list them in the article's YAML `sources:` array
+   - Also use `search_web` and `read_webpage` for additional sources beyond Reddit
+   - NEVER fabricate a URL. If a source has no public URL, do not use it.
+   - Register sources with `register_sources` before writing
+5. **These wikilink rules:**
+   - Use `[[slug|Display Text]]` syntax for entities that exist (scaffolded or published)
+   - Before linking to a new entity NOT on the map: `search_articles` to check, then scaffold with `new()` if needed
+   - Prefer existing slugs over inventing new ones
+6. **Writing quality:**
+   - Fetch guidelines from `https://openalmanac.org/writing-guidelines` using `read_webpage`
+   - Write with the community's voice — cite Reddit discussions, not just Wikipedia
+   - Include `[@citation_key]` markers throughout, especially for claims from the corpus
+   - Articles should feel like they were written by someone who lives in this community
+
+**While agents work**, narrate what's happening. Share interesting things you see them finding. Example:
+
+```
+Agent 2 found a heated 2019 thread about whether LockPickingLawyer's
+speed picks are realistic for beginners — 400 upvotes, great discussion.
+Working that into the article...
+```
+
+### Image pass
+
+After all writing agents finish, run parallel haiku-model image agents (one per article):
+
+Each image agent:
+1. Reads the article
+2. `search_images` for 1-2 hero image queries
+3. `view_images` to verify the best candidate
+4. Adds the image URL to the article's frontmatter as `image_url`
+
+### Publish
+
+```
+publish(community_slug: "<subreddit>")
+```
+
+This batch-publishes all articles in the community folder. The backend auto-creates stubs from any dead wikilinks in the articles.
+
+Share the results with enthusiasm:
+
+```
+17 articles live! The wiki now has 35 articles total, plus
+12 new stubs that emerged from wikilinks.
+
+Check it out: openalmanac.org/communities/<subreddit>/wiki
+
+You can also browse it in the Almanac desktop app — best way
+to explore and keep contributing.
+```
+
+## Step 4: Phase 2 — Deep Absorb
+
+After Phase 1, check in with the user:
+
+```
+That was Phase 1 — the foundation. There are still X,000+
+corpus entries I haven't processed yet. Lots of niche stuff
+hiding in there — topics that didn't make the top 20 but
+the community clearly cares about.
+
+Want me to start Phase 2? I can either:
+
+  › Keep going and check in every few batches
+  › Go batch by batch so you can see what emerges
+```
+
+Wait for the user to choose.
+
+### Absorb loop
+
+Read `~/.openalmanac/corpus/<subreddit>/absorb_log.json` to know what's been processed.
+
+For each batch:
+
+1. **Read 50 unabsorbed entries** from the corpus directory (skip any listed in absorb_log)
+2. **Cluster by theme** — what topics do these entries cover?
+3. **Decide:** Create new articles? Enrich existing ones? Both?
+4. **For existing articles:** `download` them first, then expand with new details/sections
+5. **For new articles:** Scaffold → write → add to wiki
+6. **Image pass** on any new articles (haiku agents)
+7. **Publish** the batch
+8. **Update absorb_log.json:**
+   ```json
+   {
+     "entries": {
+       "<filename>": {
+         "absorbed_at": "<ISO timestamp>",
+         "absorbed_into": ["article-slug-1", "article-slug-2"]
+       }
+     },
+     "stats": {
+       "total_entries": <total>,
+       "absorbed": <count>,
+       "remaining": <count>
+     }
+   }
+   ```
+
+**Between batches**, share what you found:
+
+```
+Batches 1-5 done. Found some gems:
+  • "Lock Lubricants in Cold Weather" — apparently Houdini
+    lube freezes below -20°F, community recommends graphite
+  • Expanded the American 1100 article with a detailed
+    teardown thread from 2017
+  • New article: "Lockpicking Competitions" — there's a
+    whole competitive scene
+
+3 new articles, 4 enriched. Continuing...
+```
+
+### When to stop
+
+- If the user said "keep going with check-ins": continue until all entries are absorbed or the user says stop
+- If the user said "batch by batch": pause after each batch and ask if they want to continue
+- At the end, show a final tally:
+
+```
+Phase 2 complete. Processed X,XXX entries across N batches.
+
+Final wiki:
+  XX articles (was YY)
+  XX remaining stubs
+  XXX+ citations from the community
+
+openalmanac.org/communities/<subreddit>/wiki
+```
+
+## Important rules
+
+### Citations
+- Every source MUST have a public URL. Reddit permalinks, web pages, YouTube — all fine.
+- If a source has no public URL, do NOT use it and do NOT cite it. Inform the user.
+- Never fabricate or construct URLs.
+- Corpus entries have `citation_key` and `source` in their frontmatter — these are ready to use.
+
+### Entity linking
+- Always `search_articles` before creating new entities — check what already exists
+- Check the local `~/.openalmanac/articles/<subreddit>/` folder for scaffolded files
+- Only scaffold with `new()` if the entity doesn't exist anywhere
+- Use `[[slug|Display Text]]` wikilink syntax
+- Prefer existing slugs over inventing new ones to avoid duplicates
+
+### Community creation
+- If the community doesn't exist on Almanac yet, create it with `create_community`
+- The description should have personality — capture the community's vibe, not a generic taxonomy
+- Find a good cover image with `search_images`
+
+### What NOT to do
+- Don't estimate how long things will take
+- Don't make small talk or ask personal questions
+- Don't force enthusiasm — if something isn't interesting, don't pretend
+- Don't go silent for long stretches — narrate what's happening
+- Don't ask permission for every article — the user approved the plan, that's consent
+- Don't skip Reddit as a source — the corpus IS the community's voice, cite it

package/skills/reddit-wiki/scripts/ingest.js

@@ -0,0 +1,663 @@
+#!/usr/bin/env node
+
+/**
+ * Reddit Subreddit Ingest — Download + Filter + Convert
+ *
+ * Two-step pipeline:
+ *   1. download — fetch raw posts/comments from Arctic Shift API, save as JSONL
+ *   2. filter   — score posts by quality, convert qualifying ones to markdown
+ *
+ * Usage:
+ *   node ingest.js <subreddit> download [options]
+ *   node ingest.js <subreddit> filter [options]
+ *   node ingest.js <subreddit> count              (just show stats)
+ *
+ * Download options:
+ *   --since <year>       Only download from this year onward
+ *   --posts-only         Skip comments
+ *
+ * Filter options:
+ *   --quality <level>    high (top 10%), medium (top 30%), low (top 60%), all
+ *   --stats-only         Show quality distribution without writing files
+ *
+ * Common:
+ *   --output <dir>       Base directory (default: ~/.openalmanac/corpus/<subreddit>/)
+ */
+
+import { writeFileSync, readFileSync, mkdirSync, existsSync, createWriteStream } from "node:fs";
+import { createReadStream } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { createInterface } from "node:readline";
+
+const ARCTIC_SHIFT_BASE = "https://arctic-shift.photon-reddit.com";
+const KB_PER_POST = 3.4;
+const KB_PER_COMMENT = 1.4;
+
+/* ── CLI parsing ───────────────────────────────────────────────── */
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+  if (args.length < 2 || args[0].startsWith("-")) {
+    console.error("Usage:");
+    console.error("  node ingest.js <subreddit> download [--since <year>] [--posts-only]");
+    console.error("  node ingest.js <subreddit> filter [--quality high|medium|low|all] [--stats-only]");
+    console.error("  node ingest.js <subreddit> count");
+    console.error("");
+    console.error("Options:");
+    console.error("  --output <dir>       Base directory (default: ~/.openalmanac/corpus/<subreddit>/)");
+    console.error("  --since <year>       Only download from this year onward");
+    console.error("  --posts-only         Skip comments during download");
+    console.error("  --quality <level>    high (top 10%), medium (top 30%), low (top 60%), all");
+    console.error("  --stats-only         Show quality stats without writing files");
+    process.exit(1);
+  }
+
+  const subreddit = args[0].replace(/^r\//, "");
+  const command = args[1]; // download, filter, or count
+
+  const opts = {
+    subreddit,
+    command,
+    output: join(homedir(), ".openalmanac", "corpus", subreddit),
+    since: null,
+    postsOnly: false,
+    quality: "medium",
+    statsOnly: false,
+  };
+
+  for (let i = 2; i < args.length; i++) {
+    switch (args[i]) {
+      case "--output":
+        opts.output = args[++i];
+        break;
+      case "--since":
+        opts.since = parseInt(args[++i], 10);
+        break;
+      case "--posts-only":
+        opts.postsOnly = true;
+        break;
+      case "--quality":
+        opts.quality = args[++i];
+        break;
+      case "--stats-only":
+        opts.statsOnly = true;
+        break;
+    }
+  }
+
+  return opts;
+}
+
+/* ── Arctic Shift API helpers ──────────────────────────────────── */
+
+async function fetchJson(url) {
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`HTTP ${res.status}: ${url}`);
+  return res.json();
+}
+
+async function getSubredditCounts(subreddit, since) {
+  const afterParam = since ? `&after=${since}-01-01` : "";
+  const [posts, comments] = await Promise.all([
+    fetchJson(
+      `${ARCTIC_SHIFT_BASE}/api/time_series?key=r/${subreddit}/posts/count&precision=year${afterParam}`
+    ),
+    fetchJson(
+      `${ARCTIC_SHIFT_BASE}/api/time_series?key=r/${subreddit}/comments/count&precision=year${afterParam}`
+    ),
+  ]);
+
+  const totalPosts = (posts.data || []).reduce((sum, d) => sum + (d.value || 0), 0);
+  const totalComments = (comments.data || []).reduce((sum, d) => sum + (d.value || 0), 0);
+
+  return { totalPosts, totalComments };
+}
+
+async function* paginateSearch(subreddit, type, since, limit = 100) {
+  let after = since ? `${since}-01-01` : "2005-01-01";
+
+  while (true) {
+    const url =
+      `${ARCTIC_SHIFT_BASE}/api/${type}/search?subreddit=${subreddit}` +
+      `&after=${after}&sort=asc&sort_type=created_utc&limit=${limit}`;
+
+    let data;
+    try {
+      data = await fetchJson(url);
+    } catch (err) {
+      await new Promise((r) => setTimeout(r, 2000));
+      try {
+        data = await fetchJson(url);
+      } catch {
+        console.error(`\n  Failed to fetch page after retry: ${err.message}`);
+        break;
+      }
+    }
+
+    const items = data.data || [];
+    if (items.length === 0) break;
+
+    yield items;
+
+    const lastCreated = items[items.length - 1].created_utc;
+    if (!lastCreated) break;
+    after = new Date(lastCreated * 1000).toISOString();
+  }
+}
+
+/* ── Quality scoring ───────────────────────────────────────────── */
+
+// Flair patterns that signal educational/knowledge content
+const KNOWLEDGE_FLAIRS = /question|how.?to|guide|tutorial|tip|advice|discussion|help|info/i;
+
+function computeQualityScore(post, commentsByPost) {
+  const score = post.score || 0;
+  const text = (post.selftext || "").trim();
+  const textLen = (text === "[deleted]" || text === "[removed]") ? 0 : text.length;
+  const commentCount = post.num_comments || 0;
+  const flair = post.link_flair_text || "";
+  const isSelf = post.is_self !== false;
+
+  // Total comment text length for this post
+  const comments = commentsByPost.get(post.id) || [];
+  const totalCommentText = comments.reduce((sum, c) => {
+    const body = (c.body || "").trim();
+    return sum + (body === "[deleted]" || body === "[removed]" ? 0 : body.length);
+  }, 0);
+
+  // Normalize each signal to 0-1 range using log scale for heavy-tailed distributions
+  const normScore = Math.min(1, Math.log1p(score) / Math.log1p(500));
+  const normText = Math.min(1, Math.log1p(textLen) / Math.log1p(5000));
+  const normComments = Math.min(1, Math.log1p(commentCount) / Math.log1p(200));
+  const normCommentText = Math.min(1, Math.log1p(totalCommentText) / Math.log1p(50000));
+
+  // Weighted combination
+  let quality =
+    normScore * 0.3 +
+    normText * 0.25 +
+    normComments * 0.25 +
+    normCommentText * 0.2;
+
+  // Bonuses
+  if (KNOWLEDGE_FLAIRS.test(flair)) quality += 0.1;
+  if (isSelf && textLen > 0) quality += 0.05;
+  // Discussion exceeded the post — top comment has 2x+ the post score
+  if (comments.length > 0) {
+    const topCommentScore = Math.max(...comments.map((c) => c.score || 0));
+    if (topCommentScore > score * 2 && score > 0) quality += 0.05;
+  }
+
+  return Math.min(1, quality);
+}
+
+function getQualityThreshold(quality, scores) {
+  if (quality === "all") return 0;
+  const sorted = [...scores].sort((a, b) => b - a);
+  const percentiles = {
+    high: 0.1,   // top 10%
+    medium: 0.3, // top 30%
+    low: 0.6,    // top 60%
+  };
+  const pct = percentiles[quality] || 0.3;
+  const idx = Math.floor(sorted.length * pct);
+  return sorted[Math.min(idx, sorted.length - 1)] || 0;
+}
+
+/* ── Markdown conversion ───────────────────────────────────────── */
+
+function slugify(text, maxLen = 60) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, "")
+    .replace(/[\s-]+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, maxLen)
+    .replace(/-$/, "");
+}
+
+function formatDate(utcTimestamp) {
+  return new Date(utcTimestamp * 1000).toISOString().slice(0, 10);
+}
+
+function escapeYaml(text) {
+  if (/[:#{}[\]&*?|>!%@`"'\\,\n]/.test(text)) {
+    return JSON.stringify(text);
+  }
+  return text;
+}
+
+function buildCommentTree(comments) {
+  const byId = new Map();
+  const roots = [];
+
+  for (const c of comments) {
+    c.children = [];
+    byId.set(`t1_${c.id}`, c);
+  }
+
+  for (const c of comments) {
+    const parent = c.parent_id || "";
+    if (parent.startsWith("t1_") && byId.has(parent)) {
+      byId.get(parent).children.push(c);
+    } else {
+      roots.push(c);
+    }
+  }
+
+  function sortTree(nodes) {
+    nodes.sort((a, b) => (b.score || 0) - (a.score || 0));
+    for (const n of nodes) sortTree(n.children);
+    return nodes;
+  }
+
+  return sortTree(roots);
+}
+
+function renderComment(comment, depth = 0, minScore = 3) {
+  if ((comment.score || 0) < minScore) return "";
+
+  const author = comment.author || "[deleted]";
+  const score = comment.score || 0;
+  const body = (comment.body || "").trim();
+  const created = comment.created_utc;
+
+  if (!body || body === "[deleted]" || body === "[removed]") {
+    return (comment.children || [])
+      .map((c) => renderComment(c, depth + 1, minScore))
+      .join("");
+  }
+
+  const prefix = depth > 0 ? ">".repeat(depth) + " " : "";
+  const dateStr = created ? ` \u00b7 ${formatDate(created)}` : "";
+  const lines = [`${prefix}**${author}** (score: ${score}${dateStr})`];
+
+  for (const line of body.split("\n")) {
+    lines.push(prefix ? `${prefix}${line}` : line);
+  }
+  lines.push("");
+
+  for (const child of comment.children || []) {
+    const rendered = renderComment(child, depth + 1, minScore);
+    if (rendered) lines.push(rendered);
+  }
+
+  return lines.join("\n");
+}
+
+function renderPost(post, comments, subreddit) {
+  const postId = post.id;
+  const title = post.title || "(untitled)";
+  const author = post.author || "[deleted]";
+  const score = post.score || 0;
+  const created = post.created_utc || 0;
+  const permalink = post.permalink || "";
+  const url = post.url || "";
+  const domain = post.domain || "";
+  const flair = post.link_flair_text || "";
+  const isSelf = post.is_self !== false;
+  const selftext = (post.selftext || "").trim();
+  const numComments = post.num_comments || 0;
+
+  const sourceUrl = permalink
+    ? `https://www.reddit.com${permalink}`
+    : `https://www.reddit.com/r/${subreddit}/comments/${postId}/`;
+
+  const fm = [
+    "---",
+    `source: ${escapeYaml(sourceUrl)}`,
+    `subreddit: ${subreddit}`,
+    `post_id: ${postId}`,
+    `author: ${escapeYaml(author)}`,
+    `score: ${score}`,
+    `created: ${formatDate(created)}`,
+    `comment_count: ${numComments}`,
+    `citation_key: reddit-${subreddit}-${postId}`,
+  ];
+  if (flair) fm.push(`flair: ${escapeYaml(flair)}`);
+  if (!isSelf && url) {
+    fm.push(`external_url: ${escapeYaml(url)}`);
+    fm.push(`domain: ${domain}`);
+  }
+  fm.push("---");
+
+  const body = [fm.join("\n"), "", `# ${title}`, ""];
+
+  if (!isSelf && url) {
+    body.push(`**Link:** [${domain}](${url})`, "");
+  }
+
+  if (selftext && selftext !== "[deleted]" && selftext !== "[removed]") {
+    body.push(selftext, "");
+  }
+
+  const tree = buildCommentTree(comments);
+  const rendered = tree
+    .map((c) => renderComment(c, 0, 3))
+    .filter((r) => r.trim());
+
+  if (rendered.length > 0) {
+    body.push("---", "", "## Comments", "", ...rendered);
+  }
+
+  return body.join("\n");
+}
+
+/* ── JSONL helpers ─────────────────────────────────────────────── */
+
+async function* readJsonl(filepath) {
+  const rl = createInterface({
+    input: createReadStream(filepath),
+    crlfDelay: Infinity,
+  });
+  for await (const line of rl) {
+    if (line.trim()) {
+      try {
+        yield JSON.parse(line);
+      } catch {
+        // skip malformed lines
+      }
+    }
+  }
+}
+
+/* ── Progress display ──────────────────────────────────────────── */
+
+function printProgress(label, current, total) {
+  if (total > 0) {
+    const pct = Math.round((current / total) * 100);
+    const filled = Math.round(pct / 5);
+    const bar = "\u2588".repeat(filled) + "\u2591".repeat(20 - filled);
+    process.stderr.write(`\r  ${label}: ${current.toLocaleString()} / ~${total.toLocaleString()} ${bar} ${pct}%`);
+  } else {
+    process.stderr.write(`\r  ${label}: ${current.toLocaleString()} downloaded...`);
+  }
+}
+
+/* ── Commands ──────────────────────────────────────────────────── */
+
+async function runCount(opts) {
+  const { subreddit } = opts;
+
+  console.error("\nFetching subreddit stats...");
+  const totalCounts = await getSubredditCounts(subreddit, null);
+  const filteredCounts = opts.since ? await getSubredditCounts(subreddit, opts.since) : totalCounts;
+
+  const estPosts = filteredCounts.totalPosts > 0 ? filteredCounts.totalPosts : totalCounts.totalPosts;
+  const estComments = filteredCounts.totalComments > 0 ? filteredCounts.totalComments : totalCounts.totalComments;
+  const estSizeMB = Math.round(
+    (estPosts * KB_PER_POST + estComments * KB_PER_COMMENT) / 1024
+  );
+
+  console.error(`  Total posts:    ~${totalCounts.totalPosts.toLocaleString()}`);
+  console.error(`  Total comments: ~${totalCounts.totalComments.toLocaleString()}`);
+  if (opts.since) {
+    console.error(`  Since ${opts.since}:    ~${estPosts.toLocaleString()} posts, ~${estComments.toLocaleString()} comments`);
+  }
+  console.error(`  Est. download: ~${estSizeMB >= 1024 ? (estSizeMB / 1024).toFixed(1) + " GB" : estSizeMB + " MB"}`);
+
+  console.log(
+    JSON.stringify({
+      subreddit,
+      total_posts: totalCounts.totalPosts,
+      total_comments: totalCounts.totalComments,
+      filtered_posts: estPosts,
+      filtered_comments: estComments,
+      estimated_size_mb: estSizeMB,
+      since: opts.since,
+    })
+  );
+}
+
+async function runDownload(opts) {
+  const { subreddit } = opts;
+  const rawDir = join(opts.output, "raw");
+  mkdirSync(rawDir, { recursive: true });
+
+  // Get estimated counts for progress display
+  console.error("\nFetching subreddit stats...");
+  const totalCounts = await getSubredditCounts(subreddit, null);
+  const filteredCounts = opts.since ? await getSubredditCounts(subreddit, opts.since) : totalCounts;
+  const estPosts = filteredCounts.totalPosts > 0 ? filteredCounts.totalPosts : totalCounts.totalPosts;
+  const estComments = filteredCounts.totalComments > 0 ? filteredCounts.totalComments : totalCounts.totalComments;
+
+  console.error(`  ~${estPosts.toLocaleString()} posts, ~${estComments.toLocaleString()} comments to download`);
+
+  // Download posts → raw/posts.jsonl
+  const postsPath = join(rawDir, "posts.jsonl");
+  const postsStream = createWriteStream(postsPath);
+  let postCount = 0;
+
+  console.error("\nDownloading posts...");
+  for await (const batch of paginateSearch(subreddit, "posts", opts.since)) {
+    for (const post of batch) {
+      postsStream.write(JSON.stringify(post) + "\n");
+    }
+    postCount += batch.length;
+    printProgress("Posts", postCount, estPosts);
+  }
+  postsStream.end();
+  console.error("");
+
+  // Download comments → raw/comments.jsonl
+  let commentCount = 0;
+  if (!opts.postsOnly) {
+    const commentsPath = join(rawDir, "comments.jsonl");
+    const commentsStream = createWriteStream(commentsPath);
+
+    console.error("Downloading comments...");
+    for await (const batch of paginateSearch(subreddit, "comments", opts.since)) {
+      for (const comment of batch) {
+        commentsStream.write(JSON.stringify(comment) + "\n");
+      }
+      commentCount += batch.length;
+      printProgress("Comments", commentCount, estComments);
+    }
+    commentsStream.end();
+    console.error("");
+  }
+
+  // Write download metadata
+  const metaPath = join(rawDir, "meta.json");
+  const meta = {
+    subreddit,
+    downloaded_at: new Date().toISOString(),
+    since: opts.since,
+    posts_downloaded: postCount,
+    comments_downloaded: commentCount,
+    posts_only: opts.postsOnly,
+  };
+  writeFileSync(metaPath, JSON.stringify(meta, null, 2));
+
+  console.error(`\n${"─".repeat(40)}`);
+  console.error(`Done. ${postCount.toLocaleString()} posts, ${commentCount.toLocaleString()} comments saved to ${rawDir}`);
+
+  console.log(JSON.stringify(meta));
+}
+
+async function runFilter(opts) {
+  const { subreddit } = opts;
+  const rawDir = join(opts.output, "raw");
+  const entriesDir = join(opts.output, "entries");
+
+  // Check raw data exists
+  const postsPath = join(rawDir, "posts.jsonl");
+  if (!existsSync(postsPath)) {
+    console.error(`Error: No raw data found at ${rawDir}`);
+    console.error(`Run 'node ingest.js ${subreddit} download' first.`);
+    process.exit(1);
+  }
+
+  // Load all posts
+  console.error("\nLoading posts...");
+  const allPosts = new Map();
+  for await (const post of readJsonl(postsPath)) {
+    allPosts.set(post.id, post);
+  }
+  console.error(`  ${allPosts.size.toLocaleString()} posts loaded`);
+
+  // Load comments grouped by post
+  const commentsByPost = new Map();
+  const commentsPath = join(rawDir, "comments.jsonl");
+  if (existsSync(commentsPath)) {
+    console.error("Loading comments...");
+    let commentCount = 0;
+    for await (const comment of readJsonl(commentsPath)) {
+      const linkId = comment.link_id || "";
+      if (!linkId.startsWith("t3_")) continue;
+      const postId = linkId.slice(3);
+      if (!commentsByPost.has(postId)) commentsByPost.set(postId, []);
+      commentsByPost.get(postId).push(comment);
+      commentCount++;
+    }
+    console.error(`  ${commentCount.toLocaleString()} comments loaded`);
+  }
+
+  // Score every post
+  console.error("\nScoring posts...");
+  const scored = [];
+  for (const [postId, post] of allPosts) {
+    const text = (post.selftext || "").trim();
+    if (text === "[deleted]" || text === "[removed]") continue;
+    if ((post.author || "") === "[deleted]") continue;
+
+    const quality = computeQualityScore(post, commentsByPost);
+    scored.push({ post, quality });
+  }
+  scored.sort((a, b) => b.quality - a.quality);
+
+  const allScores = scored.map((s) => s.quality);
+
+  // Compute stats for each quality level
+  const levels = ["high", "medium", "low", "all"];
+  const levelCounts = {};
+  for (const level of levels) {
+    const threshold = getQualityThreshold(level, allScores);
+    levelCounts[level] = scored.filter((s) => s.quality >= threshold).length;
+  }
+
+  // Show distribution
+  console.error(`\n  Quality distribution (${scored.length.toLocaleString()} scorable posts):`);
+  console.error(`    high   (top 10%):  ${levelCounts.high.toLocaleString()} posts`);
+  console.error(`    medium (top 30%):  ${levelCounts.medium.toLocaleString()} posts`);
+  console.error(`    low    (top 60%):  ${levelCounts.low.toLocaleString()} posts`);
+  console.error(`    all:               ${levelCounts.all.toLocaleString()} posts`);
+
+  // Show sample posts at each level
+  const showSample = (label, idx) => {
+    if (idx < scored.length) {
+      const s = scored[idx];
+      const title = (s.post.title || "").slice(0, 60);
+      const score = s.post.score || 0;
+      const comments = s.post.num_comments || 0;
+      console.error(`    ${label}: "${title}" (score: ${score}, comments: ${comments}, quality: ${s.quality.toFixed(2)})`);
+    }
+  };
+
+  console.error(`\n  Samples:`);
+  showSample("top     ", 0);
+  showSample("10%     ", Math.floor(scored.length * 0.1));
+  showSample("30%     ", Math.floor(scored.length * 0.3));
+  showSample("60%     ", Math.floor(scored.length * 0.6));
+  showSample("bottom  ", scored.length - 1);
+
+  if (opts.statsOnly) {
+    console.log(
+      JSON.stringify({
+        subreddit,
+        total_posts: allPosts.size,
+        scorable_posts: scored.length,
+        levels: levelCounts,
+        top_post: scored[0] ? { title: scored[0].post.title, score: scored[0].post.score, quality: scored[0].quality } : null,
+      })
+    );
+    return;
+  }
+
+  // Apply quality filter
+  const threshold = getQualityThreshold(opts.quality, allScores);
+  const qualifying = scored.filter((s) => s.quality >= threshold);
+
+  console.error(`\n  Filtering at "${opts.quality}" → ${qualifying.length.toLocaleString()} posts`);
+
+  // Write markdown entries
+  console.error("Writing entries...");
+  mkdirSync(entriesDir, { recursive: true });
+
+  let written = 0;
+  for (const { post } of qualifying) {
+    const title = post.title || "untitled";
+    const slug = slugify(title);
+    const filename = slug ? `${post.id}-${slug}.md` : `${post.id}.md`;
+    const filepath = join(entriesDir, filename);
+
+    const comments = commentsByPost.get(post.id) || [];
+    const md = renderPost(post, comments, subreddit);
+    writeFileSync(filepath, md);
+    written++;
+  }
+
+  // Initialize absorb log
+  const absorbLogPath = join(opts.output, "absorb_log.json");
+  if (!existsSync(absorbLogPath)) {
+    writeFileSync(
+      absorbLogPath,
+      JSON.stringify(
+        { entries: {}, stats: { total_entries: written, absorbed: 0, remaining: written } },
+        null,
+        2
+      )
+    );
+  } else {
+    // Update total count in existing log
+    const log = JSON.parse(readFileSync(absorbLogPath, "utf-8"));
+    log.stats.total_entries = written;
+    log.stats.remaining = written - log.stats.absorbed;
+    writeFileSync(absorbLogPath, JSON.stringify(log, null, 2));
+  }
+
+  console.error(`\n${"─".repeat(40)}`);
+  console.error(`Done. ${written.toLocaleString()} entries written to ${entriesDir}`);
+
+  console.log(
+    JSON.stringify({
+      subreddit,
+      total_posts: allPosts.size,
+      quality_level: opts.quality,
+      qualifying_posts: qualifying.length,
+      entries_written: written,
+      entries_dir: entriesDir,
+      absorb_log: absorbLogPath,
+    })
+  );
+}
+
+/* ── Main ──────────────────────────────────────────────────────── */
+
+async function main() {
+  const opts = parseArgs();
+
+  console.error(`\nReddit Ingest — r/${opts.subreddit}`);
+  console.error("─".repeat(40));
+
+  switch (opts.command) {
+    case "count":
+      await runCount(opts);
+      break;
+    case "download":
+      await runDownload(opts);
+      break;
+    case "filter":
+      await runFilter(opts);
+      break;
+    default:
+      console.error(`Unknown command: ${opts.command}`);
+      console.error("Use: download, filter, or count");
+      process.exit(1);
+  }
+}
+
+main().catch((err) => {
+  console.error(`\nError: ${err.message}`);
+  process.exit(1);
+});