mulmoclaude 0.1.0

package/server/workspace/sources/pipeline/plan.ts

@@ -0,0 +1,66 @@
+// Pipeline planner — pure.
+//
+// Given the current source registry + per-source state + the
+// schedule type being run (daily / hourly / weekly), decide
+// which sources should fetch this cycle:
+//
+//   1. Schedule match — a daily run skips hourly sources and
+//      vice versa. "on-demand" sources are never picked up by
+//      any scheduled run.
+//   2. Backoff respect — sources with a `nextAttemptAt` in the
+//      future are skipped until that time arrives, so a flapping
+//      source doesn't monopolize the rate limit.
+//
+// Separate module so tests can pin the filtering semantics
+// without touching the rest of the pipeline.
+
+import type { Source, SourceSchedule, SourceState } from "../types.js";
+
+export interface PlanInput {
+  sources: readonly Source[];
+  statesBySlug: ReadonlyMap<string, SourceState>;
+  // Schedule type this run is handling. The caller (task-manager
+  // cron or the manual `rebuild` endpoint) knows which kind it is.
+  scheduleType: SourceSchedule;
+  // Wall-clock ms since epoch. Passed in (rather than calling
+  // Date.now() internally) so tests can drive a deterministic
+  // clock.
+  nowMs: number;
+}
+
+// Sort key: slug, ascending. Deterministic ordering keeps the
+// daily summary's item sequence stable across runs for the same
+// input, which makes markdown diffs readable.
+function bySlug(a: Source, b: Source): number {
+  return a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0;
+}
+
+// Returns the subset of sources eligible for this cycle. Pure.
+export function planEligibleSources(input: PlanInput): Source[] {
+  const eligible: Source[] = [];
+  for (const source of input.sources) {
+    if (source.schedule !== input.scheduleType) continue;
+    if (!isWithinBackoff(input.statesBySlug.get(source.slug), input.nowMs)) {
+      eligible.push(source);
+    }
+  }
+  eligible.sort(bySlug);
+  return eligible;
+}
+
+// True when the state indicates the source is STILL in backoff
+// (so we should SKIP it). False means eligible to run now.
+//
+// - No state at all → run.
+// - No nextAttemptAt → run.
+// - nextAttemptAt unparseable → run (don't let a corrupt state
+//   file permanently lock out a source).
+// - nextAttemptAt in the future → skip.
+// - nextAttemptAt at or before now → run.
+function isWithinBackoff(state: SourceState | undefined, nowMs: number): boolean {
+  if (!state) return false;
+  if (!state.nextAttemptAt) return false;
+  const ts = Date.parse(state.nextAttemptAt);
+  if (!Number.isFinite(ts)) return false;
+  return ts > nowMs;
+}

package/server/workspace/sources/pipeline/summarize.ts

@@ -0,0 +1,189 @@
+// Daily-summary generator.
+//
+// Takes the cross-source-deduped list of new items and asks
+// `claude` (haiku, budget-capped) to produce the human-readable
+// daily brief markdown. The pipeline then pairs that markdown
+// with a machine-readable JSON block (see write.ts) so the
+// dashboard can consume item metadata without parsing markdown.
+//
+// Shape mirrors `chat-index/summarizer.ts` — same CLI flags, same
+// "errors on STDOUT not stderr" handling, same injectable
+// `SummarizeFn` so tests never invoke the real CLI.
+
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { ClaudeCliNotFoundError } from "../../journal/archivist.js";
+import { formatSpawnFailure } from "../../../utils/spawn.js";
+import { errorMessage } from "../../../utils/errors.js";
+import type { SourceItem } from "../types.js";
+import { CLI_SUBPROCESS_TIMEOUT_MS } from "../../../utils/time.js";
+
+// A function that takes items and returns markdown. The
+// production implementation spawns claude; tests pass a
+// deterministic stub.
+export type SummarizeFn = (items: readonly SourceItem[]) => Promise<string>;
+
+// Wall-clock cap per summarize call. 5 minutes is plenty for a
+// daily brief across a few dozen items; beyond that the call is
+// almost certainly wedged.
+export const DEFAULT_TIMEOUT_MS = CLI_SUBPROCESS_TIMEOUT_MS;
+
+// Budget per summarize call. A daily brief is longer and more
+// expensive than a classify call, so the cap is higher than the
+// classifier's $0.05. $0.25 covers several hundred items
+// comfortably.
+const MAX_BUDGET_USD = 0.25;
+
+const SYSTEM_PROMPT =
+  "You write a daily information brief from a JSON list of items. " +
+  "Group items by the `categories` field (one heading per category you see), " +
+  "sorted by the most items per category first; within each category, list newest-first by `publishedAt`. " +
+  "Use Markdown headings: `# Daily brief — YYYY-MM-DD` as the top heading, then `## <Category>` per group. " +
+  "Each item is one bullet: `- [title](url) — one-line summary`. " +
+  "Keep summaries under 140 characters. Prefer the item's own summary when present; otherwise paraphrase the title. " +
+  "Do NOT emit a JSON block, table of contents, or anything outside the brief itself — the caller appends machine-readable data separately. " +
+  "Output Markdown only — no code fences, no prose commentary.";
+
+// Shape passed to claude. Kept deliberately compact so the
+// prompt stays within budget even for a busy day: no `content`
+// field (full body), just `summary` truncated to 200 chars.
+interface PromptItem {
+  title: string;
+  url: string;
+  publishedAt: string;
+  categories: string[];
+  sourceSlug: string;
+  summary?: string;
+  severity?: string;
+}
+
+// Build the user-prompt JSON body. Exported so tests can verify
+// the exact shape the CLI sees, and so a future "generate a
+// test brief without summarizing" workflow can use the same
+// input format.
+export function buildSummarizePromptBody(items: readonly SourceItem[], isoDate: string): string {
+  const compactItems: PromptItem[] = items.map((item) => {
+    const base: PromptItem = {
+      title: item.title,
+      url: item.url,
+      publishedAt: item.publishedAt,
+      categories: [...item.categories],
+      sourceSlug: item.sourceSlug,
+    };
+    if (item.summary) base.summary = item.summary.slice(0, 200);
+    if (item.severity) base.severity = item.severity;
+    return base;
+  });
+  return [`DATE: ${isoDate}`, "", "ITEMS (JSON):", JSON.stringify(compactItems, null, 2)].join("\n");
+}
+
+// Fallback markdown when there are zero new items today.
+// Writing a file even on an empty day makes it clear the pipeline
+// ran; dashboards can still read it and show "no new items".
+export function buildEmptyDayMarkdown(isoDate: string): string {
+  return `# Daily brief — ${isoDate}\n\n_No new items today._\n`;
+}
+
+// Pure: parse the CLI envelope, surface structured errors,
+// return the markdown body.
+export function parseSummarizeOutput(stdout: string): string {
+  let parsed: {
+    is_error?: boolean;
+    result?: string;
+    errors?: unknown;
+  };
+  try {
+    parsed = JSON.parse(stdout.trim());
+  } catch (err) {
+    throw new Error(`[sources/summarize] failed to parse claude json: ${errorMessage(err)}`);
+  }
+  if (parsed.is_error) {
+    const msg = Array.isArray(parsed.errors) && parsed.errors.length > 0 ? parsed.errors.join("; ") : (parsed.result ?? "unknown");
+    throw new Error(`[sources/summarize] claude error: ${msg}`);
+  }
+  const result = typeof parsed.result === "string" ? parsed.result : "";
+  if (!result) {
+    throw new Error("[sources/summarize] claude returned empty / missing result");
+  }
+  return result;
+}
+
+// --- spawn layer --------------------------------------------------------
+
+function spawnClaudeSummarize(userPrompt: string, timeoutMs: number): Promise<string> {
+  return new Promise((resolve, reject) => {
+    // `--output-format json` returns a result envelope containing
+    // the model's text response as `.result` — we don't use
+    // `--json-schema` here because the model produces free-form
+    // markdown. Same "errors on stdout" handling as the
+    // classifier / chat-index summarizer.
+    const args = [
+      "--print",
+      "--no-session-persistence",
+      "--output-format",
+      "json",
+      "--model",
+      "haiku",
+      "--max-budget-usd",
+      String(MAX_BUDGET_USD),
+      "--system-prompt",
+      SYSTEM_PROMPT,
+      "-p",
+      userPrompt,
+    ];
+    const proc = spawn("claude", args, {
+      cwd: tmpdir(),
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+
+    const timer = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      proc.kill("SIGKILL");
+      reject(new Error(`[sources/summarize] claude timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+
+    proc.stdout.on("data", (chunk: Buffer) => {
+      stdout += chunk.toString();
+    });
+    proc.stderr.on("data", (chunk: Buffer) => {
+      stderr += chunk.toString();
+    });
+    proc.on("error", (err: Error & { code?: string }) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (err.code === "ENOENT") {
+        reject(new ClaudeCliNotFoundError());
+      } else {
+        reject(err);
+      }
+    });
+    proc.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (code !== 0) {
+        reject(new Error(formatSpawnFailure("[sources/summarize]", code, stdout, stderr)));
+        return;
+      }
+      resolve(stdout);
+    });
+  });
+}
+
+// Build the production SummarizeFn. `isoDate` is captured once
+// per pipeline run so every call in that run uses the same date
+// header (even if the run crosses midnight).
+export function makeDefaultSummarize(isoDate: string, timeoutMs: number = DEFAULT_TIMEOUT_MS): SummarizeFn {
+  return async (items) => {
+    if (items.length === 0) return buildEmptyDayMarkdown(isoDate);
+    const prompt = buildSummarizePromptBody(items, isoDate);
+    const stdout = await spawnClaudeSummarize(prompt, timeoutMs);
+    return parseSummarizeOutput(stdout);
+  };
+}

package/server/workspace/sources/pipeline/write.ts

@@ -0,0 +1,185 @@
+// Write-phase helpers for the source pipeline.
+//
+// Two responsibilities (kept pure where possible, with small
+// async wrappers for the filesystem side):
+//
+//   1. Compose the daily file content — the LLM-written markdown
+//      brief + a trailing fenced JSON block the dashboard reads
+//      without re-parsing markdown (#188 Q2).
+//   2. Write the daily file + append every item to its
+//      source-specific monthly archive under
+//      `archive/<slug>/YYYY/MM.md` (#188 Q4).
+
+import fsp from "node:fs/promises";
+import path from "node:path";
+import { archivePath, dailyNewsPath } from "../paths.js";
+import { errorMessage } from "../../../utils/errors.js";
+import type { SourceItem } from "../types.js";
+import { writeFileAtomic } from "../../../utils/files/index.js";
+
+// --- JSON index --------------------------------------------------------
+
+// Compact shape of the JSON block appended at the bottom of each
+// daily file. Mirrors what the dashboard (#143) expects:
+//   - itemCount: total items across all categories
+//   - byCategory: per-slug counts, sorted-by-count-desc for
+//     quick "which genre was hot today" read
+//   - items: the raw per-item metadata (no body, no summary) so
+//     the dashboard can render compact cards
+export interface DailyJsonIndex {
+  itemCount: number;
+  byCategory: Record<string, number>;
+  items: Array<{
+    id: string;
+    title: string;
+    url: string;
+    publishedAt: string;
+    categories: string[];
+    sourceSlug: string;
+    severity?: string;
+  }>;
+}
+
+export function buildDailyJsonIndex(items: readonly SourceItem[]): DailyJsonIndex {
+  const byCategory: Record<string, number> = {};
+  for (const item of items) {
+    for (const cat of item.categories) {
+      byCategory[cat] = (byCategory[cat] ?? 0) + 1;
+    }
+  }
+  return {
+    itemCount: items.length,
+    byCategory,
+    items: items.map((item) => ({
+      id: item.id,
+      title: item.title,
+      url: item.url,
+      publishedAt: item.publishedAt,
+      categories: [...item.categories],
+      sourceSlug: item.sourceSlug,
+      ...(item.severity !== undefined && { severity: item.severity }),
+    })),
+  };
+}
+
+// Assemble the full markdown file: claude's brief + a trailing
+// ```json block with the structured index. Pure.
+//
+// `markdown` is the LLM output verbatim (it's supposed to end
+// with a trailing newline already; we guard both cases). The
+// JSON block always ends with a final newline so editors don't
+// complain about "no newline at end of file".
+export function assembleDailyFile(markdown: string, items: readonly SourceItem[]): string {
+  const trimmed = markdown.endsWith("\n") ? markdown.slice(0, -1) : markdown;
+  const index = buildDailyJsonIndex(items);
+  // Pretty-printed JSON — the daily file is meant to be
+  // readable in a text editor as much as machine-consumable.
+  const json = JSON.stringify(index, null, 2);
+  return `${trimmed}\n\n\`\`\`json\n${json}\n\`\`\`\n`;
+}
+
+// --- daily file write ---------------------------------------------------
+
+// Atomic write: stage to a sibling `.tmp` then rename. Crash
+// mid-write can't leave a half-written daily file visible to
+// downstream readers.
+export async function writeDailyFile(workspaceRoot: string, isoDate: string, markdown: string, items: readonly SourceItem[]): Promise<string> {
+  const target = dailyNewsPath(workspaceRoot, isoDate);
+  await writeFileAtomic(target, assembleDailyFile(markdown, items));
+  return target;
+}
+
+// --- per-source archive append -----------------------------------------
+
+// Each item lands in its source's monthly archive file under
+// `archive/<slug>/YYYY/MM.md`. Per-month bucket chosen in
+// #188 Q4 — keeps single-year browsing doable without per-day
+// explosion.
+
+// Pure: render one item as the markdown block we append to the
+// archive. Exported for tests; idempotent-safe so re-appending
+// the same item produces the same bytes.
+export function renderItemForArchive(item: SourceItem): string {
+  const lines: string[] = [];
+  lines.push(`## ${item.title}`);
+  lines.push("");
+  lines.push(`- **Published:** ${item.publishedAt}`);
+  lines.push(`- **Source:** ${item.sourceSlug}`);
+  lines.push(`- **URL:** ${item.url}`);
+  if (item.categories.length > 0) {
+    lines.push(`- **Categories:** ${item.categories.join(", ")}`);
+  }
+  if (item.severity) {
+    lines.push(`- **Severity:** ${item.severity}`);
+  }
+  if (item.summary) {
+    lines.push("");
+    lines.push(item.summary);
+  }
+  if (item.content && item.content !== item.summary) {
+    lines.push("");
+    lines.push(item.content);
+  }
+  lines.push("");
+  lines.push("---");
+  lines.push("");
+  return lines.join("\n");
+}
+
+// ISO `publishedAt` → `YYYY-MM` used as the archive-month key.
+// Malformed dates fall back to the caller-supplied default
+// (typically the current YYYY-MM) so we don't drop items.
+export function archiveMonthFor(isoPublishedAt: string, fallbackMonth: string): string {
+  const ts = Date.parse(isoPublishedAt);
+  if (!Number.isFinite(ts)) return fallbackMonth;
+  const d = new Date(ts);
+  const year = d.getUTCFullYear();
+  const month = String(d.getUTCMonth() + 1).padStart(2, "0");
+  return `${year}-${String(month).padStart(2, "0")}`;
+}
+
+// Group items by (sourceSlug, YYYY-MM) so each destination file
+// gets one append instead of many. Exported so the orchestrator
+// can size the concurrency and tests can pin the bucketing logic.
+export function groupItemsForArchive(items: readonly SourceItem[], fallbackMonth: string): Map<string, SourceItem[]> {
+  const groups = new Map<string, SourceItem[]>();
+  for (const item of items) {
+    const month = archiveMonthFor(item.publishedAt, fallbackMonth);
+    const key = `${item.sourceSlug}::${month}`;
+    const existing = groups.get(key);
+    if (existing) existing.push(item);
+    else groups.set(key, [item]);
+  }
+  return groups;
+}
+
+// Append every item's rendered markdown block to the appropriate
+// `archive/<slug>/YYYY/MM.md`. Idempotency is the caller's
+// responsibility (deduping via `stableItemId` earlier in the
+// pipeline) — this helper blindly appends.
+//
+// Errors on individual source groups are collected, not thrown,
+// so one bad group can't lose the others. Returns the list of
+// successfully-written archive paths.
+export async function appendItemsToArchives(
+  workspaceRoot: string,
+  items: readonly SourceItem[],
+  fallbackMonth: string,
+): Promise<{ writtenPaths: string[]; errors: string[] }> {
+  const writtenPaths: string[] = [];
+  const errors: string[] = [];
+  const groups = groupItemsForArchive(items, fallbackMonth);
+  for (const [key, groupItems] of groups) {
+    const [slug, month] = key.split("::");
+    try {
+      const target = archivePath(workspaceRoot, slug, month);
+      await fsp.mkdir(path.dirname(target), { recursive: true });
+      const body = groupItems.map(renderItemForArchive).join("");
+      await fsp.appendFile(target, body, "utf-8");
+      writtenPaths.push(target);
+    } catch (err) {
+      errors.push(`[sources/archive] ${slug}/${month}: ${errorMessage(err)}`);
+    }
+  }
+  return { writtenPaths, errors };
+}

package/server/workspace/sources/rateLimiter.ts

@@ -0,0 +1,148 @@
+// Per-host rate limiter for outbound HTTP fetches.
+//
+// Design decision from #188 Q7: fetchers run parallel across hosts
+// but serial per host. This module is the "serial per host" part.
+// Two concurrent requests to the same hostname wait in FIFO order;
+// different hostnames proceed independently.
+//
+// Mechanism:
+//
+// - Per-host chain: one `Promise<void>` per hostname. Each new
+//   request `await`s the previous promise, runs the work, then
+//   releases the next waiter.
+// - Minimum delay: optional per-host floor between the end of one
+//   request and the start of the next. Defaults to
+//   DEFAULT_MIN_DELAY_MS to keep us well-behaved on cooperative
+//   hosts; robots.txt `Crawl-delay` callers pass their host's
+//   value.
+// - Clock-injectable: the `now` + `sleep` deps make this testable
+//   without wall-clock waits.
+//
+// Pure in the sense that all state is captured in the limiter
+// instance — no module-level globals. Tests can spin up a fresh
+// limiter per case and fully observe the ordering.
+
+import { ONE_SECOND_MS } from "../../utils/time.js";
+
+// Seconds of quiet between requests to the same host when the
+// caller doesn't specify an explicit `minDelayMs` for the host.
+// One second is polite-default for public feeds.
+export const DEFAULT_MIN_DELAY_MS = ONE_SECOND_MS;
+
+export interface RateLimiterDeps {
+  // Returns current wall-clock milliseconds. `Date.now` in prod,
+  // tests inject a controllable counter.
+  now: () => number;
+  // Sleep for `ms` milliseconds. `setTimeout` wrapper in prod,
+  // tests inject a faketimer-backed implementation.
+  sleep: (ms: number) => Promise<void>;
+}
+
+export function defaultRateLimiterDeps(): RateLimiterDeps {
+  return {
+    now: () => Date.now(),
+    sleep: (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)),
+  };
+}
+
+interface HostState {
+  // Tail of the per-host promise chain. New work waits for this,
+  // then replaces it with its own completion promise.
+  tail: Promise<void>;
+  // Wall-clock time the most-recent completed request returned,
+  // or `null` when no request has completed yet. Using `null`
+  // rather than 0 as the sentinel avoids ambiguity against fake
+  // test clocks that legitimately start at t=0.
+  lastFinishedAt: number | null;
+  // Number of tasks currently queued or in-flight for this host.
+  // Must hit zero before the host is eligible for eviction —
+  // otherwise evictIdle can delete a state with pending work,
+  // letting a later run() recreate a fresh chain and break serial
+  // ordering.
+  activeCount: number;
+}
+
+export class HostRateLimiter {
+  private readonly hosts = new Map<string, HostState>();
+  private readonly deps: RateLimiterDeps;
+
+  constructor(deps: RateLimiterDeps = defaultRateLimiterDeps()) {
+    this.deps = deps;
+  }
+
+  // Run `task` under the host's rate-limit slot. Resolves with the
+  // task's return value, or rejects with the task's error (without
+  // poisoning the queue — the next waiter still gets to run).
+  //
+  // `minDelayMs` is the minimum ms between the END of the previous
+  // request to this host and the START of this one. Defaults to
+  // DEFAULT_MIN_DELAY_MS.
+  run<T>(host: string, task: () => Promise<T>, minDelayMs: number = DEFAULT_MIN_DELAY_MS): Promise<T> {
+    const key = host.toLowerCase();
+    const state: HostState = this.hosts.get(key) ?? {
+      tail: Promise.resolve(),
+      lastFinishedAt: null,
+      activeCount: 0,
+    };
+    const prev = state.tail;
+
+    // Build the new tail: wait for prev, enforce delay, run task.
+    let resolveTail: () => void = () => {};
+    const newTail = new Promise<void>((resolve) => {
+      resolveTail = resolve;
+    });
+    state.tail = newTail;
+    state.activeCount++;
+    this.hosts.set(key, state);
+
+    return (async () => {
+      try {
+        await prev;
+        const wait = state.lastFinishedAt === null ? 0 : minDelayMs - (this.deps.now() - state.lastFinishedAt);
+        if (wait > 0) await this.deps.sleep(wait);
+        try {
+          return await task();
+        } finally {
+          // Mark finished time even on error so a flapping host
+          // doesn't get spammed with retries. Read state from the
+          // map (rather than the closure-captured variable) in
+          // case a parallel call has since updated it.
+          const fresh = this.hosts.get(key);
+          if (fresh) fresh.lastFinishedAt = this.deps.now();
+        }
+      } finally {
+        resolveTail();
+        const fresh = this.hosts.get(key);
+        if (fresh) fresh.activeCount--;
+      }
+    })();
+  }
+
+  // Test / debug: how many hosts currently have a chain. Returns
+  // 0 when the limiter is fresh.
+  hostCount(): number {
+    return this.hosts.size;
+  }
+
+  // Release internal state for hosts that have been idle longer
+  // than `idleMs`. Not strictly required for correctness (the
+  // map grows linearly with distinct hosts, which is bounded for
+  // any real workspace), but handy for long-lived processes.
+  evictIdle(idleMs: number): number {
+    const cutoff = this.deps.now() - idleMs;
+    let removed = 0;
+    for (const [key, state] of this.hosts) {
+      // Only evict states whose queue is empty. An idle
+      // `lastFinishedAt` alone isn't enough — if `tail` still has
+      // queued or in-flight work we'd delete live state and a
+      // later run() would recreate a fresh chain, breaking serial
+      // per-host ordering.
+      if (state.activeCount > 0) continue;
+      if (state.lastFinishedAt !== null && state.lastFinishedAt < cutoff) {
+        this.hosts.delete(key);
+        removed++;
+      }
+    }
+    return removed;
+  }
+}