mulmoclaude 0.1.0

package/server/workspace/sources/classifier.ts

@@ -0,0 +1,268 @@
+// Auto-categorizer for newly-registered sources.
+//
+// When a user registers a new source, the UI / CLI calls
+// `classifySource({ title, url, sampleTitles, sampleSummaries })`.
+// The classifier spawns `claude` with a strict JSON schema that
+// limits output to the fixed 25-slug taxonomy (see
+// server/sources/taxonomy.ts), so the model can't invent
+// `artificial-intelligence` when we already have `ai` — the whole
+// point of the closed enum.
+//
+// Shape of the spawn layer mirrors `server/chat-index/summarizer.ts`
+// so we reuse the "errors on stdout not stderr", "budget
+// exhaustion surfaces cleanly" behaviour that we already got
+// right once.
+//
+// Injection-friendly: production `classifySource` goes through
+// `defaultClassify`. Tests pass their own `ClassifyFn` that
+// skips the CLI entirely.
+
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { ClaudeCliNotFoundError } from "../journal/archivist.js";
+import { formatSpawnFailure } from "../../utils/spawn.js";
+import { ONE_MINUTE_MS } from "../../utils/time.js";
+import { CATEGORY_SLUGS, normalizeCategories, type CategorySlug } from "./taxonomy.js";
+import { errorMessage } from "../../utils/errors.js";
+import { isRecord } from "../../utils/types.js";
+
+// Structured input passed to the classifier. Kept small (not the
+// full source content) so the prompt stays cheap — a couple of
+// sample titles is enough for the model to tell "tech-news" from
+// "culture".
+export interface ClassifyInput {
+  title: string;
+  url: string;
+  // Optional extra context from a sample fetch — the first few
+  // item titles and (optionally) their summaries give the LLM
+  // enough signal to distinguish subject matter. When the
+  // pre-register handshake can't fetch yet (or the source is
+  // brand new), these can be empty arrays.
+  sampleTitles?: string[];
+  sampleSummaries?: string[];
+  // Optional human notes from the YAML frontmatter body — users
+  // sometimes describe why they registered the source.
+  notes?: string;
+}
+
+export interface ClassifyResult {
+  // 1-5 slugs from CATEGORY_SLUGS. Order matches model output
+  // (usually "most specific → most general"). Always validated
+  // through `normalizeCategories` so a hallucinated slug never
+  // leaks into the registry.
+  categories: CategorySlug[];
+  // Model's free-text one-line explanation — stored in the source
+  // file's notes body for human review.
+  rationale: string;
+}
+
+// Injection point for tests: any function matching this
+// signature is accepted by `classifySource`. Production passes
+// `defaultClassify`.
+export type ClassifyFn = (input: ClassifyInput) => Promise<ClassifyResult>;
+
+// Max time we let `claude` run during registration. Registration
+// is a foreground user action, so anything longer than 2 min is
+// effectively broken anyway.
+export const DEFAULT_TIMEOUT_MS = 2 * ONE_MINUTE_MS;
+
+// Budget cap. Classification is one small call per source (once
+// at registration, rarely re-classified) so $0.05 is fine — we
+// don't pay the first-call cache-creation cost on every source
+// because we warm it once and reuse.
+const MAX_BUDGET_USD = 0.05;
+
+const SYSTEM_PROMPT =
+  "You classify an information source (RSS feed, GitHub repo, web site, etc.) into a fixed taxonomy. " +
+  "You MUST pick between 1 and 5 categories from the provided enum — no synonyms, no new slugs, no invented labels. " +
+  "Choose the most specific matches first, then add broader ones only if they add signal. " +
+  "Output strict JSON matching the provided schema. Respond with structured output only.";
+
+// The category enum is inlined into the prompt schema so the
+// model sees the exact list of allowed values. Kept as a readonly
+// array so mutations at runtime are impossible.
+function classifySchema(): Record<string, unknown> {
+  return {
+    type: "object",
+    required: ["categories", "rationale"],
+    properties: {
+      categories: {
+        type: "array",
+        minItems: 1,
+        maxItems: 5,
+        items: { type: "string", enum: [...CATEGORY_SLUGS] },
+      },
+      rationale: {
+        type: "string",
+        maxLength: 200,
+      },
+    },
+  };
+}
+
+// Build the user-prompt text from the structured input. Pure —
+// exported so tests can pin the shape of what the model sees.
+export function buildClassifyPrompt(input: ClassifyInput): string {
+  const lines: string[] = [];
+  lines.push(`TITLE: ${input.title}`);
+  lines.push(`URL: ${input.url}`);
+  if (input.notes && input.notes.trim().length > 0) {
+    lines.push("");
+    lines.push("USER NOTES:");
+    lines.push(input.notes.trim().slice(0, 400));
+  }
+  const titles = input.sampleTitles ?? [];
+  if (titles.length > 0) {
+    lines.push("");
+    lines.push("RECENT ITEM TITLES:");
+    for (const t of titles.slice(0, 5)) {
+      lines.push(`- ${t}`);
+    }
+  }
+  const summaries = input.sampleSummaries ?? [];
+  if (summaries.length > 0) {
+    lines.push("");
+    lines.push("RECENT ITEM SUMMARIES:");
+    for (const s of summaries.slice(0, 3)) {
+      // One-line truncation so a single long abstract doesn't
+      // dominate the prompt budget.
+      lines.push(`- ${s.replace(/\s+/g, " ").slice(0, 200)}`);
+    }
+  }
+  return lines.join("\n");
+}
+
+interface ClaudeJsonResult {
+  type?: string;
+  is_error?: boolean;
+  structured_output?: unknown;
+  result?: string;
+}
+
+// Pure: parse the claude `--output-format json` envelope and
+// validate against our result shape. Exported so tests cover the
+// envelope-handling + normalization paths without spawning CLI.
+export function parseClassifyOutput(stdout: string): ClassifyResult {
+  let parsed: ClaudeJsonResult;
+  try {
+    parsed = JSON.parse(stdout.trim());
+  } catch (err) {
+    throw new Error(`[sources/classifier] failed to parse claude json: ${errorMessage(err)}`);
+  }
+  if (parsed.is_error) {
+    throw new Error(`[sources/classifier] claude returned error: ${parsed.result ?? "unknown"}`);
+  }
+  return validateClassifyResult(parsed.structured_output);
+}
+
+// Runtime-validate the `structured_output` field into a
+// ClassifyResult. Invalid categories are silently dropped
+// (normalizeCategories filter); rationale falls back to empty
+// string. Either the enum-constrained JSON schema OR this
+// validator would catch a bad slug — defense-in-depth.
+export function validateClassifyResult(obj: unknown): ClassifyResult {
+  // Arrays are `typeof === "object"` but aren't a valid
+  // structured_output shape — reject them explicitly so the
+  // error message stays accurate.
+  if (!isRecord(obj)) {
+    throw new Error("[sources/classifier] output is not an object");
+  }
+  const o = obj as Record<string, unknown>;
+  const categories = normalizeCategories(o.categories);
+  if (categories.length === 0) {
+    // The model is required to pick at least one (min_items=1 in
+    // the schema). If we end up here, something went wrong upstream
+    // — either the model ignored the schema or normalizeCategories
+    // filtered every slug as invalid. Throw so the caller treats
+    // the registration as failed rather than registering a source
+    // with no categories.
+    throw new Error("[sources/classifier] output has no valid categories from the taxonomy");
+  }
+  const rationale = typeof o.rationale === "string" ? o.rationale.slice(0, 400) : "";
+  return { categories, rationale };
+}
+
+// --- spawn layer --------------------------------------------------------
+
+function spawnClaudeClassify(userPrompt: string, timeoutMs: number): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const args = [
+      "--print",
+      "--no-session-persistence",
+      "--output-format",
+      "json",
+      "--model",
+      "haiku",
+      "--max-budget-usd",
+      String(MAX_BUDGET_USD),
+      "--json-schema",
+      JSON.stringify(classifySchema()),
+      "--system-prompt",
+      SYSTEM_PROMPT,
+      "-p",
+      userPrompt,
+    ];
+    // Run from tmpdir so claude doesn't load the project's
+    // CLAUDE.md / plugins / memory and inflate the context.
+    const proc = spawn("claude", args, {
+      cwd: tmpdir(),
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+
+    const timer = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      proc.kill("SIGKILL");
+      reject(new Error(`[sources/classifier] claude timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+
+    proc.stdout.on("data", (chunk: Buffer) => {
+      stdout += chunk.toString();
+    });
+    proc.stderr.on("data", (chunk: Buffer) => {
+      stderr += chunk.toString();
+    });
+    proc.on("error", (err: Error & { code?: string }) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (err.code === "ENOENT") {
+        reject(new ClaudeCliNotFoundError());
+      } else {
+        reject(err);
+      }
+    });
+    proc.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (code !== 0) {
+        // claude writes structured errors (incl.
+        // error_max_budget_usd) to STDOUT in JSON form — same
+        // lesson we learned in chat-index/summarizer. Prefer the
+        // structured message when we can parse it.
+        reject(new Error(formatSpawnFailure("[sources/classifier]", code, stdout, stderr)));
+        return;
+      }
+      resolve(stdout);
+    });
+  });
+}
+
+// Production ClassifyFn — spawns the real claude CLI.
+export const defaultClassify: ClassifyFn = async (input) => {
+  const userPrompt = buildClassifyPrompt(input);
+  const stdout = await spawnClaudeClassify(userPrompt, DEFAULT_TIMEOUT_MS);
+  return parseClassifyOutput(stdout);
+};
+
+// Public entry. Thin wrapper so tests can inject a ClassifyFn
+// without reaching into spawn internals, and the call site in
+// the manageSource plugin / pipeline stays a single symbol.
+export async function classifySource(input: ClassifyInput, classify: ClassifyFn = defaultClassify): Promise<ClassifyResult> {
+  return classify(input);
+}

package/server/workspace/sources/fetchers/arxiv.ts

@@ -0,0 +1,170 @@
+// arXiv fetcher.
+//
+// arXiv's query API returns Atom 1.0 XML, which parseFeed already
+// handles — so this fetcher is mostly "build URL + parse + filter
+// by cursor". The arxiv-specific value-adds are:
+//
+//   1. Query-URL builder with sensible defaults
+//      (sort=submittedDate, descending) and a defensive cap on
+//      max_results (arXiv's own cap is 2000 but our phase-1
+//      source model caps at maxItemsPerFetch)
+//   2. Separate cursor key so a source that migrates between
+//      fetcher kinds doesn't mishandle state
+//
+// Source config:
+//
+//   fetcher_kind: arxiv
+//   arxiv_query: cat:cs.CL OR ti:"large language model"
+//   arxiv_sort: submittedDate   # optional: submittedDate | lastUpdatedDate | relevance
+//   arxiv_order: descending     # optional: ascending | descending
+//
+// The `arxiv_query` string goes straight into the API's
+// `search_query` parameter (URL-encoded). arXiv's own query
+// syntax supports boolean AND/OR/ANDNOT, field prefixes
+// (ti:, au:, abs:, cat:, ...), and quoted phrases — we pass
+// through verbatim.
+
+import { normalizeUrl, stableItemId } from "../urls.js";
+import type { Source, SourceItem, SourceState } from "../types.js";
+import type { FetcherDeps, FetchResult, SourceFetcher } from "./index.js";
+import { registerFetcher } from "./index.js";
+import { fetchPolite } from "../httpFetcher.js";
+import { parseFeed, type ParsedFeed } from "./rssParser.js";
+
+export const ARXIV_CURSOR_KEY = "arxiv_last_published_at";
+
+export const ARXIV_API_BASE = "https://export.arxiv.org/api/query";
+
+const ALLOWED_SORT = new Set(["submittedDate", "lastUpdatedDate", "relevance"]);
+const ALLOWED_ORDER = new Set(["ascending", "descending"]);
+
+export class ArxivFetcherError extends Error {
+  readonly url: string;
+  readonly status: number | null;
+  constructor(url: string, status: number | null, message: string) {
+    super(message);
+    this.name = "ArxivFetcherError";
+    this.url = url;
+    this.status = status;
+  }
+}
+
+// Build the arXiv query URL. Validates and defaults sort / order
+// so a typo in a source file falls back to a safe configuration
+// rather than 400-ing from the API.
+export function arxivUrl(query: string, sort: string, order: string, maxResults: number): string {
+  const safeSort = ALLOWED_SORT.has(sort) ? sort : "submittedDate";
+  const safeOrder = ALLOWED_ORDER.has(order) ? order : "descending";
+  // arXiv caps at 2000 items per response; our maxItemsPerFetch
+  // is usually 30-ish but clamp defensively so a mistyped large
+  // value doesn't waste bandwidth.
+  const clamped = Math.max(1, Math.min(200, Math.floor(maxResults)));
+  const params = new URLSearchParams();
+  params.set("search_query", query);
+  params.set("start", "0");
+  params.set("max_results", String(clamped));
+  params.set("sortBy", safeSort);
+  params.set("sortOrder", safeOrder);
+  return `${ARXIV_API_BASE}?${params.toString()}`;
+}
+
+// Pure: given a parsed feed and the parent source, apply the
+// cursor filter and normalize to SourceItem[]. Capped at
+// `source.maxItemsPerFetch`. arXiv's feed items carry ISO
+// publishedAt so the same comparison semantics as RSS apply —
+// items at-or-older than the cursor are dropped.
+export function normalizeArxivFeed(feed: ParsedFeed, source: Source, cursor: Record<string, string>): SourceItem[] {
+  const lastSeenTs = parseCursorTs(cursor);
+  const items: SourceItem[] = [];
+  for (const entry of feed.items) {
+    if (items.length >= source.maxItemsPerFetch) break;
+    const item = feedItemToSourceItem(entry, source, lastSeenTs);
+    if (item) items.push(item);
+  }
+  return items;
+}
+
+// Extract the cursor's ISO timestamp into a ms-since-epoch number
+// usable for `<=` comparison. Returns null when absent or invalid
+// (either case means "no cursor filtering, emit everything").
+function parseCursorTs(cursor: Record<string, string>): number | null {
+  const raw = cursor[ARXIV_CURSOR_KEY];
+  if (!raw) return null;
+  const ts = Date.parse(raw);
+  return Number.isFinite(ts) ? ts : null;
+}
+
+// Decide whether one ParsedFeedItem produces a SourceItem given
+// the cursor. Returns null when the item should be skipped
+// (missing link, unparseable URL, at-or-older than cursor).
+// Extracted so `normalizeArxivFeed` stays under the cognitive-
+// complexity threshold.
+function feedItemToSourceItem(entry: ParsedFeed["items"][number], source: Source, lastSeenTs: number | null): SourceItem | null {
+  if (!entry.link) return null;
+  const normalizedUrl = normalizeUrl(entry.link);
+  if (!normalizedUrl) return null;
+  if (entry.publishedAt && lastSeenTs !== null) {
+    const ts = Date.parse(entry.publishedAt);
+    if (Number.isFinite(ts) && ts <= lastSeenTs) return null;
+  }
+  const publishedAt = entry.publishedAt ?? new Date().toISOString();
+  return {
+    id: stableItemId(normalizedUrl),
+    title: entry.title,
+    url: normalizedUrl,
+    publishedAt,
+    ...(entry.summary !== null && { summary: entry.summary }),
+    ...(entry.content !== null && { content: entry.content }),
+    categories: source.categories,
+    sourceSlug: source.slug,
+  };
+}
+
+// Advance the cursor to the newest publishedAt across the parsed
+// feed (not just the emitted items), same pattern as the RSS /
+// GitHub fetchers so a quiet arXiv query doesn't keep re-emitting
+// the same papers after a one-off republish.
+export function updateArxivCursor(current: Record<string, string>, feed: ParsedFeed): Record<string, string> {
+  let newest: number | null = null;
+  for (const entry of feed.items) {
+    if (!entry.publishedAt) continue;
+    const ts = Date.parse(entry.publishedAt);
+    if (!Number.isFinite(ts)) continue;
+    if (newest === null || ts > newest) newest = ts;
+  }
+  if (newest === null) return current;
+  const currentTs = current[ARXIV_CURSOR_KEY] ? Date.parse(current[ARXIV_CURSOR_KEY]) : -Infinity;
+  if (newest <= currentTs) return current;
+  return {
+    ...current,
+    [ARXIV_CURSOR_KEY]: new Date(newest).toISOString(),
+  };
+}
+
+export const arxivFetcher: SourceFetcher = {
+  kind: "arxiv",
+  async fetch(source: Source, state: SourceState, deps: FetcherDeps): Promise<FetchResult> {
+    const query = source.fetcherParams["arxiv_query"];
+    if (typeof query !== "string" || query.trim().length === 0) {
+      throw new ArxivFetcherError(source.url, null, "arxiv_query param is required");
+    }
+    const sort = source.fetcherParams["arxiv_sort"] ?? "submittedDate";
+    const order = source.fetcherParams["arxiv_order"] ?? "descending";
+    const url = arxivUrl(query, sort, order, source.maxItemsPerFetch);
+    const res = await fetchPolite(url, deps.http);
+    if (!res.ok) {
+      throw new ArxivFetcherError(url, res.status, `arXiv fetch failed with HTTP ${res.status}`);
+    }
+    const body = await res.text();
+    const feed = parseFeed(body);
+    if (!feed) {
+      throw new ArxivFetcherError(url, res.status, `arXiv response did not parse as Atom / RSS`);
+    }
+    return {
+      items: normalizeArxivFeed(feed, source, state.cursor),
+      cursor: updateArxivCursor(state.cursor, feed),
+    };
+  },
+};
+
+registerFetcher(arxivFetcher);

package/server/workspace/sources/fetchers/github.ts

@@ -0,0 +1,106 @@
+// Shared helpers for GitHub fetchers (releases + issues).
+//
+// Phase-1 scope: UNAUTHENTICATED public REST access only. GitHub
+// grants 60 req/hour/IP without a token, which is plenty for a
+// personal workspace with a handful of registered repos. Adding
+// a `github-authed` fetcher with a PAT lives in phase 3.
+//
+// Everything in this module is pure or uses the already-injected
+// `fetchPolite` — no direct HTTP. Tests stub HTTP at the
+// FetcherDeps boundary.
+
+import type { HttpFetcherDeps } from "../httpFetcher.js";
+import { fetchPolite } from "../httpFetcher.js";
+import { hasStringProp } from "../../../utils/types.js";
+
+// GitHub REST API base. Factored out so tests / local dev can
+// point at a stub server by patching this module — rare enough
+// that we don't bother with an env var today.
+export const GITHUB_API_BASE = "https://api.github.com";
+
+// Owner / repo slugs on GitHub accept letters, digits, hyphen,
+// underscore, dot. Lengths are loose but we cap at 100 each —
+// any real repo comes in well under that.
+//
+// The slug doubles as a URL path segment: rejecting `..` / `/`
+// / whitespace defends against a user-supplied (or LLM-suggested)
+// `github_repo` that would craft a malicious request URL.
+// Owner must start with an alphanumeric (GitHub usernames /
+// org names can't begin with a dot).
+const OWNER_SEGMENT_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/;
+// Repo can start with a dot — GitHub has the special `.github`
+// repository used for org-wide community health files, so the
+// repo-name regex is slightly looser than the owner one.
+const REPO_SEGMENT_RE = /^[A-Za-z0-9.][A-Za-z0-9._-]{0,99}$/;
+
+export interface RepoSlug {
+  owner: string;
+  repo: string;
+}
+
+// Validate + parse an `owner/repo` string from source
+// frontmatter. Returns null on any shape violation — callers
+// treat null as "skip this source" rather than crashing the pass.
+export function parseRepoSlug(raw: string): RepoSlug | null {
+  if (typeof raw !== "string") return null;
+  const parts = raw.trim().split("/");
+  if (parts.length !== 2) return null;
+  const [owner, repo] = parts;
+  if (!OWNER_SEGMENT_RE.test(owner) || !REPO_SEGMENT_RE.test(repo)) return null;
+  // Reject path-traversal-ish repo names even though the regex
+  // would accept them (`.`, `..` can both start with a dot).
+  if (repo === "." || repo === "..") return null;
+  if (owner.endsWith(".") || repo.endsWith(".")) return null;
+  return { owner, repo };
+}
+
+// Named error for any non-2xx GitHub response. Carries the status
+// code so the pipeline can decide whether to backoff harder (403
+// rate-limit) or surface a "source broken" warning (404 /
+// repo-not-found).
+export class GithubFetcherError extends Error {
+  readonly url: string;
+  readonly status: number;
+  readonly apiMessage: string | null;
+  constructor(url: string, status: number, apiMessage: string | null) {
+    const suffix = apiMessage ? ` — ${apiMessage}` : "";
+    super(`GitHub fetch ${url} failed with HTTP ${status}${suffix}`);
+    this.name = "GithubFetcherError";
+    this.url = url;
+    this.status = status;
+    this.apiMessage = apiMessage;
+  }
+}
+
+// Tight helper: issue a GET against the GitHub API, decode the
+// JSON body, and surface non-2xx responses as typed errors.
+// Errors from fetchPolite (including `RobotsDisallowedError`) pass
+// through untouched.
+//
+// GitHub API responses always include an `X-GitHub-Request-Id` and
+// many include a `message` body field on errors. We include the
+// body message in the thrown error for easier log reading.
+export async function githubFetchJson(url: string, http: HttpFetcherDeps): Promise<unknown> {
+  const res = await fetchPolite(url, http);
+  if (!res.ok) {
+    // Body may or may not be JSON — try both. Not throwing on
+    // a failed body read so the HTTP status is always reported.
+    let apiMessage: string | null = null;
+    try {
+      const bodyJson: unknown = await res.json();
+      if (hasStringProp(bodyJson, "message")) {
+        apiMessage = bodyJson.message;
+      }
+    } catch {
+      // Ignore — just means the body wasn't JSON.
+    }
+    throw new GithubFetcherError(url, res.status, apiMessage);
+  }
+  return res.json();
+}
+
+// Small type-guard used by the json-shape parsers below. Keeping
+// it here (rather than duplicated per fetcher) so adding a new
+// GitHub endpoint stays boilerplate-free.
+// Re-export so existing callers that import from this module keep working.
+export { isRecord } from "../../../utils/types.js";