mulmoclaude 0.1.0

package/server/workspace/sources/registry.ts

@@ -0,0 +1,326 @@
+// Read / write source registry files under `workspace/sources/`.
+//
+// On-disk format — one markdown file per source:
+//
+//   ---
+//   slug: hn-front-page
+//   title: Hacker News front page
+//   url: https://news.ycombinator.com/rss
+//   fetcher_kind: rss
+//   schedule: daily
+//   categories: [tech-news, general, english]
+//   max_items_per_fetch: 30
+//   added_at: 2026-04-13T09:00:00Z
+//   <fetcher-specific params as flat key: value>
+//   ---
+//
+//   # Notes
+//
+//   Free-form markdown body. Claude reads it for context when
+//   summarizing.
+//
+// Parser policy:
+//
+// - Flat YAML only. Nested mappings are not supported by design —
+//   the frontmatter is hand-edited by humans and the LLM, both of
+//   which routinely get nesting wrong. Fetcher params are flat
+//   strings (e.g. `github_repo: foo/bar`) so the fetcher itself
+//   interprets them.
+// - Unknown frontmatter keys are preserved as opaque strings in
+//   `fetcherParams`, so future fetchers can add fields without
+//   round-trip data loss.
+// - Missing required fields → the loader returns `null` and logs
+//   a warning; the caller skips that source rather than crashing
+//   the pass.
+//
+// The writer preserves the body text verbatim so re-saving a file
+// doesn't rewrite the user's notes.
+
+import fsp from "node:fs/promises";
+import { isFetcherKind, isSourceSchedule, type Source, type FetcherParams, type FetcherKind, type SourceSchedule } from "./types.js";
+import { normalizeCategories } from "./taxonomy.js";
+import type { CategorySlug } from "./taxonomy.js";
+import { writeFileAtomic } from "../../utils/files/index.js";
+import { sourceFilePath, sourcesRoot } from "./paths.js";
+import { isValidSlug } from "../../utils/slug.js";
+import { isNonEmptyString } from "../../utils/types.js";
+import { log } from "../../system/logger/index.js";
+
+// --- Frontmatter parsing ------------------------------------------------
+
+// Fields we recognize as first-class on every source. Anything else
+// in the frontmatter ends up in `fetcherParams` so a fetcher kind
+// that needs extra config can read it without us adding yet
+// another typed field for every new fetcher.
+const RESERVED_KEYS = new Set(["slug", "title", "url", "fetcher_kind", "schedule", "categories", "max_items_per_fetch", "added_at"]);
+
+const FRONTMATTER_OPEN = /^---\r?\n/;
+const FRONTMATTER_CLOSE = /\r?\n---\s*(\r?\n|$)/;
+
+interface ParsedFrontmatter {
+  fields: Map<string, string | string[]>;
+  body: string;
+}
+
+// Extract YAML frontmatter + body. Returns null when the file has
+// no frontmatter at all — that's an error condition for source
+// files (we always write frontmatter), not a degraded mode.
+export function parseSourceFile(raw: string): ParsedFrontmatter | null {
+  if (!FRONTMATTER_OPEN.test(raw)) return null;
+  const afterOpen = raw.replace(FRONTMATTER_OPEN, "");
+  const closeMatch = FRONTMATTER_CLOSE.exec(afterOpen);
+  if (!closeMatch || closeMatch.index === undefined) return null;
+  const fmText = afterOpen.slice(0, closeMatch.index);
+  const body = afterOpen.slice(closeMatch.index + closeMatch[0].length);
+  return { fields: parseFields(fmText), body };
+}
+
+function parseFields(fmText: string): Map<string, string | string[]> {
+  const fields = new Map<string, string | string[]>();
+  for (const line of fmText.split(/\r?\n/)) {
+    const parsed = parseLine(line);
+    if (parsed) fields.set(parsed.key, parsed.value);
+  }
+  return fields;
+}
+
+function parseLine(line: string): { key: string; value: string | string[] } | null {
+  if (!line.trim() || line.trimStart().startsWith("#")) return null;
+  const colonIdx = line.indexOf(":");
+  if (colonIdx <= 0) return null;
+  const key = line.slice(0, colonIdx).trim();
+  const rawValue = line.slice(colonIdx + 1).trim();
+  if (!key) return null;
+  return { key, value: parseValue(rawValue) };
+}
+
+function parseValue(raw: string): string | string[] {
+  if (!raw) return "";
+  const arrayMatch = /^\[(.*)\]$/.exec(raw);
+  if (arrayMatch) {
+    return arrayMatch[1]
+      .split(",")
+      .map((s) => unquote(s.trim()))
+      .filter((s) => s.length > 0);
+  }
+  return unquote(raw);
+}
+
+function unquote(s: string): string {
+  // Double-quoted strings: yamlScalar writes JSON-compatible escape
+  // sequences (\\ for \, \" for "), so JSON.parse reverses them in
+  // one shot. Fall back to a plain strip if the string is
+  // double-quoted but somehow malformed.
+  if (s.length >= 2 && s.startsWith('"') && s.endsWith('"')) {
+    try {
+      return JSON.parse(s);
+    } catch {
+      return s.slice(1, -1);
+    }
+  }
+  // Single-quoted scalars follow YAML's doubling convention: '' → '.
+  if (s.length >= 2 && s.startsWith("'") && s.endsWith("'")) {
+    return s.slice(1, -1).replace(/''/g, "'");
+  }
+  return s;
+}
+
+// --- Source validation / construction -----------------------------------
+
+function stringField(fields: Map<string, string | string[]>, key: string): string | null {
+  const v = fields.get(key);
+  return isNonEmptyString(v) ? v : null;
+}
+
+function numberField(fields: Map<string, string | string[]>, key: string, defaultValue: number): number {
+  const v = fields.get(key);
+  if (typeof v !== "string") return defaultValue;
+  const n = Number(v);
+  return Number.isFinite(n) && n > 0 ? Math.floor(n) : defaultValue;
+}
+
+// Default per-fetch cap. Fetchers treat it as a hint — if the
+// upstream API returns fewer items naturally the fetcher MAY
+// return fewer, but must NEVER return more than this.
+export const DEFAULT_MAX_ITEMS_PER_FETCH = 30;
+
+// Construct a Source from parsed frontmatter fields. Returns null
+// on required-field validation failure. The `body` arg is inlined
+// into the Source as `notes`.
+export function buildSource(fields: Map<string, string | string[]>, body: string): Source | null {
+  const slug = stringField(fields, "slug");
+  if (!slug || !isValidSlug(slug)) return null;
+
+  const title = stringField(fields, "title");
+  if (!title) return null;
+
+  const url = stringField(fields, "url");
+  if (!url) return null;
+
+  const fetcherKindRaw = stringField(fields, "fetcher_kind");
+  if (!isFetcherKind(fetcherKindRaw)) return null;
+  const fetcherKind: FetcherKind = fetcherKindRaw;
+
+  const scheduleRaw = stringField(fields, "schedule");
+  if (!isSourceSchedule(scheduleRaw)) return null;
+  const schedule: SourceSchedule = scheduleRaw;
+
+  const categoriesRaw = fields.get("categories");
+  const categories: CategorySlug[] = normalizeCategories(categoriesRaw);
+
+  const maxItemsPerFetch = numberField(fields, "max_items_per_fetch", DEFAULT_MAX_ITEMS_PER_FETCH);
+
+  const addedAt = stringField(fields, "added_at") ?? new Date(0).toISOString();
+
+  // Collect unrecognized fields into fetcherParams. Only flat
+  // string values — array values would indicate a schema mismatch
+  // since no fetcher param is a list today.
+  const fetcherParams: FetcherParams = {};
+  for (const [key, value] of fields.entries()) {
+    if (RESERVED_KEYS.has(key)) continue;
+    if (typeof value === "string") fetcherParams[key] = value;
+  }
+
+  return {
+    slug,
+    title,
+    url,
+    fetcherKind,
+    fetcherParams,
+    schedule,
+    categories,
+    maxItemsPerFetch,
+    addedAt,
+    notes: body,
+  };
+}
+
+// --- Serialization ------------------------------------------------------
+
+// Escape a scalar for use as a YAML value. Very conservative —
+// wraps in double-quotes whenever the value contains any character
+// that could be mis-parsed. Idempotent-safe: a round-trip through
+// parseValue → yamlScalar preserves the semantic string.
+function yamlScalar(value: string): string {
+  // Quote whenever the raw value contains characters that would
+  // confuse the flat-YAML parser or collide with a YAML reserved
+  // glyph. Numbers, dates, booleans, null all get quoted too so
+  // the reader always treats them as strings.
+  const needsQuote =
+    value === "" ||
+    /[:#[\]{},&*?|<>=!%@`]/.test(value) ||
+    /^\s|\s$/.test(value) ||
+    /^(true|false|null|~|yes|no|on|off)$/i.test(value) ||
+    /^[+-]?[\d.]/.test(value);
+  if (needsQuote) {
+    return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+  }
+  return value;
+}
+
+function yamlList(values: readonly string[]): string {
+  return `[${values.map(yamlScalar).join(", ")}]`;
+}
+
+// Serialize a Source back to the canonical markdown-with-
+// frontmatter shape. Reserved-key ordering is stable (nice for
+// diffs) and fetcher-specific params come after in alphabetical
+// order.
+export function serializeSource(source: Source): string {
+  const lines: string[] = [];
+  lines.push("---");
+  lines.push(`slug: ${yamlScalar(source.slug)}`);
+  lines.push(`title: ${yamlScalar(source.title)}`);
+  lines.push(`url: ${yamlScalar(source.url)}`);
+  lines.push(`fetcher_kind: ${yamlScalar(source.fetcherKind)}`);
+  lines.push(`schedule: ${yamlScalar(source.schedule)}`);
+  lines.push(`categories: ${yamlList(source.categories)}`);
+  lines.push(`max_items_per_fetch: ${String(source.maxItemsPerFetch)}`);
+  lines.push(`added_at: ${yamlScalar(source.addedAt)}`);
+  const paramKeys = Object.keys(source.fetcherParams).sort();
+  for (const key of paramKeys) {
+    lines.push(`${key}: ${yamlScalar(source.fetcherParams[key])}`);
+  }
+  lines.push("---");
+  lines.push("");
+  // Preserve trailing newline semantics — if the notes were empty,
+  // emit exactly one newline after the closing fence; otherwise
+  // append the notes verbatim.
+  if (source.notes.length > 0) {
+    lines.push(source.notes.endsWith("\n") ? source.notes : `${source.notes}\n`);
+  } else {
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+// --- Filesystem I/O -----------------------------------------------------
+
+// Load one source by slug. Returns null if missing, malformed, or
+// fails required-field validation. Never throws — consumer code
+// just skips null entries.
+export async function readSource(workspaceRoot: string, slug: string): Promise<Source | null> {
+  if (!isValidSlug(slug)) return null;
+  let raw: string;
+  try {
+    raw = await fsp.readFile(sourceFilePath(workspaceRoot, slug), "utf-8");
+  } catch {
+    return null;
+  }
+  const parsed = parseSourceFile(raw);
+  if (!parsed) return null;
+  const source = buildSource(parsed.fields, parsed.body);
+  // Sanity: filename slug must match frontmatter slug. A mismatch
+  // indicates the user renamed the file without editing the header
+  // (or vice-versa) — refuse the load rather than silently using
+  // the wrong slug.
+  if (source && source.slug !== slug) return null;
+  return source;
+}
+
+// List every source in the registry. Files that fail to parse are
+// logged and skipped; a single bad source file must not break the
+// daily pipeline for all the others.
+export async function listSources(workspaceRoot: string): Promise<Source[]> {
+  const dir = sourcesRoot(workspaceRoot);
+  let entries: string[];
+  try {
+    entries = await fsp.readdir(dir);
+  } catch {
+    return [];
+  }
+  const out: Source[] = [];
+  for (const name of entries) {
+    // Skip meta files and the `_state/` subdirectory.
+    if (name.startsWith("_")) continue;
+    if (!name.endsWith(".md")) continue;
+    const slug = name.slice(0, -".md".length);
+    const source = await readSource(workspaceRoot, slug);
+    if (source) out.push(source);
+    else {
+      log.warn("sources", "failed to load source, skipping", { slug });
+    }
+  }
+  // Deterministic sort by slug so callers can rely on stable order.
+  out.sort((a, b) => (a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0));
+  return out;
+}
+
+// Atomic write: stage to a sibling `.tmp` file then rename. Crash
+// mid-write cannot leave a half-written source file behind.
+export async function writeSource(workspaceRoot: string, source: Source): Promise<void> {
+  if (!isValidSlug(source.slug)) {
+    throw new Error(`[sources] invalid slug: ${source.slug}`);
+  }
+  await writeFileAtomic(sourceFilePath(workspaceRoot, source.slug), serializeSource(source));
+}
+
+export async function deleteSource(workspaceRoot: string, slug: string): Promise<boolean> {
+  if (!isValidSlug(slug)) return false;
+  try {
+    await fsp.unlink(sourceFilePath(workspaceRoot, slug));
+    return true;
+  } catch {
+    return false;
+  }
+}

package/server/workspace/sources/robots.ts

@@ -0,0 +1,271 @@
+// robots.txt parser + rule evaluator.
+//
+// Phase-1 fetchers call `isAllowedByRobots(robotsText, userAgent, path)`
+// before GET-ing a URL on a host we haven't fetched from recently.
+// The robots text itself comes from a 24h cache populated elsewhere —
+// this module only deals with parsing and rule evaluation.
+//
+// Supported directives:
+//   User-agent: <name>   — group selector
+//   Disallow:   <path>   — path-prefix block (empty value == allow all)
+//   Allow:      <path>   — path-prefix exception
+//   Crawl-delay: <secs>  — minimum seconds between fetches for this UA
+//
+// Deliberately NOT supported:
+//   Sitemap:             — irrelevant for fetchers
+//   Request-rate:        — rarely used in the wild
+//   Visit-time:          — same
+//   Host:                — Yandex-only extension
+//
+// Matching semantics follow the de-facto Google robots.txt rules
+// (draft IETF "robotstxt-00"): longest-prefix wins between Allow
+// and Disallow; `*` in paths is treated as a wildcard; `$` at
+// end-of-path anchors to end-of-URL.
+//
+// Pure — no I/O, no network, fully testable.
+
+export interface RobotsGroup {
+  // Lowercased user-agent names this group applies to. May contain
+  // "*" meaning "any agent not matched by a more-specific group".
+  userAgents: string[];
+  rules: RobotsRule[];
+  crawlDelaySec: number | null;
+}
+
+export interface RobotsRule {
+  kind: "allow" | "disallow";
+  // Raw path pattern as it appeared in the file, minus the leading
+  // directive. Wildcards preserved.
+  pattern: string;
+}
+
+export interface ParsedRobots {
+  groups: RobotsGroup[];
+}
+
+// Parse a robots.txt body into structured groups. Completely
+// lenient: unknown directives are skipped, malformed lines are
+// skipped, empty input yields an empty group list (which means
+// "everything allowed" downstream).
+export function parseRobots(text: string): ParsedRobots {
+  // State machine driven by one helper per directive kind — keeps
+  // the main loop free of nested branching.
+  //
+  // `collectingAgents` is true while we're inside a run of
+  // consecutive User-agent lines before the first rule. Additional
+  // User-agent lines extend the same group; once a rule appears,
+  // the next User-agent starts a new group.
+  const state: ParseState = {
+    groups: [],
+    current: null,
+    collectingAgents: false,
+  };
+  for (const rawLine of text.split(/\r?\n/)) {
+    const line = stripComment(rawLine).trim();
+    if (!line) continue;
+    const parsed = parseDirective(line);
+    if (!parsed) continue;
+    applyDirective(state, parsed.name, parsed.value);
+  }
+  return { groups: state.groups };
+}
+
+interface ParseState {
+  groups: RobotsGroup[];
+  current: RobotsGroup | null;
+  collectingAgents: boolean;
+}
+
+function applyDirective(state: ParseState, name: string, value: string): void {
+  if (name === "user-agent") {
+    applyUserAgent(state, value);
+    return;
+  }
+  // Any non-user-agent directive ends the "collecting agents"
+  // window. If we see a rule before any User-agent (malformed),
+  // drop it — robots.txt without a User-agent scope is
+  // meaningless.
+  state.collectingAgents = false;
+  if (!state.current) return;
+  applyRule(state.current, name, value);
+}
+
+function applyUserAgent(state: ParseState, value: string): void {
+  if (!state.collectingAgents || state.current === null) {
+    state.current = { userAgents: [], rules: [], crawlDelaySec: null };
+    state.groups.push(state.current);
+    state.collectingAgents = true;
+  }
+  state.current.userAgents.push(value.toLowerCase());
+}
+
+function applyRule(group: RobotsGroup, name: string, value: string): void {
+  if (name === "disallow") {
+    group.rules.push({ kind: "disallow", pattern: value });
+  } else if (name === "allow") {
+    group.rules.push({ kind: "allow", pattern: value });
+  } else if (name === "crawl-delay") {
+    const n = Number(value);
+    if (Number.isFinite(n) && n >= 0) group.crawlDelaySec = n;
+  }
+  // Any other directive: ignored.
+}
+
+function stripComment(line: string): string {
+  const hashIdx = line.indexOf("#");
+  return hashIdx === -1 ? line : line.slice(0, hashIdx);
+}
+
+function parseDirective(line: string): { name: string; value: string } | null {
+  const colonIdx = line.indexOf(":");
+  if (colonIdx <= 0) return null;
+  const name = line.slice(0, colonIdx).trim().toLowerCase();
+  const value = line.slice(colonIdx + 1).trim();
+  return { name, value };
+}
+
+// Pick the group whose User-agent directive best matches `userAgent`.
+// Ordering of preference:
+//   1. All exact-match groups (case-insensitive) merged together.
+//   2. All prefix-match groups tied for longest prefix, merged.
+//   3. All `*` groups merged together.
+//   4. No group at all → null (caller treats as "everything allowed").
+//
+// Per REP (IETF draft `robotstxt-00`): when multiple groups apply
+// equally to the same agent, their rules are combined into a single
+// rule set before the Allow/Disallow decision is made. Returning the
+// first match only (old behaviour) let a later Disallow in a
+// duplicate group get ignored, which could silently let a fetcher
+// hit a path the site explicitly blocked.
+export function selectGroup(robots: ParsedRobots, userAgent: string): RobotsGroup | null {
+  const ua = userAgent.toLowerCase();
+  const exacts: RobotsGroup[] = [];
+  const stars: RobotsGroup[] = [];
+  let bestPrefixScore = -1;
+  let prefixMatches: RobotsGroup[] = [];
+  for (const group of robots.groups) {
+    const outcome = scoreGroupAgainstAgent(group, ua);
+    if (outcome.kind === "exact") exacts.push(outcome.group);
+    else if (outcome.kind === "star") stars.push(outcome.group);
+    else if (outcome.kind === "prefix") {
+      if (outcome.score > bestPrefixScore) {
+        bestPrefixScore = outcome.score;
+        prefixMatches = [outcome.group];
+      } else if (outcome.score === bestPrefixScore) {
+        prefixMatches.push(outcome.group);
+      }
+    }
+  }
+  if (exacts.length > 0) return mergeGroups(exacts);
+  if (prefixMatches.length > 0) return mergeGroups(prefixMatches);
+  if (stars.length > 0) return mergeGroups(stars);
+  return null;
+}
+
+// Merge an array of groups into one. Concatenates rules (preserving
+// per-group order) and takes the smallest non-null crawlDelaySec
+// (the most conservative value) if any group specifies one.
+function mergeGroups(groups: readonly RobotsGroup[]): RobotsGroup {
+  if (groups.length === 1) return groups[0];
+  const rules: RobotsRule[] = [];
+  const userAgents: string[] = [];
+  let crawlDelaySec: number | null = null;
+  for (const g of groups) {
+    rules.push(...g.rules);
+    userAgents.push(...g.userAgents);
+    if (g.crawlDelaySec !== null) {
+      crawlDelaySec = crawlDelaySec === null ? g.crawlDelaySec : Math.min(crawlDelaySec, g.crawlDelaySec);
+    }
+  }
+  return { userAgents, rules, crawlDelaySec };
+}
+
+type AgentMatch =
+  | { kind: "exact"; group: RobotsGroup }
+  | { kind: "prefix"; group: RobotsGroup; score: number }
+  | { kind: "star"; group: RobotsGroup }
+  | { kind: "none" };
+
+function scoreGroupAgainstAgent(group: RobotsGroup, ua: string): AgentMatch {
+  let bestPrefix = -1;
+  let hasStar = false;
+  for (const listed of group.userAgents) {
+    if (listed === "*") {
+      hasStar = true;
+      continue;
+    }
+    if (listed === ua) return { kind: "exact", group };
+    if (ua.startsWith(listed) && listed.length > bestPrefix) {
+      bestPrefix = listed.length;
+    }
+  }
+  if (bestPrefix >= 0) return { kind: "prefix", group, score: bestPrefix };
+  if (hasStar) return { kind: "star", group };
+  return { kind: "none" };
+}
+
+// Decide whether `path` (the URL's path + query, e.g. "/a/b?c=d")
+// is permitted for `userAgent` by the parsed robots text. Returns
+// `true` when allowed, `false` when disallowed. Empty / missing
+// groups default to allowed, matching the robots.txt convention.
+//
+// Rule resolution follows the Google / IETF robots draft:
+// longest-prefix match wins between Allow and Disallow; tie goes
+// to Allow (the more permissive outcome).
+export function isAllowedByRobots(robots: ParsedRobots, userAgent: string, path: string): boolean {
+  const group = selectGroup(robots, userAgent);
+  if (!group) return true;
+  const { bestAllow, bestDisallow } = scoreRules(group, path);
+  if (bestAllow < 0 && bestDisallow < 0) return true;
+  return bestAllow >= bestDisallow;
+}
+
+// For each rule in `group`, compute the length of the longest
+// matching prefix; return the best Allow and Disallow lengths.
+// Returns -1 in either slot when no rule of that kind matched.
+// Empty patterns (from `Disallow:` with no value) never match in
+// `matchesPattern`, so they correctly fall through to the
+// allow-all default.
+function scoreRules(group: RobotsGroup, path: string): { bestAllow: number; bestDisallow: number } {
+  let bestAllow = -1;
+  let bestDisallow = -1;
+  for (const rule of group.rules) {
+    const matchLen = matchesPattern(rule.pattern, path);
+    if (matchLen < 0) continue;
+    if (rule.kind === "allow" && matchLen > bestAllow) bestAllow = matchLen;
+    else if (rule.kind === "disallow" && matchLen > bestDisallow) {
+      bestDisallow = matchLen;
+    }
+  }
+  return { bestAllow, bestDisallow };
+}
+
+// Returns the length of the matched prefix (for longest-prefix
+// arbitration), or -1 if the pattern doesn't match. An empty
+// pattern never matches (special-cased so `Disallow:` with empty
+// value doesn't block everything). Wildcards:
+//
+//   `*`  — matches any sequence of characters
+//   `$`  — at end of pattern, anchors the match to end-of-path
+//
+// The return value is the length of the pattern consumed (i.e.
+// pattern length with wildcards counted literally). This ranking
+// isn't perfect for patterns with multiple `*` but good enough
+// for real-world robots.txt where rule specificity rarely
+// depends on wildcard placement.
+export function matchesPattern(pattern: string, path: string): number {
+  if (pattern === "") return -1;
+  // Fast path: no wildcards means literal prefix match.
+  if (!pattern.includes("*") && !pattern.endsWith("$")) {
+    return path.startsWith(pattern) ? pattern.length : -1;
+  }
+  // Compile to a regex. Escape everything except `*` / end-$.
+  const endAnchored = pattern.endsWith("$");
+  const body = endAnchored ? pattern.slice(0, -1) : pattern;
+  const regexBody = body
+    .split("*")
+    .map((chunk) => chunk.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
+    .join(".*");
+  const re = new RegExp("^" + regexBody + (endAnchored ? "$" : ""));
+  return re.test(path) ? pattern.length : -1;
+}