mulmoclaude 0.1.0

package/server/workspace/sources/sourceState.ts

@@ -0,0 +1,135 @@
+// Per-source runtime state I/O.
+//
+// State lives at `workspace/sources/_state/<slug>.json`, kept
+// separate from the source config (`<slug>.md`) so the config
+// is the git-tracked source of truth while state can grow /
+// reset / get cleared without touching committed history.
+//
+// All functions take an explicit `workspaceRoot` so tests use
+// mkdtempSync without touching real workspace state.
+
+import fsp from "node:fs/promises";
+import { defaultSourceState, type SourceState } from "./types.js";
+import { sourceStatePath } from "./paths.js";
+import { isValidSlug } from "../../utils/slug.js";
+import { errorMessage } from "../../utils/errors.js";
+import { writeJsonAtomic } from "../../utils/files/index.js";
+import { isRecord } from "../../utils/types.js";
+
+// Shallow-parse + type-guard one state record. Returns a
+// default state (zeroed counters, empty cursor) when the file
+// is missing, malformed, or any required field fails. Never
+// throws.
+export async function readSourceState(workspaceRoot: string, slug: string): Promise<SourceState> {
+  if (!isValidSlug(slug)) return defaultSourceState(slug);
+  let raw: string;
+  try {
+    raw = await fsp.readFile(sourceStatePath(workspaceRoot, slug), "utf-8");
+  } catch {
+    return defaultSourceState(slug);
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return defaultSourceState(slug);
+  }
+  return validateSourceState(parsed, slug);
+}
+
+// Runtime-validate an arbitrary parse result into a
+// SourceState. Unknown fields are dropped; missing fields get
+// default values; wrong-typed fields collapse to the default.
+// Defensive: a hand-edited / corrupted state file should NOT
+// crash the pipeline, it should quietly get rebuilt on the
+// next successful run.
+export function validateSourceState(raw: unknown, slug: string): SourceState {
+  if (!isRecord(raw)) {
+    return defaultSourceState(slug);
+  }
+  const o = raw as Record<string, unknown>;
+  const lastFetchedAt = typeof o.lastFetchedAt === "string" ? o.lastFetchedAt : null;
+  const nextAttemptAt = typeof o.nextAttemptAt === "string" ? o.nextAttemptAt : null;
+  const consecutiveFailures =
+    typeof o.consecutiveFailures === "number" && Number.isFinite(o.consecutiveFailures) && o.consecutiveFailures >= 0 ? Math.floor(o.consecutiveFailures) : 0;
+  const cursor = validateCursor(o.cursor);
+  return {
+    slug,
+    lastFetchedAt,
+    cursor,
+    consecutiveFailures,
+    nextAttemptAt,
+  };
+}
+
+function validateCursor(raw: unknown): Record<string, string> {
+  if (!isRecord(raw)) {
+    return {};
+  }
+  const out: Record<string, string> = {};
+  for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
+    if (typeof value === "string") out[key] = value;
+  }
+  return out;
+}
+
+// Atomic write: stage to `.tmp` then rename. Parent directory
+// created as needed.
+export async function writeSourceState(workspaceRoot: string, state: SourceState): Promise<void> {
+  if (!isValidSlug(state.slug)) {
+    throw new Error(`[sources/state] invalid slug: ${state.slug}`);
+  }
+  await writeJsonAtomic(sourceStatePath(workspaceRoot, state.slug), state);
+}
+
+// Convenience: read every state file listed for the given
+// slugs. Used by the pipeline to gather per-source state before
+// the fetch phase.
+export async function readManyStates(workspaceRoot: string, slugs: readonly string[]): Promise<Map<string, SourceState>> {
+  const out = new Map<string, SourceState>();
+  const reads = await Promise.all(slugs.map((slug) => readSourceState(workspaceRoot, slug)));
+  for (const state of reads) out.set(state.slug, state);
+  return out;
+}
+
+// Convenience: write every state back in parallel. Failure on
+// one state write is logged and absorbed — the daily run's
+// summary has already landed on disk, a lost state update just
+// means the next run re-fetches slightly more than needed.
+export async function writeManyStates(workspaceRoot: string, states: readonly SourceState[]): Promise<{ written: number; errors: string[] }> {
+  const errors: string[] = [];
+  let written = 0;
+  for (const state of states) {
+    try {
+      await writeSourceState(workspaceRoot, state);
+      written++;
+    } catch (err) {
+      errors.push(`[sources/state] ${state.slug}: ${errorMessage(err)}`);
+    }
+  }
+  return { written, errors };
+}
+
+// Delete the state file for a slug. Used by `manageSource
+// delete` so a removed source doesn't leave orphan state.
+// Missing file is fine — returns false rather than throwing.
+export async function deleteSourceState(workspaceRoot: string, slug: string): Promise<boolean> {
+  if (!isValidSlug(slug)) return false;
+  try {
+    await fsp.unlink(sourceStatePath(workspaceRoot, slug));
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// Utility: sort outcomes by source slug so results are
+// deterministic regardless of which fetcher finished first.
+// Used by reporting / logging code.
+export function sortBySlug<T extends { sourceSlug?: string; slug?: string }>(items: readonly T[]): T[] {
+  return [...items].sort((a, b) => {
+    const ak = a.sourceSlug ?? a.slug ?? "";
+    const bk = b.sourceSlug ?? b.slug ?? "";
+    return ak < bk ? -1 : ak > bk ? 1 : 0;
+  });
+}

package/server/workspace/sources/taxonomy.ts

@@ -0,0 +1,74 @@
+// Fixed taxonomy of source categories. Keeping this a closed enum
+// (rather than accepting free-form LLM-generated tags) prevents
+// synonym sprawl — `ai` vs `artificial-intelligence` vs `AI` would
+// otherwise all coexist and make filtering useless.
+//
+// The auto-categorizer (see plans/feat-source-registry.md §"Auto-
+// categorization") classifies each new source into 1-5 of these
+// slugs and writes them into the source file's frontmatter. Users
+// can override by editing the file; the next daily run picks up
+// the edits.
+//
+// Pin-tested so a silent enum mutation doesn't sneak past review.
+
+export const CATEGORY_SLUGS = [
+  "tech-news",
+  "business-news",
+  "ai",
+  "security",
+  "devops",
+  "frontend",
+  "backend",
+  "ml-research",
+  "dependencies",
+  "product-updates",
+  "japanese",
+  "english",
+  "papers",
+  "general",
+  "startup",
+  "personal",
+  // --- Phase-1 expansion (resolved from #188 open-question Q1) ---
+  // Added to cover common genres the original 16 couldn't capture
+  // (which were tech-centric and collapsed everything non-tech
+  // into `general`). See plans/feat-source-registry.md §Resolved
+  // decisions for rationale per slug.
+  "finance",
+  "design",
+  "productivity",
+  "science",
+  "health",
+  "gaming",
+  "climate",
+  "culture",
+  "policy",
+] as const;
+
+export type CategorySlug = (typeof CATEGORY_SLUGS)[number];
+
+const CATEGORY_SET: ReadonlySet<string> = new Set(CATEGORY_SLUGS);
+
+// Runtime type-guard. Used when reading a source file from disk to
+// drop any legacy / typo categories so downstream code only ever
+// deals in the current enum.
+export function isCategorySlug(value: unknown): value is CategorySlug {
+  return typeof value === "string" && CATEGORY_SET.has(value);
+}
+
+// Normalize an unknown list of category candidates into a clean,
+// deduplicated array of valid slugs. Used when reading from
+// frontmatter (where the user may have typo'd) and when receiving
+// classifier output (where the LLM may have hallucinated a slug
+// outside the taxonomy).
+export function normalizeCategories(raw: unknown): CategorySlug[] {
+  if (!Array.isArray(raw)) return [];
+  const seen = new Set<CategorySlug>();
+  const out: CategorySlug[] = [];
+  for (const item of raw) {
+    if (isCategorySlug(item) && !seen.has(item)) {
+      seen.add(item);
+      out.push(item);
+    }
+  }
+  return out;
+}

package/server/workspace/sources/types.ts

@@ -0,0 +1,144 @@
+// Data model for the information-source registry. Every source
+// lives as one markdown file under `workspace/sources/<slug>.md`,
+// with these fields in the YAML frontmatter plus optional free-form
+// markdown notes in the body.
+//
+// Design invariants the consumer code relies on:
+//
+// - `slug` is the primary key and matches the filename exactly.
+//   Enforced by `registry.ts` on both read and write.
+// - `url` is always the normalized form (see urls.ts) so dedup
+//   across sources works by string equality.
+// - `fetcherKind` is one of a closed set so the fetcher dispatcher
+//   can look up the right handler without `any`.
+// - `schedule` drives the daily / weekly aggregation pipeline.
+// - `categories` contains only valid CategorySlug values
+//   (runtime-validated on read).
+//
+// Secrets (API tokens, bearer auth) are NEVER stored here —
+// phase-1 scope is public sources only. Phase-3 authed fetchers
+// will read credentials from `.env` at runtime by name; the name
+// reference lives in `fetcherParams` as an `envVar` field.
+
+import type { CategorySlug } from "./taxonomy.js";
+
+// Closed set of fetcher kinds we can dispatch on. Adding a new
+// fetcher means: add the string literal here, implement a matching
+// module under `server/sources/fetchers/<kind>.ts`, and register
+// it in the fetcher index. Nothing else in the framework needs to
+// change.
+//
+// Phase-1 surface:
+//   "rss"                — public RSS / Atom feeds (server-side fetch)
+//   "github-releases"    — GitHub /releases endpoint, unauthenticated
+//   "github-issues"      — GitHub /issues + /pulls, unauthenticated
+//   "arxiv"              — arXiv query API
+//   "web-fetch"          — one-shot page fetch via Claude's web_fetch
+//   "web-search"         — ad-hoc query via Claude's web_search
+export const FETCHER_KINDS = ["rss", "github-releases", "github-issues", "arxiv", "web-fetch", "web-search"] as const;
+
+export type FetcherKind = (typeof FETCHER_KINDS)[number];
+
+const FETCHER_KIND_SET: ReadonlySet<string> = new Set(FETCHER_KINDS);
+
+export function isFetcherKind(value: unknown): value is FetcherKind {
+  return typeof value === "string" && FETCHER_KIND_SET.has(value);
+}
+
+// How often the daily pipeline is expected to refresh this source.
+// `on-demand` sources are never auto-fetched; they only respond to
+// the `manageSource fetch` action or the on-demand research
+// workflow.
+export const SOURCE_SCHEDULES = ["hourly", "daily", "weekly", "on-demand"] as const;
+
+export type SourceSchedule = (typeof SOURCE_SCHEDULES)[number];
+
+const SOURCE_SCHEDULE_SET: ReadonlySet<string> = new Set(SOURCE_SCHEDULES);
+
+export function isSourceSchedule(value: unknown): value is SourceSchedule {
+  return typeof value === "string" && SOURCE_SCHEDULE_SET.has(value);
+}
+
+// Per-fetcher extra parameters carried on the Source file. Flat
+// string map on disk so the minimal frontmatter parser can handle
+// it without nested YAML. Fetchers interpret the keys they care
+// about and ignore the rest — keeps cross-fetcher rewiring cheap.
+export type FetcherParams = Record<string, string>;
+
+// The on-disk configuration for one source. This is the exact
+// shape serialized into the YAML frontmatter of
+// `workspace/sources/<slug>.md` — state (cursors, etags, failure
+// counts) lives separately under `_state/<slug>.json`.
+export interface Source {
+  slug: string;
+  title: string;
+  url: string;
+  fetcherKind: FetcherKind;
+  fetcherParams: FetcherParams;
+  schedule: SourceSchedule;
+  categories: CategorySlug[];
+  maxItemsPerFetch: number;
+  addedAt: string; // ISO timestamp
+  notes: string; // markdown body of the file
+}
+
+// One normalized item after a fetch. All fetchers produce this
+// shape regardless of source type so the pipeline / dedup / summary
+// layers don't care where items came from.
+export interface SourceItem {
+  // Stable unique id for dedup. Hash of the normalized URL, or
+  // the fetcher's native id (e.g. GitHub release id) when one is
+  // available.
+  id: string;
+  title: string;
+  url: string;
+  publishedAt: string; // ISO timestamp
+  // Short one-line summary if the fetcher can produce one without
+  // LLM help. The pipeline's summarize step may replace this with
+  // a richer LLM-generated version.
+  summary?: string;
+  // Full body content if available (RSS description, GitHub release
+  // body, etc.). The summarize step reads this when the short
+  // summary is insufficient.
+  content?: string;
+  // Categories inherited from the source this item came from.
+  // Duplicated on the item so per-category daily rollups don't
+  // need a re-join.
+  categories: CategorySlug[];
+  // Slug of the parent Source so the dashboard / notification
+  // layer can link back.
+  sourceSlug: string;
+  // Optional severity hint set by the classifier or the fetcher
+  // itself (security advisories set `critical`). Daily pipeline
+  // uses this to decide whether to notify.
+  severity?: "info" | "warn" | "critical";
+}
+
+// Per-source runtime state, NOT committed to git. Mirrors the
+// Source-vs-_state split described in plans/feat-source-registry.md.
+export interface SourceState {
+  slug: string;
+  // Last successful fetch.
+  lastFetchedAt: string | null;
+  // Fetcher-specific cursor — ISO timestamp, etag, GitHub release
+  // id, arXiv last-seen, whatever the fetcher persists to
+  // de-duplicate across runs. Free-form string map so the fetcher
+  // interface doesn't need to know the shape upfront.
+  cursor: Record<string, string>;
+  // Consecutive failure count. Incremented per failed fetch,
+  // reset to 0 on success. Drives exponential backoff.
+  consecutiveFailures: number;
+  // Timestamp after which the next attempt is allowed, so backoff
+  // survives server restarts.
+  nextAttemptAt: string | null;
+}
+
+export function defaultSourceState(slug: string): SourceState {
+  return {
+    slug,
+    lastFetchedAt: null,
+    cursor: {},
+    consecutiveFailures: 0,
+    nextAttemptAt: null,
+  };
+}

package/server/workspace/sources/urls.ts

@@ -0,0 +1,112 @@
+// URL normalization utilities for dedup + cache keys.
+//
+// The same article commonly arrives from multiple sources with
+// different query strings (utm_source, fbclid, gclid, mc_cid, ...)
+// or trailing slashes. We normalize before dedup so "same article
+// different feed" collapses into one item.
+//
+// Pure — no I/O, fully testable.
+
+import { createHash } from "node:crypto";
+
+// Tracking parameters we always strip. Case-insensitive on the
+// parameter NAME only.
+const TRACKING_PARAM_PREFIXES: readonly string[] = ["utm_", "mc_", "pk_", "hsa_"];
+
+const TRACKING_PARAMS: ReadonlySet<string> = new Set([
+  "fbclid",
+  "gclid",
+  "msclkid",
+  "dclid",
+  "gbraid",
+  "wbraid",
+  "yclid",
+  "ref",
+  "ref_src",
+  "ref_url",
+  "share",
+  "share_source",
+  "trk",
+  "igshid",
+  "cmp",
+]);
+
+function isTrackingParam(name: string): boolean {
+  const lower = name.toLowerCase();
+  if (TRACKING_PARAMS.has(lower)) return true;
+  for (const prefix of TRACKING_PARAM_PREFIXES) {
+    if (lower.startsWith(prefix)) return true;
+  }
+  return false;
+}
+
+// Normalize a URL for use as a dedup key:
+//
+//   1. Parse it. Invalid → return null (caller decides fallback).
+//   2. Lowercase the protocol + host.
+//   3. Drop the fragment.
+//   4. Drop tracking query params.
+//   5. Sort remaining query params so different orderings hash the
+//      same.
+//   6. Collapse trailing slash on the pathname (except root "/").
+//   7. Drop default ports (80 for http, 443 for https).
+//
+// Returns the normalized href on success, null on unparseable
+// input.
+export function normalizeUrl(raw: string): string | null {
+  if (typeof raw !== "string" || raw.trim() === "") return null;
+  let url: URL;
+  try {
+    url = new URL(raw.trim());
+  } catch {
+    return null;
+  }
+
+  // protocol + host lowercase (URL already normalizes these but
+  // doing it explicitly guards against future WHATWG tweaks).
+  url.protocol = url.protocol.toLowerCase();
+  url.hostname = url.hostname.toLowerCase();
+
+  // Drop fragment.
+  url.hash = "";
+
+  // Drop tracking params. Iterate on a snapshot because delete
+  // mutates the underlying list.
+  const paramNames = Array.from(url.searchParams.keys());
+  for (const name of paramNames) {
+    if (isTrackingParam(name)) url.searchParams.delete(name);
+  }
+
+  // Sort remaining params for deterministic ordering. Preserve
+  // multi-value params by iterating all entries.
+  const entries = Array.from(url.searchParams.entries());
+  entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  // Clear and reinsert.
+  for (const name of Array.from(url.searchParams.keys())) {
+    url.searchParams.delete(name);
+  }
+  for (const [name, value] of entries) {
+    url.searchParams.append(name, value);
+  }
+
+  // Collapse trailing slash on non-root paths.
+  if (url.pathname.length > 1 && url.pathname.endsWith("/")) {
+    url.pathname = url.pathname.slice(0, -1);
+  }
+
+  // Drop default ports.
+  if ((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443")) {
+    url.port = "";
+  }
+
+  return url.toString();
+}
+
+// Stable content-addressed id for an item. Used for persistent
+// cross-run dedup of news items. Truncated SHA-256 gives ~64 bits
+// of collision resistance — safe into the billions of items, vs.
+// FNV-1a 32-bit which starts colliding in the tens of thousands
+// (birthday-paradox territory at our projected volume).
+export function stableItemId(normalizedUrl: string): string {
+  return createHash("sha256").update(normalizedUrl, "utf8").digest("hex").slice(0, 16);
+}

package/server/workspace/tool-trace/classify.ts

@@ -0,0 +1,114 @@
+// Pure classification for built-in Claude tool_result events.
+// Decides whether a given tool result should be stored in the session
+// jsonl as a pointer to a real workspace file, inlined verbatim, or
+// inlined with truncation. No filesystem access — callers do I/O
+// separately and feed the result here.
+//
+// See plans/done/feat-tool-trace-persistence.md for the design rationale.
+
+import { isRecord } from "../../utils/types.js";
+
+export type Classification = { kind: "pointer"; contentRef: string } | { kind: "inline"; content: string; truncated: boolean };
+
+// Max characters kept when content is stored inline in the jsonl.
+// Picked to keep per-turn jsonl size sane while still capturing
+// enough of a small Bash/Grep output to be useful for debugging.
+export const MAX_INLINE_CONTENT_CHARS = 4096;
+
+// Tools whose `args.file_path` already points at an existing file —
+// the jsonl can simply reference that path instead of duplicating the
+// content. Matches Claude Code's built-in file tool names exactly.
+const FILE_POINTER_TOOLS = new Set(["Read", "Write", "Edit"]);
+
+// Image-generation MCP tools. The tool result already carries the
+// saved path so we extract it; the raw bytes/base64 never leave the
+// agent stream memory.
+const IMAGE_TOOLS = new Set(["generateImage", "editImage"]);
+
+// Tool name we always route through `writeSearch.ts` before
+// classifying. Exposed so callers know which tools need a
+// pre-computed `searchContentRef` injected.
+export const WEB_SEARCH_TOOL_NAME = "WebSearch";
+
+export interface ClassifyInput {
+  toolName: string;
+  args: unknown;
+  content: string;
+  // Optional pre-computed contentRef for WebSearch — the caller saves
+  // the result file first (in `writeSearch.ts`) and passes the
+  // workspace-relative path in here.
+  searchContentRef?: string;
+}
+
+export function classifyToolResult(input: ClassifyInput): Classification {
+  const { toolName, args, content, searchContentRef } = input;
+
+  if (toolName === WEB_SEARCH_TOOL_NAME && searchContentRef) {
+    return { kind: "pointer", contentRef: searchContentRef };
+  }
+
+  if (FILE_POINTER_TOOLS.has(toolName)) {
+    const ref = filePointerFromArgs(args);
+    if (ref) return { kind: "pointer", contentRef: ref };
+  }
+
+  if (IMAGE_TOOLS.has(toolName)) {
+    const ref = imagePointerFromContent(content);
+    if (ref) return { kind: "pointer", contentRef: ref };
+  }
+
+  return inlineWithTruncation(content);
+}
+
+function filePointerFromArgs(args: unknown): string | null {
+  if (!isRecord(args)) return null;
+  const record = args;
+  const raw = record.file_path;
+  if (typeof raw !== "string" || raw.length === 0) return null;
+  return normalizeWorkspacePath(raw);
+}
+
+// Image MCP tool results typically include a saved path somewhere in
+// the stringified result. Be conservative: only treat it as a pointer
+// when we can confidently extract an `images/` or absolute path. No
+// match → fall back to inline (truncated) handling so the record
+// still carries *something* useful.
+function imagePointerFromContent(content: string): string | null {
+  // Matches a JSON-ish "filePath": "..." or "path": "..." value
+  // without using regex backtracking. Look for known keys then scan
+  // the quoted value.
+  for (const key of ['"filePath":', '"path":', '"file":']) {
+    const idx = content.indexOf(key);
+    if (idx === -1) continue;
+    const afterKey = content.slice(idx + key.length);
+    const quoteStart = afterKey.indexOf('"');
+    if (quoteStart === -1) continue;
+    const quoteEnd = afterKey.indexOf('"', quoteStart + 1);
+    if (quoteEnd === -1) continue;
+    const raw = afterKey.slice(quoteStart + 1, quoteEnd);
+    if (raw.length === 0) continue;
+    return normalizeWorkspacePath(raw);
+  }
+  return null;
+}
+
+// Strip a leading "/" or "./" so the stored ref is workspace-relative
+// regardless of how the tool happened to quote it. Leave "../"
+// prefixes alone — a relative escape is a bug and we want it visible
+// rather than silently fixed up.
+function normalizeWorkspacePath(p: string): string {
+  if (p.startsWith("./")) return p.slice(2);
+  if (p.startsWith("/")) return p.slice(1);
+  return p;
+}
+
+function inlineWithTruncation(content: string): Classification {
+  if (content.length <= MAX_INLINE_CONTENT_CHARS) {
+    return { kind: "inline", content, truncated: false };
+  }
+  return {
+    kind: "inline",
+    content: content.slice(0, MAX_INLINE_CONTENT_CHARS),
+    truncated: true,
+  };
+}