mulmoclaude 0.9.0 → 0.9.2

package/server/utils/files/accounting-io.ts

@@ -1,294 +0,0 @@
-// Single fs gateway for the accounting plugin. Every read / write
-// against `data/accounting/...` lives here so callers don't sprinkle
-// raw `fs` / path concatenation across the codebase (CLAUDE.md rule:
-// raw `fs.readFile` / `fs.writeFile` is forbidden in route handlers).
-//
-// Snapshot cache rule: snapshots are derived state. Any write that
-// touches past data must call `invalidateSnapshotsFrom(...)` to drop
-// stale snapshot files; the next read regenerates lazily via
-// `server/accounting/snapshotCache.ts`. The journal JSONL files are
-// the single source of truth.
-
-import { promises as fsPromises } from "node:fs";
-import path from "node:path";
-
-import { workspacePath, WORKSPACE_DIRS } from "../../workspace/paths.js";
-import { writeJsonAtomic } from "./json.js";
-import { isEnoent } from "./safe.js";
-import type { AccountingConfig, Account, JournalEntry, MonthSnapshot } from "../../accounting/types.js";
-
-const root = (workspaceRoot?: string): string => workspaceRoot ?? workspacePath;
-
-function accountingRoot(workspaceRoot?: string): string {
-  return path.join(root(workspaceRoot), WORKSPACE_DIRS.accounting);
-}
-
-function configPath(workspaceRoot?: string): string {
-  return path.join(accountingRoot(workspaceRoot), "config.json");
-}
-
-/** Allowed shape for a book id used as a directory name. Defense
- *  against path traversal: a crafted id like "../../config" or
- *  "/tmp/x" would otherwise let `bookRoot` escape the
- *  `data/accounting/books/` tree, since every write path joins
- *  `bookId` directly into the filesystem. The first character is
- *  alphanumeric to forbid leading dashes / underscores that some
- *  shells / docs render confusingly; `_` and `-` are allowed inside.
- *  64 chars is plenty for any reasonable book name. */
-const SAFE_BOOK_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/;
-
-export function isSafeBookId(bookId: string): boolean {
-  return typeof bookId === "string" && SAFE_BOOK_ID_RE.test(bookId);
-}
-
-function assertSafeBookId(bookId: string): void {
-  if (!isSafeBookId(bookId)) {
-    throw new Error(`accounting: invalid bookId ${JSON.stringify(bookId)} (allowed: alphanumeric / _ / -; 1-64 chars; cannot start with _ or -)`);
-  }
-}
-
-export function bookRoot(bookId: string, workspaceRoot?: string): string {
-  assertSafeBookId(bookId);
-  return path.join(root(workspaceRoot), WORKSPACE_DIRS.accountingBooks, bookId);
-}
-
-function accountsPath(bookId: string, workspaceRoot?: string): string {
-  return path.join(bookRoot(bookId, workspaceRoot), "accounts.json");
-}
-
-function journalDir(bookId: string, workspaceRoot?: string): string {
-  return path.join(bookRoot(bookId, workspaceRoot), "journal");
-}
-
-function journalFileFor(bookId: string, period: string, workspaceRoot?: string): string {
-  return path.join(journalDir(bookId, workspaceRoot), `${period}.jsonl`);
-}
-
-function snapshotsDir(bookId: string, workspaceRoot?: string): string {
-  return path.join(bookRoot(bookId, workspaceRoot), "snapshots");
-}
-
-function snapshotFileFor(bookId: string, period: string, workspaceRoot?: string): string {
-  return path.join(snapshotsDir(bookId, workspaceRoot), `${period}.json`);
-}
-
-async function fileExists(filePath: string): Promise<boolean> {
-  try {
-    await fsPromises.access(filePath);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/** Strict variant of `readJsonOrNull` from `./json.ts`: returns null
- *  on ENOENT but RETHROWS other read errors and parse failures so a
- *  corrupted accounting journal surfaces rather than silently
- *  collapsing to "no data". `./json.ts` keeps the permissive
- *  variant for user-config files where a single bad keystroke
- *  shouldn't 500 the server. */
-async function readJsonStrict<T>(filePath: string): Promise<T | null> {
-  try {
-    const raw = await fsPromises.readFile(filePath, "utf-8");
-    return JSON.parse(raw) as T;
-  } catch (err) {
-    if (isEnoent(err)) return null;
-    throw err;
-  }
-}
-
-// ── config.json ────────────────────────────────────────────────────
-
-export async function readConfig(workspaceRoot?: string): Promise<AccountingConfig | null> {
-  return readJsonStrict<AccountingConfig>(configPath(workspaceRoot));
-}
-
-export async function writeConfig(config: AccountingConfig, workspaceRoot?: string): Promise<void> {
-  await writeJsonAtomic(configPath(workspaceRoot), config);
-}
-
-// ── accounts.json ──────────────────────────────────────────────────
-
-export async function readAccounts(bookId: string, workspaceRoot?: string): Promise<Account[]> {
-  const accounts = await readJsonStrict<Account[]>(accountsPath(bookId, workspaceRoot));
-  return accounts ?? [];
-}
-
-export async function writeAccounts(bookId: string, accounts: Account[], workspaceRoot?: string): Promise<void> {
-  await writeJsonAtomic(accountsPath(bookId, workspaceRoot), accounts);
-}
-
-// ── journal/YYYY-MM.jsonl (append-only) ────────────────────────────
-
-/** Convert a YYYY-MM-DD date string to its YYYY-MM month bucket. The
- *  month bucket dictates which JSONL file the entry lives in. */
-export function periodFromDate(date: string): string {
-  // YYYY-MM-DD → YYYY-MM. Validate the prefix shape so a malformed
-  // input fails early instead of silently bucketing into "1970-01"
-  // or similar.
-  if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) {
-    throw new Error(`accounting: invalid date format ${JSON.stringify(date)} (expected YYYY-MM-DD)`);
-  }
-  return date.slice(0, 7);
-}
-
-/** Append one entry to the appropriate month's JSONL.
- *
- *  Uses POSIX append-only semantics (`fs.appendFile` → `O_APPEND`).
- *  Two concurrent callers landing in the same month file are
- *  serialised by the kernel — neither overwrites the other, which
- *  is the bug the previous read-modify-write implementation had.
- *
- *  Crash mid-write: an entry shorter than `PIPE_BUF` (≥ 512 bytes
- *  on every supported platform) writes atomically; a single
- *  serialised `JournalEntry` is comfortably under that. If the
- *  process is killed during the syscall the worst case is a torn
- *  trailing line, which `readJournalMonth` already tolerates by
- *  skipping unparseable lines and surfacing a `skipped` count to
- *  the caller. */
-export async function appendJournal(bookId: string, entry: JournalEntry, workspaceRoot?: string): Promise<void> {
-  const period = periodFromDate(entry.date);
-  const file = journalFileFor(bookId, period, workspaceRoot);
-  await fsPromises.mkdir(path.dirname(file), { recursive: true });
-  await fsPromises.appendFile(file, `${JSON.stringify(entry)}\n`, { encoding: "utf-8" });
-}
-
-function groupEntriesByPeriod(entries: readonly JournalEntry[]): Map<string, JournalEntry[]> {
-  const byPeriod = new Map<string, JournalEntry[]>();
-  for (const entry of entries) {
-    const period = periodFromDate(entry.date);
-    const list = byPeriod.get(period) ?? [];
-    list.push(entry);
-    byPeriod.set(period, list);
-  }
-  return byPeriod;
-}
-
-/** Append a batch of entries: same-period entries are concatenated
- *  into one `appendFile` call so the whole same-period chunk hits
- *  the kernel as a single `O_APPEND` write — small chunks (under
- *  `PIPE_BUF`, ≥ 512 bytes on every supported platform) are
- *  guaranteed atomic by POSIX, and `O_APPEND` serialises with any
- *  concurrent appender (a parallel `appendJournal` / `addEntries`
- *  call can never overwrite our write or vice versa). Cross-period
- *  batches loop one append per period; each is independently
- *  concurrency-safe but their union is not transactional across
- *  files (out of scope for the append-only JSONL design). */
-export async function appendJournalBatch(bookId: string, entries: readonly JournalEntry[], workspaceRoot?: string): Promise<void> {
-  if (entries.length === 0) return;
-  const byPeriod = groupEntriesByPeriod(entries);
-  for (const [period, items] of byPeriod) {
-    const file = journalFileFor(bookId, period, workspaceRoot);
-    await fsPromises.mkdir(path.dirname(file), { recursive: true });
-    const chunk = items.map((entry) => `${JSON.stringify(entry)}\n`).join("");
-    await fsPromises.appendFile(file, chunk, { encoding: "utf-8" });
-  }
-}
-
-/** Read a single month's JSONL. Malformed lines are skipped (logged
- *  by the caller; this layer just returns the parseable subset) so
- *  one bad line doesn't lock the user out of their book. */
-export async function readJournalMonth(bookId: string, period: string, workspaceRoot?: string): Promise<{ entries: JournalEntry[]; skipped: number }> {
-  const file = journalFileFor(bookId, period, workspaceRoot);
-  let raw: string;
-  try {
-    raw = await fsPromises.readFile(file, "utf-8");
-  } catch (err) {
-    if (isEnoent(err)) return { entries: [], skipped: 0 };
-    throw err;
-  }
-  const entries: JournalEntry[] = [];
-  let skipped = 0;
-  for (const line of raw.split("\n")) {
-    if (line.trim() === "") continue;
-    try {
-      entries.push(JSON.parse(line) as JournalEntry);
-    } catch {
-      skipped += 1;
-    }
-  }
-  return { entries, skipped };
-}
-
-/** List the YYYY-MM periods that have a journal file on disk, sorted
- *  ascending. Useful for full-history scans (rebuilding snapshots
- *  from scratch). */
-export async function listJournalPeriods(bookId: string, workspaceRoot?: string): Promise<string[]> {
-  let names: string[];
-  try {
-    names = await fsPromises.readdir(journalDir(bookId, workspaceRoot));
-  } catch (err) {
-    if (isEnoent(err)) return [];
-    throw err;
-  }
-  return names
-    .filter((name) => /^\d{4}-\d{2}\.jsonl$/.test(name))
-    .map((name) => name.slice(0, 7))
-    .sort();
-}
-
-// ── snapshots/YYYY-MM.json (cache, not source of truth) ────────────
-
-export async function readSnapshot(bookId: string, period: string, workspaceRoot?: string): Promise<MonthSnapshot | null> {
-  return readJsonStrict<MonthSnapshot>(snapshotFileFor(bookId, period, workspaceRoot));
-}
-
-export async function writeSnapshot(bookId: string, snapshot: MonthSnapshot, workspaceRoot?: string): Promise<void> {
-  const file = snapshotFileFor(bookId, snapshot.period, workspaceRoot);
-  await fsPromises.mkdir(path.dirname(file), { recursive: true });
-  // `uniqueTmp` guards against the lazy fallback in `getOrBuildSnapshot`
-  // racing with the background rebuild — both can land in
-  // `writeSnapshot` for the same period at once. Distinct tmp file
-  // names mean each writer renames its own; the destination
-  // overwrites idempotently (both derive from the same journal).
-  await writeJsonAtomic(file, snapshot, { uniqueTmp: true });
-}
-
-/** Drop snapshot files for all periods >= `fromPeriod`. The next
- *  read regenerates them. Idempotent: missing files are silently
- *  ignored. */
-export async function invalidateSnapshotsFrom(bookId: string, fromPeriod: string, workspaceRoot?: string): Promise<{ removed: string[] }> {
-  let names: string[];
-  try {
-    names = await fsPromises.readdir(snapshotsDir(bookId, workspaceRoot));
-  } catch (err) {
-    if (isEnoent(err)) return { removed: [] };
-    throw err;
-  }
-  const removed: string[] = [];
-  for (const name of names) {
-    const match = /^(\d{4}-\d{2})\.json$/.exec(name);
-    if (!match) continue;
-    const [, period] = match;
-    if (period >= fromPeriod) {
-      await fsPromises.rm(path.join(snapshotsDir(bookId, workspaceRoot), name), { force: true });
-      removed.push(period);
-    }
-  }
-  return { removed: removed.sort() };
-}
-
-/** Drop ALL snapshots for a book — used by `rebuildSnapshots()`
- *  with no `from`. Equivalent to `invalidateSnapshotsFrom("0000-00")`
- *  but reads more clearly at call sites. */
-export async function invalidateAllSnapshots(bookId: string, workspaceRoot?: string): Promise<{ removed: string[] }> {
-  return invalidateSnapshotsFrom(bookId, "0000-00", workspaceRoot);
-}
-
-// ── book directory housekeeping ────────────────────────────────────
-
-export async function bookExists(bookId: string, workspaceRoot?: string): Promise<boolean> {
-  return fileExists(bookRoot(bookId, workspaceRoot));
-}
-
-export async function ensureBookDir(bookId: string, workspaceRoot?: string): Promise<void> {
-  await fsPromises.mkdir(bookRoot(bookId, workspaceRoot), { recursive: true });
-  await fsPromises.mkdir(journalDir(bookId, workspaceRoot), { recursive: true });
-  await fsPromises.mkdir(snapshotsDir(bookId, workspaceRoot), { recursive: true });
-}
-
-/** Recursively delete a book's directory. Used by `deleteBook` after
- *  the config has been updated to drop the entry. */
-export async function removeBookDir(bookId: string, workspaceRoot?: string): Promise<void> {
-  await fsPromises.rm(bookRoot(bookId, workspaceRoot), { recursive: true, force: true });
-}

package/server/workspace/feeds/engine.ts

@@ -1,143 +0,0 @@
-// Retrieval engine: fetch a feed, upsert its records into the
-// collection's data dir (keyed by primaryKey, so re-fetches replace in
-// place / accumulate by id), and persist per-feed state. Per-feed
-// failures are isolated — `refreshOne` never throws; `refreshDue`
-// processes feeds sequentially to stay gentle on remote hosts (the
-// fetch client does no rate-limiting yet).
-
-import { workspacePath } from "../workspace.js";
-import { log } from "../../system/logger/index.js";
-import { deleteItem, listItems, writeItem, type CollectionItem, type CollectionSchema, type LoadedCollection } from "../collections/index.js";
-import { ONE_HOUR_MS, ONE_DAY_MS } from "../../utils/time.js";
-import { getRetriever } from "./retrievers/index.js";
-import "./retrievers/registerAll.js";
-import { listFeeds } from "./registry.js";
-import { readFeedState, writeFeedState, type FeedState } from "./state.js";
-import { DEFAULT_FEED_MAX_ITEMS, type FeedSchedule, type IngestSpec } from "./ingestTypes.js";
-
-/** Feed schemas carry the rich `IngestSpec` (validated at discovery —
- *  `source === "feed"` requires `ingest`), but the canonical
- *  `CollectionSchema.ingest` only promises the minimal `CollectionIngest`.
- *  Narrow here so the engine can read the retrieval fields type-safely. */
-function feedIngest(schema: CollectionSchema): IngestSpec | undefined {
-  return schema.ingest as IngestSpec | undefined;
-}
-
-export interface RefreshResult {
-  slug: string;
-  written: number;
-  /** Old records deleted by the maxItems cap this run. */
-  removed: number;
-  errors: string[];
-}
-
-async function upsertItems(workspaceRoot: string, feed: LoadedCollection, items: CollectionItem[]): Promise<number> {
-  let written = 0;
-  for (const item of items) {
-    const itemId = item[feed.schema.primaryKey];
-    if (typeof itemId !== "string" || itemId.length === 0) continue;
-    const result = await writeItem(feed.dataDir, itemId, item, { refuseOverwrite: false, workspaceRoot, slug: feed.slug });
-    if (result.kind === "ok") written += 1;
-    else log.warn("feeds", "feed item write skipped", { slug: feed.slug, itemId, kind: result.kind });
-  }
-  return written;
-}
-
-/** The schema's first `date` field, used to order records for the
- *  maxItems cap. Null when the schema declares none. */
-function firstDateField(schema: CollectionSchema): string | null {
-  for (const [key, spec] of Object.entries(schema.fields)) {
-    if (spec.type === "date") return key;
-  }
-  return null;
-}
-
-// Epoch ms for a record's date value; missing / unparseable sorts oldest.
-function recordTime(item: CollectionItem, field: string): number {
-  const value = item[field];
-  const millis = typeof value === "string" ? Date.parse(value) : NaN;
-  return Number.isFinite(millis) ? millis : Number.NEGATIVE_INFINITY;
-}
-
-/** Enforce `ingest.maxItems` (default 100): keep the newest N records by
- *  the schema's date field, delete the rest. No-op when the cap is 0/absent
- *  of a date field, or when under the cap. Returns the number deleted. */
-async function pruneFeed(workspaceRoot: string, feed: LoadedCollection): Promise<number> {
-  const cap = feedIngest(feed.schema)?.maxItems ?? DEFAULT_FEED_MAX_ITEMS;
-  if (cap <= 0) return 0;
-  const dateField = firstDateField(feed.schema);
-  if (!dateField) {
-    log.warn("feeds", "maxItems prune skipped: schema has no date field to order by", { slug: feed.slug });
-    return 0;
-  }
-  const items = await listItems(feed.dataDir, { workspaceRoot });
-  if (items.length <= cap) return 0;
-  const stale = [...items].sort((left, right) => recordTime(right, dateField) - recordTime(left, dateField)).slice(cap);
-  let removed = 0;
-  for (const item of stale) {
-    const itemId = item[feed.schema.primaryKey];
-    if (typeof itemId !== "string" || itemId.length === 0) continue;
-    if ((await deleteItem(feed.dataDir, itemId, { workspaceRoot, slug: feed.slug })).kind === "ok") removed += 1;
-  }
-  if (removed > 0) log.info("feeds", "pruned old feed records", { slug: feed.slug, removed, cap });
-  return removed;
-}
-
-/** Fetch one feed now, upsert its records, then enforce the maxItems cap.
- *  Failure-isolated: returns an errors array rather than throwing. */
-export async function refreshOne(workspaceRoot: string, feed: LoadedCollection): Promise<RefreshResult> {
-  const { slug } = feed;
-  const ingest = feedIngest(feed.schema);
-  if (!ingest) return { slug, written: 0, removed: 0, errors: ["collection has no ingest config"] };
-  const retriever = getRetriever(ingest.kind);
-  if (!retriever) return { slug, written: 0, removed: 0, errors: [`no retriever registered for kind '${ingest.kind}'`] };
-  const state = await readFeedState(workspaceRoot, slug);
-  try {
-    const result = await retriever(ingest, feed.schema, state);
-    const written = await upsertItems(workspaceRoot, feed, result.items);
-    await writeFeedState(workspaceRoot, slug, { ...state, lastFetchedAt: new Date().toISOString(), cursor: result.cursor, consecutiveFailures: 0 });
-    const removed = await pruneFeed(workspaceRoot, feed);
-    log.info("feeds", "feed refreshed", { slug, written, removed, fetched: result.items.length });
-    return { slug, written, removed, errors: [] };
-  } catch (error) {
-    await writeFeedState(workspaceRoot, slug, { ...state, consecutiveFailures: state.consecutiveFailures + 1 });
-    const message = String(error);
-    log.warn("feeds", "feed refresh failed", { slug, error: message });
-    return { slug, written: 0, removed: 0, errors: [message] };
-  }
-}
-
-function dueIntervalMs(schedule: FeedSchedule): number {
-  switch (schedule) {
-    case "daily":
-      return ONE_DAY_MS;
-    case "weekly":
-      return 7 * ONE_DAY_MS;
-    default:
-      return ONE_HOUR_MS;
-  }
-}
-
-/** True iff a feed is due to refresh given its schedule + last fetch.
- *  `on-demand` feeds are never auto-due. */
-function isFeedDue(feed: LoadedCollection, state: FeedState): boolean {
-  const schedule = feedIngest(feed.schema)?.schedule;
-  if (!schedule || schedule === "on-demand") return false;
-  if (!state.lastFetchedAt) return true;
-  const elapsed = Date.now() - Date.parse(state.lastFetchedAt);
-  if (!Number.isFinite(elapsed)) return true;
-  return elapsed >= dueIntervalMs(schedule);
-}
-
-/** Refresh every feed whose schedule says it's due. Called by the
- *  hourly system task. Sequential + failure-isolated. */
-export async function refreshDue(workspaceRoot: string = workspacePath): Promise<RefreshResult[]> {
-  const feeds = await listFeeds(workspaceRoot);
-  const results: RefreshResult[] = [];
-  for (const feed of feeds) {
-    const state = await readFeedState(workspaceRoot, feed.slug);
-    if (!isFeedDue(feed, state)) continue;
-    results.push(await refreshOne(workspaceRoot, feed));
-  }
-  return results;
-}

package/server/workspace/feeds/fetch/httpClient.ts

@@ -1,127 +0,0 @@
-// Minimal HTTP client for feed retrievers. Feed URLs are model-authored /
-// user-supplied, so beyond a User-Agent + timeout this guards against SSRF:
-// every URL (and every redirect hop) is DNS-resolved and rejected if it
-// points at a loopback / private / link-local / cloud-metadata address.
-// Redirects are followed MANUALLY so a public URL can't 302 to an internal
-// one and bypass the guard. It does NOT do robots.txt / rate limiting (the
-// engine fetches feeds sequentially to stay gentle).
-
-import { lookup } from "node:dns/promises";
-import { isIP } from "node:net";
-import { ONE_SECOND_MS } from "../../../utils/time.js";
-
-/** Identifies the bot to site operators. */
-export const FEED_USER_AGENT = "MulmoClaude-FeedBot/1.0 (+https://github.com/receptron/mulmoclaude)";
-
-/** Per-request wall-clock cap so a hung server can't wedge a refresh. */
-export const DEFAULT_FEED_TIMEOUT_MS = 30 * ONE_SECOND_MS;
-
-/** Cap on redirect hops followed (each re-checked for SSRF). */
-const MAX_REDIRECTS = 5;
-
-// CIDR blocks we refuse to fetch: unspecified, loopback, RFC1918, CGNAT,
-// and link-local (which also covers the 169.254.169.254 metadata IP).
-/* eslint-disable sonarjs/no-hardcoded-ip -- intentional SSRF deny-list of loopback / private / link-local / CGNAT CIDRs */
-const BLOCKED_V4_CIDRS: readonly (readonly [string, number])[] = [
-  ["0.0.0.0", 8],
-  ["10.0.0.0", 8],
-  ["100.64.0.0", 10],
-  ["127.0.0.0", 8],
-  ["169.254.0.0", 16],
-  ["172.16.0.0", 12],
-  ["192.168.0.0", 16],
-];
-/* eslint-enable sonarjs/no-hardcoded-ip */
-
-function ipv4ToInt(address: string): number | null {
-  const octets = address.split(".").map(Number);
-  if (octets.length !== 4 || octets.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return null;
-  return ((octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3]) >>> 0;
-}
-
-function isBlockedIpv4(address: string): boolean {
-  const value = ipv4ToInt(address);
-  if (value === null) return true; // malformed → block
-  return BLOCKED_V4_CIDRS.some(([base, bits]) => {
-    const baseInt = ipv4ToInt(base) ?? 0;
-    const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0;
-    return (value & mask) === (baseInt & mask);
-  });
-}
-
-function isBlockedIpv6(address: string): boolean {
-  const lower = address.toLowerCase();
-  if (lower === "::1" || lower === "::") return true; // loopback, unspecified
-  if (lower.startsWith("fe80")) return true; // link-local fe80::/10
-  if (lower.startsWith("fc") || lower.startsWith("fd")) return true; // ULA fc00::/7
-  const mapped = /^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(lower); // IPv4-mapped
-  return mapped ? isBlockedIpv4(mapped[1]) : false;
-}
-
-/** True for any address we must not fetch (also blocks non-IP input). */
-function isBlockedIp(address: string): boolean {
-  const kind = isIP(address);
-  if (kind === 4) return isBlockedIpv4(address);
-  if (kind === 6) return isBlockedIpv6(address);
-  return true;
-}
-
-/** Reject non-http(s) URLs and URLs that resolve to a private/loopback
- *  address (SSRF guard). Throws with a clear reason; returns void on pass. */
-async function assertFetchableUrl(rawUrl: string): Promise<void> {
-  if (!/^https?:\/\//i.test(rawUrl)) throw new Error(`refusing non-http(s) URL: ${rawUrl}`);
-  const { hostname } = new URL(rawUrl);
-  // URL.hostname keeps the brackets for IPv6 literals (`[::1]`); strip them
-  // so isIP / the block check see the bare address.
-  const host = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
-  if (isIP(host)) {
-    if (isBlockedIp(host)) throw new Error(`refusing to fetch a private/loopback address: ${host}`);
-    return;
-  }
-  let addresses: { address: string }[];
-  try {
-    addresses = await lookup(host, { all: true });
-  } catch (err) {
-    throw new Error(`could not resolve host '${host}': ${String(err)}`);
-  }
-  const blocked = addresses.find((entry) => isBlockedIp(entry.address));
-  if (blocked) throw new Error(`refusing to fetch '${host}' — resolves to a private/loopback address (${blocked.address})`);
-}
-
-async function fetchOnce(url: string, timeoutMs: number): Promise<Response> {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(new DOMException(`feed fetch timed out after ${timeoutMs}ms`, "TimeoutError")), timeoutMs);
-  try {
-    return await fetch(url, { headers: { "User-Agent": FEED_USER_AGENT }, signal: controller.signal, redirect: "manual" });
-  } finally {
-    clearTimeout(timer);
-  }
-}
-
-// Follow redirects manually, re-running the SSRF guard on every hop so a
-// public URL cannot bounce to an internal target.
-async function fetchGuarded(rawUrl: string, timeoutMs: number): Promise<Response> {
-  let current = rawUrl;
-  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-    await assertFetchableUrl(current);
-    const response = await fetchOnce(current, timeoutMs);
-    const redirect = response.status >= 300 && response.status < 400 && response.status !== 304 ? response.headers.get("location") : null;
-    if (!redirect) return response;
-    current = new URL(redirect, current).toString();
-  }
-  throw new Error(`too many redirects (>${MAX_REDIRECTS}) starting from ${rawUrl}`);
-}
-
-/** Fetch a URL as text, throwing on guard rejection, network error, or non-2xx. */
-export async function fetchText(url: string, timeoutMs: number = DEFAULT_FEED_TIMEOUT_MS): Promise<string> {
-  const response = await fetchGuarded(url, timeoutMs);
-  if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText} fetching ${url}`);
-  return response.text();
-}
-
-/** Fetch a URL as parsed JSON, throwing on guard rejection, network error, or non-2xx. */
-export async function fetchJson(url: string, timeoutMs: number = DEFAULT_FEED_TIMEOUT_MS): Promise<unknown> {
-  const response = await fetchGuarded(url, timeoutMs);
-  if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText} fetching ${url}`);
-  return response.json();
-}

package/server/workspace/feeds/fetch/rssParser.ts

@@ -1,117 +0,0 @@
-// RSS 2.0 + Atom 1.0 + RSS 1.0 (RDF) parser for the Feeds engine.
-//
-// Deliberately NON-normalizing: it locates the feed's items and hands
-// each one back as its RAW parsed XML element. The host hard-codes no
-// per-feed field list — `ingest.map` resolves source paths against the
-// raw item (tags are keys; attributes are prefixed `@_`; namespaced tags
-// keep their prefix), and the caller inspects the feed to decide what to
-// map. Generic value handling (text-node unwrapping, date parsing by the
-// target field's declared type) lives in `projectItem.ts`, not here.
-//
-// Own copy of the XML plumbing — the Feeds tree does not import the
-// legacy `sources` tree. Pure; unit-testable with fixture strings.
-
-import { XMLParser } from "fast-xml-parser";
-import { isNonEmptyString, isRecord } from "../../../utils/types.js";
-
-export interface ParsedFeedItem {
-  /** The raw parsed XML <item>/<entry> object, verbatim. */
-  raw: Record<string, unknown>;
-}
-
-export interface ParsedFeed {
-  kind: "rss" | "atom";
-  title: string | null;
-  items: ParsedFeedItem[];
-}
-
-const xml = new XMLParser({
-  ignoreAttributes: false,
-  attributeNamePrefix: "@_",
-  cdataPropName: "#cdata",
-  parseTagValue: false,
-  parseAttributeValue: false,
-  trimValues: true,
-  // Only the item containers are forced to arrays (single vs. many).
-  // Everything else stays in its natural parsed shape so `ingest.map`
-  // resolves paths the way the caller sees them in the feed.
-  isArray: (name) => name === "item" || name === "entry",
-});
-
-/** Parse an RSS/Atom/RDF feed body. Returns null when the input doesn't
- *  look like a feed we understand. */
-export function parseFeed(body: string): ParsedFeed | null {
-  const text = stripBom(body);
-  if (!text.trim()) return null;
-  let parsed: unknown;
-  try {
-    parsed = xml.parse(text);
-  } catch {
-    return null;
-  }
-  if (!isRecord(parsed)) return null;
-  if (isRecord(parsed.rss)) return parseRss(parsed.rss);
-  if (isRecord(parsed.feed)) return parseAtom(parsed.feed);
-  const rdf = parsed["rdf:RDF"] ?? parsed.RDF;
-  if (isRecord(rdf)) return parseRss10(rdf);
-  return null;
-}
-
-// Collect the raw item objects, skipping anything that isn't a record or
-// carries no <title> (drops feed-level noise / empty entries).
-function collectItems(value: unknown): ParsedFeedItem[] {
-  const rawItems = Array.isArray(value) ? value : [];
-  const items: ParsedFeedItem[] = [];
-  for (const raw of rawItems) {
-    if (isRecord(raw) && readString(raw.title) !== null) items.push({ raw });
-  }
-  return items;
-}
-
-function parseRss(rss: Record<string, unknown>): ParsedFeed | null {
-  const { channel } = rss;
-  if (!isRecord(channel)) return null;
-  return { kind: "rss", title: readString(channel.title), items: collectItems(channel.item) };
-}
-
-function parseRss10(rdf: Record<string, unknown>): ParsedFeed | null {
-  const channel = isRecord(rdf.channel) ? rdf.channel : null;
-  return { kind: "rss", title: channel ? readString(channel.title) : null, items: collectItems(rdf.item) };
-}
-
-function parseAtom(feed: Record<string, unknown>): ParsedFeed | null {
-  return { kind: "atom", title: readString(feed.title), items: collectItems(feed.entry) };
-}
-
-// --- helpers ------------------------------------------------------------
-
-// Extract a string from a plain string, a `{ "#text" }` / `{ "#cdata" }`
-// node, or an array (first non-empty). Used only to read the feed/item
-// title for the channel name + the empty-item filter.
-function readString(value: unknown): string | null {
-  if (isNonEmptyString(value)) return value;
-  if (typeof value === "string") return null;
-  if (isRecord(value)) return readStringFromRecord(value);
-  if (Array.isArray(value)) return readStringFromArray(value);
-  return null;
-}
-
-function readStringFromRecord(record: Record<string, unknown>): string | null {
-  const text = record["#text"];
-  if (isNonEmptyString(text)) return text;
-  const cdata = record["#cdata"];
-  if (isNonEmptyString(cdata)) return cdata;
-  return null;
-}
-
-function readStringFromArray(array: readonly unknown[]): string | null {
-  for (const entry of array) {
-    const resolved = readString(entry);
-    if (resolved !== null) return resolved;
-  }
-  return null;
-}
-
-function stripBom(text: string): string {
-  return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
-}