moshcode 0.56.0 → 0.58.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "moshcode",
-  "version": "0.56.0",
+  "version": "0.58.0",
   "type": "module",
   "description": "moshcode — a metal wrapper for coding engines and native UGig/CoinPay workflow CLIs, with OpenPRD and moshscript",
   "repository": {

package/src/news-sources.mjs

@@ -272,16 +272,24 @@ export const DEFAULT_FEEDS = [
  * one. They are somebody else's lists, and that is the point: the feeds inside
  * them stay current without moshcode shipping a release.
  *
- * Two shapes, because the lists worth reading come in two:
+ * Three shapes, because the lists worth reading come in three:
  *
  *   `format: "opml"` — an OPML document, parsed by parseOpml.
- *   `format: "text"` — one feed URL per line, `#` comments ignored. Kagi's
- *     smallweb list is published this way and there is no OPML of it.
+ *   `format: "text"` — one feed URL per line, `#` comments ignored.
+ *   `format: "feed"` — the URL is not a catalogue at all but a single feed that
+ *     already aggregates one. Nothing is fetched to list it; `add` subscribes
+ *     to the one URL. This is how smallweb is carried — see below.
  *
- * `searchOnly` marks a list too large to subscribe to wholesale. smallweb is
- * 32,000+ feeds: importing it would write a 32,000-entry news.opml and then try
- * to fetch every one of them on the next `/news`. It is a catalogue to search,
- * not a subscription — `/rss search <keyword>` is how you get feeds out of it.
+ * Kagi's Small Web used to be here as its catalogue: a 5MB OPML of 33,000
+ * personal blogs, plus a 1.1MB plain-text fork of the same thing. Both were
+ * marked search-only, because subscribing to 33,000 feeds is not a thing a
+ * reader can do — but `find` reads every list on every search, so *searching*
+ * loaded and parsed 33,000 outlines each time, which is what took `/rss` and
+ * `/news find` down. Lazily paging a catalogue that size is a feature nobody
+ * asked for, so the catalogue is gone and the firehose stands in for it:
+ * kagi.com/smallweb/feed is those same blogs already merged into one Atom
+ * document, newest first, with titles and summaries. One feed, one fetch, and
+ * `/news --feed smallweb` reads it on its own.
  */
 export const FEED_LISTS = [
   {
@@ -310,17 +318,11 @@ export const FEED_LISTS = [
   },
   {
     name: "smallweb",
-    description: "Kagi Small Web — 33k personal blogs, with titles",
-    url: "https://kagi.com/smallweb/opml",
-    format: "opml",
-    searchOnly: true,
-  },
-  {
-    name: "smallweb-txt",
-    description: "Kagi Small Web, profullstack's fork — the plain-text list",
-    url: "https://raw.githubusercontent.com/ralyodio/smallweb/refs/heads/main/smallweb.txt",
-    format: "text",
-    searchOnly: true,
+    description: "Kagi Small Web — 33k personal blogs, merged into one feed",
+    url: "https://kagi.com/smallweb/feed",
+    format: "feed",
+    title: "Kagi Small Web",
+    site: "https://kagi.com/smallweb",
   },
 ];
 

package/src/news.mjs

@@ -99,6 +99,40 @@ const FILE_MODE = 0o600;
 /** Enough for a large feed, small enough that one bad URL cannot eat the pit. */
 const MAX_BYTES = 8 * 1024 * 1024;
 
+/**
+ * Where a *feed* read stops, and — unlike MAX_BYTES — stopping is not failing.
+ *
+ * An aggregated feed has no natural size: smallweb's firehose is 10MB of
+ * today's posts, which the 8MB cap would refuse outright, leaving the reader
+ * with nothing rather than with the newest few hundred entries. Feeds are
+ * newest-first, so the first two megabytes are the part worth having; the rest
+ * of the transfer is cancelled mid-stream and what arrived is parsed. parseFeed
+ * matches whole `<item>`/`<entry>` blocks, so a cut tail simply is not an item.
+ *
+ * Ordinary feeds never reach this — the largest in the defaults is under 1MB.
+ */
+const FEED_MAX_BYTES = 2 * 1024 * 1024;
+
+/**
+ * The most headlines one feed may put into a merged listing.
+ *
+ * Without it, subscribing to an aggregator is the same as unsubscribing from
+ * everything else: the merge sorts by date, and 4,000 small-web posts from
+ * today's firehose sit above every headline the other feeds have. Set at
+ * MAX_LIMIT so no listing can be starved by the cap — `--feed smallweb --limit
+ * 200` still fills, because a single feed can supply every row.
+ */
+const PER_FEED_ITEMS = 200;
+
+/**
+ * The most feeds one `add` may subscribe to at once.
+ *
+ * Whatever the source — a named list, a URL, a local file. The largest list
+ * published by name holds 611 feeds, so this refuses only the thing that is a
+ * catalogue rather than a reading list.
+ */
+const MAX_IMPORT_FEEDS = 2000;
+
 /** How many feeds are in flight at once. Politeness, not throughput. */
 const CONCURRENCY = 6;
 
@@ -413,11 +447,11 @@ export function findFeed(feeds, needle) {
 /**
  * Parse a newline-delimited feed list: one URL per line, `#` comments ignored.
  *
- * The other shape a published list comes in. Kagi's smallweb list is a plain
- * text file of 32,000 URLs and there is no OPML of it, so a reader that only
- * speaks OPML cannot read the largest list there is. No titles are invented
- * beyond the hostname — naming 32,000 feeds properly would mean fetching
- * 32,000 feeds, and the host is what the URL already tells us for free.
+ * The other shape a published list comes in: plenty of them are a text file of
+ * URLs and nothing else, so a reader that only speaks OPML cannot read them.
+ * No titles are invented beyond the hostname — naming a list's feeds properly
+ * would mean fetching every one of them, and the host is what the URL already
+ * tells us for free.
  */
 export function parseFeedList(text) {
   const feeds = [];
@@ -440,7 +474,7 @@ export function parseListDocument(body, format = "") {
   return looksLikeOpml(body) ? parseOpml(body) : parseFeedList(body);
 }
 
-/** Where a fetched list is cached, so searching does not refetch 32k feeds. */
+/** Where a fetched list is cached, so searching does not refetch every list. */
 export function listCacheFile(name, env = process.env) {
   return path.join(path.dirname(opmlFile(env)), "lists", `${slugify(name) || "list"}.json`);
 }
@@ -462,6 +496,24 @@ const LIST_CACHE_MS = 24 * 60 * 60 * 1000;
 export async function loadListFeeds(list, {
   fetchImpl, env = process.env, timeoutMs = DEFAULT_TIMEOUT_MS, now = Date.now(), refresh = false,
 } = {}) {
+  // A `format: "feed"` list is a single feed that has already done the
+  // aggregating — smallweb's firehose. There is no catalogue to fetch or cache:
+  // the entry itself is the one feed, so a search costs nothing and can never
+  // be the thing that makes `find` slow.
+  if (list.format === "feed") {
+    return {
+      ok: true,
+      cached: false,
+      feeds: [{
+        name: list.name,
+        title: list.title || list.name,
+        url: list.url,
+        site: list.site || "",
+        category: list.name,
+      }],
+    };
+  }
+
   const file = listCacheFile(list.name, env);
   const cached = () => {
     try {
@@ -512,7 +564,7 @@ export function parseKeywords(raw) {
  *
  * Ranked rather than filtered, because filtering cannot win both halves of this:
  *
- *   · Plain substring matching searches 32,000 feeds for `rust` and returns
+ *   · Plain substring matching searches every list for `rust` and returns
  *     Trust Machines, Trustnodes, frustrat.com and popthruster.com.
  *   · Requiring the keyword to start a word fixes that and then finds nothing
  *     at all for `homelab`, because the list's only metadata is the hostname
@@ -689,8 +741,15 @@ export function tidyTitle(title) {
  *
  * Size-capped while streaming rather than after: a feed that turns out to be a
  * disk image should cost a few megabytes of transfer, not all of it.
+ *
+ * What happens at the cap is the caller's to choose. A document read whole —
+ * an OPML catalogue — fails, because half a catalogue is a wrong answer rather
+ * than a partial one. A feed read for its newest entries truncates instead:
+ * see FEED_MAX_BYTES.
  */
-export async function fetchDocument(url, { fetchImpl, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
+export async function fetchDocument(url, {
+  fetchImpl, timeoutMs = DEFAULT_TIMEOUT_MS, maxBytes = MAX_BYTES, truncate = false,
+} = {}) {
   const impl = fetchImpl || globalThis.fetch;
   if (typeof impl !== "function") return { ok: false, error: "no fetch available in this runtime" };
   const safe = safeUrl(url);
@@ -713,7 +772,10 @@ export async function fetchDocument(url, { fetchImpl, timeoutMs = DEFAULT_TIMEOU
     // text() for any fetch implementation (tests included) that has no body.
     if (!res.body || typeof res.body.getReader !== "function") {
       const body = await res.text();
-      if (body.length > MAX_BYTES) return { ok: false, error: `feed is larger than ${MAX_BYTES} bytes` };
+      if (body.length > maxBytes) {
+        if (!truncate) return { ok: false, error: `feed is larger than ${maxBytes} bytes` };
+        return { ok: true, body: body.slice(0, maxBytes), truncated: true };
+      }
       return { ok: true, body };
     }
     const reader = res.body.getReader();
@@ -724,9 +786,11 @@ export async function fetchDocument(url, { fetchImpl, timeoutMs = DEFAULT_TIMEOU
       const { done, value } = await reader.read();
       if (done) break;
       bytes += value.byteLength;
-      if (bytes > MAX_BYTES) {
+      if (bytes > maxBytes) {
         try { await reader.cancel(); } catch { /* already gone */ }
-        return { ok: false, error: `feed is larger than ${MAX_BYTES} bytes` };
+        if (!truncate) return { ok: false, error: `feed is larger than ${maxBytes} bytes` };
+        body += decoder.decode(value, { stream: true });
+        return { ok: true, body: body + decoder.decode(), truncated: true };
       }
       body += decoder.decode(value, { stream: true });
     }
@@ -772,7 +836,7 @@ async function mapLimit(items, limit, worker) {
  */
 export async function collectNews(feeds, { fetchImpl, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
   const results = await mapLimit(feeds, CONCURRENCY, async (feed) => {
-    const res = await fetchDocument(feed.url, { fetchImpl, timeoutMs });
+    const res = await fetchDocument(feed.url, { fetchImpl, timeoutMs, maxBytes: FEED_MAX_BYTES, truncate: true });
     if (!res.ok) return { feed, error: res.error };
     let parsed;
     try { parsed = parseFeed(res.body, { url: feed.url }); }
@@ -787,7 +851,8 @@ export async function collectNews(feeds, { fetchImpl, timeoutMs = DEFAULT_TIMEOU
   let skipped = 0;
   for (const result of results) {
     if (result.error) { failures.push({ name: result.feed.name, url: result.feed.url, error: result.error }); continue; }
-    for (const item of result.parsed.items) {
+    // Newest first inside a feed, so the cap keeps the entries worth keeping.
+    for (const item of result.parsed.items.slice(0, PER_FEED_ITEMS)) {
       // Aggregator feeds wrap the publisher's URL in one of their own. Unwrap
       // before deduping, so the same story arriving via Google News and via the
       // publisher's own feed is recognised as one story rather than two.
@@ -1398,9 +1463,13 @@ const FIND_LIMIT = 25;
 /**
  * `news find <keyword,…>` — search every published list for feeds to subscribe to.
  *
- * This is the way into the big lists. smallweb alone is 32,000 feeds, which is
- * a fine thing to search and an impossible thing to subscribe to, so the
- * catalogue is read here and only the matches become candidates for `add`.
+ * This is the way into the lists: they hold a few thousand feeds between them,
+ * which is a fine thing to search and a poor thing to read end to end, so the
+ * catalogues are read here and only the matches become candidates for `add`.
+ *
+ * Every list is loaded on every search, which is why none of them may be
+ * enormous — a 33,000-feed catalogue in here made this the slowest thing in the
+ * pit and took `/rss` down with it.
  */
 async function findCommand(request, { out, fail, fetchImpl, env }) {
   const hits = [];
@@ -1418,7 +1487,7 @@ async function findCommand(request, { out, fail, fetchImpl, env }) {
   }
 
   // Ranked across every list, not within each one. Sorting per list and then
-  // concatenating puts all of web3's weak matches above smallweb's exact ones
+  // concatenating puts all of web3's weak matches above journalists' exact ones
   // purely because web3 is fetched first.
   hits.sort((a, b) => scoreFeed(b, request.keywords) - scoreFeed(a, request.keywords));
 
@@ -1472,6 +1541,32 @@ async function addCommand(request, { out, fail, fetchImpl, env }) {
     fail(`  ${ash("too large to subscribe to wholesale — search it instead:")} ${bone(`/rss search <keyword>`)}`);
     return 1;
   }
+  // A `format: "feed"` list is one feed wearing a list's name. Subscribed
+  // without fetching it first: smallweb's firehose is ten megabytes of
+  // headlines and not one of them is needed to decide the feed exists. Tagged
+  // with the list name like any other list, so `rm smallweb` still takes it
+  // back out.
+  if (bundle?.format === "feed") {
+    const { feeds, added, existed } = withFeed(loadFeeds(env), {
+      name: bundle.name,
+      title: bundle.title || bundle.name,
+      url: bundle.url,
+      site: bundle.site || "",
+      category: bundle.name,
+    });
+    if (existed) { out(`${ash("· ")}already subscribed to ${bone(added.name)} ${ash(added.url)}`); return 0; }
+    try { saveFeeds(feeds, env); }
+    catch (e) { fail(danger(`✗ can't write ${opmlFile(env)}: ${e.message}`)); return 1; }
+    // Named from the OPML on the way back in, like every other feed, so the
+    // name printed here is the one `--feed` will answer to rather than the one
+    // that went in.
+    const saved = findFeed(loadFeeds(env), bundle.url) ?? added;
+    if (request.json) { out(JSON.stringify({ added: saved, skipped: 0 }, null, 2)); return 0; }
+    out(`${acid("✓ ")}subscribed to ${bone(saved.name)} ${ash(`— ${bundle.description}`)}`);
+    out(`  ${ash("read it on its own with")} ${bone(`/news --feed ${saved.name}`)}`);
+    return 0;
+  }
+
   const target = bundle ? bundle.url : request.target;
   if (bundle) out(`${ash("· ")}fetching ${bone(bundle.name)} ${ash(`— ${bundle.description}`)}`);
 
@@ -1491,6 +1586,17 @@ async function addCommand(request, { out, fail, fetchImpl, env }) {
     // Tagged with the list it came from, so `rm <list>` can take it back out.
     const incoming = bundle ? tagWithList(parsedList, bundle.name) : parsedList;
     if (!incoming.length) { fail(danger(`✗ that ${bundle?.format === "text" ? "list" : "OPML file"} lists no feeds`)); return 1; }
+    // A catalogue is not a subscription. `searchOnly` guarded the *name* but
+    // never the URL, so `add https://kagi.com/smallweb/opml` wrote 32,969
+    // outlines into news.opml and every `/rss` afterwards tried to fetch all of
+    // them — the crash this was supposed to prevent, through the one door left
+    // open. The limit sits well above every published list (the largest is 611)
+    // and well below a catalogue.
+    if (incoming.length > MAX_IMPORT_FEEDS) {
+      fail(danger(`✗ that list holds ${incoming.length.toLocaleString()} feeds — more than ${MAX_IMPORT_FEEDS.toLocaleString()}, which is more than a reader can fetch`));
+      fail(`  ${ash("it is a catalogue, not a subscription — search it instead:")} ${bone("/rss search <keyword>")}`);
+      return 1;
+    }
     let feeds = existing;
     const added = [];
     let skipped = 0;

package/src/tui.mjs

@@ -464,6 +464,35 @@ let activeMirror = null;
  */
 const childSink = () => (activeMirror ? (chunk) => activeMirror?.write(chunk) : undefined);
 
+/**
+ * How fast an exit has to be before it is worth remarking on.
+ *
+ * A person who opens an agent and immediately quits takes longer than this.
+ * Nothing that actually started a session lands under it.
+ */
+const INSTANT_EXIT_MS = 1500;
+
+/**
+ * The note for a hand-off that ended the moment it began, or null.
+ *
+ * A CLI that exits 0 without doing anything is indistinguishable, from out
+ * here, from one the operator opened and closed — both are "exited (code 0)",
+ * which reads as success and sent somebody looking for the bug in the wrong
+ * program. It happens for real: @serjm/deepseek-code 0.5.0 compares
+ * `resolve(process.argv[1])` against `import.meta.url` to decide whether it is
+ * the entrypoint, and npm installs every global bin as a symlink, so the
+ * comparison fails and the whole CLI silently runs nothing.
+ *
+ * Timing is all we have — the child owns the terminal, so its output is not
+ * ours to inspect — and it is enough to say "this looks wrong" without
+ * claiming to know why.
+ */
+export function instantExitNote({ key, bin, code, ms }) {
+  if (code !== 0 || ms >= INSTANT_EXIT_MS) return null;
+  return `${key} exited instantly without running — that usually means a broken install, not a clean session.`
+    + ` check it directly with \`${bin} --version\`, and reinstall with /install ${key} if that prints nothing.`;
+}
+
 async function openEngine(key, engine, args, { agentMode = false } = {}) {
   if (!engine.installed && !args.length) {
     console.log(info(`${key} isn't installed — try ${acid("/install " + key)} first.`));
@@ -479,7 +508,9 @@ async function openEngine(key, engine, args, { agentMode = false } = {}) {
   console.log(info(`opening ${bone(key)}${agentMode ? " autonomously" : " raw"} — hand-off to its CLI, exit it to come back…`));
   console.log(hr());
   activeMirror?.setEngine(key);
+  const startedAt = Date.now();
   const r = await openSession(engine, agentMode ? agentLaunchArgs(engine, args) : args, { onOutput: childSink() });
+  const elapsed = Date.now() - startedAt;
   activeMirror?.setEngine(null);
   console.log(hr());
   if (!r.ok) {
@@ -488,6 +519,8 @@ async function openEngine(key, engine, args, { agentMode = false } = {}) {
       : err(`couldn't launch ${key}: ${r.error?.message || r.error}`));
   } else {
     console.log(info(`${key} exited${r.code != null ? ` (code ${r.code})` : ""}. back in the pit.`));
+    const note = instantExitNote({ key, bin: engine.bin, code: r.code, ms: elapsed });
+    if (note) console.log(warn(note));
   }
 }
 
@@ -497,7 +530,9 @@ async function openWorkflowTool(key, tool, args) {
   }
   console.log(info(`opening ${bone(key)} — native CLI owns the terminal until it exits…`));
   console.log(hr());
+  const toolStartedAt = Date.now();
   const result = await openTool(tool, args, { onOutput: childSink() });
+  const toolElapsed = Date.now() - toolStartedAt;
   console.log(hr());
   if (!result.ok) {
     console.log(result.error?.code === "ENOENT"
@@ -505,6 +540,8 @@ async function openWorkflowTool(key, tool, args) {
       : err(`couldn't launch ${key}: ${result.error?.message || result.error}`));
   } else {
     console.log(info(`${key} exited${result.code != null ? ` (code ${result.code})` : result.signal ? ` (${result.signal})` : ""}. back in the pit.`));
+    const note = instantExitNote({ key, bin: tool.bin, code: result.code, ms: toolElapsed });
+    if (note) console.log(warn(note));
   }
 }