moshcode 0.47.0 → 0.48.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "moshcode",
3
- "version": "0.47.0",
3
+ "version": "0.48.0",
4
4
  "type": "module",
5
5
  "description": "moshcode — a metal wrapper for coding engines and native UGig/CoinPay workflow CLIs, with OpenPRD and moshscript",
6
6
  "repository": {
@@ -550,10 +550,14 @@ export const CORE_CLI_COMMANDS = [
550
550
  synopsis: [
551
551
  ["moshcode rss", "open the reader on your feeds"],
552
552
  ["moshcode rss <keyword…>", "open it on a search"],
553
+ ["moshcode rss search <keyword[,keyword…]>", "find feeds to subscribe to"],
554
+ ["moshcode rss add <url|list|#n>", "subscribe — same as `moshcode news add`"],
553
555
  ],
554
556
  examples: [
555
557
  ["moshcode rss", "feeds on the left, headlines in the middle"],
556
558
  ["moshcode rss tariffs", "open straight into a search"],
559
+ ["moshcode rss search rust,compilers", "search 32k published feeds for one to add"],
560
+ ["moshcode rss add 3", "subscribe to the third result"],
557
561
  ],
558
562
  seeAlso: ["news"],
559
563
  note: "needs an interactive terminal. ↑↓/jk move · ⏎ read · o open in a browser · "
@@ -800,12 +804,17 @@ export const NEWS_VERBS = [
800
804
  },
801
805
  { name: "list", description: "the feeds you are subscribed to", synopsis: [["moshcode news list [--json]", ""]] },
802
806
  {
803
- name: "add", description: "subscribe to a feed, an OPML list, or a bundle",
804
- synopsis: [["moshcode news add <url|file|bundle>", "an RSS/Atom link, an OPML list, or journalists|web3|blockchain"]],
807
+ name: "find", description: "search the published lists for feeds to subscribe to",
808
+ synopsis: [["moshcode news find <keyword[,keyword…]>", "same as `moshcode rss search <keyword…>`"]],
805
809
  },
806
- { name: "rm", description: "unsubscribe", synopsis: [["moshcode news rm <name|url>", ""]] },
810
+ {
811
+ name: "add", description: "subscribe to a feed, a list, or a numbered result",
812
+ synopsis: [["moshcode news add <url|file|list|#n>", "an RSS/Atom link, an OPML file, a list by name, or #n from the last find"]],
813
+ },
814
+ { name: "rm", description: "unsubscribe from a feed, or a whole list", synopsis: [["moshcode news rm <name|url|list>", ""]] },
807
815
  { name: "open", description: "open a headline from the last listing", synopsis: [["moshcode news open <n>", ""]] },
808
- { name: "sources", description: "the default feeds and the bundles on offer", synopsis: [["moshcode news sources", ""]] },
816
+ { name: "lists", description: "the published lists, and which you have added", synopsis: [["moshcode news lists [--json]", ""]] },
817
+ { name: "sources", description: "the default feeds and the lists on offer", synopsis: [["moshcode news sources", ""]] },
809
818
  { name: "export", description: "print the subscription list as OPML", synopsis: [["moshcode news export > feeds.opml", ""]] },
810
819
  ];
811
820
 
@@ -7,58 +7,26 @@
7
7
  // brisk.news — apps/web/src/lib/news/fetch-feed.ts builds Google News feeds,
8
8
  // top stories for the front page and one per category, and
9
9
  // scripts/import-opml-feeds.ts seeds its publisher table from four public
10
- // OPML lists. Both are represented here: the Google feeds as the defaults,
11
- // the OPML lists as named bundles `/news add` accepts by name.
10
+ // OPML lists. The lists are represented here as named entries `/news add`
11
+ // accepts; the Google feeds are not, for the reason isDeadEndLink() gives.
12
12
  //
13
13
  // advis0r.com — src/providers/news/rss.ts pairs a Google News query feed with
14
14
  // a Bing one (Google's links are interstitials, Bing's carry the publisher
15
- // URL in a `url=` parameter), and reads two newswires directly. The search
16
- // pairing is why `/news <keyword>` queries both, and the wires are here
17
- // under `markets`.
15
+ // URL in a `url=` parameter), and reads two newswires directly. Only the
16
+ // Bing half of that pairing survives here see isDeadEndLink() and the
17
+ // wires are under `markets`.
18
18
  //
19
19
  // Deliberately small. A default list is a claim that every entry works, so it
20
20
  // holds feeds with stable well-known URLs and defers everything else to the
21
- // OPML bundles, where the list is somebody else's to maintain.
22
-
23
- /** Google News locale. One place, because every builder below needs it. */
24
- const GOOGLE_LOCALE = "hl=en-US&gl=US&ceid=US:en";
21
+ // published lists, where the list is somebody else's to maintain.
25
22
 
26
23
  /**
27
- * Google News category feeds, keyed the way brisk.news keys them.
24
+ * Bing News query feed the only search feed left.
28
25
  *
29
- * `general` is null there and means "top stories", which is a different URL
30
- * rather than a search for the word "general" the same distinction is kept.
31
- */
32
- export const GOOGLE_NEWS_CATEGORIES = {
33
- general: null,
34
- science: "science",
35
- sports: "sports",
36
- business: "business",
37
- health: "health",
38
- entertainment: "entertainment",
39
- tech: "technology",
40
- politics: "politics",
41
- food: "food",
42
- travel: "travel",
43
- };
44
-
45
- /** Google News top stories, or one category from the map above. */
46
- export function googleNewsFeed(category = null) {
47
- const mapped = category ? GOOGLE_NEWS_CATEGORIES[category] ?? null : null;
48
- if (!mapped) return `https://news.google.com/rss?${GOOGLE_LOCALE}`;
49
- return `https://news.google.com/rss/search?q=${encodeURIComponent(mapped)}&${GOOGLE_LOCALE}`;
50
- }
51
-
52
- /**
53
- * Google News query feed. `when:7d` style windows keep results recent —
54
- * advis0r's googleNewsFeed() does the same, for the same reason.
26
+ * The Google News builders that used to sit here were removed rather than left
27
+ * unused: keeping them would imply a Google News feed is still a thing this can
28
+ * read, and isDeadEndLink() explains at length why it is not.
55
29
  */
56
- export function googleNewsSearch(query, { window = "7d" } = {}) {
57
- const q = window ? `${query} when:${window}` : String(query);
58
- return `https://news.google.com/rss/search?q=${encodeURIComponent(q)}&${GOOGLE_LOCALE}`;
59
- }
60
-
61
- /** Bing News query feed — the half of a search whose links are real articles. */
62
30
  export function bingNewsSearch(query) {
63
31
  return `https://www.bing.com/news/search?q=${encodeURIComponent(query)}&format=RSS`;
64
32
  }
@@ -84,68 +52,176 @@ export function unwrapRedirect(url) {
84
52
  }
85
53
  }
86
54
 
55
+ /**
56
+ * Is this a link that can never be resolved to the article it stands for?
57
+ *
58
+ * Google News item links are `news.google.com/rss/articles/CBMi…?oc=5`, and
59
+ * they are a dead end in the strict sense — this was measured, not assumed:
60
+ *
61
+ * · A HEAD returns 302 whose `location` is the *same* URL with the locale
62
+ * appended, which then 200s on a Google page. There is no publisher URL in
63
+ * any header, so unwrapping by redirect does not work.
64
+ * · The base64 segment decodes to a protobuf holding an opaque `AU_yqL…`
65
+ * token and no URL.
66
+ * · A full GET returns a 600KB Angular shell — the redirect is client-side,
67
+ * and the target is not in the HTML.
68
+ * · That leaves Google's internal `/_/DotsSplashUi/data/batchexecute`, which
69
+ * answered 429 Too Many Requests on the first unauthenticated call. A
70
+ * reader cannot be built on an endpoint that rate-limits a single request.
71
+ * · Their `<description>` is an `<ol><li><a href=…>` list of more Google
72
+ * links rather than a summary, so there is no content to show either.
73
+ *
74
+ * So an item behind one of these is a headline that cannot be read and cannot
75
+ * be opened. collectNews drops them rather than listing rows that do nothing.
76
+ * Bing's search feed is unaffected: it carries the publisher URL in `url=` and
77
+ * a real summary, which is why it is the one kept for searching.
78
+ */
79
+ export function isDeadEndLink(url) {
80
+ try {
81
+ const parsed = new URL(String(url));
82
+ return /(^|\.)news\.google\.com$/i.test(parsed.hostname)
83
+ && /^\/rss\/articles\//i.test(parsed.pathname);
84
+ } catch {
85
+ return false;
86
+ }
87
+ }
88
+
87
89
  /**
88
90
  * The feeds a fresh install reads.
89
91
  *
90
- * Google News carries the general desks because that is exactly what brisk.news
91
- * serves its front page from, and it means one URL shape covers ten sections
92
- * without ten publisher relationships to keep working. The named publishers are
93
- * tier-1 hosts from advis0r's own tiering table that publish a stable feed, and
94
- * they earn their place by being readable end to end a Google News item is a
95
- * headline behind an interstitial, theirs is the article.
92
+ * Every one of them is a publisher speaking for itself. Google News used to
93
+ * carry the general desks here one URL shape covering ten sections without
94
+ * ten publisher relationships and it was dropped precisely because of what
95
+ * isDeadEndLink() documents: its items cannot be opened and carry no summary,
96
+ * so a default list built on it is a list of rows that do nothing. The desks it
97
+ * covered (world, science, politics) are named publishers now instead.
98
+ *
99
+ * The profullstack blogs are here too, so `profullstack.com/feeds.opml` is read
100
+ * out of the box rather than only after `/news add profullstack`.
96
101
  */
97
102
  export const DEFAULT_FEEDS = [
98
- { name: "top-stories", title: "Google News — Top Stories", url: googleNewsFeed(), site: "https://news.google.com", category: "" },
99
- { name: "world", title: "Google News — World", url: googleNewsSearch("world news", { window: "2d" }), site: "https://news.google.com", category: "" },
100
-
101
- { name: "tech", title: "Google News — Technology", url: googleNewsFeed("tech"), site: "https://news.google.com", category: "tech" },
102
103
  { name: "ars-technica", title: "Ars Technica", url: "https://feeds.arstechnica.com/arstechnica/index", site: "https://arstechnica.com", category: "tech" },
103
104
  { name: "techcrunch", title: "TechCrunch", url: "https://techcrunch.com/feed/", site: "https://techcrunch.com", category: "tech" },
104
105
  { name: "the-register", title: "The Register", url: "https://www.theregister.com/headlines.atom", site: "https://www.theregister.com", category: "tech" },
105
106
  { name: "hacker-news", title: "Hacker News — Front Page", url: "https://hnrss.org/frontpage", site: "https://news.ycombinator.com", category: "tech" },
106
107
 
107
- { name: "business", title: "Google News — Business", url: googleNewsFeed("business"), site: "https://news.google.com", category: "markets" },
108
108
  { name: "marketwatch", title: "MarketWatch — Top Stories", url: "https://feeds.content.dowjones.io/public/rss/mw_topstories", site: "https://www.marketwatch.com", category: "markets" },
109
109
  // The two newswires advis0r reads directly. Tier 0 there: issuers speaking
110
110
  // for themselves rather than a publication speaking about them.
111
111
  { name: "pr-newswire", title: "PR Newswire — Financial Services", url: "https://www.prnewswire.com/rss/financial-services-latest-news/financial-services-latest-news-list.rss", site: "https://www.prnewswire.com", category: "markets" },
112
112
  { name: "globenewswire", title: "GlobeNewswire — Public Companies", url: "https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies", site: "https://www.globenewswire.com", category: "markets" },
113
113
 
114
- { name: "science", title: "Google News Science", url: googleNewsFeed("science"), site: "https://news.google.com", category: "science" },
115
- { name: "politics", title: "Google News — Politics", url: googleNewsFeed("politics"), site: "https://news.google.com", category: "politics" },
114
+ { name: "npr-world", title: "NPRWorld", url: "https://feeds.npr.org/1004/rss.xml", site: "https://www.npr.org", category: "world" },
115
+ { name: "bbc-world", title: "BBC News — World", url: "https://feeds.bbci.co.uk/news/world/rss.xml", site: "https://www.bbc.co.uk/news", category: "world" },
116
+ { name: "guardian-world", title: "The Guardian — World", url: "https://www.theguardian.com/world/rss", site: "https://www.theguardian.com", category: "world" },
117
+
118
+ { name: "science-daily", title: "ScienceDaily — Top Science", url: "https://www.sciencedaily.com/rss/top/science.xml", site: "https://www.sciencedaily.com", category: "science" },
119
+ { name: "phys-org", title: "Phys.org", url: "https://phys.org/rss-feed/", site: "https://phys.org", category: "science" },
120
+
121
+ { name: "npr-politics", title: "NPR — Politics", url: "https://feeds.npr.org/1014/rss.xml", site: "https://www.npr.org", category: "politics" },
122
+
123
+ // The profullstack blogs, read out of the box. This is the one published list
124
+ // small enough to be a default: profullstack.com/feeds.opml is 14 feeds, where
125
+ // smallweb is 33,000 and could only ever be searched. Kept in step with that
126
+ // file by hand rather than fetched, because defaultFeeds() is synchronous and
127
+ // a fresh install must not wait on a network call to show anything at all.
128
+ { name: "bittorrented-blog", title: "BitTorrented Blog", url: "https://bittorrented.com/blog/rss.xml", site: "https://bittorrented.com/blog", category: "profullstack" },
129
+ { name: "bl0ggers-blog", title: "bl0ggers Blog", url: "https://bl0ggers.com/blog/rss.xml", site: "https://bl0ggers.com/blog", category: "profullstack" },
130
+ { name: "c0mpute-blog", title: "c0mpute Blog", url: "https://c0mpute.com/blog/rss.xml", site: "https://c0mpute.com/blog", category: "profullstack" },
131
+ { name: "c0upons-blog", title: "c0upons Blog", url: "https://c0upons.com/blog/rss.xml", site: "https://c0upons.com/blog", category: "profullstack" },
132
+ { name: "coinpay-blog", title: "CoinPay Blog", url: "https://coinpayportal.com/blog/rss.xml", site: "https://coinpayportal.com/blog", category: "profullstack" },
133
+ { name: "crawlproof-blog", title: "CrawlProof Blog", url: "https://crawlproof.com/blog/rss.xml", site: "https://crawlproof.com/blog", category: "profullstack" },
134
+ { name: "d0rz-blog", title: "d0rz Blog", url: "https://d0rz.com/blog/rss.xml", site: "https://d0rz.com/blog", category: "profullstack" },
135
+ { name: "logicsrc-blog", title: "LogicSRC Blog", url: "https://logicsrc.com/blog/rss.xml", site: "https://logicsrc.com/blog", category: "profullstack" },
136
+ { name: "pairux-blog", title: "PairUX Blog", url: "https://pairux.com/blog/rss.xml", site: "https://pairux.com/blog", category: "profullstack" },
137
+ { name: "qryptchat-blog", title: "QryptChat Blog", url: "https://qrypt.chat/blog/rss.xml", site: "https://qrypt.chat/blog", category: "profullstack" },
138
+ { name: "saasrow-blog", title: "SaaSRow Blog", url: "https://www.saasrow.com/blog/rss.xml", site: "https://www.saasrow.com/blog", category: "profullstack" },
139
+ { name: "sh1pt-blog", title: "sh1pt Blog", url: "https://sh1pt.com/blog/rss.xml", site: "https://sh1pt.com/blog", category: "profullstack" },
140
+ { name: "threatcrush-blog", title: "ThreatCrush Blog", url: "https://threatcrush.com/blog/rss.xml", site: "https://threatcrush.com/blog", category: "profullstack" },
141
+ { name: "ugig-blog", title: "ugig Blog", url: "https://ugig.net/blog/rss.xml", site: "https://ugig.net/blog", category: "profullstack" },
116
142
  ];
117
143
 
118
144
  /**
119
- * The public OPML lists brisk.news imports its publisher table from.
145
+ * The public feed lists, offered by name.
120
146
  *
121
147
  * Offered by name (`/news add journalists`) rather than only by URL because
122
148
  * these are long, exact raw.githubusercontent paths that nobody is going to
123
149
  * retype, and importing one is the fastest way from an empty reader to a real
124
150
  * one. They are somebody else's lists, and that is the point: the feeds inside
125
151
  * them stay current without moshcode shipping a release.
152
+ *
153
+ * Two shapes, because the lists worth reading come in two:
154
+ *
155
+ * `format: "opml"` — an OPML document, parsed by parseOpml.
156
+ * `format: "text"` — one feed URL per line, `#` comments ignored. Kagi's
157
+ * smallweb list is published this way and there is no OPML of it.
158
+ *
159
+ * `searchOnly` marks a list too large to subscribe to wholesale. smallweb is
160
+ * 32,000+ feeds: importing it would write a 32,000-entry news.opml and then try
161
+ * to fetch every one of them on the next `/news`. It is a catalogue to search,
162
+ * not a subscription — `/rss search <keyword>` is how you get feeds out of it.
126
163
  */
127
- export const OPML_BUNDLES = [
164
+ export const FEED_LISTS = [
128
165
  {
129
166
  name: "journalists",
130
167
  description: "Dave Winer's feedsForJournalists — mainstream desks",
131
168
  url: "https://raw.githubusercontent.com/scripting/feedsForJournalists/master/list.opml",
169
+ format: "opml",
132
170
  },
133
171
  {
134
172
  name: "web3",
135
173
  description: "ChainFeeds RSSAggregatorforWeb3 — crypto and web3",
136
174
  url: "https://raw.githubusercontent.com/chainfeeds/RSSAggregatorforWeb3/main/RAW.opml",
175
+ format: "opml",
137
176
  },
138
177
  {
139
178
  name: "blockchain",
140
179
  description: "CoinFabrik decentralized-and-blockchain-feeds",
141
180
  url: "https://raw.githubusercontent.com/CoinFabrik/resources/master/decentralized-and-blockchain-feeds.opml",
181
+ format: "opml",
182
+ },
183
+ {
184
+ name: "profullstack",
185
+ description: "Profullstack product blogs",
186
+ url: "https://profullstack.com/feeds.opml",
187
+ format: "opml",
188
+ },
189
+ {
190
+ name: "smallweb",
191
+ description: "Kagi Small Web — 33k personal blogs, with titles",
192
+ url: "https://kagi.com/smallweb/opml",
193
+ format: "opml",
194
+ searchOnly: true,
195
+ },
196
+ {
197
+ name: "smallweb-txt",
198
+ description: "Kagi Small Web, profullstack's fork — the plain-text list",
199
+ url: "https://raw.githubusercontent.com/ralyodio/smallweb/refs/heads/main/smallweb.txt",
200
+ format: "text",
201
+ searchOnly: true,
142
202
  },
143
203
  ];
144
204
 
145
- /** Resolve a bundle name to its OPML URL, or null. */
146
- export function resolveBundle(name) {
205
+ /**
206
+ * The old name for the list catalogue.
207
+ *
208
+ * Kept because `/news sources --json` published it and a caller may be reading
209
+ * that key; the shape is unchanged for the three lists that were in it.
210
+ */
211
+ export const OPML_BUNDLES = FEED_LISTS;
212
+
213
+ /** Resolve a list name to its entry, or null. */
214
+ export function resolveList(name) {
147
215
  const wanted = String(name ?? "").trim().toLowerCase();
148
- return OPML_BUNDLES.find((b) => b.name === wanted) ?? null;
216
+ return FEED_LISTS.find((b) => b.name === wanted) ?? null;
217
+ }
218
+
219
+ /** The former name of resolveList. */
220
+ export const resolveBundle = resolveList;
221
+
222
+ /** The lists `/news add <name>` will import in full. */
223
+ export function subscribableLists() {
224
+ return FEED_LISTS.filter((list) => !list.searchOnly);
149
225
  }
150
226
 
151
227
  /** A fresh copy of the defaults — callers mutate feed lists. */
package/src/news.mjs CHANGED
@@ -26,9 +26,10 @@ import { acid, ash, amber, bone, danger } from "./ui.mjs";
26
26
  import {
27
27
  bingNewsSearch,
28
28
  defaultFeeds,
29
- googleNewsSearch,
30
- OPML_BUNDLES,
31
- resolveBundle,
29
+ FEED_LISTS,
30
+ isDeadEndLink,
31
+ resolveList,
32
+ subscribableLists,
32
33
  unwrapRedirect,
33
34
  } from "./news-sources.mjs";
34
35
 
@@ -39,11 +40,14 @@ const USAGE = `usage: moshcode news [verb|keyword…] [args…]
39
40
  <keyword…> search the news for a word or phrase
40
41
  <url> read one feed without subscribing to it
41
42
  list the feeds you are subscribed to
42
- add <url|file|bundle> subscribe an RSS/Atom link, an OPML list, or
43
- a bundle: ${OPML_BUNDLES.map((b) => b.name).join(", ")}
44
- rm <name|url> unsubscribe
43
+ find <keyword[,keyword…]> search every published list for feeds to add
44
+ add <url|file|list|#n> subscribe an RSS/Atom link, an OPML file, a
45
+ list by name (${subscribableLists().map((b) => b.name).join(", ")}),
46
+ or #n from the last find
47
+ rm <name|url|list> unsubscribe — one feed, or a whole list at once
45
48
  open <n> open headline <n> from the last listing
46
- sources the default feeds and the bundles on offer
49
+ lists the published lists, and which you have added
50
+ sources the default feeds and the lists on offer
47
51
  export print the subscription list as OPML
48
52
 
49
53
  --json print structured data instead of headlines
@@ -63,7 +67,7 @@ export function newsUsage() {
63
67
  }
64
68
 
65
69
  /** Verb names, in help order. cli-schema's NEWS_VERBS must match (drift test). */
66
- export const NEWS_VERB_NAMES = ["latest", "search", "list", "add", "rm", "open", "sources", "export"];
70
+ export const NEWS_VERB_NAMES = ["latest", "search", "list", "find", "add", "rm", "open", "lists", "sources", "export"];
67
71
 
68
72
  // The same reasoning as crypto's alias table: the obvious synonym should not be
69
73
  // an error. `import` is the word an OPML file invites, and it is the same verb
@@ -75,7 +79,9 @@ const VERB_ALIASES = {
75
79
  sub: "add", subscribe: "add", import: "add", follow: "add",
76
80
  remove: "rm", unsub: "rm", unsubscribe: "rm", del: "rm", delete: "rm",
77
81
  read: "open", browse: "open", www: "open",
78
- bundles: "sources", defaults: "sources",
82
+ discover: "find", catalog: "find", catalogue: "find",
83
+ bundles: "lists",
84
+ defaults: "sources",
79
85
  opml: "export", dump: "export",
80
86
  };
81
87
 
@@ -399,6 +405,176 @@ export function findFeed(feeds, needle) {
399
405
  ?? null;
400
406
  }
401
407
 
408
+ // ---------------------------------------------------------------------------
409
+ // Published lists — a catalogue to search, and a unit to subscribe to
410
+ // ---------------------------------------------------------------------------
411
+
412
+ /**
413
+ * Parse a newline-delimited feed list: one URL per line, `#` comments ignored.
414
+ *
415
+ * The other shape a published list comes in. Kagi's smallweb list is a plain
416
+ * text file of 32,000 URLs and there is no OPML of it, so a reader that only
417
+ * speaks OPML cannot read the largest list there is. No titles are invented
418
+ * beyond the hostname — naming 32,000 feeds properly would mean fetching
419
+ * 32,000 feeds, and the host is what the URL already tells us for free.
420
+ */
421
+ export function parseFeedList(text) {
422
+ const feeds = [];
423
+ const seen = new Set();
424
+ for (const line of String(text ?? "").split(/\r?\n/)) {
425
+ const trimmed = line.trim();
426
+ if (!trimmed || trimmed.startsWith("#")) continue;
427
+ const url = safeUrl(trimmed);
428
+ if (!url || seen.has(url)) continue;
429
+ seen.add(url);
430
+ feeds.push({ name: hostSlug(url), title: hostOf(url), url, site: "", category: "" });
431
+ }
432
+ return feeds;
433
+ }
434
+
435
+ /** Parse a list document as whichever of the two shapes it is. */
436
+ export function parseListDocument(body, format = "") {
437
+ if (format === "text") return parseFeedList(body);
438
+ if (format === "opml") return parseOpml(body);
439
+ return looksLikeOpml(body) ? parseOpml(body) : parseFeedList(body);
440
+ }
441
+
442
+ /** Where a fetched list is cached, so searching does not refetch 32k feeds. */
443
+ export function listCacheFile(name, env = process.env) {
444
+ return path.join(path.dirname(opmlFile(env)), "lists", `${slugify(name) || "list"}.json`);
445
+ }
446
+
447
+ /** Where the last `find` results are remembered, so `add 3` knows what 3 was. */
448
+ export function findCacheFile(env = process.env) {
449
+ return path.join(path.dirname(opmlFile(env)), "news-found.json");
450
+ }
451
+
452
+ /** A day. These lists change on the order of weeks; a search must not wait on the network. */
453
+ const LIST_CACHE_MS = 24 * 60 * 60 * 1000;
454
+
455
+ /**
456
+ * The feeds in a published list, from cache when it is fresh.
457
+ *
458
+ * A stale cache is preferred to an error when the fetch fails: the list was
459
+ * good yesterday and a search that works offline beats one that does not.
460
+ */
461
+ export async function loadListFeeds(list, {
462
+ fetchImpl, env = process.env, timeoutMs = DEFAULT_TIMEOUT_MS, now = Date.now(), refresh = false,
463
+ } = {}) {
464
+ const file = listCacheFile(list.name, env);
465
+ const cached = () => {
466
+ try {
467
+ const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
468
+ return Array.isArray(parsed?.feeds) ? parsed : null;
469
+ } catch { return null; }
470
+ };
471
+
472
+ if (!refresh) {
473
+ const hit = cached();
474
+ if (hit && now - Number(hit.at || 0) < LIST_CACHE_MS) {
475
+ return { ok: true, feeds: hit.feeds, cached: true };
476
+ }
477
+ }
478
+
479
+ const res = await readSource(list.url, { fetchImpl, timeoutMs });
480
+ if (!res.ok) {
481
+ const hit = cached();
482
+ if (hit) return { ok: true, feeds: hit.feeds, cached: true, stale: true };
483
+ return { ok: false, error: res.error, feeds: [] };
484
+ }
485
+
486
+ const feeds = parseListDocument(res.body, list.format);
487
+ try {
488
+ fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
489
+ fs.writeFileSync(file, `${JSON.stringify({ at: now, url: list.url, feeds })}\n`, { mode: FILE_MODE });
490
+ try { fs.chmodSync(file, FILE_MODE); } catch { /* best effort */ }
491
+ } catch { /* a cache that cannot be written must not fail the search */ }
492
+ return { ok: true, feeds, cached: false };
493
+ }
494
+
495
+ /**
496
+ * Split `ai,crypto` into keywords.
497
+ *
498
+ * Comma-separated rather than space-separated because feed titles have spaces
499
+ * in them: `/rss search hacker news` means one thing to a person and two to a
500
+ * tokenizer, and the comma settles it. Spaces around a comma are trimmed.
501
+ */
502
+ export function parseKeywords(raw) {
503
+ return String(raw ?? "")
504
+ .split(",")
505
+ .map((s) => s.trim().toLowerCase())
506
+ .filter(Boolean);
507
+ }
508
+
509
+ /**
510
+ * Feeds matching any keyword, best match first.
511
+ *
512
+ * Ranked rather than filtered, because filtering cannot win both halves of this:
513
+ *
514
+ * · Plain substring matching searches 32,000 feeds for `rust` and returns
515
+ * Trust Machines, Trustnodes, frustrat.com and popthruster.com.
516
+ * · Requiring the keyword to start a word fixes that and then finds nothing
517
+ * at all for `homelab`, because the list's only metadata is the hostname
518
+ * and the blog is called myhomelab.net.
519
+ *
520
+ * So everything containing the keyword is kept, and a whole-word match sorts
521
+ * above a word that starts with it, which sorts above a match buried anywhere.
522
+ * With the results capped for display, the good ones are the ones you see.
523
+ * Sorting is stable, so the list's own order survives within a rank.
524
+ */
525
+ export function scoreFeed(feed, keywords) {
526
+ const hay = `${feed.title || ""} ${feed.url} ${feed.category || ""}`.toLowerCase();
527
+ let best = 0;
528
+ for (const k of keywords) {
529
+ if (!hay.includes(k)) continue;
530
+ const tokens = hay.split(/[^a-z0-9]+/).filter(Boolean);
531
+ if (tokens.some((t) => t === k)) { best = Math.max(best, 3); continue; }
532
+ if (tokens.some((t) => t.startsWith(k))) { best = Math.max(best, 2); continue; }
533
+ best = Math.max(best, 1);
534
+ }
535
+ return best;
536
+ }
537
+
538
+ export function matchFeeds(feeds, keywords) {
539
+ if (!keywords.length) return [];
540
+ return feeds
541
+ .map((feed, i) => ({ feed, rank: scoreFeed(feed, keywords), i }))
542
+ .filter((row) => row.rank > 0)
543
+ .sort((a, b) => (b.rank - a.rank) || (a.i - b.i))
544
+ .map((row) => row.feed);
545
+ }
546
+
547
+ /**
548
+ * The list a subscribed feed came from, or "".
549
+ *
550
+ * Provenance rides in the OPML folder — the first path segment of `category` —
551
+ * rather than in a sidecar file, so it survives a round trip through any other
552
+ * reader and needs no format of our own. Only catalogue names count, so a
553
+ * folder someone happens to call "tech" is not mistaken for a list.
554
+ */
555
+ export function listOf(feed) {
556
+ const head = String(feed?.category || "").split("/")[0];
557
+ return resolveList(head) ? head : "";
558
+ }
559
+
560
+ /** How many subscribed feeds came from each published list. */
561
+ export function subscribedLists(feeds = []) {
562
+ const counts = new Map();
563
+ for (const feed of feeds) {
564
+ const name = listOf(feed);
565
+ if (name) counts.set(name, (counts.get(name) || 0) + 1);
566
+ }
567
+ return counts;
568
+ }
569
+
570
+ /** Tag a list's feeds with where they came from, keeping any folder they had. */
571
+ export function tagWithList(feeds, listName) {
572
+ return feeds.map((feed) => ({
573
+ ...feed,
574
+ category: feed.category ? `${listName}/${feed.category}` : listName,
575
+ }));
576
+ }
577
+
402
578
  // ---------------------------------------------------------------------------
403
579
  // Feeds — RSS 2.0, Atom, and RSS 1.0/RDF
404
580
  // ---------------------------------------------------------------------------
@@ -597,6 +773,11 @@ export async function collectNews(feeds, { fetchImpl, timeoutMs = DEFAULT_TIMEOU
597
773
  // before deduping, so the same story arriving via Google News and via the
598
774
  // publisher's own feed is recognised as one story rather than two.
599
775
  const link = item.link ? unwrapRedirect(item.link) : null;
776
+ // A link that unwrapping cannot rescue is a row that would do nothing
777
+ // when opened, so the headline is dropped rather than shown. Only
778
+ // aggregator interstitials qualify; an item with no link at all is still
779
+ // worth listing, because its title carries the news.
780
+ if (link && isDeadEndLink(link)) continue;
600
781
  const key = link || `${result.feed.name}:${item.title}`;
601
782
  if (seen.has(key)) continue;
602
783
  seen.add(key);
@@ -621,8 +802,11 @@ export async function collectNews(feeds, { fetchImpl, timeoutMs = DEFAULT_TIMEOU
621
802
  * openable links wherever the two overlap.
622
803
  */
623
804
  export function searchFeeds(query) {
805
+ // Bing only. The Google half of advis0r's pairing returned headlines whose
806
+ // links go nowhere and whose descriptions are lists of more Google links, so
807
+ // every one of its results was dropped downstream anyway — see
808
+ // isDeadEndLink(). Bing carries the publisher URL and a real summary.
624
809
  return [
625
- { name: "google", title: `Google News — ${query}`, url: googleNewsSearch(query), site: "", category: "" },
626
810
  { name: "bing", title: `Bing News — ${query}`, url: bingNewsSearch(query), site: "", category: "" },
627
811
  ];
628
812
  }
@@ -723,13 +907,28 @@ export function newsArgs(argv = []) {
723
907
  if (!query) return { error: "usage: moshcode news search <keyword…>" };
724
908
  return { ...base, verb, query };
725
909
  }
910
+ if (verb === "find") {
911
+ const query = tail.join(" ").trim();
912
+ if (!query) return { error: "usage: moshcode news find <keyword[,keyword…]>" };
913
+ const keywords = parseKeywords(query);
914
+ if (!keywords.length) return { error: "find needs a keyword" };
915
+ return { ...base, verb, keywords };
916
+ }
726
917
  if (verb === "add") {
727
- if (!tail.length) return { error: "usage: moshcode news add <url|file>" };
728
- if (tail.length > 1) return { error: "add takes one URL or file at a time" };
918
+ if (!tail.length) return { error: "usage: moshcode news add <url|file|list|#n>" };
919
+ if (tail.length > 1) return { error: "add takes one URL, list, or number at a time" };
920
+ // `add 3` and `add #3` take the third result of the last find. A feed URL
921
+ // is never a bare number, so this cannot shadow one.
922
+ const asIndex = /^#?(\d+)$/.exec(String(tail[0]).trim());
923
+ if (asIndex) {
924
+ const n = Number(asIndex[1]);
925
+ if (n < 1) return { error: `add takes a result number from 1, got ${JSON.stringify(tail[0])}` };
926
+ return { ...base, verb, target: null, index: n };
927
+ }
729
928
  return { ...base, verb, target: tail[0] };
730
929
  }
731
930
  if (verb === "rm") {
732
- if (!tail.length) return { error: "usage: moshcode news rm <name|url>" };
931
+ if (!tail.length) return { error: "usage: moshcode news rm <name|url|list>" };
733
932
  return { ...base, verb, target: tail.join(" ") };
734
933
  }
735
934
  if (verb === "open") {
@@ -808,8 +1007,9 @@ export function renderFeeds(feeds, { file = "", usingDefaults = false } = {}) {
808
1007
  if (!feeds.length) {
809
1008
  return ["", ` ${ash("no feeds yet")}`, "",
810
1009
  ` ${ash("add one:")} ${bone("/news add https://example.com/feed.xml")}`,
811
- ` ${ash("or a list:")} ${bone("/news add ~/subscriptions.opml")}`,
812
- ` ${ash("or a bundle:")}${bone(" /news add journalists")}`].join("\n");
1010
+ ` ${ash("or a file:")} ${bone("/news add ~/subscriptions.opml")}`,
1011
+ ` ${ash("or a list:")} ${bone("/news add journalists")}`,
1012
+ ` ${ash("or search:")} ${bone("/rss search ai,crypto")}`].join("\n");
813
1013
  }
814
1014
  const width = Math.max(...feeds.map((f) => f.name.length));
815
1015
  const header = usingDefaults
@@ -825,7 +1025,7 @@ export function renderFeeds(feeds, { file = "", usingDefaults = false } = {}) {
825
1025
  lines.push(` ${acid(feed.name.padEnd(width))} ${bone(clip(feed.title || "", 34).padEnd(36))}${ash(feed.url)}`);
826
1026
  }
827
1027
  lines.push("", usingDefaults
828
- ? ` ${ash("subscribe to your own with")} ${bone("/news add <url|bundle>")} ${ash("· see them with")} ${bone("/news sources")}`
1028
+ ? ` ${ash("subscribe to your own with")} ${bone("/news add <url|list>")} ${ash("· see them with")} ${bone("/news lists")}`
829
1029
  : ` ${ash("read them with")} ${bone("/news")} ${ash("· one of them with")} ${bone("/news --feed <name>")}`);
830
1030
  return lines.join("\n");
831
1031
  }
@@ -843,12 +1043,60 @@ export function renderSources() {
843
1043
  }
844
1044
  lines.push(` ${acid(feed.name.padEnd(width))} ${ash(clip(feed.title || "", 44))}`);
845
1045
  }
846
- lines.push("", ` ${bone("bundles")} ${ash("— public OPML lists, pull one in by name")}`, "");
847
- const bw = Math.max(...OPML_BUNDLES.map((b) => b.name.length));
848
- for (const bundle of OPML_BUNDLES) {
849
- lines.push(` ${acid(bundle.name.padEnd(bw))} ${ash(bundle.description)}`);
1046
+ lines.push("", ` ${bone("lists")} ${ash("— published feed lists, pull one in by name")}`, "");
1047
+ const bw = Math.max(...FEED_LISTS.map((b) => b.name.length));
1048
+ for (const list of FEED_LISTS) {
1049
+ const note = list.searchOnly ? ash(" (search only)") : "";
1050
+ lines.push(` ${acid(list.name.padEnd(bw))} ${ash(clip(list.description, 52))}${note}`);
850
1051
  }
851
- lines.push("", ` ${ash("pull one in with")} ${bone(`/news add ${OPML_BUNDLES[0].name}`)}`);
1052
+ lines.push("", ` ${ash("pull one in with")} ${bone(`/news add ${subscribableLists()[0].name}`)} ${ash("· search them all with")} ${bone("/rss search <keyword>")}`);
1053
+ return lines.join("\n");
1054
+ }
1055
+
1056
+ /**
1057
+ * The published lists, and how many feeds you have from each.
1058
+ *
1059
+ * `subscribed` is counted from the subscription file rather than remembered
1060
+ * separately, so a list stays "added" exactly as long as its feeds are there —
1061
+ * removing them by hand cannot leave a phantom membership behind.
1062
+ */
1063
+ export function renderLists(feeds = []) {
1064
+ const mine = subscribedLists(feeds);
1065
+ const width = Math.max(...FEED_LISTS.map((b) => b.name.length));
1066
+ const lines = ["", ` ${ash("published lists")}`, ""];
1067
+ for (const list of FEED_LISTS) {
1068
+ const count = mine.get(list.name) || 0;
1069
+ const state = count
1070
+ ? acid(`✓ ${count} feed${count === 1 ? "" : "s"}`)
1071
+ : (list.searchOnly ? ash("search only") : ash("—"));
1072
+ lines.push(` ${acid(list.name.padEnd(width))} ${bone(clip(list.description, 46).padEnd(48))}${state}`);
1073
+ }
1074
+ lines.push("",
1075
+ ` ${ash("add one:")} ${bone("/news add profullstack")}`,
1076
+ ` ${ash("drop one:")} ${bone("/news rm profullstack")} ${ash("— removes every feed it brought in")}`,
1077
+ ` ${ash("search them:")} ${bone("/rss search ai,crypto")}`);
1078
+ return lines.join("\n");
1079
+ }
1080
+
1081
+ /** Feeds found in the published lists, numbered so `add #n` can take one. */
1082
+ export function renderFindHits(hits, { keywords = [], total = 0, subscribed = [] } = {}) {
1083
+ if (!hits.length) {
1084
+ return ["", ` ${ash(`nothing in the published lists matches ${keywords.join(", ")}`)}`, "",
1085
+ ` ${ash("the lists searched:")} ${bone(FEED_LISTS.map((l) => l.name).join(", "))}`,
1086
+ ` ${ash("or subscribe by URL:")} ${bone("/rss add https://example.com/feed.xml")}`].join("\n");
1087
+ }
1088
+ const have = new Set(subscribed.map((f) => f.url));
1089
+ const width = String(hits.length).length;
1090
+ const lines = ["",
1091
+ ` ${ash(`${total} feed${total === 1 ? "" : "s"} match ${keywords.join(", ")}${total > hits.length ? ` · showing ${hits.length}` : ""}`)}`,
1092
+ ""];
1093
+ hits.forEach((feed, i) => {
1094
+ const n = String(i + 1).padStart(width);
1095
+ const mark = have.has(feed.url) ? acid(" ✓") : " ";
1096
+ lines.push(` ${bone(n)}.${mark} ${acid(clip(feed.title || hostOf(feed.url), 30).padEnd(32))}${ash(clip(feed.url, 58))}`);
1097
+ if (feed.list) lines.push(` ${ash(`from ${feed.list}`)}`);
1098
+ });
1099
+ lines.push("", ` ${ash("subscribe with")} ${bone("/rss add 1")} ${ash("· or the URL ·")} ${acid("✓")} ${ash("means already subscribed")}`);
852
1100
  return lines.join("\n");
853
1101
  }
854
1102
 
@@ -903,11 +1151,28 @@ export async function newsCommand(argv = [], deps = {}) {
903
1151
  }
904
1152
 
905
1153
  if (request.verb === "sources") {
906
- if (request.json) { out(JSON.stringify({ defaults: defaultFeeds(), bundles: OPML_BUNDLES }, null, 2)); return 0; }
1154
+ // `bundles` is the old key for the same array; kept so a caller reading it
1155
+ // does not break on the rename.
1156
+ if (request.json) { out(JSON.stringify({ defaults: defaultFeeds(), lists: FEED_LISTS, bundles: FEED_LISTS }, null, 2)); return 0; }
907
1157
  out(renderSources());
908
1158
  return 0;
909
1159
  }
910
1160
 
1161
+ if (request.verb === "lists") {
1162
+ const feeds = loadFeeds(env);
1163
+ if (request.json) {
1164
+ const mine = subscribedLists(feeds);
1165
+ out(JSON.stringify({
1166
+ lists: FEED_LISTS.map((list) => ({ ...list, subscribed: mine.get(list.name) || 0 })),
1167
+ }, null, 2));
1168
+ return 0;
1169
+ }
1170
+ out(renderLists(feeds));
1171
+ return 0;
1172
+ }
1173
+
1174
+ if (request.verb === "find") return findCommand(request, { out, fail, fetchImpl, env });
1175
+
911
1176
  if (request.verb === "export") {
912
1177
  // The defaults deliberately: exporting an empty file to hand to a reader is
913
1178
  // not what anyone means by "export my feeds" on a fresh install.
@@ -979,11 +1244,86 @@ export async function newsCommand(argv = [], deps = {}) {
979
1244
  return items.length === 0 && failures.length === feeds.length ? 1 : 0;
980
1245
  }
981
1246
 
982
- /** `news add <url|file|bundle>` one feed, or every feed in an OPML list. */
1247
+ /** How many matches a find prints. The rest are counted, not listed. */
1248
+ const FIND_LIMIT = 25;
1249
+
1250
+ /**
1251
+ * `news find <keyword,…>` — search every published list for feeds to subscribe to.
1252
+ *
1253
+ * This is the way into the big lists. smallweb alone is 32,000 feeds, which is
1254
+ * a fine thing to search and an impossible thing to subscribe to, so the
1255
+ * catalogue is read here and only the matches become candidates for `add`.
1256
+ */
1257
+ async function findCommand(request, { out, fail, fetchImpl, env }) {
1258
+ const hits = [];
1259
+ const seen = new Set();
1260
+ const failures = [];
1261
+
1262
+ for (const list of FEED_LISTS) {
1263
+ const loaded = await loadListFeeds(list, { fetchImpl, env, timeoutMs: request.timeoutMs });
1264
+ if (!loaded.ok) { failures.push(`${list.name}: ${loaded.error}`); continue; }
1265
+ for (const feed of matchFeeds(loaded.feeds, request.keywords)) {
1266
+ if (seen.has(feed.url)) continue;
1267
+ seen.add(feed.url);
1268
+ hits.push({ ...feed, list: list.name });
1269
+ }
1270
+ }
1271
+
1272
+ // Ranked across every list, not within each one. Sorting per list and then
1273
+ // concatenating puts all of web3's weak matches above smallweb's exact ones
1274
+ // purely because web3 is fetched first.
1275
+ hits.sort((a, b) => scoreFeed(b, request.keywords) - scoreFeed(a, request.keywords));
1276
+
1277
+ if (!hits.length && failures.length === FEED_LISTS.length) {
1278
+ fail(danger("✗ could not read any of the published lists"));
1279
+ for (const line of failures) fail(` ${ash(line)}`);
1280
+ return 1;
1281
+ }
1282
+
1283
+ const shown = hits.slice(0, FIND_LIMIT);
1284
+ // Remembered so `add 3` resolves, and only the shown ones — a number the
1285
+ // operator never saw is not a number they can have meant.
1286
+ try {
1287
+ const file = findCacheFile(env);
1288
+ fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
1289
+ fs.writeFileSync(file, `${JSON.stringify({ at: Date.now(), keywords: request.keywords, hits: shown }, null, 2)}\n`, { mode: FILE_MODE });
1290
+ try { fs.chmodSync(file, FILE_MODE); } catch { /* best effort */ }
1291
+ } catch { /* a cache that cannot be written must not fail the search */ }
1292
+
1293
+ if (request.json) { out(JSON.stringify({ keywords: request.keywords, total: hits.length, hits: shown, failures }, null, 2)); return 0; }
1294
+ out(renderFindHits(shown, { keywords: request.keywords, total: hits.length, subscribed: loadFeeds(env) }));
1295
+ for (const line of failures) fail(` ${ash(`· ${line}`)}`);
1296
+ return 0;
1297
+ }
1298
+
1299
+ /** The feed a previous `find` numbered `n`, or null. */
1300
+ function foundAt(index, env) {
1301
+ try {
1302
+ const parsed = JSON.parse(fs.readFileSync(findCacheFile(env), "utf8"));
1303
+ return Array.isArray(parsed?.hits) ? parsed.hits[index - 1] ?? null : null;
1304
+ } catch { return null; }
1305
+ }
1306
+
1307
+ /** `news add <url|file|list|#n>` — one feed, or every feed in a list. */
983
1308
  async function addCommand(request, { out, fail, fetchImpl, env }) {
984
- // A bundle name resolves to somebody else's OPML list. Checked before the URL
985
- // and the path so `journalists` is a name rather than a missing file.
986
- const bundle = resolveBundle(request.target);
1309
+ // `add 3` the third result of the last find, subscribed by its URL.
1310
+ if (request.index != null) {
1311
+ const found = foundAt(request.index, env);
1312
+ if (!found) {
1313
+ fail(danger(`✗ there is no result ${request.index} — run \`/rss search <keyword>\` first`));
1314
+ return 1;
1315
+ }
1316
+ request = { ...request, target: found.url, index: null };
1317
+ }
1318
+
1319
+ // A list name resolves to somebody else's published list. Checked before the
1320
+ // URL and the path so `journalists` is a name rather than a missing file.
1321
+ const bundle = resolveList(request.target);
1322
+ if (bundle?.searchOnly) {
1323
+ fail(danger(`✗ ${bundle.name} is ${bundle.description}`));
1324
+ fail(` ${ash("too large to subscribe to wholesale — search it instead:")} ${bone(`/rss search <keyword>`)}`);
1325
+ return 1;
1326
+ }
987
1327
  const target = bundle ? bundle.url : request.target;
988
1328
  if (bundle) out(`${ash("· ")}fetching ${bone(bundle.name)} ${ash(`— ${bundle.description}`)}`);
989
1329
 
@@ -996,9 +1336,13 @@ async function addCommand(request, { out, fail, fetchImpl, env }) {
996
1336
  const existing = loadFeeds(env);
997
1337
  const asUrl = safeUrl(target);
998
1338
 
999
- if (looksLikeOpml(res.body)) {
1000
- const incoming = parseOpml(res.body);
1001
- if (!incoming.length) { fail(danger("✗ that OPML file lists no feeds")); return 1; }
1339
+ // A named list is whichever shape it says it is; anything else is a list only
1340
+ // if it announces itself as OPML, so a plain feed is never read as one.
1341
+ if (bundle || looksLikeOpml(res.body)) {
1342
+ const parsedList = parseListDocument(res.body, bundle?.format);
1343
+ // Tagged with the list it came from, so `rm <list>` can take it back out.
1344
+ const incoming = bundle ? tagWithList(parsedList, bundle.name) : parsedList;
1345
+ if (!incoming.length) { fail(danger(`✗ that ${bundle?.format === "text" ? "list" : "OPML file"} lists no feeds`)); return 1; }
1002
1346
  let feeds = existing;
1003
1347
  const added = [];
1004
1348
  let skipped = 0;
@@ -1054,9 +1398,27 @@ async function addCommand(request, { out, fail, fetchImpl, env }) {
1054
1398
  return 0;
1055
1399
  }
1056
1400
 
1057
- /** `news rm <name|url>` — unsubscribe. */
1401
+ /** `news rm <name|url|list>` — unsubscribe from one feed, or a whole list. */
1058
1402
  function removeCommand(request, { out, fail, env }) {
1059
1403
  const feeds = loadFeeds(env);
1404
+
1405
+ // A list name takes the whole list back out. Checked first: subscribing to a
1406
+ // list is one action and unsubscribing from it should be one action too,
1407
+ // rather than the operator deleting forty feeds by hand.
1408
+ const asList = resolveList(request.target);
1409
+ if (asList) {
1410
+ const mine = feeds.filter((f) => listOf(f) === asList.name);
1411
+ if (!mine.length) {
1412
+ fail(danger(`✗ you have no feeds from "${asList.name}"`));
1413
+ return 1;
1414
+ }
1415
+ try { saveFeeds(feeds.filter((f) => listOf(f) !== asList.name), env); }
1416
+ catch (e) { fail(danger(`✗ can't write ${opmlFile(env)}: ${e.message}`)); return 1; }
1417
+ if (request.json) { out(JSON.stringify({ removed: mine, list: asList.name }, null, 2)); return 0; }
1418
+ out(`${acid("✓ ")}unsubscribed from ${bone(asList.name)} ${ash(`— ${mine.length} feed${mine.length === 1 ? "" : "s"}`)}`);
1419
+ return 0;
1420
+ }
1421
+
1060
1422
  const feed = findFeed(feeds, request.target);
1061
1423
  if (!feed) {
1062
1424
  fail(danger(`✗ no feed named "${request.target}"`));
package/src/rss-ui.mjs CHANGED
@@ -18,10 +18,21 @@ import {
18
18
  ago,
19
19
  collectNews,
20
20
  findFeed,
21
+ newsCommand,
21
22
  readingList,
23
+ resolveVerb,
22
24
  searchFeeds,
23
25
  } from "./news.mjs";
24
26
 
27
+ /**
28
+ * Verbs `/rss` hands straight to `moshcode news`.
29
+ *
30
+ * Everything that manages the subscription list rather than reading it. They
31
+ * print and exit, so they work with no TTY — which is why this runs before the
32
+ * interactive-terminal check.
33
+ */
34
+ const MANAGEMENT_VERBS = new Set(["add", "rm", "list", "lists", "find", "sources", "export", "open"]);
35
+
25
36
  const ESC = {
26
37
  altOn: "\x1b[?1049h", altOff: "\x1b[?1049l",
27
38
  hideCursor: "\x1b[?25l", showCursor: "\x1b[?25h",
@@ -305,14 +316,37 @@ export async function rssUi(argv = [], deps = {}) {
305
316
  write = (s) => process.stdout.write(`${s}\n`),
306
317
  } = deps;
307
318
 
319
+ const args = (Array.isArray(argv) ? argv : []).map(String);
320
+ const words = args.filter((a) => !a.startsWith("-"));
321
+ const verb = resolveVerb(words[0]);
322
+
323
+ // Managing subscriptions is the same work under either name, and `/rss add
324
+ // <url>` is what people type — the reader is where you are when you decide to
325
+ // subscribe to something. Handing these to newsCommand rather than reading
326
+ // them as a search is the whole difference between `/rss add <url>` working
327
+ // and it silently opening a reader on the literal phrase "add <url>".
328
+ //
329
+ // `search` is deliberately not among them. On `/news` it searches headlines;
330
+ // here it searches the published lists for feeds to add, because a reader
331
+ // already has `/` for searching what it is showing.
332
+ if (verb && MANAGEMENT_VERBS.has(verb)) {
333
+ return newsCommand(args, { fetchImpl, openUrl, env, out: write, fail: write });
334
+ }
335
+ if (verb === "search") {
336
+ const keywords = words.slice(1).join(" ").trim();
337
+ if (!keywords) { write("usage: moshcode rss search <keyword[,keyword…]>"); return 1; }
338
+ return newsCommand(["find", ...words.slice(1)], { fetchImpl, openUrl, env, out: write, fail: write });
339
+ }
340
+
308
341
  if (!stdin.isTTY || !stdout.isTTY) {
309
342
  write("moshcode rss needs an interactive terminal — try `moshcode news`");
310
343
  return 1;
311
344
  }
312
345
 
313
346
  // A query on the command line (`moshcode rss tariffs`) opens straight into
314
- // the search, which is the same shape `/news <keyword>` has.
315
- const query = argv.filter((a) => !String(a).startsWith("-")).join(" ").trim();
347
+ // the search, which is the same shape `/news <keyword>` has. `latest` is the
348
+ // verb for "no query", so it is dropped rather than searched for.
349
+ const query = (verb === "latest" ? "" : words.join(" ")).trim();
316
350
  const list = readingList(env);
317
351
 
318
352
  const state = {
@@ -58,6 +58,12 @@ export const MAX_TOTAL_BYTES = 256 * 1024;
58
58
  export const SYNCED_FILES = [
59
59
  { path: "aliases.json", json: true, label: "pit aliases" },
60
60
  { path: "herd/rules.json", json: true, label: "herd state rules" },
61
+ // OPML rather than JSON, so `json: false`: it is the interchange format every
62
+ // feed reader already imports and exports, which is the whole reason a feed
63
+ // list is worth carrying between machines at all. Written by `tcfeed rss add`
64
+ // and read by nothing here — moshcode's interest in it begins and ends with
65
+ // moving it, and a file this does not parse cannot be broken by this.
66
+ { path: "feeds.opml", json: false, label: "rss feeds" },
61
67
  ];
62
68
 
63
69
  /**
@@ -349,7 +355,7 @@ function endpoint(creds) {
349
355
  * code; a thrown network error inside the pit's dispatch loop would take the
350
356
  * prompt down instead, which is a lost session over a dropped wifi connection.
351
357
  */
352
- async function request(method, route, { creds, body = null, fetchImpl = fetch, timeoutMs = 20_000 } = {}) {
358
+ async function attempt(method, route, { creds, body, fetchImpl, timeoutMs }) {
353
359
  const controller = new AbortController();
354
360
  const timer = setTimeout(() => controller.abort(), timeoutMs);
355
361
  try {
@@ -374,6 +380,36 @@ async function request(method, route, { creds, body = null, fetchImpl = fetch, t
374
380
  }
375
381
  }
376
382
 
383
+ /**
384
+ * Retried only where a retry can be right: the app not answering.
385
+ *
386
+ * 502, 503, 504 and a dead socket say nothing about the request — the platform
387
+ * returned them without the app seeing it, so the same bytes sent a second
388
+ * later are as likely to work as they were the first time. Every other status
389
+ * is an answer: 400 will be 400 again, 401 wants `/login`, and 409 is another
390
+ * machine having saved first, which retrying would only re-ask.
391
+ *
392
+ * Unretried, one blip is a visible failure — and it looked frequent because
393
+ * every one of them was. `/save` had no retry at all while the scan workflow
394
+ * this same repository ships retries `npm install` three times for precisely
395
+ * this reasoning.
396
+ *
397
+ * Safe to repeat a PUT because the write is conditional: `ifRevision` pins the
398
+ * revision the caller last agreed on, so a retry that lands after a first
399
+ * attempt secretly succeeded is refused with 409 rather than writing twice.
400
+ */
401
+ const RETRY_STATUS = new Set([0, 502, 503, 504]);
402
+ const RETRY_BACKOFF_MS = [400, 1200];
403
+
404
+ async function request(method, route, { creds, body = null, fetchImpl = fetch, timeoutMs = 20_000, retries = RETRY_BACKOFF_MS.length } = {}) {
405
+ let last;
406
+ for (let i = 0; ; i++) {
407
+ last = await attempt(method, route, { creds, body, fetchImpl, timeoutMs });
408
+ if (!RETRY_STATUS.has(last.status) || i >= retries) return last;
409
+ await new Promise((r) => setTimeout(r, RETRY_BACKOFF_MS[i] ?? RETRY_BACKOFF_MS.at(-1)));
410
+ }
411
+ }
412
+
377
413
  export const pushSnapshot = (snapshot, { ifRevision = null, ...opts }) =>
378
414
  request("PUT", "/api/settings", { ...opts, body: { snapshot, ifRevision } });
379
415