moshcode 0.46.0 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli-schema.mjs +13 -4
- package/src/games-breakout.mjs +33 -7
- package/src/games-draw.mjs +78 -13
- package/src/games-pong.mjs +25 -5
- package/src/news-sources.mjs +136 -60
- package/src/news.mjs +392 -30
- package/src/rss-ui.mjs +36 -2
- package/src/settings-sync.mjs +37 -1
package/package.json
CHANGED
package/src/cli-schema.mjs
CHANGED
|
@@ -550,10 +550,14 @@ export const CORE_CLI_COMMANDS = [
|
|
|
550
550
|
synopsis: [
|
|
551
551
|
["moshcode rss", "open the reader on your feeds"],
|
|
552
552
|
["moshcode rss <keyword…>", "open it on a search"],
|
|
553
|
+
["moshcode rss search <keyword[,keyword…]>", "find feeds to subscribe to"],
|
|
554
|
+
["moshcode rss add <url|list|#n>", "subscribe — same as `moshcode news add`"],
|
|
553
555
|
],
|
|
554
556
|
examples: [
|
|
555
557
|
["moshcode rss", "feeds on the left, headlines in the middle"],
|
|
556
558
|
["moshcode rss tariffs", "open straight into a search"],
|
|
559
|
+
["moshcode rss search rust,compilers", "search 32k published feeds for one to add"],
|
|
560
|
+
["moshcode rss add 3", "subscribe to the third result"],
|
|
557
561
|
],
|
|
558
562
|
seeAlso: ["news"],
|
|
559
563
|
note: "needs an interactive terminal. ↑↓/jk move · ⏎ read · o open in a browser · "
|
|
@@ -800,12 +804,17 @@ export const NEWS_VERBS = [
|
|
|
800
804
|
},
|
|
801
805
|
{ name: "list", description: "the feeds you are subscribed to", synopsis: [["moshcode news list [--json]", ""]] },
|
|
802
806
|
{
|
|
803
|
-
name: "
|
|
804
|
-
synopsis: [["moshcode news
|
|
807
|
+
name: "find", description: "search the published lists for feeds to subscribe to",
|
|
808
|
+
synopsis: [["moshcode news find <keyword[,keyword…]>", "same as `moshcode rss search <keyword…>`"]],
|
|
805
809
|
},
|
|
806
|
-
{
|
|
810
|
+
{
|
|
811
|
+
name: "add", description: "subscribe to a feed, a list, or a numbered result",
|
|
812
|
+
synopsis: [["moshcode news add <url|file|list|#n>", "an RSS/Atom link, an OPML file, a list by name, or #n from the last find"]],
|
|
813
|
+
},
|
|
814
|
+
{ name: "rm", description: "unsubscribe from a feed, or a whole list", synopsis: [["moshcode news rm <name|url|list>", ""]] },
|
|
807
815
|
{ name: "open", description: "open a headline from the last listing", synopsis: [["moshcode news open <n>", ""]] },
|
|
808
|
-
{ name: "
|
|
816
|
+
{ name: "lists", description: "the published lists, and which you have added", synopsis: [["moshcode news lists [--json]", ""]] },
|
|
817
|
+
{ name: "sources", description: "the default feeds and the lists on offer", synopsis: [["moshcode news sources", ""]] },
|
|
809
818
|
{ name: "export", description: "print the subscription list as OPML", synopsis: [["moshcode news export > feeds.opml", ""]] },
|
|
810
819
|
];
|
|
811
820
|
|
package/src/games-breakout.mjs
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// decides the angle it leaves at, so the paddle is a steering wheel rather than
|
|
5
5
|
// a wall. Without that you cannot dig a channel up the side of the wall, and
|
|
6
6
|
// digging a channel is the entire reason anybody still plays this.
|
|
7
|
-
import {
|
|
7
|
+
import { advanceBall, drawnBall, drawnCell, snapBall } from "./games-draw.mjs";
|
|
8
8
|
import { acid, amber, bone, danger, rgb } from "./ui.mjs";
|
|
9
9
|
|
|
10
10
|
export const WIDTH = 40;
|
|
@@ -17,7 +17,11 @@ export const BRICK_TOP = 1;
|
|
|
17
17
|
|
|
18
18
|
export const PADDLE_W = 7;
|
|
19
19
|
export const PADDLE_ROW = HEIGHT - 1;
|
|
20
|
-
|
|
20
|
+
// A keypress, not a tick, so this is unchanged by the tick rate — but it does
|
|
21
|
+
// have to keep up with the ball, and a ball taken off the end of the paddle now
|
|
22
|
+
// crosses a column in under two ticks. Three columns a press stays ahead of it
|
|
23
|
+
// at a terminal's key-repeat rate; two only just did.
|
|
24
|
+
const PADDLE_STEP = 3;
|
|
21
25
|
|
|
22
26
|
/**
|
|
23
27
|
* How often the wall is stepped. See the note in games-pong.mjs: the ball can
|
|
@@ -32,8 +36,20 @@ const PADDLE_STEP = 2; // a keypress, not a tick — unchanged by the tick rate
|
|
|
32
36
|
*/
|
|
33
37
|
export const TICK_MS = 16;
|
|
34
38
|
|
|
39
|
+
/**
|
|
40
|
+
* How hard the wall is played, against the pace it was first tuned at.
|
|
41
|
+
*
|
|
42
|
+
* Launched, the old ball took three seconds to cross the board and two and a
|
|
43
|
+
* half to fall the height of it, which is a slow enough ball that you can put
|
|
44
|
+
* the paddle under it and go and make a cup of tea. It also meant the drawn
|
|
45
|
+
* ball moved twenty times a second, and twenty steps a second does not read as
|
|
46
|
+
* travel however even they are. Everything below scales together, so a ball off
|
|
47
|
+
* the end of the paddle leaves at the angle it always did.
|
|
48
|
+
*/
|
|
49
|
+
const PACE = 1.9;
|
|
50
|
+
|
|
35
51
|
/** The speeds below are still written per 50ms, the rate this was tuned at. */
|
|
36
|
-
const SCALE = TICK_MS / 50;
|
|
52
|
+
const SCALE = (TICK_MS / 50) * PACE;
|
|
37
53
|
|
|
38
54
|
const LIVES = 3;
|
|
39
55
|
const BASE_VX = 0.62 * SCALE;
|
|
@@ -67,6 +83,7 @@ export function brickAt(wall, x, y) {
|
|
|
67
83
|
function rest(state) {
|
|
68
84
|
state.ball = { x: state.paddle + PADDLE_W / 2, y: PADDLE_ROW - 1, vx: 0, vy: 0 };
|
|
69
85
|
state.stuck = true;
|
|
86
|
+
state.drawn = drawnBall(state.ball.x, state.ball.y);
|
|
70
87
|
return state;
|
|
71
88
|
}
|
|
72
89
|
|
|
@@ -82,8 +99,12 @@ export function launch(state) {
|
|
|
82
99
|
export function step(state) {
|
|
83
100
|
if (state.stuck) {
|
|
84
101
|
// A ball that has not been launched rides the paddle, so moving before you
|
|
85
|
-
// serve aims the serve.
|
|
102
|
+
// serve aims the serve. It is carried rather than travelling, so it is put
|
|
103
|
+
// where the paddle is rather than paced there — a stationary ball earns no
|
|
104
|
+
// steps, and would otherwise sit still while the paddle slid out from under
|
|
105
|
+
// it.
|
|
86
106
|
state.ball.x = state.paddle + PADDLE_W / 2;
|
|
107
|
+
snapBall(state.drawn, state.ball.x, state.ball.y);
|
|
87
108
|
return state;
|
|
88
109
|
}
|
|
89
110
|
|
|
@@ -122,11 +143,16 @@ export function step(state) {
|
|
|
122
143
|
}
|
|
123
144
|
}
|
|
124
145
|
|
|
146
|
+
advanceBall(state.drawn, ball);
|
|
147
|
+
|
|
125
148
|
if (ball.y > PADDLE_ROW) {
|
|
126
149
|
state.lives--;
|
|
127
150
|
if (state.lives <= 0) {
|
|
128
151
|
state.lives = 0;
|
|
129
152
|
state.over = `out of balls · ${state.score} points`;
|
|
153
|
+
// The last ball is left where it went, below the board and so off it,
|
|
154
|
+
// rather than resting on the row it fell past.
|
|
155
|
+
snapBall(state.drawn, ball.x, ball.y);
|
|
130
156
|
return state;
|
|
131
157
|
}
|
|
132
158
|
rest(state);
|
|
@@ -198,9 +224,9 @@ export const BREAKOUT = {
|
|
|
198
224
|
}
|
|
199
225
|
|
|
200
226
|
for (let i = 0; i < PADDLE_W; i++) put(state.paddle + i, PADDLE_ROW, bone("▀"));
|
|
201
|
-
// Drawn on half-rows, so the ball steps the same
|
|
202
|
-
//
|
|
203
|
-
const ball =
|
|
227
|
+
// Drawn on half-rows and on its own even clock, so the ball steps the same
|
|
228
|
+
// distance down the wall as across it, and at a rate. See games-draw.mjs.
|
|
229
|
+
const ball = drawnCell(state.drawn);
|
|
204
230
|
put(ball.col, ball.row, (state.stuck ? amber : bone)(ball.glyph));
|
|
205
231
|
|
|
206
232
|
return grid.map((row) => row.map((cell) => cell ?? " ").join(""));
|
package/src/games-draw.mjs
CHANGED
|
@@ -24,27 +24,92 @@
|
|
|
24
24
|
// one, and the corner it turns is half as wide. It is also a better ball than
|
|
25
25
|
// `●` was — a square pixel moving on a square grid, rather than a round dot
|
|
26
26
|
// snapping between cells twice its own height apart.
|
|
27
|
+
//
|
|
28
|
+
// Half blocks fix the size of the steps. They do not fix when the steps happen,
|
|
29
|
+
// which turned out to be the other half of it — see `drawnBall` below.
|
|
27
30
|
|
|
28
31
|
/**
|
|
29
|
-
*
|
|
32
|
+
* The ball's position in half-rows — the unit it is actually drawn in.
|
|
30
33
|
*
|
|
31
34
|
* `y` is a row centre, so row r covers y from r - 0.5 up to r + 0.5: below the
|
|
32
|
-
* centre the ball is in the top half of
|
|
33
|
-
*
|
|
35
|
+
* centre the ball is in the top half of that cell, at or above it the bottom.
|
|
36
|
+
* Half-rows and columns are the same size on screen, so this and the column
|
|
37
|
+
* together are a square lattice, and a step is a step whichever way it goes.
|
|
34
38
|
*/
|
|
35
|
-
export
|
|
36
|
-
const col = Math.round(x);
|
|
39
|
+
export const halfRow = (y) => {
|
|
37
40
|
const row = Math.round(y);
|
|
38
|
-
return
|
|
41
|
+
return y < row ? row * 2 : row * 2 + 1;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* A drawn ball, released on its own even clock.
|
|
46
|
+
*
|
|
47
|
+
* Equal pitch fixed the size of the ball's steps but not their timing, and the
|
|
48
|
+
* timing is the rest of the jiggle. Rounding the true position to the lattice
|
|
49
|
+
* moves the ball whenever it happens to cross a column edge or a half-row edge,
|
|
50
|
+
* and those two are on unrelated schedules: a ball with the two periods close
|
|
51
|
+
* but not equal — which is what a near-diagonal is, and what breakout launches
|
|
52
|
+
* at — beats between them. Measured, breakout held a cell for 16ms, then 80ms,
|
|
53
|
+
* then 48ms, five times a second. The steps were the right size and still the
|
|
54
|
+
* ball looked like it was struggling, because nothing was moving at a rate.
|
|
55
|
+
*
|
|
56
|
+
* So the true position is not what is drawn. It decides only which way the
|
|
57
|
+
* drawn ball owes a step; when that step is paid is decided by a clock that
|
|
58
|
+
* ticks at the ball's own speed. `owed` accrues at |vx| + 2|vy| lattice units
|
|
59
|
+
* per tick — the distance the true ball covers, measured the way the drawn one
|
|
60
|
+
* has to travel it — and a whole unit buys one step. The drawn ball therefore
|
|
61
|
+
* moves every 1/speed ticks whatever angle it is on, trailing the true one by
|
|
62
|
+
* under a unit, which is under half a character.
|
|
63
|
+
*
|
|
64
|
+
* It cannot fall behind: the same accrual that paces the ball also lets it pay
|
|
65
|
+
* two steps in a tick when the ball is genuinely moving that fast.
|
|
66
|
+
*/
|
|
67
|
+
export function drawnBall(x, y) {
|
|
68
|
+
return snapBall({ col: 0, half: 0, owed: 0 }, x, y);
|
|
39
69
|
}
|
|
40
70
|
|
|
41
71
|
/**
|
|
42
|
-
*
|
|
72
|
+
* Put the drawn ball exactly where the real one is, with no debt either way.
|
|
43
73
|
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
74
|
+
* For the moves that are not travel and so have nothing to smooth: a serve, a
|
|
75
|
+
* fresh ball on the paddle, the ball riding a paddle that is being aimed.
|
|
46
76
|
*/
|
|
47
|
-
export
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
77
|
+
export function snapBall(drawn, x, y) {
|
|
78
|
+
drawn.col = Math.round(x);
|
|
79
|
+
drawn.half = halfRow(y);
|
|
80
|
+
drawn.owed = 0;
|
|
81
|
+
return drawn;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Far enough apart that the ball was put there rather than travelled there. */
|
|
85
|
+
const TELEPORT = 4;
|
|
86
|
+
|
|
87
|
+
/** Pay out whatever steps the ball has earned this tick. */
|
|
88
|
+
export function advanceBall(drawn, ball) {
|
|
89
|
+
const col = Math.round(ball.x);
|
|
90
|
+
const half = halfRow(ball.y);
|
|
91
|
+
if (Math.abs(col - drawn.col) + Math.abs(half - drawn.half) >= TELEPORT) return snapBall(drawn, ball.x, ball.y);
|
|
92
|
+
|
|
93
|
+
drawn.owed += Math.abs(ball.vx) + Math.abs(ball.vy) * 2;
|
|
94
|
+
while (drawn.owed >= 1) {
|
|
95
|
+
const dcol = col - drawn.col;
|
|
96
|
+
const dhalf = half - drawn.half;
|
|
97
|
+
if (dcol === 0 && dhalf === 0) break;
|
|
98
|
+
// Whichever axis is further behind goes first, which is what keeps a
|
|
99
|
+
// diagonal a staircase instead of a sideways run and then a drop.
|
|
100
|
+
if (Math.abs(dcol) >= Math.abs(dhalf)) drawn.col += Math.sign(dcol);
|
|
101
|
+
else drawn.half += Math.sign(dhalf);
|
|
102
|
+
drawn.owed -= 1;
|
|
103
|
+
}
|
|
104
|
+
// A ball that has caught up banks at most one step, so that standing still
|
|
105
|
+
// for a moment cannot be turned into a lurch later.
|
|
106
|
+
if (drawn.owed > 1) drawn.owed = 1;
|
|
107
|
+
return drawn;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The cell and half block to write for a drawn ball. */
|
|
111
|
+
export const drawnCell = (drawn) => ({
|
|
112
|
+
col: drawn.col,
|
|
113
|
+
row: drawn.half >> 1,
|
|
114
|
+
glyph: drawn.half % 2 === 0 ? "▀" : "▄",
|
|
115
|
+
});
|
package/src/games-pong.mjs
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// A flat return it will always get; one taken off the end of your paddle it will
|
|
8
8
|
// not. That is the whole game, and it is why the angle off the paddle depends on
|
|
9
9
|
// where the ball hit it.
|
|
10
|
-
import {
|
|
10
|
+
import { advanceBall, drawnBall, drawnCell, snapBall } from "./games-draw.mjs";
|
|
11
11
|
import { acid, bone, danger, dim } from "./ui.mjs";
|
|
12
12
|
|
|
13
13
|
export const WIDTH = 44;
|
|
@@ -37,13 +37,25 @@ export const TARGET = 7; // first to this many
|
|
|
37
37
|
*/
|
|
38
38
|
export const TICK_MS = 16;
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* How hard the table is played, against the pace it was first tuned at.
|
|
42
|
+
*
|
|
43
|
+
* The old pace put the ball across the table in just under three seconds. That
|
|
44
|
+
* is not a ball being hit, it is a ball being carried, and it was also why the
|
|
45
|
+
* drawn ball only moved twenty times a second — too few steps for any of them
|
|
46
|
+
* to be smooth. Everything below is scaled by this, the machine along with the
|
|
47
|
+
* ball, so the balance is exactly the one that was tuned; only the clock it is
|
|
48
|
+
* played against changes.
|
|
49
|
+
*/
|
|
50
|
+
const PACE = 1.9;
|
|
51
|
+
|
|
40
52
|
/**
|
|
41
53
|
* The speeds below are still written per 55ms — the rate this game was tuned
|
|
42
54
|
* at — and scaled to the tick. Keeping the tuned numbers legible matters more
|
|
43
55
|
* than saving a multiply: they are what makes the machine beatable off the end
|
|
44
56
|
* of the paddle and not from the middle, and that balance is the game.
|
|
45
57
|
*/
|
|
46
|
-
const SCALE = TICK_MS / 55;
|
|
58
|
+
const SCALE = (TICK_MS / 55) * PACE;
|
|
47
59
|
|
|
48
60
|
const SERVE_SPEED = 0.85 * SCALE;
|
|
49
61
|
const MAX_SPEED = 1.7 * SCALE;
|
|
@@ -65,6 +77,8 @@ export function serve(state, toward) {
|
|
|
65
77
|
// Never dead flat: a ball with no angle is a rally nobody can lose.
|
|
66
78
|
vy: (state.rng() < 0.5 ? -1 : 1) * (0.15 + state.rng() * 0.2) * SCALE,
|
|
67
79
|
};
|
|
80
|
+
// A serve is a ball put on the table, not a ball that travelled there.
|
|
81
|
+
state.drawn = drawnBall(state.ball.x, state.ball.y);
|
|
68
82
|
return state;
|
|
69
83
|
}
|
|
70
84
|
|
|
@@ -106,6 +120,12 @@ export function step(state) {
|
|
|
106
120
|
|
|
107
121
|
if (ball.x < 0) { state.theirs++; point(state, 1); }
|
|
108
122
|
else if (ball.x > WIDTH - 1) { state.yours++; point(state, -1); }
|
|
123
|
+
// Anything else is the ball travelling, which is the only thing the drawn
|
|
124
|
+
// ball is asked to follow — a serve puts it back itself.
|
|
125
|
+
else advanceBall(state.drawn, ball);
|
|
126
|
+
// A match ends with the ball where it went out, off the table and so off the
|
|
127
|
+
// board, rather than parked on the edge it left by.
|
|
128
|
+
if (state.over) snapBall(state.drawn, ball.x, ball.y);
|
|
109
129
|
|
|
110
130
|
// The machine: idle in the middle until the ball is on its half, then chase
|
|
111
131
|
// the ball's row. Perfect tracking here would make the game unloseable for it,
|
|
@@ -165,9 +185,9 @@ export const PONG = {
|
|
|
165
185
|
|
|
166
186
|
for (const row of paddleRows(state.you)) put(YOU_COL, row, acid("█"));
|
|
167
187
|
for (const row of paddleRows(state.them)) put(THEM_COL, row, danger("█"));
|
|
168
|
-
// Drawn on half-rows, so the ball steps the same
|
|
169
|
-
//
|
|
170
|
-
const ball =
|
|
188
|
+
// Drawn on half-rows and on its own even clock, so the ball steps the same
|
|
189
|
+
// distance up the table as across it, and at a rate. See games-draw.mjs.
|
|
190
|
+
const ball = drawnCell(state.drawn);
|
|
171
191
|
put(ball.col, ball.row, bone(ball.glyph));
|
|
172
192
|
|
|
173
193
|
return grid.map((row, y) => row.map((cell, x) => (
|
package/src/news-sources.mjs
CHANGED
|
@@ -7,58 +7,26 @@
|
|
|
7
7
|
// brisk.news — apps/web/src/lib/news/fetch-feed.ts builds Google News feeds,
|
|
8
8
|
// top stories for the front page and one per category, and
|
|
9
9
|
// scripts/import-opml-feeds.ts seeds its publisher table from four public
|
|
10
|
-
// OPML lists.
|
|
11
|
-
// the
|
|
10
|
+
// OPML lists. The lists are represented here as named entries `/news add`
|
|
11
|
+
// accepts; the Google feeds are not, for the reason isDeadEndLink() gives.
|
|
12
12
|
//
|
|
13
13
|
// advis0r.com — src/providers/news/rss.ts pairs a Google News query feed with
|
|
14
14
|
// a Bing one (Google's links are interstitials, Bing's carry the publisher
|
|
15
|
-
// URL in a `url=` parameter), and reads two newswires directly.
|
|
16
|
-
// pairing
|
|
17
|
-
// under `markets`.
|
|
15
|
+
// URL in a `url=` parameter), and reads two newswires directly. Only the
|
|
16
|
+
// Bing half of that pairing survives here — see isDeadEndLink() — and the
|
|
17
|
+
// wires are under `markets`.
|
|
18
18
|
//
|
|
19
19
|
// Deliberately small. A default list is a claim that every entry works, so it
|
|
20
20
|
// holds feeds with stable well-known URLs and defers everything else to the
|
|
21
|
-
//
|
|
22
|
-
|
|
23
|
-
/** Google News locale. One place, because every builder below needs it. */
|
|
24
|
-
const GOOGLE_LOCALE = "hl=en-US&gl=US&ceid=US:en";
|
|
21
|
+
// published lists, where the list is somebody else's to maintain.
|
|
25
22
|
|
|
26
23
|
/**
|
|
27
|
-
*
|
|
24
|
+
* Bing News query feed — the only search feed left.
|
|
28
25
|
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
|
|
32
|
-
export const GOOGLE_NEWS_CATEGORIES = {
|
|
33
|
-
general: null,
|
|
34
|
-
science: "science",
|
|
35
|
-
sports: "sports",
|
|
36
|
-
business: "business",
|
|
37
|
-
health: "health",
|
|
38
|
-
entertainment: "entertainment",
|
|
39
|
-
tech: "technology",
|
|
40
|
-
politics: "politics",
|
|
41
|
-
food: "food",
|
|
42
|
-
travel: "travel",
|
|
43
|
-
};
|
|
44
|
-
|
|
45
|
-
/** Google News top stories, or one category from the map above. */
|
|
46
|
-
export function googleNewsFeed(category = null) {
|
|
47
|
-
const mapped = category ? GOOGLE_NEWS_CATEGORIES[category] ?? null : null;
|
|
48
|
-
if (!mapped) return `https://news.google.com/rss?${GOOGLE_LOCALE}`;
|
|
49
|
-
return `https://news.google.com/rss/search?q=${encodeURIComponent(mapped)}&${GOOGLE_LOCALE}`;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
/**
|
|
53
|
-
* Google News query feed. `when:7d` style windows keep results recent —
|
|
54
|
-
* advis0r's googleNewsFeed() does the same, for the same reason.
|
|
26
|
+
* The Google News builders that used to sit here were removed rather than left
|
|
27
|
+
* unused: keeping them would imply a Google News feed is still a thing this can
|
|
28
|
+
* read, and isDeadEndLink() explains at length why it is not.
|
|
55
29
|
*/
|
|
56
|
-
export function googleNewsSearch(query, { window = "7d" } = {}) {
|
|
57
|
-
const q = window ? `${query} when:${window}` : String(query);
|
|
58
|
-
return `https://news.google.com/rss/search?q=${encodeURIComponent(q)}&${GOOGLE_LOCALE}`;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/** Bing News query feed — the half of a search whose links are real articles. */
|
|
62
30
|
export function bingNewsSearch(query) {
|
|
63
31
|
return `https://www.bing.com/news/search?q=${encodeURIComponent(query)}&format=RSS`;
|
|
64
32
|
}
|
|
@@ -84,68 +52,176 @@ export function unwrapRedirect(url) {
|
|
|
84
52
|
}
|
|
85
53
|
}
|
|
86
54
|
|
|
55
|
+
/**
|
|
56
|
+
* Is this a link that can never be resolved to the article it stands for?
|
|
57
|
+
*
|
|
58
|
+
* Google News item links are `news.google.com/rss/articles/CBMi…?oc=5`, and
|
|
59
|
+
* they are a dead end in the strict sense — this was measured, not assumed:
|
|
60
|
+
*
|
|
61
|
+
* · A HEAD returns 302 whose `location` is the *same* URL with the locale
|
|
62
|
+
* appended, which then 200s on a Google page. There is no publisher URL in
|
|
63
|
+
* any header, so unwrapping by redirect does not work.
|
|
64
|
+
* · The base64 segment decodes to a protobuf holding an opaque `AU_yqL…`
|
|
65
|
+
* token and no URL.
|
|
66
|
+
* · A full GET returns a 600KB Angular shell — the redirect is client-side,
|
|
67
|
+
* and the target is not in the HTML.
|
|
68
|
+
* · That leaves Google's internal `/_/DotsSplashUi/data/batchexecute`, which
|
|
69
|
+
* answered 429 Too Many Requests on the first unauthenticated call. A
|
|
70
|
+
* reader cannot be built on an endpoint that rate-limits a single request.
|
|
71
|
+
* · Their `<description>` is an `<ol><li><a href=…>` list of more Google
|
|
72
|
+
* links rather than a summary, so there is no content to show either.
|
|
73
|
+
*
|
|
74
|
+
* So an item behind one of these is a headline that cannot be read and cannot
|
|
75
|
+
* be opened. collectNews drops them rather than listing rows that do nothing.
|
|
76
|
+
* Bing's search feed is unaffected: it carries the publisher URL in `url=` and
|
|
77
|
+
* a real summary, which is why it is the one kept for searching.
|
|
78
|
+
*/
|
|
79
|
+
export function isDeadEndLink(url) {
|
|
80
|
+
try {
|
|
81
|
+
const parsed = new URL(String(url));
|
|
82
|
+
return /(^|\.)news\.google\.com$/i.test(parsed.hostname)
|
|
83
|
+
&& /^\/rss\/articles\//i.test(parsed.pathname);
|
|
84
|
+
} catch {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
87
89
|
/**
|
|
88
90
|
* The feeds a fresh install reads.
|
|
89
91
|
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
92
|
+
* Every one of them is a publisher speaking for itself. Google News used to
|
|
93
|
+
* carry the general desks here — one URL shape covering ten sections without
|
|
94
|
+
* ten publisher relationships — and it was dropped precisely because of what
|
|
95
|
+
* isDeadEndLink() documents: its items cannot be opened and carry no summary,
|
|
96
|
+
* so a default list built on it is a list of rows that do nothing. The desks it
|
|
97
|
+
* covered (world, science, politics) are named publishers now instead.
|
|
98
|
+
*
|
|
99
|
+
* The profullstack blogs are here too, so `profullstack.com/feeds.opml` is read
|
|
100
|
+
* out of the box rather than only after `/news add profullstack`.
|
|
96
101
|
*/
|
|
97
102
|
export const DEFAULT_FEEDS = [
|
|
98
|
-
{ name: "top-stories", title: "Google News — Top Stories", url: googleNewsFeed(), site: "https://news.google.com", category: "" },
|
|
99
|
-
{ name: "world", title: "Google News — World", url: googleNewsSearch("world news", { window: "2d" }), site: "https://news.google.com", category: "" },
|
|
100
|
-
|
|
101
|
-
{ name: "tech", title: "Google News — Technology", url: googleNewsFeed("tech"), site: "https://news.google.com", category: "tech" },
|
|
102
103
|
{ name: "ars-technica", title: "Ars Technica", url: "https://feeds.arstechnica.com/arstechnica/index", site: "https://arstechnica.com", category: "tech" },
|
|
103
104
|
{ name: "techcrunch", title: "TechCrunch", url: "https://techcrunch.com/feed/", site: "https://techcrunch.com", category: "tech" },
|
|
104
105
|
{ name: "the-register", title: "The Register", url: "https://www.theregister.com/headlines.atom", site: "https://www.theregister.com", category: "tech" },
|
|
105
106
|
{ name: "hacker-news", title: "Hacker News — Front Page", url: "https://hnrss.org/frontpage", site: "https://news.ycombinator.com", category: "tech" },
|
|
106
107
|
|
|
107
|
-
{ name: "business", title: "Google News — Business", url: googleNewsFeed("business"), site: "https://news.google.com", category: "markets" },
|
|
108
108
|
{ name: "marketwatch", title: "MarketWatch — Top Stories", url: "https://feeds.content.dowjones.io/public/rss/mw_topstories", site: "https://www.marketwatch.com", category: "markets" },
|
|
109
109
|
// The two newswires advis0r reads directly. Tier 0 there: issuers speaking
|
|
110
110
|
// for themselves rather than a publication speaking about them.
|
|
111
111
|
{ name: "pr-newswire", title: "PR Newswire — Financial Services", url: "https://www.prnewswire.com/rss/financial-services-latest-news/financial-services-latest-news-list.rss", site: "https://www.prnewswire.com", category: "markets" },
|
|
112
112
|
{ name: "globenewswire", title: "GlobeNewswire — Public Companies", url: "https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies", site: "https://www.globenewswire.com", category: "markets" },
|
|
113
113
|
|
|
114
|
-
{ name: "
|
|
115
|
-
{ name: "
|
|
114
|
+
{ name: "npr-world", title: "NPR — World", url: "https://feeds.npr.org/1004/rss.xml", site: "https://www.npr.org", category: "world" },
|
|
115
|
+
{ name: "bbc-world", title: "BBC News — World", url: "https://feeds.bbci.co.uk/news/world/rss.xml", site: "https://www.bbc.co.uk/news", category: "world" },
|
|
116
|
+
{ name: "guardian-world", title: "The Guardian — World", url: "https://www.theguardian.com/world/rss", site: "https://www.theguardian.com", category: "world" },
|
|
117
|
+
|
|
118
|
+
{ name: "science-daily", title: "ScienceDaily — Top Science", url: "https://www.sciencedaily.com/rss/top/science.xml", site: "https://www.sciencedaily.com", category: "science" },
|
|
119
|
+
{ name: "phys-org", title: "Phys.org", url: "https://phys.org/rss-feed/", site: "https://phys.org", category: "science" },
|
|
120
|
+
|
|
121
|
+
{ name: "npr-politics", title: "NPR — Politics", url: "https://feeds.npr.org/1014/rss.xml", site: "https://www.npr.org", category: "politics" },
|
|
122
|
+
|
|
123
|
+
// The profullstack blogs, read out of the box. This is the one published list
|
|
124
|
+
// small enough to be a default: profullstack.com/feeds.opml is 14 feeds, where
|
|
125
|
+
// smallweb is 33,000 and could only ever be searched. Kept in step with that
|
|
126
|
+
// file by hand rather than fetched, because defaultFeeds() is synchronous and
|
|
127
|
+
// a fresh install must not wait on a network call to show anything at all.
|
|
128
|
+
{ name: "bittorrented-blog", title: "BitTorrented Blog", url: "https://bittorrented.com/blog/rss.xml", site: "https://bittorrented.com/blog", category: "profullstack" },
|
|
129
|
+
{ name: "bl0ggers-blog", title: "bl0ggers Blog", url: "https://bl0ggers.com/blog/rss.xml", site: "https://bl0ggers.com/blog", category: "profullstack" },
|
|
130
|
+
{ name: "c0mpute-blog", title: "c0mpute Blog", url: "https://c0mpute.com/blog/rss.xml", site: "https://c0mpute.com/blog", category: "profullstack" },
|
|
131
|
+
{ name: "c0upons-blog", title: "c0upons Blog", url: "https://c0upons.com/blog/rss.xml", site: "https://c0upons.com/blog", category: "profullstack" },
|
|
132
|
+
{ name: "coinpay-blog", title: "CoinPay Blog", url: "https://coinpayportal.com/blog/rss.xml", site: "https://coinpayportal.com/blog", category: "profullstack" },
|
|
133
|
+
{ name: "crawlproof-blog", title: "CrawlProof Blog", url: "https://crawlproof.com/blog/rss.xml", site: "https://crawlproof.com/blog", category: "profullstack" },
|
|
134
|
+
{ name: "d0rz-blog", title: "d0rz Blog", url: "https://d0rz.com/blog/rss.xml", site: "https://d0rz.com/blog", category: "profullstack" },
|
|
135
|
+
{ name: "logicsrc-blog", title: "LogicSRC Blog", url: "https://logicsrc.com/blog/rss.xml", site: "https://logicsrc.com/blog", category: "profullstack" },
|
|
136
|
+
{ name: "pairux-blog", title: "PairUX Blog", url: "https://pairux.com/blog/rss.xml", site: "https://pairux.com/blog", category: "profullstack" },
|
|
137
|
+
{ name: "qryptchat-blog", title: "QryptChat Blog", url: "https://qrypt.chat/blog/rss.xml", site: "https://qrypt.chat/blog", category: "profullstack" },
|
|
138
|
+
{ name: "saasrow-blog", title: "SaaSRow Blog", url: "https://www.saasrow.com/blog/rss.xml", site: "https://www.saasrow.com/blog", category: "profullstack" },
|
|
139
|
+
{ name: "sh1pt-blog", title: "sh1pt Blog", url: "https://sh1pt.com/blog/rss.xml", site: "https://sh1pt.com/blog", category: "profullstack" },
|
|
140
|
+
{ name: "threatcrush-blog", title: "ThreatCrush Blog", url: "https://threatcrush.com/blog/rss.xml", site: "https://threatcrush.com/blog", category: "profullstack" },
|
|
141
|
+
{ name: "ugig-blog", title: "ugig Blog", url: "https://ugig.net/blog/rss.xml", site: "https://ugig.net/blog", category: "profullstack" },
|
|
116
142
|
];
|
|
117
143
|
|
|
118
144
|
/**
|
|
119
|
-
* The public
|
|
145
|
+
* The public feed lists, offered by name.
|
|
120
146
|
*
|
|
121
147
|
* Offered by name (`/news add journalists`) rather than only by URL because
|
|
122
148
|
* these are long, exact raw.githubusercontent paths that nobody is going to
|
|
123
149
|
* retype, and importing one is the fastest way from an empty reader to a real
|
|
124
150
|
* one. They are somebody else's lists, and that is the point: the feeds inside
|
|
125
151
|
* them stay current without moshcode shipping a release.
|
|
152
|
+
*
|
|
153
|
+
* Two shapes, because the lists worth reading come in two:
|
|
154
|
+
*
|
|
155
|
+
* `format: "opml"` — an OPML document, parsed by parseOpml.
|
|
156
|
+
* `format: "text"` — one feed URL per line, `#` comments ignored. Kagi's
|
|
157
|
+
* smallweb list is published this way and there is no OPML of it.
|
|
158
|
+
*
|
|
159
|
+
* `searchOnly` marks a list too large to subscribe to wholesale. smallweb is
|
|
160
|
+
* 32,000+ feeds: importing it would write a 32,000-entry news.opml and then try
|
|
161
|
+
* to fetch every one of them on the next `/news`. It is a catalogue to search,
|
|
162
|
+
* not a subscription — `/rss search <keyword>` is how you get feeds out of it.
|
|
126
163
|
*/
|
|
127
|
-
export const
|
|
164
|
+
export const FEED_LISTS = [
|
|
128
165
|
{
|
|
129
166
|
name: "journalists",
|
|
130
167
|
description: "Dave Winer's feedsForJournalists — mainstream desks",
|
|
131
168
|
url: "https://raw.githubusercontent.com/scripting/feedsForJournalists/master/list.opml",
|
|
169
|
+
format: "opml",
|
|
132
170
|
},
|
|
133
171
|
{
|
|
134
172
|
name: "web3",
|
|
135
173
|
description: "ChainFeeds RSSAggregatorforWeb3 — crypto and web3",
|
|
136
174
|
url: "https://raw.githubusercontent.com/chainfeeds/RSSAggregatorforWeb3/main/RAW.opml",
|
|
175
|
+
format: "opml",
|
|
137
176
|
},
|
|
138
177
|
{
|
|
139
178
|
name: "blockchain",
|
|
140
179
|
description: "CoinFabrik decentralized-and-blockchain-feeds",
|
|
141
180
|
url: "https://raw.githubusercontent.com/CoinFabrik/resources/master/decentralized-and-blockchain-feeds.opml",
|
|
181
|
+
format: "opml",
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
name: "profullstack",
|
|
185
|
+
description: "Profullstack product blogs",
|
|
186
|
+
url: "https://profullstack.com/feeds.opml",
|
|
187
|
+
format: "opml",
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
name: "smallweb",
|
|
191
|
+
description: "Kagi Small Web — 33k personal blogs, with titles",
|
|
192
|
+
url: "https://kagi.com/smallweb/opml",
|
|
193
|
+
format: "opml",
|
|
194
|
+
searchOnly: true,
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
name: "smallweb-txt",
|
|
198
|
+
description: "Kagi Small Web, profullstack's fork — the plain-text list",
|
|
199
|
+
url: "https://raw.githubusercontent.com/ralyodio/smallweb/refs/heads/main/smallweb.txt",
|
|
200
|
+
format: "text",
|
|
201
|
+
searchOnly: true,
|
|
142
202
|
},
|
|
143
203
|
];
|
|
144
204
|
|
|
145
|
-
/**
|
|
146
|
-
|
|
205
|
+
/**
|
|
206
|
+
* The old name for the list catalogue.
|
|
207
|
+
*
|
|
208
|
+
* Kept because `/news sources --json` published it and a caller may be reading
|
|
209
|
+
* that key; the shape is unchanged for the three lists that were in it.
|
|
210
|
+
*/
|
|
211
|
+
export const OPML_BUNDLES = FEED_LISTS;
|
|
212
|
+
|
|
213
|
+
/** Resolve a list name to its entry, or null. */
|
|
214
|
+
export function resolveList(name) {
|
|
147
215
|
const wanted = String(name ?? "").trim().toLowerCase();
|
|
148
|
-
return
|
|
216
|
+
return FEED_LISTS.find((b) => b.name === wanted) ?? null;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** The former name of resolveList. */
|
|
220
|
+
export const resolveBundle = resolveList;
|
|
221
|
+
|
|
222
|
+
/** The lists `/news add <name>` will import in full. */
|
|
223
|
+
export function subscribableLists() {
|
|
224
|
+
return FEED_LISTS.filter((list) => !list.searchOnly);
|
|
149
225
|
}
|
|
150
226
|
|
|
151
227
|
/** A fresh copy of the defaults — callers mutate feed lists. */
|
package/src/news.mjs
CHANGED
|
@@ -26,9 +26,10 @@ import { acid, ash, amber, bone, danger } from "./ui.mjs";
|
|
|
26
26
|
import {
|
|
27
27
|
bingNewsSearch,
|
|
28
28
|
defaultFeeds,
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
FEED_LISTS,
|
|
30
|
+
isDeadEndLink,
|
|
31
|
+
resolveList,
|
|
32
|
+
subscribableLists,
|
|
32
33
|
unwrapRedirect,
|
|
33
34
|
} from "./news-sources.mjs";
|
|
34
35
|
|
|
@@ -39,11 +40,14 @@ const USAGE = `usage: moshcode news [verb|keyword…] [args…]
|
|
|
39
40
|
<keyword…> search the news for a word or phrase
|
|
40
41
|
<url> read one feed without subscribing to it
|
|
41
42
|
list the feeds you are subscribed to
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
find <keyword[,keyword…]> search every published list for feeds to add
|
|
44
|
+
add <url|file|list|#n> subscribe — an RSS/Atom link, an OPML file, a
|
|
45
|
+
list by name (${subscribableLists().map((b) => b.name).join(", ")}),
|
|
46
|
+
or #n from the last find
|
|
47
|
+
rm <name|url|list> unsubscribe — one feed, or a whole list at once
|
|
45
48
|
open <n> open headline <n> from the last listing
|
|
46
|
-
|
|
49
|
+
lists the published lists, and which you have added
|
|
50
|
+
sources the default feeds and the lists on offer
|
|
47
51
|
export print the subscription list as OPML
|
|
48
52
|
|
|
49
53
|
--json print structured data instead of headlines
|
|
@@ -63,7 +67,7 @@ export function newsUsage() {
|
|
|
63
67
|
}
|
|
64
68
|
|
|
65
69
|
/** Verb names, in help order. cli-schema's NEWS_VERBS must match (drift test). */
|
|
66
|
-
export const NEWS_VERB_NAMES = ["latest", "search", "list", "add", "rm", "open", "sources", "export"];
|
|
70
|
+
export const NEWS_VERB_NAMES = ["latest", "search", "list", "find", "add", "rm", "open", "lists", "sources", "export"];
|
|
67
71
|
|
|
68
72
|
// The same reasoning as crypto's alias table: the obvious synonym should not be
|
|
69
73
|
// an error. `import` is the word an OPML file invites, and it is the same verb
|
|
@@ -75,7 +79,9 @@ const VERB_ALIASES = {
|
|
|
75
79
|
sub: "add", subscribe: "add", import: "add", follow: "add",
|
|
76
80
|
remove: "rm", unsub: "rm", unsubscribe: "rm", del: "rm", delete: "rm",
|
|
77
81
|
read: "open", browse: "open", www: "open",
|
|
78
|
-
|
|
82
|
+
discover: "find", catalog: "find", catalogue: "find",
|
|
83
|
+
bundles: "lists",
|
|
84
|
+
defaults: "sources",
|
|
79
85
|
opml: "export", dump: "export",
|
|
80
86
|
};
|
|
81
87
|
|
|
@@ -399,6 +405,176 @@ export function findFeed(feeds, needle) {
|
|
|
399
405
|
?? null;
|
|
400
406
|
}
|
|
401
407
|
|
|
408
|
+
// ---------------------------------------------------------------------------
|
|
409
|
+
// Published lists — a catalogue to search, and a unit to subscribe to
|
|
410
|
+
// ---------------------------------------------------------------------------
|
|
411
|
+
|
|
412
|
+
/**
|
|
413
|
+
* Parse a newline-delimited feed list: one URL per line, `#` comments ignored.
|
|
414
|
+
*
|
|
415
|
+
* The other shape a published list comes in. Kagi's smallweb list is a plain
|
|
416
|
+
* text file of 32,000 URLs and there is no OPML of it, so a reader that only
|
|
417
|
+
* speaks OPML cannot read the largest list there is. No titles are invented
|
|
418
|
+
* beyond the hostname — naming 32,000 feeds properly would mean fetching
|
|
419
|
+
* 32,000 feeds, and the host is what the URL already tells us for free.
|
|
420
|
+
*/
|
|
421
|
+
export function parseFeedList(text) {
|
|
422
|
+
const feeds = [];
|
|
423
|
+
const seen = new Set();
|
|
424
|
+
for (const line of String(text ?? "").split(/\r?\n/)) {
|
|
425
|
+
const trimmed = line.trim();
|
|
426
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
427
|
+
const url = safeUrl(trimmed);
|
|
428
|
+
if (!url || seen.has(url)) continue;
|
|
429
|
+
seen.add(url);
|
|
430
|
+
feeds.push({ name: hostSlug(url), title: hostOf(url), url, site: "", category: "" });
|
|
431
|
+
}
|
|
432
|
+
return feeds;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/** Parse a list document as whichever of the two shapes it is. */
|
|
436
|
+
export function parseListDocument(body, format = "") {
|
|
437
|
+
if (format === "text") return parseFeedList(body);
|
|
438
|
+
if (format === "opml") return parseOpml(body);
|
|
439
|
+
return looksLikeOpml(body) ? parseOpml(body) : parseFeedList(body);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/** Where a fetched list is cached, so searching does not refetch 32k feeds. */
|
|
443
|
+
export function listCacheFile(name, env = process.env) {
|
|
444
|
+
return path.join(path.dirname(opmlFile(env)), "lists", `${slugify(name) || "list"}.json`);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/** Where the last `find` results are remembered, so `add 3` knows what 3 was. */
|
|
448
|
+
export function findCacheFile(env = process.env) {
|
|
449
|
+
return path.join(path.dirname(opmlFile(env)), "news-found.json");
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** A day. These lists change on the order of weeks; a search must not wait on the network. */
|
|
453
|
+
const LIST_CACHE_MS = 24 * 60 * 60 * 1000;
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* The feeds in a published list, from cache when it is fresh.
|
|
457
|
+
*
|
|
458
|
+
* A stale cache is preferred to an error when the fetch fails: the list was
|
|
459
|
+
* good yesterday and a search that works offline beats one that does not.
|
|
460
|
+
*/
|
|
461
|
+
export async function loadListFeeds(list, {
|
|
462
|
+
fetchImpl, env = process.env, timeoutMs = DEFAULT_TIMEOUT_MS, now = Date.now(), refresh = false,
|
|
463
|
+
} = {}) {
|
|
464
|
+
const file = listCacheFile(list.name, env);
|
|
465
|
+
const cached = () => {
|
|
466
|
+
try {
|
|
467
|
+
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
468
|
+
return Array.isArray(parsed?.feeds) ? parsed : null;
|
|
469
|
+
} catch { return null; }
|
|
470
|
+
};
|
|
471
|
+
|
|
472
|
+
if (!refresh) {
|
|
473
|
+
const hit = cached();
|
|
474
|
+
if (hit && now - Number(hit.at || 0) < LIST_CACHE_MS) {
|
|
475
|
+
return { ok: true, feeds: hit.feeds, cached: true };
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const res = await readSource(list.url, { fetchImpl, timeoutMs });
|
|
480
|
+
if (!res.ok) {
|
|
481
|
+
const hit = cached();
|
|
482
|
+
if (hit) return { ok: true, feeds: hit.feeds, cached: true, stale: true };
|
|
483
|
+
return { ok: false, error: res.error, feeds: [] };
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
const feeds = parseListDocument(res.body, list.format);
|
|
487
|
+
try {
|
|
488
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
489
|
+
fs.writeFileSync(file, `${JSON.stringify({ at: now, url: list.url, feeds })}\n`, { mode: FILE_MODE });
|
|
490
|
+
try { fs.chmodSync(file, FILE_MODE); } catch { /* best effort */ }
|
|
491
|
+
} catch { /* a cache that cannot be written must not fail the search */ }
|
|
492
|
+
return { ok: true, feeds, cached: false };
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* Split `ai,crypto` into keywords.
|
|
497
|
+
*
|
|
498
|
+
* Comma-separated rather than space-separated because feed titles have spaces
|
|
499
|
+
* in them: `/rss search hacker news` means one thing to a person and two to a
|
|
500
|
+
* tokenizer, and the comma settles it. Spaces around a comma are trimmed.
|
|
501
|
+
*/
|
|
502
|
+
export function parseKeywords(raw) {
|
|
503
|
+
return String(raw ?? "")
|
|
504
|
+
.split(",")
|
|
505
|
+
.map((s) => s.trim().toLowerCase())
|
|
506
|
+
.filter(Boolean);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Feeds matching any keyword, best match first.
|
|
511
|
+
*
|
|
512
|
+
* Ranked rather than filtered, because filtering cannot win both halves of this:
|
|
513
|
+
*
|
|
514
|
+
* · Plain substring matching searches 32,000 feeds for `rust` and returns
|
|
515
|
+
* Trust Machines, Trustnodes, frustrat.com and popthruster.com.
|
|
516
|
+
* · Requiring the keyword to start a word fixes that and then finds nothing
|
|
517
|
+
* at all for `homelab`, because the list's only metadata is the hostname
|
|
518
|
+
* and the blog is called myhomelab.net.
|
|
519
|
+
*
|
|
520
|
+
* So everything containing the keyword is kept, and a whole-word match sorts
|
|
521
|
+
* above a word that starts with it, which sorts above a match buried anywhere.
|
|
522
|
+
* With the results capped for display, the good ones are the ones you see.
|
|
523
|
+
* Sorting is stable, so the list's own order survives within a rank.
|
|
524
|
+
*/
|
|
525
|
+
export function scoreFeed(feed, keywords) {
|
|
526
|
+
const hay = `${feed.title || ""} ${feed.url} ${feed.category || ""}`.toLowerCase();
|
|
527
|
+
let best = 0;
|
|
528
|
+
for (const k of keywords) {
|
|
529
|
+
if (!hay.includes(k)) continue;
|
|
530
|
+
const tokens = hay.split(/[^a-z0-9]+/).filter(Boolean);
|
|
531
|
+
if (tokens.some((t) => t === k)) { best = Math.max(best, 3); continue; }
|
|
532
|
+
if (tokens.some((t) => t.startsWith(k))) { best = Math.max(best, 2); continue; }
|
|
533
|
+
best = Math.max(best, 1);
|
|
534
|
+
}
|
|
535
|
+
return best;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
export function matchFeeds(feeds, keywords) {
|
|
539
|
+
if (!keywords.length) return [];
|
|
540
|
+
return feeds
|
|
541
|
+
.map((feed, i) => ({ feed, rank: scoreFeed(feed, keywords), i }))
|
|
542
|
+
.filter((row) => row.rank > 0)
|
|
543
|
+
.sort((a, b) => (b.rank - a.rank) || (a.i - b.i))
|
|
544
|
+
.map((row) => row.feed);
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* The list a subscribed feed came from, or "".
|
|
549
|
+
*
|
|
550
|
+
* Provenance rides in the OPML folder — the first path segment of `category` —
|
|
551
|
+
* rather than in a sidecar file, so it survives a round trip through any other
|
|
552
|
+
* reader and needs no format of our own. Only catalogue names count, so a
|
|
553
|
+
* folder someone happens to call "tech" is not mistaken for a list.
|
|
554
|
+
*/
|
|
555
|
+
export function listOf(feed) {
|
|
556
|
+
const head = String(feed?.category || "").split("/")[0];
|
|
557
|
+
return resolveList(head) ? head : "";
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** How many subscribed feeds came from each published list. */
|
|
561
|
+
export function subscribedLists(feeds = []) {
|
|
562
|
+
const counts = new Map();
|
|
563
|
+
for (const feed of feeds) {
|
|
564
|
+
const name = listOf(feed);
|
|
565
|
+
if (name) counts.set(name, (counts.get(name) || 0) + 1);
|
|
566
|
+
}
|
|
567
|
+
return counts;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
/** Tag a list's feeds with where they came from, keeping any folder they had. */
|
|
571
|
+
export function tagWithList(feeds, listName) {
|
|
572
|
+
return feeds.map((feed) => ({
|
|
573
|
+
...feed,
|
|
574
|
+
category: feed.category ? `${listName}/${feed.category}` : listName,
|
|
575
|
+
}));
|
|
576
|
+
}
|
|
577
|
+
|
|
402
578
|
// ---------------------------------------------------------------------------
|
|
403
579
|
// Feeds — RSS 2.0, Atom, and RSS 1.0/RDF
|
|
404
580
|
// ---------------------------------------------------------------------------
|
|
@@ -597,6 +773,11 @@ export async function collectNews(feeds, { fetchImpl, timeoutMs = DEFAULT_TIMEOU
|
|
|
597
773
|
// before deduping, so the same story arriving via Google News and via the
|
|
598
774
|
// publisher's own feed is recognised as one story rather than two.
|
|
599
775
|
const link = item.link ? unwrapRedirect(item.link) : null;
|
|
776
|
+
// A link that unwrapping cannot rescue is a row that would do nothing
|
|
777
|
+
// when opened, so the headline is dropped rather than shown. Only
|
|
778
|
+
// aggregator interstitials qualify; an item with no link at all is still
|
|
779
|
+
// worth listing, because its title carries the news.
|
|
780
|
+
if (link && isDeadEndLink(link)) continue;
|
|
600
781
|
const key = link || `${result.feed.name}:${item.title}`;
|
|
601
782
|
if (seen.has(key)) continue;
|
|
602
783
|
seen.add(key);
|
|
@@ -621,8 +802,11 @@ export async function collectNews(feeds, { fetchImpl, timeoutMs = DEFAULT_TIMEOU
|
|
|
621
802
|
* openable links wherever the two overlap.
|
|
622
803
|
*/
|
|
623
804
|
export function searchFeeds(query) {
|
|
805
|
+
// Bing only. The Google half of advis0r's pairing returned headlines whose
|
|
806
|
+
// links go nowhere and whose descriptions are lists of more Google links, so
|
|
807
|
+
// every one of its results was dropped downstream anyway — see
|
|
808
|
+
// isDeadEndLink(). Bing carries the publisher URL and a real summary.
|
|
624
809
|
return [
|
|
625
|
-
{ name: "google", title: `Google News — ${query}`, url: googleNewsSearch(query), site: "", category: "" },
|
|
626
810
|
{ name: "bing", title: `Bing News — ${query}`, url: bingNewsSearch(query), site: "", category: "" },
|
|
627
811
|
];
|
|
628
812
|
}
|
|
@@ -723,13 +907,28 @@ export function newsArgs(argv = []) {
|
|
|
723
907
|
if (!query) return { error: "usage: moshcode news search <keyword…>" };
|
|
724
908
|
return { ...base, verb, query };
|
|
725
909
|
}
|
|
910
|
+
if (verb === "find") {
|
|
911
|
+
const query = tail.join(" ").trim();
|
|
912
|
+
if (!query) return { error: "usage: moshcode news find <keyword[,keyword…]>" };
|
|
913
|
+
const keywords = parseKeywords(query);
|
|
914
|
+
if (!keywords.length) return { error: "find needs a keyword" };
|
|
915
|
+
return { ...base, verb, keywords };
|
|
916
|
+
}
|
|
726
917
|
if (verb === "add") {
|
|
727
|
-
if (!tail.length) return { error: "usage: moshcode news add <url|file>" };
|
|
728
|
-
if (tail.length > 1) return { error: "add takes one URL or
|
|
918
|
+
if (!tail.length) return { error: "usage: moshcode news add <url|file|list|#n>" };
|
|
919
|
+
if (tail.length > 1) return { error: "add takes one URL, list, or number at a time" };
|
|
920
|
+
// `add 3` and `add #3` take the third result of the last find. A feed URL
|
|
921
|
+
// is never a bare number, so this cannot shadow one.
|
|
922
|
+
const asIndex = /^#?(\d+)$/.exec(String(tail[0]).trim());
|
|
923
|
+
if (asIndex) {
|
|
924
|
+
const n = Number(asIndex[1]);
|
|
925
|
+
if (n < 1) return { error: `add takes a result number from 1, got ${JSON.stringify(tail[0])}` };
|
|
926
|
+
return { ...base, verb, target: null, index: n };
|
|
927
|
+
}
|
|
729
928
|
return { ...base, verb, target: tail[0] };
|
|
730
929
|
}
|
|
731
930
|
if (verb === "rm") {
|
|
732
|
-
if (!tail.length) return { error: "usage: moshcode news rm <name|url>" };
|
|
931
|
+
if (!tail.length) return { error: "usage: moshcode news rm <name|url|list>" };
|
|
733
932
|
return { ...base, verb, target: tail.join(" ") };
|
|
734
933
|
}
|
|
735
934
|
if (verb === "open") {
|
|
@@ -808,8 +1007,9 @@ export function renderFeeds(feeds, { file = "", usingDefaults = false } = {}) {
|
|
|
808
1007
|
if (!feeds.length) {
|
|
809
1008
|
return ["", ` ${ash("no feeds yet")}`, "",
|
|
810
1009
|
` ${ash("add one:")} ${bone("/news add https://example.com/feed.xml")}`,
|
|
811
|
-
` ${ash("or a
|
|
812
|
-
` ${ash("or a
|
|
1010
|
+
` ${ash("or a file:")} ${bone("/news add ~/subscriptions.opml")}`,
|
|
1011
|
+
` ${ash("or a list:")} ${bone("/news add journalists")}`,
|
|
1012
|
+
` ${ash("or search:")} ${bone("/rss search ai,crypto")}`].join("\n");
|
|
813
1013
|
}
|
|
814
1014
|
const width = Math.max(...feeds.map((f) => f.name.length));
|
|
815
1015
|
const header = usingDefaults
|
|
@@ -825,7 +1025,7 @@ export function renderFeeds(feeds, { file = "", usingDefaults = false } = {}) {
|
|
|
825
1025
|
lines.push(` ${acid(feed.name.padEnd(width))} ${bone(clip(feed.title || "", 34).padEnd(36))}${ash(feed.url)}`);
|
|
826
1026
|
}
|
|
827
1027
|
lines.push("", usingDefaults
|
|
828
|
-
? ` ${ash("subscribe to your own with")} ${bone("/news add <url|
|
|
1028
|
+
? ` ${ash("subscribe to your own with")} ${bone("/news add <url|list>")} ${ash("· see them with")} ${bone("/news lists")}`
|
|
829
1029
|
: ` ${ash("read them with")} ${bone("/news")} ${ash("· one of them with")} ${bone("/news --feed <name>")}`);
|
|
830
1030
|
return lines.join("\n");
|
|
831
1031
|
}
|
|
@@ -843,12 +1043,60 @@ export function renderSources() {
|
|
|
843
1043
|
}
|
|
844
1044
|
lines.push(` ${acid(feed.name.padEnd(width))} ${ash(clip(feed.title || "", 44))}`);
|
|
845
1045
|
}
|
|
846
|
-
lines.push("", ` ${bone("
|
|
847
|
-
const bw = Math.max(...
|
|
848
|
-
for (const
|
|
849
|
-
|
|
1046
|
+
lines.push("", ` ${bone("lists")} ${ash("— published feed lists, pull one in by name")}`, "");
|
|
1047
|
+
const bw = Math.max(...FEED_LISTS.map((b) => b.name.length));
|
|
1048
|
+
for (const list of FEED_LISTS) {
|
|
1049
|
+
const note = list.searchOnly ? ash(" (search only)") : "";
|
|
1050
|
+
lines.push(` ${acid(list.name.padEnd(bw))} ${ash(clip(list.description, 52))}${note}`);
|
|
850
1051
|
}
|
|
851
|
-
lines.push("", ` ${ash("pull one in with")} ${bone(`/news add ${
|
|
1052
|
+
lines.push("", ` ${ash("pull one in with")} ${bone(`/news add ${subscribableLists()[0].name}`)} ${ash("· search them all with")} ${bone("/rss search <keyword>")}`);
|
|
1053
|
+
return lines.join("\n");
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* The published lists, and how many feeds you have from each.
|
|
1058
|
+
*
|
|
1059
|
+
* `subscribed` is counted from the subscription file rather than remembered
|
|
1060
|
+
* separately, so a list stays "added" exactly as long as its feeds are there —
|
|
1061
|
+
* removing them by hand cannot leave a phantom membership behind.
|
|
1062
|
+
*/
|
|
1063
|
+
export function renderLists(feeds = []) {
|
|
1064
|
+
const mine = subscribedLists(feeds);
|
|
1065
|
+
const width = Math.max(...FEED_LISTS.map((b) => b.name.length));
|
|
1066
|
+
const lines = ["", ` ${ash("published lists")}`, ""];
|
|
1067
|
+
for (const list of FEED_LISTS) {
|
|
1068
|
+
const count = mine.get(list.name) || 0;
|
|
1069
|
+
const state = count
|
|
1070
|
+
? acid(`✓ ${count} feed${count === 1 ? "" : "s"}`)
|
|
1071
|
+
: (list.searchOnly ? ash("search only") : ash("—"));
|
|
1072
|
+
lines.push(` ${acid(list.name.padEnd(width))} ${bone(clip(list.description, 46).padEnd(48))}${state}`);
|
|
1073
|
+
}
|
|
1074
|
+
lines.push("",
|
|
1075
|
+
` ${ash("add one:")} ${bone("/news add profullstack")}`,
|
|
1076
|
+
` ${ash("drop one:")} ${bone("/news rm profullstack")} ${ash("— removes every feed it brought in")}`,
|
|
1077
|
+
` ${ash("search them:")} ${bone("/rss search ai,crypto")}`);
|
|
1078
|
+
return lines.join("\n");
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
/** Feeds found in the published lists, numbered so `add #n` can take one. */
|
|
1082
|
+
export function renderFindHits(hits, { keywords = [], total = 0, subscribed = [] } = {}) {
|
|
1083
|
+
if (!hits.length) {
|
|
1084
|
+
return ["", ` ${ash(`nothing in the published lists matches ${keywords.join(", ")}`)}`, "",
|
|
1085
|
+
` ${ash("the lists searched:")} ${bone(FEED_LISTS.map((l) => l.name).join(", "))}`,
|
|
1086
|
+
` ${ash("or subscribe by URL:")} ${bone("/rss add https://example.com/feed.xml")}`].join("\n");
|
|
1087
|
+
}
|
|
1088
|
+
const have = new Set(subscribed.map((f) => f.url));
|
|
1089
|
+
const width = String(hits.length).length;
|
|
1090
|
+
const lines = ["",
|
|
1091
|
+
` ${ash(`${total} feed${total === 1 ? "" : "s"} match ${keywords.join(", ")}${total > hits.length ? ` · showing ${hits.length}` : ""}`)}`,
|
|
1092
|
+
""];
|
|
1093
|
+
hits.forEach((feed, i) => {
|
|
1094
|
+
const n = String(i + 1).padStart(width);
|
|
1095
|
+
const mark = have.has(feed.url) ? acid(" ✓") : " ";
|
|
1096
|
+
lines.push(` ${bone(n)}.${mark} ${acid(clip(feed.title || hostOf(feed.url), 30).padEnd(32))}${ash(clip(feed.url, 58))}`);
|
|
1097
|
+
if (feed.list) lines.push(` ${ash(`from ${feed.list}`)}`);
|
|
1098
|
+
});
|
|
1099
|
+
lines.push("", ` ${ash("subscribe with")} ${bone("/rss add 1")} ${ash("· or the URL ·")} ${acid("✓")} ${ash("means already subscribed")}`);
|
|
852
1100
|
return lines.join("\n");
|
|
853
1101
|
}
|
|
854
1102
|
|
|
@@ -903,11 +1151,28 @@ export async function newsCommand(argv = [], deps = {}) {
|
|
|
903
1151
|
}
|
|
904
1152
|
|
|
905
1153
|
if (request.verb === "sources") {
|
|
906
|
-
|
|
1154
|
+
// `bundles` is the old key for the same array; kept so a caller reading it
|
|
1155
|
+
// does not break on the rename.
|
|
1156
|
+
if (request.json) { out(JSON.stringify({ defaults: defaultFeeds(), lists: FEED_LISTS, bundles: FEED_LISTS }, null, 2)); return 0; }
|
|
907
1157
|
out(renderSources());
|
|
908
1158
|
return 0;
|
|
909
1159
|
}
|
|
910
1160
|
|
|
1161
|
+
if (request.verb === "lists") {
|
|
1162
|
+
const feeds = loadFeeds(env);
|
|
1163
|
+
if (request.json) {
|
|
1164
|
+
const mine = subscribedLists(feeds);
|
|
1165
|
+
out(JSON.stringify({
|
|
1166
|
+
lists: FEED_LISTS.map((list) => ({ ...list, subscribed: mine.get(list.name) || 0 })),
|
|
1167
|
+
}, null, 2));
|
|
1168
|
+
return 0;
|
|
1169
|
+
}
|
|
1170
|
+
out(renderLists(feeds));
|
|
1171
|
+
return 0;
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
if (request.verb === "find") return findCommand(request, { out, fail, fetchImpl, env });
|
|
1175
|
+
|
|
911
1176
|
if (request.verb === "export") {
|
|
912
1177
|
// The defaults deliberately: exporting an empty file to hand to a reader is
|
|
913
1178
|
// not what anyone means by "export my feeds" on a fresh install.
|
|
@@ -979,11 +1244,86 @@ export async function newsCommand(argv = [], deps = {}) {
|
|
|
979
1244
|
return items.length === 0 && failures.length === feeds.length ? 1 : 0;
|
|
980
1245
|
}
|
|
981
1246
|
|
|
982
|
-
/**
|
|
1247
|
+
/** How many matches a find prints. The rest are counted, not listed. */
|
|
1248
|
+
const FIND_LIMIT = 25;
|
|
1249
|
+
|
|
1250
|
+
/**
|
|
1251
|
+
* `news find <keyword,…>` — search every published list for feeds to subscribe to.
|
|
1252
|
+
*
|
|
1253
|
+
* This is the way into the big lists. smallweb alone is 32,000 feeds, which is
|
|
1254
|
+
* a fine thing to search and an impossible thing to subscribe to, so the
|
|
1255
|
+
* catalogue is read here and only the matches become candidates for `add`.
|
|
1256
|
+
*/
|
|
1257
|
+
async function findCommand(request, { out, fail, fetchImpl, env }) {
|
|
1258
|
+
const hits = [];
|
|
1259
|
+
const seen = new Set();
|
|
1260
|
+
const failures = [];
|
|
1261
|
+
|
|
1262
|
+
for (const list of FEED_LISTS) {
|
|
1263
|
+
const loaded = await loadListFeeds(list, { fetchImpl, env, timeoutMs: request.timeoutMs });
|
|
1264
|
+
if (!loaded.ok) { failures.push(`${list.name}: ${loaded.error}`); continue; }
|
|
1265
|
+
for (const feed of matchFeeds(loaded.feeds, request.keywords)) {
|
|
1266
|
+
if (seen.has(feed.url)) continue;
|
|
1267
|
+
seen.add(feed.url);
|
|
1268
|
+
hits.push({ ...feed, list: list.name });
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
// Ranked across every list, not within each one. Sorting per list and then
|
|
1273
|
+
// concatenating puts all of web3's weak matches above smallweb's exact ones
|
|
1274
|
+
// purely because web3 is fetched first.
|
|
1275
|
+
hits.sort((a, b) => scoreFeed(b, request.keywords) - scoreFeed(a, request.keywords));
|
|
1276
|
+
|
|
1277
|
+
if (!hits.length && failures.length === FEED_LISTS.length) {
|
|
1278
|
+
fail(danger("✗ could not read any of the published lists"));
|
|
1279
|
+
for (const line of failures) fail(` ${ash(line)}`);
|
|
1280
|
+
return 1;
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
const shown = hits.slice(0, FIND_LIMIT);
|
|
1284
|
+
// Remembered so `add 3` resolves, and only the shown ones — a number the
|
|
1285
|
+
// operator never saw is not a number they can have meant.
|
|
1286
|
+
try {
|
|
1287
|
+
const file = findCacheFile(env);
|
|
1288
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
1289
|
+
fs.writeFileSync(file, `${JSON.stringify({ at: Date.now(), keywords: request.keywords, hits: shown }, null, 2)}\n`, { mode: FILE_MODE });
|
|
1290
|
+
try { fs.chmodSync(file, FILE_MODE); } catch { /* best effort */ }
|
|
1291
|
+
} catch { /* a cache that cannot be written must not fail the search */ }
|
|
1292
|
+
|
|
1293
|
+
if (request.json) { out(JSON.stringify({ keywords: request.keywords, total: hits.length, hits: shown, failures }, null, 2)); return 0; }
|
|
1294
|
+
out(renderFindHits(shown, { keywords: request.keywords, total: hits.length, subscribed: loadFeeds(env) }));
|
|
1295
|
+
for (const line of failures) fail(` ${ash(`· ${line}`)}`);
|
|
1296
|
+
return 0;
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
/** The feed a previous `find` numbered `n`, or null. */
|
|
1300
|
+
function foundAt(index, env) {
|
|
1301
|
+
try {
|
|
1302
|
+
const parsed = JSON.parse(fs.readFileSync(findCacheFile(env), "utf8"));
|
|
1303
|
+
return Array.isArray(parsed?.hits) ? parsed.hits[index - 1] ?? null : null;
|
|
1304
|
+
} catch { return null; }
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
/** `news add <url|file|list|#n>` — one feed, or every feed in a list. */
|
|
983
1308
|
async function addCommand(request, { out, fail, fetchImpl, env }) {
|
|
984
|
-
//
|
|
985
|
-
|
|
986
|
-
|
|
1309
|
+
// `add 3` — the third result of the last find, subscribed by its URL.
|
|
1310
|
+
if (request.index != null) {
|
|
1311
|
+
const found = foundAt(request.index, env);
|
|
1312
|
+
if (!found) {
|
|
1313
|
+
fail(danger(`✗ there is no result ${request.index} — run \`/rss search <keyword>\` first`));
|
|
1314
|
+
return 1;
|
|
1315
|
+
}
|
|
1316
|
+
request = { ...request, target: found.url, index: null };
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
// A list name resolves to somebody else's published list. Checked before the
|
|
1320
|
+
// URL and the path so `journalists` is a name rather than a missing file.
|
|
1321
|
+
const bundle = resolveList(request.target);
|
|
1322
|
+
if (bundle?.searchOnly) {
|
|
1323
|
+
fail(danger(`✗ ${bundle.name} is ${bundle.description}`));
|
|
1324
|
+
fail(` ${ash("too large to subscribe to wholesale — search it instead:")} ${bone(`/rss search <keyword>`)}`);
|
|
1325
|
+
return 1;
|
|
1326
|
+
}
|
|
987
1327
|
const target = bundle ? bundle.url : request.target;
|
|
988
1328
|
if (bundle) out(`${ash("· ")}fetching ${bone(bundle.name)} ${ash(`— ${bundle.description}`)}`);
|
|
989
1329
|
|
|
@@ -996,9 +1336,13 @@ async function addCommand(request, { out, fail, fetchImpl, env }) {
|
|
|
996
1336
|
const existing = loadFeeds(env);
|
|
997
1337
|
const asUrl = safeUrl(target);
|
|
998
1338
|
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1339
|
+
// A named list is whichever shape it says it is; anything else is a list only
|
|
1340
|
+
// if it announces itself as OPML, so a plain feed is never read as one.
|
|
1341
|
+
if (bundle || looksLikeOpml(res.body)) {
|
|
1342
|
+
const parsedList = parseListDocument(res.body, bundle?.format);
|
|
1343
|
+
// Tagged with the list it came from, so `rm <list>` can take it back out.
|
|
1344
|
+
const incoming = bundle ? tagWithList(parsedList, bundle.name) : parsedList;
|
|
1345
|
+
if (!incoming.length) { fail(danger(`✗ that ${bundle?.format === "text" ? "list" : "OPML file"} lists no feeds`)); return 1; }
|
|
1002
1346
|
let feeds = existing;
|
|
1003
1347
|
const added = [];
|
|
1004
1348
|
let skipped = 0;
|
|
@@ -1054,9 +1398,27 @@ async function addCommand(request, { out, fail, fetchImpl, env }) {
|
|
|
1054
1398
|
return 0;
|
|
1055
1399
|
}
|
|
1056
1400
|
|
|
1057
|
-
/** `news rm <name|url>` — unsubscribe. */
|
|
1401
|
+
/** `news rm <name|url|list>` — unsubscribe from one feed, or a whole list. */
|
|
1058
1402
|
function removeCommand(request, { out, fail, env }) {
|
|
1059
1403
|
const feeds = loadFeeds(env);
|
|
1404
|
+
|
|
1405
|
+
// A list name takes the whole list back out. Checked first: subscribing to a
|
|
1406
|
+
// list is one action and unsubscribing from it should be one action too,
|
|
1407
|
+
// rather than the operator deleting forty feeds by hand.
|
|
1408
|
+
const asList = resolveList(request.target);
|
|
1409
|
+
if (asList) {
|
|
1410
|
+
const mine = feeds.filter((f) => listOf(f) === asList.name);
|
|
1411
|
+
if (!mine.length) {
|
|
1412
|
+
fail(danger(`✗ you have no feeds from "${asList.name}"`));
|
|
1413
|
+
return 1;
|
|
1414
|
+
}
|
|
1415
|
+
try { saveFeeds(feeds.filter((f) => listOf(f) !== asList.name), env); }
|
|
1416
|
+
catch (e) { fail(danger(`✗ can't write ${opmlFile(env)}: ${e.message}`)); return 1; }
|
|
1417
|
+
if (request.json) { out(JSON.stringify({ removed: mine, list: asList.name }, null, 2)); return 0; }
|
|
1418
|
+
out(`${acid("✓ ")}unsubscribed from ${bone(asList.name)} ${ash(`— ${mine.length} feed${mine.length === 1 ? "" : "s"}`)}`);
|
|
1419
|
+
return 0;
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1060
1422
|
const feed = findFeed(feeds, request.target);
|
|
1061
1423
|
if (!feed) {
|
|
1062
1424
|
fail(danger(`✗ no feed named "${request.target}"`));
|
package/src/rss-ui.mjs
CHANGED
|
@@ -18,10 +18,21 @@ import {
|
|
|
18
18
|
ago,
|
|
19
19
|
collectNews,
|
|
20
20
|
findFeed,
|
|
21
|
+
newsCommand,
|
|
21
22
|
readingList,
|
|
23
|
+
resolveVerb,
|
|
22
24
|
searchFeeds,
|
|
23
25
|
} from "./news.mjs";
|
|
24
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Verbs `/rss` hands straight to `moshcode news`.
|
|
29
|
+
*
|
|
30
|
+
* Everything that manages the subscription list rather than reading it. They
|
|
31
|
+
* print and exit, so they work with no TTY — which is why this runs before the
|
|
32
|
+
* interactive-terminal check.
|
|
33
|
+
*/
|
|
34
|
+
const MANAGEMENT_VERBS = new Set(["add", "rm", "list", "lists", "find", "sources", "export", "open"]);
|
|
35
|
+
|
|
25
36
|
const ESC = {
|
|
26
37
|
altOn: "\x1b[?1049h", altOff: "\x1b[?1049l",
|
|
27
38
|
hideCursor: "\x1b[?25l", showCursor: "\x1b[?25h",
|
|
@@ -305,14 +316,37 @@ export async function rssUi(argv = [], deps = {}) {
|
|
|
305
316
|
write = (s) => process.stdout.write(`${s}\n`),
|
|
306
317
|
} = deps;
|
|
307
318
|
|
|
319
|
+
const args = (Array.isArray(argv) ? argv : []).map(String);
|
|
320
|
+
const words = args.filter((a) => !a.startsWith("-"));
|
|
321
|
+
const verb = resolveVerb(words[0]);
|
|
322
|
+
|
|
323
|
+
// Managing subscriptions is the same work under either name, and `/rss add
|
|
324
|
+
// <url>` is what people type — the reader is where you are when you decide to
|
|
325
|
+
// subscribe to something. Handing these to newsCommand rather than reading
|
|
326
|
+
// them as a search is the whole difference between `/rss add <url>` working
|
|
327
|
+
// and it silently opening a reader on the literal phrase "add <url>".
|
|
328
|
+
//
|
|
329
|
+
// `search` is deliberately not among them. On `/news` it searches headlines;
|
|
330
|
+
// here it searches the published lists for feeds to add, because a reader
|
|
331
|
+
// already has `/` for searching what it is showing.
|
|
332
|
+
if (verb && MANAGEMENT_VERBS.has(verb)) {
|
|
333
|
+
return newsCommand(args, { fetchImpl, openUrl, env, out: write, fail: write });
|
|
334
|
+
}
|
|
335
|
+
if (verb === "search") {
|
|
336
|
+
const keywords = words.slice(1).join(" ").trim();
|
|
337
|
+
if (!keywords) { write("usage: moshcode rss search <keyword[,keyword…]>"); return 1; }
|
|
338
|
+
return newsCommand(["find", ...words.slice(1)], { fetchImpl, openUrl, env, out: write, fail: write });
|
|
339
|
+
}
|
|
340
|
+
|
|
308
341
|
if (!stdin.isTTY || !stdout.isTTY) {
|
|
309
342
|
write("moshcode rss needs an interactive terminal — try `moshcode news`");
|
|
310
343
|
return 1;
|
|
311
344
|
}
|
|
312
345
|
|
|
313
346
|
// A query on the command line (`moshcode rss tariffs`) opens straight into
|
|
314
|
-
// the search, which is the same shape `/news <keyword>` has.
|
|
315
|
-
|
|
347
|
+
// the search, which is the same shape `/news <keyword>` has. `latest` is the
|
|
348
|
+
// verb for "no query", so it is dropped rather than searched for.
|
|
349
|
+
const query = (verb === "latest" ? "" : words.join(" ")).trim();
|
|
316
350
|
const list = readingList(env);
|
|
317
351
|
|
|
318
352
|
const state = {
|
package/src/settings-sync.mjs
CHANGED
|
@@ -58,6 +58,12 @@ export const MAX_TOTAL_BYTES = 256 * 1024;
|
|
|
58
58
|
export const SYNCED_FILES = [
|
|
59
59
|
{ path: "aliases.json", json: true, label: "pit aliases" },
|
|
60
60
|
{ path: "herd/rules.json", json: true, label: "herd state rules" },
|
|
61
|
+
// OPML rather than JSON, so `json: false`: it is the interchange format every
|
|
62
|
+
// feed reader already imports and exports, which is the whole reason a feed
|
|
63
|
+
// list is worth carrying between machines at all. Written by `tcfeed rss add`
|
|
64
|
+
// and read by nothing here — moshcode's interest in it begins and ends with
|
|
65
|
+
// moving it, and a file this does not parse cannot be broken by this.
|
|
66
|
+
{ path: "feeds.opml", json: false, label: "rss feeds" },
|
|
61
67
|
];
|
|
62
68
|
|
|
63
69
|
/**
|
|
@@ -349,7 +355,7 @@ function endpoint(creds) {
|
|
|
349
355
|
* code; a thrown network error inside the pit's dispatch loop would take the
|
|
350
356
|
* prompt down instead, which is a lost session over a dropped wifi connection.
|
|
351
357
|
*/
|
|
352
|
-
async function
|
|
358
|
+
async function attempt(method, route, { creds, body, fetchImpl, timeoutMs }) {
|
|
353
359
|
const controller = new AbortController();
|
|
354
360
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
355
361
|
try {
|
|
@@ -374,6 +380,36 @@ async function request(method, route, { creds, body = null, fetchImpl = fetch, t
|
|
|
374
380
|
}
|
|
375
381
|
}
|
|
376
382
|
|
|
383
|
+
/**
|
|
384
|
+
* Retried only where a retry can be right: the app not answering.
|
|
385
|
+
*
|
|
386
|
+
* 502, 503, 504 and a dead socket say nothing about the request — the platform
|
|
387
|
+
* returned them without the app seeing it, so the same bytes sent a second
|
|
388
|
+
* later are as likely to work as they were the first time. Every other status
|
|
389
|
+
* is an answer: 400 will be 400 again, 401 wants `/login`, and 409 is another
|
|
390
|
+
* machine having saved first, which retrying would only re-ask.
|
|
391
|
+
*
|
|
392
|
+
* Unretried, one blip is a visible failure — and it looked frequent because
|
|
393
|
+
* every one of them was. `/save` had no retry at all while the scan workflow
|
|
394
|
+
* this same repository ships retries `npm install` three times for precisely
|
|
395
|
+
* this reasoning.
|
|
396
|
+
*
|
|
397
|
+
* Safe to repeat a PUT because the write is conditional: `ifRevision` pins the
|
|
398
|
+
* revision the caller last agreed on, so a retry that lands after a first
|
|
399
|
+
* attempt secretly succeeded is refused with 409 rather than writing twice.
|
|
400
|
+
*/
|
|
401
|
+
const RETRY_STATUS = new Set([0, 502, 503, 504]);
|
|
402
|
+
const RETRY_BACKOFF_MS = [400, 1200];
|
|
403
|
+
|
|
404
|
+
async function request(method, route, { creds, body = null, fetchImpl = fetch, timeoutMs = 20_000, retries = RETRY_BACKOFF_MS.length } = {}) {
|
|
405
|
+
let last;
|
|
406
|
+
for (let i = 0; ; i++) {
|
|
407
|
+
last = await attempt(method, route, { creds, body, fetchImpl, timeoutMs });
|
|
408
|
+
if (!RETRY_STATUS.has(last.status) || i >= retries) return last;
|
|
409
|
+
await new Promise((r) => setTimeout(r, RETRY_BACKOFF_MS[i] ?? RETRY_BACKOFF_MS.at(-1)));
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
377
413
|
export const pushSnapshot = (snapshot, { ifRevision = null, ...opts }) =>
|
|
378
414
|
request("PUT", "/api/settings", { ...opts, body: { snapshot, ifRevision } });
|
|
379
415
|
|