@abhi-r-27/feedwatch 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/commands/add.js +37 -0
- package/commands/list.js +49 -0
- package/commands/read.js +271 -0
- package/commands/remove.js +43 -0
- package/commands/run.js +162 -0
- package/feedwatch.js +146 -0
- package/lib/config.js +128 -0
- package/lib/errors.js +50 -0
- package/lib/fetcher.js +162 -0
- package/lib/logger.js +39 -0
- package/lib/parser.js +121 -0
- package/lib/store.js +73 -0
- package/package.json +27 -0
package/README.md
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# rss-cli
|
|
2
|
+
|
|
3
|
+
To install dependencies:
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
bun install
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
To run:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
bun run index.ts
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
This project was created using `bun init` in bun v1.3.10. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.
|
|
16
|
+
|
package/commands/add.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// commands/add.js
|
|
2
|
+
// TICKET 1.2 — Register a feed URL to the local store
|
|
3
|
+
//
|
|
4
|
+
// Usage: feedwatch add <name> <url>
|
|
5
|
+
|
|
6
|
+
import chalk from 'chalk';
|
|
7
|
+
import { loadStore, saveStore, getFeed, addFeed } from '../lib/store.js';
|
|
8
|
+
import { ValidationError, handleError } from '../lib/errors.js';
|
|
9
|
+
|
|
10
|
+
export function registerAdd(program) {
|
|
11
|
+
program
|
|
12
|
+
.command('add <name> <url>')
|
|
13
|
+
.description('Register a feed URL to watch')
|
|
14
|
+
.action((name, url) => {
|
|
15
|
+
try {
|
|
16
|
+
// Validate URL
|
|
17
|
+
try { new URL(url); } catch {
|
|
18
|
+
throw new ValidationError(`Invalid URL: ${url}`);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const store = loadStore();
|
|
22
|
+
|
|
23
|
+
if (getFeed(store, name)) {
|
|
24
|
+
throw new ValidationError(`A feed named "${name}" already exists. Use a different name.`);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
addFeed(store, name, url);
|
|
28
|
+
saveStore(store);
|
|
29
|
+
|
|
30
|
+
console.log(chalk.green(`✔ Added feed "${name}"`));
|
|
31
|
+
console.log(chalk.gray(` URL: ${url}`));
|
|
32
|
+
console.log(chalk.gray(` Run "feedwatch run" to fetch it.`));
|
|
33
|
+
} catch (err) {
|
|
34
|
+
handleError(err);
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
}
|
package/commands/list.js
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// commands/list.js
|
|
2
|
+
// TICKET 1.2 — Display all registered feeds
|
|
3
|
+
//
|
|
4
|
+
// Usage: feedwatch list
|
|
5
|
+
|
|
6
|
+
import chalk from 'chalk';
|
|
7
|
+
import Table from 'cli-table3';
|
|
8
|
+
import { loadStore } from '../lib/store.js';
|
|
9
|
+
import { handleError } from '../lib/errors.js';
|
|
10
|
+
|
|
11
|
+
export function registerList(program) {
|
|
12
|
+
program
|
|
13
|
+
.command('list')
|
|
14
|
+
.description('List all registered feeds')
|
|
15
|
+
.action(() => {
|
|
16
|
+
try {
|
|
17
|
+
const store = loadStore();
|
|
18
|
+
|
|
19
|
+
if (store.feeds.length === 0) {
|
|
20
|
+
console.log(chalk.yellow('No feeds registered.'));
|
|
21
|
+
console.log(chalk.gray(' Add one with: feedwatch add <n> <url>'));
|
|
22
|
+
process.exit(0);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const table = new Table({
|
|
26
|
+
head: ['Name', 'URL', 'Last Fetched', 'New Items (last run)'],
|
|
27
|
+
style: { head: ['cyan'] },
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
for (const feed of store.feeds) {
|
|
31
|
+
table.push([
|
|
32
|
+
chalk.white(feed.name),
|
|
33
|
+
chalk.gray(feed.url.length > 50 ? feed.url.slice(0, 47) + '...' : feed.url),
|
|
34
|
+
feed.lastFetchedAt
|
|
35
|
+
? chalk.gray(new Date(feed.lastFetchedAt).toLocaleString())
|
|
36
|
+
: chalk.gray('never'),
|
|
37
|
+
feed.lastNewCount > 0
|
|
38
|
+
? chalk.green(String(feed.lastNewCount))
|
|
39
|
+
: chalk.gray('0'),
|
|
40
|
+
]);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
console.log(table.toString());
|
|
44
|
+
console.log(chalk.gray(` ${store.feeds.length} feed(s) registered`));
|
|
45
|
+
} catch (err) {
|
|
46
|
+
handleError(err);
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
}
|
package/commands/read.js
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
// commands/read.js
|
|
2
|
+
// TICKET 1.6 — Navigable read mode for a single feed
|
|
3
|
+
//
|
|
4
|
+
// Usage:
|
|
5
|
+
// feedwatch read <n> — fetch and browse latest items
|
|
6
|
+
// feedwatch read <n> --max 20 — show up to 20 items
|
|
7
|
+
//
|
|
8
|
+
// UX flow:
|
|
9
|
+
// 1. Fetch the feed
|
|
10
|
+
// 2. Show item list as arrow-key select menu
|
|
11
|
+
// 3. On selection — display full description in read mode
|
|
12
|
+
// 4. Press any key to return to the item list
|
|
13
|
+
// 5. Select "Exit" to quit
|
|
14
|
+
|
|
15
|
+
import { select } from '@inquirer/prompts';
|
|
16
|
+
import chalk from 'chalk';
|
|
17
|
+
import ora from 'ora';
|
|
18
|
+
import axios from 'axios';
|
|
19
|
+
import { fetchAll } from '../lib/fetcher.js';
|
|
20
|
+
import { parseXML } from '../lib/parser.js';
|
|
21
|
+
import { loadStore } from '../lib/store.js';
|
|
22
|
+
import { handleError, NotFoundError } from '../lib/errors.js';
|
|
23
|
+
import { resolveConfig } from '../lib/config.js';
|
|
24
|
+
import { createLogger } from '../lib/logger.js';
|
|
25
|
+
|
|
26
|
+
// ── Fetch full article body ───────────────────────────────────────────────────
|
|
27
|
+
|
|
28
|
+
async function fetchArticleBody(url, timeout) {
|
|
29
|
+
try {
|
|
30
|
+
const response = await axios.get(url, {
|
|
31
|
+
timeout,
|
|
32
|
+
headers: { 'User-Agent': 'feedwatch/1.0 (feed reader)' },
|
|
33
|
+
responseType: 'text',
|
|
34
|
+
});
|
|
35
|
+
return response.data;
|
|
36
|
+
} catch {
|
|
37
|
+
return null; // fall back to feed description on any error
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// ── Content extraction ────────────────────────────────────────────────────────
|
|
42
|
+
//
|
|
43
|
+
// Raw article pages contain nav bars, inline CSS, inline JS, JSON blobs,
|
|
44
|
+
// and footer markup alongside the actual article text. Passing the full
|
|
45
|
+
// HTML through stripHTML() produces unusable output — nav items, CSS class
|
|
46
|
+
// names, and script tokens all end up in the rendered text.
|
|
47
|
+
//
|
|
48
|
+
// Strategy (applied in order, stops at first match):
|
|
49
|
+
// 1. Strip <script> and <style> blocks entirely — they are never content.
|
|
50
|
+
// 2. Extract the <article> element if present — most editorial sites use it.
|
|
51
|
+
// 3. Extract the <main> element as a fallback.
|
|
52
|
+
// 4. If neither exists, collect all <p> and <h2> tags whose stripped text
|
|
53
|
+
// is longer than 20 characters. This threshold filters single-word nav
|
|
54
|
+
// items ("Home", "Sport", "Terms of Use") while preserving short but
|
|
55
|
+
// real sentences ("He feels it is quite ironic.").
|
|
56
|
+
|
|
57
|
+
function extractContent(html) {
|
|
58
|
+
// Step 1: remove script and style blocks — these are never readable content
|
|
59
|
+
let cleaned = html
|
|
60
|
+
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
|
61
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
|
|
62
|
+
|
|
63
|
+
// Step 2: prefer <article> — present on most modern editorial sites
|
|
64
|
+
const articleMatch = cleaned.match(/<article[^>]*>([\s\S]*?)<\/article>/i);
|
|
65
|
+
if (articleMatch) return articleMatch[1];
|
|
66
|
+
|
|
67
|
+
// Step 3: fall back to <main>
|
|
68
|
+
const mainMatch = cleaned.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
|
|
69
|
+
if (mainMatch) return mainMatch[1];
|
|
70
|
+
|
|
71
|
+
// Step 4: no semantic container — harvest individual <p> and <h2> tags.
|
|
72
|
+
// Threshold of 20 chars drops nav atoms while keeping short real sentences.
|
|
73
|
+
const MIN_CHARS = 20;
|
|
74
|
+
const blocks = [];
|
|
75
|
+
|
|
76
|
+
const tagPattern = /<(p|h2)[^>]*>([\s\S]*?)<\/\1>/gi;
|
|
77
|
+
let match;
|
|
78
|
+
while ((match = tagPattern.exec(cleaned)) !== null) {
|
|
79
|
+
const [full, tag, inner] = match;
|
|
80
|
+
const text = inner.replace(/<[^>]*>/g, '').trim();
|
|
81
|
+
if (text.length > MIN_CHARS) blocks.push(full);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return blocks.join('\n\n');
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// ── HTML → plain text ─────────────────────────────────────────────────────────
|
|
88
|
+
|
|
89
|
+
function stripHTML(html) {
|
|
90
|
+
return (html || '')
|
|
91
|
+
.replace(/<br\s*\/?>/gi, '\n')
|
|
92
|
+
.replace(/<\/p>/gi, '\n\n')
|
|
93
|
+
.replace(/<\/h2>/gi, '\n\n')
|
|
94
|
+
.replace(/<[^>]*>/g, '')
|
|
95
|
+
.replace(/&/g, '&')
|
|
96
|
+
.replace(/</g, '<')
|
|
97
|
+
.replace(/>/g, '>')
|
|
98
|
+
.replace(/"/g, '"')
|
|
99
|
+
.replace(/'/g, "'")
|
|
100
|
+
.replace(/ /g, ' ')
|
|
101
|
+
// Collapse runs of blank lines down to a single blank line
|
|
102
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
103
|
+
.trim();
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ── Read mode display ─────────────────────────────────────────────────────────
|
|
107
|
+
|
|
108
|
+
function wordWrap(text, width = 72) {
|
|
109
|
+
const lines = [];
|
|
110
|
+
for (const para of text.split('\n')) {
|
|
111
|
+
if (para.trim() === '') { lines.push(''); continue; }
|
|
112
|
+
const words = para.split(' ');
|
|
113
|
+
let current = '';
|
|
114
|
+
for (const word of words) {
|
|
115
|
+
if ((current + ' ' + word).trim().length > width) {
|
|
116
|
+
if (current) lines.push(current);
|
|
117
|
+
current = word;
|
|
118
|
+
} else {
|
|
119
|
+
current = current ? current + ' ' + word : word;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (current) lines.push(current);
|
|
123
|
+
}
|
|
124
|
+
return lines;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async function showArticle(item, config) {
|
|
128
|
+
console.clear();
|
|
129
|
+
console.log('');
|
|
130
|
+
console.log(chalk.bold.white(' ' + (item.title || '(no title)')));
|
|
131
|
+
console.log('');
|
|
132
|
+
|
|
133
|
+
if (item.pubDate) {
|
|
134
|
+
const date = new Date(item.pubDate).toLocaleString();
|
|
135
|
+
console.log(chalk.gray(` Published: ${date}`));
|
|
136
|
+
}
|
|
137
|
+
if (item.link) {
|
|
138
|
+
console.log(chalk.gray(` Link: ${item.link}`));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
console.log('');
|
|
142
|
+
console.log(chalk.gray(' ' + '─'.repeat(68)));
|
|
143
|
+
console.log('');
|
|
144
|
+
|
|
145
|
+
// Prefer the full article page over the feed description snippet.
|
|
146
|
+
// extractContent() isolates article body before stripHTML runs,
|
|
147
|
+
// so the reader sees article text — not nav bars or inline CSS.
|
|
148
|
+
let body = null;
|
|
149
|
+
if (item.link) {
|
|
150
|
+
const spinner = ora({ text: chalk.gray(' Fetching article...'), color: 'gray' }).start();
|
|
151
|
+
const html = await fetchArticleBody(item.link, config.timeout);
|
|
152
|
+
spinner.stop();
|
|
153
|
+
if (html) body = stripHTML(extractContent(html));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Fall back to feed description if fetch failed or item has no link.
|
|
157
|
+
// The feed description is already plain text (or light HTML), so
|
|
158
|
+
// extractContent is not needed — a direct stripHTML is sufficient.
|
|
159
|
+
if (!body) body = stripHTML(item.description || '');
|
|
160
|
+
|
|
161
|
+
if (!body) {
|
|
162
|
+
console.log(chalk.gray(' (No content available for this article.)'));
|
|
163
|
+
} else {
|
|
164
|
+
const wrapped = wordWrap(body);
|
|
165
|
+
for (const line of wrapped) {
|
|
166
|
+
console.log(line ? ' ' + chalk.white(line) : '');
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
console.log('');
|
|
171
|
+
console.log(chalk.gray(' ' + '─'.repeat(68)));
|
|
172
|
+
console.log('');
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// ── Wait for keypress ─────────────────────────────────────────────────────────
|
|
176
|
+
|
|
177
|
+
function waitForKey(prompt = 'Press any key to go back...') {
|
|
178
|
+
return new Promise(resolve => {
|
|
179
|
+
process.stdout.write(chalk.gray(` ${prompt}`));
|
|
180
|
+
process.stdin.setRawMode(true);
|
|
181
|
+
process.stdin.resume();
|
|
182
|
+
process.stdin.once('data', () => {
|
|
183
|
+
process.stdin.setRawMode(false);
|
|
184
|
+
process.stdin.pause();
|
|
185
|
+
process.stdout.write('\n');
|
|
186
|
+
resolve();
|
|
187
|
+
});
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// ── Main command ──────────────────────────────────────────────────────────────
|
|
192
|
+
|
|
193
|
+
export function registerRead(program) {
|
|
194
|
+
program
|
|
195
|
+
.command('read <n>')
|
|
196
|
+
.description('Browse and read items from a registered feed')
|
|
197
|
+
.option('--max <n>', 'Max items to load', '20')
|
|
198
|
+
.action(async (name, opts) => {
|
|
199
|
+
try {
|
|
200
|
+
const store = loadStore();
|
|
201
|
+
const feed = store.feeds.find(f => f.name === name);
|
|
202
|
+
|
|
203
|
+
if (!feed) {
|
|
204
|
+
throw new NotFoundError(`No feed named "${name}". Run "feedwatch list" to see registered feeds.`);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const { config } = resolveConfig();
|
|
208
|
+
const logger = createLogger(config.logLevel);
|
|
209
|
+
const spinner = ora(`Fetching "${name}"...`).start();
|
|
210
|
+
|
|
211
|
+
const results = await fetchAll([feed], config, logger);
|
|
212
|
+
spinner.stop();
|
|
213
|
+
|
|
214
|
+
const result = results[0];
|
|
215
|
+
if (result.status === 'failed') {
|
|
216
|
+
console.log(chalk.red(`✖ Failed to fetch "${name}": ${result.error.message}`));
|
|
217
|
+
process.exit(1);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const maxItems = parseInt(opts.max, 10) || 20;
|
|
221
|
+
const items = parseXML(result.xml).slice(0, maxItems);
|
|
222
|
+
|
|
223
|
+
if (items.length === 0) {
|
|
224
|
+
console.log(chalk.yellow(`No items found in "${name}".`));
|
|
225
|
+
process.exit(0);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// ── Interactive browse loop ───────────────────────────────────────────
|
|
229
|
+
|
|
230
|
+
while (true) {
|
|
231
|
+
console.clear();
|
|
232
|
+
console.log('');
|
|
233
|
+
console.log(chalk.bold(` ${name}`) + chalk.gray(` (${items.length} items)`));
|
|
234
|
+
console.log(chalk.gray(` ${feed.url}`));
|
|
235
|
+
console.log('');
|
|
236
|
+
|
|
237
|
+
const choices = [
|
|
238
|
+
...items.map((item, i) => {
|
|
239
|
+
const title = (item.title || '(no title)').slice(0, 60);
|
|
240
|
+
const date = item.pubDate
|
|
241
|
+
? chalk.gray(' · ' + new Date(item.pubDate).toLocaleDateString())
|
|
242
|
+
: '';
|
|
243
|
+
return { name: `${chalk.cyan(String(i + 1).padStart(2, '0'))} ${title}${date}`, value: i };
|
|
244
|
+
}),
|
|
245
|
+
{ name: chalk.gray('─── Exit'), value: 'exit' },
|
|
246
|
+
];
|
|
247
|
+
|
|
248
|
+
const choice = await select({
|
|
249
|
+
message: 'Select an item to read',
|
|
250
|
+
choices,
|
|
251
|
+
pageSize: 15,
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
if (choice === 'exit') {
|
|
255
|
+
console.clear();
|
|
256
|
+
process.exit(0);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
await showArticle(items[choice], config);
|
|
260
|
+
await waitForKey();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
} catch (err) {
|
|
264
|
+
if (err.name === 'ExitPromptError') {
|
|
265
|
+
console.clear();
|
|
266
|
+
process.exit(0);
|
|
267
|
+
}
|
|
268
|
+
handleError(err);
|
|
269
|
+
}
|
|
270
|
+
});
|
|
271
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// commands/remove.js
|
|
2
|
+
// TICKET 1.2 — Remove a registered feed
|
|
3
|
+
//
|
|
4
|
+
// Usage: feedwatch remove <n>
|
|
5
|
+
|
|
6
|
+
import { confirm } from '@inquirer/prompts';
|
|
7
|
+
import chalk from 'chalk';
|
|
8
|
+
import { loadStore, saveStore, getFeed, removeFeed } from '../lib/store.js';
|
|
9
|
+
import { NotFoundError, handleError } from '../lib/errors.js';
|
|
10
|
+
|
|
11
|
+
export function registerRemove(program) {
|
|
12
|
+
program
|
|
13
|
+
.command('remove <n>')
|
|
14
|
+
.description('Remove a registered feed')
|
|
15
|
+
.option('-y, --yes', 'Skip confirmation prompt')
|
|
16
|
+
.action(async (name, opts) => {
|
|
17
|
+
try {
|
|
18
|
+
const store = loadStore();
|
|
19
|
+
const feed = getFeed(store, name);
|
|
20
|
+
|
|
21
|
+
if (!feed) {
|
|
22
|
+
throw new NotFoundError(`No feed named "${name}". Run "feedwatch list" to see registered feeds.`);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (!opts.yes) {
|
|
26
|
+
const confirmed = await confirm({
|
|
27
|
+
message: `Remove feed "${name}" (${feed.url})?`,
|
|
28
|
+
default: false,
|
|
29
|
+
});
|
|
30
|
+
if (!confirmed) {
|
|
31
|
+
console.log(chalk.gray('Cancelled.'));
|
|
32
|
+
process.exit(0);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
removeFeed(store, name);
|
|
37
|
+
saveStore(store);
|
|
38
|
+
console.log(chalk.green(`✔ Removed feed "${name}"`));
|
|
39
|
+
} catch (err) {
|
|
40
|
+
handleError(err);
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
}
|
package/commands/run.js
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
// commands/run.js
|
|
2
|
+
// TICKET 1.3 — Fetch engine (Promise.allSettled, retry)
|
|
3
|
+
// TICKET 1.4 — RSS/Atom parsing and normalisation
|
|
4
|
+
// TICKET 1.5 — State diffing and change detection
|
|
5
|
+
// TICKET 1.6 — Output formatting, --json, exit codes
|
|
6
|
+
//
|
|
7
|
+
// Usage:
|
|
8
|
+
// feedwatch run
|
|
9
|
+
// feedwatch run --all show all items, not just new
|
|
10
|
+
// feedwatch run --json output raw JSON
|
|
11
|
+
// feedwatch run --timeout 5000
|
|
12
|
+
|
|
13
|
+
import chalk from 'chalk';
|
|
14
|
+
import ora from 'ora';
|
|
15
|
+
import Table from 'cli-table3';
|
|
16
|
+
import { fetchAll } from '../lib/fetcher.js';
|
|
17
|
+
import { parseXML } from '../lib/parser.js';
|
|
18
|
+
import { loadStore, saveStore, getSeenGuids,
|
|
19
|
+
updateSeen, updateFeedMeta } from '../lib/store.js';
|
|
20
|
+
import { handleError } from '../lib/errors.js';
|
|
21
|
+
import { resolveConfig } from '../lib/config.js';
|
|
22
|
+
import { createLogger } from '../lib/logger.js';
|
|
23
|
+
|
|
24
|
+
export function registerRun(program) {
|
|
25
|
+
program
|
|
26
|
+
.command('run')
|
|
27
|
+
.description('Fetch all registered feeds and report new items')
|
|
28
|
+
.option('--all', 'Show all items, not just new ones')
|
|
29
|
+
.option('--json', 'Output results as JSON')
|
|
30
|
+
.option('--timeout <ms>', 'Request timeout in ms')
|
|
31
|
+
.option('--retries <n>', 'Max retry attempts')
|
|
32
|
+
.option('--log-level <level>','Log level (debug|info|warn|error)')
|
|
33
|
+
.action(async (opts) => {
|
|
34
|
+
try {
|
|
35
|
+
const { config } = resolveConfig({
|
|
36
|
+
timeout: opts.timeout,
|
|
37
|
+
retries: opts.retries,
|
|
38
|
+
logLevel: opts.logLevel,
|
|
39
|
+
});
|
|
40
|
+
const logger = createLogger(config.logLevel);
|
|
41
|
+
const store = loadStore();
|
|
42
|
+
|
|
43
|
+
if (store.feeds.length === 0) {
|
|
44
|
+
console.log(chalk.yellow('No feeds registered.'));
|
|
45
|
+
console.log(chalk.gray(' Add one with: feedwatch add <n> <url>'));
|
|
46
|
+
process.exit(0);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const spinner = ora(`Fetching ${store.feeds.length} feed(s)...`).start();
|
|
50
|
+
|
|
51
|
+
// TICKET 1.3: concurrent fetch with allSettled + retry
|
|
52
|
+
const fetchResults = await fetchAll(store.feeds, config, logger);
|
|
53
|
+
|
|
54
|
+
spinner.stop();
|
|
55
|
+
|
|
56
|
+
// TICKET 1.4 + 1.5: parse, diff, update seen state
|
|
57
|
+
const runResults = [];
|
|
58
|
+
let anyFailed = false;
|
|
59
|
+
|
|
60
|
+
for (const result of fetchResults) {
|
|
61
|
+
if (result.status === 'failed') {
|
|
62
|
+
anyFailed = true;
|
|
63
|
+
runResults.push({ name: result.name, status: 'failed', items: [], error: result.error });
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const items = parseXML(result.xml);
|
|
68
|
+
const seenSet = getSeenGuids(store, result.name);
|
|
69
|
+
const allGuids = items.map(i => i.guid || i.link || i.title);
|
|
70
|
+
|
|
71
|
+
const newItems = items.filter((item, idx) => !seenSet.has(allGuids[idx]));
|
|
72
|
+
const seenItems = items.filter((item, idx) => seenSet.has(allGuids[idx]));
|
|
73
|
+
|
|
74
|
+
// Update seen state with all current guids
|
|
75
|
+
updateSeen(store, result.name, allGuids);
|
|
76
|
+
updateFeedMeta(store, result.name, {
|
|
77
|
+
lastFetchedAt: new Date().toISOString(),
|
|
78
|
+
lastNewCount: newItems.length,
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
runResults.push({
|
|
82
|
+
name: result.name,
|
|
83
|
+
status: 'ok',
|
|
84
|
+
newItems,
|
|
85
|
+
seenItems,
|
|
86
|
+
allItems: items,
|
|
87
|
+
error: null,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
saveStore(store);
|
|
92
|
+
|
|
93
|
+
// TICKET 1.6: output
|
|
94
|
+
if (opts.json) {
|
|
95
|
+
console.log(JSON.stringify(runResults.map(r => ({
|
|
96
|
+
name: r.name,
|
|
97
|
+
status: r.status,
|
|
98
|
+
new: r.newItems?.length ?? 0,
|
|
99
|
+
items: (opts.all ? r.allItems : r.newItems) ?? [],
|
|
100
|
+
error: r.error?.message ?? null,
|
|
101
|
+
})), null, 2));
|
|
102
|
+
} else {
|
|
103
|
+
printResults(runResults, opts.all);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
process.exit(anyFailed ? 1 : 0);
|
|
107
|
+
} catch (err) {
|
|
108
|
+
handleError(err);
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// ── Output helpers ────────────────────────────────────────────────────────────
|
|
114
|
+
|
|
115
|
+
function printResults(results, showAll) {
|
|
116
|
+
let totalNew = 0;
|
|
117
|
+
|
|
118
|
+
for (const r of results) {
|
|
119
|
+
console.log('');
|
|
120
|
+
|
|
121
|
+
if (r.status === 'failed') {
|
|
122
|
+
console.log(chalk.red(`✖ ${r.name}`) + chalk.gray(` — FAILED: ${r.error.message}`));
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const items = showAll ? r.allItems : r.newItems;
|
|
127
|
+
totalNew += r.newItems.length;
|
|
128
|
+
|
|
129
|
+
if (items.length === 0) {
|
|
130
|
+
console.log(chalk.gray(` ${r.name} — no ${showAll ? '' : 'new '}items`));
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
console.log(chalk.bold(` ${r.name}`) + chalk.gray(` — ${r.newItems.length} new`));
|
|
135
|
+
|
|
136
|
+
const table = new Table({
|
|
137
|
+
head: ['Status', 'Title', 'Published', 'Link'],
|
|
138
|
+
style: { head: ['cyan'] },
|
|
139
|
+
colWidths: [8, 40, 22, 40],
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
for (const item of items) {
|
|
143
|
+
const isNew = r.newItems.includes(item);
|
|
144
|
+
const status = isNew ? chalk.green('NEW') : chalk.gray('SEEN');
|
|
145
|
+
const title = truncate(item.title || '(no title)', 38);
|
|
146
|
+
const date = item.pubDate ? new Date(item.pubDate).toLocaleDateString() : '';
|
|
147
|
+
const link = truncate(item.link || '', 38);
|
|
148
|
+
table.push([status, title, date, chalk.gray(link)]);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
console.log(table.toString());
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
console.log('');
|
|
155
|
+
console.log(totalNew > 0
|
|
156
|
+
? chalk.green(` ${totalNew} new item(s) found`)
|
|
157
|
+
: chalk.gray(' No new items'));
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function truncate(str, max) {
|
|
161
|
+
return str.length > max ? str.slice(0, max - 1) + '…' : str;
|
|
162
|
+
}
|
package/feedwatch.js
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// feedwatch.js — Entry point
|
|
3
|
+
// TICKET 1.7 — shebang + bin field wired in package.json
|
|
4
|
+
//
|
|
5
|
+
// Run:
|
|
6
|
+
// feedwatch add <n> <url> Register a feed
|
|
7
|
+
// feedwatch remove <n> Remove a feed
|
|
8
|
+
// feedwatch list List all registered feeds
|
|
9
|
+
// feedwatch run Fetch all feeds, report new items
|
|
10
|
+
// feedwatch run --all Show all items, not just new
|
|
11
|
+
// feedwatch run --json Output raw JSON
|
|
12
|
+
// feedwatch read <n> Show latest items for one feed
|
|
13
|
+
// feedwatch config show Display resolved config
|
|
14
|
+
|
|
15
|
+
import { Command } from 'commander';
|
|
16
|
+
import chalk from 'chalk';
|
|
17
|
+
import Table from 'cli-table3';
|
|
18
|
+
|
|
19
|
+
import { registerAdd } from './commands/add.js';
|
|
20
|
+
import { registerRemove } from './commands/remove.js';
|
|
21
|
+
import { registerList } from './commands/list.js';
|
|
22
|
+
import { registerRun } from './commands/run.js';
|
|
23
|
+
import { registerRead } from './commands/read.js';
|
|
24
|
+
import { resolveConfig } from './lib/config.js';
|
|
25
|
+
import { handleError } from './lib/errors.js';
|
|
26
|
+
|
|
27
|
+
const program = new Command();
|
|
28
|
+
|
|
29
|
+
program
|
|
30
|
+
.name('feedwatch')
|
|
31
|
+
.description('Monitor RSS and Atom feeds for new items')
|
|
32
|
+
.version('1.0.0');
|
|
33
|
+
|
|
34
|
+
// ── Register subcommands ──────────────────────────────────────────────────────
|
|
35
|
+
|
|
36
|
+
registerAdd(program);
|
|
37
|
+
registerRemove(program);
|
|
38
|
+
registerList(program);
|
|
39
|
+
registerRun(program);
|
|
40
|
+
registerRead(program);
|
|
41
|
+
|
|
42
|
+
// ── config show — TICKET 1.1 ──────────────────────────────────────────────────
|
|
43
|
+
|
|
44
|
+
const configCmd = program.command('config').description('Config management');
|
|
45
|
+
|
|
46
|
+
configCmd
|
|
47
|
+
.command('show')
|
|
48
|
+
.description('Display resolved configuration with source per key')
|
|
49
|
+
.option('--config <path>', 'Path to config file')
|
|
50
|
+
.action((opts) => {
|
|
51
|
+
try {
|
|
52
|
+
// opts - parsed command-line options
|
|
53
|
+
const { config, sources, configPath, fileFound } = resolveConfig({}, opts.config || null);
|
|
54
|
+
|
|
55
|
+
console.log('');
|
|
56
|
+
console.log(chalk.bold(' Resolved Configuration'));
|
|
57
|
+
console.log(chalk.gray(` Config file: ${configPath} ${fileFound ? chalk.green('(found)') : chalk.yellow('(not found — using defaults)')}`));
|
|
58
|
+
console.log('');
|
|
59
|
+
|
|
60
|
+
const SOURCE_COLOR = {
|
|
61
|
+
default: chalk.gray,
|
|
62
|
+
file: chalk.blue,
|
|
63
|
+
env: chalk.yellow,
|
|
64
|
+
flag: chalk.green,
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
const table = new Table({
|
|
68
|
+
head: ['Key', 'Value', 'Source'],
|
|
69
|
+
style: { head: ['cyan'] },
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
for (const [key, value] of Object.entries(config)) {
|
|
73
|
+
const source = sources[key] || 'default';
|
|
74
|
+
const colorFn = SOURCE_COLOR[source] || chalk.white;
|
|
75
|
+
table.push([key, String(value), colorFn(source)]);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
console.log(table.toString());
|
|
79
|
+
console.log(
|
|
80
|
+
chalk.gray(' Sources: ') +
|
|
81
|
+
chalk.gray('default') + ' ' +
|
|
82
|
+
chalk.blue('file') + ' ' +
|
|
83
|
+
chalk.yellow('env') + ' ' +
|
|
84
|
+
chalk.green('flag')
|
|
85
|
+
);
|
|
86
|
+
console.log('');
|
|
87
|
+
} catch (err) {
|
|
88
|
+
handleError(err);
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
// ── SIGINT / SIGTERM graceful shutdown ────────────────────────────────────────
|
|
93
|
+
|
|
94
|
+
function shutdown(signal) {
|
|
95
|
+
process.stderr.write(chalk.gray(`\n Received ${signal} — shutting down cleanly.\n`));
|
|
96
|
+
process.exit(0);
|
|
97
|
+
}
|
|
98
|
+
process.on('SIGINT', () => shutdown('SIGINT'));
|
|
99
|
+
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
|
100
|
+
|
|
101
|
+
// ── Parse ─────────────────────────────────────────────────────────────────────
|
|
102
|
+
|
|
103
|
+
program.parse(process.argv);
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
/*
|
|
107
|
+
|
|
108
|
+
Tech/Dev
|
|
109
|
+
|
|
110
|
+
Hacker News: https://news.ycombinator.com/rss
|
|
111
|
+
GitHub Blog: https://github.blog/feed/
|
|
112
|
+
CSS Tricks: https://css-tricks.com/feed/
|
|
113
|
+
|
|
114
|
+
News
|
|
115
|
+
|
|
116
|
+
BBC Top Stories: http://feeds.bbci.co.uk/news/rss.xml
|
|
117
|
+
Reuters: https://feeds.reuters.com/reuters/topNews
|
|
118
|
+
|
|
119
|
+
# 1. Install dependencies
|
|
120
|
+
cd feedwatch
|
|
121
|
+
bun install
|
|
122
|
+
|
|
123
|
+
# 2. Register the feed
|
|
124
|
+
bun feedwatch.js add hn https://news.ycombinator.com/rss
|
|
125
|
+
|
|
126
|
+
# 3. Confirm it was saved
|
|
127
|
+
bun feedwatch.js list
|
|
128
|
+
|
|
129
|
+
# 4. First run — all items will be NEW (nothing seen yet)
|
|
130
|
+
bun feedwatch.js run
|
|
131
|
+
|
|
132
|
+
# 5. Run again immediately — all items now SEEN (state was saved)
|
|
133
|
+
bun feedwatch.js run
|
|
134
|
+
|
|
135
|
+
# 6. Wait a few minutes, run again — new HN posts will appear as NEW
|
|
136
|
+
bun feedwatch.js run
|
|
137
|
+
|
|
138
|
+
# 7. Read items for a single feed without updating seen state
|
|
139
|
+
bun feedwatch.js read hn
|
|
140
|
+
|
|
141
|
+
# 8. See the full config
|
|
142
|
+
bun feedwatch.js config show
|
|
143
|
+
|
|
144
|
+
# 9. Override config via env var — observe source column changes
|
|
145
|
+
FEEDWATCH_TIMEOUT=3000 bun feedwatch.js config show
|
|
146
|
+
*/
|
package/lib/config.js
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// lib/config.js
|
|
2
|
+
// TICKET 1.1 — Layered configuration
|
|
3
|
+
//
|
|
4
|
+
// Resolution order (each layer overrides the previous):
|
|
5
|
+
// 1. Hardcoded defaults
|
|
6
|
+
// 2. feedwatch.config.json (default location, or --config <path>)
|
|
7
|
+
// 3. Environment variables (FEEDWATCH_RETRIES, FEEDWATCH_TIMEOUT, etc.)
|
|
8
|
+
// 4. CLI flags (--retries, --timeout, --max-items, --log-level)
|
|
9
|
+
|
|
10
|
+
import fs from 'fs';
|
|
11
|
+
import path from 'path';
|
|
12
|
+
import os from 'os';
|
|
13
|
+
|
|
14
|
+
// ── Layer 1: Defaults ─────────────────────────────────────────────────────────
|
|
15
|
+
|
|
16
|
+
const DEFAULTS = {
|
|
17
|
+
retries: 3,
|
|
18
|
+
timeout: 8000,
|
|
19
|
+
maxItems: 10,
|
|
20
|
+
logLevel: 'info',
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const DEFAULT_CONFIG_PATH = path.join(os.homedir(), '.feedwatch', 'config.json');
|
|
24
|
+
|
|
25
|
+
// ── Schema ────────────────────────────────────────────────────────────────────
|
|
26
|
+
|
|
27
|
+
const SCHEMA = {
|
|
28
|
+
retries: { type: 'number', min: 0, max: 10 },
|
|
29
|
+
timeout: { type: 'number', min: 500, max: 60000 },
|
|
30
|
+
maxItems: { type: 'number', min: 1, max: 100 },
|
|
31
|
+
logLevel: { type: 'string', allowed: ['debug', 'info', 'warn', 'error'] },
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
function validate(config, source) {
|
|
35
|
+
const errors = [];
|
|
36
|
+
for (const [key, val] of Object.entries(config)) {
|
|
37
|
+
if (!SCHEMA[key]) { errors.push(`Unknown key "${key}" in ${source}`); continue; }
|
|
38
|
+
const s = SCHEMA[key];
|
|
39
|
+
if (s.type === 'number') {
|
|
40
|
+
if (typeof val !== 'number' || isNaN(val)) errors.push(`"${key}" must be a number`);
|
|
41
|
+
else if (s.min !== undefined && val < s.min) errors.push(`"${key}" must be >= ${s.min}`);
|
|
42
|
+
else if (s.max !== undefined && val > s.max) errors.push(`"${key}" must be <= ${s.max}`);
|
|
43
|
+
}
|
|
44
|
+
if (s.type === 'string') {
|
|
45
|
+
if (typeof val !== 'string') errors.push(`"${key}" must be a string`);
|
|
46
|
+
else if (s.allowed && !s.allowed.includes(val))
|
|
47
|
+
errors.push(`"${key}" must be one of: ${s.allowed.join(', ')}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
if (errors.length) {
|
|
51
|
+
errors.forEach(e => process.stderr.write(`Config error: ${e}\n`));
|
|
52
|
+
process.exit(1);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// ── Layer 2: Config file ──────────────────────────────────────────────────────
|
|
57
|
+
|
|
58
|
+
function loadFile(configPath) {
|
|
59
|
+
if (!fs.existsSync(configPath)) return { values: {}, found: false };
|
|
60
|
+
let parsed;
|
|
61
|
+
try {
|
|
62
|
+
parsed = JSON.parse(fs.readFileSync(configPath, 'utf8'));
|
|
63
|
+
} catch (err) {
|
|
64
|
+
process.stderr.write(`Error reading config file ${configPath}: ${err.message}\n`);
|
|
65
|
+
process.exit(1);
|
|
66
|
+
}
|
|
67
|
+
validate(parsed, configPath);
|
|
68
|
+
return { values: parsed, found: true };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// ── Layer 3: Environment variables ───────────────────────────────────────────
|
|
72
|
+
|
|
73
|
+
function loadEnv() {
|
|
74
|
+
const env = {};
|
|
75
|
+
// load process ENV variables
|
|
76
|
+
const num = (key, envKey) => {
|
|
77
|
+
if (!process.env[envKey]) return;
|
|
78
|
+
const n = parseInt(process.env[envKey], 10);
|
|
79
|
+
if (isNaN(n)) process.stderr.write(`Warning: ${envKey} is not a valid number — ignored\n`);
|
|
80
|
+
else env[key] = n;
|
|
81
|
+
};
|
|
82
|
+
num('retries', 'FEEDWATCH_RETRIES');
|
|
83
|
+
num('timeout', 'FEEDWATCH_TIMEOUT');
|
|
84
|
+
num('maxItems', 'FEEDWATCH_MAX_ITEMS');
|
|
85
|
+
if (process.env.FEEDWATCH_LOG_LEVEL) {
|
|
86
|
+
const v = process.env.FEEDWATCH_LOG_LEVEL.toLowerCase();
|
|
87
|
+
if (SCHEMA.logLevel.allowed.includes(v)) env.logLevel = v;
|
|
88
|
+
else process.stderr.write(`Warning: FEEDWATCH_LOG_LEVEL="${v}" is not valid — ignored\n`);
|
|
89
|
+
}
|
|
90
|
+
return env;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ── Layer 4: CLI flags ────────────────────────────────────────────────────────
|
|
94
|
+
|
|
95
|
+
function loadFlags(opts = {}) {
|
|
96
|
+
const flags = {};
|
|
97
|
+
if (opts.retries !== undefined) flags.retries = parseInt(opts.retries, 10);
|
|
98
|
+
if (opts.timeout !== undefined) flags.timeout = parseInt(opts.timeout, 10);
|
|
99
|
+
if (opts.maxItems !== undefined) flags.maxItems = parseInt(opts.maxItems, 10);
|
|
100
|
+
if (opts.logLevel !== undefined) flags.logLevel = opts.logLevel;
|
|
101
|
+
return flags;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// ── Resolve ───────────────────────────────────────────────────────────────────
|
|
105
|
+
|
|
106
|
+
export function resolveConfig(opts = {}, configPathOverride = null) {
|
|
107
|
+
const configPath = configPathOverride || DEFAULT_CONFIG_PATH;
|
|
108
|
+
const file = loadFile(configPath);
|
|
109
|
+
const env = loadEnv();
|
|
110
|
+
console.log(env);
|
|
111
|
+
|
|
112
|
+
const flags = loadFlags(opts);
|
|
113
|
+
|
|
114
|
+
const config = { ...DEFAULTS, ...file.values, ...env, ...flags };
|
|
115
|
+
|
|
116
|
+
// Track source of each key for `config show`
|
|
117
|
+
const sources = {};
|
|
118
|
+
for (const key of Object.keys(DEFAULTS)) {
|
|
119
|
+
if (flags[key] !== undefined) sources[key] = 'flag';
|
|
120
|
+
else if (env[key] !== undefined) sources[key] = 'env';
|
|
121
|
+
else if (file.values[key]!== undefined) sources[key] = 'file';
|
|
122
|
+
else sources[key] = 'default';
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return { config, sources, configPath, fileFound: file.found };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export { DEFAULT_CONFIG_PATH, DEFAULTS };
|
package/lib/errors.js
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// lib/errors.js
|
|
2
|
+
// TICKET 1.1 (production requirements) — Custom error classes + centralised handler
|
|
3
|
+
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import fs from 'fs';
|
|
6
|
+
import path from 'path';
|
|
7
|
+
|
|
8
|
+
const LOG_FILE = path.join(process.cwd(), 'feedwatch.log');
|
|
9
|
+
|
|
10
|
+
// ── Custom error classes ──────────────────────────────────────────────────────
|
|
11
|
+
|
|
12
|
+
/*
|
|
13
|
+
The sequence is:
|
|
14
|
+
- super(message) → create the base Error object
|
|
15
|
+
|
|
16
|
+
Error sets properties like:
|
|
17
|
+
- message
|
|
18
|
+
- stack
|
|
19
|
+
|
|
20
|
+
Then your class adds:
|
|
21
|
+
- name
|
|
22
|
+
- exitCode
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
export class ValidationError extends Error {
|
|
26
|
+
constructor(message) { super(message); this.name = 'ValidationError'; this.exitCode = 1; }
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export class NotFoundError extends Error {
|
|
30
|
+
constructor(message) { super(message); this.name = 'NotFoundError'; this.exitCode = 1; }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export class FileSystemError extends Error {
|
|
34
|
+
constructor(message) { super(message); this.name = 'FileSystemError'; this.exitCode = 1; }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// ── Centralised handler ───────────────────────────────────────────────────────
|
|
38
|
+
|
|
39
|
+
export function handleError(err) {
|
|
40
|
+
const ts = new Date().toISOString();
|
|
41
|
+
const line = `[${ts}] [ERROR] ${err.name}: ${err.message}\n${err.stack}\n`;
|
|
42
|
+
|
|
43
|
+
// Always log to file
|
|
44
|
+
try { fs.appendFileSync(LOG_FILE, line); } catch { /* ignore log write failures */ }
|
|
45
|
+
|
|
46
|
+
// User-facing message
|
|
47
|
+
process.stderr.write(chalk.red(`Error: ${err.message}\n`));
|
|
48
|
+
|
|
49
|
+
process.exit(err.exitCode ?? 1);
|
|
50
|
+
}
|
package/lib/fetcher.js
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
// lib/fetcher.js
|
|
2
|
+
// TICKET 1.3 — Concurrent fetch engine
|
|
3
|
+
//
|
|
4
|
+
// fetchAll(feeds, config, logger):
|
|
5
|
+
// Fetches all feeds concurrently with Promise.allSettled.
|
|
6
|
+
// Each request retries on network errors and 5xx responses.
|
|
7
|
+
// Returns array of { name, url, status, xml, error }
|
|
8
|
+
|
|
9
|
+
/*
|
|
10
|
+
Error Propagation and Handling
|
|
11
|
+
|
|
12
|
+
The custom errors defined in `lib/fetcher.js` (`NetworkError`, `FeedError`) are **not handled inside the fetcher itself**. Instead, they are intentionally **propagated upward** to the CLI layer.
|
|
13
|
+
|
|
14
|
+
The fetch engine uses `Promise.allSettled()` when fetching feeds concurrently. This means that if a single feed fails, the error is captured in the result rather than crashing the entire program.
|
|
15
|
+
|
|
16
|
+
Each settled result is converted into a structured object:
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
{ name, url, status, xml, error }
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
If a request fails, the `error` field contains the thrown `NetworkError` or `FeedError`. The CLI command (for example `feedwatch run`) then formats and displays this information to the user.
|
|
23
|
+
|
|
24
|
+
This design ensures that:
|
|
25
|
+
|
|
26
|
+
* one failing feed does **not stop other feeds from processing**
|
|
27
|
+
* errors remain **structured and inspectable**
|
|
28
|
+
* the CLI layer controls **how failures are presented to the user**
|
|
29
|
+
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import axios from 'axios';
|
|
33
|
+
|
|
34
|
+
// ── Custom error classes ──────────────────────────────────────────────────────
|
|
35
|
+
|
|
36
|
+
export class NetworkError extends Error {
|
|
37
|
+
constructor(message, feedName, originalError) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.name = 'NetworkError';
|
|
40
|
+
this.feedName = feedName;
|
|
41
|
+
this.originalError = originalError; // Preserve for debugging
|
|
42
|
+
this.exitCode = 1;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export class FeedError extends Error {
|
|
47
|
+
constructor(message, status, feedName, originalError) {
|
|
48
|
+
super(message);
|
|
49
|
+
this.name = 'FeedError';
|
|
50
|
+
this.status = status;
|
|
51
|
+
this.feedName = feedName;
|
|
52
|
+
this.originalError = originalError; // Preserve for debugging
|
|
53
|
+
this.exitCode = 1;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// ── Retry logic ───────────────────────────────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
function isRetryable(err) {
|
|
60
|
+
if (axios.isAxiosError(err)) {
|
|
61
|
+
if (!err.response) return true; // network error
|
|
62
|
+
const s = err.response.status;
|
|
63
|
+
return s === 429 || s >= 500; // too many requests or server errors
|
|
64
|
+
}
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function withRetry(fn, feedName, maxRetries, logger) {
|
|
69
|
+
let lastErr;
|
|
70
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
71
|
+
try {
|
|
72
|
+
// invokes fetch function - with retries handled in catch block
|
|
73
|
+
return await fn();
|
|
74
|
+
} catch (err) {
|
|
75
|
+
lastErr = err;
|
|
76
|
+
|
|
77
|
+
// Check retryability BEFORE wrapping
|
|
78
|
+
if (!isRetryable(err) || attempt === maxRetries) {
|
|
79
|
+
// Now wrap based on error type
|
|
80
|
+
if (axios.isAxiosError(err)) {
|
|
81
|
+
if (!err.response) {
|
|
82
|
+
throw new NetworkError(err.message, feedName, err);
|
|
83
|
+
}
|
|
84
|
+
throw new FeedError(`HTTP ${err.response.status}`, err.response.status, feedName, err);
|
|
85
|
+
}
|
|
86
|
+
throw err; // Non-Axios error
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const delay = Math.pow(2, attempt) * 200 + Math.random() * 300;
|
|
90
|
+
logger.debug(`[${feedName}] retry ${attempt + 1}/${maxRetries}`, { delayMs: Math.round(delay) });
|
|
91
|
+
await new Promise(r => setTimeout(r, delay));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
throw lastErr; // Should never reach here
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// ── Fetch one feed ────────────────────────────────────────────────────────────
|
|
98
|
+
|
|
99
|
+
async function fetchOne(client, feed, maxRetries, logger) {
|
|
100
|
+
// fetch function (async callback), feed, retries count, logger instance - passed to withRetry
|
|
101
|
+
const result = await withRetry(async () => {
|
|
102
|
+
try {
|
|
103
|
+
const response = await client.get(feed.url);
|
|
104
|
+
return response.data;
|
|
105
|
+
} catch (err) {
|
|
106
|
+
// Don't wrap error type here - let withRetry handle it
|
|
107
|
+
throw err;
|
|
108
|
+
}
|
|
109
|
+
}, feed.name, maxRetries, logger);
|
|
110
|
+
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// ── Fetch all feeds ───────────────────────────────────────────────────────────
|
|
115
|
+
|
|
116
|
+
// receives feeds from (store), config, instantiated logger - invoked when reading from an RSS source
|
|
117
|
+
export async function fetchAll(feeds, config, logger) {
|
|
118
|
+
const client = axios.create({ timeout: config.timeout });
|
|
119
|
+
/*
|
|
120
|
+
Promise.allSettled sets it automatically. When a promise rejects,
|
|
121
|
+
the settled result object is shaped as:
|
|
122
|
+
{ status: 'rejected', reason: <the thrown value> }
|
|
123
|
+
|
|
124
|
+
*/
|
|
125
|
+
const settled = await Promise.allSettled(
|
|
126
|
+
feeds.map(feed => fetchOne(client, feed, config.retries, logger))
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
return settled.map((result, i) => {
|
|
130
|
+
const feed = feeds[i];
|
|
131
|
+
if (result.status === 'fulfilled') {
|
|
132
|
+
// all settled fetch promises provide xml values from feed
|
|
133
|
+
return { name: feed.name, url: feed.url, status: 'ok', xml: result.value, error: null };
|
|
134
|
+
} else {
|
|
135
|
+
return { name: feed.name, url: feed.url, status: 'failed', xml: null, error: result.reason };
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/*
|
|
141
|
+
|
|
142
|
+
Trace for a failing feed:
|
|
143
|
+
```
|
|
144
|
+
fetchOne()
|
|
145
|
+
└─ withRetry()
|
|
146
|
+
└─ fn() → axios throws (e.g. network error)
|
|
147
|
+
└─ isRetryable = true → retries...
|
|
148
|
+
└─ maxRetries exhausted → throw new NetworkError(...)
|
|
149
|
+
← NetworkError propagates out of withRetry
|
|
150
|
+
← NetworkError propagates out of fetchOne
|
|
151
|
+
← Promise rejects with NetworkError
|
|
152
|
+
|
|
153
|
+
Promise.allSettled captures it:
|
|
154
|
+
{ status: 'rejected', reason: NetworkError }
|
|
155
|
+
|
|
156
|
+
Your map reads result.reason → stored as error field:
|
|
157
|
+
{ name, url, status: 'failed', xml: null, error: NetworkError }
|
|
158
|
+
|
|
159
|
+
The key insight is that allSettled never throws, it catches every rejection and boxes
|
|
160
|
+
it into { status, reason }. That's what makes one feed failure safe, other feeds'
|
|
161
|
+
promises are unaffected and still resolve normally.
|
|
162
|
+
*/
|
package/lib/logger.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// lib/logger.js
|
|
2
|
+
// TICKET 1.1 — structured logger wired to logLevel from resolved config
|
|
3
|
+
//
|
|
4
|
+
// Usage:
|
|
5
|
+
// import { createLogger } from './lib/logger.js';
|
|
6
|
+
// const log = createLogger('info');
|
|
7
|
+
// log.info('Feed added');
|
|
8
|
+
// log.debug('Retrying request', { attempt: 2 });
|
|
9
|
+
|
|
10
|
+
import fs from 'fs';
|
|
11
|
+
import path from 'path';
|
|
12
|
+
|
|
13
|
+
const LEVELS = { debug: 0, info: 1, warn: 2, error: 3 };
|
|
14
|
+
const LOG_FILE = path.join(process.cwd(), 'feedwatch.log');
|
|
15
|
+
|
|
16
|
+
export function createLogger(level = 'info') {
|
|
17
|
+
const minLevel = LEVELS[level] ?? LEVELS.info;
|
|
18
|
+
|
|
19
|
+
function write(lvl, msg, data) {
|
|
20
|
+
if (LEVELS[lvl] < minLevel) return;
|
|
21
|
+
const ts = new Date().toISOString();
|
|
22
|
+
const extra = data ? ' ' + JSON.stringify(data) : '';
|
|
23
|
+
const line = `[${ts}] [${lvl.toUpperCase()}] ${msg}${extra}`;
|
|
24
|
+
// debug and info go to stderr so they don't pollute stdout JSON output
|
|
25
|
+
if (lvl === 'error' || lvl === 'warn') {
|
|
26
|
+
process.stderr.write(line + '\n');
|
|
27
|
+
fs.appendFileSync(LOG_FILE, line + '\n');
|
|
28
|
+
} else {
|
|
29
|
+
process.stderr.write(line + '\n');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
debug: (msg, data) => write('debug', msg, data),
|
|
35
|
+
info: (msg, data) => write('info', msg, data),
|
|
36
|
+
warn: (msg, data) => write('warn', msg, data),
|
|
37
|
+
error: (msg, data) => write('error', msg, data),
|
|
38
|
+
};
|
|
39
|
+
}
|
package/lib/parser.js
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// lib/parser.js
|
|
2
|
+
// TICKET 1.4 — RSS/Atom XML parsing and normalisation
|
|
3
|
+
//
|
|
4
|
+
// Normalised item schema:
|
|
5
|
+
// { title, link, pubDate, description, guid }
|
|
6
|
+
//
|
|
7
|
+
// Handles:
|
|
8
|
+
// - RSS 2.0 (<channel><item>)
|
|
9
|
+
// - Atom 1.0 (<feed><entry>)
|
|
10
|
+
// - Missing fields default to empty string — no crash on malformed feeds
|
|
11
|
+
// - pubDate normalised to ISO 8601
|
|
12
|
+
|
|
13
|
+
import { XMLParser } from 'fast-xml-parser';
|
|
14
|
+
|
|
15
|
+
const parser = new XMLParser({
|
|
16
|
+
ignoreAttributes: false,
|
|
17
|
+
attributeNamePrefix: '@_',
|
|
18
|
+
parseTagValue: true,
|
|
19
|
+
trimValues: true,
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
// ── Date normalisation ────────────────────────────────────────────────────────
|
|
23
|
+
|
|
24
|
+
function toISO(raw) {
|
|
25
|
+
if (!raw) return '';
|
|
26
|
+
const d = new Date(raw);
|
|
27
|
+
return isNaN(d.getTime()) ? String(raw) : d.toISOString();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// ── RSS 2.0 ───────────────────────────────────────────────────────────────────
|
|
31
|
+
|
|
32
|
+
function parseRSS(parsed) {
|
|
33
|
+
const channel = parsed?.rss?.channel;
|
|
34
|
+
if (!channel) return null;
|
|
35
|
+
|
|
36
|
+
const rawItems = channel.item || [];
|
|
37
|
+
const items = Array.isArray(rawItems) ? rawItems : [rawItems];
|
|
38
|
+
|
|
39
|
+
return items.map(item => ({
|
|
40
|
+
title: String(item.title || ''),
|
|
41
|
+
link: String(item.link || ''),
|
|
42
|
+
pubDate: toISO(item.pubDate || item['dc:date'] || ''),
|
|
43
|
+
description: String(item.description || item.summary || ''),
|
|
44
|
+
guid: String(item.guid?.['#text'] || item.guid || item.link || item.title || ''),
|
|
45
|
+
}));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// ── Atom 1.0 ──────────────────────────────────────────────────────────────────
|
|
49
|
+
|
|
50
|
+
function parseAtom(parsed) {
|
|
51
|
+
const feed = parsed?.feed;
|
|
52
|
+
if (!feed) return null;
|
|
53
|
+
|
|
54
|
+
const rawEntries = feed.entry || [];
|
|
55
|
+
const entries = Array.isArray(rawEntries) ? rawEntries : [rawEntries];
|
|
56
|
+
|
|
57
|
+
return entries.map(entry => {
|
|
58
|
+
// <link> in Atom can be an object with @_href or a plain string
|
|
59
|
+
const link = entry.link?.['@_href'] || entry.link || '';
|
|
60
|
+
return {
|
|
61
|
+
title: String(entry.title?.['#text'] || entry.title || ''),
|
|
62
|
+
link: String(link),
|
|
63
|
+
pubDate: toISO(entry.updated || entry.published || ''),
|
|
64
|
+
description: String(entry.summary?.['#text'] || entry.summary || entry.content?.['#text'] || ''),
|
|
65
|
+
guid: String(entry.id || link || entry.title || ''),
|
|
66
|
+
};
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// ── Public API ────────────────────────────────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Parse raw XML string into normalised items.
|
|
74
|
+
* Returns [] on any parse error — never throws.
|
|
75
|
+
*/
|
|
76
|
+
export function parseXML(xml) {
|
|
77
|
+
if (!xml || typeof xml !== 'string') return [];
|
|
78
|
+
let parsed;
|
|
79
|
+
try {
|
|
80
|
+
// parser.parse runs once and just converts the raw XML string into a plain JS object.
|
|
81
|
+
parsed = parser.parse(xml);
|
|
82
|
+
} catch {
|
|
83
|
+
return [];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const rssItems = parseRSS(parsed); // looks for parsed?.rss?.channel
|
|
87
|
+
if (rssItems) return rssItems; // found it — return early
|
|
88
|
+
|
|
89
|
+
const atomItems = parseAtom(parsed); // looks for parsed?.feed
|
|
90
|
+
if (atomItems) return atomItems; // found it — return early
|
|
91
|
+
|
|
92
|
+
return []; // neither matched
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
/*
|
|
97
|
+
RSS 2.0 structure:
|
|
98
|
+
xml<rss>
|
|
99
|
+
<channel>
|
|
100
|
+
<item>
|
|
101
|
+
<title>...</title>
|
|
102
|
+
<link>...</link>
|
|
103
|
+
<pubDate>...</pubDate>
|
|
104
|
+
<guid>...</guid>
|
|
105
|
+
</item>
|
|
106
|
+
</channel>
|
|
107
|
+
</rss>
|
|
108
|
+
|
|
109
|
+
Atom 1.0 structure:
|
|
110
|
+
xml<feed>
|
|
111
|
+
<entry>
|
|
112
|
+
<title type="text">...</title>
|
|
113
|
+
<link href="..."/> <!-- href is an attribute, not text content -->
|
|
114
|
+
<updated>...</updated> <!-- different date field name -->
|
|
115
|
+
<id>...</id> <!-- guid equivalent -->
|
|
116
|
+
</entry>
|
|
117
|
+
</feed>
|
|
118
|
+
|
|
119
|
+
parseRSS and parseAtom each know their own shape, and parseXML just tries both RSS first, Atom second;
|
|
120
|
+
returning whichever one finds its root element. If neither matches, it returns [] rather than crashing.
|
|
121
|
+
*/
|
package/lib/store.js
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// lib/store.js
|
|
2
|
+
// TICKET 1.2 — Feed registration persistence
|
|
3
|
+
// TICKET 1.5 — Seen-state persistence
|
|
4
|
+
//
|
|
5
|
+
// Store layout (feedwatch-store.json):
|
|
6
|
+
// {
|
|
7
|
+
// feeds: [{ name, url, addedAt, lastFetchedAt, lastNewCount }],
|
|
8
|
+
// seen: { "<feedName>": ["guid1", "guid2", ...] }
|
|
9
|
+
// }
|
|
10
|
+
|
|
11
|
+
import fs from 'fs';
|
|
12
|
+
import path from 'path';
|
|
13
|
+
import os from 'os';
|
|
14
|
+
|
|
15
|
+
function getStorePaths() {
|
|
16
|
+
const dir = process.env.FEEDWATCH_STORE_DIR || path.join(os.homedir(), '.feedwatch');
|
|
17
|
+
return { dir, path: path.join(dir, 'store.json') };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function ensureDir(dir) {
|
|
21
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function loadStore() {
|
|
25
|
+
const { dir, path: storePath } = getStorePaths();
|
|
26
|
+
ensureDir(dir);
|
|
27
|
+
if (!fs.existsSync(storePath)) return { feeds: [], seen: {} };
|
|
28
|
+
try {
|
|
29
|
+
return JSON.parse(fs.readFileSync(storePath, 'utf8'));
|
|
30
|
+
} catch {
|
|
31
|
+
return { feeds: [], seen: {} };
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function saveStore(store) {
|
|
36
|
+
const { dir, path: storePath } = getStorePaths();
|
|
37
|
+
ensureDir(dir);
|
|
38
|
+
const tmp = storePath + '.tmp';
|
|
39
|
+
fs.writeFileSync(tmp, JSON.stringify(store, null, 2), 'utf8');
|
|
40
|
+
fs.renameSync(tmp, storePath);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// ── Feed helpers ──────────────────────────────────────────────────────────────
|
|
44
|
+
|
|
45
|
+
export function getFeed(store, name) {
|
|
46
|
+
return store.feeds.find(f => f.name === name);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function addFeed(store, name, url) {
|
|
50
|
+
store.feeds.push({ name, url, addedAt: new Date().toISOString(), lastFetchedAt: null, lastNewCount: 0 });
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function removeFeed(store, name) {
|
|
54
|
+
store.feeds = store.feeds.filter(f => f.name !== name);
|
|
55
|
+
delete store.seen[name];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// ── Seen-state helpers ────────────────────────────────────────────────────────
|
|
59
|
+
|
|
60
|
+
export function getSeenGuids(store, feedName) {
|
|
61
|
+
return new Set(store.seen[feedName] || []);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function updateSeen(store, feedName, guids) {
|
|
65
|
+
store.seen[feedName] = guids;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function updateFeedMeta(store, feedName, { lastFetchedAt, lastNewCount }) {
|
|
69
|
+
const feed = getFeed(store, feedName);
|
|
70
|
+
if (!feed) return;
|
|
71
|
+
feed.lastFetchedAt = lastFetchedAt;
|
|
72
|
+
feed.lastNewCount = lastNewCount;
|
|
73
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@abhi-r-27/feedwatch",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "CLI tool to monitor RSS and Atom feeds for new items",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"feedwatch": "./feedwatch.js"
|
|
8
|
+
},
|
|
9
|
+
"files": [
|
|
10
|
+
"feedwatch.js",
|
|
11
|
+
"commands/",
|
|
12
|
+
"lib/",
|
|
13
|
+
"README.md"
|
|
14
|
+
],
|
|
15
|
+
"scripts": {
|
|
16
|
+
"test": "bun test tests/feedwatch.test.js"
|
|
17
|
+
},
|
|
18
|
+
"dependencies": {
|
|
19
|
+
"@inquirer/prompts": "^7.0.0",
|
|
20
|
+
"axios": "^1.7.0",
|
|
21
|
+
"chalk": "^5.3.0",
|
|
22
|
+
"cli-table3": "^0.6.5",
|
|
23
|
+
"commander": "^12.0.0",
|
|
24
|
+
"fast-xml-parser": "^4.4.0",
|
|
25
|
+
"ora": "^8.0.0"
|
|
26
|
+
}
|
|
27
|
+
}
|