@ozzylabs/feedradar 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +31 -6
- package/README.md +31 -6
- package/dist/agents/claude-code.d.ts +12 -1
- package/dist/agents/claude-code.d.ts.map +1 -1
- package/dist/agents/claude-code.js +9 -5
- package/dist/agents/claude-code.js.map +1 -1
- package/dist/agents/codex-cli.d.ts +7 -1
- package/dist/agents/codex-cli.d.ts.map +1 -1
- package/dist/agents/codex-cli.js +9 -5
- package/dist/agents/codex-cli.js.map +1 -1
- package/dist/agents/copilot.d.ts +7 -1
- package/dist/agents/copilot.d.ts.map +1 -1
- package/dist/agents/copilot.js +9 -5
- package/dist/agents/copilot.js.map +1 -1
- package/dist/agents/gemini-cli.d.ts +7 -1
- package/dist/agents/gemini-cli.d.ts.map +1 -1
- package/dist/agents/gemini-cli.js +9 -5
- package/dist/agents/gemini-cli.js.map +1 -1
- package/dist/agents/index.d.ts +1 -1
- package/dist/agents/index.d.ts.map +1 -1
- package/dist/agents/types.d.ts +33 -0
- package/dist/agents/types.d.ts.map +1 -1
- package/dist/cli/_progress.d.ts +138 -0
- package/dist/cli/_progress.d.ts.map +1 -0
- package/dist/cli/_progress.js +176 -0
- package/dist/cli/_progress.js.map +1 -0
- package/dist/cli/doctor.d.ts +20 -0
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +291 -2
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/research.d.ts +18 -20
- package/dist/cli/research.d.ts.map +1 -1
- package/dist/cli/research.js +318 -203
- package/dist/cli/research.js.map +1 -1
- package/dist/cli/respawn.d.ts +53 -0
- package/dist/cli/respawn.d.ts.map +1 -0
- package/dist/cli/respawn.js +120 -0
- package/dist/cli/respawn.js.map +1 -0
- package/dist/cli/review.d.ts +7 -0
- package/dist/cli/review.d.ts.map +1 -1
- package/dist/cli/review.js +46 -1
- package/dist/cli/review.js.map +1 -1
- package/dist/cli/source.d.ts +23 -2
- package/dist/cli/source.d.ts.map +1 -1
- package/dist/cli/source.js +425 -7
- package/dist/cli/source.js.map +1 -1
- package/dist/cli/update.d.ts +7 -0
- package/dist/cli/update.d.ts.map +1 -1
- package/dist/cli/update.js +41 -1
- package/dist/cli/update.js.map +1 -1
- package/dist/cli/watch.d.ts.map +1 -1
- package/dist/cli/watch.js +65 -3
- package/dist/cli/watch.js.map +1 -1
- package/dist/cli/workflow/generate-combined.d.ts +100 -0
- package/dist/cli/workflow/generate-combined.d.ts.map +1 -0
- package/dist/cli/workflow/generate-combined.js +387 -0
- package/dist/cli/workflow/generate-combined.js.map +1 -0
- package/dist/cli/workflow/generate-watch.d.ts +142 -0
- package/dist/cli/workflow/generate-watch.d.ts.map +1 -0
- package/dist/cli/workflow/generate-watch.js +338 -0
- package/dist/cli/workflow/generate-watch.js.map +1 -0
- package/dist/cli/workflow.d.ts +29 -0
- package/dist/cli/workflow.d.ts.map +1 -0
- package/dist/cli/workflow.js +66 -0
- package/dist/cli/workflow.js.map +1 -0
- package/dist/core/feeds/_fetch.d.ts +103 -0
- package/dist/core/feeds/_fetch.d.ts.map +1 -0
- package/dist/core/feeds/_fetch.js +364 -0
- package/dist/core/feeds/_fetch.js.map +1 -0
- package/dist/core/feeds/_jsonpath.d.ts +57 -0
- package/dist/core/feeds/_jsonpath.d.ts.map +1 -0
- package/dist/core/feeds/_jsonpath.js +207 -0
- package/dist/core/feeds/_jsonpath.js.map +1 -0
- package/dist/core/feeds/github-api.d.ts.map +1 -1
- package/dist/core/feeds/github-api.js +2 -1
- package/dist/core/feeds/github-api.js.map +1 -1
- package/dist/core/feeds/html-js.d.ts +29 -0
- package/dist/core/feeds/html-js.d.ts.map +1 -1
- package/dist/core/feeds/html-js.js +86 -2
- package/dist/core/feeds/html-js.js.map +1 -1
- package/dist/core/feeds/html.d.ts.map +1 -1
- package/dist/core/feeds/html.js +2 -1
- package/dist/core/feeds/html.js.map +1 -1
- package/dist/core/feeds/index.d.ts +1 -1
- package/dist/core/feeds/index.d.ts.map +1 -1
- package/dist/core/feeds/index.js +4 -0
- package/dist/core/feeds/index.js.map +1 -1
- package/dist/core/feeds/json-api.d.ts +3 -0
- package/dist/core/feeds/json-api.d.ts.map +1 -0
- package/dist/core/feeds/json-api.js +723 -0
- package/dist/core/feeds/json-api.js.map +1 -0
- package/dist/core/feeds/json-feed.d.ts +11 -0
- package/dist/core/feeds/json-feed.d.ts.map +1 -0
- package/dist/core/feeds/json-feed.js +242 -0
- package/dist/core/feeds/json-feed.js.map +1 -0
- package/dist/core/feeds/npm-registry.d.ts.map +1 -1
- package/dist/core/feeds/npm-registry.js +2 -1
- package/dist/core/feeds/npm-registry.js.map +1 -1
- package/dist/core/feeds/rss.d.ts.map +1 -1
- package/dist/core/feeds/rss.js +2 -1
- package/dist/core/feeds/rss.js.map +1 -1
- package/dist/core/feeds/types.d.ts +123 -0
- package/dist/core/feeds/types.d.ts.map +1 -1
- package/dist/core/progress.d.ts +101 -0
- package/dist/core/progress.d.ts.map +1 -0
- package/dist/core/progress.js +212 -0
- package/dist/core/progress.js.map +1 -0
- package/dist/core/proxy.d.ts +87 -0
- package/dist/core/proxy.d.ts.map +1 -0
- package/dist/core/proxy.js +146 -0
- package/dist/core/proxy.js.map +1 -0
- package/dist/core/recipes.d.ts +138 -0
- package/dist/core/recipes.d.ts.map +1 -0
- package/dist/core/recipes.js +238 -0
- package/dist/core/recipes.js.map +1 -0
- package/dist/core/watcher.d.ts +61 -1
- package/dist/core/watcher.d.ts.map +1 -1
- package/dist/core/watcher.js +99 -2
- package/dist/core/watcher.js.map +1 -1
- package/dist/index.js +17 -4
- package/dist/index.js.map +1 -1
- package/dist/recipes/aws-whats-new.yaml +61 -0
- package/dist/recipes/dev-to.yaml +40 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/index.js.map +1 -1
- package/dist/schemas/recipe.d.ts +115 -0
- package/dist/schemas/recipe.d.ts.map +1 -0
- package/dist/schemas/recipe.js +54 -0
- package/dist/schemas/recipe.js.map +1 -0
- package/dist/schemas/source.d.ts +130 -0
- package/dist/schemas/source.d.ts.map +1 -1
- package/dist/schemas/source.js +130 -0
- package/dist/schemas/source.js.map +1 -1
- package/dist/templates/agents/AGENTS.md +31 -3
- package/dist/templates/feedradar.md +23 -8
- package/dist/templates/workflows/combined.template.yaml.tmpl +110 -0
- package/dist/templates/workflows/watch.template.yaml.tmpl +103 -0
- package/dist/templates/workflows/watch.yaml +5 -1
- package/package.json +2 -3
package/dist/cli/source.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { access, readdir, readFile, unlink, writeFile } from "node:fs/promises";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
|
|
4
|
+
import { createProgressReporter } from "../core/progress.js";
|
|
5
|
+
import { listRecipes, loadRecipe, mergeRecipeWithOverrides, } from "../core/recipes.js";
|
|
4
6
|
import { loadSourceState } from "../core/state.js";
|
|
5
7
|
import { watchRun } from "../core/watcher.js";
|
|
6
8
|
import { SourceKindSchema, SourceSchema, SourceSelectorsSchema } from "../schemas/source.js";
|
|
@@ -40,6 +42,35 @@ function splitCsv(value) {
|
|
|
40
42
|
.map((s) => s.trim())
|
|
41
43
|
.filter((s) => s.length > 0);
|
|
42
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Parse an `--<option> N` integer argument with a configurable minimum.
|
|
47
|
+
*
|
|
48
|
+
* Used for `--pagination-page-size` / `--max-pages` / etc. where Zod will
|
|
49
|
+
* also validate the resulting Source object, but throwing here means the
|
|
50
|
+
* user sees a single targeted error referencing the flag name they typed
|
|
51
|
+
* instead of a deeper schema-path message.
|
|
52
|
+
*
|
|
53
|
+
* `min` defaults to 1 because every json-api pagination integer the schema
|
|
54
|
+
* accepts is `z.number().int().positive()`; the `--pagination-start` flag
|
|
55
|
+
* passes `min: 0` for offset/page indices that legitimately begin at 0.
|
|
56
|
+
*/
|
|
57
|
+
function parseIntFlag(flag, raw, min) {
|
|
58
|
+
if (raw === undefined)
|
|
59
|
+
throw new Error(`option ${flag} requires a value`);
|
|
60
|
+
const n = Number(raw);
|
|
61
|
+
if (!Number.isInteger(n) || n < min) {
|
|
62
|
+
throw new Error(`option ${flag} expects an integer >= ${min}, got '${raw}'`);
|
|
63
|
+
}
|
|
64
|
+
return n;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Pagination strategies accepted by `--pagination-strategy` on
|
|
68
|
+
* `radar source add --kind json-api`. Mirrors `SourcePaginationSchema.type`
|
|
69
|
+
* — kept as a local const so the CLI validator emits a user-friendly enum
|
|
70
|
+
* list before Zod runs (Zod surfaces the same set, but a leading "did you
|
|
71
|
+
* mean …?" message at the CLI layer reads better in shell context).
|
|
72
|
+
*/
|
|
73
|
+
const PAGINATION_STRATEGIES = ["page", "offset", "cursor", "link-header", "token", "none"];
|
|
43
74
|
/**
|
|
44
75
|
* Parse `source add` flags.
|
|
45
76
|
*
|
|
@@ -62,6 +93,13 @@ function parseAddArgs(args) {
|
|
|
62
93
|
out.url = args[++i];
|
|
63
94
|
continue;
|
|
64
95
|
}
|
|
96
|
+
if (a === "--recipe") {
|
|
97
|
+
const value = args[++i];
|
|
98
|
+
if (value === undefined)
|
|
99
|
+
throw new Error(`option ${a} requires a value`);
|
|
100
|
+
out.recipe = value;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
65
103
|
if (a === "--name") {
|
|
66
104
|
out.name = args[++i];
|
|
67
105
|
continue;
|
|
@@ -94,6 +132,56 @@ function parseAddArgs(args) {
|
|
|
94
132
|
out.selectors[field] = value;
|
|
95
133
|
continue;
|
|
96
134
|
}
|
|
135
|
+
if (a === "--pagination-strategy") {
|
|
136
|
+
const value = args[++i];
|
|
137
|
+
if (value === undefined)
|
|
138
|
+
throw new Error(`option ${a} requires a value`);
|
|
139
|
+
if (!PAGINATION_STRATEGIES.includes(value)) {
|
|
140
|
+
throw new Error(`option --pagination-strategy expects one of: ${PAGINATION_STRATEGIES.join(" | ")}, got '${value}'`);
|
|
141
|
+
}
|
|
142
|
+
out.paginationStrategy = value;
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (a === "--pagination-param") {
|
|
146
|
+
const value = args[++i];
|
|
147
|
+
if (value === undefined)
|
|
148
|
+
throw new Error(`option ${a} requires a value`);
|
|
149
|
+
out.paginationParam = value;
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (a === "--pagination-start") {
|
|
153
|
+
out.paginationStart = parseIntFlag(a, args[++i], 0);
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
if (a === "--page-size") {
|
|
157
|
+
out.paginationPageSize = parseIntFlag(a, args[++i], 1);
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
if (a === "--page-size-param") {
|
|
161
|
+
const value = args[++i];
|
|
162
|
+
if (value === undefined)
|
|
163
|
+
throw new Error(`option ${a} requires a value`);
|
|
164
|
+
out.paginationPageSizeParam = value;
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
if (a === "--max-pages") {
|
|
168
|
+
out.paginationMaxPages = parseIntFlag(a, args[++i], 1);
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (a === "--next-cursor-path") {
|
|
172
|
+
const value = args[++i];
|
|
173
|
+
if (value === undefined)
|
|
174
|
+
throw new Error(`option ${a} requires a value`);
|
|
175
|
+
out.paginationNextCursorPath = value;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (a === "--total-path") {
|
|
179
|
+
const value = args[++i];
|
|
180
|
+
if (value === undefined)
|
|
181
|
+
throw new Error(`option ${a} requires a value`);
|
|
182
|
+
out.paginationTotalPath = value;
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
97
185
|
if (a?.startsWith("--")) {
|
|
98
186
|
throw new Error(`unknown option: ${a}`);
|
|
99
187
|
}
|
|
@@ -154,6 +242,14 @@ function parseTestArgs(args) {
|
|
|
154
242
|
out.showContent = true;
|
|
155
243
|
continue;
|
|
156
244
|
}
|
|
245
|
+
if (a === "--verbose" || a === "-v") {
|
|
246
|
+
out.verbose = true;
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
if (a === "--quiet" || a === "-q") {
|
|
250
|
+
out.quiet = true;
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
157
253
|
if (a?.startsWith("--")) {
|
|
158
254
|
throw new Error(`unknown option: ${a}`);
|
|
159
255
|
}
|
|
@@ -163,6 +259,9 @@ function parseTestArgs(args) {
|
|
|
163
259
|
}
|
|
164
260
|
throw new Error(`unexpected positional argument: ${a}`);
|
|
165
261
|
}
|
|
262
|
+
if (out.verbose && out.quiet) {
|
|
263
|
+
throw new Error("--verbose and --quiet are mutually exclusive");
|
|
264
|
+
}
|
|
166
265
|
return out;
|
|
167
266
|
}
|
|
168
267
|
function parseRemoveArgs(args) {
|
|
@@ -185,10 +284,15 @@ function parseRemoveArgs(args) {
|
|
|
185
284
|
}
|
|
186
285
|
function printAddHelp(log) {
|
|
187
286
|
log("Usage: radar source add <id> --kind <kind> --url <url> [options]");
|
|
287
|
+
log(" radar source add <id> --recipe <name> [overrides]");
|
|
188
288
|
log("");
|
|
189
289
|
log("Options:");
|
|
190
|
-
log(" --kind <kind> rss | html | html-js | github-releases | npm-registry");
|
|
290
|
+
log(" --kind <kind> rss | html | html-js | github-releases | npm-registry | json-feed | json-api");
|
|
191
291
|
log(" --url <url> fetch target URL");
|
|
292
|
+
log(" --recipe <name> apply a bundled recipe (see `radar source recipes`).");
|
|
293
|
+
log(" Mutually exclusive with --kind / --url / --selector-* /");
|
|
294
|
+
log(" --pagination-*; --name / --tags / --keywords /");
|
|
295
|
+
log(" --exclude-keywords still override the recipe defaults.");
|
|
192
296
|
log(" --name <name> display name (defaults to <id>)");
|
|
193
297
|
log(" --tags <a,b> comma-separated tags");
|
|
194
298
|
log(" --keywords <a,b> comma-separated include keywords");
|
|
@@ -199,6 +303,21 @@ function printAddHelp(log) {
|
|
|
199
303
|
log(" For kind=html-js, selectors evaluate against the post-JS DOM.");
|
|
200
304
|
log(" The `js:` block (waitFor / timeout / userAgent) cannot be set");
|
|
201
305
|
log(" via flags; edit sources/<id>.yaml after add. See ADR-0010.");
|
|
306
|
+
log("");
|
|
307
|
+
log(" For kind=json-api (ADR-0012 / #174):");
|
|
308
|
+
log(" --pagination-strategy <s> page | offset | cursor | link-header | token | none (default: page)");
|
|
309
|
+
log(" --pagination-param <name> query param name for the page/offset/cursor value");
|
|
310
|
+
log(" --pagination-start N initial page/offset value (default: 0)");
|
|
311
|
+
log(" --page-size N items per page");
|
|
312
|
+
log(" --page-size-param <name> query param name for the page-size value");
|
|
313
|
+
log(" --max-pages N hard cap on pages traversed (default: 20)");
|
|
314
|
+
log(" --next-cursor-path <jp> JSONPath-lite to the next-cursor value (cursor/token strategy)");
|
|
315
|
+
log(" --total-path <jp> JSONPath-lite to the total-count value (backfill early-stop hint)");
|
|
316
|
+
log("");
|
|
317
|
+
log(" Selector fields (`jsonSelectors.*`) for kind=json-api cannot be set via flags;");
|
|
318
|
+
log(" the schema has a default fallback chain (items / title / link / publishedAt / summary),");
|
|
319
|
+
log(" so simple APIs work without selectors. Edit sources/<id>.yaml directly when explicit");
|
|
320
|
+
log(" selectors are needed (nested fields, non-standard envelopes).");
|
|
202
321
|
}
|
|
203
322
|
function printListHelp(log) {
|
|
204
323
|
log("Usage: radar source list [--enabled-only] [-v|--verbose]");
|
|
@@ -222,16 +341,42 @@ function printTestHelp(log) {
|
|
|
222
341
|
log("state/ and items/ are not touched (no persistence). Useful for tuning");
|
|
223
342
|
log("keywords when adding a new source.");
|
|
224
343
|
log("");
|
|
344
|
+
log("For kind=json-api (ADR-0012 / #174), `source test` fetches PAGE 0 ONLY.");
|
|
345
|
+
log("Pagination is NOT walked even when the recipe declares multiple pages —");
|
|
346
|
+
log("`--limit N` caps how many matched items are PRINTED, it does not change");
|
|
347
|
+
log("the page budget. Use `radar watch run --backfill` for full-history ingest.");
|
|
348
|
+
log("Page 0's `Link` header / `nextCursor` extraction is surfaced via");
|
|
349
|
+
log("`--show-content` for pagination tuning without state mutation.");
|
|
350
|
+
log("");
|
|
225
351
|
log("Options:");
|
|
226
352
|
log(" --limit N Maximum number of matched items to print (default 10)");
|
|
227
|
-
log(" --show-content Also print the first 200
|
|
353
|
+
log(" --show-content Also print the first 200 chars of each item's body, plus");
|
|
354
|
+
log(" (kind=json-api) the selector adoption table and pagination");
|
|
355
|
+
log(" preview (would-be next URL / Link header / nextCursor).");
|
|
356
|
+
log(" -v, --verbose Enable progress-reporter raw() pass-through (adapter stdout).");
|
|
357
|
+
log(" Most useful with kind=html-js (Playwright phase markers).");
|
|
358
|
+
log(" -q, --quiet Suppress the progress reporter entirely. RADAR_NO_PROGRESS=1");
|
|
359
|
+
log(" has the same effect.");
|
|
360
|
+
}
|
|
361
|
+
function printRecipesHelp(log) {
|
|
362
|
+
log("Usage: radar source recipes");
|
|
363
|
+
log("");
|
|
364
|
+
log("List bundled recipes (recipes/*.yaml in the radar package — ADR-0012 §D3).");
|
|
365
|
+
log("Each recipe can be applied via:");
|
|
366
|
+
log(" radar source add <id> --recipe <name> [--keywords <kw>] [--tags <t>] [--name <display>]");
|
|
367
|
+
log("");
|
|
368
|
+
log("Bundled recipes ship with the radar npm package; user-authored recipes are");
|
|
369
|
+
log("not yet supported. To add a new bundled recipe, contribute a YAML to the");
|
|
370
|
+
log("radar repo's recipes/ directory.");
|
|
228
371
|
}
|
|
229
372
|
function printSourceHelp(log) {
|
|
230
|
-
log("Usage: radar source <add|list|remove|test> [...]");
|
|
373
|
+
log("Usage: radar source <add|list|recipes|remove|test> [...]");
|
|
231
374
|
log("");
|
|
232
375
|
log("Subcommands:");
|
|
233
376
|
log(" add <id> --kind <kind> --url <url> [...]");
|
|
377
|
+
log(" add <id> --recipe <name> [--keywords <kw>] [--tags <t>] [--name <display>]");
|
|
234
378
|
log(" list [--enabled-only]");
|
|
379
|
+
log(" recipes");
|
|
235
380
|
log(" remove <id>");
|
|
236
381
|
log(" test <id> [--limit N] [--show-content]");
|
|
237
382
|
}
|
|
@@ -269,6 +414,16 @@ export async function addSource(args, options = {}) {
|
|
|
269
414
|
error(`source add: invalid <id> '${parsed.id}' (must match [A-Za-z0-9][A-Za-z0-9._-]*)`);
|
|
270
415
|
return 2;
|
|
271
416
|
}
|
|
417
|
+
// `--recipe <name>` short-circuits the flag-based composition: the
|
|
418
|
+
// recipe supplies kind / url / pagination / selectors / etc., and only
|
|
419
|
+
// a narrow whitelist of CLI flags is allowed to override
|
|
420
|
+
// (`--name` / `--tags` / `--keywords` / `--exclude-keywords`).
|
|
421
|
+
// Everything else (incl. `--kind` / `--url` / `--selector-*` /
|
|
422
|
+
// `--pagination-*`) is rejected so the user gets an immediate, targeted
|
|
423
|
+
// error instead of silently-ignored flags. ADR-0012 §D3.
|
|
424
|
+
if (parsed.recipe !== undefined) {
|
|
425
|
+
return addSourceFromRecipe(parsed, cwd, options, log, warn, error);
|
|
426
|
+
}
|
|
272
427
|
if (!parsed.kind) {
|
|
273
428
|
error("source add: --kind is required");
|
|
274
429
|
return 2;
|
|
@@ -279,7 +434,7 @@ export async function addSource(args, options = {}) {
|
|
|
279
434
|
}
|
|
280
435
|
const kindResult = SourceKindSchema.safeParse(parsed.kind);
|
|
281
436
|
if (!kindResult.success) {
|
|
282
|
-
error(`source add: invalid --kind '${parsed.kind}' (expected: rss | html | html-js | github-releases | npm-registry)`);
|
|
437
|
+
error(`source add: invalid --kind '${parsed.kind}' (expected: rss | html | html-js | github-releases | npm-registry | json-feed | json-api)`);
|
|
283
438
|
return 2;
|
|
284
439
|
}
|
|
285
440
|
// Compose the object before schema validation so url-format errors et al.
|
|
@@ -318,6 +473,52 @@ export async function addSource(args, options = {}) {
|
|
|
318
473
|
}
|
|
319
474
|
candidate.selectors = selectorsResult.data;
|
|
320
475
|
}
|
|
476
|
+
// For kind=json-api, build the `pagination:` block from --pagination-*
|
|
477
|
+
// flags. The schema requires `pagination` for this kind; if the user
|
|
478
|
+
// omitted --pagination-strategy entirely we default to `page` (the most
|
|
479
|
+
// common shape for AWS What's New / dev.to / Anthropic news). We do NOT
|
|
480
|
+
// generate a `jsonSelectors:` block — the default fallback chain covers
|
|
481
|
+
// simple APIs, and recipe authors edit the YAML directly when explicit
|
|
482
|
+
// selectors are needed (the field count makes flag-based mutation
|
|
483
|
+
// impractical).
|
|
484
|
+
if (kindResult.data === "json-api") {
|
|
485
|
+
const strategy = parsed.paginationStrategy ?? "page";
|
|
486
|
+
const pagination = { type: strategy };
|
|
487
|
+
if (parsed.paginationParam !== undefined)
|
|
488
|
+
pagination.param = parsed.paginationParam;
|
|
489
|
+
if (parsed.paginationStart !== undefined)
|
|
490
|
+
pagination.start = parsed.paginationStart;
|
|
491
|
+
if (parsed.paginationPageSize !== undefined) {
|
|
492
|
+
pagination.pageSize = parsed.paginationPageSize;
|
|
493
|
+
}
|
|
494
|
+
if (parsed.paginationPageSizeParam !== undefined) {
|
|
495
|
+
pagination.pageSizeParam = parsed.paginationPageSizeParam;
|
|
496
|
+
}
|
|
497
|
+
if (parsed.paginationMaxPages !== undefined) {
|
|
498
|
+
pagination.maxPages = parsed.paginationMaxPages;
|
|
499
|
+
}
|
|
500
|
+
if (parsed.paginationNextCursorPath !== undefined) {
|
|
501
|
+
pagination.nextCursorPath = parsed.paginationNextCursorPath;
|
|
502
|
+
}
|
|
503
|
+
if (parsed.paginationTotalPath !== undefined) {
|
|
504
|
+
pagination.totalPath = parsed.paginationTotalPath;
|
|
505
|
+
}
|
|
506
|
+
candidate.pagination = pagination;
|
|
507
|
+
}
|
|
508
|
+
else if (parsed.paginationStrategy !== undefined ||
|
|
509
|
+
parsed.paginationParam !== undefined ||
|
|
510
|
+
parsed.paginationStart !== undefined ||
|
|
511
|
+
parsed.paginationPageSize !== undefined ||
|
|
512
|
+
parsed.paginationPageSizeParam !== undefined ||
|
|
513
|
+
parsed.paginationMaxPages !== undefined ||
|
|
514
|
+
parsed.paginationNextCursorPath !== undefined ||
|
|
515
|
+
parsed.paginationTotalPath !== undefined) {
|
|
516
|
+
// Reject pagination flags on non-json-api kinds early so the user sees
|
|
517
|
+
// a targeted hint instead of a deep schema refinement error ("pagination
|
|
518
|
+
// is required when kind is 'json-api'" makes no sense for `kind: rss`).
|
|
519
|
+
error(`source add: --pagination-* flags are only valid with --kind json-api (got --kind '${kindResult.data}')`);
|
|
520
|
+
return 2;
|
|
521
|
+
}
|
|
321
522
|
const validated = SourceSchema.safeParse(candidate);
|
|
322
523
|
if (!validated.success) {
|
|
323
524
|
const issues = validated.error.issues.map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`);
|
|
@@ -346,6 +547,104 @@ export async function addSource(args, options = {}) {
|
|
|
346
547
|
}
|
|
347
548
|
return 0;
|
|
348
549
|
}
|
|
550
|
+
/**
|
|
551
|
+
* Apply a bundled recipe (`--recipe <name>`) to produce a new
|
|
552
|
+
* `sources/<id>.yaml` (ADR-0012 §D3, strategy A).
|
|
553
|
+
*
|
|
554
|
+
* Validation discipline:
|
|
555
|
+
*
|
|
556
|
+
* - The recipe supplies `kind` / `url` / structural fields. Re-passing
|
|
557
|
+
* them as CLI flags is rejected to prevent the "recipe says one thing,
|
|
558
|
+
* flag says another, who wins?" footgun. Only the explicit override
|
|
559
|
+
* whitelist (`--name` / `--tags` / `--keywords` /
|
|
560
|
+
* `--exclude-keywords`) is honoured.
|
|
561
|
+
* - `--selector-*` / `--pagination-*` are also rejected on the recipe
|
|
562
|
+
* path — these belong to the recipe author. Users who want to deviate
|
|
563
|
+
* structurally edit `sources/<id>.yaml` after generation, same as for
|
|
564
|
+
* any other source.
|
|
565
|
+
*/
|
|
566
|
+
async function addSourceFromRecipe(parsed, cwd, options, log, warn, error) {
|
|
567
|
+
const recipeName = parsed.recipe;
|
|
568
|
+
// Defensive — should be guaranteed by the caller, but `parsed.recipe`
|
|
569
|
+
// is typed as `string | undefined` so a quick narrow here keeps the
|
|
570
|
+
// rest of the function tidy.
|
|
571
|
+
if (recipeName === undefined) {
|
|
572
|
+
error("source add: --recipe is required (internal dispatch error)");
|
|
573
|
+
return 2;
|
|
574
|
+
}
|
|
575
|
+
// Reject flags that the recipe owns. Surfacing each forbidden flag
|
|
576
|
+
// individually beats a generic "incompatible flags" message because
|
|
577
|
+
// the user sees exactly what to remove.
|
|
578
|
+
const forbidden = [];
|
|
579
|
+
if (parsed.kind !== undefined)
|
|
580
|
+
forbidden.push("--kind");
|
|
581
|
+
if (parsed.url !== undefined)
|
|
582
|
+
forbidden.push("--url");
|
|
583
|
+
if (parsed.selectors !== undefined)
|
|
584
|
+
forbidden.push("--selector-<field>");
|
|
585
|
+
if (parsed.paginationStrategy !== undefined ||
|
|
586
|
+
parsed.paginationParam !== undefined ||
|
|
587
|
+
parsed.paginationStart !== undefined ||
|
|
588
|
+
parsed.paginationPageSize !== undefined ||
|
|
589
|
+
parsed.paginationPageSizeParam !== undefined ||
|
|
590
|
+
parsed.paginationMaxPages !== undefined ||
|
|
591
|
+
parsed.paginationNextCursorPath !== undefined ||
|
|
592
|
+
parsed.paginationTotalPath !== undefined) {
|
|
593
|
+
forbidden.push("--pagination-*");
|
|
594
|
+
}
|
|
595
|
+
if (forbidden.length > 0) {
|
|
596
|
+
error(`source add: --recipe '${recipeName}' supplies kind / url / structural fields; the following flags are not allowed with --recipe: ${forbidden.join(", ")}`);
|
|
597
|
+
return 2;
|
|
598
|
+
}
|
|
599
|
+
let loaded;
|
|
600
|
+
try {
|
|
601
|
+
loaded = await loadRecipe(recipeName, { recipesRoot: options.recipesRoot });
|
|
602
|
+
}
|
|
603
|
+
catch (e) {
|
|
604
|
+
error(`source add: ${e instanceof Error ? e.message : String(e)}`);
|
|
605
|
+
return 1;
|
|
606
|
+
}
|
|
607
|
+
const candidate = mergeRecipeWithOverrides(loaded.recipe, {
|
|
608
|
+
// `parsed.id` is asserted non-undefined by the caller before this
|
|
609
|
+
// function is reached, but TypeScript cannot see that across the
|
|
610
|
+
// dispatch boundary. The non-null assertion mirrors what the
|
|
611
|
+
// flag-based path expects from the same guard.
|
|
612
|
+
// biome-ignore lint/style/noNonNullAssertion: caller-enforced invariant
|
|
613
|
+
id: parsed.id,
|
|
614
|
+
name: parsed.name,
|
|
615
|
+
tags: parsed.tags,
|
|
616
|
+
keywords: parsed.keywords,
|
|
617
|
+
excludeKeywords: parsed.excludeKeywords,
|
|
618
|
+
});
|
|
619
|
+
const validated = SourceSchema.safeParse(candidate);
|
|
620
|
+
if (!validated.success) {
|
|
621
|
+
// A Zod failure on a recipe-derived candidate means the recipe
|
|
622
|
+
// itself is malformed (or, more rarely, an override produced an
|
|
623
|
+
// illegal combination). Surface every issue verbatim so recipe
|
|
624
|
+
// authors and end users can both diagnose.
|
|
625
|
+
const issues = validated.error.issues.map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`);
|
|
626
|
+
error(`source add: recipe '${recipeName}' produced an invalid source`);
|
|
627
|
+
for (const issue of issues) {
|
|
628
|
+
error(` - ${issue}`);
|
|
629
|
+
}
|
|
630
|
+
return 2;
|
|
631
|
+
}
|
|
632
|
+
const file = sourceFile(cwd, validated.data.id);
|
|
633
|
+
if (await pathExists(file)) {
|
|
634
|
+
error(`source add: '${validated.data.id}' already exists (sources/${validated.data.id}.yaml)`);
|
|
635
|
+
return 1;
|
|
636
|
+
}
|
|
637
|
+
await writeFile(file, stringifyYaml(validated.data), "utf8");
|
|
638
|
+
log(`source add: created sources/${validated.data.id}.yaml from recipe '${recipeName}'`);
|
|
639
|
+
// Same firehose-guard hint as the flag-based path: an empty
|
|
640
|
+
// include-keyword list silently drops every fetched item, which
|
|
641
|
+
// surprises users when they thought the recipe came with sensible
|
|
642
|
+
// defaults.
|
|
643
|
+
if (validated.data.filters.keywords.length === 0) {
|
|
644
|
+
warn(`source add: warning — '${validated.data.id}' has no keywords; all fetched items will be filtered out. Re-add with --keywords or edit sources/${validated.data.id}.yaml to start ingesting.`);
|
|
645
|
+
}
|
|
646
|
+
return 0;
|
|
647
|
+
}
|
|
349
648
|
/**
|
|
350
649
|
* Load and validate a single `sources/<id>.yaml` file. Returns `null` and
|
|
351
650
|
* reports through `onError` when the file is malformed, so `list` can keep
|
|
@@ -592,6 +891,13 @@ export async function testSource(args, options = {}) {
|
|
|
592
891
|
return 1;
|
|
593
892
|
}
|
|
594
893
|
const limit = parsed.limit ?? 10;
|
|
894
|
+
// Build the progress reporter (#198). `source test` runs exactly one
|
|
895
|
+
// source so the watcher heuristic only enables narration when the kind
|
|
896
|
+
// is `html-js` / `json-api`; for rss / html / npm-registry / etc. the
|
|
897
|
+
// legacy 1-line summary remains the only output. Tests pin the level
|
|
898
|
+
// explicitly via `--quiet` or `RADAR_NO_PROGRESS=1`.
|
|
899
|
+
const level = parsed.quiet ? "quiet" : parsed.verbose ? "verbose" : "normal";
|
|
900
|
+
const progress = createProgressReporter({ level });
|
|
595
901
|
let result;
|
|
596
902
|
try {
|
|
597
903
|
result = await watchRun({
|
|
@@ -602,6 +908,7 @@ export async function testSource(args, options = {}) {
|
|
|
602
908
|
log,
|
|
603
909
|
warn,
|
|
604
910
|
error,
|
|
911
|
+
progress,
|
|
605
912
|
});
|
|
606
913
|
}
|
|
607
914
|
catch (e) {
|
|
@@ -620,6 +927,41 @@ export async function testSource(args, options = {}) {
|
|
|
620
927
|
log("");
|
|
621
928
|
log(`source test: ${parsed.id}`);
|
|
622
929
|
log(` fetched: ${fetched} / filtered: ${filtered} / matched: ${matched.length}`);
|
|
930
|
+
// Render the adapter diag for `kind: json-api` when --show-content is on.
|
|
931
|
+
// The diag block is intentionally gated behind --show-content so the
|
|
932
|
+
// default `source test` output (used by scripts that just want a quick
|
|
933
|
+
// matched-items dump) stays narrow. Adapters that do not return diag
|
|
934
|
+
// simply skip this block.
|
|
935
|
+
if (parsed.showContent) {
|
|
936
|
+
const diag = result.diag[parsed.id];
|
|
937
|
+
if (diag) {
|
|
938
|
+
if (diag.selectorAdoption) {
|
|
939
|
+
log("");
|
|
940
|
+
log(" selector adoption:");
|
|
941
|
+
for (const [field, path] of Object.entries(diag.selectorAdoption)) {
|
|
942
|
+
if (path === null) {
|
|
943
|
+
log(` ${field}: (no candidate matched)`);
|
|
944
|
+
}
|
|
945
|
+
else {
|
|
946
|
+
log(` ${field} ← ${path} を採用`);
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
if (diag.paginationPreview) {
|
|
951
|
+
const p = diag.paginationPreview;
|
|
952
|
+
log("");
|
|
953
|
+
log(" pagination preview (page 0 only — state not mutated):");
|
|
954
|
+
log(` strategy: ${p.strategy}`);
|
|
955
|
+
log(` nextUrl: ${p.nextUrl ?? "(end of pagination)"}`);
|
|
956
|
+
if (p.linkHeaderNext !== undefined) {
|
|
957
|
+
log(` Link rel=next: ${p.linkHeaderNext ?? "(absent)"}`);
|
|
958
|
+
}
|
|
959
|
+
if (p.nextCursor !== undefined) {
|
|
960
|
+
log(` nextCursor: ${p.nextCursor ?? "(absent)"}`);
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
}
|
|
623
965
|
if (matched.length === 0) {
|
|
624
966
|
log(" (no matched items)");
|
|
625
967
|
return 0;
|
|
@@ -650,12 +992,86 @@ export async function testSource(args, options = {}) {
|
|
|
650
992
|
}
|
|
651
993
|
return 0;
|
|
652
994
|
}
|
|
995
|
+
/**
|
|
996
|
+
* Implementation of `source recipes` — list bundled recipes.
|
|
997
|
+
*
|
|
998
|
+
* Prints a fixed-width table (NAME / KIND / DESCRIPTION) for each
|
|
999
|
+
* `recipes/<name>.yaml` that loads cleanly. Recipes that fail to parse
|
|
1000
|
+
* are appended after the valid set with their error so the user can fix
|
|
1001
|
+
* (or report) the recipe without losing the rest of the listing.
|
|
1002
|
+
*
|
|
1003
|
+
* When the bundle is empty or absent (the bootstrap state before #178
|
|
1004
|
+
* ships the actual recipe content), a friendly "no recipes" message is
|
|
1005
|
+
* printed and exit code 0 is returned — the CLI is functional even when
|
|
1006
|
+
* the recipe library is empty.
|
|
1007
|
+
*/
|
|
1008
|
+
export async function recipesSubcommand(args, options = {}) {
|
|
1009
|
+
const log = options.io?.log ?? ((m) => console.log(m));
|
|
1010
|
+
const error = options.io?.error ?? ((m) => console.error(m));
|
|
1011
|
+
// The only flag accepted today is `-h` / `--help`. Keep the parser
|
|
1012
|
+
// tiny rather than introducing a typed args struct for a single
|
|
1013
|
+
// option — easier to extend if/when filters land.
|
|
1014
|
+
for (const a of args) {
|
|
1015
|
+
if (a === "-h" || a === "--help") {
|
|
1016
|
+
printRecipesHelp(log);
|
|
1017
|
+
return 0;
|
|
1018
|
+
}
|
|
1019
|
+
if (a.startsWith("--")) {
|
|
1020
|
+
error(`source recipes: unknown option: ${a}`);
|
|
1021
|
+
return 2;
|
|
1022
|
+
}
|
|
1023
|
+
error(`source recipes: unexpected positional argument: ${a}`);
|
|
1024
|
+
return 2;
|
|
1025
|
+
}
|
|
1026
|
+
let entries;
|
|
1027
|
+
try {
|
|
1028
|
+
entries = await listRecipes({ recipesRoot: options.recipesRoot });
|
|
1029
|
+
}
|
|
1030
|
+
catch (e) {
|
|
1031
|
+
error(`source recipes: ${e instanceof Error ? e.message : String(e)}`);
|
|
1032
|
+
return 1;
|
|
1033
|
+
}
|
|
1034
|
+
if (entries.length === 0) {
|
|
1035
|
+
log("source recipes: no recipes bundled (recipes/ is empty or absent)");
|
|
1036
|
+
return 0;
|
|
1037
|
+
}
|
|
1038
|
+
const valid = entries.filter((e) => e.recipe !== null);
|
|
1039
|
+
const invalid = entries.filter((e) => e.recipe === null);
|
|
1040
|
+
if (valid.length > 0) {
|
|
1041
|
+
const nameWidth = Math.max(4, ...valid.map((e) => e.name.length));
|
|
1042
|
+
const kindWidth = Math.max(4, ...valid.map((e) => (e.recipe ? e.recipe.kind.length : 0)));
|
|
1043
|
+
log(`${pad("NAME", nameWidth)} ${pad("KIND", kindWidth)} DESCRIPTION`);
|
|
1044
|
+
for (const e of valid) {
|
|
1045
|
+
if (!e.recipe)
|
|
1046
|
+
continue;
|
|
1047
|
+
const desc = e.recipe.description ?? "";
|
|
1048
|
+
log(`${pad(e.name, nameWidth)} ${pad(e.recipe.kind, kindWidth)} ${desc}`);
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
else {
|
|
1052
|
+
log("source recipes: no valid recipes found (all bundled entries failed to load)");
|
|
1053
|
+
}
|
|
1054
|
+
if (invalid.length > 0) {
|
|
1055
|
+
log("");
|
|
1056
|
+
log("Recipes with errors:");
|
|
1057
|
+
for (const e of invalid) {
|
|
1058
|
+
log(` ${e.name}: ${e.error ?? "(unknown error)"}`);
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
log("");
|
|
1062
|
+
log("Apply a recipe with:");
|
|
1063
|
+
log(" radar source add <id> --recipe <name> [--keywords <kw>] [--tags <t>] [--name <display>]");
|
|
1064
|
+
// Returning 0 even when individual recipes have errors keeps the
|
|
1065
|
+
// listing useful in CI: a single malformed recipe should not break
|
|
1066
|
+
// the discovery command for the rest of the bundle.
|
|
1067
|
+
return 0;
|
|
1068
|
+
}
|
|
653
1069
|
/**
|
|
654
1070
|
* Top-level dispatcher for `radar source <subcommand>`.
|
|
655
1071
|
*
|
|
656
1072
|
* Sub-commands are kept as named functions (addSource/listSources/removeSource/
|
|
657
|
-
* testSource) so tests can call them directly with injected
|
|
658
|
-
* spawning the full CLI.
|
|
1073
|
+
* testSource/recipesSubcommand) so tests can call them directly with injected
|
|
1074
|
+
* IO sinks without spawning the full CLI.
|
|
659
1075
|
*/
|
|
660
1076
|
export async function runSource(args, options = {}) {
|
|
661
1077
|
const log = options.io?.log ?? ((m) => console.log(m));
|
|
@@ -670,6 +1086,8 @@ export async function runSource(args, options = {}) {
|
|
|
670
1086
|
return addSource(rest, options);
|
|
671
1087
|
case "list":
|
|
672
1088
|
return listSources(rest, options);
|
|
1089
|
+
case "recipes":
|
|
1090
|
+
return recipesSubcommand(rest, options);
|
|
673
1091
|
case "remove":
|
|
674
1092
|
return removeSource(rest, options);
|
|
675
1093
|
case "test":
|
|
@@ -682,7 +1100,7 @@ export async function runSource(args, options = {}) {
|
|
|
682
1100
|
}
|
|
683
1101
|
export const sourceCommand = {
|
|
684
1102
|
name: "source",
|
|
685
|
-
summary: "Manage feed sources (add | list | remove | test)",
|
|
1103
|
+
summary: "Manage feed sources (add | list | recipes | remove | test)",
|
|
686
1104
|
run: (args) => runSource(args),
|
|
687
1105
|
};
|
|
688
1106
|
//# sourceMappingURL=source.js.map
|