@ozzylabs/feedradar 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +12 -6
- package/README.md +11 -6
- package/dist/agents/claude-code.d.ts +12 -1
- package/dist/agents/claude-code.d.ts.map +1 -1
- package/dist/agents/claude-code.js +9 -5
- package/dist/agents/claude-code.js.map +1 -1
- package/dist/agents/codex-cli.d.ts +7 -1
- package/dist/agents/codex-cli.d.ts.map +1 -1
- package/dist/agents/codex-cli.js +9 -5
- package/dist/agents/codex-cli.js.map +1 -1
- package/dist/agents/copilot.d.ts +7 -1
- package/dist/agents/copilot.d.ts.map +1 -1
- package/dist/agents/copilot.js +9 -5
- package/dist/agents/copilot.js.map +1 -1
- package/dist/agents/gemini-cli.d.ts +7 -1
- package/dist/agents/gemini-cli.d.ts.map +1 -1
- package/dist/agents/gemini-cli.js +9 -5
- package/dist/agents/gemini-cli.js.map +1 -1
- package/dist/agents/index.d.ts +1 -1
- package/dist/agents/index.d.ts.map +1 -1
- package/dist/agents/types.d.ts +33 -0
- package/dist/agents/types.d.ts.map +1 -1
- package/dist/cli/_progress.d.ts +138 -0
- package/dist/cli/_progress.d.ts.map +1 -0
- package/dist/cli/_progress.js +176 -0
- package/dist/cli/_progress.js.map +1 -0
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/research.d.ts +18 -20
- package/dist/cli/research.d.ts.map +1 -1
- package/dist/cli/research.js +318 -203
- package/dist/cli/research.js.map +1 -1
- package/dist/cli/review.d.ts +7 -0
- package/dist/cli/review.d.ts.map +1 -1
- package/dist/cli/review.js +46 -1
- package/dist/cli/review.js.map +1 -1
- package/dist/cli/source.d.ts +23 -2
- package/dist/cli/source.d.ts.map +1 -1
- package/dist/cli/source.js +428 -7
- package/dist/cli/source.js.map +1 -1
- package/dist/cli/update.d.ts +7 -0
- package/dist/cli/update.d.ts.map +1 -1
- package/dist/cli/update.js +41 -1
- package/dist/cli/update.js.map +1 -1
- package/dist/cli/watch.d.ts.map +1 -1
- package/dist/cli/watch.js +67 -3
- package/dist/cli/watch.js.map +1 -1
- package/dist/cli/workflow/generate-combined.d.ts +100 -0
- package/dist/cli/workflow/generate-combined.d.ts.map +1 -0
- package/dist/cli/workflow/generate-combined.js +387 -0
- package/dist/cli/workflow/generate-combined.js.map +1 -0
- package/dist/cli/workflow/generate-watch.d.ts +142 -0
- package/dist/cli/workflow/generate-watch.d.ts.map +1 -0
- package/dist/cli/workflow/generate-watch.js +338 -0
- package/dist/cli/workflow/generate-watch.js.map +1 -0
- package/dist/cli/workflow.d.ts +29 -0
- package/dist/cli/workflow.d.ts.map +1 -0
- package/dist/cli/workflow.js +66 -0
- package/dist/cli/workflow.js.map +1 -0
- package/dist/core/feeds/_fetch.d.ts +10 -0
- package/dist/core/feeds/_fetch.d.ts.map +1 -1
- package/dist/core/feeds/_fetch.js +182 -0
- package/dist/core/feeds/_fetch.js.map +1 -1
- package/dist/core/feeds/_jsonpath.d.ts +57 -0
- package/dist/core/feeds/_jsonpath.d.ts.map +1 -0
- package/dist/core/feeds/_jsonpath.js +207 -0
- package/dist/core/feeds/_jsonpath.js.map +1 -0
- package/dist/core/feeds/html-js.d.ts +8 -0
- package/dist/core/feeds/html-js.d.ts.map +1 -1
- package/dist/core/feeds/html-js.js +47 -1
- package/dist/core/feeds/html-js.js.map +1 -1
- package/dist/core/feeds/index.d.ts +1 -1
- package/dist/core/feeds/index.d.ts.map +1 -1
- package/dist/core/feeds/index.js +4 -0
- package/dist/core/feeds/index.js.map +1 -1
- package/dist/core/feeds/json-api.d.ts +29 -0
- package/dist/core/feeds/json-api.d.ts.map +1 -0
- package/dist/core/feeds/json-api.js +860 -0
- package/dist/core/feeds/json-api.js.map +1 -0
- package/dist/core/feeds/json-feed.d.ts +11 -0
- package/dist/core/feeds/json-feed.d.ts.map +1 -0
- package/dist/core/feeds/json-feed.js +242 -0
- package/dist/core/feeds/json-feed.js.map +1 -0
- package/dist/core/feeds/types.d.ts +123 -0
- package/dist/core/feeds/types.d.ts.map +1 -1
- package/dist/core/progress.d.ts +101 -0
- package/dist/core/progress.d.ts.map +1 -0
- package/dist/core/progress.js +212 -0
- package/dist/core/progress.js.map +1 -0
- package/dist/core/recipes.d.ts +138 -0
- package/dist/core/recipes.d.ts.map +1 -0
- package/dist/core/recipes.js +242 -0
- package/dist/core/recipes.js.map +1 -0
- package/dist/core/watcher.d.ts +61 -1
- package/dist/core/watcher.d.ts.map +1 -1
- package/dist/core/watcher.js +99 -2
- package/dist/core/watcher.js.map +1 -1
- package/dist/recipes/aws-whats-new.yaml +87 -0
- package/dist/recipes/dev-to.yaml +40 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/index.js.map +1 -1
- package/dist/schemas/recipe.d.ts +127 -0
- package/dist/schemas/recipe.d.ts.map +1 -0
- package/dist/schemas/recipe.js +57 -0
- package/dist/schemas/recipe.js.map +1 -0
- package/dist/schemas/source.d.ts +222 -0
- package/dist/schemas/source.d.ts.map +1 -1
- package/dist/schemas/source.js +234 -0
- package/dist/schemas/source.js.map +1 -1
- package/dist/templates/agents/AGENTS.md +33 -3
- package/dist/templates/feedradar.md +23 -8
- package/dist/templates/workflows/combined.template.yaml.tmpl +110 -0
- package/dist/templates/workflows/watch.template.yaml.tmpl +103 -0
- package/package.json +1 -2
package/dist/cli/source.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { access, readdir, readFile, unlink, writeFile } from "node:fs/promises";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
|
|
4
|
+
import { createProgressReporter } from "../core/progress.js";
|
|
5
|
+
import { listRecipes, loadRecipe, mergeRecipeWithOverrides, } from "../core/recipes.js";
|
|
4
6
|
import { loadSourceState } from "../core/state.js";
|
|
5
7
|
import { watchRun } from "../core/watcher.js";
|
|
6
8
|
import { SourceKindSchema, SourceSchema, SourceSelectorsSchema } from "../schemas/source.js";
|
|
@@ -40,6 +42,35 @@ function splitCsv(value) {
|
|
|
40
42
|
.map((s) => s.trim())
|
|
41
43
|
.filter((s) => s.length > 0);
|
|
42
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Parse an `--<option> N` integer argument with a configurable minimum.
|
|
47
|
+
*
|
|
48
|
+
* Used for `--pagination-page-size` / `--max-pages` / etc. where Zod will
|
|
49
|
+
* also validate the resulting Source object, but throwing here means the
|
|
50
|
+
* user sees a single targeted error referencing the flag name they typed
|
|
51
|
+
* instead of a deeper schema-path message.
|
|
52
|
+
*
|
|
53
|
+
* `min` defaults to 1 because every json-api pagination integer the schema
|
|
54
|
+
* accepts is `z.number().int().positive()`; the `--pagination-start` flag
|
|
55
|
+
* passes `min: 0` for offset/page indices that legitimately begin at 0.
|
|
56
|
+
*/
|
|
57
|
+
function parseIntFlag(flag, raw, min) {
|
|
58
|
+
if (raw === undefined)
|
|
59
|
+
throw new Error(`option ${flag} requires a value`);
|
|
60
|
+
const n = Number(raw);
|
|
61
|
+
if (!Number.isInteger(n) || n < min) {
|
|
62
|
+
throw new Error(`option ${flag} expects an integer >= ${min}, got '${raw}'`);
|
|
63
|
+
}
|
|
64
|
+
return n;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Pagination strategies accepted by `--pagination-strategy` on
|
|
68
|
+
* `radar source add --kind json-api`. Mirrors `SourcePaginationSchema.type`
|
|
69
|
+
* — kept as a local const so the CLI validator emits a user-friendly enum
|
|
70
|
+
* list before Zod runs (Zod surfaces the same set, but a leading "did you
|
|
71
|
+
* mean …?" message at the CLI layer reads better in shell context).
|
|
72
|
+
*/
|
|
73
|
+
const PAGINATION_STRATEGIES = ["page", "offset", "cursor", "link-header", "token", "none"];
|
|
43
74
|
/**
|
|
44
75
|
* Parse `source add` flags.
|
|
45
76
|
*
|
|
@@ -62,6 +93,13 @@ function parseAddArgs(args) {
|
|
|
62
93
|
out.url = args[++i];
|
|
63
94
|
continue;
|
|
64
95
|
}
|
|
96
|
+
if (a === "--recipe") {
|
|
97
|
+
const value = args[++i];
|
|
98
|
+
if (value === undefined)
|
|
99
|
+
throw new Error(`option ${a} requires a value`);
|
|
100
|
+
out.recipe = value;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
65
103
|
if (a === "--name") {
|
|
66
104
|
out.name = args[++i];
|
|
67
105
|
continue;
|
|
@@ -94,6 +132,56 @@ function parseAddArgs(args) {
|
|
|
94
132
|
out.selectors[field] = value;
|
|
95
133
|
continue;
|
|
96
134
|
}
|
|
135
|
+
if (a === "--pagination-strategy") {
|
|
136
|
+
const value = args[++i];
|
|
137
|
+
if (value === undefined)
|
|
138
|
+
throw new Error(`option ${a} requires a value`);
|
|
139
|
+
if (!PAGINATION_STRATEGIES.includes(value)) {
|
|
140
|
+
throw new Error(`option --pagination-strategy expects one of: ${PAGINATION_STRATEGIES.join(" | ")}, got '${value}'`);
|
|
141
|
+
}
|
|
142
|
+
out.paginationStrategy = value;
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (a === "--pagination-param") {
|
|
146
|
+
const value = args[++i];
|
|
147
|
+
if (value === undefined)
|
|
148
|
+
throw new Error(`option ${a} requires a value`);
|
|
149
|
+
out.paginationParam = value;
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (a === "--pagination-start") {
|
|
153
|
+
out.paginationStart = parseIntFlag(a, args[++i], 0);
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
if (a === "--page-size") {
|
|
157
|
+
out.paginationPageSize = parseIntFlag(a, args[++i], 1);
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
if (a === "--page-size-param") {
|
|
161
|
+
const value = args[++i];
|
|
162
|
+
if (value === undefined)
|
|
163
|
+
throw new Error(`option ${a} requires a value`);
|
|
164
|
+
out.paginationPageSizeParam = value;
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
if (a === "--max-pages") {
|
|
168
|
+
out.paginationMaxPages = parseIntFlag(a, args[++i], 1);
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (a === "--next-cursor-path") {
|
|
172
|
+
const value = args[++i];
|
|
173
|
+
if (value === undefined)
|
|
174
|
+
throw new Error(`option ${a} requires a value`);
|
|
175
|
+
out.paginationNextCursorPath = value;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (a === "--total-path") {
|
|
179
|
+
const value = args[++i];
|
|
180
|
+
if (value === undefined)
|
|
181
|
+
throw new Error(`option ${a} requires a value`);
|
|
182
|
+
out.paginationTotalPath = value;
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
97
185
|
if (a?.startsWith("--")) {
|
|
98
186
|
throw new Error(`unknown option: ${a}`);
|
|
99
187
|
}
|
|
@@ -154,6 +242,14 @@ function parseTestArgs(args) {
|
|
|
154
242
|
out.showContent = true;
|
|
155
243
|
continue;
|
|
156
244
|
}
|
|
245
|
+
if (a === "--verbose" || a === "-v") {
|
|
246
|
+
out.verbose = true;
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
if (a === "--quiet" || a === "-q") {
|
|
250
|
+
out.quiet = true;
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
157
253
|
if (a?.startsWith("--")) {
|
|
158
254
|
throw new Error(`unknown option: ${a}`);
|
|
159
255
|
}
|
|
@@ -163,6 +259,9 @@ function parseTestArgs(args) {
|
|
|
163
259
|
}
|
|
164
260
|
throw new Error(`unexpected positional argument: ${a}`);
|
|
165
261
|
}
|
|
262
|
+
if (out.verbose && out.quiet) {
|
|
263
|
+
throw new Error("--verbose and --quiet are mutually exclusive");
|
|
264
|
+
}
|
|
166
265
|
return out;
|
|
167
266
|
}
|
|
168
267
|
function parseRemoveArgs(args) {
|
|
@@ -185,10 +284,15 @@ function parseRemoveArgs(args) {
|
|
|
185
284
|
}
|
|
186
285
|
function printAddHelp(log) {
|
|
187
286
|
log("Usage: radar source add <id> --kind <kind> --url <url> [options]");
|
|
287
|
+
log(" radar source add <id> --recipe <name> [overrides]");
|
|
188
288
|
log("");
|
|
189
289
|
log("Options:");
|
|
190
|
-
log(" --kind <kind> rss | html | html-js | github-releases | npm-registry");
|
|
290
|
+
log(" --kind <kind> rss | html | html-js | github-releases | npm-registry | json-feed | json-api");
|
|
191
291
|
log(" --url <url> fetch target URL");
|
|
292
|
+
log(" --recipe <name> apply a bundled recipe (see `radar source recipes`).");
|
|
293
|
+
log(" Mutually exclusive with --kind / --url / --selector-* /");
|
|
294
|
+
log(" --pagination-*; --name / --tags / --keywords /");
|
|
295
|
+
log(" --exclude-keywords still override the recipe defaults.");
|
|
192
296
|
log(" --name <name> display name (defaults to <id>)");
|
|
193
297
|
log(" --tags <a,b> comma-separated tags");
|
|
194
298
|
log(" --keywords <a,b> comma-separated include keywords");
|
|
@@ -199,6 +303,24 @@ function printAddHelp(log) {
|
|
|
199
303
|
log(" For kind=html-js, selectors evaluate against the post-JS DOM.");
|
|
200
304
|
log(" The `js:` block (waitFor / timeout / userAgent) cannot be set");
|
|
201
305
|
log(" via flags; edit sources/<id>.yaml after add. See ADR-0010.");
|
|
306
|
+
log("");
|
|
307
|
+
log(" For kind=json-api (ADR-0012 / #174):");
|
|
308
|
+
log(" --pagination-strategy <s> page | offset | cursor | link-header | token | none (default: page)");
|
|
309
|
+
log(" --pagination-param <name> query param name for the page/offset/cursor value");
|
|
310
|
+
log(" --pagination-start N initial page/offset value (default: 0)");
|
|
311
|
+
log(" --page-size N items per page");
|
|
312
|
+
log(" --page-size-param <name> query param name for the page-size value");
|
|
313
|
+
log(" --max-pages N hard cap on pages traversed (default: 20)");
|
|
314
|
+
log(" --next-cursor-path <jp> JSONPath-lite to the next-cursor value (cursor/token strategy)");
|
|
315
|
+
log(" --total-path <jp> JSONPath-lite to the total-count value (backfill early-stop hint)");
|
|
316
|
+
log("");
|
|
317
|
+
log(" Selector fields (`jsonSelectors.*`) for kind=json-api cannot be set via flags;");
|
|
318
|
+
log(" the schema has a default fallback chain (items / title / link / publishedAt / summary),");
|
|
319
|
+
log(" so simple APIs work without selectors. Edit sources/<id>.yaml directly when explicit");
|
|
320
|
+
log(" selectors are needed (nested fields, non-standard envelopes).");
|
|
321
|
+
log("");
|
|
322
|
+
log(" Facet sweep (e.g. year-by-year sweep) cannot be configured via flags; see ADR-0017");
|
|
323
|
+
log(" and bundle the year sweep through `--recipe aws-whats-new`. Recipe-only structural field.");
|
|
202
324
|
}
|
|
203
325
|
function printListHelp(log) {
|
|
204
326
|
log("Usage: radar source list [--enabled-only] [-v|--verbose]");
|
|
@@ -222,16 +344,42 @@ function printTestHelp(log) {
|
|
|
222
344
|
log("state/ and items/ are not touched (no persistence). Useful for tuning");
|
|
223
345
|
log("keywords when adding a new source.");
|
|
224
346
|
log("");
|
|
347
|
+
log("For kind=json-api (ADR-0012 / #174), `source test` fetches PAGE 0 ONLY.");
|
|
348
|
+
log("Pagination is NOT walked even when the recipe declares multiple pages —");
|
|
349
|
+
log("`--limit N` caps how many matched items are PRINTED, it does not change");
|
|
350
|
+
log("the page budget. Use `radar watch run --backfill` for full-history ingest.");
|
|
351
|
+
log("Page 0's `Link` header / `nextCursor` extraction is surfaced via");
|
|
352
|
+
log("`--show-content` for pagination tuning without state mutation.");
|
|
353
|
+
log("");
|
|
225
354
|
log("Options:");
|
|
226
355
|
log(" --limit N Maximum number of matched items to print (default 10)");
|
|
227
|
-
log(" --show-content Also print the first 200
|
|
356
|
+
log(" --show-content Also print the first 200 chars of each item's body, plus");
|
|
357
|
+
log(" (kind=json-api) the selector adoption table and pagination");
|
|
358
|
+
log(" preview (would-be next URL / Link header / nextCursor).");
|
|
359
|
+
log(" -v, --verbose Enable progress-reporter raw() pass-through (adapter stdout).");
|
|
360
|
+
log(" Most useful with kind=html-js (Playwright phase markers).");
|
|
361
|
+
log(" -q, --quiet Suppress the progress reporter entirely. RADAR_NO_PROGRESS=1");
|
|
362
|
+
log(" has the same effect.");
|
|
363
|
+
}
|
|
364
|
+
function printRecipesHelp(log) {
|
|
365
|
+
log("Usage: radar source recipes");
|
|
366
|
+
log("");
|
|
367
|
+
log("List bundled recipes (recipes/*.yaml in the radar package — ADR-0012 §D3).");
|
|
368
|
+
log("Each recipe can be applied via:");
|
|
369
|
+
log(" radar source add <id> --recipe <name> [--keywords <kw>] [--tags <t>] [--name <display>]");
|
|
370
|
+
log("");
|
|
371
|
+
log("Bundled recipes ship with the radar npm package; user-authored recipes are");
|
|
372
|
+
log("not yet supported. To add a new bundled recipe, contribute a YAML to the");
|
|
373
|
+
log("radar repo's recipes/ directory.");
|
|
228
374
|
}
|
|
229
375
|
function printSourceHelp(log) {
|
|
230
|
-
log("Usage: radar source <add|list|remove|test> [...]");
|
|
376
|
+
log("Usage: radar source <add|list|recipes|remove|test> [...]");
|
|
231
377
|
log("");
|
|
232
378
|
log("Subcommands:");
|
|
233
379
|
log(" add <id> --kind <kind> --url <url> [...]");
|
|
380
|
+
log(" add <id> --recipe <name> [--keywords <kw>] [--tags <t>] [--name <display>]");
|
|
234
381
|
log(" list [--enabled-only]");
|
|
382
|
+
log(" recipes");
|
|
235
383
|
log(" remove <id>");
|
|
236
384
|
log(" test <id> [--limit N] [--show-content]");
|
|
237
385
|
}
|
|
@@ -269,6 +417,16 @@ export async function addSource(args, options = {}) {
|
|
|
269
417
|
error(`source add: invalid <id> '${parsed.id}' (must match [A-Za-z0-9][A-Za-z0-9._-]*)`);
|
|
270
418
|
return 2;
|
|
271
419
|
}
|
|
420
|
+
// `--recipe <name>` short-circuits the flag-based composition: the
|
|
421
|
+
// recipe supplies kind / url / pagination / selectors / etc., and only
|
|
422
|
+
// a narrow whitelist of CLI flags is allowed to override
|
|
423
|
+
// (`--name` / `--tags` / `--keywords` / `--exclude-keywords`).
|
|
424
|
+
// Everything else (incl. `--kind` / `--url` / `--selector-*` /
|
|
425
|
+
// `--pagination-*`) is rejected so the user gets an immediate, targeted
|
|
426
|
+
// error instead of silently-ignored flags. ADR-0012 §D3.
|
|
427
|
+
if (parsed.recipe !== undefined) {
|
|
428
|
+
return addSourceFromRecipe(parsed, cwd, options, log, warn, error);
|
|
429
|
+
}
|
|
272
430
|
if (!parsed.kind) {
|
|
273
431
|
error("source add: --kind is required");
|
|
274
432
|
return 2;
|
|
@@ -279,7 +437,7 @@ export async function addSource(args, options = {}) {
|
|
|
279
437
|
}
|
|
280
438
|
const kindResult = SourceKindSchema.safeParse(parsed.kind);
|
|
281
439
|
if (!kindResult.success) {
|
|
282
|
-
error(`source add: invalid --kind '${parsed.kind}' (expected: rss | html | html-js | github-releases | npm-registry)`);
|
|
440
|
+
error(`source add: invalid --kind '${parsed.kind}' (expected: rss | html | html-js | github-releases | npm-registry | json-feed | json-api)`);
|
|
283
441
|
return 2;
|
|
284
442
|
}
|
|
285
443
|
// Compose the object before schema validation so url-format errors et al.
|
|
@@ -318,6 +476,52 @@ export async function addSource(args, options = {}) {
|
|
|
318
476
|
}
|
|
319
477
|
candidate.selectors = selectorsResult.data;
|
|
320
478
|
}
|
|
479
|
+
// For kind=json-api, build the `pagination:` block from --pagination-*
|
|
480
|
+
// flags. The schema requires `pagination` for this kind; if the user
|
|
481
|
+
// omitted --pagination-strategy entirely we default to `page` (the most
|
|
482
|
+
// common shape for AWS What's New / dev.to / Anthropic news). We do NOT
|
|
483
|
+
// generate a `jsonSelectors:` block — the default fallback chain covers
|
|
484
|
+
// simple APIs, and recipe authors edit the YAML directly when explicit
|
|
485
|
+
// selectors are needed (the field count makes flag-based mutation
|
|
486
|
+
// impractical).
|
|
487
|
+
if (kindResult.data === "json-api") {
|
|
488
|
+
const strategy = parsed.paginationStrategy ?? "page";
|
|
489
|
+
const pagination = { type: strategy };
|
|
490
|
+
if (parsed.paginationParam !== undefined)
|
|
491
|
+
pagination.param = parsed.paginationParam;
|
|
492
|
+
if (parsed.paginationStart !== undefined)
|
|
493
|
+
pagination.start = parsed.paginationStart;
|
|
494
|
+
if (parsed.paginationPageSize !== undefined) {
|
|
495
|
+
pagination.pageSize = parsed.paginationPageSize;
|
|
496
|
+
}
|
|
497
|
+
if (parsed.paginationPageSizeParam !== undefined) {
|
|
498
|
+
pagination.pageSizeParam = parsed.paginationPageSizeParam;
|
|
499
|
+
}
|
|
500
|
+
if (parsed.paginationMaxPages !== undefined) {
|
|
501
|
+
pagination.maxPages = parsed.paginationMaxPages;
|
|
502
|
+
}
|
|
503
|
+
if (parsed.paginationNextCursorPath !== undefined) {
|
|
504
|
+
pagination.nextCursorPath = parsed.paginationNextCursorPath;
|
|
505
|
+
}
|
|
506
|
+
if (parsed.paginationTotalPath !== undefined) {
|
|
507
|
+
pagination.totalPath = parsed.paginationTotalPath;
|
|
508
|
+
}
|
|
509
|
+
candidate.pagination = pagination;
|
|
510
|
+
}
|
|
511
|
+
else if (parsed.paginationStrategy !== undefined ||
|
|
512
|
+
parsed.paginationParam !== undefined ||
|
|
513
|
+
parsed.paginationStart !== undefined ||
|
|
514
|
+
parsed.paginationPageSize !== undefined ||
|
|
515
|
+
parsed.paginationPageSizeParam !== undefined ||
|
|
516
|
+
parsed.paginationMaxPages !== undefined ||
|
|
517
|
+
parsed.paginationNextCursorPath !== undefined ||
|
|
518
|
+
parsed.paginationTotalPath !== undefined) {
|
|
519
|
+
// Reject pagination flags on non-json-api kinds early so the user sees
|
|
520
|
+
// a targeted hint instead of a deep schema refinement error ("pagination
|
|
521
|
+
// is required when kind is 'json-api'" makes no sense for `kind: rss`).
|
|
522
|
+
error(`source add: --pagination-* flags are only valid with --kind json-api (got --kind '${kindResult.data}')`);
|
|
523
|
+
return 2;
|
|
524
|
+
}
|
|
321
525
|
const validated = SourceSchema.safeParse(candidate);
|
|
322
526
|
if (!validated.success) {
|
|
323
527
|
const issues = validated.error.issues.map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`);
|
|
@@ -346,6 +550,104 @@ export async function addSource(args, options = {}) {
|
|
|
346
550
|
}
|
|
347
551
|
return 0;
|
|
348
552
|
}
|
|
553
|
+
/**
|
|
554
|
+
* Apply a bundled recipe (`--recipe <name>`) to produce a new
|
|
555
|
+
* `sources/<id>.yaml` (ADR-0012 §D3, strategy A).
|
|
556
|
+
*
|
|
557
|
+
* Validation discipline:
|
|
558
|
+
*
|
|
559
|
+
* - The recipe supplies `kind` / `url` / structural fields. Re-passing
|
|
560
|
+
* them as CLI flags is rejected to prevent the "recipe says one thing,
|
|
561
|
+
* flag says another, who wins?" footgun. Only the explicit override
|
|
562
|
+
* whitelist (`--name` / `--tags` / `--keywords` /
|
|
563
|
+
* `--exclude-keywords`) is honoured.
|
|
564
|
+
* - `--selector-*` / `--pagination-*` are also rejected on the recipe
|
|
565
|
+
* path — these belong to the recipe author. Users who want to deviate
|
|
566
|
+
* structurally edit `sources/<id>.yaml` after generation, same as for
|
|
567
|
+
* any other source.
|
|
568
|
+
*/
|
|
569
|
+
async function addSourceFromRecipe(parsed, cwd, options, log, warn, error) {
|
|
570
|
+
const recipeName = parsed.recipe;
|
|
571
|
+
// Defensive — should be guaranteed by the caller, but `parsed.recipe`
|
|
572
|
+
// is typed as `string | undefined` so a quick narrow here keeps the
|
|
573
|
+
// rest of the function tidy.
|
|
574
|
+
if (recipeName === undefined) {
|
|
575
|
+
error("source add: --recipe is required (internal dispatch error)");
|
|
576
|
+
return 2;
|
|
577
|
+
}
|
|
578
|
+
// Reject flags that the recipe owns. Surfacing each forbidden flag
|
|
579
|
+
// individually beats a generic "incompatible flags" message because
|
|
580
|
+
// the user sees exactly what to remove.
|
|
581
|
+
const forbidden = [];
|
|
582
|
+
if (parsed.kind !== undefined)
|
|
583
|
+
forbidden.push("--kind");
|
|
584
|
+
if (parsed.url !== undefined)
|
|
585
|
+
forbidden.push("--url");
|
|
586
|
+
if (parsed.selectors !== undefined)
|
|
587
|
+
forbidden.push("--selector-<field>");
|
|
588
|
+
if (parsed.paginationStrategy !== undefined ||
|
|
589
|
+
parsed.paginationParam !== undefined ||
|
|
590
|
+
parsed.paginationStart !== undefined ||
|
|
591
|
+
parsed.paginationPageSize !== undefined ||
|
|
592
|
+
parsed.paginationPageSizeParam !== undefined ||
|
|
593
|
+
parsed.paginationMaxPages !== undefined ||
|
|
594
|
+
parsed.paginationNextCursorPath !== undefined ||
|
|
595
|
+
parsed.paginationTotalPath !== undefined) {
|
|
596
|
+
forbidden.push("--pagination-*");
|
|
597
|
+
}
|
|
598
|
+
if (forbidden.length > 0) {
|
|
599
|
+
error(`source add: --recipe '${recipeName}' supplies kind / url / structural fields; the following flags are not allowed with --recipe: ${forbidden.join(", ")}`);
|
|
600
|
+
return 2;
|
|
601
|
+
}
|
|
602
|
+
let loaded;
|
|
603
|
+
try {
|
|
604
|
+
loaded = await loadRecipe(recipeName, { recipesRoot: options.recipesRoot });
|
|
605
|
+
}
|
|
606
|
+
catch (e) {
|
|
607
|
+
error(`source add: ${e instanceof Error ? e.message : String(e)}`);
|
|
608
|
+
return 1;
|
|
609
|
+
}
|
|
610
|
+
const candidate = mergeRecipeWithOverrides(loaded.recipe, {
|
|
611
|
+
// `parsed.id` is asserted non-undefined by the caller before this
|
|
612
|
+
// function is reached, but TypeScript cannot see that across the
|
|
613
|
+
// dispatch boundary. The non-null assertion mirrors what the
|
|
614
|
+
// flag-based path expects from the same guard.
|
|
615
|
+
// biome-ignore lint/style/noNonNullAssertion: caller-enforced invariant
|
|
616
|
+
id: parsed.id,
|
|
617
|
+
name: parsed.name,
|
|
618
|
+
tags: parsed.tags,
|
|
619
|
+
keywords: parsed.keywords,
|
|
620
|
+
excludeKeywords: parsed.excludeKeywords,
|
|
621
|
+
});
|
|
622
|
+
const validated = SourceSchema.safeParse(candidate);
|
|
623
|
+
if (!validated.success) {
|
|
624
|
+
// A Zod failure on a recipe-derived candidate means the recipe
|
|
625
|
+
// itself is malformed (or, more rarely, an override produced an
|
|
626
|
+
// illegal combination). Surface every issue verbatim so recipe
|
|
627
|
+
// authors and end users can both diagnose.
|
|
628
|
+
const issues = validated.error.issues.map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`);
|
|
629
|
+
error(`source add: recipe '${recipeName}' produced an invalid source`);
|
|
630
|
+
for (const issue of issues) {
|
|
631
|
+
error(` - ${issue}`);
|
|
632
|
+
}
|
|
633
|
+
return 2;
|
|
634
|
+
}
|
|
635
|
+
const file = sourceFile(cwd, validated.data.id);
|
|
636
|
+
if (await pathExists(file)) {
|
|
637
|
+
error(`source add: '${validated.data.id}' already exists (sources/${validated.data.id}.yaml)`);
|
|
638
|
+
return 1;
|
|
639
|
+
}
|
|
640
|
+
await writeFile(file, stringifyYaml(validated.data), "utf8");
|
|
641
|
+
log(`source add: created sources/${validated.data.id}.yaml from recipe '${recipeName}'`);
|
|
642
|
+
// Same firehose-guard hint as the flag-based path: an empty
|
|
643
|
+
// include-keyword list silently drops every fetched item, which
|
|
644
|
+
// surprises users when they thought the recipe came with sensible
|
|
645
|
+
// defaults.
|
|
646
|
+
if (validated.data.filters.keywords.length === 0) {
|
|
647
|
+
warn(`source add: warning — '${validated.data.id}' has no keywords; all fetched items will be filtered out. Re-add with --keywords or edit sources/${validated.data.id}.yaml to start ingesting.`);
|
|
648
|
+
}
|
|
649
|
+
return 0;
|
|
650
|
+
}
|
|
349
651
|
/**
|
|
350
652
|
* Load and validate a single `sources/<id>.yaml` file. Returns `null` and
|
|
351
653
|
* reports through `onError` when the file is malformed, so `list` can keep
|
|
@@ -592,6 +894,13 @@ export async function testSource(args, options = {}) {
|
|
|
592
894
|
return 1;
|
|
593
895
|
}
|
|
594
896
|
const limit = parsed.limit ?? 10;
|
|
897
|
+
// Build the progress reporter (#198). `source test` runs exactly one
|
|
898
|
+
// source so the watcher heuristic only enables narration when the kind
|
|
899
|
+
// is `html-js` / `json-api`; for rss / html / npm-registry / etc. the
|
|
900
|
+
// legacy 1-line summary remains the only output. Tests pin the level
|
|
901
|
+
// explicitly via `--quiet` or `RADAR_NO_PROGRESS=1`.
|
|
902
|
+
const level = parsed.quiet ? "quiet" : parsed.verbose ? "verbose" : "normal";
|
|
903
|
+
const progress = createProgressReporter({ level });
|
|
595
904
|
let result;
|
|
596
905
|
try {
|
|
597
906
|
result = await watchRun({
|
|
@@ -602,6 +911,7 @@ export async function testSource(args, options = {}) {
|
|
|
602
911
|
log,
|
|
603
912
|
warn,
|
|
604
913
|
error,
|
|
914
|
+
progress,
|
|
605
915
|
});
|
|
606
916
|
}
|
|
607
917
|
catch (e) {
|
|
@@ -620,6 +930,41 @@ export async function testSource(args, options = {}) {
|
|
|
620
930
|
log("");
|
|
621
931
|
log(`source test: ${parsed.id}`);
|
|
622
932
|
log(` fetched: ${fetched} / filtered: ${filtered} / matched: ${matched.length}`);
|
|
933
|
+
// Render the adapter diag for `kind: json-api` when --show-content is on.
|
|
934
|
+
// The diag block is intentionally gated behind --show-content so the
|
|
935
|
+
// default `source test` output (used by scripts that just want a quick
|
|
936
|
+
// matched-items dump) stays narrow. Adapters that do not return diag
|
|
937
|
+
// simply skip this block.
|
|
938
|
+
if (parsed.showContent) {
|
|
939
|
+
const diag = result.diag[parsed.id];
|
|
940
|
+
if (diag) {
|
|
941
|
+
if (diag.selectorAdoption) {
|
|
942
|
+
log("");
|
|
943
|
+
log(" selector adoption:");
|
|
944
|
+
for (const [field, path] of Object.entries(diag.selectorAdoption)) {
|
|
945
|
+
if (path === null) {
|
|
946
|
+
log(` ${field}: (no candidate matched)`);
|
|
947
|
+
}
|
|
948
|
+
else {
|
|
949
|
+
log(` ${field} ← ${path} を採用`);
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
if (diag.paginationPreview) {
|
|
954
|
+
const p = diag.paginationPreview;
|
|
955
|
+
log("");
|
|
956
|
+
log(" pagination preview (page 0 only — state not mutated):");
|
|
957
|
+
log(` strategy: ${p.strategy}`);
|
|
958
|
+
log(` nextUrl: ${p.nextUrl ?? "(end of pagination)"}`);
|
|
959
|
+
if (p.linkHeaderNext !== undefined) {
|
|
960
|
+
log(` Link rel=next: ${p.linkHeaderNext ?? "(absent)"}`);
|
|
961
|
+
}
|
|
962
|
+
if (p.nextCursor !== undefined) {
|
|
963
|
+
log(` nextCursor: ${p.nextCursor ?? "(absent)"}`);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
}
|
|
623
968
|
if (matched.length === 0) {
|
|
624
969
|
log(" (no matched items)");
|
|
625
970
|
return 0;
|
|
@@ -650,12 +995,86 @@ export async function testSource(args, options = {}) {
|
|
|
650
995
|
}
|
|
651
996
|
return 0;
|
|
652
997
|
}
|
|
998
|
+
/**
|
|
999
|
+
* Implementation of `source recipes` — list bundled recipes.
|
|
1000
|
+
*
|
|
1001
|
+
* Prints a fixed-width table (NAME / KIND / DESCRIPTION) for each
|
|
1002
|
+
* `recipes/<name>.yaml` that loads cleanly. Recipes that fail to parse
|
|
1003
|
+
* are appended after the valid set with their error so the user can fix
|
|
1004
|
+
* (or report) the recipe without losing the rest of the listing.
|
|
1005
|
+
*
|
|
1006
|
+
* When the bundle is empty or absent (the bootstrap state before #178
|
|
1007
|
+
* ships the actual recipe content), a friendly "no recipes" message is
|
|
1008
|
+
* printed and exit code 0 is returned — the CLI is functional even when
|
|
1009
|
+
* the recipe library is empty.
|
|
1010
|
+
*/
|
|
1011
|
+
export async function recipesSubcommand(args, options = {}) {
|
|
1012
|
+
const log = options.io?.log ?? ((m) => console.log(m));
|
|
1013
|
+
const error = options.io?.error ?? ((m) => console.error(m));
|
|
1014
|
+
// The only flag accepted today is `-h` / `--help`. Keep the parser
|
|
1015
|
+
// tiny rather than introducing a typed args struct for a single
|
|
1016
|
+
// option — easier to extend if/when filters land.
|
|
1017
|
+
for (const a of args) {
|
|
1018
|
+
if (a === "-h" || a === "--help") {
|
|
1019
|
+
printRecipesHelp(log);
|
|
1020
|
+
return 0;
|
|
1021
|
+
}
|
|
1022
|
+
if (a.startsWith("--")) {
|
|
1023
|
+
error(`source recipes: unknown option: ${a}`);
|
|
1024
|
+
return 2;
|
|
1025
|
+
}
|
|
1026
|
+
error(`source recipes: unexpected positional argument: ${a}`);
|
|
1027
|
+
return 2;
|
|
1028
|
+
}
|
|
1029
|
+
let entries;
|
|
1030
|
+
try {
|
|
1031
|
+
entries = await listRecipes({ recipesRoot: options.recipesRoot });
|
|
1032
|
+
}
|
|
1033
|
+
catch (e) {
|
|
1034
|
+
error(`source recipes: ${e instanceof Error ? e.message : String(e)}`);
|
|
1035
|
+
return 1;
|
|
1036
|
+
}
|
|
1037
|
+
if (entries.length === 0) {
|
|
1038
|
+
log("source recipes: no recipes bundled (recipes/ is empty or absent)");
|
|
1039
|
+
return 0;
|
|
1040
|
+
}
|
|
1041
|
+
const valid = entries.filter((e) => e.recipe !== null);
|
|
1042
|
+
const invalid = entries.filter((e) => e.recipe === null);
|
|
1043
|
+
if (valid.length > 0) {
|
|
1044
|
+
const nameWidth = Math.max(4, ...valid.map((e) => e.name.length));
|
|
1045
|
+
const kindWidth = Math.max(4, ...valid.map((e) => (e.recipe ? e.recipe.kind.length : 0)));
|
|
1046
|
+
log(`${pad("NAME", nameWidth)} ${pad("KIND", kindWidth)} DESCRIPTION`);
|
|
1047
|
+
for (const e of valid) {
|
|
1048
|
+
if (!e.recipe)
|
|
1049
|
+
continue;
|
|
1050
|
+
const desc = e.recipe.description ?? "";
|
|
1051
|
+
log(`${pad(e.name, nameWidth)} ${pad(e.recipe.kind, kindWidth)} ${desc}`);
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
else {
|
|
1055
|
+
log("source recipes: no valid recipes found (all bundled entries failed to load)");
|
|
1056
|
+
}
|
|
1057
|
+
if (invalid.length > 0) {
|
|
1058
|
+
log("");
|
|
1059
|
+
log("Recipes with errors:");
|
|
1060
|
+
for (const e of invalid) {
|
|
1061
|
+
log(` ${e.name}: ${e.error ?? "(unknown error)"}`);
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
log("");
|
|
1065
|
+
log("Apply a recipe with:");
|
|
1066
|
+
log(" radar source add <id> --recipe <name> [--keywords <kw>] [--tags <t>] [--name <display>]");
|
|
1067
|
+
// Returning 0 even when individual recipes have errors keeps the
|
|
1068
|
+
// listing useful in CI: a single malformed recipe should not break
|
|
1069
|
+
// the discovery command for the rest of the bundle.
|
|
1070
|
+
return 0;
|
|
1071
|
+
}
|
|
653
1072
|
/**
|
|
654
1073
|
* Top-level dispatcher for `radar source <subcommand>`.
|
|
655
1074
|
*
|
|
656
1075
|
* Sub-commands are kept as named functions (addSource/listSources/removeSource/
|
|
657
|
-
* testSource) so tests can call them directly with injected
|
|
658
|
-
* spawning the full CLI.
|
|
1076
|
+
* testSource/recipesSubcommand) so tests can call them directly with injected
|
|
1077
|
+
* IO sinks without spawning the full CLI.
|
|
659
1078
|
*/
|
|
660
1079
|
export async function runSource(args, options = {}) {
|
|
661
1080
|
const log = options.io?.log ?? ((m) => console.log(m));
|
|
@@ -670,6 +1089,8 @@ export async function runSource(args, options = {}) {
|
|
|
670
1089
|
return addSource(rest, options);
|
|
671
1090
|
case "list":
|
|
672
1091
|
return listSources(rest, options);
|
|
1092
|
+
case "recipes":
|
|
1093
|
+
return recipesSubcommand(rest, options);
|
|
673
1094
|
case "remove":
|
|
674
1095
|
return removeSource(rest, options);
|
|
675
1096
|
case "test":
|
|
@@ -682,7 +1103,7 @@ export async function runSource(args, options = {}) {
|
|
|
682
1103
|
}
|
|
683
1104
|
export const sourceCommand = {
|
|
684
1105
|
name: "source",
|
|
685
|
-
summary: "Manage feed sources (add | list | remove | test)",
|
|
1106
|
+
summary: "Manage feed sources (add | list | recipes | remove | test)",
|
|
686
1107
|
run: (args) => runSource(args),
|
|
687
1108
|
};
|
|
688
1109
|
//# sourceMappingURL=source.js.map
|