@ozzylabs/feedradar 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +114 -0
- package/README.md +63 -53
- package/dist/cli/doctor.d.ts +83 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +260 -0
- package/dist/cli/doctor.js.map +1 -0
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +2 -2
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/source.d.ts.map +1 -1
- package/dist/cli/source.js +6 -3
- package/dist/cli/source.js.map +1 -1
- package/dist/cli/watch.d.ts +16 -0
- package/dist/cli/watch.d.ts.map +1 -1
- package/dist/cli/watch.js +3 -0
- package/dist/cli/watch.js.map +1 -1
- package/dist/core/feeds/_html-common.d.ts +30 -0
- package/dist/core/feeds/_html-common.d.ts.map +1 -0
- package/dist/core/feeds/_html-common.js +192 -0
- package/dist/core/feeds/_html-common.js.map +1 -0
- package/dist/core/feeds/html-js.d.ts +50 -0
- package/dist/core/feeds/html-js.d.ts.map +1 -0
- package/dist/core/feeds/html-js.js +135 -0
- package/dist/core/feeds/html-js.js.map +1 -0
- package/dist/core/feeds/html.d.ts +1 -7
- package/dist/core/feeds/html.d.ts.map +1 -1
- package/dist/core/feeds/html.js +5 -180
- package/dist/core/feeds/html.js.map +1 -1
- package/dist/core/feeds/index.d.ts.map +1 -1
- package/dist/core/feeds/index.js +2 -0
- package/dist/core/feeds/index.js.map +1 -1
- package/dist/core/playwright-check.d.ts +134 -0
- package/dist/core/playwright-check.d.ts.map +1 -0
- package/dist/core/playwright-check.js +98 -0
- package/dist/core/playwright-check.js.map +1 -0
- package/dist/core/watcher.d.ts +17 -0
- package/dist/core/watcher.d.ts.map +1 -1
- package/dist/core/watcher.js +59 -0
- package/dist/core/watcher.js.map +1 -1
- package/dist/index.js +0 -0
- package/dist/schemas/source.d.ts +42 -0
- package/dist/schemas/source.d.ts.map +1 -1
- package/dist/schemas/source.js +42 -7
- package/dist/schemas/source.js.map +1 -1
- package/dist/templates/agents/AGENTS.md +2 -2
- package/dist/templates/feedradar.md +2 -2
- package/package.json +11 -1
package/dist/core/watcher.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Item, Source, SourceState } from "../schemas/index.js";
|
|
2
2
|
import type { FeedAdapter, FetchLike } from "./feeds/index.js";
|
|
3
|
+
import { installChromium, type ProbeOptions } from "./playwright-check.js";
|
|
3
4
|
export interface WorkspacePaths {
|
|
4
5
|
/** Workspace root; defaults to process.cwd() at the CLI layer. */
|
|
5
6
|
cwd: string;
|
|
@@ -24,6 +25,22 @@ export interface WatchRunOptions extends WorkspacePaths {
|
|
|
24
25
|
log?: (message: string) => void;
|
|
25
26
|
warn?: (message: string) => void;
|
|
26
27
|
error?: (message: string) => void;
|
|
28
|
+
/**
|
|
29
|
+
* Override `process.env` lookup. Tests use this to toggle
|
|
30
|
+
* `RADAR_AUTO_INSTALL_CHROMIUM=1` without poking at the real environment.
|
|
31
|
+
*/
|
|
32
|
+
env?: NodeJS.ProcessEnv;
|
|
33
|
+
/**
|
|
34
|
+
* Test seam: replace the Playwright probe used by the lazy `html-js`
|
|
35
|
+
* pre-check. Production callers leave this unset and the real
|
|
36
|
+
* `import("playwright")` path runs.
|
|
37
|
+
*/
|
|
38
|
+
playwrightProbeOptions?: ProbeOptions;
|
|
39
|
+
/**
|
|
40
|
+
* Test seam: replace the auto-install function. Tests inject a stub that
|
|
41
|
+
* records invocation without actually spawning `npx playwright install`.
|
|
42
|
+
*/
|
|
43
|
+
installChromiumImpl?: typeof installChromium;
|
|
27
44
|
}
|
|
28
45
|
export interface WatchRunResult {
|
|
29
46
|
/** Map of sourceId → detected (filter-passing, not previously seen) items. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"watcher.d.ts","sourceRoot":"","sources":["../../src/core/watcher.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAErE,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"watcher.d.ts","sourceRoot":"","sources":["../../src/core/watcher.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAErE,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAK/D,OAAO,EAEL,eAAe,EAGf,KAAK,YAAY,EAElB,MAAM,uBAAuB,CAAC;AA2C/B,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6CAA6C;IAC7C,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,WAAW,CAAC;IACnD,yCAAyC;IACzC,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,2DAA2D;IAC3D,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACjC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,YAAY,CAAC;IACtC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,eAAe,CAAC;CAC9C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACjC,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACpC,6EAA6E;IAC7E,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACtD;AAWD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GACjC,OAAO,CAAC,MAAM,EAAE,CAAC,CA6BnB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CA+KhF;AAED;;;;;;GAMG;AACH,wBAAsB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAQ9D"}
|
package/dist/core/watcher.js
CHANGED
|
@@ -6,6 +6,7 @@ import { getFeedAdapter } from "./feeds/index.js";
|
|
|
6
6
|
import { filterItems } from "./filter.js";
|
|
7
7
|
import { detectInjection } from "./injection-detector.js";
|
|
8
8
|
import { saveItems } from "./items.js";
|
|
9
|
+
import { CHROMIUM_MISSING_HINT, installChromium, PLAYWRIGHT_MODULE_MISSING_HINT, probePlaywright, } from "./playwright-check.js";
|
|
9
10
|
import { loadSourceState, saveSourceState } from "./state.js";
|
|
10
11
|
async function pathExists(p) {
|
|
11
12
|
try {
|
|
@@ -118,6 +119,8 @@ export async function watchRun(options) {
|
|
|
118
119
|
const warn = options.warn ?? ((m) => console.warn(m));
|
|
119
120
|
const error = options.error ?? ((m) => console.error(m));
|
|
120
121
|
const getAdapter = options.getAdapter ?? getFeedAdapter;
|
|
122
|
+
const env = options.env ?? process.env;
|
|
123
|
+
const installImpl = options.installChromiumImpl ?? installChromium;
|
|
121
124
|
const sources = await loadSources(paths.sourcesDir, (m) => warn(`watch run: ${m}`));
|
|
122
125
|
const filtered = options.sourceId ? sources.filter((s) => s.id === options.sourceId) : sources;
|
|
123
126
|
if (filtered.length === 0) {
|
|
@@ -130,6 +133,41 @@ export async function watchRun(options) {
|
|
|
130
133
|
return { detected: {}, states: {}, errors: [] };
|
|
131
134
|
}
|
|
132
135
|
const result = { detected: {}, states: {}, errors: [] };
|
|
136
|
+
// Lazy Playwright probe cache. We only run the probe when the first
|
|
137
|
+
// `html-js` source comes up so RSS / GitHub / npm-only workspaces never pay
|
|
138
|
+
// for the dynamic import. The result is reused across every subsequent
|
|
139
|
+
// `html-js` source in the same run — both because the install state cannot
|
|
140
|
+
// realistically change mid-run and because re-probing per source would be
|
|
141
|
+
// wasteful (and would spawn `npx playwright install` repeatedly when the
|
|
142
|
+
// auto-install hatch is on but fails for some reason).
|
|
143
|
+
let playwrightProbe = null;
|
|
144
|
+
const ensurePlaywrightReady = async () => {
|
|
145
|
+
if (playwrightProbe !== null)
|
|
146
|
+
return playwrightProbe;
|
|
147
|
+
playwrightProbe = await probePlaywright(options.playwrightProbeOptions);
|
|
148
|
+
// Auto-install escape hatch (CI-friendly, see playwright-check.ts policy).
|
|
149
|
+
// Triggered only when (a) Playwright itself is present, (b) Chromium is
|
|
150
|
+
// missing, and (c) the user opted in via env. We re-probe after install
|
|
151
|
+
// so the cached result reflects post-install reality; if the install
|
|
152
|
+
// failed the result stays at `chromium-missing` and the source is skipped
|
|
153
|
+
// with the usual hint.
|
|
154
|
+
if (playwrightProbe.status === "chromium-missing" && env.RADAR_AUTO_INSTALL_CHROMIUM === "1") {
|
|
155
|
+
log("watch run: RADAR_AUTO_INSTALL_CHROMIUM=1 detected — attempting to install Chromium...");
|
|
156
|
+
try {
|
|
157
|
+
const code = await installImpl({ cwd: paths.cwd, log });
|
|
158
|
+
if (code === 0) {
|
|
159
|
+
playwrightProbe = await probePlaywright(options.playwrightProbeOptions);
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
warn(`watch run: chromium auto-install exited with code ${code}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
catch (e) {
|
|
166
|
+
warn(`watch run: chromium auto-install failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return playwrightProbe;
|
|
170
|
+
};
|
|
133
171
|
for (const source of filtered) {
|
|
134
172
|
const previousState = await loadSourceState(paths.stateDir, source.id);
|
|
135
173
|
let adapter;
|
|
@@ -142,6 +180,27 @@ export async function watchRun(options) {
|
|
|
142
180
|
result.errors.push({ sourceId: source.id, message });
|
|
143
181
|
continue;
|
|
144
182
|
}
|
|
183
|
+
// Lazy Playwright pre-check for `html-js` sources. Skipping the source
|
|
184
|
+
// (rather than aborting the whole run) preserves the contract that one
|
|
185
|
+
// misbehaving source must not block the others — the same shape used for
|
|
186
|
+
// adapter errors / fetch failures above. The error message embeds the
|
|
187
|
+
// canonical install hint so the user sees the same wording here as in
|
|
188
|
+
// `radar doctor`.
|
|
189
|
+
if (source.kind === "html-js") {
|
|
190
|
+
const probe = await ensurePlaywrightReady();
|
|
191
|
+
if (probe.status !== "ok") {
|
|
192
|
+
const hint = probe.status === "module-missing"
|
|
193
|
+
? PLAYWRIGHT_MODULE_MISSING_HINT
|
|
194
|
+
: CHROMIUM_MISSING_HINT;
|
|
195
|
+
const detail = probe.status === "module-missing"
|
|
196
|
+
? "playwright module not installed"
|
|
197
|
+
: `chromium binary missing at '${probe.executablePath}'`;
|
|
198
|
+
const message = `${detail}\n${hint}`;
|
|
199
|
+
error(`watch run: '${source.id}' skipped: ${message}`);
|
|
200
|
+
result.errors.push({ sourceId: source.id, message });
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
145
204
|
let fetched;
|
|
146
205
|
let nextStatePatch;
|
|
147
206
|
let notModified = false;
|
package/dist/core/watcher.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"watcher.js","sourceRoot":"","sources":["../../src/core/watcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAE9D,KAAK,UAAU,UAAU,CAAC,CAAS;IACjC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,sBAAsB,CAAC,IAAU;IACxC,MAAM,KAAK,GAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,uEAAuE;YACvE,6DAA6D;YAC7D,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAC9C,OAAO,EAAE,GAAG,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC;
|
|
1
|
+
{"version":3,"file":"watcher.js","sourceRoot":"","sources":["../../src/core/watcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EACL,qBAAqB,EACrB,eAAe,EACf,8BAA8B,EAG9B,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAE9D,KAAK,UAAU,UAAU,CAAC,CAAS;IACjC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,sBAAsB,CAAC,IAAU;IACxC,MAAM,KAAK,GAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,uEAAuE;YACvE,6DAA6D;YAC7D,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAC9C,OAAO,EAAE,GAAG,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC;AAsDD,SAAS,YAAY,CAAC,IAAoB;IACxC,OAAO;QACL,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC;QACxD,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC;QAClD,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC;KACnD,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,UAAkB,EAClB,OAAkC;IAElC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACxC,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,kBAAkB,QAAQ,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrF,SAAS;QACX,CAAC;QACD,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,mBAAmB,QAAQ,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACtF,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,sBAAsB,QAAQ,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,SAAS,EAAE,CAAC,CAAC;YAC3F,SAAS;QACX,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB;IACrD,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,cAAc,CAAC;IACxD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,WAAW,GAAG,OAAO,CAAC,mBAAmB,IAAI,eAAe,CAAC;IAEnE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;IACpF,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAE/F,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,IAAI,CAAC,iCAAiC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,4DAA4D,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAClD,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAExE,oEAAoE;IACpE,4EAA4E;IAC5E,uEAAuE;IACvE,2EAA2E;IAC3E,0EAA0E;IAC1E,yEAAyE;IACzE,uDAAuD;IACvD,IAAI,eAAe,GAAiC,IAAI,CAAC;IACzD,MAAM,qBAAqB,GAAG,KAAK,IAAoC,EAAE;QACvE,IAAI,eAAe,KAAK,IAAI;YAAE,OAAO,eAAe,CAAC;QACrD,eAAe,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QACxE,2EAA2E;QAC3E,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,0EAA0E;QAC1E,uBAAuB;QACvB,IAAI,eAAe,CAAC,MAAM,KAAK,kBAAkB,IAAI,GAAG,CAAC,2BAA2B,KAAK,GAAG,EAAE,CAAC;YAC7F,GAAG,CAAC,uFAAuF,CAAC,CAAC;YAC7F,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;gBACxD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,eAAe,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;gBAC1E,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,qDAAqD,IAAI,EAAE,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CACF,4CAA4C,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACzF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACvE,IAAI,OAAoB,CAAC;QACzB,IAAI,CAAC;YACH,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3D,KAAK,CAAC,eAAe,MAAM,CAAC,EAAE,oBAAoB,OAAO,EAAE,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,uEAAuE;QACvE,uEAAuE;QACvE,yEAAyE;QACzE,sEAAsE;QACtE,sEAAsE;QACtE,kBAAkB;QAClB,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,qBAAqB,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,GACR,KAAK,CAAC,MAAM,KAAK,gBAAgB;oBAC/B,CAAC,CAAC,8BAA8B;oBAChC,CAAC,CAAC,qBAAqB,CAAC;gBAC5B,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,KAAK,gBAAgB;oBAC/B,CAAC,CAAC,iCAAiC;oBACnC,CAAC,CAAC,+BAA+B,KAAK,CAAC,cAAc,GAAG,CAAC;gBAC7D,MAAM,OAAO,GAAG,GAAG,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrC,KAAK,CAAC,eAAe,MAAM,CAAC,EAAE,cAAc,OAAO,EAAE,CAAC,CAAC;gBACvD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;gBACrD,SAAS;YACX,CAAC;QACH,CAAC;QACD,IAAI,OAAe,CAAC;QACpB,IAAI,cAAoC,CAAC;QACzC,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE;gBAC9C,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,KAAK,EAAE,aAAa;aACrB,CAAC,CAAC;YACH,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC;YAC5B,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC;YACnC,WAAW,GAAG,WAAW,CAAC,WAAW,IAAI,KAAK,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3D,KAAK,CAAC,eAAe,MAAM,CAAC,EAAE,mBAAmB,OAAO,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QACnD,IAAI,aAAa,GAAW,EAAE,CAAC;QAE/B,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,uEAAuE;YACvE,wEAAwE;YACxE,kEAAkE;YAClE,KAAK,MAAM,IAAI,IAAI,OAAO;gBAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjD,GAAG,CAAC,yBAAyB,MAAM,CAAC,EAAE,OAAO,OAAO,CAAC,MAAM,iCAAiC,CAAC,CAAC;QAChG,CAAC;aAAM,IAAI,WAAW,EAAE,CAAC;YACvB,GAAG,CAAC,eAAe,MAAM,CAAC,EAAE,mBAAmB,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,MAAM;iBACjB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;iBACvC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACvC,KAAK,MAAM,IAAI,IAAI,KAAK;oBAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjD,CAAC;YACD,mEAAmE;YACnE,sEAAsE;YACtE,qEAAqE;YACrE,2CAA2C;YAC3C,KAAK,MAAM,IAAI,IAAI,OAAO;gBAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjD,aAAa,GAAG,KAAK,CAAC;YACtB,oEAAoE;YACpE,mEAAmE;YACnE,qEAAqE;YACrE,iEAAiE;YACjE,gEAAgE;YAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrF,IAAI,CACF,eAAe,MAAM,CAAC,EAAE,sCAAsC,OAAO,CAAC,MAAM,+DAA+D,MAAM,CAAC,EAAE,2BAA2B,CAChL,CAAC;YACJ,CAAC;YACD,GAAG,CACD,eAAe,MAAM,CAAC,EAAE,aAAa,OAAO,CAAC,MAAM,WAAW,KAAK,CAAC,MAAM,mBAAmB,CAC9F,CAAC;YACF,gEAAgE;YAChE,qEAAqE;YACrE,uEAAuE;YACvE,8DAA8D;YAC9D,8DAA8D;YAC9D,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACjE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CACF,eAAe,MAAM,CAAC,EAAE,KAAK,OAAO,CAAC,MAAM,yHAAyH,CACrK,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAgB;YAC7B,QAAQ,EAAE,MAAM,CAAC,EAAE;YACnB,aAAa,EAAE,cAAc,CAAC,aAAa,IAAI,aAAa,CAAC,aAAa;YAC1E,QAAQ,EAAE,cAAc,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;YAC3D,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;SACjC,CAAC;QACF,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC;IACvC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,OAAiB;IAC3C,MAAM,OAAO,GAAW,EAAE,CAAC;IAC3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/index.js
CHANGED
|
File without changes
|
package/dist/schemas/source.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { z } from "zod";
|
|
|
2
2
|
export declare const SourceKindSchema: z.ZodEnum<{
|
|
3
3
|
rss: "rss";
|
|
4
4
|
html: "html";
|
|
5
|
+
"html-js": "html-js";
|
|
5
6
|
"github-releases": "github-releases";
|
|
6
7
|
"npm-registry": "npm-registry";
|
|
7
8
|
}>;
|
|
@@ -94,11 +95,42 @@ export declare const SourceSelectorsSchema: z.ZodObject<{
|
|
|
94
95
|
tags: z.ZodOptional<z.ZodString>;
|
|
95
96
|
}, z.core.$strip>;
|
|
96
97
|
export type SourceSelectors = z.infer<typeof SourceSelectorsSchema>;
|
|
98
|
+
/**
|
|
99
|
+
* Optional JS rendering options for `kind: html-js` sources (ADR-0010).
|
|
100
|
+
*
|
|
101
|
+
* The adapter delegates fetching to a headless Chromium via Playwright. These
|
|
102
|
+
* options expose the few knobs users actually need to tune per source; all
|
|
103
|
+
* hardening policy (headless / acceptDownloads / fresh context / viewport)
|
|
104
|
+
* is hardcoded in the adapter and intentionally NOT user-configurable.
|
|
105
|
+
*
|
|
106
|
+
* - `waitFor`: CSS selector to wait for before reading `page.content()`.
|
|
107
|
+
* Defaults at adapter level to `selectors.item` so the common case "wait
|
|
108
|
+
* until the item list has rendered" needs no extra config.
|
|
109
|
+
* - `waitUntil`: Playwright `page.goto()` lifecycle event. `networkidle` is
|
|
110
|
+
* the safest default for SPA/CSR pages where item data arrives via XHR
|
|
111
|
+
* after the document has loaded.
|
|
112
|
+
* - `timeout`: Per-step timeout (goto, waitForSelector) in milliseconds.
|
|
113
|
+
* Caps OOM / infinite-loop risk on pathological pages.
|
|
114
|
+
* - `userAgent`: Optional UA override. Most sites accept the default
|
|
115
|
+
* Chromium UA; override only when a site gates content behind a UA check.
|
|
116
|
+
*/
|
|
117
|
+
export declare const SourceJsOptionsSchema: z.ZodObject<{
|
|
118
|
+
waitFor: z.ZodOptional<z.ZodString>;
|
|
119
|
+
waitUntil: z.ZodDefault<z.ZodEnum<{
|
|
120
|
+
load: "load";
|
|
121
|
+
domcontentloaded: "domcontentloaded";
|
|
122
|
+
networkidle: "networkidle";
|
|
123
|
+
}>>;
|
|
124
|
+
timeout: z.ZodDefault<z.ZodNumber>;
|
|
125
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
126
|
+
}, z.core.$strip>;
|
|
127
|
+
export type SourceJsOptions = z.infer<typeof SourceJsOptionsSchema>;
|
|
97
128
|
export declare const SourceSchema: z.ZodObject<{
|
|
98
129
|
id: z.ZodString;
|
|
99
130
|
kind: z.ZodEnum<{
|
|
100
131
|
rss: "rss";
|
|
101
132
|
html: "html";
|
|
133
|
+
"html-js": "html-js";
|
|
102
134
|
"github-releases": "github-releases";
|
|
103
135
|
"npm-registry": "npm-registry";
|
|
104
136
|
}>;
|
|
@@ -130,6 +162,16 @@ export declare const SourceSchema: z.ZodObject<{
|
|
|
130
162
|
body: z.ZodOptional<z.ZodString>;
|
|
131
163
|
tags: z.ZodOptional<z.ZodString>;
|
|
132
164
|
}, z.core.$strip>>;
|
|
165
|
+
js: z.ZodOptional<z.ZodObject<{
|
|
166
|
+
waitFor: z.ZodOptional<z.ZodString>;
|
|
167
|
+
waitUntil: z.ZodDefault<z.ZodEnum<{
|
|
168
|
+
load: "load";
|
|
169
|
+
domcontentloaded: "domcontentloaded";
|
|
170
|
+
networkidle: "networkidle";
|
|
171
|
+
}>>;
|
|
172
|
+
timeout: z.ZodDefault<z.ZodNumber>;
|
|
173
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
174
|
+
}, z.core.$strip>>;
|
|
133
175
|
trustLevel: z.ZodDefault<z.ZodEnum<{
|
|
134
176
|
trusted: "trusted";
|
|
135
177
|
untrusted: "untrusted";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../../src/schemas/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,gBAAgB
|
|
1
|
+
{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../../src/schemas/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,gBAAgB;;;;;;EAM3B,CAAC;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe;;;;EAAyC,CAAC;AACtE,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,qGAAqG;AACrG,eAAO,MAAM,gBAAgB;;;;;EAA+C,CAAC;AAC7E,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,gBAAgB;;;EAAmC,CAAC;AACjE,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;iBAM9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;iBAQhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;iBAKhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAoBpE,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA+CrB,CAAC;AACL,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC"}
|
package/dist/schemas/source.js
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
export const SourceKindSchema = z.enum([
|
|
2
|
+
export const SourceKindSchema = z.enum([
|
|
3
|
+
"rss",
|
|
4
|
+
"html",
|
|
5
|
+
"html-js",
|
|
6
|
+
"github-releases",
|
|
7
|
+
"npm-registry",
|
|
8
|
+
]);
|
|
3
9
|
/**
|
|
4
10
|
* Match modes for keyword evaluation (ADR-0006).
|
|
5
11
|
*
|
|
@@ -62,6 +68,31 @@ export const SourceSelectorsSchema = z.object({
|
|
|
62
68
|
body: z.string().optional(),
|
|
63
69
|
tags: z.string().optional(),
|
|
64
70
|
});
|
|
71
|
+
/**
|
|
72
|
+
* Optional JS rendering options for `kind: html-js` sources (ADR-0010).
|
|
73
|
+
*
|
|
74
|
+
* The adapter delegates fetching to a headless Chromium via Playwright. These
|
|
75
|
+
* options expose the few knobs users actually need to tune per source; all
|
|
76
|
+
* hardening policy (headless / acceptDownloads / fresh context / viewport)
|
|
77
|
+
* is hardcoded in the adapter and intentionally NOT user-configurable.
|
|
78
|
+
*
|
|
79
|
+
* - `waitFor`: CSS selector to wait for before reading `page.content()`.
|
|
80
|
+
* Defaults at adapter level to `selectors.item` so the common case "wait
|
|
81
|
+
* until the item list has rendered" needs no extra config.
|
|
82
|
+
* - `waitUntil`: Playwright `page.goto()` lifecycle event. `networkidle` is
|
|
83
|
+
* the safest default for SPA/CSR pages where item data arrives via XHR
|
|
84
|
+
* after the document has loaded.
|
|
85
|
+
* - `timeout`: Per-step timeout (goto, waitForSelector) in milliseconds.
|
|
86
|
+
* Caps OOM / infinite-loop risk on pathological pages.
|
|
87
|
+
* - `userAgent`: Optional UA override. Most sites accept the default
|
|
88
|
+
* Chromium UA; override only when a site gates content behind a UA check.
|
|
89
|
+
*/
|
|
90
|
+
export const SourceJsOptionsSchema = z.object({
|
|
91
|
+
waitFor: z.string().optional(),
|
|
92
|
+
waitUntil: z.enum(["load", "domcontentloaded", "networkidle"]).default("networkidle"),
|
|
93
|
+
timeout: z.number().int().positive().default(30000),
|
|
94
|
+
userAgent: z.string().optional(),
|
|
95
|
+
});
|
|
65
96
|
/**
|
|
66
97
|
* Validate `Source.url` per kind.
|
|
67
98
|
*
|
|
@@ -97,11 +128,15 @@ export const SourceSchema = z
|
|
|
97
128
|
matchFields: ["title", "summary"],
|
|
98
129
|
caseSensitive: false,
|
|
99
130
|
}),
|
|
100
|
-
// `selectors` is required for `kind: html` and
|
|
101
|
-
// kinds. We model it as optional at the field level and
|
|
102
|
-
// "required when html" rule via a refinement so the
|
|
103
|
-
// serializes cleanly for both cases.
|
|
131
|
+
// `selectors` is required for `kind: html` and `kind: html-js`, ignored
|
|
132
|
+
// for the other kinds. We model it as optional at the field level and
|
|
133
|
+
// enforce the "required when html / html-js" rule via a refinement so the
|
|
134
|
+
// same Source type serializes cleanly for both cases.
|
|
104
135
|
selectors: SourceSelectorsSchema.optional(),
|
|
136
|
+
// `js` is only consulted by the `html-js` adapter. Marked optional so
|
|
137
|
+
// existing source YAMLs (and `kind: html` ones) parse unchanged; the
|
|
138
|
+
// adapter applies defaults when the field is omitted entirely.
|
|
139
|
+
js: SourceJsOptionsSchema.optional(),
|
|
105
140
|
// `trustLevel` defaults to `"untrusted"` so existing source YAMLs (which
|
|
106
141
|
// omit the field entirely) keep their current treatment. Per ADR-0009 M4
|
|
107
142
|
// this is schema-only; policy branches that read `trustLevel` arrive in a
|
|
@@ -116,11 +151,11 @@ export const SourceSchema = z
|
|
|
116
151
|
message: "Invalid url",
|
|
117
152
|
});
|
|
118
153
|
}
|
|
119
|
-
if (value.kind === "html" && value.selectors === undefined) {
|
|
154
|
+
if ((value.kind === "html" || value.kind === "html-js") && value.selectors === undefined) {
|
|
120
155
|
ctx.addIssue({
|
|
121
156
|
code: "custom",
|
|
122
157
|
path: ["selectors"],
|
|
123
|
-
message:
|
|
158
|
+
message: `selectors is required when kind is '${value.kind}'`,
|
|
124
159
|
});
|
|
125
160
|
}
|
|
126
161
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"source.js","sourceRoot":"","sources":["../../src/schemas/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"source.js","sourceRoot":"","sources":["../../src/schemas/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,IAAI,CAAC;IACrC,KAAK;IACL,MAAM;IACN,SAAS;IACT,iBAAiB;IACjB,cAAc;CACf,CAAC,CAAC;AAGH;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;AAGtE,qGAAqG;AACrG,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAG7E;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;AAGjE,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACzC,eAAe,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAChD,SAAS,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC;IAC1C,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACpE,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;CAC1C,CAAC,CAAC;AAGH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAC;AAGH;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC;IACrF,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC,CAAC;AAGH;;;;;;;;GAQG;AACH,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,OAAO,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC;KAC1B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,IAAI,EAAE,gBAAgB;IACtB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,wEAAwE;IACxE,sEAAsE;IACtE,mEAAmE;IACnE,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC;QACnC,QAAQ,EAAE,EAAE;QACZ,eAAe,EAAE,EAAE;QACnB,SAAS,EAAE,MAAM;QACjB,WAAW,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;QACjC,aAAa,EAAE,KAAK;KACrB,CAAC;IACF,wEAAwE;IACxE,sEAAsE;IACtE,0EAA0E;IAC1E,sDAAsD;IACtD,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IAC3C,sEAAsE;IACtE,qEAAqE;IACrE,+DAA+D;IAC/D,EAAE,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IACpC,yEAAyE;IACzE,yEAAyE;IACzE,0EAA0E;IAC1E,sBAAsB;IACtB,UAAU,EAAE,gBAAgB,CAAC,OAAO,CAAC,WAAW,CAAC;CAClD,CAAC;KACD,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,CAAC,KAAK,CAAC;YACb,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;IACL,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACzF,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,CAAC,WAAW,CAAC;YACnB,OAAO,EAAE,uCAAuC,KAAK,CAAC,IAAI,GAAG;SAC9D,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC,CAAC"}
|
|
@@ -48,7 +48,7 @@ radar init --with-actions # .github/workflows/watch.yaml を生成
|
|
|
48
48
|
radar init --force # 既存ファイルを上書き
|
|
49
49
|
|
|
50
50
|
# 監視対象の管理
|
|
51
|
-
radar source add <id> --kind <rss|html|github-releases|npm-registry> --url <url> [options]
|
|
51
|
+
radar source add <id> --kind <rss|html|html-js|github-releases|npm-registry> --url <url> [options]
|
|
52
52
|
radar source list
|
|
53
53
|
radar source remove <id>
|
|
54
54
|
|
|
@@ -143,7 +143,7 @@ agent の選択は CLI が強制せず、ユーザー判断です。
|
|
|
143
143
|
|
|
144
144
|
## セキュリティ警告 (untrusted external content)
|
|
145
145
|
|
|
146
|
-
`radar` が fetch する外部 feed (RSS / HTML / GitHub Releases / npm registry) のコンテンツは **untrusted** として扱われます ([ADR-0009](https://github.com/ozzy-labs/feedradar/blob/main/docs/adr/0009-untrusted-external-content-handling.md))。攻撃者が feed 内容に prompt injection を仕込む可能性があるため:
|
|
146
|
+
`radar` が fetch する外部 feed (RSS / HTML / HTML (JS rendered, `kind: html-js`) / GitHub Releases / npm registry) のコンテンツは **untrusted** として扱われます ([ADR-0009](https://github.com/ozzy-labs/feedradar/blob/main/docs/adr/0009-untrusted-external-content-handling.md))。攻撃者が feed 内容に prompt injection を仕込む可能性があるため:
|
|
147
147
|
|
|
148
148
|
- agent に渡すコンテンツは boundary marker で囲まれ、procedure 本体と分離される
|
|
149
149
|
- `sources/<id>.yaml` の `trustLevel` で `"trusted" | "untrusted"` を per-source で指定可能 (既定 `"untrusted"`)
|
|
@@ -111,7 +111,7 @@ slash で直接呼ぶなら:
|
|
|
111
111
|
エージェントを起動しない自動化文脈では CLI を直接呼びます。`radar <subcommand> --help` で全コマンドのヘルプが出ます。
|
|
112
112
|
|
|
113
113
|
```bash
|
|
114
|
-
radar source add <id> --kind <rss|html|github-releases|npm-registry> --url <url> [options]
|
|
114
|
+
radar source add <id> --kind <rss|html|html-js|github-releases|npm-registry> --url <url> [options]
|
|
115
115
|
radar source list
|
|
116
116
|
radar source remove <id>
|
|
117
117
|
radar watch run [--source <id>] [--bootstrap]
|
|
@@ -152,7 +152,7 @@ radar dismiss <item-id>
|
|
|
152
152
|
|
|
153
153
|
## セキュリティ警告
|
|
154
154
|
|
|
155
|
-
FeedRadar が fetch する外部 feed (RSS / HTML / GitHub Releases / npm registry) は **untrusted** として扱われます ([ADR-0009](https://github.com/ozzy-labs/feedradar/blob/main/docs/adr/0009-untrusted-external-content-handling.md))。攻撃者が feed 内容に prompt injection を仕込む可能性があるため:
|
|
155
|
+
FeedRadar が fetch する外部 feed (RSS / HTML / HTML (JS rendered, `kind: html-js`) / GitHub Releases / npm registry) は **untrusted** として扱われます ([ADR-0009](https://github.com/ozzy-labs/feedradar/blob/main/docs/adr/0009-untrusted-external-content-handling.md))。攻撃者が feed 内容に prompt injection を仕込む可能性があるため:
|
|
156
156
|
|
|
157
157
|
- 信頼できる公式 source のみ登録するのが第一の防御線
|
|
158
158
|
- `sources/<id>.yaml` の `trustLevel: trusted` で個別 opt-in 可 (既定 `untrusted`)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ozzylabs/feedradar",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Multi-agent CLI that watches blogs and release feeds, then turns keyword hits into Markdown research reports",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"publishConfig": {
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
"test": "vitest run",
|
|
24
24
|
"test:watch": "vitest",
|
|
25
25
|
"test:coverage": "vitest run --coverage",
|
|
26
|
+
"test:integration": "vitest run --config vitest.integration.config.ts",
|
|
26
27
|
"radar": "node dist/index.js",
|
|
27
28
|
"prepublishOnly": "pnpm run build && pnpm run typecheck && pnpm run test"
|
|
28
29
|
},
|
|
@@ -33,9 +34,18 @@
|
|
|
33
34
|
"yaml": "^2.8.4",
|
|
34
35
|
"zod": "^4.4.3"
|
|
35
36
|
},
|
|
37
|
+
"peerDependencies": {
|
|
38
|
+
"playwright": "^1.60.0"
|
|
39
|
+
},
|
|
40
|
+
"peerDependenciesMeta": {
|
|
41
|
+
"playwright": {
|
|
42
|
+
"optional": true
|
|
43
|
+
}
|
|
44
|
+
},
|
|
36
45
|
"devDependencies": {
|
|
37
46
|
"@types/node": "^24.12.2",
|
|
38
47
|
"@vitest/coverage-v8": "^4.1.6",
|
|
48
|
+
"playwright": "^1.60.0",
|
|
39
49
|
"typescript": "^6.0.3",
|
|
40
50
|
"vitest": "^4.1.5"
|
|
41
51
|
},
|