@ozzylabs/feedradar 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +31 -6
- package/README.md +31 -6
- package/dist/agents/claude-code.d.ts +12 -1
- package/dist/agents/claude-code.d.ts.map +1 -1
- package/dist/agents/claude-code.js +9 -5
- package/dist/agents/claude-code.js.map +1 -1
- package/dist/agents/codex-cli.d.ts +7 -1
- package/dist/agents/codex-cli.d.ts.map +1 -1
- package/dist/agents/codex-cli.js +9 -5
- package/dist/agents/codex-cli.js.map +1 -1
- package/dist/agents/copilot.d.ts +7 -1
- package/dist/agents/copilot.d.ts.map +1 -1
- package/dist/agents/copilot.js +9 -5
- package/dist/agents/copilot.js.map +1 -1
- package/dist/agents/gemini-cli.d.ts +7 -1
- package/dist/agents/gemini-cli.d.ts.map +1 -1
- package/dist/agents/gemini-cli.js +9 -5
- package/dist/agents/gemini-cli.js.map +1 -1
- package/dist/agents/index.d.ts +1 -1
- package/dist/agents/index.d.ts.map +1 -1
- package/dist/agents/types.d.ts +33 -0
- package/dist/agents/types.d.ts.map +1 -1
- package/dist/cli/_progress.d.ts +138 -0
- package/dist/cli/_progress.d.ts.map +1 -0
- package/dist/cli/_progress.js +176 -0
- package/dist/cli/_progress.js.map +1 -0
- package/dist/cli/doctor.d.ts +20 -0
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +291 -2
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/research.d.ts +18 -20
- package/dist/cli/research.d.ts.map +1 -1
- package/dist/cli/research.js +318 -203
- package/dist/cli/research.js.map +1 -1
- package/dist/cli/respawn.d.ts +53 -0
- package/dist/cli/respawn.d.ts.map +1 -0
- package/dist/cli/respawn.js +120 -0
- package/dist/cli/respawn.js.map +1 -0
- package/dist/cli/review.d.ts +7 -0
- package/dist/cli/review.d.ts.map +1 -1
- package/dist/cli/review.js +46 -1
- package/dist/cli/review.js.map +1 -1
- package/dist/cli/source.d.ts +23 -2
- package/dist/cli/source.d.ts.map +1 -1
- package/dist/cli/source.js +425 -7
- package/dist/cli/source.js.map +1 -1
- package/dist/cli/update.d.ts +7 -0
- package/dist/cli/update.d.ts.map +1 -1
- package/dist/cli/update.js +41 -1
- package/dist/cli/update.js.map +1 -1
- package/dist/cli/watch.d.ts.map +1 -1
- package/dist/cli/watch.js +65 -3
- package/dist/cli/watch.js.map +1 -1
- package/dist/cli/workflow/generate-combined.d.ts +100 -0
- package/dist/cli/workflow/generate-combined.d.ts.map +1 -0
- package/dist/cli/workflow/generate-combined.js +387 -0
- package/dist/cli/workflow/generate-combined.js.map +1 -0
- package/dist/cli/workflow/generate-watch.d.ts +142 -0
- package/dist/cli/workflow/generate-watch.d.ts.map +1 -0
- package/dist/cli/workflow/generate-watch.js +338 -0
- package/dist/cli/workflow/generate-watch.js.map +1 -0
- package/dist/cli/workflow.d.ts +29 -0
- package/dist/cli/workflow.d.ts.map +1 -0
- package/dist/cli/workflow.js +66 -0
- package/dist/cli/workflow.js.map +1 -0
- package/dist/core/feeds/_fetch.d.ts +103 -0
- package/dist/core/feeds/_fetch.d.ts.map +1 -0
- package/dist/core/feeds/_fetch.js +364 -0
- package/dist/core/feeds/_fetch.js.map +1 -0
- package/dist/core/feeds/_jsonpath.d.ts +57 -0
- package/dist/core/feeds/_jsonpath.d.ts.map +1 -0
- package/dist/core/feeds/_jsonpath.js +207 -0
- package/dist/core/feeds/_jsonpath.js.map +1 -0
- package/dist/core/feeds/github-api.d.ts.map +1 -1
- package/dist/core/feeds/github-api.js +2 -1
- package/dist/core/feeds/github-api.js.map +1 -1
- package/dist/core/feeds/html-js.d.ts +29 -0
- package/dist/core/feeds/html-js.d.ts.map +1 -1
- package/dist/core/feeds/html-js.js +86 -2
- package/dist/core/feeds/html-js.js.map +1 -1
- package/dist/core/feeds/html.d.ts.map +1 -1
- package/dist/core/feeds/html.js +2 -1
- package/dist/core/feeds/html.js.map +1 -1
- package/dist/core/feeds/index.d.ts +1 -1
- package/dist/core/feeds/index.d.ts.map +1 -1
- package/dist/core/feeds/index.js +4 -0
- package/dist/core/feeds/index.js.map +1 -1
- package/dist/core/feeds/json-api.d.ts +3 -0
- package/dist/core/feeds/json-api.d.ts.map +1 -0
- package/dist/core/feeds/json-api.js +723 -0
- package/dist/core/feeds/json-api.js.map +1 -0
- package/dist/core/feeds/json-feed.d.ts +11 -0
- package/dist/core/feeds/json-feed.d.ts.map +1 -0
- package/dist/core/feeds/json-feed.js +242 -0
- package/dist/core/feeds/json-feed.js.map +1 -0
- package/dist/core/feeds/npm-registry.d.ts.map +1 -1
- package/dist/core/feeds/npm-registry.js +2 -1
- package/dist/core/feeds/npm-registry.js.map +1 -1
- package/dist/core/feeds/rss.d.ts.map +1 -1
- package/dist/core/feeds/rss.js +2 -1
- package/dist/core/feeds/rss.js.map +1 -1
- package/dist/core/feeds/types.d.ts +123 -0
- package/dist/core/feeds/types.d.ts.map +1 -1
- package/dist/core/progress.d.ts +101 -0
- package/dist/core/progress.d.ts.map +1 -0
- package/dist/core/progress.js +212 -0
- package/dist/core/progress.js.map +1 -0
- package/dist/core/proxy.d.ts +87 -0
- package/dist/core/proxy.d.ts.map +1 -0
- package/dist/core/proxy.js +146 -0
- package/dist/core/proxy.js.map +1 -0
- package/dist/core/recipes.d.ts +138 -0
- package/dist/core/recipes.d.ts.map +1 -0
- package/dist/core/recipes.js +238 -0
- package/dist/core/recipes.js.map +1 -0
- package/dist/core/watcher.d.ts +61 -1
- package/dist/core/watcher.d.ts.map +1 -1
- package/dist/core/watcher.js +99 -2
- package/dist/core/watcher.js.map +1 -1
- package/dist/index.js +17 -4
- package/dist/index.js.map +1 -1
- package/dist/recipes/aws-whats-new.yaml +61 -0
- package/dist/recipes/dev-to.yaml +40 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/index.js.map +1 -1
- package/dist/schemas/recipe.d.ts +115 -0
- package/dist/schemas/recipe.d.ts.map +1 -0
- package/dist/schemas/recipe.js +54 -0
- package/dist/schemas/recipe.js.map +1 -0
- package/dist/schemas/source.d.ts +130 -0
- package/dist/schemas/source.d.ts.map +1 -1
- package/dist/schemas/source.js +130 -0
- package/dist/schemas/source.js.map +1 -1
- package/dist/templates/agents/AGENTS.md +31 -3
- package/dist/templates/feedradar.md +23 -8
- package/dist/templates/workflows/combined.template.yaml.tmpl +110 -0
- package/dist/templates/workflows/watch.template.yaml.tmpl +103 -0
- package/dist/templates/workflows/watch.yaml +5 -1
- package/package.json +2 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github-api.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/github-api.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"github-api.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/github-api.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAY5C,2FAA2F;AAC3F,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;IACf,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uEAAuE;AACvE,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CA+BvD;AAED,gEAAgE;AAChE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED,iFAAiF;AACjF,MAAM,WAAW,oBAAoB;IACnC,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,4FAA4F;IAC5F,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,kDAAkD;AAClD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,yDAAyD;IACzD,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,yDAAyD;AACzD,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gEAAgE;IAChE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,uEAAuE;IACvE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,mBAAmB,CAAC,CA+E9B"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { fetchWithRetry } from "./_fetch.js";
|
|
1
2
|
const USER_AGENT = "feedradar/0.0.0 (+https://github.com/ozzy-labs/feedradar)";
|
|
2
3
|
const GITHUB_API_BASE = "https://api.github.com";
|
|
3
4
|
/**
|
|
@@ -79,7 +80,7 @@ export async function fetchReleases(owner, repo, options = {}) {
|
|
|
79
80
|
if (options.etag)
|
|
80
81
|
headers["if-none-match"] = options.etag;
|
|
81
82
|
const url = buildReleasesUrl(owner, repo);
|
|
82
|
-
const response = await fetchImpl
|
|
83
|
+
const response = await fetchWithRetry(fetchImpl, url, { headers, signal: options.signal });
|
|
83
84
|
const rateLimit = {
|
|
84
85
|
remaining: parseIntHeader(response.headers.get("x-ratelimit-remaining")),
|
|
85
86
|
limit: parseIntHeader(response.headers.get("x-ratelimit-limit")),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github-api.js","sourceRoot":"","sources":["../../../src/core/feeds/github-api.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"github-api.js","sourceRoot":"","sources":["../../../src/core/feeds/github-api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAG7C,MAAM,UAAU,GAAG,2DAA2D,CAAC;AAC/E,MAAM,eAAe,GAAG,wBAAwB,CAAC;AAEjD;;;;GAIG;AACH,MAAM,4BAA4B,GAAG,EAAE,CAAC;AAqBxC;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAE3E,IAAI,SAAS,GAAG,OAAO,CAAC;IACxB,wDAAwD;IACxD,IAAI,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,IAAI,GAAQ,CAAC;QACb,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,qFAAqF;QACrF,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAC/E,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,kFAAkF;IAClF,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC;IAC7C,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,0DAA0D,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,0DAA0D,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAAY;IAC1D,OAAO,GAAG,eAAe,UAAU,kBAAkB,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC;AACtG,CAAC;AAkCD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAa,EACb,IAAY,EACZ,UAAgC,EAAE;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,IAAK,UAAU,CAAC,KAA8B,CAAC;IAC9E,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CACb,gFAAgF,CACjF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IACxD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtD,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,6BAA6B;QACrC,YAAY,EAAE,UAAU;QACxB,sEAAsE;QACtE,sDAAsD;QACtD,sBAAsB,EAAE,YAAY;KACrC,CAAC;IACF,IAAI,KAAK;QAAE,OAAO,CAAC,aAAa,GAAG,UAAU,KAAK,EAAE,CAAC;IACrD,IAAI,OAAO,CAAC,IAAI;QAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1D,MAAM,GAAG,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE3F,MAAM,SAAS,GAAkB;QAC/B,SAAS,EAAE,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACxE,KAAK,EAAE,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAChE,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;KACnE,CAAC;IACF,oBAAoB,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,IAAI,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE1C,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAC3E,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QACzD,kEAAkE;QAClE,0EAA0E;QAC1E,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,IAAI,KAAK,CACb,qDAAqD,KAAK,IAAI,IAAI,GAAG,SAAS,IAAI;YAChF,CAAC,KAAK;gBACJ,CAAC,CAAC,oCAAoC;gBACtC,CAAC,CAAC,4DAA4D,CAAC,CACpE,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,sDAAsD,KAAK,IAAI,IAAI,uBAAuB,CAC3F,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,iCAAiC,QAAQ,CAAC,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,kDAAkD,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC/F,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,qDAAqD,GAAG,SAAS,OAAO,MAAM,EAAE,CACjF,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAChD,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AACpF,CAAC;AAED,yFAAyF;AACzF,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC7D,MAAM,CAAC,GAAG,KAAgC,CAAC;IAC3C,OAAO,CACL,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ;QACxB,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;QAC9B,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;QAC9B,OAAO,CAAC,CAAC,KAAK,KAAK,SAAS;QAC5B,OAAO,CAAC,CAAC,UAAU,KAAK,SAAS,CAClC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAoB;IAC1C,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC;IAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,oBAAoB,CAC3B,SAAwB,EACxB,KAAa,EACb,IAAY,EACZ,aAAsB,EACtB,IAA+B;IAE/B,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC;IACvC,IAAI,SAAS,IAAI,IAAI,IAAI,SAAS,GAAG,4BAA4B;QAAE,OAAO;IAC1E,MAAM,SAAS,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnD,MAAM,QAAQ,GAAG,aAAa;QAC5B,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,6DAA6D,CAAC;IAClE,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACrD,IAAI,CACF,oCAAoC,SAAS,GAAG,SAAS,mBAAmB,KAAK,IAAI,IAAI,GAAG,SAAS,IAAI,QAAQ,EAAE,CACpH,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,OAAsB;IAC7C,IAAI,OAAO,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5C,OAAO,eAAe,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC;AAC9C,CAAC"}
|
|
@@ -6,10 +6,25 @@ import type { FeedAdapter, FeedAdapterOptions } from "./types.js";
|
|
|
6
6
|
* pull in the full Playwright type tree (which is itself an optional peer
|
|
7
7
|
* dep and therefore not guaranteed to be installed in dev).
|
|
8
8
|
*/
|
|
9
|
+
/**
|
|
10
|
+
* Subset of Playwright's `BrowserType.launch()` proxy options used by this
|
|
11
|
+
* adapter. Playwright does NOT auto-honor `HTTPS_PROXY` / `HTTP_PROXY` the way
|
|
12
|
+
* Node's `--use-env-proxy` does for built-in `fetch()` — it requires the proxy
|
|
13
|
+
* to be passed explicitly into launch options. The adapter therefore probes
|
|
14
|
+
* the env (via `detectProxyUrl`) on every fetch and forwards the URL to
|
|
15
|
+
* Chromium so corporate proxies / NO_PROXY rules are respected end-to-end.
|
|
16
|
+
*/
|
|
17
|
+
export interface PlaywrightProxyOptions {
|
|
18
|
+
/** Proxy URL (e.g. `http://proxy.corp:8080`). Playwright resolves SOCKS / HTTP automatically. */
|
|
19
|
+
server: string;
|
|
20
|
+
/** Optional semicolon-separated bypass list (NOT comma — see `noProxyToPlaywrightBypass`). */
|
|
21
|
+
bypass?: string;
|
|
22
|
+
}
|
|
9
23
|
export interface PlaywrightLike {
|
|
10
24
|
chromium: {
|
|
11
25
|
launch(options?: {
|
|
12
26
|
headless?: boolean;
|
|
27
|
+
proxy?: PlaywrightProxyOptions;
|
|
13
28
|
}): Promise<PlaywrightBrowserLike>;
|
|
14
29
|
};
|
|
15
30
|
}
|
|
@@ -45,6 +60,20 @@ export interface PlaywrightPageLike {
|
|
|
45
60
|
export interface HtmlJsAdapterOptions extends FeedAdapterOptions {
|
|
46
61
|
/** Injected Playwright module (tests only). Production uses dynamic import. */
|
|
47
62
|
playwright?: PlaywrightLike;
|
|
63
|
+
/**
|
|
64
|
+
* Injected env (tests only) — production reads `process.env`. Used to probe
|
|
65
|
+
* `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` so the proxy injection branch is
|
|
66
|
+
* deterministically testable without polluting the real process env.
|
|
67
|
+
*/
|
|
68
|
+
env?: NodeJS.ProcessEnv;
|
|
69
|
+
/**
|
|
70
|
+
* Test seam (#198): override the `Still waiting…` reminder threshold in
|
|
71
|
+
* milliseconds. Production hardcodes 10s (a third of the default 30s
|
|
72
|
+
* timeout) so the prediction triggers well before the actual selector
|
|
73
|
+
* timeout. Tests shrink it so the reminder fires deterministically without
|
|
74
|
+
* blocking the suite on real time.
|
|
75
|
+
*/
|
|
76
|
+
stillWaitingMs?: number;
|
|
48
77
|
}
|
|
49
78
|
export declare const htmlJsAdapter: FeedAdapter;
|
|
50
79
|
//# sourceMappingURL=html-js.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html-js.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/html-js.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAU,eAAe,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"html-js.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/html-js.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAU,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAGtE,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAoElE;;;;;GAKG;AACH;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB;IACrC,iGAAiG;IACjG,MAAM,EAAE,MAAM,CAAC;IACf,8FAA8F;IAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE;QACR,MAAM,CAAC,OAAO,CAAC,EAAE;YACf,QAAQ,CAAC,EAAE,OAAO,CAAC;YACnB,KAAK,CAAC,EAAE,sBAAsB,CAAC;SAChC,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;KACpC,CAAC;CACH;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,OAAO,CAAC,EAAE;QACnB,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACvC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,CACF,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,eAAe,CAAC,WAAW,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GACvE,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACpF,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAqB,SAAQ,kBAAkB;IAC9D,+EAA+E;IAC/E,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B;;;;OAIG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAqBD,eAAO,MAAM,aAAa,EAAE,WAsI3B,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { detectProxyUrl, noProxyToPlaywrightBypass } from "../proxy.js";
|
|
1
2
|
import { contentHash, parseHtmlDocument } from "./_html-common.js";
|
|
2
3
|
/**
|
|
3
4
|
* `kind: html-js` adapter — Playwright-rendered HTML scraping (ADR-0010).
|
|
@@ -29,6 +30,18 @@ import { contentHash, parseHtmlDocument } from "./_html-common.js";
|
|
|
29
30
|
* Chromium. The import is therefore `await import("playwright")` and resolves
|
|
30
31
|
* lazily on the first `html-js` fetch; missing-module errors are translated
|
|
31
32
|
* into a user-friendly install hint.
|
|
33
|
+
*
|
|
34
|
+
* ## Proxy auto-injection
|
|
35
|
+
*
|
|
36
|
+
* Unlike Node's built-in `fetch()` (which honors `HTTPS_PROXY` once
|
|
37
|
+
* `--use-env-proxy` is on), Playwright's `chromium.launch()` does NOT read
|
|
38
|
+
* proxy env vars automatically. The adapter probes `HTTPS_PROXY` /
|
|
39
|
+
* `HTTP_PROXY` / `ALL_PROXY` on every fetch and forwards the URL through
|
|
40
|
+
* `launch({ proxy: { server, bypass } })`. `NO_PROXY` is translated from
|
|
41
|
+
* the Node convention (`,`-separated, leading-dot subdomain match) to the
|
|
42
|
+
* Playwright convention (`;`-separated, `*.example.com` glob) by
|
|
43
|
+
* `noProxyToPlaywrightBypass`. In unproxied environments the `proxy` field
|
|
44
|
+
* is omitted entirely so launch behavior is bit-identical to before.
|
|
32
45
|
*/
|
|
33
46
|
/**
|
|
34
47
|
* Default per-step timeout in ms when `Source.js?.timeout` is omitted.
|
|
@@ -36,6 +49,15 @@ import { contentHash, parseHtmlDocument } from "./_html-common.js";
|
|
|
36
49
|
* going through schema parse) still get the documented behavior.
|
|
37
50
|
*/
|
|
38
51
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
52
|
+
/**
|
|
53
|
+
* Default delay before the adapter emits a `Still waiting for "<sel>"…`
|
|
54
|
+
* reminder during `waitForSelector` (#198). Chosen at ~33% of the default
|
|
55
|
+
* 30s timeout so the user gets a heads-up well before the actual timeout
|
|
56
|
+
* fires; the reminder is purely informational and never causes the wait
|
|
57
|
+
* itself to abort. Override via `HtmlJsAdapterOptions.stillWaitingMs` in
|
|
58
|
+
* tests.
|
|
59
|
+
*/
|
|
60
|
+
const DEFAULT_STILL_WAITING_MS = 10_000;
|
|
39
61
|
/**
|
|
40
62
|
* Default Playwright `page.goto()` waitUntil mode. `networkidle` is the
|
|
41
63
|
* safest default for SPA / CSR pages where item data arrives via XHR after
|
|
@@ -75,9 +97,41 @@ export const htmlJsAdapter = {
|
|
|
75
97
|
const playwright = options.playwright ?? (await loadPlaywright());
|
|
76
98
|
const previous = options.state;
|
|
77
99
|
const fetchedAt = new Date().toISOString();
|
|
100
|
+
// Progress reporter is optional — callers that did not opt in see the
|
|
101
|
+
// pre-#198 no-op behavior. Marker names follow ADR-0015 D4
|
|
102
|
+
// ("Chromium launching" / "Page navigated to <url>" etc.) verbatim so
|
|
103
|
+
// the user-guide stays accurate without per-call docstrings.
|
|
104
|
+
const progress = options.onProgress;
|
|
105
|
+
const stillWaitingMs = options.stillWaitingMs ?? DEFAULT_STILL_WAITING_MS;
|
|
106
|
+
// Probe proxy env BEFORE launch. Playwright's `BrowserType.launch()` does
|
|
107
|
+
// not honor `HTTPS_PROXY` / `HTTP_PROXY` automatically (unlike Node's
|
|
108
|
+
// `--use-env-proxy` flag for built-in `fetch()`) — we must pass the URL
|
|
109
|
+
// through `launch({ proxy: { server, bypass } })` explicitly. When no
|
|
110
|
+
// proxy env is set, omit the field entirely so the launch path is bit-
|
|
111
|
+
// identical to the pre-proxy behavior in unproxied environments.
|
|
112
|
+
const env = options.env ?? process.env;
|
|
113
|
+
const detection = detectProxyUrl(env);
|
|
114
|
+
// Honor `NO_PROXY` end-to-end so internal hosts skip the corporate
|
|
115
|
+
// proxy. Computed once even when proxy detection fails — the helper is
|
|
116
|
+
// pure and the result is discarded with `detection`, which is cheaper
|
|
117
|
+
// than guarding the call site.
|
|
118
|
+
const bypass = noProxyToPlaywrightBypass(env.NO_PROXY ?? env.no_proxy);
|
|
119
|
+
const proxyOption = detection
|
|
120
|
+
? {
|
|
121
|
+
server: detection.url,
|
|
122
|
+
// Spreading `undefined` is a no-op so the `bypass` field is absent
|
|
123
|
+
// when NO_PROXY is unset (Playwright treats omitted vs `""`
|
|
124
|
+
// slightly differently — we want omitted).
|
|
125
|
+
...(bypass !== undefined ? { bypass } : {}),
|
|
126
|
+
}
|
|
127
|
+
: undefined;
|
|
78
128
|
// Hardening: headless is forced true. Even if a future Playwright default
|
|
79
129
|
// changes, the adapter pins it explicitly here.
|
|
80
|
-
|
|
130
|
+
progress?.phase("Launching Chromium…");
|
|
131
|
+
const browser = await playwright.chromium.launch({
|
|
132
|
+
headless: true,
|
|
133
|
+
...(proxyOption ? { proxy: proxyOption } : {}),
|
|
134
|
+
});
|
|
81
135
|
let html;
|
|
82
136
|
try {
|
|
83
137
|
// Hardening: fresh context per fetch (no SW / IndexedDB / localStorage
|
|
@@ -90,8 +144,37 @@ export const htmlJsAdapter = {
|
|
|
90
144
|
try {
|
|
91
145
|
const page = await context.newPage();
|
|
92
146
|
try {
|
|
147
|
+
progress?.phase(`Navigating to ${source.url}…`);
|
|
93
148
|
await page.goto(source.url, { waitUntil, timeout });
|
|
94
|
-
|
|
149
|
+
progress?.phase(`Waiting for selector "${waitFor}" (timeout: ${timeout}ms)…`);
|
|
150
|
+
// Long-running selector waits warrant a `Still waiting…` reminder
|
|
151
|
+
// so the user knows the adapter is alive (not hung). The timer is
|
|
152
|
+
// set up only when a progress reporter is wired so we do not keep
|
|
153
|
+
// an idle interval running in the no-progress fast path. `unref()`
|
|
154
|
+
// (when supported) lets the event loop exit if the underlying
|
|
155
|
+
// wait resolves quickly — we still clear the timer below in
|
|
156
|
+
// `finally`.
|
|
157
|
+
let stillWaitingTimer;
|
|
158
|
+
if (progress && stillWaitingMs > 0) {
|
|
159
|
+
const startedAt = Date.now();
|
|
160
|
+
stillWaitingTimer = setTimeout(() => {
|
|
161
|
+
const elapsed = Date.now() - startedAt;
|
|
162
|
+
const mm = String(Math.floor(elapsed / 60_000)).padStart(2, "0");
|
|
163
|
+
const ss = String(Math.floor((elapsed % 60_000) / 1000)).padStart(2, "0");
|
|
164
|
+
progress.phase(`Still waiting for "${waitFor}"… [${mm}:${ss}]`);
|
|
165
|
+
}, stillWaitingMs);
|
|
166
|
+
if (typeof stillWaitingTimer.unref === "function") {
|
|
167
|
+
stillWaitingTimer.unref();
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
try {
|
|
171
|
+
await page.waitForSelector(waitFor, { timeout });
|
|
172
|
+
}
|
|
173
|
+
finally {
|
|
174
|
+
if (stillWaitingTimer)
|
|
175
|
+
clearTimeout(stillWaitingTimer);
|
|
176
|
+
}
|
|
177
|
+
progress?.phase("Capturing page content…");
|
|
95
178
|
html = await page.content();
|
|
96
179
|
}
|
|
97
180
|
finally {
|
|
@@ -105,6 +188,7 @@ export const htmlJsAdapter = {
|
|
|
105
188
|
}
|
|
106
189
|
}
|
|
107
190
|
finally {
|
|
191
|
+
progress?.phase("Closing browser…");
|
|
108
192
|
await browser.close();
|
|
109
193
|
}
|
|
110
194
|
// Dedup via content hash stored in the `lastEtag` slot (same convention
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html-js.js","sourceRoot":"","sources":["../../../src/core/feeds/html-js.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGnE
|
|
1
|
+
{"version":3,"file":"html-js.js","sourceRoot":"","sources":["../../../src/core/feeds/html-js.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC;;;;;;;GAOG;AACH,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC;;;;GAIG;AACH,MAAM,kBAAkB,GAAiC,aAAa,CAAC;AAiFvE;;;GAGG;AACH,KAAK,UAAU,cAAc;IAC3B,IAAI,CAAC;QACH,wEAAwE;QACxE,kEAAkE;QAClE,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAA8B,CAAC;QACtE,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,IAAI,KAAK,CACb,+CAA+C,OAAO,KAAK;YACzD,wEAAwE,CAC3E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAgB;IACxC,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,KAAK,EAAE,MAAc,EAAE,UAAgC,EAAE,EAAE,EAAE;QAClE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACnC,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,EAAE,EAAE,OAAO,IAAI,kBAAkB,CAAC;QAClD,MAAM,SAAS,GAAG,EAAE,EAAE,SAAS,IAAI,kBAAkB,CAAC;QACtD,uEAAuE;QACvE,0EAA0E;QAC1E,MAAM,OAAO,GAAG,EAAE,EAAE,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC;QAE9C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,sEAAsE;QACtE,2DAA2D;QAC3D,sEAAsE;QACtE,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;QACpC,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QAE1E,0EAA0E;QAC1E,sEAAsE;QACtE,wEAAwE;QACxE,sEAAsE;QACtE,uEAAuE;QACvE,iEAAiE;QACjE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;QACvC,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACtC,mEAAmE;QACnE,uEAAuE;QACvE,sEAAsE;QACtE,+BAA+B;QAC/B,MAAM,MAAM,GAAG,yBAAyB,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvE,MAAM,WAAW,GAAG,SAAS;YAC3B,CAAC,CAAC;gBACE,MAAM,EAAE,SAAS,CAAC,GAAG;gBACrB,mEAAmE;gBACnE,4DAA4D;gBAC5D,2CAA2C;gBAC3C,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C;YACH,CAAC,CAAC,SAAS,CAAC;QAEd,0EAA0E;QAC1E,gDAAgD;QAChD,QAAQ,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC/C,QAAQ,EAAE,IAAI;YACd,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,uEAAuE;YACvE,mEAAmE;YACnE,mEAAmE;YACnE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;gBACvC,eAAe,EAAE,KAAK;gBACtB,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtD,CAAC,CAAC;YACH,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,QAAQ,EAAE,KAAK,CAAC,iBAAiB,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;oBACpD,QAAQ,EAAE,KAAK,CAAC,yBAAyB,OAAO,eAAe,OAAO,MAAM,CAAC,CAAC;oBAC9E,kEAAkE;oBAClE,kEAAkE;oBAClE,kEAAkE;oBAClE,mEAAmE;oBACnE,8DAA8D;oBAC9D,4DAA4D;oBAC5D,aAAa;oBACb,IAAI,iBAA6C,CAAC;oBAClD,IAAI,QAAQ,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;wBACnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC7B,iBAAiB,GAAG,UAAU,CAAC,GAAG,EAAE;4BAClC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;4BACvC,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;4BACjE,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;4BAC1E,QAAQ,CAAC,KAAK,CAAC,sBAAsB,OAAO,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;wBAClE,CAAC,EAAE,cAAc,CAAC,CAAC;wBACnB,IAAI,OAAQ,iBAA4C,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;4BAC7E,iBAA2C,CAAC,KAAK,EAAE,CAAC;wBACvD,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;oBACnD,CAAC;4BAAS,CAAC;wBACT,IAAI,iBAAiB;4BAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;oBACzD,CAAC;oBACD,QAAQ,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAC;oBAC3C,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC9B,CAAC;wBAAS,CAAC;oBACT,mEAAmE;oBACnE,6DAA6D;oBAC7D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,QAAQ,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;YACpC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAED,wEAAwE;QACxE,sEAAsE;QACtE,oEAAoE;QACpE,+BAA+B;QAC/B,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,QAAQ,EAAE,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACpC,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE;oBACL,aAAa,EAAE,SAAS;oBACxB,QAAQ,EAAE,QAAQ;iBACnB;aACF,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACzD,OAAO;YACL,KAAK;YACL,KAAK,EAAE;gBACL,aAAa,EAAE,SAAS;gBACxB,QAAQ,EAAE,QAAQ;aACnB;SACF,CAAC;IACJ,CAAC;CACF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/html.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/html.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAiC,MAAM,YAAY,CAAC;AAK7E,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AA8CtD,eAAO,MAAM,WAAW,EAAE,WAmEzB,CAAC"}
|
package/dist/core/feeds/html.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { fetchWithRetry } from "./_fetch.js";
|
|
1
2
|
import { CONTENT_HASH_PREFIX, contentHash, parseHtmlDocument } from "./_html-common.js";
|
|
2
3
|
// Re-export shared primitives so existing imports (and tests) that pulled
|
|
3
4
|
// `parseHtmlDocument` from this module keep working after the
|
|
@@ -24,7 +25,7 @@ async function fetchHtml(url, fetchImpl, options = {}) {
|
|
|
24
25
|
}
|
|
25
26
|
if (options.lastModified)
|
|
26
27
|
headers["if-modified-since"] = options.lastModified;
|
|
27
|
-
const response = await fetchImpl
|
|
28
|
+
const response = await fetchWithRetry(fetchImpl, url, { headers, signal: options.signal });
|
|
28
29
|
const etag = response.headers.get("etag");
|
|
29
30
|
const lastModified = response.headers.get("last-modified");
|
|
30
31
|
if (response.status === 304) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html.js","sourceRoot":"","sources":["../../../src/core/feeds/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGxF,0EAA0E;AAC1E,8DAA8D;AAC9D,8DAA8D;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,UAAU,GAAG,2DAA2D,CAAC;AAE/E;;;;;;GAMG;AACH,KAAK,UAAU,SAAS,CACtB,GAAW,EACX,SAAoB,EACpB,UAA0E,EAAE;IAO5E,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,mDAAmD;QAC3D,YAAY,EAAE,UAAU;KACzB,CAAC;IACF,0EAA0E;IAC1E,2EAA2E;IAC3E,WAAW;IACX,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAClE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1C,CAAC;IACD,IAAI,OAAO,CAAC,YAAY;QAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAE9E,MAAM,QAAQ,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"html.js","sourceRoot":"","sources":["../../../src/core/feeds/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGxF,0EAA0E;AAC1E,8DAA8D;AAC9D,8DAA8D;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,UAAU,GAAG,2DAA2D,CAAC;AAE/E;;;;;;GAMG;AACH,KAAK,UAAU,SAAS,CACtB,GAAW,EACX,SAAoB,EACpB,UAA0E,EAAE;IAO5E,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,mDAAmD;QAC3D,YAAY,EAAE,UAAU;KACzB,CAAC;IACF,0EAA0E;IAC1E,2EAA2E;IAC3E,WAAW;IACX,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAClE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1C,CAAC;IACD,IAAI,OAAO,CAAC,YAAY;QAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAE9E,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3F,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;IACvD,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;AAC/D,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAgB;IACtC,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,KAAK,EAAE,MAAc,EAAE,UAA8B,EAAE,EAAE,EAAE;QAChE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,IAAK,UAAU,CAAC,KAA8B,CAAC;QAC9E,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,SAAS,EAAE;YACtD,IAAI,EAAE,QAAQ,EAAE,QAAQ;YACxB,YAAY,EAAE,QAAQ,EAAE,YAAY;SACrC,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE;oBACL,aAAa,EAAE,SAAS;oBACxB,iEAAiE;oBACjE,oEAAoE;oBACpE,QAAQ,EAAE,QAAQ,CAAC,IAAI,IAAI,QAAQ,EAAE,QAAQ;oBAC7C,qEAAqE;oBACrE,iEAAiE;oBACjE,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,EAAE,YAAY;iBAC9D;aACF,CAAC;QACJ,CAAC;QAED,yEAAyE;QACzE,uEAAuE;QACvE,wDAAwD;QACxD,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,cAAc,GAAG,QAAQ,EAAE,QAAQ,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,cAAc,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE;oBACL,aAAa,EAAE,SAAS;oBACxB,QAAQ,EAAE,QAAQ;oBAClB,iEAAiE;oBACjE,kEAAkE;oBAClE,oCAAoC;oBACpC,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,EAAE,YAAY;iBAC9D;aACF,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAClE,OAAO;YACL,KAAK;YACL,KAAK,EAAE;gBACL,aAAa,EAAE,SAAS;gBACxB,qEAAqE;gBACrE,wDAAwD;gBACxD,QAAQ,EAAE,QAAQ,CAAC,IAAI,IAAI,QAAQ;gBACnC,oEAAoE;gBACpE,mEAAmE;gBACnE,yDAAyD;gBACzD,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,EAAE,YAAY;aAC9D;SACF,CAAC;IACJ,CAAC;CACF,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Source } from "../../schemas/index.js";
|
|
2
2
|
import type { FeedAdapter } from "./types.js";
|
|
3
3
|
export declare function getFeedAdapter(kind: Source["kind"]): FeedAdapter;
|
|
4
|
-
export type { FeedAdapter, FeedAdapterOptions, FeedFetchResult, FetchLike } from "./types.js";
|
|
4
|
+
export type { FeedAdapter, FeedAdapterOptions, FeedFetchDiag, FeedFetchResult, FetchLike, } from "./types.js";
|
|
5
5
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAQrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAY9C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,WAAW,CAMhE;AAED,YAAY,EACV,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,SAAS,GACV,MAAM,YAAY,CAAC"}
|
package/dist/core/feeds/index.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { githubReleasesAdapter } from "./github-releases.js";
|
|
2
2
|
import { htmlAdapter } from "./html.js";
|
|
3
3
|
import { htmlJsAdapter } from "./html-js.js";
|
|
4
|
+
import { jsonApiAdapter } from "./json-api.js";
|
|
5
|
+
import { jsonFeedAdapter } from "./json-feed.js";
|
|
4
6
|
import { npmRegistryAdapter } from "./npm-registry.js";
|
|
5
7
|
import { rssAdapter } from "./rss.js";
|
|
6
8
|
const adapters = new Map([
|
|
@@ -9,6 +11,8 @@ const adapters = new Map([
|
|
|
9
11
|
[htmlJsAdapter.kind, htmlJsAdapter],
|
|
10
12
|
[githubReleasesAdapter.kind, githubReleasesAdapter],
|
|
11
13
|
[npmRegistryAdapter.kind, npmRegistryAdapter],
|
|
14
|
+
[jsonFeedAdapter.kind, jsonFeedAdapter],
|
|
15
|
+
[jsonApiAdapter.kind, jsonApiAdapter],
|
|
12
16
|
]);
|
|
13
17
|
export function getFeedAdapter(kind) {
|
|
14
18
|
const adapter = adapters.get(kind);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/feeds/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAA8B;IACpD,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;IAC7B,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC;IAC/B,CAAC,aAAa,CAAC,IAAI,EAAE,aAAa,CAAC;IACnC,CAAC,qBAAqB,CAAC,IAAI,EAAE,qBAAqB,CAAC;IACnD,CAAC,kBAAkB,CAAC,IAAI,EAAE,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/feeds/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAA8B;IACpD,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;IAC7B,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC;IAC/B,CAAC,aAAa,CAAC,IAAI,EAAE,aAAa,CAAC;IACnC,CAAC,qBAAqB,CAAC,IAAI,EAAE,qBAAqB,CAAC;IACnD,CAAC,kBAAkB,CAAC,IAAI,EAAE,kBAAkB,CAAC;IAC7C,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC;IACvC,CAAC,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC;CACtC,CAAC,CAAC;AAEH,MAAM,UAAU,cAAc,CAAC,IAAoB;IACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-api.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/json-api.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAgD,MAAM,YAAY,CAAC;AAyhB5F,eAAO,MAAM,cAAc,EAAE,WAqR5B,CAAC"}
|