@ozzylabs/feedradar 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +19 -0
- package/README.md +21 -1
- package/dist/cli/doctor.d.ts +20 -0
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +291 -2
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/respawn.d.ts +53 -0
- package/dist/cli/respawn.d.ts.map +1 -0
- package/dist/cli/respawn.js +120 -0
- package/dist/cli/respawn.js.map +1 -0
- package/dist/core/feeds/_fetch.d.ts +93 -0
- package/dist/core/feeds/_fetch.d.ts.map +1 -0
- package/dist/core/feeds/_fetch.js +182 -0
- package/dist/core/feeds/_fetch.js.map +1 -0
- package/dist/core/feeds/github-api.d.ts.map +1 -1
- package/dist/core/feeds/github-api.js +2 -1
- package/dist/core/feeds/github-api.js.map +1 -1
- package/dist/core/feeds/html-js.d.ts +21 -0
- package/dist/core/feeds/html-js.d.ts.map +1 -1
- package/dist/core/feeds/html-js.js +39 -1
- package/dist/core/feeds/html-js.js.map +1 -1
- package/dist/core/feeds/html.d.ts.map +1 -1
- package/dist/core/feeds/html.js +2 -1
- package/dist/core/feeds/html.js.map +1 -1
- package/dist/core/feeds/npm-registry.d.ts.map +1 -1
- package/dist/core/feeds/npm-registry.js +2 -1
- package/dist/core/feeds/npm-registry.js.map +1 -1
- package/dist/core/feeds/rss.d.ts.map +1 -1
- package/dist/core/feeds/rss.js +2 -1
- package/dist/core/feeds/rss.js.map +1 -1
- package/dist/core/proxy.d.ts +87 -0
- package/dist/core/proxy.d.ts.map +1 -0
- package/dist/core/proxy.js +146 -0
- package/dist/core/proxy.js.map +1 -0
- package/dist/index.js +17 -4
- package/dist/index.js.map +1 -1
- package/dist/templates/workflows/watch.yaml +5 -1
- package/package.json +2 -2
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { detectProxyUrl, mergeNodeOptions } from "../core/proxy.js";
|
|
3
|
+
/**
|
|
4
|
+
* Sentinel env var. Set on the **child** to signal that the current process is
|
|
5
|
+
* already the respawned instance — without this, the bin would respawn itself
|
|
6
|
+
* indefinitely.
|
|
7
|
+
*/
|
|
8
|
+
export const RESPAWN_SENTINEL = "RADAR_PROXY_RESPAWNED";
|
|
9
|
+
/**
|
|
10
|
+
* User opt-out. Setting `RADAR_AUTO_PROXY=0` disables the self-respawn entirely
|
|
11
|
+
* (useful when the user wants to manage `NODE_OPTIONS` themselves, or when the
|
|
12
|
+
* proxy env vars are present but `radar` should bypass them for some reason).
|
|
13
|
+
*/
|
|
14
|
+
export const OPT_OUT_ENV = "RADAR_AUTO_PROXY";
|
|
15
|
+
/**
|
|
16
|
+
* Flag injected into `NODE_OPTIONS`. Tells Node to honor `HTTPS_PROXY` /
|
|
17
|
+
* `HTTP_PROXY` for the built-in `fetch()` / `undici` global agent. Available
|
|
18
|
+
* in Node 22.21+ and 24.5+ (see `engines.node` in package.json).
|
|
19
|
+
*/
|
|
20
|
+
export const PROXY_FLAG = "--use-env-proxy";
|
|
21
|
+
/**
|
|
22
|
+
* Env hint propagated to spawned subprocesses (agent CLI helpers). Mirrors
|
|
23
|
+
* Node's own convention so any tooling that reads the var sees a consistent
|
|
24
|
+
* signal that the user wants proxy-aware HTTP.
|
|
25
|
+
*/
|
|
26
|
+
export const NODE_USE_ENV_PROXY = "NODE_USE_ENV_PROXY";
|
|
27
|
+
/**
|
|
28
|
+
* Decide whether to respawn, and if so, do it.
|
|
29
|
+
*
|
|
30
|
+
* Returns synchronously with `{ respawned: true }` after spawning the child
|
|
31
|
+
* and wiring up signal / exit propagation. Callers MUST treat that as a
|
|
32
|
+
* terminal state and not continue executing the normal CLI path; otherwise
|
|
33
|
+
* both the parent and child would race to write to stdout. The opposite
|
|
34
|
+
* (`respawned: false`) is the green-light to run the CLI inline.
|
|
35
|
+
*/
|
|
36
|
+
export function maybeRespawnForProxy(deps) {
|
|
37
|
+
const { env, argv, execPath, spawn: spawnImpl = spawn, onSignal = (sig, h) => process.on(sig, h), exit = (code) => process.exit(code), warn = (m) => console.warn(m), } = deps;
|
|
38
|
+
// Centralized opt-out check. Documented as `RADAR_AUTO_PROXY=0`, but any
|
|
39
|
+
// falsy-looking value ("0" / "false" / "off") disables the feature for
|
|
40
|
+
// user ergonomics. Empty string is treated as "not set" (env shells often
|
|
41
|
+
// collapse unset vars to "").
|
|
42
|
+
const optOut = env[OPT_OUT_ENV];
|
|
43
|
+
if (optOut !== undefined && optOut !== "") {
|
|
44
|
+
if (optOut === "0" || optOut.toLowerCase() === "false" || optOut.toLowerCase() === "off") {
|
|
45
|
+
return { respawned: false, reason: "opt-out" };
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// Sentinel: the child of a previous respawn must not respawn again,
|
|
49
|
+
// otherwise NODE_OPTIONS grows by one token per generation.
|
|
50
|
+
if (env[RESPAWN_SENTINEL] === "1") {
|
|
51
|
+
return { respawned: false, reason: "already-respawned" };
|
|
52
|
+
}
|
|
53
|
+
const detection = detectProxyUrl(env);
|
|
54
|
+
if (!detection)
|
|
55
|
+
return { respawned: false, reason: "no-proxy" };
|
|
56
|
+
// `--use-env-proxy` ignores `ALL_PROXY`. Warn the user that respawning
|
|
57
|
+
// alone won't make fetch hit the proxy — they need HTTPS_PROXY or HTTP_PROXY.
|
|
58
|
+
// We still respawn so any *downstream* helper that reads ALL_PROXY benefits,
|
|
59
|
+
// but the warning prevents silent failure when the user expected fetch to
|
|
60
|
+
// pick it up.
|
|
61
|
+
if (detection.allProxyOnly) {
|
|
62
|
+
warn("radar: ALL_PROXY is set but HTTPS_PROXY / HTTP_PROXY is not. " +
|
|
63
|
+
"Node's --use-env-proxy ignores ALL_PROXY; set HTTPS_PROXY=<url> " +
|
|
64
|
+
"(and/or HTTP_PROXY) so fetch() can route through the proxy.");
|
|
65
|
+
}
|
|
66
|
+
const nextEnv = {
|
|
67
|
+
...env,
|
|
68
|
+
NODE_OPTIONS: mergeNodeOptions(env.NODE_OPTIONS, PROXY_FLAG),
|
|
69
|
+
[NODE_USE_ENV_PROXY]: "1",
|
|
70
|
+
[RESPAWN_SENTINEL]: "1",
|
|
71
|
+
};
|
|
72
|
+
// argv[0] = node binary, argv[1] = bin script (radar). We respawn with the
|
|
73
|
+
// same script and forward the user's original arguments untouched.
|
|
74
|
+
const child = spawnImpl(execPath, argv.slice(1), {
|
|
75
|
+
env: nextEnv,
|
|
76
|
+
stdio: "inherit",
|
|
77
|
+
});
|
|
78
|
+
forwardSignals(child, onSignal);
|
|
79
|
+
child.on("exit", (code, signal) => {
|
|
80
|
+
if (signal) {
|
|
81
|
+
// Re-raise the signal on ourselves so our exit status matches the
|
|
82
|
+
// child's (Node maps signal exits to 128 + sig#). Best-effort: if the
|
|
83
|
+
// platform refuses, fall back to a generic non-zero code.
|
|
84
|
+
try {
|
|
85
|
+
process.kill(process.pid, signal);
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
exit(1);
|
|
89
|
+
}
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
exit(code ?? 0);
|
|
93
|
+
});
|
|
94
|
+
child.on("error", (err) => {
|
|
95
|
+
warn(`radar: failed to respawn with ${PROXY_FLAG}: ${err.message}`);
|
|
96
|
+
exit(1);
|
|
97
|
+
});
|
|
98
|
+
return { respawned: true, reason: "spawned" };
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Forward terminal signals to the child so Ctrl-C / `kill` propagate cleanly.
|
|
102
|
+
* Without this, the parent would intercept SIGINT and the child (which owns
|
|
103
|
+
* stdin/stdout via `stdio: inherit`) would stay alive until manually killed.
|
|
104
|
+
*/
|
|
105
|
+
function forwardSignals(child, onSignal) {
|
|
106
|
+
const signals = ["SIGINT", "SIGTERM", "SIGHUP"];
|
|
107
|
+
for (const sig of signals) {
|
|
108
|
+
onSignal(sig, () => {
|
|
109
|
+
// child.kill may throw if the child is already gone; swallow so we
|
|
110
|
+
// don't crash on the very last tick before exit.
|
|
111
|
+
try {
|
|
112
|
+
child.kill(sig);
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
// ignore
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=respawn.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"respawn.js","sourceRoot":"","sources":["../../src/cli/respawn.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpE;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;AAExD;;;;GAIG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,kBAAkB,CAAC;AAE9C;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,iBAAiB,CAAC;AAE5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,oBAAoB,CAAC;AAqBvD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAiB;IACpD,MAAM,EACJ,GAAG,EACH,IAAI,EACJ,QAAQ,EACR,KAAK,EAAE,SAAS,GAAG,KAAK,EACxB,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,EACzC,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EACnC,IAAI,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAC9B,GAAG,IAAI,CAAC;IAET,yEAAyE;IACzE,uEAAuE;IACvE,0EAA0E;IAC1E,8BAA8B;IAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAC1C,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,KAAK,EAAE,CAAC;YACzF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QACjD,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,4DAA4D;IAC5D,IAAI,GAAG,CAAC,gBAAgB,CAAC,KAAK,GAAG,EAAE,CAAC;QAClC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC3D,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,CAAC,SAAS;QAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAEhE,uEAAuE;IACvE,8EAA8E;IAC9E,6EAA6E;IAC7E,0EAA0E;IAC1E,cAAc;IACd,IAAI,SAAS,CAAC,YAAY,EAAE,CAAC;QAC3B,IAAI,CACF,+DAA+D;YAC7D,kEAAkE;YAClE,6DAA6D,CAChE,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAsB;QACjC,GAAG,GAAG;QACN,YAAY,EAAE,gBAAgB,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC;QAC5D,CAAC,kBAAkB,CAAC,EAAE,GAAG;QACzB,CAAC,gBAAgB,CAAC,EAAE,GAAG;KACxB,CAAC;IAEF,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QAC/C,GAAG,EAAE,OAAO;QACZ,KAAK,EAAE,SAAS;KACjB,CAAC,CAAC;IAEH,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAEhC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;QAChC,IAAI,MAAM,EAAE,CAAC;YACX,kEAAkE;YAClE,sEAAsE;YACtE,0DAA0D;YAC1D,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,CAAC,CAAC,CAAC;YACV,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACxB,IAAI,CAAC,iCAAiC,UAAU,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,IAAI,CAAC,CAAC,CAAC,CAAC;IACV,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AAChD,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CACrB,KAAmB,EACnB,QAA+D;IAE/D,MAAM,OAAO,GAAqB,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAClE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE;YACjB,mEAAmE;YACnE,iDAAiD;YACjD,IAAI,CAAC;gBACH,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { FetchLike } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Shared HTTP fetch wrapper for every feed adapter (`rss` / `html` /
|
|
4
|
+
* `npm-registry` / `github-releases`).
|
|
5
|
+
*
|
|
6
|
+
* Why a wrapper instead of having each adapter call `fetch` directly:
|
|
7
|
+
* - Proxy environments and flaky public RSS endpoints regularly hang or 5xx,
|
|
8
|
+
* so a default 30s **timeout** and a small **retry loop** for transient
|
|
9
|
+
* errors are needed everywhere — not just one adapter (issue #165).
|
|
10
|
+
* - 4xx responses (404 / 401 / 403) are permanent: retrying them just burns
|
|
11
|
+
* the user's GitHub rate-limit / wall time. We only retry transient signals.
|
|
12
|
+
* - Each adapter still passes its own `headers` and reads its own response
|
|
13
|
+
* body, so the wrapper stays a thin "fetch + timeout + retry" layer rather
|
|
14
|
+
* than a request-builder. The `options.fetch` test seam keeps working
|
|
15
|
+
* because the wrapper takes a `FetchLike` from the caller; mocks injected
|
|
16
|
+
* by tests flow through unchanged.
|
|
17
|
+
*
|
|
18
|
+
* Default tuning (see `resolveFetchConfig`):
|
|
19
|
+
* - Timeout: 30s per attempt.
|
|
20
|
+
* - Retries: 2 (so up to 3 attempts total).
|
|
21
|
+
* - Backoff: 200ms → 800ms (×4 exponential).
|
|
22
|
+
*
|
|
23
|
+
* Both timeout and retries can be overridden via env vars `RADAR_FETCH_TIMEOUT_MS`
|
|
24
|
+
* and `RADAR_FETCH_RETRIES`. We resolve them at call time (not module load)
|
|
25
|
+
* so tests can mutate `process.env` between runs without import cache games.
|
|
26
|
+
*/
|
|
27
|
+
/** Default per-attempt timeout in milliseconds. */
|
|
28
|
+
export declare const DEFAULT_FETCH_TIMEOUT_MS = 30000;
|
|
29
|
+
/** Default number of retries after the initial attempt. */
|
|
30
|
+
export declare const DEFAULT_FETCH_RETRIES = 2;
|
|
31
|
+
/** Initial backoff in ms; doubled-and-quadrupled each retry (200 → 800 → ...). */
|
|
32
|
+
export declare const DEFAULT_FETCH_BACKOFF_BASE_MS = 200;
|
|
33
|
+
/** Multiplier between successive backoff attempts. */
|
|
34
|
+
export declare const DEFAULT_FETCH_BACKOFF_FACTOR = 4;
|
|
35
|
+
/**
|
|
36
|
+
* Resolved fetch configuration. Exposed so tests can assert env parsing
|
|
37
|
+
* behavior without touching the wrapper.
|
|
38
|
+
*/
|
|
39
|
+
export interface FetchConfig {
|
|
40
|
+
timeoutMs: number;
|
|
41
|
+
retries: number;
|
|
42
|
+
backoffBaseMs: number;
|
|
43
|
+
backoffFactor: number;
|
|
44
|
+
}
|
|
45
|
+
/** Optional overrides that take precedence over env-var defaults. */
|
|
46
|
+
export interface FetchWithRetryOptions {
|
|
47
|
+
timeoutMs?: number;
|
|
48
|
+
retries?: number;
|
|
49
|
+
backoffBaseMs?: number;
|
|
50
|
+
backoffFactor?: number;
|
|
51
|
+
/** Caller-provided abort signal (composed with the per-attempt timeout signal). */
|
|
52
|
+
signal?: AbortSignal;
|
|
53
|
+
/**
|
|
54
|
+
* Sleep function for backoff. Injected by tests so the retry test does not
|
|
55
|
+
* stall the suite waiting on a real timer.
|
|
56
|
+
*/
|
|
57
|
+
sleep?: (ms: number) => Promise<void>;
|
|
58
|
+
/** Override env for `RADAR_FETCH_*` parsing. Defaults to `process.env`. */
|
|
59
|
+
env?: NodeJS.ProcessEnv;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Build a `FetchConfig` from env vars + explicit overrides.
|
|
63
|
+
*
|
|
64
|
+
* Precedence: explicit `options` → env vars → defaults. Negative / non-numeric
|
|
65
|
+
* env values fall through to defaults so a typo never breaks fetching.
|
|
66
|
+
*/
|
|
67
|
+
export declare function resolveFetchConfig(options?: FetchWithRetryOptions): FetchConfig;
|
|
68
|
+
/** The fetch response shape adapters consume (matches `FetchLike` return). */
|
|
69
|
+
type FetchResponse = Awaited<ReturnType<FetchLike>>;
|
|
70
|
+
/**
|
|
71
|
+
* Issue an HTTP request via `fetchImpl` with timeout + retry semantics.
|
|
72
|
+
*
|
|
73
|
+
* The caller still owns the request shape (headers / method) and the response
|
|
74
|
+
* body — we only retry the network round-trip. 5xx responses are returned to
|
|
75
|
+
* the retry loop, but on the **final** attempt the response is returned to
|
|
76
|
+
* the caller as-is (so the caller's adapter-specific error message wins).
|
|
77
|
+
*
|
|
78
|
+
* Retries kick in for:
|
|
79
|
+
* - Thrown errors that look transient (ECONNRESET / ETIMEDOUT / ENETUNREACH
|
|
80
|
+
* / EAI_AGAIN, or our own timeout AbortError).
|
|
81
|
+
* - HTTP 5xx responses.
|
|
82
|
+
*
|
|
83
|
+
* Retries do NOT happen for:
|
|
84
|
+
* - HTTP 4xx (permanent — fail fast so users see real config issues).
|
|
85
|
+
* - HTTP 2xx / 3xx (already success / handled by the adapter).
|
|
86
|
+
* - Caller-initiated aborts (we re-throw to surface the cancellation).
|
|
87
|
+
*/
|
|
88
|
+
export declare function fetchWithRetry(fetchImpl: FetchLike, url: string | URL, init?: {
|
|
89
|
+
headers?: Record<string, string>;
|
|
90
|
+
signal?: AbortSignal;
|
|
91
|
+
}, options?: FetchWithRetryOptions): Promise<FetchResponse>;
|
|
92
|
+
export {};
|
|
93
|
+
//# sourceMappingURL=_fetch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"_fetch.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/_fetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,mDAAmD;AACnD,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAE/C,2DAA2D;AAC3D,eAAO,MAAM,qBAAqB,IAAI,CAAC;AAEvC,kFAAkF;AAClF,eAAO,MAAM,6BAA6B,MAAM,CAAC;AAEjD,sDAAsD;AACtD,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAE9C;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,qEAAqE;AACrE,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mFAAmF;IACnF,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,2EAA2E;IAC3E,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CACzB;AAUD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,GAAE,qBAA0B,GAAG,WAAW,CAYnF;AAuDD,8EAA8E;AAC9E,KAAK,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;AAEpD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,cAAc,CAClC,SAAS,EAAE,SAAS,EACpB,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,EACrE,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,aAAa,CAAC,CA8CxB"}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared HTTP fetch wrapper for every feed adapter (`rss` / `html` /
|
|
3
|
+
* `npm-registry` / `github-releases`).
|
|
4
|
+
*
|
|
5
|
+
* Why a wrapper instead of having each adapter call `fetch` directly:
|
|
6
|
+
* - Proxy environments and flaky public RSS endpoints regularly hang or 5xx,
|
|
7
|
+
* so a default 30s **timeout** and a small **retry loop** for transient
|
|
8
|
+
* errors are needed everywhere — not just one adapter (issue #165).
|
|
9
|
+
* - 4xx responses (404 / 401 / 403) are permanent: retrying them just burns
|
|
10
|
+
* the user's GitHub rate-limit / wall time. We only retry transient signals.
|
|
11
|
+
* - Each adapter still passes its own `headers` and reads its own response
|
|
12
|
+
* body, so the wrapper stays a thin "fetch + timeout + retry" layer rather
|
|
13
|
+
* than a request-builder. The `options.fetch` test seam keeps working
|
|
14
|
+
* because the wrapper takes a `FetchLike` from the caller; mocks injected
|
|
15
|
+
* by tests flow through unchanged.
|
|
16
|
+
*
|
|
17
|
+
* Default tuning (see `resolveFetchConfig`):
|
|
18
|
+
* - Timeout: 30s per attempt.
|
|
19
|
+
* - Retries: 2 (so up to 3 attempts total).
|
|
20
|
+
* - Backoff: 200ms → 800ms (×4 exponential).
|
|
21
|
+
*
|
|
22
|
+
* Both timeout and retries can be overridden via env vars `RADAR_FETCH_TIMEOUT_MS`
|
|
23
|
+
* and `RADAR_FETCH_RETRIES`. We resolve them at call time (not module load)
|
|
24
|
+
* so tests can mutate `process.env` between runs without import cache games.
|
|
25
|
+
*/
|
|
26
|
+
/** Default per-attempt timeout in milliseconds. */
|
|
27
|
+
export const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
|
28
|
+
/** Default number of retries after the initial attempt. */
|
|
29
|
+
export const DEFAULT_FETCH_RETRIES = 2;
|
|
30
|
+
/** Initial backoff in ms; doubled-and-quadrupled each retry (200 → 800 → ...). */
|
|
31
|
+
export const DEFAULT_FETCH_BACKOFF_BASE_MS = 200;
|
|
32
|
+
/** Multiplier between successive backoff attempts. */
|
|
33
|
+
export const DEFAULT_FETCH_BACKOFF_FACTOR = 4;
|
|
34
|
+
/** Parse a positive integer env var, falling back to `fallback` on missing / invalid. */
|
|
35
|
+
function parsePositiveInt(value, fallback) {
|
|
36
|
+
if (value == null)
|
|
37
|
+
return fallback;
|
|
38
|
+
const n = Number.parseInt(value, 10);
|
|
39
|
+
if (!Number.isFinite(n) || n < 0)
|
|
40
|
+
return fallback;
|
|
41
|
+
return n;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Build a `FetchConfig` from env vars + explicit overrides.
|
|
45
|
+
*
|
|
46
|
+
* Precedence: explicit `options` → env vars → defaults. Negative / non-numeric
|
|
47
|
+
* env values fall through to defaults so a typo never breaks fetching.
|
|
48
|
+
*/
|
|
49
|
+
export function resolveFetchConfig(options = {}) {
|
|
50
|
+
const env = options.env ?? process.env;
|
|
51
|
+
const timeoutMs = options.timeoutMs ?? parsePositiveInt(env.RADAR_FETCH_TIMEOUT_MS, DEFAULT_FETCH_TIMEOUT_MS);
|
|
52
|
+
const retries = options.retries ?? parsePositiveInt(env.RADAR_FETCH_RETRIES, DEFAULT_FETCH_RETRIES);
|
|
53
|
+
return {
|
|
54
|
+
timeoutMs,
|
|
55
|
+
retries,
|
|
56
|
+
backoffBaseMs: options.backoffBaseMs ?? DEFAULT_FETCH_BACKOFF_BASE_MS,
|
|
57
|
+
backoffFactor: options.backoffFactor ?? DEFAULT_FETCH_BACKOFF_FACTOR,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Node-style error code we want to retry on. Matches the codes the runtime
|
|
62
|
+
* uses when a connection drops mid-flight or the OS routing layer rejects.
|
|
63
|
+
*/
|
|
64
|
+
const TRANSIENT_ERROR_CODES = new Set(["ECONNRESET", "ETIMEDOUT", "ENETUNREACH", "EAI_AGAIN"]);
|
|
65
|
+
/** Whether a thrown error from `fetch` looks like a transient network failure. */
|
|
66
|
+
function isTransientNetworkError(err) {
|
|
67
|
+
if (err == null || typeof err !== "object")
|
|
68
|
+
return false;
|
|
69
|
+
const e = err;
|
|
70
|
+
// Direct Node-style errors (undici occasionally throws `{ code: 'ECONNRESET' }`).
|
|
71
|
+
if (typeof e.code === "string" && TRANSIENT_ERROR_CODES.has(e.code))
|
|
72
|
+
return true;
|
|
73
|
+
// AbortError from our per-attempt timeout — treat as transient so the retry
|
|
74
|
+
// loop gets a chance. Caller-initiated aborts surface differently (their
|
|
75
|
+
// signal already aborted before our `AbortSignal.any` collapsed).
|
|
76
|
+
if (e.name === "TimeoutError" || e.name === "AbortError")
|
|
77
|
+
return true;
|
|
78
|
+
// `fetch` wraps the underlying cause in `TypeError("fetch failed", { cause })`.
|
|
79
|
+
if (e.cause != null)
|
|
80
|
+
return isTransientNetworkError(e.cause);
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
/** Whether an HTTP status code should trigger a retry (5xx is transient). */
|
|
84
|
+
function isTransientStatus(status) {
|
|
85
|
+
return status >= 500 && status <= 599;
|
|
86
|
+
}
|
|
87
|
+
/** Default sleep used between retries; injectable for tests via options.sleep. */
|
|
88
|
+
function defaultSleep(ms) {
|
|
89
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Compose two signals: returns a signal that aborts when either input aborts.
|
|
93
|
+
* Uses `AbortSignal.any` (Node 20.3+ / 22+). Falls back to a manual controller
|
|
94
|
+
* for the legacy path (kept defensive — package engines is `>=22` but we still
|
|
95
|
+
* guard so the wrapper does not hard-depend on the static method).
|
|
96
|
+
*/
|
|
97
|
+
function composeSignals(...signals) {
|
|
98
|
+
const present = signals.filter((s) => s != null);
|
|
99
|
+
if (present.length === 0)
|
|
100
|
+
return undefined;
|
|
101
|
+
if (present.length === 1)
|
|
102
|
+
return present[0];
|
|
103
|
+
if (typeof AbortSignal.any === "function")
|
|
104
|
+
return AbortSignal.any(present);
|
|
105
|
+
const controller = new AbortController();
|
|
106
|
+
for (const s of present) {
|
|
107
|
+
if (s.aborted) {
|
|
108
|
+
controller.abort(s.reason);
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
s.addEventListener("abort", () => controller.abort(s.reason), { once: true });
|
|
112
|
+
}
|
|
113
|
+
return controller.signal;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Issue an HTTP request via `fetchImpl` with timeout + retry semantics.
|
|
117
|
+
*
|
|
118
|
+
* The caller still owns the request shape (headers / method) and the response
|
|
119
|
+
* body — we only retry the network round-trip. 5xx responses are returned to
|
|
120
|
+
* the retry loop, but on the **final** attempt the response is returned to
|
|
121
|
+
* the caller as-is (so the caller's adapter-specific error message wins).
|
|
122
|
+
*
|
|
123
|
+
* Retries kick in for:
|
|
124
|
+
* - Thrown errors that look transient (ECONNRESET / ETIMEDOUT / ENETUNREACH
|
|
125
|
+
* / EAI_AGAIN, or our own timeout AbortError).
|
|
126
|
+
* - HTTP 5xx responses.
|
|
127
|
+
*
|
|
128
|
+
* Retries do NOT happen for:
|
|
129
|
+
* - HTTP 4xx (permanent — fail fast so users see real config issues).
|
|
130
|
+
* - HTTP 2xx / 3xx (already success / handled by the adapter).
|
|
131
|
+
* - Caller-initiated aborts (we re-throw to surface the cancellation).
|
|
132
|
+
*/
|
|
133
|
+
export async function fetchWithRetry(fetchImpl, url, init = {}, options = {}) {
|
|
134
|
+
const config = resolveFetchConfig(options);
|
|
135
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
136
|
+
const callerSignal = options.signal ?? init.signal;
|
|
137
|
+
let lastError;
|
|
138
|
+
let lastResponse;
|
|
139
|
+
// attempts = retries + 1; e.g. retries=2 → 3 total attempts.
|
|
140
|
+
const totalAttempts = config.retries + 1;
|
|
141
|
+
for (let attempt = 0; attempt < totalAttempts; attempt++) {
|
|
142
|
+
// Per-attempt timeout. We rebuild the signal each iteration so a slow
|
|
143
|
+
// attempt does not poison the budget for the next one.
|
|
144
|
+
const timeoutSignal = AbortSignal.timeout(config.timeoutMs);
|
|
145
|
+
const signal = composeSignals(callerSignal, timeoutSignal);
|
|
146
|
+
try {
|
|
147
|
+
const response = await fetchImpl(url, { ...init, signal });
|
|
148
|
+
// Retry transient 5xx; permanent 4xx / 2xx / 3xx fall through to the caller.
|
|
149
|
+
if (isTransientStatus(response.status) && attempt < totalAttempts - 1) {
|
|
150
|
+
lastResponse = response;
|
|
151
|
+
await sleep(backoffDelay(attempt, config));
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
return response;
|
|
155
|
+
}
|
|
156
|
+
catch (err) {
|
|
157
|
+
// Caller-initiated abort short-circuits the retry loop — re-throw so the
|
|
158
|
+
// adapter / consumer can see the cancellation.
|
|
159
|
+
if (callerSignal?.aborted)
|
|
160
|
+
throw err;
|
|
161
|
+
if (isTransientNetworkError(err) && attempt < totalAttempts - 1) {
|
|
162
|
+
lastError = err;
|
|
163
|
+
await sleep(backoffDelay(attempt, config));
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
throw err;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
// We only reach here if every attempt was a retryable 5xx (no exception
|
|
170
|
+
// was thrown). Surface the last response so the adapter's HTTP-status error
|
|
171
|
+
// message wins (the wrapper does not know each adapter's preferred phrasing).
|
|
172
|
+
if (lastResponse)
|
|
173
|
+
return lastResponse;
|
|
174
|
+
// Defensive — should be unreachable, but if every attempt threw and we
|
|
175
|
+
// exited the loop without returning, re-throw the last error.
|
|
176
|
+
throw lastError ?? new Error("fetchWithRetry: exhausted retries with no response");
|
|
177
|
+
}
|
|
178
|
+
/** Compute the backoff delay for the given attempt index (0-based). */
|
|
179
|
+
function backoffDelay(attemptIndex, config) {
|
|
180
|
+
return config.backoffBaseMs * config.backoffFactor ** attemptIndex;
|
|
181
|
+
}
|
|
182
|
+
//# sourceMappingURL=_fetch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"_fetch.js","sourceRoot":"","sources":["../../../src/core/feeds/_fetch.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,mDAAmD;AACnD,MAAM,CAAC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AAE/C,2DAA2D;AAC3D,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAEvC,kFAAkF;AAClF,MAAM,CAAC,MAAM,6BAA6B,GAAG,GAAG,CAAC;AAEjD,sDAAsD;AACtD,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC;AA8B9C,yFAAyF;AACzF,SAAS,gBAAgB,CAAC,KAAyB,EAAE,QAAgB;IACnE,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,QAAQ,CAAC;IACnC,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACrC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,UAAiC,EAAE;IACpE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,SAAS,GACb,OAAO,CAAC,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,sBAAsB,EAAE,wBAAwB,CAAC,CAAC;IAC9F,MAAM,OAAO,GACX,OAAO,CAAC,OAAO,IAAI,gBAAgB,CAAC,GAAG,CAAC,mBAAmB,EAAE,qBAAqB,CAAC,CAAC;IACtF,OAAO;QACL,SAAS;QACT,OAAO;QACP,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,6BAA6B;QACrE,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,4BAA4B;KACrE,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC,CAAC;AAE/F,kFAAkF;AAClF,SAAS,uBAAuB,CAAC,GAAY;IAC3C,IAAI,GAAG,IAAI,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACzD,MAAM,CAAC,GAAG,GAA0D,CAAC;IACrE,kFAAkF;IAClF,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACjF,4EAA4E;IAC5E,yEAAyE;IACzE,kEAAkE;IAClE,IAAI,CAAC,CAAC,IAAI,KAAK,cAAc,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC;IACtE,gFAAgF;IAChF,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI;QAAE,OAAO,uBAAuB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC7D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6EAA6E;AAC7E,SAAS,iBAAiB,CAAC,MAAc;IACvC,OAAO,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,CAAC;AACxC,CAAC;AAED,kFAAkF;AAClF,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CAAC,GAAG,OAAoC;IAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAoB,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACnE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5C,IAAI,OAAO,WAAW,CAAC,GAAG,KAAK,UAAU;QAAE,OAAO,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3E,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC3B,MAAM;QACR,CAAC;QACD,CAAC,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,UAAU,CAAC,MAAM,CAAC;AAC3B,CAAC;AAKD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAoB,EACpB,GAAiB,EACjB,OAAmE,EAAE,EACrE,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAC5C,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IAEnD,IAAI,SAAkB,CAAC;IACvB,IAAI,YAAuC,CAAC;IAE5C,6DAA6D;IAC7D,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC;IACzC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,aAAa,EAAE,OAAO,EAAE,EAAE,CAAC;QACzD,sEAAsE;QACtE,uDAAuD;QACvD,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,cAAc,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAE3D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3D,6EAA6E;YAC7E,IAAI,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,aAAa,GAAG,CAAC,EAAE,CAAC;gBACtE,YAAY,GAAG,QAAQ,CAAC;gBACxB,MAAM,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,yEAAyE;YACzE,+CAA+C;YAC/C,IAAI,YAAY,EAAE,OAAO;gBAAE,MAAM,GAAG,CAAC;YAErC,IAAI,uBAAuB,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,aAAa,GAAG,CAAC,EAAE,CAAC;gBAChE,SAAS,GAAG,GAAG,CAAC;gBAChB,MAAM,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,4EAA4E;IAC5E,8EAA8E;IAC9E,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,uEAAuE;IACvE,8DAA8D;IAC9D,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACrF,CAAC;AAED,uEAAuE;AACvE,SAAS,YAAY,CAAC,YAAoB,EAAE,MAAmB;IAC7D,OAAO,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,YAAY,CAAC;AACrE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github-api.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/github-api.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"github-api.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/github-api.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAY5C,2FAA2F;AAC3F,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;IACf,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uEAAuE;AACvE,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CA+BvD;AAED,gEAAgE;AAChE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED,iFAAiF;AACjF,MAAM,WAAW,oBAAoB;IACnC,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,4FAA4F;IAC5F,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,kDAAkD;AAClD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,yDAAyD;IACzD,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,yDAAyD;AACzD,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gEAAgE;IAChE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,uEAAuE;IACvE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,mBAAmB,CAAC,CA+E9B"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { fetchWithRetry } from "./_fetch.js";
|
|
1
2
|
const USER_AGENT = "feedradar/0.0.0 (+https://github.com/ozzy-labs/feedradar)";
|
|
2
3
|
const GITHUB_API_BASE = "https://api.github.com";
|
|
3
4
|
/**
|
|
@@ -79,7 +80,7 @@ export async function fetchReleases(owner, repo, options = {}) {
|
|
|
79
80
|
if (options.etag)
|
|
80
81
|
headers["if-none-match"] = options.etag;
|
|
81
82
|
const url = buildReleasesUrl(owner, repo);
|
|
82
|
-
const response = await fetchImpl
|
|
83
|
+
const response = await fetchWithRetry(fetchImpl, url, { headers, signal: options.signal });
|
|
83
84
|
const rateLimit = {
|
|
84
85
|
remaining: parseIntHeader(response.headers.get("x-ratelimit-remaining")),
|
|
85
86
|
limit: parseIntHeader(response.headers.get("x-ratelimit-limit")),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github-api.js","sourceRoot":"","sources":["../../../src/core/feeds/github-api.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"github-api.js","sourceRoot":"","sources":["../../../src/core/feeds/github-api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAG7C,MAAM,UAAU,GAAG,2DAA2D,CAAC;AAC/E,MAAM,eAAe,GAAG,wBAAwB,CAAC;AAEjD;;;;GAIG;AACH,MAAM,4BAA4B,GAAG,EAAE,CAAC;AAqBxC;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAE3E,IAAI,SAAS,GAAG,OAAO,CAAC;IACxB,wDAAwD;IACxD,IAAI,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,IAAI,GAAQ,CAAC;QACb,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,qFAAqF;QACrF,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAC/E,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,kFAAkF;IAClF,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC;IAC7C,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,0DAA0D,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,0DAA0D,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAAY;IAC1D,OAAO,GAAG,eAAe,UAAU,kBAAkB,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC;AACtG,CAAC;AAkCD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAa,EACb,IAAY,EACZ,UAAgC,EAAE;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,IAAK,UAAU,CAAC,KAA8B,CAAC;IAC9E,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CACb,gFAAgF,CACjF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IACxD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtD,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,6BAA6B;QACrC,YAAY,EAAE,UAAU;QACxB,sEAAsE;QACtE,sDAAsD;QACtD,sBAAsB,EAAE,YAAY;KACrC,CAAC;IACF,IAAI,KAAK;QAAE,OAAO,CAAC,aAAa,GAAG,UAAU,KAAK,EAAE,CAAC;IACrD,IAAI,OAAO,CAAC,IAAI;QAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1D,MAAM,GAAG,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE3F,MAAM,SAAS,GAAkB;QAC/B,SAAS,EAAE,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACxE,KAAK,EAAE,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAChE,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;KACnE,CAAC;IACF,oBAAoB,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,IAAI,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE1C,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAC3E,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QACzD,kEAAkE;QAClE,0EAA0E;QAC1E,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,IAAI,KAAK,CACb,qDAAqD,KAAK,IAAI,IAAI,GAAG,SAAS,IAAI;YAChF,CAAC,KAAK;gBACJ,CAAC,CAAC,oCAAoC;gBACtC,CAAC,CAAC,4DAA4D,CAAC,CACpE,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,sDAAsD,KAAK,IAAI,IAAI,uBAAuB,CAC3F,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,iCAAiC,QAAQ,CAAC,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,kDAAkD,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC/F,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,qDAAqD,GAAG,SAAS,OAAO,MAAM,EAAE,CACjF,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAChD,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AACpF,CAAC;AAED,yFAAyF;AACzF,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC7D,MAAM,CAAC,GAAG,KAAgC,CAAC;IAC3C,OAAO,CACL,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ;QACxB,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;QAC9B,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;QAC9B,OAAO,CAAC,CAAC,KAAK,KAAK,SAAS;QAC5B,OAAO,CAAC,CAAC,UAAU,KAAK,SAAS,CAClC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAoB;IAC1C,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC;IAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,oBAAoB,CAC3B,SAAwB,EACxB,KAAa,EACb,IAAY,EACZ,aAAsB,EACtB,IAA+B;IAE/B,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC;IACvC,IAAI,SAAS,IAAI,IAAI,IAAI,SAAS,GAAG,4BAA4B;QAAE,OAAO;IAC1E,MAAM,SAAS,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnD,MAAM,QAAQ,GAAG,aAAa;QAC5B,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,6DAA6D,CAAC;IAClE,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACrD,IAAI,CACF,oCAAoC,SAAS,GAAG,SAAS,mBAAmB,KAAK,IAAI,IAAI,GAAG,SAAS,IAAI,QAAQ,EAAE,CACpH,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,OAAsB;IAC7C,IAAI,OAAO,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5C,OAAO,eAAe,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC;AAC9C,CAAC"}
|
|
@@ -6,10 +6,25 @@ import type { FeedAdapter, FeedAdapterOptions } from "./types.js";
|
|
|
6
6
|
* pull in the full Playwright type tree (which is itself an optional peer
|
|
7
7
|
* dep and therefore not guaranteed to be installed in dev).
|
|
8
8
|
*/
|
|
9
|
+
/**
|
|
10
|
+
* Subset of Playwright's `BrowserType.launch()` proxy options used by this
|
|
11
|
+
* adapter. Playwright does NOT auto-honor `HTTPS_PROXY` / `HTTP_PROXY` the way
|
|
12
|
+
* Node's `--use-env-proxy` does for built-in `fetch()` — it requires the proxy
|
|
13
|
+
* to be passed explicitly into launch options. The adapter therefore probes
|
|
14
|
+
* the env (via `detectProxyUrl`) on every fetch and forwards the URL to
|
|
15
|
+
* Chromium so corporate proxies / NO_PROXY rules are respected end-to-end.
|
|
16
|
+
*/
|
|
17
|
+
export interface PlaywrightProxyOptions {
|
|
18
|
+
/** Proxy URL (e.g. `http://proxy.corp:8080`). Playwright resolves SOCKS / HTTP automatically. */
|
|
19
|
+
server: string;
|
|
20
|
+
/** Optional semicolon-separated bypass list (NOT comma — see `noProxyToPlaywrightBypass`). */
|
|
21
|
+
bypass?: string;
|
|
22
|
+
}
|
|
9
23
|
export interface PlaywrightLike {
|
|
10
24
|
chromium: {
|
|
11
25
|
launch(options?: {
|
|
12
26
|
headless?: boolean;
|
|
27
|
+
proxy?: PlaywrightProxyOptions;
|
|
13
28
|
}): Promise<PlaywrightBrowserLike>;
|
|
14
29
|
};
|
|
15
30
|
}
|
|
@@ -45,6 +60,12 @@ export interface PlaywrightPageLike {
|
|
|
45
60
|
export interface HtmlJsAdapterOptions extends FeedAdapterOptions {
|
|
46
61
|
/** Injected Playwright module (tests only). Production uses dynamic import. */
|
|
47
62
|
playwright?: PlaywrightLike;
|
|
63
|
+
/**
|
|
64
|
+
* Injected env (tests only) — production reads `process.env`. Used to probe
|
|
65
|
+
* `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` so the proxy injection branch is
|
|
66
|
+
* deterministically testable without polluting the real process env.
|
|
67
|
+
*/
|
|
68
|
+
env?: NodeJS.ProcessEnv;
|
|
48
69
|
}
|
|
49
70
|
export declare const htmlJsAdapter: FeedAdapter;
|
|
50
71
|
//# sourceMappingURL=html-js.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html-js.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/html-js.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAU,eAAe,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"html-js.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/html-js.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAU,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAGtE,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AA2DlE;;;;;GAKG;AACH;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB;IACrC,iGAAiG;IACjG,MAAM,EAAE,MAAM,CAAC;IACf,8FAA8F;IAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE;QACR,MAAM,CAAC,OAAO,CAAC,EAAE;YACf,QAAQ,CAAC,EAAE,OAAO,CAAC;YACnB,KAAK,CAAC,EAAE,sBAAsB,CAAC;SAChC,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;KACpC,CAAC;CACH;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,OAAO,CAAC,EAAE;QACnB,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACvC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,CACF,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,eAAe,CAAC,WAAW,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GACvE,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACpF,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAqB,SAAQ,kBAAkB;IAC9D,+EAA+E;IAC/E,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B;;;;OAIG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CACzB;AAqBD,eAAO,MAAM,aAAa,EAAE,WAmG3B,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { detectProxyUrl, noProxyToPlaywrightBypass } from "../proxy.js";
|
|
1
2
|
import { contentHash, parseHtmlDocument } from "./_html-common.js";
|
|
2
3
|
/**
|
|
3
4
|
* `kind: html-js` adapter — Playwright-rendered HTML scraping (ADR-0010).
|
|
@@ -29,6 +30,18 @@ import { contentHash, parseHtmlDocument } from "./_html-common.js";
|
|
|
29
30
|
* Chromium. The import is therefore `await import("playwright")` and resolves
|
|
30
31
|
* lazily on the first `html-js` fetch; missing-module errors are translated
|
|
31
32
|
* into a user-friendly install hint.
|
|
33
|
+
*
|
|
34
|
+
* ## Proxy auto-injection
|
|
35
|
+
*
|
|
36
|
+
* Unlike Node's built-in `fetch()` (which honors `HTTPS_PROXY` once
|
|
37
|
+
* `--use-env-proxy` is on), Playwright's `chromium.launch()` does NOT read
|
|
38
|
+
* proxy env vars automatically. The adapter probes `HTTPS_PROXY` /
|
|
39
|
+
* `HTTP_PROXY` / `ALL_PROXY` on every fetch and forwards the URL through
|
|
40
|
+
* `launch({ proxy: { server, bypass } })`. `NO_PROXY` is translated from
|
|
41
|
+
* the Node convention (`,`-separated, leading-dot subdomain match) to the
|
|
42
|
+
* Playwright convention (`;`-separated, `*.example.com` glob) by
|
|
43
|
+
* `noProxyToPlaywrightBypass`. In unproxied environments the `proxy` field
|
|
44
|
+
* is omitted entirely so launch behavior is bit-identical to before.
|
|
32
45
|
*/
|
|
33
46
|
/**
|
|
34
47
|
* Default per-step timeout in ms when `Source.js?.timeout` is omitted.
|
|
@@ -75,9 +88,34 @@ export const htmlJsAdapter = {
|
|
|
75
88
|
const playwright = options.playwright ?? (await loadPlaywright());
|
|
76
89
|
const previous = options.state;
|
|
77
90
|
const fetchedAt = new Date().toISOString();
|
|
91
|
+
// Probe proxy env BEFORE launch. Playwright's `BrowserType.launch()` does
|
|
92
|
+
// not honor `HTTPS_PROXY` / `HTTP_PROXY` automatically (unlike Node's
|
|
93
|
+
// `--use-env-proxy` flag for built-in `fetch()`) — we must pass the URL
|
|
94
|
+
// through `launch({ proxy: { server, bypass } })` explicitly. When no
|
|
95
|
+
// proxy env is set, omit the field entirely so the launch path is bit-
|
|
96
|
+
// identical to the pre-proxy behavior in unproxied environments.
|
|
97
|
+
const env = options.env ?? process.env;
|
|
98
|
+
const detection = detectProxyUrl(env);
|
|
99
|
+
// Honor `NO_PROXY` end-to-end so internal hosts skip the corporate
|
|
100
|
+
// proxy. Computed once even when proxy detection fails — the helper is
|
|
101
|
+
// pure and the result is discarded with `detection`, which is cheaper
|
|
102
|
+
// than guarding the call site.
|
|
103
|
+
const bypass = noProxyToPlaywrightBypass(env.NO_PROXY ?? env.no_proxy);
|
|
104
|
+
const proxyOption = detection
|
|
105
|
+
? {
|
|
106
|
+
server: detection.url,
|
|
107
|
+
// Spreading `undefined` is a no-op so the `bypass` field is absent
|
|
108
|
+
// when NO_PROXY is unset (Playwright treats omitted vs `""`
|
|
109
|
+
// slightly differently — we want omitted).
|
|
110
|
+
...(bypass !== undefined ? { bypass } : {}),
|
|
111
|
+
}
|
|
112
|
+
: undefined;
|
|
78
113
|
// Hardening: headless is forced true. Even if a future Playwright default
|
|
79
114
|
// changes, the adapter pins it explicitly here.
|
|
80
|
-
const browser = await playwright.chromium.launch({
|
|
115
|
+
const browser = await playwright.chromium.launch({
|
|
116
|
+
headless: true,
|
|
117
|
+
...(proxyOption ? { proxy: proxyOption } : {}),
|
|
118
|
+
});
|
|
81
119
|
let html;
|
|
82
120
|
try {
|
|
83
121
|
// Hardening: fresh context per fetch (no SW / IndexedDB / localStorage
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html-js.js","sourceRoot":"","sources":["../../../src/core/feeds/html-js.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGnE
|
|
1
|
+
{"version":3,"file":"html-js.js","sourceRoot":"","sources":["../../../src/core/feeds/html-js.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC;;;;GAIG;AACH,MAAM,kBAAkB,GAAiC,aAAa,CAAC;AAyEvE;;;GAGG;AACH,KAAK,UAAU,cAAc;IAC3B,IAAI,CAAC;QACH,wEAAwE;QACxE,kEAAkE;QAClE,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAA8B,CAAC;QACtE,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,IAAI,KAAK,CACb,+CAA+C,OAAO,KAAK;YACzD,wEAAwE,CAC3E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAgB;IACxC,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,KAAK,EAAE,MAAc,EAAE,UAAgC,EAAE,EAAE,EAAE;QAClE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACnC,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,EAAE,EAAE,OAAO,IAAI,kBAAkB,CAAC;QAClD,MAAM,SAAS,GAAG,EAAE,EAAE,SAAS,IAAI,kBAAkB,CAAC;QACtD,uEAAuE;QACvE,0EAA0E;QAC1E,MAAM,OAAO,GAAG,EAAE,EAAE,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC;QAE9C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3C,0EAA0E;QAC1E,sEAAsE;QACtE,wEAAwE;QACxE,sEAAsE;QACtE,uEAAuE;QACvE,iEAAiE;QACjE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;QACvC,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACtC,mEAAmE;QACnE,uEAAuE;QACvE,sEAAsE;QACtE,+BAA+B;QAC/B,MAAM,MAAM,GAAG,yBAAyB,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvE,MAAM,WAAW,GAAG,SAAS;YAC3B,CAAC,CAAC;gBACE,MAAM,EAAE,SAAS,CAAC,GAAG;gBACrB,mEAAmE;gBACnE,4DAA4D;gBAC5D,2CAA2C;gBAC3C,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C;YACH,CAAC,CAAC,SAAS,CAAC;QAEd,0EAA0E;QAC1E,gDAAgD;QAChD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC/C,QAAQ,EAAE,IAAI;YACd,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,uEAAuE;YACvE,mEAAmE;YACnE,mEAAmE;YACnE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;gBACvC,eAAe,EAAE,KAAK;gBACtB,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtD,CAAC,CAAC;YACH,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;oBACpD,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;oBACjD,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC9B,CAAC;wBAAS,CAAC;oBACT,mEAAmE;oBACnE,6DAA6D;oBAC7D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAED,wEAAwE;QACxE,sEAAsE;QACtE,oEAAoE;QACpE,+BAA+B;QAC/B,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,QAAQ,EAAE,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACpC,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE;oBACL,aAAa,EAAE,SAAS;oBACxB,QAAQ,EAAE,QAAQ;iBACnB;aACF,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACzD,OAAO;YACL,KAAK;YACL,KAAK,EAAE;gBACL,aAAa,EAAE,SAAS;gBACxB,QAAQ,EAAE,QAAQ;aACnB;SACF,CAAC;IACJ,CAAC;CACF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/html.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/core/feeds/html.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAiC,MAAM,YAAY,CAAC;AAK7E,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AA8CtD,eAAO,MAAM,WAAW,EAAE,WAmEzB,CAAC"}
|
package/dist/core/feeds/html.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { fetchWithRetry } from "./_fetch.js";
|
|
1
2
|
import { CONTENT_HASH_PREFIX, contentHash, parseHtmlDocument } from "./_html-common.js";
|
|
2
3
|
// Re-export shared primitives so existing imports (and tests) that pulled
|
|
3
4
|
// `parseHtmlDocument` from this module keep working after the
|
|
@@ -24,7 +25,7 @@ async function fetchHtml(url, fetchImpl, options = {}) {
|
|
|
24
25
|
}
|
|
25
26
|
if (options.lastModified)
|
|
26
27
|
headers["if-modified-since"] = options.lastModified;
|
|
27
|
-
const response = await fetchImpl
|
|
28
|
+
const response = await fetchWithRetry(fetchImpl, url, { headers, signal: options.signal });
|
|
28
29
|
const etag = response.headers.get("etag");
|
|
29
30
|
const lastModified = response.headers.get("last-modified");
|
|
30
31
|
if (response.status === 304) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html.js","sourceRoot":"","sources":["../../../src/core/feeds/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGxF,0EAA0E;AAC1E,8DAA8D;AAC9D,8DAA8D;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,UAAU,GAAG,2DAA2D,CAAC;AAE/E;;;;;;GAMG;AACH,KAAK,UAAU,SAAS,CACtB,GAAW,EACX,SAAoB,EACpB,UAA0E,EAAE;IAO5E,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,mDAAmD;QAC3D,YAAY,EAAE,UAAU;KACzB,CAAC;IACF,0EAA0E;IAC1E,2EAA2E;IAC3E,WAAW;IACX,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAClE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1C,CAAC;IACD,IAAI,OAAO,CAAC,YAAY;QAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAE9E,MAAM,QAAQ,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"html.js","sourceRoot":"","sources":["../../../src/core/feeds/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGxF,0EAA0E;AAC1E,8DAA8D;AAC9D,8DAA8D;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,UAAU,GAAG,2DAA2D,CAAC;AAE/E;;;;;;GAMG;AACH,KAAK,UAAU,SAAS,CACtB,GAAW,EACX,SAAoB,EACpB,UAA0E,EAAE;IAO5E,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,mDAAmD;QAC3D,YAAY,EAAE,UAAU;KACzB,CAAC;IACF,0EAA0E;IAC1E,2EAA2E;IAC3E,WAAW;IACX,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAClE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1C,CAAC;IACD,IAAI,OAAO,CAAC,YAAY;QAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAE9E,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3F,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;IACvD,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;AAC/D,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAgB;IACtC,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,KAAK,EAAE,MAAc,EAAE,UAA8B,EAAE,EAAE,EAAE;QAChE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,IAAK,UAAU,CAAC,KAA8B,CAAC;QAC9E,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,SAAS,EAAE;YACtD,IAAI,EAAE,QAAQ,EAAE,QAAQ;YACxB,YAAY,EAAE,QAAQ,EAAE,YAAY;SACrC,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE;oBACL,aAAa,EAAE,SAAS;oBACxB,iEAAiE;oBACjE,oEAAoE;oBACpE,QAAQ,EAAE,QAAQ,CAAC,IAAI,IAAI,QAAQ,EAAE,QAAQ;oBAC7C,qEAAqE;oBACrE,iEAAiE;oBACjE,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,EAAE,YAAY;iBAC9D;aACF,CAAC;QACJ,CAAC;QAED,yEAAyE;QACzE,uEAAuE;QACvE,wDAAwD;QACxD,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,cAAc,GAAG,QAAQ,EAAE,QAAQ,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,cAAc,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE;oBACL,aAAa,EAAE,SAAS;oBACxB,QAAQ,EAAE,QAAQ;oBAClB,iEAAiE;oBACjE,kEAAkE;oBAClE,oCAAoC;oBACpC,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,EAAE,YAAY;iBAC9D;aACF,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAClE,OAAO;YACL,KAAK;YACL,KAAK,EAAE;gBACL,aAAa,EAAE,SAAS;gBACxB,qEAAqE;gBACrE,wDAAwD;gBACxD,QAAQ,EAAE,QAAQ,CAAC,IAAI,IAAI,QAAQ;gBACnC,oEAAoE;gBACpE,mEAAmE;gBACnE,yDAAyD;gBACzD,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,EAAE,YAAY;aAC9D;SACF,CAAC;IACJ,CAAC;CACF,CAAC"}
|