@j0hanz/superfetch 1.1.9 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -469
- package/dist/config/constants.d.ts +19 -0
- package/dist/config/constants.d.ts.map +1 -0
- package/dist/config/constants.js +24 -0
- package/dist/config/constants.js.map +1 -0
- package/dist/config/formatting.d.ts +0 -2
- package/dist/config/formatting.d.ts.map +1 -1
- package/dist/config/formatting.js +1 -3
- package/dist/config/formatting.js.map +1 -1
- package/dist/config/index.d.ts +9 -3
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +19 -16
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +1 -20
- package/dist/config/types/content.d.ts.map +1 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/runtime.d.ts +7 -5
- package/dist/config/types/runtime.d.ts.map +1 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/tools.d.ts +5 -50
- package/dist/config/types/tools.d.ts.map +1 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/errors/app-error.d.ts +0 -1
- package/dist/errors/app-error.js +0 -1
- package/dist/http/auth.d.ts +0 -1
- package/dist/http/auth.d.ts.map +1 -1
- package/dist/http/auth.js +17 -13
- package/dist/http/auth.js.map +1 -1
- package/dist/http/cors.d.ts +0 -1
- package/dist/http/cors.js +4 -1
- package/dist/http/cors.js.map +1 -1
- package/dist/http/download-routes.d.ts +14 -0
- package/dist/http/download-routes.d.ts.map +1 -0
- package/dist/http/download-routes.js +131 -0
- package/dist/http/download-routes.js.map +1 -0
- package/dist/http/mcp-routes.d.ts +1 -2
- package/dist/http/mcp-routes.d.ts.map +1 -1
- package/dist/http/mcp-routes.js +1 -2
- package/dist/http/mcp-routes.js.map +1 -1
- package/dist/http/mcp-session-helpers.d.ts +13 -0
- package/dist/http/mcp-session-helpers.d.ts.map +1 -0
- package/dist/http/mcp-session-helpers.js +64 -0
- package/dist/http/mcp-session-helpers.js.map +1 -0
- package/dist/http/mcp-session.d.ts +1 -3
- package/dist/http/mcp-session.d.ts.map +1 -1
- package/dist/http/mcp-session.js +7 -71
- package/dist/http/mcp-session.js.map +1 -1
- package/dist/http/mcp-validation.d.ts +1 -2
- package/dist/http/mcp-validation.d.ts.map +1 -1
- package/dist/http/mcp-validation.js +6 -27
- package/dist/http/mcp-validation.js.map +1 -1
- package/dist/http/rate-limit.d.ts +1 -2
- package/dist/http/rate-limit.d.ts.map +1 -1
- package/dist/http/rate-limit.js +0 -1
- package/dist/http/rate-limit.js.map +1 -1
- package/dist/http/server-middleware.d.ts +9 -0
- package/dist/http/server-middleware.d.ts.map +1 -0
- package/dist/http/server-middleware.js +111 -0
- package/dist/http/server-middleware.js.map +1 -0
- package/dist/http/server.d.ts +0 -1
- package/dist/http/server.d.ts.map +1 -1
- package/dist/http/server.js +20 -99
- package/dist/http/server.js.map +1 -1
- package/dist/http/session-cleanup.d.ts +2 -0
- package/dist/http/session-cleanup.d.ts.map +1 -0
- package/dist/http/session-cleanup.js +37 -0
- package/dist/http/session-cleanup.js.map +1 -0
- package/dist/http/sessions.d.ts +1 -2
- package/dist/http/sessions.d.ts.map +1 -1
- package/dist/http/sessions.js +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +13 -6
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +0 -1
- package/dist/middleware/error-handler.js +0 -1
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +76 -12
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/index.js +0 -1
- package/dist/server.d.ts +0 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +8 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +7 -4
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +86 -26
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +2 -2
- package/dist/services/context.d.ts.map +1 -1
- package/dist/services/context.js +0 -1
- package/dist/services/extractor.d.ts +1 -2
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +45 -18
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher/agents.d.ts +0 -1
- package/dist/services/fetcher/agents.d.ts.map +1 -1
- package/dist/services/fetcher/agents.js +3 -7
- package/dist/services/fetcher/agents.js.map +1 -1
- package/dist/services/fetcher/errors.d.ts +0 -1
- package/dist/services/fetcher/errors.js +0 -1
- package/dist/services/fetcher/headers.d.ts.map +1 -1
- package/dist/services/fetcher/headers.js +2 -24
- package/dist/services/fetcher/headers.js.map +1 -1
- package/dist/services/fetcher/interceptors.d.ts +2 -2
- package/dist/services/fetcher/interceptors.d.ts.map +1 -1
- package/dist/services/fetcher/interceptors.js +30 -21
- package/dist/services/fetcher/interceptors.js.map +1 -1
- package/dist/services/fetcher/redirects.d.ts +0 -2
- package/dist/services/fetcher/redirects.d.ts.map +1 -1
- package/dist/services/fetcher/redirects.js +20 -18
- package/dist/services/fetcher/redirects.js.map +1 -1
- package/dist/services/fetcher/response.d.ts +0 -1
- package/dist/services/fetcher/response.js +4 -5
- package/dist/services/fetcher/retry-policy.d.ts +1 -28
- package/dist/services/fetcher/retry-policy.d.ts.map +1 -1
- package/dist/services/fetcher/retry-policy.js +119 -126
- package/dist/services/fetcher/retry-policy.js.map +1 -1
- package/dist/services/fetcher.d.ts +1 -2
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +18 -13
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +1 -2
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +0 -1
- package/dist/services/parser.d.ts +1 -3
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +5 -39
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +15 -19
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -2
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +50 -20
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +14 -3
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.js +66 -3
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +39 -17
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls/validation.js +1 -1
- package/dist/tools/handlers/fetch-urls/validation.js.map +1 -1
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +1 -20
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/schemas.d.ts +57 -250
- package/dist/tools/schemas.d.ts.map +1 -1
- package/dist/tools/schemas.js +38 -198
- package/dist/tools/schemas.js.map +1 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -2
- package/dist/tools/utils/cache-vary.d.ts.map +1 -1
- package/dist/tools/utils/cache-vary.js +8 -40
- package/dist/tools/utils/cache-vary.js.map +1 -1
- package/dist/tools/utils/common.d.ts +2 -4
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +6 -7
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +1 -3
- package/dist/tools/utils/content-transform.d.ts.map +1 -1
- package/dist/tools/utils/content-transform.js +65 -14
- package/dist/tools/utils/content-transform.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +1 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +25 -21
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/tools/utils/inline-content.d.ts +3 -3
- package/dist/tools/utils/inline-content.d.ts.map +1 -1
- package/dist/tools/utils/inline-content.js +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +0 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -2
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +11 -7
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/code-language.d.ts +2 -0
- package/dist/utils/code-language.d.ts.map +1 -0
- package/dist/utils/code-language.js +56 -0
- package/dist/utils/code-language.js.map +1 -0
- package/dist/utils/content-cleaner.d.ts +0 -2
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +0 -4
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/crypto.d.ts +2 -0
- package/dist/utils/crypto.d.ts.map +1 -0
- package/dist/utils/crypto.js +32 -0
- package/dist/utils/crypto.js.map +1 -0
- package/dist/utils/download-url.d.ts +8 -0
- package/dist/utils/download-url.d.ts.map +1 -0
- package/dist/utils/download-url.js +27 -0
- package/dist/utils/download-url.js.map +1 -0
- package/dist/utils/error-utils.d.ts +3 -0
- package/dist/utils/error-utils.d.ts.map +1 -0
- package/dist/utils/error-utils.js +12 -0
- package/dist/utils/error-utils.js.map +1 -0
- package/dist/utils/filename-generator.d.ts +1 -0
- package/dist/utils/filename-generator.d.ts.map +1 -0
- package/dist/utils/filename-generator.js +59 -0
- package/dist/utils/filename-generator.js.map +1 -0
- package/dist/utils/header-normalizer.d.ts +7 -4
- package/dist/utils/header-normalizer.d.ts.map +1 -1
- package/dist/utils/header-normalizer.js +23 -17
- package/dist/utils/header-normalizer.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.js +0 -1
- package/dist/utils/sanitizer.d.ts +0 -1
- package/dist/utils/sanitizer.js +0 -1
- package/dist/utils/tool-error-handler.d.ts +1 -3
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +11 -6
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-sanitizer.d.ts +2 -0
- package/dist/utils/url-sanitizer.d.ts.map +1 -0
- package/dist/utils/url-sanitizer.js +12 -0
- package/dist/utils/url-sanitizer.js.map +1 -0
- package/dist/utils/url-validator.d.ts +1 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +89 -53
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +7 -9
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
interface FetchTelemetryContext {
|
|
2
2
|
requestId: string;
|
|
3
3
|
startTime: number;
|
|
4
4
|
url: string;
|
|
@@ -7,4 +7,4 @@ export interface FetchTelemetryContext {
|
|
|
7
7
|
export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
|
|
8
8
|
export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
|
|
9
9
|
export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
|
|
10
|
-
|
|
10
|
+
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interceptors.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/interceptors.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"interceptors.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/interceptors.ts"],"names":[],"mappings":"AAUA,UAAU,qBAAqB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,qBAAqB,CAwBvB;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,QAAQ,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAYN;AA4CD,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,qBAAqB,EAC9B,KAAK,EAAE,OAAO,EACd,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAyBN"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import diagnosticsChannel from 'node:diagnostics_channel';
|
|
3
3
|
import { performance } from 'node:perf_hooks';
|
|
4
|
+
import { isSystemError } from '../../utils/error-utils.js';
|
|
4
5
|
import { logDebug, logError, logWarn } from '../logger.js';
|
|
5
6
|
const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
|
|
6
7
|
export function startFetchTelemetry(url, method) {
|
|
@@ -27,38 +28,47 @@ export function startFetchTelemetry(url, method) {
|
|
|
27
28
|
}
|
|
28
29
|
export function recordFetchResponse(context, response, contentSize) {
|
|
29
30
|
const duration = performance.now() - context.startTime;
|
|
30
|
-
|
|
31
|
-
const contentLength = response.headers.get('content-length') ?? contentSize?.toString();
|
|
32
|
-
if (fetchChannel.hasSubscribers) {
|
|
33
|
-
fetchChannel.publish({
|
|
34
|
-
type: 'end',
|
|
35
|
-
requestId: context.requestId,
|
|
36
|
-
status: response.status,
|
|
37
|
-
duration,
|
|
38
|
-
});
|
|
39
|
-
}
|
|
31
|
+
publishFetchEnd(context, response.status, duration);
|
|
40
32
|
logDebug('HTTP Response', {
|
|
41
33
|
requestId: context.requestId,
|
|
42
34
|
status: response.status,
|
|
43
35
|
url: context.url,
|
|
36
|
+
...buildResponseMeta(response, contentSize, duration),
|
|
37
|
+
});
|
|
38
|
+
logSlowRequestIfNeeded(context, duration);
|
|
39
|
+
}
|
|
40
|
+
function publishFetchEnd(context, status, duration) {
|
|
41
|
+
if (!fetchChannel.hasSubscribers)
|
|
42
|
+
return;
|
|
43
|
+
fetchChannel.publish({
|
|
44
|
+
type: 'end',
|
|
45
|
+
requestId: context.requestId,
|
|
46
|
+
status,
|
|
47
|
+
duration,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
function buildResponseMeta(response, contentSize, duration) {
|
|
51
|
+
const contentType = response.headers.get('content-type') ?? undefined;
|
|
52
|
+
const contentLength = response.headers.get('content-length') ?? contentSize?.toString();
|
|
53
|
+
return {
|
|
44
54
|
contentType,
|
|
45
55
|
duration: `${Math.round(duration)}ms`,
|
|
46
56
|
size: contentLength,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
function logSlowRequestIfNeeded(context, duration) {
|
|
60
|
+
if (duration <= 5000)
|
|
61
|
+
return;
|
|
62
|
+
logWarn('Slow HTTP request detected', {
|
|
63
|
+
requestId: context.requestId,
|
|
64
|
+
url: context.url,
|
|
65
|
+
duration: `${Math.round(duration)}ms`,
|
|
47
66
|
});
|
|
48
|
-
if (duration > 5000) {
|
|
49
|
-
logWarn('Slow HTTP request detected', {
|
|
50
|
-
requestId: context.requestId,
|
|
51
|
-
url: context.url,
|
|
52
|
-
duration: `${Math.round(duration)}ms`,
|
|
53
|
-
});
|
|
54
|
-
}
|
|
55
67
|
}
|
|
56
68
|
export function recordFetchError(context, error, status) {
|
|
57
69
|
const duration = performance.now() - context.startTime;
|
|
58
70
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
59
|
-
const code =
|
|
60
|
-
? err.code
|
|
61
|
-
: undefined;
|
|
71
|
+
const code = isSystemError(err) ? err.code : undefined;
|
|
62
72
|
if (fetchChannel.hasSubscribers) {
|
|
63
73
|
fetchChannel.publish({
|
|
64
74
|
type: 'error',
|
|
@@ -79,4 +89,3 @@ export function recordFetchError(context, error, status) {
|
|
|
79
89
|
error: err.message,
|
|
80
90
|
});
|
|
81
91
|
}
|
|
82
|
-
//# sourceMappingURL=interceptors.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interceptors.js","sourceRoot":"","sources":["../../../src/services/fetcher/interceptors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,kBAAkB,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE3D,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AASpE,MAAM,UAAU,mBAAmB,CACjC,GAAW,EACX,MAAc;IAEd,MAAM,OAAO,GAA0B;QACrC,SAAS,EAAE,UAAU,EAAE;QACvB,SAAS,EAAE,WAAW,CAAC,GAAG,EAAE;QAC5B,GAAG;QACH,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;KAC7B,CAAC;IAEF,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,cAAc,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAA8B,EAC9B,QAAkB,EAClB,WAAoB;IAEpB,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IACvD,
|
|
1
|
+
{"version":3,"file":"interceptors.js","sourceRoot":"","sources":["../../../src/services/fetcher/interceptors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,kBAAkB,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE3D,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AASpE,MAAM,UAAU,mBAAmB,CACjC,GAAW,EACX,MAAc;IAEd,MAAM,OAAO,GAA0B;QACrC,SAAS,EAAE,UAAU,EAAE;QACvB,SAAS,EAAE,WAAW,CAAC,GAAG,EAAE;QAC5B,GAAG;QACH,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;KAC7B,CAAC;IAEF,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,cAAc,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAA8B,EAC9B,QAAkB,EAClB,WAAoB;IAEpB,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IACvD,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEpD,QAAQ,CAAC,eAAe,EAAE;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,iBAAiB,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC;KACtD,CAAC,CAAC;IAEH,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,eAAe,CACtB,OAA8B,EAC9B,MAAc,EACd,QAAgB;IAEhB,IAAI,CAAC,YAAY,CAAC,cAAc;QAAE,OAAO;IACzC,YAAY,CAAC,OAAO,CAAC;QACnB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM;QACN,QAAQ;KACT,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CACxB,QAAkB,EAClB,WAA+B,EAC/B,QAAgB;IAEhB,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,CAAC;IACtE,MAAM,aAAa,GACjB,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,WAAW,EAAE,QAAQ,EAAE,CAAC;IAEpE,OAAO;QACL,WAAW;QACX,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI;QACrC,IAAI,EAAE,aAAa;KACpB,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAA8B,EAC9B,QAAgB;IAEhB,IAAI,QAAQ,IAAI,IAAI;QAAE,OAAO;IAC7B,OAAO,CAAC,4BAA4B,EAAE;QACpC,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI;KACtC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAA8B,EAC9B,KAAc,EACd,MAAe;IAEf,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IACvD,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,KAAK,EAAE,GAAG,CAAC,OAAO;YAClB,IAAI;YACJ,MAAM;YACN,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;IAChD,GAAG,CAAC,oBAAoB,EAAE;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,MAAM;QACN,IAAI;QACJ,KAAK,EAAE,GAAG,CAAC,OAAO;KACnB,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
export declare function resolveRedirectTarget(baseUrl: string, location: string): string;
|
|
2
1
|
export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
|
|
3
2
|
response: Response;
|
|
4
3
|
url: string;
|
|
5
4
|
}>;
|
|
6
|
-
//# sourceMappingURL=redirects.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redirects.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/redirects.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"redirects.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/redirects.ts"],"names":[],"mappings":"AA8EA,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAwB9C"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { FetchError } from '../../errors/app-error.js';
|
|
2
|
+
import { createErrorWithCode } from '../../utils/error-utils.js';
|
|
2
3
|
import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
|
|
3
4
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
4
5
|
function isRedirectStatus(status) {
|
|
@@ -9,37 +10,39 @@ async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount)
|
|
|
9
10
|
if (!isRedirectStatus(response.status)) {
|
|
10
11
|
return { response };
|
|
11
12
|
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
throw new FetchError('Too many redirects', currentUrl);
|
|
15
|
-
}
|
|
16
|
-
const location = response.headers.get('location');
|
|
17
|
-
if (!location) {
|
|
18
|
-
void response.body?.cancel();
|
|
19
|
-
throw new FetchError('Redirect response missing Location header', currentUrl);
|
|
20
|
-
}
|
|
13
|
+
assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
|
|
14
|
+
const location = getRedirectLocation(response, currentUrl);
|
|
21
15
|
void response.body?.cancel();
|
|
22
16
|
return {
|
|
23
17
|
response,
|
|
24
|
-
nextUrl: resolveRedirectTarget(currentUrl, location),
|
|
18
|
+
nextUrl: await resolveRedirectTarget(currentUrl, location),
|
|
25
19
|
};
|
|
26
20
|
}
|
|
21
|
+
function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
|
|
22
|
+
if (redirectCount < redirectLimit)
|
|
23
|
+
return;
|
|
24
|
+
void response.body?.cancel();
|
|
25
|
+
throw new FetchError('Too many redirects', currentUrl);
|
|
26
|
+
}
|
|
27
|
+
function getRedirectLocation(response, currentUrl) {
|
|
28
|
+
const location = response.headers.get('location');
|
|
29
|
+
if (location)
|
|
30
|
+
return location;
|
|
31
|
+
void response.body?.cancel();
|
|
32
|
+
throw new FetchError('Redirect response missing Location header', currentUrl);
|
|
33
|
+
}
|
|
27
34
|
function annotateRedirectError(error, url) {
|
|
28
35
|
if (!error || typeof error !== 'object')
|
|
29
36
|
return;
|
|
30
37
|
error.requestUrl = url;
|
|
31
38
|
}
|
|
32
|
-
|
|
39
|
+
async function resolveRedirectTarget(baseUrl, location) {
|
|
33
40
|
if (!URL.canParse(location, baseUrl)) {
|
|
34
|
-
|
|
35
|
-
error.code = 'EBADREDIRECT';
|
|
36
|
-
throw error;
|
|
41
|
+
throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
|
|
37
42
|
}
|
|
38
43
|
const resolved = new URL(location, baseUrl);
|
|
39
44
|
if (resolved.username || resolved.password) {
|
|
40
|
-
|
|
41
|
-
error.code = 'EBADREDIRECT';
|
|
42
|
-
throw error;
|
|
45
|
+
throw createErrorWithCode('Redirect target includes credentials', 'EBADREDIRECT');
|
|
43
46
|
}
|
|
44
47
|
return validateAndNormalizeUrl(resolved.href);
|
|
45
48
|
}
|
|
@@ -64,4 +67,3 @@ async function performFetchCycleSafely(currentUrl, init, redirectLimit, redirect
|
|
|
64
67
|
throw error;
|
|
65
68
|
}
|
|
66
69
|
}
|
|
67
|
-
//# sourceMappingURL=redirects.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redirects.js","sourceRoot":"","sources":["../../../src/services/fetcher/redirects.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,SAAS,gBAAgB,CAAC,MAAc;IACtC,OAAO,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC;AAOD,KAAK,UAAU,iBAAiB,CAC9B,UAAkB,EAClB,IAAiB,EACjB,aAAqB,EACrB,aAAqB;IAErB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE1E,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,QAAQ,EAAE,CAAC;IACtB,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"redirects.js","sourceRoot":"","sources":["../../../src/services/fetcher/redirects.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,SAAS,gBAAgB,CAAC,MAAc;IACtC,OAAO,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC;AAOD,KAAK,UAAU,iBAAiB,CAC9B,UAAkB,EAClB,IAAiB,EACjB,aAAqB,EACrB,aAAqB;IAErB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE1E,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,QAAQ,EAAE,CAAC;IACtB,CAAC;IAED,yBAAyB,CAAC,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE3D,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAC7B,OAAO;QACL,QAAQ;QACR,OAAO,EAAE,qBAAqB,CAAC,UAAU,EAAE,QAAQ,CAAC;KACrD,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAChC,QAAkB,EAClB,UAAkB,EAClB,aAAqB,EACrB,aAAqB;IAErB,IAAI,aAAa,GAAG,aAAa;QAAE,OAAO;IAC1C,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAC7B,MAAM,IAAI,UAAU,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAkB,EAAE,UAAkB;IACjE,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAClD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAE9B,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAC7B,MAAM,IAAI,UAAU,CAAC,2CAA2C,EAAE,UAAU,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc,EAAE,GAAW;IACxD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO;IAC/C,KAAiC,CAAC,UAAU,GAAG,GAAG,CAAC;AACtD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,QAAgB;IAC9D,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,mBAAmB,CAAC,yBAAyB,EAAE,cAAc,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,mBAAmB,CACvB,sCAAsC,EACtC,cAAc,CACf,CAAC;IACJ,CAAC;IAED,OAAO,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,IAAiB,EACjB,YAAoB;IAEpB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAEhD,KACE,IAAI,aAAa,GAAG,CAAC,EACrB,aAAa,IAAI,aAAa,EAC9B,aAAa,IAAI,CAAC,EAClB,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,uBAAuB,CACzD,UAAU,EACV,IAAI,EACJ,aAAa,EACb,aAAa,CACd,CAAC;QAEF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;QACvC,CAAC;QAED,UAAU,GAAG,OAAO,CAAC;IACvB,CAAC;IAED,MAAM,IAAI,UAAU,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,uBAAuB,CACpC,UAAkB,EAClB,IAAiB,EACjB,aAAqB,EACrB,aAAqB;IAErB,IAAI,CAAC;QACH,OAAO,MAAM,iBAAiB,CAC5B,UAAU,EACV,IAAI,EACJ,aAAa,EACb,aAAa,CACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,qBAAqB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -13,7 +13,7 @@ async function readStreamWithLimit(stream, url, maxBytes) {
|
|
|
13
13
|
const reader = stream.getReader();
|
|
14
14
|
const decoder = new TextDecoder();
|
|
15
15
|
let total = 0;
|
|
16
|
-
|
|
16
|
+
const chunks = [];
|
|
17
17
|
for (;;) {
|
|
18
18
|
const { value, done } = await reader.read();
|
|
19
19
|
if (done)
|
|
@@ -23,10 +23,10 @@ async function readStreamWithLimit(stream, url, maxBytes) {
|
|
|
23
23
|
await reader.cancel();
|
|
24
24
|
throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
|
|
25
25
|
}
|
|
26
|
-
|
|
26
|
+
chunks.push(decoder.decode(value, { stream: true }));
|
|
27
27
|
}
|
|
28
|
-
|
|
29
|
-
return { text, size: total };
|
|
28
|
+
chunks.push(decoder.decode());
|
|
29
|
+
return { text: chunks.join(''), size: total };
|
|
30
30
|
}
|
|
31
31
|
export async function readResponseText(response, url, maxBytes) {
|
|
32
32
|
assertContentLengthWithinLimit(response, url, maxBytes);
|
|
@@ -36,4 +36,3 @@ export async function readResponseText(response, url, maxBytes) {
|
|
|
36
36
|
}
|
|
37
37
|
return readStreamWithLimit(response.body, url, maxBytes);
|
|
38
38
|
}
|
|
39
|
-
//# sourceMappingURL=response.js.map
|
|
@@ -1,28 +1 @@
|
|
|
1
|
-
export declare
|
|
2
|
-
private readonly maxRetries;
|
|
3
|
-
private readonly url;
|
|
4
|
-
private static readonly BASE_DELAY_MS;
|
|
5
|
-
private static readonly MAX_DELAY_MS;
|
|
6
|
-
private static readonly JITTER_FACTOR;
|
|
7
|
-
constructor(maxRetries: number, url: string);
|
|
8
|
-
execute<T>(operation: () => Promise<T>, signal?: AbortSignal): Promise<T>;
|
|
9
|
-
private runAttempt;
|
|
10
|
-
private throwIfNotRetryable;
|
|
11
|
-
private shouldRetry;
|
|
12
|
-
private wait;
|
|
13
|
-
private calculateDelay;
|
|
14
|
-
private normalizeRetries;
|
|
15
|
-
private throwIfAborted;
|
|
16
|
-
private normalizeError;
|
|
17
|
-
private buildFinalError;
|
|
18
|
-
private isAbortError;
|
|
19
|
-
private isRateLimited;
|
|
20
|
-
private isClientError;
|
|
21
|
-
private logRetryDelay;
|
|
22
|
-
private isRateLimitLog;
|
|
23
|
-
private sleep;
|
|
24
|
-
private handleSleepError;
|
|
25
|
-
private isAbortTimeout;
|
|
26
|
-
private getRateLimitDelay;
|
|
27
|
-
}
|
|
28
|
-
//# sourceMappingURL=retry-policy.d.ts.map
|
|
1
|
+
export declare function executeWithRetry<T>(url: string, maxRetries: number, operation: () => Promise<T>, signal?: AbortSignal): Promise<T>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retry-policy.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/retry-policy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"retry-policy.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/retry-policy.ts"],"names":[],"mappings":"AAUA,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,CAAC,CAAC,CAWZ"}
|
|
@@ -1,138 +1,131 @@
|
|
|
1
1
|
import { setTimeout } from 'node:timers/promises';
|
|
2
2
|
import { FetchError } from '../../errors/app-error.js';
|
|
3
3
|
import { logDebug, logWarn } from '../logger.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
return { done: true, value };
|
|
30
|
-
}
|
|
31
|
-
catch (error) {
|
|
32
|
-
const normalizedError = this.normalizeError(error);
|
|
33
|
-
this.throwIfNotRetryable(attempt, retries, normalizedError);
|
|
34
|
-
await this.wait(attempt, normalizedError, signal);
|
|
35
|
-
return { done: false, error: normalizedError };
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
throwIfNotRetryable(attempt, retries, error) {
|
|
39
|
-
if (!this.shouldRetry(attempt, retries, error)) {
|
|
40
|
-
throw error;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
shouldRetry(attempt, maxRetries, error) {
|
|
44
|
-
if (attempt >= maxRetries)
|
|
45
|
-
return false;
|
|
46
|
-
if (!(error instanceof FetchError))
|
|
47
|
-
return true;
|
|
48
|
-
if (this.isAbortError(error))
|
|
49
|
-
return false;
|
|
50
|
-
if (this.isRateLimited(error))
|
|
51
|
-
return true;
|
|
52
|
-
return !this.isClientError(error);
|
|
53
|
-
}
|
|
54
|
-
async wait(attempt, error, signal) {
|
|
55
|
-
const delay = this.calculateDelay(attempt, error);
|
|
56
|
-
this.logRetryDelay(attempt, delay, error);
|
|
57
|
-
await this.sleep(delay, signal);
|
|
58
|
-
}
|
|
59
|
-
calculateDelay(attempt, error) {
|
|
60
|
-
const rateLimitDelay = this.getRateLimitDelay(error);
|
|
61
|
-
if (rateLimitDelay !== null)
|
|
62
|
-
return rateLimitDelay;
|
|
63
|
-
const exponentialDelay = Math.min(RetryPolicy.BASE_DELAY_MS * Math.pow(2, attempt - 1), RetryPolicy.MAX_DELAY_MS);
|
|
64
|
-
const jitter = exponentialDelay * RetryPolicy.JITTER_FACTOR * (Math.random() * 2 - 1);
|
|
65
|
-
return Math.round(exponentialDelay + jitter);
|
|
66
|
-
}
|
|
67
|
-
normalizeRetries() {
|
|
68
|
-
return Math.min(Math.max(1, this.maxRetries), 10);
|
|
69
|
-
}
|
|
70
|
-
throwIfAborted(signal) {
|
|
71
|
-
if (!signal?.aborted)
|
|
72
|
-
return;
|
|
73
|
-
throw new FetchError('Request was aborted before execution', this.url);
|
|
74
|
-
}
|
|
75
|
-
normalizeError(error) {
|
|
76
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
77
|
-
}
|
|
78
|
-
buildFinalError(retries, error) {
|
|
79
|
-
return new FetchError(`Failed after ${retries} attempts: ${error.message}`, this.url);
|
|
80
|
-
}
|
|
81
|
-
isAbortError(error) {
|
|
82
|
-
return error.details.reason === 'aborted';
|
|
83
|
-
}
|
|
84
|
-
isRateLimited(error) {
|
|
85
|
-
return error.details.httpStatus === 429;
|
|
4
|
+
const BASE_DELAY_MS = 1000;
|
|
5
|
+
const MAX_DELAY_MS = 10000;
|
|
6
|
+
const JITTER_FACTOR = 0.25;
|
|
7
|
+
export async function executeWithRetry(url, maxRetries, operation, signal) {
|
|
8
|
+
let lastError = new Error(`Failed to fetch ${url}`);
|
|
9
|
+
const retries = normalizeRetries(maxRetries);
|
|
10
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
11
|
+
const result = await runAttempt(url, operation, attempt, retries, signal);
|
|
12
|
+
if (result.done)
|
|
13
|
+
return result.value;
|
|
14
|
+
lastError = result.error;
|
|
15
|
+
}
|
|
16
|
+
throw buildFinalError(url, retries, lastError);
|
|
17
|
+
}
|
|
18
|
+
async function runAttempt(url, operation, attempt, retries, signal) {
|
|
19
|
+
throwIfAborted(url, signal);
|
|
20
|
+
try {
|
|
21
|
+
const value = await operation();
|
|
22
|
+
return { done: true, value };
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
const normalizedError = normalizeError(error);
|
|
26
|
+
throwIfNotRetryable(attempt, retries, normalizedError);
|
|
27
|
+
await wait(url, attempt, normalizedError, signal);
|
|
28
|
+
return { done: false, error: normalizedError };
|
|
86
29
|
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
30
|
+
}
|
|
31
|
+
function throwIfNotRetryable(attempt, retries, error) {
|
|
32
|
+
if (!shouldRetry(attempt, retries, error)) {
|
|
33
|
+
throw error;
|
|
90
34
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
35
|
+
}
|
|
36
|
+
function shouldRetry(attempt, maxRetries, error) {
|
|
37
|
+
if (attempt >= maxRetries)
|
|
38
|
+
return false;
|
|
39
|
+
if (!(error instanceof FetchError))
|
|
40
|
+
return true;
|
|
41
|
+
if (isAbortError(error))
|
|
42
|
+
return false;
|
|
43
|
+
if (isRateLimited(error))
|
|
44
|
+
return true;
|
|
45
|
+
return !isClientError(error);
|
|
46
|
+
}
|
|
47
|
+
async function wait(url, attempt, error, signal) {
|
|
48
|
+
const delay = calculateDelay(attempt, error);
|
|
49
|
+
logRetryDelay(url, attempt, delay, error);
|
|
50
|
+
await sleep(url, delay, signal);
|
|
51
|
+
}
|
|
52
|
+
function calculateDelay(attempt, error) {
|
|
53
|
+
const rateLimitDelay = getRateLimitDelay(error);
|
|
54
|
+
if (rateLimitDelay !== null)
|
|
55
|
+
return rateLimitDelay;
|
|
56
|
+
const exponentialDelay = Math.min(BASE_DELAY_MS * Math.pow(2, attempt - 1), MAX_DELAY_MS);
|
|
57
|
+
const jitter = exponentialDelay * JITTER_FACTOR * (Math.random() * 2 - 1);
|
|
58
|
+
return Math.round(exponentialDelay + jitter);
|
|
59
|
+
}
|
|
60
|
+
function normalizeRetries(maxRetries) {
|
|
61
|
+
return Math.min(Math.max(1, maxRetries), 10);
|
|
62
|
+
}
|
|
63
|
+
function throwIfAborted(url, signal) {
|
|
64
|
+
if (!signal?.aborted)
|
|
65
|
+
return;
|
|
66
|
+
throw new FetchError('Request was aborted before execution', url);
|
|
67
|
+
}
|
|
68
|
+
function normalizeError(error) {
|
|
69
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
70
|
+
}
|
|
71
|
+
function buildFinalError(url, retries, error) {
|
|
72
|
+
return new FetchError(`Failed after ${retries} attempts: ${error.message}`, url);
|
|
73
|
+
}
|
|
74
|
+
function isAbortError(error) {
|
|
75
|
+
return error.details.reason === 'aborted';
|
|
76
|
+
}
|
|
77
|
+
function isRateLimited(error) {
|
|
78
|
+
return error.details.httpStatus === 429;
|
|
79
|
+
}
|
|
80
|
+
function isClientError(error) {
|
|
81
|
+
const status = error.details.httpStatus;
|
|
82
|
+
return typeof status === 'number' && status >= 400 && status < 500;
|
|
83
|
+
}
|
|
84
|
+
function logRetryDelay(url, attempt, delay, error) {
|
|
85
|
+
if (isRateLimitLog(error)) {
|
|
86
|
+
logWarn('Rate limited, waiting before retry', {
|
|
87
|
+
url,
|
|
102
88
|
attempt,
|
|
103
|
-
|
|
89
|
+
waitTime: `${delay}ms`,
|
|
104
90
|
});
|
|
91
|
+
return;
|
|
105
92
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if (this.isAbortTimeout(error)) {
|
|
119
|
-
throw new FetchError('Request was aborted during retry wait', this.url, 499, {
|
|
120
|
-
reason: 'aborted',
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
|
-
throw error;
|
|
93
|
+
logDebug('Retrying request', {
|
|
94
|
+
url,
|
|
95
|
+
attempt,
|
|
96
|
+
delay: `${delay}ms`,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
function isRateLimitLog(error) {
|
|
100
|
+
return error instanceof FetchError && error.details.httpStatus === 429;
|
|
101
|
+
}
|
|
102
|
+
async function sleep(url, delay, signal) {
|
|
103
|
+
try {
|
|
104
|
+
await setTimeout(delay, undefined, { signal });
|
|
124
105
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
(error.name === 'AbortError' || error.name === 'TimeoutError'));
|
|
106
|
+
catch (timeoutError) {
|
|
107
|
+
handleSleepError(url, timeoutError);
|
|
128
108
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
return Math.min(retryAfter * 1000, 30000);
|
|
109
|
+
}
|
|
110
|
+
function handleSleepError(url, error) {
|
|
111
|
+
if (isAbortTimeout(error)) {
|
|
112
|
+
throw new FetchError('Request was aborted during retry wait', url, 499, {
|
|
113
|
+
reason: 'aborted',
|
|
114
|
+
});
|
|
136
115
|
}
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
function isAbortTimeout(error) {
|
|
119
|
+
return (error instanceof Error &&
|
|
120
|
+
(error.name === 'AbortError' || error.name === 'TimeoutError'));
|
|
121
|
+
}
|
|
122
|
+
function getRateLimitDelay(error) {
|
|
123
|
+
if (!(error instanceof FetchError))
|
|
124
|
+
return null;
|
|
125
|
+
if (error.details.httpStatus !== 429)
|
|
126
|
+
return null;
|
|
127
|
+
const retryAfter = typeof error.details.retryAfter === 'number'
|
|
128
|
+
? error.details.retryAfter
|
|
129
|
+
: 60;
|
|
130
|
+
return Math.min(retryAfter * 1000, 30000);
|
|
137
131
|
}
|
|
138
|
-
//# sourceMappingURL=retry-policy.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retry-policy.js","sourceRoot":"","sources":["../../../src/services/fetcher/retry-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEjD,MAAM,
|
|
1
|
+
{"version":3,"file":"retry-policy.js","sourceRoot":"","sources":["../../../src/services/fetcher/retry-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEjD,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,YAAY,GAAG,KAAK,CAAC;AAC3B,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,UAAkB,EAClB,SAA2B,EAC3B,MAAoB;IAEpB,IAAI,SAAS,GAAU,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAE7C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1E,IAAI,MAAM,CAAC,IAAI;YAAE,OAAO,MAAM,CAAC,KAAK,CAAC;QACrC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,CAAC;IAED,MAAM,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,GAAW,EACX,SAA2B,EAC3B,OAAe,EACf,OAAe,EACf,MAAoB;IAEpB,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;QAChC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,eAAe,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAC9C,mBAAmB,CAAC,OAAO,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QACvD,MAAM,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QAClD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;IACjD,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,OAAe,EACf,OAAe,EACf,KAAY;IAEZ,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAClB,OAAe,EACf,UAAkB,EAClB,KAAY;IAEZ,IAAI,OAAO,IAAI,UAAU;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,YAAY,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,IAAI,aAAa,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,IAAI,CACjB,GAAW,EACX,OAAe,EACf,KAAY,EACZ,MAAoB;IAEpB,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAE7C,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC1C,MAAM,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CAAC,OAAe,EAAE,KAAY;IACnD,MAAM,cAAc,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAChD,IAAI,cAAc,KAAK,IAAI;QAAE,OAAO,cAAc,CAAC;IAEnD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EACxC,YAAY,CACb,CAAC;IACF,MAAM,MAAM,GAAG,gBAAgB,GAAG,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,gBAAgB,CAAC,UAAkB;IAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,MAAoB;IACvD,IAAI,CAAC,MAAM,EAAE,OAAO;QAAE,OAAO;IAC7B,MAAM,IAAI,UAAU,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,eAAe,CACtB,GAAW,EACX,OAAe,EACf,KAAY;IAEZ,OAAO,IAAI,UAAU,CACnB,gBAAgB,OAAO,cAAc,KAAK,CAAC,OAAO,EAAE,EACpD,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAiB;IACrC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC;AAC5C,CAAC;AAED,SAAS,aAAa,CAAC,KAAiB;IACtC,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG,CAAC;AAC1C,CAAC;AAED,SAAS,aAAa,CAAC,KAAiB;IACtC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;IACxC,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAC;AACrE,CAAC;AAED,SAAS,aAAa,CACpB,GAAW,EACX,OAAe,EACf,KAAa,EACb,KAAY;IAEZ,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,oCAAoC,EAAE;YAC5C,GAAG;YACH,OAAO;YACP,QAAQ,EAAE,GAAG,KAAK,IAAI;SACvB,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,QAAQ,CAAC,kBAAkB,EAAE;QAC3B,GAAG;QACH,OAAO;QACP,KAAK,EAAE,GAAG,KAAK,IAAI;KACpB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAY;IAClC,OAAO,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,KAAK,CAClB,GAAW,EACX,KAAa,EACb,MAAoB;IAEpB,IAAI,CAAC;QACH,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,YAAY,EAAE,CAAC;QACtB,gBAAgB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,KAAc;IACnD,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,UAAU,CAAC,uCAAuC,EAAE,GAAG,EAAE,GAAG,EAAE;YACtE,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,CACL,KAAK,YAAY,KAAK;QACtB,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,CAC/D,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAY;IACrC,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAElD,MAAM,UAAU,GACd,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,QAAQ;QAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU;QAC1B,CAAC,CAAC,EAAE,CAAC;IACT,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;AAC5C,CAAC"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type { FetchOptions } from '../config/types.js';
|
|
1
|
+
import type { FetchOptions } from '../config/types/runtime.js';
|
|
2
2
|
import { destroyAgents } from './fetcher/agents.js';
|
|
3
3
|
export { destroyAgents };
|
|
4
4
|
export declare function fetchUrlWithRetry(url: string, options?: FetchOptions, maxRetries?: number): Promise<string>;
|
|
5
|
-
//# sourceMappingURL=fetcher.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAK/D,OAAO,EAAE,aAAa,EAAc,MAAM,qBAAqB,CAAC;AAehE,OAAO,EAAE,aAAa,EAAE,CAAC;AA8FzB,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,YAAY,EACtB,UAAU,SAAI,GACb,OAAO,CAAC,MAAM,CAAC,CAUjB"}
|