@j0hanz/superfetch 2.1.4 → 2.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts +1 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +1 -0
- package/dist/cache.js.map +1 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -0
- package/dist/crypto.d.ts +1 -0
- package/dist/crypto.d.ts.map +1 -0
- package/dist/crypto.js +1 -0
- package/dist/crypto.js.map +1 -0
- package/dist/errors.d.ts +1 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +1 -0
- package/dist/errors.js.map +1 -0
- package/dist/fetch.d.ts +1 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +1 -0
- package/dist/fetch.js.map +1 -0
- package/dist/http.d.ts +1 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +1 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts +1 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +1 -0
- package/dist/mcp.js.map +1 -0
- package/dist/observability.d.ts +1 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/observability.js +1 -0
- package/dist/observability.js.map +1 -0
- package/dist/tools.d.ts +1 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +1 -0
- package/dist/tools.js.map +1 -0
- package/dist/transform.d.ts +1 -0
- package/dist/transform.d.ts.map +1 -0
- package/dist/transform.js +1 -0
- package/dist/transform.js.map +1 -0
- package/dist/utils.d.ts +1 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +1 -0
- package/dist/utils.js.map +1 -0
- package/dist/workers/transform-worker.d.ts +1 -0
- package/dist/workers/transform-worker.d.ts.map +1 -0
- package/dist/workers/transform-worker.js +1 -0
- package/dist/workers/transform-worker.js.map +1 -0
- package/package.json +5 -2
- package/dist/config/auth-config.d.ts +0 -16
- package/dist/config/auth-config.js +0 -53
- package/dist/config/constants.d.ts +0 -17
- package/dist/config/constants.d.ts.map +0 -1
- package/dist/config/constants.js +0 -22
- package/dist/config/constants.js.map +0 -1
- package/dist/config/env-parsers.d.ts +0 -8
- package/dist/config/env-parsers.js +0 -96
- package/dist/config/formatting.d.ts +0 -7
- package/dist/config/formatting.d.ts.map +0 -1
- package/dist/config/formatting.js +0 -9
- package/dist/config/formatting.js.map +0 -1
- package/dist/config/index.d.ts +0 -61
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -107
- package/dist/config/index.js.map +0 -1
- package/dist/config/types/content.d.ts +0 -41
- package/dist/config/types/content.d.ts.map +0 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/content.js.map +0 -1
- package/dist/config/types/runtime.d.ts +0 -72
- package/dist/config/types/runtime.d.ts.map +0 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/runtime.js.map +0 -1
- package/dist/config/types/tools.d.ts +0 -28
- package/dist/config/types/tools.d.ts.map +0 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/config/types/tools.js.map +0 -1
- package/dist/config/types.d.ts +0 -4
- package/dist/config/types.d.ts.map +0 -1
- package/dist/config/types.js +0 -2
- package/dist/config/types.js.map +0 -1
- package/dist/errors/app-error.d.ts +0 -7
- package/dist/errors/app-error.d.ts.map +0 -1
- package/dist/errors/app-error.js +0 -16
- package/dist/errors/app-error.js.map +0 -1
- package/dist/errors/index.d.ts +0 -2
- package/dist/errors/index.d.ts.map +0 -1
- package/dist/errors/index.js +0 -2
- package/dist/errors/index.js.map +0 -1
- package/dist/http/accept-policy.d.ts +0 -3
- package/dist/http/accept-policy.js +0 -45
- package/dist/http/async-handler.d.ts +0 -2
- package/dist/http/async-handler.js +0 -5
- package/dist/http/auth-introspection.d.ts +0 -2
- package/dist/http/auth-introspection.js +0 -141
- package/dist/http/auth-static.d.ts +0 -2
- package/dist/http/auth-static.js +0 -23
- package/dist/http/auth.d.ts +0 -3
- package/dist/http/auth.d.ts.map +0 -1
- package/dist/http/auth.js +0 -269
- package/dist/http/auth.js.map +0 -1
- package/dist/http/base-middleware.d.ts +0 -7
- package/dist/http/base-middleware.js +0 -143
- package/dist/http/cors.d.ts +0 -2
- package/dist/http/cors.d.ts.map +0 -1
- package/dist/http/cors.js +0 -9
- package/dist/http/cors.js.map +0 -1
- package/dist/http/download-routes.d.ts +0 -2
- package/dist/http/download-routes.d.ts.map +0 -1
- package/dist/http/download-routes.js +0 -104
- package/dist/http/download-routes.js.map +0 -1
- package/dist/http/error-handler.d.ts +0 -2
- package/dist/http/error-handler.js +0 -55
- package/dist/http/host-allowlist.d.ts +0 -3
- package/dist/http/host-allowlist.js +0 -117
- package/dist/http/jsonrpc-http.d.ts +0 -2
- package/dist/http/jsonrpc-http.js +0 -10
- package/dist/http/mcp-routes.d.ts +0 -9
- package/dist/http/mcp-routes.d.ts.map +0 -1
- package/dist/http/mcp-routes.js +0 -204
- package/dist/http/mcp-routes.js.map +0 -1
- package/dist/http/mcp-session-eviction.d.ts +0 -3
- package/dist/http/mcp-session-eviction.js +0 -24
- package/dist/http/mcp-session-helpers.d.ts +0 -12
- package/dist/http/mcp-session-helpers.d.ts.map +0 -1
- package/dist/http/mcp-session-helpers.js +0 -64
- package/dist/http/mcp-session-helpers.js.map +0 -1
- package/dist/http/mcp-session-init.d.ts +0 -7
- package/dist/http/mcp-session-init.js +0 -94
- package/dist/http/mcp-session-slots.d.ts +0 -17
- package/dist/http/mcp-session-slots.js +0 -55
- package/dist/http/mcp-session-transport-init.d.ts +0 -7
- package/dist/http/mcp-session-transport-init.js +0 -41
- package/dist/http/mcp-session-transport.d.ts +0 -7
- package/dist/http/mcp-session-transport.js +0 -57
- package/dist/http/mcp-session-types.d.ts +0 -5
- package/dist/http/mcp-session-types.js +0 -1
- package/dist/http/mcp-session.d.ts +0 -10
- package/dist/http/mcp-session.d.ts.map +0 -1
- package/dist/http/mcp-session.js +0 -23
- package/dist/http/mcp-session.js.map +0 -1
- package/dist/http/mcp-sessions.d.ts +0 -41
- package/dist/http/mcp-sessions.js +0 -392
- package/dist/http/mcp-validation.d.ts +0 -3
- package/dist/http/mcp-validation.d.ts.map +0 -1
- package/dist/http/mcp-validation.js +0 -14
- package/dist/http/mcp-validation.js.map +0 -1
- package/dist/http/protocol-policy.d.ts +0 -2
- package/dist/http/protocol-policy.js +0 -31
- package/dist/http/rate-limit.d.ts +0 -12
- package/dist/http/rate-limit.d.ts.map +0 -1
- package/dist/http/rate-limit.js +0 -93
- package/dist/http/rate-limit.js.map +0 -1
- package/dist/http/server-config.d.ts +0 -1
- package/dist/http/server-config.js +0 -40
- package/dist/http/server-middleware.d.ts +0 -7
- package/dist/http/server-middleware.d.ts.map +0 -1
- package/dist/http/server-middleware.js +0 -52
- package/dist/http/server-middleware.js.map +0 -1
- package/dist/http/server-shutdown.d.ts +0 -4
- package/dist/http/server-shutdown.js +0 -43
- package/dist/http/server-tuning.d.ts +0 -9
- package/dist/http/server-tuning.js +0 -45
- package/dist/http/server.d.ts +0 -3
- package/dist/http/server.d.ts.map +0 -1
- package/dist/http/server.js +0 -291
- package/dist/http/server.js.map +0 -1
- package/dist/http/session-cleanup.d.ts +0 -2
- package/dist/http/session-cleanup.d.ts.map +0 -1
- package/dist/http/session-cleanup.js +0 -40
- package/dist/http/session-cleanup.js.map +0 -1
- package/dist/http/sessions.d.ts +0 -14
- package/dist/http/sessions.d.ts.map +0 -1
- package/dist/http/sessions.js +0 -63
- package/dist/http/sessions.js.map +0 -1
- package/dist/middleware/error-handler.d.ts +0 -2
- package/dist/middleware/error-handler.d.ts.map +0 -1
- package/dist/middleware/error-handler.js +0 -56
- package/dist/middleware/error-handler.js.map +0 -1
- package/dist/middleware/rate-limiter.d.ts +0 -16
- package/dist/middleware/rate-limiter.d.ts.map +0 -1
- package/dist/middleware/rate-limiter.js +0 -111
- package/dist/middleware/rate-limiter.js.map +0 -1
- package/dist/parsers/base-html-element-parser.d.ts +0 -43
- package/dist/parsers/base-html-element-parser.d.ts.map +0 -1
- package/dist/parsers/base-html-element-parser.js +0 -59
- package/dist/parsers/base-html-element-parser.js.map +0 -1
- package/dist/parsers/heading-element-parser.d.ts +0 -14
- package/dist/parsers/heading-element-parser.d.ts.map +0 -1
- package/dist/parsers/heading-element-parser.js +0 -26
- package/dist/parsers/heading-element-parser.js.map +0 -1
- package/dist/parsers/image-element-parser.d.ts +0 -16
- package/dist/parsers/image-element-parser.d.ts.map +0 -1
- package/dist/parsers/image-element-parser.js +0 -33
- package/dist/parsers/image-element-parser.js.map +0 -1
- package/dist/parsers/link-element-parser.d.ts +0 -15
- package/dist/parsers/link-element-parser.d.ts.map +0 -1
- package/dist/parsers/link-element-parser.js +0 -28
- package/dist/parsers/link-element-parser.js.map +0 -1
- package/dist/parsers/open-graph-parser.d.ts +0 -17
- package/dist/parsers/open-graph-parser.d.ts.map +0 -1
- package/dist/parsers/open-graph-parser.js +0 -41
- package/dist/parsers/open-graph-parser.js.map +0 -1
- package/dist/parsers/schema-org-parser.d.ts +0 -17
- package/dist/parsers/schema-org-parser.d.ts.map +0 -1
- package/dist/parsers/schema-org-parser.js +0 -32
- package/dist/parsers/schema-org-parser.js.map +0 -1
- package/dist/parsers/standard-meta-parser.d.ts +0 -18
- package/dist/parsers/standard-meta-parser.d.ts.map +0 -1
- package/dist/parsers/standard-meta-parser.js +0 -32
- package/dist/parsers/standard-meta-parser.js.map +0 -1
- package/dist/parsers/twitter-card-parser.d.ts +0 -17
- package/dist/parsers/twitter-card-parser.d.ts.map +0 -1
- package/dist/parsers/twitter-card-parser.js +0 -41
- package/dist/parsers/twitter-card-parser.js.map +0 -1
- package/dist/prompts/index.d.ts +0 -3
- package/dist/prompts/index.d.ts.map +0 -1
- package/dist/prompts/index.js +0 -73
- package/dist/prompts/index.js.map +0 -1
- package/dist/resources/cached-content-params.d.ts +0 -5
- package/dist/resources/cached-content-params.js +0 -36
- package/dist/resources/cached-content.d.ts +0 -2
- package/dist/resources/cached-content.d.ts.map +0 -1
- package/dist/resources/cached-content.js +0 -132
- package/dist/resources/cached-content.js.map +0 -1
- package/dist/resources/index.d.ts +0 -2
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js +0 -4
- package/dist/resources/index.js.map +0 -1
- package/dist/server.d.ts +0 -3
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js +0 -94
- package/dist/server.js.map +0 -1
- package/dist/services/cache-events.d.ts +0 -8
- package/dist/services/cache-events.js +0 -19
- package/dist/services/cache-keys.d.ts +0 -7
- package/dist/services/cache-keys.js +0 -57
- package/dist/services/cache.d.ts +0 -17
- package/dist/services/cache.d.ts.map +0 -1
- package/dist/services/cache.js +0 -145
- package/dist/services/cache.js.map +0 -1
- package/dist/services/cache.service.d.ts +0 -52
- package/dist/services/cache.service.d.ts.map +0 -1
- package/dist/services/cache.service.js +0 -113
- package/dist/services/cache.service.js.map +0 -1
- package/dist/services/card-extractor.d.ts +0 -6
- package/dist/services/card-extractor.d.ts.map +0 -1
- package/dist/services/card-extractor.js +0 -199
- package/dist/services/card-extractor.js.map +0 -1
- package/dist/services/context.d.ts +0 -10
- package/dist/services/context.d.ts.map +0 -1
- package/dist/services/context.js +0 -14
- package/dist/services/context.js.map +0 -1
- package/dist/services/extractor.d.ts +0 -5
- package/dist/services/extractor.d.ts.map +0 -1
- package/dist/services/extractor.js +0 -142
- package/dist/services/extractor.js.map +0 -1
- package/dist/services/extractor.service.d.ts +0 -18
- package/dist/services/extractor.service.d.ts.map +0 -1
- package/dist/services/extractor.service.js +0 -75
- package/dist/services/extractor.service.js.map +0 -1
- package/dist/services/fetcher/agents.d.ts +0 -3
- package/dist/services/fetcher/agents.d.ts.map +0 -1
- package/dist/services/fetcher/agents.js +0 -100
- package/dist/services/fetcher/agents.js.map +0 -1
- package/dist/services/fetcher/dns-selection.d.ts +0 -2
- package/dist/services/fetcher/dns-selection.js +0 -72
- package/dist/services/fetcher/errors.d.ts +0 -4
- package/dist/services/fetcher/errors.d.ts.map +0 -1
- package/dist/services/fetcher/errors.js +0 -70
- package/dist/services/fetcher/errors.js.map +0 -1
- package/dist/services/fetcher/headers.d.ts +0 -2
- package/dist/services/fetcher/headers.d.ts.map +0 -1
- package/dist/services/fetcher/headers.js +0 -6
- package/dist/services/fetcher/headers.js.map +0 -1
- package/dist/services/fetcher/interceptors.d.ts +0 -10
- package/dist/services/fetcher/interceptors.d.ts.map +0 -1
- package/dist/services/fetcher/interceptors.js +0 -108
- package/dist/services/fetcher/interceptors.js.map +0 -1
- package/dist/services/fetcher/redirects.d.ts +0 -4
- package/dist/services/fetcher/redirects.d.ts.map +0 -1
- package/dist/services/fetcher/redirects.js +0 -78
- package/dist/services/fetcher/redirects.js.map +0 -1
- package/dist/services/fetcher/response.d.ts +0 -4
- package/dist/services/fetcher/response.d.ts.map +0 -1
- package/dist/services/fetcher/response.js +0 -104
- package/dist/services/fetcher/response.js.map +0 -1
- package/dist/services/fetcher/retry-policy.d.ts +0 -1
- package/dist/services/fetcher/retry-policy.d.ts.map +0 -1
- package/dist/services/fetcher/retry-policy.js +0 -131
- package/dist/services/fetcher/retry-policy.js.map +0 -1
- package/dist/services/fetcher.d.ts +0 -25
- package/dist/services/fetcher.d.ts.map +0 -1
- package/dist/services/fetcher.js +0 -607
- package/dist/services/fetcher.js.map +0 -1
- package/dist/services/fetcher.service.d.ts +0 -18
- package/dist/services/fetcher.service.d.ts.map +0 -1
- package/dist/services/fetcher.service.js +0 -122
- package/dist/services/fetcher.service.js.map +0 -1
- package/dist/services/fifo-queue.d.ts +0 -8
- package/dist/services/fifo-queue.js +0 -25
- package/dist/services/logger.d.ts +0 -5
- package/dist/services/logger.d.ts.map +0 -1
- package/dist/services/logger.js +0 -52
- package/dist/services/logger.js.map +0 -1
- package/dist/services/logger.service.d.ts +0 -5
- package/dist/services/logger.service.d.ts.map +0 -1
- package/dist/services/logger.service.js +0 -57
- package/dist/services/logger.service.js.map +0 -1
- package/dist/services/metadata-collector.d.ts +0 -2
- package/dist/services/metadata-collector.js +0 -80
- package/dist/services/parser.d.ts +0 -6
- package/dist/services/parser.d.ts.map +0 -1
- package/dist/services/parser.js +0 -278
- package/dist/services/parser.js.map +0 -1
- package/dist/services/parser.service.d.ts +0 -42
- package/dist/services/parser.service.d.ts.map +0 -1
- package/dist/services/parser.service.js +0 -209
- package/dist/services/parser.service.js.map +0 -1
- package/dist/services/session-manager.d.ts +0 -18
- package/dist/services/session-manager.d.ts.map +0 -1
- package/dist/services/session-manager.js +0 -73
- package/dist/services/session-manager.js.map +0 -1
- package/dist/services/telemetry.d.ts +0 -19
- package/dist/services/telemetry.js +0 -43
- package/dist/services/transform-worker-pool.d.ts +0 -11
- package/dist/services/transform-worker-pool.js +0 -244
- package/dist/services/transform-worker-types.d.ts +0 -32
- package/dist/services/transform-worker-types.js +0 -14
- package/dist/strategies/exponential-backoff-strategy.d.ts +0 -13
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +0 -1
- package/dist/strategies/exponential-backoff-strategy.js +0 -32
- package/dist/strategies/exponential-backoff-strategy.js.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +0 -4
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +0 -159
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -98
- package/dist/tools/handlers/fetch-links.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +0 -11
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +0 -97
- package/dist/tools/handlers/fetch-markdown.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +0 -31
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-single.shared.js +0 -223
- package/dist/tools/handlers/fetch-single.shared.js.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +0 -10
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-url.tool.js +0 -119
- package/dist/tools/handlers/fetch-url.tool.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +0 -13
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/processor.js +0 -153
- package/dist/tools/handlers/fetch-urls/processor.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.d.ts +0 -3
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/response.js +0 -79
- package/dist/tools/handlers/fetch-urls/response.js.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls/validation.js +0 -18
- package/dist/tools/handlers/fetch-urls/validation.js.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +0 -5
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +0 -1
- package/dist/tools/handlers/fetch-urls.tool.js +0 -124
- package/dist/tools/handlers/fetch-urls.tool.js.map +0 -1
- package/dist/tools/index.d.ts +0 -3
- package/dist/tools/index.d.ts.map +0 -1
- package/dist/tools/index.js +0 -37
- package/dist/tools/index.js.map +0 -1
- package/dist/tools/schemas.d.ts +0 -12
- package/dist/tools/schemas.d.ts.map +0 -1
- package/dist/tools/schemas.js +0 -21
- package/dist/tools/schemas.js.map +0 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -1
- package/dist/tools/utils/cache-vary.d.ts.map +0 -1
- package/dist/tools/utils/cache-vary.js +0 -12
- package/dist/tools/utils/cache-vary.js.map +0 -1
- package/dist/tools/utils/cached-markdown.d.ts +0 -5
- package/dist/tools/utils/cached-markdown.js +0 -46
- package/dist/tools/utils/common.d.ts +0 -5
- package/dist/tools/utils/common.d.ts.map +0 -1
- package/dist/tools/utils/common.js +0 -42
- package/dist/tools/utils/common.js.map +0 -1
- package/dist/tools/utils/content-shaping.d.ts +0 -4
- package/dist/tools/utils/content-shaping.js +0 -67
- package/dist/tools/utils/content-transform-async.d.ts +0 -6
- package/dist/tools/utils/content-transform-async.js +0 -33
- package/dist/tools/utils/content-transform-core.d.ts +0 -5
- package/dist/tools/utils/content-transform-core.js +0 -180
- package/dist/tools/utils/content-transform-workers.d.ts +0 -1
- package/dist/tools/utils/content-transform-workers.js +0 -1
- package/dist/tools/utils/content-transform.d.ts +0 -3
- package/dist/tools/utils/content-transform.d.ts.map +0 -1
- package/dist/tools/utils/content-transform.js +0 -41
- package/dist/tools/utils/content-transform.js.map +0 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +0 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +0 -1
- package/dist/tools/utils/fetch-pipeline.js +0 -98
- package/dist/tools/utils/fetch-pipeline.js.map +0 -1
- package/dist/tools/utils/frontmatter.d.ts +0 -3
- package/dist/tools/utils/frontmatter.js +0 -73
- package/dist/tools/utils/index.d.ts +0 -4
- package/dist/tools/utils/index.d.ts.map +0 -1
- package/dist/tools/utils/index.js +0 -3
- package/dist/tools/utils/index.js.map +0 -1
- package/dist/tools/utils/inline-content.d.ts +0 -10
- package/dist/tools/utils/inline-content.d.ts.map +0 -1
- package/dist/tools/utils/inline-content.js +0 -35
- package/dist/tools/utils/inline-content.js.map +0 -1
- package/dist/tools/utils/markdown-heuristics.d.ts +0 -1
- package/dist/tools/utils/markdown-heuristics.js +0 -19
- package/dist/tools/utils/markdown-signals.d.ts +0 -1
- package/dist/tools/utils/markdown-signals.js +0 -19
- package/dist/tools/utils/markdown-toc.d.ts +0 -3
- package/dist/tools/utils/markdown-toc.d.ts.map +0 -1
- package/dist/tools/utils/markdown-toc.js +0 -35
- package/dist/tools/utils/markdown-toc.js.map +0 -1
- package/dist/tools/utils/raw-markdown-frontmatter.d.ts +0 -3
- package/dist/tools/utils/raw-markdown-frontmatter.js +0 -73
- package/dist/tools/utils/raw-markdown.d.ts +0 -6
- package/dist/tools/utils/raw-markdown.js +0 -149
- package/dist/tools/utils/response-builder.d.ts +0 -3
- package/dist/tools/utils/response-builder.d.ts.map +0 -1
- package/dist/tools/utils/response-builder.js +0 -24
- package/dist/tools/utils/response-builder.js.map +0 -1
- package/dist/tools/utils/tool-response.d.ts +0 -9
- package/dist/tools/utils/tool-response.d.ts.map +0 -1
- package/dist/tools/utils/tool-response.js +0 -19
- package/dist/tools/utils/tool-response.js.map +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +0 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +0 -1
- package/dist/transformers/jsonl.transformer.js +0 -75
- package/dist/transformers/jsonl.transformer.js.map +0 -1
- package/dist/transformers/markdown/fenced-code-rule.d.ts +0 -2
- package/dist/transformers/markdown/fenced-code-rule.js +0 -38
- package/dist/transformers/markdown/frontmatter.d.ts +0 -2
- package/dist/transformers/markdown/frontmatter.js +0 -45
- package/dist/transformers/markdown/noise-rule.d.ts +0 -2
- package/dist/transformers/markdown/noise-rule.js +0 -80
- package/dist/transformers/markdown/turndown-instance.d.ts +0 -2
- package/dist/transformers/markdown/turndown-instance.js +0 -19
- package/dist/transformers/markdown.d.ts +0 -5
- package/dist/transformers/markdown.js +0 -314
- package/dist/transformers/markdown.transformer.d.ts +0 -2
- package/dist/transformers/markdown.transformer.d.ts.map +0 -1
- package/dist/transformers/markdown.transformer.js +0 -14
- package/dist/transformers/markdown.transformer.js.map +0 -1
- package/dist/types/content.types.d.ts +0 -63
- package/dist/types/content.types.d.ts.map +0 -1
- package/dist/types/content.types.js +0 -2
- package/dist/types/content.types.js.map +0 -1
- package/dist/types/index.d.ts +0 -2
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -2
- package/dist/types/index.js.map +0 -1
- package/dist/types/schemas.d.ts +0 -49
- package/dist/types/schemas.d.ts.map +0 -1
- package/dist/types/schemas.js +0 -5
- package/dist/types/schemas.js.map +0 -1
- package/dist/utils/cached-payload.d.ts +0 -7
- package/dist/utils/cached-payload.js +0 -36
- package/dist/utils/cancellation.d.ts +0 -1
- package/dist/utils/cancellation.js +0 -18
- package/dist/utils/code-language-bash.d.ts +0 -1
- package/dist/utils/code-language-bash.js +0 -48
- package/dist/utils/code-language-core.d.ts +0 -2
- package/dist/utils/code-language-core.js +0 -13
- package/dist/utils/code-language-detectors.d.ts +0 -5
- package/dist/utils/code-language-detectors.js +0 -142
- package/dist/utils/code-language-helpers.d.ts +0 -5
- package/dist/utils/code-language-helpers.js +0 -62
- package/dist/utils/code-language-parsing.d.ts +0 -5
- package/dist/utils/code-language-parsing.js +0 -62
- package/dist/utils/code-language.d.ts +0 -2
- package/dist/utils/code-language.d.ts.map +0 -1
- package/dist/utils/code-language.js +0 -260
- package/dist/utils/code-language.js.map +0 -1
- package/dist/utils/concurrency.d.ts +0 -3
- package/dist/utils/concurrency.d.ts.map +0 -1
- package/dist/utils/concurrency.js +0 -38
- package/dist/utils/concurrency.js.map +0 -1
- package/dist/utils/content-cleaner.d.ts +0 -5
- package/dist/utils/content-cleaner.d.ts.map +0 -1
- package/dist/utils/content-cleaner.js +0 -77
- package/dist/utils/content-cleaner.js.map +0 -1
- package/dist/utils/crypto.d.ts +0 -2
- package/dist/utils/crypto.d.ts.map +0 -1
- package/dist/utils/crypto.js +0 -32
- package/dist/utils/crypto.js.map +0 -1
- package/dist/utils/download-url.d.ts +0 -16
- package/dist/utils/download-url.d.ts.map +0 -1
- package/dist/utils/download-url.js +0 -30
- package/dist/utils/download-url.js.map +0 -1
- package/dist/utils/error-details.d.ts +0 -3
- package/dist/utils/error-details.js +0 -12
- package/dist/utils/error-utils.d.ts +0 -3
- package/dist/utils/error-utils.d.ts.map +0 -1
- package/dist/utils/error-utils.js +0 -12
- package/dist/utils/error-utils.js.map +0 -1
- package/dist/utils/filename-generator.d.ts +0 -1
- package/dist/utils/filename-generator.d.ts.map +0 -1
- package/dist/utils/filename-generator.js +0 -81
- package/dist/utils/filename-generator.js.map +0 -1
- package/dist/utils/guards.d.ts +0 -1
- package/dist/utils/guards.js +0 -3
- package/dist/utils/header-normalizer.d.ts +0 -5
- package/dist/utils/header-normalizer.d.ts.map +0 -1
- package/dist/utils/header-normalizer.js +0 -31
- package/dist/utils/header-normalizer.js.map +0 -1
- package/dist/utils/host-normalizer.d.ts +0 -1
- package/dist/utils/host-normalizer.js +0 -37
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.d.ts.map +0 -1
- package/dist/utils/html-truncator.js +0 -13
- package/dist/utils/html-truncator.js.map +0 -1
- package/dist/utils/ip-address.d.ts +0 -4
- package/dist/utils/ip-address.js +0 -6
- package/dist/utils/language-detector.d.ts +0 -2
- package/dist/utils/language-detector.d.ts.map +0 -1
- package/dist/utils/language-detector.js +0 -39
- package/dist/utils/language-detector.js.map +0 -1
- package/dist/utils/sanitizer.d.ts +0 -2
- package/dist/utils/sanitizer.d.ts.map +0 -1
- package/dist/utils/sanitizer.js +0 -20
- package/dist/utils/sanitizer.js.map +0 -1
- package/dist/utils/tool-error-handler.d.ts +0 -3
- package/dist/utils/tool-error-handler.d.ts.map +0 -1
- package/dist/utils/tool-error-handler.js +0 -31
- package/dist/utils/tool-error-handler.js.map +0 -1
- package/dist/utils/url-redactor.d.ts +0 -1
- package/dist/utils/url-redactor.js +0 -13
- package/dist/utils/url-sanitizer.d.ts +0 -2
- package/dist/utils/url-sanitizer.d.ts.map +0 -1
- package/dist/utils/url-sanitizer.js +0 -12
- package/dist/utils/url-sanitizer.js.map +0 -1
- package/dist/utils/url-transformer.d.ts +0 -7
- package/dist/utils/url-transformer.js +0 -147
- package/dist/utils/url-validator.d.ts +0 -6
- package/dist/utils/url-validator.d.ts.map +0 -1
- package/dist/utils/url-validator.js +0 -156
- package/dist/utils/url-validator.js.map +0 -1
- package/dist/workers/content-transform.worker.d.ts +0 -1
- package/dist/workers/content-transform.worker.js +0 -40
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"retry-policy.js","sourceRoot":"","sources":["../../../src/services/fetcher/retry-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEjD,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,YAAY,GAAG,KAAK,CAAC;AAC3B,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,UAAkB,EAClB,SAA2B,EAC3B,MAAoB;IAEpB,IAAI,SAAS,GAAU,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAE7C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1E,IAAI,MAAM,CAAC,IAAI;YAAE,OAAO,MAAM,CAAC,KAAK,CAAC;QACrC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,CAAC;IAED,MAAM,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,GAAW,EACX,SAA2B,EAC3B,OAAe,EACf,OAAe,EACf,MAAoB;IAEpB,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;QAChC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,eAAe,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAC9C,mBAAmB,CAAC,OAAO,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QACvD,MAAM,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QAClD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;IACjD,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,OAAe,EACf,OAAe,EACf,KAAY;IAEZ,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAClB,OAAe,EACf,UAAkB,EAClB,KAAY;IAEZ,IAAI,OAAO,IAAI,UAAU;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,YAAY,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,IAAI,aAAa,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,IAAI,CACjB,GAAW,EACX,OAAe,EACf,KAAY,EACZ,MAAoB;IAEpB,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAE7C,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC1C,MAAM,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CAAC,OAAe,EAAE,KAAY;IACnD,MAAM,cAAc,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAChD,IAAI,cAAc,KAAK,IAAI;QAAE,OAAO,cAAc,CAAC;IAEnD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,EACxC,YAAY,CACb,CAAC;IACF,MAAM,MAAM,GAAG,gBAAgB,GAAG,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,gBAAgB,CAAC,UAAkB;IAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,MAAoB;IACvD,IAAI,CAAC,MAAM,EAAE,OAAO;QAAE,OAAO;IAC7B,MAAM,IAAI,UAAU,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,eAAe,CACtB,GAAW,EACX,OAAe,EACf,KAAY;IAEZ,OAAO,IAAI,UAAU,CACnB,gBAAgB,OAAO,cAAc,KAAK,CAAC,OAAO,EAAE,EACpD,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAiB;IACrC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC;AAC5C,CAAC;AAED,SAAS,aAAa,CAAC,KAAiB;IACtC,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG,CAAC;AAC1C,CAAC;AAED,SAAS,aAAa,CAAC,KAAiB;IACtC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;IACxC,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAC;AACrE,CAAC;AAED,SAAS,aAAa,CACpB,GAAW,EACX,OAAe,EACf,KAAa,EACb,KAAY;IAEZ,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,oCAAoC,EAAE;YAC5C,GAAG;YACH,OAAO;YACP,QAAQ,EAAE,GAAG,KAAK,IAAI;SACvB,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,QAAQ,CAAC,kBAAkB,EAAE;QAC3B,GAAG;QACH,OAAO;QACP,KAAK,EAAE,GAAG,KAAK,IAAI;KACpB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAY;IAClC,OAAO,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,KAAK,CAClB,GAAW,EACX,KAAa,EACb,MAAoB;IAEpB,IAAI,CAAC;QACH,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,YAAY,EAAE,CAAC;QACtB,gBAAgB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,KAAc;IACnD,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,UAAU,CAAC,uCAAuC,EAAE,GAAG,EAAE,GAAG,EAAE;YACtE,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,CACL,KAAK,YAAY,KAAK;QACtB,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,CAC/D,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAY;IACrC,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAElD,MAAM,UAAU,GACd,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,KAAK,QAAQ;QAC1C,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU;QAC1B,CAAC,CAAC,EAAE,CAAC;IACT,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;AAC5C,CAAC"}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
import type { Dispatcher } from 'undici';
|
|
2
|
-
import type { FetchOptions } from '../config/types/runtime.js';
|
|
3
|
-
export declare const dispatcher: Dispatcher;
|
|
4
|
-
export declare function destroyAgents(): void;
|
|
5
|
-
interface FetchTelemetryContext {
|
|
6
|
-
requestId: string;
|
|
7
|
-
startTime: number;
|
|
8
|
-
url: string;
|
|
9
|
-
method: string;
|
|
10
|
-
contextRequestId?: string;
|
|
11
|
-
operationId?: string;
|
|
12
|
-
}
|
|
13
|
-
export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
|
|
14
|
-
export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
|
|
15
|
-
export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
|
|
16
|
-
export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
|
|
17
|
-
response: Response;
|
|
18
|
-
url: string;
|
|
19
|
-
}>;
|
|
20
|
-
export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal): Promise<{
|
|
21
|
-
text: string;
|
|
22
|
-
size: number;
|
|
23
|
-
}>;
|
|
24
|
-
export declare function fetchNormalizedUrl(normalizedUrl: string, options?: FetchOptions): Promise<string>;
|
|
25
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAK/D,OAAO,EAAE,aAAa,EAAc,MAAM,qBAAqB,CAAC;AAehE,OAAO,EAAE,aAAa,EAAE,CAAC;AA8FzB,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,YAAY,EACtB,UAAU,SAAI,GACb,OAAO,CAAC,MAAM,CAAC,CAUjB"}
|
package/dist/services/fetcher.js
DELETED
|
@@ -1,607 +0,0 @@
|
|
|
1
|
-
import { randomUUID } from 'node:crypto';
|
|
2
|
-
import diagnosticsChannel from 'node:diagnostics_channel';
|
|
3
|
-
import dns from 'node:dns';
|
|
4
|
-
import os from 'node:os';
|
|
5
|
-
import { performance } from 'node:perf_hooks';
|
|
6
|
-
import { Agent } from 'undici';
|
|
7
|
-
import { config } from '../config/index.js';
|
|
8
|
-
import { FetchError } from '../errors/app-error.js';
|
|
9
|
-
import { createErrorWithCode, isSystemError } from '../utils/error-details.js';
|
|
10
|
-
import { isRecord } from '../utils/guards.js';
|
|
11
|
-
import { redactUrl } from '../utils/url-redactor.js';
|
|
12
|
-
import { isBlockedIp, validateAndNormalizeUrl, } from '../utils/url-validator.js';
|
|
13
|
-
import { getOperationId, getRequestId } from './context.js';
|
|
14
|
-
import { logDebug, logError, logWarn } from './logger.js';
|
|
15
|
-
const DNS_LOOKUP_TIMEOUT_MS = 5000;
|
|
16
|
-
function normalizeLookupResults(addresses, family) {
|
|
17
|
-
if (Array.isArray(addresses)) {
|
|
18
|
-
return addresses;
|
|
19
|
-
}
|
|
20
|
-
return [{ address: addresses, family: family ?? 4 }];
|
|
21
|
-
}
|
|
22
|
-
function findBlockedIpError(list, hostname) {
|
|
23
|
-
for (const addr of list) {
|
|
24
|
-
const ip = typeof addr === 'string' ? addr : addr.address;
|
|
25
|
-
if (!isBlockedIp(ip)) {
|
|
26
|
-
continue;
|
|
27
|
-
}
|
|
28
|
-
return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
|
|
29
|
-
}
|
|
30
|
-
return null;
|
|
31
|
-
}
|
|
32
|
-
function findInvalidFamilyError(list, hostname) {
|
|
33
|
-
for (const addr of list) {
|
|
34
|
-
const family = typeof addr === 'string' ? 0 : addr.family;
|
|
35
|
-
if (family === 4 || family === 6)
|
|
36
|
-
continue;
|
|
37
|
-
return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
|
|
38
|
-
}
|
|
39
|
-
return null;
|
|
40
|
-
}
|
|
41
|
-
function createNoDnsResultsError(hostname) {
|
|
42
|
-
return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
|
|
43
|
-
}
|
|
44
|
-
function createEmptySelection(hostname) {
|
|
45
|
-
return {
|
|
46
|
-
error: createNoDnsResultsError(hostname),
|
|
47
|
-
fallback: [],
|
|
48
|
-
address: [],
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
function selectLookupResult(list, useAll, hostname) {
|
|
52
|
-
if (list.length === 0)
|
|
53
|
-
return createEmptySelection(hostname);
|
|
54
|
-
if (useAll)
|
|
55
|
-
return { address: list, fallback: list };
|
|
56
|
-
const first = list.at(0);
|
|
57
|
-
if (!first)
|
|
58
|
-
return createEmptySelection(hostname);
|
|
59
|
-
return {
|
|
60
|
-
address: first.address,
|
|
61
|
-
family: first.family,
|
|
62
|
-
fallback: list,
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
function findLookupError(list, hostname) {
|
|
66
|
-
return (findInvalidFamilyError(list, hostname) ?? findBlockedIpError(list, hostname));
|
|
67
|
-
}
|
|
68
|
-
function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
|
|
69
|
-
if (error) {
|
|
70
|
-
callback(error, addresses);
|
|
71
|
-
return;
|
|
72
|
-
}
|
|
73
|
-
const list = normalizeLookupResults(addresses, resolvedFamily);
|
|
74
|
-
const lookupError = findLookupError(list, hostname);
|
|
75
|
-
if (lookupError) {
|
|
76
|
-
callback(lookupError, list);
|
|
77
|
-
return;
|
|
78
|
-
}
|
|
79
|
-
const selection = selectLookupResult(list, useAll, hostname);
|
|
80
|
-
if (selection.error) {
|
|
81
|
-
callback(selection.error, selection.fallback);
|
|
82
|
-
return;
|
|
83
|
-
}
|
|
84
|
-
callback(null, selection.address, selection.family);
|
|
85
|
-
}
|
|
86
|
-
function resolveDns(hostname, options, callback) {
|
|
87
|
-
const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
|
|
88
|
-
const lookupOptions = buildLookupOptions(normalizedOptions);
|
|
89
|
-
const timeout = createLookupTimeout(hostname, callback);
|
|
90
|
-
const safeCallback = wrapLookupCallback(callback, timeout);
|
|
91
|
-
dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
|
|
92
|
-
}
|
|
93
|
-
function normalizeLookupOptions(options) {
|
|
94
|
-
return typeof options === 'number' ? { family: options } : options;
|
|
95
|
-
}
|
|
96
|
-
function buildLookupContext(options) {
|
|
97
|
-
const normalizedOptions = normalizeLookupOptions(options);
|
|
98
|
-
return {
|
|
99
|
-
normalizedOptions,
|
|
100
|
-
useAll: Boolean(normalizedOptions.all),
|
|
101
|
-
resolvedFamily: resolveFamily(normalizedOptions.family),
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
const DEFAULT_DNS_ORDER = 'verbatim';
|
|
105
|
-
function resolveResultOrder(options) {
|
|
106
|
-
if (options.order)
|
|
107
|
-
return options.order;
|
|
108
|
-
const legacyVerbatim = getLegacyVerbatim(options);
|
|
109
|
-
if (legacyVerbatim !== undefined) {
|
|
110
|
-
return legacyVerbatim ? 'verbatim' : 'ipv4first';
|
|
111
|
-
}
|
|
112
|
-
return DEFAULT_DNS_ORDER;
|
|
113
|
-
}
|
|
114
|
-
function getLegacyVerbatim(options) {
|
|
115
|
-
if (isRecord(options)) {
|
|
116
|
-
const { verbatim } = options;
|
|
117
|
-
return typeof verbatim === 'boolean' ? verbatim : undefined;
|
|
118
|
-
}
|
|
119
|
-
return undefined;
|
|
120
|
-
}
|
|
121
|
-
function buildLookupOptions(normalizedOptions) {
|
|
122
|
-
return {
|
|
123
|
-
family: normalizedOptions.family,
|
|
124
|
-
hints: normalizedOptions.hints,
|
|
125
|
-
all: true,
|
|
126
|
-
order: resolveResultOrder(normalizedOptions),
|
|
127
|
-
};
|
|
128
|
-
}
|
|
129
|
-
function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
|
|
130
|
-
return (err, addresses) => {
|
|
131
|
-
handleLookupResult(err, addresses, hostname, resolvedFamily, useAll, callback);
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
|
-
function resolveFamily(family) {
|
|
135
|
-
if (family === 'IPv4')
|
|
136
|
-
return 4;
|
|
137
|
-
if (family === 'IPv6')
|
|
138
|
-
return 6;
|
|
139
|
-
return family;
|
|
140
|
-
}
|
|
141
|
-
function createLookupTimeout(hostname, callback) {
|
|
142
|
-
let done = false;
|
|
143
|
-
const timer = setTimeout(() => {
|
|
144
|
-
if (done)
|
|
145
|
-
return;
|
|
146
|
-
done = true;
|
|
147
|
-
callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
|
|
148
|
-
}, DNS_LOOKUP_TIMEOUT_MS);
|
|
149
|
-
timer.unref();
|
|
150
|
-
return {
|
|
151
|
-
isDone: () => done,
|
|
152
|
-
markDone: () => {
|
|
153
|
-
done = true;
|
|
154
|
-
clearTimeout(timer);
|
|
155
|
-
},
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
function wrapLookupCallback(callback, timeout) {
|
|
159
|
-
return (err, address, family) => {
|
|
160
|
-
if (timeout.isDone())
|
|
161
|
-
return;
|
|
162
|
-
timeout.markDone();
|
|
163
|
-
callback(err, address, family);
|
|
164
|
-
};
|
|
165
|
-
}
|
|
166
|
-
function getAgentOptions() {
|
|
167
|
-
const cpuCount = os.availableParallelism();
|
|
168
|
-
return {
|
|
169
|
-
keepAliveTimeout: 60000,
|
|
170
|
-
connections: Math.max(cpuCount * 2, 25),
|
|
171
|
-
pipelining: 1,
|
|
172
|
-
connect: { lookup: resolveDns },
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
export const dispatcher = new Agent(getAgentOptions());
|
|
176
|
-
export function destroyAgents() {
|
|
177
|
-
void dispatcher.close();
|
|
178
|
-
}
|
|
179
|
-
function parseRetryAfter(header) {
|
|
180
|
-
if (!header)
|
|
181
|
-
return 60;
|
|
182
|
-
const parsed = parseInt(header, 10);
|
|
183
|
-
return Number.isNaN(parsed) ? 60 : parsed;
|
|
184
|
-
}
|
|
185
|
-
function createCanceledError(url) {
|
|
186
|
-
return new FetchError('Request was canceled', url, 499, {
|
|
187
|
-
reason: 'aborted',
|
|
188
|
-
});
|
|
189
|
-
}
|
|
190
|
-
function createTimeoutError(url, timeoutMs) {
|
|
191
|
-
return new FetchError(`Request timeout after ${timeoutMs}ms`, url, 504, {
|
|
192
|
-
timeout: timeoutMs,
|
|
193
|
-
});
|
|
194
|
-
}
|
|
195
|
-
function createRateLimitError(url, headerValue) {
|
|
196
|
-
const retryAfter = parseRetryAfter(headerValue);
|
|
197
|
-
return new FetchError('Too many requests', url, 429, { retryAfter });
|
|
198
|
-
}
|
|
199
|
-
function createHttpError(url, status, statusText) {
|
|
200
|
-
return new FetchError(`HTTP ${status}: ${statusText}`, url, status);
|
|
201
|
-
}
|
|
202
|
-
function createNetworkError(url, message) {
|
|
203
|
-
const details = message ? { message } : undefined;
|
|
204
|
-
return new FetchError(`Network error: Could not reach ${url}`, url, undefined, details ?? {});
|
|
205
|
-
}
|
|
206
|
-
function createUnknownError(url, message) {
|
|
207
|
-
return new FetchError(message, url);
|
|
208
|
-
}
|
|
209
|
-
function isAbortError(error) {
|
|
210
|
-
return (error instanceof Error &&
|
|
211
|
-
(error.name === 'AbortError' || error.name === 'TimeoutError'));
|
|
212
|
-
}
|
|
213
|
-
function isTimeoutError(error) {
|
|
214
|
-
return error instanceof Error && error.name === 'TimeoutError';
|
|
215
|
-
}
|
|
216
|
-
function getRequestUrl(record) {
|
|
217
|
-
const value = record.requestUrl;
|
|
218
|
-
return typeof value === 'string' ? value : null;
|
|
219
|
-
}
|
|
220
|
-
function resolveErrorUrl(error, fallback) {
|
|
221
|
-
if (error instanceof FetchError)
|
|
222
|
-
return error.url;
|
|
223
|
-
if (!isRecord(error))
|
|
224
|
-
return fallback;
|
|
225
|
-
const requestUrl = getRequestUrl(error);
|
|
226
|
-
if (requestUrl)
|
|
227
|
-
return requestUrl;
|
|
228
|
-
return fallback;
|
|
229
|
-
}
|
|
230
|
-
function mapFetchError(error, fallbackUrl, timeoutMs) {
|
|
231
|
-
if (error instanceof FetchError)
|
|
232
|
-
return error;
|
|
233
|
-
const url = resolveErrorUrl(error, fallbackUrl);
|
|
234
|
-
if (isAbortError(error)) {
|
|
235
|
-
if (isTimeoutError(error)) {
|
|
236
|
-
return createTimeoutError(url, timeoutMs);
|
|
237
|
-
}
|
|
238
|
-
return createCanceledError(url);
|
|
239
|
-
}
|
|
240
|
-
if (error instanceof Error) {
|
|
241
|
-
return createNetworkError(url, error.message);
|
|
242
|
-
}
|
|
243
|
-
return createUnknownError(url, 'Unexpected error');
|
|
244
|
-
}
|
|
245
|
-
const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
|
|
246
|
-
function publishFetchEvent(event) {
|
|
247
|
-
if (!fetchChannel.hasSubscribers)
|
|
248
|
-
return;
|
|
249
|
-
try {
|
|
250
|
-
fetchChannel.publish(event);
|
|
251
|
-
}
|
|
252
|
-
catch {
|
|
253
|
-
// Avoid crashing the publisher if a subscriber throws.
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
export function startFetchTelemetry(url, method) {
|
|
257
|
-
const safeUrl = redactUrl(url);
|
|
258
|
-
const contextRequestId = getRequestId();
|
|
259
|
-
const operationId = getOperationId();
|
|
260
|
-
const context = {
|
|
261
|
-
requestId: randomUUID(),
|
|
262
|
-
startTime: performance.now(),
|
|
263
|
-
url: safeUrl,
|
|
264
|
-
method: method.toUpperCase(),
|
|
265
|
-
...(contextRequestId ? { contextRequestId } : {}),
|
|
266
|
-
...(operationId ? { operationId } : {}),
|
|
267
|
-
};
|
|
268
|
-
publishFetchEvent({
|
|
269
|
-
v: 1,
|
|
270
|
-
type: 'start',
|
|
271
|
-
requestId: context.requestId,
|
|
272
|
-
method: context.method,
|
|
273
|
-
url: context.url,
|
|
274
|
-
...(context.contextRequestId
|
|
275
|
-
? { contextRequestId: context.contextRequestId }
|
|
276
|
-
: {}),
|
|
277
|
-
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
278
|
-
});
|
|
279
|
-
logDebug('HTTP Request', {
|
|
280
|
-
requestId: context.requestId,
|
|
281
|
-
method: context.method,
|
|
282
|
-
url: context.url,
|
|
283
|
-
...(context.contextRequestId
|
|
284
|
-
? { contextRequestId: context.contextRequestId }
|
|
285
|
-
: {}),
|
|
286
|
-
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
287
|
-
});
|
|
288
|
-
return context;
|
|
289
|
-
}
|
|
290
|
-
export function recordFetchResponse(context, response, contentSize) {
|
|
291
|
-
const duration = performance.now() - context.startTime;
|
|
292
|
-
const durationLabel = `${Math.round(duration)}ms`;
|
|
293
|
-
publishFetchEvent({
|
|
294
|
-
v: 1,
|
|
295
|
-
type: 'end',
|
|
296
|
-
requestId: context.requestId,
|
|
297
|
-
status: response.status,
|
|
298
|
-
duration,
|
|
299
|
-
...(context.contextRequestId
|
|
300
|
-
? { contextRequestId: context.contextRequestId }
|
|
301
|
-
: {}),
|
|
302
|
-
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
303
|
-
});
|
|
304
|
-
const contentType = response.headers.get('content-type');
|
|
305
|
-
const contentLength = response.headers.get('content-length') ??
|
|
306
|
-
(contentSize === undefined ? undefined : String(contentSize));
|
|
307
|
-
logDebug('HTTP Response', {
|
|
308
|
-
requestId: context.requestId,
|
|
309
|
-
status: response.status,
|
|
310
|
-
url: context.url,
|
|
311
|
-
duration: durationLabel,
|
|
312
|
-
...(context.contextRequestId
|
|
313
|
-
? { contextRequestId: context.contextRequestId }
|
|
314
|
-
: {}),
|
|
315
|
-
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
316
|
-
...(contentType ? { contentType } : {}),
|
|
317
|
-
...(contentLength ? { size: contentLength } : {}),
|
|
318
|
-
});
|
|
319
|
-
if (duration > 5000) {
|
|
320
|
-
logWarn('Slow HTTP request detected', {
|
|
321
|
-
requestId: context.requestId,
|
|
322
|
-
url: context.url,
|
|
323
|
-
duration: durationLabel,
|
|
324
|
-
...(context.contextRequestId
|
|
325
|
-
? { contextRequestId: context.contextRequestId }
|
|
326
|
-
: {}),
|
|
327
|
-
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
328
|
-
});
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
export function recordFetchError(context, error, status) {
|
|
332
|
-
const duration = performance.now() - context.startTime;
|
|
333
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
334
|
-
const event = {
|
|
335
|
-
v: 1,
|
|
336
|
-
type: 'error',
|
|
337
|
-
requestId: context.requestId,
|
|
338
|
-
url: context.url,
|
|
339
|
-
error: err.message,
|
|
340
|
-
duration,
|
|
341
|
-
...(context.contextRequestId
|
|
342
|
-
? { contextRequestId: context.contextRequestId }
|
|
343
|
-
: {}),
|
|
344
|
-
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
345
|
-
};
|
|
346
|
-
const code = isSystemError(err) ? err.code : undefined;
|
|
347
|
-
if (code !== undefined) {
|
|
348
|
-
event.code = code;
|
|
349
|
-
}
|
|
350
|
-
if (status !== undefined) {
|
|
351
|
-
event.status = status;
|
|
352
|
-
}
|
|
353
|
-
publishFetchEvent(event);
|
|
354
|
-
const log = status === 429 ? logWarn : logError;
|
|
355
|
-
log('HTTP Request Error', {
|
|
356
|
-
requestId: context.requestId,
|
|
357
|
-
url: context.url,
|
|
358
|
-
status,
|
|
359
|
-
code,
|
|
360
|
-
error: err.message,
|
|
361
|
-
...(context.contextRequestId
|
|
362
|
-
? { contextRequestId: context.contextRequestId }
|
|
363
|
-
: {}),
|
|
364
|
-
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
365
|
-
});
|
|
366
|
-
}
|
|
367
|
-
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
368
|
-
function isRedirectStatus(status) {
|
|
369
|
-
return REDIRECT_STATUSES.has(status);
|
|
370
|
-
}
|
|
371
|
-
function cancelResponseBody(response) {
|
|
372
|
-
const cancelPromise = response.body?.cancel();
|
|
373
|
-
if (cancelPromise) {
|
|
374
|
-
cancelPromise.catch(() => {
|
|
375
|
-
// Best-effort cancellation; ignore failures.
|
|
376
|
-
});
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount) {
|
|
380
|
-
const response = await fetch(currentUrl, { ...init, redirect: 'manual' });
|
|
381
|
-
if (!isRedirectStatus(response.status)) {
|
|
382
|
-
return { response };
|
|
383
|
-
}
|
|
384
|
-
assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
|
|
385
|
-
const location = getRedirectLocation(response, currentUrl);
|
|
386
|
-
cancelResponseBody(response);
|
|
387
|
-
return {
|
|
388
|
-
response,
|
|
389
|
-
nextUrl: resolveRedirectTarget(currentUrl, location),
|
|
390
|
-
};
|
|
391
|
-
}
|
|
392
|
-
function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
|
|
393
|
-
if (redirectCount < redirectLimit)
|
|
394
|
-
return;
|
|
395
|
-
cancelResponseBody(response);
|
|
396
|
-
throw new FetchError('Too many redirects', currentUrl);
|
|
397
|
-
}
|
|
398
|
-
function getRedirectLocation(response, currentUrl) {
|
|
399
|
-
const location = response.headers.get('location');
|
|
400
|
-
if (location)
|
|
401
|
-
return location;
|
|
402
|
-
cancelResponseBody(response);
|
|
403
|
-
throw new FetchError('Redirect response missing Location header', currentUrl);
|
|
404
|
-
}
|
|
405
|
-
function annotateRedirectError(error, url) {
|
|
406
|
-
if (!isRecord(error))
|
|
407
|
-
return;
|
|
408
|
-
error.requestUrl = url;
|
|
409
|
-
}
|
|
410
|
-
function resolveRedirectTarget(baseUrl, location) {
|
|
411
|
-
if (!URL.canParse(location, baseUrl)) {
|
|
412
|
-
throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
|
|
413
|
-
}
|
|
414
|
-
const resolved = new URL(location, baseUrl);
|
|
415
|
-
if (resolved.username || resolved.password) {
|
|
416
|
-
throw createErrorWithCode('Redirect target includes credentials', 'EBADREDIRECT');
|
|
417
|
-
}
|
|
418
|
-
return validateAndNormalizeUrl(resolved.href);
|
|
419
|
-
}
|
|
420
|
-
export async function fetchWithRedirects(url, init, maxRedirects) {
|
|
421
|
-
let currentUrl = url;
|
|
422
|
-
const redirectLimit = Math.max(0, maxRedirects);
|
|
423
|
-
for (let redirectCount = 0; redirectCount <= redirectLimit; redirectCount += 1) {
|
|
424
|
-
const { response, nextUrl } = await performFetchCycleSafely(currentUrl, init, redirectLimit, redirectCount);
|
|
425
|
-
if (!nextUrl) {
|
|
426
|
-
return { response, url: currentUrl };
|
|
427
|
-
}
|
|
428
|
-
currentUrl = nextUrl;
|
|
429
|
-
}
|
|
430
|
-
throw new FetchError('Too many redirects', currentUrl);
|
|
431
|
-
}
|
|
432
|
-
async function performFetchCycleSafely(currentUrl, init, redirectLimit, redirectCount) {
|
|
433
|
-
try {
|
|
434
|
-
return await performFetchCycle(currentUrl, init, redirectLimit, redirectCount);
|
|
435
|
-
}
|
|
436
|
-
catch (error) {
|
|
437
|
-
annotateRedirectError(error, currentUrl);
|
|
438
|
-
throw error;
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
function assertContentLengthWithinLimit(response, url, maxBytes) {
|
|
442
|
-
const contentLengthHeader = response.headers.get('content-length');
|
|
443
|
-
if (!contentLengthHeader)
|
|
444
|
-
return;
|
|
445
|
-
const contentLength = Number.parseInt(contentLengthHeader, 10);
|
|
446
|
-
if (Number.isNaN(contentLength) || contentLength <= maxBytes) {
|
|
447
|
-
return;
|
|
448
|
-
}
|
|
449
|
-
cancelResponseBody(response);
|
|
450
|
-
throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
|
|
451
|
-
}
|
|
452
|
-
function createReadState() {
|
|
453
|
-
return {
|
|
454
|
-
decoder: new TextDecoder(),
|
|
455
|
-
parts: [],
|
|
456
|
-
total: 0,
|
|
457
|
-
};
|
|
458
|
-
}
|
|
459
|
-
function appendChunk(state, chunk, maxBytes, url) {
|
|
460
|
-
state.total += chunk.byteLength;
|
|
461
|
-
if (state.total > maxBytes) {
|
|
462
|
-
throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
|
|
463
|
-
}
|
|
464
|
-
const decoded = state.decoder.decode(chunk, { stream: true });
|
|
465
|
-
if (decoded)
|
|
466
|
-
state.parts.push(decoded);
|
|
467
|
-
}
|
|
468
|
-
function finalizeRead(state) {
|
|
469
|
-
const decoded = state.decoder.decode();
|
|
470
|
-
if (decoded)
|
|
471
|
-
state.parts.push(decoded);
|
|
472
|
-
}
|
|
473
|
-
function createAbortError(url) {
|
|
474
|
-
return new FetchError('Request was aborted during response read', url, 499, {
|
|
475
|
-
reason: 'aborted',
|
|
476
|
-
});
|
|
477
|
-
}
|
|
478
|
-
async function cancelReaderQuietly(reader) {
|
|
479
|
-
try {
|
|
480
|
-
await reader.cancel();
|
|
481
|
-
}
|
|
482
|
-
catch {
|
|
483
|
-
// Ignore cancel errors; we're already failing this read.
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
async function throwIfAborted(signal, url, reader) {
|
|
487
|
-
if (!signal?.aborted)
|
|
488
|
-
return;
|
|
489
|
-
await cancelReaderQuietly(reader);
|
|
490
|
-
throw createAbortError(url);
|
|
491
|
-
}
|
|
492
|
-
async function handleReadFailure(error, signal, url, reader) {
|
|
493
|
-
const aborted = signal?.aborted ?? false;
|
|
494
|
-
await cancelReaderQuietly(reader);
|
|
495
|
-
if (aborted) {
|
|
496
|
-
throw createAbortError(url);
|
|
497
|
-
}
|
|
498
|
-
throw error;
|
|
499
|
-
}
|
|
500
|
-
async function readAllChunks(reader, state, url, maxBytes, signal) {
|
|
501
|
-
await throwIfAborted(signal, url, reader);
|
|
502
|
-
let result = await reader.read();
|
|
503
|
-
while (!result.done) {
|
|
504
|
-
appendChunk(state, result.value, maxBytes, url);
|
|
505
|
-
await throwIfAborted(signal, url, reader);
|
|
506
|
-
result = await reader.read();
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
async function readStreamWithLimit(stream, url, maxBytes, signal) {
|
|
510
|
-
const state = createReadState();
|
|
511
|
-
const reader = stream.getReader();
|
|
512
|
-
try {
|
|
513
|
-
await readAllChunks(reader, state, url, maxBytes, signal);
|
|
514
|
-
}
|
|
515
|
-
catch (error) {
|
|
516
|
-
await handleReadFailure(error, signal, url, reader);
|
|
517
|
-
}
|
|
518
|
-
finally {
|
|
519
|
-
reader.releaseLock();
|
|
520
|
-
}
|
|
521
|
-
finalizeRead(state);
|
|
522
|
-
return { text: state.parts.join(''), size: state.total };
|
|
523
|
-
}
|
|
524
|
-
export async function readResponseText(response, url, maxBytes, signal) {
|
|
525
|
-
assertContentLengthWithinLimit(response, url, maxBytes);
|
|
526
|
-
if (!response.body) {
|
|
527
|
-
const text = await response.text();
|
|
528
|
-
const size = Buffer.byteLength(text);
|
|
529
|
-
if (size > maxBytes) {
|
|
530
|
-
throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
|
|
531
|
-
}
|
|
532
|
-
return { text, size };
|
|
533
|
-
}
|
|
534
|
-
return readStreamWithLimit(response.body, url, maxBytes, signal);
|
|
535
|
-
}
|
|
536
|
-
const DEFAULT_HEADERS = {
|
|
537
|
-
'User-Agent': config.fetcher.userAgent,
|
|
538
|
-
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
539
|
-
'Accept-Language': 'en-US,en;q=0.5',
|
|
540
|
-
'Accept-Encoding': 'gzip, deflate, br',
|
|
541
|
-
Connection: 'keep-alive',
|
|
542
|
-
};
|
|
543
|
-
function buildHeaders() {
|
|
544
|
-
return { ...DEFAULT_HEADERS };
|
|
545
|
-
}
|
|
546
|
-
function buildRequestSignal(timeoutMs, external) {
|
|
547
|
-
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
548
|
-
if (!external)
|
|
549
|
-
return timeoutSignal;
|
|
550
|
-
return AbortSignal.any([external, timeoutSignal]);
|
|
551
|
-
}
|
|
552
|
-
function buildRequestInit(headers, signal) {
|
|
553
|
-
return {
|
|
554
|
-
method: 'GET',
|
|
555
|
-
headers,
|
|
556
|
-
signal,
|
|
557
|
-
dispatcher,
|
|
558
|
-
};
|
|
559
|
-
}
|
|
560
|
-
function resolveResponseError(response, finalUrl) {
|
|
561
|
-
return (resolveRateLimitError(response, finalUrl) ??
|
|
562
|
-
resolveHttpError(response, finalUrl));
|
|
563
|
-
}
|
|
564
|
-
function resolveRateLimitError(response, finalUrl) {
|
|
565
|
-
return response.status === 429
|
|
566
|
-
? createRateLimitError(finalUrl, response.headers.get('retry-after'))
|
|
567
|
-
: null;
|
|
568
|
-
}
|
|
569
|
-
function resolveHttpError(response, finalUrl) {
|
|
570
|
-
return response.ok
|
|
571
|
-
? null
|
|
572
|
-
: createHttpError(finalUrl, response.status, response.statusText);
|
|
573
|
-
}
|
|
574
|
-
async function handleFetchResponse(response, finalUrl, telemetry, signal) {
|
|
575
|
-
const responseError = resolveResponseError(response, finalUrl);
|
|
576
|
-
if (responseError) {
|
|
577
|
-
cancelResponseBody(response);
|
|
578
|
-
throw responseError;
|
|
579
|
-
}
|
|
580
|
-
const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength, signal);
|
|
581
|
-
recordFetchResponse(telemetry, response, size);
|
|
582
|
-
return text;
|
|
583
|
-
}
|
|
584
|
-
async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
|
|
585
|
-
const telemetry = startFetchTelemetry(normalizedUrl, 'GET');
|
|
586
|
-
try {
|
|
587
|
-
return await fetchAndHandle(normalizedUrl, requestInit, telemetry);
|
|
588
|
-
}
|
|
589
|
-
catch (error) {
|
|
590
|
-
const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
|
|
591
|
-
telemetry.url = mapped.url;
|
|
592
|
-
recordFetchError(telemetry, mapped, mapped.statusCode);
|
|
593
|
-
throw mapped;
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
async function fetchAndHandle(normalizedUrl, requestInit, telemetry) {
|
|
597
|
-
const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
|
|
598
|
-
telemetry.url = finalUrl;
|
|
599
|
-
return handleFetchResponse(response, finalUrl, telemetry, requestInit.signal ?? undefined);
|
|
600
|
-
}
|
|
601
|
-
export async function fetchNormalizedUrl(normalizedUrl, options) {
|
|
602
|
-
const timeoutMs = config.fetcher.timeout;
|
|
603
|
-
const headers = buildHeaders();
|
|
604
|
-
const signal = buildRequestSignal(timeoutMs, options?.signal);
|
|
605
|
-
const requestInit = buildRequestInit(headers, signal);
|
|
606
|
-
return fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs);
|
|
607
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/services/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EACL,eAAe,EACf,oBAAoB,EACpB,aAAa,GACd,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,aAAa,EAAE,CAAC;AAEzB,MAAM,eAAe,GAAG;IACtB,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;IACtC,MAAM,EACJ,4EAA4E;IAC9E,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,SAAS,YAAY,CAAC,aAAsC;IAC1D,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7C,MAAM,SAAS,GAAG,qBAAqB,CACrC,aAAa,EACb,MAAM,CAAC,QAAQ,CAAC,cAAc,CAC/B,CAAC;IACF,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CACzB,SAAiB,EACjB,QAAsB;IAEtB,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ;QAAE,OAAO,aAAa,CAAC;IACpC,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAgB,EAChB,MAAmB;IAEnB,OAAO;QACL,MAAM,EAAE,KAAK;QACb,OAAO;QACP,MAAM;QACN,UAAU;KACX,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,QAAkB,EAClB,QAAgB,EAChB,SAAiD;IAEjD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,gBAAgB,CAC3C,QAAQ,EACR,QAAQ,EACR,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAChC,CAAC;IACF,mBAAmB,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,aAAqB,EACrB,WAAwB,EACxB,SAAiB;IAEjB,MAAM,SAAS,GAAG,mBAAmB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAE5D,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,MAAM,kBAAkB,CAC1D,aAAa,EACb,WAAW,EACX,MAAM,CAAC,OAAO,CAAC,YAAY,CAC5B,CAAC;QAEF,SAAS,CAAC,GAAG,GAAG,QAAQ,CAAC;QACzB,OAAO,MAAM,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;QAC9D,SAAS,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QAC3B,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACvD,MAAM,MAAM,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,OAAsB,EACtB,UAAU,GAAG,CAAC;IAEd,MAAM,aAAa,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,OAAO,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAE7C,OAAO,gBAAgB,CACrB,aAAa,EACb,UAAU,EACV,KAAK,IAAI,EAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,EAC5C,OAAO,CAAC,MAAM,CACf,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAsB;IAKjD,OAAO;QACL,SAAS,EAAE,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO;QACrD,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,aAAa,CAAC;QAC7C,MAAM,EAAE,OAAO,EAAE,MAAM;KACxB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,aAAqB,EACrB,OAAsE;IAEtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC9D,OAAO,kBAAkB,CAAC,aAAa,EAAE,WAAW,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;AAC3E,CAAC"}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
declare class FetcherService {
|
|
2
|
-
private readonly client;
|
|
3
|
-
constructor();
|
|
4
|
-
/**
|
|
5
|
-
* Fetches HTML content from a URL
|
|
6
|
-
* @throws {FetchError} on network or HTTP errors
|
|
7
|
-
* @throws {TimeoutError} on request timeout
|
|
8
|
-
*/
|
|
9
|
-
fetchUrl(url: string, customHeaders?: Record<string, string>): Promise<string>;
|
|
10
|
-
/**
|
|
11
|
-
* Fetches URL with exponential backoff retry logic
|
|
12
|
-
* @throws {FetchError} after all retries exhausted
|
|
13
|
-
*/
|
|
14
|
-
fetchUrlWithRetry(url: string, customHeaders?: Record<string, string>, maxRetries?: number): Promise<string>;
|
|
15
|
-
}
|
|
16
|
-
export declare const fetcherService: FetcherService;
|
|
17
|
-
export {};
|
|
18
|
-
//# sourceMappingURL=fetcher.service.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.service.d.ts","sourceRoot":"","sources":["../../src/services/fetcher.service.ts"],"names":[],"mappings":"AA6CA,cAAM,cAAc;IAClB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;;IAoBvC;;;;OAIG;IACG,QAAQ,CACZ,GAAG,EAAE,MAAM,EACX,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACrC,OAAO,CAAC,MAAM,CAAC;IA8ClB;;;OAGG;IACG,iBAAiB,CACrB,GAAG,EAAE,MAAM,EACX,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACtC,UAAU,SAAI,GACb,OAAO,CAAC,MAAM,CAAC;CA6BnB;AAGD,eAAO,MAAM,cAAc,gBAAuB,CAAC"}
|