@venizia/ignis-helpers 0.4.8 → 0.5.0-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/common/constants/http.d.ts +3 -0
- package/dist/esm/common/constants/http.js +3 -0
- package/dist/esm/common/constants/http.js.map +1 -1
- package/dist/esm/common/redact.js +2 -0
- package/dist/esm/common/redact.js.map +1 -1
- package/dist/esm/core.d.ts +5 -2
- package/dist/esm/core.js +5 -2
- package/dist/esm/core.js.map +1 -1
- package/dist/esm/modules/network/http-request/base-network-request.helper.d.ts +5 -0
- package/dist/esm/modules/network/http-request/base-network-request.helper.js +14 -17
- package/dist/esm/modules/network/http-request/base-network-request.helper.js.map +1 -1
- package/dist/esm/modules/network/http-request/common/constants.d.ts +2 -1
- package/dist/esm/modules/network/http-request/common/constants.js +2 -1
- package/dist/esm/modules/network/http-request/common/constants.js.map +1 -1
- package/dist/esm/modules/network/http-request/fetcher/axios-fetcher.d.ts +37 -11
- package/dist/esm/modules/network/http-request/fetcher/axios-fetcher.js +181 -38
- package/dist/esm/modules/network/http-request/fetcher/axios-fetcher.js.map +1 -1
- package/dist/esm/modules/network/http-request/fetcher/base-fetcher.d.ts +204 -18
- package/dist/esm/modules/network/http-request/fetcher/base-fetcher.js +433 -13
- package/dist/esm/modules/network/http-request/fetcher/base-fetcher.js.map +1 -1
- package/dist/esm/modules/network/http-request/fetcher/common/body-limit.d.ts +36 -0
- package/dist/esm/modules/network/http-request/fetcher/common/body-limit.js +119 -0
- package/dist/esm/modules/network/http-request/fetcher/common/body-limit.js.map +1 -0
- package/dist/esm/modules/network/http-request/fetcher/common/errors.d.ts +40 -0
- package/dist/esm/modules/network/http-request/fetcher/common/errors.js +37 -0
- package/dist/esm/modules/network/http-request/fetcher/common/errors.js.map +1 -0
- package/dist/esm/modules/network/http-request/fetcher/common/guard.d.ts +27 -0
- package/dist/esm/modules/network/http-request/fetcher/common/guard.js +2 -0
- package/dist/esm/modules/network/http-request/fetcher/common/guard.js.map +1 -0
- package/dist/esm/modules/network/http-request/fetcher/common/index.d.ts +6 -0
- package/dist/esm/modules/network/http-request/fetcher/common/index.js +6 -0
- package/dist/esm/modules/network/http-request/fetcher/common/index.js.map +1 -1
- package/dist/esm/modules/network/http-request/fetcher/common/request-context.d.ts +15 -0
- package/dist/esm/modules/network/http-request/fetcher/common/request-context.js +11 -0
- package/dist/esm/modules/network/http-request/fetcher/common/request-context.js.map +1 -0
- package/dist/esm/modules/network/http-request/fetcher/common/response-reader.d.ts +9 -0
- package/dist/esm/modules/network/http-request/fetcher/common/response-reader.js +14 -0
- package/dist/esm/modules/network/http-request/fetcher/common/response-reader.js.map +1 -0
- package/dist/esm/modules/network/http-request/fetcher/common/types.d.ts +65 -1
- package/dist/esm/modules/network/http-request/fetcher/common/types.js.map +1 -1
- package/dist/esm/modules/network/http-request/fetcher/common/utilities.d.ts +68 -0
- package/dist/esm/modules/network/http-request/fetcher/common/utilities.js +143 -0
- package/dist/esm/modules/network/http-request/fetcher/common/utilities.js.map +1 -0
- package/dist/esm/modules/network/http-request/fetcher/index.d.ts +2 -1
- package/dist/esm/modules/network/http-request/fetcher/index.js +2 -1
- package/dist/esm/modules/network/http-request/fetcher/index.js.map +1 -1
- package/dist/esm/modules/network/http-request/fetcher/native-fetcher.d.ts +83 -0
- package/dist/esm/modules/network/http-request/fetcher/native-fetcher.js +221 -0
- package/dist/esm/modules/network/http-request/fetcher/native-fetcher.js.map +1 -0
- package/dist/esm/modules/network/utilities/index.d.ts +1 -0
- package/dist/esm/modules/network/utilities/index.js +1 -0
- package/dist/esm/modules/network/utilities/index.js.map +1 -1
- package/dist/esm/modules/network/utilities/query-string.d.ts +6 -23
- package/dist/esm/modules/network/utilities/query-string.js +3 -17
- package/dist/esm/modules/network/utilities/query-string.js.map +1 -1
- package/dist/esm/modules/network/utilities/url-builder.d.ts +39 -0
- package/dist/esm/modules/network/utilities/url-builder.js +102 -0
- package/dist/esm/modules/network/utilities/url-builder.js.map +1 -0
- package/dist/esm/modules/network/utilities/url-safety/common/constants.d.ts +0 -3
- package/dist/esm/modules/network/utilities/url-safety/common/constants.js.map +1 -1
- package/dist/esm/modules/network/utilities/url-safety/common/errors.d.ts +38 -0
- package/dist/esm/modules/network/utilities/url-safety/common/errors.js +33 -1
- package/dist/esm/modules/network/utilities/url-safety/common/errors.js.map +1 -1
- package/dist/esm/modules/network/utilities/url-safety/common/types.d.ts +26 -3
- package/dist/esm/modules/network/utilities/url-safety/common/types.js.map +1 -1
- package/dist/esm/modules/network/utilities/url-safety/guard.d.ts +46 -0
- package/dist/esm/modules/network/utilities/url-safety/guard.js +145 -0
- package/dist/esm/modules/network/utilities/url-safety/guard.js.map +1 -0
- package/dist/esm/modules/network/utilities/url-safety/index.d.ts +1 -1
- package/dist/esm/modules/network/utilities/url-safety/index.js +1 -1
- package/dist/esm/modules/network/utilities/url-safety/index.js.map +1 -1
- package/dist/esm/modules/network/utilities/url-safety/policy.d.ts +52 -2
- package/dist/esm/modules/network/utilities/url-safety/policy.js +161 -14
- package/dist/esm/modules/network/utilities/url-safety/policy.js.map +1 -1
- package/dist/esm/modules/queue/bullmq/helper.d.ts +43 -7
- package/dist/esm/modules/queue/bullmq/helper.js +70 -2
- package/dist/esm/modules/queue/bullmq/helper.js.map +1 -1
- package/dist/esm/modules/retry/helper.d.ts +3 -1
- package/dist/esm/modules/retry/helper.js +8 -4
- package/dist/esm/modules/retry/helper.js.map +1 -1
- package/package.json +3 -3
- package/dist/esm/modules/network/http-request/fetcher/node-fetcher.d.ts +0 -41
- package/dist/esm/modules/network/http-request/fetcher/node-fetcher.js +0 -111
- package/dist/esm/modules/network/http-request/fetcher/node-fetcher.js.map +0 -1
- package/dist/esm/modules/network/utilities/url-safety/ingest.d.ts +0 -95
- package/dist/esm/modules/network/utilities/url-safety/ingest.js +0 -328
- package/dist/esm/modules/network/utilities/url-safety/ingest.js.map +0 -1
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"node-fetcher.js","sourceRoot":"","sources":["../../../../../../src/modules/network/http-request/fetcher/node-fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAoB,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAEL,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EACL,8BAA8B,GAG/B,MAAM,gBAAgB,CAAC;AASxB,iNAAiN;AACjN,MAAM,SAAS,GAAG,CAAC,IAAa,EAAW,EAAE;IAC3C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QACxF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;QAC7E,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAE3C,OAAO,OAAO,IAAI,KAAK,UAAU,IAAI,IAAI,YAAY,IAAI,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,sJAAsJ;AACtJ,MAAM,OAAO,WAAY,SAAQ,8BAKhC;IAIC,YAAY,IAMX;QACC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACjE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC;QACtC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,wDAAwD,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAExF,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IAC5C,CAAC;IAED,oGAAoG;IAC1F,aAAa,CAAC,IAAuB;QAC7C,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QAEtB,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YACxC,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,IAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAChF,CAAC;IAEQ,KAAK,CAAC,IAAI,CAAC,IAA8B,EAAE,MAAuB;QACzE,MAAM,EACJ,GAAG,EACH,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EACzB,MAAM,EACN,IAAI,EACJ,OAAO,EACP,OAAO,EACP,MAAM,EACN,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;QAET,IAAI,SAAoD,CAAC;QACzD,IAAI,iBAA8C,CAAC;QACnD,IAAI,eAAe,GAAG,MAAM,CAAC;QAE7B,IAAI,OAAO,EAAE,CAAC;YACZ,iBAAiB,GAAG,IAAI,eAAe,EAAE,CAAC;YAC1C,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC1B,iBAAiB,EAAE,KAAK,EAAE,CAAC;YAC7B,CAAC,EAAE,OAAO,CAAC,CAAC;YAEZ,iQAAiQ;YACjQ,eAAe,GAAG,MAAM;gBACtB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC;gBACrD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,MAAM,cAAc,GAAgB;YAClC,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,IAAI;YACP,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;YAC5B,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,CAAC;YAClC,uPAAuP;YACvP,OAAO,EAAE,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO;YAC/C,MAAM,EAAE,eAAe;SACxB,CAAC;QAEF,sKAAsK;QACtK,MAAM,gBAAgB,GAAG,MAAM;YAC7B,CAAC,CAAC,kBAAkB,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,UAAU,GAAG,gBAAgB;YACjC,CAAC,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,KAAK,GAAG,gBAAgB,EAAE;YACpI,CAAC,CAAC,GAAG,CAAC;QAER,iGAAiG;QACjG,gGAAgG;QAChG,iGAAiG;QACjG,2FAA2F;QAC3F,MAAM,eAAe,GAAG;YACtB,GAAG,cAAc;YACjB,IAAI,EACF,OAAO,cAAc,CAAC,IAAI,KAAK,QAAQ;gBACrC,CAAC,CAAC,SAAS,cAAc,CAAC,IAAI,CAAC,MAAM,SAAS;gBAC9C,CAAC,CAAC,cAAc,CAAC,IAAI;oBACnB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,SAAS;YACjB,OAAO,EAAE,cAAc,CAAC,OAAO;SAChC,CAAC;QAEF,uOAAuO;QACvO,MAAM,EAAE,KAAK,CACX,wCAAwC,EACxC,IAAI,CAAC,IAAI,CAAC,IAAI,EACd,GAAG,EACH,aAAa,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,EACvD,OAAO,CACR,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAEnD,OAAO,EAAE,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,IAAI,SAAS,EAAE,CAAC;gBACd,YAAY,CAAC,SAAS,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAYD,+HAA+H;AAC/H,MAAM,OAAO,uBAAwB,SAAQ,kBAAgC;IAC3E,YAAY,IAAqC;QAC/C,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC;QACtC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,CAAC;QAElE,uFAAuF;QACvF,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,EAAE,cAAc,EAAE,iCAAiC,EAAE,CAAC,CAAC;QAEzF,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QAE5E,MAAM,cAAc,GAAyB;YAC3C,GAAG,IAAI;YACP,kHAAkH;YAClH,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC;SAC3C,CAAC;QAEF,KAAK,CAAC;YACJ,IAAI;YACJ,OAAO;YACP,OAAO,EAAE,IAAI,WAAW,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;SAChE,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["import { RedactUtility } from '@/common/redact';\nimport { HTTP, type THttpMethod } from '@/common/constants/http';\nimport {\n type IQueryStringOptions,\n QueryStringSeparators,\n QueryStringUtility,\n} from '../../utilities/query-string';\n\nimport { BaseNetworkRequest } from '../base-network-request.helper';\nimport {\n AbstractNetworkFetchableHelper,\n type IRequestOptions,\n type TFetcherLogger,\n} from './base-fetcher';\n\n/** Request accepted by `NodeFetcher`: `RequestInit` plus `url`; `params` are written into the url query. */\nexport interface INodeFetchRequestOptions extends RequestInit, IRequestOptions {\n url: string;\n method?: THttpMethod;\n params?: Record<string, unknown>;\n}\n\n/** What `fetch` takes as a body as it is: encoding any of these again would change the bytes sent. A constructor is looked up on `globalThis` so a runtime without one (no `FormData`) just never matches it. */\nconst isRawBody = (body: unknown): boolean => {\n if (typeof body === 'string' || ArrayBuffer.isView(body) || body instanceof ArrayBuffer) {\n return true;\n }\n\n return ['Blob', 'FormData', 'URLSearchParams', 'ReadableStream'].some((name) => {\n const ctor = Reflect.get(globalThis, name);\n\n return typeof ctor === 'function' && body instanceof ctor;\n });\n};\n\n/** Fetcher over the global `fetch` (`getWorker()`). `timeout` (milliseconds) aborts the request and is combined with a `signal` the caller passes. */\nexport class NodeFetcher extends AbstractNetworkFetchableHelper<\n 'node-fetch',\n INodeFetchRequestOptions,\n Awaited<ReturnType<typeof fetch>>,\n typeof fetch\n> {\n private defaultConfigs: RequestInit;\n private queryString: IQueryStringOptions;\n\n constructor(opts: {\n name: string;\n defaultConfigs: RequestInit;\n /** How `params` are written into the url; unset fields behave as `node:querystring` does. */\n queryString?: IQueryStringOptions;\n logger?: TFetcherLogger;\n }) {\n super({ name: opts.name, variant: 'node-fetch', worker: fetch });\n const { name, defaultConfigs } = opts;\n this.name = name;\n opts?.logger?.info('Creating new network request worker instance! Name: %s', this.name);\n\n this.defaultConfigs = defaultConfigs;\n this.queryString = opts.queryString ?? {};\n }\n\n /** A raw body passes through; a plain object or an array is JSON-encoded; no body stays no body. */\n protected serializeBody(opts: { body: unknown }): RequestInit['body'] | undefined {\n const { body } = opts;\n\n if (body === undefined || body === null) {\n return undefined;\n }\n\n return isRawBody(body) ? (body as RequestInit['body']) : JSON.stringify(body);\n }\n\n override async send(opts: INodeFetchRequestOptions, logger?: TFetcherLogger) {\n const {\n url,\n method = HTTP.Methods.GET,\n params,\n body,\n headers,\n timeout,\n signal,\n ...rest\n } = opts;\n\n let timeoutId: ReturnType<typeof setTimeout> | undefined;\n let timeoutController: AbortController | undefined;\n let effectiveSignal = signal;\n\n if (timeout) {\n timeoutController = new AbortController();\n timeoutId = setTimeout(() => {\n timeoutController?.abort();\n }, timeout);\n\n // Composed, never substituted: a timeout must not silently swallow the caller's own signal, or the caller would believe it cancelled a request that is in fact still running. Either source - and a signal already aborted before the call - aborts the request.\n effectiveSignal = signal\n ? AbortSignal.any([signal, timeoutController.signal])\n : timeoutController.signal;\n }\n\n const requestConfigs: RequestInit = {\n ...this.defaultConfigs,\n ...rest,\n method: method.toUpperCase(),\n body: this.serializeBody({ body }),\n // The call's headers REPLACE the defaults (a caller that sends a multipart body leaves the content type out so `fetch` can write the boundary); a call with none keeps them. Spreading an absent `headers` over the config used to erase the defaults.\n headers: headers ?? this.defaultConfigs.headers,\n signal: effectiveSignal,\n };\n\n // `?` only when the url carries no query string of its own - a second `?` is not a separator, so the server would read the previous parameter's value as `1?extra=2`.\n const serializedParams = params\n ? QueryStringUtility.toQueryString({ params, ...this.queryString })\n : '';\n const requestUrl = serializedParams\n ? `${url}${url.includes(QueryStringSeparators.START) ? QueryStringSeparators.PAIR : QueryStringSeparators.START}${serializedParams}`\n : url;\n\n // The body is NEVER logged, only its size. `fetch` bodies are strings, and `redactSecrets` walks\n // object keys - so a JSON credential payload passed straight through as a string was printed in\n // full, with no key for the redactor to match on. Headers are normalised through `Headers` first\n // because the `string[][]` form is a list of tuples, which key matching cannot see either.\n const loggableConfigs = {\n ...requestConfigs,\n body:\n typeof requestConfigs.body === 'string'\n ? `[body ${requestConfigs.body.length} chars]`\n : requestConfigs.body\n ? '[body]'\n : undefined,\n headers: requestConfigs.headers,\n };\n\n // Straight on the logger it was handed, never `logger.for(...)`: a host logger may carry only `debug` and `info`. With none handed, nothing is logged - no default logger is built, so a browser never reaches the node-only provider.\n logger?.debug(\n '[%s] URL: %s | Props: %s | Timeout: %s',\n this.send.name,\n url,\n RedactUtility.redactSecrets({ value: loggableConfigs }),\n timeout,\n );\n\n try {\n const rs = await fetch(requestUrl, requestConfigs);\n\n return rs;\n } finally {\n if (timeoutId) {\n clearTimeout(timeoutId);\n }\n }\n }\n}\n\n/** Options of `NodeFetchNetworkRequest`: `RequestInit` plus `baseUrl` and `queryString`, which sets how `params` are written into every url. */\nexport interface INodeFetchNetworkRequestOptions {\n name: string;\n networkOptions: RequestInit & {\n baseUrl?: string;\n /** How `params` are written into every url this request sends; see {@link IQueryStringOptions}. */\n queryString?: IQueryStringOptions;\n };\n}\n\n/** `BaseNetworkRequest` over a `NodeFetcher`. Request headers default to a JSON content type, and the caller's headers win. */\nexport class NodeFetchNetworkRequest extends BaseNetworkRequest<'node-fetch'> {\n constructor(opts: INodeFetchNetworkRequestOptions) {\n const { name, networkOptions } = opts;\n const { headers, baseUrl, queryString, ...rest } = networkOptions;\n\n // Case-insensitive, like the wire: a caller's `Content-Type` replaces the default one.\n const mergedHeaders = new Headers({ 'content-type': 'application/json; charset=utf-8' });\n\n new Headers(headers).forEach((value, key) => mergedHeaders.set(key, value));\n\n const defaultConfigs: Partial<RequestInit> = {\n ...rest,\n // A plain object, so the request log (which redacts by key) can read it; a `Headers` instance has no own entries.\n headers: Object.fromEntries(mergedHeaders),\n };\n\n super({\n name,\n baseUrl,\n fetcher: new NodeFetcher({ name, defaultConfigs, queryString }),\n });\n }\n}\n"]}
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import type { IUrlSafetyPolicy } from './common/types.js';
|
|
2
|
-
/** Fetching an untrusted url without letting it reach inside the network. */
|
|
3
|
-
export declare class UrlIngest {
|
|
4
|
-
/**
|
|
5
|
-
* Resolves `hostname` once and returns every address it answered, refusing the host when ANY of them is non-public.
|
|
6
|
-
* Checking only the first walks straight into a host that publishes one public A record beside an internal one.
|
|
7
|
-
* An IP literal answers itself. `allowPrivateAddress` skips the check and returns an empty list: nothing to pin to.
|
|
8
|
-
*/
|
|
9
|
-
static resolvePublicAddresses(opts: {
|
|
10
|
-
hostname: string;
|
|
11
|
-
policy?: IUrlSafetyPolicy;
|
|
12
|
-
signal?: AbortSignal;
|
|
13
|
-
}): Promise<string[]>;
|
|
14
|
-
/**
|
|
15
|
-
* Rejects when ANY resolved address is non-public.
|
|
16
|
-
*
|
|
17
|
-
* A rebinding window remains when this is called on its own: the address can change between this check and a
|
|
18
|
-
* later connect. {@link UrlIngest.fetchGuarded} closes it by connecting to the address it vetted.
|
|
19
|
-
*/
|
|
20
|
-
static assertPublicHost(opts: {
|
|
21
|
-
hostname: string;
|
|
22
|
-
policy?: IUrlSafetyPolicy;
|
|
23
|
-
}): Promise<void>;
|
|
24
|
-
/** Both halves, in the order they must run: shape first, then the network. */
|
|
25
|
-
static assertReachableUrl(opts: {
|
|
26
|
-
url: string;
|
|
27
|
-
policy?: IUrlSafetyPolicy;
|
|
28
|
-
}): Promise<URL>;
|
|
29
|
-
private static readTimeoutMilliseconds;
|
|
30
|
-
/** Settles with `work`, or rejects with the signal's reason when the deadline passes first: a slow resolver must not outlive it. */
|
|
31
|
-
private static withDeadline;
|
|
32
|
-
/** Request headers that name the body; they go when a redirect turns the request into a GET. */
|
|
33
|
-
private static readonly BODY_HEADERS;
|
|
34
|
-
/**
|
|
35
|
-
* The only caller headers a cross-origin redirect keeps. Every other one is dropped: a tenant's webhook secret
|
|
36
|
-
* (`x-api-key`, a signature) must not follow a redirect to a host that tenant did not name.
|
|
37
|
-
*/
|
|
38
|
-
private static readonly CROSS_ORIGIN_HEADERS;
|
|
39
|
-
/** The statuses that name a new location; a 304 or a 300 is an answer, not a redirect. */
|
|
40
|
-
private static readonly REDIRECT_STATUSES;
|
|
41
|
-
/** Connect-phase errors: nothing was sent, so trying the next vetted address cannot deliver a request twice. */
|
|
42
|
-
private static readonly CONNECT_ERROR_CODES;
|
|
43
|
-
private static onlyHeaders;
|
|
44
|
-
private static withoutHeaders;
|
|
45
|
-
/**
|
|
46
|
-
* One request to `target`. With `addresses` it connects to those vetted addresses and nowhere else, in order, and
|
|
47
|
-
* moves to the next one only when the connection itself fails: the url's host is set as the `Host` header, and for
|
|
48
|
-
* https as the TLS server name, so the certificate is still checked against the name the caller asked for.
|
|
49
|
-
*/
|
|
50
|
-
private static send;
|
|
51
|
-
/**
|
|
52
|
-
* A request that an untrusted url cannot steer inside the network. GET by default; `method`, `headers` and `body`
|
|
53
|
-
* (text or bytes, so a redirect can replay it) make it a signed webhook dispatch.
|
|
54
|
-
*
|
|
55
|
-
* The host is resolved ONCE per hop, every answer is checked, and the request connects to those addresses and to no
|
|
56
|
-
* other, so a DNS answer that changes after the check (DNS rebinding) is never looked up again. The `Host` header and,
|
|
57
|
-
* for https, the TLS server name stay the url's own, so the certificate is verified for that name. A `Host` in
|
|
58
|
-
* `headers` is ignored. The environment's `HTTP_PROXY` / `HTTPS_PROXY` is ignored on a pinned
|
|
59
|
-
* connection, because a proxy would resolve the name itself and undo the pin. `policy.proxy` sends the request through a
|
|
60
|
-
* proxy on purpose; that connection is not pinned (see `IUrlSafetyPolicy.proxy`). `allowPrivateAddress` turns the guard,
|
|
61
|
-
* and so the pin, off.
|
|
62
|
-
*
|
|
63
|
-
* Redirects are followed by hand so every hop is re-checked and re-pinned - `redirect: 'follow'` would land on an
|
|
64
|
-
* internal host with no second look. 301, 302 (for a POST) and 303 turn the request into a GET without a body. A 307 or
|
|
65
|
-
* 308 repeats the request; with a body it is refused when it leaves the origin, so a signed payload is never replayed to
|
|
66
|
-
* a host a redirect chose. When a hop changes origin every caller header is dropped except `accept`, `accept-language` and `user-agent`.
|
|
67
|
-
*
|
|
68
|
-
* The body is returned unread: the caller decides whether to stream it onward or cap it with
|
|
69
|
-
* {@link UrlIngest.readCappedBody}. Nothing is buffered here.
|
|
70
|
-
*/
|
|
71
|
-
static fetchGuarded(opts: {
|
|
72
|
-
url: string;
|
|
73
|
-
policy?: IUrlSafetyPolicy;
|
|
74
|
-
method?: string;
|
|
75
|
-
headers?: Record<string, string>;
|
|
76
|
-
body?: string | Uint8Array;
|
|
77
|
-
}): Promise<Response>;
|
|
78
|
-
/**
|
|
79
|
-
* Enforces `maxBytes` on a stream nothing reads into this process - the shape a caller wants when
|
|
80
|
-
* the body is piped onward rather than held. Without it a `maxBytes` policy is inert on that path.
|
|
81
|
-
*/
|
|
82
|
-
static capStream(opts: {
|
|
83
|
-
source: ReadableStream<Uint8Array>;
|
|
84
|
-
policy?: IUrlSafetyPolicy;
|
|
85
|
-
}): ReadableStream<Uint8Array>;
|
|
86
|
-
/**
|
|
87
|
-
* Reads a body that must not exceed `maxBytes`, cancelling the moment it does. `content-length` is
|
|
88
|
-
* checked first because it is free, and counted anyway because it may be absent or lie.
|
|
89
|
-
*/
|
|
90
|
-
static readCappedBody(opts: {
|
|
91
|
-
response: Response;
|
|
92
|
-
policy?: IUrlSafetyPolicy;
|
|
93
|
-
}): Promise<Buffer>;
|
|
94
|
-
}
|
|
95
|
-
//# sourceMappingURL=ingest.d.ts.map
|
|
@@ -1,328 +0,0 @@
|
|
|
1
|
-
// The half `policy.ts` cannot do: resolving a hostname needs `node:dns`, so nothing here may be
|
|
2
|
-
// re-exported from the `./core` subpath.
|
|
3
|
-
import { DurationMultipliers } from '../../../../common/constants/duration.js';
|
|
4
|
-
import { getError } from '../../../error/index.js';
|
|
5
|
-
import { lookup } from 'node:dns/promises';
|
|
6
|
-
import { UrlSafetyDefaults } from './common/constants.js';
|
|
7
|
-
import { UrlSafetyErrors } from './common/errors.js';
|
|
8
|
-
import { UrlPolicy } from './policy.js';
|
|
9
|
-
/** Fetching an untrusted url without letting it reach inside the network. */
|
|
10
|
-
export class UrlIngest {
|
|
11
|
-
/**
|
|
12
|
-
* Resolves `hostname` once and returns every address it answered, refusing the host when ANY of them is non-public.
|
|
13
|
-
* Checking only the first walks straight into a host that publishes one public A record beside an internal one.
|
|
14
|
-
* An IP literal answers itself. `allowPrivateAddress` skips the check and returns an empty list: nothing to pin to.
|
|
15
|
-
*/
|
|
16
|
-
static async resolvePublicAddresses(opts) {
|
|
17
|
-
const { hostname, policy, signal } = opts;
|
|
18
|
-
if (policy?.allowPrivateAddress ?? UrlSafetyDefaults.ALLOW_PRIVATE_ADDRESS) {
|
|
19
|
-
return [];
|
|
20
|
-
}
|
|
21
|
-
// `URL.hostname` keeps the brackets on an IPv6 literal; `lookup` does not accept them, and the
|
|
22
|
-
// ENOTFOUND that follows reads as a transient DNS failure rather than the refusal it should be.
|
|
23
|
-
const address = UrlPolicy.readHostAddress({ hostname });
|
|
24
|
-
const addresses = await UrlIngest.withDeadline({
|
|
25
|
-
work: lookup(address, { all: true }),
|
|
26
|
-
signal,
|
|
27
|
-
});
|
|
28
|
-
if (!addresses.length) {
|
|
29
|
-
throw getError({
|
|
30
|
-
error: UrlSafetyErrors.URL_REFUSED,
|
|
31
|
-
message: `[assertPublicHost] Host resolves to nothing | host: ${hostname}`,
|
|
32
|
-
});
|
|
33
|
-
}
|
|
34
|
-
const offender = addresses.find((entry) => UrlPolicy.isNonPublicAddress({ address: entry.address }));
|
|
35
|
-
if (offender) {
|
|
36
|
-
throw getError({
|
|
37
|
-
error: UrlSafetyErrors.URL_REFUSED,
|
|
38
|
-
message: `[assertPublicHost] Host resolves to a non-public address | host: ${hostname} | address: ${offender.address}`,
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
return addresses.map((entry) => entry.address);
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* Rejects when ANY resolved address is non-public.
|
|
45
|
-
*
|
|
46
|
-
* A rebinding window remains when this is called on its own: the address can change between this check and a
|
|
47
|
-
* later connect. {@link UrlIngest.fetchGuarded} closes it by connecting to the address it vetted.
|
|
48
|
-
*/
|
|
49
|
-
static async assertPublicHost(opts) {
|
|
50
|
-
await UrlIngest.resolvePublicAddresses(opts);
|
|
51
|
-
}
|
|
52
|
-
/** Both halves, in the order they must run: shape first, then the network. */
|
|
53
|
-
static async assertReachableUrl(opts) {
|
|
54
|
-
const parsed = UrlPolicy.assertSafeUrl(opts);
|
|
55
|
-
await UrlIngest.assertPublicHost({ hostname: parsed.hostname, policy: opts.policy });
|
|
56
|
-
return parsed;
|
|
57
|
-
}
|
|
58
|
-
static readTimeoutMilliseconds(opts) {
|
|
59
|
-
const { timeout } = opts.policy ?? {};
|
|
60
|
-
const milliseconds = timeout ? DurationMultipliers.toMilliseconds(timeout) : null;
|
|
61
|
-
return milliseconds ?? UrlSafetyDefaults.TIMEOUT_MILLISECONDS;
|
|
62
|
-
}
|
|
63
|
-
/** Settles with `work`, or rejects with the signal's reason when the deadline passes first: a slow resolver must not outlive it. */
|
|
64
|
-
static async withDeadline(opts) {
|
|
65
|
-
const { work, signal } = opts;
|
|
66
|
-
if (!signal) {
|
|
67
|
-
return work;
|
|
68
|
-
}
|
|
69
|
-
signal.throwIfAborted();
|
|
70
|
-
let onAbort;
|
|
71
|
-
const aborted = new Promise((_resolve, reject) => {
|
|
72
|
-
onAbort = () => reject(signal.reason);
|
|
73
|
-
signal.addEventListener('abort', onAbort, { once: true });
|
|
74
|
-
});
|
|
75
|
-
try {
|
|
76
|
-
const settled = await Promise.race([work, aborted]);
|
|
77
|
-
return settled;
|
|
78
|
-
}
|
|
79
|
-
finally {
|
|
80
|
-
signal.removeEventListener('abort', onAbort);
|
|
81
|
-
// The lookup that lost keeps running; its late answer or error must not surface as an unhandled rejection. Nothing
|
|
82
|
-
// is left to report: the caller already got the deadline's error.
|
|
83
|
-
// eslint-disable-next-line venizia/no-silent-catch -- the work lost the race; its outcome is irrelevant by now
|
|
84
|
-
work.catch(() => undefined);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
/** Request headers that name the body; they go when a redirect turns the request into a GET. */
|
|
88
|
-
static { this.BODY_HEADERS = ['content-type', 'content-length', 'content-encoding']; }
|
|
89
|
-
/**
|
|
90
|
-
* The only caller headers a cross-origin redirect keeps. Every other one is dropped: a tenant's webhook secret
|
|
91
|
-
* (`x-api-key`, a signature) must not follow a redirect to a host that tenant did not name.
|
|
92
|
-
*/
|
|
93
|
-
static { this.CROSS_ORIGIN_HEADERS = ['accept', 'accept-language', 'user-agent']; }
|
|
94
|
-
/** The statuses that name a new location; a 304 or a 300 is an answer, not a redirect. */
|
|
95
|
-
static { this.REDIRECT_STATUSES = [301, 302, 303, 307, 308]; }
|
|
96
|
-
/** Connect-phase errors: nothing was sent, so trying the next vetted address cannot deliver a request twice. */
|
|
97
|
-
static { this.CONNECT_ERROR_CODES = [
|
|
98
|
-
'ECONNREFUSED',
|
|
99
|
-
'ENETUNREACH',
|
|
100
|
-
'EHOSTUNREACH',
|
|
101
|
-
'EADDRNOTAVAIL',
|
|
102
|
-
]; }
|
|
103
|
-
static onlyHeaders(opts) {
|
|
104
|
-
const { headers, names } = opts;
|
|
105
|
-
return Object.fromEntries(Object.entries(headers).filter(([name]) => names.includes(name.toLowerCase())));
|
|
106
|
-
}
|
|
107
|
-
static withoutHeaders(opts) {
|
|
108
|
-
const { headers, names } = opts;
|
|
109
|
-
return Object.fromEntries(Object.entries(headers).filter(([name]) => !names.includes(name.toLowerCase())));
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* One request to `target`. With `addresses` it connects to those vetted addresses and nowhere else, in order, and
|
|
113
|
-
* moves to the next one only when the connection itself fails: the url's host is set as the `Host` header, and for
|
|
114
|
-
* https as the TLS server name, so the certificate is still checked against the name the caller asked for.
|
|
115
|
-
*/
|
|
116
|
-
static async send(opts) {
|
|
117
|
-
const { target, addresses, init } = opts;
|
|
118
|
-
if (!addresses.length) {
|
|
119
|
-
return fetch(target, init);
|
|
120
|
-
}
|
|
121
|
-
// `proxy: false` is Bun's way to ignore HTTP_PROXY / HTTPS_PROXY for this call (an empty string or undefined keeps
|
|
122
|
-
// the environment's proxy, which would resolve the name itself and undo the pin). A test pins it.
|
|
123
|
-
const noProxy = { proxy: false };
|
|
124
|
-
const isHttps = target.protocol === 'https:';
|
|
125
|
-
const hasNameToVerify = isHttps &&
|
|
126
|
-
!UrlPolicy.readHostAddress({ hostname: target.hostname }).includes(':') &&
|
|
127
|
-
!/^[\d.]+$/.test(target.hostname);
|
|
128
|
-
const headers = {
|
|
129
|
-
...UrlIngest.withoutHeaders({
|
|
130
|
-
headers: init.headers ?? {},
|
|
131
|
-
names: ['host'],
|
|
132
|
-
}),
|
|
133
|
-
Host: target.host,
|
|
134
|
-
};
|
|
135
|
-
let lastError;
|
|
136
|
-
for (const address of addresses) {
|
|
137
|
-
const pinned = new URL(target);
|
|
138
|
-
pinned.hostname = address.includes(':') ? `[${address}]` : address;
|
|
139
|
-
try {
|
|
140
|
-
const result = await fetch(pinned, {
|
|
141
|
-
...init,
|
|
142
|
-
...noProxy,
|
|
143
|
-
headers,
|
|
144
|
-
...(hasNameToVerify ? { tls: { serverName: target.hostname } } : {}),
|
|
145
|
-
});
|
|
146
|
-
return result;
|
|
147
|
-
}
|
|
148
|
-
catch (error) {
|
|
149
|
-
const name = error?.name;
|
|
150
|
-
// The deadline is not a reason to try another address: it is spent.
|
|
151
|
-
if (name === 'AbortError' || name === 'TimeoutError') {
|
|
152
|
-
throw error;
|
|
153
|
-
}
|
|
154
|
-
// A request that may have reached the server (a reset after the body went out) is not repeated on another
|
|
155
|
-
// address unless it is safe to repeat: a webhook must not be delivered twice.
|
|
156
|
-
const code = error?.code ?? '';
|
|
157
|
-
const method = (init.method ?? 'GET').toUpperCase();
|
|
158
|
-
if (method !== 'GET' &&
|
|
159
|
-
method !== 'HEAD' &&
|
|
160
|
-
!UrlIngest.CONNECT_ERROR_CODES.includes(code)) {
|
|
161
|
-
throw error;
|
|
162
|
-
}
|
|
163
|
-
lastError = error;
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
throw lastError;
|
|
167
|
-
}
|
|
168
|
-
/**
|
|
169
|
-
* A request that an untrusted url cannot steer inside the network. GET by default; `method`, `headers` and `body`
|
|
170
|
-
* (text or bytes, so a redirect can replay it) make it a signed webhook dispatch.
|
|
171
|
-
*
|
|
172
|
-
* The host is resolved ONCE per hop, every answer is checked, and the request connects to those addresses and to no
|
|
173
|
-
* other, so a DNS answer that changes after the check (DNS rebinding) is never looked up again. The `Host` header and,
|
|
174
|
-
* for https, the TLS server name stay the url's own, so the certificate is verified for that name. A `Host` in
|
|
175
|
-
* `headers` is ignored. The environment's `HTTP_PROXY` / `HTTPS_PROXY` is ignored on a pinned
|
|
176
|
-
* connection, because a proxy would resolve the name itself and undo the pin. `policy.proxy` sends the request through a
|
|
177
|
-
* proxy on purpose; that connection is not pinned (see `IUrlSafetyPolicy.proxy`). `allowPrivateAddress` turns the guard,
|
|
178
|
-
* and so the pin, off.
|
|
179
|
-
*
|
|
180
|
-
* Redirects are followed by hand so every hop is re-checked and re-pinned - `redirect: 'follow'` would land on an
|
|
181
|
-
* internal host with no second look. 301, 302 (for a POST) and 303 turn the request into a GET without a body. A 307 or
|
|
182
|
-
* 308 repeats the request; with a body it is refused when it leaves the origin, so a signed payload is never replayed to
|
|
183
|
-
* a host a redirect chose. When a hop changes origin every caller header is dropped except `accept`, `accept-language` and `user-agent`.
|
|
184
|
-
*
|
|
185
|
-
* The body is returned unread: the caller decides whether to stream it onward or cap it with
|
|
186
|
-
* {@link UrlIngest.readCappedBody}. Nothing is buffered here.
|
|
187
|
-
*/
|
|
188
|
-
static async fetchGuarded(opts) {
|
|
189
|
-
const { url, policy } = opts;
|
|
190
|
-
const maxRedirects = policy?.maxRedirects ?? UrlSafetyDefaults.MAX_REDIRECTS;
|
|
191
|
-
const timeoutMilliseconds = UrlIngest.readTimeoutMilliseconds({ policy });
|
|
192
|
-
let method = (opts.method ?? 'GET').toUpperCase();
|
|
193
|
-
let headers = opts.headers ?? {};
|
|
194
|
-
let body = opts.body;
|
|
195
|
-
if (body !== undefined && (method === 'GET' || method === 'HEAD')) {
|
|
196
|
-
throw getError({
|
|
197
|
-
message: `[fetchGuarded] A ${method} request cannot carry a body | url: ${url}`,
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
let target = UrlPolicy.assertSafeUrl({ url, policy });
|
|
201
|
-
// ONE deadline for the whole call, every lookup and every hop included, not one per hop.
|
|
202
|
-
const signal = AbortSignal.timeout(timeoutMilliseconds);
|
|
203
|
-
for (let hop = 0; hop <= maxRedirects; hop += 1) {
|
|
204
|
-
const addresses = await UrlIngest.resolvePublicAddresses({
|
|
205
|
-
hostname: target.hostname,
|
|
206
|
-
policy,
|
|
207
|
-
signal,
|
|
208
|
-
});
|
|
209
|
-
const response = await UrlIngest.send({
|
|
210
|
-
target,
|
|
211
|
-
// A proxy resolves the name itself, so there is nothing to pin: the caller chose it (`policy.proxy`).
|
|
212
|
-
addresses: policy?.proxy ? [] : addresses,
|
|
213
|
-
init: {
|
|
214
|
-
method,
|
|
215
|
-
...(policy?.proxy ? { proxy: policy.proxy } : {}),
|
|
216
|
-
redirect: 'manual',
|
|
217
|
-
signal,
|
|
218
|
-
...(Object.keys(headers).length ? { headers } : {}),
|
|
219
|
-
...(body === undefined ? {} : { body }),
|
|
220
|
-
},
|
|
221
|
-
});
|
|
222
|
-
if (!UrlIngest.REDIRECT_STATUSES.includes(response.status)) {
|
|
223
|
-
if (!response.ok) {
|
|
224
|
-
// Cancelled, or the socket stays open on every 4xx and 5xx.
|
|
225
|
-
await response.body?.cancel();
|
|
226
|
-
throw getError({
|
|
227
|
-
message: `[fetchGuarded] Request failed | status: ${response.status} | url: ${url}`,
|
|
228
|
-
});
|
|
229
|
-
}
|
|
230
|
-
return response;
|
|
231
|
-
}
|
|
232
|
-
const location = response.headers.get('location');
|
|
233
|
-
await response.body?.cancel();
|
|
234
|
-
if (!location) {
|
|
235
|
-
throw getError({
|
|
236
|
-
error: UrlSafetyErrors.URL_REFUSED,
|
|
237
|
-
message: `[fetchGuarded] Redirect carried no location | url: ${url}`,
|
|
238
|
-
});
|
|
239
|
-
}
|
|
240
|
-
const next = UrlPolicy.assertSafeUrl({ url: new URL(location, target).toString(), policy });
|
|
241
|
-
const isSameOrigin = next.origin === target.origin;
|
|
242
|
-
const becomesGet = response.status === 303 ||
|
|
243
|
-
((response.status === 301 || response.status === 302) && method === 'POST');
|
|
244
|
-
if (becomesGet) {
|
|
245
|
-
method = 'GET';
|
|
246
|
-
body = undefined;
|
|
247
|
-
headers = UrlIngest.withoutHeaders({ headers, names: UrlIngest.BODY_HEADERS });
|
|
248
|
-
}
|
|
249
|
-
else if (body !== undefined && !isSameOrigin) {
|
|
250
|
-
throw getError({
|
|
251
|
-
error: UrlSafetyErrors.URL_REFUSED,
|
|
252
|
-
message: `[fetchGuarded] Redirect would replay the request body to another origin | from: ${target.origin} | to: ${next.origin}`,
|
|
253
|
-
});
|
|
254
|
-
}
|
|
255
|
-
if (!isSameOrigin) {
|
|
256
|
-
headers = UrlIngest.onlyHeaders({ headers, names: UrlIngest.CROSS_ORIGIN_HEADERS });
|
|
257
|
-
}
|
|
258
|
-
target = next;
|
|
259
|
-
}
|
|
260
|
-
throw getError({
|
|
261
|
-
error: UrlSafetyErrors.URL_REFUSED,
|
|
262
|
-
message: `[fetchGuarded] Too many redirects | max: ${maxRedirects} | url: ${url}`,
|
|
263
|
-
});
|
|
264
|
-
}
|
|
265
|
-
/**
|
|
266
|
-
* Enforces `maxBytes` on a stream nothing reads into this process - the shape a caller wants when
|
|
267
|
-
* the body is piped onward rather than held. Without it a `maxBytes` policy is inert on that path.
|
|
268
|
-
*/
|
|
269
|
-
static capStream(opts) {
|
|
270
|
-
const { source } = opts;
|
|
271
|
-
const maxBytes = opts.policy?.maxBytes ?? UrlSafetyDefaults.MAX_BYTES;
|
|
272
|
-
let received = 0;
|
|
273
|
-
return source.pipeThrough(new TransformStream({
|
|
274
|
-
transform(chunk, controller) {
|
|
275
|
-
received += chunk.byteLength;
|
|
276
|
-
if (received > maxBytes) {
|
|
277
|
-
controller.error(getError({
|
|
278
|
-
error: UrlSafetyErrors.URL_REFUSED,
|
|
279
|
-
message: `[capStream] Body exceeds the cap | received: ${received} | max: ${maxBytes}`,
|
|
280
|
-
}));
|
|
281
|
-
return;
|
|
282
|
-
}
|
|
283
|
-
controller.enqueue(chunk);
|
|
284
|
-
},
|
|
285
|
-
}));
|
|
286
|
-
}
|
|
287
|
-
/**
|
|
288
|
-
* Reads a body that must not exceed `maxBytes`, cancelling the moment it does. `content-length` is
|
|
289
|
-
* checked first because it is free, and counted anyway because it may be absent or lie.
|
|
290
|
-
*/
|
|
291
|
-
static async readCappedBody(opts) {
|
|
292
|
-
const { response, policy } = opts;
|
|
293
|
-
const maxBytes = policy?.maxBytes ?? UrlSafetyDefaults.MAX_BYTES;
|
|
294
|
-
const declaredLength = Number(response.headers.get('content-length') ?? 0);
|
|
295
|
-
if (declaredLength > maxBytes) {
|
|
296
|
-
await response.body?.cancel();
|
|
297
|
-
// The same fact as the streaming branch below, so the same verdict: a caller must not retry an
|
|
298
|
-
// oversized body when the server sends `content-length` and drop it when the server omits one.
|
|
299
|
-
throw getError({
|
|
300
|
-
error: UrlSafetyErrors.URL_REFUSED,
|
|
301
|
-
message: `[readCappedBody] Body exceeds the cap | declared: ${declaredLength} | max: ${maxBytes}`,
|
|
302
|
-
});
|
|
303
|
-
}
|
|
304
|
-
const reader = response.body?.getReader();
|
|
305
|
-
if (!reader) {
|
|
306
|
-
throw getError({ message: '[readCappedBody] Response carried no body' });
|
|
307
|
-
}
|
|
308
|
-
const chunks = [];
|
|
309
|
-
let received = 0;
|
|
310
|
-
for (;;) {
|
|
311
|
-
const { done, value } = await reader.read();
|
|
312
|
-
if (done) {
|
|
313
|
-
break;
|
|
314
|
-
}
|
|
315
|
-
received += value.length;
|
|
316
|
-
if (received > maxBytes) {
|
|
317
|
-
await reader.cancel();
|
|
318
|
-
throw getError({
|
|
319
|
-
error: UrlSafetyErrors.URL_REFUSED,
|
|
320
|
-
message: `[readCappedBody] Body exceeds the cap | received: ${received} | max: ${maxBytes}`,
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
|
-
chunks.push(value);
|
|
324
|
-
}
|
|
325
|
-
return Buffer.concat(chunks);
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
//# sourceMappingURL=ingest.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../../../../src/modules/network/utilities/url-safety/ingest.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,yCAAyC;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAOrC,6EAA6E;AAC7E,MAAM,OAAO,SAAS;IACpB;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAInC;QACC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAE1C,IAAI,MAAM,EAAE,mBAAmB,IAAI,iBAAiB,CAAC,qBAAqB,EAAE,CAAC;YAC3E,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,+FAA+F;QAC/F,gGAAgG;QAChG,MAAM,OAAO,GAAG,SAAS,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC;YAC7C,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;YACpC,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,QAAQ,CAAC;gBACb,KAAK,EAAE,eAAe,CAAC,WAAW;gBAClC,OAAO,EAAE,uDAAuD,QAAQ,EAAE;aAC3E,CAAC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACxC,SAAS,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CACzD,CAAC;QAEF,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,QAAQ,CAAC;gBACb,KAAK,EAAE,eAAe,CAAC,WAAW;gBAClC,OAAO,EAAE,oEAAoE,QAAQ,eAAe,QAAQ,CAAC,OAAO,EAAE;aACvH,CAAC,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAG7B;QACC,MAAM,SAAS,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,8EAA8E;IAC9E,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAgD;QAC9E,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,SAAS,CAAC,gBAAgB,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAErF,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,uBAAuB,CAAC,IAAmC;QACxE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QACtC,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAElF,OAAO,YAAY,IAAI,iBAAiB,CAAC,oBAAoB,CAAC;IAChE,CAAC;IAED,oIAAoI;IAC5H,MAAM,CAAC,KAAK,CAAC,YAAY,CAAI,IAGpC;QACC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAE9B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,CAAC,cAAc,EAAE,CAAC;QAExB,IAAI,OAAiC,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YACtD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YAEpD,OAAO,OAAO,CAAC;QACjB,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAqB,CAAC,CAAC;YAC3D,mHAAmH;YACnH,kEAAkE;YAClE,+GAA+G;YAC/G,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,gGAAgG;aACxE,iBAAY,GAAG,CAAC,cAAc,EAAE,gBAAgB,EAAE,kBAAkB,CAAC,CAAC;IAE9F;;;OAGG;aACqB,yBAAoB,GAAG,CAAC,QAAQ,EAAE,iBAAiB,EAAE,YAAY,CAAC,CAAC;IAE3F,0FAA0F;aAClE,sBAAiB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAEtE,gHAAgH;aACxF,wBAAmB,GAAG;QAC5C,cAAc;QACd,aAAa;QACb,cAAc;QACd,eAAe;KAChB,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,IAG1B;QACC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;QAEhC,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAC/E,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,IAG7B;QACC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;QAEhC,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAChF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAIzB;QACC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QAEzC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,mHAAmH;QACnH,kGAAkG;QAClG,MAAM,OAAO,GAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC;QAC7C,MAAM,eAAe,GACnB,OAAO;YACP,CAAC,SAAS,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;YACvE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG;YACd,GAAG,SAAS,CAAC,cAAc,CAAC;gBAC1B,OAAO,EAAG,IAAI,CAAC,OAA8C,IAAI,EAAE;gBACnE,KAAK,EAAE,CAAC,MAAM,CAAC;aAChB,CAAC;YACF,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;QACF,IAAI,SAAkB,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YAEnE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;oBACjC,GAAG,IAAI;oBACP,GAAG,OAAO;oBACV,OAAO;oBACP,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACrE,CAAC,CAAC;gBAEH,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,GAAI,KAA2B,EAAE,IAAI,CAAC;gBAEhD,oEAAoE;gBACpE,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;oBACrD,MAAM,KAAK,CAAC;gBACd,CAAC;gBAED,0GAA0G;gBAC1G,8EAA8E;gBAC9E,MAAM,IAAI,GAAI,KAAuC,EAAE,IAAI,IAAI,EAAE,CAAC;gBAClE,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;gBAEpD,IACE,MAAM,KAAK,KAAK;oBAChB,MAAM,KAAK,MAAM;oBACjB,CAAC,SAAS,CAAC,mBAAmB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAC7C,CAAC;oBACD,MAAM,KAAK,CAAC;gBACd,CAAC;gBAED,SAAS,GAAG,KAAK,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,IAMzB;QACC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAC7B,MAAM,YAAY,GAAG,MAAM,EAAE,YAAY,IAAI,iBAAiB,CAAC,aAAa,CAAC;QAC7E,MAAM,mBAAmB,GAAG,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAE1E,IAAI,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACjC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QAErB,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;YAClE,MAAM,QAAQ,CAAC;gBACb,OAAO,EAAE,oBAAoB,MAAM,uCAAuC,GAAG,EAAE;aAChF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,yFAAyF;QACzF,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QAExD,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,sBAAsB,CAAC;gBACvD,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM;gBACN,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC;gBACpC,MAAM;gBACN,sGAAsG;gBACtG,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;gBACzC,IAAI,EAAE;oBACJ,MAAM;oBACN,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjD,QAAQ,EAAE,QAAQ;oBAClB,MAAM;oBACN,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnD,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;iBACxC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3D,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,4DAA4D;oBAC5D,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;oBAC9B,MAAM,QAAQ,CAAC;wBACb,OAAO,EAAE,2CAA2C,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE;qBACpF,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO,QAAQ,CAAC;YAClB,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;YAE9B,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,QAAQ,CAAC;oBACb,KAAK,EAAE,eAAe,CAAC,WAAW;oBAClC,OAAO,EAAE,sDAAsD,GAAG,EAAE;iBACrE,CAAC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,GAAG,SAAS,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5F,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC;YACnD,MAAM,UAAU,GACd,QAAQ,CAAC,MAAM,KAAK,GAAG;gBACvB,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,MAAM,KAAK,MAAM,CAAC,CAAC;YAE9E,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,GAAG,KAAK,CAAC;gBACf,IAAI,GAAG,SAAS,CAAC;gBACjB,OAAO,GAAG,SAAS,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC;YACjF,CAAC;iBAAM,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,YAAY,EAAE,CAAC;gBAC/C,MAAM,QAAQ,CAAC;oBACb,KAAK,EAAE,eAAe,CAAC,WAAW;oBAClC,OAAO,EAAE,mFAAmF,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,EAAE;iBACjI,CAAC,CAAC;YACL,CAAC;YAED,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,OAAO,GAAG,SAAS,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,oBAAoB,EAAE,CAAC,CAAC;YACtF,CAAC;YAED,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,QAAQ,CAAC;YACb,KAAK,EAAE,eAAe,CAAC,WAAW;YAClC,OAAO,EAAE,4CAA4C,YAAY,WAAW,GAAG,EAAE;SAClF,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,SAAS,CAAC,IAGhB;QACC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,IAAI,iBAAiB,CAAC,SAAS,CAAC;QACtE,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,OAAO,MAAM,CAAC,WAAW,CACvB,IAAI,eAAe,CAAyB;YAC1C,SAAS,CAAC,KAAK,EAAE,UAAU;gBACzB,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC;gBAE7B,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;oBACxB,UAAU,CAAC,KAAK,CACd,QAAQ,CAAC;wBACP,KAAK,EAAE,eAAe,CAAC,WAAW;wBAClC,OAAO,EAAE,gDAAgD,QAAQ,WAAW,QAAQ,EAAE;qBACvF,CAAC,CACH,CAAC;oBAEF,OAAO;gBACT,CAAC;gBAED,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;SACF,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,IAG3B;QACC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAClC,MAAM,QAAQ,GAAG,MAAM,EAAE,QAAQ,IAAI,iBAAiB,CAAC,SAAS,CAAC;QAEjE,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QAE3E,IAAI,cAAc,GAAG,QAAQ,EAAE,CAAC;YAC9B,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;YAC9B,+FAA+F;YAC/F,+FAA+F;YAC/F,MAAM,QAAQ,CAAC;gBACb,KAAK,EAAE,eAAe,CAAC,WAAW;gBAClC,OAAO,EAAE,qDAAqD,cAAc,WAAW,QAAQ,EAAE;aAClG,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QAE1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,QAAQ,CAAC,EAAE,OAAO,EAAE,2CAA2C,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,SAAS,CAAC;YACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAE5C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM;YACR,CAAC;YAED,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;YAEzB,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBACxB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,QAAQ,CAAC;oBACb,KAAK,EAAE,eAAe,CAAC,WAAW;oBAClC,OAAO,EAAE,qDAAqD,QAAQ,WAAW,QAAQ,EAAE;iBAC5F,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC","sourcesContent":["// The half `policy.ts` cannot do: resolving a hostname needs `node:dns`, so nothing here may be\n// re-exported from the `./core` subpath.\n\nimport { DurationMultipliers } from '@/common/constants/duration';\nimport { getError } from '@/modules/error';\nimport { lookup } from 'node:dns/promises';\nimport { UrlSafetyDefaults } from './common/constants';\nimport { UrlSafetyErrors } from './common/errors';\nimport type { IUrlSafetyPolicy } from './common/types';\nimport { UrlPolicy } from './policy';\n\n/** The options Bun's `fetch` takes beyond the standard ones; the standard `RequestInit` does not declare `proxy`. */\ninterface IBunFetchInit extends RequestInit {\n proxy?: string | false;\n}\n\n/** Fetching an untrusted url without letting it reach inside the network. */\nexport class UrlIngest {\n /**\n * Resolves `hostname` once and returns every address it answered, refusing the host when ANY of them is non-public.\n * Checking only the first walks straight into a host that publishes one public A record beside an internal one.\n * An IP literal answers itself. `allowPrivateAddress` skips the check and returns an empty list: nothing to pin to.\n */\n static async resolvePublicAddresses(opts: {\n hostname: string;\n policy?: IUrlSafetyPolicy;\n signal?: AbortSignal;\n }): Promise<string[]> {\n const { hostname, policy, signal } = opts;\n\n if (policy?.allowPrivateAddress ?? UrlSafetyDefaults.ALLOW_PRIVATE_ADDRESS) {\n return [];\n }\n\n // `URL.hostname` keeps the brackets on an IPv6 literal; `lookup` does not accept them, and the\n // ENOTFOUND that follows reads as a transient DNS failure rather than the refusal it should be.\n const address = UrlPolicy.readHostAddress({ hostname });\n const addresses = await UrlIngest.withDeadline({\n work: lookup(address, { all: true }),\n signal,\n });\n\n if (!addresses.length) {\n throw getError({\n error: UrlSafetyErrors.URL_REFUSED,\n message: `[assertPublicHost] Host resolves to nothing | host: ${hostname}`,\n });\n }\n\n const offender = addresses.find((entry) =>\n UrlPolicy.isNonPublicAddress({ address: entry.address }),\n );\n\n if (offender) {\n throw getError({\n error: UrlSafetyErrors.URL_REFUSED,\n message: `[assertPublicHost] Host resolves to a non-public address | host: ${hostname} | address: ${offender.address}`,\n });\n }\n\n return addresses.map((entry) => entry.address);\n }\n\n /**\n * Rejects when ANY resolved address is non-public.\n *\n * A rebinding window remains when this is called on its own: the address can change between this check and a\n * later connect. {@link UrlIngest.fetchGuarded} closes it by connecting to the address it vetted.\n */\n static async assertPublicHost(opts: {\n hostname: string;\n policy?: IUrlSafetyPolicy;\n }): Promise<void> {\n await UrlIngest.resolvePublicAddresses(opts);\n }\n\n /** Both halves, in the order they must run: shape first, then the network. */\n static async assertReachableUrl(opts: { url: string; policy?: IUrlSafetyPolicy }): Promise<URL> {\n const parsed = UrlPolicy.assertSafeUrl(opts);\n await UrlIngest.assertPublicHost({ hostname: parsed.hostname, policy: opts.policy });\n\n return parsed;\n }\n\n private static readTimeoutMilliseconds(opts: { policy?: IUrlSafetyPolicy }): number {\n const { timeout } = opts.policy ?? {};\n const milliseconds = timeout ? DurationMultipliers.toMilliseconds(timeout) : null;\n\n return milliseconds ?? UrlSafetyDefaults.TIMEOUT_MILLISECONDS;\n }\n\n /** Settles with `work`, or rejects with the signal's reason when the deadline passes first: a slow resolver must not outlive it. */\n private static async withDeadline<T>(opts: {\n work: Promise<T>;\n signal?: AbortSignal;\n }): Promise<T> {\n const { work, signal } = opts;\n\n if (!signal) {\n return work;\n }\n\n signal.throwIfAborted();\n\n let onAbort: (() => void) | undefined;\n const aborted = new Promise<never>((_resolve, reject) => {\n onAbort = () => reject(signal.reason);\n signal.addEventListener('abort', onAbort, { once: true });\n });\n\n try {\n const settled = await Promise.race([work, aborted]);\n\n return settled;\n } finally {\n signal.removeEventListener('abort', onAbort as () => void);\n // The lookup that lost keeps running; its late answer or error must not surface as an unhandled rejection. Nothing\n // is left to report: the caller already got the deadline's error.\n // eslint-disable-next-line venizia/no-silent-catch -- the work lost the race; its outcome is irrelevant by now\n work.catch(() => undefined);\n }\n }\n\n /** Request headers that name the body; they go when a redirect turns the request into a GET. */\n private static readonly BODY_HEADERS = ['content-type', 'content-length', 'content-encoding'];\n\n /**\n * The only caller headers a cross-origin redirect keeps. Every other one is dropped: a tenant's webhook secret\n * (`x-api-key`, a signature) must not follow a redirect to a host that tenant did not name.\n */\n private static readonly CROSS_ORIGIN_HEADERS = ['accept', 'accept-language', 'user-agent'];\n\n /** The statuses that name a new location; a 304 or a 300 is an answer, not a redirect. */\n private static readonly REDIRECT_STATUSES = [301, 302, 303, 307, 308];\n\n /** Connect-phase errors: nothing was sent, so trying the next vetted address cannot deliver a request twice. */\n private static readonly CONNECT_ERROR_CODES = [\n 'ECONNREFUSED',\n 'ENETUNREACH',\n 'EHOSTUNREACH',\n 'EADDRNOTAVAIL',\n ];\n\n private static onlyHeaders(opts: {\n headers: Record<string, string>;\n names: string[];\n }): Record<string, string> {\n const { headers, names } = opts;\n\n return Object.fromEntries(\n Object.entries(headers).filter(([name]) => names.includes(name.toLowerCase())),\n );\n }\n\n private static withoutHeaders(opts: {\n headers: Record<string, string>;\n names: string[];\n }): Record<string, string> {\n const { headers, names } = opts;\n\n return Object.fromEntries(\n Object.entries(headers).filter(([name]) => !names.includes(name.toLowerCase())),\n );\n }\n\n /**\n * One request to `target`. With `addresses` it connects to those vetted addresses and nowhere else, in order, and\n * moves to the next one only when the connection itself fails: the url's host is set as the `Host` header, and for\n * https as the TLS server name, so the certificate is still checked against the name the caller asked for.\n */\n private static async send(opts: {\n target: URL;\n addresses: string[];\n init: IBunFetchInit;\n }): Promise<Response> {\n const { target, addresses, init } = opts;\n\n if (!addresses.length) {\n return fetch(target, init);\n }\n\n // `proxy: false` is Bun's way to ignore HTTP_PROXY / HTTPS_PROXY for this call (an empty string or undefined keeps\n // the environment's proxy, which would resolve the name itself and undo the pin). A test pins it.\n const noProxy: IBunFetchInit = { proxy: false };\n const isHttps = target.protocol === 'https:';\n const hasNameToVerify =\n isHttps &&\n !UrlPolicy.readHostAddress({ hostname: target.hostname }).includes(':') &&\n !/^[\\d.]+$/.test(target.hostname);\n const headers = {\n ...UrlIngest.withoutHeaders({\n headers: (init.headers as Record<string, string> | undefined) ?? {},\n names: ['host'],\n }),\n Host: target.host,\n };\n let lastError: unknown;\n\n for (const address of addresses) {\n const pinned = new URL(target);\n pinned.hostname = address.includes(':') ? `[${address}]` : address;\n\n try {\n const result = await fetch(pinned, {\n ...init,\n ...noProxy,\n headers,\n ...(hasNameToVerify ? { tls: { serverName: target.hostname } } : {}),\n });\n\n return result;\n } catch (error) {\n const name = (error as Error | undefined)?.name;\n\n // The deadline is not a reason to try another address: it is spent.\n if (name === 'AbortError' || name === 'TimeoutError') {\n throw error;\n }\n\n // A request that may have reached the server (a reset after the body went out) is not repeated on another\n // address unless it is safe to repeat: a webhook must not be delivered twice.\n const code = (error as { code?: string } | undefined)?.code ?? '';\n const method = (init.method ?? 'GET').toUpperCase();\n\n if (\n method !== 'GET' &&\n method !== 'HEAD' &&\n !UrlIngest.CONNECT_ERROR_CODES.includes(code)\n ) {\n throw error;\n }\n\n lastError = error;\n }\n }\n\n throw lastError;\n }\n\n /**\n * A request that an untrusted url cannot steer inside the network. GET by default; `method`, `headers` and `body`\n * (text or bytes, so a redirect can replay it) make it a signed webhook dispatch.\n *\n * The host is resolved ONCE per hop, every answer is checked, and the request connects to those addresses and to no\n * other, so a DNS answer that changes after the check (DNS rebinding) is never looked up again. The `Host` header and,\n * for https, the TLS server name stay the url's own, so the certificate is verified for that name. A `Host` in\n * `headers` is ignored. The environment's `HTTP_PROXY` / `HTTPS_PROXY` is ignored on a pinned\n * connection, because a proxy would resolve the name itself and undo the pin. `policy.proxy` sends the request through a\n * proxy on purpose; that connection is not pinned (see `IUrlSafetyPolicy.proxy`). `allowPrivateAddress` turns the guard,\n * and so the pin, off.\n *\n * Redirects are followed by hand so every hop is re-checked and re-pinned - `redirect: 'follow'` would land on an\n * internal host with no second look. 301, 302 (for a POST) and 303 turn the request into a GET without a body. A 307 or\n * 308 repeats the request; with a body it is refused when it leaves the origin, so a signed payload is never replayed to\n * a host a redirect chose. When a hop changes origin every caller header is dropped except `accept`, `accept-language` and `user-agent`.\n *\n * The body is returned unread: the caller decides whether to stream it onward or cap it with\n * {@link UrlIngest.readCappedBody}. Nothing is buffered here.\n */\n static async fetchGuarded(opts: {\n url: string;\n policy?: IUrlSafetyPolicy;\n method?: string;\n headers?: Record<string, string>;\n body?: string | Uint8Array;\n }): Promise<Response> {\n const { url, policy } = opts;\n const maxRedirects = policy?.maxRedirects ?? UrlSafetyDefaults.MAX_REDIRECTS;\n const timeoutMilliseconds = UrlIngest.readTimeoutMilliseconds({ policy });\n\n let method = (opts.method ?? 'GET').toUpperCase();\n let headers = opts.headers ?? {};\n let body = opts.body;\n\n if (body !== undefined && (method === 'GET' || method === 'HEAD')) {\n throw getError({\n message: `[fetchGuarded] A ${method} request cannot carry a body | url: ${url}`,\n });\n }\n\n let target = UrlPolicy.assertSafeUrl({ url, policy });\n // ONE deadline for the whole call, every lookup and every hop included, not one per hop.\n const signal = AbortSignal.timeout(timeoutMilliseconds);\n\n for (let hop = 0; hop <= maxRedirects; hop += 1) {\n const addresses = await UrlIngest.resolvePublicAddresses({\n hostname: target.hostname,\n policy,\n signal,\n });\n\n const response = await UrlIngest.send({\n target,\n // A proxy resolves the name itself, so there is nothing to pin: the caller chose it (`policy.proxy`).\n addresses: policy?.proxy ? [] : addresses,\n init: {\n method,\n ...(policy?.proxy ? { proxy: policy.proxy } : {}),\n redirect: 'manual',\n signal,\n ...(Object.keys(headers).length ? { headers } : {}),\n ...(body === undefined ? {} : { body }),\n },\n });\n\n if (!UrlIngest.REDIRECT_STATUSES.includes(response.status)) {\n if (!response.ok) {\n // Cancelled, or the socket stays open on every 4xx and 5xx.\n await response.body?.cancel();\n throw getError({\n message: `[fetchGuarded] Request failed | status: ${response.status} | url: ${url}`,\n });\n }\n\n return response;\n }\n\n const location = response.headers.get('location');\n await response.body?.cancel();\n\n if (!location) {\n throw getError({\n error: UrlSafetyErrors.URL_REFUSED,\n message: `[fetchGuarded] Redirect carried no location | url: ${url}`,\n });\n }\n\n const next = UrlPolicy.assertSafeUrl({ url: new URL(location, target).toString(), policy });\n const isSameOrigin = next.origin === target.origin;\n const becomesGet =\n response.status === 303 ||\n ((response.status === 301 || response.status === 302) && method === 'POST');\n\n if (becomesGet) {\n method = 'GET';\n body = undefined;\n headers = UrlIngest.withoutHeaders({ headers, names: UrlIngest.BODY_HEADERS });\n } else if (body !== undefined && !isSameOrigin) {\n throw getError({\n error: UrlSafetyErrors.URL_REFUSED,\n message: `[fetchGuarded] Redirect would replay the request body to another origin | from: ${target.origin} | to: ${next.origin}`,\n });\n }\n\n if (!isSameOrigin) {\n headers = UrlIngest.onlyHeaders({ headers, names: UrlIngest.CROSS_ORIGIN_HEADERS });\n }\n\n target = next;\n }\n\n throw getError({\n error: UrlSafetyErrors.URL_REFUSED,\n message: `[fetchGuarded] Too many redirects | max: ${maxRedirects} | url: ${url}`,\n });\n }\n\n /**\n * Enforces `maxBytes` on a stream nothing reads into this process - the shape a caller wants when\n * the body is piped onward rather than held. Without it a `maxBytes` policy is inert on that path.\n */\n static capStream(opts: {\n source: ReadableStream<Uint8Array>;\n policy?: IUrlSafetyPolicy;\n }): ReadableStream<Uint8Array> {\n const { source } = opts;\n const maxBytes = opts.policy?.maxBytes ?? UrlSafetyDefaults.MAX_BYTES;\n let received = 0;\n\n return source.pipeThrough(\n new TransformStream<Uint8Array, Uint8Array>({\n transform(chunk, controller) {\n received += chunk.byteLength;\n\n if (received > maxBytes) {\n controller.error(\n getError({\n error: UrlSafetyErrors.URL_REFUSED,\n message: `[capStream] Body exceeds the cap | received: ${received} | max: ${maxBytes}`,\n }),\n );\n\n return;\n }\n\n controller.enqueue(chunk);\n },\n }),\n );\n }\n\n /**\n * Reads a body that must not exceed `maxBytes`, cancelling the moment it does. `content-length` is\n * checked first because it is free, and counted anyway because it may be absent or lie.\n */\n static async readCappedBody(opts: {\n response: Response;\n policy?: IUrlSafetyPolicy;\n }): Promise<Buffer> {\n const { response, policy } = opts;\n const maxBytes = policy?.maxBytes ?? UrlSafetyDefaults.MAX_BYTES;\n\n const declaredLength = Number(response.headers.get('content-length') ?? 0);\n\n if (declaredLength > maxBytes) {\n await response.body?.cancel();\n // The same fact as the streaming branch below, so the same verdict: a caller must not retry an\n // oversized body when the server sends `content-length` and drop it when the server omits one.\n throw getError({\n error: UrlSafetyErrors.URL_REFUSED,\n message: `[readCappedBody] Body exceeds the cap | declared: ${declaredLength} | max: ${maxBytes}`,\n });\n }\n\n const reader = response.body?.getReader();\n\n if (!reader) {\n throw getError({ message: '[readCappedBody] Response carried no body' });\n }\n\n const chunks: Uint8Array[] = [];\n let received = 0;\n\n for (;;) {\n const { done, value } = await reader.read();\n\n if (done) {\n break;\n }\n\n received += value.length;\n\n if (received > maxBytes) {\n await reader.cancel();\n throw getError({\n error: UrlSafetyErrors.URL_REFUSED,\n message: `[readCappedBody] Body exceeds the cap | received: ${received} | max: ${maxBytes}`,\n });\n }\n\n chunks.push(value);\n }\n\n return Buffer.concat(chunks);\n }\n}\n"]}
|