@johnhenry/sitefile 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +152 -0
- package/LICENSE +21 -0
- package/README.md +513 -2
- package/lib/src/archive-writers.d.ts +31 -0
- package/lib/src/archive-writers.js +197 -0
- package/lib/src/attest.d.ts +126 -0
- package/lib/src/attest.js +220 -0
- package/lib/src/bake.d.ts +74 -0
- package/lib/src/bake.js +372 -0
- package/lib/src/base-image.d.ts +76 -0
- package/lib/src/base-image.js +138 -0
- package/lib/src/build-args.d.ts +26 -0
- package/lib/src/build-args.js +164 -0
- package/lib/src/cache-limit.d.ts +54 -0
- package/lib/src/cache-limit.js +239 -0
- package/lib/src/casefold-data.d.ts +2 -0
- package/lib/src/casefold-data.js +217 -0
- package/lib/src/cbor.d.ts +39 -0
- package/lib/src/cbor.js +174 -0
- package/lib/src/checklinks.d.ts +42 -0
- package/lib/src/checklinks.js +120 -0
- package/lib/src/cli/bake.d.ts +2 -0
- package/lib/src/cli/bake.js +179 -0
- package/lib/src/cli/build-image.d.ts +21 -0
- package/lib/src/cli/build-image.js +42 -0
- package/lib/src/cli/cache.d.ts +8 -0
- package/lib/src/cli/cache.js +119 -0
- package/lib/src/cli/common.d.ts +29 -0
- package/lib/src/cli/common.js +100 -0
- package/lib/src/cli/git-context.d.ts +18 -0
- package/lib/src/cli/git-context.js +48 -0
- package/lib/src/cli/images.d.ts +6 -0
- package/lib/src/cli/images.js +285 -0
- package/lib/src/cli/index.d.ts +10 -0
- package/lib/src/cli/index.js +95 -0
- package/lib/src/cli/network.d.ts +34 -0
- package/lib/src/cli/network.js +56 -0
- package/lib/src/cli/outdated.d.ts +5 -0
- package/lib/src/cli/outdated.js +99 -0
- package/lib/src/cli/registry.d.ts +2 -0
- package/lib/src/cli/registry.js +127 -0
- package/lib/src/cli/serve-worker.d.ts +2 -0
- package/lib/src/cli/serve-worker.js +42 -0
- package/lib/src/cli/servers.d.ts +15 -0
- package/lib/src/cli/servers.js +467 -0
- package/lib/src/cli/vendor.d.ts +2 -0
- package/lib/src/cli/vendor.js +97 -0
- package/lib/src/cli/watch.d.ts +26 -0
- package/lib/src/cli/watch.js +81 -0
- package/lib/src/cli.d.ts +12 -0
- package/lib/src/cli.js +522 -0
- package/lib/src/csv.d.ts +27 -0
- package/lib/src/csv.js +117 -0
- package/lib/src/deflate.d.ts +76 -0
- package/lib/src/deflate.js +604 -0
- package/lib/src/digest.d.ts +3 -0
- package/lib/src/digest.js +7 -0
- package/lib/src/errors.d.ts +56 -0
- package/lib/src/errors.js +29 -0
- package/lib/src/executor.d.ts +393 -0
- package/lib/src/executor.js +2201 -0
- package/lib/src/explain.d.ts +64 -0
- package/lib/src/explain.js +445 -0
- package/lib/src/export-formats.d.ts +23 -0
- package/lib/src/export-formats.js +63 -0
- package/lib/src/exporter.d.ts +103 -0
- package/lib/src/exporter.js +676 -0
- package/lib/src/fingerprint.d.ts +40 -0
- package/lib/src/fingerprint.js +168 -0
- package/lib/src/frontmatter.d.ts +22 -0
- package/lib/src/frontmatter.js +45 -0
- package/lib/src/index.d.ts +13 -0
- package/lib/src/index.js +13 -0
- package/lib/src/language/ast.d.ts +360 -0
- package/lib/src/language/ast.js +57 -0
- package/lib/src/language/parser.d.ts +3 -0
- package/lib/src/language/parser.js +612 -0
- package/lib/src/language/scanner.d.ts +110 -0
- package/lib/src/language/scanner.js +449 -0
- package/lib/src/language/validate.d.ts +13 -0
- package/lib/src/language/validate.js +367 -0
- package/lib/src/links.d.ts +79 -0
- package/lib/src/links.js +583 -0
- package/lib/src/lint.d.ts +10 -0
- package/lib/src/lint.js +223 -0
- package/lib/src/markdown.d.ts +29 -0
- package/lib/src/markdown.js +612 -0
- package/lib/src/media.d.ts +82 -0
- package/lib/src/media.js +650 -0
- package/lib/src/memo.d.ts +57 -0
- package/lib/src/memo.js +93 -0
- package/lib/src/oci.d.ts +178 -0
- package/lib/src/oci.js +380 -0
- package/lib/src/outdated.d.ts +46 -0
- package/lib/src/outdated.js +195 -0
- package/lib/src/pages.d.ts +10 -0
- package/lib/src/pages.js +54 -0
- package/lib/src/paths.d.ts +39 -0
- package/lib/src/paths.js +101 -0
- package/lib/src/planner.d.ts +297 -0
- package/lib/src/planner.js +410 -0
- package/lib/src/plugins/cache.d.ts +96 -0
- package/lib/src/plugins/cache.js +325 -0
- package/lib/src/plugins/exec.d.ts +51 -0
- package/lib/src/plugins/exec.js +144 -0
- package/lib/src/plugins/host.d.ts +75 -0
- package/lib/src/plugins/host.js +277 -0
- package/lib/src/plugins/wasm.d.ts +23 -0
- package/lib/src/plugins/wasm.js +249 -0
- package/lib/src/profile.d.ts +42 -0
- package/lib/src/profile.js +68 -0
- package/lib/src/ref.d.ts +40 -0
- package/lib/src/ref.js +79 -0
- package/lib/src/registry-auth.d.ts +32 -0
- package/lib/src/registry-auth.js +151 -0
- package/lib/src/registry-cache.d.ts +71 -0
- package/lib/src/registry-cache.js +333 -0
- package/lib/src/registry-ops.d.ts +51 -0
- package/lib/src/registry-ops.js +248 -0
- package/lib/src/registry.d.ts +115 -0
- package/lib/src/registry.js +524 -0
- package/lib/src/render.d.ts +134 -0
- package/lib/src/render.js +576 -0
- package/lib/src/resources/archive.d.ts +11 -0
- package/lib/src/resources/archive.js +306 -0
- package/lib/src/resources/bundle-data.d.ts +16 -0
- package/lib/src/resources/bundle-data.js +50 -0
- package/lib/src/resources/convert.d.ts +12 -0
- package/lib/src/resources/convert.js +63 -0
- package/lib/src/resources/documents.d.ts +35 -0
- package/lib/src/resources/documents.js +122 -0
- package/lib/src/resources/git.d.ts +103 -0
- package/lib/src/resources/git.js +720 -0
- package/lib/src/resources/https.d.ts +74 -0
- package/lib/src/resources/https.js +413 -0
- package/lib/src/resources/ignore.d.ts +33 -0
- package/lib/src/resources/ignore.js +119 -0
- package/lib/src/resources/local.d.ts +64 -0
- package/lib/src/resources/local.js +248 -0
- package/lib/src/resources/lockfile.d.ts +52 -0
- package/lib/src/resources/lockfile.js +172 -0
- package/lib/src/resources/netpolicy.d.ts +6 -0
- package/lib/src/resources/netpolicy.js +82 -0
- package/lib/src/resources/network.d.ts +70 -0
- package/lib/src/resources/network.js +304 -0
- package/lib/src/resources/types.d.ts +69 -0
- package/lib/src/resources/types.js +13 -0
- package/lib/src/resources/vendor.d.ts +94 -0
- package/lib/src/resources/vendor.js +296 -0
- package/lib/src/resources/wbn-reader.d.ts +44 -0
- package/lib/src/resources/wbn-reader.js +302 -0
- package/lib/src/schema.d.ts +25 -0
- package/lib/src/schema.js +169 -0
- package/lib/src/secrets.d.ts +21 -0
- package/lib/src/secrets.js +103 -0
- package/lib/src/serve-rules.d.ts +29 -0
- package/lib/src/serve-rules.js +123 -0
- package/lib/src/server.d.ts +45 -0
- package/lib/src/server.js +246 -0
- package/lib/src/servers.d.ts +85 -0
- package/lib/src/servers.js +225 -0
- package/lib/src/site.d.ts +60 -0
- package/lib/src/site.js +169 -0
- package/lib/src/store.d.ts +151 -0
- package/lib/src/store.js +596 -0
- package/lib/src/tree.d.ts +50 -0
- package/lib/src/tree.js +134 -0
- package/lib/src/values.d.ts +7 -0
- package/lib/src/values.js +37 -0
- package/lib/src/vendor.d.ts +19 -0
- package/lib/src/vendor.js +145 -0
- package/lib/src/watch.d.ts +44 -0
- package/lib/src/watch.js +137 -0
- package/lib/src/wbn.d.ts +26 -0
- package/lib/src/wbn.js +241 -0
- package/lib/src/yaml.d.ts +34 -0
- package/lib/src/yaml.js +562 -0
- package/package.json +59 -4
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { request as httpRequest, type ClientRequest, type RequestOptions } from 'node:http';
|
|
2
|
+
import { type LookupFunction } from 'node:net';
|
|
3
|
+
import { SitefileError, type SourceSpan } from '../errors.js';
|
|
4
|
+
import type { FetchPolicy } from './types.js';
|
|
5
|
+
import { type NetworkSettings } from './network.js';
|
|
6
|
+
export interface FetchResult {
|
|
7
|
+
readonly bytes: Uint8Array;
|
|
8
|
+
/** URL of the final hop after redirects. */
|
|
9
|
+
readonly finalUrl: string;
|
|
10
|
+
/** Media type essence from `content-type` (lowercase, parameters removed). */
|
|
11
|
+
readonly mediaType?: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* What to send. `headers` go to every hop; `originHeaders` (credentials: IMPORT … SECRET) only to hops
|
|
15
|
+
* on the first URL's origin (scheme, host and port), so a redirect elsewhere never receives them.
|
|
16
|
+
* A request with a body (POST) never follows redirects (E_REDIRECT).
|
|
17
|
+
*/
|
|
18
|
+
export interface FetchRequest {
|
|
19
|
+
readonly method?: 'GET' | 'POST';
|
|
20
|
+
readonly body?: Uint8Array;
|
|
21
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
22
|
+
readonly originHeaders?: Readonly<Record<string, string>>;
|
|
23
|
+
}
|
|
24
|
+
/** A URL rendered without any userinfo, safe for error messages. */
|
|
25
|
+
export declare function redactUrl(url: URL | string): string;
|
|
26
|
+
/**
|
|
27
|
+
* Validate one hop before connecting: scheme, credentials and IP-literal
|
|
28
|
+
* hosts. Hostnames are checked at connect time by the guarded lookup.
|
|
29
|
+
*/
|
|
30
|
+
export declare function validateHop(url: URL, policy: FetchPolicy, span?: SourceSpan): void;
|
|
31
|
+
/**
|
|
32
|
+
* A `lookup` for http(s).request that resolves every address and fails with
|
|
33
|
+
* E_SSRF_BLOCKED if any is non-public. Runs at connect time, so DNS rebinding
|
|
34
|
+
* between validation and connection cannot bypass it.
|
|
35
|
+
*/
|
|
36
|
+
export declare function createGuardedLookup(policy: FetchPolicy, span?: SourceSpan): LookupFunction;
|
|
37
|
+
/**
|
|
38
|
+
* Fetch a remote resource under the given network policy. Redirects are
|
|
39
|
+
* followed manually and every hop is re-validated; one deadline covers the
|
|
40
|
+
* whole fetch.
|
|
41
|
+
*/
|
|
42
|
+
export declare function fetchRemote(url: string, policy: FetchPolicy, span?: SourceSpan, request?: FetchRequest): Promise<FetchResult>;
|
|
43
|
+
/** How to send one request: directly, or through the configured proxy. */
|
|
44
|
+
export interface Route {
|
|
45
|
+
readonly fn: typeof httpRequest;
|
|
46
|
+
/** The URL to pass to `fn`; undefined when `options` carry the proxy's host, port and absolute path. */
|
|
47
|
+
readonly url?: URL | undefined;
|
|
48
|
+
readonly options: RequestOptions;
|
|
49
|
+
/** True when the connection is to a proxy (the connect timeout then covers the tunnel). */
|
|
50
|
+
readonly proxied: boolean;
|
|
51
|
+
}
|
|
52
|
+
export interface RouteContext {
|
|
53
|
+
readonly signal?: AbortSignal | undefined;
|
|
54
|
+
readonly span?: SourceSpan | undefined;
|
|
55
|
+
/** The hop's URL as shown in messages. */
|
|
56
|
+
readonly shown: string;
|
|
57
|
+
/** Records a timeout error before the request is destroyed with it. */
|
|
58
|
+
readonly fail?: ((e: SitefileError) => SitefileError) | undefined;
|
|
59
|
+
}
|
|
60
|
+
/** E_FETCH_TIMEOUT for a connect or read (idle) timeout. */
|
|
61
|
+
export declare function hopTimeout(kind: 'connect' | 'read', shown: string, network: NetworkSettings | undefined, span?: SourceSpan): SitefileError;
|
|
62
|
+
/**
|
|
63
|
+
* Apply `network`'s connect timeout (until the TCP connection is up) and read timeout (socket idle time
|
|
64
|
+
* once connected) to a request; `onTimeout` destroys it. For a tunnelled request the socket is already
|
|
65
|
+
* connected (the tunnel had the connect timeout), so only the read timeout applies.
|
|
66
|
+
*/
|
|
67
|
+
export declare function armTimeouts(req: ClientRequest, network: NetworkSettings | undefined, _proxied: boolean, onTimeout: (kind: 'connect' | 'read') => void): void;
|
|
68
|
+
/**
|
|
69
|
+
* Route one request: directly (with the guarded lookup in `options`), or through `policy.network.proxy`
|
|
70
|
+
* unless the host is in its no-proxy list. Through a proxy, the target host is still resolved here and
|
|
71
|
+
* refused if it is non-public (unless `allowPrivateNetwork`), so a proxy never widens the policy; the
|
|
72
|
+
* proxy itself is named by the user and may be on a private address.
|
|
73
|
+
*/
|
|
74
|
+
export declare function routeRequest(url: URL, options: RequestOptions, policy: FetchPolicy, ctx: RouteContext): Promise<Route>;
|
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
import { lookup as dnsLookup } from 'node:dns';
|
|
2
|
+
import { request as httpRequest } from 'node:http';
|
|
3
|
+
import { request as httpsRequest } from 'node:https';
|
|
4
|
+
import * as tls from 'node:tls';
|
|
5
|
+
import { isIP } from 'node:net';
|
|
6
|
+
import { SitefileError, isSitefileError } from '../errors.js';
|
|
7
|
+
import { isPublicAddress } from './netpolicy.js';
|
|
8
|
+
import { checkHostRules, proxyFor } from './network.js';
|
|
9
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
10
|
+
const USER_AGENT = 'sitefile/0.1';
|
|
11
|
+
/** A URL rendered without any userinfo, safe for error messages. */
|
|
12
|
+
export function redactUrl(url) {
|
|
13
|
+
try {
|
|
14
|
+
const u = new URL(String(url));
|
|
15
|
+
u.username = '';
|
|
16
|
+
u.password = '';
|
|
17
|
+
return u.href;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return '<invalid url>';
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function stripBrackets(host) {
|
|
24
|
+
return host.startsWith('[') && host.endsWith(']') ? host.slice(1, -1) : host;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Validate one hop before connecting: scheme, credentials and IP-literal
|
|
28
|
+
* hosts. Hostnames are checked at connect time by the guarded lookup.
|
|
29
|
+
*/
|
|
30
|
+
export function validateHop(url, policy, span) {
|
|
31
|
+
if (url.protocol !== 'https:' && !(url.protocol === 'http:' && policy.allowHttp)) {
|
|
32
|
+
const hint = url.protocol === 'http:' ? ' (plain http requires --allow-http)' : '';
|
|
33
|
+
throw new SitefileError('E_PROTOCOL', `unsupported protocol '${url.protocol}' for ${redactUrl(url)}${hint}`, span);
|
|
34
|
+
}
|
|
35
|
+
if (url.username !== '' || url.password !== '') {
|
|
36
|
+
throw new SitefileError('E_PROTOCOL', `URL ${redactUrl(url)} contains credentials; credentials in URLs are not allowed`, span);
|
|
37
|
+
}
|
|
38
|
+
const host = stripBrackets(url.hostname);
|
|
39
|
+
if (host === '')
|
|
40
|
+
throw new SitefileError('E_PROTOCOL', `URL ${redactUrl(url)} has no host`, span);
|
|
41
|
+
checkHostRules(url, policy.network, redactUrl(url), span);
|
|
42
|
+
if (isIP(host) !== 0 && !policy.allowPrivateNetwork && !isPublicAddress(host)) {
|
|
43
|
+
throw new SitefileError('E_SSRF_BLOCKED', `refusing to connect to non-public address ${host} (${redactUrl(url)}); use --allow-private-network to permit it`, span);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* A `lookup` for http(s).request that resolves every address and fails with
|
|
48
|
+
* E_SSRF_BLOCKED if any is non-public. Runs at connect time, so DNS rebinding
|
|
49
|
+
* between validation and connection cannot bypass it.
|
|
50
|
+
*/
|
|
51
|
+
export function createGuardedLookup(policy, span) {
|
|
52
|
+
return (hostname, options, callback) => {
|
|
53
|
+
dnsLookup(hostname, { ...options, all: true }, (err, addresses) => {
|
|
54
|
+
if (err) {
|
|
55
|
+
callback(err);
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
const list = addresses;
|
|
59
|
+
if (list.length === 0) {
|
|
60
|
+
callback(new SitefileError('E_FETCH', `no addresses found for ${hostname}`, span));
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
if (!policy.allowPrivateNetwork) {
|
|
64
|
+
const bad = list.find((a) => !isPublicAddress(a.address));
|
|
65
|
+
if (bad) {
|
|
66
|
+
callback(new SitefileError('E_SSRF_BLOCKED', `refusing to connect to ${hostname}: it resolves to non-public address ${bad.address}; use --allow-private-network to permit it`, span));
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (options.all) {
|
|
71
|
+
callback(null, list);
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
const first = list[0];
|
|
75
|
+
callback(null, first.address, first.family);
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/** Find a SitefileError carried by a socket/request error, if any. */
|
|
81
|
+
function findSitefileError(e, depth = 0) {
|
|
82
|
+
if (depth > 4 || e === null || typeof e !== 'object')
|
|
83
|
+
return undefined;
|
|
84
|
+
if (isSitefileError(e))
|
|
85
|
+
return e;
|
|
86
|
+
const anyE = e;
|
|
87
|
+
if (Array.isArray(anyE.errors)) {
|
|
88
|
+
for (const inner of anyE.errors) {
|
|
89
|
+
const found = findSitefileError(inner, depth + 1);
|
|
90
|
+
if (found)
|
|
91
|
+
return found;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return findSitefileError(anyE.cause, depth + 1);
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Fetch a remote resource under the given network policy. Redirects are
|
|
98
|
+
* followed manually and every hop is re-validated; one deadline covers the
|
|
99
|
+
* whole fetch.
|
|
100
|
+
*/
|
|
101
|
+
export async function fetchRemote(url, policy, span, request = {}) {
|
|
102
|
+
let current;
|
|
103
|
+
try {
|
|
104
|
+
current = new URL(url);
|
|
105
|
+
}
|
|
106
|
+
catch (e) {
|
|
107
|
+
throw new SitefileError('E_PROTOCOL', `invalid URL '${redactUrl(url)}'`, span, { cause: e });
|
|
108
|
+
}
|
|
109
|
+
const origin = current.origin;
|
|
110
|
+
const method = request.method ?? 'GET';
|
|
111
|
+
const controller = new AbortController();
|
|
112
|
+
const state = {};
|
|
113
|
+
const timer = setTimeout(() => {
|
|
114
|
+
controller.abort();
|
|
115
|
+
state.req?.destroy();
|
|
116
|
+
state.res?.destroy();
|
|
117
|
+
}, policy.timeoutMs);
|
|
118
|
+
timer.unref?.();
|
|
119
|
+
const timeoutError = () => new SitefileError('E_FETCH_TIMEOUT', `fetching ${redactUrl(url)} timed out after ${policy.timeoutMs} ms`, span);
|
|
120
|
+
try {
|
|
121
|
+
let redirects = 0;
|
|
122
|
+
for (;;) {
|
|
123
|
+
validateHop(current, policy, span);
|
|
124
|
+
if (controller.signal.aborted)
|
|
125
|
+
throw timeoutError();
|
|
126
|
+
// Credentials only ever go to the origin they were given for (not after a redirect elsewhere).
|
|
127
|
+
const headers = current.origin === origin ? { ...request.headers, ...request.originHeaders } : { ...request.headers };
|
|
128
|
+
const res = await openHop(current, policy, controller.signal, state, span, timeoutError, method, headers, request.body);
|
|
129
|
+
const status = res.statusCode ?? 0;
|
|
130
|
+
if (REDIRECT_STATUSES.has(status) && method !== 'GET') {
|
|
131
|
+
res.destroy();
|
|
132
|
+
throw new SitefileError('E_REDIRECT', `${method} ${redactUrl(current)} was redirected (HTTP ${status}); redirects are only followed for GET`, span);
|
|
133
|
+
}
|
|
134
|
+
if (REDIRECT_STATUSES.has(status)) {
|
|
135
|
+
const location = res.headers.location;
|
|
136
|
+
res.destroy();
|
|
137
|
+
state.req?.destroy();
|
|
138
|
+
if (!location) {
|
|
139
|
+
throw new SitefileError('E_REDIRECT', `redirect ${status} from ${redactUrl(current)} has no Location header`, span);
|
|
140
|
+
}
|
|
141
|
+
redirects++;
|
|
142
|
+
if (redirects > policy.maxRedirects) {
|
|
143
|
+
throw new SitefileError('E_REDIRECT', `too many redirects fetching ${redactUrl(url)} (limit ${policy.maxRedirects})`, span);
|
|
144
|
+
}
|
|
145
|
+
let next;
|
|
146
|
+
try {
|
|
147
|
+
next = new URL(location, current);
|
|
148
|
+
}
|
|
149
|
+
catch (e) {
|
|
150
|
+
throw new SitefileError('E_REDIRECT', `invalid redirect Location from ${redactUrl(current)}`, span, { cause: e });
|
|
151
|
+
}
|
|
152
|
+
current = next;
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
if (status !== 200) {
|
|
156
|
+
res.destroy();
|
|
157
|
+
throw new SitefileError('E_FETCH_STATUS', `${method} ${redactUrl(current)} returned HTTP ${status}`, span);
|
|
158
|
+
}
|
|
159
|
+
const encoding = String(res.headers['content-encoding'] ?? '').trim().toLowerCase();
|
|
160
|
+
if (encoding !== '' && encoding !== 'identity') {
|
|
161
|
+
res.destroy();
|
|
162
|
+
throw new SitefileError('E_FETCH', `${redactUrl(current)} responded with content-encoding '${encoding}'; only identity is accepted`, span);
|
|
163
|
+
}
|
|
164
|
+
const lengthHeader = res.headers['content-length'];
|
|
165
|
+
if (lengthHeader !== undefined) {
|
|
166
|
+
const declared = Number(lengthHeader);
|
|
167
|
+
if (Number.isFinite(declared) && declared > policy.maxBytes) {
|
|
168
|
+
res.destroy();
|
|
169
|
+
throw new SitefileError('E_TOO_LARGE', `${redactUrl(current)} is ${declared} bytes, exceeding the limit of ${policy.maxBytes} bytes`, span);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
const chunks = [];
|
|
173
|
+
let total = 0;
|
|
174
|
+
try {
|
|
175
|
+
for await (const chunk of res) {
|
|
176
|
+
total += chunk.length;
|
|
177
|
+
if (total > policy.maxBytes) {
|
|
178
|
+
res.destroy();
|
|
179
|
+
throw new SitefileError('E_TOO_LARGE', `${redactUrl(current)} exceeded the limit of ${policy.maxBytes} bytes`, span);
|
|
180
|
+
}
|
|
181
|
+
chunks.push(chunk);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
catch (e) {
|
|
185
|
+
if (isSitefileError(e))
|
|
186
|
+
throw e;
|
|
187
|
+
if (controller.signal.aborted)
|
|
188
|
+
throw timeoutError();
|
|
189
|
+
if (state.failure)
|
|
190
|
+
throw state.failure;
|
|
191
|
+
throw new SitefileError('E_FETCH', `error reading ${redactUrl(current)}: ${e.message}`, span, {
|
|
192
|
+
cause: e,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
if (controller.signal.aborted)
|
|
196
|
+
throw timeoutError();
|
|
197
|
+
const buf = Buffer.concat(chunks, total);
|
|
198
|
+
const bytes = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
199
|
+
const ct = res.headers['content-type'];
|
|
200
|
+
const mediaType = ct ? ct.split(';')[0].trim().toLowerCase() || undefined : undefined;
|
|
201
|
+
return mediaType === undefined ? { bytes, finalUrl: current.href } : { bytes, finalUrl: current.href, mediaType };
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
finally {
|
|
205
|
+
clearTimeout(timer);
|
|
206
|
+
state.res?.destroy();
|
|
207
|
+
state.req?.destroy();
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async function openHop(url, policy, signal, state, span, timeoutError, method, extra, body) {
|
|
211
|
+
const options = {
|
|
212
|
+
method,
|
|
213
|
+
headers: { 'accept-encoding': 'identity', 'user-agent': USER_AGENT, accept: '*/*', ...extra, ...(body ? { 'content-length': String(body.byteLength) } : {}) },
|
|
214
|
+
lookup: createGuardedLookup(policy, span),
|
|
215
|
+
agent: false,
|
|
216
|
+
signal,
|
|
217
|
+
};
|
|
218
|
+
if (url.protocol === 'https:' && policy.ca !== undefined) {
|
|
219
|
+
// Additive: setting `ca` replaces Node's whole default store, so start from
|
|
220
|
+
// that store (bundled roots plus NODE_EXTRA_CA_CERTS / --use-system-ca),
|
|
221
|
+
// not just the bundled roots, and add the extra CA(s).
|
|
222
|
+
options.ca = [...defaultCaCertificates(), policy.ca];
|
|
223
|
+
}
|
|
224
|
+
const shown = redactUrl(url);
|
|
225
|
+
const fail = (e) => {
|
|
226
|
+
state.failure = e;
|
|
227
|
+
return e;
|
|
228
|
+
};
|
|
229
|
+
let route;
|
|
230
|
+
try {
|
|
231
|
+
route = await routeRequest(url, options, policy, { signal, span, shown, fail });
|
|
232
|
+
}
|
|
233
|
+
catch (e) {
|
|
234
|
+
if (signal.aborted)
|
|
235
|
+
throw timeoutError();
|
|
236
|
+
throw e;
|
|
237
|
+
}
|
|
238
|
+
return new Promise((resolve, reject) => {
|
|
239
|
+
let req;
|
|
240
|
+
try {
|
|
241
|
+
req = route.url ? route.fn(route.url, route.options) : route.fn(route.options);
|
|
242
|
+
}
|
|
243
|
+
catch (e) {
|
|
244
|
+
reject(new SitefileError('E_FETCH', `cannot request ${shown}: ${e.message}`, span, { cause: e }));
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
state.req = req;
|
|
248
|
+
state.res = undefined;
|
|
249
|
+
state.failure = undefined;
|
|
250
|
+
armTimeouts(req, policy.network, route.proxied, (kind) => req.destroy(fail(hopTimeout(kind, shown, policy.network, span))));
|
|
251
|
+
req.on('response', (res) => {
|
|
252
|
+
state.res = res;
|
|
253
|
+
resolve(res);
|
|
254
|
+
});
|
|
255
|
+
req.on('error', (e) => {
|
|
256
|
+
if (signal.aborted) {
|
|
257
|
+
reject(timeoutError());
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
const inner = findSitefileError(e);
|
|
261
|
+
if (inner) {
|
|
262
|
+
reject(inner);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
reject(new SitefileError('E_FETCH', `fetching ${shown} failed: ${e.message}`, span, { cause: e }));
|
|
266
|
+
});
|
|
267
|
+
req.end(body);
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
/** E_FETCH_TIMEOUT for a connect or read (idle) timeout. */
|
|
271
|
+
export function hopTimeout(kind, shown, network, span) {
|
|
272
|
+
return kind === 'connect'
|
|
273
|
+
? new SitefileError('E_FETCH_TIMEOUT', `fetching ${shown}: connecting timed out after ${network?.connectTimeoutMs} ms (connect timeout)`, span)
|
|
274
|
+
: new SitefileError('E_FETCH_TIMEOUT', `fetching ${shown}: no data for ${network?.readTimeoutMs} ms (read timeout)`, span);
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Apply `network`'s connect timeout (until the TCP connection is up) and read timeout (socket idle time
|
|
278
|
+
* once connected) to a request; `onTimeout` destroys it. For a tunnelled request the socket is already
|
|
279
|
+
* connected (the tunnel had the connect timeout), so only the read timeout applies.
|
|
280
|
+
*/
|
|
281
|
+
export function armTimeouts(req, network, _proxied, onTimeout) {
|
|
282
|
+
const connectMs = network?.connectTimeoutMs;
|
|
283
|
+
if (connectMs) {
|
|
284
|
+
const t = setTimeout(() => onTimeout('connect'), connectMs);
|
|
285
|
+
t.unref?.();
|
|
286
|
+
const clear = () => clearTimeout(t);
|
|
287
|
+
req.once('socket', (socket) => {
|
|
288
|
+
if (socket.connecting)
|
|
289
|
+
socket.once('connect', clear);
|
|
290
|
+
else
|
|
291
|
+
clear();
|
|
292
|
+
});
|
|
293
|
+
req.once('close', clear);
|
|
294
|
+
req.once('error', clear);
|
|
295
|
+
}
|
|
296
|
+
const readMs = network?.readTimeoutMs;
|
|
297
|
+
if (readMs)
|
|
298
|
+
req.setTimeout(readMs, () => onTimeout('read'));
|
|
299
|
+
}
|
|
300
|
+
/** Resolve `hostname` and apply the private-address rule, as a direct connection would at connect time. */
|
|
301
|
+
function checkResolved(hostname, policy, span) {
|
|
302
|
+
return new Promise((resolve, reject) => {
|
|
303
|
+
createGuardedLookup(policy, span)(hostname, { all: true }, (err) => {
|
|
304
|
+
if (!err) {
|
|
305
|
+
resolve();
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
if (isSitefileError(err)) {
|
|
309
|
+
reject(err);
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
reject(new SitefileError('E_FETCH', `cannot resolve ${hostname} to check it against the private-network rule before using the proxy: ${err.code ?? err.message}`, span));
|
|
313
|
+
});
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Route one request: directly (with the guarded lookup in `options`), or through `policy.network.proxy`
|
|
318
|
+
* unless the host is in its no-proxy list. Through a proxy, the target host is still resolved here and
|
|
319
|
+
* refused if it is non-public (unless `allowPrivateNetwork`), so a proxy never widens the policy; the
|
|
320
|
+
* proxy itself is named by the user and may be on a private address.
|
|
321
|
+
*/
|
|
322
|
+
export async function routeRequest(url, options, policy, ctx) {
|
|
323
|
+
const https = url.protocol === 'https:';
|
|
324
|
+
const proxy = proxyFor(url, policy.network);
|
|
325
|
+
if (!proxy)
|
|
326
|
+
return { fn: https ? httpsRequest : httpRequest, url, options, proxied: false };
|
|
327
|
+
const host = url.hostname.replace(/^\[|\]$/g, '');
|
|
328
|
+
if (!policy.allowPrivateNetwork && isIP(host) === 0)
|
|
329
|
+
await checkResolved(host, policy, ctx.span);
|
|
330
|
+
const port = url.port !== '' ? Number(url.port) : https ? 443 : 80;
|
|
331
|
+
const { lookup: _lookup, agent: _agent, ...rest } = options;
|
|
332
|
+
if (!https) {
|
|
333
|
+
const target = new URL(url.href);
|
|
334
|
+
target.hash = '';
|
|
335
|
+
return {
|
|
336
|
+
fn: httpRequest,
|
|
337
|
+
options: {
|
|
338
|
+
...rest,
|
|
339
|
+
host: proxy.host,
|
|
340
|
+
port: proxy.port,
|
|
341
|
+
path: target.href,
|
|
342
|
+
agent: false,
|
|
343
|
+
headers: { ...options.headers, host: url.host, ...(proxy.authorization ? { 'proxy-authorization': proxy.authorization } : {}) },
|
|
344
|
+
},
|
|
345
|
+
proxied: true,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
const socket = await openTunnel(proxy, host, port, policy.network, ctx);
|
|
349
|
+
const ca = options.ca;
|
|
350
|
+
return {
|
|
351
|
+
fn: httpsRequest,
|
|
352
|
+
url,
|
|
353
|
+
options: {
|
|
354
|
+
...rest,
|
|
355
|
+
createConnection: () => tls.connect({ socket, host, ...(isIP(host) === 0 ? { servername: host } : {}), ...(ca !== undefined ? { ca } : {}), ALPNProtocols: ['http/1.1'] }),
|
|
356
|
+
},
|
|
357
|
+
proxied: true,
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
/** A CONNECT tunnel to `host:port` through `proxy`. */
|
|
361
|
+
function openTunnel(proxy, host, port, network, ctx) {
|
|
362
|
+
const authority = `${isIP(host) === 6 ? `[${host}]` : host}:${port}`;
|
|
363
|
+
return new Promise((resolve, reject) => {
|
|
364
|
+
let req;
|
|
365
|
+
try {
|
|
366
|
+
req = httpRequest({
|
|
367
|
+
host: proxy.host,
|
|
368
|
+
port: proxy.port,
|
|
369
|
+
method: 'CONNECT',
|
|
370
|
+
path: authority,
|
|
371
|
+
agent: false,
|
|
372
|
+
headers: { host: authority, 'user-agent': USER_AGENT, ...(proxy.authorization ? { 'proxy-authorization': proxy.authorization } : {}) },
|
|
373
|
+
...(ctx.signal ? { signal: ctx.signal } : {}),
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
catch (e) {
|
|
377
|
+
reject(new SitefileError('E_FETCH', `fetching ${ctx.shown} failed: proxy ${proxy.shown}: ${e.message}`, ctx.span, { cause: e }));
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
const record = ctx.fail ?? ((e) => e);
|
|
381
|
+
armTimeouts(req, network, true, (kind) => req.destroy(record(hopTimeout(kind, ctx.shown, network, ctx.span))));
|
|
382
|
+
const refused = (status) => status === 407
|
|
383
|
+
? new SitefileError('E_FETCH', `proxy ${proxy.shown} requires authentication (HTTP 407)`, ctx.span)
|
|
384
|
+
: new SitefileError('E_FETCH', `proxy ${proxy.shown} refused to connect to ${authority}`, ctx.span);
|
|
385
|
+
req.on('connect', (res, socket, head) => {
|
|
386
|
+
if (res.statusCode !== 200) {
|
|
387
|
+
socket.destroy();
|
|
388
|
+
reject(refused(res.statusCode));
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
socket.setTimeout(0);
|
|
392
|
+
socket.removeAllListeners('timeout');
|
|
393
|
+
if (head.length)
|
|
394
|
+
socket.unshift(head);
|
|
395
|
+
resolve(socket);
|
|
396
|
+
});
|
|
397
|
+
req.on('response', (res) => {
|
|
398
|
+
res.resume();
|
|
399
|
+
reject(refused(res.statusCode));
|
|
400
|
+
});
|
|
401
|
+
req.on('error', (e) => {
|
|
402
|
+
const inner = findSitefileError(e);
|
|
403
|
+
reject(inner ?? new SitefileError('E_FETCH', `fetching ${ctx.shown} failed: proxy ${proxy.shown}: ${e.message}`, ctx.span, { cause: e }));
|
|
404
|
+
});
|
|
405
|
+
req.end();
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
/** Node's default CA store; `tls.getCACertificates` exists from Node 22.15/23.5. */
|
|
409
|
+
function defaultCaCertificates() {
|
|
410
|
+
const get = tls.getCACertificates;
|
|
411
|
+
return get ? get('default') : tls.rootCertificates;
|
|
412
|
+
}
|
|
413
|
+
//# sourceMappingURL=https.js.map
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `.sitefileignore`: a gitignore-style subset, implemented without dependencies.
|
|
3
|
+
*
|
|
4
|
+
* - Blank lines and lines starting with `#` are skipped (`\#` escapes a literal `#`).
|
|
5
|
+
* - `!` negates a rule (`\!` escapes a literal `!`). The last matching rule wins.
|
|
6
|
+
* - A trailing `/` makes the rule match directories only.
|
|
7
|
+
* - A leading `/` or a `/` in the middle anchors the rule to the context root;
|
|
8
|
+
* otherwise it matches at any depth.
|
|
9
|
+
* - `*` and `?` match within one path segment; `**` as a whole segment matches
|
|
10
|
+
* any number of segments.
|
|
11
|
+
* - Unescaped trailing spaces are ignored. Character classes are not supported.
|
|
12
|
+
*
|
|
13
|
+
* As in gitignore, a file cannot be re-included once a parent directory is
|
|
14
|
+
* excluded; callers get this by not descending into excluded directories.
|
|
15
|
+
*/
|
|
16
|
+
export declare class IgnoreRules {
|
|
17
|
+
private readonly rules;
|
|
18
|
+
private constructor();
|
|
19
|
+
static readonly empty: IgnoreRules;
|
|
20
|
+
static parse(text: string): IgnoreRules;
|
|
21
|
+
/** Rules from `COPY --exclude` patterns, one `.sitefileignore` line each (see `isIgnorePattern`). */
|
|
22
|
+
static fromPatterns(patterns: readonly string[]): IgnoreRules;
|
|
23
|
+
/**
|
|
24
|
+
* Whether `rel` (a POSIX path relative to the rules' root) is excluded, counting excluded parent
|
|
25
|
+
* directories: what a directory walk that does not descend into excluded directories would skip.
|
|
26
|
+
*/
|
|
27
|
+
excludes(rel: string): boolean;
|
|
28
|
+
get size(): number;
|
|
29
|
+
/** Whether the context-relative POSIX path (no leading `./`) is excluded by the last matching rule. */
|
|
30
|
+
ignores(rel: string, isDirectory: boolean): boolean;
|
|
31
|
+
}
|
|
32
|
+
/** Whether `text` is a `.sitefileignore` rule (not blank, a `#` comment, or only `!` and `/`). */
|
|
33
|
+
export declare function isIgnorePattern(text: string): boolean;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `.sitefileignore`: a gitignore-style subset, implemented without dependencies.
|
|
3
|
+
*
|
|
4
|
+
* - Blank lines and lines starting with `#` are skipped (`\#` escapes a literal `#`).
|
|
5
|
+
* - `!` negates a rule (`\!` escapes a literal `!`). The last matching rule wins.
|
|
6
|
+
* - A trailing `/` makes the rule match directories only.
|
|
7
|
+
* - A leading `/` or a `/` in the middle anchors the rule to the context root;
|
|
8
|
+
* otherwise it matches at any depth.
|
|
9
|
+
* - `*` and `?` match within one path segment; `**` as a whole segment matches
|
|
10
|
+
* any number of segments.
|
|
11
|
+
* - Unescaped trailing spaces are ignored. Character classes are not supported.
|
|
12
|
+
*
|
|
13
|
+
* As in gitignore, a file cannot be re-included once a parent directory is
|
|
14
|
+
* excluded; callers get this by not descending into excluded directories.
|
|
15
|
+
*/
|
|
16
|
+
export class IgnoreRules {
|
|
17
|
+
rules;
|
|
18
|
+
constructor(rules) {
|
|
19
|
+
this.rules = rules;
|
|
20
|
+
}
|
|
21
|
+
static empty = new IgnoreRules([]);
|
|
22
|
+
static parse(text) {
|
|
23
|
+
const rules = [];
|
|
24
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
25
|
+
const rule = parseLine(raw);
|
|
26
|
+
if (rule)
|
|
27
|
+
rules.push(rule);
|
|
28
|
+
}
|
|
29
|
+
return new IgnoreRules(rules);
|
|
30
|
+
}
|
|
31
|
+
/** Rules from `COPY --exclude` patterns, one `.sitefileignore` line each (see `isIgnorePattern`). */
|
|
32
|
+
static fromPatterns(patterns) {
|
|
33
|
+
return new IgnoreRules(patterns.map(parseLine).filter((r) => r !== undefined));
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Whether `rel` (a POSIX path relative to the rules' root) is excluded, counting excluded parent
|
|
37
|
+
* directories: what a directory walk that does not descend into excluded directories would skip.
|
|
38
|
+
*/
|
|
39
|
+
excludes(rel) {
|
|
40
|
+
const segs = rel.split('/');
|
|
41
|
+
for (let i = 1; i < segs.length; i++)
|
|
42
|
+
if (this.ignores(segs.slice(0, i).join('/'), true))
|
|
43
|
+
return true;
|
|
44
|
+
return this.ignores(rel, false);
|
|
45
|
+
}
|
|
46
|
+
get size() {
|
|
47
|
+
return this.rules.length;
|
|
48
|
+
}
|
|
49
|
+
/** Whether the context-relative POSIX path (no leading `./`) is excluded by the last matching rule. */
|
|
50
|
+
ignores(rel, isDirectory) {
|
|
51
|
+
let ignored = false;
|
|
52
|
+
for (const r of this.rules) {
|
|
53
|
+
if (r.dirOnly && !isDirectory)
|
|
54
|
+
continue;
|
|
55
|
+
if (r.re.test(rel))
|
|
56
|
+
ignored = !r.negate;
|
|
57
|
+
}
|
|
58
|
+
return ignored;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/** Whether `text` is a `.sitefileignore` rule (not blank, a `#` comment, or only `!` and `/`). */
|
|
62
|
+
export function isIgnorePattern(text) {
|
|
63
|
+
return parseLine(text) !== undefined;
|
|
64
|
+
}
|
|
65
|
+
function parseLine(raw) {
|
|
66
|
+
let line = raw.replace(/(?<!\\)\s+$/, '');
|
|
67
|
+
if (line === '' || line.startsWith('#'))
|
|
68
|
+
return undefined;
|
|
69
|
+
let negate = false;
|
|
70
|
+
if (line.startsWith('!')) {
|
|
71
|
+
negate = true;
|
|
72
|
+
line = line.slice(1);
|
|
73
|
+
}
|
|
74
|
+
const dirOnly = line.endsWith('/');
|
|
75
|
+
if (dirOnly)
|
|
76
|
+
line = line.replace(/\/+$/, '');
|
|
77
|
+
const anchored = line.includes('/');
|
|
78
|
+
line = line.replace(/^\/+/, '');
|
|
79
|
+
if (line === '')
|
|
80
|
+
return undefined;
|
|
81
|
+
const segs = line.split('/');
|
|
82
|
+
let src = '';
|
|
83
|
+
segs.forEach((seg, i) => {
|
|
84
|
+
const last = i === segs.length - 1;
|
|
85
|
+
if (seg === '**') {
|
|
86
|
+
// A trailing `**` needs at least one more segment; elsewhere it may match none.
|
|
87
|
+
src += last ? '.+' : '(?:[^/]+/)*';
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
src += globSegment(seg) + (last ? '' : '/');
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
return { negate, dirOnly, re: new RegExp(`^${anchored ? '' : '(?:.*/)?'}${src}$`) };
|
|
94
|
+
}
|
|
95
|
+
function globSegment(seg) {
|
|
96
|
+
let out = '';
|
|
97
|
+
for (let i = 0; i < seg.length; i++) {
|
|
98
|
+
const c = seg[i];
|
|
99
|
+
if (c === '\\' && i + 1 < seg.length) {
|
|
100
|
+
out += escapeRe(seg[++i]);
|
|
101
|
+
}
|
|
102
|
+
else if (c === '*') {
|
|
103
|
+
while (seg[i + 1] === '*')
|
|
104
|
+
i++; // `**` inside a segment behaves like `*`
|
|
105
|
+
out += '[^/]*';
|
|
106
|
+
}
|
|
107
|
+
else if (c === '?') {
|
|
108
|
+
out += '[^/]';
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
out += escapeRe(c);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
function escapeRe(c) {
|
|
117
|
+
return c.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=ignore.js.map
|