@johnhenry/sitefile 0.0.0 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -1
- package/README.md +62 -24
- package/lib/src/checklinks.js +2 -0
- package/lib/src/cli/build-image.js +2 -2
- package/lib/src/cli/cache.js +15 -5
- package/lib/src/cli/images.js +3 -3
- package/lib/src/cli/index.d.ts +1 -1
- package/lib/src/cli/index.js +1 -1
- package/lib/src/cli/network.d.ts +34 -0
- package/lib/src/cli/network.js +56 -0
- package/lib/src/cli/outdated.js +5 -0
- package/lib/src/cli/registry.js +6 -1
- package/lib/src/cli/servers.js +23 -7
- package/lib/src/cli/vendor.js +5 -0
- package/lib/src/cli.js +44 -8
- package/lib/src/errors.d.ts +25 -3
- package/lib/src/errors.js +9 -0
- package/lib/src/executor.d.ts +7 -0
- package/lib/src/executor.js +18 -10
- package/lib/src/export-formats.d.ts +10 -3
- package/lib/src/export-formats.js +17 -6
- package/lib/src/index.d.ts +2 -1
- package/lib/src/index.js +2 -1
- package/lib/src/language/ast.d.ts +9 -2
- package/lib/src/outdated.js +1 -1
- package/lib/src/planner.d.ts +1 -1
- package/lib/src/planner.js +2 -2
- package/lib/src/plugins/cache.d.ts +29 -3
- package/lib/src/plugins/cache.js +104 -5
- package/lib/src/plugins/host.js +3 -2
- package/lib/src/registry.d.ts +6 -0
- package/lib/src/registry.js +38 -19
- package/lib/src/resources/https.d.ts +35 -1
- package/lib/src/resources/https.js +175 -18
- package/lib/src/resources/network.d.ts +70 -0
- package/lib/src/resources/network.js +304 -0
- package/lib/src/resources/types.d.ts +6 -0
- package/lib/src/secrets.js +1 -1
- package/lib/src/serve-rules.js +2 -1
- package/lib/src/servers.d.ts +14 -0
- package/lib/src/servers.js +89 -3
- package/lib/src/vendor.js +1 -1
- package/lib/src/wbn.d.ts +21 -0
- package/lib/src/wbn.js +111 -17
- package/package.json +2 -1
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { type SourceSpan } from '../errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* Explicit network settings for everything sitefile fetches (IMPORT over http(s), Git, registries).
|
|
4
|
+
* Nothing here is read from the environment implicitly: `HTTP(S)_PROXY` stays ignored, and the CLI
|
|
5
|
+
* reads `SITEFILE_PROXY` and friends through {@link networkFromEnv}. Embedders pass this in code
|
|
6
|
+
* (`FetchPolicy.network`, `RegistryOptions.network`).
|
|
7
|
+
*/
|
|
8
|
+
export interface NetworkSettings {
|
|
9
|
+
/** `http://[user:password@]host[:port]`: a CONNECT tunnel for https targets, absolute-form requests for http. */
|
|
10
|
+
readonly proxy?: string | undefined;
|
|
11
|
+
/** Hosts reached directly, never through the proxy (`NO_PROXY` syntax, see {@link hostMatches}). */
|
|
12
|
+
readonly noProxy?: readonly string[] | undefined;
|
|
13
|
+
/** When non-empty, only hosts matching one of these patterns may be contacted. */
|
|
14
|
+
readonly allowHosts?: readonly string[] | undefined;
|
|
15
|
+
/** Hosts never contacted; a deny rule beats an allow rule. */
|
|
16
|
+
readonly denyHosts?: readonly string[] | undefined;
|
|
17
|
+
/** Time limit for establishing the TCP connection (to the target, or to the proxy), ms. */
|
|
18
|
+
readonly connectTimeoutMs?: number | undefined;
|
|
19
|
+
/** Longest wait for data once connected (idle time), ms. */
|
|
20
|
+
readonly readTimeoutMs?: number | undefined;
|
|
21
|
+
}
|
|
22
|
+
/** A configured HTTP proxy, parsed. */
|
|
23
|
+
export interface ProxyConfig {
|
|
24
|
+
readonly host: string;
|
|
25
|
+
readonly port: number;
|
|
26
|
+
/** `Basic …` when the URL carries credentials. */
|
|
27
|
+
readonly authorization?: string | undefined;
|
|
28
|
+
/** The proxy URL without credentials, for messages. */
|
|
29
|
+
readonly shown: string;
|
|
30
|
+
}
|
|
31
|
+
/** Split a list given as one string (commas and/or whitespace). */
|
|
32
|
+
export declare function splitList(raw: string | undefined): string[];
|
|
33
|
+
/** Parse a proxy URL, or throw E_USAGE. Credentials never appear in the message. */
|
|
34
|
+
export declare function parseProxy(raw: string): ProxyConfig;
|
|
35
|
+
/**
|
|
36
|
+
* True when `host` (as in a URL, IPv6 without brackets) on `port` matches `pattern`:
|
|
37
|
+
* `*` every host
|
|
38
|
+
* `example.com` example.com and every subdomain (`.example.com` and `*.example.com` too)
|
|
39
|
+
* `10.0.0.0/8`, `fd00::/8` IP-literal hosts in that network; `192.0.2.1`, `::1` that address
|
|
40
|
+
* any of these + `:port` only that port (`[::1]:8080` for IPv6)
|
|
41
|
+
* Patterns match the host as written in the URL: a name pattern never matches an IP literal and an
|
|
42
|
+
* IP pattern never matches a name, whatever it resolves to (the private-address rule covers that).
|
|
43
|
+
*/
|
|
44
|
+
export declare function hostMatches(host: string, port: number, pattern: string): boolean;
|
|
45
|
+
/** Default port of an http(s) URL. */
|
|
46
|
+
export declare function urlPort(url: URL): number;
|
|
47
|
+
/**
|
|
48
|
+
* Apply the allow/deny host rules to one hop (every redirect, token realm and blob CDN hop included).
|
|
49
|
+
* Throws E_HOST_BLOCKED naming the rule; a deny rule beats an allow rule.
|
|
50
|
+
*/
|
|
51
|
+
export declare function checkHostRules(url: URL, network: NetworkSettings | undefined, shown: string, span?: SourceSpan): void;
|
|
52
|
+
/** The proxy to use for `url`, or undefined for a direct connection (no proxy, or a no-proxy match). */
|
|
53
|
+
export declare function proxyFor(url: URL, network: NetworkSettings | undefined): ProxyConfig | undefined;
|
|
54
|
+
/** A usage problem in `network` (bad proxy URL, host pattern or timeout), or undefined. */
|
|
55
|
+
export declare function networkProblem(network: NetworkSettings): string | undefined;
|
|
56
|
+
/** Parse `SITEFILE_FETCH_TIMEOUT`: `<ms>` (connect and read) or `connect=<ms>,read=<ms>` (either part optional). */
|
|
57
|
+
export declare function parseFetchTimeout(raw: string): {
|
|
58
|
+
connectTimeoutMs?: number;
|
|
59
|
+
readTimeoutMs?: number;
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* Network settings from `SITEFILE_PROXY`, `SITEFILE_NO_PROXY`, `SITEFILE_ALLOW_HOSTS`,
|
|
63
|
+
* `SITEFILE_DENY_HOSTS` (comma- or space-separated) and `SITEFILE_FETCH_TIMEOUT`. Empty values are unset.
|
|
64
|
+
* `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` are never read.
|
|
65
|
+
*/
|
|
66
|
+
export declare function networkFromEnv(env?: NodeJS.ProcessEnv): NetworkSettings;
|
|
67
|
+
/** `over` on top of `base`: the proxy and timeouts are replaced, the host lists are joined (duplicates dropped). */
|
|
68
|
+
export declare function mergeNetwork(base: NetworkSettings, over: NetworkSettings): NetworkSettings;
|
|
69
|
+
/** True when any setting is present. */
|
|
70
|
+
export declare function hasNetworkSettings(n: NetworkSettings | undefined): boolean;
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import { isIP } from 'node:net';
|
|
2
|
+
import { SitefileError } from '../errors.js';
|
|
3
|
+
/** Split a list given as one string (commas and/or whitespace). */
|
|
4
|
+
export function splitList(raw) {
|
|
5
|
+
return (raw ?? '')
|
|
6
|
+
.split(/[\s,]+/)
|
|
7
|
+
.map((s) => s.trim())
|
|
8
|
+
.filter(Boolean);
|
|
9
|
+
}
|
|
10
|
+
/** A proxy URL with any credentials removed. */
|
|
11
|
+
function shownProxy(raw) {
|
|
12
|
+
try {
|
|
13
|
+
const u = new URL(raw);
|
|
14
|
+
u.username = '';
|
|
15
|
+
u.password = '';
|
|
16
|
+
return u.href.replace(/\/$/, '');
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return '<invalid url>';
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/** Percent-decode, keeping invalid escapes as written (bytes decoded as UTF-8, lossily). */
|
|
23
|
+
function percentDecode(s) {
|
|
24
|
+
const bytes = [];
|
|
25
|
+
for (let i = 0; i < s.length;) {
|
|
26
|
+
if (s[i] === '%' && /^[0-9a-fA-F]{2}$/.test(s.slice(i + 1, i + 3))) {
|
|
27
|
+
bytes.push(parseInt(s.slice(i + 1, i + 3), 16));
|
|
28
|
+
i += 3;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
const c = Buffer.from(s[i], 'utf8');
|
|
32
|
+
bytes.push(...c);
|
|
33
|
+
i += 1;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return Buffer.from(bytes).toString('utf8');
|
|
37
|
+
}
|
|
38
|
+
/** Parse a proxy URL, or throw E_USAGE. Credentials never appear in the message. */
|
|
39
|
+
export function parseProxy(raw) {
|
|
40
|
+
const bad = () => new SitefileError('E_USAGE', `invalid proxy URL '${shownProxy(raw)}': expected http://[user:password@]host[:port]`);
|
|
41
|
+
let u;
|
|
42
|
+
try {
|
|
43
|
+
u = new URL(raw);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
throw bad();
|
|
47
|
+
}
|
|
48
|
+
if (u.protocol !== 'http:' || u.hostname === '' || (u.pathname !== '/' && u.pathname !== '') || u.search !== '' || u.hash !== '')
|
|
49
|
+
throw bad();
|
|
50
|
+
const host = u.hostname.replace(/^\[|\]$/g, '');
|
|
51
|
+
const port = u.port === '' ? 80 : Number(u.port);
|
|
52
|
+
const user = percentDecode(u.username);
|
|
53
|
+
const pass = percentDecode(u.password);
|
|
54
|
+
const authorization = user !== '' || pass !== '' ? `Basic ${Buffer.from(`${user}:${pass}`).toString('base64')}` : undefined;
|
|
55
|
+
return { host, port, authorization, shown: shownProxy(raw) };
|
|
56
|
+
}
|
|
57
|
+
function ipToBig(ip) {
|
|
58
|
+
const family = isIP(ip);
|
|
59
|
+
if (family === 4) {
|
|
60
|
+
const addr = ip.split('.').reduce((acc, b) => (acc << 8n) | BigInt(Number(b)), 0n);
|
|
61
|
+
return { family: 4, addr };
|
|
62
|
+
}
|
|
63
|
+
if (family === 6) {
|
|
64
|
+
let s = ip.toLowerCase();
|
|
65
|
+
const v4 = /(\d+)\.(\d+)\.(\d+)\.(\d+)$/.exec(s);
|
|
66
|
+
if (v4) {
|
|
67
|
+
const [b0, b1, b2, b3] = v4.slice(1).map(Number);
|
|
68
|
+
s = s.slice(0, v4.index) + ((b0 << 8) | b1).toString(16) + ':' + ((b2 << 8) | b3).toString(16);
|
|
69
|
+
}
|
|
70
|
+
const [head, tail] = s.split('::');
|
|
71
|
+
const h = head ? head.split(':') : [];
|
|
72
|
+
const t = tail ? tail.split(':') : [];
|
|
73
|
+
const groups = [...h, ...(tail === undefined ? [] : new Array(8 - h.length - t.length).fill('0')), ...t];
|
|
74
|
+
if (groups.length !== 8)
|
|
75
|
+
return undefined;
|
|
76
|
+
return { family: 6, addr: groups.reduce((acc, g) => (acc << 16n) | BigInt(parseInt(g, 16)), 0n) };
|
|
77
|
+
}
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
const HOSTNAME_RE = /^[a-z0-9_-]+(?:\.[a-z0-9_-]+)*$/;
|
|
81
|
+
/** Parse one pattern; undefined when it is not valid. */
|
|
82
|
+
function parsePattern(raw) {
|
|
83
|
+
let p = raw.trim().toLowerCase();
|
|
84
|
+
if (p === '')
|
|
85
|
+
return undefined;
|
|
86
|
+
if (p === '*')
|
|
87
|
+
return { any: true };
|
|
88
|
+
let port;
|
|
89
|
+
// [v6]:port, [v6], v4:port, name:port. A bare IPv6 (two or more colons) has no port.
|
|
90
|
+
const bracket = /^\[([^\]]+)\](?::(\d+))?$/.exec(p);
|
|
91
|
+
if (bracket) {
|
|
92
|
+
p = bracket[1];
|
|
93
|
+
if (bracket[2] !== undefined)
|
|
94
|
+
port = Number(bracket[2]);
|
|
95
|
+
}
|
|
96
|
+
else if ((p.match(/:/g) ?? []).length === 1) {
|
|
97
|
+
const i = p.indexOf(':');
|
|
98
|
+
const ps = p.slice(i + 1);
|
|
99
|
+
if (!/^\d+$/.test(ps))
|
|
100
|
+
return undefined;
|
|
101
|
+
port = Number(ps);
|
|
102
|
+
p = p.slice(0, i);
|
|
103
|
+
}
|
|
104
|
+
if (port !== undefined && (port < 1 || port > 65535))
|
|
105
|
+
return undefined;
|
|
106
|
+
const withPort = (x) => (port === undefined ? x : { ...x, port });
|
|
107
|
+
const slash = p.indexOf('/');
|
|
108
|
+
if (slash !== -1) {
|
|
109
|
+
const ip = ipToBig(p.slice(0, slash));
|
|
110
|
+
const ps = p.slice(slash + 1);
|
|
111
|
+
if (!ip || !/^\d+$/.test(ps))
|
|
112
|
+
return undefined;
|
|
113
|
+
const prefix = Number(ps);
|
|
114
|
+
if (prefix > (ip.family === 4 ? 32 : 128))
|
|
115
|
+
return undefined;
|
|
116
|
+
return withPort({ net: { ...ip, prefix } });
|
|
117
|
+
}
|
|
118
|
+
const ip = ipToBig(p);
|
|
119
|
+
if (ip)
|
|
120
|
+
return withPort({ net: { ...ip, prefix: ip.family === 4 ? 32 : 128 } });
|
|
121
|
+
if (p.startsWith('*.'))
|
|
122
|
+
p = p.slice(2);
|
|
123
|
+
else if (p.startsWith('.'))
|
|
124
|
+
p = p.slice(1);
|
|
125
|
+
if (p.endsWith('.'))
|
|
126
|
+
p = p.slice(0, -1);
|
|
127
|
+
if (!HOSTNAME_RE.test(p))
|
|
128
|
+
return undefined;
|
|
129
|
+
return withPort({ domain: p });
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* True when `host` (as in a URL, IPv6 without brackets) on `port` matches `pattern`:
|
|
133
|
+
* `*` every host
|
|
134
|
+
* `example.com` example.com and every subdomain (`.example.com` and `*.example.com` too)
|
|
135
|
+
* `10.0.0.0/8`, `fd00::/8` IP-literal hosts in that network; `192.0.2.1`, `::1` that address
|
|
136
|
+
* any of these + `:port` only that port (`[::1]:8080` for IPv6)
|
|
137
|
+
* Patterns match the host as written in the URL: a name pattern never matches an IP literal and an
|
|
138
|
+
* IP pattern never matches a name, whatever it resolves to (the private-address rule covers that).
|
|
139
|
+
*/
|
|
140
|
+
export function hostMatches(host, port, pattern) {
|
|
141
|
+
const p = parsePattern(pattern);
|
|
142
|
+
return p !== undefined && matches(normalizeHost(host), port, p);
|
|
143
|
+
}
|
|
144
|
+
function normalizeHost(host) {
|
|
145
|
+
let h = host.toLowerCase();
|
|
146
|
+
if (h.startsWith('[') && h.endsWith(']'))
|
|
147
|
+
h = h.slice(1, -1);
|
|
148
|
+
if (h.endsWith('.'))
|
|
149
|
+
h = h.slice(0, -1);
|
|
150
|
+
return h;
|
|
151
|
+
}
|
|
152
|
+
function matches(host, port, p) {
|
|
153
|
+
if (p.any)
|
|
154
|
+
return true;
|
|
155
|
+
if (p.port !== undefined && p.port !== port)
|
|
156
|
+
return false;
|
|
157
|
+
if (p.domain !== undefined)
|
|
158
|
+
return host === p.domain || host.endsWith(`.${p.domain}`);
|
|
159
|
+
if (p.net) {
|
|
160
|
+
const ip = ipToBig(host);
|
|
161
|
+
if (!ip || ip.family !== p.net.family)
|
|
162
|
+
return false;
|
|
163
|
+
const bits = BigInt(ip.family === 4 ? 32 : 128);
|
|
164
|
+
const shift = bits - BigInt(p.net.prefix);
|
|
165
|
+
return ip.addr >> shift === p.net.addr >> shift;
|
|
166
|
+
}
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
/** The first pattern of `list` that matches, if any. */
|
|
170
|
+
function firstMatch(host, port, list) {
|
|
171
|
+
const h = normalizeHost(host);
|
|
172
|
+
for (const raw of list ?? []) {
|
|
173
|
+
const p = parsePattern(raw);
|
|
174
|
+
if (p && matches(h, port, p))
|
|
175
|
+
return raw.trim();
|
|
176
|
+
}
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
/** Default port of an http(s) URL. */
|
|
180
|
+
export function urlPort(url) {
|
|
181
|
+
return url.port !== '' ? Number(url.port) : url.protocol === 'http:' ? 80 : 443;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Apply the allow/deny host rules to one hop (every redirect, token realm and blob CDN hop included).
|
|
185
|
+
* Throws E_HOST_BLOCKED naming the rule; a deny rule beats an allow rule.
|
|
186
|
+
*/
|
|
187
|
+
export function checkHostRules(url, network, shown, span) {
|
|
188
|
+
if (!network)
|
|
189
|
+
return;
|
|
190
|
+
const host = normalizeHost(url.hostname);
|
|
191
|
+
const port = urlPort(url);
|
|
192
|
+
const denied = firstMatch(host, port, network.denyHosts);
|
|
193
|
+
if (denied !== undefined) {
|
|
194
|
+
throw new SitefileError('E_HOST_BLOCKED', `refusing to connect to ${host}: it matches deny-hosts rule '${denied}' (${shown})`, span);
|
|
195
|
+
}
|
|
196
|
+
if ((network.allowHosts ?? []).length > 0 && firstMatch(host, port, network.allowHosts) === undefined) {
|
|
197
|
+
throw new SitefileError('E_HOST_BLOCKED', `refusing to connect to ${host}: it matches no allow-hosts rule (${shown})`, span);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/** The proxy to use for `url`, or undefined for a direct connection (no proxy, or a no-proxy match). */
|
|
201
|
+
export function proxyFor(url, network) {
|
|
202
|
+
if (!network?.proxy)
|
|
203
|
+
return undefined;
|
|
204
|
+
if (firstMatch(url.hostname, urlPort(url), network.noProxy) !== undefined)
|
|
205
|
+
return undefined;
|
|
206
|
+
return parseProxy(network.proxy);
|
|
207
|
+
}
|
|
208
|
+
// ------------------------------------------------------------------ configuration
|
|
209
|
+
/** A usage problem in `network` (bad proxy URL, host pattern or timeout), or undefined. */
|
|
210
|
+
export function networkProblem(network) {
|
|
211
|
+
if (network.proxy) {
|
|
212
|
+
try {
|
|
213
|
+
parseProxy(network.proxy);
|
|
214
|
+
}
|
|
215
|
+
catch (e) {
|
|
216
|
+
return e.message;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
for (const [name, list] of [
|
|
220
|
+
['no-proxy', network.noProxy],
|
|
221
|
+
['allow-hosts', network.allowHosts],
|
|
222
|
+
['deny-hosts', network.denyHosts],
|
|
223
|
+
]) {
|
|
224
|
+
for (const p of list ?? []) {
|
|
225
|
+
if (!parsePattern(p))
|
|
226
|
+
return `invalid ${name} pattern '${p}': expected *, a host name (example.com, .example.com, *.example.com), an IP address or a CIDR network, optionally with :port`;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
for (const [name, v] of [
|
|
230
|
+
['connect timeout', network.connectTimeoutMs],
|
|
231
|
+
['read timeout', network.readTimeoutMs],
|
|
232
|
+
]) {
|
|
233
|
+
if (v !== undefined && (!Number.isSafeInteger(v) || v <= 0))
|
|
234
|
+
return `the ${name} must be a positive integer (ms)`;
|
|
235
|
+
}
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
238
|
+
function positiveMs(raw, what) {
|
|
239
|
+
if (!/^\d+$/.test(raw) || Number(raw) <= 0 || !Number.isSafeInteger(Number(raw))) {
|
|
240
|
+
throw new SitefileError('E_USAGE', `${what} expects a positive integer (ms), got '${raw}'`);
|
|
241
|
+
}
|
|
242
|
+
return Number(raw);
|
|
243
|
+
}
|
|
244
|
+
/** Parse `SITEFILE_FETCH_TIMEOUT`: `<ms>` (connect and read) or `connect=<ms>,read=<ms>` (either part optional). */
|
|
245
|
+
export function parseFetchTimeout(raw) {
|
|
246
|
+
const s = raw.trim();
|
|
247
|
+
if (s === '')
|
|
248
|
+
return {};
|
|
249
|
+
if (/^\d+$/.test(s)) {
|
|
250
|
+
const ms = positiveMs(s, 'SITEFILE_FETCH_TIMEOUT');
|
|
251
|
+
return { connectTimeoutMs: ms, readTimeoutMs: ms };
|
|
252
|
+
}
|
|
253
|
+
const out = {};
|
|
254
|
+
for (const part of s.split(',')) {
|
|
255
|
+
const m = /^\s*(connect|read)\s*=\s*(\S*)\s*$/.exec(part);
|
|
256
|
+
if (!m)
|
|
257
|
+
throw new SitefileError('E_USAGE', `SITEFILE_FETCH_TIMEOUT expects <ms> or connect=<ms>,read=<ms>, got '${raw}'`);
|
|
258
|
+
const ms = positiveMs(m[2], `SITEFILE_FETCH_TIMEOUT ${m[1]}`);
|
|
259
|
+
if (m[1] === 'connect')
|
|
260
|
+
out.connectTimeoutMs = ms;
|
|
261
|
+
else
|
|
262
|
+
out.readTimeoutMs = ms;
|
|
263
|
+
}
|
|
264
|
+
return out;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Network settings from `SITEFILE_PROXY`, `SITEFILE_NO_PROXY`, `SITEFILE_ALLOW_HOSTS`,
|
|
268
|
+
* `SITEFILE_DENY_HOSTS` (comma- or space-separated) and `SITEFILE_FETCH_TIMEOUT`. Empty values are unset.
|
|
269
|
+
* `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` are never read.
|
|
270
|
+
*/
|
|
271
|
+
export function networkFromEnv(env = process.env) {
|
|
272
|
+
const proxy = (env.SITEFILE_PROXY ?? '').trim();
|
|
273
|
+
return {
|
|
274
|
+
...(proxy !== '' ? { proxy } : {}),
|
|
275
|
+
...(splitList(env.SITEFILE_NO_PROXY).length ? { noProxy: splitList(env.SITEFILE_NO_PROXY) } : {}),
|
|
276
|
+
...(splitList(env.SITEFILE_ALLOW_HOSTS).length ? { allowHosts: splitList(env.SITEFILE_ALLOW_HOSTS) } : {}),
|
|
277
|
+
...(splitList(env.SITEFILE_DENY_HOSTS).length ? { denyHosts: splitList(env.SITEFILE_DENY_HOSTS) } : {}),
|
|
278
|
+
...parseFetchTimeout(env.SITEFILE_FETCH_TIMEOUT ?? ''),
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
/** `over` on top of `base`: the proxy and timeouts are replaced, the host lists are joined (duplicates dropped). */
|
|
282
|
+
export function mergeNetwork(base, over) {
|
|
283
|
+
const join = (a, b) => {
|
|
284
|
+
const all = [...new Set([...(a ?? []), ...(b ?? [])])];
|
|
285
|
+
return all.length ? all : undefined;
|
|
286
|
+
};
|
|
287
|
+
const out = {
|
|
288
|
+
proxy: over.proxy ?? base.proxy,
|
|
289
|
+
noProxy: join(base.noProxy, over.noProxy),
|
|
290
|
+
allowHosts: join(base.allowHosts, over.allowHosts),
|
|
291
|
+
denyHosts: join(base.denyHosts, over.denyHosts),
|
|
292
|
+
connectTimeoutMs: over.connectTimeoutMs ?? base.connectTimeoutMs,
|
|
293
|
+
readTimeoutMs: over.readTimeoutMs ?? base.readTimeoutMs,
|
|
294
|
+
};
|
|
295
|
+
for (const k of Object.keys(out))
|
|
296
|
+
if (out[k] === undefined)
|
|
297
|
+
delete out[k];
|
|
298
|
+
return out;
|
|
299
|
+
}
|
|
300
|
+
/** True when any setting is present. */
|
|
301
|
+
export function hasNetworkSettings(n) {
|
|
302
|
+
return !!n && Object.values(n).some((v) => v !== undefined && !(Array.isArray(v) && v.length === 0));
|
|
303
|
+
}
|
|
304
|
+
//# sourceMappingURL=network.js.map
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { NetworkSettings } from './network.js';
|
|
1
2
|
import type { Tree } from '../tree.js';
|
|
2
3
|
export interface Provenance {
|
|
3
4
|
/** The URI as written (after variable interpolation), or `./relative/path`. */
|
|
@@ -49,6 +50,11 @@ export interface FetchPolicy {
|
|
|
49
50
|
* private CAs; certificate and hostname verification stay on.
|
|
50
51
|
*/
|
|
51
52
|
readonly ca?: string | Buffer | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* Proxy, host rules and connect/read timeouts (see {@link NetworkSettings}). Unset: direct
|
|
55
|
+
* connections, every public host, only the total `timeoutMs`.
|
|
56
|
+
*/
|
|
57
|
+
readonly network?: NetworkSettings | undefined;
|
|
52
58
|
}
|
|
53
59
|
export declare const DEFAULT_FETCH_POLICY: FetchPolicy;
|
|
54
60
|
/** Limits applied when an archive is extracted with `IMPORT … AS tree`. Defaults are secure. */
|
package/lib/src/secrets.js
CHANGED
|
@@ -84,7 +84,7 @@ export function redactError(e, secrets) {
|
|
|
84
84
|
if (!isSitefileError(e) || !secrets || secrets.size === 0)
|
|
85
85
|
return e;
|
|
86
86
|
const detail = redactSecrets(e.detail, secrets);
|
|
87
|
-
return detail === e.detail ? e : new SitefileError(e.code, detail, e.span);
|
|
87
|
+
return detail === e.detail ? e : new SitefileError(e.code, detail, e.span, e.needs ? { needs: e.needs } : undefined);
|
|
88
88
|
}
|
|
89
89
|
/** The request headers of an import's SECRET clauses; a secret that was not given is E_SECRET. */
|
|
90
90
|
export function secretHeaders(spec, secrets) {
|
package/lib/src/serve-rules.js
CHANGED
|
@@ -100,7 +100,8 @@ export function globMatch(pattern, s) {
|
|
|
100
100
|
}
|
|
101
101
|
/** Every matching block's headers; a name set by several blocks gets its values joined with `, `. */
|
|
102
102
|
export function headersFor(rules, path) {
|
|
103
|
-
|
|
103
|
+
// No prototype: a header named `__proto__` is an ordinary name (as in the Rust port).
|
|
104
|
+
const out = Object.create(null);
|
|
104
105
|
for (const r of rules) {
|
|
105
106
|
if (!globMatch(r.pattern, path))
|
|
106
107
|
continue;
|
package/lib/src/servers.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ArchiveLimits } from './resources/types.js';
|
|
1
2
|
import { Store } from './store.js';
|
|
2
3
|
import { Tree } from './tree.js';
|
|
3
4
|
/**
|
|
@@ -22,6 +23,11 @@ export interface ServerState {
|
|
|
22
23
|
} | {
|
|
23
24
|
readonly kind: 'dir';
|
|
24
25
|
readonly path: string;
|
|
26
|
+
}
|
|
27
|
+
/** zip, tar, tar.gz/tgz, a Web Bundle or an OCI image archive; read again on every start, as a directory is. */
|
|
28
|
+
| {
|
|
29
|
+
readonly kind: 'archive';
|
|
30
|
+
readonly path: string;
|
|
25
31
|
};
|
|
26
32
|
readonly spa: boolean;
|
|
27
33
|
/** The requested bind address (port 0 = pick a free port). */
|
|
@@ -67,5 +73,13 @@ export declare function pidAlive(pid: number): boolean;
|
|
|
67
73
|
export declare function isRunning(s: ServerState): Promise<boolean>;
|
|
68
74
|
/** Load a directory into a Tree through the sandboxed local reader; hidden files (except .well-known) are not served. */
|
|
69
75
|
export declare function loadDirectoryTree(dir: string): Promise<Tree>;
|
|
76
|
+
/**
|
|
77
|
+
* Load an archive file into a Tree. The format comes from the bytes, never the name: zip, tar, tar.gz,
|
|
78
|
+
* a Web Bundle, or an OCI image archive (a tar holding an OCI layout, as `build -o x.oci.tar` and `save`
|
|
79
|
+
* write it). A zip, tar or Web Bundle is extracted with `limits` (unsafe entries are refused) and served
|
|
80
|
+
* like a directory, without hidden files except `.well-known`; an OCI archive serves its first image's
|
|
81
|
+
* files, like that image from the store, after checking every blob's digest and its site layer against `limits`.
|
|
82
|
+
*/
|
|
83
|
+
export declare function loadArchiveTree(file: string, limits?: ArchiveLimits): Promise<Tree>;
|
|
70
84
|
/** The tree a server state serves. */
|
|
71
85
|
export declare function loadServedTree(store: Store, source: ServerState['source']): Promise<Tree>;
|
package/lib/src/servers.js
CHANGED
|
@@ -2,8 +2,12 @@ import { randomBytes } from 'node:crypto';
|
|
|
2
2
|
import { promises as fs } from 'node:fs';
|
|
3
3
|
import http from 'node:http';
|
|
4
4
|
import path from 'node:path';
|
|
5
|
+
import { sha256 } from './digest.js';
|
|
5
6
|
import { SitefileError } from './errors.js';
|
|
7
|
+
import { blobPathOf, decodeIndex, extractImageFiles, MEDIA, parseConfig, parseManifest, siteLayerIndex } from './oci.js';
|
|
8
|
+
import { extractArchive } from './resources/archive.js';
|
|
6
9
|
import { LocalContext } from './resources/local.js';
|
|
10
|
+
import { DEFAULT_ARCHIVE_LIMITS } from './resources/types.js';
|
|
7
11
|
import { Store } from './store.js';
|
|
8
12
|
import { Tree } from './tree.js';
|
|
9
13
|
const NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$/;
|
|
@@ -125,15 +129,97 @@ export async function isRunning(s) {
|
|
|
125
129
|
req.end();
|
|
126
130
|
});
|
|
127
131
|
}
|
|
132
|
+
/** Paths of `tree` a directory or archive serves: no hidden files except `.well-known`. */
|
|
133
|
+
function visible(tree) {
|
|
134
|
+
return Tree.fromFiles(tree.entries().filter(([p]) => p.split('/').every((seg) => !seg.startsWith('.') || seg === '' || seg === '.well-known')));
|
|
135
|
+
}
|
|
128
136
|
/** Load a directory into a Tree through the sandboxed local reader; hidden files (except .well-known) are not served. */
|
|
129
137
|
export async function loadDirectoryTree(dir) {
|
|
130
138
|
const ctx = await LocalContext.open(dir, { sitefileignore: false });
|
|
131
139
|
const { tree } = await ctx.readTree('', 'serve');
|
|
132
|
-
|
|
133
|
-
|
|
140
|
+
return visible(tree);
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Load an archive file into a Tree. The format comes from the bytes, never the name: zip, tar, tar.gz,
|
|
144
|
+
* a Web Bundle, or an OCI image archive (a tar holding an OCI layout, as `build -o x.oci.tar` and `save`
|
|
145
|
+
* write it). A zip, tar or Web Bundle is extracted with `limits` (unsafe entries are refused) and served
|
|
146
|
+
* like a directory, without hidden files except `.well-known`; an OCI archive serves its first image's
|
|
147
|
+
* files, like that image from the store, after checking every blob's digest and its site layer against `limits`.
|
|
148
|
+
*/
|
|
149
|
+
export async function loadArchiveTree(file, limits = DEFAULT_ARCHIVE_LIMITS) {
|
|
150
|
+
const origin = file;
|
|
151
|
+
const unreadable = () => new SitefileError('E_ARCHIVE', `cannot read '${origin}'`);
|
|
152
|
+
const st = await fs.stat(file).catch((e) => {
|
|
153
|
+
throw e.code === 'ENOENT' ? new SitefileError('E_NOT_FOUND', `'${origin}' does not exist`) : unreadable();
|
|
154
|
+
});
|
|
155
|
+
if (!st.isFile())
|
|
156
|
+
throw new SitefileError('E_TYPE', `'${origin}' is not a file`);
|
|
157
|
+
// The archive is at most its expanded size plus tar framing for every entry.
|
|
158
|
+
if (st.size > limits.maxTotalBytes + (limits.maxEntries + 2) * 4 * 512) {
|
|
159
|
+
throw new SitefileError('E_ARCHIVE_LIMIT', `${origin}: archive expands beyond ${limits.maxTotalBytes} bytes`);
|
|
160
|
+
}
|
|
161
|
+
const bytes = new Uint8Array(await fs.readFile(file).catch(() => { throw unreadable(); }));
|
|
162
|
+
const tree = await extractArchive(bytes, origin, limits);
|
|
163
|
+
return tree.get('/oci-layout') ? ociArchiveTree(tree, origin, limits) : visible(tree);
|
|
164
|
+
}
|
|
165
|
+
/** The files of the first image in an extracted OCI layout. */
|
|
166
|
+
async function ociArchiveTree(layout, origin, limits) {
|
|
167
|
+
const bad = (why) => new SitefileError('E_ARCHIVE', `${origin}: ${why}`);
|
|
168
|
+
const blob = (digest, size) => {
|
|
169
|
+
const node = layout.get(blobPathOf(digest));
|
|
170
|
+
if (!node)
|
|
171
|
+
throw bad(`OCI archive has no blob ${digest}`);
|
|
172
|
+
if (sha256(node.bytes) !== digest || node.bytes.byteLength !== size)
|
|
173
|
+
throw bad(`blob ${digest} does not match its digest`);
|
|
174
|
+
return node.bytes;
|
|
175
|
+
};
|
|
176
|
+
const index = layout.get('/index.json');
|
|
177
|
+
if (!index)
|
|
178
|
+
throw bad('OCI archive has no index.json');
|
|
179
|
+
let entries;
|
|
180
|
+
try {
|
|
181
|
+
entries = decodeIndex(index.bytes);
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
throw bad('OCI archive index.json is corrupt');
|
|
185
|
+
}
|
|
186
|
+
const first = entries[0];
|
|
187
|
+
if (!first)
|
|
188
|
+
throw bad('OCI archive holds no image');
|
|
189
|
+
const manifestBytes = blob(first.digest, first.size);
|
|
190
|
+
let m;
|
|
191
|
+
try {
|
|
192
|
+
m = parseManifest(manifestBytes);
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
throw bad("OCI archive's first manifest is not an image manifest");
|
|
196
|
+
}
|
|
197
|
+
const configBytes = blob(m.config.digest, m.config.size);
|
|
198
|
+
let cfg;
|
|
199
|
+
try {
|
|
200
|
+
cfg = parseConfig(configBytes);
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
throw bad("OCI archive's image config is corrupt");
|
|
204
|
+
}
|
|
205
|
+
const layer = m.layers[siteLayerIndex(m, cfg)];
|
|
206
|
+
let layerBytes = new Uint8Array(0); // extractImageFiles reports "not a Sitefile image"
|
|
207
|
+
if (layer && (layer.mediaType === MEDIA.layer || layer.mediaType === MEDIA.layerTar)) {
|
|
208
|
+
layerBytes = blob(layer.digest, layer.size);
|
|
209
|
+
// Hold the site layer to the archive limits before it is fully decoded.
|
|
210
|
+
await extractArchive(layerBytes, origin, limits);
|
|
211
|
+
}
|
|
212
|
+
return extractImageFiles(m, cfg, layerBytes, origin);
|
|
134
213
|
}
|
|
135
214
|
/** The tree a server state serves. */
|
|
136
215
|
export async function loadServedTree(store, source) {
|
|
137
|
-
|
|
216
|
+
switch (source.kind) {
|
|
217
|
+
case 'image':
|
|
218
|
+
return store.loadTree(source.imageId);
|
|
219
|
+
case 'dir':
|
|
220
|
+
return loadDirectoryTree(source.path);
|
|
221
|
+
case 'archive':
|
|
222
|
+
return loadArchiveTree(source.path);
|
|
223
|
+
}
|
|
138
224
|
}
|
|
139
225
|
//# sourceMappingURL=servers.js.map
|
package/lib/src/vendor.js
CHANGED
|
@@ -104,7 +104,7 @@ async function vendorUnredacted(options) {
|
|
|
104
104
|
lock,
|
|
105
105
|
frozen,
|
|
106
106
|
update,
|
|
107
|
-
registry: { ...options.registry, allowPrivateNetwork: options.registry?.allowPrivateNetwork ?? policy.allowPrivateNetwork },
|
|
107
|
+
registry: { ...options.registry, allowPrivateNetwork: options.registry?.allowPrivateNetwork ?? policy.allowPrivateNetwork, network: options.registry?.network ?? policy.network },
|
|
108
108
|
warn,
|
|
109
109
|
lead,
|
|
110
110
|
enforcePlatform,
|
package/lib/src/wbn.d.ts
CHANGED
|
@@ -3,3 +3,24 @@ export declare function mimeTypeFor(path: string): string;
|
|
|
3
3
|
/** An absolute https URL ending in `/`, without credentials, query or fragment. Returns its normalized form. */
|
|
4
4
|
export declare function validateBaseUrl(raw: string | undefined): string;
|
|
5
5
|
export declare function writeWebBundle(tree: Tree, baseUrl: string | undefined): Uint8Array;
|
|
6
|
+
/**
|
|
7
|
+
* The responses section and its URLs (tree paths, percent-encoded by the caller), as `sitefile
|
|
8
|
+
* serve` would answer them:
|
|
9
|
+
*
|
|
10
|
+
* - Each file is a 200 with `content-type` and the `/_headers` blocks whose pattern matches its
|
|
11
|
+
* URL path (`headersFor`; a `content-type` there replaces the inferred one). A directory URL
|
|
12
|
+
* (`/docs/` for `/docs/index.html`) matches with its trailing slash, so it gets its own
|
|
13
|
+
* response when its headers differ from the file URL's.
|
|
14
|
+
* - An exact-path `/_redirects` rule with a 3xx status becomes a response with that `:status`, a
|
|
15
|
+
* `location` of the rule's target as written, and an empty body: at its path when no file
|
|
16
|
+
* would be served there, and over the file when forced (`301!`). Like `serve`, the first rule
|
|
17
|
+
* matching the path decides; when that is a splat or a 200/404 rewrite, the bundle leaves the
|
|
18
|
+
* path to `/_redirects`. Splat rules and rewrites are not baked in.
|
|
19
|
+
*
|
|
20
|
+
* `/_headers` and `/_redirects` stay in the bundle as files. Responses are deduplicated (both URLs
|
|
21
|
+
* of an index page share one when their headers agree; identical redirects share one).
|
|
22
|
+
*/
|
|
23
|
+
export declare function bundleResponses(tree: Tree): {
|
|
24
|
+
responses: Uint8Array[];
|
|
25
|
+
urls: Array<[string, number]>;
|
|
26
|
+
};
|