@intellectual-frontiers/net 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/net.ts ADDED
@@ -0,0 +1,332 @@
1
+ // Spec: 0001-net FR-001
2
+ /**
3
+ * Downloads over a network that fails now and works in a minute (commons 0001-net FR-001), on the web standard `fetch` and
4
+ * `node:fs`. Taken from the machine setup's `net.ts`: a failure is classified (the name, the connection, a busy server, a certificate),
5
+ * retried only where another try can help, with a longer wait each time (or as long as the server's `Retry-After` asks, up to the cap) and every retry said; a download that stops part way is resumed
6
+ * with `Range`; and a file is kept only where its SHA-256 is the one expected. A download that fails is an expected outcome, so these
7
+ * return a `Result`.
8
+ *
9
+ * @example
10
+ * ```ts
11
+ * import { downloadVerified, Retries } from "./net.ts";
12
+ * import { sha256Of } from "@intellectual-frontiers/crypto/sha256";
13
+ * const sum = sha256Of("…64 hex digits…");
14
+ * if (sum !== undefined) {
15
+ * const r = await downloadVerified(new Retries({ tries: 5, base: 2, cap: 30 }, console.error), "https://example.com/tool.tgz", sum, "/tmp/cache");
16
+ * if (r.kind === "ok") console.log("kept at", r.value);
17
+ * }
18
+ * ```
19
+ *
20
+ * @module
21
+ */
22
+ import * as fsp from "node:fs/promises";
23
+ import { join } from "node:path";
24
+ import { absurd, err, ok, type Result } from "@intellectual-frontiers/core";
25
+ import { retryAfterMs } from "@intellectual-frontiers/formats/http";
26
+ import type { Sha256 } from "@intellectual-frontiers/crypto/sha256";
27
+ import { exists, mkdir, remove, rename, sha256File, stat } from "@intellectual-frontiers/system/fs";
28
+
29
+ /** Why a request failed, as far as the network is concerned. */
30
+ export type NetFailure =
31
+ | { readonly kind: "dns" }
32
+ | { readonly kind: "connect" }
33
+ | { readonly kind: "broken" }
34
+ | { readonly kind: "timeout" }
35
+ | { readonly kind: "tls" }
36
+ | { readonly kind: "certificate" }
37
+ | { readonly kind: "http"; readonly status: number; readonly retryAfterMs?: number }
38
+ | { readonly kind: "other"; readonly message: string };
39
+
40
+ const busy = (status: number): boolean => status === 408 || status === 425 || status === 429 || (status >= 500 && status < 600);
41
+
42
+ /** A failure as a person reads it. */
43
+ export function describeFailure(f: NetFailure): string {
44
+ switch (f.kind) {
45
+ case "dns":
46
+ return "the name could not be looked up (DNS)";
47
+ case "connect":
48
+ return "no connection could be made";
49
+ case "broken":
50
+ return "the connection broke or the download stopped early";
51
+ case "timeout":
52
+ return "it took too long";
53
+ case "tls":
54
+ return "the secure connection could not be made";
55
+ case "certificate":
56
+ return "the certificate was not accepted (a proxy, or the computer's clock?)";
57
+ case "http":
58
+ return f.status === 416
59
+ ? "the part already fetched did not fit, so it starts again"
60
+ : busy(f.status)
61
+ ? `the server answered ${f.status} (busy or not ready)`
62
+ : `the server answered ${f.status}`;
63
+ case "other":
64
+ return f.message;
65
+ default:
66
+ return absurd(f);
67
+ }
68
+ }
69
+
70
+ /** Whether another try may help: the name, the connection, a busy server; never a refusal or a certificate. */
71
+ export function retryable(f: NetFailure): boolean {
72
+ switch (f.kind) {
73
+ case "dns":
74
+ case "connect":
75
+ case "broken":
76
+ case "timeout":
77
+ case "tls":
78
+ return true;
79
+ case "certificate":
80
+ case "other":
81
+ return false;
82
+ case "http":
83
+ return busy(f.status) || f.status === 416;
84
+ default:
85
+ return absurd(f);
86
+ }
87
+ }
88
+
89
+ const CODES: Readonly<Record<string, NetFailure["kind"]>> = {
90
+ ENOTFOUND: "dns",
91
+ EAI_AGAIN: "dns",
92
+ ECONNREFUSED: "connect",
93
+ ENETUNREACH: "connect",
94
+ EHOSTUNREACH: "connect",
95
+ UND_ERR_CONNECT_TIMEOUT: "timeout",
96
+ UND_ERR_HEADERS_TIMEOUT: "timeout",
97
+ UND_ERR_BODY_TIMEOUT: "timeout",
98
+ ETIMEDOUT: "timeout",
99
+ ECONNRESET: "broken",
100
+ EPIPE: "broken",
101
+ UND_ERR_SOCKET: "broken",
102
+ ERR_TLS_CERT_ALTNAME_INVALID: "certificate",
103
+ CERT_HAS_EXPIRED: "certificate",
104
+ UNABLE_TO_VERIFY_LEAF_SIGNATURE: "certificate",
105
+ UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "certificate",
106
+ DEPTH_ZERO_SELF_SIGNED_CERT: "certificate",
107
+ SELF_SIGNED_CERT_IN_CHAIN: "certificate",
108
+ };
109
+
110
+ const codeOf = (e: unknown): string | undefined => {
111
+ if (typeof e !== "object" || e === null) return undefined;
112
+ const c: unknown = Reflect.get(e, "code");
113
+ return typeof c === "string" ? c : undefined;
114
+ };
115
+
116
+ /** What an error of `fetch` says, as a failure of the network: by its code (or its cause's) where it has one, else by its words. */
117
+ export function classify(e: unknown): NetFailure {
118
+ if (e instanceof DOMException && (e.name === "TimeoutError" || e.name === "AbortError")) return { kind: "timeout" };
119
+ const cause: unknown = e instanceof Error ? e.cause : undefined;
120
+ const code = codeOf(e) ?? codeOf(cause);
121
+ const byCode = code === undefined ? undefined : CODES[code];
122
+ if (byCode !== undefined && byCode !== "http" && byCode !== "other") return { kind: byCode };
123
+ const m = e instanceof Error ? `${e.message} ${cause instanceof Error ? cause.message : ""}` : String(e);
124
+ if (/certificate|UnknownIssuer|invalid peer|self[ -]signed|CertNotValid/i.test(m)) return { kind: "certificate" };
125
+ if (
126
+ /dns error|failed to lookup|name or service not known|nodename nor servname|no such host|temporary failure in name resolution|ENOTFOUND/i.test(
127
+ m,
128
+ )
129
+ ) {
130
+ return { kind: "dns" };
131
+ }
132
+ if (/connection refused|ConnectionRefused|network is unreachable|no route to host|connect error|ECONNREFUSED/i.test(m))
133
+ return { kind: "connect" };
134
+ if (/timed? ?out|deadline/i.test(m)) return { kind: "timeout" };
135
+ if (/tls|ssl|handshake/i.test(m)) return { kind: "tls" };
136
+ if (
137
+ /reset|broken pipe|unexpected eof|connection closed|IncompleteMessage|incomplete|error reading a body|body from connection|terminated|other side closed/i.test(
138
+ m,
139
+ )
140
+ ) {
141
+ return { kind: "broken" };
142
+ }
143
+ return { kind: "other", message: m.trim().slice(0, 200) };
144
+ }
145
+
146
+ /** How often and how patiently to try: at most `tries` attempts, waiting `base` seconds after the first failure, doubled each time up to `cap`. */
147
+ export type RetryPolicy = { readonly tries: number; readonly base: number; readonly cap: number };
148
+
149
+ /** One retry: what was retried, why, and which attempt comes next. */
150
+ export type RetryNote = { readonly what: string; readonly why: string; readonly attempt: number };
151
+
152
+ /** The retries of a run: how long to wait, each retry said through `report` and kept for a summary. */
153
+ export class Retries {
154
+ /** The retries so far, in order. */
155
+ readonly notes: RetryNote[] = [];
156
+ /** The policy. */
157
+ readonly policy: RetryPolicy;
158
+ /** Where each retry is said. */
159
+ readonly report: (line: string) => void;
160
+ /** Seconds added to each wait so that many computers do not try together (0 to 2 by default). */
161
+ readonly jitter: () => number;
162
+ /** How a wait is spent (a timer by default; a test passes one that returns at once). */
163
+ readonly pause: (seconds: number) => Promise<void>;
164
+
165
+ /** Retries under a policy, said through `report`. */
166
+ constructor(
167
+ policy: RetryPolicy,
168
+ report: (line: string) => void,
169
+ jitter: () => number = () => Math.floor(Math.random() * 3),
170
+ pause: (seconds: number) => Promise<void> = (s) => new Promise((r) => setTimeout(r, s * 1000)),
171
+ ) {
172
+ this.policy = policy;
173
+ this.report = report;
174
+ this.jitter = jitter;
175
+ this.pause = pause;
176
+ }
177
+
178
+ /** The seconds to wait after attempt `attempt` failed: the base doubled each time, no more than the cap, plus the jitter. */
179
+ wait(attempt: number): number {
180
+ let d = this.policy.base;
181
+ for (let i = 1; i < attempt && d < this.policy.cap; i++) d *= 2;
182
+ return Math.min(d, this.policy.cap) + this.jitter();
183
+ }
184
+
185
+ /** Say that an attempt failed and when the next one is, keep it, and wait; a server's `Retry-After` (in seconds, up to the cap) wins. */
186
+ async again(what: string, why: string, attempt: number, retryAfterSeconds?: number): Promise<void> {
187
+ const w = retryAfterSeconds === undefined ? this.wait(attempt) : Math.min(Math.ceil(retryAfterSeconds), this.policy.cap);
188
+ this.notes.push({ what, why, attempt: attempt + 1 });
189
+ this.report(`${what}: ${why}; trying again in ${w}s (attempt ${attempt + 1} of ${this.policy.tries})`);
190
+ await this.pause(w);
191
+ }
192
+
193
+ /** What the network cost, most retried first: `github.com (3), example.org (1)`. */
194
+ top(n = 4): string {
195
+ const by = new Map<string, number>();
196
+ for (const x of this.notes) by.set(x.what, (by.get(x.what) ?? 0) + 1);
197
+ return [...by.entries()]
198
+ .sort((a, b) => b[1] - a[1])
199
+ .slice(0, n)
200
+ .map(([w, c]) => `${w} (${c})`)
201
+ .join(", ");
202
+ }
203
+ }
204
+
205
+ /** Limits of one try. */
206
+ export type FetchOptions = {
207
+ /** No answer within this many seconds fails the try (20 by default). */
208
+ readonly answerSeconds?: number;
209
+ /** Less than 1 KB a second over this many seconds fails the try (60 by default). */
210
+ readonly stallSeconds?: number;
211
+ /** Headers to send. */
212
+ readonly headers?: Readonly<Record<string, string>>;
213
+ };
214
+
215
+ /** One try: the file at `url` written to `dest`; a `dest` ending `.part` that already has bytes is resumed where the server can. */
216
+ export async function fetchOnce(url: string, dest: string, o: FetchOptions = {}): Promise<Result<void, NetFailure>> {
217
+ const ctl = new AbortController();
218
+ const answer = setTimeout(() => ctl.abort(new DOMException("no answer", "TimeoutError")), (o.answerSeconds ?? 20) * 1000);
219
+ const have = dest.endsWith(".part") && (await exists(dest)) ? (await stat(dest)).size : 0;
220
+ let res: Response;
221
+ try {
222
+ const range: Record<string, string> = have > 0 && !url.startsWith("file:") ? { Range: `bytes=${have}-` } : {};
223
+ res = await fetch(url, { signal: ctl.signal, headers: { ...o.headers, ...range } });
224
+ } catch (e) {
225
+ return err(classify(e));
226
+ } finally {
227
+ clearTimeout(answer);
228
+ }
229
+ if (!res.ok) {
230
+ await res.body?.cancel();
231
+ const after = busy(res.status) ? retryAfterMs(res.headers.get("retry-after"), Date.now()) : undefined;
232
+ return err({ kind: "http", status: res.status, ...(after === undefined ? {} : { retryAfterMs: after }) });
233
+ }
234
+ const append = res.status === 206;
235
+ const stall = (o.stallSeconds ?? 60) * 1000;
236
+ let window = 0;
237
+ let windowStart = Date.now();
238
+ const watch = setInterval(
239
+ () => {
240
+ if (Date.now() - windowStart < stall) return;
241
+ if (window < (stall / 1000) * 1024) ctl.abort(new DOMException("stalled", "TimeoutError"));
242
+ window = 0;
243
+ windowStart = Date.now();
244
+ },
245
+ Math.min(5000, stall),
246
+ );
247
+ let file: fsp.FileHandle | undefined;
248
+ try {
249
+ file = await fsp.open(dest, append ? "a" : "w");
250
+ if (res.body !== null) {
251
+ const reader = res.body.getReader();
252
+ for (;;) {
253
+ const { done, value } = await reader.read();
254
+ if (done) break;
255
+ window += value.length;
256
+ await file.write(value);
257
+ }
258
+ }
259
+ return ok(undefined);
260
+ } catch (e) {
261
+ return err(classify(e));
262
+ } finally {
263
+ clearInterval(watch);
264
+ await file?.close();
265
+ }
266
+ }
267
+
268
+ /** The host a URL names, for saying what was retried. */
269
+ export function hostOf(url: string): string {
270
+ try {
271
+ return new URL(url).host || url;
272
+ } catch {
273
+ return url;
274
+ }
275
+ }
276
+
277
+ /** The file at `url` written to `dest`, tried again where the network failed and another try can help; or why it was not. */
278
+ export async function fetchTo(retries: Retries, url: string, dest: string, o: FetchOptions = {}): Promise<Result<void, string>> {
279
+ const name = hostOf(url);
280
+ for (let n = 1; ; n++) {
281
+ const r = await fetchOnce(url, dest, o);
282
+ if (r.kind === "ok") return r;
283
+ const f = r.error;
284
+ if (!retryable(f) || n >= retries.policy.tries)
285
+ return err(`${name}: ${describeFailure(f)} (after ${n} of ${retries.policy.tries} attempts)`);
286
+ if (f.kind === "http" && f.status === 416) await remove(dest).catch(() => {});
287
+ await retries.again(name, describeFailure(f), n, f.kind === "http" && f.retryAfterMs !== undefined ? f.retryAfterMs / 1000 : undefined);
288
+ }
289
+ }
290
+
291
+ /**
292
+ * The file in `cache/downloads/<sha256>`, fetched where it is not there, and kept only where its SHA-256 is `sum`. A download that stops
293
+ * part way is kept as `.part` and resumed by the next try or the next run; a file that is not the one expected is removed, never kept.
294
+ */
295
+ export async function downloadVerified(
296
+ retries: Retries,
297
+ url: string,
298
+ sum: Sha256,
299
+ cache: string,
300
+ o: FetchOptions = {},
301
+ ): Promise<Result<string, string>> {
302
+ const dir = join(cache, "downloads");
303
+ await mkdir(dir, { recursive: true });
304
+ const f = join(dir, sum);
305
+ if ((await exists(f)) && (await sha256File(f)) === sum) return ok(f);
306
+ const part = `${f}.part`;
307
+ const got = await fetchTo(retries, url, part, o);
308
+ if (got.kind === "err") return err(`could not fetch ${url}: ${got.error}`);
309
+ const actual = await sha256File(part);
310
+ if (actual !== sum) {
311
+ await remove(part).catch(() => {});
312
+ return err(`the file at ${url} is not the one expected (its SHA-256 is ${actual}, expected ${sum}); nothing was kept`);
313
+ }
314
+ await rename(part, f);
315
+ return ok(f);
316
+ }
317
+
318
+ /** Whether `host` answers over HTTPS (any answer counts, even a refusal), tried again where it does not, at most 5 times. */
319
+ export async function reach(retries: Retries, host: string, scheme: "https" | "http" = "https"): Promise<Result<void, NetFailure>> {
320
+ const max = Math.min(5, retries.policy.tries);
321
+ for (let n = 1; ; n++) {
322
+ try {
323
+ const res = await fetch(`${scheme}://${host}/`, { method: "HEAD", signal: AbortSignal.timeout(20_000) });
324
+ await res.body?.cancel();
325
+ return ok(undefined);
326
+ } catch (e) {
327
+ const f = classify(e);
328
+ if (n >= max) return err(f);
329
+ await retries.again(host, describeFailure(f), n);
330
+ }
331
+ }
332
+ }
package/src/serve.ts ADDED
@@ -0,0 +1,135 @@
1
+ // Spec: 0001-net FR-002
2
+ /**
3
+ * An HTTP server whose handler takes a web `Request` and gives a web `Response` on `node:http`, so the same handler
4
+ * runs on Bun and Node (commons 0001-net FR-002). Request bodies and response bodies stream; a client that goes away
5
+ * aborts `request.signal`. `serve` returns a promise, settled once the port is bound, so `addr` is known.
6
+ *
7
+ * @example
8
+ * ```ts
9
+ * import { serve } from "./serve.ts";
10
+ * const server = await serve({ port: 0, hostname: "127.0.0.1" }, (req) => new Response(`you asked for ${new URL(req.url).pathname}`));
11
+ * const text = await (await fetch(`http://127.0.0.1:${server.addr.port}/x`)).text(); // "you asked for /x"
12
+ * await server.shutdown();
13
+ * ```
14
+ *
15
+ * @module
16
+ */
17
+ import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
18
+ import type { AddressInfo } from "node:net";
19
+ import { Readable } from "node:stream";
20
+ import { mapError } from "@intellectual-frontiers/system/errors";
21
+ import { traceServer } from "@intellectual-frontiers/telemetry/host/otel";
22
+
23
+ /** Where the server listens, or where a request came from. */
24
+ export type NetAddr = { readonly hostname: string; readonly port: number };
25
+
26
+ /** What a handler knows of a request besides the request itself. */
27
+ export type ServeInfo = { readonly remoteAddr: NetAddr };
28
+
29
+ /** A function from a request to its response. */
30
+ export type ServeHandler = (request: Request, info: ServeInfo) => Response | Promise<Response>;
31
+
32
+ /** Where and how to listen. */
33
+ export type ServeOptions = {
34
+ /** The port; 0 (the default) asks the system for a free one. */
35
+ readonly port?: number;
36
+ /** The address to bind; `127.0.0.1` by default, never every interface unless asked. */
37
+ readonly hostname?: string;
38
+ /** Aborting it shuts the server down. */
39
+ readonly signal?: AbortSignal;
40
+ /** The response where the handler throws (a plain 500 by default). */
41
+ readonly onError?: (error: unknown) => Response | Promise<Response>;
42
+ };
43
+
44
+ /** A running server. */
45
+ export type Server = {
46
+ /** Where it listens. */
47
+ readonly addr: NetAddr;
48
+ /** Settles when it has shut down. */
49
+ readonly finished: Promise<void>;
50
+ /** Stop listening and close every connection; settles when it has shut down. */
51
+ shutdown(): Promise<void>;
52
+ };
53
+
54
+ const NO_BODY = new Set(["GET", "HEAD"]);
55
+
56
+ function toRequest(req: IncomingMessage, fallbackHost: string): { readonly request: Request; readonly abort: AbortController } {
57
+ const abort = new AbortController();
58
+ const headers = new Headers();
59
+ for (let i = 0; i + 1 < req.rawHeaders.length; i += 2) headers.append(req.rawHeaders[i] ?? "", req.rawHeaders[i + 1] ?? "");
60
+ const method = req.method ?? "GET";
61
+ const url = new URL(req.url ?? "/", `http://${req.headers.host ?? fallbackHost}`);
62
+ const init: RequestInit & { duplex?: "half" } = { method, headers, signal: abort.signal };
63
+ if (!NO_BODY.has(method)) {
64
+ init.body = Readable.toWeb(req) as ReadableStream<Uint8Array>;
65
+ init.duplex = "half";
66
+ }
67
+ return { request: new Request(url, init), abort };
68
+ }
69
+
70
+ async function send(res: ServerResponse, response: Response, head: boolean): Promise<void> {
71
+ res.statusCode = response.status;
72
+ if (response.statusText !== "") res.statusMessage = response.statusText;
73
+ const cookies = response.headers.getSetCookie();
74
+ response.headers.forEach((value, key) => {
75
+ if (key !== "set-cookie") res.setHeader(key, value);
76
+ });
77
+ if (cookies.length > 0) res.setHeader("set-cookie", cookies);
78
+ if (response.body === null || head) {
79
+ await response.body?.cancel();
80
+ res.end();
81
+ return;
82
+ }
83
+ res.flushHeaders();
84
+ const reader = response.body.getReader();
85
+ try {
86
+ for (;;) {
87
+ const { done, value } = await reader.read();
88
+ if (done) break;
89
+ if (!res.write(value)) await new Promise<void>((r) => res.once("drain", r));
90
+ if (res.destroyed) break;
91
+ }
92
+ } finally {
93
+ reader.releaseLock();
94
+ res.end();
95
+ }
96
+ }
97
+
98
+ /** A server that answers each request with what `handler` gives, once its port is bound. `AddrInUse` where the port is taken. */
99
+ export function serve(options: ServeOptions, handler: ServeHandler): Promise<Server> {
100
+ const hostname = options.hostname ?? "127.0.0.1";
101
+ const onError = options.onError ?? (() => new Response("internal server error", { status: 500 }));
102
+ const server = createServer((req, res) => {
103
+ const { request, abort } = toRequest(req, `${hostname}:${options.port ?? 0}`);
104
+ res.on("close", () => {
105
+ if (!res.writableFinished) abort.abort();
106
+ });
107
+ const remoteAddr = { hostname: req.socket.remoteAddress ?? "", port: req.socket.remotePort ?? 0 };
108
+ void (async () => {
109
+ let response: Response;
110
+ try {
111
+ // Where a telemetry pipeline is installed, an `http.server` span, the child of the request's `traceparent` (0001-net FR-016).
112
+ response = await traceServer(request, () => handler(request, { remoteAddr }), handler);
113
+ } catch (e) {
114
+ response = await onError(e);
115
+ }
116
+ await send(res, response, request.method === "HEAD").catch(() => res.destroy());
117
+ })();
118
+ });
119
+ const finished = new Promise<void>((resolve) => server.on("close", () => resolve()));
120
+ const shutdown = async (): Promise<void> => {
121
+ if (server.listening) {
122
+ server.close();
123
+ server.closeAllConnections();
124
+ }
125
+ await finished;
126
+ };
127
+ return new Promise<Server>((resolve, reject) => {
128
+ server.once("error", (e) => reject(mapError(e, `${hostname}:${options.port ?? 0}`)));
129
+ server.listen(options.port ?? 0, hostname, () => {
130
+ const a = server.address() as AddressInfo;
131
+ options.signal?.addEventListener("abort", () => void shutdown(), { once: true });
132
+ resolve({ addr: { hostname: a.address, port: a.port }, finished, shutdown });
133
+ });
134
+ });
135
+ }