@intellectual-frontiers/net 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/dist/acme.d.ts +71 -0
- package/dist/acme.js +475 -0
- package/dist/dns.d.ts +57 -0
- package/dist/dns.js +381 -0
- package/dist/http-client.d.ts +149 -0
- package/dist/http-client.js +248 -0
- package/dist/mod.d.ts +21 -0
- package/dist/mod.js +21 -0
- package/dist/net.d.ts +84 -0
- package/dist/net.js +288 -0
- package/dist/serve.d.ts +33 -0
- package/dist/serve.js +109 -0
- package/dist/tcp.d.ts +77 -0
- package/dist/tcp.js +274 -0
- package/package.json +72 -4
- package/src/acme.ts +546 -0
- package/src/dns.ts +423 -0
- package/src/http-client.ts +369 -0
- package/src/mod.ts +22 -0
- package/src/net.ts +332 -0
- package/src/serve.ts +135 -0
- package/src/tcp.ts +344 -0
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
// Spec: 0001-net FR-003
|
|
2
|
+
/**
|
|
3
|
+
* How a typed client reaches a system over HTTP (commons 0001-net FR-003): a transport, live (`fetch`) or recorded (the
|
|
4
|
+
* exchanges a test keeps, so it reaches no host), and over it the guards and helpers every client otherwise rewrites: a read-only guard
|
|
5
|
+
* (safe methods only, POST to the token endpoints named, and the POSTs a predicate says only read, FR-066), AWS Signature Version 4 on each
|
|
6
|
+
* request (FR-069), backoff on 429 and 503 that honours `Retry-After`, a JSON read decoded at the boundary with typed failures (a denied
|
|
7
|
+
* call is told apart, so a caller can record a gap instead of failing), and every item of a list that pages by `Link: rel="next"` or by a
|
|
8
|
+
* cursor in the body. A request that carries a credential is refused before it is sent unless the transport is made to allow it, so a
|
|
9
|
+
* harness whose proxy injects credentials never sends one by mistake. Taken from erga's `lib/ts/http-client` and Stolos's connector SDK
|
|
10
|
+
* (`src/archon/connect/sdk.ts`, `connector.ts`).
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```ts
|
|
14
|
+
* import { array, object, string } from "@intellectual-frontiers/core";
|
|
15
|
+
* import { getJson, liveTransport, withBackoff } from "./http-client.ts";
|
|
16
|
+
* const http = withBackoff(liveTransport());
|
|
17
|
+
* const repo = await getJson(http, "https://api.github.com/repos/oven-sh/bun", object({ full_name: string }));
|
|
18
|
+
* ```
|
|
19
|
+
*
|
|
20
|
+
* @module
|
|
21
|
+
*/
|
|
22
|
+
import { absurd, array, formatDecodeError, json, literal, number, object, record, string, err, ok, } from "@intellectual-frontiers/core";
|
|
23
|
+
import { isSafeMethod, nextLink, retryAfterMs } from "@intellectual-frontiers/formats/http";
|
|
24
|
+
import { formatJson5Error, parseJson5, toJson5 } from "@intellectual-frontiers/formats/json5";
|
|
25
|
+
import { sign } from "@intellectual-frontiers/crypto/sigv4";
|
|
26
|
+
import { traceHttpClient } from "@intellectual-frontiers/telemetry/host/otel";
|
|
27
|
+
import { sleep } from "@intellectual-frontiers/system/proc";
|
|
28
|
+
/** The headers a credential travels in. */
|
|
29
|
+
export const CREDENTIAL_HEADERS = [
|
|
30
|
+
"authorization",
|
|
31
|
+
"proxy-authorization",
|
|
32
|
+
"cookie",
|
|
33
|
+
"x-api-key",
|
|
34
|
+
"x-goog-api-key",
|
|
35
|
+
"api-key",
|
|
36
|
+
"private-token",
|
|
37
|
+
"job-token",
|
|
38
|
+
];
|
|
39
|
+
/** Why a request may not be sent where credentials are refused: it carries one in a header or in its query. Undefined where it carries none. */
|
|
40
|
+
export function carriesCredential(r) {
|
|
41
|
+
const named = Object.keys(r.headers)
|
|
42
|
+
.map((h) => h.toLowerCase())
|
|
43
|
+
.find((h) => CREDENTIAL_HEADERS.includes(h));
|
|
44
|
+
if (named !== undefined)
|
|
45
|
+
return `${r.method} ${r.url.split("?")[0] ?? r.url} carries the header ${named}, and this transport sends no credential`;
|
|
46
|
+
if (/[?&](key|api_key|apikey|access_token|token|password|passwd)=/i.test(r.url)) {
|
|
47
|
+
return `${r.method} ${r.url.split("?")[0] ?? r.url} carries a credential in its query, and this transport sends no credential`;
|
|
48
|
+
}
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
const KEPT = ["content-type", "link", "retry-after", "x-ratelimit-remaining", "x-ratelimit-reset", "etag", "location"];
|
|
52
|
+
/** The live transport, over `fetch`. */
|
|
53
|
+
export function liveTransport(options = {}) {
|
|
54
|
+
const fetcher = options.fetch ?? fetch;
|
|
55
|
+
return {
|
|
56
|
+
kind: "live",
|
|
57
|
+
async send(r) {
|
|
58
|
+
if (options.credentials !== "allow") {
|
|
59
|
+
const why = carriesCredential(r);
|
|
60
|
+
if (why !== undefined)
|
|
61
|
+
return err({ kind: "refused", message: why });
|
|
62
|
+
}
|
|
63
|
+
// Where a telemetry pipeline is installed, the request is an `http.client` span and carries `traceparent` (0001-net FR-016).
|
|
64
|
+
return traceHttpClient(r.method, r.url, async (trace) => {
|
|
65
|
+
try {
|
|
66
|
+
const res = await fetcher(r.url, {
|
|
67
|
+
method: r.method,
|
|
68
|
+
headers: { "user-agent": options.userAgent ?? "ifc-ts", ...trace, ...r.headers },
|
|
69
|
+
...(r.body === undefined ? {} : { body: r.body }),
|
|
70
|
+
signal: AbortSignal.timeout((options.timeoutSeconds ?? 30) * 1000),
|
|
71
|
+
});
|
|
72
|
+
const headers = {};
|
|
73
|
+
for (const k of KEPT) {
|
|
74
|
+
const v = res.headers.get(k);
|
|
75
|
+
if (v !== null)
|
|
76
|
+
headers[k] = v;
|
|
77
|
+
}
|
|
78
|
+
if (r.binary === true) {
|
|
79
|
+
return ok({
|
|
80
|
+
method: r.method,
|
|
81
|
+
url: r.url,
|
|
82
|
+
status: res.status,
|
|
83
|
+
headers,
|
|
84
|
+
body: "",
|
|
85
|
+
bytes: new Uint8Array(await res.arrayBuffer()),
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
return ok({ method: r.method, url: r.url, status: res.status, headers, body: await res.text() });
|
|
89
|
+
}
|
|
90
|
+
catch (e) {
|
|
91
|
+
return err({
|
|
92
|
+
kind: "network",
|
|
93
|
+
message: e instanceof Error ? `${e.message}${e.cause instanceof Error ? `: ${e.cause.message}` : ""}` : String(e),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
}, (x) => (x.kind === "ok" ? x.value.status : undefined));
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
/** A transport that answers from recorded exchanges, by method and URL, and refuses a request it has none for, naming it. */
|
|
101
|
+
export function recordedTransport(exchanges) {
|
|
102
|
+
return {
|
|
103
|
+
kind: "recorded",
|
|
104
|
+
send(r) {
|
|
105
|
+
const x = exchanges.find((e) => e.method === r.method && e.url === r.url);
|
|
106
|
+
return Promise.resolve(x === undefined ? err({ kind: "refused", message: `no recorded exchange for ${r.method} ${r.url}` }) : ok(x));
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* A transport that sends only safe methods (GET, HEAD, ...), POST to URLs that start with one of `tokenEndpoints`, and the POSTs that
|
|
112
|
+
* `readPost` says only read (FR-066). Every other request (PUT, PATCH, DELETE, any other POST) is refused without being sent.
|
|
113
|
+
*/
|
|
114
|
+
export function readOnly(t, tokenEndpoints = [], readPost) {
|
|
115
|
+
return {
|
|
116
|
+
kind: t.kind,
|
|
117
|
+
send(r) {
|
|
118
|
+
const post = r.method === "POST" && (tokenEndpoints.some((p) => r.url.startsWith(p)) || readPost?.(r) === true);
|
|
119
|
+
if (isSafeMethod(r.method) || post)
|
|
120
|
+
return t.send(r);
|
|
121
|
+
return Promise.resolve(err({ kind: "refused", message: `read-only: ${r.method} ${new URL(r.url).origin} is refused` }));
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* A transport that signs each request by AWS Signature Version 4 (FR-069, by the kernel's `sigv4`, FR-068) when it is sent. A signed
|
|
127
|
+
* request carries a credential, so the live transport under it must be made with `credentials: "allow"`, or it refuses the request.
|
|
128
|
+
*/
|
|
129
|
+
export function withSigV4(t, options) {
|
|
130
|
+
return {
|
|
131
|
+
kind: t.kind,
|
|
132
|
+
async send(r) {
|
|
133
|
+
const signed = await sign(r, {
|
|
134
|
+
credentials: options.credentials,
|
|
135
|
+
region: options.region,
|
|
136
|
+
service: options.service,
|
|
137
|
+
date: (options.now ?? (() => new Date()))(),
|
|
138
|
+
...(options.unsignedPayload === undefined ? {} : { unsignedPayload: options.unsignedPayload }),
|
|
139
|
+
});
|
|
140
|
+
return await t.send({ ...r, headers: signed.headers });
|
|
141
|
+
},
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
/** A transport that waits and tries again on 429 and 503: as long as `Retry-After` asks (up to the cap), else 1, 2, 4 seconds. */
|
|
145
|
+
export function withBackoff(t, options = {}) {
|
|
146
|
+
const pause = options.pause ?? ((ms) => sleep(ms));
|
|
147
|
+
const cap = options.capMs ?? 60_000;
|
|
148
|
+
return {
|
|
149
|
+
kind: t.kind,
|
|
150
|
+
async send(r) {
|
|
151
|
+
for (let n = 0;; n++) {
|
|
152
|
+
const x = await t.send(r);
|
|
153
|
+
if (x.kind === "err" || (x.value.status !== 429 && x.value.status !== 503) || n >= (options.retries ?? 3))
|
|
154
|
+
return x;
|
|
155
|
+
const asked = retryAfterMs(x.value.headers["retry-after"], Date.now());
|
|
156
|
+
await pause(Math.min(cap, asked ?? 1000 * 2 ** n));
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/** A read error as one sentence; never a header or a body, so no credential or personal datum is said. */
|
|
162
|
+
export function formatReadError(e) {
|
|
163
|
+
switch (e.kind) {
|
|
164
|
+
case "refused":
|
|
165
|
+
case "network":
|
|
166
|
+
return e.message;
|
|
167
|
+
case "denied":
|
|
168
|
+
return `${new URL(e.url).pathname}: ${e.status} (the credential may not read this)`;
|
|
169
|
+
case "status":
|
|
170
|
+
return `${new URL(e.url).pathname}: HTTP ${e.status}`;
|
|
171
|
+
case "decode":
|
|
172
|
+
return `${new URL(e.url).pathname}: ${e.message}`;
|
|
173
|
+
default:
|
|
174
|
+
return absurd(e);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/** A GET whose JSON body is decoded, with the exchange; 401 and 403 are `denied`, any other status from 400 up is `status`. */
|
|
178
|
+
export async function getExchange(t, url, decoder, headers = {}) {
|
|
179
|
+
const x = await t.send({ method: "GET", url, headers: { accept: "application/json", ...headers } });
|
|
180
|
+
if (x.kind === "err")
|
|
181
|
+
return x;
|
|
182
|
+
const { status } = x.value;
|
|
183
|
+
if (status === 401 || status === 403)
|
|
184
|
+
return err({ kind: "denied", status, url });
|
|
185
|
+
if (status >= 400)
|
|
186
|
+
return err({ kind: "status", status, url });
|
|
187
|
+
const v = json(x.value.body, decoder);
|
|
188
|
+
if (v.kind === "err")
|
|
189
|
+
return err({ kind: "decode", url, message: formatDecodeError(v.error) });
|
|
190
|
+
return ok({ value: v.value, exchange: x.value });
|
|
191
|
+
}
|
|
192
|
+
/** A GET whose JSON body is decoded. */
|
|
193
|
+
export async function getJson(t, url, decoder, headers = {}) {
|
|
194
|
+
const r = await getExchange(t, url, decoder, headers);
|
|
195
|
+
return r.kind === "ok" ? ok(r.value.value) : r;
|
|
196
|
+
}
|
|
197
|
+
/** Every page of a list that pages by `Link: <...>; rel="next"`, at most `maxPages` (200 by default); the first failure ends it, as an error. */
|
|
198
|
+
export async function* linkPages(t, url, item, options = {}) {
|
|
199
|
+
const page = array(item);
|
|
200
|
+
let next = url;
|
|
201
|
+
let pages = 0;
|
|
202
|
+
while (next !== undefined && pages < (options.maxPages ?? 200)) {
|
|
203
|
+
const r = await getExchange(t, next, page, options.headers);
|
|
204
|
+
if (r.kind === "err")
|
|
205
|
+
return r;
|
|
206
|
+
yield* r.value.value;
|
|
207
|
+
pages++;
|
|
208
|
+
next = nextLink(r.value.exchange.headers["link"], "next", next);
|
|
209
|
+
}
|
|
210
|
+
return ok(pages);
|
|
211
|
+
}
|
|
212
|
+
/** Every item of a list that pages by a cursor in its body: `items` takes a page's items, `next` the next page's URL or undefined. */
|
|
213
|
+
export async function* cursorPages(t, first, page, items, next, options = {}) {
|
|
214
|
+
let url = first;
|
|
215
|
+
let pages = 0;
|
|
216
|
+
while (url !== undefined && pages < (options.maxPages ?? 200)) {
|
|
217
|
+
const r = await getJson(t, url, page, options.headers);
|
|
218
|
+
if (r.kind === "err")
|
|
219
|
+
return r;
|
|
220
|
+
yield* items(r.value);
|
|
221
|
+
pages++;
|
|
222
|
+
url = next(r.value, url);
|
|
223
|
+
}
|
|
224
|
+
return ok(pages);
|
|
225
|
+
}
|
|
226
|
+
const exchangeDecoder = object({
|
|
227
|
+
method: literal("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"),
|
|
228
|
+
url: string,
|
|
229
|
+
status: number,
|
|
230
|
+
headers: record(string),
|
|
231
|
+
body: string,
|
|
232
|
+
});
|
|
233
|
+
/** A recorded exchange read from the JSON5 text `recordedText` writes, or why it is not one. */
|
|
234
|
+
export function decodeRecorded(text) {
|
|
235
|
+
const v = parseJson5(text);
|
|
236
|
+
if (v.kind === "err")
|
|
237
|
+
return err(`not JSON5: ${formatJson5Error(v.error)}`);
|
|
238
|
+
const x = exchangeDecoder(v.value);
|
|
239
|
+
return x.kind === "ok" ? ok(x.value) : err(formatDecodeError(x.error));
|
|
240
|
+
}
|
|
241
|
+
/** The text a recorded exchange is kept in: a comment that says where and when it was recorded, then the exchange as JSON5. */
|
|
242
|
+
export function recordedText(x, note) {
|
|
243
|
+
return toJson5({ method: x.method, url: x.url, status: x.status, headers: x.headers, body: x.body }, note);
|
|
244
|
+
}
|
|
245
|
+
/** Headers that give a bearer token (for a transport made to allow credentials). */
|
|
246
|
+
export function bearer(token, scheme = "Bearer") {
|
|
247
|
+
return { authorization: `${scheme} ${token}` };
|
|
248
|
+
}
|
package/dist/mod.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Net, the network of Intellectual Frontiers Commons as one typed library (its specification 0001-net): downloads that retry and resume (as `net`), an HTTP server over web `Request` and `Response` (as `serve`),
|
|
3
|
+
* typed HTTP clients with a live or recorded transport (as `http`), TCP and TLS (as `tcp`), DNS records (as `dns`) and ACME certificates by DNS-01 (as `acme`). It names `node:` builtins and no vendor's global,
|
|
4
|
+
* depends on core, formats, crypto, system and telemetry, and runs the same on Bun and on Node 26.
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* ```ts
|
|
8
|
+
* import { serve } from "@intellectual-frontiers/net";
|
|
9
|
+
* const server = await serve({ port: 0 }, () => new Response("ok"));
|
|
10
|
+
* await server.shutdown();
|
|
11
|
+
* ```
|
|
12
|
+
*
|
|
13
|
+
* @module
|
|
14
|
+
*/
|
|
15
|
+
export * as acme from "./acme.js";
|
|
16
|
+
export * as dns from "./dns.js";
|
|
17
|
+
export * as http from "./http-client.js";
|
|
18
|
+
export * as net from "./net.js";
|
|
19
|
+
export * as tcp from "./tcp.js";
|
|
20
|
+
export { serve } from "./serve.js";
|
|
21
|
+
export type { NetAddr, Server, ServeHandler, ServeInfo, ServeOptions } from "./serve.js";
|
package/dist/mod.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Spec: 0001-net FR-017
|
|
2
|
+
/**
|
|
3
|
+
* Net, the network of Intellectual Frontiers Commons as one typed library (its specification 0001-net): downloads that retry and resume (as `net`), an HTTP server over web `Request` and `Response` (as `serve`),
|
|
4
|
+
* typed HTTP clients with a live or recorded transport (as `http`), TCP and TLS (as `tcp`), DNS records (as `dns`) and ACME certificates by DNS-01 (as `acme`). It names `node:` builtins and no vendor's global,
|
|
5
|
+
* depends on core, formats, crypto, system and telemetry, and runs the same on Bun and on Node 26.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```ts
|
|
9
|
+
* import { serve } from "@intellectual-frontiers/net";
|
|
10
|
+
* const server = await serve({ port: 0 }, () => new Response("ok"));
|
|
11
|
+
* await server.shutdown();
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* @module
|
|
15
|
+
*/
|
|
16
|
+
export * as acme from "./acme.js";
|
|
17
|
+
export * as dns from "./dns.js";
|
|
18
|
+
export * as http from "./http-client.js";
|
|
19
|
+
export * as net from "./net.js";
|
|
20
|
+
export * as tcp from "./tcp.js";
|
|
21
|
+
export { serve } from "./serve.js";
|
package/dist/net.d.ts
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { type Result } from "@intellectual-frontiers/core";
|
|
2
|
+
import type { Sha256 } from "@intellectual-frontiers/crypto/sha256";
|
|
3
|
+
/** Why a request failed, as far as the network is concerned. */
|
|
4
|
+
export type NetFailure = {
|
|
5
|
+
readonly kind: "dns";
|
|
6
|
+
} | {
|
|
7
|
+
readonly kind: "connect";
|
|
8
|
+
} | {
|
|
9
|
+
readonly kind: "broken";
|
|
10
|
+
} | {
|
|
11
|
+
readonly kind: "timeout";
|
|
12
|
+
} | {
|
|
13
|
+
readonly kind: "tls";
|
|
14
|
+
} | {
|
|
15
|
+
readonly kind: "certificate";
|
|
16
|
+
} | {
|
|
17
|
+
readonly kind: "http";
|
|
18
|
+
readonly status: number;
|
|
19
|
+
readonly retryAfterMs?: number;
|
|
20
|
+
} | {
|
|
21
|
+
readonly kind: "other";
|
|
22
|
+
readonly message: string;
|
|
23
|
+
};
|
|
24
|
+
/** A failure as a person reads it. */
|
|
25
|
+
export declare function describeFailure(f: NetFailure): string;
|
|
26
|
+
/** Whether another try may help: the name, the connection, a busy server; never a refusal or a certificate. */
|
|
27
|
+
export declare function retryable(f: NetFailure): boolean;
|
|
28
|
+
/** What an error of `fetch` says, as a failure of the network: by its code (or its cause's) where it has one, else by its words. */
|
|
29
|
+
export declare function classify(e: unknown): NetFailure;
|
|
30
|
+
/** How often and how patiently to try: at most `tries` attempts, waiting `base` seconds after the first failure, doubled each time up to `cap`. */
|
|
31
|
+
export type RetryPolicy = {
|
|
32
|
+
readonly tries: number;
|
|
33
|
+
readonly base: number;
|
|
34
|
+
readonly cap: number;
|
|
35
|
+
};
|
|
36
|
+
/** One retry: what was retried, why, and which attempt comes next. */
|
|
37
|
+
export type RetryNote = {
|
|
38
|
+
readonly what: string;
|
|
39
|
+
readonly why: string;
|
|
40
|
+
readonly attempt: number;
|
|
41
|
+
};
|
|
42
|
+
/** The retries of a run: how long to wait, each retry said through `report` and kept for a summary. */
|
|
43
|
+
export declare class Retries {
|
|
44
|
+
/** The retries so far, in order. */
|
|
45
|
+
readonly notes: RetryNote[];
|
|
46
|
+
/** The policy. */
|
|
47
|
+
readonly policy: RetryPolicy;
|
|
48
|
+
/** Where each retry is said. */
|
|
49
|
+
readonly report: (line: string) => void;
|
|
50
|
+
/** Seconds added to each wait so that many computers do not try together (0 to 2 by default). */
|
|
51
|
+
readonly jitter: () => number;
|
|
52
|
+
/** How a wait is spent (a timer by default; a test passes one that returns at once). */
|
|
53
|
+
readonly pause: (seconds: number) => Promise<void>;
|
|
54
|
+
/** Retries under a policy, said through `report`. */
|
|
55
|
+
constructor(policy: RetryPolicy, report: (line: string) => void, jitter?: () => number, pause?: (seconds: number) => Promise<void>);
|
|
56
|
+
/** The seconds to wait after attempt `attempt` failed: the base doubled each time, no more than the cap, plus the jitter. */
|
|
57
|
+
wait(attempt: number): number;
|
|
58
|
+
/** Say that an attempt failed and when the next one is, keep it, and wait; a server's `Retry-After` (in seconds, up to the cap) wins. */
|
|
59
|
+
again(what: string, why: string, attempt: number, retryAfterSeconds?: number): Promise<void>;
|
|
60
|
+
/** What the network cost, most retried first: `github.com (3), example.org (1)`. */
|
|
61
|
+
top(n?: number): string;
|
|
62
|
+
}
|
|
63
|
+
/** Limits of one try. */
|
|
64
|
+
export type FetchOptions = {
|
|
65
|
+
/** No answer within this many seconds fails the try (20 by default). */
|
|
66
|
+
readonly answerSeconds?: number;
|
|
67
|
+
/** Less than 1 KB a second over this many seconds fails the try (60 by default). */
|
|
68
|
+
readonly stallSeconds?: number;
|
|
69
|
+
/** Headers to send. */
|
|
70
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
71
|
+
};
|
|
72
|
+
/** One try: the file at `url` written to `dest`; a `dest` ending `.part` that already has bytes is resumed where the server can. */
|
|
73
|
+
export declare function fetchOnce(url: string, dest: string, o?: FetchOptions): Promise<Result<void, NetFailure>>;
|
|
74
|
+
/** The host a URL names, for saying what was retried. */
|
|
75
|
+
export declare function hostOf(url: string): string;
|
|
76
|
+
/** The file at `url` written to `dest`, tried again where the network failed and another try can help; or why it was not. */
|
|
77
|
+
export declare function fetchTo(retries: Retries, url: string, dest: string, o?: FetchOptions): Promise<Result<void, string>>;
|
|
78
|
+
/**
|
|
79
|
+
* The file in `cache/downloads/<sha256>`, fetched where it is not there, and kept only where its SHA-256 is `sum`. A download that stops
|
|
80
|
+
* part way is kept as `.part` and resumed by the next try or the next run; a file that is not the one expected is removed, never kept.
|
|
81
|
+
*/
|
|
82
|
+
export declare function downloadVerified(retries: Retries, url: string, sum: Sha256, cache: string, o?: FetchOptions): Promise<Result<string, string>>;
|
|
83
|
+
/** Whether `host` answers over HTTPS (any answer counts, even a refusal), tried again where it does not, at most 5 times. */
|
|
84
|
+
export declare function reach(retries: Retries, host: string, scheme?: "https" | "http"): Promise<Result<void, NetFailure>>;
|
package/dist/net.js
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
// Spec: 0001-net FR-001
|
|
2
|
+
/**
|
|
3
|
+
* Downloads over a network that fails now and works in a minute (commons 0001-net FR-001), on the web standard `fetch` and
|
|
4
|
+
* `node:fs`. Taken from the machine setup's `net.ts`: a failure is classified (the name, the connection, a busy server, a certificate),
|
|
5
|
+
* retried only where another try can help, with a longer wait each time (or as long as the server's `Retry-After` asks, up to the cap) and every retry said; a download that stops part way is resumed
|
|
6
|
+
* with `Range`; and a file is kept only where its SHA-256 is the one expected. A download that fails is an expected outcome, so these
|
|
7
|
+
* return a `Result`.
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* ```ts
|
|
11
|
+
* import { downloadVerified, Retries } from "./net.ts";
|
|
12
|
+
* import { sha256Of } from "@intellectual-frontiers/crypto/sha256";
|
|
13
|
+
* const sum = sha256Of("…64 hex digits…");
|
|
14
|
+
* if (sum !== undefined) {
|
|
15
|
+
* const r = await downloadVerified(new Retries({ tries: 5, base: 2, cap: 30 }, console.error), "https://example.com/tool.tgz", sum, "/tmp/cache");
|
|
16
|
+
* if (r.kind === "ok") console.log("kept at", r.value);
|
|
17
|
+
* }
|
|
18
|
+
* ```
|
|
19
|
+
*
|
|
20
|
+
* @module
|
|
21
|
+
*/
|
|
22
|
+
import * as fsp from "node:fs/promises";
|
|
23
|
+
import { join } from "node:path";
|
|
24
|
+
import { absurd, err, ok } from "@intellectual-frontiers/core";
|
|
25
|
+
import { retryAfterMs } from "@intellectual-frontiers/formats/http";
|
|
26
|
+
import { exists, mkdir, remove, rename, sha256File, stat } from "@intellectual-frontiers/system/fs";
|
|
27
|
+
const busy = (status) => status === 408 || status === 425 || status === 429 || (status >= 500 && status < 600);
|
|
28
|
+
/** A failure as a person reads it. */
|
|
29
|
+
export function describeFailure(f) {
|
|
30
|
+
switch (f.kind) {
|
|
31
|
+
case "dns":
|
|
32
|
+
return "the name could not be looked up (DNS)";
|
|
33
|
+
case "connect":
|
|
34
|
+
return "no connection could be made";
|
|
35
|
+
case "broken":
|
|
36
|
+
return "the connection broke or the download stopped early";
|
|
37
|
+
case "timeout":
|
|
38
|
+
return "it took too long";
|
|
39
|
+
case "tls":
|
|
40
|
+
return "the secure connection could not be made";
|
|
41
|
+
case "certificate":
|
|
42
|
+
return "the certificate was not accepted (a proxy, or the computer's clock?)";
|
|
43
|
+
case "http":
|
|
44
|
+
return f.status === 416
|
|
45
|
+
? "the part already fetched did not fit, so it starts again"
|
|
46
|
+
: busy(f.status)
|
|
47
|
+
? `the server answered ${f.status} (busy or not ready)`
|
|
48
|
+
: `the server answered ${f.status}`;
|
|
49
|
+
case "other":
|
|
50
|
+
return f.message;
|
|
51
|
+
default:
|
|
52
|
+
return absurd(f);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** Whether another try may help: the name, the connection, a busy server; never a refusal or a certificate. */
|
|
56
|
+
export function retryable(f) {
|
|
57
|
+
switch (f.kind) {
|
|
58
|
+
case "dns":
|
|
59
|
+
case "connect":
|
|
60
|
+
case "broken":
|
|
61
|
+
case "timeout":
|
|
62
|
+
case "tls":
|
|
63
|
+
return true;
|
|
64
|
+
case "certificate":
|
|
65
|
+
case "other":
|
|
66
|
+
return false;
|
|
67
|
+
case "http":
|
|
68
|
+
return busy(f.status) || f.status === 416;
|
|
69
|
+
default:
|
|
70
|
+
return absurd(f);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const CODES = {
|
|
74
|
+
ENOTFOUND: "dns",
|
|
75
|
+
EAI_AGAIN: "dns",
|
|
76
|
+
ECONNREFUSED: "connect",
|
|
77
|
+
ENETUNREACH: "connect",
|
|
78
|
+
EHOSTUNREACH: "connect",
|
|
79
|
+
UND_ERR_CONNECT_TIMEOUT: "timeout",
|
|
80
|
+
UND_ERR_HEADERS_TIMEOUT: "timeout",
|
|
81
|
+
UND_ERR_BODY_TIMEOUT: "timeout",
|
|
82
|
+
ETIMEDOUT: "timeout",
|
|
83
|
+
ECONNRESET: "broken",
|
|
84
|
+
EPIPE: "broken",
|
|
85
|
+
UND_ERR_SOCKET: "broken",
|
|
86
|
+
ERR_TLS_CERT_ALTNAME_INVALID: "certificate",
|
|
87
|
+
CERT_HAS_EXPIRED: "certificate",
|
|
88
|
+
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "certificate",
|
|
89
|
+
UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "certificate",
|
|
90
|
+
DEPTH_ZERO_SELF_SIGNED_CERT: "certificate",
|
|
91
|
+
SELF_SIGNED_CERT_IN_CHAIN: "certificate",
|
|
92
|
+
};
|
|
93
|
+
const codeOf = (e) => {
|
|
94
|
+
if (typeof e !== "object" || e === null)
|
|
95
|
+
return undefined;
|
|
96
|
+
const c = Reflect.get(e, "code");
|
|
97
|
+
return typeof c === "string" ? c : undefined;
|
|
98
|
+
};
|
|
99
|
+
/** What an error of `fetch` says, as a failure of the network: by its code (or its cause's) where it has one, else by its words. */
|
|
100
|
+
export function classify(e) {
|
|
101
|
+
if (e instanceof DOMException && (e.name === "TimeoutError" || e.name === "AbortError"))
|
|
102
|
+
return { kind: "timeout" };
|
|
103
|
+
const cause = e instanceof Error ? e.cause : undefined;
|
|
104
|
+
const code = codeOf(e) ?? codeOf(cause);
|
|
105
|
+
const byCode = code === undefined ? undefined : CODES[code];
|
|
106
|
+
if (byCode !== undefined && byCode !== "http" && byCode !== "other")
|
|
107
|
+
return { kind: byCode };
|
|
108
|
+
const m = e instanceof Error ? `${e.message} ${cause instanceof Error ? cause.message : ""}` : String(e);
|
|
109
|
+
if (/certificate|UnknownIssuer|invalid peer|self[ -]signed|CertNotValid/i.test(m))
|
|
110
|
+
return { kind: "certificate" };
|
|
111
|
+
if (/dns error|failed to lookup|name or service not known|nodename nor servname|no such host|temporary failure in name resolution|ENOTFOUND/i.test(m)) {
|
|
112
|
+
return { kind: "dns" };
|
|
113
|
+
}
|
|
114
|
+
if (/connection refused|ConnectionRefused|network is unreachable|no route to host|connect error|ECONNREFUSED/i.test(m))
|
|
115
|
+
return { kind: "connect" };
|
|
116
|
+
if (/timed? ?out|deadline/i.test(m))
|
|
117
|
+
return { kind: "timeout" };
|
|
118
|
+
if (/tls|ssl|handshake/i.test(m))
|
|
119
|
+
return { kind: "tls" };
|
|
120
|
+
if (/reset|broken pipe|unexpected eof|connection closed|IncompleteMessage|incomplete|error reading a body|body from connection|terminated|other side closed/i.test(m)) {
|
|
121
|
+
return { kind: "broken" };
|
|
122
|
+
}
|
|
123
|
+
return { kind: "other", message: m.trim().slice(0, 200) };
|
|
124
|
+
}
|
|
125
|
+
/** The retries of a run: how long to wait, each retry said through `report` and kept for a summary. */
|
|
126
|
+
export class Retries {
|
|
127
|
+
/** The retries so far, in order. */
|
|
128
|
+
notes = [];
|
|
129
|
+
/** The policy. */
|
|
130
|
+
policy;
|
|
131
|
+
/** Where each retry is said. */
|
|
132
|
+
report;
|
|
133
|
+
/** Seconds added to each wait so that many computers do not try together (0 to 2 by default). */
|
|
134
|
+
jitter;
|
|
135
|
+
/** How a wait is spent (a timer by default; a test passes one that returns at once). */
|
|
136
|
+
pause;
|
|
137
|
+
/** Retries under a policy, said through `report`. */
|
|
138
|
+
constructor(policy, report, jitter = () => Math.floor(Math.random() * 3), pause = (s) => new Promise((r) => setTimeout(r, s * 1000))) {
|
|
139
|
+
this.policy = policy;
|
|
140
|
+
this.report = report;
|
|
141
|
+
this.jitter = jitter;
|
|
142
|
+
this.pause = pause;
|
|
143
|
+
}
|
|
144
|
+
/** The seconds to wait after attempt `attempt` failed: the base doubled each time, no more than the cap, plus the jitter. */
|
|
145
|
+
wait(attempt) {
|
|
146
|
+
let d = this.policy.base;
|
|
147
|
+
for (let i = 1; i < attempt && d < this.policy.cap; i++)
|
|
148
|
+
d *= 2;
|
|
149
|
+
return Math.min(d, this.policy.cap) + this.jitter();
|
|
150
|
+
}
|
|
151
|
+
/** Say that an attempt failed and when the next one is, keep it, and wait; a server's `Retry-After` (in seconds, up to the cap) wins. */
|
|
152
|
+
async again(what, why, attempt, retryAfterSeconds) {
|
|
153
|
+
const w = retryAfterSeconds === undefined ? this.wait(attempt) : Math.min(Math.ceil(retryAfterSeconds), this.policy.cap);
|
|
154
|
+
this.notes.push({ what, why, attempt: attempt + 1 });
|
|
155
|
+
this.report(`${what}: ${why}; trying again in ${w}s (attempt ${attempt + 1} of ${this.policy.tries})`);
|
|
156
|
+
await this.pause(w);
|
|
157
|
+
}
|
|
158
|
+
/** What the network cost, most retried first: `github.com (3), example.org (1)`. */
|
|
159
|
+
top(n = 4) {
|
|
160
|
+
const by = new Map();
|
|
161
|
+
for (const x of this.notes)
|
|
162
|
+
by.set(x.what, (by.get(x.what) ?? 0) + 1);
|
|
163
|
+
return [...by.entries()]
|
|
164
|
+
.sort((a, b) => b[1] - a[1])
|
|
165
|
+
.slice(0, n)
|
|
166
|
+
.map(([w, c]) => `${w} (${c})`)
|
|
167
|
+
.join(", ");
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/** One try: the file at `url` written to `dest`; a `dest` ending `.part` that already has bytes is resumed where the server can. */
|
|
171
|
+
export async function fetchOnce(url, dest, o = {}) {
|
|
172
|
+
const ctl = new AbortController();
|
|
173
|
+
const answer = setTimeout(() => ctl.abort(new DOMException("no answer", "TimeoutError")), (o.answerSeconds ?? 20) * 1000);
|
|
174
|
+
const have = dest.endsWith(".part") && (await exists(dest)) ? (await stat(dest)).size : 0;
|
|
175
|
+
let res;
|
|
176
|
+
try {
|
|
177
|
+
const range = have > 0 && !url.startsWith("file:") ? { Range: `bytes=${have}-` } : {};
|
|
178
|
+
res = await fetch(url, { signal: ctl.signal, headers: { ...o.headers, ...range } });
|
|
179
|
+
}
|
|
180
|
+
catch (e) {
|
|
181
|
+
return err(classify(e));
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
clearTimeout(answer);
|
|
185
|
+
}
|
|
186
|
+
if (!res.ok) {
|
|
187
|
+
await res.body?.cancel();
|
|
188
|
+
const after = busy(res.status) ? retryAfterMs(res.headers.get("retry-after"), Date.now()) : undefined;
|
|
189
|
+
return err({ kind: "http", status: res.status, ...(after === undefined ? {} : { retryAfterMs: after }) });
|
|
190
|
+
}
|
|
191
|
+
const append = res.status === 206;
|
|
192
|
+
const stall = (o.stallSeconds ?? 60) * 1000;
|
|
193
|
+
let window = 0;
|
|
194
|
+
let windowStart = Date.now();
|
|
195
|
+
const watch = setInterval(() => {
|
|
196
|
+
if (Date.now() - windowStart < stall)
|
|
197
|
+
return;
|
|
198
|
+
if (window < (stall / 1000) * 1024)
|
|
199
|
+
ctl.abort(new DOMException("stalled", "TimeoutError"));
|
|
200
|
+
window = 0;
|
|
201
|
+
windowStart = Date.now();
|
|
202
|
+
}, Math.min(5000, stall));
|
|
203
|
+
let file;
|
|
204
|
+
try {
|
|
205
|
+
file = await fsp.open(dest, append ? "a" : "w");
|
|
206
|
+
if (res.body !== null) {
|
|
207
|
+
const reader = res.body.getReader();
|
|
208
|
+
for (;;) {
|
|
209
|
+
const { done, value } = await reader.read();
|
|
210
|
+
if (done)
|
|
211
|
+
break;
|
|
212
|
+
window += value.length;
|
|
213
|
+
await file.write(value);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return ok(undefined);
|
|
217
|
+
}
|
|
218
|
+
catch (e) {
|
|
219
|
+
return err(classify(e));
|
|
220
|
+
}
|
|
221
|
+
finally {
|
|
222
|
+
clearInterval(watch);
|
|
223
|
+
await file?.close();
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
/** The host a URL names, for saying what was retried. */
|
|
227
|
+
export function hostOf(url) {
|
|
228
|
+
try {
|
|
229
|
+
return new URL(url).host || url;
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
return url;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
/** The file at `url` written to `dest`, tried again where the network failed and another try can help; or why it was not. */
|
|
236
|
+
export async function fetchTo(retries, url, dest, o = {}) {
|
|
237
|
+
const name = hostOf(url);
|
|
238
|
+
for (let n = 1;; n++) {
|
|
239
|
+
const r = await fetchOnce(url, dest, o);
|
|
240
|
+
if (r.kind === "ok")
|
|
241
|
+
return r;
|
|
242
|
+
const f = r.error;
|
|
243
|
+
if (!retryable(f) || n >= retries.policy.tries)
|
|
244
|
+
return err(`${name}: ${describeFailure(f)} (after ${n} of ${retries.policy.tries} attempts)`);
|
|
245
|
+
if (f.kind === "http" && f.status === 416)
|
|
246
|
+
await remove(dest).catch(() => { });
|
|
247
|
+
await retries.again(name, describeFailure(f), n, f.kind === "http" && f.retryAfterMs !== undefined ? f.retryAfterMs / 1000 : undefined);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* The file in `cache/downloads/<sha256>`, fetched where it is not there, and kept only where its SHA-256 is `sum`. A download that stops
|
|
252
|
+
* part way is kept as `.part` and resumed by the next try or the next run; a file that is not the one expected is removed, never kept.
|
|
253
|
+
*/
|
|
254
|
+
export async function downloadVerified(retries, url, sum, cache, o = {}) {
|
|
255
|
+
const dir = join(cache, "downloads");
|
|
256
|
+
await mkdir(dir, { recursive: true });
|
|
257
|
+
const f = join(dir, sum);
|
|
258
|
+
if ((await exists(f)) && (await sha256File(f)) === sum)
|
|
259
|
+
return ok(f);
|
|
260
|
+
const part = `${f}.part`;
|
|
261
|
+
const got = await fetchTo(retries, url, part, o);
|
|
262
|
+
if (got.kind === "err")
|
|
263
|
+
return err(`could not fetch ${url}: ${got.error}`);
|
|
264
|
+
const actual = await sha256File(part);
|
|
265
|
+
if (actual !== sum) {
|
|
266
|
+
await remove(part).catch(() => { });
|
|
267
|
+
return err(`the file at ${url} is not the one expected (its SHA-256 is ${actual}, expected ${sum}); nothing was kept`);
|
|
268
|
+
}
|
|
269
|
+
await rename(part, f);
|
|
270
|
+
return ok(f);
|
|
271
|
+
}
|
|
272
|
+
/** Whether `host` answers over HTTPS (any answer counts, even a refusal), tried again where it does not, at most 5 times. */
|
|
273
|
+
export async function reach(retries, host, scheme = "https") {
|
|
274
|
+
const max = Math.min(5, retries.policy.tries);
|
|
275
|
+
for (let n = 1;; n++) {
|
|
276
|
+
try {
|
|
277
|
+
const res = await fetch(`${scheme}://${host}/`, { method: "HEAD", signal: AbortSignal.timeout(20_000) });
|
|
278
|
+
await res.body?.cancel();
|
|
279
|
+
return ok(undefined);
|
|
280
|
+
}
|
|
281
|
+
catch (e) {
|
|
282
|
+
const f = classify(e);
|
|
283
|
+
if (n >= max)
|
|
284
|
+
return err(f);
|
|
285
|
+
await retries.again(host, describeFailure(f), n);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|