@intellectual-frontiers/net 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,369 @@
1
+ // Spec: 0001-net FR-003
2
+ /**
3
+ * How a typed client reaches a system over HTTP (commons 0001-net FR-003): a transport, live (`fetch`) or recorded (the
4
+ * exchanges a test keeps, so it reaches no host), and over it the guards and helpers every client otherwise rewrites: a read-only guard
5
+ * (safe methods only, POST to the token endpoints named, and the POSTs a predicate says only read, FR-066), AWS Signature Version 4 on each
6
+ * request (FR-069), backoff on 429 and 503 that honours `Retry-After`, a JSON read decoded at the boundary with typed failures (a denied
7
+ * call is told apart, so a caller can record a gap instead of failing), and every item of a list that pages by `Link: rel="next"` or by a
8
+ * cursor in the body. A request that carries a credential is refused before it is sent unless the transport is made to allow it, so a
9
+ * harness whose proxy injects credentials never sends one by mistake. Taken from erga's `lib/ts/http-client` and Stolos's connector SDK
10
+ * (`src/archon/connect/sdk.ts`, `connector.ts`).
11
+ *
12
+ * @example
13
+ * ```ts
14
+ * import { array, object, string } from "@intellectual-frontiers/core";
15
+ * import { getJson, liveTransport, withBackoff } from "./http-client.ts";
16
+ * const http = withBackoff(liveTransport());
17
+ * const repo = await getJson(http, "https://api.github.com/repos/oven-sh/bun", object({ full_name: string }));
18
+ * ```
19
+ *
20
+ * @module
21
+ */
22
+ import {
23
+ absurd,
24
+ array,
25
+ type Decoder,
26
+ formatDecodeError,
27
+ json,
28
+ literal,
29
+ number,
30
+ object,
31
+ record,
32
+ string,
33
+ err,
34
+ ok,
35
+ type Result,
36
+ } from "@intellectual-frontiers/core";
37
+ import { isSafeMethod, nextLink, retryAfterMs } from "@intellectual-frontiers/formats/http";
38
+ import { formatJson5Error, parseJson5, toJson5 } from "@intellectual-frontiers/formats/json5";
39
+ import { type AwsCredentials, sign } from "@intellectual-frontiers/crypto/sigv4";
40
+ import { traceHttpClient } from "@intellectual-frontiers/telemetry/host/otel";
41
+ import { sleep } from "@intellectual-frontiers/system/proc";
42
+
43
+ /** A request a client makes. */
44
+ export type OutRequest = {
45
+ readonly method: "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE";
46
+ readonly url: string;
47
+ readonly headers: Readonly<Record<string, string>>;
48
+ readonly body?: string;
49
+ /** Read the answer as bytes (`Exchange.bytes`), for an artifact that is not text (a tarball, a zip); the text `body` of the answer is then empty. */
50
+ readonly binary?: boolean;
51
+ };
52
+
53
+ /** One exchange: a request and what answered it. */
54
+ export type Exchange = {
55
+ readonly method: OutRequest["method"];
56
+ readonly url: string;
57
+ readonly status: number;
58
+ /** The headers kept: the content type, paging, rate limits and `Retry-After`. */
59
+ readonly headers: Readonly<Record<string, string>>;
60
+ readonly body: string;
61
+ /** The answer's bytes: present where the request asked for `binary` (a recorded exchange of an artifact holds them). */
62
+ readonly bytes?: Uint8Array;
63
+ };
64
+
65
+ /** Why a request got no answer at all (a refusal before it was sent, or the network). */
66
+ export type TransportError =
67
+ { readonly kind: "refused"; readonly message: string } | { readonly kind: "network"; readonly message: string };
68
+
69
+ /** A way to reach a system. */
70
+ export type Transport = { readonly kind: "live" | "recorded"; readonly send: (r: OutRequest) => Promise<Result<Exchange, TransportError>> };
71
+
72
+ /** The headers a credential travels in. */
73
+ export const CREDENTIAL_HEADERS: readonly string[] = [
74
+ "authorization",
75
+ "proxy-authorization",
76
+ "cookie",
77
+ "x-api-key",
78
+ "x-goog-api-key",
79
+ "api-key",
80
+ "private-token",
81
+ "job-token",
82
+ ];
83
+
84
+ /** Why a request may not be sent where credentials are refused: it carries one in a header or in its query. Undefined where it carries none. */
85
+ export function carriesCredential(r: OutRequest): string | undefined {
86
+ const named = Object.keys(r.headers)
87
+ .map((h) => h.toLowerCase())
88
+ .find((h) => CREDENTIAL_HEADERS.includes(h));
89
+ if (named !== undefined)
90
+ return `${r.method} ${r.url.split("?")[0] ?? r.url} carries the header ${named}, and this transport sends no credential`;
91
+ if (/[?&](key|api_key|apikey|access_token|token|password|passwd)=/i.test(r.url)) {
92
+ return `${r.method} ${r.url.split("?")[0] ?? r.url} carries a credential in its query, and this transport sends no credential`;
93
+ }
94
+ return undefined;
95
+ }
96
+
97
+ const KEPT = ["content-type", "link", "retry-after", "x-ratelimit-remaining", "x-ratelimit-reset", "etag", "location"];
98
+
99
+ /** How the live transport sends. */
100
+ export type LiveOptions = {
101
+ /** The `fetch` to use (the platform's by default). */
102
+ readonly fetch?: typeof fetch;
103
+ /** Refuse a request that carries a credential (the default), or send it. */
104
+ readonly credentials?: "refuse" | "allow";
105
+ /** The `User-Agent` said to every system: a generic name, never a person's. */
106
+ readonly userAgent?: string;
107
+ /** No answer within this many seconds fails the request (30 by default). */
108
+ readonly timeoutSeconds?: number;
109
+ };
110
+
111
+ /** The live transport, over `fetch`. */
112
+ export function liveTransport(options: LiveOptions = {}): Transport {
113
+ const fetcher = options.fetch ?? fetch;
114
+ return {
115
+ kind: "live",
116
+ async send(r) {
117
+ if (options.credentials !== "allow") {
118
+ const why = carriesCredential(r);
119
+ if (why !== undefined) return err({ kind: "refused", message: why });
120
+ }
121
+ // Where a telemetry pipeline is installed, the request is an `http.client` span and carries `traceparent` (0001-net FR-016).
122
+ return traceHttpClient(
123
+ r.method,
124
+ r.url,
125
+ async (trace): Promise<Result<Exchange, TransportError>> => {
126
+ try {
127
+ const res = await fetcher(r.url, {
128
+ method: r.method,
129
+ headers: { "user-agent": options.userAgent ?? "ifc-ts", ...trace, ...r.headers },
130
+ ...(r.body === undefined ? {} : { body: r.body }),
131
+ signal: AbortSignal.timeout((options.timeoutSeconds ?? 30) * 1000),
132
+ });
133
+ const headers: Record<string, string> = {};
134
+ for (const k of KEPT) {
135
+ const v = res.headers.get(k);
136
+ if (v !== null) headers[k] = v;
137
+ }
138
+ if (r.binary === true) {
139
+ return ok({
140
+ method: r.method,
141
+ url: r.url,
142
+ status: res.status,
143
+ headers,
144
+ body: "",
145
+ bytes: new Uint8Array(await res.arrayBuffer()),
146
+ });
147
+ }
148
+ return ok({ method: r.method, url: r.url, status: res.status, headers, body: await res.text() });
149
+ } catch (e) {
150
+ return err({
151
+ kind: "network",
152
+ message: e instanceof Error ? `${e.message}${e.cause instanceof Error ? `: ${e.cause.message}` : ""}` : String(e),
153
+ });
154
+ }
155
+ },
156
+ (x) => (x.kind === "ok" ? x.value.status : undefined),
157
+ );
158
+ },
159
+ };
160
+ }
161
+
162
+ /** A transport that answers from recorded exchanges, by method and URL, and refuses a request it has none for, naming it. */
163
+ export function recordedTransport(exchanges: readonly Exchange[]): Transport {
164
+ return {
165
+ kind: "recorded",
166
+ send(r) {
167
+ const x = exchanges.find((e) => e.method === r.method && e.url === r.url);
168
+ return Promise.resolve(x === undefined ? err({ kind: "refused", message: `no recorded exchange for ${r.method} ${r.url}` }) : ok(x));
169
+ },
170
+ };
171
+ }
172
+
173
+ /**
174
+ * Whether a `POST` only reads (FR-066): a GraphQL query, a search, a JSON API's `Describe` and `Get` actions. It is given the whole
175
+ * request (method, URL, headers and body), as an action may be named in a header (`x-amz-target`) or the body.
176
+ */
177
+ export type ReadPost = (r: OutRequest) => boolean;
178
+
179
+ /**
180
+ * A transport that sends only safe methods (GET, HEAD, ...), POST to URLs that start with one of `tokenEndpoints`, and the POSTs that
181
+ * `readPost` says only read (FR-066). Every other request (PUT, PATCH, DELETE, any other POST) is refused without being sent.
182
+ */
183
+ export function readOnly(t: Transport, tokenEndpoints: readonly string[] = [], readPost?: ReadPost): Transport {
184
+ return {
185
+ kind: t.kind,
186
+ send(r) {
187
+ const post = r.method === "POST" && (tokenEndpoints.some((p) => r.url.startsWith(p)) || readPost?.(r) === true);
188
+ if (isSafeMethod(r.method) || post) return t.send(r);
189
+ return Promise.resolve(err({ kind: "refused", message: `read-only: ${r.method} ${new URL(r.url).origin} is refused` }));
190
+ },
191
+ };
192
+ }
193
+
194
+ /** How {@link withSigV4} signs. */
195
+ export type SigV4TransportOptions = {
196
+ readonly credentials: AwsCredentials;
197
+ readonly region: string;
198
+ readonly service: string;
199
+ /** The clock read as each request is sent (`new Date()` by default; a test gives its own). */
200
+ readonly now?: () => Date;
201
+ /** Sign `UNSIGNED-PAYLOAD` instead of the body's hash (S3 allows it). */
202
+ readonly unsignedPayload?: boolean;
203
+ };
204
+
205
+ /**
206
+ * A transport that signs each request by AWS Signature Version 4 (FR-069, by the kernel's `sigv4`, FR-068) when it is sent. A signed
207
+ * request carries a credential, so the live transport under it must be made with `credentials: "allow"`, or it refuses the request.
208
+ */
209
+ export function withSigV4(t: Transport, options: SigV4TransportOptions): Transport {
210
+ return {
211
+ kind: t.kind,
212
+ async send(r) {
213
+ const signed = await sign(r, {
214
+ credentials: options.credentials,
215
+ region: options.region,
216
+ service: options.service,
217
+ date: (options.now ?? (() => new Date()))(),
218
+ ...(options.unsignedPayload === undefined ? {} : { unsignedPayload: options.unsignedPayload }),
219
+ });
220
+ return await t.send({ ...r, headers: signed.headers });
221
+ },
222
+ };
223
+ }
224
+
225
+ /** How long to back off. */
226
+ export type BackoffOptions = {
227
+ /** Tries after the first (3 by default). */
228
+ readonly retries?: number;
229
+ /** How a wait is spent (a timer by default; a test passes one that returns at once). */
230
+ readonly pause?: (ms: number) => Promise<void>;
231
+ /** The longest wait, whatever `Retry-After` asks (60 s by default). */
232
+ readonly capMs?: number;
233
+ };
234
+
235
+ /** A transport that waits and tries again on 429 and 503: as long as `Retry-After` asks (up to the cap), else 1, 2, 4 seconds. */
236
+ export function withBackoff(t: Transport, options: BackoffOptions = {}): Transport {
237
+ const pause = options.pause ?? ((ms: number) => sleep(ms));
238
+ const cap = options.capMs ?? 60_000;
239
+ return {
240
+ kind: t.kind,
241
+ async send(r) {
242
+ for (let n = 0; ; n++) {
243
+ const x = await t.send(r);
244
+ if (x.kind === "err" || (x.value.status !== 429 && x.value.status !== 503) || n >= (options.retries ?? 3)) return x;
245
+ const asked = retryAfterMs(x.value.headers["retry-after"], Date.now());
246
+ await pause(Math.min(cap, asked ?? 1000 * 2 ** n));
247
+ }
248
+ },
249
+ };
250
+ }
251
+
252
+ /** Why a JSON read failed. */
253
+ export type ReadError =
254
+ | TransportError
255
+ | { readonly kind: "denied"; readonly status: 401 | 403; readonly url: string }
256
+ | { readonly kind: "status"; readonly status: number; readonly url: string }
257
+ | { readonly kind: "decode"; readonly url: string; readonly message: string };
258
+
259
+ /** A read error as one sentence; never a header or a body, so no credential or personal datum is said. */
260
+ export function formatReadError(e: ReadError): string {
261
+ switch (e.kind) {
262
+ case "refused":
263
+ case "network":
264
+ return e.message;
265
+ case "denied":
266
+ return `${new URL(e.url).pathname}: ${e.status} (the credential may not read this)`;
267
+ case "status":
268
+ return `${new URL(e.url).pathname}: HTTP ${e.status}`;
269
+ case "decode":
270
+ return `${new URL(e.url).pathname}: ${e.message}`;
271
+ default:
272
+ return absurd(e);
273
+ }
274
+ }
275
+
276
+ /** A GET whose JSON body is decoded, with the exchange; 401 and 403 are `denied`, any other status from 400 up is `status`. */
277
+ export async function getExchange<T>(
278
+ t: Transport,
279
+ url: string,
280
+ decoder: Decoder<T>,
281
+ headers: Readonly<Record<string, string>> = {},
282
+ ): Promise<Result<{ readonly value: T; readonly exchange: Exchange }, ReadError>> {
283
+ const x = await t.send({ method: "GET", url, headers: { accept: "application/json", ...headers } });
284
+ if (x.kind === "err") return x;
285
+ const { status } = x.value;
286
+ if (status === 401 || status === 403) return err({ kind: "denied", status, url });
287
+ if (status >= 400) return err({ kind: "status", status, url });
288
+ const v = json(x.value.body, decoder);
289
+ if (v.kind === "err") return err({ kind: "decode", url, message: formatDecodeError(v.error) });
290
+ return ok({ value: v.value, exchange: x.value });
291
+ }
292
+
293
+ /** A GET whose JSON body is decoded. */
294
+ export async function getJson<T>(
295
+ t: Transport,
296
+ url: string,
297
+ decoder: Decoder<T>,
298
+ headers: Readonly<Record<string, string>> = {},
299
+ ): Promise<Result<T, ReadError>> {
300
+ const r = await getExchange(t, url, decoder, headers);
301
+ return r.kind === "ok" ? ok(r.value.value) : r;
302
+ }
303
+
304
+ /** Every page of a list that pages by `Link: <...>; rel="next"`, at most `maxPages` (200 by default); the first failure ends it, as an error. */
305
+ export async function* linkPages<T>(
306
+ t: Transport,
307
+ url: string,
308
+ item: Decoder<T>,
309
+ options: { readonly headers?: Readonly<Record<string, string>>; readonly maxPages?: number } = {},
310
+ ): AsyncGenerator<T, Result<number, ReadError>> {
311
+ const page = array(item);
312
+ let next: string | undefined = url;
313
+ let pages = 0;
314
+ while (next !== undefined && pages < (options.maxPages ?? 200)) {
315
+ const r = await getExchange(t, next, page, options.headers);
316
+ if (r.kind === "err") return r;
317
+ yield* r.value.value;
318
+ pages++;
319
+ next = nextLink(r.value.exchange.headers["link"], "next", next);
320
+ }
321
+ return ok(pages);
322
+ }
323
+
324
+ /** Every item of a list that pages by a cursor in its body: `items` takes a page's items, `next` the next page's URL or undefined. */
325
+ export async function* cursorPages<B, T>(
326
+ t: Transport,
327
+ first: string,
328
+ page: Decoder<B>,
329
+ items: (body: B) => readonly T[],
330
+ next: (body: B, url: string) => string | undefined,
331
+ options: { readonly headers?: Readonly<Record<string, string>>; readonly maxPages?: number } = {},
332
+ ): AsyncGenerator<T, Result<number, ReadError>> {
333
+ let url: string | undefined = first;
334
+ let pages = 0;
335
+ while (url !== undefined && pages < (options.maxPages ?? 200)) {
336
+ const r = await getJson(t, url, page, options.headers);
337
+ if (r.kind === "err") return r;
338
+ yield* items(r.value);
339
+ pages++;
340
+ url = next(r.value, url);
341
+ }
342
+ return ok(pages);
343
+ }
344
+
345
+ const exchangeDecoder: Decoder<Exchange> = object({
346
+ method: literal("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"),
347
+ url: string,
348
+ status: number,
349
+ headers: record(string),
350
+ body: string,
351
+ });
352
+
353
+ /** A recorded exchange read from the JSON5 text `recordedText` writes, or why it is not one. */
354
+ export function decodeRecorded(text: string): Result<Exchange, string> {
355
+ const v = parseJson5(text);
356
+ if (v.kind === "err") return err(`not JSON5: ${formatJson5Error(v.error)}`);
357
+ const x = exchangeDecoder(v.value);
358
+ return x.kind === "ok" ? ok(x.value) : err(formatDecodeError(x.error));
359
+ }
360
+
361
+ /** The text a recorded exchange is kept in: a comment that says where and when it was recorded, then the exchange as JSON5. */
362
+ export function recordedText(x: Exchange, note: string): string {
363
+ return toJson5({ method: x.method, url: x.url, status: x.status, headers: x.headers, body: x.body }, note);
364
+ }
365
+
366
+ /** Headers that give a bearer token (for a transport made to allow credentials). */
367
+ export function bearer(token: string, scheme = "Bearer"): Readonly<Record<string, string>> {
368
+ return { authorization: `${scheme} ${token}` };
369
+ }
package/src/mod.ts ADDED
@@ -0,0 +1,22 @@
1
+ // Spec: 0001-net FR-017
2
+ /**
3
+ * Net, the network of Intellectual Frontiers Commons as one typed library (its specification 0001-net): downloads that retry and resume (as `net`), an HTTP server over web `Request` and `Response` (as `serve`),
4
+ * typed HTTP clients with a live or recorded transport (as `http`), TCP and TLS (as `tcp`), DNS records (as `dns`) and ACME certificates by DNS-01 (as `acme`). It names `node:` builtins and no vendor's global,
5
+ * depends on core, formats, crypto, system and telemetry, and runs the same on Bun and on Node 26.
6
+ *
7
+ * @example
8
+ * ```ts
9
+ * import { serve } from "@intellectual-frontiers/net";
10
+ * const server = await serve({ port: 0 }, () => new Response("ok"));
11
+ * await server.shutdown();
12
+ * ```
13
+ *
14
+ * @module
15
+ */
16
+ export * as acme from "./acme.ts";
17
+ export * as dns from "./dns.ts";
18
+ export * as http from "./http-client.ts";
19
+ export * as net from "./net.ts";
20
+ export * as tcp from "./tcp.ts";
21
+ export { serve } from "./serve.ts";
22
+ export type { NetAddr, Server, ServeHandler, ServeInfo, ServeOptions } from "./serve.ts";