@procedures/http 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/CHANGELOG.md +46 -0
  2. package/LICENSE +21 -0
  3. package/README.md +118 -2
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/address.d.ts +31 -0
  6. package/dist/address.d.ts.map +1 -0
  7. package/dist/address.js +214 -0
  8. package/dist/address.js.map +1 -0
  9. package/dist/errors.d.ts +39 -0
  10. package/dist/errors.d.ts.map +1 -0
  11. package/dist/errors.js +18 -0
  12. package/dist/errors.js.map +1 -0
  13. package/dist/exchange.d.ts +102 -0
  14. package/dist/exchange.d.ts.map +1 -0
  15. package/dist/exchange.js +322 -0
  16. package/dist/exchange.js.map +1 -0
  17. package/dist/index.d.ts +10 -0
  18. package/dist/index.d.ts.map +1 -0
  19. package/dist/index.js +11 -0
  20. package/dist/index.js.map +1 -0
  21. package/dist/module.d.ts +41 -0
  22. package/dist/module.d.ts.map +1 -0
  23. package/dist/module.js +205 -0
  24. package/dist/module.js.map +1 -0
  25. package/dist/parsers.d.ts +37 -0
  26. package/dist/parsers.d.ts.map +1 -0
  27. package/dist/parsers.js +175 -0
  28. package/dist/parsers.js.map +1 -0
  29. package/dist/policy.d.ts +94 -0
  30. package/dist/policy.d.ts.map +1 -0
  31. package/dist/policy.js +140 -0
  32. package/dist/policy.js.map +1 -0
  33. package/dist/procedures/http/index.d.ts +5 -0
  34. package/dist/procedures/http/index.d.ts.map +1 -0
  35. package/dist/procedures/http/index.js +5 -0
  36. package/dist/procedures/http/index.js.map +1 -0
  37. package/dist/procedures/http/json.d.ts +14 -0
  38. package/dist/procedures/http/json.d.ts.map +1 -0
  39. package/dist/procedures/http/json.js +56 -0
  40. package/dist/procedures/http/json.js.map +1 -0
  41. package/dist/procedures/http/ndjson.d.ts +18 -0
  42. package/dist/procedures/http/ndjson.d.ts.map +1 -0
  43. package/dist/procedures/http/ndjson.js +65 -0
  44. package/dist/procedures/http/ndjson.js.map +1 -0
  45. package/dist/procedures/http/request.d.ts +14 -0
  46. package/dist/procedures/http/request.d.ts.map +1 -0
  47. package/dist/procedures/http/request.js +79 -0
  48. package/dist/procedures/http/request.js.map +1 -0
  49. package/dist/procedures/http/sse.d.ts +19 -0
  50. package/dist/procedures/http/sse.d.ts.map +1 -0
  51. package/dist/procedures/http/sse.js +66 -0
  52. package/dist/procedures/http/sse.js.map +1 -0
  53. package/dist/register.d.ts +6 -0
  54. package/dist/register.d.ts.map +1 -0
  55. package/dist/register.js +8 -0
  56. package/dist/register.js.map +1 -0
  57. package/dist/scope.d.ts +53 -0
  58. package/dist/scope.d.ts.map +1 -0
  59. package/dist/scope.js +107 -0
  60. package/dist/scope.js.map +1 -0
  61. package/dist/types.d.ts +102 -0
  62. package/dist/types.d.ts.map +1 -0
  63. package/dist/types.js +8 -0
  64. package/dist/types.js.map +1 -0
  65. package/package.json +79 -3
  66. package/src/address.ts +206 -0
  67. package/src/errors.ts +44 -0
  68. package/src/exchange.ts +381 -0
  69. package/src/index.ts +11 -0
  70. package/src/module.ts +257 -0
  71. package/src/parsers.ts +184 -0
  72. package/src/policy.ts +192 -0
  73. package/src/procedures/http/index.ts +4 -0
  74. package/src/procedures/http/json.ts +52 -0
  75. package/src/procedures/http/ndjson.ts +64 -0
  76. package/src/procedures/http/request.ts +72 -0
  77. package/src/procedures/http/sse.ts +65 -0
  78. package/src/register.ts +10 -0
  79. package/src/scope.ts +129 -0
  80. package/src/types.ts +127 -0
package/src/parsers.ts ADDED
@@ -0,0 +1,184 @@
1
+ /**
2
+ * The stream parsers of the HTTP procedures: the lines of NDJSON and the events of an event
3
+ * stream (the `text/event-stream` format of the HTML standard). Each parser reads chunks of
4
+ * bytes, so a line or a character can arrive in two chunks.
5
+ */
6
+
7
+ import { HttpError } from "./errors.js";
8
+
9
+ /** One line of an NDJSON stream. */
10
+ export interface NdjsonLine {
11
+ /** The number of the line: the first line is 1. Blank lines count. */
12
+ readonly line: number;
13
+ readonly text: string;
14
+ }
15
+
16
+ const LF = 0x0a;
17
+ const CR = 0x0d;
18
+
19
+ function bytesOf(chunk: Uint8Array | string): Buffer {
20
+ return typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
21
+ }
22
+
23
+ /**
24
+ * This function gives each line of a stream of bytes that is not blank. A line ends at LF, and a
25
+ * CR before the LF is removed. A line of more than `maxLineBytes` bytes throws `LINE_TOO_LARGE`.
26
+ * The last line can end without LF. A byte order mark at the start is removed.
27
+ */
28
+ export async function* ndjsonLines(chunks: AsyncIterable<Uint8Array | string>, maxLineBytes: number, label = "http.ndjson"): AsyncGenerator<NdjsonLine, void, undefined> {
29
+ let pending: Buffer[] = [];
30
+ let pendingSize = 0;
31
+ let number = 0;
32
+ const tooLarge = (): HttpError => new HttpError("LINE_TOO_LARGE", `${label}: line ${number + 1} has more than ${maxLineBytes} bytes (maxLineBytes)`);
33
+ const finish = (last: Buffer): NdjsonLine | undefined => {
34
+ let bytes = pending.length === 0 ? last : Buffer.concat([...pending, last]);
35
+ pending = [];
36
+ pendingSize = 0;
37
+ if (bytes.length > 0 && bytes[bytes.length - 1] === CR) bytes = bytes.subarray(0, bytes.length - 1);
38
+ if (bytes.length > maxLineBytes) throw tooLarge();
39
+ number++;
40
+ let text = bytes.toString("utf8");
41
+ if (number === 1 && text.charCodeAt(0) === 0xfeff) text = text.slice(1);
42
+ return text.trim() === "" ? undefined : { line: number, text };
43
+ };
44
+ for await (const chunk of chunks) {
45
+ const buffer = bytesOf(chunk);
46
+ let start = 0;
47
+ for (;;) {
48
+ const end = buffer.indexOf(LF, start);
49
+ if (end < 0) break;
50
+ if (pendingSize + (end - start) > maxLineBytes + 1) throw tooLarge();
51
+ const line = finish(buffer.subarray(start, end));
52
+ if (line) yield line;
53
+ start = end + 1;
54
+ }
55
+ if (start < buffer.length) {
56
+ const rest = buffer.subarray(start);
57
+ pendingSize += rest.length;
58
+ // One more byte for a CR before the LF
59
+ if (pendingSize > maxLineBytes + 1) throw tooLarge();
60
+ pending.push(Buffer.from(rest));
61
+ }
62
+ }
63
+ if (pendingSize > 0) {
64
+ const line = finish(Buffer.alloc(0));
65
+ if (line) yield line;
66
+ }
67
+ }
68
+
69
+ /** One server-sent event. */
70
+ export interface ServerSentEvent {
71
+ /** The type of the event: the `event` field, or `message`. */
72
+ readonly event: string;
73
+ /** The data: the `data` fields of the event, joined with LF. */
74
+ readonly data: string;
75
+ /** The last event ID: the last `id` field of the stream, when it is not empty. */
76
+ readonly id?: string;
77
+ /** The reconnection time that the event gave with a `retry` field, in milliseconds. */
78
+ readonly retry?: number;
79
+ }
80
+
81
+ /**
82
+ * This function gives each event of an event stream. It obeys the parse rules of the HTML
83
+ * standard: a line ends at CR, LF or CRLF, a blank line sends the event, a line that starts with a
84
+ * colon is a comment, and an event without data is not sent. An event of more than
85
+ * `maxEventBytes` bytes throws `EVENT_TOO_LARGE`. At the end of the stream, an event without its
86
+ * blank line is not sent.
87
+ */
88
+ export async function* sseEvents(chunks: AsyncIterable<Uint8Array | string>, maxEventBytes: number, label = "http.sse"): AsyncGenerator<ServerSentEvent, void, undefined> {
89
+ // The decoder removes the byte order mark and keeps a character whose bytes are in two chunks
90
+ const decoder = new TextDecoder("utf-8");
91
+ let buffer = "";
92
+ let skipLf = false;
93
+ let data = "";
94
+ let dataBytes = 0;
95
+ let eventType = "";
96
+ let lastEventId = "";
97
+ let retry: number | undefined;
98
+ const tooLarge = (): HttpError => new HttpError("EVENT_TOO_LARGE", `${label}: an event has more than ${maxEventBytes} bytes (maxEventBytes)`);
99
+
100
+ /** The event of a blank line, or undefined. */
101
+ const dispatch = (): ServerSentEvent | undefined => {
102
+ const event: ServerSentEvent | undefined =
103
+ data === ""
104
+ ? undefined
105
+ : {
106
+ event: eventType === "" ? "message" : eventType,
107
+ data: data.endsWith("\n") ? data.slice(0, -1) : data,
108
+ ...(lastEventId !== "" ? { id: lastEventId } : {}),
109
+ ...(retry !== undefined ? { retry } : {}),
110
+ };
111
+ data = "";
112
+ dataBytes = 0;
113
+ eventType = "";
114
+ retry = undefined;
115
+ return event;
116
+ };
117
+
118
+ const field = (line: string): void => {
119
+ if (line.startsWith(":")) return;
120
+ const colon = line.indexOf(":");
121
+ const name = colon < 0 ? line : line.slice(0, colon);
122
+ let value = colon < 0 ? "" : line.slice(colon + 1);
123
+ if (value.startsWith(" ")) value = value.slice(1);
124
+ switch (name) {
125
+ case "event":
126
+ eventType = value;
127
+ break;
128
+ case "data":
129
+ dataBytes += Buffer.byteLength(value) + 1;
130
+ if (dataBytes > maxEventBytes + 1) throw tooLarge();
131
+ data += `${value}\n`;
132
+ break;
133
+ case "id":
134
+ if (!value.includes("\u0000")) lastEventId = value;
135
+ break;
136
+ case "retry":
137
+ if (/^[0-9]+$/.test(value)) retry = Number(value);
138
+ break;
139
+ default:
140
+ break;
141
+ }
142
+ };
143
+
144
+ const lines = function* (text: string, end: boolean): Generator<string> {
145
+ buffer += text;
146
+ let start = 0;
147
+ if (skipLf && buffer.length > 0) {
148
+ if (buffer.charCodeAt(0) === 0x0a) start = 1;
149
+ skipLf = false;
150
+ }
151
+ for (let i = start; i < buffer.length; i++) {
152
+ const code = buffer.charCodeAt(i);
153
+ if (code !== 0x0a && code !== 0x0d) continue;
154
+ yield buffer.slice(start, i);
155
+ if (code === 0x0d) {
156
+ if (i + 1 < buffer.length) {
157
+ if (buffer.charCodeAt(i + 1) === 0x0a) i++;
158
+ } else {
159
+ // A CR at the end of a chunk: an LF at the start of the next chunk is the same line end
160
+ skipLf = true;
161
+ }
162
+ }
163
+ start = i + 1;
164
+ }
165
+ buffer = buffer.slice(start);
166
+ if (!end && buffer.length > maxEventBytes) throw tooLarge();
167
+ };
168
+
169
+ const feed = function* (text: string, end: boolean): Generator<ServerSentEvent> {
170
+ for (const line of lines(text, end)) {
171
+ if (line === "") {
172
+ const event = dispatch();
173
+ if (event) yield event;
174
+ } else {
175
+ field(line);
176
+ }
177
+ }
178
+ };
179
+
180
+ for await (const chunk of chunks) {
181
+ yield* feed(typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true }), false);
182
+ }
183
+ yield* feed(decoder.decode(), true);
184
+ }
package/src/policy.ts ADDED
@@ -0,0 +1,192 @@
1
+ /**
2
+ * The host policy of the HTTP procedures (roadmap R38): the hosts that a request can reach.
3
+ *
4
+ * - The default policy allows each public host and refuses the other addresses: loopback,
5
+ * private, link-local, shared, multicast and reserved (see `address.ts`).
6
+ * - `allow` names more hosts: a host name (`api.example.com`), a wildcard (`*.example.com`), an
7
+ * IP address (`127.0.0.1`, `::1`) or a CIDR range (`10.0.0.0/8`). An entry can end with a port:
8
+ * `localhost:8080`, `[::1]:8080`, `10.0.0.0/8:443`. Without a port, an entry allows each port.
9
+ * - `publicHosts: false` refuses the public hosts too: only the entries of `allow` stay.
10
+ *
11
+ * The check resolves the host name and examines each address. The connection uses only the
12
+ * addresses that the check allowed, so a second answer of the DNS cannot change the address
13
+ * (DNS rebinding). A redirect goes through the same check. A refusal is `FORBIDDEN` with the
14
+ * reason `host-not-allowed`.
15
+ */
16
+
17
+ import { lookup } from "node:dns/promises";
18
+ import { InvocationError, type ProcedurePath } from "@procedures/client";
19
+ import { formatAddress, parseAddress, parseRange, rangeHas, specialClass, type AddressBytes, type AddressRange } from "./address.js";
20
+
21
+ /** The reason of the `FORBIDDEN` error for a host that the policy does not allow. */
22
+ export const HOST_NOT_ALLOWED = "host-not-allowed";
23
+
24
+ /** One address of a host name. */
25
+ export interface ResolvedAddress {
26
+ readonly address: string;
27
+ readonly family: 4 | 6;
28
+ }
29
+
30
+ /**
31
+ * A resolver: it gives the addresses of a host name. The default uses the resolver of the
32
+ * operating system (`dns.lookup`), so it reads the hosts file too. A test gives its own resolver.
33
+ */
34
+ export type HostResolver = (hostname: string) => Promise<readonly ResolvedAddress[]>;
35
+
36
+ /** The options of the host policy: the options of `createHttpModule()`. */
37
+ export interface HostPolicyOptions {
38
+ /**
39
+ * More hosts that the procedures can reach: host names, wildcards (`*.example.com`), IP
40
+ * addresses and CIDR ranges, each with an optional port. An entry that names a private or a
41
+ * loopback address allows it.
42
+ */
43
+ readonly allow?: readonly string[] | undefined;
44
+ /** False: only the hosts of `allow`. The default is true: each public host too. */
45
+ readonly publicHosts?: boolean | undefined;
46
+ /** The resolver of host names. The default is `dns.lookup` with all the addresses. */
47
+ readonly resolve?: HostResolver | undefined;
48
+ }
49
+
50
+ /** One entry of `allow`. */
51
+ type AllowEntry =
52
+ | { readonly kind: "name"; readonly name: string; readonly port: number | undefined; readonly text: string }
53
+ | { readonly kind: "wildcard"; readonly suffix: string; readonly port: number | undefined; readonly text: string }
54
+ | { readonly kind: "range"; readonly range: AddressRange; readonly port: number | undefined; readonly text: string };
55
+
56
+ /** A host policy: the parsed options. */
57
+ export interface HostPolicy {
58
+ readonly entries: readonly AllowEntry[];
59
+ readonly publicHosts: boolean;
60
+ readonly resolve: HostResolver;
61
+ }
62
+
63
+ /** The resolver of the operating system: each address of the name, in the order of the system. */
64
+ export const systemResolver: HostResolver = async (hostname) => {
65
+ const found = await lookup(hostname, { all: true, verbatim: true });
66
+ return found.map((entry) => ({ address: entry.address, family: entry.family === 6 ? 6 : 4 }));
67
+ };
68
+
69
+ /** A host name in the form of a comparison: lowercase, without a final dot. */
70
+ function normalName(name: string): string {
71
+ return name.toLowerCase().replace(/\.$/, "");
72
+ }
73
+
74
+ const NAME = /^[a-z0-9_]([a-z0-9_-]*[a-z0-9_])?(\.[a-z0-9_]([a-z0-9_-]*[a-z0-9_])?)*$/;
75
+
76
+ /** This function parses one entry of `allow`. A bad entry throws, so a host finds it at the start. */
77
+ function parseEntry(text: string): AllowEntry {
78
+ let head = text.trim();
79
+ let port: number | undefined;
80
+ // A port: "host:80", "[::1]:80", "10.0.0.0/8:80". A bare IPv6 address has no port.
81
+ const withPort = /^(.*[^:]):([0-9]{1,5})$/.exec(head);
82
+ if (withPort && (!withPort[1]!.includes(":") || withPort[1]!.endsWith("]"))) {
83
+ head = withPort[1]!;
84
+ port = Number(withPort[2]);
85
+ if (port < 1 || port > 65535) throw new Error(`The port of the allow entry "${text}" is not in 1 to 65535`);
86
+ }
87
+ const range = parseRange(head);
88
+ if (range) return { kind: "range", range, port, text };
89
+ const name = normalName(head);
90
+ if (name.startsWith("*.") && NAME.test(name.slice(2))) return { kind: "wildcard", suffix: name.slice(1), port, text };
91
+ if (NAME.test(name)) return { kind: "name", name, port, text };
92
+ throw new Error(`The allow entry "${text}" is not a host name, a wildcard, an IP address or a CIDR range`);
93
+ }
94
+
95
+ /** This function makes a host policy from the options. A bad entry of `allow` throws. */
96
+ export function createHostPolicy(options: HostPolicyOptions = {}): HostPolicy {
97
+ return {
98
+ entries: (options.allow ?? []).map(parseEntry),
99
+ publicHosts: options.publicHosts ?? true,
100
+ resolve: options.resolve ?? systemResolver,
101
+ };
102
+ }
103
+
104
+ /** The default policy: each public host, and no other host. */
105
+ export const DEFAULT_HOST_POLICY: HostPolicy = createHostPolicy();
106
+
107
+ /** The host of a URL to connect to, after the check. */
108
+ export interface AllowedTarget {
109
+ /** The host name of the URL, or the IP address without brackets. */
110
+ readonly hostname: string;
111
+ readonly port: number;
112
+ /** The addresses that the connection can use: each one passed the check. */
113
+ readonly addresses: readonly ResolvedAddress[];
114
+ }
115
+
116
+ /** The `FORBIDDEN` error for a host that the policy does not allow. */
117
+ export function hostNotAllowed(procedure: ProcedurePath, message: string): InvocationError {
118
+ return new InvocationError("FORBIDDEN", `${procedure.join(".")}: ${message}`, procedure, HOST_NOT_ALLOWED);
119
+ }
120
+
121
+ /** True for the `FORBIDDEN` error of a host that the policy does not allow. */
122
+ export function isHostNotAllowed(error: unknown): boolean {
123
+ return error instanceof InvocationError && error.code === "FORBIDDEN" && error.reason === HOST_NOT_ALLOWED;
124
+ }
125
+
126
+ function portMatches(entry: AllowEntry, port: number): boolean {
127
+ return entry.port === undefined || entry.port === port;
128
+ }
129
+
130
+ function nameAllowed(policy: HostPolicy, hostname: string, port: number): boolean {
131
+ const name = normalName(hostname);
132
+ return policy.entries.some(
133
+ (entry) =>
134
+ portMatches(entry, port) &&
135
+ ((entry.kind === "name" && entry.name === name) || (entry.kind === "wildcard" && name.endsWith(entry.suffix) && name.length > entry.suffix.length)),
136
+ );
137
+ }
138
+
139
+ /** Undefined when the policy allows the address, or the reason of the refusal. */
140
+ function addressRefusal(policy: HostPolicy, address: AddressBytes, port: number): string | undefined {
141
+ if (policy.entries.some((entry) => entry.kind === "range" && portMatches(entry, port) && rangeHas(entry.range, address))) return undefined;
142
+ const special = specialClass(address);
143
+ if (special !== undefined) return `${formatAddress(address)} is a ${special} address`;
144
+ return policy.publicHosts ? undefined : `${formatAddress(address)} is not in the allowed hosts`;
145
+ }
146
+
147
+ /** The default port of a URL protocol. */
148
+ export function portOf(url: URL): number {
149
+ if (url.port !== "") return Number(url.port);
150
+ return url.protocol === "https:" ? 443 : 80;
151
+ }
152
+
153
+ /**
154
+ * This function examines the host of a URL. It gives the addresses that the connection can use,
155
+ * or it throws `FORBIDDEN` with the reason `host-not-allowed`. A name that `allow` names keeps
156
+ * each address. For another name, each address must be public or in a range of `allow`. The
157
+ * connection uses only the addresses that pass, and the check fails when none passes.
158
+ */
159
+ export async function checkHost(policy: HostPolicy, url: URL, procedure: ProcedurePath, what = "the URL"): Promise<AllowedTarget> {
160
+ const hostname = url.hostname.startsWith("[") ? url.hostname.slice(1, -1) : url.hostname;
161
+ const port = portOf(url);
162
+ const literal = parseAddress(hostname);
163
+ if (literal) {
164
+ const refusal = addressRefusal(policy, literal, port);
165
+ if (refusal !== undefined) throw hostNotAllowed(procedure, `the host of ${what} is not allowed: ${refusal}`);
166
+ return { hostname, port, addresses: [{ address: formatAddress(literal), family: literal.length === 4 ? 4 : 6 }] };
167
+ }
168
+ if (!policy.publicHosts && !nameAllowed(policy, hostname, port) && !policy.entries.some((entry) => entry.kind === "range")) {
169
+ throw hostNotAllowed(procedure, `the host ${hostname}:${port} of ${what} is not in the allowed hosts`);
170
+ }
171
+ let resolved: readonly ResolvedAddress[];
172
+ try {
173
+ resolved = await policy.resolve(hostname);
174
+ } catch (error) {
175
+ const code = (error as NodeJS.ErrnoException | undefined)?.code;
176
+ throw new Error(`${procedure.join(".")}: the host ${hostname} of ${what} has no address${code ? ` (${code})` : ""}`, { cause: error });
177
+ }
178
+ if (resolved.length === 0) throw new Error(`${procedure.join(".")}: the host ${hostname} of ${what} has no address`);
179
+ if (nameAllowed(policy, hostname, port)) return { hostname, port, addresses: resolved };
180
+ const allowed: ResolvedAddress[] = [];
181
+ const refusals: string[] = [];
182
+ for (const entry of resolved) {
183
+ const bytes = parseAddress(entry.address);
184
+ const refusal = bytes ? addressRefusal(policy, bytes, port) : `${entry.address} is not an IP address`;
185
+ if (refusal === undefined) allowed.push(entry);
186
+ else refusals.push(refusal);
187
+ }
188
+ if (allowed.length === 0) {
189
+ throw hostNotAllowed(procedure, `the host ${hostname} of ${what} is not allowed: ${refusals.join(", ")}`);
190
+ }
191
+ return { hostname, port, addresses: allowed };
192
+ }
@@ -0,0 +1,4 @@
1
+ export { httpRequest } from "./request.js";
2
+ export { httpJson } from "./json.js";
3
+ export { httpNdjson } from "./ndjson.js";
4
+ export { httpSse } from "./sse.js";
@@ -0,0 +1,52 @@
1
+ /**
2
+ * http.json procedure
3
+ *
4
+ * Send a GET or a POST of JSON and give the parsed JSON response. By default, a status outside
5
+ * 200 to 299 is an error (`HTTP_STATUS`) with the start of the body in its message.
6
+ */
7
+
8
+ import { HttpError } from "../../errors.js";
9
+ import { DEFAULT_SETTINGS, DEFAULT_TIMEOUT_MS, excerpt, exchange, isOk, prepare, readBody, responseHeaders, type HttpSettings } from "../../exchange.js";
10
+ import { CallScope, type ScopeContext } from "../../scope.js";
11
+ import type { HttpJsonInput, HttpJsonOutput } from "../../types.js";
12
+
13
+ const PATH = ["http", "json"];
14
+
15
+ /**
16
+ * Send a GET or a POST of JSON and give the parsed JSON response
17
+ */
18
+ export async function httpJson(input: HttpJsonInput, ctx: ScopeContext = {}, settings: HttpSettings = DEFAULT_SETTINGS): Promise<HttpJsonOutput> {
19
+ const method = input.method ?? (input.json !== undefined ? "POST" : "GET");
20
+ if (method === "GET" && input.json !== undefined) throw new Error("http.json: a GET request has no body: use the method POST");
21
+ const body = input.json !== undefined ? Buffer.from(JSON.stringify(input.json), "utf8") : undefined;
22
+ const headers: Record<string, string> = { accept: "application/json" };
23
+ if (body !== undefined) headers["content-type"] = "application/json";
24
+ const prepared = prepare({ ...input, method }, PATH, settings, { headers, body });
25
+ const scope = new CallScope(PATH, ctx, { timeoutMs: input.timeoutMs ?? DEFAULT_TIMEOUT_MS });
26
+ try {
27
+ const opened = await exchange(prepared, settings.policy, scope);
28
+ const status = opened.response.statusCode ?? 0;
29
+ const limit = Math.min(input.maxResponseBytes ?? settings.maxResponseBytes, settings.maxResponseBytes);
30
+ const bytes = await readBody(opened.body, limit, scope, opened.response);
31
+ const text = bytes.toString("utf8");
32
+ if (!isOk(status) && (input.failOnStatus ?? true)) {
33
+ const reason = opened.response.statusMessage ? ` ${opened.response.statusMessage}` : "";
34
+ throw new HttpError("HTTP_STATUS", `http.json: the server answered ${status}${reason}${text ? `: ${excerpt(text)}` : ""}`, status);
35
+ }
36
+ let data: unknown = null;
37
+ if (text.trim() !== "") {
38
+ try {
39
+ data = JSON.parse(text) as unknown;
40
+ } catch {
41
+ // An error page is often HTML: with failOnStatus: false, the output keeps its text
42
+ if (!isOk(status)) data = text;
43
+ else throw new HttpError("BAD_JSON", `http.json: the body of the response is not JSON: ${excerpt(text)}`, status);
44
+ }
45
+ }
46
+ return { status, ok: isOk(status), url: opened.url, headers: responseHeaders(opened.response.headers), data };
47
+ } catch (error) {
48
+ throw scope.translate(error);
49
+ } finally {
50
+ scope.dispose();
51
+ }
52
+ }
@@ -0,0 +1,64 @@
1
+ /**
2
+ * http.ndjson procedure
3
+ *
4
+ * Read an NDJSON response as a stream of parsed lines. The procedure sends a GET. Each line that
5
+ * is not blank is one item: its JSON value. The stream can run without an end, so the procedure
6
+ * declares an unbounded stream (roadmap R15). When the reader stops, the connection closes.
7
+ *
8
+ * The call waits for the status of the response. A refused host, a failed connection and a status
9
+ * outside 200 to 299 fail the call itself, before the first item.
10
+ */
11
+
12
+ import { HttpError } from "../../errors.js";
13
+ import { DEFAULT_ITEM_BYTES, DEFAULT_SETTINGS, excerpt, exchange, isOk, prepare, statusError, touching, type Exchange, type HttpSettings } from "../../exchange.js";
14
+ import { ndjsonLines } from "../../parsers.js";
15
+ import { CallScope, type ScopeContext } from "../../scope.js";
16
+ import type { HttpNdjsonInput, HttpNdjsonOutput } from "../../types.js";
17
+
18
+ const PATH = ["http", "ndjson"];
19
+
20
+ /** The items of an open response. The end of the generator closes the connection. */
21
+ async function* items(opened: Exchange, scope: CallScope, maxLineBytes: number): AsyncGenerator<HttpNdjsonOutput, void, undefined> {
22
+ try {
23
+ scope.touch();
24
+ for await (const line of ndjsonLines(touching(opened.body, scope), maxLineBytes, "http.ndjson")) {
25
+ let value: unknown;
26
+ try {
27
+ value = JSON.parse(line.text) as unknown;
28
+ } catch {
29
+ throw new HttpError("BAD_JSON", `http.ndjson: line ${line.line} is not JSON: ${excerpt(line.text, 80)}`, opened.response.statusCode);
30
+ }
31
+ // The idle time does not run while the reader holds the item
32
+ scope.pause();
33
+ yield value;
34
+ scope.touch();
35
+ }
36
+ } catch (error) {
37
+ throw scope.translate(error);
38
+ } finally {
39
+ opened.body.destroy();
40
+ opened.close();
41
+ scope.dispose();
42
+ }
43
+ }
44
+
45
+ /**
46
+ * Read an NDJSON response as a stream of parsed lines
47
+ */
48
+ export async function httpNdjson(input: HttpNdjsonInput, ctx: ScopeContext = {}, settings: HttpSettings = DEFAULT_SETTINGS): Promise<AsyncGenerator<HttpNdjsonOutput, void, undefined>> {
49
+ const headers = { accept: "application/x-ndjson, application/jsonl;q=0.9, application/json;q=0.8" };
50
+ const prepared = prepare({ ...input, method: "GET" }, PATH, settings, { headers });
51
+ const scope = new CallScope(PATH, ctx, { timeoutMs: input.timeoutMs, idleTimeoutMs: input.idleTimeoutMs });
52
+ let opened: Exchange | undefined;
53
+ try {
54
+ opened = await exchange(prepared, settings.policy, scope);
55
+ if (!isOk(opened.response.statusCode ?? 0)) throw await statusError(opened, scope);
56
+ } catch (error) {
57
+ opened?.close();
58
+ scope.dispose();
59
+ throw scope.translate(error);
60
+ }
61
+ // The idle time starts again when the reader asks for the first item
62
+ scope.pause();
63
+ return items(opened, scope, input.maxLineBytes ?? DEFAULT_ITEM_BYTES);
64
+ }
@@ -0,0 +1,72 @@
1
+ /**
2
+ * http.request procedure
3
+ *
4
+ * Send an HTTP request and give the status, the headers and the body. The procedure gives each
5
+ * status: a status of 400 or more is not an error. The body of the response has a size limit.
6
+ */
7
+
8
+ import { HttpError } from "../../errors.js";
9
+ import { DEFAULT_SETTINGS, DEFAULT_TIMEOUT_MS, exchange, isOk, prepare, readBody, responseHeaders, type HttpSettings } from "../../exchange.js";
10
+ import { CallScope, type ScopeContext } from "../../scope.js";
11
+ import type { HttpRequestInput, HttpRequestOutput } from "../../types.js";
12
+
13
+ const PATH = ["http", "request"];
14
+
15
+ /** The body of an input and its content type, or undefined. */
16
+ function bodyOf(input: HttpRequestInput): { body: Buffer; type: string } | undefined {
17
+ const given = [input.body, input.json, input.bodyBase64].filter((value) => value !== undefined).length;
18
+ if (given > 1) throw new Error("http.request: give only one of body, json and bodyBase64");
19
+ if (input.body !== undefined) return { body: Buffer.from(input.body, "utf8"), type: "text/plain; charset=utf-8" };
20
+ if (input.json !== undefined) return { body: Buffer.from(JSON.stringify(input.json), "utf8"), type: "application/json" };
21
+ if (input.bodyBase64 !== undefined) return { body: Buffer.from(input.bodyBase64, "base64"), type: "application/octet-stream" };
22
+ return undefined;
23
+ }
24
+
25
+ /** The body of the response in the form of `responseType`. */
26
+ function decode(bytes: Buffer, responseType: HttpRequestInput["responseType"], status: number): unknown {
27
+ if (responseType === "bytes") return bytes.toString("base64");
28
+ const text = bytes.toString("utf8");
29
+ if (responseType !== "json") return text;
30
+ if (text.trim() === "") return null;
31
+ try {
32
+ return JSON.parse(text) as unknown;
33
+ } catch {
34
+ throw new HttpError("BAD_JSON", `http.request: the body of the response is not JSON: ${text.slice(0, 200)}`, status);
35
+ }
36
+ }
37
+
38
+ /**
39
+ * Send an HTTP request and give the status, the headers and the body
40
+ */
41
+ export async function httpRequest(input: HttpRequestInput, ctx: ScopeContext = {}, settings: HttpSettings = DEFAULT_SETTINGS): Promise<HttpRequestOutput> {
42
+ const method = input.method ?? "GET";
43
+ const body = bodyOf(input);
44
+ if (body !== undefined && (method === "GET" || method === "HEAD")) {
45
+ throw new Error(`http.request: a ${method} request has no body`);
46
+ }
47
+ const headers: Record<string, string> = {};
48
+ if (body !== undefined) headers["content-type"] = body.type;
49
+ if (input.responseType === "json") headers["accept"] = "application/json";
50
+ const prepared = prepare({ ...input, method }, PATH, settings, { headers, body: body?.body });
51
+ const scope = new CallScope(PATH, ctx, { timeoutMs: input.timeoutMs ?? DEFAULT_TIMEOUT_MS });
52
+ try {
53
+ const opened = await exchange(prepared, settings.policy, scope);
54
+ const limit = Math.min(input.maxResponseBytes ?? settings.maxResponseBytes, settings.maxResponseBytes);
55
+ const bytes = await readBody(opened.body, limit, scope, opened.response);
56
+ const status = opened.response.statusCode ?? 0;
57
+ return {
58
+ status,
59
+ statusText: opened.response.statusMessage ?? "",
60
+ ok: isOk(status),
61
+ url: opened.url,
62
+ redirected: opened.redirects.length > 0,
63
+ headers: responseHeaders(opened.response.headers),
64
+ body: decode(bytes, input.responseType, status),
65
+ bytes: bytes.length,
66
+ };
67
+ } catch (error) {
68
+ throw scope.translate(error);
69
+ } finally {
70
+ scope.dispose();
71
+ }
72
+ }
@@ -0,0 +1,65 @@
1
+ /**
2
+ * http.sse procedure
3
+ *
4
+ * Read a stream of server-sent events. The procedure sends a GET with
5
+ * `accept: text/event-stream`, and each event is one item: its type, its data and the last event
6
+ * ID. The procedure does not connect again after the end of the stream. To continue, a caller
7
+ * sends the last ID in `lastEventId`. The stream can run without an end (roadmap R15).
8
+ *
9
+ * The call waits for the status of the response. A refused host, a failed connection, a status
10
+ * outside 200 to 299 and another content type fail the call itself, before the first item.
11
+ */
12
+
13
+ import { HttpError } from "../../errors.js";
14
+ import { DEFAULT_ITEM_BYTES, DEFAULT_SETTINGS, exchange, headerText, isOk, prepare, statusError, touching, type Exchange, type HttpSettings } from "../../exchange.js";
15
+ import { sseEvents } from "../../parsers.js";
16
+ import { CallScope, type ScopeContext } from "../../scope.js";
17
+ import type { HttpSseInput, HttpSseOutput } from "../../types.js";
18
+
19
+ const PATH = ["http", "sse"];
20
+
21
+ /** The events of an open response. The end of the generator closes the connection. */
22
+ async function* events(opened: Exchange, scope: CallScope, maxEventBytes: number): AsyncGenerator<HttpSseOutput, void, undefined> {
23
+ try {
24
+ scope.touch();
25
+ for await (const event of sseEvents(touching(opened.body, scope), maxEventBytes, "http.sse")) {
26
+ // The idle time does not run while the reader holds the item
27
+ scope.pause();
28
+ yield event;
29
+ scope.touch();
30
+ }
31
+ } catch (error) {
32
+ throw scope.translate(error);
33
+ } finally {
34
+ opened.body.destroy();
35
+ opened.close();
36
+ scope.dispose();
37
+ }
38
+ }
39
+
40
+ /**
41
+ * Read a stream of server-sent events
42
+ */
43
+ export async function httpSse(input: HttpSseInput, ctx: ScopeContext = {}, settings: HttpSettings = DEFAULT_SETTINGS): Promise<AsyncGenerator<HttpSseOutput, void, undefined>> {
44
+ const headers: Record<string, string> = { accept: "text/event-stream", "cache-control": "no-cache" };
45
+ if (input.lastEventId !== undefined) headers["last-event-id"] = input.lastEventId;
46
+ const prepared = prepare({ ...input, method: "GET" }, PATH, settings, { headers });
47
+ const scope = new CallScope(PATH, ctx, { timeoutMs: input.timeoutMs, idleTimeoutMs: input.idleTimeoutMs });
48
+ let opened: Exchange | undefined;
49
+ try {
50
+ opened = await exchange(prepared, settings.policy, scope);
51
+ const status = opened.response.statusCode ?? 0;
52
+ if (!isOk(status)) throw await statusError(opened, scope);
53
+ const type = headerText(opened.response.headers["content-type"]) ?? "";
54
+ if (!type.toLowerCase().startsWith("text/event-stream")) {
55
+ throw new HttpError("BAD_CONTENT_TYPE", `http.sse: the content type of the response is ${JSON.stringify(type)}, not text/event-stream`, status);
56
+ }
57
+ } catch (error) {
58
+ opened?.close();
59
+ scope.dispose();
60
+ throw scope.translate(error);
61
+ }
62
+ // The idle time starts again when the reader asks for the first event
63
+ scope.pause();
64
+ return events(opened, scope, input.maxEventBytes ?? DEFAULT_ITEM_BYTES);
65
+ }
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Importing this file registers the procedures of @procedures/http in the global registry:
3
+ * `import "@procedures/http/register"`. The package root registers nothing.
4
+ */
5
+
6
+ import { registerHttpProcedures } from "./module.js";
7
+
8
+ export * from "./module.js";
9
+
10
+ registerHttpProcedures();