@amritk/nish-x86_64-linux 0.27.2 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTALL.md +16 -8
- package/bin/nish +0 -0
- package/docs/AI.md +118 -26
- package/package.json +1 -1
- package/runtime/core/host.d.ts +2 -1
- package/runtime/core/host.mjs +4 -0
- package/runtime/core.d.ts +47 -0
- package/runtime/core.mjs +67 -0
- package/runtime/nish.d.ts +176 -0
- package/runtime/nish.h +120 -6
- package/runtime/nish.mjs +24 -4
- package/runtime/runtime-checksum.c +270 -0
- package/runtime/runtime-net.c +386 -5
- package/runtime/runtime-os.c +59 -8
- package/runtime/runtime-parallel.c +81 -5
- package/runtime/runtime-sandbox.c +479 -0
- package/runtime/runtime-sha256.c +239 -0
- package/runtime/runtime-shm.c +541 -0
- package/runtime/runtime.c +827 -791
- package/scripts/build.sh +79 -28
- package/std/README.md +25 -13
- package/std/collections.ts +5 -5
- package/std/compress/checksum.ts +88 -75
- package/std/compress/deflate.ts +224 -0
- package/std/compress/gzip.ts +52 -0
- package/std/crypto/bcrypt.ts +816 -280
- package/std/crypto/hmac.ts +305 -2
- package/std/crypto/jwt.ts +1661 -0
- package/std/json-equal.ts +335 -0
- package/std/json-format.ts +1070 -0
- package/std/json-number.ts +428 -0
- package/std/json-pointer.ts +245 -0
- package/std/json-typed.ts +34 -0
- package/std/json-value.ts +510 -17
- package/std/json.ts +35 -6
- package/std/net/handler.ts +1348 -0
- package/std/net/http-client.ts +97 -16
- package/std/net/http-compress.ts +301 -16
- package/std/net/http1-client.ts +104 -19
- package/std/net/http1-server.ts +127 -24
- package/std/net/http1.ts +24 -4
- package/std/net/outbound.ts +409 -0
- package/std/net/postgres.ts +405 -0
- package/std/net/prefork.ts +344 -20
- package/std/net/websocket.ts +295 -14
- package/std/regex.ts +613 -52
- package/std/secret.ts +1 -1
- package/std/sync/ring.ts +587 -0
- package/std/sync/shm.ts +284 -0
- package/std/term-io.ts +35 -8
- package/std/term-rate.ts +13 -1
- package/std/threads.ts +35 -3
- package/std/yaml-json.ts +841 -0
- /package/runtime/{LICENSE-ryu → LICENSE-zmij} +0 -0
package/std/net/http-client.ts
CHANGED
|
@@ -83,6 +83,7 @@ import { Url, urlDecode, urlDefaultPort, urlOrigin, urlParse, urlResolve } from
|
|
|
83
83
|
import { resolveHost } from "nish/net/dns"
|
|
84
84
|
import { DNS_ADDRESS_BYTES } from "nish/net/dns-message"
|
|
85
85
|
import { DNS_RESOLVE_MAX_ADDRESSES, DNS_RESOLVE_TIMEOUT } from "nish/net/dns-resolver"
|
|
86
|
+
import { OutboundPolicy, outboundAddressText, outboundFilter, outboundHostAllowed } from "nish/net/outbound"
|
|
86
87
|
import {
|
|
87
88
|
connectResult,
|
|
88
89
|
netClose,
|
|
@@ -136,6 +137,8 @@ export const HTTP_E_REQUEST: i32 = 14
|
|
|
136
137
|
export const HTTP_E_IO: i32 = 15
|
|
137
138
|
/** A proxy setting that is not an `http://` URL with a host. */
|
|
138
139
|
export const HTTP_E_PROXY: i32 = 16
|
|
140
|
+
/** The outbound policy refused the URL's host or every address it resolved to. */
|
|
141
|
+
export const HTTP_E_BLOCKED: i32 = 17
|
|
139
142
|
|
|
140
143
|
/** A typed zero: a bare literal is an `f64` under `--number-mode f64`. */
|
|
141
144
|
const HTTP_CLIENT_ZERO: i32 = 0
|
|
@@ -185,6 +188,7 @@ export const httpErrorName = (code: i32): string => {
|
|
|
185
188
|
"request",
|
|
186
189
|
"io",
|
|
187
190
|
"proxy",
|
|
191
|
+
"blocked",
|
|
188
192
|
]
|
|
189
193
|
return code >= 0 && code < toI32(names.length) ? names[code] : "unknown"
|
|
190
194
|
}
|
|
@@ -230,6 +234,12 @@ export class HttpOptions {
|
|
|
230
234
|
maxBodyBytes: i64 = 268435456
|
|
231
235
|
/** A PEM file of the roots an https server's chain must end at; null for the system's. */
|
|
232
236
|
rootsPemPath: string | null = null
|
|
237
|
+
/**
|
|
238
|
+
* For a URL the program did not choose: only the addresses and hosts the
|
|
239
|
+
* policy allows are reached, no proxy is used, and its caps apply (see
|
|
240
|
+
* `nish/net/outbound`). Null for none.
|
|
241
|
+
*/
|
|
242
|
+
outbound: OutboundPolicy | null = null
|
|
233
243
|
/** The budget for the lookup and the connect together; 0 or less for none but the total. */
|
|
234
244
|
connectTimeoutMs: i32 = 10000
|
|
235
245
|
/** The longest wait for the server to send or take a byte; 0 or less for none. */
|
|
@@ -371,6 +381,8 @@ export class HttpClient {
|
|
|
371
381
|
noProxy: string = ""
|
|
372
382
|
/** Where `fd` leads: an origin, or the proxy an http request went through. */
|
|
373
383
|
route: string = ""
|
|
384
|
+
/** The outbound policy `fd` was dialled under; a connection is reused only under the same one. */
|
|
385
|
+
routePolicy: OutboundPolicy | null = null
|
|
374
386
|
parser: Http1ResponseParser
|
|
375
387
|
/** The TLS connection over `fd` to an https origin; null over plain http. */
|
|
376
388
|
tls: TlsRecordClient | null = null
|
|
@@ -442,13 +454,18 @@ export class HttpClient {
|
|
|
442
454
|
this.drop()
|
|
443
455
|
}
|
|
444
456
|
this.generation += 1
|
|
445
|
-
const total: i32 = this.options
|
|
457
|
+
const total: i32 = httpClientTotalMs(this.options)
|
|
446
458
|
this.deadline = total > 0 ? monotonicNanos() + toI64(total) * HTTP_CLIENT_NS_PER_MS : HTTP_CLIENT_NEVER
|
|
447
459
|
let url: Url | null = urlParse(req.url)
|
|
448
460
|
if (url === null || (url.scheme !== "http" && url.scheme !== "https")) {
|
|
449
461
|
this.fail(HTTP_E_URL, `not an absolute http or https URL: ${httpClientRedact(url)}`)
|
|
450
462
|
return null
|
|
451
463
|
}
|
|
464
|
+
const refusal: string = this.hostRefusal(url)
|
|
465
|
+
if (refusal !== "") {
|
|
466
|
+
this.fail(HTTP_E_BLOCKED, refusal)
|
|
467
|
+
return null
|
|
468
|
+
}
|
|
452
469
|
let method: string = req.method
|
|
453
470
|
let body: u8[] | null = req.body
|
|
454
471
|
const names: string[] = []
|
|
@@ -468,10 +485,11 @@ export class HttpClient {
|
|
|
468
485
|
return res
|
|
469
486
|
}
|
|
470
487
|
const next: Url | null = urlResolve(url, location)
|
|
471
|
-
|
|
488
|
+
const maxRedirects: i32 = httpClientMaxRedirects(this.options)
|
|
489
|
+
if (hops >= maxRedirects) {
|
|
472
490
|
return this.refuse(
|
|
473
491
|
HTTP_E_TOO_MANY_REDIRECTS,
|
|
474
|
-
`more than ${
|
|
492
|
+
`more than ${maxRedirects} redirects, the last to ${httpClientRedact(next)}`
|
|
475
493
|
)
|
|
476
494
|
}
|
|
477
495
|
if (next === null || (next.scheme !== "http" && next.scheme !== "https")) {
|
|
@@ -486,6 +504,10 @@ export class HttpClient {
|
|
|
486
504
|
`a redirect from https to http (${httpClientRedact(next)})`
|
|
487
505
|
)
|
|
488
506
|
}
|
|
507
|
+
const hopRefusal: string = this.hostRefusal(next)
|
|
508
|
+
if (hopRefusal !== "") {
|
|
509
|
+
return this.refuse(HTTP_E_BLOCKED, hopRefusal)
|
|
510
|
+
}
|
|
489
511
|
const from: string | null = urlOrigin(url)
|
|
490
512
|
const to: string | null = urlOrigin(next)
|
|
491
513
|
if (from === null || to === null || from !== to) {
|
|
@@ -517,6 +539,15 @@ export class HttpClient {
|
|
|
517
539
|
return null
|
|
518
540
|
}
|
|
519
541
|
|
|
542
|
+
/** Why the outbound policy refuses to let a URL name `url`'s host; "" when there is no policy or it allows the host. */
|
|
543
|
+
hostRefusal(url: Url): string {
|
|
544
|
+
const policy: OutboundPolicy | null = this.options.outbound
|
|
545
|
+
if (policy === null || outboundHostAllowed(policy, url.host)) {
|
|
546
|
+
return ""
|
|
547
|
+
}
|
|
548
|
+
return `${httpClientRedact(url)} names a host the outbound policy does not allow`
|
|
549
|
+
}
|
|
550
|
+
|
|
520
551
|
/** Closes the connection; the client can still fetch, on a new one. */
|
|
521
552
|
close(): void {
|
|
522
553
|
this.drop()
|
|
@@ -564,14 +595,11 @@ export class HttpClient {
|
|
|
564
595
|
*/
|
|
565
596
|
open(url: Url): boolean {
|
|
566
597
|
const port: i32 = url.port >= 0 ? url.port : urlDefaultPort(url.scheme)
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
this.httpsProxy,
|
|
573
|
-
this.noProxy
|
|
574
|
-
)
|
|
598
|
+
// A proxy would look the name up itself, past the policy's address check.
|
|
599
|
+
const proxyText: string =
|
|
600
|
+
this.options.outbound !== null
|
|
601
|
+
? ""
|
|
602
|
+
: httpProxyFor(url.scheme, url.host, port, this.httpProxy, this.httpsProxy, this.noProxy)
|
|
575
603
|
let proxy: Url | null = null
|
|
576
604
|
if (proxyText !== "") {
|
|
577
605
|
proxy = httpClientProxyUrl(proxyText)
|
|
@@ -592,7 +620,12 @@ export class HttpClient {
|
|
|
592
620
|
} else if (proxy !== null) {
|
|
593
621
|
route = `${origin} via ${httpClientHostText(proxy.host)}:${proxy.port}`
|
|
594
622
|
}
|
|
595
|
-
if (
|
|
623
|
+
if (
|
|
624
|
+
this.fd >= 0 &&
|
|
625
|
+
this.route === route &&
|
|
626
|
+
httpClientSamePolicy(this.routePolicy, this.options.outbound) &&
|
|
627
|
+
this.quiet()
|
|
628
|
+
) {
|
|
596
629
|
this.reused = true
|
|
597
630
|
return true
|
|
598
631
|
}
|
|
@@ -610,6 +643,7 @@ export class HttpClient {
|
|
|
610
643
|
}
|
|
611
644
|
this.fd = fd
|
|
612
645
|
this.route = route
|
|
646
|
+
this.routePolicy = this.options.outbound
|
|
613
647
|
this.parser = httpClientParser(this.options)
|
|
614
648
|
if (url.scheme === "https") {
|
|
615
649
|
let early: u8[] = []
|
|
@@ -863,7 +897,7 @@ export class HttpClient {
|
|
|
863
897
|
if (code === HTTP1_E_BODY_LIMIT) {
|
|
864
898
|
this.fail(
|
|
865
899
|
HTTP_E_BODY_TOO_LARGE,
|
|
866
|
-
`the body is larger than maxBodyBytes (${this.options
|
|
900
|
+
`the body is larger than maxBodyBytes (${httpClientBodyCap(this.options)})`
|
|
867
901
|
)
|
|
868
902
|
} else {
|
|
869
903
|
// A response the server closed part way through is a connection lost,
|
|
@@ -1080,9 +1114,22 @@ export class HttpClient {
|
|
|
1080
1114
|
)
|
|
1081
1115
|
return -1
|
|
1082
1116
|
}
|
|
1117
|
+
let usable: i32 = count < DNS_RESOLVE_MAX_ADDRESSES ? count : DNS_RESOLVE_MAX_ADDRESSES
|
|
1118
|
+
const policy: OutboundPolicy | null = this.options.outbound
|
|
1119
|
+
if (policy !== null) {
|
|
1120
|
+
// Only the bytes checked here are connected to: no second lookup a rebinding answer could change.
|
|
1121
|
+
usable = outboundFilter(policy, addresses, usable)
|
|
1122
|
+
if (usable === 0) {
|
|
1123
|
+
this.fail(
|
|
1124
|
+
HTTP_E_BLOCKED,
|
|
1125
|
+
`${host} resolves to ${outboundAddressText(addresses, 0)}, which the outbound policy does not allow`
|
|
1126
|
+
)
|
|
1127
|
+
return -1
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1083
1130
|
const one: u8[] = new Array<u8>(DNS_ADDRESS_BYTES)
|
|
1084
1131
|
let last: i32 = 0
|
|
1085
|
-
for (let i: i32 = 0; i <
|
|
1132
|
+
for (let i: i32 = 0; i < usable && i < DNS_RESOLVE_MAX_ADDRESSES; i += 1) {
|
|
1086
1133
|
copyBytes(one, 0, addresses, i * DNS_ADDRESS_BYTES, DNS_ADDRESS_BYTES)
|
|
1087
1134
|
const fd: i32 = tcpConnect(one)
|
|
1088
1135
|
if (fd < 0) {
|
|
@@ -1320,7 +1367,7 @@ export class HttpClient {
|
|
|
1320
1367
|
/** Sets the error for a wait that ended with `code`, `text` saying what timed out. */
|
|
1321
1368
|
failWait(code: i32, text: string): void {
|
|
1322
1369
|
if (code === HTTP_E_TOTAL_TIMEOUT) {
|
|
1323
|
-
this.fail(code, `the fetch ran past ${this.options
|
|
1370
|
+
this.fail(code, `the fetch ran past ${httpClientTotalMs(this.options)} ms`)
|
|
1324
1371
|
} else if (code === HTTP_E_IO) {
|
|
1325
1372
|
this.fail(code, "the poll loop failed")
|
|
1326
1373
|
} else {
|
|
@@ -1468,10 +1515,44 @@ const httpClientParser = (options: HttpOptions): Http1ResponseParser =>
|
|
|
1468
1515
|
options.maxHeaderBytes,
|
|
1469
1516
|
options.maxHeaderBytes,
|
|
1470
1517
|
HTTP_CLIENT_MAX_HEADERS,
|
|
1471
|
-
options
|
|
1518
|
+
httpClientBodyCap(options),
|
|
1472
1519
|
options.maxHeaderBytes
|
|
1473
1520
|
)
|
|
1474
1521
|
|
|
1522
|
+
/** Whether `a` and `b` are the same outbound policy, or both none. */
|
|
1523
|
+
const httpClientSamePolicy = (a: OutboundPolicy | null, b: OutboundPolicy | null): boolean => {
|
|
1524
|
+
if (a === null || b === null) {
|
|
1525
|
+
return a === null && b === null
|
|
1526
|
+
}
|
|
1527
|
+
return a === b
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
/** `maxBodyBytes`, or the outbound policy's cap when it is lower. */
|
|
1531
|
+
const httpClientBodyCap = (options: HttpOptions): i64 => {
|
|
1532
|
+
const policy: OutboundPolicy | null = options.outbound
|
|
1533
|
+
return policy !== null && policy.maxBodyBytes < options.maxBodyBytes
|
|
1534
|
+
? policy.maxBodyBytes
|
|
1535
|
+
: options.maxBodyBytes
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
/** `totalTimeoutMs`, or the outbound policy's budget when it is set and the options' is not, or is longer. */
|
|
1539
|
+
const httpClientTotalMs = (options: HttpOptions): i32 => {
|
|
1540
|
+
const policy: OutboundPolicy | null = options.outbound
|
|
1541
|
+
const own: i32 = options.totalTimeoutMs
|
|
1542
|
+
if (policy === null || policy.totalTimeoutMs <= 0) {
|
|
1543
|
+
return own
|
|
1544
|
+
}
|
|
1545
|
+
return own > 0 && own < policy.totalTimeoutMs ? own : policy.totalTimeoutMs
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
/** `maxRedirects`, or the outbound policy's limit when it is lower. */
|
|
1549
|
+
const httpClientMaxRedirects = (options: HttpOptions): i32 => {
|
|
1550
|
+
const policy: OutboundPolicy | null = options.outbound
|
|
1551
|
+
return policy !== null && policy.maxRedirects < options.maxRedirects
|
|
1552
|
+
? policy.maxRedirects
|
|
1553
|
+
: options.maxRedirects
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1475
1556
|
/** The statuses a redirect is followed for: 301, 302, 303, 307 and 308. */
|
|
1476
1557
|
const httpClientIsRedirect = (status: i32): boolean =>
|
|
1477
1558
|
status === 301 || status === 302 || status === 303 || status === 307 || status === 308
|
package/std/net/http-compress.ts
CHANGED
|
@@ -23,6 +23,32 @@
|
|
|
23
23
|
* its `minLength`, longer than its `maxLength`, or that gzip would not make
|
|
24
24
|
* smaller, answering -1 so the program sends the bytes as they are.
|
|
25
25
|
*
|
|
26
|
+
* **Compression that pays for itself.** A body is sent compressed only when
|
|
27
|
+
* it saves at least `minSaving` bytes and `minSavingPercent` percent of the
|
|
28
|
+
* body; otherwise the compressed bytes are dropped and the body goes as it
|
|
29
|
+
* is. With `segmentMode` set, `respondEncoded` also sends it compressed only
|
|
30
|
+
* when the whole response, head included, takes fewer TCP segments of `mss`
|
|
31
|
+
* bytes (the connection's `mss` if set, else the compressor's, 1448 by
|
|
32
|
+
* default): a 1300-byte body that gzips to 700 still fits in one segment
|
|
33
|
+
* either way. After each call `decision` says why (`HTTP_COMPRESS_SENT`,
|
|
34
|
+
* `_NOT_ACCEPTED`, `_TOO_SMALL`, `_TOO_LARGE`, `_TOO_LITTLE_SAVED`,
|
|
35
|
+
* `_NO_FEWER_SEGMENTS`), for a program to count. A response whose identity
|
|
36
|
+
* form already fits one segment is `_NO_FEWER_SEGMENTS` without running gzip
|
|
37
|
+
* at all, as no compression could take fewer.
|
|
38
|
+
*
|
|
39
|
+
* **Two encoders.** Bodies up to `fixedLimit` (16 KB) go through
|
|
40
|
+
* `GzipFixedCompressor`: one fixed-Huffman block, a 16 KB window, no state
|
|
41
|
+
* between calls, two to four times faster than `DEFLATE_FAST` on 0.5 to
|
|
42
|
+
* 16 KB of JSON for output about 1.35 times larger. Larger bodies go through
|
|
43
|
+
* the `GzipCompressor` at the level given; `encoder` forces either one.
|
|
44
|
+
*
|
|
45
|
+
* **`Vary: Accept-Encoding`** (with `vary`, on by default) goes on every
|
|
46
|
+
* response whose coding depended on the request: one sent compressed, and
|
|
47
|
+
* one sent as it is because the request did not accept gzip. A body that is
|
|
48
|
+
* too small, too large or did not pay is sent the same to every client, so
|
|
49
|
+
* it carries no `Vary`. A request that refuses identity (`identity;q=0`)
|
|
50
|
+
* and accepts gzip is sent gzip whatever the policy (`compressAnyway`).
|
|
51
|
+
*
|
|
26
52
|
* **Negotiation** (`httpAcceptEncoding`) follows §12.5.3: each listed coding
|
|
27
53
|
* with its q-value, `x-gzip` as `gzip`, `*` for every coding not listed, and
|
|
28
54
|
* `identity` acceptable unless it is given q=0, directly or through `*`. Of
|
|
@@ -35,15 +61,17 @@
|
|
|
35
61
|
* and identity is refused: a server may answer 406 or send identity anyway.
|
|
36
62
|
* A malformed element is skipped. The parser allocates nothing.
|
|
37
63
|
*
|
|
38
|
-
* **Compressing allocates nothing
|
|
39
|
-
* `
|
|
64
|
+
* **Compressing allocates nothing** and caches nothing between responses:
|
|
65
|
+
* `HttpCompressor` holds a `GzipCompressor`, a `GzipFixedCompressor`, an
|
|
66
|
+
* output buffer of `compressBound(maxLength)` bytes and
|
|
40
67
|
* the field arrays `addFields` fills, all built once.
|
|
41
68
|
* `HTTP_CODING_DEFLATE` (zlib framing) is negotiated but not produced: a
|
|
42
69
|
* client that accepts only `deflate` is sent identity, which every client
|
|
43
70
|
* reads.
|
|
44
71
|
*/
|
|
45
72
|
import { DEFLATE_FAST, compressBound } from "nish/compress/deflate"
|
|
46
|
-
import { GzipCompressor } from "nish/compress/gzip"
|
|
73
|
+
import { GzipCompressor, GzipFixedCompressor } from "nish/compress/gzip"
|
|
74
|
+
import { wrappingAdd } from "nish:unsafe"
|
|
47
75
|
|
|
48
76
|
/** No content coding: the body as it is. */
|
|
49
77
|
export const HTTP_CODING_IDENTITY: i32 = 0
|
|
@@ -61,6 +89,63 @@ export const HTTP_CODING_NONE: i32 = -1
|
|
|
61
89
|
export const HTTP_OFFER_GZIP: i32 = 1
|
|
62
90
|
export const HTTP_OFFER_DEFLATE: i32 = 2
|
|
63
91
|
|
|
92
|
+
/**
|
|
93
|
+
* What `HttpCompressor.compress` (and `respondEncoded`'s segment check) last
|
|
94
|
+
* decided, in `HttpCompressor.decision`, for a program to count: the body
|
|
95
|
+
* went out compressed; the client did not take gzip; it was shorter than
|
|
96
|
+
* `minLength`; longer than `maxLength`; gzip did not save `minSaving` bytes
|
|
97
|
+
* and `minSavingPercent` percent; or the compressed response would not have
|
|
98
|
+
* taken fewer TCP segments (`segmentMode`), which includes an identity
|
|
99
|
+
* response that already fits one segment, decided without compressing.
|
|
100
|
+
*/
|
|
101
|
+
export const HTTP_COMPRESS_SENT: i32 = 0
|
|
102
|
+
export const HTTP_COMPRESS_NOT_ACCEPTED: i32 = 1
|
|
103
|
+
export const HTTP_COMPRESS_TOO_SMALL: i32 = 2
|
|
104
|
+
export const HTTP_COMPRESS_TOO_LARGE: i32 = 3
|
|
105
|
+
export const HTTP_COMPRESS_TOO_LITTLE_SAVED: i32 = 4
|
|
106
|
+
export const HTTP_COMPRESS_NO_FEWER_SEGMENTS: i32 = 5
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* `HttpCompressor.encoder`: by the body's length (the fixed-Huffman encoder up
|
|
110
|
+
* to `fixedLimit` bytes, the constructor's level above), the fixed-Huffman
|
|
111
|
+
* encoder always, or the constructor's level always.
|
|
112
|
+
*/
|
|
113
|
+
export const HTTP_ENCODER_AUTO: i32 = 0
|
|
114
|
+
export const HTTP_ENCODER_FIXED: i32 = 1
|
|
115
|
+
export const HTTP_ENCODER_LEVEL: i32 = 2
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* The maximum segment size `segmentMode` counts with unless told the
|
|
119
|
+
* connection's: a 1500-byte Ethernet MTU less 20 bytes of IPv4 header, 20 of
|
|
120
|
+
* TCP header and 12 of the timestamp option.
|
|
121
|
+
*/
|
|
122
|
+
export const HTTP_DEFAULT_MSS: i32 = 1448
|
|
123
|
+
|
|
124
|
+
/** The bytes `content-encoding: gzip` adds to an HTTP/1.1 head: name, value, ": " and CRLF. */
|
|
125
|
+
export const HTTP1_GZIP_FIELD_BYTES: i32 = 24
|
|
126
|
+
|
|
127
|
+
/** The number of TCP segments `bytes` bytes take at `mss` bytes a segment. */
|
|
128
|
+
export const httpSegments = (bytes: i32, mss: i32): i32 => {
|
|
129
|
+
if (mss <= 0) {
|
|
130
|
+
panic("httpSegments: a segment size of zero or less")
|
|
131
|
+
}
|
|
132
|
+
if (bytes <= 0) {
|
|
133
|
+
return 0
|
|
134
|
+
}
|
|
135
|
+
return (bytes - 1) / mss + 1
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** The digits of `n` in decimal, `n` 0 or more. */
|
|
139
|
+
const httpDecimalDigits = (n: i32): i32 => {
|
|
140
|
+
let digits: i32 = 1
|
|
141
|
+
let rest: i32 = n
|
|
142
|
+
while (rest >= 10) {
|
|
143
|
+
rest = rest / 10
|
|
144
|
+
digits += 1
|
|
145
|
+
}
|
|
146
|
+
return digits
|
|
147
|
+
}
|
|
148
|
+
|
|
64
149
|
/** A q-value of 1, in thousandths. */
|
|
65
150
|
const HTTP_Q_ONE: i32 = 1000
|
|
66
151
|
|
|
@@ -255,6 +340,15 @@ export const httpAcceptEncoding = (buf: u8[], start: i32, end: i32, offered: i32
|
|
|
255
340
|
return HTTP_CODING_NONE
|
|
256
341
|
}
|
|
257
342
|
|
|
343
|
+
/**
|
|
344
|
+
* Whether the `Accept-Encoding` value in `buf[start .. end)` refuses the
|
|
345
|
+
* body as it is: `identity;q=0`, or `*;q=0` with no `identity` listed. A
|
|
346
|
+
* server should then send a coding the client takes even where it would not
|
|
347
|
+
* bother (`HttpCompressor.compressAnyway`).
|
|
348
|
+
*/
|
|
349
|
+
export const httpIdentityRefused = (buf: u8[], start: i32, end: i32): boolean =>
|
|
350
|
+
httpAcceptEncoding(buf, start, end, 0) === HTTP_CODING_NONE
|
|
351
|
+
|
|
258
352
|
/** The bytes of an ASCII string. */
|
|
259
353
|
const httpCompressBytes = (text: string): u8[] => {
|
|
260
354
|
const out: u8[] = new Array<u8>(toI32(text.length))
|
|
@@ -270,6 +364,8 @@ const httpCompressBytes = (text: string): u8[] => {
|
|
|
270
364
|
*/
|
|
271
365
|
export class HttpCompressor {
|
|
272
366
|
readonly gzip: GzipCompressor
|
|
367
|
+
/** The small-body encoder: one fixed-Huffman block, nothing kept between calls. */
|
|
368
|
+
readonly fixed: GzipFixedCompressor
|
|
273
369
|
/** Where `compress` writes; valid until the next call. */
|
|
274
370
|
readonly output: u8[]
|
|
275
371
|
/** The fields `addFields` built: the program's, then `Content-Encoding` and `Vary`. */
|
|
@@ -282,12 +378,40 @@ export class HttpCompressor {
|
|
|
282
378
|
/** The shortest body worth compressing, and the longest the output can hold. */
|
|
283
379
|
readonly minLength: i32
|
|
284
380
|
readonly maxLength: i32
|
|
381
|
+
/**
|
|
382
|
+
* The least a compressed body must save to be sent: `minSaving` bytes and
|
|
383
|
+
* `minSavingPercent` percent of the body, whichever is more. 1 and 0 by
|
|
384
|
+
* default (any saving at all); a value below those counts as them.
|
|
385
|
+
*/
|
|
386
|
+
minSaving: i32
|
|
387
|
+
minSavingPercent: i32
|
|
285
388
|
/**
|
|
286
389
|
* Whether `addFields` adds `Vary: Accept-Encoding` (RFC 9110 §12.5.5), which
|
|
287
390
|
* a cache needs whenever the same resource may be sent either way. On by
|
|
288
391
|
* default; turn it off when the program sets its own `Vary`.
|
|
289
392
|
*/
|
|
290
393
|
vary: boolean
|
|
394
|
+
/**
|
|
395
|
+
* Whether `respondEncoded` also sends the compressed response only when,
|
|
396
|
+
* head included, it takes fewer TCP segments of `mss` bytes than the body
|
|
397
|
+
* as it is; off by default. `mss` is `HTTP_DEFAULT_MSS` unless the program
|
|
398
|
+
* sets it, or a connection's own `mss` overrides it.
|
|
399
|
+
*/
|
|
400
|
+
segmentMode: boolean
|
|
401
|
+
mss: i32
|
|
402
|
+
/** `HTTP_ENCODER_AUTO` (the default), `HTTP_ENCODER_FIXED` or `HTTP_ENCODER_LEVEL`. */
|
|
403
|
+
encoder: i32
|
|
404
|
+
/** The longest body `HTTP_ENCODER_AUTO` gives the fixed-Huffman encoder: 16384 by default. */
|
|
405
|
+
fixedLimit: i32
|
|
406
|
+
/** The last decision, an `HTTP_COMPRESS_*`; `HTTP_COMPRESS_NOT_ACCEPTED` before any. */
|
|
407
|
+
decision: i32
|
|
408
|
+
/**
|
|
409
|
+
* How many times `compress` and `compressAnyway` ran an encoder, from 0: a
|
|
410
|
+
* count for a test or a profile to read, never consulted by a decision. It
|
|
411
|
+
* wraps past 2^31 - 1 to -2^31 rather than panic, as a server can run that
|
|
412
|
+
* many; read it as a difference between two reads.
|
|
413
|
+
*/
|
|
414
|
+
encoderRuns: i32
|
|
291
415
|
|
|
292
416
|
/**
|
|
293
417
|
* A compressor at `level` (`DEFLATE_FAST` suits a server) for bodies of
|
|
@@ -299,6 +423,7 @@ export class HttpCompressor {
|
|
|
299
423
|
panic("HttpCompressor: the lengths are out of order")
|
|
300
424
|
}
|
|
301
425
|
this.gzip = new GzipCompressor(level)
|
|
426
|
+
this.fixed = new GzipFixedCompressor()
|
|
302
427
|
this.output = new Array<u8>(compressBound(maxLength))
|
|
303
428
|
this.names = []
|
|
304
429
|
this.values = []
|
|
@@ -309,28 +434,182 @@ export class HttpCompressor {
|
|
|
309
434
|
this.minLength = minLength
|
|
310
435
|
this.maxLength = maxLength
|
|
311
436
|
this.vary = true
|
|
437
|
+
this.minSaving = 1
|
|
438
|
+
this.minSavingPercent = 0
|
|
439
|
+
this.segmentMode = false
|
|
440
|
+
this.mss = HTTP_DEFAULT_MSS
|
|
441
|
+
this.encoder = HTTP_ENCODER_AUTO
|
|
442
|
+
this.fixedLimit = 16384
|
|
443
|
+
this.decision = HTTP_COMPRESS_NOT_ACCEPTED
|
|
444
|
+
this.encoderRuns = 0
|
|
312
445
|
}
|
|
313
446
|
|
|
314
447
|
/**
|
|
315
|
-
* Compresses `body[off .. off + len)` into `output` when
|
|
316
|
-
* `
|
|
317
|
-
*
|
|
318
|
-
* as it is
|
|
448
|
+
* Compresses `body[off .. off + len)` into `output` when the length is in
|
|
449
|
+
* `[minLength, maxLength]`, `coding` is `HTTP_CODING_GZIP` and gzip saves
|
|
450
|
+
* what `minSaving` and `minSavingPercent` ask; answers the compressed
|
|
451
|
+
* length, or -1 to send the body as it is, and sets `decision` either way.
|
|
452
|
+
* `encoder` picks the encoder; the fixed-Huffman one stops as soon as its
|
|
453
|
+
* output passes what would be kept. A window outside `body` panics.
|
|
319
454
|
*/
|
|
320
455
|
compress(coding: i32, body: u8[], off: i32, len: i32): i32 {
|
|
321
456
|
if (off < 0 || len < 0 || off > toI32(body.length) - len) {
|
|
322
457
|
panic("HttpCompressor.compress: the window lies outside its body")
|
|
323
458
|
}
|
|
324
|
-
if (
|
|
459
|
+
if (len < this.minLength) {
|
|
460
|
+
this.decision = HTTP_COMPRESS_TOO_SMALL
|
|
461
|
+
return -1
|
|
462
|
+
}
|
|
463
|
+
if (len > this.maxLength) {
|
|
464
|
+
this.decision = HTTP_COMPRESS_TOO_LARGE
|
|
465
|
+
return -1
|
|
466
|
+
}
|
|
467
|
+
if (coding !== HTTP_CODING_GZIP) {
|
|
468
|
+
this.decision = HTTP_COMPRESS_NOT_ACCEPTED
|
|
469
|
+
return -1
|
|
470
|
+
}
|
|
471
|
+
// The longest output kept; `output` holds `compressBound(maxLength)`, more.
|
|
472
|
+
const keep: i32 = len - this.leastSaving(len)
|
|
473
|
+
let n: i32 = -1
|
|
474
|
+
if (keep > 0) {
|
|
475
|
+
const fixed: boolean =
|
|
476
|
+
this.encoder === HTTP_ENCODER_FIXED || (this.encoder !== HTTP_ENCODER_LEVEL && len <= this.fixedLimit)
|
|
477
|
+
this.encoderRuns = wrappingAdd(this.encoderRuns, 1)
|
|
478
|
+
n = fixed
|
|
479
|
+
? this.fixed.compress(body, off, len, this.output, toI32(0), keep)
|
|
480
|
+
: this.gzip.compress(body, off, len, this.output, toI32(0))
|
|
481
|
+
}
|
|
482
|
+
if (n < 0 || n > keep) {
|
|
483
|
+
this.decision = HTTP_COMPRESS_TOO_LITTLE_SAVED
|
|
484
|
+
return -1
|
|
485
|
+
}
|
|
486
|
+
this.decision = HTTP_COMPRESS_SENT
|
|
487
|
+
return n
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* `compress` for a client that refuses identity (`httpIdentityRefused`):
|
|
492
|
+
* gzip whatever the body's length, the saving or the
|
|
493
|
+
* segments, so long as it is no longer than `maxLength`. Answers the
|
|
494
|
+
* compressed length, or -1 when `coding` is not gzip or the body is too
|
|
495
|
+
* long (sent as it is, which RFC 9110 §12.5.3 allows).
|
|
496
|
+
*/
|
|
497
|
+
compressAnyway(coding: i32, body: u8[], off: i32, len: i32): i32 {
|
|
498
|
+
if (off < 0 || len < 0 || off > toI32(body.length) - len) {
|
|
499
|
+
panic("HttpCompressor.compressAnyway: the window lies outside its body")
|
|
500
|
+
}
|
|
501
|
+
if (len > this.maxLength) {
|
|
502
|
+
this.decision = HTTP_COMPRESS_TOO_LARGE
|
|
325
503
|
return -1
|
|
326
504
|
}
|
|
327
|
-
|
|
328
|
-
|
|
505
|
+
if (coding !== HTTP_CODING_GZIP) {
|
|
506
|
+
this.decision = HTTP_COMPRESS_NOT_ACCEPTED
|
|
329
507
|
return -1
|
|
330
508
|
}
|
|
509
|
+
this.encoderRuns = wrappingAdd(this.encoderRuns, 1)
|
|
510
|
+
let n: i32 = -1
|
|
511
|
+
if (
|
|
512
|
+
this.encoder === HTTP_ENCODER_FIXED ||
|
|
513
|
+
(this.encoder !== HTTP_ENCODER_LEVEL && len <= this.fixedLimit)
|
|
514
|
+
) {
|
|
515
|
+
n = this.fixed.compress(body, off, len, this.output, toI32(0), toI32(this.output.length))
|
|
516
|
+
}
|
|
517
|
+
if (n < 0) {
|
|
518
|
+
n = this.gzip.compress(body, off, len, this.output, toI32(0))
|
|
519
|
+
}
|
|
520
|
+
this.decision = HTTP_COMPRESS_SENT
|
|
331
521
|
return n
|
|
332
522
|
}
|
|
333
523
|
|
|
524
|
+
/** The bytes a compressed `len`-byte body must save: `minSaving` and `minSavingPercent`, at least 1. */
|
|
525
|
+
leastSaving(len: i32): i32 {
|
|
526
|
+
let least: i32 = this.minSaving > 1 ? this.minSaving : 1
|
|
527
|
+
let pct: i32 = this.minSavingPercent
|
|
528
|
+
if (pct < 0) {
|
|
529
|
+
pct = 0
|
|
530
|
+
} else if (pct > 100) {
|
|
531
|
+
pct = 100
|
|
532
|
+
}
|
|
533
|
+
const share: i32 = toI32((toI64(len) * toI64(pct) + toI64(99)) / toI64(100))
|
|
534
|
+
if (share > least) {
|
|
535
|
+
least = share
|
|
536
|
+
}
|
|
537
|
+
return least
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* `segmentMode`'s rule, for a response of `identityBytes` bytes sent as it
|
|
542
|
+
* is or `encodedBytes` compressed, heads included, at `mss` bytes a segment
|
|
543
|
+
* (`this.mss` when 0 or less): whether the compressed one takes fewer
|
|
544
|
+
* segments. When it does not, `decision` becomes
|
|
545
|
+
* `HTTP_COMPRESS_NO_FEWER_SEGMENTS`. `respondEncoded` calls it; a program
|
|
546
|
+
* driving HTTP/2 or HTTP/3 by hand may with its own head sizes.
|
|
547
|
+
*/
|
|
548
|
+
fewerSegments(identityBytes: i32, encodedBytes: i32, mss: i32): boolean {
|
|
549
|
+
const size: i32 = this.segmentSize(mss)
|
|
550
|
+
if (httpSegments(encodedBytes, size) < httpSegments(identityBytes, size)) {
|
|
551
|
+
return true
|
|
552
|
+
}
|
|
553
|
+
this.decision = HTTP_COMPRESS_NO_FEWER_SEGMENTS
|
|
554
|
+
return false
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/** The segment `segmentMode` counts in: `mss` when above 0, else `this.mss` when above 0, else `HTTP_DEFAULT_MSS`. */
|
|
558
|
+
segmentSize(mss: i32): i32 {
|
|
559
|
+
if (mss > 0) {
|
|
560
|
+
return mss
|
|
561
|
+
}
|
|
562
|
+
return this.mss > 0 ? this.mss : HTTP_DEFAULT_MSS
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* `segmentMode`'s shortcut, before any encoder runs: whether a `len`-byte
|
|
567
|
+
* body that `compress` would try (its length in `[minLength, maxLength]`
|
|
568
|
+
* and `coding` gzip) is no longer than one segment of `mss` bytes (as
|
|
569
|
+
* `fewerSegments` reads it). Sent as it is, such a response takes one
|
|
570
|
+
* segment once its head fits beside it, and no compression takes fewer, so
|
|
571
|
+
* `decision` becomes `HTTP_COMPRESS_NO_FEWER_SEGMENTS` and the caller checks
|
|
572
|
+
* the head: when it does not fit, the caller goes on to `compress`, which
|
|
573
|
+
* decides again. Anything else answers false and leaves `decision` alone.
|
|
574
|
+
*/
|
|
575
|
+
oneSegment(coding: i32, len: i32, mss: i32): boolean {
|
|
576
|
+
if (
|
|
577
|
+
coding !== HTTP_CODING_GZIP ||
|
|
578
|
+
len < this.minLength ||
|
|
579
|
+
len > this.maxLength ||
|
|
580
|
+
len > this.segmentSize(mss)
|
|
581
|
+
) {
|
|
582
|
+
return false
|
|
583
|
+
}
|
|
584
|
+
this.decision = HTTP_COMPRESS_NO_FEWER_SEGMENTS
|
|
585
|
+
return true
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/**
|
|
589
|
+
* The bytes of the HTTP/1.1 head a `length`-byte body sent as it is would
|
|
590
|
+
* get, from `encodedHead`, that of the same response sent compressed as
|
|
591
|
+
* `encodedLength` bytes with the fields `addFields` built: less
|
|
592
|
+
* `Content-Encoding: gzip`, less `Vary` (a body the segment count kept as
|
|
593
|
+
* it is would go out so to any client), and the other `Content-Length`.
|
|
594
|
+
*/
|
|
595
|
+
http1IdentityHead(encodedHead: i32, encodedLength: i32, length: i32): i32 {
|
|
596
|
+
let head: i32 = encodedHead - HTTP1_GZIP_FIELD_BYTES
|
|
597
|
+
if (this.vary) {
|
|
598
|
+
head -= toI32(this.varyName.length) + toI32(this.acceptEncodingValue.length) + 4
|
|
599
|
+
}
|
|
600
|
+
return head - httpDecimalDigits(encodedLength) + httpDecimalDigits(length)
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* Whether the last decision depended on the request's `Accept-Encoding`, so
|
|
605
|
+
* the response needs `Vary: Accept-Encoding`: a body sent compressed, or
|
|
606
|
+
* one sent as it is because the client did not take gzip. A body the
|
|
607
|
+
* lengths or the saving kept as it is would go out the same to any client.
|
|
608
|
+
*/
|
|
609
|
+
varies(): boolean {
|
|
610
|
+
return this.decision === HTTP_COMPRESS_SENT || this.decision === HTTP_COMPRESS_NOT_ACCEPTED
|
|
611
|
+
}
|
|
612
|
+
|
|
334
613
|
/**
|
|
335
614
|
* Sets `names` / `values` to the program's fields followed by
|
|
336
615
|
* `Content-Encoding: gzip` and, when `vary`, `Vary: Accept-Encoding`. The
|
|
@@ -354,21 +633,27 @@ export class HttpCompressor {
|
|
|
354
633
|
}
|
|
355
634
|
this.names.push(this.contentEncodingName)
|
|
356
635
|
this.values.push(this.gzipValue)
|
|
357
|
-
this.
|
|
636
|
+
if (this.vary) {
|
|
637
|
+
this.names.push(this.varyName)
|
|
638
|
+
this.values.push(this.acceptEncodingValue)
|
|
639
|
+
}
|
|
358
640
|
}
|
|
359
641
|
|
|
360
|
-
/**
|
|
642
|
+
/**
|
|
643
|
+
* Appends `Vary: Accept-Encoding` to the arrays `identityFields` built when
|
|
644
|
+
* `vary` is set and the last decision `varies`.
|
|
645
|
+
*/
|
|
361
646
|
addVary(): void {
|
|
362
|
-
if (this.vary) {
|
|
647
|
+
if (this.vary && this.varies()) {
|
|
363
648
|
this.names.push(this.varyName)
|
|
364
649
|
this.values.push(this.acceptEncodingValue)
|
|
365
650
|
}
|
|
366
651
|
}
|
|
367
652
|
|
|
368
653
|
/**
|
|
369
|
-
* Sets `names` / `values` to the program's fields and, when `vary
|
|
370
|
-
* `Vary: Accept-Encoding`: the fields of a body
|
|
371
|
-
* resource that is sometimes compressed.
|
|
654
|
+
* Sets `names` / `values` to the program's fields and, when `vary` and the
|
|
655
|
+
* last decision `varies`, `Vary: Accept-Encoding`: the fields of a body
|
|
656
|
+
* sent as it is from a resource that is sometimes compressed.
|
|
372
657
|
*/
|
|
373
658
|
identityFields(names: u8[][], values: u8[][]): void {
|
|
374
659
|
while (this.names.length > 0) {
|