@sproutboat/runtime 0.26.0 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/package.json +1 -1
- package/src/ingress-utf8.test.js +5 -7
- package/src/native-fetch-prelude.js +451 -3
- package/src/r2-object.test.js +16 -7
- package/src/to-bytes.test.js +1 -1
- package/src/transport-embedded.js +65 -2
- package/src/utf8-decode.test.js +4 -2
- package/test/native-byte-text.ts +165 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,43 @@
|
|
|
1
1
|
# @sproutboat/runtime
|
|
2
2
|
|
|
3
|
+
## 0.27.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 0d2d079: Bodies of megabytes no longer take hundreds of megabytes of memory (sproutd #22). Converting bytes to text and text to bytes made one string per character, so reading a 5 MiB R2 object with `text()` peaked at about 400 MiB, and R2 multipart uploads with real (5 MiB) parts ran sproutd's workers out of memory. The conversions now run in one C pass and give exactly the same results. On a standalone build, at 5 MiB:
|
|
8
|
+
|
|
9
|
+
| | before | after |
|
|
10
|
+
|---|---|---|
|
|
11
|
+
| `put()` of a string | 82 MiB | 16 MiB |
|
|
12
|
+
| `put()` then `get()` and `text()` | 400 MiB | 26 MiB |
|
|
13
|
+
| multipart upload, then `get()` and `text()` | 396 MiB | 21 MiB |
|
|
14
|
+
| `request.text()` of the request body | 402 MiB | 24 MiB |
|
|
15
|
+
|
|
16
|
+
Also:
|
|
17
|
+
|
|
18
|
+
- At every date, `uploadPart()` stores an `ArrayBuffer` or a typed array as its bytes. It stored the text `[object ArrayBuffer]`.
|
|
19
|
+
- From `2026-10-26`, `uploadPart()` takes what `put()` takes (a `ReadableStream` or a `Blob` too), and stores text as its UTF-8 bytes, as in workerd. It stored text whose characters all fit in one byte as Latin-1.
|
|
20
|
+
- From `2026-10-26`, an R2 object body's `body` stream is made when first read, and a whole-object `get()` keeps one copy of the bytes, not two.
|
|
21
|
+
|
|
22
|
+
## 0.27.0
|
|
23
|
+
|
|
24
|
+
### Minor Changes
|
|
25
|
+
|
|
26
|
+
- 9be9c74: R2 matches workerd from compatibility date `2026-10-26`:
|
|
27
|
+
|
|
28
|
+
- `get`, `put`, `head`, `delete`, `list` and `createMultipartUpload` return Promises, and errors reject them.
|
|
29
|
+
- An object's `etag` is the MD5 of its bytes, with `checksums.md5`. Objects written before keep their old etag.
|
|
30
|
+
- Objects have `uploaded` as a `Date`, `version`, `storageClass`, `range` and `writeHttpMetadata(headers)`. A body has `body` (a `ReadableStream`), `bodyUsed`, `text()`, `json()`, `arrayBuffer()` and `blob()`.
|
|
31
|
+
- `get(key, { range, onlyIf })` takes `{ offset, length }`, `{ suffix }` or a `Range` header, and `etagMatches`, `etagDoesNotMatch`, `uploadedBefore`, `uploadedAfter` or `If-*` headers. When the condition fails it returns the object without a body.
|
|
32
|
+
- `put(key, value, { onlyIf, httpMetadata })` takes a `ReadableStream` or a `Blob`, and `httpMetadata` as `Headers`. When the condition fails it returns `null`.
|
|
33
|
+
- `delete([keys])` deletes several keys.
|
|
34
|
+
- `list({ prefix, limit, cursor, delimiter, include })` pages at up to 1,000 entries, groups keys into `delimitedPrefixes`, and returns `httpMetadata` and `customMetadata` only when `include` asks for them.
|
|
35
|
+
- A value of the wrong type and a key longer than 1,024 bytes fail with R2's errors.
|
|
36
|
+
|
|
37
|
+
Earlier dates keep the synchronous shim. One change applies at every date: **a `list()` prefix now matches exactly.** It used SQL `LIKE`, so it ignored case and read `_` and `%` as wildcards. Both transports store an object's MD5 in a new column, added to existing stores when they open.
|
|
38
|
+
|
|
39
|
+
Against miniflare 4.20260730, `conformance/bindings/r2.js` in sproutd passes 48/48 at `2026-10-26` (16/48 before).
|
|
40
|
+
|
|
3
41
|
## 0.26.0
|
|
4
42
|
|
|
5
43
|
### Minor Changes
|
package/package.json
CHANGED
package/src/ingress-utf8.test.js
CHANGED
|
@@ -2,17 +2,15 @@ import { expect, test } from "bun:test";
|
|
|
2
2
|
import { readFileSync } from "node:fs";
|
|
3
3
|
const src = readFileSync(new URL("./native-fetch-prelude.js", import.meta.url), "utf8");
|
|
4
4
|
const decoder = src.slice(src.indexOf("function __sbFromUtf8("), src.indexOf("// sproutboat #181"));
|
|
5
|
-
const ingress = src.slice(
|
|
6
|
-
src.indexOf("function __sbDecodeIngress("),
|
|
7
|
-
src.indexOf("globalThis.__sbEntry ="),
|
|
8
|
-
);
|
|
5
|
+
const ingress = src.slice(src.indexOf("function __sbDecodeIngress("), src.indexOf("globalThis.__sbEntry ="));
|
|
9
6
|
const rawBody = src.slice(src.indexOf("function __sbRawBody("), src.indexOf("function __sbHasStreamBody("));
|
|
10
7
|
if (!decoder || !ingress || !rawBody) throw new Error("Ingress decoder source not found");
|
|
11
8
|
// oxlint-disable-next-line no-function-constructor -- Exercise the text-only runtime prelude.
|
|
12
|
-
const load = new Function(
|
|
9
|
+
const load = new Function(
|
|
10
|
+
`// The C fast paths (Porffor.c) cannot run here: the stubs make the JS fallback run.\nfunction __sbFromUtf8Native() { return ""; }\nfunction __sbToBytesNative() { return ""; }\n${decoder}\n${rawBody}\n${ingress}\nreturn __sbDecodeIngress;`,
|
|
11
|
+
);
|
|
13
12
|
const decode = load();
|
|
14
|
-
const raw = (text) =>
|
|
15
|
-
Array.from(new TextEncoder().encode(text), (byte) => String.fromCharCode(byte)).join("");
|
|
13
|
+
const raw = (text) => Array.from(new TextEncoder().encode(text), (byte) => String.fromCharCode(byte)).join("");
|
|
16
14
|
|
|
17
15
|
test("native ingress text and JSON decode Unicode without changing raw bytes", async () => {
|
|
18
16
|
const body = raw('{"id":"caffè-🚤"}');
|
|
@@ -3034,7 +3034,10 @@ if (globalThis.clearInterval == null) globalThis.clearInterval = clearInterval;
|
|
|
3034
3034
|
// these express the same spec-mandated checks and are verified under Porffor
|
|
3035
3035
|
// alpha-10 by examples/kitchen-sink/harness.ts).
|
|
3036
3036
|
function __sbIsStr(v) {
|
|
3037
|
-
|
|
3037
|
+
// sproutd #22: not `Object(v) !== v`, which wraps (copies) a string in Porffor: two such
|
|
3038
|
+
// checks on a 5 MiB part took 20 MiB.
|
|
3039
|
+
// oxlint-disable-next-line anti-slop/no-runtime-typeof -- the one check that does not copy its argument
|
|
3040
|
+
return typeof v === "string";
|
|
3038
3041
|
}
|
|
3039
3042
|
function __sbIsFn(v) {
|
|
3040
3043
|
return v instanceof Function;
|
|
@@ -3936,6 +3939,9 @@ const __SB_BYTES_WINDOW = 512;
|
|
|
3936
3939
|
// TextEncoder); ArrayBuffer / typed-array input is copied byte for byte.
|
|
3937
3940
|
function __sbToBytes(input) {
|
|
3938
3941
|
if (input == null) return "";
|
|
3942
|
+
// sproutd #22: one C pass when the input allows it (see __sbToBytesNative).
|
|
3943
|
+
const native = __sbToBytesNative(input);
|
|
3944
|
+
if (native !== "") return native;
|
|
3939
3945
|
// Windowed accumulation (#180, then #181). Plain `s +=` in a loop is O(n^2)
|
|
3940
3946
|
// here, since Porffor's strings have no rope/cons optimization and every
|
|
3941
3947
|
// append copies the whole accumulated string -- that's #180, a 43KB body
|
|
@@ -4015,6 +4021,9 @@ function __sbBufFrom(latin1) {
|
|
|
4015
4021
|
// regardless of whether it holds real Latin-1 text or raw bytes, so this
|
|
4016
4022
|
// decode is only safe where the caller already knows which one it has.
|
|
4017
4023
|
function __sbFromUtf8(bytes) {
|
|
4024
|
+
// sproutd #22: one C pass for a string of bytes (see __sbFromUtf8Native).
|
|
4025
|
+
const native = __sbFromUtf8Native(bytes);
|
|
4026
|
+
if (native !== "") return native;
|
|
4018
4027
|
// Collected into an array and joined once at the end, not built with
|
|
4019
4028
|
// repeated `out +=` -- Porffor's strings have no rope/cons optimization, so
|
|
4020
4029
|
// `+=` in a loop copies the whole accumulated string on every append,
|
|
@@ -4672,8 +4681,167 @@ static int sb_os_random(unsigned char* buf, size_t n) {
|
|
|
4672
4681
|
return 0;
|
|
4673
4682
|
}
|
|
4674
4683
|
|
|
4684
|
+
// sproutd #22: text and bytes converted in one C pass. The JS loops in
|
|
4685
|
+
// __sbFromUtf8 and __sbToBytes made one string per character, so a 5 MiB R2
|
|
4686
|
+
// object read with text() took about 400 MiB of heap before the next collection.
|
|
4687
|
+
u32 porf_native_fetch_alloc(u32 bytes, u32 type);
|
|
4688
|
+
|
|
4689
|
+
// Lenient UTF-8 decode, exactly as __sbFromUtf8's JS: a lead byte whose
|
|
4690
|
+
// continuation bytes are not all 10xxxxxx passes through as its own unit.
|
|
4691
|
+
// OUT has room for LEN units. Returns the unit count; *WIDE is set when a unit
|
|
4692
|
+
// is above 0xff (the result then needs a UTF-16 string).
|
|
4693
|
+
static size_t sb_utf8_decode_units(const unsigned char* b, size_t len, uint16_t* out, int* wide) {
|
|
4694
|
+
size_t k = 0;
|
|
4695
|
+
*wide = 0;
|
|
4696
|
+
for (size_t i = 0; i < len; i++) {
|
|
4697
|
+
const unsigned b0 = b[i];
|
|
4698
|
+
if (b0 >= 0x80) {
|
|
4699
|
+
if ((b0 & 0xe0) == 0xc0 && i + 1 < len && (b[i + 1] & 0xc0) == 0x80) {
|
|
4700
|
+
out[k] = (uint16_t)(((b0 & 0x1f) << 6) | (b[i + 1] & 0x3f));
|
|
4701
|
+
if (out[k] > 0xff) *wide = 1;
|
|
4702
|
+
k++; i += 1; continue;
|
|
4703
|
+
}
|
|
4704
|
+
if ((b0 & 0xf0) == 0xe0 && i + 2 < len && (b[i + 1] & 0xc0) == 0x80 && (b[i + 2] & 0xc0) == 0x80) {
|
|
4705
|
+
out[k] = (uint16_t)(((b0 & 0x0f) << 12) | ((b[i + 1] & 0x3f) << 6) | (b[i + 2] & 0x3f));
|
|
4706
|
+
if (out[k] > 0xff) *wide = 1;
|
|
4707
|
+
k++; i += 2; continue;
|
|
4708
|
+
}
|
|
4709
|
+
if ((b0 & 0xf8) == 0xf0 && i + 3 < len && (b[i + 1] & 0xc0) == 0x80 && (b[i + 2] & 0xc0) == 0x80 && (b[i + 3] & 0xc0) == 0x80) {
|
|
4710
|
+
int32_t cp = (int32_t)(((b0 & 0x07) << 18) | ((b[i + 1] & 0x3f) << 12) | ((b[i + 2] & 0x3f) << 6) | (b[i + 3] & 0x3f));
|
|
4711
|
+
cp -= 0x10000;
|
|
4712
|
+
out[k++] = (uint16_t)(0xd800 + (cp >> 10));
|
|
4713
|
+
out[k++] = (uint16_t)(0xdc00 + (cp & 0x3ff));
|
|
4714
|
+
*wide = 1;
|
|
4715
|
+
i += 3; continue;
|
|
4716
|
+
}
|
|
4717
|
+
}
|
|
4718
|
+
out[k++] = (uint16_t)b0;
|
|
4719
|
+
}
|
|
4720
|
+
return k;
|
|
4721
|
+
}
|
|
4722
|
+
|
|
4723
|
+
// UTF-8 encode, exactly as __sbToBytes's JS: a high surrogate takes the next
|
|
4724
|
+
// unit as its pair. OUT has room for 3 bytes per unit. Returns the byte count.
|
|
4725
|
+
static size_t sb_utf8_encode_units(const uint16_t* s, size_t len, unsigned char* out) {
|
|
4726
|
+
size_t k = 0;
|
|
4727
|
+
for (size_t i = 0; i < len; i++) {
|
|
4728
|
+
const uint32_t c = s[i];
|
|
4729
|
+
if (c < 0x80) out[k++] = (unsigned char)c;
|
|
4730
|
+
else if (c < 0x800) { out[k++] = (unsigned char)(0xc0 | (c >> 6)); out[k++] = (unsigned char)(0x80 | (c & 0x3f)); }
|
|
4731
|
+
else if (c >= 0xd800 && c < 0xdc00 && i + 1 < len) {
|
|
4732
|
+
const int32_t cp = 0x10000 + (int32_t)((c - 0xd800) << 10) + ((int32_t)s[++i] - 0xdc00);
|
|
4733
|
+
out[k++] = (unsigned char)(0xf0 | ((cp >> 18) & 0xff));
|
|
4734
|
+
out[k++] = (unsigned char)(0x80 | ((cp >> 12) & 0x3f));
|
|
4735
|
+
out[k++] = (unsigned char)(0x80 | ((cp >> 6) & 0x3f));
|
|
4736
|
+
out[k++] = (unsigned char)(0x80 | (cp & 0x3f));
|
|
4737
|
+
} else {
|
|
4738
|
+
out[k++] = (unsigned char)(0xe0 | ((c >> 12) & 0xff));
|
|
4739
|
+
out[k++] = (unsigned char)(0x80 | ((c >> 6) & 0x3f));
|
|
4740
|
+
out[k++] = (unsigned char)(0x80 | (c & 0x3f));
|
|
4741
|
+
}
|
|
4742
|
+
}
|
|
4743
|
+
return k;
|
|
4744
|
+
}
|
|
4745
|
+
|
|
4675
4746
|
`;
|
|
4676
4747
|
|
|
4748
|
+
// sproutd #22: __sbFromUtf8 for a string of bytes (one byte per unit), in C.
|
|
4749
|
+
// Returns "" when it cannot (not a string, or out of memory), so the caller
|
|
4750
|
+
// falls back to the JS loop. All-ASCII byte strings are already their text.
|
|
4751
|
+
// oxlint-disable-next-line no-unused-vars -- `bytes` is read inside the RawC block below, not by JS.
|
|
4752
|
+
function __sbFromUtf8Native(bytes) {
|
|
4753
|
+
let out = "";
|
|
4754
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
4755
|
+
Porffor.c`
|
|
4756
|
+
if (bytes.type == 195 || bytes.type == 67) {
|
|
4757
|
+
const u32 __p = (u32)bytes.val;
|
|
4758
|
+
const size_t __n = (size_t)*(u32*)(MEM + __p);
|
|
4759
|
+
int __ascii = bytes.type == 195;
|
|
4760
|
+
if (__ascii) {
|
|
4761
|
+
const unsigned char* __s = (const unsigned char*)(MEM + __p + 4);
|
|
4762
|
+
for (size_t __i = 0; __i < __n; __i++) if (__s[__i] >= 0x80) { __ascii = 0; break; }
|
|
4763
|
+
}
|
|
4764
|
+
if (__ascii) out = bytes;
|
|
4765
|
+
else {
|
|
4766
|
+
unsigned char* __b = (unsigned char*)malloc(__n + 1);
|
|
4767
|
+
uint16_t* __u = (uint16_t*)malloc(__n * 2 + 2);
|
|
4768
|
+
if (__b && __u) {
|
|
4769
|
+
if (bytes.type == 195) memcpy(__b, MEM + __p + 4, __n);
|
|
4770
|
+
else { const uint16_t* __w = (const uint16_t*)(MEM + __p + 4); for (size_t __i = 0; __i < __n; __i++) __b[__i] = (unsigned char)(__w[__i] & 0xff); }
|
|
4771
|
+
int __wide = 0;
|
|
4772
|
+
const size_t __k = sb_utf8_decode_units(__b, __n, __u, &__wide);
|
|
4773
|
+
if (__wide) {
|
|
4774
|
+
const u32 __s = porf_native_fetch_alloc((u32)(4 + __k * 2), 67);
|
|
4775
|
+
*(u32*)(MEM + __s) = (u32)__k;
|
|
4776
|
+
memcpy(MEM + __s + 4, __u, __k * 2);
|
|
4777
|
+
out = porf_box((f64)__s, 67);
|
|
4778
|
+
} else {
|
|
4779
|
+
for (size_t __i = 0; __i < __k; __i++) __b[__i] = (unsigned char)__u[__i];
|
|
4780
|
+
out = porf_box((f64)porf_native_fetch_alloc_bytestring((const char*)__b, __k), 195);
|
|
4781
|
+
}
|
|
4782
|
+
}
|
|
4783
|
+
free(__b); free(__u);
|
|
4784
|
+
}
|
|
4785
|
+
}
|
|
4786
|
+
`;
|
|
4787
|
+
return out;
|
|
4788
|
+
}
|
|
4789
|
+
|
|
4790
|
+
// sproutd #22: __sbToBytes for a string, a Uint8Array or an ArrayBuffer, in C.
|
|
4791
|
+
// Returns "" when it cannot (another type, a length-tracking view, out of
|
|
4792
|
+
// memory), so the caller falls back to the JS loop.
|
|
4793
|
+
// oxlint-disable-next-line no-unused-vars -- `input` is read inside the RawC block below, not by JS.
|
|
4794
|
+
function __sbToBytesNative(input) {
|
|
4795
|
+
let out = "";
|
|
4796
|
+
// oxlint-disable-next-line no-unused-expressions -- Porffor.c`...` is inline C the compiler consumes, not a JS expression.
|
|
4797
|
+
Porffor.c`
|
|
4798
|
+
const u32 __p = (u32)input.val;
|
|
4799
|
+
if (input.type == 195) {
|
|
4800
|
+
const size_t __n = (size_t)*(u32*)(MEM + __p);
|
|
4801
|
+
const unsigned char* __s = (const unsigned char*)(MEM + __p + 4);
|
|
4802
|
+
size_t __high = 0;
|
|
4803
|
+
for (size_t __i = 0; __i < __n; __i++) if (__s[__i] >= 0x80) __high++;
|
|
4804
|
+
if (!__high) out = input;
|
|
4805
|
+
else {
|
|
4806
|
+
unsigned char* __o = (unsigned char*)malloc(__n + __high + 1);
|
|
4807
|
+
if (__o) {
|
|
4808
|
+
const unsigned char* __src = (const unsigned char*)(MEM + __p + 4);
|
|
4809
|
+
size_t __k = 0;
|
|
4810
|
+
for (size_t __i = 0; __i < __n; __i++) {
|
|
4811
|
+
const unsigned char __c = __src[__i];
|
|
4812
|
+
if (__c < 0x80) __o[__k++] = __c;
|
|
4813
|
+
else { __o[__k++] = (unsigned char)(0xc0 | (__c >> 6)); __o[__k++] = (unsigned char)(0x80 | (__c & 0x3f)); }
|
|
4814
|
+
}
|
|
4815
|
+
out = porf_box((f64)porf_native_fetch_alloc_bytestring((const char*)__o, __k), 195);
|
|
4816
|
+
free(__o);
|
|
4817
|
+
}
|
|
4818
|
+
}
|
|
4819
|
+
} else if (input.type == 67) {
|
|
4820
|
+
const size_t __n = (size_t)*(u32*)(MEM + __p);
|
|
4821
|
+
uint16_t* __w = (uint16_t*)malloc(__n * 2 + 2);
|
|
4822
|
+
unsigned char* __o = (unsigned char*)malloc(__n * 3 + 1);
|
|
4823
|
+
if (__w && __o) {
|
|
4824
|
+
memcpy(__w, MEM + __p + 4, __n * 2);
|
|
4825
|
+
const size_t __k = sb_utf8_encode_units(__w, __n, __o);
|
|
4826
|
+
out = porf_box((f64)porf_native_fetch_alloc_bytestring((const char*)__o, __k), 195);
|
|
4827
|
+
}
|
|
4828
|
+
free(__w); free(__o);
|
|
4829
|
+
} else if (input.type == 13 || input.type == 81) {
|
|
4830
|
+
const int32_t __len = *(int32_t*)(MEM + __p);
|
|
4831
|
+
if (__len > 0) {
|
|
4832
|
+
const u32 __data = input.type == 13 ? __p + 8 : *(u32*)(MEM + __p + 4) + 4;
|
|
4833
|
+
unsigned char* __o = (unsigned char*)malloc((size_t)__len);
|
|
4834
|
+
if (__o) {
|
|
4835
|
+
memcpy(__o, MEM + __data, (size_t)__len);
|
|
4836
|
+
out = porf_box((f64)porf_native_fetch_alloc_bytestring((const char*)__o, (size_t)__len), 195);
|
|
4837
|
+
free(__o);
|
|
4838
|
+
}
|
|
4839
|
+
}
|
|
4840
|
+
}
|
|
4841
|
+
`;
|
|
4842
|
+
return out;
|
|
4843
|
+
}
|
|
4844
|
+
|
|
4677
4845
|
// TRANSPORT: wrap.ts splices one of transport-broker.js / transport-embedded.js here.
|
|
4678
4846
|
|
|
4679
4847
|
// `nStr` is the decimal byte count as a string (same string-param pattern as
|
|
@@ -5218,6 +5386,15 @@ function __sbR2Object(meta, body) {
|
|
|
5218
5386
|
return obj;
|
|
5219
5387
|
}
|
|
5220
5388
|
|
|
5389
|
+
// A part's value for the transport: an ArrayBuffer or a view as its bytes (String() of one
|
|
5390
|
+
// stored "[object ArrayBuffer]", at every date), anything else as text, as before.
|
|
5391
|
+
function __sbR2PartValue(value) {
|
|
5392
|
+
value = __sbUnwrapBody(value);
|
|
5393
|
+
if (value == null) return "";
|
|
5394
|
+
if (!__sbIsStr(value) && value.byteLength !== undefined)
|
|
5395
|
+
return __sbToBytes(value.buffer !== undefined ? new Uint8Array(value.buffer, value.byteOffset, value.byteLength) : new Uint8Array(value));
|
|
5396
|
+
return String(value);
|
|
5397
|
+
}
|
|
5221
5398
|
function __sbR2Multipart(bucket, key, uploadId) {
|
|
5222
5399
|
return {
|
|
5223
5400
|
key,
|
|
@@ -5226,7 +5403,7 @@ function __sbR2Multipart(bucket, key, uploadId) {
|
|
|
5226
5403
|
const number = Number(partNumber);
|
|
5227
5404
|
if (!Number.isInteger(number) || number < 1 || number > 10000)
|
|
5228
5405
|
throw new TypeError("partNumber must be an integer between 1 and 10000");
|
|
5229
|
-
return __sbR2MultipartPut(bucket, key, uploadId, number,
|
|
5406
|
+
return __sbR2MultipartPut(bucket, key, uploadId, number, __sbR2PartValue(value)).part;
|
|
5230
5407
|
},
|
|
5231
5408
|
complete(parts) {
|
|
5232
5409
|
const r = __sbRpc("r2.multipart.complete", { bucket, key, uploadId, parts: parts || [] });
|
|
@@ -5238,6 +5415,277 @@ function __sbR2Multipart(bucket, key, uploadId) {
|
|
|
5238
5415
|
};
|
|
5239
5416
|
}
|
|
5240
5417
|
|
|
5418
|
+
// R2 from compatibility date 2026-10-26: workerd's surface. Every method returns a
|
|
5419
|
+
// Promise; an object's etag is the MD5 of its bytes (sha256 before; objects written
|
|
5420
|
+
// earlier keep theirs), with `checksums.md5`, `uploaded` as a Date and `range`; get()
|
|
5421
|
+
// takes `range` and `onlyIf`, put() takes `onlyIf`, a ReadableStream or a Blob and
|
|
5422
|
+
// httpMetadata as Headers; delete() takes a list of keys; list() pages with limit and
|
|
5423
|
+
// cursor, groups by delimiter and returns metadata only when `include` asks for it.
|
|
5424
|
+
// Earlier dates keep the synchronous shim below.
|
|
5425
|
+
function __sbR2Strict() {
|
|
5426
|
+
return globalThis.__sbCompat >= "2026-10-26";
|
|
5427
|
+
}
|
|
5428
|
+
function __sbR2Key(key, verb) {
|
|
5429
|
+
const name = String(key);
|
|
5430
|
+
if (new TextEncoder().encode(name).length > 1024) throw new Error(verb + ": The specified object name is not valid. (10020)");
|
|
5431
|
+
return name;
|
|
5432
|
+
}
|
|
5433
|
+
function __sbR2HexOf(bytes) {
|
|
5434
|
+
let out = "";
|
|
5435
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
5436
|
+
const b = bytes.charCodeAt(i) & 0xff;
|
|
5437
|
+
out += (b < 16 ? "0" : "") + b.toString(16);
|
|
5438
|
+
}
|
|
5439
|
+
return out;
|
|
5440
|
+
}
|
|
5441
|
+
// R2's httpMetadata, in workerd's key order, from an object or a Headers.
|
|
5442
|
+
function __sbR2Http(h) {
|
|
5443
|
+
if (h == null) return {};
|
|
5444
|
+
const src = h instanceof Headers
|
|
5445
|
+
? {
|
|
5446
|
+
contentType: h.get("content-type"),
|
|
5447
|
+
contentLanguage: h.get("content-language"),
|
|
5448
|
+
contentDisposition: h.get("content-disposition"),
|
|
5449
|
+
contentEncoding: h.get("content-encoding"),
|
|
5450
|
+
cacheControl: h.get("cache-control"),
|
|
5451
|
+
cacheExpiry: h.get("expires"),
|
|
5452
|
+
}
|
|
5453
|
+
: h;
|
|
5454
|
+
const out = {};
|
|
5455
|
+
const names = ["contentType", "contentLanguage", "contentDisposition", "contentEncoding", "cacheControl", "cacheExpiry"];
|
|
5456
|
+
for (let i = 0; i < names.length; i++) if (src[names[i]] != null) out[names[i]] = src[names[i]];
|
|
5457
|
+
return out;
|
|
5458
|
+
}
|
|
5459
|
+
function __sbR2EtagIn(list, etag) {
|
|
5460
|
+
const items = Array.isArray(list) ? list : String(list).split(",");
|
|
5461
|
+
for (let i = 0; i < items.length; i++) {
|
|
5462
|
+
const e = String(items[i]).trim().replace(/^W\//, "").replace(/^"|"$/g, "");
|
|
5463
|
+
if (e === "*" || e === etag) return true;
|
|
5464
|
+
}
|
|
5465
|
+
return false;
|
|
5466
|
+
}
|
|
5467
|
+
// Whether an object passes onlyIf (an object or a Headers with If-* headers).
|
|
5468
|
+
function __sbR2OnlyIf(cond, etag, uploaded) {
|
|
5469
|
+
if (cond == null) return true;
|
|
5470
|
+
const c = cond instanceof Headers
|
|
5471
|
+
? {
|
|
5472
|
+
etagMatches: cond.get("if-match"),
|
|
5473
|
+
etagDoesNotMatch: cond.get("if-none-match"),
|
|
5474
|
+
uploadedBefore: cond.get("if-unmodified-since"),
|
|
5475
|
+
uploadedAfter: cond.get("if-modified-since"),
|
|
5476
|
+
}
|
|
5477
|
+
: cond;
|
|
5478
|
+
const at = new Date(uploaded).getTime();
|
|
5479
|
+
if (c.etagMatches != null && !__sbR2EtagIn(c.etagMatches, etag)) return false;
|
|
5480
|
+
if (c.etagDoesNotMatch != null && __sbR2EtagIn(c.etagDoesNotMatch, etag)) return false;
|
|
5481
|
+
if (c.uploadedBefore != null && !(at < new Date(c.uploadedBefore).getTime())) return false;
|
|
5482
|
+
if (c.uploadedAfter != null && !(at > new Date(c.uploadedAfter).getTime())) return false;
|
|
5483
|
+
return true;
|
|
5484
|
+
}
|
|
5485
|
+
// {offset, length} of a range (an object or a Headers with Range) on an object of SIZE.
|
|
5486
|
+
function __sbR2Range(range, size) {
|
|
5487
|
+
let offset = 0, length = size;
|
|
5488
|
+
if (range instanceof Headers) {
|
|
5489
|
+
const m = /^bytes=(\d*)-(\d*)$/.exec(String(range.get("range") || "").trim());
|
|
5490
|
+
if (!m) return { offset, length };
|
|
5491
|
+
if (m[1] === "") range = { suffix: Number(m[2]) };
|
|
5492
|
+
else range = m[2] === "" ? { offset: Number(m[1]) } : { offset: Number(m[1]), length: Number(m[2]) - Number(m[1]) + 1 };
|
|
5493
|
+
}
|
|
5494
|
+
if (range.suffix != null) {
|
|
5495
|
+
length = Math.min(Number(range.suffix), size);
|
|
5496
|
+
offset = size - length;
|
|
5497
|
+
} else {
|
|
5498
|
+
offset = Math.min(Number(range.offset || 0), size);
|
|
5499
|
+
length = range.length == null ? size - offset : Math.min(Number(range.length), size - offset);
|
|
5500
|
+
}
|
|
5501
|
+
return { offset, length };
|
|
5502
|
+
}
|
|
5503
|
+
// An R2Object (or an R2ObjectBody, given BYTES) as workerd returns it.
|
|
5504
|
+
function __sbR2StrictObject(meta, bytes, range, include) {
|
|
5505
|
+
const etag = meta.md5 || meta.etag;
|
|
5506
|
+
const md5 = meta.md5;
|
|
5507
|
+
const obj = {
|
|
5508
|
+
key: meta.key,
|
|
5509
|
+
version: meta.etag,
|
|
5510
|
+
size: meta.size,
|
|
5511
|
+
etag,
|
|
5512
|
+
httpEtag: '"' + etag + '"',
|
|
5513
|
+
checksums: {
|
|
5514
|
+
md5: md5 ? __sbBufFrom(__sbR2Bytes(md5)) : undefined,
|
|
5515
|
+
toJSON() {
|
|
5516
|
+
return { md5: md5 || undefined, sha1: undefined, sha256: undefined, sha384: undefined, sha512: undefined };
|
|
5517
|
+
},
|
|
5518
|
+
},
|
|
5519
|
+
uploaded: new Date(meta.uploaded),
|
|
5520
|
+
httpMetadata: include === false ? {} : __sbR2Http(meta.httpMetadata),
|
|
5521
|
+
customMetadata: include === false ? {} : meta.customMetadata || {},
|
|
5522
|
+
storageClass: "",
|
|
5523
|
+
range,
|
|
5524
|
+
writeHttpMetadata(headers) {
|
|
5525
|
+
const h = obj.httpMetadata;
|
|
5526
|
+
if (h.contentType != null) headers.set("content-type", h.contentType);
|
|
5527
|
+
if (h.contentLanguage != null) headers.set("content-language", h.contentLanguage);
|
|
5528
|
+
if (h.contentDisposition != null) headers.set("content-disposition", h.contentDisposition);
|
|
5529
|
+
if (h.contentEncoding != null) headers.set("content-encoding", h.contentEncoding);
|
|
5530
|
+
if (h.cacheControl != null) headers.set("cache-control", h.cacheControl);
|
|
5531
|
+
if (h.cacheExpiry != null) headers.set("expires", new Date(h.cacheExpiry).toUTCString());
|
|
5532
|
+
},
|
|
5533
|
+
};
|
|
5534
|
+
if (bytes == null) return obj;
|
|
5535
|
+
let used = false;
|
|
5536
|
+
const take = () => {
|
|
5537
|
+
if (used) throw new TypeError("Body has already been used. It can only be used once. Use tee() first if you need to read it twice.");
|
|
5538
|
+
used = true;
|
|
5539
|
+
return bytes;
|
|
5540
|
+
};
|
|
5541
|
+
// The stream is made when first asked for: text() and friends read the bytes directly,
|
|
5542
|
+
// so an object read as text never holds a second copy (sproutd #22).
|
|
5543
|
+
let stream = null;
|
|
5544
|
+
Object.defineProperty(obj, "body", {
|
|
5545
|
+
get: () => {
|
|
5546
|
+
if (stream === null) stream = new Response(__sbBufFrom(bytes)).body;
|
|
5547
|
+
return stream;
|
|
5548
|
+
},
|
|
5549
|
+
enumerable: true,
|
|
5550
|
+
});
|
|
5551
|
+
Object.defineProperty(obj, "bodyUsed", { get: () => used, enumerable: true });
|
|
5552
|
+
obj.text = async () => __sbFromUtf8(take());
|
|
5553
|
+
obj.json = async () => JSON.parse(__sbFromUtf8(take()));
|
|
5554
|
+
obj.arrayBuffer = async () => __sbBufFrom(take());
|
|
5555
|
+
obj.blob = async () => new Blob([__sbBufFrom(take())], { type: obj.httpMetadata.contentType || "" });
|
|
5556
|
+
return obj;
|
|
5557
|
+
}
|
|
5558
|
+
// A hex string's bytes, one char per byte.
|
|
5559
|
+
function __sbR2Bytes(hex) {
|
|
5560
|
+
let out = "";
|
|
5561
|
+
for (let i = 0; i + 1 < hex.length; i += 2) out += String.fromCharCode(parseInt(hex.slice(i, i + 2), 16));
|
|
5562
|
+
return out;
|
|
5563
|
+
}
|
|
5564
|
+
function __sbMakeR2(name) {
|
|
5565
|
+
const attempt = (run) => {
|
|
5566
|
+
try {
|
|
5567
|
+
return Promise.resolve(run());
|
|
5568
|
+
} catch (err) {
|
|
5569
|
+
return Promise.reject(err);
|
|
5570
|
+
}
|
|
5571
|
+
};
|
|
5572
|
+
const head = (key) => {
|
|
5573
|
+
const r = __sbRpc("r2.head", { bucket: name, key });
|
|
5574
|
+
return r.found ? r.object : null;
|
|
5575
|
+
};
|
|
5576
|
+
const upload = (key, uploadId) => {
|
|
5577
|
+
const inner = __sbR2Multipart(name, key, uploadId);
|
|
5578
|
+
return {
|
|
5579
|
+
key,
|
|
5580
|
+
uploadId,
|
|
5581
|
+
// A part is any value put() takes; text is stored as its UTF-8 bytes, as in workerd.
|
|
5582
|
+
uploadPart: async (partNumber, value) => {
|
|
5583
|
+
value = __sbUnwrapBody(value);
|
|
5584
|
+
if (value instanceof Blob) value = await value.arrayBuffer();
|
|
5585
|
+
else if (value != null && __sbIsFn(value.getReader)) value = await new Response(value).arrayBuffer();
|
|
5586
|
+
return inner.uploadPart(partNumber, __sbIsStr(value) ? __sbToBytes(value) : value);
|
|
5587
|
+
},
|
|
5588
|
+
// ponytail: a multipart object's etag stays the transport's sha256; workerd's is
|
|
5589
|
+
// MD5-of-part-MD5s-N. Add when an app compares multipart etags.
|
|
5590
|
+
complete: (parts) => attempt(() => __sbR2StrictObject(__sbRpc("r2.multipart.complete", { bucket: name, key, uploadId, parts: parts || [] }).object, null)),
|
|
5591
|
+
abort: () => attempt(() => inner.abort()),
|
|
5592
|
+
};
|
|
5593
|
+
};
|
|
5594
|
+
return {
|
|
5595
|
+
head(key) {
|
|
5596
|
+
return attempt(() => {
|
|
5597
|
+
const meta = head(__sbR2Key(key, "head"));
|
|
5598
|
+
return meta ? __sbR2StrictObject(meta, null, { offset: 0, length: meta.size }) : null;
|
|
5599
|
+
});
|
|
5600
|
+
},
|
|
5601
|
+
get(key, options) {
|
|
5602
|
+
return attempt(() => {
|
|
5603
|
+
const o = options || {};
|
|
5604
|
+
const r = __sbR2Get(name, __sbR2Key(key, "get"));
|
|
5605
|
+
if (!r.found) return null;
|
|
5606
|
+
const meta = r.object;
|
|
5607
|
+
if (!__sbR2OnlyIf(o.onlyIf, meta.md5 || meta.etag, meta.uploaded)) return __sbR2StrictObject(meta, null, { offset: 0, length: meta.size });
|
|
5608
|
+
const bytes = r.body == null ? "" : r.body;
|
|
5609
|
+
const range = o.range == null ? { offset: 0, length: meta.size } : __sbR2Range(o.range, meta.size);
|
|
5610
|
+
const whole = range.offset === 0 && range.length >= bytes.length;
|
|
5611
|
+
return __sbR2StrictObject(meta, whole ? bytes : bytes.slice(range.offset, range.offset + range.length), range);
|
|
5612
|
+
});
|
|
5613
|
+
},
|
|
5614
|
+
async put(key, value, options) {
|
|
5615
|
+
const objectKey = __sbR2Key(key, "put");
|
|
5616
|
+
const o = options || {};
|
|
5617
|
+
value = __sbUnwrapBody(value);
|
|
5618
|
+
if (value instanceof Blob) value = await value.arrayBuffer();
|
|
5619
|
+
else if (value != null && __sbIsFn(value.getReader)) value = await new Response(value).arrayBuffer();
|
|
5620
|
+
let bytes;
|
|
5621
|
+
if (value == null) bytes = "";
|
|
5622
|
+
else if (__sbIsStr(value)) bytes = __sbToBytes(value);
|
|
5623
|
+
else if (value.byteLength !== undefined)
|
|
5624
|
+
bytes = __sbToBytes(value.buffer !== undefined ? new Uint8Array(value.buffer, value.byteOffset, value.byteLength) : new Uint8Array(value));
|
|
5625
|
+
else
|
|
5626
|
+
throw new TypeError("Failed to execute 'put' on 'R2Bucket': parameter 2 is not of type 'ReadableStream or ArrayBuffer or ArrayBufferView or string or Blob'.");
|
|
5627
|
+
if (o.onlyIf != null) {
|
|
5628
|
+
// ponytail: check, then write; one worker per app, so nothing writes in between.
|
|
5629
|
+
const current = head(objectKey);
|
|
5630
|
+
if (current && !__sbR2OnlyIf(o.onlyIf, current.md5 || current.etag, current.uploaded)) return null;
|
|
5631
|
+
if (!current && o.onlyIf.etagMatches != null) return null;
|
|
5632
|
+
}
|
|
5633
|
+
const http = __sbR2Http(o.httpMetadata);
|
|
5634
|
+
const custom = o.customMetadata || {};
|
|
5635
|
+
const meta = __sbR2Put(name, objectKey, bytes, http, custom).object;
|
|
5636
|
+
meta.md5 = __sbR2HexOf(__sbDigestRaw("5", bytes));
|
|
5637
|
+
__sbRpc("r2.md5", { bucket: name, key: objectKey, etag: meta.etag, md5: meta.md5 });
|
|
5638
|
+
meta.httpMetadata = http;
|
|
5639
|
+
meta.customMetadata = custom;
|
|
5640
|
+
return __sbR2StrictObject(meta, null, undefined);
|
|
5641
|
+
},
|
|
5642
|
+
delete(keys) {
|
|
5643
|
+
return attempt(() => {
|
|
5644
|
+
const list = Array.isArray(keys) ? keys : [keys];
|
|
5645
|
+
for (let i = 0; i < list.length; i++) __sbRpc("r2.delete", { bucket: name, key: __sbR2Key(list[i], "delete") });
|
|
5646
|
+
});
|
|
5647
|
+
},
|
|
5648
|
+
list(options) {
|
|
5649
|
+
return attempt(() => {
|
|
5650
|
+
const o = options || {};
|
|
5651
|
+
const r = __sbRpc("r2.list", {
|
|
5652
|
+
bucket: name,
|
|
5653
|
+
page: true,
|
|
5654
|
+
prefix: o.prefix == null ? "" : String(o.prefix),
|
|
5655
|
+
cursor: o.cursor == null ? "" : String(o.cursor),
|
|
5656
|
+
limit: o.limit == null ? 1000 : Number(o.limit),
|
|
5657
|
+
delimiter: o.delimiter == null ? "" : String(o.delimiter),
|
|
5658
|
+
});
|
|
5659
|
+
const include = o.include || [];
|
|
5660
|
+
const objects = [];
|
|
5661
|
+
for (let j = 0; j < (r.objects || []).length; j++) {
|
|
5662
|
+
const obj = __sbR2StrictObject(r.objects[j], null, undefined);
|
|
5663
|
+
if (include.indexOf("httpMetadata") < 0) obj.httpMetadata = {};
|
|
5664
|
+
if (include.indexOf("customMetadata") < 0) obj.customMetadata = {};
|
|
5665
|
+
objects.push(obj);
|
|
5666
|
+
}
|
|
5667
|
+
return { objects, truncated: !!r.truncated, cursor: r.truncated ? r.cursor : undefined, delimitedPrefixes: r.delimitedPrefixes || [] };
|
|
5668
|
+
});
|
|
5669
|
+
},
|
|
5670
|
+
createMultipartUpload(key, options) {
|
|
5671
|
+
return attempt(() => {
|
|
5672
|
+
const objectKey = __sbR2Key(key, "createMultipartUpload");
|
|
5673
|
+
const o = options || {};
|
|
5674
|
+
const r = __sbRpc("r2.multipart.create", {
|
|
5675
|
+
bucket: name,
|
|
5676
|
+
key: objectKey,
|
|
5677
|
+
httpMetadata: __sbR2Http(o.httpMetadata),
|
|
5678
|
+
customMetadata: o.customMetadata || {},
|
|
5679
|
+
});
|
|
5680
|
+
return upload(objectKey, r.uploadId);
|
|
5681
|
+
});
|
|
5682
|
+
},
|
|
5683
|
+
resumeMultipartUpload(key, uploadId) {
|
|
5684
|
+
return upload(String(key), String(uploadId));
|
|
5685
|
+
},
|
|
5686
|
+
};
|
|
5687
|
+
}
|
|
5688
|
+
|
|
5241
5689
|
// Asset rules use the same parsed manifest in both transports. Matching mirrors
|
|
5242
5690
|
// @sproutboat/assets; parity tests cover the portable native implementation.
|
|
5243
5691
|
function __sbAssetRedirect(rule, path) {
|
|
@@ -5364,7 +5812,7 @@ globalThis.__sbInstallBindings = function (target, bindings) {
|
|
|
5364
5812
|
|
|
5365
5813
|
for (let i = 0; i < (bindings.r2 || []).length; i++) {
|
|
5366
5814
|
const name = bindings.r2[i];
|
|
5367
|
-
target[name] = {
|
|
5815
|
+
target[name] = __sbR2Strict() ? __sbMakeR2(name) : {
|
|
5368
5816
|
put(key, value, options) {
|
|
5369
5817
|
value = __sbUnwrapBody(value);
|
|
5370
5818
|
const o = options || {};
|
package/src/r2-object.test.js
CHANGED
|
@@ -18,6 +18,9 @@ const rawStart = src.indexOf("function __sbRawBodyResponse(");
|
|
|
18
18
|
const utf8End = rawStart;
|
|
19
19
|
const rawEnd = src.indexOf("function __sbEqCt(");
|
|
20
20
|
const r2Start = src.indexOf("function __sbR2Object(");
|
|
21
|
+
const isStrStart = src.indexOf("function __sbIsStr(");
|
|
22
|
+
const isStrEnd = src.indexOf("function __sbIsFn(");
|
|
23
|
+
const bytesStart = src.indexOf("const __SB_BYTES_WINDOW");
|
|
21
24
|
const r2End = src.indexOf("// Installed only when the project declares bindings.");
|
|
22
25
|
if (utf8Start === -1 || utf8End === -1) throw new Error("__sbFromUtf8 block not found in prelude");
|
|
23
26
|
if (rawStart === -1 || rawEnd === -1) throw new Error("__sbRawBodyResponse block not found in prelude");
|
|
@@ -27,7 +30,8 @@ if (r2Start === -1 || r2End === -1 || r2End < r2Start) throw new Error("__sbR2Ob
|
|
|
27
30
|
const load = new Function(
|
|
28
31
|
"__sbRpc",
|
|
29
32
|
"__sbR2MultipartPut",
|
|
30
|
-
|
|
33
|
+
// __sbIsStr and __sbToBytes (for uploadPart's values), with the C fast paths stubbed out.
|
|
34
|
+
`function __sbFromUtf8Native() { return ""; }\nfunction __sbToBytesNative() { return ""; }\n${src.slice(isStrStart, isStrEnd)}\n${src.slice(bytesStart, utf8Start)}\n${src.slice(utf8Start, utf8End)}\n${src.slice(rawStart, rawEnd)}\n${src.slice(r2Start, r2End)}\nreturn { __sbR2Object, __sbR2Multipart };`,
|
|
31
35
|
);
|
|
32
36
|
const calls = [];
|
|
33
37
|
const rpc = (op, value) => {
|
|
@@ -70,7 +74,7 @@ test("toResponse() lets caller-supplied headers/status override the defaults", (
|
|
|
70
74
|
expect(resp.headers.get("content-type")).toBe("application/octet-stream");
|
|
71
75
|
});
|
|
72
76
|
|
|
73
|
-
test(
|
|
77
|
+
test('toResponse() on an empty body still returns bytes, not the string ""', async () => {
|
|
74
78
|
const obj = __sbR2Object(meta, "");
|
|
75
79
|
const resp = obj.toResponse();
|
|
76
80
|
const buf = new Uint8Array(await resp.arrayBuffer());
|
|
@@ -84,9 +88,14 @@ test("multipart uploads expose the Cloudflare-shaped uploadPart, complete, and a
|
|
|
84
88
|
expect(upload.uploadPart(1, "first")).toEqual({ partNumber: 1, etag: "part-1" });
|
|
85
89
|
expect(upload.complete([{ partNumber: 1, etag: "part-1" }])).toMatchObject(meta);
|
|
86
90
|
upload.abort();
|
|
87
|
-
expect(calls.map((call) => call.op)).toEqual([
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
91
|
+
expect(calls.map((call) => call.op)).toEqual(["r2.multipart.put", "r2.multipart.complete", "r2.multipart.abort"]);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("uploadPart stores an ArrayBuffer or a view as its bytes, not as [object ArrayBuffer]", () => {
|
|
95
|
+
calls.length = 0;
|
|
96
|
+
const upload = __sbR2Multipart("FILES", "archive.bin", "upload-1");
|
|
97
|
+
upload.uploadPart(1, new Uint8Array([0, 255, 65]).buffer);
|
|
98
|
+
upload.uploadPart(2, new Uint8Array([9, 8, 7, 6]).subarray(1, 3));
|
|
99
|
+
upload.uploadPart(3, "text");
|
|
100
|
+
expect(calls.map((call) => call.value.body)).toEqual(["\u0000ÿA", "\u0008\u0007", "text"]);
|
|
92
101
|
});
|
package/src/to-bytes.test.js
CHANGED
|
@@ -22,7 +22,7 @@ if (hexStart === -1 || hexEnd === -1 || hexEnd < hexStart) throw new Error("__sb
|
|
|
22
22
|
|
|
23
23
|
// oxlint-disable-next-line no-function-constructor -- lifting pure blocks out of the text-only prelude for test
|
|
24
24
|
const load = new Function(
|
|
25
|
-
|
|
25
|
+
`// The C fast paths (Porffor.c) cannot run here: the stubs make the JS fallback run.\nfunction __sbFromUtf8Native() { return ""; }\nfunction __sbToBytesNative() { return ""; }\n${src.slice(isStrStart, isStrEnd)}\n${src.slice(start, end)}\n${src.slice(hexStart, hexEnd)}\nreturn { __sbToBytes, __sbHexOrBytes };`,
|
|
26
26
|
);
|
|
27
27
|
const { __sbToBytes, __sbHexOrBytes } = load();
|
|
28
28
|
|
|
@@ -3590,6 +3590,10 @@ function __sbEnsureSchema() {
|
|
|
3590
3590
|
let hasBlobId = false;
|
|
3591
3591
|
for (let i = 0; i < r2Columns.length; i++) if (r2Columns[i][1] === "blob_id") hasBlobId = true;
|
|
3592
3592
|
if (!hasBlobId) __sbSql(s, "ALTER TABLE r2 ADD COLUMN blob_id TEXT", []);
|
|
3593
|
+
// The MD5 of an object's bytes, its etag from compatibility date 2026-10-26 (r2.md5).
|
|
3594
|
+
let hasR2Md5 = false;
|
|
3595
|
+
for (let i = 0; i < r2Columns.length; i++) if (r2Columns[i][1] === "md5") hasR2Md5 = true;
|
|
3596
|
+
if (!hasR2Md5) __sbSql(s, "ALTER TABLE r2 ADD COLUMN md5 TEXT", []);
|
|
3593
3597
|
__sbSql(
|
|
3594
3598
|
s,
|
|
3595
3599
|
"CREATE TABLE IF NOT EXISTS r2_upload (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
|
|
@@ -3661,6 +3665,51 @@ function __sbEnsureSchema() {
|
|
|
3661
3665
|
);
|
|
3662
3666
|
}
|
|
3663
3667
|
|
|
3668
|
+
// One page of r2.list for the shim from 2026-10-26: at most LIMIT entries after CURSOR,
|
|
3669
|
+
// keys under PREFIX, those holding DELIMITER after it grouped into delimitedPrefixes (a
|
|
3670
|
+
// group counts as one entry). Same rules as the broker's r2ListPage.
|
|
3671
|
+
// ponytail: with a delimiter every key after the cursor is read; an index on the
|
|
3672
|
+
// delimited prefix if buckets grow past ~100k keys.
|
|
3673
|
+
function __sbR2ListPage(store, msg) {
|
|
3674
|
+
const prefix = String(msg.prefix || "");
|
|
3675
|
+
const delimiter = String(msg.delimiter || "");
|
|
3676
|
+
const limit = Math.min(Math.max(Math.floor(Number(msg.limit)) || 1000, 1), 1000);
|
|
3677
|
+
const r = __sbSql(
|
|
3678
|
+
store,
|
|
3679
|
+
"SELECT key, size, etag, uploaded, http_json, custom_json, md5 FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 " +
|
|
3680
|
+
"AND key > ?3 ORDER BY key" + (delimiter ? "" : " LIMIT " + (limit + 1)),
|
|
3681
|
+
[msg.bucket, prefix, String(msg.cursor || "")],
|
|
3682
|
+
);
|
|
3683
|
+
const objects = [];
|
|
3684
|
+
const delimitedPrefixes = [];
|
|
3685
|
+
let last = "";
|
|
3686
|
+
let truncated = false;
|
|
3687
|
+
for (let i = 0; i < r.rows.length; i++) {
|
|
3688
|
+
const row = r.rows[i];
|
|
3689
|
+
const at = delimiter ? row[0].indexOf(delimiter, prefix.length) : -1;
|
|
3690
|
+
const group = at < 0 ? "" : row[0].slice(0, at + delimiter.length);
|
|
3691
|
+
if (!group || delimitedPrefixes[delimitedPrefixes.length - 1] !== group) {
|
|
3692
|
+
if (objects.length + delimitedPrefixes.length >= limit) {
|
|
3693
|
+
truncated = true;
|
|
3694
|
+
break;
|
|
3695
|
+
}
|
|
3696
|
+
if (group) delimitedPrefixes.push(group);
|
|
3697
|
+
else
|
|
3698
|
+
objects.push({
|
|
3699
|
+
key: row[0],
|
|
3700
|
+
size: Number(row[1]),
|
|
3701
|
+
etag: row[2],
|
|
3702
|
+
uploaded: row[3],
|
|
3703
|
+
httpMetadata: JSON.parse(row[4] || "{}"),
|
|
3704
|
+
customMetadata: JSON.parse(row[5] || "{}"),
|
|
3705
|
+
md5: row[6],
|
|
3706
|
+
});
|
|
3707
|
+
}
|
|
3708
|
+
last = row[0];
|
|
3709
|
+
}
|
|
3710
|
+
return { ok: true, objects, delimitedPrefixes, truncated, cursor: truncated ? last : "" };
|
|
3711
|
+
}
|
|
3712
|
+
|
|
3664
3713
|
function __sbHex(n) {
|
|
3665
3714
|
let out = "";
|
|
3666
3715
|
const bytes = __sbRandomBytes(String(n));
|
|
@@ -4072,7 +4121,7 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
4072
4121
|
const wantsBody = op === "r2.get";
|
|
4073
4122
|
const r = __sbSql(
|
|
4074
4123
|
store,
|
|
4075
|
-
"SELECT size, etag, uploaded, http_json, custom_json, blob_id FROM r2 WHERE bucket = ? AND key = ?",
|
|
4124
|
+
"SELECT size, etag, uploaded, http_json, custom_json, blob_id, md5 FROM r2 WHERE bucket = ? AND key = ?",
|
|
4076
4125
|
[msg.bucket, msg.key],
|
|
4077
4126
|
);
|
|
4078
4127
|
if (!r.rows.length) return { ok: true, found: false };
|
|
@@ -4088,6 +4137,7 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
4088
4137
|
uploaded: row[2],
|
|
4089
4138
|
httpMetadata: JSON.parse(row[3] || "{}"),
|
|
4090
4139
|
customMetadata: JSON.parse(row[4] || "{}"),
|
|
4140
|
+
md5: row[6],
|
|
4091
4141
|
},
|
|
4092
4142
|
blobId: row[5] || "",
|
|
4093
4143
|
body: wantsBody
|
|
@@ -4110,10 +4160,23 @@ function __sbEmbeddedDispatch(msg) {
|
|
|
4110
4160
|
);
|
|
4111
4161
|
return { ok: true };
|
|
4112
4162
|
}
|
|
4163
|
+
if (op === "r2.md5") {
|
|
4164
|
+
// Recorded after the put that wrote these bytes (etag): the shim from 2026-10-26 hashes them.
|
|
4165
|
+
__sbSql(store, "UPDATE r2 SET md5 = ?1 WHERE bucket = ?2 AND key = ?3 AND etag = ?4", [
|
|
4166
|
+
String(msg.md5),
|
|
4167
|
+
msg.bucket,
|
|
4168
|
+
msg.key,
|
|
4169
|
+
String(msg.etag),
|
|
4170
|
+
]);
|
|
4171
|
+
return { ok: true };
|
|
4172
|
+
}
|
|
4113
4173
|
if (op === "r2.list") {
|
|
4174
|
+
// The prefix matches exactly at every date: LIKE ignored case and read "_" and "%" as
|
|
4175
|
+
// wildcards. With `page` (the shim from 2026-10-26), a page after the cursor.
|
|
4176
|
+
if (msg.page) return __sbR2ListPage(store, msg);
|
|
4114
4177
|
const r = __sbSql(
|
|
4115
4178
|
store,
|
|
4116
|
-
"SELECT key, size, etag, uploaded FROM r2 WHERE bucket = ? AND key
|
|
4179
|
+
"SELECT key, size, etag, uploaded FROM r2 WHERE bucket = ?1 AND substr(key, 1, length(?2)) = ?2 ORDER BY key",
|
|
4117
4180
|
[msg.bucket, msg.prefix || ""],
|
|
4118
4181
|
);
|
|
4119
4182
|
const objects = [];
|
package/src/utf8-decode.test.js
CHANGED
|
@@ -12,7 +12,9 @@ const end = src.indexOf("// sproutboat #181 — wrap a Response");
|
|
|
12
12
|
if (start === -1 || end === -1 || end < start) throw new Error("__sbFromUtf8 block not found in prelude");
|
|
13
13
|
|
|
14
14
|
// oxlint-disable-next-line no-function-constructor -- lifting a pure block out of the text-only prelude for test
|
|
15
|
-
const load = new Function(
|
|
15
|
+
const load = new Function(
|
|
16
|
+
`// The C fast paths (Porffor.c) cannot run here: the stubs make the JS fallback run.\nfunction __sbFromUtf8Native() { return ""; }\nfunction __sbToBytesNative() { return ""; }\n${src.slice(start, end)}\nreturn { __sbFromUtf8 };`,
|
|
17
|
+
);
|
|
16
18
|
const { __sbFromUtf8 } = load();
|
|
17
19
|
|
|
18
20
|
/** Build a bytestring (one char per raw byte) the way __sbToBytes/porf_native_fetch_alloc_bytestring would, from real UTF-8 bytes. */
|
|
@@ -78,7 +80,7 @@ if (rbrStart === -1 || rbrEnd === -1 || rbrEnd < rbrStart) {
|
|
|
78
80
|
// oxlint-disable-next-line no-function-constructor -- lifting a pure block out of the text-only prelude for test
|
|
79
81
|
const loadRbr = new Function(
|
|
80
82
|
"Response",
|
|
81
|
-
|
|
83
|
+
`// The C fast paths (Porffor.c) cannot run here: the stubs make the JS fallback run.\nfunction __sbFromUtf8Native() { return ""; }\nfunction __sbToBytesNative() { return ""; }\n${src.slice(start, end)}\n${src.slice(rbrStart, rbrEnd)}\nreturn { __sbRawBodyResponse };`,
|
|
82
84
|
);
|
|
83
85
|
// A minimal Response stand-in: only what __sbRawBodyResponse touches (the
|
|
84
86
|
// constructor call and setting .text/.json on the instance), no Porffor
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
5
|
+
import { ensurePorffor } from "../../toolchain/src/acquire";
|
|
6
|
+
import { preludePath, TRANSPORT_MARKER, transportPath, wrapNativeFetchHandler } from "../src/wrap";
|
|
7
|
+
|
|
8
|
+
// sproutd #22: __sbFromUtf8 and __sbToBytes run in C (__sbFromUtf8Native, __sbToBytesNative).
|
|
9
|
+
// They must give exactly what the JS loops gave, and a body of megabytes must not take
|
|
10
|
+
// hundreds of megabytes of heap (the JS made one string per character: a 5 MiB R2 object
|
|
11
|
+
// read with text() took about 400 MiB).
|
|
12
|
+
const handler = `const bytes = (list) => { let s = ""; for (const b of list) s += String.fromCharCode(b); return s; };
|
|
13
|
+
const codes = (s) => { const out = []; for (let i = 0; i < s.length; i++) out.push(s.charCodeAt(i)); return out; };
|
|
14
|
+
export default {fetch(request) {
|
|
15
|
+
if (new URL(request.url).pathname === "/big") {
|
|
16
|
+
const text = __sbFromUtf8("\\u00c3\\u00a9".repeat(4 * 1024 * 1024));
|
|
17
|
+
const utf8 = __sbToBytes("\\u2713".repeat(1024 * 1024));
|
|
18
|
+
const ascii = __sbToBytes("a".repeat(8 * 1024 * 1024));
|
|
19
|
+
return new Response(JSON.stringify([text.length, text.charCodeAt(0), utf8.length, utf8.charCodeAt(2), ascii.length]));
|
|
20
|
+
}
|
|
21
|
+
const decode = [
|
|
22
|
+
"hello", bytes([0x63, 0x61, 0x66, 0xc3, 0xa9, 0x20, 0xf0, 0x9f, 0x8c, 0xb1, 0x20, 0xe2, 0x9c, 0x93]),
|
|
23
|
+
bytes([0xc2]), bytes([0xe9, 0x41]), bytes([0xf0, 0x9f, 0x8c]), bytes([0xe2, 0x28, 0xa1]), bytes([0xc0, 0x80]),
|
|
24
|
+
bytes([0xf8, 0x88, 0x80, 0x80, 0x80]), bytes([0xed, 0xa0, 0x80]), bytes([0xf4, 0x90, 0x80, 0x80]), bytes([0xff, 0xfe, 0x00]),
|
|
25
|
+
JSON.parse('"caf\\\\u00c3\\\\u00a9 \\\\u0100"'), JSON.parse('"\\\\u00e2\\\\u009c\\\\u0093"'), "",
|
|
26
|
+
];
|
|
27
|
+
const encode = [
|
|
28
|
+
"hello", "caf\\u00e9", "\\u2713 \\ud83c\\udf31", "\\ud800x", "\\udc00", "a\\ud800", "\\u0100\\u07ff\\u0800\\uffff", "",
|
|
29
|
+
new Uint8Array([0, 255, 128]), new Uint8Array([1, 2, 3]).buffer, new Uint8Array(new Uint8Array([9, 8, 7, 6]).buffer, 1, 2), new Uint8Array(0),
|
|
30
|
+
];
|
|
31
|
+
return new Response(JSON.stringify({ decode: decode.map((s) => codes(__sbFromUtf8(s))), encode: encode.map((v) => codes(__sbToBytes(v))) }));
|
|
32
|
+
}};`;
|
|
33
|
+
|
|
34
|
+
// The JS the C replaces, lifted from the prelude with the C stubbed out, as the reference.
|
|
35
|
+
const core = await readFile(preludePath, "utf8");
|
|
36
|
+
const lift = (from: string, to: string) => {
|
|
37
|
+
const start = core.indexOf(from);
|
|
38
|
+
const end = core.indexOf(to, start);
|
|
39
|
+
assert(start >= 0 && end > start, `${from} block not found in prelude`);
|
|
40
|
+
return core.slice(start, end);
|
|
41
|
+
};
|
|
42
|
+
// oxlint-disable-next-line no-function-constructor -- lifting pure blocks out of the text-only prelude for the reference
|
|
43
|
+
const reference = new Function(
|
|
44
|
+
`function __sbFromUtf8Native() { return ""; }\nfunction __sbToBytesNative() { return ""; }\n` +
|
|
45
|
+
lift("function __sbIsStr(", "function __sbIsFn(") +
|
|
46
|
+
lift("const __SB_BYTES_WINDOW", "function __sbBufFrom(") +
|
|
47
|
+
lift("function __sbFromUtf8(", "// sproutboat #181") +
|
|
48
|
+
"return { __sbFromUtf8, __sbToBytes };",
|
|
49
|
+
)();
|
|
50
|
+
const bytes = (list: number[]) => String.fromCharCode(...list);
|
|
51
|
+
const codes = (s: string) => [...s].flatMap((c) => [...Array(c.length).keys()].map((i) => c.charCodeAt(i)));
|
|
52
|
+
const want = {
|
|
53
|
+
decode: [
|
|
54
|
+
"hello",
|
|
55
|
+
bytes([0x63, 0x61, 0x66, 0xc3, 0xa9, 0x20, 0xf0, 0x9f, 0x8c, 0xb1, 0x20, 0xe2, 0x9c, 0x93]),
|
|
56
|
+
bytes([0xc2]),
|
|
57
|
+
bytes([0xe9, 0x41]),
|
|
58
|
+
bytes([0xf0, 0x9f, 0x8c]),
|
|
59
|
+
bytes([0xe2, 0x28, 0xa1]),
|
|
60
|
+
bytes([0xc0, 0x80]),
|
|
61
|
+
bytes([0xf8, 0x88, 0x80, 0x80, 0x80]),
|
|
62
|
+
bytes([0xed, 0xa0, 0x80]),
|
|
63
|
+
bytes([0xf4, 0x90, 0x80, 0x80]),
|
|
64
|
+
bytes([0xff, 0xfe, 0x00]),
|
|
65
|
+
"café Ā",
|
|
66
|
+
"â\u009c\u0093",
|
|
67
|
+
"",
|
|
68
|
+
].map((s) => codes(reference.__sbFromUtf8(s))),
|
|
69
|
+
encode: [
|
|
70
|
+
"hello",
|
|
71
|
+
"café",
|
|
72
|
+
"✓ 🌱",
|
|
73
|
+
"\ud800x",
|
|
74
|
+
"\udc00",
|
|
75
|
+
"a\ud800",
|
|
76
|
+
"Ā߿ࠀ",
|
|
77
|
+
"",
|
|
78
|
+
new Uint8Array([0, 255, 128]),
|
|
79
|
+
new Uint8Array([1, 2, 3]).buffer,
|
|
80
|
+
new Uint8Array(new Uint8Array([9, 8, 7, 6]).buffer, 1, 2),
|
|
81
|
+
new Uint8Array(0),
|
|
82
|
+
].map((v) => codes(reference.__sbToBytes(v))),
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
function freePort(): number {
|
|
86
|
+
const listener = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } });
|
|
87
|
+
const port = listener.port;
|
|
88
|
+
listener.stop(true);
|
|
89
|
+
return port;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const workdir = await mkdtemp(join(tmpdir(), "sb-native-byte-text-"));
|
|
93
|
+
let server: ReturnType<typeof Bun.spawn> | undefined;
|
|
94
|
+
try {
|
|
95
|
+
const porffor = await ensurePorffor();
|
|
96
|
+
const node = Bun.which("node");
|
|
97
|
+
if (!node) throw new Error("native byte-text test needs node on PATH");
|
|
98
|
+
const broker = await readFile(transportPath("broker"), "utf8");
|
|
99
|
+
assert(core.includes(TRANSPORT_MARKER), "runtime prelude is missing its transport marker");
|
|
100
|
+
const generated = join(workdir, "bytes.generated.js");
|
|
101
|
+
const binary = join(workdir, "bytes-native");
|
|
102
|
+
await writeFile(generated, wrapNativeFetchHandler(handler, core.replace(TRANSPORT_MARKER, broker)));
|
|
103
|
+
const compile = Bun.spawn(
|
|
104
|
+
[node, resolve(porffor, "runtime/index.js"), "native", generated, "-o", binary, "-s", "-O0"],
|
|
105
|
+
{
|
|
106
|
+
cwd: workdir,
|
|
107
|
+
stdout: "pipe",
|
|
108
|
+
stderr: "pipe",
|
|
109
|
+
},
|
|
110
|
+
);
|
|
111
|
+
const [exitCode, stdout, stderr] = await Promise.all([
|
|
112
|
+
compile.exited,
|
|
113
|
+
new Response(compile.stdout).text(),
|
|
114
|
+
new Response(compile.stderr).text(),
|
|
115
|
+
]);
|
|
116
|
+
assert.equal(exitCode, 0, `Porffor native compile failed:\n${stderr}\n${stdout}`);
|
|
117
|
+
|
|
118
|
+
const port = freePort();
|
|
119
|
+
// Peak RSS: /proc's high-water mark on Linux, time(1) on macOS.
|
|
120
|
+
const linux = process.platform === "linux";
|
|
121
|
+
server = Bun.spawn(linux ? [binary] : ["/usr/bin/time", "-l", binary], {
|
|
122
|
+
cwd: workdir,
|
|
123
|
+
env: { ...process.env, PORT: String(port) },
|
|
124
|
+
stdout: "ignore",
|
|
125
|
+
stderr: "pipe",
|
|
126
|
+
});
|
|
127
|
+
let ready = false;
|
|
128
|
+
for (let attempt = 0; attempt < 50 && !ready; attempt++) {
|
|
129
|
+
try {
|
|
130
|
+
await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(500) });
|
|
131
|
+
ready = true;
|
|
132
|
+
} catch {
|
|
133
|
+
if (server.exitCode !== null) break;
|
|
134
|
+
await Bun.sleep(100);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
assert(ready, "compiled handler did not start");
|
|
138
|
+
const got = await (await fetch(`http://127.0.0.1:${port}/`)).json();
|
|
139
|
+
assert.deepEqual(got.decode, want.decode, "__sbFromUtf8 in C decodes as the JS did");
|
|
140
|
+
assert.deepEqual(got.encode, want.encode, "__sbToBytes in C encodes as the JS did");
|
|
141
|
+
console.log("C byte/text conversions match the JS ones");
|
|
142
|
+
|
|
143
|
+
const big = await (await fetch(`http://127.0.0.1:${port}/big`)).json();
|
|
144
|
+
assert.deepEqual(big, [4 * 1024 * 1024, 0xe9, 3 * 1024 * 1024, 0x93, 8 * 1024 * 1024]);
|
|
145
|
+
let peak = 0;
|
|
146
|
+
if (linux) {
|
|
147
|
+
const status = await readFile(`/proc/${server.pid}/status`, "utf8");
|
|
148
|
+
peak = Number(/VmHWM:\s+(\d+) kB/.exec(status)?.[1] ?? 0) * 1024;
|
|
149
|
+
}
|
|
150
|
+
server.kill("SIGINT");
|
|
151
|
+
const timing = linux ? "" : await new Response(server.stderr).text();
|
|
152
|
+
await server.exited;
|
|
153
|
+
server = undefined;
|
|
154
|
+
if (!linux) peak = Number(/(\d+)\s+maximum resident set size/.exec(timing)?.[1] ?? 0);
|
|
155
|
+
assert(peak > 0, "peak RSS was measured");
|
|
156
|
+
// The JS took over 600 MiB for these; the C needs the strings plus one scratch copy.
|
|
157
|
+
assert(peak < 96 * 1024 * 1024, `8 MiB conversions peaked at ${(peak / 1048576).toFixed(1)} MiB`);
|
|
158
|
+
console.log(`8 MiB conversions peak at ${(peak / 1048576).toFixed(1)} MiB RSS`);
|
|
159
|
+
} finally {
|
|
160
|
+
if (server) {
|
|
161
|
+
server.kill(9);
|
|
162
|
+
await server.exited;
|
|
163
|
+
}
|
|
164
|
+
await rm(workdir, { recursive: true, force: true });
|
|
165
|
+
}
|