@runeditrun/rer 0.2.34 → 0.2.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/cli/demo-native.mjs +1 -12
- package/global/CHANGELOG.md +21 -0
- package/global/README.md +21 -7
- package/install.sh +7 -4
- package/modules/demo-host/README.md +2 -1
- package/modules/demo-host/native/config.mjs +1 -1
- package/modules/demo-host/native/deploy-r2.mjs +4 -14
- package/modules/demo-host/native/r2-client.mjs +27 -10
- package/modules/demo-host/native/r2-store.mjs +112 -0
- package/modules/demo-host/native/r2.mjs +173 -39
- package/modules/demo-host/native/resources.mjs +6 -3
- package/modules/demo-host/upgrade.mjs +8 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -6,14 +6,25 @@ checkout always gets the behaviour it was built and tested with.
|
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
9
|
-
Start a seed from an empty folder with
|
|
9
|
+
Start a seed from an empty folder, with or without Node:
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
curl -fsSL https://runeditrun.com/install.sh | sh
|
|
13
|
+
rer dev just-every/run-inkwell
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
The installer needs no Node: without Node 22 or later on PATH it installs a
|
|
17
|
+
private one for `rer` (see below). It edits no shell profile, so when
|
|
18
|
+
`~/.local/bin` is not on PATH, run the `export PATH=…` line it prints before
|
|
19
|
+
`rer dev`. Its last line names that next command. The same start also fits on
|
|
20
|
+
one line:
|
|
10
21
|
|
|
11
22
|
```sh
|
|
12
|
-
npx -y @runeditrun/rer dev just-every/run-inkwell
|
|
13
23
|
curl -fsSL https://runeditrun.com/install.sh | sh -s -- dev just-every/run-inkwell
|
|
24
|
+
npx -y @runeditrun/rer dev just-every/run-inkwell # only with Node 22 or later on PATH
|
|
14
25
|
```
|
|
15
26
|
|
|
16
|
-
|
|
27
|
+
Each of these installs the user CLI, clones into `./run-inkwell`, install dependencies, set
|
|
17
28
|
up a local owner and start the app. Open the printed `RER_OPEN` link to sign in.
|
|
18
29
|
Use `setup <repo>` to stop after setup, or `install <repo>` to stop after
|
|
19
30
|
dependency installation; each prints the next command. The repo can be
|
|
@@ -27,9 +38,9 @@ leaves the app running. The restart command includes `cd` into the checkout.
|
|
|
27
38
|
Install `rer` with one of these commands:
|
|
28
39
|
|
|
29
40
|
```sh
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
npm install -g @runeditrun/rer # npm's own global layout
|
|
41
|
+
curl -fsSL https://runeditrun.com/install.sh | sh # user install in ~/.local/share/rer, launcher ~/.local/bin/rer; no Node needed
|
|
42
|
+
npx -y @runeditrun/rer cli install # the same user install, with Node 22 or later on PATH
|
|
43
|
+
npm install -g @runeditrun/rer # npm's own global layout, with Node 22 or later on PATH
|
|
33
44
|
```
|
|
34
45
|
|
|
35
46
|
`install.sh` is the file at the root of this package. It reads the version, the
|
|
@@ -44,7 +55,10 @@ With no Node 22 or later on PATH, the script installs Node 22.20.0 from
|
|
|
44
55
|
`nodejs.org` into `~/.local/share/rer/runtimes` (under `XDG_DATA_HOME` when set),
|
|
45
56
|
verifies the tarball against that release's published `SHASUMS256.txt`, and
|
|
46
57
|
prints its location. The launcher uses this private runtime on later runs.
|
|
47
|
-
System Node and other PATH directories are left alone
|
|
58
|
+
System Node and other PATH directories are left alone: the private runtime's
|
|
59
|
+
`node`, `npm` and `npx` are not put on PATH, because `rer` is the one command
|
|
60
|
+
the installer provides. With no arguments the script ends by naming the next
|
|
61
|
+
command, `rer dev <owner>/<repo>`.
|
|
48
62
|
|
|
49
63
|
No installer edits shell profiles. The canonical launcher is `~/.local/bin/rer`.
|
|
50
64
|
Installation creates that directory if needed and never writes into another
|
package/cli/demo-native.mjs
CHANGED
|
@@ -7,7 +7,6 @@ import { upgradeDemo } from '../modules/demo-host/upgrade.mjs';
|
|
|
7
7
|
import { missing, invalid, RerError } from './errors.mjs';
|
|
8
8
|
import { DEMO_FILES } from '../modules/demo-host/plan.mjs';
|
|
9
9
|
import { cloudflare } from '../modules/deploy-kit/provider.mjs';
|
|
10
|
-
import { demoBucketName, prepareBucket } from '../modules/demo-host/native/deploy-r2.mjs';
|
|
11
10
|
import { RUNTIME_FILES } from '../modules/demo-host/native/config.mjs';
|
|
12
11
|
import { prepareNative } from '../modules/demo-host/native/prepare.mjs';
|
|
13
12
|
import { TESTING_SECRET } from '../modules/demo-host/measurement.mjs';
|
|
@@ -37,7 +36,6 @@ export async function nativeDemoCommand(project, options, store, signal, out, op
|
|
|
37
36
|
const config = { ...plan.wrangler, main: 'modules/demo-host/native/worker.mjs', compatibility_flags: ['nodejs_compat'],
|
|
38
37
|
assets: { directory: './assets', binding: 'ASSETS', run_worker_first: true }, worker_loaders: [{ binding: 'LOADER' }],
|
|
39
38
|
};
|
|
40
|
-
if (bundle.bindings.buckets.length) config.r2_buckets = [{ binding: 'NATIVE_BUCKET', bucket_name: demoBucketName(plan.worker) }];
|
|
41
39
|
config.vars.NATIVE_SOURCE_DIGEST = bundle.digest;
|
|
42
40
|
if (process.env.RER_NATIVE_TEST_TTL_MS && TESTING_SECRET.test(process.env.TURNSTILE_SECRET_KEY ?? '')) {
|
|
43
41
|
if (!/^[1-9][0-9]{2,6}$/.test(process.env.RER_NATIVE_TEST_TTL_MS)) throw invalid('RER_NATIVE_TEST_TTL_MS must be positive milliseconds');
|
|
@@ -53,16 +51,7 @@ export async function nativeDemoCommand(project, options, store, signal, out, op
|
|
|
53
51
|
const required = ['CLOUDFLARE_ACCOUNT_ID', 'CLOUDFLARE_API_TOKEN', 'TURNSTILE_SITE_KEY', 'TURNSTILE_SECRET_KEY'];
|
|
54
52
|
if (required.some(name => !process.env[name]?.trim())) throw missing(`${required.join(', ')} are required`);
|
|
55
53
|
const api = cloudflare({ accountId: process.env.CLOUDFLARE_ACCOUNT_ID, token: process.env.CLOUDFLARE_API_TOKEN });
|
|
56
|
-
config.migrations = await upgradeDemo({ api, plan });
|
|
57
|
-
if (bundle.bindings.buckets.length) {
|
|
58
|
-
const statePath = join(project.root, '.rer/demo-buckets', plan.worker + '.json');
|
|
59
|
-
const pending = await readFile(statePath, 'utf8').then(JSON.parse).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
60
|
-
const bucket = await prepareBucket(api, plan, pending);
|
|
61
|
-
await mkdir(dirname(statePath), { recursive: true, mode: 0o700 });
|
|
62
|
-
await writeFile(statePath, JSON.stringify({ record: plan.record, bucket }), { mode: 0o600 });
|
|
63
|
-
config.vars.NATIVE_BUCKET_CREATED = bucket.creation_date;
|
|
64
|
-
await writeFile(join(folder, 'wrangler.json'), JSON.stringify(config, null, 2) + '\n');
|
|
65
|
-
}
|
|
54
|
+
config.migrations = await upgradeDemo({ api, plan, report: line => store.report(line) });
|
|
66
55
|
await writeFile(join(folder, 'wrangler.json'), JSON.stringify(config, null, 2) + '\n');
|
|
67
56
|
const testing = TESTING_SECRET.test(process.env.TURNSTILE_SECRET_KEY);
|
|
68
57
|
const measurement = testing ? process.env.RER_DEMO_MEASUREMENT_TOKEN || randomBytes(32).toString('base64url') : undefined;
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.36
|
|
4
|
+
|
|
5
|
+
- **A native demo keeps each visitor's R2 files in that visitor's own storage.** A native hosted demo now stores R2 objects, their metadata and multipart uploads per binding in the visitor's own Durable Object storage, next to the app. Before, every visitor shared one demo-owned bucket, often far from the visitor. File reads and writes now answer from where the visitor is. Measured from Queensland on Beam: opening a file took 55 ms instead of 256 ms, uploading 207 ms instead of 1543 ms, and starting the demo 1.64 s instead of 2.65 s. One visitor's objects may total 256 MiB; a write past that fails with R2's error for an oversized upload. Expiry erases a visitor's objects along with the rest of their storage. A new demo creates no R2 bucket.
|
|
6
|
+
- **Paired-runtime tests survive a port clash.** When a loopback port is taken, the seed's paired-runtime test helpers retry on a fresh port. When one of a pair fails to start, they close the one that did start, so a clash fails within seconds and leaves nothing running.
|
|
7
|
+
|
|
8
|
+
### Upgrade notes
|
|
9
|
+
|
|
10
|
+
- On the next `rer deploy --demo`, a host deployed by rer 0.2.30 to 0.2.35 has its shared demo R2 bucket emptied and deleted, and the command reports this. The bucket is removed only while the deployed Worker still proves it owns it; a bucket it does not own is left alone. `rer deploy --demo --remove` still removes a bucket left by an older host.
|
|
11
|
+
- Visitors' files from the shared bucket are not carried over: each demo visitor already starts from seed data.
|
|
12
|
+
- Seeds that worked around a distant bucket (for example, by putting objects in parallel) can drop the workaround.
|
|
13
|
+
|
|
14
|
+
## 0.2.35
|
|
15
|
+
|
|
16
|
+
- **The Run page starts a seed without Node.** The shared /run page has two steps. Step 1 is `curl -fsSL https://runeditrun.com/install.sh | sh`, which brings its own Node and changes no shell settings. Step 2 is `rer dev <owner>/<repo>`, with `cd <folder> && rer dev` to start it again later. `npx -y @runeditrun/rer dev <owner>/<repo>` remains only as the alternative under "Already have Node 22 or later?". Before this, a newcomer without Node got `sh: npx: command not found` at step 2, because the installer keeps its Node private. Step 2 is built from Run data's existing `clone` line, so seeds need no edits. RUN-LOCALLY, HOMEPAGE-VARIANTS, SEED-COMMAND-CONTRACT and the READMEs describe the two steps, and the default and foundation Run requirements no longer say "Needs Node 22".
|
|
17
|
+
- **install.sh names the next step.** Run with no arguments, `install.sh` now ends with `Next, start a seed: rer dev <owner>/<repo>`. It still puts no `node`, `npm` or `npx` on PATH.
|
|
18
|
+
|
|
19
|
+
### Upgrade notes
|
|
20
|
+
|
|
21
|
+
- A seed refreshes its copy of the /run page and docs from Base to get the two steps.
|
|
22
|
+
- The installer served at runeditrun.com takes its text from the published package, so the Next line arrives with this release.
|
|
23
|
+
|
|
3
24
|
## 0.2.34
|
|
4
25
|
|
|
5
26
|
- **Optional deploy secrets.** A deploy declaration's secret entry may add `"optional": true` for an integration's credential (Pier, Stripe). When neither the Worker nor the deploy inputs hold it, the first deploy succeeds without binding it and reports `Optional secrets not set: NAME`, and a dry run names optional secrets that were not supplied. Health waits only for required secrets, and a missing required secret still refuses before any write. `rer deploy --set-secret NAME` and `--unset-secret NAME` record the Worker version they make, so the next deploy accepts that Worker and an unchanged repeat writes nothing. A declaration without `optional` plans and fingerprints exactly as before.
|
package/global/README.md
CHANGED
|
@@ -6,14 +6,25 @@ checkout always gets the behaviour it was built and tested with.
|
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
9
|
-
Start a seed from an empty folder with
|
|
9
|
+
Start a seed from an empty folder, with or without Node:
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
curl -fsSL https://runeditrun.com/install.sh | sh
|
|
13
|
+
rer dev just-every/run-inkwell
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
The installer needs no Node: without Node 22 or later on PATH it installs a
|
|
17
|
+
private one for `rer` (see below). It edits no shell profile, so when
|
|
18
|
+
`~/.local/bin` is not on PATH, run the `export PATH=…` line it prints before
|
|
19
|
+
`rer dev`. Its last line names that next command. The same start also fits on
|
|
20
|
+
one line:
|
|
10
21
|
|
|
11
22
|
```sh
|
|
12
|
-
npx -y @runeditrun/rer dev just-every/run-inkwell
|
|
13
23
|
curl -fsSL https://runeditrun.com/install.sh | sh -s -- dev just-every/run-inkwell
|
|
24
|
+
npx -y @runeditrun/rer dev just-every/run-inkwell # only with Node 22 or later on PATH
|
|
14
25
|
```
|
|
15
26
|
|
|
16
|
-
|
|
27
|
+
Each of these installs the user CLI, clones into `./run-inkwell`, install dependencies, set
|
|
17
28
|
up a local owner and start the app. Open the printed `RER_OPEN` link to sign in.
|
|
18
29
|
Use `setup <repo>` to stop after setup, or `install <repo>` to stop after
|
|
19
30
|
dependency installation; each prints the next command. The repo can be
|
|
@@ -27,9 +38,9 @@ leaves the app running. The restart command includes `cd` into the checkout.
|
|
|
27
38
|
Install `rer` with one of these commands:
|
|
28
39
|
|
|
29
40
|
```sh
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
npm install -g @runeditrun/rer # npm's own global layout
|
|
41
|
+
curl -fsSL https://runeditrun.com/install.sh | sh # user install in ~/.local/share/rer, launcher ~/.local/bin/rer; no Node needed
|
|
42
|
+
npx -y @runeditrun/rer cli install # the same user install, with Node 22 or later on PATH
|
|
43
|
+
npm install -g @runeditrun/rer # npm's own global layout, with Node 22 or later on PATH
|
|
33
44
|
```
|
|
34
45
|
|
|
35
46
|
`install.sh` is the file at the root of this package. It reads the version, the
|
|
@@ -44,7 +55,10 @@ With no Node 22 or later on PATH, the script installs Node 22.20.0 from
|
|
|
44
55
|
`nodejs.org` into `~/.local/share/rer/runtimes` (under `XDG_DATA_HOME` when set),
|
|
45
56
|
verifies the tarball against that release's published `SHASUMS256.txt`, and
|
|
46
57
|
prints its location. The launcher uses this private runtime on later runs.
|
|
47
|
-
System Node and other PATH directories are left alone
|
|
58
|
+
System Node and other PATH directories are left alone: the private runtime's
|
|
59
|
+
`node`, `npm` and `npx` are not put on PATH, because `rer` is the one command
|
|
60
|
+
the installer provides. With no arguments the script ends by naming the next
|
|
61
|
+
command, `rer dev <owner>/<repo>`.
|
|
48
62
|
|
|
49
63
|
No installer edits shell profiles. The canonical launcher is `~/.local/bin/rer`.
|
|
50
64
|
Installation creates that directory if needed and never writes into another
|
package/install.sh
CHANGED
|
@@ -141,11 +141,14 @@ NODE
|
|
|
141
141
|
tar -xzf "$work/rer.tgz" -C "$work/stage" || fail "could not unpack rer $version; nothing was installed."
|
|
142
142
|
RER_INSTALL_CHANNEL=script node "$work/stage/package/global/rer.mjs" cli install </dev/null \
|
|
143
143
|
|| fail "rer cli install failed; any previous install is still selected."
|
|
144
|
-
if [ "$#" -
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
144
|
+
if [ "$#" -eq 0 ]; then
|
|
145
|
+
# The same next step a Run page gives; no npx, since Node may be rer's own.
|
|
146
|
+
printf 'Next, start a seed: rer dev <owner>/<repo>\n'
|
|
147
|
+
return
|
|
148
148
|
fi
|
|
149
|
+
PATH="$HOME/.local/bin:$PATH"
|
|
150
|
+
export PATH
|
|
151
|
+
"$HOME/.local/bin/rer" "$@"
|
|
149
152
|
}
|
|
150
153
|
|
|
151
154
|
install_node() {
|
|
@@ -5,7 +5,8 @@ host. See [adoption and deployment](../../export/DEMO.md#hosting-the-demo).
|
|
|
5
5
|
|
|
6
6
|
- `build.mjs`: ordinary Vite output, declared SQL and public configuration,
|
|
7
7
|
using the shared container migration-inventory and image-variable checks.
|
|
8
|
-
- `native/`: bundle preparation, per-visitor D1/seed-DO facets,
|
|
8
|
+
- `native/`: bundle preparation, per-visitor D1/seed-DO facets, R2 in the
|
|
9
|
+
visitor's own storage (`r2.mjs` semantics, `r2-store.mjs` rows and chunks),
|
|
9
10
|
durable queue delivery, seed-DO alarms and declared UTC crons.
|
|
10
11
|
A narrow bridge exposes only visitor resource operations to the seed; own-host
|
|
11
12
|
outbound requests return to the same visitor, including declared secondary hosts.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parseCron } from './cron.mjs';
|
|
2
2
|
|
|
3
|
-
export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
|
|
3
|
+
export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'r2-store', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
|
|
4
4
|
export const ISOLATE_FILES = ['bindings', 'namespace', 'r2-client', 'd1', 'sql', 'seed-object'];
|
|
5
5
|
export function nativeBindings(config) {
|
|
6
6
|
for (const kind of ['kv_namespaces', 'services', 'workflows', 'ai', 'vectorize', 'hyperdrive', 'analytics_engine_datasets', 'browser', 'images', 'send_email', 'dispatch_namespaces', 'mtls_certificates', 'pipelines', 'secrets_store_secrets', 'unsafe', 'vpc_services', 'vpc_networks', 'ratelimits', 'worker_loaders', 'containers', 'ai_search_namespaces', 'ai_search', 'agent_memory', 'connect', 'artifacts', 'unsafe_hello_world', 'flagship', 'logfwdr', 'version_metadata']) {
|
|
@@ -1,23 +1,13 @@
|
|
|
1
|
-
import { workerSettings } from '../../deploy-kit/owner.mjs';
|
|
2
|
-
import { servedVersion } from '../../deploy-kit/site.mjs';
|
|
3
1
|
import { r2ObjectPages, r2ObjectPath } from '../../deploy-kit/r2-objects.mjs';
|
|
4
2
|
|
|
5
|
-
|
|
3
|
+
// rer 0.2.30-0.2.33 kept every visitor's R2 objects in one demo bucket. Demo
|
|
4
|
+
// R2 now lives in each visitor's storage; an upgrade or removal empties and
|
|
5
|
+
// deletes that bucket when the served Worker still proves it owns it.
|
|
6
|
+
const demoBucketName = worker => `${worker.replace(/-+/g, '-').slice(0, 48)}-objects`;
|
|
6
7
|
const missing = async (api, name) => {
|
|
7
8
|
try { return await api.request(`/r2/buckets/${name}`); }
|
|
8
9
|
catch (error) { if (error.status === 404) return null; throw error; }
|
|
9
10
|
};
|
|
10
|
-
export async function prepareBucket(api, plan, pending) {
|
|
11
|
-
const name = demoBucketName(plan.worker), bucket = await missing(api, name), settings = await workerSettings(api, plan.worker);
|
|
12
|
-
if (!bucket && settings?.bindings?.some(item => item.name === 'NATIVE_BUCKET')) throw new Error(`Demo R2 bucket is missing: ${name}`);
|
|
13
|
-
if (bucket) {
|
|
14
|
-
const version = settings && await servedVersion(api, plan.worker);
|
|
15
|
-
if (!settings && pending?.record === plan.record && pending.bucket?.name === name && pending.bucket.creation_date === bucket.creation_date) return bucket;
|
|
16
|
-
if (!version?.current || version.annotation !== plan.record || !ownsBucket(settings, bucket)) throw new Error(`Demo R2 bucket ownership is unproven: ${name}`);
|
|
17
|
-
return bucket;
|
|
18
|
-
}
|
|
19
|
-
return api.request('/r2/buckets', { method: 'POST', body: { name } });
|
|
20
|
-
}
|
|
21
11
|
function ownsBucket(settings, bucket) {
|
|
22
12
|
return settings?.bindings?.some(item => item.type === 'r2_bucket' && item.name === 'NATIVE_BUCKET' && item.bucket_name === bucket.name)
|
|
23
13
|
&& settings.bindings.some(item => item.name === 'NATIVE_BUCKET_CREATED' && item.text === bucket.creation_date);
|
|
@@ -1,35 +1,52 @@
|
|
|
1
|
+
const HEADERS = { contentType: 'content-type', contentLanguage: 'content-language', contentDisposition: 'content-disposition', contentEncoding: 'content-encoding', cacheControl: 'cache-control', cacheExpiry: 'expires' };
|
|
1
2
|
function object(value) {
|
|
2
3
|
if (!value) return value;
|
|
3
4
|
value.writeHttpMetadata = headers => {
|
|
4
5
|
for (const [key, content] of Object.entries(value.httpMetadata ?? {})) {
|
|
5
|
-
const header =
|
|
6
|
+
const header = HEADERS[key];
|
|
6
7
|
if (header) headers.set(header, content instanceof Date ? content.toUTCString() : content);
|
|
7
8
|
}
|
|
8
9
|
};
|
|
9
|
-
if (value
|
|
10
|
+
if ('body' in value) {
|
|
10
11
|
const response = new Response(value.body);
|
|
12
|
+
value.body = response.body;
|
|
11
13
|
Object.defineProperty(value, 'bodyUsed', { get: () => response.bodyUsed });
|
|
12
14
|
for (const method of ['arrayBuffer', 'text', 'json', 'blob']) value[method] = () => response[method]();
|
|
13
15
|
value.bytes = async () => new Uint8Array(await response.arrayBuffer());
|
|
14
16
|
}
|
|
15
17
|
return value;
|
|
16
18
|
}
|
|
17
|
-
//
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const
|
|
19
|
+
// Headers forms of R2 options become their plain forms before crossing RPC.
|
|
20
|
+
function httpMetadata(value) {
|
|
21
|
+
if (!(value instanceof Headers)) return value;
|
|
22
|
+
const result = {};
|
|
23
|
+
for (const [key, header] of Object.entries(HEADERS)) if (value.has(header)) result[key] = key === 'cacheExpiry' ? new Date(value.get(header)) : value.get(header);
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
function conditional(value) {
|
|
27
|
+
if (!(value instanceof Headers)) return value;
|
|
28
|
+
const date = name => value.has(name) ? new Date(value.get(name)) : undefined;
|
|
29
|
+
return { etagMatches: value.get('if-match') ?? undefined, etagDoesNotMatch: value.get('if-none-match') ?? undefined,
|
|
30
|
+
uploadedAfter: date('if-modified-since'), uploadedBefore: date('if-unmodified-since'), secondsGranularity: true };
|
|
31
|
+
}
|
|
32
|
+
const plain = (options = {}) => ({ ...options,
|
|
33
|
+
...(options.httpMetadata ? { httpMetadata: httpMetadata(options.httpMetadata) } : {}),
|
|
34
|
+
...(options.onlyIf ? { onlyIf: conditional(options.onlyIf) } : {}),
|
|
35
|
+
...(options.range instanceof Headers ? { range: options.range.has('range') ? { header: options.range.get('range') } : undefined } : {}) });
|
|
36
|
+
// Bodies stream to the visitor's storage; only Blobs are read first.
|
|
37
|
+
const body = value => value instanceof Blob ? value.arrayBuffer() : value;
|
|
21
38
|
export class DemoR2 {
|
|
22
39
|
constructor(host, binding) { this.host = host; this.binding = binding; }
|
|
23
40
|
call(method, ...args) { return this.host.r2(this.binding, method, args); }
|
|
24
|
-
async get(key, options) { return object(await this.call('get', key, options)); }
|
|
41
|
+
async get(key, options) { return object(await this.call('get', key, plain(options))); }
|
|
25
42
|
async head(key) { return object(await this.call('head', key)); }
|
|
26
|
-
async put(key,
|
|
43
|
+
async put(key, value, options) { return object(await this.call('put', key, await body(value), plain(options))); }
|
|
27
44
|
delete(keys) { return this.call('delete', keys); }
|
|
28
45
|
async list(options) { const page = await this.call('list', options); page.objects = page.objects.map(object); return page; }
|
|
29
|
-
async createMultipartUpload(key, options) { return this.resumeMultipartUpload(key, await this.call('createMultipartUpload', key, options)); }
|
|
46
|
+
async createMultipartUpload(key, options) { return this.resumeMultipartUpload(key, await this.call('createMultipartUpload', key, plain(options))); }
|
|
30
47
|
resumeMultipartUpload(key, uploadId) {
|
|
31
48
|
return { key, uploadId,
|
|
32
|
-
uploadPart: async (part,
|
|
49
|
+
uploadPart: async (part, value, options) => this.call('uploadPart', key, uploadId, part, await body(value), options),
|
|
33
50
|
abort: () => this.call('abort', key, uploadId),
|
|
34
51
|
complete: async parts => object(await this.call('complete', key, uploadId, parts)),
|
|
35
52
|
};
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
// Demo R2 objects live in the visitor's own SQLite storage, beside the app,
|
|
4
|
+
// so every call is local to wherever the visitor's Durable Object runs.
|
|
5
|
+
// Bodies are split into chunks well under the 2 MB per-value limit.
|
|
6
|
+
export const CHUNK = 1 << 20;
|
|
7
|
+
// Seeded data is kilobytes and seed upload limits are at most ~20 MB per file.
|
|
8
|
+
// 256 MiB leaves room for real demo uploads while bounding one hour's storage.
|
|
9
|
+
export const VISITOR_LIMIT = 256 * CHUNK;
|
|
10
|
+
const SCHEMA = `CREATE TABLE IF NOT EXISTS rer_r2_objects(binding TEXT NOT NULL, key TEXT NOT NULL, version TEXT NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, uploaded INTEGER NOT NULL, http TEXT NOT NULL, custom TEXT NOT NULL, checksums TEXT NOT NULL, class TEXT NOT NULL, PRIMARY KEY(binding, key));
|
|
11
|
+
CREATE TABLE IF NOT EXISTS rer_r2_chunks(version TEXT NOT NULL, seq INTEGER NOT NULL, start INTEGER NOT NULL, size INTEGER NOT NULL, data BLOB NOT NULL, PRIMARY KEY(version, seq));
|
|
12
|
+
CREATE TABLE IF NOT EXISTS rer_r2_uploads(id TEXT PRIMARY KEY, binding TEXT NOT NULL, key TEXT NOT NULL, options TEXT NOT NULL);
|
|
13
|
+
CREATE TABLE IF NOT EXISTS rer_r2_parts(upload TEXT NOT NULL, part INTEGER NOT NULL, version TEXT NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, PRIMARY KEY(upload, part));`;
|
|
14
|
+
const HASHES = { md5: 'MD5', sha1: 'SHA-1', sha256: 'SHA-256', sha384: 'SHA-384', sha512: 'SHA-512' };
|
|
15
|
+
|
|
16
|
+
export const randomId = () => [...crypto.getRandomValues(new Uint8Array(16))].map(byte => byte.toString(16).padStart(2, '0')).join('');
|
|
17
|
+
export function store(storage) {
|
|
18
|
+
const sql = storage.sql;
|
|
19
|
+
sql.exec(SCHEMA);
|
|
20
|
+
const rows = (query, ...args) => sql.exec(query, ...args).toArray();
|
|
21
|
+
return { sql, rows, transaction: run => storage.transactionSync(run),
|
|
22
|
+
object: (binding, key) => rows('SELECT * FROM rer_r2_objects WHERE binding = ? AND key = ?', binding, key)[0] ?? null,
|
|
23
|
+
used: () => rows('SELECT COALESCE(SUM(size), 0) AS n FROM rer_r2_chunks')[0].n,
|
|
24
|
+
drop(version) { sql.exec('DELETE FROM rer_r2_chunks WHERE version = ?', version); },
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const bytesOf = body => {
|
|
29
|
+
if (body === null || body === undefined) return new Uint8Array();
|
|
30
|
+
if (typeof body === 'string') return new TextEncoder().encode(body);
|
|
31
|
+
if (body instanceof ArrayBuffer) return new Uint8Array(body);
|
|
32
|
+
if (ArrayBuffer.isView(body)) return new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
|
|
33
|
+
return null;
|
|
34
|
+
};
|
|
35
|
+
async function* pieces(body) {
|
|
36
|
+
const whole = bytesOf(body);
|
|
37
|
+
if (whole) { yield whole; return; }
|
|
38
|
+
if (!(body instanceof ReadableStream)) throw new TypeError('put: The body must be a string, ArrayBuffer, ArrayBufferView, Blob or ReadableStream.');
|
|
39
|
+
const reader = body.getReader();
|
|
40
|
+
try {
|
|
41
|
+
for (;;) {
|
|
42
|
+
const { done, value } = await reader.read();
|
|
43
|
+
if (done) return;
|
|
44
|
+
yield bytesOf(value) ?? (() => { throw new TypeError('put: The body stream must contain bytes.'); })();
|
|
45
|
+
}
|
|
46
|
+
} finally { reader.releaseLock(); }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Writes a body under a fresh version. The object row is swapped separately,
|
|
50
|
+
// so a failed or oversized write leaves the previous object untouched.
|
|
51
|
+
export async function writeBody(db, method, body, { hashes = [], replaces = 0 } = {}) {
|
|
52
|
+
const version = randomId(), digests = Object.fromEntries(['md5', ...hashes].map(name => [name, createHash(name)]));
|
|
53
|
+
let size = 0, seq = 0, pending = [], pendingSize = 0;
|
|
54
|
+
const flush = () => {
|
|
55
|
+
if (!pendingSize) return;
|
|
56
|
+
if (db.used() - replaces + pendingSize > VISITOR_LIMIT) {
|
|
57
|
+
const error = new Error(`${method}: Your proposed upload exceeds this demo's storage limit of ${VISITOR_LIMIT / CHUNK} MiB per visitor. (100100)`);
|
|
58
|
+
error.code = 100100;
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
const data = new Uint8Array(pendingSize);
|
|
62
|
+
let at = 0;
|
|
63
|
+
for (const piece of pending) { data.set(piece, at); at += piece.length; }
|
|
64
|
+
db.sql.exec('INSERT INTO rer_r2_chunks(version, seq, start, size, data) VALUES(?, ?, ?, ?, ?)', version, seq++, size - pendingSize, pendingSize, data.buffer);
|
|
65
|
+
pending = []; pendingSize = 0;
|
|
66
|
+
};
|
|
67
|
+
try {
|
|
68
|
+
for await (const piece of pieces(body)) {
|
|
69
|
+
for (let at = 0; at < piece.length;) {
|
|
70
|
+
const take = Math.min(CHUNK - pendingSize, piece.length - at), part = piece.subarray(at, at + take);
|
|
71
|
+
for (const digest of Object.values(digests)) digest.update(part);
|
|
72
|
+
pending.push(part); pendingSize += take; size += take; at += take;
|
|
73
|
+
if (pendingSize === CHUNK) flush();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
flush();
|
|
77
|
+
} catch (error) { db.drop(version); throw error; }
|
|
78
|
+
const sums = Object.fromEntries(Object.entries(digests).map(([name, digest]) => [name, digest.digest('hex')]));
|
|
79
|
+
return { version, size, sums };
|
|
80
|
+
}
|
|
81
|
+
export const checksumNames = options => Object.keys(HASHES).filter(name => name !== 'md5' && options?.[name] !== undefined);
|
|
82
|
+
export function verifyChecksums(method, options, sums) {
|
|
83
|
+
for (const name of Object.keys(HASHES)) {
|
|
84
|
+
const expected = options?.[name];
|
|
85
|
+
if (expected === undefined) continue;
|
|
86
|
+
const hex = typeof expected === 'string' ? expected.toLowerCase() : Buffer.from(bytesOf(expected)).toString('hex');
|
|
87
|
+
if (hex !== sums[name]) throw new Error(`${method}: The ${HASHES[name]} checksum you specified did not match what we received. (10037)`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// A range read of one version; small reads are returned whole, larger ones
|
|
92
|
+
// stream chunk by chunk so a visitor's big object never sits in memory.
|
|
93
|
+
export function readBody(db, version, offset, length) {
|
|
94
|
+
const end = offset + length, query = (from, limit) => db.rows('SELECT seq, start, data FROM rer_r2_chunks WHERE version = ? AND seq >= ? AND start < ? AND start + size > ? ORDER BY seq LIMIT ?', version, from, end, offset, limit);
|
|
95
|
+
const slice = row => { const data = new Uint8Array(row.data); return data.subarray(Math.max(0, offset - row.start), Math.min(data.length, end - row.start)); };
|
|
96
|
+
if (length <= 4 * CHUNK) {
|
|
97
|
+
const result = new Uint8Array(length);
|
|
98
|
+
let at = 0;
|
|
99
|
+
for (const row of query(0, 1e9)) { const part = slice(row); result.set(part, at); at += part.length; }
|
|
100
|
+
if (at !== length) throw new Error('get: The object changed while it was read.');
|
|
101
|
+
return result;
|
|
102
|
+
}
|
|
103
|
+
let next = 0, sent = 0;
|
|
104
|
+
return new ReadableStream({ type: 'bytes', pull(controller) {
|
|
105
|
+
const [row] = query(next, 1);
|
|
106
|
+
if (!row) { if (sent === length) controller.close(); else controller.error(new Error('get: The object changed while it was read.')); return; }
|
|
107
|
+
next = row.seq + 1;
|
|
108
|
+
const part = slice(row); sent += part.length;
|
|
109
|
+
controller.enqueue(part.slice());
|
|
110
|
+
if (sent === length) controller.close();
|
|
111
|
+
} });
|
|
112
|
+
}
|
|
@@ -1,43 +1,177 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { writeBody, readBody, randomId, checksumNames, verifyChecksums } from './r2-store.mjs';
|
|
3
|
+
|
|
4
|
+
// R2 semantics over the visitor's own storage: one namespace per binding,
|
|
5
|
+
// conditional and ranged reads, delimited listing and multipart uploads.
|
|
6
|
+
const failure = (method, message, code) => Object.assign(new Error(`${method}: ${message} (${code})`), { code });
|
|
7
|
+
const json = value => JSON.stringify(value ?? {});
|
|
8
|
+
function httpMetadata(text) {
|
|
9
|
+
const value = JSON.parse(text);
|
|
10
|
+
if (value.cacheExpiry) value.cacheExpiry = new Date(value.cacheExpiry);
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
const buffer = hex => Uint8Array.from(Buffer.from(hex, 'hex')).buffer;
|
|
14
|
+
function metadata(row, include = ['httpMetadata', 'customMetadata']) {
|
|
15
|
+
return { key: row.key, version: row.version, size: row.size, etag: row.etag, httpEtag: `"${row.etag}"`, uploaded: new Date(row.uploaded),
|
|
16
|
+
httpMetadata: include.includes('httpMetadata') ? httpMetadata(row.http) : {}, customMetadata: include.includes('customMetadata') ? JSON.parse(row.custom) : {},
|
|
17
|
+
checksums: Object.fromEntries(Object.entries(JSON.parse(row.checksums)).map(([name, hex]) => [name, buffer(hex)])), storageClass: row.class };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const etags = value => [].concat(value).flatMap(item => String(item).split(',')).map(item => item.trim()).filter(Boolean)
|
|
21
|
+
.map(item => item === '*' ? { wildcard: true } : { weak: item.startsWith('W/'), value: item.replace(/^W\//, '').replace(/^"(.*)"$/, '$1') });
|
|
22
|
+
const includes = (list, etag, weak) => etags(list).some(item => item.wildcard || (item.value === etag && (weak || !item.weak)));
|
|
23
|
+
// The precedence of If-Match over If-Unmodified-Since (and so on) follows R2.
|
|
24
|
+
export function satisfied(condition, row) {
|
|
25
|
+
if (!condition) return true;
|
|
26
|
+
if (!row) return condition.etagMatches === undefined && condition.uploadedAfter === undefined;
|
|
27
|
+
const truncate = condition.secondsGranularity ? time => Math.floor(time / 1000) * 1000 : time => time, uploaded = truncate(row.uploaded);
|
|
28
|
+
const match = condition.etagMatches === undefined || includes(condition.etagMatches, row.etag, false);
|
|
29
|
+
const noneMatch = condition.etagDoesNotMatch === undefined || !includes(condition.etagDoesNotMatch, row.etag, true);
|
|
30
|
+
const after = condition.uploadedAfter === undefined || truncate(new Date(condition.uploadedAfter).getTime()) < uploaded || (condition.etagDoesNotMatch !== undefined && noneMatch);
|
|
31
|
+
const before = condition.uploadedBefore === undefined || uploaded < truncate(new Date(condition.uploadedBefore).getTime()) || (condition.etagMatches !== undefined && match);
|
|
32
|
+
return match && noneMatch && after && before;
|
|
33
|
+
}
|
|
34
|
+
function range(value, size) {
|
|
35
|
+
if (!value) return null;
|
|
36
|
+
let { offset, length, suffix } = value;
|
|
37
|
+
if (value.header !== undefined) {
|
|
38
|
+
const found = /^bytes=(\d*)-(\d*)$/.exec(String(value.header).trim());
|
|
39
|
+
if (!found || (!found[1] && !found[2])) return null;
|
|
40
|
+
if (!found[1]) suffix = Number(found[2]);
|
|
41
|
+
else { offset = Number(found[1]); if (found[2]) length = Number(found[2]) - offset + 1; }
|
|
26
42
|
}
|
|
27
|
-
if (
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
if (method !== 'uploadPart') await storage.delete(`upload:${args[1]}`);
|
|
31
|
-
return method === 'complete' ? metadata(result, prefix) : result;
|
|
43
|
+
if (suffix !== undefined) {
|
|
44
|
+
if (suffix <= 0) throw failure('get', 'The requested range is not satisfiable', 10039);
|
|
45
|
+
suffix = Math.min(suffix, size); offset = size - suffix; length = suffix;
|
|
32
46
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
47
|
+
offset ??= 0; length ??= size - offset;
|
|
48
|
+
if (offset < 0 || offset > size || length <= 0) throw failure('get', 'The requested range is not satisfiable', 10039);
|
|
49
|
+
return { offset, length: Math.min(length, size - offset) };
|
|
50
|
+
}
|
|
51
|
+
const cancel = body => body instanceof ReadableStream ? body.cancel().catch(() => {}) : undefined;
|
|
52
|
+
// Swaps the object row to a written version; run inside a transaction.
|
|
53
|
+
function save(db, binding, key, written, fields) {
|
|
54
|
+
const previous = db.object(binding, key);
|
|
55
|
+
if (previous) db.drop(previous.version);
|
|
56
|
+
db.sql.exec('INSERT OR REPLACE INTO rer_r2_objects(binding, key, version, size, etag, uploaded, http, custom, checksums, class) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
57
|
+
binding, key, written.version, written.size, fields.etag, Date.now(), json(fields.httpMetadata), json(fields.customMetadata), json(fields.checksums), fields.storageClass ?? 'Standard');
|
|
58
|
+
return metadata(db.object(binding, key));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function put(db, binding, key, body, options = {}) {
|
|
62
|
+
const current = db.object(binding, key);
|
|
63
|
+
if (!satisfied(options.onlyIf, current)) { await cancel(body); return null; }
|
|
64
|
+
const written = await writeBody(db, 'put', body, { hashes: checksumNames(options), replaces: current?.size ?? 0 });
|
|
65
|
+
try { verifyChecksums('put', options, written.sums); } catch (error) { db.drop(written.version); throw error; }
|
|
66
|
+
// The condition is checked again against whatever is current once the body is in.
|
|
67
|
+
const saved = db.transaction(() => satisfied(options.onlyIf, db.object(binding, key))
|
|
68
|
+
? save(db, binding, key, written, { ...options, etag: written.sums.md5, checksums: written.sums }) : null);
|
|
69
|
+
if (!saved) db.drop(written.version);
|
|
70
|
+
return saved;
|
|
71
|
+
}
|
|
72
|
+
function get(db, binding, key, options = {}) {
|
|
73
|
+
const row = db.object(binding, key);
|
|
74
|
+
if (!row) return null;
|
|
75
|
+
if (!satisfied(options.onlyIf, row)) return metadata(row);
|
|
76
|
+
const part = range(options.range, row.size);
|
|
77
|
+
return { ...metadata(row), ...(part ? { range: part } : {}), body: readBody(db, row.version, part?.offset ?? 0, part?.length ?? row.size) };
|
|
78
|
+
}
|
|
79
|
+
function remove(db, binding, keys) {
|
|
80
|
+
db.transaction(() => {
|
|
81
|
+
for (const key of [].concat(keys).map(String)) {
|
|
82
|
+
const row = db.object(binding, key);
|
|
83
|
+
if (!row) continue;
|
|
84
|
+
db.drop(row.version);
|
|
85
|
+
db.sql.exec('DELETE FROM rer_r2_objects WHERE binding = ? AND key = ?', binding, key);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
function list(db, binding, options = {}) {
|
|
90
|
+
const prefix = String(options.prefix ?? ''), include = options.include ?? [], delimiter = options.delimiter || undefined;
|
|
91
|
+
let limit = options.limit ?? 1000;
|
|
92
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1000) throw failure('list', 'MaxKeys params must be positive integer <= 1000.', 10022);
|
|
93
|
+
if (include.length) limit = Math.min(limit, 100);
|
|
94
|
+
let after = options.startAfter;
|
|
95
|
+
if (options.cursor) { const cursor = Buffer.from(options.cursor, 'base64url').toString('utf8'); if (after === undefined || cursor > after) after = cursor; }
|
|
96
|
+
const filter = 'binding = ?1 AND substr(key, 1, length(?2)) = ?2 AND (?3 IS NULL OR key > ?3)';
|
|
97
|
+
// As in R2, each delimited prefix counts once toward the limit, and the
|
|
98
|
+
// cursor skips past the last key it covers.
|
|
99
|
+
const rows = delimiter
|
|
100
|
+
? db.rows(`SELECT *, max(key) AS last, iif(instr(substr(key, length(?2) + 1), ?5), substr(key, 1, length(?2) + instr(substr(key, length(?2) + 1), ?5) + length(?5) - 1), NULL) AS grouped,
|
|
101
|
+
iif(instr(substr(key, length(?2) + 1), ?5), 'p:' || substr(key, 1, length(?2) + instr(substr(key, length(?2) + 1), ?5) + length(?5) - 1), 'k:' || key) AS item
|
|
102
|
+
FROM rer_r2_objects WHERE ${filter} GROUP BY item ORDER BY last LIMIT ?4`, binding, prefix, after ?? null, limit + 1, delimiter)
|
|
103
|
+
: db.rows(`SELECT *, key AS last, NULL AS grouped FROM rer_r2_objects WHERE ${filter} ORDER BY key LIMIT ?4`, binding, prefix, after ?? null, limit + 1);
|
|
104
|
+
const truncated = rows.length > limit, page = rows.slice(0, limit);
|
|
105
|
+
const objects = [], delimitedPrefixes = [];
|
|
106
|
+
for (const row of page) {
|
|
107
|
+
if (row.grouped !== null) delimitedPrefixes.push(row.grouped);
|
|
108
|
+
else objects.push(metadata({ ...row, key: row.last }, include));
|
|
42
109
|
}
|
|
110
|
+
return { objects, truncated, ...(truncated ? { cursor: Buffer.from(page.at(-1).last, 'utf8').toString('base64url') } : {}), delimitedPrefixes };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const upload = (db, method, binding, key, id) => {
|
|
114
|
+
const row = db.rows('SELECT * FROM rer_r2_uploads WHERE id = ? AND binding = ? AND key = ?', String(id), binding, key)[0];
|
|
115
|
+
if (!row) throw failure(method, 'The specified multipart upload does not exist.', 10024);
|
|
116
|
+
return row;
|
|
117
|
+
};
|
|
118
|
+
function createMultipartUpload(db, binding, key, options = {}) {
|
|
119
|
+
const id = randomId() + randomId();
|
|
120
|
+
db.sql.exec('INSERT INTO rer_r2_uploads(id, binding, key, options) VALUES(?, ?, ?, ?)', id, binding, key,
|
|
121
|
+
JSON.stringify({ httpMetadata: options.httpMetadata ?? {}, customMetadata: options.customMetadata ?? {}, storageClass: options.storageClass }));
|
|
122
|
+
return id;
|
|
123
|
+
}
|
|
124
|
+
async function uploadPart(db, binding, key, id, part, body) {
|
|
125
|
+
id = upload(db, 'uploadPart', binding, key, id).id;
|
|
126
|
+
const written = await writeBody(db, 'uploadPart', body);
|
|
127
|
+
// An abort while the body streamed in leaves nothing behind.
|
|
128
|
+
const saved = db.transaction(() => {
|
|
129
|
+
if (!db.rows('SELECT 1 FROM rer_r2_uploads WHERE id = ?', id).length) return null;
|
|
130
|
+
const previous = db.rows('SELECT version FROM rer_r2_parts WHERE upload = ? AND part = ?', id, part)[0];
|
|
131
|
+
if (previous) db.drop(previous.version);
|
|
132
|
+
db.sql.exec('INSERT OR REPLACE INTO rer_r2_parts(upload, part, version, size, etag) VALUES(?, ?, ?, ?, ?)', id, part, written.version, written.size, written.sums.md5);
|
|
133
|
+
return { partNumber: part, etag: written.sums.md5 };
|
|
134
|
+
});
|
|
135
|
+
if (!saved) { db.drop(written.version); upload(db, 'uploadPart', binding, key, id); }
|
|
136
|
+
return saved;
|
|
137
|
+
}
|
|
138
|
+
function complete(db, binding, key, id, selected) {
|
|
139
|
+
return db.transaction(() => {
|
|
140
|
+
const options = JSON.parse(upload(db, 'complete', binding, key, id).options);
|
|
141
|
+
const uploaded = new Map(db.rows('SELECT * FROM rer_r2_parts WHERE upload = ?', id).map(row => [row.part, row]));
|
|
142
|
+
const numbers = selected.map(item => item.partNumber);
|
|
143
|
+
if (new Set(numbers).size !== numbers.length) throw failure('complete', 'We encountered an internal error. Please try again.', 10001);
|
|
144
|
+
const parts = selected.map(item => {
|
|
145
|
+
const row = uploaded.get(item.partNumber);
|
|
146
|
+
if (row?.etag !== item.etag) throw failure('complete', 'One or more of the specified parts could not be found.', 10025);
|
|
147
|
+
return row;
|
|
148
|
+
}).sort((a, b) => a.part - b.part);
|
|
149
|
+
const version = randomId();
|
|
150
|
+
let seq = 0, start = 0;
|
|
151
|
+
for (const part of parts) {
|
|
152
|
+
const count = db.rows('SELECT count(*) AS n FROM rer_r2_chunks WHERE version = ?', part.version)[0].n;
|
|
153
|
+
db.sql.exec('UPDATE rer_r2_chunks SET version = ?, seq = seq + ?, start = start + ? WHERE version = ?', version, seq, start, part.version);
|
|
154
|
+
seq += count; start += part.size;
|
|
155
|
+
}
|
|
156
|
+
for (const row of uploaded.values()) if (!parts.includes(row)) db.drop(row.version);
|
|
157
|
+
db.sql.exec('DELETE FROM rer_r2_parts WHERE upload = ?', id);
|
|
158
|
+
db.sql.exec('DELETE FROM rer_r2_uploads WHERE id = ?', id);
|
|
159
|
+
const etag = createHash('md5').update(Buffer.concat(parts.map(part => Buffer.from(part.etag, 'hex')))).digest('hex') + `-${parts.length}`;
|
|
160
|
+
return save(db, binding, key, { version, size: start }, { ...options, etag, checksums: {} });
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
function abort(db, binding, key, id) {
|
|
164
|
+
db.transaction(() => {
|
|
165
|
+
for (const row of db.rows('SELECT version FROM rer_r2_parts WHERE upload = ?', String(id))) db.drop(row.version);
|
|
166
|
+
db.sql.exec('DELETE FROM rer_r2_parts WHERE upload = ?', String(id));
|
|
167
|
+
db.sql.exec('DELETE FROM rer_r2_uploads WHERE id = ? AND binding = ? AND key = ?', String(id), binding, key);
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const methods = { put, get, head: (db, binding, key) => { const row = db.object(binding, key); return row && metadata(row); },
|
|
172
|
+
delete: remove, list, createMultipartUpload, uploadPart, complete, abort };
|
|
173
|
+
export function r2Call(db, binding, method, args) {
|
|
174
|
+
if (!Object.hasOwn(methods, method)) throw new Error(`Unsupported R2 method: ${method}`);
|
|
175
|
+
const [key, ...rest] = args;
|
|
176
|
+
return methods[method](db, binding, method === 'list' ? key : method === 'delete' ? key : String(key), ...rest);
|
|
43
177
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { objectClass } from './config.mjs';
|
|
2
2
|
import { application, loaded, bundle } from './loader.mjs';
|
|
3
|
-
import { r2Call
|
|
3
|
+
import { r2Call } from './r2.mjs';
|
|
4
|
+
import { store } from './r2-store.mjs';
|
|
4
5
|
|
|
5
6
|
export async function database(host, binding, method, args) {
|
|
6
7
|
await host.active();
|
|
@@ -24,7 +25,8 @@ export async function object(host, binding, id, method, args) {
|
|
|
24
25
|
export async function r2(host, binding, method, args) {
|
|
25
26
|
await host.active();
|
|
26
27
|
if (!bundle.bindings.buckets.includes(binding)) throw new Error(`Unknown R2 binding: ${binding}`);
|
|
27
|
-
|
|
28
|
+
host.r2Store ??= store(host.ctx.storage);
|
|
29
|
+
const operation = r2Call(host.r2Store, binding, method, args);
|
|
28
30
|
host.r2Pending ??= new Set(); host.r2Pending.add(operation);
|
|
29
31
|
try { return await operation; } finally { host.r2Pending.delete(operation); }
|
|
30
32
|
}
|
|
@@ -32,6 +34,7 @@ export async function eraseResources(host) {
|
|
|
32
34
|
host.ctx.facets.delete('app');
|
|
33
35
|
for (const binding of bundle.bindings.databases.slice(1)) host.ctx.facets.delete(`database:${binding}`);
|
|
34
36
|
for (const key of (await host.ctx.storage.list({ prefix: 'facet:' })).keys()) host.ctx.facets.delete(key.slice(6));
|
|
37
|
+
// R2 objects are rows of the visitor's own storage, erased with it.
|
|
35
38
|
await Promise.allSettled(host.r2Pending ?? []);
|
|
36
|
-
|
|
39
|
+
host.r2Store = undefined;
|
|
37
40
|
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { demoInventory } from './remove.mjs';
|
|
2
|
+
import { removeBucket } from './native/deploy-r2.mjs';
|
|
2
3
|
|
|
3
4
|
// Keep the Worker and custom domains throughout the transition. Container apps
|
|
4
5
|
// must go before deleting their backing class; Wrangler then applies the class
|
|
5
6
|
// migration atomically with the native code. A failed upload can be rerun.
|
|
6
|
-
export async function upgradeDemo({ api, plan }) {
|
|
7
|
+
export async function upgradeDemo({ api, plan, report = () => {} }) {
|
|
7
8
|
const found = await demoInventory(api, plan);
|
|
8
9
|
const classes = found.namespaces.map(row => row.class);
|
|
9
10
|
const unknown = classes.filter(name => !['DemoContainer', 'NativeDemo', 'DemoCapacity'].includes(name));
|
|
@@ -21,6 +22,12 @@ export async function upgradeDemo({ api, plan }) {
|
|
|
21
22
|
const remaining = await api.request('/containers/applications');
|
|
22
23
|
if (remaining.some(row => found.apps.some(app => app.id === row.id))) throw new Error('demo-upgrade-container-remove-incomplete');
|
|
23
24
|
}
|
|
25
|
+
// The retired shared R2 bucket goes while its binding still proves ownership;
|
|
26
|
+
// the new code binds no bucket. A rerun after a partial removal resumes.
|
|
27
|
+
if (found.bucket) {
|
|
28
|
+
await removeBucket(api, found.bucket);
|
|
29
|
+
report(`Removed the retired shared demo R2 bucket ${found.bucket.name}; R2 now lives in each visitor's storage.`);
|
|
30
|
+
}
|
|
24
31
|
if (classes.includes('DemoContainer')) {
|
|
25
32
|
// The API checks the currently deployed bindings before class deletion.
|
|
26
33
|
// Detach the retired binding first; inherit every other value by name,
|