@runeditrun/rer 0.2.35 → 0.2.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/demo-native.mjs +1 -12
- package/global/CHANGELOG.md +17 -0
- package/modules/demo-host/README.md +2 -1
- package/modules/demo-host/native/config.mjs +1 -1
- package/modules/demo-host/native/d1.mjs +45 -7
- package/modules/demo-host/native/deploy-r2.mjs +4 -14
- package/modules/demo-host/native/r2-client.mjs +27 -10
- package/modules/demo-host/native/r2-store.mjs +112 -0
- package/modules/demo-host/native/r2.mjs +173 -39
- package/modules/demo-host/native/resources.mjs +6 -3
- package/modules/demo-host/native/sql.mjs +2 -1
- package/modules/demo-host/upgrade.mjs +8 -1
- package/package.json +1 -1
package/cli/demo-native.mjs
CHANGED
|
@@ -7,7 +7,6 @@ import { upgradeDemo } from '../modules/demo-host/upgrade.mjs';
|
|
|
7
7
|
import { missing, invalid, RerError } from './errors.mjs';
|
|
8
8
|
import { DEMO_FILES } from '../modules/demo-host/plan.mjs';
|
|
9
9
|
import { cloudflare } from '../modules/deploy-kit/provider.mjs';
|
|
10
|
-
import { demoBucketName, prepareBucket } from '../modules/demo-host/native/deploy-r2.mjs';
|
|
11
10
|
import { RUNTIME_FILES } from '../modules/demo-host/native/config.mjs';
|
|
12
11
|
import { prepareNative } from '../modules/demo-host/native/prepare.mjs';
|
|
13
12
|
import { TESTING_SECRET } from '../modules/demo-host/measurement.mjs';
|
|
@@ -37,7 +36,6 @@ export async function nativeDemoCommand(project, options, store, signal, out, op
|
|
|
37
36
|
const config = { ...plan.wrangler, main: 'modules/demo-host/native/worker.mjs', compatibility_flags: ['nodejs_compat'],
|
|
38
37
|
assets: { directory: './assets', binding: 'ASSETS', run_worker_first: true }, worker_loaders: [{ binding: 'LOADER' }],
|
|
39
38
|
};
|
|
40
|
-
if (bundle.bindings.buckets.length) config.r2_buckets = [{ binding: 'NATIVE_BUCKET', bucket_name: demoBucketName(plan.worker) }];
|
|
41
39
|
config.vars.NATIVE_SOURCE_DIGEST = bundle.digest;
|
|
42
40
|
if (process.env.RER_NATIVE_TEST_TTL_MS && TESTING_SECRET.test(process.env.TURNSTILE_SECRET_KEY ?? '')) {
|
|
43
41
|
if (!/^[1-9][0-9]{2,6}$/.test(process.env.RER_NATIVE_TEST_TTL_MS)) throw invalid('RER_NATIVE_TEST_TTL_MS must be positive milliseconds');
|
|
@@ -53,16 +51,7 @@ export async function nativeDemoCommand(project, options, store, signal, out, op
|
|
|
53
51
|
const required = ['CLOUDFLARE_ACCOUNT_ID', 'CLOUDFLARE_API_TOKEN', 'TURNSTILE_SITE_KEY', 'TURNSTILE_SECRET_KEY'];
|
|
54
52
|
if (required.some(name => !process.env[name]?.trim())) throw missing(`${required.join(', ')} are required`);
|
|
55
53
|
const api = cloudflare({ accountId: process.env.CLOUDFLARE_ACCOUNT_ID, token: process.env.CLOUDFLARE_API_TOKEN });
|
|
56
|
-
config.migrations = await upgradeDemo({ api, plan });
|
|
57
|
-
if (bundle.bindings.buckets.length) {
|
|
58
|
-
const statePath = join(project.root, '.rer/demo-buckets', plan.worker + '.json');
|
|
59
|
-
const pending = await readFile(statePath, 'utf8').then(JSON.parse).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
60
|
-
const bucket = await prepareBucket(api, plan, pending);
|
|
61
|
-
await mkdir(dirname(statePath), { recursive: true, mode: 0o700 });
|
|
62
|
-
await writeFile(statePath, JSON.stringify({ record: plan.record, bucket }), { mode: 0o600 });
|
|
63
|
-
config.vars.NATIVE_BUCKET_CREATED = bucket.creation_date;
|
|
64
|
-
await writeFile(join(folder, 'wrangler.json'), JSON.stringify(config, null, 2) + '\n');
|
|
65
|
-
}
|
|
54
|
+
config.migrations = await upgradeDemo({ api, plan, report: line => store.report(line) });
|
|
66
55
|
await writeFile(join(folder, 'wrangler.json'), JSON.stringify(config, null, 2) + '\n');
|
|
67
56
|
const testing = TESTING_SECRET.test(process.env.TURNSTILE_SECRET_KEY);
|
|
68
57
|
const measurement = testing ? process.env.RER_DEMO_MEASUREMENT_TOKEN || randomBytes(32).toString('base64url') : undefined;
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.37
|
|
4
|
+
|
|
5
|
+
- **A native demo's D1 answers `withSession`, `dump()` and `meta.size_after`, as production D1 does.** In a native hosted demo, `env.DB.withSession(...)` now returns a session whose `prepare`, `batch` and `getBookmark` work against the visitor's own database. Each visitor's database is a single primary, so every constraint or bookmark is already met and the bookmark stays the same. Before, a seed that read through a session hung: Slate's feature routes never answered. `dump()` now exists and rejects with `D1_DUMP_ERROR`, as production D1 does. Before, Uptime's owner-session revoke bound `db.dump` and answered 503 `auth_unavailable`; it now answers 404 `session_not_found`, once Uptime takes this release. Query `meta` adds `size_after`, which Orbit's archive budget reads.
|
|
6
|
+
- **A native demo's D1 errors read as D1's do.** SQL errors are `D1_ERROR: <SQLite message>`, and `exec`, type and missing-column errors carry D1's codes. Each carries the original error as its `cause`. Seeds that classify errors by their D1 text, such as Mango, Oncall, Spool, Uptime and Chest, behave as they do in production.
|
|
7
|
+
- **An unknown D1 property reads `undefined`, as on real D1.** Feature detection takes its fallback. The first read of each unknown name per isolate logs `D1_UNSUPPORTED: <Kind>.<name> is not available in hosted demos` in the Worker's tail, so a seed that calls a missing method leaves a trace.
|
|
8
|
+
|
|
9
|
+
## 0.2.36
|
|
10
|
+
|
|
11
|
+
- **A native demo keeps each visitor's R2 files in that visitor's own storage.** A native hosted demo now stores R2 objects, their metadata and multipart uploads per binding in the visitor's own Durable Object storage, next to the app. Before, every visitor shared one demo-owned bucket, often far from the visitor. File reads and writes now answer from where the visitor is. Measured from Queensland on Beam: opening a file took 55 ms instead of 256 ms, uploading 207 ms instead of 1543 ms, and starting the demo 1.64 s instead of 2.65 s. One visitor's objects may total 256 MiB; a write past that fails with R2's error for an oversized upload. Expiry erases a visitor's objects along with the rest of their storage. A new demo creates no R2 bucket.
|
|
12
|
+
- **Paired-runtime tests survive a port clash.** When a loopback port is taken, the seed's paired-runtime test helpers retry on a fresh port. When one of a pair fails to start, they close the one that did start, so a clash fails within seconds and leaves nothing running.
|
|
13
|
+
|
|
14
|
+
### Upgrade notes
|
|
15
|
+
|
|
16
|
+
- On the next `rer deploy --demo`, a host deployed by rer 0.2.30 to 0.2.35 has its shared demo R2 bucket emptied and deleted, and the command reports this. The bucket is removed only while the deployed Worker still proves it owns it; a bucket it does not own is left alone. `rer deploy --demo --remove` still removes a bucket left by an older host.
|
|
17
|
+
- Visitors' files from the shared bucket are not carried over: each demo visitor already starts from seed data.
|
|
18
|
+
- Seeds that worked around a distant bucket (for example, by putting objects in parallel) can drop the workaround.
|
|
19
|
+
|
|
3
20
|
## 0.2.35
|
|
4
21
|
|
|
5
22
|
- **The Run page starts a seed without Node.** The shared /run page has two steps. Step 1 is `curl -fsSL https://runeditrun.com/install.sh | sh`, which brings its own Node and changes no shell settings. Step 2 is `rer dev <owner>/<repo>`, with `cd <folder> && rer dev` to start it again later. `npx -y @runeditrun/rer dev <owner>/<repo>` remains only as the alternative under "Already have Node 22 or later?". Before this, a newcomer without Node got `sh: npx: command not found` at step 2, because the installer keeps its Node private. Step 2 is built from Run data's existing `clone` line, so seeds need no edits. RUN-LOCALLY, HOMEPAGE-VARIANTS, SEED-COMMAND-CONTRACT and the READMEs describe the two steps, and the default and foundation Run requirements no longer say "Needs Node 22".
|
|
@@ -5,7 +5,8 @@ host. See [adoption and deployment](../../export/DEMO.md#hosting-the-demo).
|
|
|
5
5
|
|
|
6
6
|
- `build.mjs`: ordinary Vite output, declared SQL and public configuration,
|
|
7
7
|
using the shared container migration-inventory and image-variable checks.
|
|
8
|
-
- `native/`: bundle preparation, per-visitor D1/seed-DO facets,
|
|
8
|
+
- `native/`: bundle preparation, per-visitor D1/seed-DO facets, R2 in the
|
|
9
|
+
visitor's own storage (`r2.mjs` semantics, `r2-store.mjs` rows and chunks),
|
|
9
10
|
durable queue delivery, seed-DO alarms and declared UTC crons.
|
|
10
11
|
A narrow bridge exposes only visitor resource operations to the seed; own-host
|
|
11
12
|
outbound requests return to the same visitor, including declared secondary hosts.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parseCron } from './cron.mjs';
|
|
2
2
|
|
|
3
|
-
export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
|
|
3
|
+
export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'r2-store', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
|
|
4
4
|
export const ISOLATE_FILES = ['bindings', 'namespace', 'r2-client', 'd1', 'sql', 'seed-object'];
|
|
5
5
|
export function nativeBindings(config) {
|
|
6
6
|
for (const kind of ['kv_namespaces', 'services', 'workflows', 'ai', 'vectorize', 'hyperdrive', 'analytics_engine_datasets', 'browser', 'images', 'send_email', 'dispatch_namespaces', 'mtls_certificates', 'pipelines', 'secrets_store_secrets', 'unsafe', 'vpc_services', 'vpc_networks', 'ratelimits', 'worker_loaders', 'containers', 'ai_search_namespaces', 'ai_search', 'agent_memory', 'connect', 'artifacts', 'unsafe_hello_world', 'flagship', 'logfwdr', 'version_metadata']) {
|
|
@@ -1,33 +1,46 @@
|
|
|
1
1
|
// The synchronous D1 statement builder stays inside the app isolate. Only SQL
|
|
2
2
|
// and bound values cross RPC; a batch remains one SQLite transaction.
|
|
3
3
|
export class DemoD1 {
|
|
4
|
-
constructor(sql) { this.sql = sql; }
|
|
4
|
+
constructor(sql) { this.sql = sql; return strict(this, 'D1Database'); }
|
|
5
5
|
prepare(query) { return new Statement(this, query); }
|
|
6
6
|
async batch(statements) {
|
|
7
7
|
if (statements.some(statement => statement.db !== this)) throw new Error('Foreign D1 statement');
|
|
8
|
-
return (await this.sql.batch(statements.map(({ query, args }) => ({ query, args })))).map(result);
|
|
8
|
+
return (await d1(() => this.sql.batch(statements.map(({ query, args }) => ({ query, args }))))).map(result);
|
|
9
9
|
}
|
|
10
|
-
exec(query) { return this.sql.exec(query); }
|
|
10
|
+
exec(query) { return d1(() => this.sql.exec(query), 'D1_EXEC_ERROR'); }
|
|
11
|
+
// One visitor's SQLite has no replicas: every session reads the primary,
|
|
12
|
+
// so any constraint or bookmark is already satisfied.
|
|
13
|
+
withSession(constraintOrBookmark = 'first-unconstrained') {
|
|
14
|
+
if (typeof constraintOrBookmark !== 'string' || !constraintOrBookmark) throw new Error('D1_SESSION_ERROR: invalid constraint or bookmark');
|
|
15
|
+
return strict(new Session(this), 'D1DatabaseSession');
|
|
16
|
+
}
|
|
17
|
+
async dump() { throw failure('D1_DUMP_ERROR', 'dump() is not available in hosted demos'); }
|
|
18
|
+
}
|
|
19
|
+
class Session {
|
|
20
|
+
constructor(db) { this.db = db; }
|
|
21
|
+
prepare(query) { return this.db.prepare(query); }
|
|
22
|
+
batch(statements) { return this.db.batch(statements); }
|
|
23
|
+
getBookmark() { return '00000000-00000000-00000000-rer-native-demo'; }
|
|
11
24
|
}
|
|
12
25
|
const result = value => ({ success: value.success, results: value.results, meta: value.meta });
|
|
13
26
|
class Statement {
|
|
14
|
-
constructor(db, query, args = []) { this.db = db; this.query = query; this.args = args; }
|
|
27
|
+
constructor(db, query, args = []) { this.db = db; this.query = query; this.args = args; return strict(this, 'D1PreparedStatement'); }
|
|
15
28
|
bind(...args) {
|
|
16
29
|
return new Statement(this.db, this.query, args.map(value => {
|
|
17
30
|
if (typeof value === 'boolean') return Number(value);
|
|
18
31
|
if (value === null || typeof value === 'string' || typeof value === 'number' || value instanceof ArrayBuffer || ArrayBuffer.isView(value)) return value;
|
|
19
32
|
if (Array.isArray(value) && value.every(byte => Number.isInteger(byte) && byte >= 0 && byte <= 255)) return new Uint8Array(value);
|
|
20
|
-
throw
|
|
33
|
+
throw failure('D1_TYPE_ERROR', `Type '${typeof value}' not supported for value '${value}'`);
|
|
21
34
|
}));
|
|
22
35
|
}
|
|
23
|
-
execute() { return this.db.sql.query(this.query, this.args); }
|
|
36
|
+
execute() { return d1(() => this.db.sql.query(this.query, this.args)); }
|
|
24
37
|
async all() { return result(await this.execute()); }
|
|
25
38
|
run() { return this.all(); }
|
|
26
39
|
async first(column) {
|
|
27
40
|
const row = (await this.execute()).results[0];
|
|
28
41
|
if (!row) return null;
|
|
29
42
|
if (column === undefined) return row;
|
|
30
|
-
if (!Object.hasOwn(row, column)) throw new Error(`D1_COLUMN_NOTFOUND: ${column}
|
|
43
|
+
if (!Object.hasOwn(row, column)) throw new Error(`D1_COLUMN_NOTFOUND: Column not found (${column})`, { cause: new Error('Column not found') });
|
|
31
44
|
return row[column];
|
|
32
45
|
}
|
|
33
46
|
async raw({ columnNames = false } = {}) {
|
|
@@ -35,3 +48,28 @@ class Statement {
|
|
|
35
48
|
return columnNames ? [columns, ...rows] : rows;
|
|
36
49
|
}
|
|
37
50
|
}
|
|
51
|
+
// Errors read as D1's do ("D1_ERROR: UNIQUE constraint failed: ...: SQLITE_CONSTRAINT"),
|
|
52
|
+
// with the unprefixed error as their cause.
|
|
53
|
+
const failure = (code, message, cause = new Error(message)) => new Error(`${code}: ${message}`, { cause });
|
|
54
|
+
async function d1(operation, code = 'D1_ERROR') {
|
|
55
|
+
try { return await operation(); }
|
|
56
|
+
catch (error) {
|
|
57
|
+
const message = String(error?.message ?? error);
|
|
58
|
+
throw /^D1_[A-Z_]+:/.test(message) ? error : failure(code, message, error);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
// An unknown property reads undefined, as on real D1, so feature detection
|
|
62
|
+
// takes its fallback. The first read of each unknown name per isolate warns in
|
|
63
|
+
// the Worker tail: a seed calling a method this facade lacks inside its own
|
|
64
|
+
// queueMicrotask (Slate's read batching) otherwise hangs with no trace.
|
|
65
|
+
const warned = new Set();
|
|
66
|
+
function strict(target, kind) {
|
|
67
|
+
return new Proxy(target, { get(object, key, receiver) {
|
|
68
|
+
const name = `${kind}.${String(key)}`;
|
|
69
|
+
if (typeof key === 'string' && !(key in object) && key !== 'then' && key !== 'toJSON' && !warned.has(name)) {
|
|
70
|
+
warned.add(name);
|
|
71
|
+
console.warn(`D1_UNSUPPORTED: ${name} is not available in hosted demos`);
|
|
72
|
+
}
|
|
73
|
+
return Reflect.get(object, key, receiver);
|
|
74
|
+
} });
|
|
75
|
+
}
|
|
@@ -1,23 +1,13 @@
|
|
|
1
|
-
import { workerSettings } from '../../deploy-kit/owner.mjs';
|
|
2
|
-
import { servedVersion } from '../../deploy-kit/site.mjs';
|
|
3
1
|
import { r2ObjectPages, r2ObjectPath } from '../../deploy-kit/r2-objects.mjs';
|
|
4
2
|
|
|
5
|
-
|
|
3
|
+
// rer 0.2.30-0.2.33 kept every visitor's R2 objects in one demo bucket. Demo
|
|
4
|
+
// R2 now lives in each visitor's storage; an upgrade or removal empties and
|
|
5
|
+
// deletes that bucket when the served Worker still proves it owns it.
|
|
6
|
+
const demoBucketName = worker => `${worker.replace(/-+/g, '-').slice(0, 48)}-objects`;
|
|
6
7
|
const missing = async (api, name) => {
|
|
7
8
|
try { return await api.request(`/r2/buckets/${name}`); }
|
|
8
9
|
catch (error) { if (error.status === 404) return null; throw error; }
|
|
9
10
|
};
|
|
10
|
-
export async function prepareBucket(api, plan, pending) {
|
|
11
|
-
const name = demoBucketName(plan.worker), bucket = await missing(api, name), settings = await workerSettings(api, plan.worker);
|
|
12
|
-
if (!bucket && settings?.bindings?.some(item => item.name === 'NATIVE_BUCKET')) throw new Error(`Demo R2 bucket is missing: ${name}`);
|
|
13
|
-
if (bucket) {
|
|
14
|
-
const version = settings && await servedVersion(api, plan.worker);
|
|
15
|
-
if (!settings && pending?.record === plan.record && pending.bucket?.name === name && pending.bucket.creation_date === bucket.creation_date) return bucket;
|
|
16
|
-
if (!version?.current || version.annotation !== plan.record || !ownsBucket(settings, bucket)) throw new Error(`Demo R2 bucket ownership is unproven: ${name}`);
|
|
17
|
-
return bucket;
|
|
18
|
-
}
|
|
19
|
-
return api.request('/r2/buckets', { method: 'POST', body: { name } });
|
|
20
|
-
}
|
|
21
11
|
function ownsBucket(settings, bucket) {
|
|
22
12
|
return settings?.bindings?.some(item => item.type === 'r2_bucket' && item.name === 'NATIVE_BUCKET' && item.bucket_name === bucket.name)
|
|
23
13
|
&& settings.bindings.some(item => item.name === 'NATIVE_BUCKET_CREATED' && item.text === bucket.creation_date);
|
|
@@ -1,35 +1,52 @@
|
|
|
1
|
+
const HEADERS = { contentType: 'content-type', contentLanguage: 'content-language', contentDisposition: 'content-disposition', contentEncoding: 'content-encoding', cacheControl: 'cache-control', cacheExpiry: 'expires' };
|
|
1
2
|
function object(value) {
|
|
2
3
|
if (!value) return value;
|
|
3
4
|
value.writeHttpMetadata = headers => {
|
|
4
5
|
for (const [key, content] of Object.entries(value.httpMetadata ?? {})) {
|
|
5
|
-
const header =
|
|
6
|
+
const header = HEADERS[key];
|
|
6
7
|
if (header) headers.set(header, content instanceof Date ? content.toUTCString() : content);
|
|
7
8
|
}
|
|
8
9
|
};
|
|
9
|
-
if (value
|
|
10
|
+
if ('body' in value) {
|
|
10
11
|
const response = new Response(value.body);
|
|
12
|
+
value.body = response.body;
|
|
11
13
|
Object.defineProperty(value, 'bodyUsed', { get: () => response.bodyUsed });
|
|
12
14
|
for (const method of ['arrayBuffer', 'text', 'json', 'blob']) value[method] = () => response[method]();
|
|
13
15
|
value.bytes = async () => new Uint8Array(await response.arrayBuffer());
|
|
14
16
|
}
|
|
15
17
|
return value;
|
|
16
18
|
}
|
|
17
|
-
//
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const
|
|
19
|
+
// Headers forms of R2 options become their plain forms before crossing RPC.
|
|
20
|
+
function httpMetadata(value) {
|
|
21
|
+
if (!(value instanceof Headers)) return value;
|
|
22
|
+
const result = {};
|
|
23
|
+
for (const [key, header] of Object.entries(HEADERS)) if (value.has(header)) result[key] = key === 'cacheExpiry' ? new Date(value.get(header)) : value.get(header);
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
function conditional(value) {
|
|
27
|
+
if (!(value instanceof Headers)) return value;
|
|
28
|
+
const date = name => value.has(name) ? new Date(value.get(name)) : undefined;
|
|
29
|
+
return { etagMatches: value.get('if-match') ?? undefined, etagDoesNotMatch: value.get('if-none-match') ?? undefined,
|
|
30
|
+
uploadedAfter: date('if-modified-since'), uploadedBefore: date('if-unmodified-since'), secondsGranularity: true };
|
|
31
|
+
}
|
|
32
|
+
const plain = (options = {}) => ({ ...options,
|
|
33
|
+
...(options.httpMetadata ? { httpMetadata: httpMetadata(options.httpMetadata) } : {}),
|
|
34
|
+
...(options.onlyIf ? { onlyIf: conditional(options.onlyIf) } : {}),
|
|
35
|
+
...(options.range instanceof Headers ? { range: options.range.has('range') ? { header: options.range.get('range') } : undefined } : {}) });
|
|
36
|
+
// Bodies stream to the visitor's storage; only Blobs are read first.
|
|
37
|
+
const body = value => value instanceof Blob ? value.arrayBuffer() : value;
|
|
21
38
|
export class DemoR2 {
|
|
22
39
|
constructor(host, binding) { this.host = host; this.binding = binding; }
|
|
23
40
|
call(method, ...args) { return this.host.r2(this.binding, method, args); }
|
|
24
|
-
async get(key, options) { return object(await this.call('get', key, options)); }
|
|
41
|
+
async get(key, options) { return object(await this.call('get', key, plain(options))); }
|
|
25
42
|
async head(key) { return object(await this.call('head', key)); }
|
|
26
|
-
async put(key,
|
|
43
|
+
async put(key, value, options) { return object(await this.call('put', key, await body(value), plain(options))); }
|
|
27
44
|
delete(keys) { return this.call('delete', keys); }
|
|
28
45
|
async list(options) { const page = await this.call('list', options); page.objects = page.objects.map(object); return page; }
|
|
29
|
-
async createMultipartUpload(key, options) { return this.resumeMultipartUpload(key, await this.call('createMultipartUpload', key, options)); }
|
|
46
|
+
async createMultipartUpload(key, options) { return this.resumeMultipartUpload(key, await this.call('createMultipartUpload', key, plain(options))); }
|
|
30
47
|
resumeMultipartUpload(key, uploadId) {
|
|
31
48
|
return { key, uploadId,
|
|
32
|
-
uploadPart: async (part,
|
|
49
|
+
uploadPart: async (part, value, options) => this.call('uploadPart', key, uploadId, part, await body(value), options),
|
|
33
50
|
abort: () => this.call('abort', key, uploadId),
|
|
34
51
|
complete: async parts => object(await this.call('complete', key, uploadId, parts)),
|
|
35
52
|
};
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
// Demo R2 objects live in the visitor's own SQLite storage, beside the app,
|
|
4
|
+
// so every call is local to wherever the visitor's Durable Object runs.
|
|
5
|
+
// Bodies are split into chunks well under the 2 MB per-value limit.
|
|
6
|
+
export const CHUNK = 1 << 20;
|
|
7
|
+
// Seeded data is kilobytes and seed upload limits are at most ~20 MB per file.
|
|
8
|
+
// 256 MiB leaves room for real demo uploads while bounding one hour's storage.
|
|
9
|
+
export const VISITOR_LIMIT = 256 * CHUNK;
|
|
10
|
+
const SCHEMA = `CREATE TABLE IF NOT EXISTS rer_r2_objects(binding TEXT NOT NULL, key TEXT NOT NULL, version TEXT NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, uploaded INTEGER NOT NULL, http TEXT NOT NULL, custom TEXT NOT NULL, checksums TEXT NOT NULL, class TEXT NOT NULL, PRIMARY KEY(binding, key));
|
|
11
|
+
CREATE TABLE IF NOT EXISTS rer_r2_chunks(version TEXT NOT NULL, seq INTEGER NOT NULL, start INTEGER NOT NULL, size INTEGER NOT NULL, data BLOB NOT NULL, PRIMARY KEY(version, seq));
|
|
12
|
+
CREATE TABLE IF NOT EXISTS rer_r2_uploads(id TEXT PRIMARY KEY, binding TEXT NOT NULL, key TEXT NOT NULL, options TEXT NOT NULL);
|
|
13
|
+
CREATE TABLE IF NOT EXISTS rer_r2_parts(upload TEXT NOT NULL, part INTEGER NOT NULL, version TEXT NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, PRIMARY KEY(upload, part));`;
|
|
14
|
+
const HASHES = { md5: 'MD5', sha1: 'SHA-1', sha256: 'SHA-256', sha384: 'SHA-384', sha512: 'SHA-512' };
|
|
15
|
+
|
|
16
|
+
export const randomId = () => [...crypto.getRandomValues(new Uint8Array(16))].map(byte => byte.toString(16).padStart(2, '0')).join('');
|
|
17
|
+
export function store(storage) {
|
|
18
|
+
const sql = storage.sql;
|
|
19
|
+
sql.exec(SCHEMA);
|
|
20
|
+
const rows = (query, ...args) => sql.exec(query, ...args).toArray();
|
|
21
|
+
return { sql, rows, transaction: run => storage.transactionSync(run),
|
|
22
|
+
object: (binding, key) => rows('SELECT * FROM rer_r2_objects WHERE binding = ? AND key = ?', binding, key)[0] ?? null,
|
|
23
|
+
used: () => rows('SELECT COALESCE(SUM(size), 0) AS n FROM rer_r2_chunks')[0].n,
|
|
24
|
+
drop(version) { sql.exec('DELETE FROM rer_r2_chunks WHERE version = ?', version); },
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const bytesOf = body => {
|
|
29
|
+
if (body === null || body === undefined) return new Uint8Array();
|
|
30
|
+
if (typeof body === 'string') return new TextEncoder().encode(body);
|
|
31
|
+
if (body instanceof ArrayBuffer) return new Uint8Array(body);
|
|
32
|
+
if (ArrayBuffer.isView(body)) return new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
|
|
33
|
+
return null;
|
|
34
|
+
};
|
|
35
|
+
async function* pieces(body) {
|
|
36
|
+
const whole = bytesOf(body);
|
|
37
|
+
if (whole) { yield whole; return; }
|
|
38
|
+
if (!(body instanceof ReadableStream)) throw new TypeError('put: The body must be a string, ArrayBuffer, ArrayBufferView, Blob or ReadableStream.');
|
|
39
|
+
const reader = body.getReader();
|
|
40
|
+
try {
|
|
41
|
+
for (;;) {
|
|
42
|
+
const { done, value } = await reader.read();
|
|
43
|
+
if (done) return;
|
|
44
|
+
yield bytesOf(value) ?? (() => { throw new TypeError('put: The body stream must contain bytes.'); })();
|
|
45
|
+
}
|
|
46
|
+
} finally { reader.releaseLock(); }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Writes a body under a fresh version. The object row is swapped separately,
|
|
50
|
+
// so a failed or oversized write leaves the previous object untouched.
|
|
51
|
+
export async function writeBody(db, method, body, { hashes = [], replaces = 0 } = {}) {
|
|
52
|
+
const version = randomId(), digests = Object.fromEntries(['md5', ...hashes].map(name => [name, createHash(name)]));
|
|
53
|
+
let size = 0, seq = 0, pending = [], pendingSize = 0;
|
|
54
|
+
const flush = () => {
|
|
55
|
+
if (!pendingSize) return;
|
|
56
|
+
if (db.used() - replaces + pendingSize > VISITOR_LIMIT) {
|
|
57
|
+
const error = new Error(`${method}: Your proposed upload exceeds this demo's storage limit of ${VISITOR_LIMIT / CHUNK} MiB per visitor. (100100)`);
|
|
58
|
+
error.code = 100100;
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
const data = new Uint8Array(pendingSize);
|
|
62
|
+
let at = 0;
|
|
63
|
+
for (const piece of pending) { data.set(piece, at); at += piece.length; }
|
|
64
|
+
db.sql.exec('INSERT INTO rer_r2_chunks(version, seq, start, size, data) VALUES(?, ?, ?, ?, ?)', version, seq++, size - pendingSize, pendingSize, data.buffer);
|
|
65
|
+
pending = []; pendingSize = 0;
|
|
66
|
+
};
|
|
67
|
+
try {
|
|
68
|
+
for await (const piece of pieces(body)) {
|
|
69
|
+
for (let at = 0; at < piece.length;) {
|
|
70
|
+
const take = Math.min(CHUNK - pendingSize, piece.length - at), part = piece.subarray(at, at + take);
|
|
71
|
+
for (const digest of Object.values(digests)) digest.update(part);
|
|
72
|
+
pending.push(part); pendingSize += take; size += take; at += take;
|
|
73
|
+
if (pendingSize === CHUNK) flush();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
flush();
|
|
77
|
+
} catch (error) { db.drop(version); throw error; }
|
|
78
|
+
const sums = Object.fromEntries(Object.entries(digests).map(([name, digest]) => [name, digest.digest('hex')]));
|
|
79
|
+
return { version, size, sums };
|
|
80
|
+
}
|
|
81
|
+
export const checksumNames = options => Object.keys(HASHES).filter(name => name !== 'md5' && options?.[name] !== undefined);
|
|
82
|
+
export function verifyChecksums(method, options, sums) {
|
|
83
|
+
for (const name of Object.keys(HASHES)) {
|
|
84
|
+
const expected = options?.[name];
|
|
85
|
+
if (expected === undefined) continue;
|
|
86
|
+
const hex = typeof expected === 'string' ? expected.toLowerCase() : Buffer.from(bytesOf(expected)).toString('hex');
|
|
87
|
+
if (hex !== sums[name]) throw new Error(`${method}: The ${HASHES[name]} checksum you specified did not match what we received. (10037)`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// A range read of one version; small reads are returned whole, larger ones
|
|
92
|
+
// stream chunk by chunk so a visitor's big object never sits in memory.
|
|
93
|
+
export function readBody(db, version, offset, length) {
|
|
94
|
+
const end = offset + length, query = (from, limit) => db.rows('SELECT seq, start, data FROM rer_r2_chunks WHERE version = ? AND seq >= ? AND start < ? AND start + size > ? ORDER BY seq LIMIT ?', version, from, end, offset, limit);
|
|
95
|
+
const slice = row => { const data = new Uint8Array(row.data); return data.subarray(Math.max(0, offset - row.start), Math.min(data.length, end - row.start)); };
|
|
96
|
+
if (length <= 4 * CHUNK) {
|
|
97
|
+
const result = new Uint8Array(length);
|
|
98
|
+
let at = 0;
|
|
99
|
+
for (const row of query(0, 1e9)) { const part = slice(row); result.set(part, at); at += part.length; }
|
|
100
|
+
if (at !== length) throw new Error('get: The object changed while it was read.');
|
|
101
|
+
return result;
|
|
102
|
+
}
|
|
103
|
+
let next = 0, sent = 0;
|
|
104
|
+
return new ReadableStream({ type: 'bytes', pull(controller) {
|
|
105
|
+
const [row] = query(next, 1);
|
|
106
|
+
if (!row) { if (sent === length) controller.close(); else controller.error(new Error('get: The object changed while it was read.')); return; }
|
|
107
|
+
next = row.seq + 1;
|
|
108
|
+
const part = slice(row); sent += part.length;
|
|
109
|
+
controller.enqueue(part.slice());
|
|
110
|
+
if (sent === length) controller.close();
|
|
111
|
+
} });
|
|
112
|
+
}
|
|
@@ -1,43 +1,177 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { writeBody, readBody, randomId, checksumNames, verifyChecksums } from './r2-store.mjs';
|
|
3
|
+
|
|
4
|
+
// R2 semantics over the visitor's own storage: one namespace per binding,
|
|
5
|
+
// conditional and ranged reads, delimited listing and multipart uploads.
|
|
6
|
+
const failure = (method, message, code) => Object.assign(new Error(`${method}: ${message} (${code})`), { code });
|
|
7
|
+
const json = value => JSON.stringify(value ?? {});
|
|
8
|
+
function httpMetadata(text) {
|
|
9
|
+
const value = JSON.parse(text);
|
|
10
|
+
if (value.cacheExpiry) value.cacheExpiry = new Date(value.cacheExpiry);
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
const buffer = hex => Uint8Array.from(Buffer.from(hex, 'hex')).buffer;
|
|
14
|
+
function metadata(row, include = ['httpMetadata', 'customMetadata']) {
|
|
15
|
+
return { key: row.key, version: row.version, size: row.size, etag: row.etag, httpEtag: `"${row.etag}"`, uploaded: new Date(row.uploaded),
|
|
16
|
+
httpMetadata: include.includes('httpMetadata') ? httpMetadata(row.http) : {}, customMetadata: include.includes('customMetadata') ? JSON.parse(row.custom) : {},
|
|
17
|
+
checksums: Object.fromEntries(Object.entries(JSON.parse(row.checksums)).map(([name, hex]) => [name, buffer(hex)])), storageClass: row.class };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const etags = value => [].concat(value).flatMap(item => String(item).split(',')).map(item => item.trim()).filter(Boolean)
|
|
21
|
+
.map(item => item === '*' ? { wildcard: true } : { weak: item.startsWith('W/'), value: item.replace(/^W\//, '').replace(/^"(.*)"$/, '$1') });
|
|
22
|
+
const includes = (list, etag, weak) => etags(list).some(item => item.wildcard || (item.value === etag && (weak || !item.weak)));
|
|
23
|
+
// The precedence of If-Match over If-Unmodified-Since (and so on) follows R2.
|
|
24
|
+
export function satisfied(condition, row) {
|
|
25
|
+
if (!condition) return true;
|
|
26
|
+
if (!row) return condition.etagMatches === undefined && condition.uploadedAfter === undefined;
|
|
27
|
+
const truncate = condition.secondsGranularity ? time => Math.floor(time / 1000) * 1000 : time => time, uploaded = truncate(row.uploaded);
|
|
28
|
+
const match = condition.etagMatches === undefined || includes(condition.etagMatches, row.etag, false);
|
|
29
|
+
const noneMatch = condition.etagDoesNotMatch === undefined || !includes(condition.etagDoesNotMatch, row.etag, true);
|
|
30
|
+
const after = condition.uploadedAfter === undefined || truncate(new Date(condition.uploadedAfter).getTime()) < uploaded || (condition.etagDoesNotMatch !== undefined && noneMatch);
|
|
31
|
+
const before = condition.uploadedBefore === undefined || uploaded < truncate(new Date(condition.uploadedBefore).getTime()) || (condition.etagMatches !== undefined && match);
|
|
32
|
+
return match && noneMatch && after && before;
|
|
33
|
+
}
|
|
34
|
+
function range(value, size) {
|
|
35
|
+
if (!value) return null;
|
|
36
|
+
let { offset, length, suffix } = value;
|
|
37
|
+
if (value.header !== undefined) {
|
|
38
|
+
const found = /^bytes=(\d*)-(\d*)$/.exec(String(value.header).trim());
|
|
39
|
+
if (!found || (!found[1] && !found[2])) return null;
|
|
40
|
+
if (!found[1]) suffix = Number(found[2]);
|
|
41
|
+
else { offset = Number(found[1]); if (found[2]) length = Number(found[2]) - offset + 1; }
|
|
26
42
|
}
|
|
27
|
-
if (
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
if (method !== 'uploadPart') await storage.delete(`upload:${args[1]}`);
|
|
31
|
-
return method === 'complete' ? metadata(result, prefix) : result;
|
|
43
|
+
if (suffix !== undefined) {
|
|
44
|
+
if (suffix <= 0) throw failure('get', 'The requested range is not satisfiable', 10039);
|
|
45
|
+
suffix = Math.min(suffix, size); offset = size - suffix; length = suffix;
|
|
32
46
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
47
|
+
offset ??= 0; length ??= size - offset;
|
|
48
|
+
if (offset < 0 || offset > size || length <= 0) throw failure('get', 'The requested range is not satisfiable', 10039);
|
|
49
|
+
return { offset, length: Math.min(length, size - offset) };
|
|
50
|
+
}
|
|
51
|
+
const cancel = body => body instanceof ReadableStream ? body.cancel().catch(() => {}) : undefined;
|
|
52
|
+
// Swaps the object row to a written version; run inside a transaction.
|
|
53
|
+
function save(db, binding, key, written, fields) {
|
|
54
|
+
const previous = db.object(binding, key);
|
|
55
|
+
if (previous) db.drop(previous.version);
|
|
56
|
+
db.sql.exec('INSERT OR REPLACE INTO rer_r2_objects(binding, key, version, size, etag, uploaded, http, custom, checksums, class) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
57
|
+
binding, key, written.version, written.size, fields.etag, Date.now(), json(fields.httpMetadata), json(fields.customMetadata), json(fields.checksums), fields.storageClass ?? 'Standard');
|
|
58
|
+
return metadata(db.object(binding, key));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function put(db, binding, key, body, options = {}) {
|
|
62
|
+
const current = db.object(binding, key);
|
|
63
|
+
if (!satisfied(options.onlyIf, current)) { await cancel(body); return null; }
|
|
64
|
+
const written = await writeBody(db, 'put', body, { hashes: checksumNames(options), replaces: current?.size ?? 0 });
|
|
65
|
+
try { verifyChecksums('put', options, written.sums); } catch (error) { db.drop(written.version); throw error; }
|
|
66
|
+
// The condition is checked again against whatever is current once the body is in.
|
|
67
|
+
const saved = db.transaction(() => satisfied(options.onlyIf, db.object(binding, key))
|
|
68
|
+
? save(db, binding, key, written, { ...options, etag: written.sums.md5, checksums: written.sums }) : null);
|
|
69
|
+
if (!saved) db.drop(written.version);
|
|
70
|
+
return saved;
|
|
71
|
+
}
|
|
72
|
+
function get(db, binding, key, options = {}) {
|
|
73
|
+
const row = db.object(binding, key);
|
|
74
|
+
if (!row) return null;
|
|
75
|
+
if (!satisfied(options.onlyIf, row)) return metadata(row);
|
|
76
|
+
const part = range(options.range, row.size);
|
|
77
|
+
return { ...metadata(row), ...(part ? { range: part } : {}), body: readBody(db, row.version, part?.offset ?? 0, part?.length ?? row.size) };
|
|
78
|
+
}
|
|
79
|
+
function remove(db, binding, keys) {
|
|
80
|
+
db.transaction(() => {
|
|
81
|
+
for (const key of [].concat(keys).map(String)) {
|
|
82
|
+
const row = db.object(binding, key);
|
|
83
|
+
if (!row) continue;
|
|
84
|
+
db.drop(row.version);
|
|
85
|
+
db.sql.exec('DELETE FROM rer_r2_objects WHERE binding = ? AND key = ?', binding, key);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
function list(db, binding, options = {}) {
|
|
90
|
+
const prefix = String(options.prefix ?? ''), include = options.include ?? [], delimiter = options.delimiter || undefined;
|
|
91
|
+
let limit = options.limit ?? 1000;
|
|
92
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1000) throw failure('list', 'MaxKeys params must be positive integer <= 1000.', 10022);
|
|
93
|
+
if (include.length) limit = Math.min(limit, 100);
|
|
94
|
+
let after = options.startAfter;
|
|
95
|
+
if (options.cursor) { const cursor = Buffer.from(options.cursor, 'base64url').toString('utf8'); if (after === undefined || cursor > after) after = cursor; }
|
|
96
|
+
const filter = 'binding = ?1 AND substr(key, 1, length(?2)) = ?2 AND (?3 IS NULL OR key > ?3)';
|
|
97
|
+
// As in R2, each delimited prefix counts once toward the limit, and the
|
|
98
|
+
// cursor skips past the last key it covers.
|
|
99
|
+
const rows = delimiter
|
|
100
|
+
? db.rows(`SELECT *, max(key) AS last, iif(instr(substr(key, length(?2) + 1), ?5), substr(key, 1, length(?2) + instr(substr(key, length(?2) + 1), ?5) + length(?5) - 1), NULL) AS grouped,
|
|
101
|
+
iif(instr(substr(key, length(?2) + 1), ?5), 'p:' || substr(key, 1, length(?2) + instr(substr(key, length(?2) + 1), ?5) + length(?5) - 1), 'k:' || key) AS item
|
|
102
|
+
FROM rer_r2_objects WHERE ${filter} GROUP BY item ORDER BY last LIMIT ?4`, binding, prefix, after ?? null, limit + 1, delimiter)
|
|
103
|
+
: db.rows(`SELECT *, key AS last, NULL AS grouped FROM rer_r2_objects WHERE ${filter} ORDER BY key LIMIT ?4`, binding, prefix, after ?? null, limit + 1);
|
|
104
|
+
const truncated = rows.length > limit, page = rows.slice(0, limit);
|
|
105
|
+
const objects = [], delimitedPrefixes = [];
|
|
106
|
+
for (const row of page) {
|
|
107
|
+
if (row.grouped !== null) delimitedPrefixes.push(row.grouped);
|
|
108
|
+
else objects.push(metadata({ ...row, key: row.last }, include));
|
|
42
109
|
}
|
|
110
|
+
return { objects, truncated, ...(truncated ? { cursor: Buffer.from(page.at(-1).last, 'utf8').toString('base64url') } : {}), delimitedPrefixes };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const upload = (db, method, binding, key, id) => {
|
|
114
|
+
const row = db.rows('SELECT * FROM rer_r2_uploads WHERE id = ? AND binding = ? AND key = ?', String(id), binding, key)[0];
|
|
115
|
+
if (!row) throw failure(method, 'The specified multipart upload does not exist.', 10024);
|
|
116
|
+
return row;
|
|
117
|
+
};
|
|
118
|
+
function createMultipartUpload(db, binding, key, options = {}) {
|
|
119
|
+
const id = randomId() + randomId();
|
|
120
|
+
db.sql.exec('INSERT INTO rer_r2_uploads(id, binding, key, options) VALUES(?, ?, ?, ?)', id, binding, key,
|
|
121
|
+
JSON.stringify({ httpMetadata: options.httpMetadata ?? {}, customMetadata: options.customMetadata ?? {}, storageClass: options.storageClass }));
|
|
122
|
+
return id;
|
|
123
|
+
}
|
|
124
|
+
async function uploadPart(db, binding, key, id, part, body) {
|
|
125
|
+
id = upload(db, 'uploadPart', binding, key, id).id;
|
|
126
|
+
const written = await writeBody(db, 'uploadPart', body);
|
|
127
|
+
// An abort while the body streamed in leaves nothing behind.
|
|
128
|
+
const saved = db.transaction(() => {
|
|
129
|
+
if (!db.rows('SELECT 1 FROM rer_r2_uploads WHERE id = ?', id).length) return null;
|
|
130
|
+
const previous = db.rows('SELECT version FROM rer_r2_parts WHERE upload = ? AND part = ?', id, part)[0];
|
|
131
|
+
if (previous) db.drop(previous.version);
|
|
132
|
+
db.sql.exec('INSERT OR REPLACE INTO rer_r2_parts(upload, part, version, size, etag) VALUES(?, ?, ?, ?, ?)', id, part, written.version, written.size, written.sums.md5);
|
|
133
|
+
return { partNumber: part, etag: written.sums.md5 };
|
|
134
|
+
});
|
|
135
|
+
if (!saved) { db.drop(written.version); upload(db, 'uploadPart', binding, key, id); }
|
|
136
|
+
return saved;
|
|
137
|
+
}
|
|
138
|
+
function complete(db, binding, key, id, selected) {
|
|
139
|
+
return db.transaction(() => {
|
|
140
|
+
const options = JSON.parse(upload(db, 'complete', binding, key, id).options);
|
|
141
|
+
const uploaded = new Map(db.rows('SELECT * FROM rer_r2_parts WHERE upload = ?', id).map(row => [row.part, row]));
|
|
142
|
+
const numbers = selected.map(item => item.partNumber);
|
|
143
|
+
if (new Set(numbers).size !== numbers.length) throw failure('complete', 'We encountered an internal error. Please try again.', 10001);
|
|
144
|
+
const parts = selected.map(item => {
|
|
145
|
+
const row = uploaded.get(item.partNumber);
|
|
146
|
+
if (row?.etag !== item.etag) throw failure('complete', 'One or more of the specified parts could not be found.', 10025);
|
|
147
|
+
return row;
|
|
148
|
+
}).sort((a, b) => a.part - b.part);
|
|
149
|
+
const version = randomId();
|
|
150
|
+
let seq = 0, start = 0;
|
|
151
|
+
for (const part of parts) {
|
|
152
|
+
const count = db.rows('SELECT count(*) AS n FROM rer_r2_chunks WHERE version = ?', part.version)[0].n;
|
|
153
|
+
db.sql.exec('UPDATE rer_r2_chunks SET version = ?, seq = seq + ?, start = start + ? WHERE version = ?', version, seq, start, part.version);
|
|
154
|
+
seq += count; start += part.size;
|
|
155
|
+
}
|
|
156
|
+
for (const row of uploaded.values()) if (!parts.includes(row)) db.drop(row.version);
|
|
157
|
+
db.sql.exec('DELETE FROM rer_r2_parts WHERE upload = ?', id);
|
|
158
|
+
db.sql.exec('DELETE FROM rer_r2_uploads WHERE id = ?', id);
|
|
159
|
+
const etag = createHash('md5').update(Buffer.concat(parts.map(part => Buffer.from(part.etag, 'hex')))).digest('hex') + `-${parts.length}`;
|
|
160
|
+
return save(db, binding, key, { version, size: start }, { ...options, etag, checksums: {} });
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
function abort(db, binding, key, id) {
|
|
164
|
+
db.transaction(() => {
|
|
165
|
+
for (const row of db.rows('SELECT version FROM rer_r2_parts WHERE upload = ?', String(id))) db.drop(row.version);
|
|
166
|
+
db.sql.exec('DELETE FROM rer_r2_parts WHERE upload = ?', String(id));
|
|
167
|
+
db.sql.exec('DELETE FROM rer_r2_uploads WHERE id = ? AND binding = ? AND key = ?', String(id), binding, key);
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const methods = { put, get, head: (db, binding, key) => { const row = db.object(binding, key); return row && metadata(row); },
|
|
172
|
+
delete: remove, list, createMultipartUpload, uploadPart, complete, abort };
|
|
173
|
+
export function r2Call(db, binding, method, args) {
|
|
174
|
+
if (!Object.hasOwn(methods, method)) throw new Error(`Unsupported R2 method: ${method}`);
|
|
175
|
+
const [key, ...rest] = args;
|
|
176
|
+
return methods[method](db, binding, method === 'list' ? key : method === 'delete' ? key : String(key), ...rest);
|
|
43
177
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { objectClass } from './config.mjs';
|
|
2
2
|
import { application, loaded, bundle } from './loader.mjs';
|
|
3
|
-
import { r2Call
|
|
3
|
+
import { r2Call } from './r2.mjs';
|
|
4
|
+
import { store } from './r2-store.mjs';
|
|
4
5
|
|
|
5
6
|
export async function database(host, binding, method, args) {
|
|
6
7
|
await host.active();
|
|
@@ -24,7 +25,8 @@ export async function object(host, binding, id, method, args) {
|
|
|
24
25
|
export async function r2(host, binding, method, args) {
|
|
25
26
|
await host.active();
|
|
26
27
|
if (!bundle.bindings.buckets.includes(binding)) throw new Error(`Unknown R2 binding: ${binding}`);
|
|
27
|
-
|
|
28
|
+
host.r2Store ??= store(host.ctx.storage);
|
|
29
|
+
const operation = r2Call(host.r2Store, binding, method, args);
|
|
28
30
|
host.r2Pending ??= new Set(); host.r2Pending.add(operation);
|
|
29
31
|
try { return await operation; } finally { host.r2Pending.delete(operation); }
|
|
30
32
|
}
|
|
@@ -32,6 +34,7 @@ export async function eraseResources(host) {
|
|
|
32
34
|
host.ctx.facets.delete('app');
|
|
33
35
|
for (const binding of bundle.bindings.databases.slice(1)) host.ctx.facets.delete(`database:${binding}`);
|
|
34
36
|
for (const key of (await host.ctx.storage.list({ prefix: 'facet:' })).keys()) host.ctx.facets.delete(key.slice(6));
|
|
37
|
+
// R2 objects are rows of the visitor's own storage, erased with it.
|
|
35
38
|
await Promise.allSettled(host.r2Pending ?? []);
|
|
36
|
-
|
|
39
|
+
host.r2Store = undefined;
|
|
37
40
|
}
|
|
@@ -10,7 +10,8 @@ export function sqlite(storage) {
|
|
|
10
10
|
const after = read ? { n: 0, id: lastRowId } : storage.sql.exec('SELECT total_changes() AS n, last_insert_rowid() AS id').one();
|
|
11
11
|
lastRowId = after.id;
|
|
12
12
|
return { success: true, results: rows.map(row => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), columns, rows,
|
|
13
|
-
meta: { duration: performance.now() - started, changes: after.n - before, last_row_id: after.id, changed_db: after.n !== before, rows_read: cursor.rowsRead, rows_written: cursor.rowsWritten
|
|
13
|
+
meta: { duration: performance.now() - started, changes: after.n - before, last_row_id: after.id, changed_db: after.n !== before, rows_read: cursor.rowsRead, rows_written: cursor.rowsWritten,
|
|
14
|
+
size_after: storage.sql.databaseSize, served_by: 'rer-native-demo', served_by_primary: true } };
|
|
14
15
|
}
|
|
15
16
|
return { query, batch: statements => storage.transactionSync(() => statements.map(statement => query(statement.query, statement.args))),
|
|
16
17
|
exec(query) { const started = performance.now(); storage.sql.exec(query).toArray(); return { count: query.split('\n').filter(line => line.trim()).length, duration: performance.now() - started }; } };
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { demoInventory } from './remove.mjs';
|
|
2
|
+
import { removeBucket } from './native/deploy-r2.mjs';
|
|
2
3
|
|
|
3
4
|
// Keep the Worker and custom domains throughout the transition. Container apps
|
|
4
5
|
// must go before deleting their backing class; Wrangler then applies the class
|
|
5
6
|
// migration atomically with the native code. A failed upload can be rerun.
|
|
6
|
-
export async function upgradeDemo({ api, plan }) {
|
|
7
|
+
export async function upgradeDemo({ api, plan, report = () => {} }) {
|
|
7
8
|
const found = await demoInventory(api, plan);
|
|
8
9
|
const classes = found.namespaces.map(row => row.class);
|
|
9
10
|
const unknown = classes.filter(name => !['DemoContainer', 'NativeDemo', 'DemoCapacity'].includes(name));
|
|
@@ -21,6 +22,12 @@ export async function upgradeDemo({ api, plan }) {
|
|
|
21
22
|
const remaining = await api.request('/containers/applications');
|
|
22
23
|
if (remaining.some(row => found.apps.some(app => app.id === row.id))) throw new Error('demo-upgrade-container-remove-incomplete');
|
|
23
24
|
}
|
|
25
|
+
// The retired shared R2 bucket goes while its binding still proves ownership;
|
|
26
|
+
// the new code binds no bucket. A rerun after a partial removal resumes.
|
|
27
|
+
if (found.bucket) {
|
|
28
|
+
await removeBucket(api, found.bucket);
|
|
29
|
+
report(`Removed the retired shared demo R2 bucket ${found.bucket.name}; R2 now lives in each visitor's storage.`);
|
|
30
|
+
}
|
|
24
31
|
if (classes.includes('DemoContainer')) {
|
|
25
32
|
// The API checks the currently deployed bindings before class deletion.
|
|
26
33
|
// Detach the retired binding first; inherit every other value by name,
|