beam.com 0.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +97 -0
- package/README.md +369 -0
- package/js/beam.com.mjs +33 -0
- package/js/download.mjs +78 -0
- package/js/node.mjs +75 -0
- package/package.json +36 -0
- package/runtime/app-com.js +203 -0
- package/runtime/beam.mjs +2 -0
- package/runtime/beam.wasm +0 -0
- package/runtime/browser/beam-wasm.js +2 -0
- package/runtime/browser/net.js +7 -0
- package/runtime/browser/none.js +4 -0
- package/runtime/browser.js +161 -0
- package/runtime/cloudflare/durable.js +361 -0
- package/runtime/cloudflare/index.js +76 -0
- package/runtime/cloudflare/release.js +42 -0
- package/runtime/cloudflare/snapshot.js +3 -0
- package/runtime/cloudflare/worker.js +1891 -0
- package/runtime/deno/app.js +8 -0
- package/runtime/deno/beam-wasm.js +3 -0
- package/runtime/deno/none.js +3 -0
- package/runtime/deno/release-bin.js +46 -0
- package/runtime/deno/snapshot-bin.js +11 -0
- package/runtime/deno/worker.js +1891 -0
- package/runtime/deno.js +346 -0
- package/runtime/deno.json +5 -0
- package/runtime/durable-global.js +40 -0
- package/runtime/durable.js +361 -0
- package/runtime/global.js +26 -0
- package/runtime/licenses/LICENSE +202 -0
- package/runtime/licenses/NOTICE +97 -0
- package/runtime/licenses/bcrypt_elixir.txt +87 -0
- package/runtime/licenses/cosmopolitan.txt +16 -0
- package/runtime/licenses/elixir/Apache-2.0.txt +73 -0
- package/runtime/licenses/elixir/LicenseRef-elixir-trademark-policy.txt +98 -0
- package/runtime/licenses/elixir/LicenseRef-scancode-unicode.txt +58 -0
- package/runtime/licenses/emscripten.txt +102 -0
- package/runtime/licenses/exqlite.txt +21 -0
- package/runtime/licenses/llvm.txt +311 -0
- package/runtime/licenses/musl.txt +193 -0
- package/runtime/licenses/otp/BSD-2-Clause.txt +22 -0
- package/runtime/licenses/otp/BSD-3-Clause.txt +25 -0
- package/runtime/licenses/otp/LicenseRef-IETF-MIB.txt +9 -0
- package/runtime/licenses/otp/LicenseRef-RSA-PKCS3.txt +23 -0
- package/runtime/licenses/otp/LicenseRef-RSA-PKCS5v2-0.txt +17 -0
- package/runtime/licenses/otp/LicenseRef-scancode-ietf-trust.txt +296 -0
- package/runtime/licenses/otp/LicenseRef-scancode-ietf.txt +23 -0
- package/runtime/licenses/otp/LicenseRef-scancode-w3c-docs-19990405.txt +26 -0
- package/runtime/licenses/otp/LicenseRef-scancode-wxwindows-free-doc-3.txt +57 -0
- package/runtime/licenses/otp/MIT.txt +17 -0
- package/runtime/licenses/otp/SGI-B-2.0.txt +25 -0
- package/runtime/licenses/otp/W3C-19980720.txt +55 -0
- package/runtime/licenses/picosat.txt +19 -0
- package/runtime/licenses/wamr.txt +219 -0
- package/runtime/nifs.js +2 -0
- package/runtime/package.json +1 -0
- package/runtime/page/404.html +49 -0
- package/runtime/page/app-com.js +203 -0
- package/runtime/page/beam.mjs +2 -0
- package/runtime/page/browser/beam-wasm.js +2 -0
- package/runtime/page/browser/net.js +7 -0
- package/runtime/page/browser/none.js +4 -0
- package/runtime/page/browser.js +161 -0
- package/runtime/page/index.html +50 -0
- package/runtime/page/licenses/LICENSE +202 -0
- package/runtime/page/licenses/NOTICE +97 -0
- package/runtime/page/licenses/bcrypt_elixir.txt +87 -0
- package/runtime/page/licenses/cosmopolitan.txt +16 -0
- package/runtime/page/licenses/elixir/Apache-2.0.txt +73 -0
- package/runtime/page/licenses/elixir/LicenseRef-elixir-trademark-policy.txt +98 -0
- package/runtime/page/licenses/elixir/LicenseRef-scancode-unicode.txt +58 -0
- package/runtime/page/licenses/emscripten.txt +102 -0
- package/runtime/page/licenses/exqlite.txt +21 -0
- package/runtime/page/licenses/llvm.txt +311 -0
- package/runtime/page/licenses/musl.txt +193 -0
- package/runtime/page/licenses/otp/BSD-2-Clause.txt +22 -0
- package/runtime/page/licenses/otp/BSD-3-Clause.txt +25 -0
- package/runtime/page/licenses/otp/LicenseRef-IETF-MIB.txt +9 -0
- package/runtime/page/licenses/otp/LicenseRef-RSA-PKCS3.txt +23 -0
- package/runtime/page/licenses/otp/LicenseRef-RSA-PKCS5v2-0.txt +17 -0
- package/runtime/page/licenses/otp/LicenseRef-scancode-ietf-trust.txt +296 -0
- package/runtime/page/licenses/otp/LicenseRef-scancode-ietf.txt +23 -0
- package/runtime/page/licenses/otp/LicenseRef-scancode-w3c-docs-19990405.txt +26 -0
- package/runtime/page/licenses/otp/LicenseRef-scancode-wxwindows-free-doc-3.txt +57 -0
- package/runtime/page/licenses/otp/MIT.txt +17 -0
- package/runtime/page/licenses/otp/SGI-B-2.0.txt +25 -0
- package/runtime/page/licenses/otp/W3C-19980720.txt +55 -0
- package/runtime/page/licenses/picosat.txt +19 -0
- package/runtime/page/licenses/wamr.txt +219 -0
- package/runtime/page/main.js +176 -0
- package/runtime/page/runtime-id.js +3 -0
- package/runtime/page/scope.js +17 -0
- package/runtime/page/sw.js +111 -0
- package/runtime/page/vm.js +148 -0
- package/runtime/page/worker.js +1891 -0
- package/runtime/page/ws-shim.js +143 -0
- package/runtime/release/app.js +17 -0
- package/runtime/release.json +1 -0
- package/runtime/runtime-id.js +3 -0
- package/runtime/tcp-proxy.mjs +30 -0
- package/runtime/worker.js +1891 -0
|
@@ -0,0 +1,1891 @@
|
|
|
1
|
+
// The BEAM on Cloudflare Workers: a Worker with the WebAssembly emulator
|
|
2
|
+
// (threads on JSPI, the runtime of wasm/erts, with no files) and no
|
|
3
|
+
// application. At the first request of an isolate, it gets a release
|
|
4
|
+
// (release.bin of beam_com_wasm), writes it into its file system at /app and boots
|
|
5
|
+
// it; the next requests to the isolate use the same VM. The release comes
|
|
6
|
+
// from:
|
|
7
|
+
// - a service binding APP (another Worker, as app.js): GET /release.bin;
|
|
8
|
+
// - else a text binding RELEASE_URL (R2, any URL): a fetch of that URL;
|
|
9
|
+
// - else a module release.bin in this Worker.
|
|
10
|
+
// A Durable Object can hold the VM in place of the isolate: see Vm.
|
|
11
|
+
//
|
|
12
|
+
// The HTTP server of the app (Bandit, with gen_tcp of wasm_tcp) listens on
|
|
13
|
+
// PORT (4000), and each request is a TCP connection to it (bridge): the
|
|
14
|
+
// request as HTTP/1.1 bytes, the response read back, and WebSocket frames
|
|
15
|
+
// turned into messages.
|
|
16
|
+
//
|
|
17
|
+
// Workers get no TCP connections: a WebSocket to /.tcp/PORT is a connection
|
|
18
|
+
// to the listener of PORT (gen_tcp:listen of wasm_tcp), and its binary
|
|
19
|
+
// messages are the bytes (a client proxy, as tcp-proxy.mjs or websocat -b,
|
|
20
|
+
// makes a local TCP port of it).
|
|
21
|
+
//
|
|
22
|
+
// The events between the host and Erlang (wasm_host) are those of
|
|
23
|
+
// wasm_host_server.erl; outgoing wasm_tcp sockets use node:net. Node.js
|
|
24
|
+
// and Deno have it, and Workers have it with nodejs_compat (on
|
|
25
|
+
// cloudflare:sockets), the default from the compatibility date 2026-08-04.
|
|
26
|
+
// TLS runs in Erlang (ssl), over this socket.
|
|
27
|
+
//
|
|
28
|
+
// "beam.com INPUT -o DIR --target wasm32" writes this file into DIR, with
|
|
29
|
+
// the runtime (beam.mjs, beam.wasm) and the release (release.bin).
|
|
30
|
+
import net from './browser/net.js';
|
|
31
|
+
import createBeam from './beam.mjs';
|
|
32
|
+
import wasm from './browser/beam-wasm.js';
|
|
33
|
+
|
|
34
|
+
// The NIF libraries in WebAssembly of the release (docs/NIFS.md), compiled
|
|
35
|
+
// by Wrangler: a Worker cannot compile WebAssembly at run time. nifs.js of
|
|
36
|
+
// "beam.com --target wasm32" maps the path of each file in /app to its
|
|
37
|
+
// module; the other hosts compile the files at run time (null).
|
|
38
|
+
async function nifModules() {
|
|
39
|
+
return (await import('./browser/none.js')).default;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// The module of the NIF library FILE: the key of nifs.js is its path in
|
|
43
|
+
// the release (lib/APP-VSN/priv/...). FILE can also be in a copy of priv
|
|
44
|
+
// (the priv files of a NIF app that beam.com copies to a directory), so
|
|
45
|
+
// the end of FILE must be APP-VSN/priv/...
|
|
46
|
+
function nifModule(nifs, file) {
|
|
47
|
+
for (const [path, module] of Object.entries(nifs ?? {})) {
|
|
48
|
+
if (file.endsWith('/' + path.slice(path.indexOf('/') + 1))) return module;
|
|
49
|
+
}
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function loadRelease(env) {
|
|
54
|
+
if (env.APP) return (await env.APP.fetch('http://app/release.bin')).arrayBuffer();
|
|
55
|
+
if (env.RELEASE_URL) return (await fetch(env.RELEASE_URL)).arrayBuffer();
|
|
56
|
+
return (await import('./browser/none.js')).default;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// release.bin: "BEAMFS1\n", then (length, path, length, data) for each file.
|
|
60
|
+
// Or the files of appFiles (app-com.js): {meta, files}, with no copy.
|
|
61
|
+
function unpack(FS, bytes) {
|
|
62
|
+
if (bytes.files) {
|
|
63
|
+
const dirs = new Set();
|
|
64
|
+
for (const [path, data] of bytes.files) {
|
|
65
|
+
const full = '/app/' + path;
|
|
66
|
+
const dir = full.slice(0, full.lastIndexOf('/'));
|
|
67
|
+
if (!dirs.has(dir)) { FS.mkdirTree(dir); dirs.add(dir); }
|
|
68
|
+
FS.writeFile(full, data, { canOwn: true });
|
|
69
|
+
}
|
|
70
|
+
return releaseMeta(bytes);
|
|
71
|
+
}
|
|
72
|
+
const b = new Uint8Array(bytes);
|
|
73
|
+
const view = new DataView(b.buffer, b.byteOffset, b.byteLength);
|
|
74
|
+
const text = new TextDecoder();
|
|
75
|
+
if (text.decode(b.subarray(0, 8)) !== 'BEAMFS1\n') throw new Error('release.bin: not a release');
|
|
76
|
+
let meta = null;
|
|
77
|
+
const dirs = new Set();
|
|
78
|
+
for (let i = 8; i < b.length;) {
|
|
79
|
+
const plen = view.getUint32(i); i += 4;
|
|
80
|
+
const path = text.decode(b.subarray(i, i + plen)); i += plen;
|
|
81
|
+
const dlen = view.getUint32(i); i += 4;
|
|
82
|
+
const data = b.subarray(i, i + dlen); i += dlen;
|
|
83
|
+
if (path === '.release.json') { meta = JSON.parse(text.decode(data)); continue; }
|
|
84
|
+
const full = '/app/' + path;
|
|
85
|
+
const dir = full.slice(0, full.lastIndexOf('/'));
|
|
86
|
+
if (!dirs.has(dir)) { FS.mkdirTree(dir); dirs.add(dir); }
|
|
87
|
+
// canOwn: the file is a view of release.bin, not a copy.
|
|
88
|
+
FS.writeFile(full, data, { canOwn: true });
|
|
89
|
+
}
|
|
90
|
+
return meta;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// .release.json, the first file of release.bin.
|
|
94
|
+
export function releaseMeta(bytes) {
|
|
95
|
+
if (bytes.meta) return JSON.parse(new TextDecoder().decode(bytes.meta));
|
|
96
|
+
const b = new Uint8Array(bytes);
|
|
97
|
+
const view = new DataView(b.buffer, b.byteOffset, b.byteLength);
|
|
98
|
+
const plen = view.getUint32(8);
|
|
99
|
+
const dlen = view.getUint32(12 + plen);
|
|
100
|
+
return JSON.parse(new TextDecoder().decode(b.subarray(16 + plen, 16 + plen + dlen)));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// The snapshots that the Worker makes itself (see Vm.makeSnapshot): in the
|
|
104
|
+
// R2 bucket of the binding SNAPSHOTS, else in the Cache API (of each data
|
|
105
|
+
// center). The key: the runtime, worker.js and the release (snapshot_key of
|
|
106
|
+
// .release.json, from the build), the text bindings, which the boot
|
|
107
|
+
// reads (a new secret gives a new snapshot), the host (a Worker or a
|
|
108
|
+
// Durable Object) and the version of the deploy (the binding BEAM_VERSION,
|
|
109
|
+
// version_metadata). The Workers of an account share the Cache API, and
|
|
110
|
+
// the boot of a snapshot ran its migrations on the database of its own
|
|
111
|
+
// Worker: so each deploy boots once, on its own database.
|
|
112
|
+
const snapshots = {
|
|
113
|
+
url: (key) => `https://beam-snapshot.invalid/${key}`,
|
|
114
|
+
async get(env, key) {
|
|
115
|
+
try {
|
|
116
|
+
if (env.SNAPSHOTS) return (await env.SNAPSHOTS.get(key))?.arrayBuffer() ?? null;
|
|
117
|
+
return (await caches.default.match(snapshots.url(key)))?.arrayBuffer() ?? null;
|
|
118
|
+
} catch (e) {
|
|
119
|
+
// No store here (workerd with no cache): no snapshot to make either.
|
|
120
|
+
console.log(`beam: no snapshot store (${e.message})`);
|
|
121
|
+
snapshots.unavailable = true;
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
},
|
|
125
|
+
async put(env, key, bytes) {
|
|
126
|
+
if (env.SNAPSHOTS) return env.SNAPSHOTS.put(key, bytes);
|
|
127
|
+
return caches.default.put(snapshots.url(key), new Response(bytes, {
|
|
128
|
+
headers: { 'content-type': 'application/octet-stream', 'cache-control': 'public, max-age=2592000' },
|
|
129
|
+
}));
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
async function snapshotKey(env, meta, host) {
|
|
134
|
+
const vars = Object.entries(env).filter(([, v]) => typeof v === 'string').sort(([a], [b]) => (a < b ? -1 : 1));
|
|
135
|
+
const id = JSON.stringify([meta.snapshot_key, vars, host, env.BEAM_VERSION?.id ?? null]);
|
|
136
|
+
const d = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(id));
|
|
137
|
+
return [...new Uint8Array(d)].map((x) => x.toString(16).padStart(2, '0')).join('');
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// A buffer of the files of the release: release.bin, or a buffer of
|
|
141
|
+
// appFiles (app-com.js).
|
|
142
|
+
const ofRelease = (release, buffer) => (release.buffers ? release.buffers.has(buffer) : buffer === release);
|
|
143
|
+
|
|
144
|
+
// The memory and the open files of a VM whose threads all returned
|
|
145
|
+
// (erts_wasm_hibernate), as snapshot.bin. The files that the boot wrote
|
|
146
|
+
// are the ones that are not views of release.bin (unpack: canOwn).
|
|
147
|
+
function capture(m, release, listeners, bootPoint = false, skip = () => false) {
|
|
148
|
+
const PAGE = 65536;
|
|
149
|
+
const heap = m.HEAPU8;
|
|
150
|
+
const words = new BigUint64Array(heap.buffer, 0, heap.length / 8);
|
|
151
|
+
const pages = [];
|
|
152
|
+
for (let p = 0, w = PAGE / 8; p < heap.length / PAGE; p++) {
|
|
153
|
+
for (let i = p * w; i < (p + 1) * w; i++) if (words[i]) { pages.push(p); break; }
|
|
154
|
+
}
|
|
155
|
+
const files = {};
|
|
156
|
+
const walk = (d) => {
|
|
157
|
+
for (const n of m.FS.readdir(d)) {
|
|
158
|
+
if (n === '.' || n === '..') continue;
|
|
159
|
+
const p = d === '/' ? '/' + n : d + '/' + n;
|
|
160
|
+
if (p === '/dev' || p === '/proc' || skip(p)) continue;
|
|
161
|
+
const node = m.FS.lookupPath(p).node;
|
|
162
|
+
if (m.FS.isDir(node.mode)) walk(p);
|
|
163
|
+
else if (m.FS.isFile(node.mode) && !ofRelease(release, node.contents?.buffer)) {
|
|
164
|
+
let bin = '';
|
|
165
|
+
for (const c of m.FS.readFile(p)) bin += String.fromCharCode(c);
|
|
166
|
+
files[p] = btoa(bin);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
walk('/');
|
|
171
|
+
const pipes = new Map();
|
|
172
|
+
const streams = m.FS.streams.map((st, fd) => {
|
|
173
|
+
if (!st) return null;
|
|
174
|
+
if (st.node?.pipe) {
|
|
175
|
+
if (!pipes.has(st.node.pipe)) pipes.set(st.node.pipe, pipes.size);
|
|
176
|
+
return { fd, pipe: pipes.get(st.node.pipe), flags: st.flags };
|
|
177
|
+
}
|
|
178
|
+
return { fd, path: st.path, flags: st.flags, position: st.position };
|
|
179
|
+
}).filter(Boolean);
|
|
180
|
+
// The NIF libraries in WebAssembly (nif_wasm_host.js): their pages come
|
|
181
|
+
// after the pages of the VM.
|
|
182
|
+
const libs = m.nifHost?.loaded() ? m.nifHost.save() : null;
|
|
183
|
+
const libPages = libs ? libs.libs.reduce((n, l) => n + l.pages.length, 0) : 0;
|
|
184
|
+
const head = new TextEncoder().encode(JSON.stringify({
|
|
185
|
+
size: heap.length, pages, fs: { files, streams }, listeners: Object.fromEntries(listeners),
|
|
186
|
+
boot_point: bootPoint,
|
|
187
|
+
nifs: libs && { count: libs.count, libs: libs.libs.map(({ data, ...l }) => l) },
|
|
188
|
+
}));
|
|
189
|
+
const out = new Uint8Array(12 + head.length + (pages.length + libPages) * PAGE);
|
|
190
|
+
out.set(new TextEncoder().encode('BEAMSNP1'));
|
|
191
|
+
new DataView(out.buffer).setUint32(8, head.length);
|
|
192
|
+
out.set(head, 12);
|
|
193
|
+
pages.forEach((p, i) => out.set(heap.subarray(p * PAGE, (p + 1) * PAGE), 12 + head.length + i * PAGE));
|
|
194
|
+
let at = 12 + head.length + pages.length * PAGE;
|
|
195
|
+
for (const l of libs?.libs ?? []) for (const d of l.data) { out.set(d, at); at += PAGE; }
|
|
196
|
+
return out;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// snapshot.bin (optional, beside release.bin): the memory of a booted VM
|
|
200
|
+
// whose threads all returned to the host (erts_wasm_hibernate), and the
|
|
201
|
+
// files and pipes of that moment. "BEAMSNP1", a 32-bit length and a JSON
|
|
202
|
+
// header, then the 64 KiB pages of the header (the others are zero): the
|
|
203
|
+
// pages of the VM, then the pages of each NIF library in WebAssembly
|
|
204
|
+
// (nifs of the header).
|
|
205
|
+
async function loadSnapshot(env) {
|
|
206
|
+
try {
|
|
207
|
+
if (env.APP) {
|
|
208
|
+
const r = await env.APP.fetch('http://app/snapshot.bin');
|
|
209
|
+
return r.ok ? r.arrayBuffer() : null;
|
|
210
|
+
}
|
|
211
|
+
return (await import('./browser/none.js')).default;
|
|
212
|
+
} catch {
|
|
213
|
+
return null;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function parseSnapshot(bytes) {
|
|
218
|
+
const b = new Uint8Array(bytes);
|
|
219
|
+
if (new TextDecoder().decode(b.subarray(0, 8)) !== 'BEAMSNP1') throw new Error('snapshot.bin: not a snapshot');
|
|
220
|
+
const len = new DataView(b.buffer, b.byteOffset).getUint32(8);
|
|
221
|
+
const head = JSON.parse(new TextDecoder().decode(b.subarray(12, 12 + len)));
|
|
222
|
+
return { ...head, pagesData: b.subarray(12 + len) };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// BEAM_CONNECT: the hosts that the VM can connect to, separated by commas:
|
|
226
|
+
// "host", "host:port", or "*.domain" (the subdomains of domain). The host
|
|
227
|
+
// resolves a name, so the VM cannot reach another address through it.
|
|
228
|
+
// With no BEAM_CONNECT, the VM can connect to all hosts.
|
|
229
|
+
function connectAllowed(list, host, port) {
|
|
230
|
+
if (list === undefined) return true;
|
|
231
|
+
const name = String(host).toLowerCase().replace(/\.$/, '');
|
|
232
|
+
return list.split(',').map((r) => r.trim().toLowerCase()).filter(Boolean).some((rule) => {
|
|
233
|
+
const i = rule.lastIndexOf(':');
|
|
234
|
+
const [pattern, p] = i > 0 && !rule.includes(']') ? [rule.slice(0, i), rule.slice(i + 1)] : [rule, undefined];
|
|
235
|
+
if (p !== undefined && Number(p) !== port) return false;
|
|
236
|
+
return pattern.startsWith('*.') ? name.endsWith(pattern.slice(1)) : name === pattern;
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// The memory and the open files of the snapshot, in a new instance (before
|
|
241
|
+
// main(), which does not run): then the threads start again.
|
|
242
|
+
function restore(m, exports, snap) {
|
|
243
|
+
const PAGE = 65536;
|
|
244
|
+
// The pages of the new instance that are not in the snapshot: zero (the
|
|
245
|
+
// memory that grows is zero already).
|
|
246
|
+
const first = m.HEAPU8.length / PAGE, have = new Set(snap.pages);
|
|
247
|
+
for (let p = 0; p < first; p++) if (!have.has(p)) m.HEAPU8.fill(0, p * PAGE, (p + 1) * PAGE);
|
|
248
|
+
if (!exports.jspi_snapshot_grow(snap.size)) throw new Error('snapshot: no memory');
|
|
249
|
+
const heap = m.HEAPU8;
|
|
250
|
+
snap.pages.forEach((p, i) => heap.set(snap.pagesData.subarray(i * PAGE, (i + 1) * PAGE), p * PAGE));
|
|
251
|
+
// The NIF libraries in WebAssembly, from their files (capture).
|
|
252
|
+
if (snap.nifs) {
|
|
253
|
+
let at = snap.pages.length * PAGE;
|
|
254
|
+
const libs = snap.nifs.libs.map((l) => ({
|
|
255
|
+
...l, data: l.pages.map(() => { const d = snap.pagesData.subarray(at, at + PAGE); at += PAGE; return d; }),
|
|
256
|
+
}));
|
|
257
|
+
m.nifHost.restore({ count: snap.nifs.count, libs }, (f) => m.FS.readFile(f));
|
|
258
|
+
}
|
|
259
|
+
const made = new Set();
|
|
260
|
+
for (const s of snap.fs.streams) {
|
|
261
|
+
if (s.fd <= 2) continue;
|
|
262
|
+
if (s.pipe !== undefined) {
|
|
263
|
+
if (!made.has(s.pipe)) { made.add(s.pipe); exports.jspi_snapshot_pipe(); }
|
|
264
|
+
const got = m.FS.getStream(s.fd);
|
|
265
|
+
if (!got?.node?.pipe) throw new Error(`snapshot: fd ${s.fd} is not a pipe`);
|
|
266
|
+
got.flags = s.flags;
|
|
267
|
+
} else {
|
|
268
|
+
const st = m.FS.open(s.path, s.flags);
|
|
269
|
+
if (st.fd !== s.fd) throw new Error(`snapshot: fd ${s.fd} is ${st.fd}`);
|
|
270
|
+
st.position = s.position;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
exports.wasm_host_restore();
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// The threads of a restored VM start again (erts_wasm_resume), and OpenSSL
|
|
277
|
+
// gets new random bytes. The global scope of a Worker has no random
|
|
278
|
+
// values: there, warm() starts the threads, and the first request sends
|
|
279
|
+
// the bytes (seed()).
|
|
280
|
+
function resume(m, exports) {
|
|
281
|
+
const n = exports.erts_wasm_resume();
|
|
282
|
+
seed(m);
|
|
283
|
+
return n;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function seed(m) {
|
|
287
|
+
// New random bytes for OpenSSL, before any request (wasm_host_server).
|
|
288
|
+
const h = new TextEncoder().encode('{"t":"restored"}\n');
|
|
289
|
+
const b = new Uint8Array(h.length + 48);
|
|
290
|
+
b.set(h);
|
|
291
|
+
crypto.getRandomValues(b.subarray(h.length));
|
|
292
|
+
m.beamHost.push(b);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// The directories of BEAM_PERSIST (a Durable Object): their files are in
|
|
296
|
+
// the SQLite storage of the object, so they stay when the object leaves
|
|
297
|
+
// memory. The host writes them into the file system of the VM before it
|
|
298
|
+
// starts, and saves a file when the VM closes it after a write (or 1 s
|
|
299
|
+
// after a write, for a file that stays open), and a delete, a rename or a
|
|
300
|
+
// new directory at once. A file is in chunks of 1 MB (a row has 2 MB at
|
|
301
|
+
// most).
|
|
302
|
+
const CHUNK = 1 << 20;
|
|
303
|
+
|
|
304
|
+
class Persist {
|
|
305
|
+
constructor(sql, dirs) {
|
|
306
|
+
this.sql = sql;
|
|
307
|
+
this.dirs = dirs.map((d) => d.replace(/\/+$/, '')).filter((d) => d.startsWith('/') && d.length > 1);
|
|
308
|
+
this.dirty = new Set();
|
|
309
|
+
sql.exec('CREATE TABLE IF NOT EXISTS beam_fs (path TEXT PRIMARY KEY, dir INTEGER, mode INTEGER, size INTEGER)');
|
|
310
|
+
sql.exec('CREATE TABLE IF NOT EXISTS beam_fs_chunk (path TEXT, n INTEGER, data BLOB, PRIMARY KEY (path, n))');
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
under(path) {
|
|
314
|
+
return typeof path === 'string' && this.dirs.some((d) => path === d || path.startsWith(d + '/'));
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// The files of the storage, into FS (before the VM starts).
|
|
318
|
+
load(FS) {
|
|
319
|
+
for (const d of this.dirs) FS.mkdirTree(d);
|
|
320
|
+
let files = 0, bytes = 0;
|
|
321
|
+
for (const { path, dir, size } of this.sql.exec('SELECT path, dir, size FROM beam_fs ORDER BY path')) {
|
|
322
|
+
if (!this.under(path)) continue;
|
|
323
|
+
if (dir) { FS.mkdirTree(path); continue; }
|
|
324
|
+
const data = new Uint8Array(size);
|
|
325
|
+
for (const { n, data: part } of this.sql.exec('SELECT n, data FROM beam_fs_chunk WHERE path = ? ORDER BY n', path)) {
|
|
326
|
+
data.set(new Uint8Array(part), n * CHUNK);
|
|
327
|
+
}
|
|
328
|
+
FS.mkdirTree(path.slice(0, path.lastIndexOf('/')) || '/');
|
|
329
|
+
FS.writeFile(path, data, { canOwn: true });
|
|
330
|
+
files++;
|
|
331
|
+
bytes += size;
|
|
332
|
+
}
|
|
333
|
+
console.log(`beam: persist ${this.dirs.join(', ')}: ${files} files, ${bytes >> 10} KB`);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
save(FS, path) {
|
|
337
|
+
this.dirty.delete(path);
|
|
338
|
+
let node;
|
|
339
|
+
try { node = FS.lookupPath(path).node; } catch { return this.remove(path); }
|
|
340
|
+
if (FS.isDir(node.mode)) return this.sql.exec('INSERT OR REPLACE INTO beam_fs VALUES (?, 1, ?, 0)', path, node.mode);
|
|
341
|
+
if (!FS.isFile(node.mode)) return;
|
|
342
|
+
const data = FS.readFile(path);
|
|
343
|
+
this.sql.exec('DELETE FROM beam_fs_chunk WHERE path = ?', path);
|
|
344
|
+
this.sql.exec('INSERT OR REPLACE INTO beam_fs VALUES (?, 0, ?, ?)', path, node.mode, data.length);
|
|
345
|
+
for (let n = 0; n * CHUNK < data.length; n++) {
|
|
346
|
+
this.sql.exec('INSERT INTO beam_fs_chunk VALUES (?, ?, ?)', path, n, data.slice(n * CHUNK, (n + 1) * CHUNK).buffer);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// A file or a directory, and all under it.
|
|
351
|
+
remove(path) {
|
|
352
|
+
for (const t of ['beam_fs', 'beam_fs_chunk']) {
|
|
353
|
+
this.sql.exec(`DELETE FROM ${t} WHERE path = ? OR substr(path, 1, ?) = ?`, path, path.length + 1, path + '/');
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// A directory and all under it, or one file.
|
|
358
|
+
saveTree(FS, path) {
|
|
359
|
+
this.save(FS, path);
|
|
360
|
+
let node;
|
|
361
|
+
try { node = FS.lookupPath(path).node; } catch { return; }
|
|
362
|
+
if (!FS.isDir(node.mode)) return;
|
|
363
|
+
for (const n of FS.readdir(path)) if (n !== '.' && n !== '..') this.saveTree(FS, `${path}/${n}`);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// Wrap the functions of FS that change files (the system calls of the
|
|
367
|
+
// VM call them).
|
|
368
|
+
attach(FS) {
|
|
369
|
+
const abs = (p) => (typeof p === 'string' && !p.startsWith('/') ? `${FS.cwd()}/${p}` : p);
|
|
370
|
+
const wrap = (name, after) => {
|
|
371
|
+
const f = FS[name];
|
|
372
|
+
FS[name] = (...a) => {
|
|
373
|
+
const r = f.apply(FS, a);
|
|
374
|
+
try { after(...a); } catch (e) { console.log(`beam: persist ${name}: ${e.message}`); }
|
|
375
|
+
return r;
|
|
376
|
+
};
|
|
377
|
+
};
|
|
378
|
+
wrap('write', (stream) => {
|
|
379
|
+
if (!this.under(stream.path)) return;
|
|
380
|
+
this.dirty.add(stream.path);
|
|
381
|
+
this.flush ??= setTimeout(() => {
|
|
382
|
+
this.flush = null;
|
|
383
|
+
for (const p of [...this.dirty]) this.save(FS, p);
|
|
384
|
+
}, 1000);
|
|
385
|
+
});
|
|
386
|
+
wrap('close', (stream) => {
|
|
387
|
+
if (this.under(stream.path) && ((stream.flags & 3) !== 0 || this.dirty.has(stream.path))) this.save(FS, stream.path);
|
|
388
|
+
});
|
|
389
|
+
wrap('truncate', (path) => { if (this.under(abs(path))) this.save(FS, abs(path)); });
|
|
390
|
+
wrap('mkdir', (path) => { if (this.under(abs(path))) this.save(FS, abs(path)); });
|
|
391
|
+
wrap('unlink', (path) => { if (this.under(abs(path))) this.remove(abs(path)); });
|
|
392
|
+
wrap('rmdir', (path) => { if (this.under(abs(path))) this.remove(abs(path)); });
|
|
393
|
+
wrap('rename', (from, to) => {
|
|
394
|
+
if (this.under(abs(from))) this.remove(abs(from));
|
|
395
|
+
if (this.under(abs(to))) this.saveTree(FS, abs(to));
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// Ecto SQLite (wasm_host_sqlite): the host runs each statement, on the
|
|
401
|
+
// SQLite storage of a Durable Object (ctx.storage.sql: the option sql of
|
|
402
|
+
// Vm), else on the D1 database of the binding BEAM_D1 (default DB). A
|
|
403
|
+
// value on the wire: a blob as {b: base64}, a big integer as {i: "..."}.
|
|
404
|
+
function toWire(v) {
|
|
405
|
+
// D1 gives a BLOB as an array of bytes (a SQL value is no array else).
|
|
406
|
+
if (Array.isArray(v)) v = Uint8Array.from(v);
|
|
407
|
+
if (v instanceof ArrayBuffer || ArrayBuffer.isView(v)) {
|
|
408
|
+
let bin = '';
|
|
409
|
+
for (const c of new Uint8Array(v.buffer ?? v, v.byteOffset ?? 0, v.byteLength)) bin += String.fromCharCode(c);
|
|
410
|
+
return { b: btoa(bin) };
|
|
411
|
+
}
|
|
412
|
+
if (typeof v === 'bigint') return { i: String(v) };
|
|
413
|
+
return v;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function fromWire(v) {
|
|
417
|
+
if (v && typeof v === 'object' && 'b' in v) return Uint8Array.from(atob(v.b), (c) => c.charCodeAt(0)).buffer;
|
|
418
|
+
if (v && typeof v === 'object' && 'i' in v) return BigInt(v.i);
|
|
419
|
+
return v;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// The statements that give rows (the others give changes and a row id).
|
|
423
|
+
const givesRows = (sql) => /^\s*(select|with|pragma|values|explain)\b/i.test(sql) || /\breturning\b/i.test(sql);
|
|
424
|
+
|
|
425
|
+
// The changes of SQLite (rowsWritten counts the writes of the indexes and
|
|
426
|
+
// of sqlite_sequence too).
|
|
427
|
+
const reads = (sql) => /^\s*(select|with|pragma|values|explain)\b/i.test(sql);
|
|
428
|
+
|
|
429
|
+
function sqlDurable(storage, sql, params) {
|
|
430
|
+
const c = storage.exec(sql, ...params);
|
|
431
|
+
const rows = [...c.raw()].map((r) => r.map(toWire));
|
|
432
|
+
const out = { columns: c.columnNames, rows, changes: 0 };
|
|
433
|
+
if (!reads(sql)) {
|
|
434
|
+
const [[changes, rowid]] = [...storage.exec('SELECT changes(), last_insert_rowid()').raw()];
|
|
435
|
+
Object.assign(out, { changes, last_row_id: rowid });
|
|
436
|
+
}
|
|
437
|
+
return out;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
async function sqlD1(env, sql, params) {
|
|
441
|
+
const name = env.BEAM_D1 ?? 'DB';
|
|
442
|
+
if (!env[name]) throw new Error(`no D1 binding ${name} (or set BEAM_D1)`);
|
|
443
|
+
const st = env[name].prepare(sql).bind(...params);
|
|
444
|
+
if (givesRows(sql)) {
|
|
445
|
+
const [columns = [], ...rows] = await st.raw({ columnNames: true });
|
|
446
|
+
return { columns, rows: rows.map((r) => r.map(toWire)), changes: reads(sql) ? 0 : rows.length };
|
|
447
|
+
}
|
|
448
|
+
const { meta } = await st.run();
|
|
449
|
+
return { columns: [], rows: [], changes: meta.changes ?? 0, last_row_id: meta.last_row_id ?? 0 };
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
let vm; // the VM of this isolate
|
|
453
|
+
|
|
454
|
+
export default {
|
|
455
|
+
fetch(request, env, ctx) {
|
|
456
|
+
if (!vm) {
|
|
457
|
+
const v = vm = new Vm(env, { host: new URL(request.url).hostname });
|
|
458
|
+
v.ready.catch(() => { if (vm === v) vm = undefined; });
|
|
459
|
+
}
|
|
460
|
+
return vm.fetch(request, ctx);
|
|
461
|
+
},
|
|
462
|
+
};
|
|
463
|
+
|
|
464
|
+
// The files of the host for SQLite in the VM (exqlite, the NIF of the
|
|
465
|
+
// runtime, with c_src/erts_wasm/sqlite_vfs.c): the main database files. A store
|
|
466
|
+
// keeps their blocks of 4 KiB, each with the version of the database that
|
|
467
|
+
// wrote it:
|
|
468
|
+
// meta(name) -> {version, size, stamp}: version 0 and size 0 for no file
|
|
469
|
+
// block(name, n, version) -> {data, v, prev}: the newest block n at or
|
|
470
|
+
// before version, or null
|
|
471
|
+
// commit(name, stamp, {version, size, blocks: [{n, data, prev, drop}]}, release)
|
|
472
|
+
// -> the new stamp, or null when another commit came first; with
|
|
473
|
+
// release, the commit also removes the write lock
|
|
474
|
+
// lock(name, stamp, owner, ms) -> a lease, or null when another owner
|
|
475
|
+
// has the write lock or the stamp is not the last one
|
|
476
|
+
// unlock(name, lease): removes the write lock of that lease
|
|
477
|
+
// remove(name)
|
|
478
|
+
// A transaction reads one version of the database: the version at its
|
|
479
|
+
// shared lock. Its writes go to the store in one commit, which fails when
|
|
480
|
+
// another VM committed first (SQLITE_BUSY, and nothing changes). A block
|
|
481
|
+
// keeps its last two versions, for the reads of an older transaction.
|
|
482
|
+
const BLOCK = 4096;
|
|
483
|
+
const FOP = { OPEN: 1, CLOSE: 2, READ: 3, WRITE: 4, TRUNCATE: 5, SYNC: 6, SIZE: 7, LOCK: 8, UNLOCK: 9,
|
|
484
|
+
DELETE: 10, ACCESS: 11, BEGIN_BATCH: 12, COMMIT_BATCH: 13, ROLLBACK_BATCH: 14 };
|
|
485
|
+
const SQLITE_BUSY = 5, SQLITE_IOERR = 10, SQLITE_FULL = 13;
|
|
486
|
+
|
|
487
|
+
export class HostFiles {
|
|
488
|
+
// maxBlocks: the blocks of one commit (a commit of the store has limits).
|
|
489
|
+
// lease: the time of the write lock of a database (ms). The lock only
|
|
490
|
+
// makes a conflict less frequent; the commit checks the version. A
|
|
491
|
+
// write transaction is three operations of the store: the version (at
|
|
492
|
+
// the shared lock), the lock, and the commit, which also unlocks.
|
|
493
|
+
constructor(store, { maxBlocks = 160, lease = 10000, debug = false } = {}) {
|
|
494
|
+
this.store = store;
|
|
495
|
+
this.debug = debug;
|
|
496
|
+
this.maxBlocks = maxBlocks;
|
|
497
|
+
this.lease = lease;
|
|
498
|
+
this.owner = crypto.randomUUID();
|
|
499
|
+
this.files = new Map(); // id -> {db, snap, dirty, size}
|
|
500
|
+
this.dbs = new Map(); // name -> {name, version, cache}: blocks of one version
|
|
501
|
+
this.next = 1;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// An operation of sqlite_vfs.c (jspi_file_wait): an integer.
|
|
505
|
+
async call(op, id, offset, buf, n, mem) {
|
|
506
|
+
try {
|
|
507
|
+
const r = await this.op(op, id, offset, buf, n, mem);
|
|
508
|
+
if (this.debug) console.log(`beam: SQLite file: op ${op} file ${id} at ${offset} n ${n}: ${r}`);
|
|
509
|
+
return r;
|
|
510
|
+
} catch (e) {
|
|
511
|
+
console.log(`beam: SQLite file: ${e.message}`);
|
|
512
|
+
return op === FOP.READ ? -1 : SQLITE_IOERR;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
db(name) {
|
|
517
|
+
let d = this.dbs.get(name);
|
|
518
|
+
if (!d) this.dbs.set(name, d = { name, version: -1, cache: new Map() });
|
|
519
|
+
return d;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
async op(op, id, offset, buf, n, mem) {
|
|
523
|
+
switch (op) {
|
|
524
|
+
case FOP.OPEN: {
|
|
525
|
+
const fid = this.next++;
|
|
526
|
+
this.files.set(fid, { db: this.db(mem.string(buf)), snap: null, dirty: new Map(), size: null });
|
|
527
|
+
return fid;
|
|
528
|
+
}
|
|
529
|
+
case FOP.ACCESS: return (await this.store.meta(mem.string(buf))).size > 0 ? 1 : 0;
|
|
530
|
+
case FOP.DELETE: {
|
|
531
|
+
const name = mem.string(buf);
|
|
532
|
+
await this.store.remove(name);
|
|
533
|
+
this.dbs.delete(name);
|
|
534
|
+
return 0;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
const f = this.files.get(id);
|
|
538
|
+
if (!f) return op === FOP.READ ? -1 : SQLITE_IOERR;
|
|
539
|
+
switch (op) {
|
|
540
|
+
case FOP.CLOSE: await this.release(f, id); this.files.delete(id); return 0;
|
|
541
|
+
case FOP.LOCK:
|
|
542
|
+
if (n === 1) await this.begin(f);
|
|
543
|
+
else if (n >= 2 && !f.lease) return this.reserve(f, id);
|
|
544
|
+
return 0;
|
|
545
|
+
case FOP.UNLOCK:
|
|
546
|
+
if (n < 2) await this.release(f, id);
|
|
547
|
+
if (n === 0) { f.snap = null; f.dirty.clear(); f.size = null; }
|
|
548
|
+
return 0;
|
|
549
|
+
case FOP.READ: return this.read(f, offset, buf, n, mem);
|
|
550
|
+
case FOP.WRITE: await this.write(f, offset, buf, n, mem); return 0;
|
|
551
|
+
case FOP.TRUNCATE:
|
|
552
|
+
if (!f.snap) await this.begin(f);
|
|
553
|
+
f.size = offset;
|
|
554
|
+
for (const k of f.dirty.keys()) if (k * BLOCK >= offset) f.dirty.delete(k);
|
|
555
|
+
return 0;
|
|
556
|
+
case FOP.SIZE:
|
|
557
|
+
if (!f.snap) await this.begin(f);
|
|
558
|
+
new DataView(mem.heap().buffer).setFloat64(buf, f.size ?? f.snap.size, true);
|
|
559
|
+
return 0;
|
|
560
|
+
case FOP.BEGIN_BATCH: return 0;
|
|
561
|
+
case FOP.SYNC: case FOP.COMMIT_BATCH: return this.commit(f);
|
|
562
|
+
case FOP.ROLLBACK_BATCH: f.dirty.clear(); f.size = null; return 0;
|
|
563
|
+
}
|
|
564
|
+
return SQLITE_IOERR;
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
// A shared lock: the last version of the database. The cache of the
|
|
568
|
+
// blocks keeps one version.
|
|
569
|
+
async begin(f) {
|
|
570
|
+
f.snap = await this.store.meta(f.db.name);
|
|
571
|
+
f.dirty.clear();
|
|
572
|
+
f.size = null;
|
|
573
|
+
if (f.db.version !== f.snap.version) {
|
|
574
|
+
f.db.cache.clear();
|
|
575
|
+
f.db.version = f.snap.version;
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
// A reserved lock starts a write. It fails with SQLITE_BUSY when another
|
|
580
|
+
// file has the write lock, or when the version of the read is not the
|
|
581
|
+
// last one. Then the busy handler of SQLite tries again from a new read.
|
|
582
|
+
async reserve(f, id) {
|
|
583
|
+
if (!f.snap) await this.begin(f);
|
|
584
|
+
f.lease = await this.store.lock(f.db.name, f.snap.stamp, `${this.owner}:${id}`, this.lease);
|
|
585
|
+
return f.lease ? 0 : SQLITE_BUSY;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
async release(f, id) {
|
|
589
|
+
const lease = f.lease;
|
|
590
|
+
if (!lease) return;
|
|
591
|
+
f.lease = null;
|
|
592
|
+
await this.store.unlock(f.db.name, lease);
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
// Block k as this transaction sees it (zeros for no block).
|
|
596
|
+
async block(f, k) {
|
|
597
|
+
const d = f.dirty.get(k);
|
|
598
|
+
if (d) return d;
|
|
599
|
+
const db = f.db, same = db.version === f.snap.version;
|
|
600
|
+
let b = same ? db.cache.get(k) : undefined;
|
|
601
|
+
if (!b) {
|
|
602
|
+
b = (await this.store.block(db.name, k, f.snap.version)) ?? { data: null, v: 0, prev: 0 };
|
|
603
|
+
if (same && db.version === f.snap.version) db.cache.set(k, b);
|
|
604
|
+
}
|
|
605
|
+
return b.data ?? new Uint8Array(BLOCK);
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
async read(f, offset, buf, n, mem) {
|
|
609
|
+
if (!f.snap) await this.begin(f);
|
|
610
|
+
const end = Math.min(offset + n, f.size ?? f.snap.size);
|
|
611
|
+
for (let pos = offset; pos < end;) {
|
|
612
|
+
const k = Math.floor(pos / BLOCK), at = pos - k * BLOCK, len = Math.min(BLOCK - at, end - pos);
|
|
613
|
+
const data = await this.block(f, k);
|
|
614
|
+
mem.heap().set(data.subarray(at, at + len), buf + (pos - offset));
|
|
615
|
+
pos += len;
|
|
616
|
+
}
|
|
617
|
+
return Math.max(0, end - offset);
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
async write(f, offset, buf, n, mem) {
|
|
621
|
+
if (!f.snap) await this.begin(f);
|
|
622
|
+
for (let pos = offset; pos < offset + n;) {
|
|
623
|
+
const k = Math.floor(pos / BLOCK), at = pos - k * BLOCK, len = Math.min(BLOCK - at, offset + n - pos);
|
|
624
|
+
const data = new Uint8Array(await this.block(f, k));
|
|
625
|
+
data.set(mem.heap().subarray(buf + (pos - offset), buf + (pos - offset) + len), at);
|
|
626
|
+
f.dirty.set(k, data);
|
|
627
|
+
pos += len;
|
|
628
|
+
}
|
|
629
|
+
f.size = Math.max(f.size ?? f.snap.size, offset + n);
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
// The writes of the transaction, in one commit of the store.
|
|
633
|
+
async commit(f) {
|
|
634
|
+
if (!f.snap) return 0;
|
|
635
|
+
const size = f.size ?? f.snap.size;
|
|
636
|
+
if (!f.dirty.size && size === f.snap.size) return 0;
|
|
637
|
+
const db = f.db, version = f.snap.version + 1;
|
|
638
|
+
if (f.dirty.size > this.maxBlocks) {
|
|
639
|
+
console.log(`beam: SQLite: a commit of ${f.dirty.size} blocks of 4 KiB (at most ${this.maxBlocks})`);
|
|
640
|
+
f.dirty.clear();
|
|
641
|
+
f.size = null;
|
|
642
|
+
return SQLITE_FULL;
|
|
643
|
+
}
|
|
644
|
+
const same = db.version === f.snap.version;
|
|
645
|
+
const blocks = [...f.dirty].map(([n, data]) => {
|
|
646
|
+
const old = same ? db.cache.get(n) : undefined;
|
|
647
|
+
return { n, data, prev: old?.v ?? 0, drop: old?.prev || 0 };
|
|
648
|
+
});
|
|
649
|
+
const stamp = await this.store.commit(db.name, f.snap.stamp, { version, size, blocks }, !!f.lease);
|
|
650
|
+
f.dirty.clear();
|
|
651
|
+
f.size = null;
|
|
652
|
+
if (stamp === null) {
|
|
653
|
+
db.cache.clear();
|
|
654
|
+
db.version = -1;
|
|
655
|
+
return SQLITE_BUSY;
|
|
656
|
+
}
|
|
657
|
+
f.lease = null; // the commit removed it
|
|
658
|
+
if (!same) db.cache.clear();
|
|
659
|
+
db.version = version;
|
|
660
|
+
for (const b of blocks) db.cache.set(b.n, { data: b.data, v: version, prev: b.prev });
|
|
661
|
+
f.snap = { version, size, stamp };
|
|
662
|
+
return 0;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
// A store of HostFiles in memory: for a page, and for the tests.
|
|
667
|
+
export class MemoryStore {
|
|
668
|
+
constructor() { this.dbs = new Map(); }
|
|
669
|
+
get(name) {
|
|
670
|
+
let d = this.dbs.get(name);
|
|
671
|
+
if (!d) this.dbs.set(name, d = { version: 0, size: 0, stamp: 0, blocks: new Map() });
|
|
672
|
+
return d;
|
|
673
|
+
}
|
|
674
|
+
async meta(name) { const d = this.get(name); return { version: d.version, size: d.size, stamp: d.stamp }; }
|
|
675
|
+
async block(name, n, version) {
|
|
676
|
+
const vs = this.get(name).blocks.get(n) ?? [];
|
|
677
|
+
for (let i = vs.length - 1; i >= 0; i--) if (vs[i].v <= version) return vs[i];
|
|
678
|
+
return null;
|
|
679
|
+
}
|
|
680
|
+
async commit(name, stamp, { version, size, blocks }, release = false) {
|
|
681
|
+
const d = this.get(name);
|
|
682
|
+
if (d.stamp !== stamp) return null;
|
|
683
|
+
if (release) d.lock = null;
|
|
684
|
+
for (const b of blocks) {
|
|
685
|
+
const vs = (d.blocks.get(b.n) ?? []).filter((x) => !b.drop || x.v !== b.drop);
|
|
686
|
+
vs.push({ data: b.data, v: version, prev: b.prev });
|
|
687
|
+
d.blocks.set(b.n, vs);
|
|
688
|
+
}
|
|
689
|
+
Object.assign(d, { version, size, stamp: d.stamp + 1 });
|
|
690
|
+
return d.stamp;
|
|
691
|
+
}
|
|
692
|
+
async lock(name, stamp, owner, ms) {
|
|
693
|
+
const d = this.get(name), now = Date.now();
|
|
694
|
+
if (d.stamp !== stamp || (d.lock && d.lock.owner !== owner && d.lock.until > now)) return null;
|
|
695
|
+
d.lock = { owner, until: now + ms, lease: (this.leases = (this.leases ?? 0) + 1) };
|
|
696
|
+
return d.lock.lease;
|
|
697
|
+
}
|
|
698
|
+
async unlock(name, lease) {
|
|
699
|
+
const d = this.get(name);
|
|
700
|
+
if (d.lock?.lease === lease) d.lock = null;
|
|
701
|
+
}
|
|
702
|
+
async remove(name) { this.dbs.delete(name); }
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// WebAssembly for the VM (wasm_host_wasm.erl, the API of the application
|
|
706
|
+
// wasm of beam.com): the engine of the host compiles the modules and runs
|
|
707
|
+
// them, with WASI preview 1 for programs. The standard output and error of
|
|
708
|
+
// a program go back to the caller with each reply. There is no file
|
|
709
|
+
// system: stdin is empty, and there are no preopens. A Cloudflare Worker
|
|
710
|
+
// cannot compile WebAssembly at run time, so there compile gives an error.
|
|
711
|
+
// A module or an instance has an id; they stay until the VM stops.
|
|
712
|
+
const ERRNO = { SUCCESS: 0, BADF: 8, FAULT: 21, NOSYS: 52, SPIPE: 70 };
|
|
713
|
+
// The limits of the WebAssembly host: the bytes of a module, the bytes of
|
|
714
|
+
// a read of the memory, the output of a program that the host keeps for
|
|
715
|
+
// each call, and the modules and instances that it keeps at one time.
|
|
716
|
+
const WASM_MAX_MODULE = 64 << 20, WASM_MAX_READ = 16 << 20, WASM_MAX_OUTPUT = 1 << 20, WASM_MAX_HANDLES = 1024;
|
|
717
|
+
const isOffset = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
718
|
+
|
|
719
|
+
class WasiExit {
|
|
720
|
+
constructor(code) { this.code = code; }
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
function b64(bytes) {
|
|
724
|
+
let s = '';
|
|
725
|
+
for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
|
|
726
|
+
return btoa(s);
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
function unb64(text) {
|
|
730
|
+
const s = atob(text), b = new Uint8Array(s.length);
|
|
731
|
+
for (let i = 0; i < s.length; i++) b[i] = s.charCodeAt(i);
|
|
732
|
+
return b;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
// The types of the exported functions of a module: name -> {params,
|
|
736
|
+
// results}, with the value types as bytes (0x7f i32, 0x7e i64, 0x7d f32,
|
|
737
|
+
// 0x7c f64). The JavaScript API does not give them, and an i64 needs a
|
|
738
|
+
// BigInt.
|
|
739
|
+
export function wasmSignatures(bytes) {
|
|
740
|
+
let at = 8;
|
|
741
|
+
const u = () => { let r = 0, s = 0, b; do { b = bytes[at++]; r |= (b & 0x7f) << s; s += 7; } while (b & 0x80); return r >>> 0; };
|
|
742
|
+
const name = () => { const n = u(); const t = new TextDecoder().decode(bytes.subarray(at, at + n)); at += n; return t; };
|
|
743
|
+
const limits = () => { const f = u(); u(); if (f & 1) u(); };
|
|
744
|
+
const types = [], funcs = [], out = {};
|
|
745
|
+
let exports = [];
|
|
746
|
+
while (at < bytes.length) {
|
|
747
|
+
const id = bytes[at++], size = u(), end = at + size;
|
|
748
|
+
if (id === 1) {
|
|
749
|
+
for (let n = u(); n > 0; n--) {
|
|
750
|
+
if (bytes[at++] !== 0x60) return out; // not a plain function type
|
|
751
|
+
const params = [], results = [];
|
|
752
|
+
for (let k = u(); k > 0; k--) params.push(bytes[at++]);
|
|
753
|
+
for (let k = u(); k > 0; k--) results.push(bytes[at++]);
|
|
754
|
+
types.push({ params, results });
|
|
755
|
+
}
|
|
756
|
+
} else if (id === 2) {
|
|
757
|
+
for (let n = u(); n > 0; n--) {
|
|
758
|
+
name(); name();
|
|
759
|
+
const kind = bytes[at++];
|
|
760
|
+
if (kind === 0) funcs.push(u());
|
|
761
|
+
else if (kind === 1) { at++; limits(); }
|
|
762
|
+
else if (kind === 2) limits();
|
|
763
|
+
else if (kind === 3) at += 2;
|
|
764
|
+
else if (kind === 4) { at++; u(); }
|
|
765
|
+
}
|
|
766
|
+
} else if (id === 3) {
|
|
767
|
+
for (let n = u(); n > 0; n--) funcs.push(u());
|
|
768
|
+
} else if (id === 7) {
|
|
769
|
+
for (let n = u(); n > 0; n--) exports.push({ name: name(), kind: bytes[at++], index: u() });
|
|
770
|
+
}
|
|
771
|
+
at = end;
|
|
772
|
+
}
|
|
773
|
+
for (const e of exports) if (e.kind === 0 && types[funcs[e.index]]) out[e.name] = types[funcs[e.index]];
|
|
774
|
+
return out;
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
// WASI preview 1, the functions that a small program needs. The others
|
|
778
|
+
// give ENOSYS.
|
|
779
|
+
class Wasi {
|
|
780
|
+
constructor(args = [], env = []) {
|
|
781
|
+
this.args = args.map((a) => new TextEncoder().encode(`${a}\0`));
|
|
782
|
+
this.env = env.map(([k, v]) => new TextEncoder().encode(`${k}=${v}\0`));
|
|
783
|
+
this.out = []; this.err = [];
|
|
784
|
+
this.kept = 0; // the bytes of out and err
|
|
785
|
+
this.memory = null;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
take() {
|
|
789
|
+
const join = (parts) => { const n = parts.reduce((a, p) => a + p.length, 0), b = new Uint8Array(n); let o = 0; for (const p of parts) { b.set(p, o); o += p.length; } return b; };
|
|
790
|
+
const r = { stdout: b64(join(this.out)), stderr: b64(join(this.err)) };
|
|
791
|
+
this.out = []; this.err = []; this.kept = 0;
|
|
792
|
+
return r;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
imports(module) {
|
|
796
|
+
const dv = () => new DataView(this.memory.buffer), mem = () => new Uint8Array(this.memory.buffer);
|
|
797
|
+
const strings = (list, ptrs, buf) => { let o = buf; list.forEach((s, i) => { dv().setUint32(ptrs + i * 4, o, true); mem().set(s, o); o += s.length; }); return ERRNO.SUCCESS; };
|
|
798
|
+
const sizes = (list, count, size) => { dv().setUint32(count, list.length, true); dv().setUint32(size, list.reduce((a, s) => a + s.length, 0), true); return ERRNO.SUCCESS; };
|
|
799
|
+
const f = {
|
|
800
|
+
args_sizes_get: (c, s) => sizes(this.args, c, s),
|
|
801
|
+
args_get: (p, b) => strings(this.args, p, b),
|
|
802
|
+
environ_sizes_get: (c, s) => sizes(this.env, c, s),
|
|
803
|
+
environ_get: (p, b) => strings(this.env, p, b),
|
|
804
|
+
fd_write: (fd, iovs, n, written) => {
|
|
805
|
+
if (fd !== 1 && fd !== 2) return ERRNO.BADF;
|
|
806
|
+
let total = 0;
|
|
807
|
+
for (let i = 0; i < n; i++) {
|
|
808
|
+
const ptr = dv().getUint32(iovs + i * 8, true), len = dv().getUint32(iovs + i * 8 + 4, true);
|
|
809
|
+
if (ptr + len > this.memory.buffer.byteLength) return ERRNO.FAULT;
|
|
810
|
+
// Past WASM_MAX_OUTPUT, the output of this call is dropped.
|
|
811
|
+
const keep = Math.min(len, WASM_MAX_OUTPUT - this.kept);
|
|
812
|
+
if (keep > 0) {
|
|
813
|
+
(fd === 1 ? this.out : this.err).push(mem().slice(ptr, ptr + keep));
|
|
814
|
+
this.kept += keep;
|
|
815
|
+
}
|
|
816
|
+
total += len;
|
|
817
|
+
}
|
|
818
|
+
dv().setUint32(written, total, true);
|
|
819
|
+
return ERRNO.SUCCESS;
|
|
820
|
+
},
|
|
821
|
+
fd_read: (fd, iovs, n, read) => { if (fd !== 0) return ERRNO.BADF; dv().setUint32(read, 0, true); return ERRNO.SUCCESS; },
|
|
822
|
+
fd_close: (fd) => (fd <= 2 ? ERRNO.SUCCESS : ERRNO.BADF),
|
|
823
|
+
fd_seek: () => ERRNO.SPIPE,
|
|
824
|
+
fd_fdstat_get: (fd, buf) => {
|
|
825
|
+
if (fd > 2) return ERRNO.BADF;
|
|
826
|
+
dv().setUint8(buf, 2); // a character device
|
|
827
|
+
dv().setUint16(buf + 2, 0, true);
|
|
828
|
+
dv().setBigUint64(buf + 8, 0xffffffffffffffffn, true);
|
|
829
|
+
dv().setBigUint64(buf + 16, 0xffffffffffffffffn, true);
|
|
830
|
+
return ERRNO.SUCCESS;
|
|
831
|
+
},
|
|
832
|
+
fd_fdstat_set_flags: () => ERRNO.SUCCESS,
|
|
833
|
+
fd_prestat_get: () => ERRNO.BADF,
|
|
834
|
+
fd_prestat_dir_name: () => ERRNO.BADF,
|
|
835
|
+
proc_exit: (code) => { throw new WasiExit(code); },
|
|
836
|
+
clock_res_get: (id, ptr) => { dv().setBigUint64(ptr, 1000n, true); return ERRNO.SUCCESS; },
|
|
837
|
+
clock_time_get: (id, precision, ptr) => {
|
|
838
|
+
const ns = id === 0 ? BigInt(Date.now()) * 1000000n : BigInt(Math.round(performance.now() * 1e6));
|
|
839
|
+
dv().setBigUint64(ptr, ns, true);
|
|
840
|
+
return ERRNO.SUCCESS;
|
|
841
|
+
},
|
|
842
|
+
random_get: (buf, len) => { for (let o = 0; o < len; o += 65536) crypto.getRandomValues(mem().subarray(buf + o, buf + Math.min(len, o + 65536))); return ERRNO.SUCCESS; },
|
|
843
|
+
sched_yield: () => ERRNO.SUCCESS,
|
|
844
|
+
};
|
|
845
|
+
const imports = {};
|
|
846
|
+
for (const i of WebAssembly.Module.imports(module)) {
|
|
847
|
+
if (i.module !== 'wasi_snapshot_preview1' && i.module !== 'wasi_unstable') {
|
|
848
|
+
throw new Error(`the module imports ${i.module}.${i.name}: only WASI is supported`);
|
|
849
|
+
}
|
|
850
|
+
(imports[i.module] ??= {})[i.name] = f[i.name] ?? (() => ERRNO.NOSYS);
|
|
851
|
+
}
|
|
852
|
+
return imports;
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
export class WasmHost {
|
|
857
|
+
constructor() {
|
|
858
|
+
this.modules = new Map();
|
|
859
|
+
this.instances = new Map();
|
|
860
|
+
this.next = 1;
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
async op(op, q) {
|
|
864
|
+
switch (op) {
|
|
865
|
+
case 'release':
|
|
866
|
+
this.modules.delete(q.id);
|
|
867
|
+
this.instances.delete(q.id);
|
|
868
|
+
return { ok: true };
|
|
869
|
+
case 'compile': {
|
|
870
|
+
if (this.modules.size + this.instances.size >= WASM_MAX_HANDLES) return { error: 'too many modules and instances' };
|
|
871
|
+
const bytes = unb64(q.bytes);
|
|
872
|
+
if (bytes.length > WASM_MAX_MODULE) return { error: 'the module is too large' };
|
|
873
|
+
const module = await WebAssembly.compile(bytes);
|
|
874
|
+
const id = `m${this.next++}`;
|
|
875
|
+
this.modules.set(id, { module, sig: wasmSignatures(bytes) });
|
|
876
|
+
return { ok: id };
|
|
877
|
+
}
|
|
878
|
+
case 'instantiate': {
|
|
879
|
+
const m = this.modules.get(q.module);
|
|
880
|
+
if (!m) return { error: 'unknown module' };
|
|
881
|
+
if (this.modules.size + this.instances.size >= WASM_MAX_HANDLES) return { error: 'too many modules and instances' };
|
|
882
|
+
const wasi = new Wasi(q.args, q.env);
|
|
883
|
+
const instance = await WebAssembly.instantiate(m.module, wasi.imports(m.module));
|
|
884
|
+
wasi.memory = Object.values(instance.exports).find((e) => e instanceof WebAssembly.Memory) ?? null;
|
|
885
|
+
const id = `i${this.next++}`;
|
|
886
|
+
this.instances.set(id, { instance, wasi, sig: m.sig });
|
|
887
|
+
return { ok: id };
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
const inst = this.instances.get(q.instance);
|
|
891
|
+
if (!inst) return { error: 'unknown instance' };
|
|
892
|
+
const exports = inst.instance.exports, memory = inst.wasi.memory;
|
|
893
|
+
switch (op) {
|
|
894
|
+
case 'exists': return { ok: typeof exports[q.name] === 'function' };
|
|
895
|
+
case 'call': {
|
|
896
|
+
const fn = exports[q.name];
|
|
897
|
+
if (typeof fn !== 'function') return { error: 'not_found' };
|
|
898
|
+
const type = inst.sig[q.name];
|
|
899
|
+
const args = q.args.map((a, i) => toWasm(a, type?.params[i]));
|
|
900
|
+
try {
|
|
901
|
+
const r = fn(...args);
|
|
902
|
+
const list = r === undefined ? [] : type && type.results.length > 1 ? [...r] : [r];
|
|
903
|
+
return { ok: list.map(fromWasm), ...inst.wasi.take() };
|
|
904
|
+
} catch (e) {
|
|
905
|
+
if (e instanceof WasiExit) return { exit: e.code, ...inst.wasi.take() };
|
|
906
|
+
if (e instanceof WebAssembly.RuntimeError) return { trap: e.message, ...inst.wasi.take() };
|
|
907
|
+
return { error: String(e?.message ?? e), ...inst.wasi.take() };
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
case 'memory_size': return memory ? { ok: memory.buffer.byteLength } : { error: 'not_found' };
|
|
911
|
+
case 'memory_grow':
|
|
912
|
+
if (!memory) return { error: 'not_found' };
|
|
913
|
+
if (!isOffset(q.pages)) return { error: 'out_of_bounds' };
|
|
914
|
+
try { return { ok: memory.grow(q.pages) }; } catch { return { error: 'out_of_bounds' }; }
|
|
915
|
+
case 'read':
|
|
916
|
+
if (!memory) return { error: 'not_found' };
|
|
917
|
+
if (!isOffset(q.offset) || !isOffset(q.length) || q.length > WASM_MAX_READ) return { error: 'out_of_bounds' };
|
|
918
|
+
if (q.offset + q.length > memory.buffer.byteLength) return { error: 'out_of_bounds' };
|
|
919
|
+
return { ok: b64(new Uint8Array(memory.buffer, q.offset, q.length)) };
|
|
920
|
+
case 'write': {
|
|
921
|
+
if (!memory) return { error: 'not_found' };
|
|
922
|
+
const data = unb64(q.data);
|
|
923
|
+
if (!isOffset(q.offset) || q.offset + data.length > memory.buffer.byteLength) return { error: 'out_of_bounds' };
|
|
924
|
+
new Uint8Array(memory.buffer).set(data, q.offset);
|
|
925
|
+
return { ok: true };
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
return { error: `unknown operation ${op}` };
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
// A value of Erlang (JSON) for a parameter of type t, and back.
|
|
933
|
+
function toWasm(v, t) {
|
|
934
|
+
const n = v === 'nan' ? NaN : v === 'infinity' ? Infinity : v === '-infinity' ? -Infinity : v?.i !== undefined ? v.i : v;
|
|
935
|
+
return t === 0x7e ? BigInt(n) : Number(n);
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
function fromWasm(v) {
|
|
939
|
+
if (typeof v === 'bigint') return v >= -9007199254740991n && v <= 9007199254740991n ? Number(v) : { i: v.toString() };
|
|
940
|
+
if (Number.isNaN(v)) return 'nan';
|
|
941
|
+
if (v === Infinity) return 'infinity';
|
|
942
|
+
if (v === -Infinity) return '-infinity';
|
|
943
|
+
return v;
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// A VM and its release. In a Worker (plain), the VM runs only in the handlers
|
|
947
|
+
// of open requests (serve). A Durable Object has one context for all its
|
|
948
|
+
// requests, and runs the VM all the time:
|
|
949
|
+
//
|
|
950
|
+
// import { DurableObject } from 'cloudflare:workers';
|
|
951
|
+
// import { Vm } from './worker.js';
|
|
952
|
+
// export class Beam extends DurableObject {
|
|
953
|
+
// constructor(ctx, env) { super(ctx, env); this.vm = new Vm(env, { plain: false, sql: ctx.storage.sql }); }
|
|
954
|
+
// fetch(request) { return this.vm.fetch(request); }
|
|
955
|
+
// }
|
|
956
|
+
// The key of SECRET_KEY_BASE in the store of secrets (autoVars).
|
|
957
|
+
const SECRET = 'beam:SECRET_KEY_BASE';
|
|
958
|
+
|
|
959
|
+
export class Vm {
|
|
960
|
+
// release and snapshot: the bytes of release.bin and snapshot.bin, for a
|
|
961
|
+
// VM that the global scope of a Worker restores (global.js).
|
|
962
|
+
// id: the id of the Durable Object (with sql).
|
|
963
|
+
// vars: more environment of the VM for this object only (the tenant, for
|
|
964
|
+
// example). They are not part of the key of the snapshot, so the object
|
|
965
|
+
// makes its snapshot at the boot point, and "go" gives them.
|
|
966
|
+
// files: a store of HostFiles (for example the Deno KV of deno.js):
|
|
967
|
+
// SQLite in the VM keeps its databases there, when the host does not run
|
|
968
|
+
// the SQL itself (sql, or a D1 binding).
|
|
969
|
+
// secrets: a store of the secrets that the VM makes itself ({get(name),
|
|
970
|
+
// put(name, value)}: the storage of a Durable Object, the Deno KV of
|
|
971
|
+
// deno.js), and host: the host of the first request (autoVars).
|
|
972
|
+
constructor(env, { plain = true, sql = null, id = null, release = null, snapshot = null, vars = {}, files = null,
|
|
973
|
+
secrets = null, host = null } = {}) {
|
|
974
|
+
this.vars = vars;
|
|
975
|
+
this.secrets = secrets;
|
|
976
|
+
this.host = host;
|
|
977
|
+
this.hostFiles = files && new HostFiles(files, { debug: env.BEAM_SQLITE_DEBUG === '1' });
|
|
978
|
+
this.given = release && { release, snapshot };
|
|
979
|
+
this.handles = new Map(); // id -> setTimeout handle: timers after adopt()
|
|
980
|
+
this.id = id;
|
|
981
|
+
this.sql = sql; // ctx.storage.sql of a Durable Object (Ecto SQLite)
|
|
982
|
+
this.tcps = new Map(); // id -> {send, close, h}: a TCP socket of wasm_tcp
|
|
983
|
+
this.listeners = new Map(); // port -> the id of its listener (wasm_tcp)
|
|
984
|
+
this.nextId = 1;
|
|
985
|
+
this.plain = plain;
|
|
986
|
+
this.handlers = []; // the open requests (plain)
|
|
987
|
+
this.jobs = []; // the wake-ups of the threads (plain)
|
|
988
|
+
this.timers = new Map(); // id -> {at, f}: the timers of the threads (plain)
|
|
989
|
+
this.nextTimer = 1;
|
|
990
|
+
this.env = env;
|
|
991
|
+
this.waitListen = new Map(); // port -> the resolve functions of listening()
|
|
992
|
+
// The static files of the release (appFiles of app-com.js), when the
|
|
993
|
+
// release is loaded: a request for one of them needs no VM.
|
|
994
|
+
this.statics = new Promise((resolve) => { this.gotStatics = resolve; });
|
|
995
|
+
this.ready = this.boot(env);
|
|
996
|
+
this.ready.catch(() => this.gotStatics(null));
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
// The variables that a Phoenix app needs, when the host does not give
|
|
1000
|
+
// them, so that it runs with no setup:
|
|
1001
|
+
// - SECRET_KEY_BASE: a random key, made one time and kept in the store
|
|
1002
|
+
// of secrets, so that all the VMs of the app share it. With no store,
|
|
1003
|
+
// each VM makes its own key (the sessions of one VM only).
|
|
1004
|
+
// - PHX_HOST: the host name of the first request (with no port: the app
|
|
1005
|
+
// gets the origin of PHX_HOST on port 443, see appOrigin).
|
|
1006
|
+
// They go to the VM as vars: not in the key of a snapshot.
|
|
1007
|
+
async autoVars(env, meta) {
|
|
1008
|
+
if (meta.env?.PHX_SERVER !== 'true') return;
|
|
1009
|
+
if (!env.SECRET_KEY_BASE && !this.vars.SECRET_KEY_BASE) {
|
|
1010
|
+
let key = await this.secrets?.get(SECRET);
|
|
1011
|
+
if (!key) {
|
|
1012
|
+
const made = btoa(String.fromCharCode(...crypto.getRandomValues(new Uint8Array(48))));
|
|
1013
|
+
key = this.secrets ? (await this.secrets.put(SECRET, made)) ?? made : made;
|
|
1014
|
+
if (!this.secrets) console.log('beam: SECRET_KEY_BASE is not set, and the host has no store: a new key for this VM');
|
|
1015
|
+
}
|
|
1016
|
+
this.vars.SECRET_KEY_BASE = key;
|
|
1017
|
+
}
|
|
1018
|
+
if (!env.PHX_HOST && !this.vars.PHX_HOST && this.host) this.vars.PHX_HOST = this.host;
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
async boot(env) {
|
|
1022
|
+
const t0 = Date.now();
|
|
1023
|
+
const [release, bundled] = this.given
|
|
1024
|
+
? [this.given.release, this.given.snapshot]
|
|
1025
|
+
: await Promise.all([loadRelease(env), loadSnapshot(env)]);
|
|
1026
|
+
this.gotStatics(release.statics ?? null);
|
|
1027
|
+
// The vars of the host choose a snapshot at the boot point (below);
|
|
1028
|
+
// the vars of autoVars do not.
|
|
1029
|
+
const ownVars = Object.keys(this.vars).length;
|
|
1030
|
+
await this.autoVars(env, releaseMeta(release));
|
|
1031
|
+
// A snapshot of the build (snapshot.bin), else one that a Worker made
|
|
1032
|
+
// (BEAM_SNAPSHOT = "off" turns them off).
|
|
1033
|
+
let snapBytes = bundled, key = null;
|
|
1034
|
+
if (!snapBytes && env.BEAM_SNAPSHOT !== 'off') {
|
|
1035
|
+
// A Durable Object with Ecto SQLite (sql of .release.json): the
|
|
1036
|
+
// snapshot is made at the boot point (wasm_host_server), before the
|
|
1037
|
+
// program starts and runs its migrations. So all the objects (the
|
|
1038
|
+
// tenants) share it, and each one runs the program on its own storage.
|
|
1039
|
+
// Tenants (BEAM_TENANTS) share the snapshot too, so it is made at the
|
|
1040
|
+
// boot point, before the program has the state of one tenant.
|
|
1041
|
+
const meta = releaseMeta(release);
|
|
1042
|
+
const atBoot = !this.plain && (this.sql || this.hostFiles)
|
|
1043
|
+
&& ((meta.sql ?? true) || !!env.BEAM_PERSIST || !!env.BEAM_TENANTS || ownVars > 0);
|
|
1044
|
+
key = await snapshotKey(env, meta, this.plain ? 'worker' : atBoot ? 'durable boot-point' : 'durable');
|
|
1045
|
+
snapBytes = await snapshots.get(env, key);
|
|
1046
|
+
if (!snapBytes && !snapshots.unavailable && atBoot) this.bootKey = key;
|
|
1047
|
+
}
|
|
1048
|
+
const snap = snapBytes && parseSnapshot(snapBytes);
|
|
1049
|
+
const nifs = await nifModules();
|
|
1050
|
+
// restored: this VM comes from a snapshot (the page shows it).
|
|
1051
|
+
this.restored = !!snap;
|
|
1052
|
+
this.bootPointSnap = !!snap?.boot_point;
|
|
1053
|
+
// No snapshot yet: this VM makes it, before its first request.
|
|
1054
|
+
this.makeKey = !snap && !snapshots.unavailable && !this.bootKey && key;
|
|
1055
|
+
this.release = release;
|
|
1056
|
+
const t1 = Date.now();
|
|
1057
|
+
return new Promise((resolve, reject) => {
|
|
1058
|
+
const snapKB = snapBytes ? snapBytes.byteLength >> 10 : 0;
|
|
1059
|
+
this.onready = () => { console.log(`beam: ready in ${Date.now() - t0} ms (release ${release.byteLength >> 10} KB${snap ? `, snapshot ${snapKB} KB` : ''} in ${t1 - t0} ms), ${this.memory()}`); resolve(); };
|
|
1060
|
+
createBeam({
|
|
1061
|
+
noInitialRun: !!snap,
|
|
1062
|
+
onRuntimeInitialized: snap ? () => {
|
|
1063
|
+
try {
|
|
1064
|
+
this.listeners = new Map(Object.entries(snap.listeners ?? {}).map(([p, id]) => [Number(p), id]));
|
|
1065
|
+
restore(this.beam, this.exports, snap);
|
|
1066
|
+
// In the global scope (the bytes given), the first request
|
|
1067
|
+
// starts the threads. A snapshot of the boot point then goes on
|
|
1068
|
+
// with the boot (go).
|
|
1069
|
+
const start = () => {
|
|
1070
|
+
resume(this.beam, this.exports);
|
|
1071
|
+
if (snap.boot_point) this.go();
|
|
1072
|
+
};
|
|
1073
|
+
if (this.given) this.resume = start;
|
|
1074
|
+
else start();
|
|
1075
|
+
// The copy is in the memory of the VM now: free the bytes of the
|
|
1076
|
+
// snapshot (an isolate has 128 MB). The closures of the VM keep
|
|
1077
|
+
// this scope, so snapBytes must not keep them either.
|
|
1078
|
+
snap.pagesData = null;
|
|
1079
|
+
snap.fs = null;
|
|
1080
|
+
snapBytes = null;
|
|
1081
|
+
this.onready();
|
|
1082
|
+
} catch (e) { reject(e); }
|
|
1083
|
+
} : undefined,
|
|
1084
|
+
// The boot arguments are set in preRun, after the release is unpacked.
|
|
1085
|
+
arguments: [],
|
|
1086
|
+
// A plain Worker runs the jobs and timers of the threads in its
|
|
1087
|
+
// open requests. After adopt() (a Durable Object), they run on a
|
|
1088
|
+
// MessageChannel and on setTimeout.
|
|
1089
|
+
jspiSchedule: this.plain ? {
|
|
1090
|
+
later: (f) => {
|
|
1091
|
+
if (!this.plain) return this.post(f);
|
|
1092
|
+
this.jobs.push(f);
|
|
1093
|
+
this.handlers.at(-1)?.wake?.();
|
|
1094
|
+
},
|
|
1095
|
+
// 1 ms at least: the clock of a Worker moves only by the delay of
|
|
1096
|
+
// a timer (see jspiTimer in jspi_lib.js).
|
|
1097
|
+
timer: (f, ms) => {
|
|
1098
|
+
const id = this.nextTimer++;
|
|
1099
|
+
if (!this.plain) {
|
|
1100
|
+
this.handles.set(id, setTimeout(() => { this.handles.delete(id); f(); }, Math.max(1, ms)));
|
|
1101
|
+
return id;
|
|
1102
|
+
}
|
|
1103
|
+
this.timers.set(id, { at: Date.now() + Math.max(1, ms), f });
|
|
1104
|
+
this.handlers.at(-1)?.wake?.();
|
|
1105
|
+
return id;
|
|
1106
|
+
},
|
|
1107
|
+
clear: (id) => {
|
|
1108
|
+
this.timers.delete(id);
|
|
1109
|
+
clearTimeout(this.handles.get(id));
|
|
1110
|
+
this.handles.delete(id);
|
|
1111
|
+
},
|
|
1112
|
+
} : undefined,
|
|
1113
|
+
preRun: [(m) => {
|
|
1114
|
+
this.beam = m;
|
|
1115
|
+
// .release.json: the name, the version, the boot arguments and the
|
|
1116
|
+
// environment of the release (beam_com_wasm).
|
|
1117
|
+
const { name, vsn, args, env: relEnv } = unpack(m.FS, release);
|
|
1118
|
+
// BEAM_PERSIST: directories in the SQLite storage of the object.
|
|
1119
|
+
if (this.sql && env.BEAM_PERSIST) {
|
|
1120
|
+
this.persist = new Persist(this.sql, env.BEAM_PERSIST.split(','));
|
|
1121
|
+
}
|
|
1122
|
+
// The files that the boot of the snapshot wrote.
|
|
1123
|
+
for (const [p, b64] of Object.entries(snap?.fs.files ?? {})) {
|
|
1124
|
+
if (this.persist?.under(p)) continue;
|
|
1125
|
+
m.FS.mkdirTree(p.slice(0, p.lastIndexOf('/')));
|
|
1126
|
+
m.FS.writeFile(p, Uint8Array.from(atob(b64), (c) => c.charCodeAt(0)));
|
|
1127
|
+
}
|
|
1128
|
+
if (this.persist) {
|
|
1129
|
+
this.persist.load(m.FS);
|
|
1130
|
+
this.persist.attach(m.FS);
|
|
1131
|
+
}
|
|
1132
|
+
// -c false (no time correction) for a snapshot: the monotonic time
|
|
1133
|
+
// then follows the system time, which goes on after a restore (the
|
|
1134
|
+
// OS monotonic time of a new instance starts again at 0).
|
|
1135
|
+
// BEAM_ERL_FLAGS: more flags of the emulator, as beam takes them
|
|
1136
|
+
// (-Mea min: no allocators of ERTS, only malloc; the VM of
|
|
1137
|
+
// Livebook starts with 40 MB, not 70 MB, but :erlang.memory/0 is
|
|
1138
|
+
// not supported).
|
|
1139
|
+
const flags = (env.BEAM_ERL_FLAGS ?? '').split(/\s+/).filter(Boolean);
|
|
1140
|
+
m.arguments.push('-S', '1', '-SDcpu', '1', '-A', '0', ...flags, ...(this.makeKey || this.bootKey ? ['-c', 'false'] : []), '--',
|
|
1141
|
+
'-root', '/app', '-bindir', '/app/bin', '-progname', 'erl', '--',
|
|
1142
|
+
'-home', '/', ...args, '-noshell');
|
|
1143
|
+
// Distributed Erlang over wasm_tcp, with no epmd (all nodes on
|
|
1144
|
+
// DIST_PORT): wasm_host_server starts it after the boot (DIST_NAME,
|
|
1145
|
+
// DIST_COOKIE, DIST_LISTEN, DIST_CONNECT), when wasm_tcp works.
|
|
1146
|
+
if (env.DIST_NAME) {
|
|
1147
|
+
m.arguments.push('-proto_dist', 'wasm_tcp', '-erl_epmd_port', env.DIST_PORT ?? '4370', '-start_epmd', 'false');
|
|
1148
|
+
}
|
|
1149
|
+
// The text bindings of the Worker are the environment of the release
|
|
1150
|
+
// (SECRET_KEY_BASE, PHX_HOST, DATABASE_URL, ...).
|
|
1151
|
+
const vars = { ...Object.fromEntries(Object.entries(env).filter(([, v]) => typeof v === 'string')), ...this.vars };
|
|
1152
|
+
// The environment that "go" gives to a VM of a boot point snapshot.
|
|
1153
|
+
this.envVars = { ...relEnv, ...vars };
|
|
1154
|
+
Object.assign(m.ENV, {
|
|
1155
|
+
ROOTDIR: '/app', BINDIR: '/app/bin', EMU: 'beam', PROGNAME: 'erl', HOME: '/',
|
|
1156
|
+
RELEASE_ROOT: '/app', RELEASE_NAME: name, RELEASE_VSN: vsn, RELEASE_MODE: 'interactive',
|
|
1157
|
+
RELEASE_TMP: '/app/tmp', RELEASE_SYS_CONFIG: '/app/tmp/run.runtime', RELEASE_PROG: name,
|
|
1158
|
+
// The host of the runtime, for the app: cloudflare, or the value
|
|
1159
|
+
// of deno.js (deno, deno-deploy) or browser.js (browser).
|
|
1160
|
+
WASM_HOST: '1', BEAM_HOST: 'cloudflare',
|
|
1161
|
+
}, relEnv, vars, { WASM_HOST_SQL: this.hostSql() }, this.bootKey ? { WASM_HOST_BOOT_POINT: 'wait' } : {});
|
|
1162
|
+
m.beamHost.onsend = (bytes) => this.onsend(bytes);
|
|
1163
|
+
if (this.hostFiles) m.beamHost.files = this.hostFiles;
|
|
1164
|
+
}],
|
|
1165
|
+
print: (s) => console.log(s),
|
|
1166
|
+
printErr: (s) => console.log(s),
|
|
1167
|
+
nifModule: (file) => nifModule(nifs, file),
|
|
1168
|
+
// Workers compile no WebAssembly at run time: use the imported module.
|
|
1169
|
+
instantiateWasm: (imports, done) => {
|
|
1170
|
+
WebAssembly.instantiate(wasm, imports).then((instance) => { this.exports = instance.exports; done(instance); });
|
|
1171
|
+
return {};
|
|
1172
|
+
},
|
|
1173
|
+
onExit: (code) => reject(new Error(`beam exited with status ${code}`)),
|
|
1174
|
+
}).catch(reject);
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
// The first request of a VM with no snapshot: all the threads return
|
|
1179
|
+
// (erts_wasm_hibernate), the memory is copied, the threads go on, and the
|
|
1180
|
+
// copy goes to the store in the background. About 1 ms of the VM, and the
|
|
1181
|
+
// time of the copy.
|
|
1182
|
+
async makeSnapshot() {
|
|
1183
|
+
const key = this.makeKey;
|
|
1184
|
+
this.makeKey = null;
|
|
1185
|
+
// Not before the app is up (its server listens), and not while the host
|
|
1186
|
+
// has I/O of the VM (a SQL call, a socket): that I/O would not be in
|
|
1187
|
+
// the snapshot, and a restored VM would wait for it forever.
|
|
1188
|
+
const port = Number(this.env.PORT ?? 4000);
|
|
1189
|
+
await Promise.race([this.listening(port), new Promise((r) => setTimeout(r, 10000))]);
|
|
1190
|
+
const bytes = await this.snapshot(false);
|
|
1191
|
+
// No second try: after this, the VM has served requests, and the
|
|
1192
|
+
// snapshot could hold their state. The next new VM tries again.
|
|
1193
|
+
if (bytes && bytes !== 'busy') this.store(key, bytes);
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
// The boot point (wasm_host_server): the snapshot of a VM that loaded the
|
|
1197
|
+
// modules of its boot and did not start the program; then the boot goes
|
|
1198
|
+
// on (go).
|
|
1199
|
+
async bootPoint() {
|
|
1200
|
+
const key = this.bootKey;
|
|
1201
|
+
this.bootKey = null;
|
|
1202
|
+
if (key) {
|
|
1203
|
+
const bytes = await this.snapshot(true);
|
|
1204
|
+
if (bytes && bytes !== 'busy') this.store(key, bytes);
|
|
1205
|
+
else console.log(`beam: no snapshot at the boot point (${bytes})`);
|
|
1206
|
+
}
|
|
1207
|
+
this.go();
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
go() {
|
|
1211
|
+
this.event({ t: 'go' }, new TextEncoder().encode(JSON.stringify({ ...this.envVars, WASM_HOST_SQL: this.hostSql() })));
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
store(key, bytes) {
|
|
1215
|
+
console.log(`beam: snapshot ${bytes.length >> 10} KB (${key.slice(0, 12)})`);
|
|
1216
|
+
const put = snapshots.put(this.env, key, bytes).catch((e) => console.log(`beam: snapshot not stored: ${e.message}`));
|
|
1217
|
+
this.waitUntil?.(put);
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
// All the threads return (erts_wasm_hibernate), the memory is copied, and
|
|
1221
|
+
// the threads go on: about 1 ms of the VM, and the time of the copy.
|
|
1222
|
+
// 'busy': not a quiet moment (I/O of the host); null: no snapshot.
|
|
1223
|
+
async snapshot(bootPoint) {
|
|
1224
|
+
const x = this.exports;
|
|
1225
|
+
const tick = () => new Promise((r) => setTimeout(r, 1));
|
|
1226
|
+
const busy = () => this.sqlPending > 0 || this.tcps.size > 0;
|
|
1227
|
+
// Data in a pipe (an event of the host that Erlang did not take yet,
|
|
1228
|
+
// or a wake-up of ERTS) would not be in the snapshot either.
|
|
1229
|
+
const unread = () => this.beam.FS.streams.some((st) => st?.node?.pipe?.buckets.some((b) => b.offset > b.roffset));
|
|
1230
|
+
for (let attempt = 0; ; attempt++) {
|
|
1231
|
+
for (let i = 0; busy() && i < 2000; i++) await tick();
|
|
1232
|
+
if (busy()) return 'busy';
|
|
1233
|
+
x.erts_wasm_hibernate();
|
|
1234
|
+
// 1 ms steps (5 s at most): on Cloudflare, setTimeout(0) does not wait
|
|
1235
|
+
// and does not move the clock, so 5000 steps of 0 ms ended at once.
|
|
1236
|
+
for (let i = 0; x.jspi_live_threads() > 0; i++) {
|
|
1237
|
+
if (i > 5000) {
|
|
1238
|
+
x.jspi_report_live();
|
|
1239
|
+
x.erts_wasm_resume();
|
|
1240
|
+
return null;
|
|
1241
|
+
}
|
|
1242
|
+
await new Promise((r) => setTimeout(r, 1));
|
|
1243
|
+
}
|
|
1244
|
+
if (!busy() && !unread()) break;
|
|
1245
|
+
// Not a quiet moment: go on, and try again.
|
|
1246
|
+
x.erts_wasm_resume();
|
|
1247
|
+
if (attempt >= 20) return 'busy';
|
|
1248
|
+
for (let i = 0; i < 5; i++) await tick();
|
|
1249
|
+
}
|
|
1250
|
+
try {
|
|
1251
|
+
return capture(this.beam, this.release, this.listeners, bootPoint, (p) => !!this.persist?.under(p));
|
|
1252
|
+
} finally {
|
|
1253
|
+
x.erts_wasm_resume();
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// "1" when the host runs the SQL of Ecto SQLite (the storage of a Durable
|
|
1258
|
+
// Object, or D1), else "0": SQLite in the VM (wasm_host_sqlite).
|
|
1259
|
+
hostSql() {
|
|
1260
|
+
return this.sql || this.env[this.env.BEAM_D1 ?? 'DB'] ? '1' : '0';
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
// A VM that the global scope restored (plain: its jobs ran between
|
|
1264
|
+
// microtasks) becomes the VM of a Durable Object: its jobs run on a
|
|
1265
|
+
// MessageChannel, and its timers on setTimeout (durable-global.js).
|
|
1266
|
+
adopt({ sql = null, id = null, vars = {} } = {}) {
|
|
1267
|
+
this.sql = sql;
|
|
1268
|
+
this.id = id;
|
|
1269
|
+
this.vars = vars;
|
|
1270
|
+
this.envVars = { ...this.envVars, ...vars };
|
|
1271
|
+
this.plain = false;
|
|
1272
|
+
for (const f of this.jobs.splice(0)) this.post(f);
|
|
1273
|
+
for (const [tid, t] of this.timers) {
|
|
1274
|
+
this.handles.set(tid, setTimeout(() => { this.handles.delete(tid); t.f(); }, Math.max(1, t.at - Date.now())));
|
|
1275
|
+
}
|
|
1276
|
+
this.timers.clear();
|
|
1277
|
+
return this;
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
// A job after the pending I/O and timers (a macrotask), as jspiLater.
|
|
1281
|
+
post(f) {
|
|
1282
|
+
if (!this.queue) {
|
|
1283
|
+
const ch = new MessageChannel();
|
|
1284
|
+
const fns = [];
|
|
1285
|
+
ch.port1.onmessage = () => fns.shift()?.();
|
|
1286
|
+
// Both ports: a port that only its handler holds can be collected.
|
|
1287
|
+
this.queue = { ch, fns };
|
|
1288
|
+
}
|
|
1289
|
+
this.queue.fns.push(f);
|
|
1290
|
+
this.queue.ch.port2.postMessage(0);
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1293
|
+
// In the global scope of a Worker, after a restore (global.js): the
|
|
1294
|
+
// threads start, and a GET request of path goes to the app. So V8
|
|
1295
|
+
// compiles the functions of a request here, not in the first request
|
|
1296
|
+
// (the global scope has its own time limit). No timers here: the jobs
|
|
1297
|
+
// of the threads run between microtasks. The request must not use I/O
|
|
1298
|
+
// of the host (SQL, sockets).
|
|
1299
|
+
//
|
|
1300
|
+
// The reseed of OpenSSL costs about 30 ms of CPU at its first run in an
|
|
1301
|
+
// isolate. So the warm-up also reseeds, with zero bytes: the global
|
|
1302
|
+
// scope has no random values, and the host gives zeros to the VM while
|
|
1303
|
+
// it warms up. The random bytes of the first request then reseed OpenSSL
|
|
1304
|
+
// again, before that request (seed()). A value that the warm-up makes
|
|
1305
|
+
// is the same in each isolate, as a value of the snapshot is.
|
|
1306
|
+
async warm(path) {
|
|
1307
|
+
if (this.bootPointSnap) return console.log('beam: no warm-up: the program starts at the first request');
|
|
1308
|
+
this.resume = () => seed(this.beam);
|
|
1309
|
+
const random = crypto.getRandomValues;
|
|
1310
|
+
crypto.getRandomValues = (v) => v.fill(0);
|
|
1311
|
+
try {
|
|
1312
|
+
this.exports.erts_wasm_resume();
|
|
1313
|
+
this.event({ t: 'restored' }, new Uint8Array(48));
|
|
1314
|
+
const url = new URL(path, 'http://localhost');
|
|
1315
|
+
let done = false;
|
|
1316
|
+
this.bridge(new Request(url), url, false, undefined, () => {})
|
|
1317
|
+
.then((r) => r.arrayBuffer()).finally(() => { done = true; });
|
|
1318
|
+
const idle = { jobs: [] };
|
|
1319
|
+
for (let i = 0; !done && i < 100000; i++) {
|
|
1320
|
+
this.runJobs(idle);
|
|
1321
|
+
await null;
|
|
1322
|
+
}
|
|
1323
|
+
if (!done) console.log(`beam: warm ${path}: no response`);
|
|
1324
|
+
} finally {
|
|
1325
|
+
crypto.getRandomValues = random;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
memory() {
|
|
1330
|
+
return `memory ${this.beam.HEAPU8.length >> 20} MB`;
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
event(header, body) {
|
|
1334
|
+
const h = new TextEncoder().encode(JSON.stringify(header) + '\n');
|
|
1335
|
+
if (!body || !body.byteLength) return this.beam.beamHost.push(h);
|
|
1336
|
+
const b = new Uint8Array(h.length + body.byteLength);
|
|
1337
|
+
b.set(h);
|
|
1338
|
+
b.set(new Uint8Array(body), h.length);
|
|
1339
|
+
this.beam.beamHost.push(b);
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
// ctx: the context of a request to a plain Worker (none in a Durable Object).
|
|
1343
|
+
async fetch(request, ctx) {
|
|
1344
|
+
if (ctx) this.waitUntil = (p) => ctx.waitUntil(p);
|
|
1345
|
+
let finished = () => {};
|
|
1346
|
+
const h = this.plain ? this.serve(ctx, new Promise((r) => { finished = r; })) : undefined;
|
|
1347
|
+
try {
|
|
1348
|
+
return await this.request(request, h, finished);
|
|
1349
|
+
} catch (e) {
|
|
1350
|
+
finished();
|
|
1351
|
+
throw e;
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
async request(request, h, finished) {
|
|
1356
|
+
const statics = await this.statics;
|
|
1357
|
+
const file = statics && staticResponse(statics, request);
|
|
1358
|
+
if (file) {
|
|
1359
|
+
finished();
|
|
1360
|
+
return file;
|
|
1361
|
+
}
|
|
1362
|
+
await this.ready;
|
|
1363
|
+
if (this.resume) {
|
|
1364
|
+
const f = this.resume;
|
|
1365
|
+
this.resume = null;
|
|
1366
|
+
f();
|
|
1367
|
+
}
|
|
1368
|
+
// The first request makes the snapshot, and the other requests wait for
|
|
1369
|
+
// it, so that the snapshot has the state of no request.
|
|
1370
|
+
if (this.makeKey) this.snapping = this.makeSnapshot().finally(() => { this.snapping = null; });
|
|
1371
|
+
if (this.snapping) await this.snapping;
|
|
1372
|
+
const url = new URL(request.url);
|
|
1373
|
+
const upgrade = request.headers.get('upgrade')?.toLowerCase() === 'websocket';
|
|
1374
|
+
const tcp = upgrade && url.pathname.match(/^\/\.tcp\/(\d+)$/);
|
|
1375
|
+
if (tcp) return this.tcpAccept(Number(tcp[1]), request, h, finished);
|
|
1376
|
+
return this.bridge(request, url, upgrade, h, finished);
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
// A plain Worker runs code only for an open request, and an I/O object (a
|
|
1380
|
+
// stream, a socket) or a timer belongs to the request that made it. So the
|
|
1381
|
+
// handler of each open request is the event loop of the VM: it runs the
|
|
1382
|
+
// wake-ups and timers of the threads (jspiSchedule), and the host calls
|
|
1383
|
+
// that make or use I/O objects (run), until the request has its response
|
|
1384
|
+
// and its sockets are closed. With no open request, the VM does not run.
|
|
1385
|
+
serve(ctx, finished) {
|
|
1386
|
+
const h = { jobs: [], wake: null, timeout: null, sockets: 0, done: false };
|
|
1387
|
+
this.handlers.push(h);
|
|
1388
|
+
finished.then(() => { h.done = true; h.wake?.(); });
|
|
1389
|
+
// A macrotask of this request, after its pending I/O (as jspiLater).
|
|
1390
|
+
const ch = new MessageChannel();
|
|
1391
|
+
const tick = () => new Promise((r) => { ch.port1.onmessage = r; ch.port2.postMessage(0); });
|
|
1392
|
+
ctx.waitUntil((async () => {
|
|
1393
|
+
try {
|
|
1394
|
+
for (;;) {
|
|
1395
|
+
this.runJobs(h);
|
|
1396
|
+
if (h.done && !h.sockets) break;
|
|
1397
|
+
if (h.jobs.length || this.jobs.length || this.nextTimerAt() <= Date.now()) {
|
|
1398
|
+
await tick();
|
|
1399
|
+
continue;
|
|
1400
|
+
}
|
|
1401
|
+
// A timer of this request at least each second: else the runtime can
|
|
1402
|
+
// take a request that waits only for the VM (in another request) as
|
|
1403
|
+
// hung, and cancel it with the work of the VM that waits in it.
|
|
1404
|
+
const at = Math.min(this.nextTimerAt(), Date.now() + 1000);
|
|
1405
|
+
await new Promise((wake) => {
|
|
1406
|
+
h.wake = wake;
|
|
1407
|
+
h.timeout = setTimeout(wake, Math.max(0, at - Date.now()));
|
|
1408
|
+
});
|
|
1409
|
+
h.wake = null;
|
|
1410
|
+
if (h.timeout !== null) { clearTimeout(h.timeout); h.timeout = null; }
|
|
1411
|
+
}
|
|
1412
|
+
} finally {
|
|
1413
|
+
this.handlers.splice(this.handlers.indexOf(h), 1);
|
|
1414
|
+
ch.port1.close();
|
|
1415
|
+
this.handlers.at(-1)?.wake?.(); // the next one takes the jobs and timers
|
|
1416
|
+
}
|
|
1417
|
+
})());
|
|
1418
|
+
return h;
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
// The jobs of h, the wake-ups, and the timers that are due, as they are now.
|
|
1422
|
+
runJobs(h) {
|
|
1423
|
+
const call = (f) => {
|
|
1424
|
+
try { f(); } catch (e) { console.log(`beam: host call: ${e.message}`); }
|
|
1425
|
+
};
|
|
1426
|
+
for (const f of h.jobs.splice(0)) call(f);
|
|
1427
|
+
for (const f of this.jobs.splice(0)) call(f);
|
|
1428
|
+
const now = Date.now();
|
|
1429
|
+
for (const [id, t] of this.timers) {
|
|
1430
|
+
if (t.at <= now) { this.timers.delete(id); call(t.f); }
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
nextTimerAt() {
|
|
1435
|
+
let at = Infinity;
|
|
1436
|
+
for (const t of this.timers.values()) at = Math.min(at, t.at);
|
|
1437
|
+
return at;
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
// Runs fn in the handler h (default: the newest request), or at once in a
|
|
1441
|
+
// Durable Object.
|
|
1442
|
+
run(fn, h = this.handlers.at(-1)) {
|
|
1443
|
+
if (!this.plain) return fn();
|
|
1444
|
+
if (!h) return void this.jobs.push(fn);
|
|
1445
|
+
h.jobs.push(fn);
|
|
1446
|
+
h.wake?.();
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
// One statement of wasm_host_sqlite. A D1 call is I/O of the request h:
|
|
1450
|
+
// it keeps h open (as a socket) until the answer.
|
|
1451
|
+
async sqlQuery(msg, body, h) {
|
|
1452
|
+
if (h) h.sockets++;
|
|
1453
|
+
this.sqlPending = (this.sqlPending ?? 0) + 1;
|
|
1454
|
+
let reply;
|
|
1455
|
+
try {
|
|
1456
|
+
const q = JSON.parse(new TextDecoder().decode(body));
|
|
1457
|
+
const params = q.params.map(fromWire);
|
|
1458
|
+
reply = this.sql ? sqlDurable(this.sql, q.sql, params) : await sqlD1(this.env, q.sql, params);
|
|
1459
|
+
} catch (e) {
|
|
1460
|
+
reply = { error: String(e?.message ?? e) };
|
|
1461
|
+
console.log(`beam: sql: ${reply.error}`);
|
|
1462
|
+
} finally {
|
|
1463
|
+
this.sqlPending--;
|
|
1464
|
+
if (h) { h.sockets--; h.wake?.(); }
|
|
1465
|
+
}
|
|
1466
|
+
this.event({ t: 'sql_reply', id: msg.id }, new TextEncoder().encode(JSON.stringify(reply)));
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
// One operation of wasm_host_wasm (WasmHost). It keeps the request h open
|
|
1470
|
+
// until the answer, as a D1 call does.
|
|
1471
|
+
async wasmRequest(msg, body, h) {
|
|
1472
|
+
if (h) h.sockets++;
|
|
1473
|
+
let reply;
|
|
1474
|
+
try {
|
|
1475
|
+
this.wasm ??= new WasmHost();
|
|
1476
|
+
reply = await this.wasm.op(msg.op, JSON.parse(new TextDecoder().decode(body)));
|
|
1477
|
+
} catch (e) {
|
|
1478
|
+
reply = { error: String(e?.message ?? e) };
|
|
1479
|
+
} finally {
|
|
1480
|
+
if (h) { h.sockets--; h.wake?.(); }
|
|
1481
|
+
}
|
|
1482
|
+
this.event({ t: 'wasm_reply', id: msg.id }, new TextEncoder().encode(JSON.stringify(reply)));
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
// A listener on port, once there is one.
|
|
1486
|
+
listening(port) {
|
|
1487
|
+
if (this.listeners.has(port)) return Promise.resolve();
|
|
1488
|
+
return new Promise((r) => this.waitListen.set(port, [...(this.waitListen.get(port) ?? []), r]));
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
// The Origin that the app gets. Phoenix compares the Origin of a
|
|
1492
|
+
// WebSocket with the host of its config (PHX_HOST), so a page of the app on
|
|
1493
|
+
// another name of the Worker (wrangler dev, a custom domain, a preview URL,
|
|
1494
|
+
// a host tenant) got 403, and LiveView did not connect. A request from a
|
|
1495
|
+
// page of its own origin comes from the app itself: the app gets the origin
|
|
1496
|
+
// of PHX_HOST. The browser sets Origin, so the Origin of another site goes
|
|
1497
|
+
// as it is, and the app refuses it as before (check_origin: :conn of
|
|
1498
|
+
// Phoenix does the same check).
|
|
1499
|
+
appOrigin(origin, url) {
|
|
1500
|
+
const host = this.env.PHX_HOST ?? this.vars.PHX_HOST;
|
|
1501
|
+
return origin && host && origin === url.origin ? `https://${host}` : origin;
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
// The request as a TCP connection to the HTTP server of the
|
|
1505
|
+
// app on PORT. Bandit does the HTTP; here only the bytes are framed.
|
|
1506
|
+
async bridge(request, url, upgrade, h, finished) {
|
|
1507
|
+
const port = Number(this.env.PORT ?? 4000);
|
|
1508
|
+
await this.listening(port);
|
|
1509
|
+
const id = `b${this.nextId++}`;
|
|
1510
|
+
const headers = new Headers(request.headers);
|
|
1511
|
+
headers.set('host', url.host);
|
|
1512
|
+
const origin = this.appOrigin(headers.get('origin'), url);
|
|
1513
|
+
if (origin) headers.set('origin', origin);
|
|
1514
|
+
headers.delete('transfer-encoding');
|
|
1515
|
+
headers.delete('sec-websocket-extensions'); // no compression: frames as they are
|
|
1516
|
+
// The Workers runtime compresses the response for each client. The app
|
|
1517
|
+
// gets only gzip of Accept-Encoding: Plug.Static serves the files that
|
|
1518
|
+
// are gzipped already (Livebook has no other ones), and the response
|
|
1519
|
+
// goes as it is (encodeBody: 'manual').
|
|
1520
|
+
const gzip = /\bgzip\b/i.test(headers.get('accept-encoding') ?? '');
|
|
1521
|
+
headers.delete('accept-encoding');
|
|
1522
|
+
if (gzip) headers.set('accept-encoding', 'gzip');
|
|
1523
|
+
if (upgrade) {
|
|
1524
|
+
headers.set('connection', 'Upgrade');
|
|
1525
|
+
headers.set('sec-websocket-key', btoa(String.fromCharCode(...crypto.getRandomValues(new Uint8Array(16)))));
|
|
1526
|
+
headers.set('sec-websocket-version', '13');
|
|
1527
|
+
} else {
|
|
1528
|
+
headers.set('connection', 'close');
|
|
1529
|
+
}
|
|
1530
|
+
const body = request.method === 'GET' || request.method === 'HEAD' || upgrade
|
|
1531
|
+
? new Uint8Array(0) : new Uint8Array(await request.arrayBuffer());
|
|
1532
|
+
if (body.length || !['GET', 'HEAD'].includes(request.method)) headers.set('content-length', String(body.length));
|
|
1533
|
+
let head = `${request.method} ${url.pathname}${url.search} HTTP/1.1\r\n`;
|
|
1534
|
+
for (const [k, v] of headers) head += `${k}: ${v}\r\n`;
|
|
1535
|
+
head = new TextEncoder().encode(head + '\r\n');
|
|
1536
|
+
const bytes = new Uint8Array(head.length + body.length);
|
|
1537
|
+
bytes.set(head);
|
|
1538
|
+
bytes.set(body, head.length);
|
|
1539
|
+
return new Promise((resolve) => {
|
|
1540
|
+
const c = { id, buf: new Uint8Array(0), resolve, finished, upgrade, head: request.method === 'HEAD', h, origin: request.headers.get('origin'), path: url.pathname };
|
|
1541
|
+
if (h) h.sockets++;
|
|
1542
|
+
this.tcps.set(id, { send: (b) => this.bridgeData(c, b), close: () => this.bridgeEnd(c), h });
|
|
1543
|
+
this.event({ t: 'tcp_accept', id: this.listeners.get(port), conn: id, host: request.headers.get('cf-connecting-ip') ?? '0.0.0.0', port: 0 });
|
|
1544
|
+
this.event({ t: 'tcp_data', id }, bytes);
|
|
1545
|
+
});
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
// Bytes from the HTTP server of the app to the connection c of bridge().
|
|
1549
|
+
bridgeData(c, data) {
|
|
1550
|
+
const b = new Uint8Array(c.buf.length + data.length);
|
|
1551
|
+
b.set(c.buf);
|
|
1552
|
+
b.set(data, c.buf.length);
|
|
1553
|
+
c.buf = b;
|
|
1554
|
+
if (!c.status) {
|
|
1555
|
+
const end = indexOf(c.buf, [13, 10, 13, 10]);
|
|
1556
|
+
if (end < 0) return;
|
|
1557
|
+
const lines = new TextDecoder().decode(c.buf.subarray(0, end)).split('\r\n');
|
|
1558
|
+
c.status = Number(lines[0].split(' ')[1]);
|
|
1559
|
+
const headers = new Headers();
|
|
1560
|
+
for (const l of lines.slice(1)) {
|
|
1561
|
+
const i = l.indexOf(':');
|
|
1562
|
+
headers.append(l.slice(0, i).trim(), l.slice(i + 1).trim());
|
|
1563
|
+
}
|
|
1564
|
+
c.buf = c.buf.slice(end + 4);
|
|
1565
|
+
if (c.status === 101 && c.upgrade) return this.bridgeUpgrade(c);
|
|
1566
|
+
if (c.upgrade && c.status === 403) {
|
|
1567
|
+
console.log(`beam: the app refused the WebSocket of ${c.path} (403) from the origin ${c.origin}. ` +
|
|
1568
|
+
`A Phoenix app compares the Origin with the host of its config (PHX_HOST=${this.env.PHX_HOST ?? this.vars.PHX_HOST ?? ''}): see check_origin.`);
|
|
1569
|
+
}
|
|
1570
|
+
c.length = headers.has('content-length') ? Number(headers.get('content-length')) : null;
|
|
1571
|
+
c.chunked = /chunked/i.test(headers.get('transfer-encoding') ?? '');
|
|
1572
|
+
headers.delete('transfer-encoding');
|
|
1573
|
+
headers.delete('connection');
|
|
1574
|
+
if (c.head || c.status === 204 || c.status === 304) c.length = 0;
|
|
1575
|
+
const { readable, writable } = new TransformStream();
|
|
1576
|
+
c.writer = writable.getWriter();
|
|
1577
|
+
// The app compressed the body (Bandit: gzip, deflate): the runtime
|
|
1578
|
+
// must send it as it is, not compress it again.
|
|
1579
|
+
const encodeBody = headers.has('content-encoding') ? 'manual' : 'automatic';
|
|
1580
|
+
c.resolve(new Response(c.length === 0 ? null : readable, { status: c.status, headers, encodeBody }));
|
|
1581
|
+
c.finished();
|
|
1582
|
+
}
|
|
1583
|
+
if (c.ws) return this.bridgeFrames(c);
|
|
1584
|
+
this.bridgeBody(c);
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
// The body of a response: by content-length, chunked, or until the end.
|
|
1588
|
+
bridgeBody(c) {
|
|
1589
|
+
if (c.done) return;
|
|
1590
|
+
if (c.chunked) {
|
|
1591
|
+
for (;;) {
|
|
1592
|
+
const nl = indexOf(c.buf, [13, 10]);
|
|
1593
|
+
if (nl < 0) return;
|
|
1594
|
+
const size = parseInt(new TextDecoder().decode(c.buf.subarray(0, nl)), 16);
|
|
1595
|
+
if (size === 0) return this.bridgeDone(c);
|
|
1596
|
+
if (c.buf.length < nl + 2 + size + 2) return;
|
|
1597
|
+
c.writer.write(c.buf.slice(nl + 2, nl + 2 + size));
|
|
1598
|
+
c.buf = c.buf.slice(nl + 2 + size + 2);
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
if (c.buf.length) {
|
|
1602
|
+
const part = c.length === null ? c.buf : c.buf.subarray(0, c.length);
|
|
1603
|
+
if (part.length) c.writer.write(part.slice());
|
|
1604
|
+
if (c.length !== null) c.length -= part.length;
|
|
1605
|
+
c.buf = new Uint8Array(0);
|
|
1606
|
+
}
|
|
1607
|
+
if (c.length === 0) this.bridgeDone(c);
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
bridgeDone(c) {
|
|
1611
|
+
if (c.done) return;
|
|
1612
|
+
c.done = true;
|
|
1613
|
+
c.writer?.close().catch(() => {});
|
|
1614
|
+
this.tcps.delete(c.id);
|
|
1615
|
+
this.event({ t: 'tcp_closed', id: c.id });
|
|
1616
|
+
this.bridgeRelease(c);
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
// The request of c can end (once).
|
|
1620
|
+
bridgeRelease(c) {
|
|
1621
|
+
if (c.released) return;
|
|
1622
|
+
c.released = true;
|
|
1623
|
+
if (c.h) { c.h.sockets--; c.h.wake?.(); }
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
// The app closed the connection.
|
|
1627
|
+
bridgeEnd(c) {
|
|
1628
|
+
this.tcps.delete(c.id);
|
|
1629
|
+
if (!c.status) {
|
|
1630
|
+
c.resolve(new Response('bad gateway\n', { status: 502 }));
|
|
1631
|
+
c.finished();
|
|
1632
|
+
}
|
|
1633
|
+
if (c.ws) { try { c.ws.close(); } catch {} this.bridgeRelease(c); }
|
|
1634
|
+
else this.bridgeDone(c);
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
// A WebSocket: the client end to the browser, frames to the app.
|
|
1638
|
+
bridgeUpgrade(c) {
|
|
1639
|
+
const [client, server] = Object.values(new WebSocketPair());
|
|
1640
|
+
server.accept();
|
|
1641
|
+
server.binaryType = 'arraybuffer';
|
|
1642
|
+
c.ws = server;
|
|
1643
|
+
c.frames = []; // the parts of a message in fragments
|
|
1644
|
+
server.addEventListener('message', (e) => {
|
|
1645
|
+
const text = typeof e.data === 'string';
|
|
1646
|
+
this.event({ t: 'tcp_data', id: c.id }, frame(text ? 1 : 2, text ? new TextEncoder().encode(e.data) : new Uint8Array(e.data)));
|
|
1647
|
+
});
|
|
1648
|
+
server.addEventListener('close', (e) => {
|
|
1649
|
+
const code = e.code && e.code !== 1005 ? e.code : 1000;
|
|
1650
|
+
this.event({ t: 'tcp_data', id: c.id }, frame(8, new Uint8Array([code >> 8, code & 255])));
|
|
1651
|
+
});
|
|
1652
|
+
c.resolve(new Response(null, { status: 101, webSocket: client }));
|
|
1653
|
+
c.finished();
|
|
1654
|
+
this.bridgeFrames(c);
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
// WebSocket frames from the app (not masked) to messages for the client.
|
|
1658
|
+
bridgeFrames(c) {
|
|
1659
|
+
for (;;) {
|
|
1660
|
+
const b = c.buf;
|
|
1661
|
+
if (b.length < 2) return;
|
|
1662
|
+
let len = b[1] & 127, at = 2;
|
|
1663
|
+
if (len === 126) { if (b.length < 4) return; len = (b[2] << 8) | b[3]; at = 4; }
|
|
1664
|
+
else if (len === 127) { if (b.length < 10) return; len = Number(new DataView(b.buffer, b.byteOffset).getBigUint64(2)); at = 10; }
|
|
1665
|
+
if (b.length < at + len) return;
|
|
1666
|
+
const fin = b[0] & 128, op = b[0] & 15, data = b.slice(at, at + len);
|
|
1667
|
+
c.buf = b.slice(at + len);
|
|
1668
|
+
if (op === 9) { this.event({ t: 'tcp_data', id: c.id }, frame(10, data)); continue; }
|
|
1669
|
+
if (op === 10) continue;
|
|
1670
|
+
if (op === 8) { c.ws.close(len >= 2 ? (data[0] << 8) | data[1] : 1000); continue; }
|
|
1671
|
+
if (op !== 0) c.op = op;
|
|
1672
|
+
c.frames.push(data);
|
|
1673
|
+
if (!fin) continue;
|
|
1674
|
+
const all = new Uint8Array(c.frames.reduce((n, f) => n + f.length, 0));
|
|
1675
|
+
let i = 0;
|
|
1676
|
+
for (const f of c.frames) { all.set(f, i); i += f.length; }
|
|
1677
|
+
c.frames = [];
|
|
1678
|
+
c.ws.send(c.op === 1 ? new TextDecoder().decode(all) : all);
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
// A connection to a listener of wasm_tcp: a WebSocket to /.tcp/PORT. In a
|
|
1683
|
+
// plain Worker it belongs to the handler h of its request.
|
|
1684
|
+
tcpAccept(port, request, h, finished) {
|
|
1685
|
+
const listener = this.listeners.get(port);
|
|
1686
|
+
finished();
|
|
1687
|
+
if (!listener) return new Response(`no listener on ${port}\n`, { status: 404 });
|
|
1688
|
+
const id = `w${this.nextId++}`;
|
|
1689
|
+
const [client, server] = Object.values(new WebSocketPair());
|
|
1690
|
+
server.accept();
|
|
1691
|
+
server.binaryType = 'arraybuffer'; // (the default can be Blob)
|
|
1692
|
+
if (h) h.sockets++;
|
|
1693
|
+
this.tcps.set(id, { send: (b) => server.send(b), close: () => server.close(), h });
|
|
1694
|
+
this.event({ t: 'tcp_accept', id: listener, conn: id, host: request.headers.get('cf-connecting-ip') ?? '0.0.0.0', port: 0 });
|
|
1695
|
+
server.addEventListener('message', (e) => {
|
|
1696
|
+
this.tcpData(id, typeof e.data === 'string' ? new TextEncoder().encode(e.data) : e.data);
|
|
1697
|
+
});
|
|
1698
|
+
server.addEventListener('close', () => {
|
|
1699
|
+
this.tcpClosed(id);
|
|
1700
|
+
if (h) { h.sockets--; h.wake?.(); }
|
|
1701
|
+
});
|
|
1702
|
+
return new Response(null, { status: 101, webSocket: client });
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
// A TCP socket of wasm_tcp: net.connect() of node:net. In a plain
|
|
1706
|
+
// Worker it belongs to the handler h, and closes with that request.
|
|
1707
|
+
async tcpConnect({ id, host, port }, h) {
|
|
1708
|
+
let socket;
|
|
1709
|
+
if (h) h.sockets++;
|
|
1710
|
+
try {
|
|
1711
|
+
if (!connectAllowed(this.env.BEAM_CONNECT, host, port)) throw new Error('not in BEAM_CONNECT');
|
|
1712
|
+
socket = net.connect({ host, port });
|
|
1713
|
+
// A send resolves when the bytes are written; a close sends the
|
|
1714
|
+
// bytes that wait, and then ends the socket.
|
|
1715
|
+
this.tcps.set(id, {
|
|
1716
|
+
send: (b) => new Promise((resolve) => socket.write(b, () => resolve())),
|
|
1717
|
+
close: () => socket.end(() => socket.destroy()),
|
|
1718
|
+
h,
|
|
1719
|
+
});
|
|
1720
|
+
await new Promise((resolve, reject) => {
|
|
1721
|
+
socket.once('connect', resolve);
|
|
1722
|
+
socket.once('error', reject);
|
|
1723
|
+
});
|
|
1724
|
+
} catch (e) {
|
|
1725
|
+
console.log(`beam: connect ${host}:${port}: ${e.message}`);
|
|
1726
|
+
socket?.destroy();
|
|
1727
|
+
this.tcps.delete(id);
|
|
1728
|
+
this.event({ t: 'tcp_error', id, reason: 'econnrefused' });
|
|
1729
|
+
if (h) { h.sockets--; h.wake?.(); }
|
|
1730
|
+
return;
|
|
1731
|
+
}
|
|
1732
|
+
this.event({ t: 'tcp_open', id });
|
|
1733
|
+
await new Promise((resolve) => {
|
|
1734
|
+
socket.on('data', (b) => this.tcpData(id, new Uint8Array(b.buffer, b.byteOffset, b.byteLength)));
|
|
1735
|
+
socket.on('error', () => {});
|
|
1736
|
+
socket.once('close', resolve);
|
|
1737
|
+
});
|
|
1738
|
+
this.tcpClosed(id);
|
|
1739
|
+
if (h) { h.sockets--; h.wake?.(); }
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
// Data of a TCP socket: to Erlang, or to its peer after a splice.
|
|
1743
|
+
tcpData(id, bytes) {
|
|
1744
|
+
const t = this.tcps.get(id);
|
|
1745
|
+
const peer = t?.peer && this.tcps.get(t.peer);
|
|
1746
|
+
if (peer) this.run(() => peer.send(bytes), peer.h);
|
|
1747
|
+
else this.event({ t: 'tcp_data', id }, bytes);
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
// The end of a TCP socket: to Erlang, and the end of its peer too.
|
|
1751
|
+
tcpClosed(id) {
|
|
1752
|
+
const t = this.tcps.get(id);
|
|
1753
|
+
this.tcps.delete(id);
|
|
1754
|
+
this.event({ t: 'tcp_closed', id });
|
|
1755
|
+
const peer = t?.peer && this.tcps.get(t.peer);
|
|
1756
|
+
if (peer) {
|
|
1757
|
+
this.tcps.delete(t.peer);
|
|
1758
|
+
this.run(() => peer.close(), peer.h);
|
|
1759
|
+
this.event({ t: 'tcp_closed', id: t.peer });
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
onsend(bytes) {
|
|
1764
|
+
const nl = bytes.indexOf(10);
|
|
1765
|
+
const msg = JSON.parse(new TextDecoder().decode(bytes.subarray(0, nl)));
|
|
1766
|
+
const body = bytes.slice(nl + 1);
|
|
1767
|
+
switch (msg.t) {
|
|
1768
|
+
case 'ready':
|
|
1769
|
+
this.onready();
|
|
1770
|
+
break;
|
|
1771
|
+
case 'boot_point':
|
|
1772
|
+
this.run(() => this.bootPoint());
|
|
1773
|
+
break;
|
|
1774
|
+
case 'tcp_connect': {
|
|
1775
|
+
const h = this.handlers.at(-1);
|
|
1776
|
+
this.run(() => this.tcpConnect(msg, h), h);
|
|
1777
|
+
break;
|
|
1778
|
+
}
|
|
1779
|
+
case 'sql': {
|
|
1780
|
+
const h = this.handlers.at(-1);
|
|
1781
|
+
this.run(() => this.sqlQuery(msg, body, h), h);
|
|
1782
|
+
break;
|
|
1783
|
+
}
|
|
1784
|
+
case 'wasm': {
|
|
1785
|
+
const h = this.handlers.at(-1);
|
|
1786
|
+
this.run(() => this.wasmRequest(msg, body, h), h);
|
|
1787
|
+
break;
|
|
1788
|
+
}
|
|
1789
|
+
case 'tcp_send': {
|
|
1790
|
+
const t = this.tcps.get(msg.id);
|
|
1791
|
+
if (t) this.run(() => t.send(body), t.h);
|
|
1792
|
+
break;
|
|
1793
|
+
}
|
|
1794
|
+
case 'tcp_close': {
|
|
1795
|
+
const t = this.tcps.get(msg.id);
|
|
1796
|
+
this.tcps.delete(msg.id);
|
|
1797
|
+
if (t) this.run(() => t.close(), t.h);
|
|
1798
|
+
break;
|
|
1799
|
+
}
|
|
1800
|
+
// wasm_tcp:splice/2: the data of each socket goes to the other one,
|
|
1801
|
+
// no longer through Erlang.
|
|
1802
|
+
case 'tcp_splice': {
|
|
1803
|
+
const a = this.tcps.get(msg.a), b = this.tcps.get(msg.b);
|
|
1804
|
+
if (a && b) { a.peer = msg.b; b.peer = msg.a; }
|
|
1805
|
+
break;
|
|
1806
|
+
}
|
|
1807
|
+
case 'tcp_listen':
|
|
1808
|
+
if (this.listeners.has(msg.port)) {
|
|
1809
|
+
this.event({ t: 'tcp_error', id: msg.id, reason: 'eaddrinuse' });
|
|
1810
|
+
} else {
|
|
1811
|
+
this.listeners.set(msg.port, msg.id);
|
|
1812
|
+
this.event({ t: 'tcp_listening', id: msg.id });
|
|
1813
|
+
for (const r of this.waitListen.get(msg.port) ?? []) r();
|
|
1814
|
+
this.waitListen.delete(msg.port);
|
|
1815
|
+
}
|
|
1816
|
+
break;
|
|
1817
|
+
case 'tcp_unlisten':
|
|
1818
|
+
for (const [port, id] of this.listeners) if (id === msg.id) this.listeners.delete(port);
|
|
1819
|
+
break;
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
// The types of the static files, by extension.
|
|
1825
|
+
const TYPES = {
|
|
1826
|
+
html: 'text/html; charset=utf-8', htm: 'text/html; charset=utf-8', txt: 'text/plain; charset=utf-8',
|
|
1827
|
+
css: 'text/css; charset=utf-8', js: 'text/javascript; charset=utf-8', mjs: 'text/javascript; charset=utf-8',
|
|
1828
|
+
json: 'application/json', map: 'application/json', webmanifest: 'application/manifest+json',
|
|
1829
|
+
xml: 'application/xml', wasm: 'application/wasm', pdf: 'application/pdf',
|
|
1830
|
+
svg: 'image/svg+xml', png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif',
|
|
1831
|
+
webp: 'image/webp', avif: 'image/avif', ico: 'image/vnd.microsoft.icon',
|
|
1832
|
+
woff: 'font/woff', woff2: 'font/woff2', ttf: 'font/ttf', otf: 'font/otf',
|
|
1833
|
+
mp4: 'video/mp4', webm: 'video/webm', mp3: 'audio/mpeg', ogg: 'audio/ogg',
|
|
1834
|
+
};
|
|
1835
|
+
|
|
1836
|
+
// A GET or HEAD request for a static file of the release (statics of
|
|
1837
|
+
// appFiles), as Plug.Static of Phoenix gives it: an ETag, and a cache of
|
|
1838
|
+
// one year for a request with the parameter vsn. Else null: the VM
|
|
1839
|
+
// answers.
|
|
1840
|
+
export function staticResponse(statics, request) {
|
|
1841
|
+
if (request.method !== 'GET' && request.method !== 'HEAD') return null;
|
|
1842
|
+
const url = new URL(request.url);
|
|
1843
|
+
let path;
|
|
1844
|
+
try { path = decodeURIComponent(url.pathname); } catch { return null; }
|
|
1845
|
+
// Only PATH.gz (Phoenix digests give both, some packages only the
|
|
1846
|
+
// .gz file): its data, with no gzip.
|
|
1847
|
+
const plain = statics.get(path);
|
|
1848
|
+
const s = plain ?? statics.get(`${path}.gz`);
|
|
1849
|
+
if (!s) return null;
|
|
1850
|
+
const gz = !plain;
|
|
1851
|
+
// The size of the data of a stored gzip file: its last 4 bytes (ISIZE).
|
|
1852
|
+
const size = !gz ? s.size : s.deflated || s.data.length < 18 ? null
|
|
1853
|
+
: new DataView(s.data.buffer, s.data.byteOffset + s.data.length - 4, 4).getUint32(0, true);
|
|
1854
|
+
const etag = `"${s.crc.toString(16).padStart(8, '0')}${s.size.toString(16)}${gz ? '-gz' : ''}"`;
|
|
1855
|
+
const headers = {
|
|
1856
|
+
'content-type': TYPES[path.slice(path.lastIndexOf('.') + 1).toLowerCase()] ?? 'application/octet-stream',
|
|
1857
|
+
'cache-control': url.searchParams.has('vsn') ? 'public, max-age=31536000, immutable' : 'public',
|
|
1858
|
+
etag,
|
|
1859
|
+
};
|
|
1860
|
+
if (request.headers.get('if-none-match') === etag) return new Response(null, { status: 304, headers });
|
|
1861
|
+
if (size !== null) headers['content-length'] = String(size);
|
|
1862
|
+
if (request.method === 'HEAD') return new Response(null, { headers });
|
|
1863
|
+
let body = new Blob([s.data]).stream();
|
|
1864
|
+
if (s.deflated) body = body.pipeThrough(new DecompressionStream('deflate-raw'));
|
|
1865
|
+
if (gz) body = body.pipeThrough(new DecompressionStream('gzip'));
|
|
1866
|
+
return new Response(body, { headers });
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
// The position of the bytes pat in b, or -1.
|
|
1870
|
+
function indexOf(b, pat) {
|
|
1871
|
+
outer: for (let i = 0; i + pat.length <= b.length; i++) {
|
|
1872
|
+
for (let j = 0; j < pat.length; j++) if (b[i + j] !== pat[j]) continue outer;
|
|
1873
|
+
return i;
|
|
1874
|
+
}
|
|
1875
|
+
return -1;
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
// A WebSocket frame from a client (masked, as RFC 6455 asks).
|
|
1879
|
+
function frame(op, data) {
|
|
1880
|
+
const n = data.length;
|
|
1881
|
+
const ext = n < 126 ? 0 : n < 65536 ? 2 : 8;
|
|
1882
|
+
const f = new Uint8Array(2 + ext + 4 + n);
|
|
1883
|
+
f[0] = 128 | op;
|
|
1884
|
+
if (ext === 0) f[1] = 128 | n;
|
|
1885
|
+
else if (ext === 2) { f[1] = 128 | 126; f[2] = n >> 8; f[3] = n & 255; }
|
|
1886
|
+
else { f[1] = 128 | 127; new DataView(f.buffer).setBigUint64(2, BigInt(n)); }
|
|
1887
|
+
const mask = crypto.getRandomValues(new Uint8Array(4));
|
|
1888
|
+
f.set(mask, 2 + ext);
|
|
1889
|
+
for (let i = 0; i < n; i++) f[2 + ext + 4 + i] = data[i] ^ mask[i & 3];
|
|
1890
|
+
return f;
|
|
1891
|
+
}
|