@daugasauron/dolly 0.1.4 → 0.1.5-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -14
- package/coi-serviceworker.js +28 -0
- package/dist/dolly-build-id.mjs +1 -1
- package/dist/dolly-dso-0.wasm +0 -0
- package/dist/dolly-image-build-id.mjs +1 -1
- package/dist/dolly-process-0.wasm +0 -0
- package/dist/dolly-process-abi.mjs +1 -1
- package/dist/dolly-process-dso.mjs +382 -53
- package/dist/dolly-process-gate-0.wasm +0 -0
- package/dist/dolly-process-worker.mjs +164 -35
- package/dist/dolly.wasm +0 -0
- package/host/dso/abi.mjs +3 -1
- package/host/dso/dso.mjs +1 -2
- package/host/http/abi.mjs +2 -2
- package/host/http/broker.mjs +18 -8
- package/host/http/http.mjs +0 -1
- package/host/http/policy.mjs +8 -4
- package/host/input/abi.mjs +47 -0
- package/host/modules.mjs +4 -3
- package/host/nena/abi.mjs +1 -1
- package/host/nena/nena.mjs +16 -18
- package/host/packages/service.mjs +35 -15
- package/host/packages/worker.mjs +21 -0
- package/host/runtime/abi.mjs +1 -0
- package/host/runtime/boot-files.mjs +8 -2
- package/host/runtime/grower.mjs +11 -8
- package/host/runtime/lease-bridge.mjs +8 -5
- package/host/runtime/module.json +1 -1
- package/host/runtime/runtime.mjs +37 -18
- package/host/snapshot/abi.mjs +7 -9
- package/host/snapshot/snapshot.mjs +14 -12
- package/host/snapshot/transport.mjs +4 -12
- package/host/snapshot/ui.mjs +17 -7
- package/host/stack/stack.mjs +1 -2
- package/host/threads/abi.mjs +1 -1
- package/host/threads/threads.mjs +2 -1
- package/package.json +1 -1
- package/src/browser.mjs +34 -37
- package/src/build-log.mjs +13 -2
- package/src/build-worker.mjs +182 -0
- package/src/custom-image.mjs +59 -0
- package/src/dollyfile-graph.mjs +127 -0
- package/src/dollyfile-view.mjs +335 -0
- package/src/image-artifact.mjs +248 -0
- package/src/image-build.mjs +89 -0
- package/src/image-builder.mjs +63 -0
- package/src/images.mjs +13 -2
- package/src/pack-store.mjs +141 -0
- package/src/packaged-snapshot.mjs +263 -91
- package/src/page-chords.mjs +4 -2
- package/src/process-constants.mjs +4 -2
- package/src/process-supervisor.mjs +209 -122
- package/src/runtime-worker.mjs +9 -10
- package/src/session-store.mjs +5 -7
- package/src/sha256.mjs +65 -0
- package/src/snapshot-records.mjs +135 -23
- package/src/static-asset.mjs +29 -2
- package/src/terminal-text.mjs +8 -5
- package/src/version.mjs +1 -1
- package/src/worker-image.mjs +3 -5
package/src/browser.mjs
CHANGED
|
@@ -3,26 +3,28 @@ import { holdIndicators, showIndicators } from "./page-indicators.mjs";
|
|
|
3
3
|
import { hostRegistry } from "../host/modules.mjs";
|
|
4
4
|
import { pageChords } from "./page-chords.mjs";
|
|
5
5
|
import { terminalText } from "./terminal-text.mjs";
|
|
6
|
-
import {
|
|
6
|
+
import { bootstrapSources, imageBase, imageDefinition, loadImages } from "./images.mjs";
|
|
7
7
|
import * as processConstants from "./process-constants.mjs";
|
|
8
8
|
|
|
9
9
|
// The terminal's own look: its canvas, the hidden keyboard, the boot log and
|
|
10
|
-
// the status line
|
|
11
|
-
//
|
|
10
|
+
// the status line, which lie in boot()'s element and style nothing else of the
|
|
11
|
+
// page. A host module brings the look of what it adds; the page, the boot log's
|
|
12
|
+
// font (the site's is Iosevka, dist/IosevkaTerm-SemiBold.woff2).
|
|
12
13
|
const style = `
|
|
13
|
-
.dolly-terminal { position: relative; overflow: hidden; outline: none; caret-color: transparent;
|
|
14
|
-
|
|
15
|
-
#display
|
|
16
|
-
#
|
|
17
|
-
|
|
14
|
+
.dolly-terminal { position: relative; overflow: hidden; outline: none; caret-color: transparent;
|
|
15
|
+
display: flex; flex-direction: column; }
|
|
16
|
+
.dolly-terminal #display { display: block; width: 100%; flex: 1 1 0; min-height: 0; cursor: text; touch-action: none; }
|
|
17
|
+
.dolly-terminal #display[hidden] { display: none; }
|
|
18
|
+
.dolly-terminal #keyboard { position: fixed; left: -10000px; top: 0; width: 1px; height: 1px; opacity: 0; caret-color: transparent; }
|
|
19
|
+
.dolly-terminal #bootstrap-log { position: absolute; inset: 0; z-index: 1; margin: 0; padding: 1rem 1.15rem; overflow: auto;
|
|
18
20
|
white-space: pre-wrap; color: #e8e3d7; background: #262626; font: 600 20px/1.35 "Dolly IosevkaTerm SemiBold", monospace;
|
|
19
21
|
scrollbar-width: thin; scrollbar-color: #f2d45c #262626; }
|
|
20
|
-
#bootstrap-log[hidden] { display: none; }
|
|
21
|
-
#session-status { position:
|
|
22
|
+
.dolly-terminal #bootstrap-log[hidden] { display: none; }
|
|
23
|
+
.dolly-terminal #session-status { position: absolute; right: 1rem; top: 1rem; z-index: 5; max-width: calc(100% - 2rem);
|
|
22
24
|
padding: 0.5rem 0.75rem; color: #e8e3d7; background: #262626; border: 1px solid #77736c; font-size: 14px; }
|
|
23
|
-
#session-status[hidden] { display: none; }
|
|
25
|
+
.dolly-terminal #session-status[hidden] { display: none; }
|
|
24
26
|
html[data-indicators="hidden"] .page-indicator { display: none; }
|
|
25
|
-
@media (max-width: 640px) { #bootstrap-log { padding: 0.65rem 0.75rem; } }
|
|
27
|
+
@media (max-width: 640px) { .dolly-terminal #bootstrap-log { padding: 0.65rem 0.75rem; } }
|
|
26
28
|
`;
|
|
27
29
|
|
|
28
30
|
function element(tag, properties, attributes = {}) {
|
|
@@ -39,13 +41,18 @@ function element(tag, properties, attributes = {}) {
|
|
|
39
41
|
// written over the restored image before its ENTRY starts. `site`, given, is the
|
|
40
42
|
// root of the routes a module may link to (session/, sessions/, docs/). It
|
|
41
43
|
// resolves to the page's API (window.__dolly on the site) once the image's ENTRY runs.
|
|
42
|
-
export function boot(mount, { images, image, modules, site, http = {}, files = {} } = {}) {
|
|
43
|
-
return run(mount, { images, modules, site, http }, async
|
|
44
|
-
|
|
44
|
+
export function boot(mount, { images, image, modules, site, http = {}, files = {}, session } = {}) {
|
|
45
|
+
return run(mount, { images, modules, site, http }, async registry => {
|
|
46
|
+
// A saved session (snapshot@0) restores its image as it was saved: files are for a fresh start.
|
|
47
|
+
const selected = session === undefined ? null
|
|
48
|
+
: await registry.selectBoot({ image, mode: "snapshot", loadSession: true, session });
|
|
49
|
+
if (session !== undefined && !selected) throw new Error("opening a saved session needs snapshot@0 among the page's modules");
|
|
50
|
+
const definition = imageDefinition(selected?.image ?? image);
|
|
45
51
|
if (!definition || typeof files !== "object" || files === null) throw new Error("invalid Dolly boot options");
|
|
46
52
|
if (definition.entry === null) throw new Error(`${image} has no ENTRY; it only builds`);
|
|
47
|
-
return { image, label: "PRECOMPILED SYSTEM",
|
|
48
|
-
|
|
53
|
+
return { image: definition.image, label: selected?.label ?? "PRECOMPILED SYSTEM", configuration: selected?.configuration,
|
|
54
|
+
required: [...new Set([...definition.hostRequirements, ...selected ? [selected.module] : []])],
|
|
55
|
+
start: () => ({ message: { files: selected ? {} : files }, transfers: [] }) };
|
|
49
56
|
});
|
|
50
57
|
}
|
|
51
58
|
|
|
@@ -61,8 +68,8 @@ export function boot(mount, { images, image, modules, site, http = {}, files = {
|
|
|
61
68
|
// configure message ({ message, transfers }), or null once an image without
|
|
62
69
|
// ENTRY is built;
|
|
63
70
|
// - message(message): each message of the Worker;
|
|
64
|
-
// - running(
|
|
65
|
-
//
|
|
71
|
+
// - running(host): the context host modules' entryStarted is given beside
|
|
72
|
+
// `image`, and members of the page's handle.
|
|
66
73
|
export async function run(mount, { images, modules, site, http }, route) {
|
|
67
74
|
if (!document.querySelector("style[data-dolly]")) {
|
|
68
75
|
document.head.append(element("style", { textContent: style }, { "data-dolly": "" }));
|
|
@@ -75,9 +82,8 @@ export async function run(mount, { images, modules, site, http }, route) {
|
|
|
75
82
|
mount.classList.add("dolly-terminal");
|
|
76
83
|
mount.replaceChildren(canvas, keyboard, bootstrapLog, status);
|
|
77
84
|
const bootstrapOutput = buildLog(bootstrapLog);
|
|
78
|
-
bootstrapOutput.clear();
|
|
79
85
|
const appendBootstrap = text => bootstrapOutput.append(text);
|
|
80
|
-
pageChords(keyboard);
|
|
86
|
+
pageChords(mount, keyboard);
|
|
81
87
|
const bootstrapDecoder = new TextDecoder();
|
|
82
88
|
|
|
83
89
|
let runtimeWorker;
|
|
@@ -111,13 +117,15 @@ export async function run(mount, { images, modules, site, http }, route) {
|
|
|
111
117
|
const how = failure ? `failed: ${failure}` : signal ? `was ended by ${signalName ?? `signal ${signal}`}`
|
|
112
118
|
: `exited with status ${code}`;
|
|
113
119
|
const notice = mount.appendChild(element("div", { id: "image-ended" }, { role: "alert" }));
|
|
114
|
-
notice.style.cssText = "position:
|
|
120
|
+
notice.style.cssText = "position:absolute;left:0;right:0;bottom:0;z-index:9;padding:0.6rem 1rem;" +
|
|
115
121
|
"color:#e8e3d7;background:#262626;border-top:1px solid #f2d45c;font-size:16px";
|
|
116
122
|
const links = [{ text: "Reload to start again", href: location.href }, ...offers].map(({ text, href }) =>
|
|
117
123
|
Object.assign(document.createElement("a"), { textContent: text, href, style: "color:#f2d45c;margin-left:1.5ch" }));
|
|
118
124
|
notice.append(`This image has ended: its program ${how}.`, ...links);
|
|
119
|
-
|
|
120
|
-
Object.assign(canvas.style, {
|
|
125
|
+
// The notice takes the element's bottom strip; the last frame and the log keep the rest.
|
|
126
|
+
Object.assign(canvas.style, { flex: "none", height: `calc(100% - ${notice.offsetHeight}px)`, objectFit: "contain",
|
|
127
|
+
objectPosition: "left top" });
|
|
128
|
+
bootstrapLog.style.bottom = `${notice.offsetHeight}px`;
|
|
121
129
|
links[0].focus();
|
|
122
130
|
document.documentElement.dataset.dollyStatus = "exited";
|
|
123
131
|
}
|
|
@@ -139,20 +147,13 @@ export async function run(mount, { images, modules, site, http }, route) {
|
|
|
139
147
|
const registry = await hostRegistry(modules);
|
|
140
148
|
const selected = await route(registry);
|
|
141
149
|
const { image } = selected;
|
|
142
|
-
const bootstrapSources = [
|
|
143
|
-
...DOLLY_IMAGES.map((definition) => ({
|
|
144
|
-
path: `/${definition.dollyfile}`,
|
|
145
|
-
byteLength: definition.byteLength,
|
|
146
|
-
})),
|
|
147
|
-
...DOLLY_STATIC_SOURCES,
|
|
148
|
-
];
|
|
149
150
|
appendBootstrap(`DOLLY / ${image.toUpperCase()} / ${selected.label}\n\n`);
|
|
150
151
|
host = await registry.createHost("browser", selected.enabled ?? selected.required, {
|
|
151
152
|
send: (message, transfers = []) => runtimeWorker.postMessage(message, transfers),
|
|
152
153
|
resources: { mount, canvas, keyboard, applicationBase: imageBase, site: site && new URL(site), showStatus, fatal,
|
|
153
154
|
indicators: { hold: holdIndicators, show: showIndicators } },
|
|
154
155
|
configuration: { ...selected.configuration, http: { ...selected.configuration?.http, embedding: http,
|
|
155
|
-
sources: bootstrapSources, inherited: selected.inherited } },
|
|
156
|
+
sources: bootstrapSources(), inherited: selected.inherited } },
|
|
156
157
|
});
|
|
157
158
|
host.require(selected.required);
|
|
158
159
|
const started = selected.start ? await selected.start(host, appendBootstrap) : { message: {}, transfers: [] };
|
|
@@ -189,7 +190,6 @@ export async function run(mount, { images, modules, site, http }, route) {
|
|
|
189
190
|
runtimeWorker.postMessage({
|
|
190
191
|
type: "configure",
|
|
191
192
|
image,
|
|
192
|
-
mode: "snapshot",
|
|
193
193
|
images: imageBase.href,
|
|
194
194
|
modules: registry.urls,
|
|
195
195
|
hostModules: host.enabled,
|
|
@@ -213,11 +213,8 @@ export async function run(mount, { images, modules, site, http }, route) {
|
|
|
213
213
|
}).finally(() => { rejectReady = null; });
|
|
214
214
|
appendBootstrap(bootstrapDecoder.decode());
|
|
215
215
|
runtimeReady = true;
|
|
216
|
-
if (ready.routeImage !== image || (image !== "custom" && ready.image !== image)) {
|
|
217
|
-
throw new Error("runtime image mismatch");
|
|
218
|
-
}
|
|
219
216
|
document.documentElement.dataset.image = ready.image;
|
|
220
|
-
const { context, page } = await selected.running?.(
|
|
217
|
+
const { context, page } = await selected.running?.(host) ?? {};
|
|
221
218
|
await host.entryStarted({ image, ...context });
|
|
222
219
|
runtimeWorker.postMessage({ type: "entry-ready-ack" });
|
|
223
220
|
// A module that shows the canvas replaces the bootstrap log.
|
package/src/build-log.mjs
CHANGED
|
@@ -2,10 +2,21 @@
|
|
|
2
2
|
export function buildLog(element) {
|
|
3
3
|
const maximum = 1024 * 1024;
|
|
4
4
|
let length = 0, escape = "", carriageReturn = false;
|
|
5
|
+
// The log follows its end while the reader is there, as their last scroll left
|
|
6
|
+
// it, once a frame: an append reads no layout, which in a log of a megabyte
|
|
7
|
+
// took Chrome 12 ms each.
|
|
8
|
+
let following = true, scrolling = false;
|
|
9
|
+
element.addEventListener("scroll", () => {
|
|
10
|
+
following = element.scrollHeight - element.scrollTop - element.clientHeight < 24;
|
|
11
|
+
});
|
|
12
|
+
function follow() {
|
|
13
|
+
if (!following || scrolling) return;
|
|
14
|
+
scrolling = true;
|
|
15
|
+
requestAnimationFrame(() => { scrolling = false; element.scrollTop = element.scrollHeight; });
|
|
16
|
+
}
|
|
5
17
|
return {
|
|
6
18
|
clear() { element.replaceChildren(); length = 0; escape = ""; carriageReturn = false; },
|
|
7
19
|
append(text) {
|
|
8
|
-
const follow = element.scrollHeight - element.scrollTop - element.clientHeight < 24;
|
|
9
20
|
text = escape + String(text);
|
|
10
21
|
escape = /\x1b(?:\[[0-9;:]{0,128})?$/.exec(text)?.[0] ?? "";
|
|
11
22
|
if (escape) text = text.slice(0, -escape.length);
|
|
@@ -24,7 +35,7 @@ export function buildLog(element) {
|
|
|
24
35
|
if (first.length <= excess) { length -= first.length; first.remove(); }
|
|
25
36
|
else { first.deleteData(0, excess); length -= excess; }
|
|
26
37
|
}
|
|
27
|
-
|
|
38
|
+
follow();
|
|
28
39
|
},
|
|
29
40
|
};
|
|
30
41
|
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// The site's runtime Worker (worker-image.mjs): it builds an image from its
|
|
2
|
+
// recipe, from the compiler seed or the base image the page hands it, caches
|
|
3
|
+
// it and runs it, telling the page its identity, or, for the image builder,
|
|
4
|
+
// hands the page the cached artifact (buildOnly); or it restores a custom
|
|
5
|
+
// image the site built before. Images cross to and from Wasm memory a piece
|
|
6
|
+
// at a time: the page holds them as Blobs, never in a JS heap.
|
|
7
|
+
import { DOLLY_IMAGE_BUILD_ID } from "../dist/dolly-image-build-id.mjs";
|
|
8
|
+
import { imageBase, imageDefinition } from "./images.mjs";
|
|
9
|
+
import { scanSnapshotRecords, MAX_SNAPSHOT_BYTES as snapshotSizeLimit } from "./snapshot-records.mjs";
|
|
10
|
+
import { describeImageArtifact, saveImageArtifact, sha256 } from "./image-artifact.mjs";
|
|
11
|
+
import { imageInputs } from "./image-inputs.mjs";
|
|
12
|
+
import { inspectDollyfile, MAX_DOLLYFILE_BYTES, recipeRows } from "./dollyfile-view.mjs";
|
|
13
|
+
import { checkedCustomArtifact } from "./custom-image.mjs";
|
|
14
|
+
import { siteReference, decodeStaticAsset } from "./static-asset.mjs";
|
|
15
|
+
import { checkedMemoryRange, kernelArtifact, runImage } from "./worker-image.mjs";
|
|
16
|
+
|
|
17
|
+
const encoder = new TextEncoder();
|
|
18
|
+
const PIECE = 8 * 1024 * 1024;
|
|
19
|
+
|
|
20
|
+
// The kernel's capture of the image, TOTAL bytes, as unshared copies of the
|
|
21
|
+
// pieces it reads into the staging window of PIECE bytes.
|
|
22
|
+
function* capturedPieces(kernel, memory, total) {
|
|
23
|
+
const window = checkedMemoryRange(memory, kernel.dolly_snapshot_restore_address(BigInt(PIECE)), PIECE);
|
|
24
|
+
let produced = 0;
|
|
25
|
+
for (;;) {
|
|
26
|
+
const size = Number(kernel.dolly_snapshot_capture_read());
|
|
27
|
+
if (size === 0) break;
|
|
28
|
+
if (size > window.size || size > total - produced) throw new Error("Wasm published an invalid system snapshot piece");
|
|
29
|
+
produced += size;
|
|
30
|
+
yield new Uint8Array(memory.buffer, window.address, size).slice();
|
|
31
|
+
}
|
|
32
|
+
if (produced !== total) throw new Error("Dolly system snapshot capture failed");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// A catalog recipe as this site publishes it, checked against its pin.
|
|
36
|
+
async function catalogRecipe(definition) {
|
|
37
|
+
const response = await fetch(new URL(definition.dollyfile, imageBase), { credentials: "same-origin", redirect: "error" });
|
|
38
|
+
const bytes = response.ok ? await response.arrayBuffer() : new ArrayBuffer(0);
|
|
39
|
+
if (await sha256(bytes) !== definition.sha256) throw new Error(`${definition.dollyfile} does not match its pin`);
|
|
40
|
+
return inspectDollyfile(new TextDecoder().decode(bytes), definition.dollyfile);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
await runImage(async (config, host) => {
|
|
44
|
+
const { image, mode, customSource } = config;
|
|
45
|
+
const definition = imageDefinition(image);
|
|
46
|
+
const custom = image === "custom";
|
|
47
|
+
if (!(definition || custom) || !["snapshot", "rebuild"].includes(mode) || (mode === "snapshot" && !custom)) {
|
|
48
|
+
throw new Error("invalid Dolly image selection");
|
|
49
|
+
}
|
|
50
|
+
if ((custom || customSource !== undefined) && (typeof customSource !== "string" ||
|
|
51
|
+
encoder.encode(customSource).byteLength > MAX_DOLLYFILE_BYTES || customSource.includes("\0"))) {
|
|
52
|
+
throw new Error("invalid uploaded Dollyfile");
|
|
53
|
+
}
|
|
54
|
+
const recipe = custom ? inspectDollyfile(customSource) : definition;
|
|
55
|
+
const locator = custom ? "FILE:/etc/dolly/upload.Dollyfile" : siteReference(definition.dollyfile);
|
|
56
|
+
const prepareRecipe = (files, rows) => {
|
|
57
|
+
files.write("/etc/dolly/recipe.locator", locator);
|
|
58
|
+
if (custom) files.write("/etc/dolly/upload.Dollyfile", customSource);
|
|
59
|
+
if (rows !== undefined) files.write("/etc/dolly/recipe.rows", rows);
|
|
60
|
+
};
|
|
61
|
+
// Copies a whole snapshot, a Blob, into the kernel's checked staging range a
|
|
62
|
+
// piece at a time; its size.
|
|
63
|
+
const stageSnapshot = async (kernel, memory, blob) => {
|
|
64
|
+
const range = checkedMemoryRange(memory, kernel.dolly_snapshot_restore_address(BigInt(blob.size)), blob.size);
|
|
65
|
+
for (let offset = 0; offset < range.size; offset += PIECE) {
|
|
66
|
+
const piece = new Uint8Array(await blob.slice(offset, Math.min(offset + PIECE, range.size)).arrayBuffer());
|
|
67
|
+
new Uint8Array(memory.buffer, range.address + offset, piece.length).set(piece);
|
|
68
|
+
}
|
|
69
|
+
return range.size;
|
|
70
|
+
};
|
|
71
|
+
if (mode === "snapshot") {
|
|
72
|
+
const artifact = await checkedCustomArtifact(customSource, config.customArtifact);
|
|
73
|
+
config.customArtifact = undefined;
|
|
74
|
+
host.require(artifact.hostRequirements);
|
|
75
|
+
host.admit(artifact.hostRequirements);
|
|
76
|
+
return {
|
|
77
|
+
name: image,
|
|
78
|
+
prepare({ files }) {
|
|
79
|
+
prepareRecipe(files);
|
|
80
|
+
files.write("/etc/dolly/image.manifest", `${artifact.manifest.join("\n")}\n`);
|
|
81
|
+
},
|
|
82
|
+
async restore({ kernel, memory, stage }) {
|
|
83
|
+
stage("loading precompiled userspace snapshot...");
|
|
84
|
+
const size = await stageSnapshot(kernel, memory, artifact.bytes);
|
|
85
|
+
artifact.bytes = undefined;
|
|
86
|
+
return { status: kernel.dolly_bootstrap_snapshot(BigInt(size)) };
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const declared = recipe.hostRequirements;
|
|
92
|
+
const recipeSha256 = custom ? await sha256(encoder.encode(customSource)) : definition.sha256;
|
|
93
|
+
// /bin/dollyfile executes the rows this page's parser compiles.
|
|
94
|
+
const rows = recipeRows(custom ? recipe : await catalogRecipe(definition), recipeSha256);
|
|
95
|
+
const baseReference = custom ? recipe.from : definition.artifacts.find(reference => reference.operation === "from");
|
|
96
|
+
const artifacts = new Map();
|
|
97
|
+
if (config.artifacts !== undefined && (!Array.isArray(config.artifacts) || config.artifacts.length > 256)) {
|
|
98
|
+
throw new Error("invalid build artifacts");
|
|
99
|
+
}
|
|
100
|
+
for (const candidate of config.artifacts ?? []) {
|
|
101
|
+
if (candidate?.buildId !== DOLLY_IMAGE_BUILD_ID || !/^[0-9a-f]{64}$/.test(candidate.recipeSha256) ||
|
|
102
|
+
!(candidate.bytes instanceof Blob)) throw new Error("build artifact integrity mismatch");
|
|
103
|
+
const artifact = await describeImageArtifact(candidate.bytes, candidate.recipeSha256);
|
|
104
|
+
if (artifact.sha256 !== candidate.sha256) throw new Error("build artifact integrity mismatch");
|
|
105
|
+
artifacts.set(candidate.recipeSha256, artifact);
|
|
106
|
+
}
|
|
107
|
+
config.artifacts = undefined;
|
|
108
|
+
// A build runs the engine and its tools with the build host's modules, and
|
|
109
|
+
// the programs of the images it reads with their modules: one the build host
|
|
110
|
+
// does not enable answers ENOSYS.
|
|
111
|
+
host.admit([...declared, ...host.enabled, ...[...artifacts.values()].flatMap(artifact => artifact.hostRequirements)]);
|
|
112
|
+
const inputs = imageInputs([...artifacts.values()]);
|
|
113
|
+
const base = baseReference ? artifacts.get(baseReference.sha256) : null;
|
|
114
|
+
if (baseReference && !base) throw new Error("base image artifact was not provided");
|
|
115
|
+
let seed = null, built = false;
|
|
116
|
+
return {
|
|
117
|
+
name: image,
|
|
118
|
+
async prepare({ files, stage }) {
|
|
119
|
+
// A root build starts from the compiler seed, a snapshot like any image.
|
|
120
|
+
if (!base) {
|
|
121
|
+
stage("loading root compiler seed...");
|
|
122
|
+
// Static hosts may serve the seed as verified parts.
|
|
123
|
+
const url = kernelArtifact("dolly.data");
|
|
124
|
+
seed = await (await decodeStaticAsset(await fetch(url), url, {}, snapshotSizeLimit)).blob();
|
|
125
|
+
}
|
|
126
|
+
prepareRecipe(files, rows);
|
|
127
|
+
// The engine reads each input from its Blob as it restores it: the kernel never holds an input's snapshot.
|
|
128
|
+
host.kernel.setBuildInputs([...artifacts.values()].map(artifact => artifact.bytes));
|
|
129
|
+
[...artifacts.values()].forEach((artifact, input) =>
|
|
130
|
+
files.input(`/etc/dolly/artifacts/${artifact.recipeSha256}.snapshot`, input, artifact.bytes.size));
|
|
131
|
+
// The kernel restores a snapshot against the list of its paths.
|
|
132
|
+
const manifest = base?.manifest ?? [...(await scanSnapshotRecords(seed.stream(), () => false)).records.keys()];
|
|
133
|
+
files.write("/etc/dolly/image.manifest", `${manifest.join("\n")}\n`);
|
|
134
|
+
},
|
|
135
|
+
async restore({ kernel, memory, files, stage }) {
|
|
136
|
+
stage("building userspace from the Dollyfile...");
|
|
137
|
+
let status;
|
|
138
|
+
if (base) {
|
|
139
|
+
stage(`loading builder from ${baseReference.location}...`);
|
|
140
|
+
status = kernel.dolly_process_bootstrap_resume_prepare(BigInt(await stageSnapshot(kernel, memory, base.bytes)), 1);
|
|
141
|
+
} else {
|
|
142
|
+
status = kernel.dolly_process_bootstrap_prepare(BigInt(await stageSnapshot(kernel, memory, seed)));
|
|
143
|
+
seed = null;
|
|
144
|
+
}
|
|
145
|
+
let supervisor;
|
|
146
|
+
if (status === 0) {
|
|
147
|
+
supervisor = await host.kernel.supervisor(kernel);
|
|
148
|
+
status = (await supervisor.spawn(base ? ["/bin/dollyfile", locator] : ["/usr/libexec/dolly/process-bin/bootstrap"])).status;
|
|
149
|
+
}
|
|
150
|
+
for (const artifact of artifacts.values()) files.remove(`/etc/dolly/artifacts/${artifact.recipeSha256}.snapshot`);
|
|
151
|
+
artifacts.clear();
|
|
152
|
+
host.kernel.setBuildInputs([]);
|
|
153
|
+
if (status !== 0) return { status };
|
|
154
|
+
// Kernel memory never shrinks: its size now is the build's high-water mark.
|
|
155
|
+
stage(`capturing userspace image (kernel memory ${Math.round(memory.buffer.byteLength / 2 ** 20)} MiB)...`);
|
|
156
|
+
const total = Number(kernel.dolly_snapshot_capture_begin());
|
|
157
|
+
if (total === 0 || total > snapshotSizeLimit) throw new Error("Dolly system snapshot capture failed");
|
|
158
|
+
const artifact = await describeImageArtifact(capturedPieces(kernel, memory, total), recipeSha256, inputs);
|
|
159
|
+
stage("caching completed image...");
|
|
160
|
+
const saved = await saveImageArtifact(artifact, custom ? `custom:${recipe.image}` : `/${definition.dollyfile}`);
|
|
161
|
+
stage(saved ? "saved completed image artifact" : "image built; local cache unavailable");
|
|
162
|
+
if (config.buildOnly) {
|
|
163
|
+
self.postMessage({ type: "image-built", artifact });
|
|
164
|
+
return { status };
|
|
165
|
+
}
|
|
166
|
+
// The image runs here with its own modules. The page keeps its identity,
|
|
167
|
+
// which a session saved on it names, not its bytes.
|
|
168
|
+
host.require(artifact.hostRequirements);
|
|
169
|
+
host.admit(artifact.hostRequirements);
|
|
170
|
+
const { buildId, sha256: digest, byteLength } = artifact;
|
|
171
|
+
self.postMessage({ type: "image-built", artifact: { buildId, recipeSha256, sha256: digest, byteLength, inputs: artifact.inputs } });
|
|
172
|
+
built = true;
|
|
173
|
+
return { status, supervisor };
|
|
174
|
+
},
|
|
175
|
+
finish({ kernel, stage }) {
|
|
176
|
+
if (!built) return;
|
|
177
|
+
stage("finishing image bootstrap...");
|
|
178
|
+
const status = kernel.dolly_bootstrap_finish();
|
|
179
|
+
if (status !== 0) throw new Error(`Dolly bootstrap failed with status ${status}`);
|
|
180
|
+
},
|
|
181
|
+
};
|
|
182
|
+
});
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { DOLLY_IMAGE_BUILD_ID } from "../dist/dolly-image-build-id.mjs";
|
|
2
|
+
import { describeImageArtifact, loadImageArtifact, loadImageArtifactDescriptor, sha256 } from "./image-artifact.mjs";
|
|
3
|
+
import { inspectDollyfile } from "./dollyfile-view.mjs";
|
|
4
|
+
|
|
5
|
+
export async function loadCustomImage(source, descriptor) {
|
|
6
|
+
inspectDollyfile(source);
|
|
7
|
+
if (descriptor?.buildId !== DOLLY_IMAGE_BUILD_ID ||
|
|
8
|
+
descriptor.recipeSha256 !== await sha256(new TextEncoder().encode(source))) {
|
|
9
|
+
throw new Error("Completed custom image does not match this Dollyfile/runtime. Rebuild it.");
|
|
10
|
+
}
|
|
11
|
+
const artifact = await loadImageArtifact(descriptor);
|
|
12
|
+
if (!artifact) throw new Error("Completed custom image is missing or changed in the browser cache. Rebuild the exact image, or open /sessions to recover saved files. Saved sessions have not been changed.");
|
|
13
|
+
return artifact;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// Whether this browser's image cache still records the completed custom image
|
|
17
|
+
// ARTIFACT (its descriptor): a session saved on it can then be restored, which
|
|
18
|
+
// checks the image's bytes.
|
|
19
|
+
export async function customImageCached(artifact) {
|
|
20
|
+
const cached = await loadImageArtifactDescriptor(artifact.recipeSha256, artifact.inputs);
|
|
21
|
+
return cached?.sha256 === artifact.sha256 && cached.byteLength === artifact.byteLength;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export async function checkedCustomArtifact(source, candidate) {
|
|
25
|
+
inspectDollyfile(source);
|
|
26
|
+
if (candidate?.buildId !== DOLLY_IMAGE_BUILD_ID || !(candidate.bytes instanceof Blob) ||
|
|
27
|
+
candidate.recipeSha256 !== await sha256(new TextEncoder().encode(source))) {
|
|
28
|
+
throw new Error("Invalid completed custom image");
|
|
29
|
+
}
|
|
30
|
+
const artifact = await describeImageArtifact(candidate.bytes, candidate.recipeSha256, candidate.inputs);
|
|
31
|
+
if (candidate.sha256 !== artifact.sha256) throw new Error("Completed custom image integrity mismatch");
|
|
32
|
+
return artifact;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Call only from the user's Open image gesture after a build completes.
|
|
36
|
+
// Only a fixed app route is navigable; the recipe never supplies a host URL.
|
|
37
|
+
// With the Dollyfile, the tab inherits the completed artifact and what the
|
|
38
|
+
// host modules of the building page hand down (their restrictions).
|
|
39
|
+
export function openCustomImage({ source, ...inherited }) {
|
|
40
|
+
const target = window.open("about:blank", "_blank");
|
|
41
|
+
if (!target) throw new Error("Popup blocked. Choose Open image to retry.");
|
|
42
|
+
try {
|
|
43
|
+
target.sessionStorage.setItem("dolly-custom-source", source);
|
|
44
|
+
target.sessionStorage.setItem("dolly-custom-inherited", JSON.stringify(inherited));
|
|
45
|
+
target.opener = null;
|
|
46
|
+
target.location.replace(new URL("../custom/run/", import.meta.url));
|
|
47
|
+
} catch (error) {
|
|
48
|
+
target.close();
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// The custom image this tab was opened with: its Dollyfile and, after a build,
|
|
54
|
+
// what it inherits; undefined when the tab has none.
|
|
55
|
+
export function storedCustomImage() {
|
|
56
|
+
const source = sessionStorage.getItem("dolly-custom-source");
|
|
57
|
+
if (!source) return undefined;
|
|
58
|
+
return { source, ...JSON.parse(sessionStorage.getItem("dolly-custom-inherited") ?? "{}") };
|
|
59
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { inspectDollyfile, recipeFileName, imageFileName } from "./dollyfile-view.mjs";
|
|
2
|
+
import { sha256 } from "./static-asset.mjs";
|
|
3
|
+
|
|
4
|
+
const key = object => `${object.type}:${object.name}`;
|
|
5
|
+
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
6
|
+
|
|
7
|
+
// The recipe graph of one image, as /bin/dollyfile reads it: the root and the
|
|
8
|
+
// FROM/INSTALL/USE/COPY images it imports, each with its own imports. `read(url)`
|
|
9
|
+
// returns a recipe's bytes; this checks every pin. `artifacts` are the images
|
|
10
|
+
// a build of the root reads: its own rows and, before each INSTALL or USE, the rows
|
|
11
|
+
// that package's recipe installs, which the package names and does not hold. It is an inspection graph,
|
|
12
|
+
// not a dependency solver: runtime assertions may resolve against files and
|
|
13
|
+
// environment that recipes do not declare. Pass the same `recipes` cache to
|
|
14
|
+
// reuse parsed recipes across graphs.
|
|
15
|
+
export async function loadRecipeGraph(read, rootLocation, recipes = new Map()) {
|
|
16
|
+
const records = [], edges = [], artifacts = [];
|
|
17
|
+
const active = new Set(), images = new Map(), imageNames = new Map();
|
|
18
|
+
|
|
19
|
+
function parse(location) {
|
|
20
|
+
if (!recipes.has(location)) {
|
|
21
|
+
recipes.set(location, (async () => {
|
|
22
|
+
const bytes = await read(location);
|
|
23
|
+
return { ...inspectDollyfile(decoder.decode(bytes), location), sha256: await sha256(bytes) };
|
|
24
|
+
})());
|
|
25
|
+
}
|
|
26
|
+
return recipes.get(location);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async function load(location, expected, root) {
|
|
30
|
+
if (active.has(location)) throw new Error(`${location}: recipe cycle`);
|
|
31
|
+
const parsed = await parse(location);
|
|
32
|
+
if (expected && parsed.sha256 !== expected) throw new Error(`${location}: stale recipe pin`);
|
|
33
|
+
// An uploaded root is labeled "Dollyfile"; every site path names its recipe's file.
|
|
34
|
+
const file = recipeFileName(location);
|
|
35
|
+
if (file !== "" && imageFileName(file) !== parsed.name) {
|
|
36
|
+
throw new Error(`${location}: ${parsed.role.toUpperCase()} ${parsed.name} must match its file name`);
|
|
37
|
+
}
|
|
38
|
+
if ((imageNames.get(parsed.name) ?? location) !== location) {
|
|
39
|
+
throw new Error(`${location}: image ${parsed.name} is already ${imageNames.get(parsed.name)}`);
|
|
40
|
+
}
|
|
41
|
+
imageNames.set(parsed.name, location);
|
|
42
|
+
if (!root && images.has(location)) return images.get(location);
|
|
43
|
+
active.add(location);
|
|
44
|
+
const record = { ...parsed, location, dependencies: [], imports: new Map(), artifactTargets: [], installs: [], installTargets: [] };
|
|
45
|
+
records.push(record);
|
|
46
|
+
const visible = new Map();
|
|
47
|
+
const published = new Map();
|
|
48
|
+
const operations = [
|
|
49
|
+
...record.artifacts,
|
|
50
|
+
...record.requirements.map(value => ({ ...value, operation: "requirement" })),
|
|
51
|
+
...record.exports.map(value => ({ ...value, operation: "export" })),
|
|
52
|
+
].sort((a, b) => a.line - b.line);
|
|
53
|
+
for (const operation of operations) {
|
|
54
|
+
if (["from", "install", "use", "copy"].includes(operation.operation)) {
|
|
55
|
+
const target = await load(operation.location, operation.sha256, false);
|
|
56
|
+
const installs = ["install", "use"].includes(operation.operation);
|
|
57
|
+
if (operation.operation === "from" ? target.role === "package" : installs && target.role !== "package") {
|
|
58
|
+
throw new Error(`${location}:${operation.line}: ${target.name} is a ${target.role}: ` +
|
|
59
|
+
"FROM takes an application or toolchain, INSTALL and USE a package");
|
|
60
|
+
}
|
|
61
|
+
// Host requirements are never inherited: a package's must be declared here.
|
|
62
|
+
if (operation.operation === "install") {
|
|
63
|
+
const missing = target.hostRequirements.filter(name => !record.hostRequirements.includes(name));
|
|
64
|
+
if (missing.length) {
|
|
65
|
+
throw new Error(`${location}:${operation.line}: ${target.name} needs REQUIRES HOST ${missing.join(", ")}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
record.artifactTargets.push({ reference: operation, target });
|
|
69
|
+
if (root && installs) {
|
|
70
|
+
for (const needed of target.installs) {
|
|
71
|
+
if (!artifacts.some(held => held.sha256 === needed.sha256)) artifacts.push({ ...needed, line: operation.line });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
// What a package uses is the build's alone: it is not among what arrives with it.
|
|
75
|
+
if (operation.operation === "install") {
|
|
76
|
+
const row = { operation: "install", location: operation.location, sha256: operation.sha256, line: operation.line, image: target.image };
|
|
77
|
+
for (const needed of [...target.installs, row]) {
|
|
78
|
+
if (!record.installs.some(held => held.sha256 === needed.sha256)) record.installs.push({ ...needed, line: operation.line });
|
|
79
|
+
}
|
|
80
|
+
for (const needed of [...target.installTargets, target]) {
|
|
81
|
+
if (!record.installTargets.includes(needed)) record.installTargets.push(needed);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (root) artifacts.push({ ...operation, image: target.image });
|
|
85
|
+
// A package keeps nothing of its base nor of what it installs; COPY takes files only.
|
|
86
|
+
const imports = operation.operation !== "copy" && record.role !== "package";
|
|
87
|
+
const imported = installs ? [...target.installTargets, target]
|
|
88
|
+
: operation.operation === "from" ? [target] : [];
|
|
89
|
+
for (const source of imported) {
|
|
90
|
+
for (const [name, provider] of source.scopeExporters) {
|
|
91
|
+
visible.set(name, provider);
|
|
92
|
+
if (imports) published.set(name, provider);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
} else if (operation.operation === "export") {
|
|
96
|
+
if (operation.details.length !== 0 || !visible.has(key(operation))) {
|
|
97
|
+
visible.set(key(operation), { module: record, exported: operation });
|
|
98
|
+
}
|
|
99
|
+
} else if (operation.operation === "requirement") {
|
|
100
|
+
const requirement = record.requirements.find(item => item.line === operation.line);
|
|
101
|
+
const provider = visible.get(key(operation));
|
|
102
|
+
if (provider) {
|
|
103
|
+
const edge = { consumer: record, requirement, provider: provider.module, exported: provider.exported };
|
|
104
|
+
edges.push(edge);
|
|
105
|
+
record.dependencies.push(edge);
|
|
106
|
+
record.imports.set(key(operation), provider);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// A recipe's own exports describe its completed state and win over imported
|
|
111
|
+
// ones. Bare TOOL and ENV names resolve against the last provider seen: a
|
|
112
|
+
// package, an inherited base, or runtime state.
|
|
113
|
+
for (const exported of record.exports) {
|
|
114
|
+
const provider = visible.get(key(exported));
|
|
115
|
+
const resolved = (exported.details.length === 0 || exported.type === "ENV") && provider && provider.module !== record
|
|
116
|
+
? { ...exported, details: provider.exported.details } : exported;
|
|
117
|
+
published.set(key(exported), { module: record, exported: resolved });
|
|
118
|
+
}
|
|
119
|
+
record.scopeExporters = published;
|
|
120
|
+
if (!root) images.set(location, record);
|
|
121
|
+
active.delete(location);
|
|
122
|
+
return record;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const root = await load(rootLocation, null, true);
|
|
126
|
+
return { root, records, edges, exporters: root.scopeExporters, artifacts };
|
|
127
|
+
}
|