@daugasauron/dolly-build 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +259 -6
  2. package/host/requirements.mjs +1 -1
  3. package/host/runtime/abi.mjs +41 -0
  4. package/package.json +10 -4
  5. package/packages.json +101 -0
  6. package/{src → page}/image-entry.mjs +10 -6
  7. package/{src → page}/image-inputs.mjs +11 -0
  8. package/{src → page}/image-names.mjs +2 -2
  9. package/page/sha256.mjs +65 -0
  10. package/{src → page}/snapshot-records.mjs +63 -44
  11. package/{src → page}/static-asset.mjs +36 -20
  12. package/{src → page}/version.mjs +1 -1
  13. package/scripts/dolly-build-init/Dollyfile +3 -7
  14. package/scripts/dolly-build-init/public/icon.svg +1 -0
  15. package/scripts/dolly-build-init/public/index.html +6 -58
  16. package/scripts/dolly-build-init/public/main.mjs +52 -0
  17. package/scripts/dolly-build-init/public/page.css +3 -0
  18. package/scripts/dolly-build-init/public/skills/changelog/SKILL.md +5 -0
  19. package/scripts/dolly-build-init/workflow.yml +42 -0
  20. package/scripts/dolly-build.mjs +194 -62
  21. package/scripts/dollyfile-graph.mjs +3 -3
  22. package/scripts/image-directory.mjs +6 -7
  23. package/scripts/page-image-build.mjs +29 -10
  24. package/scripts/recipe-files.mjs +3 -3
  25. package/scripts/release-layout.mjs +1 -1
  26. package/scripts/serve.mjs +40 -10
  27. package/scripts/sha256.mjs +4 -0
  28. package/scripts/share-pages-snapshots.mjs +10 -30
  29. package/scripts/snapshot-identity.mjs +1 -2
  30. package/scripts/system-snapshot-format.mjs +11 -9
  31. package/scripts/wasm-interface.mjs +2 -2
  32. package/{src → site}/dollyfile-graph.mjs +1 -1
  33. package/{src → site}/dollyfile-view.mjs +1 -1
  34. /package/{src → page}/process-abi.mjs +0 -0
  35. /package/{src → page}/wasm-interface.mjs +0 -0
  36. /package/{src → site}/alphabetical.mjs +0 -0
package/README.md CHANGED
@@ -1,12 +1,265 @@
1
- # @daugasauron/dolly-build
1
+ # Dolly from npm
2
2
 
3
- A project with a Dolly image of your own: a Dollyfile `FROM` the installed
4
- `@daugasauron/dolly-image-pi-minimal` with the packages it INSTALLs (`@daugasauron/dolly-package-NAME`)
5
- and your files, built in a browser into a site that any static host serves as it lies.
6
- @daugasauron/dolly's README (`docs/embedding.md`) says how.
3
+ Dolly is a POSIX-like userspace for wasm64 that runs in a browser tab. Its npm
4
+ packages hold the runtime, prebuilt programs and a build command, so a project
5
+ of your own needs no checkout of Dolly: `npm install` is its only download.
6
+ This document is the guide to them. Its main case is a coding agent of your
7
+ own: [Pi](https://github.com/earendil-works/pi) in a sandbox with the tools you
8
+ choose and your skills, served from your machine, and built into the same
9
+ static site in CI. What changed between versions is in the
10
+ [npm changelog](npm-changelog.md), and how to move a project to a new version
11
+ in [Upgrading](npm-upgrading.md).
12
+
13
+ ## A Pi agent of your own
14
+
15
+ With Node.js 22, npm, Chrome and Python (for its static file server), in an
16
+ empty directory:
7
17
 
8
18
  ```sh
9
- npx dolly-build init my-pi git
19
+ npm init --yes
20
+ npm install @daugasauron/dolly-build
21
+ npx dolly-build init my-pi git python nvim
10
22
  npx dolly-build Dollyfile-my-pi my-pi
11
23
  python3 -m http.server -d my-pi -b 127.0.0.1 8080
12
24
  ```
25
+
26
+ - `init` writes the project: Pi with Git, Python and Neovim here. It installs
27
+ what the project needs and lacks with npm (the packages named, the host
28
+ modules they declare), at `dolly-build`'s version, and prints that command.
29
+ - The second `dolly-build` builds the image in a headless browser (45 s for
30
+ this project on a 16-core desktop) and writes the site `my-pi/` (94 MB).
31
+ - Python's server serves it. Open <http://127.0.0.1:8080/>: Pi asks for your
32
+ OpenRouter key at a prompt that does not show it, and starts in `/workspace`.
33
+ `curl https://example.com` in Pi's shell fails: the page's policy admits the
34
+ model's provider and nothing else. Ctrl+D leaves Pi for a shell. When Pi needs
35
+ a file of yours it asks with its `upload` tool: the page's file dialog opens,
36
+ and Pi gets the file's path and size, or that you cancelled. Save
37
+ (Ctrl+Shift+S) keeps the sandbox, files and all, in this browser and names it
38
+ in the address (`?session=NAME`), which opens it again as it was.
39
+
40
+ After `npm install` and `init`, nothing is downloaded: the build, its browser,
41
+ the site and its page fetch from `node_modules` and from each other alone. The
42
+ build's browser reaches only a server the command runs on 127.0.0.1; its own
43
+ requests (its vendor's updates, clock and autofill) go to a proxy of the
44
+ command's that refuses them.
45
+
46
+ ## The project
47
+
48
+ | File | What it declares | After a change |
49
+ | --- | --- | --- |
50
+ | `Dollyfile-my-pi` | The sandbox, a [recipe](dollyfile.md): `INSTALL` rows for Pi's packages and the ones you named, each pinned by its recipe's SHA-256; `FILE` rows with Pi's `AGENTS.md`, an extension with a `/hello` command, a `todo` tool, the model's entry and the script that starts Pi with its provider and model | Build again |
51
+ | `public/skills/` | Pi's skills, one directory each: a `SKILL.md` and the files it names. The page writes them into `~/.pi/agent/skills/` at each boot | Build again (the image is kept) |
52
+ | `public/workspace/` | The project the agent works on, written into `/workspace` at each boot | Build again (the image is kept) |
53
+ | `public/main.mjs` | The hosts the agent may reach ([HTTP policy](http.md)): one rule per host, OpenRouter's API to begin with. The files the page writes at boot | Build again (the image is kept) |
54
+ | `public/index.html`, `public/page.css`, `public/icon.svg` | The page, and the popups' font. No inline script or style, so the host may serve a strict Content-Security-Policy ([Embedding](embedding.md)) | Build again (the image is kept) |
55
+ | `public/theme.json` | The sandbox's colours: five values in Pi's theme format, whose background, text, accent and muted are the terminal's and the popups' too | Build again (the image is kept) |
56
+ | `.github/workflows/my-pi.yml` | The same build in GitHub Actions ([CI](#ci)) | |
57
+ | `package.json`, `package-lock.json` | The packages, at exact versions | `npm install` |
58
+ | `my-pi/` | The site `dolly-build` writes, which `.gitignore` leaves out | Reload |
59
+
60
+ A build that keeps the image takes seconds: the image is built again only when
61
+ the recipe or the installed runtime changed.
62
+
63
+ ## Tools
64
+
65
+ A tool is a package: a program built inside Dolly from its upstream source,
66
+ with the packages it needs. `init` takes the ones to start with, and `add`
67
+ puts more into the recipe of a project, installing them first:
68
+
69
+ ```sh
70
+ npx dolly-build add my-pi jq sqlite
71
+ ```
72
+
73
+ | Package | What it brings |
74
+ | --- | --- |
75
+ | `git` | Git over the page's HTTP broker, with `diff` and `patch` |
76
+ | `python` | CPython 3.14 with pip; it brings `cc` and `libffi` |
77
+ | `numpy`, `pandas`, `matplotlib`, `seaborn`, `python-dateutil` | Python's data libraries, each with the ones it needs and `python` |
78
+ | `nvim` | Neovim, with its runtime files and tree-sitter parsers |
79
+ | `cc` | The C and C++ compiler, with `ld`, `ar`, Make and the headers |
80
+ | `jq`, `sqlite`, `less` | `jq`, `sqlite3` and the pager |
81
+ | Pi's own, which `init` always installs | `core` (the shell `slop` and its commands), `posix` (grep, sed, awk, find and the others), `curl`, `display` (the terminal), `dollyfile` (the recipe engine) and `pi-dolly` (Pi, its JavaScript runtime, ripgrep, fd, the `download` and `upload` commands, and Pi's Dolly skill and tools, among them `download` and `upload`, which hand the user a file and ask the user for one) |
82
+
83
+ `npx dolly-build init` with a name that is no package lists them all. A program
84
+ that is not among them is built in the recipe, where a compiler is: a `FILE`
85
+ row with a C source and `SLOP cc /tmp/NAME.c -o /usr/bin/NAME`, or
86
+ `SLOP pip install NAME==VERSION` followed by `FOLDER
87
+ /usr/lib/python3.14/site-packages`, which keeps what pip installed (pure-Python
88
+ wheels and sdists install). A script is a `FILE` row under `/usr/bin` with
89
+ `EXPORTS TOOL NAME`, as the template's `todo` is. The recipe's comments show
90
+ each.
91
+
92
+ A package declares the host modules its programs need (`python` declares
93
+ `sockets`), and a recipe's `REQUIRES HOST` rows must name them: `init` and
94
+ `add` write those rows and install the modules' packages
95
+ (`@daugasauron/dolly-host-NAME`). The site holds the modules its recipe
96
+ declares and no other.
97
+
98
+ ## Skills and instructions
99
+
100
+ - A skill is a directory under `public/skills/` with a `SKILL.md`: front matter
101
+ with its `name` and `description`, then what the agent does when a task
102
+ matches the description (Pi's `docs/skills.md`, in the image under
103
+ `/usr/lib/node_modules/@earendil-works/pi-coding-agent/docs/`). Scripts and
104
+ other files a skill names lie beside it and are at
105
+ `~/.pi/agent/skills/NAME/` in the sandbox. Pi lists the skills when it
106
+ starts. `init` writes one, `changelog`.
107
+ - `AGENTS.md`, the instructions Pi takes into every session, is a `FILE` row of
108
+ the recipe; so are an extension (`/hello`), the model's entry
109
+ (`models.json`) and the script that starts Pi (`/etc/dolly/init.slop`).
110
+ [An agent in place of a web frontend](agent-frontend.md) goes on from here:
111
+ your backend's API, another model provider, how Pi starts.
112
+ - These files are not secrets: Pi's tools can read them, and so can its model.
113
+
114
+ ## The policy and the key
115
+
116
+ `public/main.mjs` holds the page's HTTP policy: the hosts the sandbox may
117
+ reach, one rule each, with the header that may carry a credential. It is
118
+ enforced in the browser, outside the sandbox ([HTTP](http.md)). `init` writes
119
+ one rule, OpenRouter's API, and Anthropic's as a comment beside it. The key is
120
+ typed at `pi-key`'s prompt in the sandbox each time it starts; it is in no
121
+ file of the site. A host the agent should reach (`api.github.com`, `pypi.org`)
122
+ is one more rule.
123
+
124
+ ## Build
125
+
126
+ `dolly-build Dollyfile-NAME OUT` writes the site `OUT`, replacing the one it
127
+ wrote there before: the files of `public/`, `workspace.json` and `skills.json`
128
+ (the files under `public/workspace/` and `public/skills/`), the runtime
129
+ (`dolly/`), the device host modules the recipe declares (`dolly-host-NAME/`),
130
+ `coi-serviceworker.js` and `image/`, the image's directory (its catalog,
131
+ metadata and packs, the recipes it was built from, its `amy-index.txt` and an
132
+ `index.mjs`).
133
+
134
+ - The build runs in a browser, with the installed runtime's own builder.
135
+ Without `--browser` it takes the first this machine has of Chrome, Playwright's
136
+ Chromium and Playwright's Firefox; `--browser chrome`, `chromium` or `firefox`
137
+ names one (`npx playwright-core install chromium-headless-shell` or `firefox`
138
+ installs Playwright's). It needs Chrome 138 or Firefox 146 or later, as the
139
+ page does.
140
+ - Both commands run in the directory whose `node_modules/@daugasauron/` holds
141
+ the packages (npm, pnpm and Yarn 1 alike; Yarn 2 and later need `nodeLinker:
142
+ node-modules`). `init` and `add` install with the package manager that runs
143
+ them (`npm install`, `pnpm add` or `yarn add`); their install is measured
144
+ with npm alone.
145
+ - A recipe's steps reach the network only where the recipe says so: a `SOURCE`
146
+ row, or a `SLOP` step such as `pip install`. The browser hands each such
147
+ request to `dolly-build`, which fetches it in Node, so a host without CORS
148
+ headers works, and through the proxy `HTTPS_PROXY` or `HTTP_PROXY` names; the
149
+ log names each host a build reaches. What a step fetches is not part of the
150
+ image's identity: pin versions (`pip install NAME==1.2`) where the bytes
151
+ matter. `init`'s recipe has no such step.
152
+ - A recipe is at most 128 KiB. A `FILE` under `/workspace` is part of the
153
+ image; the page's `public/workspace/` is written over it at each fresh boot.
154
+
155
+ ## CI
156
+
157
+ `init` writes `.github/workflows/my-pi.yml`: on a push to `main`, a pull
158
+ request or by hand, it checks the project out, runs `npm ci`, installs
159
+ Playwright's headless Chromium (the version `package-lock.json` pins, with the
160
+ system libraries it needs), builds the site with it and uploads it as the
161
+ artifact `github-pages`:
162
+
163
+ ```yaml
164
+ - run: npm ci
165
+ - run: npx playwright-core install --with-deps chromium-headless-shell
166
+ - run: npx dolly-build Dollyfile-my-pi my-pi --browser chromium
167
+ - uses: actions/upload-pages-artifact@v3
168
+ with:
169
+ path: my-pi
170
+ ```
171
+
172
+ Commit the project as `init` left it (`package-lock.json` too; `.gitignore`
173
+ leaves out `node_modules/` and the site). The file ends with a `deploy` job as
174
+ comments, which publishes the site on GitHub Pages once the repository's
175
+ Pages source is "GitHub Actions". Pages sends no isolation headers; the page
176
+ isolates itself ([Serving](#serving)).
177
+
178
+ Elsewhere the same three commands are the job. In Playwright's container image
179
+ of the version of `playwright-core` the lockfile holds
180
+ (`mcr.microsoft.com/playwright:vVERSION-noble`), the browser is there already.
181
+ After `npm ci` and the browser, a CI build downloads nothing either.
182
+ `PLAYWRIGHT_DOWNLOAD_HOST` points the browser's download at a mirror.
183
+
184
+ How this is checked: `test/npm-browser.mjs` commits the project `init` wrote,
185
+ clones it, and runs the workflow's own `npm ci` and build lines in the clone,
186
+ in a network namespace whose only network is loopback, with Chrome and with
187
+ Firefox; the site's image is the one the project built. The workflow file has
188
+ not been run on GitHub's runners by Dolly's tests.
189
+
190
+ ## Serving
191
+
192
+ The site is static: any file server serves it as it lies, from any path.
193
+
194
+ - Dolly needs shared memory, which a browser gives only to a cross-origin
195
+ isolated page. Where the host sends `Cross-Origin-Opener-Policy: same-origin`
196
+ and `Cross-Origin-Embedder-Policy: require-corp`, the page is isolated. Where
197
+ it does not (Python's server, GitHub Pages), `init`'s page registers
198
+ `coi-serviceworker.js`, which adds them to the site's own responses, and
199
+ loads once more under it. A service worker needs HTTPS or `127.0.0.1`.
200
+ - The host must serve `.wasm` as `application/wasm`, `.mjs` as
201
+ `text/javascript` and `.json` as `application/json`.
202
+ - The image's packs stay in the origin's private storage after the first
203
+ visit, so the page starts again without downloading them.
204
+
205
+ ## Updating
206
+
207
+ The packages of one project are of one version: `init` and `add` install at
208
+ `dolly-build`'s, and a build with two versions fails, naming both. To move a
209
+ project to a new version, install `dolly-build` at it; `add` then installs the
210
+ project's other packages at that version and writes the recipe's pins again:
211
+
212
+ ```sh
213
+ npm install --save-exact @daugasauron/dolly-build@0.1.7
214
+ npx dolly-build add my-pi
215
+ npx dolly-build Dollyfile-my-pi my-pi
216
+ ```
217
+
218
+ `add` writes every `FROM` and `INSTALL` row at the pin of the installed
219
+ package that holds its recipe. A row whose pin no installed package holds
220
+ fails the build, naming the row.
221
+
222
+ ## The packages
223
+
224
+ | Package | What it holds |
225
+ | --- | --- |
226
+ | `@daugasauron/dolly-build` | The `dolly-build` command. It depends on the runtime, on `playwright-core`, which starts the build's browser, and on Pi's packages |
227
+ | `@daugasauron/dolly` | The runtime: `boot()`, the kernel, the process runtime, the image builder, and the host modules that are its own machinery ([Embedding](embedding.md)) |
228
+ | `@daugasauron/dolly-host-NAME` | One device host module each: `display`, `input`, `html`, `sockets`, `gpu`, `audio`, `microphone`, `touch`, `clipboard`, `upload`, `pty`, `console`, and `save`, which is `download@0` (npm takes no new name with that word) |
229
+ | `@daugasauron/dolly-package-NAME` | A package for a recipe to `INSTALL` ([Tools](#tools)): its catalog, metadata, packs and recipes. It depends on the packages it installs, so npm brings them |
230
+ | `@daugasauron/dolly-image-NAME` | A whole image for `boot()`: `default` (a shell and its tools), `pi-minimal` (Pi alone) and `python-data` (Python with NumPy, pandas, Matplotlib and seaborn) |
231
+
232
+ Each host module and image depends on the runtime alone. An image or package
233
+ does not depend on the host modules it declares: the page chooses its modules.
234
+
235
+ ## A recipe on an image
236
+
237
+ A recipe may start `FROM` an installed image instead of installing packages:
238
+ the image package's README gives the row. `FROM` carries the base's files,
239
+ environment and exports but not its host modules or `ENTRY`, so the recipe
240
+ repeats those. `init` writes Pi's project only; the page in `public/` is then
241
+ [Embedding](embedding.md)'s, importing `./dolly/page/browser.mjs` and
242
+ `./image/index.mjs`, with the lines of `init`'s `main.mjs` that register
243
+ `coi-serviceworker.js`.
244
+
245
+ ```sh
246
+ npm install @daugasauron/dolly-build # with the install line of the image package's README
247
+ npx dolly-build Dollyfile-my-data my-data
248
+ ```
249
+
250
+ To run a prebuilt image in a page with no build at all, see
251
+ [Embedding](embedding.md).
252
+
253
+ ## When something fails
254
+
255
+ | Message | What to do |
256
+ | --- | --- |
257
+ | `NAME: no such package. The packages: …` | `init` and `add` take the names listed |
258
+ | `that install failed` | Run the printed `npm install` line yourself (a registry or proxy setting), then the command again |
259
+ | `no installed package holds … at that pin` | The packages were updated: `npx dolly-build add NAME` |
260
+ | `… holds X at another pin than …: install packages of one version` | Install every `@daugasauron` package at one version ([Updating](#updating)) |
261
+ | `NAME declares sockets: install @daugasauron/dolly-host-sockets` | A `REQUIRES HOST` row written by hand names a module that is not installed |
262
+ | `a browser is not installed` | Install Chrome, or `npx playwright-core install chromium-headless-shell` |
263
+ | `this browser has no WebAssembly memory64` | The build's browser is older than Chrome 138 or Firefox 146 |
264
+ | `Dollyfile-NAME:LINE: unknown directive` | A `FILE` row's body lines start with four spaces |
265
+ | The page says it is not cross-origin isolated | It is served over plain HTTP from another address than `127.0.0.1`: serve it over HTTPS, or send the two headers |
@@ -1,6 +1,6 @@
1
1
  import { DOLLY_HOST_RECORD_BYTES as recordBytes, DOLLY_HOST_NAME_BYTES as nameBytes,
2
2
  DOLLY_HOST_DIGEST_BYTES as digestBytes, DOLLY_HOST_MAX_RECORDS as maxRecords } from "./abi.mjs";
3
- import { hex } from "../src/static-asset.mjs";
3
+ import { hex } from "../page/static-asset.mjs";
4
4
 
5
5
  const pattern = /^([a-z][a-z0-9-]{0,30})@(0|[1-9][0-9]{0,4})$/;
6
6
 
@@ -0,0 +1,41 @@
1
+ // Generated from abi/dolly-supervisor-0.wat, abi/dolly-image-0.wat, abi/dolly-process-0.wat, abi/dolly-process-gate-0.wat, include/dolly/process.h.
2
+ export const DOLLY_PROCESS_EVENT_LAUNCH = 1;
3
+ export const DOLLY_PROCESS_EVENT_INTERRUPT = 2;
4
+ export const DOLLY_PROCESS_EVENT_TERMINATE = 3;
5
+ export const DOLLY_PROCESS_EVENT_WAKE = 4;
6
+ export const DOLLY_PROCESS_EVENT_KIND_SHIFT = 48;
7
+ export const DOLLY_PROCESS_EVENT_VALUE_SHIFT = 32;
8
+ export const DOLLY_TERMINAL_WORD_RESULT_SEQUENCE = 0;
9
+ export const DOLLY_TERMINAL_WORD_RESULT_STATUS = 1;
10
+ export const DOLLY_TERMINAL_WORD_FOREGROUND_PID = 2;
11
+ export const DOLLY_TERMINAL_WORD_FOREGROUND_INTERRUPTIBLE = 3;
12
+ export const DOLLY_TERMINAL_WORD_INTERRUPT_SEQUENCE = 4;
13
+ export const DOLLY_TERMINAL_WORD_INTERRUPT_TARGET_PID = 5;
14
+ export const DOLLY_FS_BLOCK_SIZE = 1048576;
15
+ export const DOLLY_DEVICE_HEADER_BYTES = 32;
16
+ export const DOLLY_DEVICE_REPLY_HEADER_BYTES = 64;
17
+ export const DOLLY_DEVICE_REPLY_WORD_STATE = 0;
18
+ export const DOLLY_DEVICE_REPLY_WORD_SCOPE = 1;
19
+ export const DOLLY_DEVICE_REPLY_WORD_SEQUENCE = 2;
20
+ export const DOLLY_DEVICE_REPLY_WORD_ERRNO = 3;
21
+ export const DOLLY_DEVICE_REPLY_WORD_LENGTH = 4;
22
+ export const DOLLY_DEVICE_REPLY_PUBLISHED = 1;
23
+ export const DOLLY_SNAPSHOT_VERSION = 2;
24
+ export const DOLLY_SNAPSHOT_HEADER_BYTES = 16;
25
+ export const DOLLY_SNAPSHOT_RECORD_HEADER_BYTES = 16;
26
+ export const DOLLY_SNAPSHOT_RECORD_DIRECTORY = 1;
27
+ export const DOLLY_SNAPSHOT_RECORD_FILE = 2;
28
+ export const DOLLY_SNAPSHOT_RECORD_SYMLINK = 3;
29
+ export const DOLLY_SNAPSHOT_RECORD_DELETED = 4;
30
+ export const DOLLY_SNAPSHOT_MAX_PATHS = 100000;
31
+ export const DOLLY_SNAPSHOT_MAX_MEBIBYTES = 2048;
32
+ export const DOLLY_SNAPSHOT_PATH_MIN_BYTES = 2;
33
+ export const DOLLY_SNAPSHOT_PATH_MAX_BYTES = 4095;
34
+ export const DOLLY_SNAPSHOT_SYMLINK_MIN_BYTES = 1;
35
+ export const DOLLY_SNAPSHOT_SYMLINK_MAX_BYTES = 4095;
36
+ export const DOLLY_ENTRY_VERSION = 1;
37
+ export const DOLLY_ENTRY_HEADER_BYTES = 16;
38
+ export const DOLLY_ENTRY_MAX_ARGUMENTS = 256;
39
+ export const DOLLY_ENTRY_ARGUMENT_MAX_BYTES = 4096;
40
+ export const DOLLY_ENTRY_MAX_BYTES = 65536;
41
+ export const DOLLY_ERRNO = Object.freeze({ E2BIG: 1, EACCES: 2, EADDRINUSE: 3, EADDRNOTAVAIL: 4, EAFNOSUPPORT: 5, EAGAIN: 6, EALREADY: 7, EBADF: 8, EBADMSG: 9, EBUSY: 10, ECANCELED: 11, ECHILD: 12, ECONNABORTED: 13, ECONNREFUSED: 14, ECONNRESET: 15, EDEADLK: 16, EDESTADDRREQ: 17, EDOM: 18, EDQUOT: 19, EEXIST: 20, EFAULT: 21, EFBIG: 22, EHOSTUNREACH: 23, EIDRM: 24, EILSEQ: 25, EINPROGRESS: 26, EINTR: 27, EINVAL: 28, EIO: 29, EISCONN: 30, EISDIR: 31, ELOOP: 32, EMFILE: 33, EMLINK: 34, EMSGSIZE: 35, EMULTIHOP: 36, ENAMETOOLONG: 37, ENETDOWN: 38, ENETRESET: 39, ENETUNREACH: 40, ENFILE: 41, ENOBUFS: 42, ENODEV: 43, ENOENT: 44, ENOEXEC: 45, ENOLCK: 46, ENOLINK: 47, ENOMEM: 48, ENOMSG: 49, ENOPROTOOPT: 50, ENOSPC: 51, ENOSYS: 52, ENOTCONN: 53, ENOTDIR: 54, ENOTEMPTY: 55, ENOTRECOVERABLE: 56, ENOTSOCK: 57, ENOTTY: 59, ENXIO: 60, EOVERFLOW: 61, EOWNERDEAD: 62, EPERM: 63, EPIPE: 64, EPROTO: 65, EPROTONOSUPPORT: 66, EPROTOTYPE: 67, ERANGE: 68, EROFS: 69, ESPIPE: 70, ESRCH: 71, ESTALE: 72, ETIMEDOUT: 73, ETXTBSY: 74, EXDEV: 75, ENODATA: 116, ENOTSUP: 138 });
package/package.json CHANGED
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "0.1.5",
2
+ "version": "0.1.6",
3
3
  "type": "module",
4
4
  "name": "@daugasauron/dolly-build",
5
5
  "license": "MIT",
@@ -7,10 +7,16 @@
7
7
  "dolly-build": "scripts/dolly-build.mjs"
8
8
  },
9
9
  "dependencies": {
10
- "@daugasauron/dolly": "0.1.5",
11
- "playwright-core": "1.63.0"
10
+ "@daugasauron/dolly": "0.1.6",
11
+ "playwright-core": "1.63.0",
12
+ "@daugasauron/dolly-package-core": "0.1.6",
13
+ "@daugasauron/dolly-package-dollyfile": "0.1.6",
14
+ "@daugasauron/dolly-package-posix": "0.1.6",
15
+ "@daugasauron/dolly-package-display": "0.1.6",
16
+ "@daugasauron/dolly-package-curl": "0.1.6",
17
+ "@daugasauron/dolly-package-pi-dolly": "0.1.6"
12
18
  },
13
- "description": "Build a Dolly image of your own, FROM an installed image package, in a browser, into a static site.",
19
+ "description": "Build a Dolly image of your own from installed packages, in a browser, into a static site.",
14
20
  "repository": {
15
21
  "type": "git",
16
22
  "url": "git+https://github.com/daugasauron/dolly.git"
package/packages.json ADDED
@@ -0,0 +1,101 @@
1
+ {
2
+ "packages": {
3
+ "cc": [],
4
+ "core": [],
5
+ "curl": [],
6
+ "display": [
7
+ "display"
8
+ ],
9
+ "dolly-docs": [],
10
+ "dollyfile": [],
11
+ "fd": [],
12
+ "git": [],
13
+ "gzip": [],
14
+ "html": [
15
+ "html"
16
+ ],
17
+ "janis": [
18
+ "download"
19
+ ],
20
+ "janis-install": [
21
+ "download"
22
+ ],
23
+ "jq": [],
24
+ "less": [],
25
+ "libdolly-audio": [],
26
+ "libdolly-clipboard": [],
27
+ "libdolly-display": [],
28
+ "libdolly-download": [],
29
+ "libdolly-gpu": [],
30
+ "libdolly-input": [],
31
+ "libdolly-microphone": [],
32
+ "libdolly-nena": [],
33
+ "libdolly-packages": [],
34
+ "libdolly-pty": [],
35
+ "libdolly-sockets": [],
36
+ "libdolly-threads": [],
37
+ "libdolly-touch": [],
38
+ "libdolly-upload": [],
39
+ "libffi": [],
40
+ "matplotlib": [
41
+ "sockets"
42
+ ],
43
+ "numpy": [
44
+ "sockets"
45
+ ],
46
+ "nvim": [],
47
+ "pandas": [
48
+ "sockets"
49
+ ],
50
+ "pi": [
51
+ "download"
52
+ ],
53
+ "pi-dolly": [
54
+ "download",
55
+ "html",
56
+ "upload"
57
+ ],
58
+ "posix": [],
59
+ "python": [
60
+ "sockets"
61
+ ],
62
+ "python-dateutil": [
63
+ "sockets"
64
+ ],
65
+ "quickjs": [],
66
+ "ripgrep": [],
67
+ "sbase": [],
68
+ "seaborn": [
69
+ "sockets"
70
+ ],
71
+ "secretenv": [],
72
+ "sqlite": [],
73
+ "transfer": [
74
+ "download",
75
+ "upload"
76
+ ]
77
+ },
78
+ "images": {
79
+ "default": [
80
+ "display",
81
+ "download",
82
+ "input",
83
+ "upload"
84
+ ],
85
+ "pi-minimal": [
86
+ "display",
87
+ "download",
88
+ "html",
89
+ "input",
90
+ "upload"
91
+ ],
92
+ "python-data": [
93
+ "display",
94
+ "download",
95
+ "html",
96
+ "input",
97
+ "sockets",
98
+ "upload"
99
+ ]
100
+ }
101
+ }
@@ -1,22 +1,26 @@
1
- // Shared by packaging and browser admission; empty arguments are valid.
1
+ import { DOLLY_ENTRY_VERSION, DOLLY_ENTRY_HEADER_BYTES, DOLLY_ENTRY_MAX_ARGUMENTS, DOLLY_ENTRY_ARGUMENT_MAX_BYTES,
2
+ DOLLY_ENTRY_MAX_BYTES } from "../host/runtime/abi.mjs";
3
+
4
+ // /etc/dolly/entry, as abi/dolly-image-0.wat lays it out. Shared by packaging and
5
+ // browser admission; empty arguments are valid.
2
6
  export function decodeImageEntry(bytes) {
3
- if (!(bytes instanceof Uint8Array) || bytes.byteLength < 16 || bytes.byteLength > 64 * 1024) {
7
+ if (!(bytes instanceof Uint8Array) || bytes.byteLength < DOLLY_ENTRY_HEADER_BYTES || bytes.byteLength > DOLLY_ENTRY_MAX_BYTES) {
4
8
  throw new TypeError("invalid image ENTRY record");
5
9
  }
6
10
  const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
7
11
  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
8
- if (decoder.decode(bytes.subarray(0, 8)) !== "DOLLYENT" || view.getUint32(8, true) !== 1) {
12
+ if (decoder.decode(bytes.subarray(0, 8)) !== "DOLLYENT" || view.getUint32(8, true) !== DOLLY_ENTRY_VERSION) {
9
13
  throw new TypeError("invalid image ENTRY version");
10
14
  }
11
15
  const count = view.getUint32(12, true);
12
- if (count === 0 || count > 256) throw new TypeError("invalid image ENTRY count");
13
- let offset = 16;
16
+ if (count === 0 || count > DOLLY_ENTRY_MAX_ARGUMENTS) throw new TypeError("invalid image ENTRY count");
17
+ let offset = DOLLY_ENTRY_HEADER_BYTES;
14
18
  const arguments_ = [];
15
19
  for (let index = 0; index < count; ++index) {
16
20
  if (offset > bytes.byteLength - 4) throw new TypeError("truncated image ENTRY");
17
21
  const size = view.getUint32(offset, true);
18
22
  offset += 4;
19
- if (size > 4096 || offset > bytes.byteLength - size) throw new TypeError("invalid image ENTRY argument");
23
+ if (size > DOLLY_ENTRY_ARGUMENT_MAX_BYTES || offset > bytes.byteLength - size) throw new TypeError("invalid image ENTRY argument");
20
24
  const value = decoder.decode(bytes.subarray(offset, offset + size));
21
25
  if (value.includes("\0")) throw new TypeError("image ENTRY has a NUL argument");
22
26
  arguments_.push(value);
@@ -1,3 +1,5 @@
1
+ import { Sha256 } from "./sha256.mjs";
2
+
1
3
  // Recipe pins identify instructions; artifact digests identify their results.
2
4
  export function imageInputs(artifacts = []) {
3
5
  if (!Array.isArray(artifacts) || artifacts.length > 256) throw new Error("invalid image inputs");
@@ -18,3 +20,12 @@ export function imageInputs(artifacts = []) {
18
20
  export function imageInputsMatch(actual, expected) {
19
21
  return JSON.stringify(imageInputs(actual)) === JSON.stringify(imageInputs(expected));
20
22
  }
23
+
24
+ // The catalog's digest of an image's recipe records (its metadata's recipes),
25
+ // one line each, "SHA256 BYTES PATH\n", in their order: a start checks the
26
+ // records against the catalog without the catalog repeating them.
27
+ export function recipesDigest(recipes) {
28
+ const hash = new Sha256();
29
+ hash.update(new TextEncoder().encode(recipes.map(({ sha256, byteLength, sourcePath }) => `${sha256} ${byteLength} ${sourcePath}\n`).join("")));
30
+ return hash.hex();
31
+ }
@@ -5,7 +5,7 @@
5
5
  // Dollyfile-example.txt is never a recipe.
6
6
  export const validName = name => /^[a-z][a-z0-9]*(?:-[a-z0-9]+|\.[0-9]+)*$/.test(name) && name.length <= 32;
7
7
 
8
- // No image retains scratch space.
8
+ // No image retains scratch space, /tmp.
9
9
  export function unretainedPath(value) {
10
- return ["/tmp", "/workspace"].some((prefix) => value === prefix || value.startsWith(`${prefix}/`));
10
+ return value === "/tmp" || value.startsWith("/tmp/");
11
11
  }
@@ -0,0 +1,65 @@
1
+ // SHA-256 over bytes given in pieces, for a digest of more bytes than the page
2
+ // should hold at once; WebCrypto digests only a whole buffer.
3
+ const K = Int32Array.of(
4
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
5
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
6
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
7
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
8
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
9
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
10
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
11
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2);
12
+
13
+ export class Sha256 {
14
+ state = Int32Array.of(0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19);
15
+ block = new Uint8Array(64);
16
+ used = 0;
17
+ length = 0;
18
+ words = new Int32Array(64);
19
+
20
+ compress(view, offset) {
21
+ const w = this.words, s = this.state;
22
+ for (let i = 0; i < 16; i++) w[i] = view.getInt32(offset + 4 * i);
23
+ for (let i = 16; i < 64; i++) {
24
+ const x = w[i - 15], y = w[i - 2];
25
+ w[i] = (((x >>> 7 | x << 25) ^ (x >>> 18 | x << 14) ^ x >>> 3) + w[i - 7] +
26
+ ((y >>> 17 | y << 15) ^ (y >>> 19 | y << 13) ^ y >>> 10) + w[i - 16]) | 0;
27
+ }
28
+ let a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7];
29
+ for (let i = 0; i < 64; i++) {
30
+ const t1 = (h + ((e >>> 6 | e << 26) ^ (e >>> 11 | e << 21) ^ (e >>> 25 | e << 7)) + (e & f ^ ~e & g) + K[i] + w[i]) | 0;
31
+ const t2 = (((a >>> 2 | a << 30) ^ (a >>> 13 | a << 19) ^ (a >>> 22 | a << 10)) + (a & b ^ a & c ^ b & c)) | 0;
32
+ h = g; g = f; f = e; e = (d + t1) | 0; d = c; c = b; b = a; a = (t1 + t2) | 0;
33
+ }
34
+ s[0] += a; s[1] += b; s[2] += c; s[3] += d; s[4] += e; s[5] += f; s[6] += g; s[7] += h;
35
+ }
36
+
37
+ update(bytes) {
38
+ this.length += bytes.length;
39
+ let offset = 0;
40
+ if (this.used !== 0) {
41
+ offset = Math.min(64 - this.used, bytes.length);
42
+ this.block.set(bytes.subarray(0, offset), this.used);
43
+ this.used += offset;
44
+ if (this.used < 64) return;
45
+ this.compress(new DataView(this.block.buffer), 0);
46
+ this.used = 0;
47
+ }
48
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
49
+ for (; offset + 64 <= bytes.length; offset += 64) this.compress(view, offset);
50
+ this.block.set(bytes.subarray(offset));
51
+ this.used = bytes.length - offset;
52
+ }
53
+
54
+ // The lowercase hex digest; the hash takes no more bytes after it.
55
+ hex() {
56
+ const bits = this.length * 8;
57
+ this.update(Uint8Array.of(0x80));
58
+ this.update(new Uint8Array((120 - this.used) % 64));
59
+ const view = new DataView(this.block.buffer);
60
+ view.setUint32(56, Math.floor(bits / 2 ** 32));
61
+ view.setUint32(60, bits >>> 0);
62
+ this.compress(view, 0);
63
+ return [...this.state].map(word => (word >>> 0).toString(16).padStart(8, "0")).join("");
64
+ }
65
+ }