@daugasauron/dolly-build 0.1.5-rc.0 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +259 -6
- package/host/requirements.mjs +1 -1
- package/host/runtime/abi.mjs +41 -0
- package/package.json +10 -4
- package/packages.json +101 -0
- package/{src → page}/image-entry.mjs +10 -6
- package/{src → page}/image-inputs.mjs +11 -0
- package/{src → page}/image-names.mjs +2 -2
- package/page/sha256.mjs +65 -0
- package/{src → page}/snapshot-records.mjs +63 -44
- package/{src → page}/static-asset.mjs +36 -20
- package/{src → page}/version.mjs +1 -1
- package/scripts/dolly-build-init/Dollyfile +3 -7
- package/scripts/dolly-build-init/public/icon.svg +1 -0
- package/scripts/dolly-build-init/public/index.html +6 -58
- package/scripts/dolly-build-init/public/main.mjs +52 -0
- package/scripts/dolly-build-init/public/page.css +3 -0
- package/scripts/dolly-build-init/public/skills/changelog/SKILL.md +5 -0
- package/scripts/dolly-build-init/workflow.yml +42 -0
- package/scripts/dolly-build.mjs +194 -62
- package/scripts/dollyfile-graph.mjs +3 -3
- package/scripts/image-directory.mjs +6 -7
- package/scripts/page-image-build.mjs +29 -10
- package/scripts/recipe-files.mjs +3 -3
- package/scripts/release-layout.mjs +1 -1
- package/scripts/serve.mjs +40 -10
- package/scripts/sha256.mjs +4 -0
- package/scripts/share-pages-snapshots.mjs +10 -30
- package/scripts/snapshot-identity.mjs +1 -2
- package/scripts/system-snapshot-format.mjs +11 -9
- package/scripts/wasm-interface.mjs +2 -2
- package/{src → site}/dollyfile-graph.mjs +1 -1
- package/{src → site}/dollyfile-view.mjs +1 -1
- /package/{src → page}/process-abi.mjs +0 -0
- /package/{src → page}/wasm-interface.mjs +0 -0
- /package/{src → site}/alphabetical.mjs +0 -0
package/README.md
CHANGED
|
@@ -1,12 +1,265 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Dolly from npm
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
Dolly is a POSIX-like userspace for wasm64 that runs in a browser tab. Its npm
|
|
4
|
+
packages hold the runtime, prebuilt programs and a build command, so a project
|
|
5
|
+
of your own needs no checkout of Dolly: `npm install` is its only download.
|
|
6
|
+
This document is the guide to them. Its main case is a coding agent of your
|
|
7
|
+
own: [Pi](https://github.com/earendil-works/pi) in a sandbox with the tools you
|
|
8
|
+
choose and your skills, served from your machine, and built into the same
|
|
9
|
+
static site in CI. What changed between versions is in the
|
|
10
|
+
[npm changelog](npm-changelog.md), and how to move a project to a new version
|
|
11
|
+
in [Upgrading](npm-upgrading.md).
|
|
12
|
+
|
|
13
|
+
## A Pi agent of your own
|
|
14
|
+
|
|
15
|
+
With Node.js 22, npm, Chrome and Python (for its static file server), in an
|
|
16
|
+
empty directory:
|
|
7
17
|
|
|
8
18
|
```sh
|
|
9
|
-
|
|
19
|
+
npm init --yes
|
|
20
|
+
npm install @daugasauron/dolly-build
|
|
21
|
+
npx dolly-build init my-pi git python nvim
|
|
10
22
|
npx dolly-build Dollyfile-my-pi my-pi
|
|
11
23
|
python3 -m http.server -d my-pi -b 127.0.0.1 8080
|
|
12
24
|
```
|
|
25
|
+
|
|
26
|
+
- `init` writes the project: Pi with Git, Python and Neovim here. It installs
|
|
27
|
+
what the project needs and lacks with npm (the packages named, the host
|
|
28
|
+
modules they declare), at `dolly-build`'s version, and prints that command.
|
|
29
|
+
- The second `dolly-build` builds the image in a headless browser (45 s for
|
|
30
|
+
this project on a 16-core desktop) and writes the site `my-pi/` (94 MB).
|
|
31
|
+
- Python's server serves it. Open <http://127.0.0.1:8080/>: Pi asks for your
|
|
32
|
+
OpenRouter key at a prompt that does not show it, and starts in `/workspace`.
|
|
33
|
+
`curl https://example.com` in Pi's shell fails: the page's policy admits the
|
|
34
|
+
model's provider and nothing else. Ctrl+D leaves Pi for a shell. When Pi needs
|
|
35
|
+
a file of yours it asks with its `upload` tool: the page's file dialog opens,
|
|
36
|
+
and Pi gets the file's path and size, or that you cancelled. Save
|
|
37
|
+
(Ctrl+Shift+S) keeps the sandbox, files and all, in this browser and names it
|
|
38
|
+
in the address (`?session=NAME`), which opens it again as it was.
|
|
39
|
+
|
|
40
|
+
After `npm install` and `init`, nothing is downloaded: the build, its browser,
|
|
41
|
+
the site and its page fetch from `node_modules` and from each other alone. The
|
|
42
|
+
build's browser reaches only a server the command runs on 127.0.0.1; its own
|
|
43
|
+
requests (its vendor's updates, clock and autofill) go to a proxy of the
|
|
44
|
+
command's that refuses them.
|
|
45
|
+
|
|
46
|
+
## The project
|
|
47
|
+
|
|
48
|
+
| File | What it declares | After a change |
|
|
49
|
+
| --- | --- | --- |
|
|
50
|
+
| `Dollyfile-my-pi` | The sandbox, a [recipe](dollyfile.md): `INSTALL` rows for Pi's packages and the ones you named, each pinned by its recipe's SHA-256; `FILE` rows with Pi's `AGENTS.md`, an extension with a `/hello` command, a `todo` tool, the model's entry and the script that starts Pi with its provider and model | Build again |
|
|
51
|
+
| `public/skills/` | Pi's skills, one directory each: a `SKILL.md` and the files it names. The page writes them into `~/.pi/agent/skills/` at each boot | Build again (the image is kept) |
|
|
52
|
+
| `public/workspace/` | The project the agent works on, written into `/workspace` at each boot | Build again (the image is kept) |
|
|
53
|
+
| `public/main.mjs` | The hosts the agent may reach ([HTTP policy](http.md)): one rule per host, OpenRouter's API to begin with. The files the page writes at boot | Build again (the image is kept) |
|
|
54
|
+
| `public/index.html`, `public/page.css`, `public/icon.svg` | The page, and the popups' font. No inline script or style, so the host may serve a strict Content-Security-Policy ([Embedding](embedding.md)) | Build again (the image is kept) |
|
|
55
|
+
| `public/theme.json` | The sandbox's colours: five values in Pi's theme format, whose background, text, accent and muted are the terminal's and the popups' too | Build again (the image is kept) |
|
|
56
|
+
| `.github/workflows/my-pi.yml` | The same build in GitHub Actions ([CI](#ci)) | |
|
|
57
|
+
| `package.json`, `package-lock.json` | The packages, at exact versions | `npm install` |
|
|
58
|
+
| `my-pi/` | The site `dolly-build` writes, which `.gitignore` leaves out | Reload |
|
|
59
|
+
|
|
60
|
+
A build that keeps the image takes seconds: the image is built again only when
|
|
61
|
+
the recipe or the installed runtime changed.
|
|
62
|
+
|
|
63
|
+
## Tools
|
|
64
|
+
|
|
65
|
+
A tool is a package: a program built inside Dolly from its upstream source,
|
|
66
|
+
with the packages it needs. `init` takes the ones to start with, and `add`
|
|
67
|
+
puts more into the recipe of a project, installing them first:
|
|
68
|
+
|
|
69
|
+
```sh
|
|
70
|
+
npx dolly-build add my-pi jq sqlite
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
| Package | What it brings |
|
|
74
|
+
| --- | --- |
|
|
75
|
+
| `git` | Git over the page's HTTP broker, with `diff` and `patch` |
|
|
76
|
+
| `python` | CPython 3.14 with pip; it brings `cc` and `libffi` |
|
|
77
|
+
| `numpy`, `pandas`, `matplotlib`, `seaborn`, `python-dateutil` | Python's data libraries, each with the ones it needs and `python` |
|
|
78
|
+
| `nvim` | Neovim, with its runtime files and tree-sitter parsers |
|
|
79
|
+
| `cc` | The C and C++ compiler, with `ld`, `ar`, Make and the headers |
|
|
80
|
+
| `jq`, `sqlite`, `less` | `jq`, `sqlite3` and the pager |
|
|
81
|
+
| Pi's own, which `init` always installs | `core` (the shell `slop` and its commands), `posix` (grep, sed, awk, find and the others), `curl`, `display` (the terminal), `dollyfile` (the recipe engine) and `pi-dolly` (Pi, its JavaScript runtime, ripgrep, fd, the `download` and `upload` commands, and Pi's Dolly skill and tools, among them `download` and `upload`, which hand the user a file and ask the user for one) |
|
|
82
|
+
|
|
83
|
+
`npx dolly-build init` with a name that is no package lists them all. A program
|
|
84
|
+
that is not among them is built in the recipe, where a compiler is: a `FILE`
|
|
85
|
+
row with a C source and `SLOP cc /tmp/NAME.c -o /usr/bin/NAME`, or
|
|
86
|
+
`SLOP pip install NAME==VERSION` followed by `FOLDER
|
|
87
|
+
/usr/lib/python3.14/site-packages`, which keeps what pip installed (pure-Python
|
|
88
|
+
wheels and sdists install). A script is a `FILE` row under `/usr/bin` with
|
|
89
|
+
`EXPORTS TOOL NAME`, as the template's `todo` is. The recipe's comments show
|
|
90
|
+
each.
|
|
91
|
+
|
|
92
|
+
A package declares the host modules its programs need (`python` declares
|
|
93
|
+
`sockets`), and a recipe's `REQUIRES HOST` rows must name them: `init` and
|
|
94
|
+
`add` write those rows and install the modules' packages
|
|
95
|
+
(`@daugasauron/dolly-host-NAME`). The site holds the modules its recipe
|
|
96
|
+
declares and no other.
|
|
97
|
+
|
|
98
|
+
## Skills and instructions
|
|
99
|
+
|
|
100
|
+
- A skill is a directory under `public/skills/` with a `SKILL.md`: front matter
|
|
101
|
+
with its `name` and `description`, then what the agent does when a task
|
|
102
|
+
matches the description (Pi's `docs/skills.md`, in the image under
|
|
103
|
+
`/usr/lib/node_modules/@earendil-works/pi-coding-agent/docs/`). Scripts and
|
|
104
|
+
other files a skill names lie beside it and are at
|
|
105
|
+
`~/.pi/agent/skills/NAME/` in the sandbox. Pi lists the skills when it
|
|
106
|
+
starts. `init` writes one, `changelog`.
|
|
107
|
+
- `AGENTS.md`, the instructions Pi takes into every session, is a `FILE` row of
|
|
108
|
+
the recipe; so are an extension (`/hello`), the model's entry
|
|
109
|
+
(`models.json`) and the script that starts Pi (`/etc/dolly/init.slop`).
|
|
110
|
+
[An agent in place of a web frontend](agent-frontend.md) goes on from here:
|
|
111
|
+
your backend's API, another model provider, how Pi starts.
|
|
112
|
+
- These files are not secrets: Pi's tools can read them, and so can its model.
|
|
113
|
+
|
|
114
|
+
## The policy and the key
|
|
115
|
+
|
|
116
|
+
`public/main.mjs` holds the page's HTTP policy: the hosts the sandbox may
|
|
117
|
+
reach, one rule each, with the header that may carry a credential. It is
|
|
118
|
+
enforced in the browser, outside the sandbox ([HTTP](http.md)). `init` writes
|
|
119
|
+
one rule, OpenRouter's API, and Anthropic's as a comment beside it. The key is
|
|
120
|
+
typed at `pi-key`'s prompt in the sandbox each time it starts; it is in no
|
|
121
|
+
file of the site. A host the agent should reach (`api.github.com`, `pypi.org`)
|
|
122
|
+
is one more rule.
|
|
123
|
+
|
|
124
|
+
## Build
|
|
125
|
+
|
|
126
|
+
`dolly-build Dollyfile-NAME OUT` writes the site `OUT`, replacing the one it
|
|
127
|
+
wrote there before: the files of `public/`, `workspace.json` and `skills.json`
|
|
128
|
+
(the files under `public/workspace/` and `public/skills/`), the runtime
|
|
129
|
+
(`dolly/`), the device host modules the recipe declares (`dolly-host-NAME/`),
|
|
130
|
+
`coi-serviceworker.js` and `image/`, the image's directory (its catalog,
|
|
131
|
+
metadata and packs, the recipes it was built from, its `amy-index.txt` and an
|
|
132
|
+
`index.mjs`).
|
|
133
|
+
|
|
134
|
+
- The build runs in a browser, with the installed runtime's own builder.
|
|
135
|
+
Without `--browser` it takes the first this machine has of Chrome, Playwright's
|
|
136
|
+
Chromium and Playwright's Firefox; `--browser chrome`, `chromium` or `firefox`
|
|
137
|
+
names one (`npx playwright-core install chromium-headless-shell` or `firefox`
|
|
138
|
+
installs Playwright's). It needs Chrome 138 or Firefox 146 or later, as the
|
|
139
|
+
page does.
|
|
140
|
+
- Both commands run in the directory whose `node_modules/@daugasauron/` holds
|
|
141
|
+
the packages (npm, pnpm and Yarn 1 alike; Yarn 2 and later need `nodeLinker:
|
|
142
|
+
node-modules`). `init` and `add` install with the package manager that runs
|
|
143
|
+
them (`npm install`, `pnpm add` or `yarn add`); their install is measured
|
|
144
|
+
with npm alone.
|
|
145
|
+
- A recipe's steps reach the network only where the recipe says so: a `SOURCE`
|
|
146
|
+
row, or a `SLOP` step such as `pip install`. The browser hands each such
|
|
147
|
+
request to `dolly-build`, which fetches it in Node, so a host without CORS
|
|
148
|
+
headers works, and through the proxy `HTTPS_PROXY` or `HTTP_PROXY` names; the
|
|
149
|
+
log names each host a build reaches. What a step fetches is not part of the
|
|
150
|
+
image's identity: pin versions (`pip install NAME==1.2`) where the bytes
|
|
151
|
+
matter. `init`'s recipe has no such step.
|
|
152
|
+
- A recipe is at most 128 KiB. A `FILE` under `/workspace` is part of the
|
|
153
|
+
image; the page's `public/workspace/` is written over it at each fresh boot.
|
|
154
|
+
|
|
155
|
+
## CI
|
|
156
|
+
|
|
157
|
+
`init` writes `.github/workflows/my-pi.yml`: on a push to `main`, a pull
|
|
158
|
+
request or by hand, it checks the project out, runs `npm ci`, installs
|
|
159
|
+
Playwright's headless Chromium (the version `package-lock.json` pins, with the
|
|
160
|
+
system libraries it needs), builds the site with it and uploads it as the
|
|
161
|
+
artifact `github-pages`:
|
|
162
|
+
|
|
163
|
+
```yaml
|
|
164
|
+
- run: npm ci
|
|
165
|
+
- run: npx playwright-core install --with-deps chromium-headless-shell
|
|
166
|
+
- run: npx dolly-build Dollyfile-my-pi my-pi --browser chromium
|
|
167
|
+
- uses: actions/upload-pages-artifact@v3
|
|
168
|
+
with:
|
|
169
|
+
path: my-pi
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Commit the project as `init` left it (`package-lock.json` too; `.gitignore`
|
|
173
|
+
leaves out `node_modules/` and the site). The file ends with a `deploy` job as
|
|
174
|
+
comments, which publishes the site on GitHub Pages once the repository's
|
|
175
|
+
Pages source is "GitHub Actions". Pages sends no isolation headers; the page
|
|
176
|
+
isolates itself ([Serving](#serving)).
|
|
177
|
+
|
|
178
|
+
Elsewhere the same three commands are the job. In Playwright's container image
|
|
179
|
+
of the version of `playwright-core` the lockfile holds
|
|
180
|
+
(`mcr.microsoft.com/playwright:vVERSION-noble`), the browser is there already.
|
|
181
|
+
After `npm ci` and the browser, a CI build downloads nothing either.
|
|
182
|
+
`PLAYWRIGHT_DOWNLOAD_HOST` points the browser's download at a mirror.
|
|
183
|
+
|
|
184
|
+
How this is checked: `test/npm-browser.mjs` commits the project `init` wrote,
|
|
185
|
+
clones it, and runs the workflow's own `npm ci` and build lines in the clone,
|
|
186
|
+
in a network namespace whose only network is loopback, with Chrome and with
|
|
187
|
+
Firefox; the site's image is the one the project built. The workflow file has
|
|
188
|
+
not been run on GitHub's runners by Dolly's tests.
|
|
189
|
+
|
|
190
|
+
## Serving
|
|
191
|
+
|
|
192
|
+
The site is static: any file server serves it as it lies, from any path.
|
|
193
|
+
|
|
194
|
+
- Dolly needs shared memory, which a browser gives only to a cross-origin
|
|
195
|
+
isolated page. Where the host sends `Cross-Origin-Opener-Policy: same-origin`
|
|
196
|
+
and `Cross-Origin-Embedder-Policy: require-corp`, the page is isolated. Where
|
|
197
|
+
it does not (Python's server, GitHub Pages), `init`'s page registers
|
|
198
|
+
`coi-serviceworker.js`, which adds them to the site's own responses, and
|
|
199
|
+
loads once more under it. A service worker needs HTTPS or `127.0.0.1`.
|
|
200
|
+
- The host must serve `.wasm` as `application/wasm`, `.mjs` as
|
|
201
|
+
`text/javascript` and `.json` as `application/json`.
|
|
202
|
+
- The image's packs stay in the origin's private storage after the first
|
|
203
|
+
visit, so the page starts again without downloading them.
|
|
204
|
+
|
|
205
|
+
## Updating
|
|
206
|
+
|
|
207
|
+
The packages of one project are of one version: `init` and `add` install at
|
|
208
|
+
`dolly-build`'s, and a build with two versions fails, naming both. To move a
|
|
209
|
+
project to a new version, install `dolly-build` at it; `add` then installs the
|
|
210
|
+
project's other packages at that version and writes the recipe's pins again:
|
|
211
|
+
|
|
212
|
+
```sh
|
|
213
|
+
npm install --save-exact @daugasauron/dolly-build@0.1.7
|
|
214
|
+
npx dolly-build add my-pi
|
|
215
|
+
npx dolly-build Dollyfile-my-pi my-pi
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
`add` writes every `FROM` and `INSTALL` row at the pin of the installed
|
|
219
|
+
package that holds its recipe. A row whose pin no installed package holds
|
|
220
|
+
fails the build, naming the row.
|
|
221
|
+
|
|
222
|
+
## The packages
|
|
223
|
+
|
|
224
|
+
| Package | What it holds |
|
|
225
|
+
| --- | --- |
|
|
226
|
+
| `@daugasauron/dolly-build` | The `dolly-build` command. It depends on the runtime, on `playwright-core`, which starts the build's browser, and on Pi's packages |
|
|
227
|
+
| `@daugasauron/dolly` | The runtime: `boot()`, the kernel, the process runtime, the image builder, and the host modules that are its own machinery ([Embedding](embedding.md)) |
|
|
228
|
+
| `@daugasauron/dolly-host-NAME` | One device host module each: `display`, `input`, `html`, `sockets`, `gpu`, `audio`, `microphone`, `touch`, `clipboard`, `upload`, `pty`, `console`, and `save`, which is `download@0` (npm takes no new name with that word) |
|
|
229
|
+
| `@daugasauron/dolly-package-NAME` | A package for a recipe to `INSTALL` ([Tools](#tools)): its catalog, metadata, packs and recipes. It depends on the packages it installs, so npm brings them |
|
|
230
|
+
| `@daugasauron/dolly-image-NAME` | A whole image for `boot()`: `default` (a shell and its tools), `pi-minimal` (Pi alone) and `python-data` (Python with NumPy, pandas, Matplotlib and seaborn) |
|
|
231
|
+
|
|
232
|
+
Each host module and image depends on the runtime alone. An image or package
|
|
233
|
+
does not depend on the host modules it declares: the page chooses its modules.
|
|
234
|
+
|
|
235
|
+
## A recipe on an image
|
|
236
|
+
|
|
237
|
+
A recipe may start `FROM` an installed image instead of installing packages:
|
|
238
|
+
the image package's README gives the row. `FROM` carries the base's files,
|
|
239
|
+
environment and exports but not its host modules or `ENTRY`, so the recipe
|
|
240
|
+
repeats those. `init` writes Pi's project only; the page in `public/` is then
|
|
241
|
+
[Embedding](embedding.md)'s, importing `./dolly/page/browser.mjs` and
|
|
242
|
+
`./image/index.mjs`, with the lines of `init`'s `main.mjs` that register
|
|
243
|
+
`coi-serviceworker.js`.
|
|
244
|
+
|
|
245
|
+
```sh
|
|
246
|
+
npm install @daugasauron/dolly-build # with the install line of the image package's README
|
|
247
|
+
npx dolly-build Dollyfile-my-data my-data
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
To run a prebuilt image in a page with no build at all, see
|
|
251
|
+
[Embedding](embedding.md).
|
|
252
|
+
|
|
253
|
+
## When something fails
|
|
254
|
+
|
|
255
|
+
| Message | What to do |
|
|
256
|
+
| --- | --- |
|
|
257
|
+
| `NAME: no such package. The packages: …` | `init` and `add` take the names listed |
|
|
258
|
+
| `that install failed` | Run the printed `npm install` line yourself (a registry or proxy setting), then the command again |
|
|
259
|
+
| `no installed package holds … at that pin` | The packages were updated: `npx dolly-build add NAME` |
|
|
260
|
+
| `… holds X at another pin than …: install packages of one version` | Install every `@daugasauron` package at one version ([Updating](#updating)) |
|
|
261
|
+
| `NAME declares sockets: install @daugasauron/dolly-host-sockets` | A `REQUIRES HOST` row written by hand names a module that is not installed |
|
|
262
|
+
| `a browser is not installed` | Install Chrome, or `npx playwright-core install chromium-headless-shell` |
|
|
263
|
+
| `this browser has no WebAssembly memory64` | The build's browser is older than Chrome 138 or Firefox 146 |
|
|
264
|
+
| `Dollyfile-NAME:LINE: unknown directive` | A `FILE` row's body lines start with four spaces |
|
|
265
|
+
| The page says it is not cross-origin isolated | It is served over plain HTTP from another address than `127.0.0.1`: serve it over HTTPS, or send the two headers |
|
package/host/requirements.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { DOLLY_HOST_RECORD_BYTES as recordBytes, DOLLY_HOST_NAME_BYTES as nameBytes,
|
|
2
2
|
DOLLY_HOST_DIGEST_BYTES as digestBytes, DOLLY_HOST_MAX_RECORDS as maxRecords } from "./abi.mjs";
|
|
3
|
-
import { hex } from "../
|
|
3
|
+
import { hex } from "../page/static-asset.mjs";
|
|
4
4
|
|
|
5
5
|
const pattern = /^([a-z][a-z0-9-]{0,30})@(0|[1-9][0-9]{0,4})$/;
|
|
6
6
|
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// Generated from abi/dolly-supervisor-0.wat, abi/dolly-image-0.wat, abi/dolly-process-0.wat, abi/dolly-process-gate-0.wat, include/dolly/process.h.
|
|
2
|
+
export const DOLLY_PROCESS_EVENT_LAUNCH = 1;
|
|
3
|
+
export const DOLLY_PROCESS_EVENT_INTERRUPT = 2;
|
|
4
|
+
export const DOLLY_PROCESS_EVENT_TERMINATE = 3;
|
|
5
|
+
export const DOLLY_PROCESS_EVENT_WAKE = 4;
|
|
6
|
+
export const DOLLY_PROCESS_EVENT_KIND_SHIFT = 48;
|
|
7
|
+
export const DOLLY_PROCESS_EVENT_VALUE_SHIFT = 32;
|
|
8
|
+
export const DOLLY_TERMINAL_WORD_RESULT_SEQUENCE = 0;
|
|
9
|
+
export const DOLLY_TERMINAL_WORD_RESULT_STATUS = 1;
|
|
10
|
+
export const DOLLY_TERMINAL_WORD_FOREGROUND_PID = 2;
|
|
11
|
+
export const DOLLY_TERMINAL_WORD_FOREGROUND_INTERRUPTIBLE = 3;
|
|
12
|
+
export const DOLLY_TERMINAL_WORD_INTERRUPT_SEQUENCE = 4;
|
|
13
|
+
export const DOLLY_TERMINAL_WORD_INTERRUPT_TARGET_PID = 5;
|
|
14
|
+
export const DOLLY_FS_BLOCK_SIZE = 1048576;
|
|
15
|
+
export const DOLLY_DEVICE_HEADER_BYTES = 32;
|
|
16
|
+
export const DOLLY_DEVICE_REPLY_HEADER_BYTES = 64;
|
|
17
|
+
export const DOLLY_DEVICE_REPLY_WORD_STATE = 0;
|
|
18
|
+
export const DOLLY_DEVICE_REPLY_WORD_SCOPE = 1;
|
|
19
|
+
export const DOLLY_DEVICE_REPLY_WORD_SEQUENCE = 2;
|
|
20
|
+
export const DOLLY_DEVICE_REPLY_WORD_ERRNO = 3;
|
|
21
|
+
export const DOLLY_DEVICE_REPLY_WORD_LENGTH = 4;
|
|
22
|
+
export const DOLLY_DEVICE_REPLY_PUBLISHED = 1;
|
|
23
|
+
export const DOLLY_SNAPSHOT_VERSION = 2;
|
|
24
|
+
export const DOLLY_SNAPSHOT_HEADER_BYTES = 16;
|
|
25
|
+
export const DOLLY_SNAPSHOT_RECORD_HEADER_BYTES = 16;
|
|
26
|
+
export const DOLLY_SNAPSHOT_RECORD_DIRECTORY = 1;
|
|
27
|
+
export const DOLLY_SNAPSHOT_RECORD_FILE = 2;
|
|
28
|
+
export const DOLLY_SNAPSHOT_RECORD_SYMLINK = 3;
|
|
29
|
+
export const DOLLY_SNAPSHOT_RECORD_DELETED = 4;
|
|
30
|
+
export const DOLLY_SNAPSHOT_MAX_PATHS = 100000;
|
|
31
|
+
export const DOLLY_SNAPSHOT_MAX_MEBIBYTES = 2048;
|
|
32
|
+
export const DOLLY_SNAPSHOT_PATH_MIN_BYTES = 2;
|
|
33
|
+
export const DOLLY_SNAPSHOT_PATH_MAX_BYTES = 4095;
|
|
34
|
+
export const DOLLY_SNAPSHOT_SYMLINK_MIN_BYTES = 1;
|
|
35
|
+
export const DOLLY_SNAPSHOT_SYMLINK_MAX_BYTES = 4095;
|
|
36
|
+
export const DOLLY_ENTRY_VERSION = 1;
|
|
37
|
+
export const DOLLY_ENTRY_HEADER_BYTES = 16;
|
|
38
|
+
export const DOLLY_ENTRY_MAX_ARGUMENTS = 256;
|
|
39
|
+
export const DOLLY_ENTRY_ARGUMENT_MAX_BYTES = 4096;
|
|
40
|
+
export const DOLLY_ENTRY_MAX_BYTES = 65536;
|
|
41
|
+
export const DOLLY_ERRNO = Object.freeze({ E2BIG: 1, EACCES: 2, EADDRINUSE: 3, EADDRNOTAVAIL: 4, EAFNOSUPPORT: 5, EAGAIN: 6, EALREADY: 7, EBADF: 8, EBADMSG: 9, EBUSY: 10, ECANCELED: 11, ECHILD: 12, ECONNABORTED: 13, ECONNREFUSED: 14, ECONNRESET: 15, EDEADLK: 16, EDESTADDRREQ: 17, EDOM: 18, EDQUOT: 19, EEXIST: 20, EFAULT: 21, EFBIG: 22, EHOSTUNREACH: 23, EIDRM: 24, EILSEQ: 25, EINPROGRESS: 26, EINTR: 27, EINVAL: 28, EIO: 29, EISCONN: 30, EISDIR: 31, ELOOP: 32, EMFILE: 33, EMLINK: 34, EMSGSIZE: 35, EMULTIHOP: 36, ENAMETOOLONG: 37, ENETDOWN: 38, ENETRESET: 39, ENETUNREACH: 40, ENFILE: 41, ENOBUFS: 42, ENODEV: 43, ENOENT: 44, ENOEXEC: 45, ENOLCK: 46, ENOLINK: 47, ENOMEM: 48, ENOMSG: 49, ENOPROTOOPT: 50, ENOSPC: 51, ENOSYS: 52, ENOTCONN: 53, ENOTDIR: 54, ENOTEMPTY: 55, ENOTRECOVERABLE: 56, ENOTSOCK: 57, ENOTTY: 59, ENXIO: 60, EOVERFLOW: 61, EOWNERDEAD: 62, EPERM: 63, EPIPE: 64, EPROTO: 65, EPROTONOSUPPORT: 66, EPROTOTYPE: 67, ERANGE: 68, EROFS: 69, ESPIPE: 70, ESRCH: 71, ESTALE: 72, ETIMEDOUT: 73, ETXTBSY: 74, EXDEV: 75, ENODATA: 116, ENOTSUP: 138 });
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "0.1.
|
|
2
|
+
"version": "0.1.6",
|
|
3
3
|
"type": "module",
|
|
4
4
|
"name": "@daugasauron/dolly-build",
|
|
5
5
|
"license": "MIT",
|
|
@@ -7,10 +7,16 @@
|
|
|
7
7
|
"dolly-build": "scripts/dolly-build.mjs"
|
|
8
8
|
},
|
|
9
9
|
"dependencies": {
|
|
10
|
-
"@daugasauron/dolly": "0.1.
|
|
11
|
-
"playwright-core": "1.63.0"
|
|
10
|
+
"@daugasauron/dolly": "0.1.6",
|
|
11
|
+
"playwright-core": "1.63.0",
|
|
12
|
+
"@daugasauron/dolly-package-core": "0.1.6",
|
|
13
|
+
"@daugasauron/dolly-package-dollyfile": "0.1.6",
|
|
14
|
+
"@daugasauron/dolly-package-posix": "0.1.6",
|
|
15
|
+
"@daugasauron/dolly-package-display": "0.1.6",
|
|
16
|
+
"@daugasauron/dolly-package-curl": "0.1.6",
|
|
17
|
+
"@daugasauron/dolly-package-pi-dolly": "0.1.6"
|
|
12
18
|
},
|
|
13
|
-
"description": "Build a Dolly image of your own
|
|
19
|
+
"description": "Build a Dolly image of your own from installed packages, in a browser, into a static site.",
|
|
14
20
|
"repository": {
|
|
15
21
|
"type": "git",
|
|
16
22
|
"url": "git+https://github.com/daugasauron/dolly.git"
|
package/packages.json
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
{
|
|
2
|
+
"packages": {
|
|
3
|
+
"cc": [],
|
|
4
|
+
"core": [],
|
|
5
|
+
"curl": [],
|
|
6
|
+
"display": [
|
|
7
|
+
"display"
|
|
8
|
+
],
|
|
9
|
+
"dolly-docs": [],
|
|
10
|
+
"dollyfile": [],
|
|
11
|
+
"fd": [],
|
|
12
|
+
"git": [],
|
|
13
|
+
"gzip": [],
|
|
14
|
+
"html": [
|
|
15
|
+
"html"
|
|
16
|
+
],
|
|
17
|
+
"janis": [
|
|
18
|
+
"download"
|
|
19
|
+
],
|
|
20
|
+
"janis-install": [
|
|
21
|
+
"download"
|
|
22
|
+
],
|
|
23
|
+
"jq": [],
|
|
24
|
+
"less": [],
|
|
25
|
+
"libdolly-audio": [],
|
|
26
|
+
"libdolly-clipboard": [],
|
|
27
|
+
"libdolly-display": [],
|
|
28
|
+
"libdolly-download": [],
|
|
29
|
+
"libdolly-gpu": [],
|
|
30
|
+
"libdolly-input": [],
|
|
31
|
+
"libdolly-microphone": [],
|
|
32
|
+
"libdolly-nena": [],
|
|
33
|
+
"libdolly-packages": [],
|
|
34
|
+
"libdolly-pty": [],
|
|
35
|
+
"libdolly-sockets": [],
|
|
36
|
+
"libdolly-threads": [],
|
|
37
|
+
"libdolly-touch": [],
|
|
38
|
+
"libdolly-upload": [],
|
|
39
|
+
"libffi": [],
|
|
40
|
+
"matplotlib": [
|
|
41
|
+
"sockets"
|
|
42
|
+
],
|
|
43
|
+
"numpy": [
|
|
44
|
+
"sockets"
|
|
45
|
+
],
|
|
46
|
+
"nvim": [],
|
|
47
|
+
"pandas": [
|
|
48
|
+
"sockets"
|
|
49
|
+
],
|
|
50
|
+
"pi": [
|
|
51
|
+
"download"
|
|
52
|
+
],
|
|
53
|
+
"pi-dolly": [
|
|
54
|
+
"download",
|
|
55
|
+
"html",
|
|
56
|
+
"upload"
|
|
57
|
+
],
|
|
58
|
+
"posix": [],
|
|
59
|
+
"python": [
|
|
60
|
+
"sockets"
|
|
61
|
+
],
|
|
62
|
+
"python-dateutil": [
|
|
63
|
+
"sockets"
|
|
64
|
+
],
|
|
65
|
+
"quickjs": [],
|
|
66
|
+
"ripgrep": [],
|
|
67
|
+
"sbase": [],
|
|
68
|
+
"seaborn": [
|
|
69
|
+
"sockets"
|
|
70
|
+
],
|
|
71
|
+
"secretenv": [],
|
|
72
|
+
"sqlite": [],
|
|
73
|
+
"transfer": [
|
|
74
|
+
"download",
|
|
75
|
+
"upload"
|
|
76
|
+
]
|
|
77
|
+
},
|
|
78
|
+
"images": {
|
|
79
|
+
"default": [
|
|
80
|
+
"display",
|
|
81
|
+
"download",
|
|
82
|
+
"input",
|
|
83
|
+
"upload"
|
|
84
|
+
],
|
|
85
|
+
"pi-minimal": [
|
|
86
|
+
"display",
|
|
87
|
+
"download",
|
|
88
|
+
"html",
|
|
89
|
+
"input",
|
|
90
|
+
"upload"
|
|
91
|
+
],
|
|
92
|
+
"python-data": [
|
|
93
|
+
"display",
|
|
94
|
+
"download",
|
|
95
|
+
"html",
|
|
96
|
+
"input",
|
|
97
|
+
"sockets",
|
|
98
|
+
"upload"
|
|
99
|
+
]
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -1,22 +1,26 @@
|
|
|
1
|
-
|
|
1
|
+
import { DOLLY_ENTRY_VERSION, DOLLY_ENTRY_HEADER_BYTES, DOLLY_ENTRY_MAX_ARGUMENTS, DOLLY_ENTRY_ARGUMENT_MAX_BYTES,
|
|
2
|
+
DOLLY_ENTRY_MAX_BYTES } from "../host/runtime/abi.mjs";
|
|
3
|
+
|
|
4
|
+
// /etc/dolly/entry, as abi/dolly-image-0.wat lays it out. Shared by packaging and
|
|
5
|
+
// browser admission; empty arguments are valid.
|
|
2
6
|
export function decodeImageEntry(bytes) {
|
|
3
|
-
if (!(bytes instanceof Uint8Array) || bytes.byteLength <
|
|
7
|
+
if (!(bytes instanceof Uint8Array) || bytes.byteLength < DOLLY_ENTRY_HEADER_BYTES || bytes.byteLength > DOLLY_ENTRY_MAX_BYTES) {
|
|
4
8
|
throw new TypeError("invalid image ENTRY record");
|
|
5
9
|
}
|
|
6
10
|
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
7
11
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
8
|
-
if (decoder.decode(bytes.subarray(0, 8)) !== "DOLLYENT" || view.getUint32(8, true) !==
|
|
12
|
+
if (decoder.decode(bytes.subarray(0, 8)) !== "DOLLYENT" || view.getUint32(8, true) !== DOLLY_ENTRY_VERSION) {
|
|
9
13
|
throw new TypeError("invalid image ENTRY version");
|
|
10
14
|
}
|
|
11
15
|
const count = view.getUint32(12, true);
|
|
12
|
-
if (count === 0 || count >
|
|
13
|
-
let offset =
|
|
16
|
+
if (count === 0 || count > DOLLY_ENTRY_MAX_ARGUMENTS) throw new TypeError("invalid image ENTRY count");
|
|
17
|
+
let offset = DOLLY_ENTRY_HEADER_BYTES;
|
|
14
18
|
const arguments_ = [];
|
|
15
19
|
for (let index = 0; index < count; ++index) {
|
|
16
20
|
if (offset > bytes.byteLength - 4) throw new TypeError("truncated image ENTRY");
|
|
17
21
|
const size = view.getUint32(offset, true);
|
|
18
22
|
offset += 4;
|
|
19
|
-
if (size >
|
|
23
|
+
if (size > DOLLY_ENTRY_ARGUMENT_MAX_BYTES || offset > bytes.byteLength - size) throw new TypeError("invalid image ENTRY argument");
|
|
20
24
|
const value = decoder.decode(bytes.subarray(offset, offset + size));
|
|
21
25
|
if (value.includes("\0")) throw new TypeError("image ENTRY has a NUL argument");
|
|
22
26
|
arguments_.push(value);
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { Sha256 } from "./sha256.mjs";
|
|
2
|
+
|
|
1
3
|
// Recipe pins identify instructions; artifact digests identify their results.
|
|
2
4
|
export function imageInputs(artifacts = []) {
|
|
3
5
|
if (!Array.isArray(artifacts) || artifacts.length > 256) throw new Error("invalid image inputs");
|
|
@@ -18,3 +20,12 @@ export function imageInputs(artifacts = []) {
|
|
|
18
20
|
export function imageInputsMatch(actual, expected) {
|
|
19
21
|
return JSON.stringify(imageInputs(actual)) === JSON.stringify(imageInputs(expected));
|
|
20
22
|
}
|
|
23
|
+
|
|
24
|
+
// The catalog's digest of an image's recipe records (its metadata's recipes),
|
|
25
|
+
// one line each, "SHA256 BYTES PATH\n", in their order: a start checks the
|
|
26
|
+
// records against the catalog without the catalog repeating them.
|
|
27
|
+
export function recipesDigest(recipes) {
|
|
28
|
+
const hash = new Sha256();
|
|
29
|
+
hash.update(new TextEncoder().encode(recipes.map(({ sha256, byteLength, sourcePath }) => `${sha256} ${byteLength} ${sourcePath}\n`).join("")));
|
|
30
|
+
return hash.hex();
|
|
31
|
+
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// Dollyfile-example.txt is never a recipe.
|
|
6
6
|
export const validName = name => /^[a-z][a-z0-9]*(?:-[a-z0-9]+|\.[0-9]+)*$/.test(name) && name.length <= 32;
|
|
7
7
|
|
|
8
|
-
// No image retains scratch space.
|
|
8
|
+
// No image retains scratch space, /tmp.
|
|
9
9
|
export function unretainedPath(value) {
|
|
10
|
-
return
|
|
10
|
+
return value === "/tmp" || value.startsWith("/tmp/");
|
|
11
11
|
}
|
package/page/sha256.mjs
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// SHA-256 over bytes given in pieces, for a digest of more bytes than the page
|
|
2
|
+
// should hold at once; WebCrypto digests only a whole buffer.
|
|
3
|
+
const K = Int32Array.of(
|
|
4
|
+
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
|
5
|
+
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
|
6
|
+
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
|
7
|
+
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
|
8
|
+
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
|
9
|
+
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
|
10
|
+
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
|
11
|
+
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2);
|
|
12
|
+
|
|
13
|
+
export class Sha256 {
|
|
14
|
+
state = Int32Array.of(0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19);
|
|
15
|
+
block = new Uint8Array(64);
|
|
16
|
+
used = 0;
|
|
17
|
+
length = 0;
|
|
18
|
+
words = new Int32Array(64);
|
|
19
|
+
|
|
20
|
+
compress(view, offset) {
|
|
21
|
+
const w = this.words, s = this.state;
|
|
22
|
+
for (let i = 0; i < 16; i++) w[i] = view.getInt32(offset + 4 * i);
|
|
23
|
+
for (let i = 16; i < 64; i++) {
|
|
24
|
+
const x = w[i - 15], y = w[i - 2];
|
|
25
|
+
w[i] = (((x >>> 7 | x << 25) ^ (x >>> 18 | x << 14) ^ x >>> 3) + w[i - 7] +
|
|
26
|
+
((y >>> 17 | y << 15) ^ (y >>> 19 | y << 13) ^ y >>> 10) + w[i - 16]) | 0;
|
|
27
|
+
}
|
|
28
|
+
let a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7];
|
|
29
|
+
for (let i = 0; i < 64; i++) {
|
|
30
|
+
const t1 = (h + ((e >>> 6 | e << 26) ^ (e >>> 11 | e << 21) ^ (e >>> 25 | e << 7)) + (e & f ^ ~e & g) + K[i] + w[i]) | 0;
|
|
31
|
+
const t2 = (((a >>> 2 | a << 30) ^ (a >>> 13 | a << 19) ^ (a >>> 22 | a << 10)) + (a & b ^ a & c ^ b & c)) | 0;
|
|
32
|
+
h = g; g = f; f = e; e = (d + t1) | 0; d = c; c = b; b = a; a = (t1 + t2) | 0;
|
|
33
|
+
}
|
|
34
|
+
s[0] += a; s[1] += b; s[2] += c; s[3] += d; s[4] += e; s[5] += f; s[6] += g; s[7] += h;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
update(bytes) {
|
|
38
|
+
this.length += bytes.length;
|
|
39
|
+
let offset = 0;
|
|
40
|
+
if (this.used !== 0) {
|
|
41
|
+
offset = Math.min(64 - this.used, bytes.length);
|
|
42
|
+
this.block.set(bytes.subarray(0, offset), this.used);
|
|
43
|
+
this.used += offset;
|
|
44
|
+
if (this.used < 64) return;
|
|
45
|
+
this.compress(new DataView(this.block.buffer), 0);
|
|
46
|
+
this.used = 0;
|
|
47
|
+
}
|
|
48
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
49
|
+
for (; offset + 64 <= bytes.length; offset += 64) this.compress(view, offset);
|
|
50
|
+
this.block.set(bytes.subarray(offset));
|
|
51
|
+
this.used = bytes.length - offset;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// The lowercase hex digest; the hash takes no more bytes after it.
|
|
55
|
+
hex() {
|
|
56
|
+
const bits = this.length * 8;
|
|
57
|
+
this.update(Uint8Array.of(0x80));
|
|
58
|
+
this.update(new Uint8Array((120 - this.used) % 64));
|
|
59
|
+
const view = new DataView(this.block.buffer);
|
|
60
|
+
view.setUint32(56, Math.floor(bits / 2 ** 32));
|
|
61
|
+
view.setUint32(60, bits >>> 0);
|
|
62
|
+
this.compress(view, 0);
|
|
63
|
+
return [...this.state].map(word => (word >>> 0).toString(16).padStart(8, "0")).join("");
|
|
64
|
+
}
|
|
65
|
+
}
|