@nimbus-sh/core 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +112 -0
- package/dist/_shared/tarball-stream.d.ts +93 -0
- package/dist/_shared/tarball-stream.d.ts.map +1 -0
- package/dist/_shared/tarball-stream.js +235 -0
- package/dist/_shared/tarball.d.ts +17 -0
- package/dist/_shared/tarball.d.ts.map +1 -0
- package/dist/_shared/tarball.js +39 -0
- package/dist/runtime/clang-runner.d.ts +38 -0
- package/dist/runtime/clang-runner.d.ts.map +1 -0
- package/dist/runtime/clang-runner.js +866 -0
- package/dist/runtime/facet-host.d.ts +12 -0
- package/dist/runtime/facet-host.d.ts.map +1 -1
- package/dist/runtime/local-facet-host.d.ts.map +1 -1
- package/dist/runtime/local-facet-host.js +29 -9
- package/dist/runtime/ruby-gems.d.ts +30 -0
- package/dist/runtime/ruby-gems.d.ts.map +1 -0
- package/dist/runtime/ruby-gems.js +636 -0
- package/dist/runtime/ruby-runner.d.ts +127 -0
- package/dist/runtime/ruby-runner.d.ts.map +1 -0
- package/dist/runtime/ruby-runner.js +1357 -0
- package/dist/runtime/session-process-supervisor.d.ts +15 -0
- package/dist/runtime/session-process-supervisor.d.ts.map +1 -1
- package/dist/runtime/session-process-supervisor.js +30 -0
- package/dist/workspace/nimbus-workspace.d.ts.map +1 -1
- package/dist/workspace/nimbus-workspace.js +9 -2
- package/package.json +4 -2
- package/src/_shared/tarball-stream.ts +263 -0
- package/src/_shared/tarball.ts +46 -0
- package/src/runtime/clang-runner.ts +924 -0
- package/src/runtime/facet-host.ts +12 -0
- package/src/runtime/local-facet-host.ts +28 -8
- package/src/runtime/ruby-gems.ts +682 -0
- package/src/runtime/ruby-runner.ts +1484 -0
- package/src/runtime/session-process-supervisor.ts +27 -0
- package/src/workspace/nimbus-workspace.ts +10 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ashish Kumar Singh and Nimbus contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# @nimbus-sh/core
|
|
2
|
+
|
|
3
|
+
> Part of [Nimbus](https://github.com/AshishKumar4/Nimbus), my hobby/research
|
|
4
|
+
> cloud OS. This README is edited and maintained with Claude (AI) and
|
|
5
|
+
> presented as-is.
|
|
6
|
+
|
|
7
|
+
The backend-agnostic half of Nimbus: a durable POSIX-like filesystem, a shell
|
|
8
|
+
with 60+ Unix commands, and the WASI runtime layer — with no Cloudflare
|
|
9
|
+
dependency. You hand it a SQLite and get back `.fs` and `.exec`. On
|
|
10
|
+
Cloudflare that SQLite is `ctx.storage.sql` inside your Durable Object; in bun
|
|
11
|
+
or node it is `bun:sqlite` or `node:sqlite`.
|
|
12
|
+
|
|
13
|
+
I extracted this package because I kept wanting Nimbus *inside* other
|
|
14
|
+
projects — a Durable Object that already does something else but needs a real
|
|
15
|
+
workspace, or a local script that needs the same filesystem semantics the
|
|
16
|
+
hosted product has. The whole of it runs on two narrow ports (`SqlDatabase`
|
|
17
|
+
and `SqlTransactions`), so the same code serves both hosts.
|
|
18
|
+
|
|
19
|
+
## Quick start
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
import { Database } from 'bun:sqlite';
|
|
23
|
+
import { NimbusWorkspace } from '@nimbus-sh/core/workspace';
|
|
24
|
+
|
|
25
|
+
const db = new Database('workspace.sqlite');
|
|
26
|
+
const sql = {
|
|
27
|
+
exec(q, ...p) {
|
|
28
|
+
const st = db.query(q);
|
|
29
|
+
if (st.columnNames.length === 0) { db.run(q, ...p); return []; }
|
|
30
|
+
return st.all(...p);
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
const transactions = { storage: { transactionSync: (cb) => db.transaction(cb)() } };
|
|
34
|
+
|
|
35
|
+
const ws = await NimbusWorkspace.create({ sql, transactions, generation: 1 });
|
|
36
|
+
|
|
37
|
+
await ws.fs.writeFile('/home/user/hello.txt', 'hi\n');
|
|
38
|
+
const out = await ws.exec('cat /home/user/hello.txt | wc -c'); // { stdout: '3\n', exitCode: 0 }
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Inside a Cloudflare Durable Object the same call is:
|
|
42
|
+
|
|
43
|
+
```ts
|
|
44
|
+
const ws = await NimbusWorkspace.create({
|
|
45
|
+
sql: this.ctx.storage.sql,
|
|
46
|
+
transactions: this.ctx,
|
|
47
|
+
generation,
|
|
48
|
+
});
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Files written through `.fs` are owned by the session user (uid 1000), not
|
|
52
|
+
root, and the shell enforces the same permission model either way: a
|
|
53
|
+
root-owned `/etc/passwd` refuses a write from `.fs`, and `id` resolves names
|
|
54
|
+
through it.
|
|
55
|
+
|
|
56
|
+
## Real runtimes, off Cloudflare
|
|
57
|
+
|
|
58
|
+
The wasm runtimes are separate npm packages so nobody downloads a Python
|
|
59
|
+
interpreter to get a filesystem. Install the ones you want and pass them in:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
npm install @nimbus-sh/runtime-bash @nimbus-sh/runtime-cpython
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
```ts
|
|
66
|
+
import bash from '@nimbus-sh/runtime-bash';
|
|
67
|
+
import cpython from '@nimbus-sh/runtime-cpython';
|
|
68
|
+
import { localFacetHost } from '@nimbus-sh/core';
|
|
69
|
+
|
|
70
|
+
const ws = await NimbusWorkspace.create({
|
|
71
|
+
sql, transactions, generation: 1,
|
|
72
|
+
facets: localFacetHost(),
|
|
73
|
+
runtimes: [bash, cpython],
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
await ws.exec('bash -c "echo $((6*7))"'); // 42 — GNU bash 5.2, real BusyBox children
|
|
77
|
+
await ws.exec(`python -c "import sqlite3; print('live')"`); // CPython 3.13, real stdlib
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`@nimbus-sh/runtime-ruby` (Ruby 3.3) and `@nimbus-sh/runtime-clang` (clang →
|
|
81
|
+
`wasm32-wasi`, compile and run C in the workspace) work the same way. Every
|
|
82
|
+
package carries the same manifest and the same sha256-verified blobs the
|
|
83
|
+
hosted product serves from R2 — one publisher, two transports.
|
|
84
|
+
|
|
85
|
+
Without `facets` and `runtimes` you still get the full shell and coreutils;
|
|
86
|
+
the wasm runtimes are a dependency you add, not a mode you enable.
|
|
87
|
+
|
|
88
|
+
## Sharing a database with your own app
|
|
89
|
+
|
|
90
|
+
The workspace is designed to be a tenant in a database you own, not the owner
|
|
91
|
+
of it:
|
|
92
|
+
|
|
93
|
+
- It creates and touches only its own tables (`inodes`, `file_chunks`,
|
|
94
|
+
`content_lifecycle`, `vfs_*`).
|
|
95
|
+
- `destroy()` drops exactly those tables. It never calls `deleteAll()`.
|
|
96
|
+
- `transactionSync` must be a real transaction. An implementation that only
|
|
97
|
+
calls the callback turns every atomic write into a torn one.
|
|
98
|
+
- `generation` must never repeat across restarts of your host — pids derive
|
|
99
|
+
from it, and a repeated generation would hand a dead process live write
|
|
100
|
+
authority.
|
|
101
|
+
|
|
102
|
+
## What needs the Worker package instead
|
|
103
|
+
|
|
104
|
+
Resident processes (long-running servers, attached TUIs), the session
|
|
105
|
+
protocol, port routing to the public internet, and the hosted terminal all
|
|
106
|
+
live in [`@nimbus-sh/worker`](https://www.npmjs.com/package/@nimbus-sh/worker),
|
|
107
|
+
which composes on this package. If you want the full hosted product shape,
|
|
108
|
+
start from `npx create-nimbus-app`.
|
|
109
|
+
|
|
110
|
+
## License
|
|
111
|
+
|
|
112
|
+
MIT.
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tarball-stream.ts — pure streaming tar primitives.
|
|
3
|
+
*
|
|
4
|
+
* A leaf with no imports at all, deliberately: `bundle-facet-workers.mjs`
|
|
5
|
+
* esbuilds this file into a string constant the loader pool injects into
|
|
6
|
+
* dynamic workers, and a facet isolate resolves no specifier. Anything this
|
|
7
|
+
* file imported would have to travel with it.
|
|
8
|
+
*
|
|
9
|
+
* Zero dependencies. Works identically on the supervisor and inside a
|
|
10
|
+
* facet isolate. Never buffers the full decompressed tarball — peak
|
|
11
|
+
* transient heap is one file's bytes plus a 512-byte carry.
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* Maximum size of a single file inside a tarball. Larger entries are skipped.
|
|
15
|
+
*
|
|
16
|
+
* History: 5 MB was too low — it silently dropped `esbuild-wasm/esbuild.wasm`
|
|
17
|
+
* (11.35 MB on v0.24.2), which made Nimbus-in-Nimbus `npm run dev` fail with
|
|
18
|
+
* `No such module "esbuild-wasm/esbuild.wasm"` since the missing file caused
|
|
19
|
+
* esbuild's VFS plugin to mark the import `external`, and workerd's LOADER
|
|
20
|
+
* has no entry for that specifier. 20 MB covers esbuild-wasm with headroom
|
|
21
|
+
* while keeping per-facet peak heap bounded for the streaming extractor.
|
|
22
|
+
*/
|
|
23
|
+
export declare const MAX_FILE_BYTES = 20000000;
|
|
24
|
+
/**
|
|
25
|
+
* Read one tar header (USTAR) out of `block`. Returns parsed fields or
|
|
26
|
+
* `null` for an end-of-archive block (all zeros).
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* Collapse "."/".." segments in a tar entry's package-relative path.
|
|
30
|
+
* Returns the canonical relative path, or '' when the entry escapes its
|
|
31
|
+
* package root (a leading ".." that pops above the root) — the caller
|
|
32
|
+
* treats '' as a no-name entry and skips it. Mirrors the segment logic in
|
|
33
|
+
* w7-frame's canonicalPath so joined write paths are always accepted.
|
|
34
|
+
*/
|
|
35
|
+
export declare function canonicalTarName(name: string): string;
|
|
36
|
+
export declare function parseTarHeader(block: Uint8Array): {
|
|
37
|
+
name: string;
|
|
38
|
+
size: number;
|
|
39
|
+
typeFlag: number;
|
|
40
|
+
} | null;
|
|
41
|
+
/**
|
|
42
|
+
* Wrap a `ReadableStream<Uint8Array>` as an async iterable. Workerd and
|
|
43
|
+
* Node both support `Symbol.asyncIterator` on ReadableStream, but we
|
|
44
|
+
* spell the reader loop out so we don't depend on ambient lib typings.
|
|
45
|
+
*/
|
|
46
|
+
export declare function readableStreamToAsyncIterable(rs: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined>;
|
|
47
|
+
/**
|
|
48
|
+
* Reason a tar entry was skipped and never yielded. Surfaced to callers
|
|
49
|
+
* via the optional `onSkip` callback on `streamTarEntries`.
|
|
50
|
+
*
|
|
51
|
+
* - 'too-large': size > MAX_FILE_BYTES. The most common reason Nimbus
|
|
52
|
+
* cares about — it's what caused esbuild-wasm/esbuild.wasm to vanish
|
|
53
|
+
* silently in Nimbus-in-Nimbus before the cap was raised.
|
|
54
|
+
* - 'non-regular': typeFlag indicates a symlink / hardlink / directory /
|
|
55
|
+
* PaxHeader / GNU LongName etc. These aren't files we stage.
|
|
56
|
+
* - 'no-name': header parsed but name was empty (malformed or a PaxHeader
|
|
57
|
+
* that our parser didn't recognize as non-regular).
|
|
58
|
+
*/
|
|
59
|
+
export type TarSkipReason = 'too-large' | 'non-regular' | 'no-name';
|
|
60
|
+
/**
|
|
61
|
+
* Optional skip-observer passed to `streamTarEntries`. Called ONCE per
|
|
62
|
+
* skipped entry, with the declared name (may be empty for 'no-name'
|
|
63
|
+
* skips) and the declared size in bytes.
|
|
64
|
+
*
|
|
65
|
+
* Consumers typically push these into a per-package warnings array so
|
|
66
|
+
* users see what wasn't installed. The callback is synchronous and
|
|
67
|
+
* must not throw — thrown errors are swallowed to keep the extractor
|
|
68
|
+
* best-effort.
|
|
69
|
+
*/
|
|
70
|
+
export type TarSkipCallback = (name: string, size: number, reason: TarSkipReason) => void;
|
|
71
|
+
/**
|
|
72
|
+
* Streaming tar extractor.
|
|
73
|
+
*
|
|
74
|
+
* Consumes an async iterable of Uint8Array chunks (the decompressed tar
|
|
75
|
+
* byte stream) and yields one `{ name, data }` entry per regular file,
|
|
76
|
+
* as each file completes.
|
|
77
|
+
*
|
|
78
|
+
* Memory invariant: holds at most one pending file's bytes (≤ MAX_FILE_BYTES)
|
|
79
|
+
* plus a small carry buffer for the tar header being assembled.
|
|
80
|
+
*
|
|
81
|
+
* Skips: symlinks, directories, hardlinks, long-name extensions (PaxHeader),
|
|
82
|
+
* and any file whose declared size exceeds MAX_FILE_BYTES.
|
|
83
|
+
*
|
|
84
|
+
* If `onSkip` is provided, it is invoked for each skipped entry with the
|
|
85
|
+
* name, declared size, and reason code. Callers that need to surface
|
|
86
|
+
* dropped-file warnings to users should pass one; legacy callers that
|
|
87
|
+
* omit the arg still behave exactly as before (silent skip).
|
|
88
|
+
*/
|
|
89
|
+
export declare function streamTarEntries(source: AsyncIterable<Uint8Array>, onSkip?: TarSkipCallback): AsyncGenerator<{
|
|
90
|
+
name: string;
|
|
91
|
+
data: Uint8Array;
|
|
92
|
+
}, void, undefined>;
|
|
93
|
+
//# sourceMappingURL=tarball-stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tarball-stream.d.ts","sourceRoot":"","sources":["../../src/_shared/tarball-stream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,cAAc,WAAa,CAAC;AAEzC;;;GAGG;AACH;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAWrD;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CA+BzG;AAED;;;;GAIG;AACH,wBAAuB,6BAA6B,CAClD,EAAE,EAAE,cAAc,CAAC,UAAU,CAAC,GAC7B,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAW7C;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,aAAa,GAAG,SAAS,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,eAAe,GAAG,CAC5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,aAAa,KAClB,IAAI,CAAC;AAEV;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAuB,gBAAgB,CACrC,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,EACjC,MAAM,CAAC,EAAE,eAAe,GACvB,cAAc,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,EAAE,IAAI,EAAE,SAAS,CAAC,CA6GrE"}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tarball-stream.ts — pure streaming tar primitives.
|
|
3
|
+
*
|
|
4
|
+
* A leaf with no imports at all, deliberately: `bundle-facet-workers.mjs`
|
|
5
|
+
* esbuilds this file into a string constant the loader pool injects into
|
|
6
|
+
* dynamic workers, and a facet isolate resolves no specifier. Anything this
|
|
7
|
+
* file imported would have to travel with it.
|
|
8
|
+
*
|
|
9
|
+
* Zero dependencies. Works identically on the supervisor and inside a
|
|
10
|
+
* facet isolate. Never buffers the full decompressed tarball — peak
|
|
11
|
+
* transient heap is one file's bytes plus a 512-byte carry.
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* Maximum size of a single file inside a tarball. Larger entries are skipped.
|
|
15
|
+
*
|
|
16
|
+
* History: 5 MB was too low — it silently dropped `esbuild-wasm/esbuild.wasm`
|
|
17
|
+
* (11.35 MB on v0.24.2), which made Nimbus-in-Nimbus `npm run dev` fail with
|
|
18
|
+
* `No such module "esbuild-wasm/esbuild.wasm"` since the missing file caused
|
|
19
|
+
* esbuild's VFS plugin to mark the import `external`, and workerd's LOADER
|
|
20
|
+
* has no entry for that specifier. 20 MB covers esbuild-wasm with headroom
|
|
21
|
+
* while keeping per-facet peak heap bounded for the streaming extractor.
|
|
22
|
+
*/
|
|
23
|
+
export const MAX_FILE_BYTES = 20_000_000;
|
|
24
|
+
/**
|
|
25
|
+
* Read one tar header (USTAR) out of `block`. Returns parsed fields or
|
|
26
|
+
* `null` for an end-of-archive block (all zeros).
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* Collapse "."/".." segments in a tar entry's package-relative path.
|
|
30
|
+
* Returns the canonical relative path, or '' when the entry escapes its
|
|
31
|
+
* package root (a leading ".." that pops above the root) — the caller
|
|
32
|
+
* treats '' as a no-name entry and skips it. Mirrors the segment logic in
|
|
33
|
+
* w7-frame's canonicalPath so joined write paths are always accepted.
|
|
34
|
+
*/
|
|
35
|
+
export function canonicalTarName(name) {
|
|
36
|
+
const out = [];
|
|
37
|
+
for (const seg of name.split('/')) {
|
|
38
|
+
if (seg === '..') {
|
|
39
|
+
if (out.length === 0)
|
|
40
|
+
return '';
|
|
41
|
+
out.pop();
|
|
42
|
+
}
|
|
43
|
+
else if (seg !== '' && seg !== '.') {
|
|
44
|
+
out.push(seg);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return out.join('/');
|
|
48
|
+
}
|
|
49
|
+
export function parseTarHeader(block) {
|
|
50
|
+
if (block[0] === 0)
|
|
51
|
+
return null;
|
|
52
|
+
let name = '';
|
|
53
|
+
for (let i = 0; i < 100 && block[i] !== 0; i++) {
|
|
54
|
+
name += String.fromCharCode(block[i]);
|
|
55
|
+
}
|
|
56
|
+
let prefix = '';
|
|
57
|
+
for (let i = 345; i < 500 && block[i] !== 0; i++) {
|
|
58
|
+
prefix += String.fromCharCode(block[i]);
|
|
59
|
+
}
|
|
60
|
+
if (prefix)
|
|
61
|
+
name = prefix + '/' + name;
|
|
62
|
+
// Strip the npm `package/` convention.
|
|
63
|
+
name = name.replace(/^package\//, '');
|
|
64
|
+
// Canonicalize the entry-relative path. npm tarballs legitimately carry
|
|
65
|
+
// entries like "./dist/index.js" (agent-base, http-proxy-agent,
|
|
66
|
+
// protobufjs, ...); left as-is the "./" survives into the VFS write path
|
|
67
|
+
// and the w7-frame writer rejects the noncanonical path, failing the
|
|
68
|
+
// whole shared install wave and dropping shard-mates' completion markers.
|
|
69
|
+
// Collapsing here (the one place entry names are assembled) keeps every
|
|
70
|
+
// downstream join canonical. An entry that escapes its package root via
|
|
71
|
+
// ".." is dropped to '' → skipped as a no-name entry.
|
|
72
|
+
name = canonicalTarName(name);
|
|
73
|
+
let sizeStr = '';
|
|
74
|
+
for (let i = 124; i < 136 && block[i] !== 0; i++) {
|
|
75
|
+
sizeStr += String.fromCharCode(block[i]);
|
|
76
|
+
}
|
|
77
|
+
const size = parseInt(sizeStr.trim(), 8) || 0;
|
|
78
|
+
const typeFlag = block[156];
|
|
79
|
+
return { name, size, typeFlag };
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Wrap a `ReadableStream<Uint8Array>` as an async iterable. Workerd and
|
|
83
|
+
* Node both support `Symbol.asyncIterator` on ReadableStream, but we
|
|
84
|
+
* spell the reader loop out so we don't depend on ambient lib typings.
|
|
85
|
+
*/
|
|
86
|
+
export async function* readableStreamToAsyncIterable(rs) {
|
|
87
|
+
const reader = rs.getReader();
|
|
88
|
+
try {
|
|
89
|
+
while (true) {
|
|
90
|
+
const { value, done } = await reader.read();
|
|
91
|
+
if (done)
|
|
92
|
+
return;
|
|
93
|
+
if (value && value.length > 0)
|
|
94
|
+
yield value;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
finally {
|
|
98
|
+
try {
|
|
99
|
+
reader.releaseLock();
|
|
100
|
+
}
|
|
101
|
+
catch { /* ignore */ }
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Streaming tar extractor.
|
|
106
|
+
*
|
|
107
|
+
* Consumes an async iterable of Uint8Array chunks (the decompressed tar
|
|
108
|
+
* byte stream) and yields one `{ name, data }` entry per regular file,
|
|
109
|
+
* as each file completes.
|
|
110
|
+
*
|
|
111
|
+
* Memory invariant: holds at most one pending file's bytes (≤ MAX_FILE_BYTES)
|
|
112
|
+
* plus a small carry buffer for the tar header being assembled.
|
|
113
|
+
*
|
|
114
|
+
* Skips: symlinks, directories, hardlinks, long-name extensions (PaxHeader),
|
|
115
|
+
* and any file whose declared size exceeds MAX_FILE_BYTES.
|
|
116
|
+
*
|
|
117
|
+
* If `onSkip` is provided, it is invoked for each skipped entry with the
|
|
118
|
+
* name, declared size, and reason code. Callers that need to surface
|
|
119
|
+
* dropped-file warnings to users should pass one; legacy callers that
|
|
120
|
+
* omit the arg still behave exactly as before (silent skip).
|
|
121
|
+
*/
|
|
122
|
+
export async function* streamTarEntries(source, onSkip) {
|
|
123
|
+
let carry = new Uint8Array(0);
|
|
124
|
+
let state = { kind: 'header' };
|
|
125
|
+
function concat(a, b) {
|
|
126
|
+
if (a.length === 0)
|
|
127
|
+
return b;
|
|
128
|
+
if (b.length === 0)
|
|
129
|
+
return a;
|
|
130
|
+
const out = new Uint8Array(a.length + b.length);
|
|
131
|
+
out.set(a, 0);
|
|
132
|
+
out.set(b, a.length);
|
|
133
|
+
return out;
|
|
134
|
+
}
|
|
135
|
+
for await (const chunkRaw of source) {
|
|
136
|
+
let buf = concat(carry, chunkRaw);
|
|
137
|
+
let cursor = 0;
|
|
138
|
+
while (true) {
|
|
139
|
+
if (state.kind === 'header') {
|
|
140
|
+
if (buf.length - cursor < 512)
|
|
141
|
+
break;
|
|
142
|
+
const header = buf.subarray(cursor, cursor + 512);
|
|
143
|
+
const parsed = parseTarHeader(header);
|
|
144
|
+
cursor += 512;
|
|
145
|
+
if (!parsed)
|
|
146
|
+
return; // end-of-archive
|
|
147
|
+
const { name, size, typeFlag } = parsed;
|
|
148
|
+
const pad = size === 0 ? 0 : (512 - (size % 512)) % 512;
|
|
149
|
+
const isRegularFile = (typeFlag === 48 /* '0' */ || typeFlag === 0);
|
|
150
|
+
if (size === 0) {
|
|
151
|
+
if (isRegularFile && name) {
|
|
152
|
+
yield { name, data: new Uint8Array(0) };
|
|
153
|
+
}
|
|
154
|
+
state = { kind: 'header' };
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
if (!isRegularFile || !name || size > MAX_FILE_BYTES) {
|
|
158
|
+
if (onSkip) {
|
|
159
|
+
// Classify before entering the skip state so the reason is
|
|
160
|
+
// exact. Precedence matches the condition order above:
|
|
161
|
+
// non-regular first (directories/symlinks/PaxHeaders are
|
|
162
|
+
// skipped regardless of size), then no-name, then too-large.
|
|
163
|
+
const reason = !isRegularFile
|
|
164
|
+
? 'non-regular'
|
|
165
|
+
: !name
|
|
166
|
+
? 'no-name'
|
|
167
|
+
: 'too-large';
|
|
168
|
+
try {
|
|
169
|
+
onSkip(name, size, reason);
|
|
170
|
+
}
|
|
171
|
+
catch { /* best-effort */ }
|
|
172
|
+
}
|
|
173
|
+
state = { kind: 'skip', remaining: size + pad };
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
state = {
|
|
177
|
+
kind: 'file',
|
|
178
|
+
name,
|
|
179
|
+
remaining: size,
|
|
180
|
+
fileBuf: new Uint8Array(size),
|
|
181
|
+
fileOffset: 0,
|
|
182
|
+
pad,
|
|
183
|
+
skip: false,
|
|
184
|
+
};
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
if (state.kind === 'file') {
|
|
188
|
+
const avail = buf.length - cursor;
|
|
189
|
+
if (avail === 0)
|
|
190
|
+
break;
|
|
191
|
+
if (state.remaining > 0) {
|
|
192
|
+
const take = Math.min(state.remaining, avail);
|
|
193
|
+
state.fileBuf.set(buf.subarray(cursor, cursor + take), state.fileOffset);
|
|
194
|
+
state.fileOffset += take;
|
|
195
|
+
state.remaining -= take;
|
|
196
|
+
cursor += take;
|
|
197
|
+
if (state.remaining > 0)
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
if (state.pad > 0) {
|
|
201
|
+
const avail2 = buf.length - cursor;
|
|
202
|
+
if (avail2 === 0)
|
|
203
|
+
break;
|
|
204
|
+
const take = Math.min(state.pad, avail2);
|
|
205
|
+
state.pad -= take;
|
|
206
|
+
cursor += take;
|
|
207
|
+
if (state.pad > 0)
|
|
208
|
+
break;
|
|
209
|
+
}
|
|
210
|
+
yield { name: state.name, data: state.fileBuf };
|
|
211
|
+
state = { kind: 'header' };
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
// state.kind === 'skip'
|
|
215
|
+
const avail = buf.length - cursor;
|
|
216
|
+
if (avail === 0)
|
|
217
|
+
break;
|
|
218
|
+
const take = Math.min(state.remaining, avail);
|
|
219
|
+
cursor += take;
|
|
220
|
+
state.remaining -= take;
|
|
221
|
+
if (state.remaining > 0)
|
|
222
|
+
break;
|
|
223
|
+
state = { kind: 'header' };
|
|
224
|
+
}
|
|
225
|
+
if (cursor >= buf.length) {
|
|
226
|
+
carry = new Uint8Array(0);
|
|
227
|
+
}
|
|
228
|
+
else if (cursor === 0) {
|
|
229
|
+
carry = buf;
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
carry = buf.slice(cursor);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tarball.ts — a whole tarball, in memory, as a path→bytes map.
|
|
3
|
+
*
|
|
4
|
+
* The streaming primitives it walks with (`parseTarHeader`, `streamTarEntries`,
|
|
5
|
+
* `readableStreamToAsyncIterable`) live in `./tarball-stream.ts` — a
|
|
6
|
+
* dependency-free leaf, because `bundle-facet-workers.mjs` esbuilds that file
|
|
7
|
+
* into a string the loader pool injects into dynamic workers, where an import
|
|
8
|
+
* would not resolve.
|
|
9
|
+
*
|
|
10
|
+
* This half is for a caller that already holds the bytes: `gem install`, which
|
|
11
|
+
* fetches a `.gem` (a tar holding a gzipped tar) and needs both layers open at
|
|
12
|
+
* once. An installer streaming a tarball it is still downloading should drive
|
|
13
|
+
* `streamTarEntries` itself and never hold the whole thing.
|
|
14
|
+
*/
|
|
15
|
+
/** Extract every regular file. Gzipped input is decompressed first. */
|
|
16
|
+
export declare function extractTarball(tarball: ArrayBuffer): Promise<Map<string, Uint8Array>>;
|
|
17
|
+
//# sourceMappingURL=tarball.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tarball.d.ts","sourceRoot":"","sources":["../../src/_shared/tarball.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,uEAAuE;AACvE,wBAAsB,cAAc,CAClC,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAsBlC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tarball.ts — a whole tarball, in memory, as a path→bytes map.
|
|
3
|
+
*
|
|
4
|
+
* The streaming primitives it walks with (`parseTarHeader`, `streamTarEntries`,
|
|
5
|
+
* `readableStreamToAsyncIterable`) live in `./tarball-stream.ts` — a
|
|
6
|
+
* dependency-free leaf, because `bundle-facet-workers.mjs` esbuilds that file
|
|
7
|
+
* into a string the loader pool injects into dynamic workers, where an import
|
|
8
|
+
* would not resolve.
|
|
9
|
+
*
|
|
10
|
+
* This half is for a caller that already holds the bytes: `gem install`, which
|
|
11
|
+
* fetches a `.gem` (a tar holding a gzipped tar) and needs both layers open at
|
|
12
|
+
* once. An installer streaming a tarball it is still downloading should drive
|
|
13
|
+
* `streamTarEntries` itself and never hold the whole thing.
|
|
14
|
+
*/
|
|
15
|
+
import { streamTarEntries, readableStreamToAsyncIterable, } from './tarball-stream.js';
|
|
16
|
+
/** Extract every regular file. Gzipped input is decompressed first. */
|
|
17
|
+
export async function extractTarball(tarball) {
|
|
18
|
+
const files = new Map();
|
|
19
|
+
const raw = new Uint8Array(tarball);
|
|
20
|
+
// Adapter: wrap the single buffer as an async iterable. If gzipped, pipe
|
|
21
|
+
// through DecompressionStream so the streaming parser still sees tar bytes.
|
|
22
|
+
let source;
|
|
23
|
+
if (raw[0] === 0x1f && raw[1] === 0x8b) {
|
|
24
|
+
const rs = new Blob([tarball]).stream().pipeThrough(new DecompressionStream('gzip'));
|
|
25
|
+
source = readableStreamToAsyncIterable(rs);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
source = (async function* () { yield raw; })();
|
|
29
|
+
}
|
|
30
|
+
try {
|
|
31
|
+
for await (const entry of streamTarEntries(source)) {
|
|
32
|
+
files.set(entry.name, entry.data);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return files;
|
|
37
|
+
}
|
|
38
|
+
return files;
|
|
39
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* clang-runner.ts — compile, link, and execute C programs for Nimbus WASI.
|
|
3
|
+
*
|
|
4
|
+
* Architecture (compile-link-run, two facet calls):
|
|
5
|
+
*
|
|
6
|
+
* compile : clang.wasm + sysroot subset for C includes + user .c
|
|
7
|
+
* → produces .o bytes (returned to supervisor).
|
|
8
|
+
* link : lld.wasm + sysroot subset for link (crt1.o + libc.a)
|
|
9
|
+
* + .o from compile → produces .wasm executable.
|
|
10
|
+
* write : final .wasm flushed to user VFS at the requested path.
|
|
11
|
+
*
|
|
12
|
+
* The filesystem both halves see is the one WASI layer every other
|
|
13
|
+
* non-node runtime uses (wasi-instance.ts), seeded and sealed.
|
|
14
|
+
*
|
|
15
|
+
* Splitting compile and link into separate facet calls keeps each
|
|
16
|
+
* call under the empirical payload ceiling. Each ships:
|
|
17
|
+
*
|
|
18
|
+
* - compile: 31 MiB clang.wasm + ~1.3 MiB sysroot subset (C includes).
|
|
19
|
+
* - link : 19 MiB lld.wasm + ~0.75 MiB libs + tiny .o.
|
|
20
|
+
*
|
|
21
|
+
* Sysroot subset extraction happens supervisor-side via a small ustar
|
|
22
|
+
* parser. The full sysroot.tar is parsed once when the clang runtime
|
|
23
|
+
* warms for a session; compile/link calls reuse the filtered subsets.
|
|
24
|
+
*
|
|
25
|
+
* Dispatch stays direct: no sleeps, no caller-side retries, and no
|
|
26
|
+
* catch-and-continue around loader failures.
|
|
27
|
+
*/
|
|
28
|
+
import type { RuntimeManifest } from './runtime-manifest.js';
|
|
29
|
+
import type { SqliteVFS } from '../vfs/sqlite-vfs.js';
|
|
30
|
+
import type { Command } from '../substrate/lifo/commands/types.js';
|
|
31
|
+
import type { FacetHost } from './facet-host.js';
|
|
32
|
+
/** Build the runner factory. Closes over the facet host + vfs. */
|
|
33
|
+
export declare function makeClangRunnerFactory(deps: {
|
|
34
|
+
facets: FacetHost;
|
|
35
|
+
vfs: SqliteVFS;
|
|
36
|
+
}): (manifest: RuntimeManifest, installRoot: string, binName: string, binKind: string | undefined) => Command;
|
|
37
|
+
export declare const CLANG_RUNNER_PREAMBLE: string;
|
|
38
|
+
//# sourceMappingURL=clang-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"clang-runner.d.ts","sourceRoot":"","sources":["../../src/runtime/clang-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAmB,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,KAAK,EAAE,OAAO,EAAkB,MAAM,qCAAqC,CAAC;AACnF,OAAO,KAAK,EAAS,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAQxD,kEAAkE;AAClE,wBAAgB,sBAAsB,CAAC,IAAI,EAAE;IAC3C,MAAM,EAAE,SAAS,CAAC;IAClB,GAAG,EAAE,SAAS,CAAC;CAChB,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,KAC7F,OAAO,CAiTV;AA8jBD,eAAO,MAAM,qBAAqB,QAAiE,CAAC"}
|