knitting 0.1.56 → 0.1.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BROWSER.md +134 -0
- package/README.md +347 -46
- package/knitting.browser.d.ts +6 -0
- package/knitting.browser.js +1 -0
- package/knitting.d.ts +3 -2
- package/knitting.js +3 -3
- package/map.md +21 -9
- package/package.json +17 -2
- package/prebuilds/darwin-arm64-node-127/knitting_buffer_pointer.node +0 -0
- package/prebuilds/darwin-arm64-node-127/knitting_shared_memory.node +0 -0
- package/prebuilds/darwin-arm64-node-127/knitting_shm.node +0 -0
- package/prebuilds/darwin-arm64-node-137/knitting_buffer_pointer.node +0 -0
- package/prebuilds/darwin-arm64-node-137/knitting_shared_memory.node +0 -0
- package/prebuilds/darwin-arm64-node-137/knitting_shm.node +0 -0
- package/prebuilds/win32-x64/knitting_windows_shared_memory.dll +0 -0
- package/prebuilds/win32-x64-node-127/knitting_buffer_pointer.node +0 -0
- package/prebuilds/win32-x64-node-127/knitting_shared_memory.node +0 -0
- package/prebuilds/win32-x64-node-127/knitting_shm.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_buffer_pointer.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_shared_memory.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_shm.node +0 -0
- package/scripts/build-compiled-worker.ts +276 -0
- package/scripts/compiled-worker/porffor.d.ts +5 -0
- package/scripts/compiled-worker/runtime.ts +529 -0
- package/scripts/compiled-worker/task-shim.ts +32 -0
- package/src/api.d.ts +24 -1
- package/src/api.js +311 -27
- package/src/common/module-url.d.ts +1 -0
- package/src/common/module-url.js +30 -2
- package/src/common/node-compat.d.ts +2 -0
- package/src/common/path-canonical.js +20 -6
- package/src/common/runtime.d.ts +3 -1
- package/src/common/runtime.js +25 -4
- package/src/common/task-source.d.ts +2 -0
- package/src/common/task-source.js +17 -1
- package/src/common/worker-runtime.js +16 -7
- package/src/connections/buffer-reference-native.js +25 -1
- package/src/connections/buffer-reference.d.ts +23 -2
- package/src/connections/buffer-reference.js +143 -29
- package/src/connections/bun.d.ts +1 -1
- package/src/connections/bun.js +25 -14
- package/src/connections/deno.d.ts +3 -1
- package/src/connections/deno.js +35 -14
- package/src/connections/external-array-buffer.d.ts +2 -0
- package/src/connections/external-array-buffer.js +33 -0
- package/src/connections/node-addons.d.ts +11 -0
- package/src/connections/node-addons.js +49 -1
- package/src/connections/node-ffi-api.d.ts +19 -0
- package/src/connections/node-ffi-api.js +28 -0
- package/src/connections/node-ffi.d.ts +27 -0
- package/src/connections/node-ffi.js +262 -0
- package/src/connections/node.d.ts +1 -1
- package/src/connections/node.js +20 -20
- package/src/connections/package-assets.js +25 -15
- package/src/connections/posix.d.ts +42 -0
- package/src/connections/posix.js +101 -0
- package/src/connections/process-shared-buffer.d.ts +1 -0
- package/src/connections/process-shared-buffer.js +11 -45
- package/src/connections/types.d.ts +4 -0
- package/src/connections/windows.d.ts +19 -10
- package/src/connections/windows.js +87 -66
- package/src/error.d.ts +7 -6
- package/src/error.js +8 -7
- package/src/memory/lock.d.ts +112 -77
- package/src/memory/lock.js +314 -87
- package/src/memory/payloadCodec.d.ts +11 -2
- package/src/memory/payloadCodec.js +129 -37
- package/src/memory/regionRegistry.d.ts +1 -3
- package/src/memory/regionRegistry.js +16 -14
- package/src/permission/compatibility.d.ts +28 -0
- package/src/permission/compatibility.js +239 -0
- package/src/permission/index.d.ts +2 -0
- package/src/permission/index.js +1 -0
- package/src/permission/protocol.d.ts +1 -0
- package/src/permission/protocol.js +59 -15
- package/src/runtime/compiled-artifact.d.ts +21 -0
- package/src/runtime/compiled-artifact.js +260 -0
- package/src/runtime/compiled-builder.d.ts +8 -0
- package/src/runtime/compiled-builder.js +42 -0
- package/src/runtime/compiled-worker.d.ts +27 -0
- package/src/runtime/compiled-worker.js +562 -0
- package/src/runtime/dispatcher.js +9 -0
- package/src/runtime/inline-executor.js +27 -15
- package/src/runtime/pool.d.ts +121 -5
- package/src/runtime/pool.js +263 -49
- package/src/runtime/process-worker.d.ts +3 -13
- package/src/runtime/process-worker.js +97 -101
- package/src/runtime/tx-queue.d.ts +18 -2
- package/src/runtime/tx-queue.js +56 -16
- package/src/runtime/worker-common.d.ts +13 -0
- package/src/runtime/worker-common.js +102 -0
- package/src/types.d.ts +89 -9
- package/src/worker/composable-runners.js +3 -2
- package/src/worker/loop.d.ts +1 -0
- package/src/worker/loop.js +52 -17
- package/src/worker/rx-queue.d.ts +11 -1
- package/src/worker/rx-queue.js +4 -1
- package/src/worker/task-loader.d.ts +5 -4
- package/src/worker/task-loader.js +13 -7
- package/src/worker/timers.d.ts +8 -1
- package/src/worker/timers.js +28 -23
package/BROWSER.md
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# Browser limitations
|
|
2
|
+
|
|
3
|
+
`knitting/browser` runs the same pool API over web workers and
|
|
4
|
+
`SharedArrayBuffer`. Tasks, typed-array payloads, parallel calls, abort signals,
|
|
5
|
+
and shutdown all behave as they do on Node, Deno, and Bun.
|
|
6
|
+
|
|
7
|
+
This page is the other half: what a page cannot do, what happens when you try,
|
|
8
|
+
and why. Everything below was checked against headless Chromium, and every
|
|
9
|
+
quoted message is the one the runtime actually produces.
|
|
10
|
+
|
|
11
|
+
## Two hard requirements
|
|
12
|
+
|
|
13
|
+
### The page must be cross-origin isolated
|
|
14
|
+
|
|
15
|
+
Browsers only expose `SharedArrayBuffer` to pages served with both:
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
Cross-Origin-Opener-Policy: same-origin
|
|
19
|
+
Cross-Origin-Embedder-Policy: require-corp
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Without them `createPool` throws:
|
|
23
|
+
|
|
24
|
+
> SharedArrayBuffer is unavailable: serve the page cross-origin isolated
|
|
25
|
+
> (Cross-Origin-Opener-Policy: same-origin, Cross-Origin-Embedder-Policy:
|
|
26
|
+
> require-corp).
|
|
27
|
+
|
|
28
|
+
This is not a knitting policy and cannot be worked around: no shared memory
|
|
29
|
+
means no pool. It also constrains the rest of the page — every cross-origin
|
|
30
|
+
subresource needs `Cross-Origin-Resource-Policy` or CORS, or the browser refuses
|
|
31
|
+
to load it once isolation is on.
|
|
32
|
+
|
|
33
|
+
### Task modules must call `setModuleUrl(import.meta.url)`
|
|
34
|
+
|
|
35
|
+
On Node, Deno, and Bun a task discovers its own module by walking the stack.
|
|
36
|
+
Bundlers erase the paths that depends on, so in a browser the module has to say
|
|
37
|
+
where it lives:
|
|
38
|
+
|
|
39
|
+
```js
|
|
40
|
+
import { setModuleUrl, task } from "knitting/browser";
|
|
41
|
+
|
|
42
|
+
setModuleUrl(import.meta.url);
|
|
43
|
+
|
|
44
|
+
export const square = task({ f: (value) => value * value });
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Without it the worker cannot import the module that defines the tasks.
|
|
48
|
+
|
|
49
|
+
## Not available in a browser
|
|
50
|
+
|
|
51
|
+
| Feature | What happens |
|
|
52
|
+
| -------------------------------------------------------------------- | ---------------------------------------------------------------- |
|
|
53
|
+
| Process workers (`worker.runtime: "process"`, `processRuntime`) | throws `process workers are unavailable in the browser build` |
|
|
54
|
+
| Compiled / Porffor workers (`runtime: "compiled"`, `.knt` artifacts) | throws `compiled workers are unavailable in the browser build` |
|
|
55
|
+
| `BufferReference` | throws `BufferReference cannot run in runtime "browser"` |
|
|
56
|
+
| `ProcessSharedBuffer`, named shared memory | throws `ProcessSharedBuffer is unavailable in the browser build` |
|
|
57
|
+
| Native addons, FFI, file descriptors | unreachable; nothing in a page can load them |
|
|
58
|
+
| `checkCompiledWorker` | not exported from `knitting/browser` |
|
|
59
|
+
| Permissions (`permission: {...}`) | **silently ignored** — see below |
|
|
60
|
+
|
|
61
|
+
All of these need a filesystem, a process to spawn, or FFI. The browser build
|
|
62
|
+
replaces them with stubs that raise the errors above rather than failing deeper
|
|
63
|
+
in with a confusing one.
|
|
64
|
+
|
|
65
|
+
### Permissions are inert, and that is a security boundary
|
|
66
|
+
|
|
67
|
+
The `permission` option is accepted and then skipped entirely. There is no
|
|
68
|
+
filesystem to restrict, no process to sandbox, and no runtime flags to pass, so
|
|
69
|
+
a strict policy that would constrain a Node worker constrains nothing here.
|
|
70
|
+
|
|
71
|
+
A web worker runs with the privileges of the page that spawned it: same origin,
|
|
72
|
+
same `fetch` reach, same storage. **Task code you would not trust with your
|
|
73
|
+
origin must not run in a browser pool.** Isolation there is the browser's job —
|
|
74
|
+
a sandboxed iframe on a separate origin — not knitting's.
|
|
75
|
+
|
|
76
|
+
### `SharedArrayBuffer` as a task argument
|
|
77
|
+
|
|
78
|
+
This one differs from every other runtime, so it is worth calling out
|
|
79
|
+
separately. Passing a `SharedArrayBuffer` _as an argument to a task_ works on
|
|
80
|
+
Node, Deno, and Bun, and throws in a browser:
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
KNT_ERROR_3: Unsupported payload type; BufferReference cannot run in runtime "browser"
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The SAB payload path shares buffers by pinning a process-local pointer through
|
|
87
|
+
FFI, which a page cannot do. The pool's own transport is unaffected — that is
|
|
88
|
+
how the workers talk at all — this is only about SABs you pass yourself.
|
|
89
|
+
|
|
90
|
+
Nothing prevents a browser-native path here: SABs are structured-cloneable in a
|
|
91
|
+
cross-origin isolated context, so `postMessage` could carry them without any
|
|
92
|
+
pointer. It simply is not implemented.
|
|
93
|
+
|
|
94
|
+
## Behaves differently, but works
|
|
95
|
+
|
|
96
|
+
- **Workers boot by message, not by `workerData`.** The pool posts the boot
|
|
97
|
+
payload after constructing the worker. Invisible in the API; relevant if you
|
|
98
|
+
are reading worker startup code.
|
|
99
|
+
- **`KNITTING_DEBUG` is ignored.** The env gate reads `Deno.env` or
|
|
100
|
+
`process.env`, neither of which exists in a page. Pass the `debug` option to
|
|
101
|
+
`createPool` explicitly instead.
|
|
102
|
+
- **`threads` still defaults to 1.** There is no auto-sizing on any runtime; in
|
|
103
|
+
a browser `navigator.hardwareConcurrency` is the number to reach for.
|
|
104
|
+
- **Every worker loads the whole bundle.** The worker URL is the bundle's own
|
|
105
|
+
URL, so each thread parses the library again. Memory cost scales with thread
|
|
106
|
+
count.
|
|
107
|
+
|
|
108
|
+
## Not browser limitations
|
|
109
|
+
|
|
110
|
+
These fail the same way on Node, so do not go looking for a browser cause:
|
|
111
|
+
|
|
112
|
+
- `BigInt` payloads — `Do not know how to serialize a BigInt`
|
|
113
|
+
- `Map` / `Set` payloads — `Unsupported object type`
|
|
114
|
+
- Functions as payloads — `KNT_ERROR_0: Function is not a valid type`
|
|
115
|
+
- `Error` values round-trip as `{ name }`, dropping the message
|
|
116
|
+
|
|
117
|
+
## Where this is tested
|
|
118
|
+
|
|
119
|
+
The browser lane (`npm run test:browser`) drives headless Chromium through two
|
|
120
|
+
layouts — one bundle containing tasks and library, and the standalone
|
|
121
|
+
single-file bundle loaded from a script tag next to a separate task module.
|
|
122
|
+
|
|
123
|
+
**Chromium is the only engine covered.** Firefox and Safari support the same
|
|
124
|
+
primitives (workers, `SharedArrayBuffer` under cross-origin isolation) and are
|
|
125
|
+
expected to work, but nothing here has been verified against them.
|
|
126
|
+
|
|
127
|
+
## How the browser build is produced
|
|
128
|
+
|
|
129
|
+
`knitting/browser` ships as one self-contained minified file (~94 KB, ~32 KB
|
|
130
|
+
gzipped). The Node-only subsystems are swapped for stubs at bundle time by
|
|
131
|
+
[`scripts/browser-stubs/plugin.ts`](scripts/browser-stubs/plugin.ts); each stub
|
|
132
|
+
keeps the behaviour the real module already had in a page — return nothing,
|
|
133
|
+
answer false, or throw the message quoted above — which is why the errors in
|
|
134
|
+
this document are precise rather than generic.
|
package/README.md
CHANGED
|
@@ -15,7 +15,8 @@ Website: [knittingdocs.netlify.app](https://knittingdocs.netlify.app/)
|
|
|
15
15
|
|
|
16
16
|
If you are an agent trying to understand the project, the website also serves an
|
|
17
17
|
[`llms.txt`](https://knittingdocs.netlify.app/llms.txt) file with a compact map
|
|
18
|
-
of the docs
|
|
18
|
+
of the docs, plus a full inlined version at
|
|
19
|
+
[`llms-full.txt`](https://knittingdocs.netlify.app/llms-full.txt).
|
|
19
20
|
|
|
20
21
|
Knitting is a worker pool built on shared-memory IPC for Node.js, Deno, and Bun.
|
|
21
22
|
It lets you call work running on other threads or processes as if it were a
|
|
@@ -60,7 +61,8 @@ cross-runtime shared memory.
|
|
|
60
61
|
|
|
61
62
|
## Requirements
|
|
62
63
|
|
|
63
|
-
- Node.js 22
|
|
64
|
+
- Node.js 22+; native features support Node.js 22 and 24 through prebuilt
|
|
65
|
+
addons, and Node.js 26 through experimental `node:ffi`
|
|
64
66
|
- Deno 2+
|
|
65
67
|
- Bun 1+
|
|
66
68
|
|
|
@@ -235,6 +237,9 @@ export const countUntilStopped = task({
|
|
|
235
237
|
});
|
|
236
238
|
```
|
|
237
239
|
|
|
240
|
+
The signal also carries `signal.now()`, a monotonic millisecond clock for
|
|
241
|
+
measuring elapsed time inside the task.
|
|
242
|
+
|
|
238
243
|
The pool also has an `abortSignalCapacity` option for sizing the shared abort
|
|
239
244
|
signal storage when many abort-aware calls may be in flight.
|
|
240
245
|
|
|
@@ -335,19 +340,29 @@ Common options you might tweak:
|
|
|
335
340
|
| `worker.resolveAfterFinishingAll` | Let submitted calls finish before shutdown resolves. |
|
|
336
341
|
| `worker.bootstrap` | Privileged async hook imported and awaited before task modules load. |
|
|
337
342
|
| `worker.hardTimeoutMs` | Force pool shutdown when a task exceeds this many milliseconds. |
|
|
338
|
-
| `worker.runtime` | Choose `"thread"` or `"
|
|
343
|
+
| `worker.runtime` | Choose `"thread"`, `"process"`, or experimental `"compiled"` workers. |
|
|
344
|
+
| `worker.processRuntime` | Choose `"node"`, `"deno"`, or `"bun"`; standalone `"porffor"` selects compilation and rebuilds once per pool. |
|
|
339
345
|
| `worker.processSharedMemory` | Process-worker memory discovery: `"inherit"` by default on POSIX, or `"named"` for wrappers/containers that cannot preserve fd 0. |
|
|
340
346
|
| `permission` | Runtime permission policy for workers. |
|
|
341
347
|
| `host.dispatcher` | Experimental host dispatcher topology: `"per-thread"` or `"serial-channel"`. |
|
|
348
|
+
| `host.steal` | Shared-submit work stealing for multi-worker thread pools; enabled by default. Set `false` to use private submit lanes. |
|
|
342
349
|
| `debug` | Enable diagnostics (`host`, `globals`, `signals`, `imports`, `lifecycle`) or use `KNITTING_DEBUG`. |
|
|
343
350
|
| `source` | Worker source override for advanced runtimes. |
|
|
344
351
|
|
|
345
|
-
Most users can leave `host.dispatcher` alone.
|
|
346
|
-
|
|
347
|
-
Node/Deno pools use `"serial-channel"
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
352
|
+
Most users can leave `host.dispatcher` alone. It selects the dispatcher for
|
|
353
|
+
private-lane pools: Bun and single-worker pools default to `"per-thread"`, while
|
|
354
|
+
multi-worker Node/Deno pools use `"serial-channel"`. Selecting a dispatcher or
|
|
355
|
+
balancer explicitly preserves that private-lane topology unless
|
|
356
|
+
`host.steal: true` is also explicit.
|
|
357
|
+
|
|
358
|
+
Ordinary multi-worker thread pools use shared-submit work stealing by default.
|
|
359
|
+
It is not used by one-worker pools, the inliner, process workers, compiled/
|
|
360
|
+
Porffor workers, or pools with an explicit balancer/dispatcher, so those modes
|
|
361
|
+
retain their existing transport. Pools above the current 31-claimant protocol
|
|
362
|
+
limit also fall back. Set `host: { steal: false }` or `KNITTING_STEAL=0` to opt
|
|
363
|
+
out for uniformly cheap, low-concurrency workloads where arbitration has
|
|
364
|
+
nothing to rebalance. `host: { steal: true }` or `KNITTING_STEAL=1` forces it
|
|
365
|
+
for an otherwise compatible thread pool.
|
|
351
366
|
|
|
352
367
|
### Worker bootstrap
|
|
353
368
|
|
|
@@ -397,6 +412,27 @@ const pool = createPool({
|
|
|
397
412
|
```
|
|
398
413
|
|
|
399
414
|
`processRuntime` can be `"node"`, `"deno"`, or `"bun"` and defaults to `"deno"`.
|
|
415
|
+
Using `processRuntime: "porffor"` is shorthand for the compiled backend and
|
|
416
|
+
forces one fresh native build whenever `createPool(...)` is called:
|
|
417
|
+
|
|
418
|
+
```ts
|
|
419
|
+
using pool = createPool({
|
|
420
|
+
worker: { processRuntime: "porffor" },
|
|
421
|
+
})({ add });
|
|
422
|
+
```
|
|
423
|
+
|
|
424
|
+
Add `runtime: "compiled"` to reuse a compatible `.knt` instead. Missing or
|
|
425
|
+
stale artifacts still build automatically:
|
|
426
|
+
|
|
427
|
+
```ts
|
|
428
|
+
using pool = createPool({
|
|
429
|
+
worker: {
|
|
430
|
+
runtime: "compiled",
|
|
431
|
+
processRuntime: "porffor",
|
|
432
|
+
},
|
|
433
|
+
})({ add });
|
|
434
|
+
```
|
|
435
|
+
|
|
400
436
|
You can also provide a `processCommandPrefix` when workers need to be launched
|
|
401
437
|
through a wrapper such as a package manager, container command, or runtime shim.
|
|
402
438
|
|
|
@@ -445,6 +481,112 @@ const pool = createPool({
|
|
|
445
481
|
})({ add });
|
|
446
482
|
```
|
|
447
483
|
|
|
484
|
+
### Experimental compiled workers
|
|
485
|
+
|
|
486
|
+
A native worker uses the same task declaration and pool call syntax. For a
|
|
487
|
+
task module named tasks.ts, Knitting looks for tasks.knt plus tasks.knt.json.
|
|
488
|
+
If they are missing, incompatible, or older than the task module, the first
|
|
489
|
+
pool builds them automatically with a pinned Porffor compiler; later pools
|
|
490
|
+
reuse the validated artifact:
|
|
491
|
+
|
|
492
|
+
| Worker settings | Compilation behavior |
|
|
493
|
+
| --- | --- |
|
|
494
|
+
| `runtime: "compiled"` | Reuse a compatible `.knt`; build when missing, stale, or incompatible. |
|
|
495
|
+
| `processRuntime: "porffor"` | Select the compiled backend and always rebuild once per pool. |
|
|
496
|
+
| `runtime: "compiled", processRuntime: "porffor"` | Reuse a compatible `.knt`; build when missing, stale, or incompatible. |
|
|
497
|
+
|
|
498
|
+
For example, this bare-function form uses `hello.knt` when it is current:
|
|
499
|
+
|
|
500
|
+
```ts
|
|
501
|
+
import { createPool, isMain } from "knitting";
|
|
502
|
+
|
|
503
|
+
export const hello = (name: string) => "Hello " + name;
|
|
504
|
+
|
|
505
|
+
using pool = createPool({
|
|
506
|
+
worker: {
|
|
507
|
+
runtime: "compiled",
|
|
508
|
+
processRuntime: "porffor",
|
|
509
|
+
},
|
|
510
|
+
})({ hello });
|
|
511
|
+
|
|
512
|
+
if (isMain) console.log(await pool.call.hello("World!"));
|
|
513
|
+
```
|
|
514
|
+
|
|
515
|
+
Remove `runtime: "compiled"` from that example when you want a fresh build on
|
|
516
|
+
every `createPool(...)`. A multi-worker pool still compiles only once, then
|
|
517
|
+
starts every native worker from the resulting artifact.
|
|
518
|
+
|
|
519
|
+
The `task(...)` declaration form works the same way:
|
|
520
|
+
|
|
521
|
+
```ts
|
|
522
|
+
import { createPool, isMain, task } from "knitting";
|
|
523
|
+
|
|
524
|
+
export const addOne = task<number, number>({
|
|
525
|
+
f: (value) => value + 1,
|
|
526
|
+
});
|
|
527
|
+
|
|
528
|
+
if (isMain) {
|
|
529
|
+
using pool = createPool({
|
|
530
|
+
threads: 2,
|
|
531
|
+
worker: { runtime: "compiled" },
|
|
532
|
+
})({ addOne });
|
|
533
|
+
|
|
534
|
+
console.log(await pool.call.addOne(41)); // 42
|
|
535
|
+
}
|
|
536
|
+
```
|
|
537
|
+
|
|
538
|
+
Use worker.compiled.artifact when artifacts live in a build directory:
|
|
539
|
+
|
|
540
|
+
```ts
|
|
541
|
+
const pool = createPool({
|
|
542
|
+
worker: {
|
|
543
|
+
runtime: "compiled",
|
|
544
|
+
compiled: { artifact: "./build/tasks-linux-x64.knt" },
|
|
545
|
+
},
|
|
546
|
+
})({ addOne });
|
|
547
|
+
```
|
|
548
|
+
|
|
549
|
+
`worker.compiled.manifest` selects a non-default sidecar location.
|
|
550
|
+
`worker.compiled.build` controls generation directly:
|
|
551
|
+
|
|
552
|
+
- `true` or omitted: build only when the artifact cannot be reused.
|
|
553
|
+
- `false`: never build; fail if the prebuilt artifact is unavailable.
|
|
554
|
+
- `"always"`: rebuild once whenever a pool is created.
|
|
555
|
+
|
|
556
|
+
The extension is .knt rather than .out because it identifies a Knitting worker
|
|
557
|
+
artifact; executability is never inferred from the suffix alone. Knitting also
|
|
558
|
+
requires the sidecar to match the protocol version, current platform and
|
|
559
|
+
architecture, source module, source timestamp, and requested task names before
|
|
560
|
+
spawning it. `checkCompiledWorker(...)` remains a read-only way to inspect that
|
|
561
|
+
state without building or executing anything.
|
|
562
|
+
|
|
563
|
+
Automatic builds use Porffor main from `worker.compiled.compiler`,
|
|
564
|
+
`PORFFOR_MAIN`/`PORF`, or `porf` on PATH. If none exists, Knitting downloads a
|
|
565
|
+
pinned compiler into `$XDG_CACHE_HOME/knitting`, or `~/.cache/knitting` when
|
|
566
|
+
`XDG_CACHE_HOME` is unset. Native builds use Porffor `-O3` and parallel LTO.
|
|
567
|
+
Set `worker.compiled.build: false` for deployment environments that must use
|
|
568
|
+
only a prebuilt artifact.
|
|
569
|
+
|
|
570
|
+
To build ahead of time, run
|
|
571
|
+
`bun run build:compiled --module tasks.ts --out tasks.knt --tasks addOne`.
|
|
572
|
+
|
|
573
|
+
Treat this backend as experimental. Porffor is a young ahead-of-time compiler,
|
|
574
|
+
the artifact format is pre-release and changes without a compatibility path, and
|
|
575
|
+
the executable is native, unsandboxed code — only run artifacts you trust.
|
|
576
|
+
|
|
577
|
+
Compiled workers accept synchronous JSON-compatible primitives, arrays, and
|
|
578
|
+
plain objects up to 1 MiB per call, plus `ArrayBuffer`, `DataView`, and typed
|
|
579
|
+
arrays, which are copied. A `ProcessSharedBuffer` is mapped by the worker rather
|
|
580
|
+
than copied. BMP Unicode is supported; supplementary code points and async
|
|
581
|
+
results are not.
|
|
582
|
+
|
|
583
|
+
Abort-aware tasks work on POSIX: the pool shares an abort bitmap through named
|
|
584
|
+
shared memory, so `signal.hasAborted()` and `signal.now()` are native reads
|
|
585
|
+
inside the worker. Windows has no implementation yet. Task timeouts, bootstrap
|
|
586
|
+
hooks, permission policies, and host inlining still fail during pool creation or
|
|
587
|
+
invocation; `worker.hardTimeoutMs` remains available because the host enforces
|
|
588
|
+
it.
|
|
589
|
+
|
|
448
590
|
### Windows process workers
|
|
449
591
|
|
|
450
592
|
On Windows, Knitting automatically uses named shared memory for the
|
|
@@ -499,6 +641,84 @@ const pool = createPool({
|
|
|
499
641
|
})({ add });
|
|
500
642
|
```
|
|
501
643
|
|
|
644
|
+
## Browsers
|
|
645
|
+
|
|
646
|
+
Knitting also runs in the browser, where the pool spawns web workers over
|
|
647
|
+
`SharedArrayBuffer` instead of threads. Two rules apply there and nowhere else.
|
|
648
|
+
|
|
649
|
+
**The page must be cross-origin isolated.** Browsers hand out
|
|
650
|
+
`SharedArrayBuffer` only under these two response headers, and `createPool`
|
|
651
|
+
fails with a clear error when they are missing:
|
|
652
|
+
|
|
653
|
+
```
|
|
654
|
+
Cross-Origin-Opener-Policy: same-origin
|
|
655
|
+
Cross-Origin-Embedder-Policy: require-corp
|
|
656
|
+
```
|
|
657
|
+
|
|
658
|
+
**Task modules must declare their own URL.** On Node, Deno, and Bun a task
|
|
659
|
+
finds its module by walking the stack; a bundler erases the paths that depends
|
|
660
|
+
on, so call `setModuleUrl(import.meta.url)` in the module that exports tasks:
|
|
661
|
+
|
|
662
|
+
```ts
|
|
663
|
+
import { createPool, isMain, setModuleUrl, task } from "knitting/browser";
|
|
664
|
+
|
|
665
|
+
setModuleUrl(import.meta.url);
|
|
666
|
+
|
|
667
|
+
export const square = task({ f: (value: number) => value * value });
|
|
668
|
+
|
|
669
|
+
if (isMain) {
|
|
670
|
+
const pool = createPool({ threads: 4 })({ square });
|
|
671
|
+
console.log(await pool.call.square(7)); // 49
|
|
672
|
+
await pool.shutdown();
|
|
673
|
+
}
|
|
674
|
+
```
|
|
675
|
+
|
|
676
|
+
Bundle that module with any browser-targeting bundler. The result is
|
|
677
|
+
self-hosting: the page loads it, and every worker the pool spawns loads the
|
|
678
|
+
same file, which is why both sides agree on the module URL.
|
|
679
|
+
|
|
680
|
+
`knitting/browser` ships as one self-contained file, so it also works without a
|
|
681
|
+
bundler at all — serve it next to a plain task module:
|
|
682
|
+
|
|
683
|
+
```html
|
|
684
|
+
<script type="module" src="./tasks.js"></script>
|
|
685
|
+
```
|
|
686
|
+
|
|
687
|
+
```js
|
|
688
|
+
// tasks.js
|
|
689
|
+
import { createPool, isMain, setModuleUrl, task } from "./knitting.browser.js";
|
|
690
|
+
|
|
691
|
+
setModuleUrl(import.meta.url);
|
|
692
|
+
|
|
693
|
+
export const square = task({ f: (value) => value * value });
|
|
694
|
+
|
|
695
|
+
if (isMain) {
|
|
696
|
+
const pool = createPool({ threads: 2 })({ square });
|
|
697
|
+
console.log(await pool.call.square(7)); // 49
|
|
698
|
+
await pool.shutdown();
|
|
699
|
+
}
|
|
700
|
+
```
|
|
701
|
+
|
|
702
|
+
It is the same API as the main entry without the compiled worker (Porffor)
|
|
703
|
+
helpers, which need a filesystem. Process workers, native addons, FFI, and the
|
|
704
|
+
permission system are inert in a browser — permissions are skipped entirely,
|
|
705
|
+
since there is no filesystem or process to police.
|
|
706
|
+
|
|
707
|
+
Process workers, compiled workers, native addons, FFI, `BufferReference`, and
|
|
708
|
+
`ProcessSharedBuffer` are all unavailable in a page, and permissions are
|
|
709
|
+
skipped rather than enforced. [BROWSER.md](BROWSER.md) documents every one of
|
|
710
|
+
those, with the error each raises.
|
|
711
|
+
|
|
712
|
+
The published file is bundled and minified, with the Node-only subsystems
|
|
713
|
+
(process workers, compiled workers, native addons, FFI, permissions) replaced
|
|
714
|
+
by stubs that keep their browser behaviour — roughly 94 KB, 32 KB over gzip.
|
|
715
|
+
Both layouts above are covered by the browser test lane:
|
|
716
|
+
|
|
717
|
+
```bash
|
|
718
|
+
npm run build:browser # build/knitting.browser.js and .min.js
|
|
719
|
+
npm run test:browser # end-to-end checks in headless Chromium
|
|
720
|
+
```
|
|
721
|
+
|
|
502
722
|
## Permissions
|
|
503
723
|
|
|
504
724
|
Knitting defaults to a strict worker permission policy:
|
|
@@ -507,8 +727,9 @@ Knitting defaults to a strict worker permission policy:
|
|
|
507
727
|
permission: { mode: "strict", allowImport: true }
|
|
508
728
|
```
|
|
509
729
|
|
|
510
|
-
|
|
511
|
-
|
|
730
|
+
For process workers, that default is translated through the selected runtime's
|
|
731
|
+
permission model. If a runtime cannot enforce one of the implicit defaults,
|
|
732
|
+
Knitting warns once. Do not treat the default as an OS sandbox.
|
|
512
733
|
|
|
513
734
|
For trusted local scripts, you can opt out:
|
|
514
735
|
|
|
@@ -518,7 +739,8 @@ const pool = createPool({
|
|
|
518
739
|
})({ add });
|
|
519
740
|
```
|
|
520
741
|
|
|
521
|
-
For production or plugin-like workloads, prefer an explicit policy
|
|
742
|
+
For production or plugin-like workloads, prefer an explicit policy. This example
|
|
743
|
+
is fully representable by a Deno process worker:
|
|
522
744
|
|
|
523
745
|
```ts
|
|
524
746
|
const pool = createPool({
|
|
@@ -534,9 +756,45 @@ const pool = createPool({
|
|
|
534
756
|
})({ add });
|
|
535
757
|
```
|
|
536
758
|
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
759
|
+
Before a process worker is spawned, Knitting checks explicit restrictions
|
|
760
|
+
against the selected runtime. A restriction that would be broader than requested
|
|
761
|
+
or ignored is rejected synchronously. Implicit strict-default gaps remain
|
|
762
|
+
backward compatible and produce a once-per-runtime warning.
|
|
763
|
+
|
|
764
|
+
| Process-worker permission | Deno | Node 22/24 | Node 26+ | Bun |
|
|
765
|
+
| ------------------------- | ------------------- | ------------------- | -------------------------- | ----------- |
|
|
766
|
+
| Filesystem allow-list | Scoped | Scoped | Scoped | Unsupported |
|
|
767
|
+
| Filesystem deny-list | Scoped | Unsupported | Unsupported | Unsupported |
|
|
768
|
+
| Network | Scoped allow/deny | Unsupported | Deny-all or allow-all only | Unsupported |
|
|
769
|
+
| Environment allow/deny | Scoped | Unsupported | Unsupported | Unsupported |
|
|
770
|
+
| Child processes | Scoped allow/deny | All-or-none | All-or-none | Unsupported |
|
|
771
|
+
| Worker creation | Unsupported | All-or-none | All-or-none | Unsupported |
|
|
772
|
+
| Import hosts | Scoped | Unsupported | Unsupported | Unsupported |
|
|
773
|
+
| System information | Scoped | Unsupported | Unsupported | Unsupported |
|
|
774
|
+
| WASI denial | Unsupported | All-or-none | All-or-none | Unsupported |
|
|
775
|
+
| Native code / FFI denial | Transport exception | Transport exception | Transport exception | Unsupported |
|
|
776
|
+
|
|
777
|
+
“All-or-none” means an explicit scoped list fails closed. Node 26's
|
|
778
|
+
`--allow-net` switch is emitted for `net: true`, but a host allow-list still
|
|
779
|
+
cannot be represented. When a wrapper or cross-runtime host hides the target
|
|
780
|
+
Node version, Knitting uses the conservative Node 22/24 capability set.
|
|
781
|
+
|
|
782
|
+
These compatibility checks currently cover process workers. Thread workers use
|
|
783
|
+
the host runtime's worker behavior and should not be treated as a sandbox.
|
|
784
|
+
Runtime permissions are guardrails, not the only security boundary for hostile
|
|
785
|
+
code.
|
|
786
|
+
|
|
787
|
+
The top-level `ffi` permission is the explicit cross-runtime native-code
|
|
788
|
+
capability. On Node it enables both native addons and `node:ffi`; Node's
|
|
789
|
+
`--allow-ffi` permission is currently unrestricted. The legacy/runtime-specific
|
|
790
|
+
`node.allowAddons` and `node.allowFfi` switches are independent—enabling addons
|
|
791
|
+
does not silently enable FFI.
|
|
792
|
+
|
|
793
|
+
Node process workers are a transport exception: Knitting needs `--allow-addons`
|
|
794
|
+
on Node 22/24 or `--allow-ffi` on Node 26 to map their shared memory. Deno
|
|
795
|
+
process workers likewise need `--allow-ffi`. Those capabilities apply to the
|
|
796
|
+
entire worker process, including task code, so explicit native-code denial fails
|
|
797
|
+
closed. Use an OS sandbox when task code is hostile.
|
|
540
798
|
|
|
541
799
|
## Payloads
|
|
542
800
|
|
|
@@ -607,12 +865,12 @@ if (isMain) {
|
|
|
607
865
|
The body is generic — `Envelope<Header, Body>` — and accepts any of the binary
|
|
608
866
|
shapes the transport understands:
|
|
609
867
|
|
|
610
|
-
| Body
|
|
611
|
-
|
|
|
612
|
-
| `ArrayBuffer`
|
|
613
|
-
| `SharedArrayBuffer`
|
|
614
|
-
| `ProcessSharedBuffer
|
|
615
|
-
| `BufferReference`
|
|
868
|
+
| Body | Copy? | Workers | Notes |
|
|
869
|
+
| --------------------- | ----------------- | ---------------- | ------------------------------------------------------------------- |
|
|
870
|
+
| `ArrayBuffer` | copied | thread + process | The default body; works everywhere. |
|
|
871
|
+
| `SharedArrayBuffer` | zero-copy, shared | thread only | Shared by reference; process workers reject it. |
|
|
872
|
+
| `ProcessSharedBuffer` | zero-copy, shared | thread + process | Cross-process shared memory. |
|
|
873
|
+
| `BufferReference` | zero-copy, moved | thread only | From `knitting/unsafe`; same constraints as bare `BufferReference`. |
|
|
616
874
|
|
|
617
875
|
The header keeps its fast paths regardless of the body: a small header is
|
|
618
876
|
written inline, and only large headers spill to the dynamic payload region. A
|
|
@@ -840,6 +1098,24 @@ worker. It is the same-process counterpart to `ProcessSharedBuffer`: reach for
|
|
|
840
1098
|
it when you hold a large `ArrayBuffer` or typed array and the copy cost to a
|
|
841
1099
|
thread worker actually matters.
|
|
842
1100
|
|
|
1101
|
+
The ownership and revocation paths are hardened against ordinary
|
|
1102
|
+
use-after-free: sources are detached on move, aliases are detached before a
|
|
1103
|
+
borrow is released, and outstanding borrowed returns are revoked during pool
|
|
1104
|
+
shutdown. This is still an **unsafe capability**, not a memory-safety or
|
|
1105
|
+
security boundary. Do not accept `BufferReferenceMetadata` or raw pointer data
|
|
1106
|
+
from untrusted code, and do not mutate the same bytes concurrently without
|
|
1107
|
+
your own synchronization.
|
|
1108
|
+
|
|
1109
|
+
| Path | Node 22/24 owning addon | Node 26 FFI | Deno/Bun FFI |
|
|
1110
|
+
| --- | --- | --- | --- |
|
|
1111
|
+
| Forward input | Moved, zero-copy | Moved, zero-copy alias | Moved, zero-copy alias |
|
|
1112
|
+
| Default returned reference | Co-owned, zero-copy | One safe copy | One safe copy |
|
|
1113
|
+
| `BufferReferenceReturn: "borrow"` | Revocable borrow | Revocable borrow | Revocable borrow |
|
|
1114
|
+
| Worker shutdown | Outstanding borrows revoked | Copy while readable, then revoke | Copy while readable, then revoke |
|
|
1115
|
+
|
|
1116
|
+
Older Node addons without the owning primitives use the non-owning fallback and
|
|
1117
|
+
therefore follow the copy/borrow rules rather than the owning-addon row.
|
|
1118
|
+
|
|
843
1119
|
```ts
|
|
844
1120
|
import { createPool, isMain, task } from "knitting";
|
|
845
1121
|
import { BufferReference } from "knitting/unsafe";
|
|
@@ -874,33 +1150,36 @@ Read these constraints before reaching for it:
|
|
|
874
1150
|
- **Move semantics.** Constructing a `BufferReference` detaches its source — the
|
|
875
1151
|
original buffer is empty afterward, and reads/writes through it are gone. The
|
|
876
1152
|
bytes now belong to the reference; to get a result back, the worker returns
|
|
877
|
-
its own `BufferReference`. Each
|
|
1153
|
+
its own `BufferReference`. Each source ownership transfer is one-shot, but an
|
|
1154
|
+
active reference may be read more than once. Forward inputs the worker
|
|
878
1155
|
materializes with `.toArrayBuffer()`/`.toUint8Array()` are borrowed for the
|
|
879
1156
|
duration of the call and detached once it settles; do not keep using them from
|
|
880
1157
|
fire-and-forget work after the task returns.
|
|
881
|
-
- **Forward is zero-copy everywhere
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
- **Borrowed Deno/Bun returns are opt-in.** The default is
|
|
1158
|
+
- **Forward is zero-copy everywhere.** Sending a buffer to the worker never
|
|
1159
|
+
copies. Node 22 and 24 with the owning addon also return the buffer without
|
|
1160
|
+
copying because the addon co-owns the V8 backing store. Node 26, Deno, and Bun
|
|
1161
|
+
take one safe copy on return because their FFI aliases cannot own the worker's
|
|
1162
|
+
backing store.
|
|
1163
|
+
- **Borrowed FFI returns are opt-in.** The default on Node 26, Deno, and Bun is
|
|
888
1164
|
`unsafe: { BufferReferenceReturn: "copy" }` — the safe single copy described
|
|
889
|
-
above. Set it to `"borrow"`
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
1165
|
+
above. Set it to `"borrow"` to skip that copy by borrowing the worker's
|
|
1166
|
+
backing store until the returned `BufferReference` is released. Call
|
|
1167
|
+
`ref.release()` or use `using`. Releasing **revokes** the borrow: the
|
|
1168
|
+
reference and every view taken from it are detached first, so later reads see
|
|
1169
|
+
empty views or throw instead of touching freed memory. A reference that is
|
|
1170
|
+
never released drops its borrow only once it and all of its views are
|
|
1171
|
+
unreachable. Pool shutdown revokes outstanding borrows too — references you
|
|
1172
|
+
still hold survive on a private copy, while stale views read as empty. If the
|
|
1173
|
+
bytes escape into HTTP responses, streams, timers, callbacks, or caches, copy
|
|
1174
|
+
them before the borrowed reference is released, or they will read as empty
|
|
1175
|
+
afterward.
|
|
896
1176
|
- **Unsafe escape hatch.** This is not a security boundary. Forged metadata or
|
|
897
1177
|
unsynchronized host/worker mutation can still be unsafe.
|
|
898
|
-
- **Node
|
|
899
|
-
`knitting_buffer_pointer`
|
|
900
|
-
|
|
901
|
-
a `BufferReference` on Node throws.
|
|
1178
|
+
- **Node backend depends on the Node line.** Node 22 and 24 use the
|
|
1179
|
+
`knitting_buffer_pointer` addon. Node 26 uses `node:ffi` and therefore needs
|
|
1180
|
+
`--experimental-ffi`.
|
|
902
1181
|
|
|
903
|
-
Borrowed returns, end to end (
|
|
1182
|
+
Borrowed returns, end to end (this opts Node 26, Deno, and Bun in):
|
|
904
1183
|
|
|
905
1184
|
```ts
|
|
906
1185
|
using pool = createPool({
|
|
@@ -914,7 +1193,7 @@ using pool = createPool({
|
|
|
914
1193
|
using result = await pool.call.invert(new BufferReference(pixels));
|
|
915
1194
|
const out = result.toUint8Array(); // borrowed — valid only while `result` lives
|
|
916
1195
|
console.log([...out]);
|
|
917
|
-
} // `using` releases the borrow here;
|
|
1196
|
+
} // `using` releases the borrow here; `out` is detached and reads as empty now
|
|
918
1197
|
```
|
|
919
1198
|
|
|
920
1199
|
When in doubt, a plain `ArrayBuffer` or typed-array payload — which knitting
|
|
@@ -927,10 +1206,32 @@ pointer setup tends to cost more than just copying, so the plain transport wins.
|
|
|
927
1206
|
|
|
928
1207
|
Knitting supports Node.js 22+, Deno 2+, and Bun 1+ on Linux, macOS, and Windows.
|
|
929
1208
|
|
|
930
|
-
|
|
931
|
-
`ProcessSharedBuffer` use
|
|
932
|
-
include
|
|
933
|
-
|
|
1209
|
+
Plain Node thread workers work without native pieces. Node process workers,
|
|
1210
|
+
`ProcessSharedBuffer`, and `BufferReference` use native support. Release
|
|
1211
|
+
packages currently include Node prebuilds for:
|
|
1212
|
+
|
|
1213
|
+
- Node.js 22 (ABI 127) and Node.js 24 (ABI 137)
|
|
1214
|
+
- Linux x64
|
|
1215
|
+
- macOS x64 and arm64
|
|
1216
|
+
- Windows x64
|
|
1217
|
+
|
|
1218
|
+
Odd-numbered Node releases are not supported by packaged native features.
|
|
1219
|
+
Node.js 26 uses `node:ffi` instead of another ABI-specific addon. Start the host
|
|
1220
|
+
with:
|
|
1221
|
+
|
|
1222
|
+
```bash
|
|
1223
|
+
node --experimental-ffi app.js
|
|
1224
|
+
```
|
|
1225
|
+
|
|
1226
|
+
When using Node's Permission Model, also grant `--allow-ffi`. Knitting passes
|
|
1227
|
+
the experimental FFI flag to Node process workers it starts, but the host must
|
|
1228
|
+
be started with the flag so it can create the shared mappings. The FFI backend
|
|
1229
|
+
uses external `ArrayBuffer` mappings, so `ProcessSharedBuffer.view()`,
|
|
1230
|
+
`.getBuffer()`, and `.bytes()` work; `.getSAB()` is available only on the Node
|
|
1231
|
+
22/24 addon backend.
|
|
1232
|
+
|
|
1233
|
+
If you are developing locally on another Node ABI or architecture, you can
|
|
1234
|
+
compile the current V8 addon for that exact runtime:
|
|
934
1235
|
|
|
935
1236
|
```bash
|
|
936
1237
|
bun run build:native
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { workerMainLoop } from "./src/worker/loop.ts";
|
|
2
|
+
import { createPool, importTask, isMain, setModuleUrl, task } from "./src/api.ts";
|
|
3
|
+
import { Envelope } from "./src/common/envelope.ts";
|
|
4
|
+
import { isNumericArray, NumericArray } from "./src/connections/numeric-array.ts";
|
|
5
|
+
export { createPool as createPool, Envelope as Envelope, importTask as importTask, isMain as isMain, isNumericArray as isNumericArray, NumericArray as NumericArray, setModuleUrl as setModuleUrl, task as task, workerMainLoop as workerMainLoop, };
|
|
6
|
+
export type { EnvelopeBody as EnvelopeBody, EnvelopeHeader as EnvelopeHeader, } from "./src/common/envelope.ts";
|