knitting 0.1.56 → 0.1.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +254 -40
- package/knitting.d.ts +3 -2
- package/knitting.js +3 -3
- package/map.md +21 -9
- package/package.json +7 -1
- package/scripts/build-compiled-worker.ts +276 -0
- package/scripts/compiled-worker/porffor.d.ts +5 -0
- package/scripts/compiled-worker/runtime.ts +529 -0
- package/scripts/compiled-worker/task-shim.ts +32 -0
- package/src/api.d.ts +24 -1
- package/src/api.js +199 -20
- package/src/common/module-url.d.ts +1 -0
- package/src/common/module-url.js +30 -2
- package/src/common/node-compat.d.ts +2 -0
- package/src/common/path-canonical.js +20 -6
- package/src/common/runtime.d.ts +2 -1
- package/src/common/runtime.js +18 -4
- package/src/common/task-source.d.ts +2 -0
- package/src/common/task-source.js +17 -1
- package/src/common/worker-runtime.js +16 -7
- package/src/connections/buffer-reference-native.js +25 -1
- package/src/connections/buffer-reference.d.ts +23 -2
- package/src/connections/buffer-reference.js +143 -29
- package/src/connections/bun.d.ts +1 -1
- package/src/connections/bun.js +25 -14
- package/src/connections/deno.d.ts +3 -1
- package/src/connections/deno.js +35 -14
- package/src/connections/external-array-buffer.d.ts +2 -0
- package/src/connections/external-array-buffer.js +33 -0
- package/src/connections/node-addons.d.ts +11 -0
- package/src/connections/node-addons.js +49 -1
- package/src/connections/node-ffi-api.d.ts +19 -0
- package/src/connections/node-ffi-api.js +28 -0
- package/src/connections/node-ffi.d.ts +27 -0
- package/src/connections/node-ffi.js +262 -0
- package/src/connections/node.d.ts +1 -1
- package/src/connections/node.js +20 -20
- package/src/connections/package-assets.js +25 -15
- package/src/connections/posix.d.ts +42 -0
- package/src/connections/posix.js +101 -0
- package/src/connections/process-shared-buffer.d.ts +1 -0
- package/src/connections/process-shared-buffer.js +11 -45
- package/src/connections/types.d.ts +4 -0
- package/src/connections/windows.d.ts +19 -10
- package/src/connections/windows.js +87 -66
- package/src/error.d.ts +7 -6
- package/src/error.js +8 -7
- package/src/memory/lock.d.ts +87 -77
- package/src/memory/lock.js +103 -84
- package/src/memory/payloadCodec.d.ts +11 -2
- package/src/memory/payloadCodec.js +127 -31
- package/src/permission/compatibility.d.ts +28 -0
- package/src/permission/compatibility.js +239 -0
- package/src/permission/index.d.ts +2 -0
- package/src/permission/index.js +1 -0
- package/src/permission/protocol.d.ts +1 -0
- package/src/permission/protocol.js +54 -14
- package/src/runtime/compiled-artifact.d.ts +21 -0
- package/src/runtime/compiled-artifact.js +260 -0
- package/src/runtime/compiled-builder.d.ts +8 -0
- package/src/runtime/compiled-builder.js +42 -0
- package/src/runtime/compiled-worker.d.ts +27 -0
- package/src/runtime/compiled-worker.js +562 -0
- package/src/runtime/inline-executor.js +27 -15
- package/src/runtime/pool.d.ts +7 -2
- package/src/runtime/pool.js +74 -6
- package/src/runtime/process-worker.d.ts +1 -1
- package/src/runtime/process-worker.js +48 -8
- package/src/runtime/tx-queue.d.ts +3 -1
- package/src/runtime/tx-queue.js +4 -1
- package/src/types.d.ts +48 -6
- package/src/worker/composable-runners.js +3 -2
- package/src/worker/loop.d.ts +1 -0
- package/src/worker/loop.js +25 -12
- package/src/worker/task-loader.d.ts +5 -4
- package/src/worker/task-loader.js +13 -7
- package/src/worker/timers.js +8 -6
package/README.md
CHANGED
|
@@ -15,7 +15,8 @@ Website: [knittingdocs.netlify.app](https://knittingdocs.netlify.app/)
|
|
|
15
15
|
|
|
16
16
|
If you are an agent trying to understand the project, the website also serves an
|
|
17
17
|
[`llms.txt`](https://knittingdocs.netlify.app/llms.txt) file with a compact map
|
|
18
|
-
of the docs
|
|
18
|
+
of the docs, plus a full inlined version at
|
|
19
|
+
[`llms-full.txt`](https://knittingdocs.netlify.app/llms-full.txt).
|
|
19
20
|
|
|
20
21
|
Knitting is a worker pool built on shared-memory IPC for Node.js, Deno, and Bun.
|
|
21
22
|
It lets you call work running on other threads or processes as if it were a
|
|
@@ -60,7 +61,8 @@ cross-runtime shared memory.
|
|
|
60
61
|
|
|
61
62
|
## Requirements
|
|
62
63
|
|
|
63
|
-
- Node.js 22
|
|
64
|
+
- Node.js 22+; native features support Node.js 22 and 24 through prebuilt
|
|
65
|
+
addons, and Node.js 26 through experimental `node:ffi`
|
|
64
66
|
- Deno 2+
|
|
65
67
|
- Bun 1+
|
|
66
68
|
|
|
@@ -235,6 +237,9 @@ export const countUntilStopped = task({
|
|
|
235
237
|
});
|
|
236
238
|
```
|
|
237
239
|
|
|
240
|
+
The signal also carries `signal.now()`, a monotonic millisecond clock for
|
|
241
|
+
measuring elapsed time inside the task.
|
|
242
|
+
|
|
238
243
|
The pool also has an `abortSignalCapacity` option for sizing the shared abort
|
|
239
244
|
signal storage when many abort-aware calls may be in flight.
|
|
240
245
|
|
|
@@ -335,7 +340,8 @@ Common options you might tweak:
|
|
|
335
340
|
| `worker.resolveAfterFinishingAll` | Let submitted calls finish before shutdown resolves. |
|
|
336
341
|
| `worker.bootstrap` | Privileged async hook imported and awaited before task modules load. |
|
|
337
342
|
| `worker.hardTimeoutMs` | Force pool shutdown when a task exceeds this many milliseconds. |
|
|
338
|
-
| `worker.runtime` | Choose `"thread"` or `"
|
|
343
|
+
| `worker.runtime` | Choose `"thread"`, `"process"`, or experimental `"compiled"` workers. |
|
|
344
|
+
| `worker.processRuntime` | Choose `"node"`, `"deno"`, or `"bun"`; standalone `"porffor"` selects compilation and rebuilds once per pool. |
|
|
339
345
|
| `worker.processSharedMemory` | Process-worker memory discovery: `"inherit"` by default on POSIX, or `"named"` for wrappers/containers that cannot preserve fd 0. |
|
|
340
346
|
| `permission` | Runtime permission policy for workers. |
|
|
341
347
|
| `host.dispatcher` | Experimental host dispatcher topology: `"per-thread"` or `"serial-channel"`. |
|
|
@@ -397,6 +403,27 @@ const pool = createPool({
|
|
|
397
403
|
```
|
|
398
404
|
|
|
399
405
|
`processRuntime` can be `"node"`, `"deno"`, or `"bun"` and defaults to `"deno"`.
|
|
406
|
+
Using `processRuntime: "porffor"` is shorthand for the compiled backend and
|
|
407
|
+
forces one fresh native build whenever `createPool(...)` is called:
|
|
408
|
+
|
|
409
|
+
```ts
|
|
410
|
+
using pool = createPool({
|
|
411
|
+
worker: { processRuntime: "porffor" },
|
|
412
|
+
})({ add });
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
Add `runtime: "compiled"` to reuse a compatible `.knt` instead. Missing or
|
|
416
|
+
stale artifacts still build automatically:
|
|
417
|
+
|
|
418
|
+
```ts
|
|
419
|
+
using pool = createPool({
|
|
420
|
+
worker: {
|
|
421
|
+
runtime: "compiled",
|
|
422
|
+
processRuntime: "porffor",
|
|
423
|
+
},
|
|
424
|
+
})({ add });
|
|
425
|
+
```
|
|
426
|
+
|
|
400
427
|
You can also provide a `processCommandPrefix` when workers need to be launched
|
|
401
428
|
through a wrapper such as a package manager, container command, or runtime shim.
|
|
402
429
|
|
|
@@ -445,6 +472,112 @@ const pool = createPool({
|
|
|
445
472
|
})({ add });
|
|
446
473
|
```
|
|
447
474
|
|
|
475
|
+
### Experimental compiled workers
|
|
476
|
+
|
|
477
|
+
A native worker uses the same task declaration and pool call syntax. For a
|
|
478
|
+
task module named tasks.ts, Knitting looks for tasks.knt plus tasks.knt.json.
|
|
479
|
+
If they are missing, incompatible, or older than the task module, the first
|
|
480
|
+
pool builds them automatically with a pinned Porffor compiler; later pools
|
|
481
|
+
reuse the validated artifact:
|
|
482
|
+
|
|
483
|
+
| Worker settings | Compilation behavior |
|
|
484
|
+
| --- | --- |
|
|
485
|
+
| `runtime: "compiled"` | Reuse a compatible `.knt`; build when missing, stale, or incompatible. |
|
|
486
|
+
| `processRuntime: "porffor"` | Select the compiled backend and always rebuild once per pool. |
|
|
487
|
+
| `runtime: "compiled", processRuntime: "porffor"` | Reuse a compatible `.knt`; build when missing, stale, or incompatible. |
|
|
488
|
+
|
|
489
|
+
For example, this bare-function form uses `hello.knt` when it is current:
|
|
490
|
+
|
|
491
|
+
```ts
|
|
492
|
+
import { createPool, isMain } from "knitting";
|
|
493
|
+
|
|
494
|
+
export const hello = (name: string) => "Hello " + name;
|
|
495
|
+
|
|
496
|
+
using pool = createPool({
|
|
497
|
+
worker: {
|
|
498
|
+
runtime: "compiled",
|
|
499
|
+
processRuntime: "porffor",
|
|
500
|
+
},
|
|
501
|
+
})({ hello });
|
|
502
|
+
|
|
503
|
+
if (isMain) console.log(await pool.call.hello("World!"));
|
|
504
|
+
```
|
|
505
|
+
|
|
506
|
+
Remove `runtime: "compiled"` from that example when you want a fresh build on
|
|
507
|
+
every `createPool(...)`. A multi-worker pool still compiles only once, then
|
|
508
|
+
starts every native worker from the resulting artifact.
|
|
509
|
+
|
|
510
|
+
The `task(...)` declaration form works the same way:
|
|
511
|
+
|
|
512
|
+
```ts
|
|
513
|
+
import { createPool, isMain, task } from "knitting";
|
|
514
|
+
|
|
515
|
+
export const addOne = task<number, number>({
|
|
516
|
+
f: (value) => value + 1,
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
if (isMain) {
|
|
520
|
+
using pool = createPool({
|
|
521
|
+
threads: 2,
|
|
522
|
+
worker: { runtime: "compiled" },
|
|
523
|
+
})({ addOne });
|
|
524
|
+
|
|
525
|
+
console.log(await pool.call.addOne(41)); // 42
|
|
526
|
+
}
|
|
527
|
+
```
|
|
528
|
+
|
|
529
|
+
Use worker.compiled.artifact when artifacts live in a build directory:
|
|
530
|
+
|
|
531
|
+
```ts
|
|
532
|
+
const pool = createPool({
|
|
533
|
+
worker: {
|
|
534
|
+
runtime: "compiled",
|
|
535
|
+
compiled: { artifact: "./build/tasks-linux-x64.knt" },
|
|
536
|
+
},
|
|
537
|
+
})({ addOne });
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
`worker.compiled.manifest` selects a non-default sidecar location.
|
|
541
|
+
`worker.compiled.build` controls generation directly:
|
|
542
|
+
|
|
543
|
+
- `true` or omitted: build only when the artifact cannot be reused.
|
|
544
|
+
- `false`: never build; fail if the prebuilt artifact is unavailable.
|
|
545
|
+
- `"always"`: rebuild once whenever a pool is created.
|
|
546
|
+
|
|
547
|
+
The extension is .knt rather than .out because it identifies a Knitting worker
|
|
548
|
+
artifact; executability is never inferred from the suffix alone. Knitting also
|
|
549
|
+
requires the sidecar to match the protocol version, current platform and
|
|
550
|
+
architecture, source module, source timestamp, and requested task names before
|
|
551
|
+
spawning it. `checkCompiledWorker(...)` remains a read-only way to inspect that
|
|
552
|
+
state without building or executing anything.
|
|
553
|
+
|
|
554
|
+
Automatic builds use Porffor main from `worker.compiled.compiler`,
|
|
555
|
+
`PORFFOR_MAIN`/`PORF`, or `porf` on PATH. If none exists, Knitting downloads a
|
|
556
|
+
pinned compiler into `$XDG_CACHE_HOME/knitting`, or `~/.cache/knitting` when
|
|
557
|
+
`XDG_CACHE_HOME` is unset. Native builds use Porffor `-O3` and parallel LTO.
|
|
558
|
+
Set `worker.compiled.build: false` for deployment environments that must use
|
|
559
|
+
only a prebuilt artifact.
|
|
560
|
+
|
|
561
|
+
To build ahead of time, run
|
|
562
|
+
`bun run build:compiled --module tasks.ts --out tasks.knt --tasks addOne`.
|
|
563
|
+
|
|
564
|
+
Treat this backend as experimental. Porffor is a young ahead-of-time compiler,
|
|
565
|
+
the artifact format is pre-release and changes without a compatibility path, and
|
|
566
|
+
the executable is native, unsandboxed code — only run artifacts you trust.
|
|
567
|
+
|
|
568
|
+
Compiled workers accept synchronous JSON-compatible primitives, arrays, and
|
|
569
|
+
plain objects up to 1 MiB per call, plus `ArrayBuffer`, `DataView`, and typed
|
|
570
|
+
arrays, which are copied. A `ProcessSharedBuffer` is mapped by the worker rather
|
|
571
|
+
than copied. BMP Unicode is supported; supplementary code points and async
|
|
572
|
+
results are not.
|
|
573
|
+
|
|
574
|
+
Abort-aware tasks work on POSIX: the pool shares an abort bitmap through named
|
|
575
|
+
shared memory, so `signal.hasAborted()` and `signal.now()` are native reads
|
|
576
|
+
inside the worker. Windows has no implementation yet. Task timeouts, bootstrap
|
|
577
|
+
hooks, permission policies, and host inlining still fail during pool creation or
|
|
578
|
+
invocation; `worker.hardTimeoutMs` remains available because the host enforces
|
|
579
|
+
it.
|
|
580
|
+
|
|
448
581
|
### Windows process workers
|
|
449
582
|
|
|
450
583
|
On Windows, Knitting automatically uses named shared memory for the
|
|
@@ -507,8 +640,9 @@ Knitting defaults to a strict worker permission policy:
|
|
|
507
640
|
permission: { mode: "strict", allowImport: true }
|
|
508
641
|
```
|
|
509
642
|
|
|
510
|
-
|
|
511
|
-
|
|
643
|
+
For process workers, that default is translated through the selected runtime's
|
|
644
|
+
permission model. If a runtime cannot enforce one of the implicit defaults,
|
|
645
|
+
Knitting warns once. Do not treat the default as an OS sandbox.
|
|
512
646
|
|
|
513
647
|
For trusted local scripts, you can opt out:
|
|
514
648
|
|
|
@@ -518,7 +652,8 @@ const pool = createPool({
|
|
|
518
652
|
})({ add });
|
|
519
653
|
```
|
|
520
654
|
|
|
521
|
-
For production or plugin-like workloads, prefer an explicit policy
|
|
655
|
+
For production or plugin-like workloads, prefer an explicit policy. This example
|
|
656
|
+
is fully representable by a Deno process worker:
|
|
522
657
|
|
|
523
658
|
```ts
|
|
524
659
|
const pool = createPool({
|
|
@@ -534,9 +669,45 @@ const pool = createPool({
|
|
|
534
669
|
})({ add });
|
|
535
670
|
```
|
|
536
671
|
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
672
|
+
Before a process worker is spawned, Knitting checks explicit restrictions
|
|
673
|
+
against the selected runtime. A restriction that would be broader than requested
|
|
674
|
+
or ignored is rejected synchronously. Implicit strict-default gaps remain
|
|
675
|
+
backward compatible and produce a once-per-runtime warning.
|
|
676
|
+
|
|
677
|
+
| Process-worker permission | Deno | Node 22/24 | Node 26+ | Bun |
|
|
678
|
+
| ------------------------- | ------------------- | ------------------- | -------------------------- | ----------- |
|
|
679
|
+
| Filesystem allow-list | Scoped | Scoped | Scoped | Unsupported |
|
|
680
|
+
| Filesystem deny-list | Scoped | Unsupported | Unsupported | Unsupported |
|
|
681
|
+
| Network | Scoped allow/deny | Unsupported | Deny-all or allow-all only | Unsupported |
|
|
682
|
+
| Environment allow/deny | Scoped | Unsupported | Unsupported | Unsupported |
|
|
683
|
+
| Child processes | Scoped allow/deny | All-or-none | All-or-none | Unsupported |
|
|
684
|
+
| Worker creation | Unsupported | All-or-none | All-or-none | Unsupported |
|
|
685
|
+
| Import hosts | Scoped | Unsupported | Unsupported | Unsupported |
|
|
686
|
+
| System information | Scoped | Unsupported | Unsupported | Unsupported |
|
|
687
|
+
| WASI denial | Unsupported | All-or-none | All-or-none | Unsupported |
|
|
688
|
+
| Native code / FFI denial | Transport exception | Transport exception | Transport exception | Unsupported |
|
|
689
|
+
|
|
690
|
+
“All-or-none” means an explicit scoped list fails closed. Node 26's
|
|
691
|
+
`--allow-net` switch is emitted for `net: true`, but a host allow-list still
|
|
692
|
+
cannot be represented. When a wrapper or cross-runtime host hides the target
|
|
693
|
+
Node version, Knitting uses the conservative Node 22/24 capability set.
|
|
694
|
+
|
|
695
|
+
These compatibility checks currently cover process workers. Thread workers use
|
|
696
|
+
the host runtime's worker behavior and should not be treated as a sandbox.
|
|
697
|
+
Runtime permissions are guardrails, not the only security boundary for hostile
|
|
698
|
+
code.
|
|
699
|
+
|
|
700
|
+
The top-level `ffi` permission is the explicit cross-runtime native-code
|
|
701
|
+
capability. On Node it enables both native addons and `node:ffi`; Node's
|
|
702
|
+
`--allow-ffi` permission is currently unrestricted. The legacy/runtime-specific
|
|
703
|
+
`node.allowAddons` and `node.allowFfi` switches are independent—enabling addons
|
|
704
|
+
does not silently enable FFI.
|
|
705
|
+
|
|
706
|
+
Node process workers are a transport exception: Knitting needs `--allow-addons`
|
|
707
|
+
on Node 22/24 or `--allow-ffi` on Node 26 to map their shared memory. Deno
|
|
708
|
+
process workers likewise need `--allow-ffi`. Those capabilities apply to the
|
|
709
|
+
entire worker process, including task code, so explicit native-code denial fails
|
|
710
|
+
closed. Use an OS sandbox when task code is hostile.
|
|
540
711
|
|
|
541
712
|
## Payloads
|
|
542
713
|
|
|
@@ -607,12 +778,12 @@ if (isMain) {
|
|
|
607
778
|
The body is generic — `Envelope<Header, Body>` — and accepts any of the binary
|
|
608
779
|
shapes the transport understands:
|
|
609
780
|
|
|
610
|
-
| Body
|
|
611
|
-
|
|
|
612
|
-
| `ArrayBuffer`
|
|
613
|
-
| `SharedArrayBuffer`
|
|
614
|
-
| `ProcessSharedBuffer
|
|
615
|
-
| `BufferReference`
|
|
781
|
+
| Body | Copy? | Workers | Notes |
|
|
782
|
+
| --------------------- | ----------------- | ---------------- | ------------------------------------------------------------------- |
|
|
783
|
+
| `ArrayBuffer` | copied | thread + process | The default body; works everywhere. |
|
|
784
|
+
| `SharedArrayBuffer` | zero-copy, shared | thread only | Shared by reference; process workers reject it. |
|
|
785
|
+
| `ProcessSharedBuffer` | zero-copy, shared | thread + process | Cross-process shared memory. |
|
|
786
|
+
| `BufferReference` | zero-copy, moved | thread only | From `knitting/unsafe`; same constraints as bare `BufferReference`. |
|
|
616
787
|
|
|
617
788
|
The header keeps its fast paths regardless of the body: a small header is
|
|
618
789
|
written inline, and only large headers spill to the dynamic payload region. A
|
|
@@ -840,6 +1011,24 @@ worker. It is the same-process counterpart to `ProcessSharedBuffer`: reach for
|
|
|
840
1011
|
it when you hold a large `ArrayBuffer` or typed array and the copy cost to a
|
|
841
1012
|
thread worker actually matters.
|
|
842
1013
|
|
|
1014
|
+
The ownership and revocation paths are hardened against ordinary
|
|
1015
|
+
use-after-free: sources are detached on move, aliases are detached before a
|
|
1016
|
+
borrow is released, and outstanding borrowed returns are revoked during pool
|
|
1017
|
+
shutdown. This is still an **unsafe capability**, not a memory-safety or
|
|
1018
|
+
security boundary. Do not accept `BufferReferenceMetadata` or raw pointer data
|
|
1019
|
+
from untrusted code, and do not mutate the same bytes concurrently without
|
|
1020
|
+
your own synchronization.
|
|
1021
|
+
|
|
1022
|
+
| Path | Node 22/24 owning addon | Node 26 FFI | Deno/Bun FFI |
|
|
1023
|
+
| --- | --- | --- | --- |
|
|
1024
|
+
| Forward input | Moved, zero-copy | Moved, zero-copy alias | Moved, zero-copy alias |
|
|
1025
|
+
| Default returned reference | Co-owned, zero-copy | One safe copy | One safe copy |
|
|
1026
|
+
| `BufferReferenceReturn: "borrow"` | Revocable borrow | Revocable borrow | Revocable borrow |
|
|
1027
|
+
| Worker shutdown | Outstanding borrows revoked | Copy while readable, then revoke | Copy while readable, then revoke |
|
|
1028
|
+
|
|
1029
|
+
Older Node addons without the owning primitives use the non-owning fallback and
|
|
1030
|
+
therefore follow the copy/borrow rules rather than the owning-addon row.
|
|
1031
|
+
|
|
843
1032
|
```ts
|
|
844
1033
|
import { createPool, isMain, task } from "knitting";
|
|
845
1034
|
import { BufferReference } from "knitting/unsafe";
|
|
@@ -874,33 +1063,36 @@ Read these constraints before reaching for it:
|
|
|
874
1063
|
- **Move semantics.** Constructing a `BufferReference` detaches its source — the
|
|
875
1064
|
original buffer is empty afterward, and reads/writes through it are gone. The
|
|
876
1065
|
bytes now belong to the reference; to get a result back, the worker returns
|
|
877
|
-
its own `BufferReference`. Each
|
|
1066
|
+
its own `BufferReference`. Each source ownership transfer is one-shot, but an
|
|
1067
|
+
active reference may be read more than once. Forward inputs the worker
|
|
878
1068
|
materializes with `.toArrayBuffer()`/`.toUint8Array()` are borrowed for the
|
|
879
1069
|
duration of the call and detached once it settles; do not keep using them from
|
|
880
1070
|
fire-and-forget work after the task returns.
|
|
881
|
-
- **Forward is zero-copy everywhere
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
- **Borrowed Deno/Bun returns are opt-in.** The default is
|
|
1071
|
+
- **Forward is zero-copy everywhere.** Sending a buffer to the worker never
|
|
1072
|
+
copies. Node 22 and 24 with the owning addon also return the buffer without
|
|
1073
|
+
copying because the addon co-owns the V8 backing store. Node 26, Deno, and Bun
|
|
1074
|
+
take one safe copy on return because their FFI aliases cannot own the worker's
|
|
1075
|
+
backing store.
|
|
1076
|
+
- **Borrowed FFI returns are opt-in.** The default on Node 26, Deno, and Bun is
|
|
888
1077
|
`unsafe: { BufferReferenceReturn: "copy" }` — the safe single copy described
|
|
889
|
-
above. Set it to `"borrow"`
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
1078
|
+
above. Set it to `"borrow"` to skip that copy by borrowing the worker's
|
|
1079
|
+
backing store until the returned `BufferReference` is released. Call
|
|
1080
|
+
`ref.release()` or use `using`. Releasing **revokes** the borrow: the
|
|
1081
|
+
reference and every view taken from it are detached first, so later reads see
|
|
1082
|
+
empty views or throw instead of touching freed memory. A reference that is
|
|
1083
|
+
never released drops its borrow only once it and all of its views are
|
|
1084
|
+
unreachable. Pool shutdown revokes outstanding borrows too — references you
|
|
1085
|
+
still hold survive on a private copy, while stale views read as empty. If the
|
|
1086
|
+
bytes escape into HTTP responses, streams, timers, callbacks, or caches, copy
|
|
1087
|
+
them before the borrowed reference is released, or they will read as empty
|
|
1088
|
+
afterward.
|
|
896
1089
|
- **Unsafe escape hatch.** This is not a security boundary. Forged metadata or
|
|
897
1090
|
unsynchronized host/worker mutation can still be unsafe.
|
|
898
|
-
- **Node
|
|
899
|
-
`knitting_buffer_pointer`
|
|
900
|
-
|
|
901
|
-
a `BufferReference` on Node throws.
|
|
1091
|
+
- **Node backend depends on the Node line.** Node 22 and 24 use the
|
|
1092
|
+
`knitting_buffer_pointer` addon. Node 26 uses `node:ffi` and therefore needs
|
|
1093
|
+
`--experimental-ffi`.
|
|
902
1094
|
|
|
903
|
-
Borrowed returns, end to end (
|
|
1095
|
+
Borrowed returns, end to end (this opts Node 26, Deno, and Bun in):
|
|
904
1096
|
|
|
905
1097
|
```ts
|
|
906
1098
|
using pool = createPool({
|
|
@@ -914,7 +1106,7 @@ using pool = createPool({
|
|
|
914
1106
|
using result = await pool.call.invert(new BufferReference(pixels));
|
|
915
1107
|
const out = result.toUint8Array(); // borrowed — valid only while `result` lives
|
|
916
1108
|
console.log([...out]);
|
|
917
|
-
} // `using` releases the borrow here;
|
|
1109
|
+
} // `using` releases the borrow here; `out` is detached and reads as empty now
|
|
918
1110
|
```
|
|
919
1111
|
|
|
920
1112
|
When in doubt, a plain `ArrayBuffer` or typed-array payload — which knitting
|
|
@@ -927,10 +1119,32 @@ pointer setup tends to cost more than just copying, so the plain transport wins.
|
|
|
927
1119
|
|
|
928
1120
|
Knitting supports Node.js 22+, Deno 2+, and Bun 1+ on Linux, macOS, and Windows.
|
|
929
1121
|
|
|
930
|
-
|
|
931
|
-
`ProcessSharedBuffer` use
|
|
932
|
-
include
|
|
933
|
-
|
|
1122
|
+
Plain Node thread workers work without native pieces. Node process workers,
|
|
1123
|
+
`ProcessSharedBuffer`, and `BufferReference` use native support. Release
|
|
1124
|
+
packages currently include Node prebuilds for:
|
|
1125
|
+
|
|
1126
|
+
- Node.js 22 (ABI 127) and Node.js 24 (ABI 137)
|
|
1127
|
+
- Linux x64
|
|
1128
|
+
- macOS x64 and arm64
|
|
1129
|
+
- Windows x64
|
|
1130
|
+
|
|
1131
|
+
Odd-numbered Node releases are not supported by packaged native features.
|
|
1132
|
+
Node.js 26 uses `node:ffi` instead of another ABI-specific addon. Start the host
|
|
1133
|
+
with:
|
|
1134
|
+
|
|
1135
|
+
```bash
|
|
1136
|
+
node --experimental-ffi app.js
|
|
1137
|
+
```
|
|
1138
|
+
|
|
1139
|
+
When using Node's Permission Model, also grant `--allow-ffi`. Knitting passes
|
|
1140
|
+
the experimental FFI flag to Node process workers it starts, but the host must
|
|
1141
|
+
be started with the flag so it can create the shared mappings. The FFI backend
|
|
1142
|
+
uses external `ArrayBuffer` mappings, so `ProcessSharedBuffer.view()`,
|
|
1143
|
+
`.getBuffer()`, and `.bytes()` work; `.getSAB()` is available only on the Node
|
|
1144
|
+
22/24 addon backend.
|
|
1145
|
+
|
|
1146
|
+
If you are developing locally on another Node ABI or architecture, you can
|
|
1147
|
+
compile the current V8 addon for that exact runtime:
|
|
934
1148
|
|
|
935
1149
|
```bash
|
|
936
1150
|
bun run build:native
|
package/knitting.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { workerMainLoop } from "./src/worker/loop.js";
|
|
2
|
-
import { createPool, importTask, isMain, task } from "./src/api.js";
|
|
2
|
+
import { checkCompiledWorker, createPool, importTask, isMain, setModuleUrl, task } from "./src/api.js";
|
|
3
3
|
import { Envelope } from "./src/common/envelope.js";
|
|
4
4
|
import { isNumericArray, NumericArray } from "./src/connections/numeric-array.js";
|
|
5
|
-
export { createPool as createPool, Envelope as Envelope, importTask as importTask, isMain as isMain, isNumericArray as isNumericArray, NumericArray as NumericArray, task as task, workerMainLoop as workerMainLoop, };
|
|
5
|
+
export { checkCompiledWorker as checkCompiledWorker, createPool as createPool, Envelope as Envelope, importTask as importTask, isMain as isMain, isNumericArray as isNumericArray, NumericArray as NumericArray, setModuleUrl as setModuleUrl, task as task, workerMainLoop as workerMainLoop, };
|
|
6
6
|
export type { EnvelopeBody as EnvelopeBody, EnvelopeHeader as EnvelopeHeader, } from "./src/common/envelope.js";
|
|
7
|
+
export type { CompiledWorkerCheck, CompiledWorkerOptions, CompiledWorkerSource, } from "./src/types.js";
|
package/knitting.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Exportables
|
|
2
2
|
import { workerMainLoop } from "./src/worker/loop.js";
|
|
3
|
-
import { createPool, importTask, isMain, task } from "./src/api.js";
|
|
3
|
+
import { checkCompiledWorker, createPool, importTask, isMain, setModuleUrl, task, } from "./src/api.js";
|
|
4
4
|
import { Envelope } from "./src/common/envelope.js";
|
|
5
|
-
import { isNumericArray, NumericArray } from "./src/connections/numeric-array.js";
|
|
6
|
-
export { createPool as createPool, Envelope as Envelope, importTask as importTask, isMain as isMain, isNumericArray as isNumericArray, NumericArray as NumericArray, task as task, workerMainLoop as workerMainLoop, };
|
|
5
|
+
import { isNumericArray, NumericArray, } from "./src/connections/numeric-array.js";
|
|
6
|
+
export { checkCompiledWorker as checkCompiledWorker, createPool as createPool, Envelope as Envelope, importTask as importTask, isMain as isMain, isNumericArray as isNumericArray, NumericArray as NumericArray, setModuleUrl as setModuleUrl, task as task, workerMainLoop as workerMainLoop, };
|
package/map.md
CHANGED
|
@@ -36,8 +36,10 @@ The core flow is:
|
|
|
36
36
|
|
|
37
37
|
## Build And Scripts
|
|
38
38
|
|
|
39
|
-
- `build.ts`:
|
|
40
|
-
`src/**/*.ts` to `.js` and `.d.ts` files with TypeScript.
|
|
39
|
+
- `build.ts`: Runs the generated-output cleaner, then compiles public entries
|
|
40
|
+
and `src/**/*.ts` to `.js` and `.d.ts` files with TypeScript.
|
|
41
|
+
- `scripts/clean-generated.ts`: Removes npm-generated root entry files and
|
|
42
|
+
generated `.js`/`.d.ts` files under `src/`; exposed as `npm run clean`.
|
|
41
43
|
- `tsconfig.npm.json`: TypeScript config used by the npm release build.
|
|
42
44
|
- `scripts/rewrite-declaration-imports.mjs`: Rewrites generated declaration
|
|
43
45
|
imports from `.ts` to `.js` so npm consumers resolve the compiled files.
|
|
@@ -65,8 +67,8 @@ The core flow is:
|
|
|
65
67
|
- `docs/AGENTS.md`: Agent/contributor orientation, invariants, and workflow
|
|
66
68
|
notes.
|
|
67
69
|
- `docs/CLAUDE.md`: Pointer to the shared agent guidance.
|
|
68
|
-
- `docs/buffer-reference-ownership-move.md`: Design record for
|
|
69
|
-
|
|
70
|
+
- `docs/buffer-reference-ownership-move.md`: Design record for `BufferReference`
|
|
71
|
+
ownership transfer.
|
|
70
72
|
- `docs/knitting.pdf`: Longer-form project design/theory document.
|
|
71
73
|
|
|
72
74
|
## Public API Layer
|
|
@@ -120,6 +122,12 @@ The core flow is:
|
|
|
120
122
|
- `src/runtime/process-worker.ts`: Process-worker spawning, command/runtime
|
|
121
123
|
selection, process shared-memory layout, inherited/named mapping metadata, and
|
|
122
124
|
child boot payload construction.
|
|
125
|
+
- `src/runtime/compiled-artifact.ts`: Resolves `.knt` executables and validates
|
|
126
|
+
their versioned manifests, source/task identity, target platform, and execute
|
|
127
|
+
permission without launching them.
|
|
128
|
+
- `src/runtime/compiled-worker.ts`: Experimental host adapter for compiled
|
|
129
|
+
worker executables. Spawns the artifact and maps pool calls onto the current
|
|
130
|
+
numeric framed-pipe protocol.
|
|
123
131
|
|
|
124
132
|
## Worker Side
|
|
125
133
|
|
|
@@ -196,14 +204,16 @@ The core flow is:
|
|
|
196
204
|
parsing, mapping support, and descriptor lifecycle helpers.
|
|
197
205
|
- `src/connections/node-addons.ts`: Native addon specifier resolution for
|
|
198
206
|
committed Node ABI prebuilds with `build/Release` fallback loading.
|
|
199
|
-
- `src/connections/node.ts`:
|
|
200
|
-
|
|
207
|
+
- `src/connections/node.ts`: Selects the Node 22/24 addon backend or the Node 26
|
|
208
|
+
FFI backend and exposes shared-memory and futex primitives.
|
|
209
|
+
- `src/connections/node-ffi.ts`: Node 26 `node:ffi` implementation for POSIX
|
|
210
|
+
shared-memory creation, mapping, cleanup, and buffer-pointer aliases.
|
|
201
211
|
- `src/connections/bun.ts`: Bun FFI implementation for POSIX shared memory.
|
|
202
212
|
- `src/connections/deno.ts`: Deno FFI implementation for POSIX shared memory.
|
|
203
213
|
- `src/connections/posix.ts`: POSIX constants, shared-memory naming, libc path
|
|
204
214
|
detection, and close-on-exec helpers.
|
|
205
215
|
- `src/connections/windows.ts`: Windows FFI loader and named file-mapping
|
|
206
|
-
primitives used by Deno and Bun.
|
|
216
|
+
primitives used by Node 26, Deno, and Bun.
|
|
207
217
|
- `src/knitting_shared_memory.cc`: Native Node addon for shared-memory create,
|
|
208
218
|
map, unlink, and descriptor operations, including Windows named mappings.
|
|
209
219
|
- `src/knitting_shm.cc`: Native Node addon for futex/wait helpers used by parked
|
|
@@ -213,8 +223,8 @@ The core flow is:
|
|
|
213
223
|
FFI runtimes.
|
|
214
224
|
- `prebuilds/*/*.node`: Tracked Node native-addon prebuilds for supported
|
|
215
225
|
platform/Node ABI combinations.
|
|
216
|
-
- `prebuilds/win32-x64/*.dll`: Tracked Windows FFI DLL prebuild used by
|
|
217
|
-
Deno shared-memory primitives on Windows.
|
|
226
|
+
- `prebuilds/win32-x64/*.dll`: Tracked Windows FFI DLL prebuild used by Node 26,
|
|
227
|
+
Bun, and Deno shared-memory primitives on Windows.
|
|
218
228
|
|
|
219
229
|
## Permissions
|
|
220
230
|
|
|
@@ -271,6 +281,8 @@ The core flow is:
|
|
|
271
281
|
- `test/_runner.ts`: Runtime-neutral test runner shim used by the test suite.
|
|
272
282
|
- `test/abortSignal.test.ts`: Shared abort bitset behavior.
|
|
273
283
|
- `test/api-cap.test.ts`: API limits such as maximum task id count.
|
|
284
|
+
- `test/compiled-worker.test.ts`: Compiled-artifact validation, public pool
|
|
285
|
+
integration, source-derived `.knt` paths, and unsupported-feature guards.
|
|
274
286
|
- `test/shared-buffer-io.test.ts`: Shared-buffer IO read/write behavior.
|
|
275
287
|
- `test/file-descriptor.test.ts`: File descriptor metadata and mapping behavior.
|
|
276
288
|
- `test/inliner.test.ts`: Inline executor behavior and thresholds.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "knitting",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.60",
|
|
4
4
|
"description": "Shared-memory IPC runtime for Node.js, Deno, and Bun.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -44,6 +44,9 @@
|
|
|
44
44
|
"prebuilds/**/*.node",
|
|
45
45
|
"prebuilds/**/*.dll",
|
|
46
46
|
"scripts/build-native-addons.ts",
|
|
47
|
+
"scripts/build-compiled-worker.ts",
|
|
48
|
+
"scripts/compiled-worker/**/*.ts",
|
|
49
|
+
"scripts/compiled-worker/**/*.d.ts",
|
|
47
50
|
"src/**/*.cc",
|
|
48
51
|
"src/**/*.d.ts",
|
|
49
52
|
"src/**/*.js",
|
|
@@ -59,11 +62,13 @@
|
|
|
59
62
|
"utils.js"
|
|
60
63
|
],
|
|
61
64
|
"scripts": {
|
|
65
|
+
"clean": "bun run scripts/clean-generated.ts",
|
|
62
66
|
"build": "bun run build.ts && node scripts/rewrite-declaration-imports.mjs",
|
|
63
67
|
"build:js": "bun run build.ts && node scripts/rewrite-declaration-imports.mjs",
|
|
64
68
|
"build:types": "bun run build.ts && node scripts/rewrite-declaration-imports.mjs",
|
|
65
69
|
"build:npm": "bun run build.ts && node scripts/rewrite-declaration-imports.mjs",
|
|
66
70
|
"build:native": "bun run scripts/build-native-addons.ts",
|
|
71
|
+
"build:compiled": "bun run scripts/build-compiled-worker.ts",
|
|
67
72
|
"build:release": "bun run build:native && npm run build",
|
|
68
73
|
"build:node": "bun run build.ts && node scripts/rewrite-declaration-imports.mjs",
|
|
69
74
|
"prepack": "npm run build",
|
|
@@ -72,6 +77,7 @@
|
|
|
72
77
|
"test": "npm run test:deno",
|
|
73
78
|
"test:deno": "deno test -A --ignore=test/runtime.node.test.ts --ignore=test/runtime.process.test.ts",
|
|
74
79
|
"test:node": "node --no-warnings --experimental-transform-types --test \"./test/*.test.ts\"",
|
|
80
|
+
"test:node:26": "node --no-warnings --experimental-ffi test/node26-ffi.integration.mjs",
|
|
75
81
|
"test:bun": "bun test ./test/*.test.ts --path-ignore-patterns=**/tx-queue.test.ts",
|
|
76
82
|
"test:runtimes": "npm run test:node && npm run test:bun",
|
|
77
83
|
"test:all": "npm run test:deno && npm run test:runtimes",
|