knitting 0.1.56 → 0.1.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +254 -40
- package/knitting.d.ts +3 -2
- package/knitting.js +3 -3
- package/map.md +21 -9
- package/package.json +7 -1
- package/scripts/build-compiled-worker.ts +276 -0
- package/scripts/compiled-worker/porffor.d.ts +5 -0
- package/scripts/compiled-worker/runtime.ts +529 -0
- package/scripts/compiled-worker/task-shim.ts +32 -0
- package/src/api.d.ts +24 -1
- package/src/api.js +199 -20
- package/src/common/module-url.d.ts +1 -0
- package/src/common/module-url.js +30 -2
- package/src/common/node-compat.d.ts +2 -0
- package/src/common/path-canonical.js +20 -6
- package/src/common/runtime.d.ts +2 -1
- package/src/common/runtime.js +18 -4
- package/src/common/task-source.d.ts +2 -0
- package/src/common/task-source.js +17 -1
- package/src/common/worker-runtime.js +16 -7
- package/src/connections/buffer-reference-native.js +25 -1
- package/src/connections/buffer-reference.d.ts +23 -2
- package/src/connections/buffer-reference.js +143 -29
- package/src/connections/bun.d.ts +1 -1
- package/src/connections/bun.js +25 -14
- package/src/connections/deno.d.ts +3 -1
- package/src/connections/deno.js +35 -14
- package/src/connections/external-array-buffer.d.ts +2 -0
- package/src/connections/external-array-buffer.js +33 -0
- package/src/connections/node-addons.d.ts +11 -0
- package/src/connections/node-addons.js +49 -1
- package/src/connections/node-ffi-api.d.ts +19 -0
- package/src/connections/node-ffi-api.js +28 -0
- package/src/connections/node-ffi.d.ts +27 -0
- package/src/connections/node-ffi.js +262 -0
- package/src/connections/node.d.ts +1 -1
- package/src/connections/node.js +20 -20
- package/src/connections/package-assets.js +25 -15
- package/src/connections/posix.d.ts +42 -0
- package/src/connections/posix.js +101 -0
- package/src/connections/process-shared-buffer.d.ts +1 -0
- package/src/connections/process-shared-buffer.js +11 -45
- package/src/connections/types.d.ts +4 -0
- package/src/connections/windows.d.ts +19 -10
- package/src/connections/windows.js +87 -66
- package/src/error.d.ts +7 -6
- package/src/error.js +8 -7
- package/src/memory/lock.d.ts +87 -77
- package/src/memory/lock.js +103 -84
- package/src/memory/payloadCodec.d.ts +11 -2
- package/src/memory/payloadCodec.js +127 -31
- package/src/permission/compatibility.d.ts +28 -0
- package/src/permission/compatibility.js +239 -0
- package/src/permission/index.d.ts +2 -0
- package/src/permission/index.js +1 -0
- package/src/permission/protocol.d.ts +1 -0
- package/src/permission/protocol.js +54 -14
- package/src/runtime/compiled-artifact.d.ts +21 -0
- package/src/runtime/compiled-artifact.js +260 -0
- package/src/runtime/compiled-builder.d.ts +8 -0
- package/src/runtime/compiled-builder.js +42 -0
- package/src/runtime/compiled-worker.d.ts +27 -0
- package/src/runtime/compiled-worker.js +562 -0
- package/src/runtime/inline-executor.js +27 -15
- package/src/runtime/pool.d.ts +7 -2
- package/src/runtime/pool.js +74 -6
- package/src/runtime/process-worker.d.ts +1 -1
- package/src/runtime/process-worker.js +48 -8
- package/src/runtime/tx-queue.d.ts +3 -1
- package/src/runtime/tx-queue.js +4 -1
- package/src/types.d.ts +48 -6
- package/src/worker/composable-runners.js +3 -2
- package/src/worker/loop.d.ts +1 -0
- package/src/worker/loop.js +25 -12
- package/src/worker/task-loader.d.ts +5 -4
- package/src/worker/task-loader.js +13 -7
- package/src/worker/timers.js +8 -6
|
@@ -37,7 +37,8 @@ const RUNTIME_NODE = 1;
|
|
|
37
37
|
const RUNTIME_DENO = 2;
|
|
38
38
|
const RUNTIME_BUN = 3;
|
|
39
39
|
const PAYLOAD_TRANSPORT_FINALIZER = Symbol.for("knitting.payloadCodec.transportFinalizer");
|
|
40
|
-
let
|
|
40
|
+
let currentReturnHooks;
|
|
41
|
+
const bufferReferenceInstances = new WeakSet();
|
|
41
42
|
const encodeRuntime = (runtime) => {
|
|
42
43
|
switch (runtime) {
|
|
43
44
|
case "node":
|
|
@@ -95,16 +96,17 @@ const numericMetadataToMetadata = (words) => {
|
|
|
95
96
|
byteLength: words[5] ?? 0,
|
|
96
97
|
};
|
|
97
98
|
};
|
|
99
|
+
const toReturnHooks = (releaser) => typeof releaser === "function" ? { release: releaser } : releaser;
|
|
98
100
|
export const withBufferReferenceReturnReleaser = (releaser, run) => {
|
|
99
101
|
if (releaser === undefined)
|
|
100
102
|
return run();
|
|
101
|
-
const previous =
|
|
102
|
-
|
|
103
|
+
const previous = currentReturnHooks;
|
|
104
|
+
currentReturnHooks = toReturnHooks(releaser);
|
|
103
105
|
try {
|
|
104
106
|
return run();
|
|
105
107
|
}
|
|
106
108
|
finally {
|
|
107
|
-
|
|
109
|
+
currentReturnHooks = previous;
|
|
108
110
|
}
|
|
109
111
|
};
|
|
110
112
|
export const createBufferReferenceReturnReleaseMessage = (token) => ({
|
|
@@ -124,11 +126,13 @@ export const readBufferReferenceReturnReleaseMessage = (value) => {
|
|
|
124
126
|
return undefined;
|
|
125
127
|
}
|
|
126
128
|
};
|
|
127
|
-
// Backstop for references that never flow through the transport.
|
|
128
129
|
const producerFinalizer = typeof FinalizationRegistry === "function"
|
|
129
|
-
? new FinalizationRegistry((
|
|
130
|
+
? new FinalizationRegistry((backstop) => {
|
|
131
|
+
backstop.count -= 1;
|
|
132
|
+
if (backstop.count > 0)
|
|
133
|
+
return;
|
|
130
134
|
try {
|
|
131
|
-
release(token);
|
|
135
|
+
backstop.release(backstop.token);
|
|
132
136
|
}
|
|
133
137
|
catch {
|
|
134
138
|
// best effort
|
|
@@ -162,7 +166,8 @@ const assertMovableSource = (source) => {
|
|
|
162
166
|
};
|
|
163
167
|
const isDetached = (buffer) => buffer.detached === true ||
|
|
164
168
|
buffer.byteLength === 0;
|
|
165
|
-
|
|
169
|
+
/** Revocation primitive: detach `buffer` so every view over it is neutralized. */
|
|
170
|
+
export const detachArrayBufferBestEffort = (runtime, buffer) => {
|
|
166
171
|
if (isDetached(buffer))
|
|
167
172
|
return true;
|
|
168
173
|
if (runtime === "node") {
|
|
@@ -233,9 +238,11 @@ export class BufferReference {
|
|
|
233
238
|
#token;
|
|
234
239
|
#isProducer;
|
|
235
240
|
#finalizerToken;
|
|
241
|
+
#producerBackstop;
|
|
236
242
|
#materializedRegions;
|
|
237
243
|
#owned;
|
|
238
244
|
#borrowedReturnReleaser;
|
|
245
|
+
#borrowedReturnTrack;
|
|
239
246
|
#borrowedReturnFinalizerToken;
|
|
240
247
|
#transportRefs = 0;
|
|
241
248
|
#released = false;
|
|
@@ -249,6 +256,7 @@ export class BufferReference {
|
|
|
249
256
|
this.byteLength = meta.byteLength;
|
|
250
257
|
this.#token = BigInt(meta.token);
|
|
251
258
|
this.#isProducer = false;
|
|
259
|
+
bufferReferenceInstances.add(this);
|
|
252
260
|
return;
|
|
253
261
|
}
|
|
254
262
|
if (source === undefined) {
|
|
@@ -264,8 +272,16 @@ export class BufferReference {
|
|
|
264
272
|
this.byteLength = produced.byteLength;
|
|
265
273
|
this.#token = produced.token;
|
|
266
274
|
this.#isProducer = true;
|
|
267
|
-
this
|
|
268
|
-
producerFinalizer
|
|
275
|
+
bufferReferenceInstances.add(this);
|
|
276
|
+
if (producerFinalizer !== undefined) {
|
|
277
|
+
this.#finalizerToken = {};
|
|
278
|
+
this.#producerBackstop = {
|
|
279
|
+
count: 1,
|
|
280
|
+
token: this.#token,
|
|
281
|
+
release: caps.release,
|
|
282
|
+
};
|
|
283
|
+
producerFinalizer.register(this, this.#producerBackstop, this.#finalizerToken);
|
|
284
|
+
}
|
|
269
285
|
}
|
|
270
286
|
static fromMetadata(meta) {
|
|
271
287
|
if (!isBufferReferenceMetadata(meta)) {
|
|
@@ -325,7 +341,7 @@ export class BufferReference {
|
|
|
325
341
|
}
|
|
326
342
|
}
|
|
327
343
|
/** Prepare a returned reference before the worker-side producer hold drains. */
|
|
328
|
-
claimOwnership(releaser =
|
|
344
|
+
claimOwnership(releaser = currentReturnHooks) {
|
|
329
345
|
if (this.#owned !== undefined || this.#borrowedReturnReleaser !== undefined ||
|
|
330
346
|
this.#released)
|
|
331
347
|
return this;
|
|
@@ -338,9 +354,18 @@ export class BufferReference {
|
|
|
338
354
|
byteLength: this.byteLength,
|
|
339
355
|
};
|
|
340
356
|
if (releaser !== undefined) {
|
|
341
|
-
|
|
357
|
+
const hooks = toReturnHooks(releaser);
|
|
358
|
+
// Borrow: adopt lazily — the alias costs GC-visible external memory on
|
|
359
|
+
// FFI runtimes, so it is only created when the bytes are actually read.
|
|
360
|
+
// Until then no view can exist, so the GC backstop safely watches the
|
|
361
|
+
// reference itself; first materialize migrates it to the alias buffer
|
|
362
|
+
// (views strongly reference their buffer, so the worker pin can only
|
|
363
|
+
// drop once no view can reach the borrowed bytes).
|
|
364
|
+
this.#borrowedReturnReleaser = hooks.release;
|
|
365
|
+
this.#borrowedReturnTrack = hooks.track;
|
|
342
366
|
this.#borrowedReturnFinalizerToken = {};
|
|
343
|
-
borrowedReturnFinalizer?.register(this, { token: this.#token, release:
|
|
367
|
+
borrowedReturnFinalizer?.register(this, { token: this.#token, release: hooks.release }, this.#borrowedReturnFinalizerToken);
|
|
368
|
+
hooks.track?.(this, this.#token);
|
|
344
369
|
return this;
|
|
345
370
|
}
|
|
346
371
|
this.#owned = caps.adopt(input, caps.supportsOwningAdopt ? undefined : { copy: true });
|
|
@@ -358,10 +383,22 @@ export class BufferReference {
|
|
|
358
383
|
byteLength: this.byteLength,
|
|
359
384
|
});
|
|
360
385
|
if (this.#borrowedReturnReleaser !== undefined) {
|
|
386
|
+
// First borrowed read: views over this alias exist from here on, so
|
|
387
|
+
// move the GC backstop from the reference onto the alias buffer.
|
|
361
388
|
this.#owned = region;
|
|
389
|
+
if (this.#borrowedReturnFinalizerToken !== undefined &&
|
|
390
|
+
borrowedReturnFinalizer !== undefined) {
|
|
391
|
+
borrowedReturnFinalizer.unregister(this.#borrowedReturnFinalizerToken);
|
|
392
|
+
borrowedReturnFinalizer.register(region.buffer, { token: this.#token, release: this.#borrowedReturnReleaser }, this.#borrowedReturnFinalizerToken);
|
|
393
|
+
}
|
|
394
|
+
this.#borrowedReturnTrack?.(this, this.#token, region.buffer);
|
|
362
395
|
return region;
|
|
363
396
|
}
|
|
364
397
|
(this.#materializedRegions ??= []).push(region);
|
|
398
|
+
if (this.#producerBackstop !== undefined && producerFinalizer !== undefined) {
|
|
399
|
+
this.#producerBackstop.count += 1;
|
|
400
|
+
producerFinalizer.register(region.buffer, this.#producerBackstop, this.#finalizerToken);
|
|
401
|
+
}
|
|
365
402
|
return region;
|
|
366
403
|
}
|
|
367
404
|
toArrayBuffer() {
|
|
@@ -383,38 +420,51 @@ export class BufferReference {
|
|
|
383
420
|
if (this.#released)
|
|
384
421
|
return;
|
|
385
422
|
this.#released = true;
|
|
423
|
+
// Revoke every alias this reference handed out before any pin can drop.
|
|
424
|
+
// Leak-not-UAF: if a detach fails, the pin must stay held — the GC
|
|
425
|
+
// backstops remain registered and retry once the buffers are unreachable.
|
|
426
|
+
let allDetached = true;
|
|
386
427
|
const regions = this.#materializedRegions;
|
|
387
428
|
this.#materializedRegions = undefined;
|
|
388
429
|
if (regions !== undefined) {
|
|
389
430
|
for (let i = 0; i < regions.length; i++) {
|
|
390
|
-
|
|
431
|
+
allDetached =
|
|
432
|
+
detachArrayBufferBestEffort(this.runtime, regions[i].buffer) &&
|
|
433
|
+
allDetached;
|
|
391
434
|
}
|
|
392
435
|
}
|
|
393
|
-
if (this.#borrowedReturnFinalizerToken !== undefined) {
|
|
394
|
-
borrowedReturnFinalizer?.unregister(this.#borrowedReturnFinalizerToken);
|
|
395
|
-
this.#borrowedReturnFinalizerToken = undefined;
|
|
396
|
-
}
|
|
397
436
|
const borrowedReturnReleaser = this.#borrowedReturnReleaser;
|
|
398
437
|
this.#borrowedReturnReleaser = undefined;
|
|
438
|
+
this.#borrowedReturnTrack = undefined;
|
|
399
439
|
if (borrowedReturnReleaser !== undefined) {
|
|
400
440
|
const owned = this.#owned;
|
|
401
441
|
this.#owned = undefined;
|
|
402
442
|
if (owned !== undefined) {
|
|
403
|
-
detachArrayBufferBestEffort(this.runtime, owned.buffer)
|
|
404
|
-
|
|
405
|
-
try {
|
|
406
|
-
borrowedReturnReleaser(this.#token);
|
|
443
|
+
allDetached = detachArrayBufferBestEffort(this.runtime, owned.buffer) &&
|
|
444
|
+
allDetached;
|
|
407
445
|
}
|
|
408
|
-
|
|
409
|
-
|
|
446
|
+
if (allDetached) {
|
|
447
|
+
if (this.#borrowedReturnFinalizerToken !== undefined) {
|
|
448
|
+
borrowedReturnFinalizer?.unregister(this.#borrowedReturnFinalizerToken);
|
|
449
|
+
this.#borrowedReturnFinalizerToken = undefined;
|
|
450
|
+
}
|
|
451
|
+
try {
|
|
452
|
+
borrowedReturnReleaser(this.#token);
|
|
453
|
+
}
|
|
454
|
+
catch {
|
|
455
|
+
// best effort
|
|
456
|
+
}
|
|
410
457
|
}
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
producerFinalizer?.unregister(this.#finalizerToken);
|
|
414
|
-
this.#finalizerToken = undefined;
|
|
458
|
+
// else: the finalizer stays registered on the alias buffer and sends
|
|
459
|
+
// the release once every escaped view is gone.
|
|
415
460
|
}
|
|
416
461
|
// Only the producer owns the registry hold; consumers must not drop it.
|
|
417
|
-
if (this.#isProducer) {
|
|
462
|
+
if (this.#isProducer && allDetached) {
|
|
463
|
+
if (this.#finalizerToken !== undefined) {
|
|
464
|
+
producerFinalizer?.unregister(this.#finalizerToken);
|
|
465
|
+
this.#finalizerToken = undefined;
|
|
466
|
+
}
|
|
467
|
+
this.#producerBackstop = undefined;
|
|
418
468
|
try {
|
|
419
469
|
getBufferReferenceCapabilities().release(this.#token);
|
|
420
470
|
}
|
|
@@ -422,6 +472,67 @@ export class BufferReference {
|
|
|
422
472
|
// best effort
|
|
423
473
|
}
|
|
424
474
|
}
|
|
475
|
+
// else on failed detach: the countdown backstop drops the pin when the
|
|
476
|
+
// reference and every materialized alias are unreachable.
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Revoke a borrowed return before its producing worker goes away: detach
|
|
480
|
+
* the borrowed alias so no view can dangle. With `copyBytes` the reference
|
|
481
|
+
* itself survives on an owned copy; without it (worker bytes already gone)
|
|
482
|
+
* later reads fail loud instead of aliasing freed memory.
|
|
483
|
+
*/
|
|
484
|
+
revokeBorrow(copyBytes) {
|
|
485
|
+
const releaser = this.#borrowedReturnReleaser;
|
|
486
|
+
if (releaser === undefined || this.#released)
|
|
487
|
+
return;
|
|
488
|
+
const owned = this.#owned;
|
|
489
|
+
let nextOwned;
|
|
490
|
+
let detached = true;
|
|
491
|
+
if (owned !== undefined) {
|
|
492
|
+
if (copyBytes) {
|
|
493
|
+
nextOwned = {
|
|
494
|
+
buffer: owned.buffer.slice(owned.byteOffset, owned.byteOffset + owned.byteLength),
|
|
495
|
+
byteOffset: 0,
|
|
496
|
+
byteLength: owned.byteLength,
|
|
497
|
+
};
|
|
498
|
+
}
|
|
499
|
+
// Without a copy, keep the (now detached) region so reads throw
|
|
500
|
+
// instead of re-materializing an alias over freed worker memory.
|
|
501
|
+
detached = detachArrayBufferBestEffort(this.runtime, owned.buffer);
|
|
502
|
+
}
|
|
503
|
+
else if (copyBytes) {
|
|
504
|
+
// Never materialized: take the bytes now, while the worker still pins
|
|
505
|
+
// them, so the reference stays readable after its worker dies.
|
|
506
|
+
const caps = getBufferReferenceCapabilities();
|
|
507
|
+
nextOwned = caps.adopt({
|
|
508
|
+
token: this.#token,
|
|
509
|
+
pointer: this.pointer,
|
|
510
|
+
byteOffset: this.byteOffset,
|
|
511
|
+
byteLength: this.byteLength,
|
|
512
|
+
}, { copy: !caps.supportsOwningAdopt });
|
|
513
|
+
}
|
|
514
|
+
else {
|
|
515
|
+
// Never materialized and the worker bytes are already gone: no view
|
|
516
|
+
// exists, so poison future reads instead of aliasing freed memory.
|
|
517
|
+
this.#released = true;
|
|
518
|
+
}
|
|
519
|
+
if (!detached)
|
|
520
|
+
return;
|
|
521
|
+
if (nextOwned !== undefined)
|
|
522
|
+
this.#owned = nextOwned;
|
|
523
|
+
this.#borrowedReturnReleaser = undefined;
|
|
524
|
+
this.#borrowedReturnTrack = undefined;
|
|
525
|
+
if (this.#borrowedReturnFinalizerToken !== undefined) {
|
|
526
|
+
borrowedReturnFinalizer?.unregister(this.#borrowedReturnFinalizerToken);
|
|
527
|
+
this.#borrowedReturnFinalizerToken = undefined;
|
|
528
|
+
}
|
|
529
|
+
// Best effort: if the worker is still alive it can drop its pin early.
|
|
530
|
+
try {
|
|
531
|
+
releaser(this.#token);
|
|
532
|
+
}
|
|
533
|
+
catch {
|
|
534
|
+
// best effort
|
|
535
|
+
}
|
|
425
536
|
}
|
|
426
537
|
[Symbol.dispose]() {
|
|
427
538
|
this.release();
|
|
@@ -445,6 +556,9 @@ export class BufferReference {
|
|
|
445
556
|
return finalizer;
|
|
446
557
|
}
|
|
447
558
|
}
|
|
559
|
+
export const isBufferReferenceValue = (value) => value !== null &&
|
|
560
|
+
typeof value === "object" &&
|
|
561
|
+
bufferReferenceInstances.has(value);
|
|
448
562
|
const codecGlobal = globalThis;
|
|
449
563
|
const codecs = codecGlobal.__KNITTING_PAYLOAD_CODECS__ ??= Object.create(null);
|
|
450
564
|
codecs[BUFFER_REFERENCE_CODEC_ID] = {
|
package/src/connections/bun.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ type BunPointer = number;
|
|
|
3
3
|
type BunLibc = {
|
|
4
4
|
symbols: {
|
|
5
5
|
memfd_create?: (name: Uint8Array, flags: number) => number;
|
|
6
|
-
shm_open?: (name: Uint8Array, flags: number,
|
|
6
|
+
shm_open?: (name: Uint8Array, flags: number, ...rest: number[]) => number;
|
|
7
7
|
shm_unlink?: (name: Uint8Array) => number;
|
|
8
8
|
ftruncate: (fd: number, length: bigint) => number;
|
|
9
9
|
dup: (fd: number) => number;
|
package/src/connections/bun.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { DARWIN_O_CREAT, DARWIN_O_EXCL, DARWIN_SHM_MODE,
|
|
2
|
-
import { expectFd, expectPositiveSize, readCreateName,
|
|
1
|
+
import { assertNamedSharedMemoryIsReopenable, checkPosixResult, DARWIN_O_CREAT, DARWIN_O_EXCL, DARWIN_SHM_MODE, detectPosixPlatform, encodeCString, getPosixLibcPath, LINUX_O_CREAT, LINUX_O_EXCL, makeDarwinSharedMemoryName, MAP_SHARED, O_RDWR, POSIX_SHM_MODE, posixError, PROT_READ, PROT_WRITE, setCloseOnExec, SHM_OPEN_STACK_PAD, shmOpenModeIsStackPassed, toPosixSharedMemoryName, } from "./posix.js";
|
|
2
|
+
import { expectFd, expectPositiveSize, readCreateMode, readCreateName, readCreateSize, readRequiredCreateName, } from "./types.js";
|
|
3
3
|
import { createBunWindowsConnectionPrimitives, isWindowsRuntime, } from "./windows.js";
|
|
4
4
|
const FFIType = {
|
|
5
5
|
i32: 5,
|
|
@@ -53,7 +53,14 @@ export const openBunLibc = () => getBunFFI().dlopen(getPosixLibcPath(), {
|
|
|
53
53
|
const getBunCreateSymbols = (platform = detectPosixPlatform()) => platform === "darwin"
|
|
54
54
|
? {
|
|
55
55
|
shm_open: {
|
|
56
|
-
args:
|
|
56
|
+
args: shmOpenModeIsStackPassed(platform)
|
|
57
|
+
? [
|
|
58
|
+
FFIType.ptr,
|
|
59
|
+
FFIType.i32,
|
|
60
|
+
...SHM_OPEN_STACK_PAD.map(() => FFIType.i32),
|
|
61
|
+
FFIType.u32,
|
|
62
|
+
]
|
|
63
|
+
: [FFIType.ptr, FFIType.i32, FFIType.u32],
|
|
57
64
|
returns: FFIType.i32,
|
|
58
65
|
},
|
|
59
66
|
shm_unlink: {
|
|
@@ -75,11 +82,9 @@ const getBunCreateSymbols = (platform = detectPosixPlatform()) => platform === "
|
|
|
75
82
|
returns: FFIType.i32,
|
|
76
83
|
},
|
|
77
84
|
};
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
return result;
|
|
82
|
-
};
|
|
85
|
+
const callShmOpen = (shmOpen, name, flags, mode, platform) => shmOpenModeIsStackPassed(platform)
|
|
86
|
+
? shmOpen(name, flags, ...SHM_OPEN_STACK_PAD, mode)
|
|
87
|
+
: shmOpen(name, flags, mode);
|
|
83
88
|
const isBunMmapFailed = (pointer) => !pointer || pointer < 0 || pointer >= Number.MAX_SAFE_INTEGER;
|
|
84
89
|
const createBunSharedMemoryFd = (name, platform, libc) => {
|
|
85
90
|
if (platform === "darwin") {
|
|
@@ -89,7 +94,7 @@ const createBunSharedMemoryFd = (name, platform, libc) => {
|
|
|
89
94
|
throw new Error("shm_open symbols are not available");
|
|
90
95
|
}
|
|
91
96
|
const shmName = encodeCString(makeDarwinSharedMemoryName(name, "bun"));
|
|
92
|
-
const fd =
|
|
97
|
+
const fd = checkPosixResult(callShmOpen(shmOpen, shmName, O_RDWR | DARWIN_O_CREAT | DARWIN_O_EXCL, DARWIN_SHM_MODE, platform), "shm_open failed");
|
|
93
98
|
shmUnlink(shmName);
|
|
94
99
|
try {
|
|
95
100
|
return setCloseOnExec(libc, fd);
|
|
@@ -103,7 +108,7 @@ const createBunSharedMemoryFd = (name, platform, libc) => {
|
|
|
103
108
|
if (memfdCreate === undefined) {
|
|
104
109
|
throw new Error("memfd_create symbol is not available");
|
|
105
110
|
}
|
|
106
|
-
const fd =
|
|
111
|
+
const fd = checkPosixResult(memfdCreate(encodeCString(name), 0), "memfd_create failed");
|
|
107
112
|
try {
|
|
108
113
|
return setCloseOnExec(libc, fd);
|
|
109
114
|
}
|
|
@@ -122,7 +127,12 @@ const createNamedBunSharedMemoryFd = (name, mode, platform, libc) => {
|
|
|
122
127
|
const flags = mode === "create"
|
|
123
128
|
? O_RDWR | createFlags | exclusiveFlags
|
|
124
129
|
: O_RDWR;
|
|
125
|
-
const
|
|
130
|
+
const shmMode = platform === "darwin" ? DARWIN_SHM_MODE : POSIX_SHM_MODE;
|
|
131
|
+
const shmName = encodeCString(toPosixSharedMemoryName(name));
|
|
132
|
+
const fd = checkPosixResult(callShmOpen(shmOpen, shmName, flags, shmMode, platform), "shm_open failed");
|
|
133
|
+
if (mode === "create") {
|
|
134
|
+
assertNamedSharedMemoryIsReopenable(platform, fd, () => callShmOpen(shmOpen, shmName, O_RDWR, shmMode, platform), libc.symbols.close);
|
|
135
|
+
}
|
|
126
136
|
try {
|
|
127
137
|
return setCloseOnExec(libc, fd);
|
|
128
138
|
}
|
|
@@ -139,7 +149,7 @@ export const mapBunSharedMemory = (options, libc = openBunLibc()) => {
|
|
|
139
149
|
fd = createNamedBunSharedMemoryFd(options.name, "open", detectPosixPlatform(), libc);
|
|
140
150
|
}
|
|
141
151
|
else if (options.duplicateFd !== false) {
|
|
142
|
-
fd =
|
|
152
|
+
fd = checkPosixResult(libc.symbols.dup(sourceFd), "dup(fd) failed");
|
|
143
153
|
try {
|
|
144
154
|
setCloseOnExec(libc, fd);
|
|
145
155
|
}
|
|
@@ -150,9 +160,10 @@ export const mapBunSharedMemory = (options, libc = openBunLibc()) => {
|
|
|
150
160
|
}
|
|
151
161
|
const pointer = libc.symbols.mmap(null, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0n);
|
|
152
162
|
if (isBunMmapFailed(pointer)) {
|
|
163
|
+
const error = posixError("mmap failed");
|
|
153
164
|
if (options.duplicateFd !== false)
|
|
154
165
|
libc.symbols.close(fd);
|
|
155
|
-
throw
|
|
166
|
+
throw error;
|
|
156
167
|
}
|
|
157
168
|
const arrayBuffer = getBunFFI().toArrayBuffer(pointer, 0, size);
|
|
158
169
|
return {
|
|
@@ -183,7 +194,7 @@ export const createBunSharedMemory = (options, libc = openBunLibc()) => {
|
|
|
183
194
|
: createNamedBunSharedMemoryFd(name, mode, platform, libc);
|
|
184
195
|
try {
|
|
185
196
|
if (mode !== "open") {
|
|
186
|
-
|
|
197
|
+
checkPosixResult(libc.symbols.ftruncate(fd, BigInt(size)), "ftruncate failed");
|
|
187
198
|
}
|
|
188
199
|
const mapping = mapBunSharedMemory({ fd, size, duplicateFd: false }, libc);
|
|
189
200
|
if (mode !== "anonymous") {
|
|
@@ -2,8 +2,10 @@ import { type CreateSharedMemoryOptions, type MapSharedMemoryOptions, type Share
|
|
|
2
2
|
type DenoLibc = {
|
|
3
3
|
symbols: {
|
|
4
4
|
memfd_create?: (name: Uint8Array, flags: number) => number;
|
|
5
|
-
shm_open?: (name: Uint8Array, flags: number,
|
|
5
|
+
shm_open?: (name: Uint8Array, flags: number, ...rest: number[]) => number;
|
|
6
6
|
shm_unlink?: (name: Uint8Array) => number;
|
|
7
|
+
__error?: () => unknown;
|
|
8
|
+
__errno_location?: () => unknown;
|
|
7
9
|
ftruncate: (fd: number, length: bigint) => number;
|
|
8
10
|
dup: (fd: number) => number;
|
|
9
11
|
fcntl: (fd: number, cmd: number, arg: number) => number;
|
package/src/connections/deno.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { DARWIN_O_CREAT, DARWIN_O_EXCL, DARWIN_SHM_MODE,
|
|
2
|
-
import { expectFd, expectPositiveSize, readCreateName,
|
|
1
|
+
import { assertNamedSharedMemoryIsReopenable, checkPosixResult, DARWIN_O_CREAT, DARWIN_O_EXCL, DARWIN_SHM_MODE, detectPosixPlatform, encodeCString, getErrnoSymbolName, getPosixLibcPath, LINUX_O_CREAT, LINUX_O_EXCL, makeDarwinSharedMemoryName, makeErrnoReader, MAP_SHARED, O_RDWR, POSIX_SHM_MODE, posixError, PROT_READ, PROT_WRITE, setCloseOnExec, SHM_OPEN_STACK_PAD, shmOpenModeIsStackPassed, toPosixSharedMemoryName, } from "./posix.js";
|
|
2
|
+
import { expectFd, expectPositiveSize, readCreateMode, readCreateName, readCreateSize, readRequiredCreateName, } from "./types.js";
|
|
3
3
|
import { createDenoWindowsConnectionPrimitives, isWindowsRuntime, } from "./windows.js";
|
|
4
4
|
const getDeno = () => {
|
|
5
5
|
const deno = globalThis.Deno;
|
|
@@ -8,8 +8,15 @@ const getDeno = () => {
|
|
|
8
8
|
}
|
|
9
9
|
return deno;
|
|
10
10
|
};
|
|
11
|
+
const getDenoErrnoSymbols = (platform = detectPosixPlatform()) => ({
|
|
12
|
+
[getErrnoSymbolName(platform)]: {
|
|
13
|
+
parameters: [],
|
|
14
|
+
result: "pointer",
|
|
15
|
+
},
|
|
16
|
+
});
|
|
11
17
|
export const openDenoLibc = () => getDeno().dlopen(getPosixLibcPath(), {
|
|
12
18
|
...getDenoCreateSymbols(),
|
|
19
|
+
...getDenoErrnoSymbols(),
|
|
13
20
|
ftruncate: {
|
|
14
21
|
parameters: ["i32", "i64"],
|
|
15
22
|
result: "i32",
|
|
@@ -38,7 +45,14 @@ export const openDenoLibc = () => getDeno().dlopen(getPosixLibcPath(), {
|
|
|
38
45
|
const getDenoCreateSymbols = (platform = detectPosixPlatform()) => platform === "darwin"
|
|
39
46
|
? {
|
|
40
47
|
shm_open: {
|
|
41
|
-
parameters:
|
|
48
|
+
parameters: shmOpenModeIsStackPassed(platform)
|
|
49
|
+
? [
|
|
50
|
+
"buffer",
|
|
51
|
+
"i32",
|
|
52
|
+
...SHM_OPEN_STACK_PAD.map(() => "i32"),
|
|
53
|
+
"u32",
|
|
54
|
+
]
|
|
55
|
+
: ["buffer", "i32", "u32"],
|
|
42
56
|
result: "i32",
|
|
43
57
|
},
|
|
44
58
|
shm_unlink: {
|
|
@@ -60,11 +74,12 @@ const getDenoCreateSymbols = (platform = detectPosixPlatform()) => platform ===
|
|
|
60
74
|
result: "i32",
|
|
61
75
|
},
|
|
62
76
|
};
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
77
|
+
const callShmOpen = (shmOpen, name, flags, mode, platform) => shmOpenModeIsStackPassed(platform)
|
|
78
|
+
? shmOpen(name, flags, ...SHM_OPEN_STACK_PAD, mode)
|
|
79
|
+
: shmOpen(name, flags, mode);
|
|
80
|
+
const errnoOf = (libc, platform = detectPosixPlatform()) => makeErrnoReader(platform === "darwin"
|
|
81
|
+
? libc.symbols.__error
|
|
82
|
+
: libc.symbols.__errno_location, (pointer) => new Int32Array(new (getDeno().UnsafePointerView)(pointer).getArrayBuffer(4))[0]);
|
|
68
83
|
const isDenoMmapFailed = (pointer) => {
|
|
69
84
|
if (pointer === null)
|
|
70
85
|
return true;
|
|
@@ -79,7 +94,7 @@ const createDenoSharedMemoryFd = (name, platform, libc) => {
|
|
|
79
94
|
throw new Error("shm_open symbols are not available");
|
|
80
95
|
}
|
|
81
96
|
const shmName = encodeCString(makeDarwinSharedMemoryName(name, "deno"));
|
|
82
|
-
const fd =
|
|
97
|
+
const fd = checkPosixResult(callShmOpen(shmOpen, shmName, O_RDWR | DARWIN_O_CREAT | DARWIN_O_EXCL, DARWIN_SHM_MODE, platform), "shm_open failed", errnoOf(libc, platform), platform);
|
|
83
98
|
shmUnlink(shmName);
|
|
84
99
|
try {
|
|
85
100
|
return setCloseOnExec(libc, fd);
|
|
@@ -93,7 +108,7 @@ const createDenoSharedMemoryFd = (name, platform, libc) => {
|
|
|
93
108
|
if (memfdCreate === undefined) {
|
|
94
109
|
throw new Error("memfd_create symbol is not available");
|
|
95
110
|
}
|
|
96
|
-
const fd =
|
|
111
|
+
const fd = checkPosixResult(memfdCreate(encodeCString(name), 0), "memfd_create failed", errnoOf(libc, platform), platform);
|
|
97
112
|
try {
|
|
98
113
|
return setCloseOnExec(libc, fd);
|
|
99
114
|
}
|
|
@@ -112,7 +127,12 @@ const createNamedDenoSharedMemoryFd = (name, mode, platform, libc) => {
|
|
|
112
127
|
const flags = mode === "create"
|
|
113
128
|
? O_RDWR | createFlags | exclusiveFlags
|
|
114
129
|
: O_RDWR;
|
|
115
|
-
const
|
|
130
|
+
const shmMode = platform === "darwin" ? DARWIN_SHM_MODE : POSIX_SHM_MODE;
|
|
131
|
+
const shmName = encodeCString(toPosixSharedMemoryName(name));
|
|
132
|
+
const fd = checkPosixResult(callShmOpen(shmOpen, shmName, flags, shmMode, platform), "shm_open failed", errnoOf(libc, platform), platform);
|
|
133
|
+
if (mode === "create") {
|
|
134
|
+
assertNamedSharedMemoryIsReopenable(platform, fd, () => callShmOpen(shmOpen, shmName, O_RDWR, shmMode, platform), libc.symbols.close, errnoOf(libc, platform));
|
|
135
|
+
}
|
|
116
136
|
try {
|
|
117
137
|
return setCloseOnExec(libc, fd);
|
|
118
138
|
}
|
|
@@ -129,7 +149,7 @@ export const mapDenoSharedMemory = (options, libc = openDenoLibc()) => {
|
|
|
129
149
|
fd = createNamedDenoSharedMemoryFd(options.name, "open", detectPosixPlatform(), libc);
|
|
130
150
|
}
|
|
131
151
|
else if (options.duplicateFd !== false) {
|
|
132
|
-
fd =
|
|
152
|
+
fd = checkPosixResult(libc.symbols.dup(sourceFd), "dup(fd) failed", errnoOf(libc));
|
|
133
153
|
try {
|
|
134
154
|
setCloseOnExec(libc, fd);
|
|
135
155
|
}
|
|
@@ -140,9 +160,10 @@ export const mapDenoSharedMemory = (options, libc = openDenoLibc()) => {
|
|
|
140
160
|
}
|
|
141
161
|
const pointer = libc.symbols.mmap(null, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0n);
|
|
142
162
|
if (isDenoMmapFailed(pointer)) {
|
|
163
|
+
const error = posixError("mmap failed", errnoOf(libc));
|
|
143
164
|
if (options.duplicateFd !== false)
|
|
144
165
|
libc.symbols.close(fd);
|
|
145
|
-
throw
|
|
166
|
+
throw error;
|
|
146
167
|
}
|
|
147
168
|
const arrayBuffer = new (getDeno().UnsafePointerView)(pointer)
|
|
148
169
|
.getArrayBuffer(size);
|
|
@@ -174,7 +195,7 @@ export const createDenoSharedMemory = (options, libc = openDenoLibc()) => {
|
|
|
174
195
|
: createNamedDenoSharedMemoryFd(name, mode, platform, libc);
|
|
175
196
|
try {
|
|
176
197
|
if (mode !== "open") {
|
|
177
|
-
|
|
198
|
+
checkPosixResult(libc.symbols.ftruncate(fd, BigInt(size)), "ftruncate failed", errnoOf(libc, platform), platform);
|
|
178
199
|
}
|
|
179
200
|
const mapping = mapDenoSharedMemory({ fd, size, duplicateFd: false }, libc);
|
|
180
201
|
if (mode !== "anonymous") {
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export const detachExternalArrayBuffer = (buffer) => {
|
|
2
|
+
if (buffer.byteLength === 0)
|
|
3
|
+
return true;
|
|
4
|
+
const detachable = buffer;
|
|
5
|
+
try {
|
|
6
|
+
if (typeof detachable.transferToFixedLength === "function") {
|
|
7
|
+
detachable.transferToFixedLength(0);
|
|
8
|
+
return buffer.byteLength === 0;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
}
|
|
13
|
+
try {
|
|
14
|
+
if (typeof detachable.transfer === "function") {
|
|
15
|
+
detachable.transfer(0);
|
|
16
|
+
return buffer.byteLength === 0;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
}
|
|
21
|
+
try {
|
|
22
|
+
structuredClone(buffer, { transfer: [buffer] });
|
|
23
|
+
return buffer.byteLength === 0;
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
export const requireDetachedExternalArrayBuffer = (buffer) => {
|
|
30
|
+
if (detachExternalArrayBuffer(buffer))
|
|
31
|
+
return;
|
|
32
|
+
throw new Error("knitting: could not detach external ArrayBuffer; refusing to unmap memory still reachable from JavaScript");
|
|
33
|
+
};
|
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
type NodeRequire = (specifier: string) => unknown;
|
|
2
2
|
export type NodeNativeAddonName = "knitting_shared_memory" | "knitting_shm" | "knitting_buffer_pointer";
|
|
3
|
+
export type NodePlatformInfo = {
|
|
4
|
+
arch: string;
|
|
5
|
+
modules?: string;
|
|
6
|
+
platform: string;
|
|
7
|
+
version?: string;
|
|
8
|
+
};
|
|
9
|
+
export declare const SUPPORTED_NODE_NATIVE_ADDON_ABIS: {
|
|
10
|
+
readonly "127": "22";
|
|
11
|
+
readonly "137": "24";
|
|
12
|
+
};
|
|
13
|
+
export declare const formatNodeNativeAddonLoadError: (name: NodeNativeAddonName, platformInfo: NodePlatformInfo | undefined, errors: readonly string[]) => string;
|
|
3
14
|
export declare const nodeNativeAddonSpecifiers: (name: NodeNativeAddonName) => readonly string[];
|
|
4
15
|
export declare const loadNodeNativeAddon: <T>(require: NodeRequire, name: NodeNativeAddonName, specifier?: string) => T;
|
|
5
16
|
export {};
|
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
import { resolveKnittingPackageAsset } from "./package-assets.js";
|
|
2
|
+
export const SUPPORTED_NODE_NATIVE_ADDON_ABIS = {
|
|
3
|
+
"127": "22",
|
|
4
|
+
"137": "24",
|
|
5
|
+
};
|
|
6
|
+
const NODE_26_MODULE_ABI = "147";
|
|
7
|
+
const SUPPORTED_NODE_NATIVE_TARGETS = new Set([
|
|
8
|
+
"darwin-arm64",
|
|
9
|
+
"darwin-x64",
|
|
10
|
+
"linux-x64",
|
|
11
|
+
"win32-x64",
|
|
12
|
+
]);
|
|
2
13
|
const readNodePlatformInfo = () => {
|
|
3
14
|
const processLike = globalThis.process;
|
|
4
15
|
if (typeof processLike?.platform !== "string" ||
|
|
@@ -9,8 +20,45 @@ const readNodePlatformInfo = () => {
|
|
|
9
20
|
arch: processLike.arch,
|
|
10
21
|
modules: processLike.versions?.modules,
|
|
11
22
|
platform: processLike.platform,
|
|
23
|
+
version: processLike.versions?.node,
|
|
12
24
|
};
|
|
13
25
|
};
|
|
26
|
+
const formatAttemptErrors = (errors) => errors.length > 0 ? `\nTried:\n${errors.join("\n")}` : "";
|
|
27
|
+
export const formatNodeNativeAddonLoadError = (name, platformInfo, errors) => {
|
|
28
|
+
const attempts = formatAttemptErrors(errors);
|
|
29
|
+
if (platformInfo === undefined) {
|
|
30
|
+
return `knitting: could not load native addon ${name}.${attempts}`;
|
|
31
|
+
}
|
|
32
|
+
const abi = platformInfo.modules;
|
|
33
|
+
const version = platformInfo.version === undefined
|
|
34
|
+
? "an unknown version"
|
|
35
|
+
: `v${platformInfo.version}`;
|
|
36
|
+
const target = `${platformInfo.platform}-${platformInfo.arch}`;
|
|
37
|
+
if (abi === NODE_26_MODULE_ABI) {
|
|
38
|
+
return (`knitting: Node.js 26 uses node:ffi instead of ABI-specific addons. ` +
|
|
39
|
+
`Restart Node with the --experimental-ffi flag` +
|
|
40
|
+
` (--allow-ffi is also required when using Node's Permission Model). ` +
|
|
41
|
+
`No Node ABI 147 addon is shipped.${attempts}`);
|
|
42
|
+
}
|
|
43
|
+
if (typeof abi === "string" &&
|
|
44
|
+
!(abi in SUPPORTED_NODE_NATIVE_ADDON_ABIS)) {
|
|
45
|
+
return (`knitting: no native addon is shipped for Node.js ${version} ` +
|
|
46
|
+
`(ABI ${abi}). Packaged native features support Node.js 22 ` +
|
|
47
|
+
`(ABI 127) and Node.js 24 (ABI 137) only. Plain thread workers that do ` +
|
|
48
|
+
`not use ProcessSharedBuffer or BufferReference do not need these ` +
|
|
49
|
+
`addons.${attempts}`);
|
|
50
|
+
}
|
|
51
|
+
if (!SUPPORTED_NODE_NATIVE_TARGETS.has(target)) {
|
|
52
|
+
return (`knitting: Node.js ${version} uses a supported ABI (${abi ?? "unknown"}), ` +
|
|
53
|
+
`but no packaged native addon target is shipped for ${target}. ` +
|
|
54
|
+
`Published targets are linux-x64, darwin-x64, darwin-arm64, and ` +
|
|
55
|
+
`win32-x64. You can build locally with "bun run build:native".${attempts}`);
|
|
56
|
+
}
|
|
57
|
+
return (`knitting: could not load native addon ${name} for Node.js ${version} ` +
|
|
58
|
+
`(ABI ${abi ?? "unknown"}, ${target}). The prebuild may be missing, ` +
|
|
59
|
+
`damaged, or incompatible; reinstall the package or build locally with ` +
|
|
60
|
+
`"bun run build:native".${attempts}`);
|
|
61
|
+
};
|
|
14
62
|
export const nodeNativeAddonSpecifiers = (name) => {
|
|
15
63
|
const platformInfo = readNodePlatformInfo();
|
|
16
64
|
const fallback = resolveKnittingPackageAsset("build", "Release", `${name}.node`);
|
|
@@ -40,5 +88,5 @@ export const loadNodeNativeAddon = (require, name, specifier) => {
|
|
|
40
88
|
errors.push(`${candidate}: ${String(error)}`);
|
|
41
89
|
}
|
|
42
90
|
}
|
|
43
|
-
throw new Error(
|
|
91
|
+
throw new Error(formatNodeNativeAddonLoadError(name, readNodePlatformInfo(), errors));
|
|
44
92
|
};
|