knitting 0.1.63 → 0.1.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +525 -335
  2. package/knitting.browser.js +1 -1
  3. package/map.md +0 -6
  4. package/package.json +3 -3
  5. package/prebuilds/darwin-arm64-node-127/knitting_buffer_pointer.node +0 -0
  6. package/prebuilds/darwin-arm64-node-127/knitting_doorbell.node +0 -0
  7. package/prebuilds/darwin-arm64-node-137/knitting_buffer_pointer.node +0 -0
  8. package/prebuilds/darwin-arm64-node-137/knitting_doorbell.node +0 -0
  9. package/prebuilds/darwin-x64-node-127/knitting_buffer_pointer.node +0 -0
  10. package/prebuilds/darwin-x64-node-127/knitting_doorbell.node +0 -0
  11. package/prebuilds/darwin-x64-node-137/knitting_buffer_pointer.node +0 -0
  12. package/prebuilds/darwin-x64-node-137/knitting_doorbell.node +0 -0
  13. package/prebuilds/linux-x64-node-127/knitting_buffer_pointer.node +0 -0
  14. package/prebuilds/linux-x64-node-127/knitting_doorbell.node +0 -0
  15. package/prebuilds/linux-x64-node-137/knitting_buffer_pointer.node +0 -0
  16. package/prebuilds/linux-x64-node-137/knitting_doorbell.node +0 -0
  17. package/prebuilds/win32-x64/knitting_windows_shared_memory.dll +0 -0
  18. package/prebuilds/win32-x64-node-127/knitting_buffer_pointer.node +0 -0
  19. package/prebuilds/win32-x64-node-127/knitting_doorbell.node +0 -0
  20. package/prebuilds/win32-x64-node-127/knitting_shared_memory.node +0 -0
  21. package/prebuilds/win32-x64-node-127/knitting_shm.node +0 -0
  22. package/prebuilds/win32-x64-node-137/knitting_buffer_pointer.node +0 -0
  23. package/prebuilds/win32-x64-node-137/knitting_doorbell.node +0 -0
  24. package/prebuilds/win32-x64-node-137/knitting_shared_memory.node +0 -0
  25. package/prebuilds/win32-x64-node-137/knitting_shm.node +0 -0
  26. package/scripts/build-native-addons.ts +5 -0
  27. package/shared-memory.d.ts +3 -0
  28. package/shared-memory.js +3 -0
  29. package/src/api.js +105 -42
  30. package/src/common/with-resolvers.js +2 -5
  31. package/src/common/worker-runtime.d.ts +7 -0
  32. package/src/common/worker-runtime.js +7 -0
  33. package/src/connections/buffer-reference.d.ts +10 -36
  34. package/src/connections/buffer-reference.js +15 -170
  35. package/src/connections/node-addons.d.ts +1 -1
  36. package/src/connections/shared-array-buffer-payload.d.ts +7 -0
  37. package/src/connections/shared-array-buffer-payload.js +27 -11
  38. package/src/knitting_buffer_pointer.cc +57 -2
  39. package/src/knitting_doorbell.cc +220 -0
  40. package/src/memory/knitting-body.d.ts +44 -0
  41. package/src/memory/knitting-body.js +51 -0
  42. package/src/memory/knitting-buffer-http.d.ts +116 -0
  43. package/src/memory/knitting-buffer-http.js +255 -0
  44. package/src/memory/knitting-buffer.d.ts +250 -0
  45. package/src/memory/knitting-buffer.js +695 -0
  46. package/src/memory/lazy-region-registry.d.ts +83 -0
  47. package/src/memory/lazy-region-registry.js +355 -0
  48. package/src/memory/lock.d.ts +38 -15
  49. package/src/memory/lock.js +205 -79
  50. package/src/memory/payloadCodec.d.ts +18 -2
  51. package/src/memory/payloadCodec.js +309 -65
  52. package/src/memory/regionRegistry.d.ts +6 -0
  53. package/src/memory/regionRegistry.js +125 -240
  54. package/src/memory/shared-buffer-io.d.ts +7 -0
  55. package/src/memory/shared-buffer-io.js +34 -8
  56. package/src/runtime/deno-doorbell.d.ts +26 -0
  57. package/src/runtime/deno-doorbell.js +117 -0
  58. package/src/runtime/dispatcher.d.ts +8 -6
  59. package/src/runtime/dispatcher.js +80 -58
  60. package/src/runtime/host-arg-arena.d.ts +3 -0
  61. package/src/runtime/host-arg-arena.js +16 -0
  62. package/src/runtime/node-doorbell.d.ts +14 -0
  63. package/src/runtime/node-doorbell.js +84 -0
  64. package/src/runtime/pool.d.ts +21 -15
  65. package/src/runtime/pool.js +104 -116
  66. package/src/runtime/process-worker.d.ts +9 -0
  67. package/src/runtime/process-worker.js +22 -2
  68. package/src/runtime/tx-queue.d.ts +2 -5
  69. package/src/runtime/tx-queue.js +52 -48
  70. package/src/types.d.ts +35 -71
  71. package/src/worker/loop.js +79 -57
  72. package/src/worker/rx-queue.d.ts +2 -3
  73. package/src/worker/rx-queue.js +34 -40
  74. package/src/worker/shared-return.d.ts +9 -0
  75. package/src/worker/shared-return.js +22 -0
  76. package/src/worker/task-loader.js +1 -2
  77. package/src/worker/timers.d.ts +2 -6
  78. package/src/worker/timers.js +14 -19
  79. package/unsafe.d.ts +2 -1
  80. package/unsafe.js +2 -1
@@ -1,20 +1,19 @@
1
1
  import { createHostTxQueue } from "./tx-queue.js";
2
- import { cleanupProcessWorkerMemoryQuietly, createProcessSharedMemoryAllocator, createProcessStealMemoryLayout, createProcessWorkerMemoryLayout, createProcessWorkerNativeSignalNotifier, readProcessSharedMemorySettings, readProcessWorkerCommandPrefix, readProcessWorkerRuntime, spawnProcessWorker, toProcessWorkerBootPayload, } from "./process-worker.js";
2
+ import { cleanupProcessWorkerMemoryQuietly, createProcessSharedMemoryAllocator, createProcessStealMemoryLayout, createProcessWorkerMemoryLayout, createProcessWorkerNativeSignalNotifier, processWorkerUsesIpc, readProcessSharedMemorySettings, readProcessWorkerCommandPrefix, readProcessWorkerRuntime, spawnProcessWorker, toProcessWorkerBootPayload, } from "./process-worker.js";
3
3
  import { serializeWorkerBootstrapData, terminateWorkerQuietly, toWorkerCompatExecArgv, toWorkerSafeExecArgv, } from "./worker-common.js";
4
4
  import { createSharedMemoryTransport, TRANSPORT_SIGNAL_BYTES, WORKER_STOP, } from "../ipc/transport/shared-memory.js";
5
5
  import { ChannelHandler, hostDispatcherLoop, } from "./dispatcher.js";
6
+ import { createNodeCompletionDoorbell } from "./node-doorbell.js";
6
7
  import { HEADER_SLOT_STRIDE_U32, lock2, LOCK_SECTOR_BYTE_LENGTH, LockBound, } from "../memory/lock.js";
7
- // Side-effect import: registers the payload codec (cycle break for Andromeda;
8
- // see lock.ts). Must run before any lock2() call.
8
+ // Registers the payload codec before any lock is built.
9
9
  import "../memory/payloadCodec.js";
10
10
  import "../worker/loop.js";
11
11
  import { createSharedArrayBuffer, createWasmSharedArrayBuffer, IS_BROWSER, RUNTIME, } from "../common/runtime.js";
12
- import { HAS_NODE_WORKER_THREADS, RUNTIME_WORKER, } from "../common/worker-runtime.js";
12
+ import { HAS_NODE_WORKER_THREADS, isProcessCompletionDoorbell, RUNTIME_WORKER, } from "../common/worker-runtime.js";
13
13
  import { probeLockBufferTextCompat } from "../common/shared-buffer-text.js";
14
14
  import { signalAbortFactory } from "../shared/abortSignal.js";
15
15
  import { createLockControlCarpet } from "../memory/byte-carpet.js";
16
16
  import { resolvePayloadBufferOptions, } from "../memory/payload-config.js";
17
- import { createBufferReferenceReturnReleaseMessage, detachArrayBufferBestEffort, } from "../connections/buffer-reference.js";
18
17
  import { spawnCompiledWorkerContext } from "./compiled-worker.js";
19
18
  const WORKER_FATAL_MESSAGE_KEY = "__knittingWorkerFatal";
20
19
  const isWorkerFatalMessage = (value) => !!value &&
@@ -22,7 +21,10 @@ const isWorkerFatalMessage = (value) => !!value &&
22
21
  typeof value[WORKER_FATAL_MESSAGE_KEY] === "string";
23
22
  // Bound the wait for a worker to acknowledge loop exit.
24
23
  const WORKER_STOP_ACK_TIMEOUT_MS = 50;
25
- const WORKER_STOP_NEEDS_ACK = RUNTIME === "deno";
24
+ // Node owns returned BufferReference backing stores in a process-global native
25
+ // registry. Give its worker loop a chance to drain deferred producer pins
26
+ // before the host falls back to terminate().
27
+ const WORKER_STOP_NEEDS_ACK = RUNTIME === "deno" || RUNTIME === "node";
26
28
  // Keep idle workers self-healing if an Atomics.notify wake is missed.
27
29
  const DEFAULT_WORKER_PARK_MS = 1;
28
30
  const SINGLE_WORKER_SPIN_US = 50;
@@ -57,16 +59,7 @@ const withFixedPayloadConfig = (config) => ({
57
59
  mode: "fixed",
58
60
  payloadInitialBytes: config.payloadMaxByteLength,
59
61
  });
60
- /**
61
- * Build the buffers, host-side locks and pending registry that a stealing pool
62
- * shares. One submit region for every worker to claim from, one private return
63
- * region per worker, and a single pool-global queue so a response arriving on
64
- * any lane settles the right promise.
65
- *
66
- * Region width follows paper §6.1: the largest power-of-two `g` leaving a spare
67
- * region for a delayed claimant (`slots / g >= consumers + 1`). At 32 slots that
68
- * caps how wide a region can be well before it caps the worker count.
69
- */
62
+ /** Build the shared submit and private return buffers for a stealing pool. */
70
63
  export const resolveStealRegionLanes = (consumers) => {
71
64
  for (let lanes = LockBound.slots; lanes >= 1; lanes >>= 1) {
72
65
  if (LockBound.slots / lanes >= consumers + 1)
@@ -74,9 +67,13 @@ export const resolveStealRegionLanes = (consumers) => {
74
67
  }
75
68
  throw new RangeError(`${consumers} stealing workers need more than ${LockBound.slots} lanes`);
76
69
  };
70
+ /** Return the widest region allowed by the selected claim discipline. */
71
+ export const resolveMaxStealRegionLanes = (consumers, stealClaim) => stealClaim !== undefined && stealClaim !== "dekker"
72
+ ? LockBound.slots
73
+ : resolveStealRegionLanes(consumers);
77
74
  /** Maximum claimants that leave the protocol's required spare region. */
78
75
  export const MAX_STEAL_CONSUMERS = LockBound.slots - 1;
79
- export const createStealPoolBuffers = ({ threads, payload, regionLanes, abortSignalCapacity, usesAbortSignal, processWorker, }) => {
76
+ export const createStealPoolBuffers = ({ threads, payload, sharedArgs, regionLanes, stealClaim, abortSignalCapacity, usesAbortSignal, processWorker, }) => {
80
77
  const basePayloadConfig = resolvePayloadBufferOptions({ options: payload });
81
78
  const payloadConfig = processWorker === undefined
82
79
  ? basePayloadConfig
@@ -100,9 +97,10 @@ export const createStealPoolBuffers = ({ threads, payload, regionLanes, abortSig
100
97
  payload: payloadSab,
101
98
  }),
102
99
  });
103
- const maxLanes = resolveStealRegionLanes(threads);
100
+ // Keep the default width valid for Dekker; wider CAS-mask regions are explicit.
101
+ const maxLanes = resolveMaxStealRegionLanes(threads, stealClaim);
104
102
  const lanes = regionLanes === undefined
105
- ? maxLanes
103
+ ? resolveStealRegionLanes(threads)
106
104
  : Math.min(Math.max(1, regionLanes | 0), maxLanes);
107
105
  const resolvedAbortSignalCapacity = resolveAbortSignalCapacity(abortSignalCapacity);
108
106
  const processMemory = processWorker === undefined ? undefined : createProcessStealMemoryLayout({
@@ -133,7 +131,11 @@ export const createStealPoolBuffers = ({ threads, payload, regionLanes, abortSig
133
131
  textCompat: submitBuffers.textCompat,
134
132
  consumers: threads,
135
133
  regionLanes: lanes,
134
+ stealClaim,
136
135
  processBoundary: processMemory !== undefined,
136
+ // Shared arguments are opt-in because the receiving task may outlive the
137
+ // borrowed region.
138
+ sharedReturn: sharedArgs === true,
137
139
  });
138
140
  const returnBuffers = [];
139
141
  const hostReturnLocks = [];
@@ -179,13 +181,14 @@ export const createStealPoolBuffers = ({ threads, payload, regionLanes, abortSig
179
181
  hostReturnLocks,
180
182
  sharedQueue,
181
183
  regionLanes: lanes,
184
+ stealClaim,
182
185
  processMemory,
183
186
  abortSignalSAB,
184
187
  abortSignalMax: processMemory?.abortSignalMax ??
185
188
  (abortSignalSAB === undefined ? undefined : resolvedAbortSignalCapacity),
186
189
  };
187
190
  };
188
- export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostDebug, totalNumberOfThread, workerCount, source, at, workerOptions, workerExecArgv, permission, host, payload, bufferReferenceReturn, abortSignalCapacity, usesAbortSignal, sharedChannelHandler, stealPool, }) => {
191
+ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostDebug, totalNumberOfThread, workerCount, source, at, workerOptions, workerExecArgv, permission, host, payload, sharedBytesEnabled, abortSignalCapacity, usesAbortSignal, sharedChannelHandler, denoCompletionDoorbell, stealPool, }) => {
189
192
  if (workerOptions?.runtime === "compiled") {
190
193
  return spawnCompiledWorkerContext({
191
194
  list,
@@ -200,12 +203,20 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
200
203
  const poliWorker = RUNTIME_WORKER;
201
204
  const resolvedWorkerOptions = serializeWorkerBootstrapData(withDefaultWorkerTimers(workerOptions, workerCount ?? totalNumberOfThread));
202
205
  const useProcessWorkerRuntime = resolvedWorkerOptions.runtime === "process";
206
+ const canUseDenoCompletionDoorbell = !useProcessWorkerRuntime &&
207
+ denoCompletionDoorbell !== undefined;
203
208
  const processWorkerRuntime = useProcessWorkerRuntime
204
209
  ? readProcessWorkerRuntime(resolvedWorkerOptions)
205
210
  : undefined;
206
211
  const processWorkerCommandPrefix = useProcessWorkerRuntime
207
212
  ? readProcessWorkerCommandPrefix(resolvedWorkerOptions)
208
213
  : undefined;
214
+ const canUseProcessCompletionDoorbell = useProcessWorkerRuntime &&
215
+ host?.doorbell !== false &&
216
+ processWorkerUsesIpc({
217
+ processRuntime: processWorkerRuntime,
218
+ commandPrefix: processWorkerCommandPrefix,
219
+ });
209
220
  const processSharedMemorySettings = useProcessWorkerRuntime
210
221
  ? readProcessSharedMemorySettings(resolvedWorkerOptions)
211
222
  : undefined;
@@ -238,10 +249,7 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
238
249
  const abortBytes = stealPool === undefined && usesAbortSignal === true
239
250
  ? abortSignalByteLength(resolvedAbortSignalCapacity)
240
251
  : 0;
241
- // A stealing process pool carves every lane out of one mapping, so lane
242
- // `thread` must find its slice there. Falling back to a private layout would
243
- // silently give this lane its own submit region — it would boot and then
244
- // never see a task the host published to the shared one.
252
+ // Stealing process workers use their slice of the pool-wide mapping.
245
253
  const stealProcessMemory = stealPool?.processMemory;
246
254
  const processWorkerMemory = !useProcessWorkerRuntime
247
255
  ? undefined
@@ -264,18 +272,14 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
264
272
  createWasmSharedArrayBuffer;
265
273
  const createPayloadBuffer = processSharedMemory?.createBuffer;
266
274
  const makePayloadBuffer = () => createPayloadBuffer
267
- // ProcessSharedBuffer is fixed-size today, so reserve the configured
268
- // payload ceiling instead of relying on SAB growth.
275
+ // Process-shared buffers are fixed-size.
269
276
  ? createPayloadBuffer(resolvedPayloadConfig.payloadMaxByteLength)
270
277
  : resolvedPayloadConfig.mode === "growable"
271
278
  ? createSharedArrayBuffer(resolvedPayloadConfig.payloadInitialBytes, resolvedPayloadConfig.payloadMaxByteLength)
272
279
  : createSharedArrayBuffer(resolvedPayloadConfig.payloadInitialBytes);
273
280
  const makeLockControlLayout = () => {
274
- // Keep the hottest control words in one compact front strip:
275
- // transport signals -> request lock -> return lock.
276
- // Request/return headers stay in separate contiguous slabs to preserve
277
- // sequential batching locality.
278
- // Abort bitmap stays at the tail.
281
+ // Place hot control words first; keep headers contiguous and the abort bitmap
282
+ // at the tail.
279
283
  return createLockControlCarpet({
280
284
  signalBytes,
281
285
  abortBytes,
@@ -350,68 +354,11 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
350
354
  processRuntime: processWorkerRuntime,
351
355
  signal: signalBox.opView,
352
356
  });
353
- const outstandingBorrowedReturns = bufferReferenceReturn === "borrow" && typeof WeakRef === "function"
354
- ? new Map()
355
- : undefined;
356
- const releaseBorrowedReturnToken = (token) => {
357
- outstandingBorrowedReturns?.delete(token);
358
- worker?.postMessage?.(createBufferReferenceReturnReleaseMessage(token));
359
- };
360
- const revokeOutstandingBorrowedReturns = (copyBytes) => {
361
- if (outstandingBorrowedReturns === undefined)
362
- return;
363
- for (const [token, record] of outstandingBorrowedReturns) {
364
- try {
365
- const ref = record.ref.deref();
366
- if (ref !== undefined) {
367
- ref.revokeBorrow(copyBytes);
368
- continue;
369
- }
370
- const aliasBuffer = record.aliasBuffer?.deref();
371
- if (aliasBuffer !== undefined &&
372
- detachArrayBufferBestEffort(record.runtime, aliasBuffer) &&
373
- copyBytes) {
374
- releaseBorrowedReturnToken(token);
375
- }
376
- }
377
- catch {
378
- // best effort
379
- }
380
- }
381
- outstandingBorrowedReturns.clear();
382
- };
383
- const returnHooks = bufferReferenceReturn === "borrow"
384
- ? {
385
- release: releaseBorrowedReturnToken,
386
- track: (ref, token, aliasBuffer) => {
387
- if (outstandingBorrowedReturns === undefined)
388
- return;
389
- const record = outstandingBorrowedReturns.get(token);
390
- if (record !== undefined) {
391
- if (aliasBuffer !== undefined) {
392
- record.aliasBuffer = new WeakRef(aliasBuffer);
393
- }
394
- return;
395
- }
396
- outstandingBorrowedReturns.set(token, {
397
- ref: new WeakRef(ref),
398
- aliasBuffer: aliasBuffer === undefined
399
- ? undefined
400
- : new WeakRef(aliasBuffer),
401
- runtime: ref.runtime,
402
- });
403
- },
404
- }
405
- : undefined;
406
357
  const queue = stealPool?.sharedQueue ?? createHostTxQueue({
407
358
  lock,
408
359
  returnLock,
409
360
  abortSignals,
410
- releaseBufferReferenceReturn: returnHooks,
411
361
  });
412
- if (stealPool !== undefined) {
413
- stealPool.sharedQueue.setReturnHooks(stealPool.consumerId, returnHooks);
414
- }
415
362
  const { enqueue, rejectAll, txIdle, } = queue;
416
363
  const thisSignal = signalBox.opView;
417
364
  const a_add = Atomics.add;
@@ -420,8 +367,7 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
420
367
  const canNotifySignal = thisSignal.buffer instanceof SharedArrayBuffer;
421
368
  const notifySignal = nativeNotifySignal ??
422
369
  (canNotifySignal ? (() => a_notify(thisSignal, 0, 1)) : undefined);
423
- // Wakes this lane's worker when it is parked (rxStatus 0). Used by send()
424
- // (new work just arrived) — the dispatcher drain wakes it the same way.
370
+ // Wake this lane when its worker is parked.
425
371
  const laneWake = () => {
426
372
  if (a_load(signalBox.rxStatus, 0) === 0) {
427
373
  a_add(thisSignal, 0, 1);
@@ -434,10 +380,16 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
434
380
  const ownsChannel = sharedChannelHandler === undefined &&
435
381
  stealPool === undefined;
436
382
  const ownChannel = sharedChannelHandler ?? new ChannelHandler();
437
- // Under stealing there is one queue, so one dispatcher drives it from the
438
- // pool; a per-lane dispatcher would double-drain the shared registry.
439
- const { check: dispatcherCheck } = stealPool !== undefined
440
- ? { check: undefined }
383
+ // Create the Node doorbell before bootstrapping the worker.
384
+ let nodeCompletionWake;
385
+ const nodeCompletionDoorbell = !useProcessWorkerRuntime &&
386
+ host?.doorbell !== false && host?.nativeDoorbell === true
387
+ ? createNodeCompletionDoorbell(() => nodeCompletionWake?.())
388
+ : undefined;
389
+ const canUseNodeCompletionDoorbell = nodeCompletionDoorbell !== undefined;
390
+ // A stealing pool has one dispatcher for its shared queue.
391
+ const { check: dispatcherCheck, wakeCompletion: directCompletionWake, } = stealPool !== undefined
392
+ ? { check: undefined, wakeCompletion: undefined }
441
393
  : hostDispatcherLoop({
442
394
  signalBox,
443
395
  queue,
@@ -445,7 +397,15 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
445
397
  dispatcherOptions: host,
446
398
  notifySignal: nativeNotifySignal,
447
399
  crossProcess: useProcessWorkerRuntime,
400
+ nativeCompletionDoorbell: canUseDenoCompletionDoorbell ||
401
+ canUseNodeCompletionDoorbell,
402
+ processCompletionDoorbell: canUseProcessCompletionDoorbell,
448
403
  });
404
+ if (canUseDenoCompletionDoorbell && dispatcherCheck !== undefined) {
405
+ denoCompletionDoorbell.listen(thread, directCompletionWake);
406
+ }
407
+ nodeCompletionWake = directCompletionWake;
408
+ let processCompletionWake = directCompletionWake;
449
409
  if (ownsChannel && dispatcherCheck !== undefined) {
450
410
  ownChannel.open(dispatcherCheck);
451
411
  channelHandler = ownChannel;
@@ -453,7 +413,7 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
453
413
  if (dispatcherCheck.isRunning === true)
454
414
  return;
455
415
  dispatcherCheck.isRunning = true;
456
- Promise.resolve().then(dispatcherCheck);
416
+ queueMicrotask(dispatcherCheck);
457
417
  laneWake();
458
418
  };
459
419
  }
@@ -484,18 +444,27 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
484
444
  lock: lockBuffers,
485
445
  returnLock: returnLockBuffers,
486
446
  payloadConfig: resolvedPayloadConfig,
487
- bufferReferenceReturn,
447
+ sharedReturn: sharedBytesEnabled === true,
488
448
  permission,
489
- // Mirrors the dispatcher's `canUseDoorbell`: a cross-process worker cannot
490
- // ring the host's waiter, so do not make it pay the atomic op.
491
- notifyOnHostPublish: !useProcessWorkerRuntime &&
492
- host?.doorbell !== false &&
493
- (RUNTIME === "bun" || RUNTIME === "node") &&
494
- typeof Atomics.waitAsync === "function",
449
+ // Select the completion doorbell supported by the worker runtime.
450
+ notifyOnHostPublish: host?.doorbell !== false &&
451
+ (canUseProcessCompletionDoorbell ||
452
+ canUseNodeCompletionDoorbell ||
453
+ (!useProcessWorkerRuntime &&
454
+ (RUNTIME === "bun" || RUNTIME === "node") &&
455
+ typeof Atomics.waitAsync === "function")),
456
+ processCompletionDoorbell: canUseProcessCompletionDoorbell,
457
+ denoCompletionDoorbell: canUseDenoCompletionDoorbell
458
+ ? denoCompletionDoorbell.pointer
459
+ : undefined,
460
+ nodeCompletionDoorbell: canUseNodeCompletionDoorbell
461
+ ? nodeCompletionDoorbell.pointer
462
+ : undefined,
495
463
  steal: stealPool === undefined ? undefined : {
496
464
  consumers: stealPool.consumers,
497
465
  consumerId: stealPool.consumerId,
498
466
  regionLanes: stealPool.regionLanes,
467
+ claim: stealPool.stealClaim,
499
468
  },
500
469
  };
501
470
  const baseWorkerOptions = {
@@ -565,6 +534,11 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
565
534
  worker.postMessage?.(workerDataPayload);
566
535
  }
567
536
  let closedReason;
537
+ // A worker that exits because the host asked it to stop is a closed thread,
538
+ // not a crash. The stop is acknowledged in shared memory before the worker
539
+ // unwinds, so on a slow host the `exit` event can land ahead of `kills()`
540
+ // and label an orderly shutdown "Worker exited with code 0".
541
+ let stopRequested = false;
568
542
  const deactivateStealConsumer = () => {
569
543
  stealPool?.hostSubmitLock.deactivateStealConsumer(stealPool.consumerId);
570
544
  };
@@ -579,21 +553,21 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
579
553
  deactivateStealConsumer();
580
554
  }
581
555
  };
582
- // `workerBytesReadable`: while the worker is still alive its pinned bytes
583
- // can be copied out; after a crash/exit they are gone and borrows are
584
- // detached so stale reads fail loud instead of aliasing freed memory.
585
- const markWorkerClosed = (reason, workerBytesReadable = false) => {
556
+ const markWorkerClosed = (reason) => {
586
557
  if (closedReason)
587
558
  return;
588
559
  closedReason = reason;
589
- revokeOutstandingBorrowedReturns(workerBytesReadable);
590
560
  rejectAll(reason);
591
561
  channelHandler?.close();
592
562
  };
593
563
  const onWorkerMessage = (message) => {
564
+ if (isProcessCompletionDoorbell(message)) {
565
+ processCompletionWake?.();
566
+ return;
567
+ }
594
568
  if (!isWorkerFatalMessage(message))
595
569
  return;
596
- markWorkerClosed(`Worker startup failed: ${message[WORKER_FATAL_MESSAGE_KEY]}`, true);
570
+ markWorkerClosed(`Worker startup failed: ${message[WORKER_FATAL_MESSAGE_KEY]}`);
597
571
  terminateFailedWorker();
598
572
  };
599
573
  const onWorkerError = (error) => {
@@ -606,11 +580,14 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
606
580
  nodeWorker.on("message", onWorkerMessage);
607
581
  nodeWorker.on("error", onWorkerError);
608
582
  nodeWorker.on("exit", (code) => {
609
- // Exit is the point at which this endpoint can no longer write WANT, so
610
- // survivors may safely ignore any intent it left behind.
611
583
  deactivateStealConsumer();
584
+ nodeCompletionDoorbell?.close();
612
585
  if (closedReason !== undefined)
613
586
  return;
587
+ if (stopRequested) {
588
+ markWorkerClosed("Thread closed");
589
+ return;
590
+ }
614
591
  const normalized = typeof code === "number" ? code : -1;
615
592
  markWorkerClosed(`Worker exited with code ${normalized}`);
616
593
  });
@@ -646,34 +623,35 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
646
623
  return pending;
647
624
  };
648
625
  };
649
- // Ask the worker to leave its loop before termination.
650
626
  const requestWorkerStop = async () => {
627
+ stopRequested = true;
651
628
  const stopView = signalBox.stopView;
652
629
  if (stopView === undefined)
653
- return;
630
+ return true;
654
631
  try {
655
632
  Atomics.store(stopView, 0, WORKER_STOP.requested);
656
633
  laneWake();
657
634
  notifySignal?.();
658
635
  }
659
636
  catch {
660
- return;
637
+ return false;
661
638
  }
662
639
  if (!WORKER_STOP_NEEDS_ACK)
663
- return;
640
+ return true;
664
641
  const deadline = Date.now() + WORKER_STOP_ACK_TIMEOUT_MS;
665
642
  while (Atomics.load(stopView, 0) !== WORKER_STOP.acknowledged) {
666
643
  if (Date.now() >= deadline)
667
- return;
644
+ return false;
668
645
  await new Promise((resolve) => setTimeout(resolve, 1));
669
646
  }
647
+ return true;
670
648
  };
671
649
  const context = {
672
650
  txIdle,
673
651
  call,
674
652
  requestStop: requestWorkerStop,
675
653
  kills: async () => {
676
- markWorkerClosed("Thread closed", true);
654
+ markWorkerClosed("Thread closed");
677
655
  // Process workers must exit before their shared memory is unmapped.
678
656
  const awaitExit = processWorkerMemory !== undefined;
679
657
  const termination = terminateWorkerQuietly(worker, awaitExit);
@@ -690,6 +668,16 @@ export const spawnWorkerContext = ({ list, ids, names, sab, thread, debug, hostD
690
668
  bindSend: sharedChannelHandler !== undefined || stealPool !== undefined
691
669
  ? ((fn) => void (dispatchSend = fn))
692
670
  : undefined,
671
+ bindCompletionWake: (canUseProcessCompletionDoorbell ||
672
+ canUseNodeCompletionDoorbell) &&
673
+ stealPool !== undefined
674
+ ? ((fn) => {
675
+ processCompletionWake = fn;
676
+ nodeCompletionWake = fn;
677
+ })
678
+ : undefined,
679
+ processCompletionDoorbell: canUseProcessCompletionDoorbell,
680
+ nodeCompletionDoorbell: canUseNodeCompletionDoorbell,
693
681
  };
694
682
  return context;
695
683
  };
@@ -92,6 +92,15 @@ export declare const createProcessWorkerNativeSignalNotifier: ({ processRuntime,
92
92
  processRuntime: ProcessWorkerRuntime | undefined;
93
93
  signal: Int32Array;
94
94
  }) => (() => void) | undefined;
95
+ /**
96
+ * Whether this host/child pair has the Node-compatible IPC channel that the
97
+ * process completion-doorbell prototype uses. Deno process workers currently
98
+ * boot through environment data and do not have that channel.
99
+ */
100
+ export declare const processWorkerUsesIpc: ({ processRuntime, commandPrefix, }: {
101
+ processRuntime: ProcessWorkerRuntime | undefined;
102
+ commandPrefix?: ProcessWorkerCommandPrefix;
103
+ }) => boolean;
95
104
  export declare const spawnProcessWorker: (options: {
96
105
  workerUrl: string | URL;
97
106
  bootPayload: ProcessWorkerBootPayload;
@@ -109,6 +109,13 @@ const isWindowsRuntimeHost = () => {
109
109
  return denoOs === "windows";
110
110
  return globalThis.process?.platform === "win32";
111
111
  };
112
+ const processWorkerNeedsNamedMemory = (sharedMemory) =>
113
+ // `Deno.Command` does not reliably carry an anonymous memfd/shm fd into a
114
+ // child as stdin: Linux can hand the child a non-mappable descriptor and
115
+ // macOS cannot always reopen the host fd through `/dev/fd`. Named mappings
116
+ // are reopened by the worker instead, which works across Deno's supported
117
+ // hosts and process-worker runtimes.
118
+ sharedMemory.mode === "named" || isWindowsRuntimeHost() || RUNTIME === "deno";
112
119
  let processWorkerMemoryNameCounter = 0;
113
120
  const makeProcessWorkerMemoryName = (thread, prefix = "kpw") => {
114
121
  const processId = globalThis.process?.pid ??
@@ -142,7 +149,7 @@ export const createProcessWorkerMemoryLayout = ({ signalBytes, abortBytes, paylo
142
149
  const requestPayloadSlice = carpet.take("requestPayload", payloadBytes);
143
150
  const returnPayloadSlice = carpet.take("returnPayload", payloadBytes);
144
151
  const primitives = getProcessWorkerSharedMemoryPrimitives();
145
- const forceNamed = sharedMemory.mode === "named" || isWindowsRuntimeHost();
152
+ const forceNamed = processWorkerNeedsNamedMemory(sharedMemory);
146
153
  const mapping = primitives.createSharedMemory(forceNamed
147
154
  ? {
148
155
  size: carpet.byteLength(),
@@ -215,7 +222,7 @@ export const createProcessStealMemoryLayout = ({ threads, signalBytes, abortByte
215
222
  returnPayload: carpet.take(`returnPayload-${thread}`, payloadBytes),
216
223
  }));
217
224
  const primitives = getProcessWorkerSharedMemoryPrimitives();
218
- const forceNamed = sharedMemory.mode === "named" || isWindowsRuntimeHost();
225
+ const forceNamed = processWorkerNeedsNamedMemory(sharedMemory);
219
226
  const mapping = primitives.createSharedMemory(forceNamed
220
227
  ? {
221
228
  size: carpet.byteLength(),
@@ -546,6 +553,19 @@ export const createProcessWorkerNativeSignalNotifier = ({ processRuntime, signal
546
553
  return undefined;
547
554
  }
548
555
  };
556
+ /**
557
+ * Whether this host/child pair has the Node-compatible IPC channel that the
558
+ * process completion-doorbell prototype uses. Deno process workers currently
559
+ * boot through environment data and do not have that channel.
560
+ */
561
+ export const processWorkerUsesIpc = ({ processRuntime, commandPrefix, }) => {
562
+ if (processRuntime === undefined || commandPrefix !== undefined)
563
+ return false;
564
+ if (RUNTIME === "node") {
565
+ return processRuntime === "node" || processRuntime === "bun";
566
+ }
567
+ return RUNTIME === "bun" && processRuntime === "bun";
568
+ };
549
569
  const createProcessWorkerEventHub = () => {
550
570
  const messageHandlers = [];
551
571
  const errorHandlers = [];
@@ -2,7 +2,6 @@ import "../memory/payloadCodec.js";
2
2
  import { type Lock2, type Task } from "../memory/lock.js";
3
3
  import type { AbortSignalOption, TaskTimeout } from "../types.js";
4
4
  import { type SignalAbortStore } from "../shared/abortSignal.js";
5
- import { type BufferReferenceReturnHooks } from "../connections/buffer-reference.js";
6
5
  type RawArguments = unknown;
7
6
  type FunctionID = number;
8
7
  type QueueTask = Task;
@@ -23,19 +22,17 @@ type CreateHostTxQueueArgs = {
23
22
  * When omitted, only `returnLock` is drained (the classic one-lane shape).
24
23
  */
25
24
  extraReturnLocks?: readonly Lock2[];
26
- releaseBufferReferenceReturn?: ((token: bigint) => void) | BufferReferenceReturnHooks;
27
25
  abortSignals?: Pick<SignalAbortStore, "getSignal" | "setSignal" | "resetSignal" | "closeNow">;
28
26
  now?: () => number;
29
27
  };
30
- type ReturnHooks = ((token: bigint) => void) | BufferReferenceReturnHooks;
31
- export declare function createHostTxQueue({ max, lock, returnLock, extraReturnLocks, releaseBufferReferenceReturn, abortSignals, now, }: CreateHostTxQueueArgs): {
28
+ export declare function createHostTxQueue({ max, lock, returnLock, extraReturnLocks, abortSignals, now, }: CreateHostTxQueueArgs): {
32
29
  rejectAll: (reason: string) => void;
33
30
  hasPendingFrames: () => boolean;
34
31
  txIdle: () => boolean;
35
32
  completeFrame: () => number;
36
33
  waitForCompletion: (onWake: () => void, timeoutMs?: number) => boolean;
34
+ armCompletionNotifier: () => boolean;
37
35
  setCompletionWaiterArmed: (armed: boolean) => void;
38
- setReturnHooks: (lane: number, hooks: ReturnHooks | undefined) => void;
39
36
  enqueue: (functionID: FunctionID, timeout?: TaskTimeout, abortSignal?: AbortSignalOption) => (rawArgs: RawArguments) => Promise<never> | import("../common/with-resolvers.js").PromiseWithMaybeReject<unknown>;
40
37
  flushToWorker: () => boolean;
41
38
  enqueueKnown: (task: QueueTask) => boolean;