knitting 0.1.70 → 0.1.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -10
- package/knitting.browser.d.ts +3 -1
- package/knitting.browser.js +1 -1
- package/knitting.d.ts +3 -1
- package/knitting.js +2 -1
- package/package.json +8 -3
- package/prebuilds/win32-x64/knitting_windows_shared_memory.dll +0 -0
- package/prebuilds/win32-x64-node-127/knitting_buffer_pointer.node +0 -0
- package/prebuilds/win32-x64-node-127/knitting_doorbell.node +0 -0
- package/prebuilds/win32-x64-node-127/knitting_shared_memory.node +0 -0
- package/prebuilds/win32-x64-node-127/knitting_shm.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_buffer_pointer.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_doorbell.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_shared_memory.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_shm.node +0 -0
- package/src/api.js +58 -34
- package/src/connections/node-addons.js +11 -1
- package/src/debug/gate.js +1 -1
- package/src/debug/handle.d.ts +6 -1
- package/src/debug/handle.js +14 -6
- package/src/error.d.ts +9 -0
- package/src/error.js +16 -2
- package/src/memory/lock.d.ts +48 -6
- package/src/memory/lock.js +350 -142
- package/src/memory/payloadCodec.js +31 -11
- package/src/permission/protocol.d.ts +1 -0
- package/src/permission/protocol.js +8 -3
- package/src/runtime/dispatcher.d.ts +6 -1
- package/src/runtime/dispatcher.js +24 -8
- package/src/runtime/inline-executor.js +2 -1
- package/src/runtime/pool.d.ts +13 -4
- package/src/runtime/pool.js +107 -47
- package/src/runtime/process-worker.js +10 -1
- package/src/runtime/tx-queue.d.ts +2 -1
- package/src/runtime/tx-queue.js +11 -0
- package/src/runtime/worker-common.d.ts +7 -0
- package/src/runtime/worker-common.js +28 -2
- package/src/types.d.ts +35 -11
- package/src/worker/loop.js +17 -4
- package/src/worker/safety/index.d.ts +1 -1
- package/src/worker/safety/index.js +1 -1
- package/src/worker/safety/process.d.ts +2 -0
- package/src/worker/safety/process.js +8 -1
- package/src/worker/safety/startup.js +11 -6
- package/src/worker/timers.js +25 -3
package/src/memory/lock.js
CHANGED
|
@@ -257,8 +257,30 @@ export const STEAL_PAYLOAD_ACK_SLOT_OFFSET_U32 = STEAL_ACK_SLOT_OFFSET_U32 + 2;
|
|
|
257
257
|
export const STEAL_LIVE_SLOT_OFFSET_U32 = STEAL_PAYLOAD_ACK_SLOT_OFFSET_U32 + 1;
|
|
258
258
|
/** Host-owned arm word for the return-lock completion doorbell. */
|
|
259
259
|
export const DOORBELL_ARMED_SLOT_OFFSET_U32 = STEAL_LIVE_SLOT_OFFSET_U32 + 1;
|
|
260
|
-
/**
|
|
261
|
-
|
|
260
|
+
/**
|
|
261
|
+
* Non-wrapping 64-bit claim head for the `ticket` discipline. The producer
|
|
262
|
+
* publishes only the low 32 bits of the tail. A successful head CAS validates
|
|
263
|
+
* the snapshot: with that head unchanged, at most 32 tickets can be pending,
|
|
264
|
+
* so unsigned subtraction recovers the exact distance even across tail wrap.
|
|
265
|
+
* Head and tail occupy different slots' header lines (slots 0 and 1).
|
|
266
|
+
*/
|
|
267
|
+
export const STEAL_TICKET_HEAD_SLOT_OFFSET_U32 = DOORBELL_ARMED_SLOT_OFFSET_U32 + 2; // Keep the 64-bit head aligned.
|
|
268
|
+
/** Negative head permanently closes a failed ticket queue. */
|
|
269
|
+
export const STEAL_TICKET_FAILED = -1n;
|
|
270
|
+
const STEAL_TICKET_LIMIT = (1n << 63n) - 1n;
|
|
271
|
+
const STEAL_TICKET_LOW_MASK = 0xffffffffn;
|
|
272
|
+
/**
|
|
273
|
+
* Publication-order ring. `order[t & 31]` is the slot the producer filled for
|
|
274
|
+
* ticket `t`, so a ticket names a lane without the ticket having to *be* the
|
|
275
|
+
* lane: the producer keeps its free-bit allocation and claims follow publication
|
|
276
|
+
* order. Cell `i` uses the former ACK word; words 14..15
|
|
277
|
+
* are reserved for the head in slot 0 and the tail in slot 1.
|
|
278
|
+
*/
|
|
279
|
+
export const STEAL_TICKET_ORDER_SLOT_OFFSET_U32 = STEAL_ACK_SLOT_OFFSET_U32;
|
|
280
|
+
/** Ring cells, and therefore the wrap mask, follow the slot count. */
|
|
281
|
+
export const STEAL_TICKET_RING_MASK = (LockBound.slots - 1) | 0;
|
|
282
|
+
/** Bounded CAS retries before a claimant re-polls from the worker loop. */
|
|
283
|
+
const STEAL_TICKET_ATTEMPTS = 2;
|
|
262
284
|
// A producer claims ARMED before ringing the host. Keeping SIGNALLED until the
|
|
263
285
|
// host drains coalesces a burst of published frames into one native/IPC wake.
|
|
264
286
|
const DOORBELL_OFF = 0;
|
|
@@ -313,7 +335,46 @@ const settleTask = (task) => {
|
|
|
313
335
|
task[TaskIndex["FlagsToHost"]] = 0;
|
|
314
336
|
}
|
|
315
337
|
};
|
|
316
|
-
|
|
338
|
+
/**
|
|
339
|
+
* Claim discipline for stealing consumers.
|
|
340
|
+
*
|
|
341
|
+
* - `dekker`: per-consumer intent words and an O(N) peer survey.
|
|
342
|
+
* - `ticket`: one monotonic counter; a CAS on the head claims lanes in
|
|
343
|
+
* publication order, with no regions and no per-consumer state.
|
|
344
|
+
*/
|
|
345
|
+
export const STEAL_CLAIM_DISCIPLINES = ["dekker", "ticket"];
|
|
346
|
+
/** Disciplines that once existed, so a stale config gets a useful error. */
|
|
347
|
+
const REMOVED_STEAL_CLAIMS = {
|
|
348
|
+
"cas-mask": "cas-mask was removed; use ticket",
|
|
349
|
+
};
|
|
350
|
+
/**
|
|
351
|
+
* Validate a claim discipline coming from an untrusted source — an env var, or
|
|
352
|
+
* a JS caller the type system never saw.
|
|
353
|
+
*
|
|
354
|
+
* This throws rather than falling back, because a silent fallback is worse than
|
|
355
|
+
* a crash here: a stale `cas-mask` setting or a typo would quietly run Dekker,
|
|
356
|
+
* and every benchmark and test built on it would report Dekker's numbers under
|
|
357
|
+
* another name. Both sides of a lock must also agree, so an unrecognised value
|
|
358
|
+
* that resolved differently on host and worker would corrupt the protocol.
|
|
359
|
+
*/
|
|
360
|
+
export const assertStealClaim = (value, source) => {
|
|
361
|
+
if (typeof value === "string" &&
|
|
362
|
+
STEAL_CLAIM_DISCIPLINES.includes(value)) {
|
|
363
|
+
return value;
|
|
364
|
+
}
|
|
365
|
+
const hint = typeof value === "string" && REMOVED_STEAL_CLAIMS[value] !== undefined
|
|
366
|
+
? ` (${REMOVED_STEAL_CLAIMS[value]})`
|
|
367
|
+
: "";
|
|
368
|
+
throw new RangeError(`${source} must be one of ${STEAL_CLAIM_DISCIPLINES.join(", ")}; ` +
|
|
369
|
+
`got ${JSON.stringify(value)}${hint}`);
|
|
370
|
+
};
|
|
371
|
+
/**
|
|
372
|
+
* The discipline chosen when nothing selects one. Host and worker default
|
|
373
|
+
* independently, so this must be a single shared constant: two sides landing on
|
|
374
|
+
* different disciplines would read the same buffer under different protocols.
|
|
375
|
+
*/
|
|
376
|
+
export const DEFAULT_STEAL_CLAIM = "ticket";
|
|
377
|
+
export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload, payloadConfig, payloadSector, textCompat, resultList, toSentList, recycleList, processBoundary, sharedReturn, moveReturn, consumers, consumerId, regionLanes, stealClaim, notifyOnHostPublish, notifyHostPublish, traceClaim, }) => {
|
|
317
378
|
// Layout within `lockSectorRegion`:
|
|
318
379
|
// - hostBits starts at byte 0
|
|
319
380
|
// - workerBits starts at byte 64
|
|
@@ -357,14 +418,20 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
357
418
|
const stealId = (consumerId ?? 0) | 0;
|
|
358
419
|
const stealRegionLanes = (regionLanes ?? 8) | 0;
|
|
359
420
|
const stealRegions = (LockBound.slots / stealRegionLanes) | 0;
|
|
360
|
-
|
|
421
|
+
// Validate before use: an unrecognised value must not degrade to Dekker.
|
|
422
|
+
const stealClaimResolved = stealClaim === undefined
|
|
423
|
+
? DEFAULT_STEAL_CLAIM
|
|
424
|
+
: assertStealClaim(stealClaim, "stealClaim");
|
|
425
|
+
const stealTicket = stealClaimResolved === "ticket";
|
|
361
426
|
if (stealEnabled) {
|
|
427
|
+
if (stealTicket && headersSlotStride % 2 !== 0) {
|
|
428
|
+
throw new RangeError("ticket headers must have an 8-byte aligned slot stride");
|
|
429
|
+
}
|
|
362
430
|
if (stealRegionLanes < 1 || (stealRegionLanes & (stealRegionLanes - 1)) !== 0) {
|
|
363
431
|
throw new RangeError("regionLanes must be a power of two");
|
|
364
432
|
}
|
|
365
|
-
// Dekker requires at least one region per
|
|
366
|
-
|
|
367
|
-
if (!stealClaimMask && stealRegions < stealConsumers) {
|
|
433
|
+
// Dekker requires at least one region per consumer.
|
|
434
|
+
if (!stealTicket && stealRegions < stealConsumers) {
|
|
368
435
|
throw new RangeError(`regionLanes=${stealRegionLanes} yields ${stealRegions} regions, ` +
|
|
369
436
|
`too few for ${stealConsumers} consumers`);
|
|
370
437
|
}
|
|
@@ -381,16 +448,41 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
381
448
|
STEAL_WANT_SLOT_OFFSET_U32;
|
|
382
449
|
}
|
|
383
450
|
const stealLiveIndex = LockBound.header + STEAL_LIVE_SLOT_OFFSET_U32;
|
|
384
|
-
/**
|
|
385
|
-
const
|
|
451
|
+
/** Head (consumer-CAS) and tail (producer-store) on separate slot lines. */
|
|
452
|
+
const stealTicketHeadIndex = LockBound.header +
|
|
453
|
+
STEAL_TICKET_HEAD_SLOT_OFFSET_U32;
|
|
454
|
+
const stealTicketTailIndex = headersSlotStride + LockBound.header +
|
|
455
|
+
STEAL_TICKET_HEAD_SLOT_OFFSET_U32;
|
|
456
|
+
const stealTicketTailWord = stealEnabled && stealTicket
|
|
457
|
+
? new Int32Array(headersRegion.sab, headersRegion.byteOffset + stealTicketTailIndex * Int32Array.BYTES_PER_ELEMENT, 1)
|
|
458
|
+
: undefined;
|
|
459
|
+
const stealTicketView = stealEnabled && stealTicket
|
|
460
|
+
? new BigInt64Array(headersRegion.sab, headersRegion.byteOffset, headersRegion.byteLength >>> 3)
|
|
461
|
+
: undefined;
|
|
462
|
+
const stealTicketHead64 = stealTicketHeadIndex >>> 1;
|
|
463
|
+
const stealTicketOrderIndex = new Int32Array(LockBound.slots);
|
|
464
|
+
for (let slot = 0; slot < LockBound.slots; slot++) {
|
|
465
|
+
stealTicketOrderIndex[slot] = (slot * headersSlotStride) +
|
|
466
|
+
LockBound.header + STEAL_TICKET_ORDER_SLOT_OFFSET_U32;
|
|
467
|
+
}
|
|
468
|
+
/** A claimant drains at most one region's worth of tickets per CAS. */
|
|
469
|
+
const stealTicketBatch = stealRegionLanes;
|
|
470
|
+
const stealTicketSlots = new Int32Array(stealTicketBatch);
|
|
471
|
+
const stealTicketPublish = stealEnabled && stealIsProducer && stealTicket;
|
|
472
|
+
/** Producer-private shadow of the tail; the producer is its only writer. */
|
|
473
|
+
let stealTicketTail = 0 | 0;
|
|
474
|
+
/** At most 32 tickets can be staged, so equality is safe across uint32 wrap. */
|
|
475
|
+
let stealTicketPublishedTail = 0 | 0;
|
|
386
476
|
const stealAllLiveMask = stealConsumers === 32
|
|
387
477
|
? -1
|
|
388
478
|
: ((1 << stealConsumers) - 1) | 0;
|
|
389
479
|
if (stealEnabled && stealIsProducer) {
|
|
390
480
|
Atomics.store(stealView, stealLiveIndex, stealAllLiveMask);
|
|
391
481
|
// A carpet can outlive one pool on a reused buffer; never inherit owners.
|
|
392
|
-
if (
|
|
393
|
-
Atomics.store(
|
|
482
|
+
if (stealTicket) {
|
|
483
|
+
Atomics.store(stealTicketView, stealTicketHead64, 0n);
|
|
484
|
+
Atomics.store(stealTicketTailWord, 0, 0);
|
|
485
|
+
}
|
|
394
486
|
}
|
|
395
487
|
const stealIsLive = (mask, consumer) => (mask & (1 << consumer)) !== 0;
|
|
396
488
|
let promiseHandler;
|
|
@@ -495,83 +587,93 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
495
587
|
const encodeManyFrom = (list) => {
|
|
496
588
|
let state = ensureSenderStateHasFree((LastLocal ^ workerShadow) | 0);
|
|
497
589
|
let encoded = 0 | 0;
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
590
|
+
try {
|
|
591
|
+
if (list === toBeSent) {
|
|
592
|
+
while (true) {
|
|
593
|
+
const task = toBeSentShift();
|
|
594
|
+
if (!task)
|
|
595
|
+
break;
|
|
596
|
+
state = ensureSenderStateHasFree(state);
|
|
597
|
+
const bit = encodeWithState(task, state) | 0;
|
|
598
|
+
if (bit === 0) {
|
|
599
|
+
toBeSentUnshift(task);
|
|
600
|
+
break;
|
|
601
|
+
}
|
|
602
|
+
state = (state ^ bit) | 0;
|
|
603
|
+
encoded = (encoded + 1) | 0;
|
|
508
604
|
}
|
|
509
|
-
state = (state ^ bit) | 0;
|
|
510
|
-
encoded = (encoded + 1) | 0;
|
|
511
605
|
}
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
606
|
+
else {
|
|
607
|
+
while (true) {
|
|
608
|
+
const task = list.shiftNoClear();
|
|
609
|
+
if (!task)
|
|
610
|
+
break;
|
|
611
|
+
state = ensureSenderStateHasFree(state);
|
|
612
|
+
const bit = encodeWithState(task, state) | 0;
|
|
613
|
+
if (bit === 0) {
|
|
614
|
+
list.unshift(task);
|
|
615
|
+
break;
|
|
616
|
+
}
|
|
617
|
+
state = (state ^ bit) | 0;
|
|
618
|
+
encoded = (encoded + 1) | 0;
|
|
523
619
|
}
|
|
524
|
-
state = (state ^ bit) | 0;
|
|
525
|
-
encoded = (encoded + 1) | 0;
|
|
526
620
|
}
|
|
621
|
+
return encoded;
|
|
622
|
+
}
|
|
623
|
+
finally {
|
|
624
|
+
flushTicketTail();
|
|
527
625
|
}
|
|
528
|
-
return encoded;
|
|
529
626
|
};
|
|
530
627
|
const encodeManyTrackedFrom = (list) => {
|
|
531
628
|
let state = ensureSenderStateHasFree((LastLocal ^ workerShadow) | 0);
|
|
532
629
|
let encoded = 0 | 0;
|
|
533
630
|
deferredCount = 0 | 0;
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
if (
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
631
|
+
try {
|
|
632
|
+
if (list === toBeSent) {
|
|
633
|
+
while (true) {
|
|
634
|
+
const task = toBeSentShift();
|
|
635
|
+
if (!task)
|
|
636
|
+
break;
|
|
637
|
+
state = ensureSenderStateHasFree(state);
|
|
638
|
+
const bit = encodeWithState(task, state) | 0;
|
|
639
|
+
if (bit === 0) {
|
|
640
|
+
if (isPromisePayloadPending(task)) {
|
|
641
|
+
deferredCount = (deferredCount + 1) | 0;
|
|
642
|
+
trackDeferredTask(task);
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
toBeSentUnshift(task);
|
|
646
|
+
break;
|
|
546
647
|
}
|
|
547
|
-
|
|
548
|
-
|
|
648
|
+
state = (state ^ bit) | 0;
|
|
649
|
+
encoded = (encoded + 1) | 0;
|
|
549
650
|
}
|
|
550
|
-
state = (state ^ bit) | 0;
|
|
551
|
-
encoded = (encoded + 1) | 0;
|
|
552
651
|
}
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
652
|
+
else {
|
|
653
|
+
while (true) {
|
|
654
|
+
const task = list.shiftNoClear();
|
|
655
|
+
if (!task)
|
|
656
|
+
break;
|
|
657
|
+
state = ensureSenderStateHasFree(state);
|
|
658
|
+
const bit = encodeWithState(task, state) | 0;
|
|
659
|
+
if (bit === 0) {
|
|
660
|
+
if (isPromisePayloadPending(task)) {
|
|
661
|
+
deferredCount = (deferredCount + 1) | 0;
|
|
662
|
+
trackDeferredTask(task);
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
list.unshift(task);
|
|
666
|
+
break;
|
|
566
667
|
}
|
|
567
|
-
|
|
568
|
-
|
|
668
|
+
state = (state ^ bit) | 0;
|
|
669
|
+
encoded = (encoded + 1) | 0;
|
|
569
670
|
}
|
|
570
|
-
state = (state ^ bit) | 0;
|
|
571
|
-
encoded = (encoded + 1) | 0;
|
|
572
671
|
}
|
|
672
|
+
return encoded;
|
|
673
|
+
}
|
|
674
|
+
finally {
|
|
675
|
+
flushTicketTail();
|
|
573
676
|
}
|
|
574
|
-
return encoded;
|
|
575
677
|
};
|
|
576
678
|
const encodeAll = () => {
|
|
577
679
|
if (toBeSent.isEmpty)
|
|
@@ -580,10 +682,8 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
580
682
|
deferredCount = 0 | 0;
|
|
581
683
|
return toBeSent.isEmpty;
|
|
582
684
|
};
|
|
583
|
-
const
|
|
584
|
-
|
|
585
|
-
if (shouldNotifyHostPublish &&
|
|
586
|
-
a_compareExchange(doorbellArmed, 0, DOORBELL_ARMED, DOORBELL_SIGNALLED) === DOORBELL_ARMED) {
|
|
685
|
+
const notifyHostPublication = () => {
|
|
686
|
+
if (a_compareExchange(doorbellArmed, 0, DOORBELL_ARMED, DOORBELL_SIGNALLED) === DOORBELL_ARMED) {
|
|
587
687
|
if (notifyHostPublish !== undefined) {
|
|
588
688
|
try {
|
|
589
689
|
notifyHostPublish();
|
|
@@ -600,6 +700,11 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
600
700
|
}
|
|
601
701
|
}
|
|
602
702
|
};
|
|
703
|
+
const storeHost = (bit) => {
|
|
704
|
+
a_store(hostBits, 0, LastLocal = (LastLocal ^ bit) | 0);
|
|
705
|
+
if (shouldNotifyHostPublish)
|
|
706
|
+
notifyHostPublication();
|
|
707
|
+
};
|
|
603
708
|
const storeWorker = (bit) => a_store(workerBits, 0, LastWorker = (LastWorker ^ bit) | 0);
|
|
604
709
|
const encode = (task, state = (LastLocal ^ workerShadow) | 0) => {
|
|
605
710
|
state = ensureSenderStateHasFree(state);
|
|
@@ -609,7 +714,12 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
609
714
|
if (!encodeTaskValue(task, selectedSlotIndex = 31 - clz32(free))) {
|
|
610
715
|
return false;
|
|
611
716
|
}
|
|
612
|
-
|
|
717
|
+
try {
|
|
718
|
+
return encodeAt(task, selectedSlotIndex, selectedSlotBit = 1 << selectedSlotIndex);
|
|
719
|
+
}
|
|
720
|
+
finally {
|
|
721
|
+
flushTicketTail();
|
|
722
|
+
}
|
|
613
723
|
};
|
|
614
724
|
const encodeTracked = (task, state = (LastLocal ^ workerShadow) | 0) => {
|
|
615
725
|
deferredCount = 0 | 0;
|
|
@@ -624,7 +734,12 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
624
734
|
}
|
|
625
735
|
return false;
|
|
626
736
|
}
|
|
627
|
-
|
|
737
|
+
try {
|
|
738
|
+
return encodeAt(task, selectedSlotIndex, selectedSlotBit = 1 << selectedSlotIndex);
|
|
739
|
+
}
|
|
740
|
+
finally {
|
|
741
|
+
flushTicketTail();
|
|
742
|
+
}
|
|
628
743
|
};
|
|
629
744
|
const encodeAt = (task, at, bit) => {
|
|
630
745
|
const off = (at * headersSlotStride) + slotBaseU32;
|
|
@@ -636,9 +751,39 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
636
751
|
headersBuffer[off + 5] = task[5];
|
|
637
752
|
headersBuffer[off + 6] = task[6];
|
|
638
753
|
headersBuffer[off + TASK_LOCAL_FLAGS_INDEX] = 0;
|
|
754
|
+
if (stealTicketPublish) {
|
|
755
|
+
// The tail's seq-cst store publishes the header, ring cell and host bit
|
|
756
|
+
// together. Ticket consumers acquire the tail before reading a lane;
|
|
757
|
+
// unlike region consumers, they never arbitrate through hostBits.
|
|
758
|
+
stealView[stealTicketOrderIndex[stealTicketTail & STEAL_TICKET_RING_MASK]] = at;
|
|
759
|
+
stealTicketTail = (stealTicketTail + 1) | 0;
|
|
760
|
+
if (shouldNotifyHostPublish) {
|
|
761
|
+
a_store(hostBits, 0, LastLocal = (LastLocal ^ bit) | 0);
|
|
762
|
+
}
|
|
763
|
+
else {
|
|
764
|
+
hostBits[0] = LastLocal = (LastLocal ^ bit) | 0;
|
|
765
|
+
}
|
|
766
|
+
// The tail store is deferred so a burst pays one atomic, not one each.
|
|
767
|
+
// Nothing is claimable until it lands, which is what makes the batch
|
|
768
|
+
// atomic; every exit from an encode path must flush, including throws.
|
|
769
|
+
return true;
|
|
770
|
+
}
|
|
639
771
|
storeHost(bit);
|
|
640
772
|
return true;
|
|
641
773
|
};
|
|
774
|
+
/**
|
|
775
|
+
* Publish accumulated tickets. Consumers cannot see any lane of the batch
|
|
776
|
+
* until this store lands, so skipping it strands the slots forever.
|
|
777
|
+
*/
|
|
778
|
+
const flushTicketTail = () => {
|
|
779
|
+
if (stealTicketPublishedTail === stealTicketTail)
|
|
780
|
+
return;
|
|
781
|
+
stealTicketPublishedTail = stealTicketTail;
|
|
782
|
+
a_store(stealTicketTailWord, 0, stealTicketTail);
|
|
783
|
+
// A notifier must not run while its tickets are still invisible.
|
|
784
|
+
if (shouldNotifyHostPublish)
|
|
785
|
+
notifyHostPublication();
|
|
786
|
+
};
|
|
642
787
|
const hasSpace = () => (hostBits[0] ^ LastWorker) !== 0;
|
|
643
788
|
const decode = () => {
|
|
644
789
|
let diff = (a_load(hostBits, 0) ^ LastWorker) | 0;
|
|
@@ -804,75 +949,135 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
804
949
|
Atomics.xor(workerBits, 0, done);
|
|
805
950
|
a_store(stealView, stealWantIndex[stealId], 0);
|
|
806
951
|
stealCursor = stealHome;
|
|
952
|
+
// Maybe in the future
|
|
953
|
+
// stealCursor = (region + 1) % stealRegions;
|
|
807
954
|
}
|
|
808
955
|
return true;
|
|
809
956
|
};
|
|
810
|
-
/**
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
for (let
|
|
819
|
-
const
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
957
|
+
/**
|
|
958
|
+
* Claim tickets from one monotonic counter.
|
|
959
|
+
*
|
|
960
|
+
* The ticket is the arbiter and the FIFO order at once: a winning CAS owns
|
|
961
|
+
* lanes nobody else can name, so there are no regions, no intent words, no
|
|
962
|
+
* peer survey and no liveness mask. An idle poll costs two loads and no RMW.
|
|
963
|
+
*/
|
|
964
|
+
const decodeStealTicket = () => {
|
|
965
|
+
for (let attempt = 0; attempt < STEAL_TICKET_ATTEMPTS; attempt++) {
|
|
966
|
+
const head = Atomics.load(stealTicketView, stealTicketHead64);
|
|
967
|
+
if (head === STEAL_TICKET_FAILED)
|
|
968
|
+
return false;
|
|
969
|
+
const lowHead = Number(head & STEAL_TICKET_LOW_MASK);
|
|
970
|
+
const available = (a_load(stealTicketTailWord, 0) - lowHead) >>> 0;
|
|
971
|
+
if (available === 0)
|
|
972
|
+
return false;
|
|
973
|
+
// A valid head has at most 32 published tickets ahead of it. Larger
|
|
974
|
+
// distances are stale snapshots; the 64-bit head is still the arbiter.
|
|
975
|
+
if (available > LockBound.slots)
|
|
823
976
|
continue;
|
|
824
|
-
const
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
const previous = a_compareExchange(stealView, stealMaskIndex, owners, (owners | bit) | 0) | 0;
|
|
830
|
-
if (previous === owners) {
|
|
831
|
-
region = candidate;
|
|
832
|
-
intent = bit;
|
|
833
|
-
break;
|
|
834
|
-
}
|
|
835
|
-
owners = previous;
|
|
977
|
+
const batch = Math.min(available, stealTicketBatch);
|
|
978
|
+
const next = head + BigInt(batch);
|
|
979
|
+
if (next > STEAL_TICKET_LIMIT) {
|
|
980
|
+
Atomics.store(stealTicketView, stealTicketHead64, STEAL_TICKET_FAILED);
|
|
981
|
+
throw new RangeError("ticket queue exhausted its 64-bit sequence");
|
|
836
982
|
}
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
Atomics.
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
let at = stealOldestLane(region, take);
|
|
852
|
-
const regionLow = (region * stealRegionLanes) | 0;
|
|
853
|
-
const regionHigh = (regionLow + stealRegionLanes - 1) | 0;
|
|
854
|
-
try {
|
|
855
|
-
for (let step = 0; step < stealRegionLanes; step++) {
|
|
856
|
-
const bit = (1 << at) | 0;
|
|
857
|
-
if ((lanes & bit) !== 0) {
|
|
983
|
+
const ringHead = lowHead & STEAL_TICKET_RING_MASK;
|
|
984
|
+
// Read the lane ids *before* the CAS. Cell `head & 31` cannot be refilled
|
|
985
|
+
// while the head still sits at `head` (refilling it needs a free slot,
|
|
986
|
+
// which needs a retire, which needs a claim), but once the CAS moves the
|
|
987
|
+
// head past a ticket the producer may reuse that cell immediately.
|
|
988
|
+
for (let i = 0; i < batch; i++) {
|
|
989
|
+
stealTicketSlots[i] = stealView[stealTicketOrderIndex[(ringHead + i) & STEAL_TICKET_RING_MASK]] & STEAL_TICKET_RING_MASK;
|
|
990
|
+
}
|
|
991
|
+
if (Atomics.compareExchange(stealTicketView, stealTicketHead64, head, next) !== head)
|
|
992
|
+
continue;
|
|
993
|
+
let done = 0 | 0;
|
|
994
|
+
try {
|
|
995
|
+
for (let i = 0; i < batch; i++) {
|
|
996
|
+
const at = stealTicketSlots[i];
|
|
858
997
|
decodeAt(at);
|
|
859
|
-
|
|
860
|
-
done = (done ^ bit) | 0;
|
|
861
|
-
if (lanes === 0)
|
|
862
|
-
break;
|
|
998
|
+
done = (done ^ (1 << at)) | 0;
|
|
863
999
|
}
|
|
864
|
-
at = at === regionLow ? regionHigh : at - 1;
|
|
865
1000
|
}
|
|
1001
|
+
catch (error) {
|
|
1002
|
+
// Decoding may already have consumed payload state. Replaying the
|
|
1003
|
+
// unfinished batch is unsafe; report the fatal error and stop claims.
|
|
1004
|
+
Atomics.store(stealTicketView, stealTicketHead64, STEAL_TICKET_FAILED);
|
|
1005
|
+
throw error;
|
|
1006
|
+
}
|
|
1007
|
+
finally {
|
|
1008
|
+
LastWorker = (LastWorker ^ done) | 0;
|
|
1009
|
+
if (done !== 0)
|
|
1010
|
+
Atomics.xor(workerBits, 0, done);
|
|
1011
|
+
}
|
|
1012
|
+
return true;
|
|
866
1013
|
}
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
1014
|
+
return false;
|
|
1015
|
+
};
|
|
1016
|
+
/**
|
|
1017
|
+
* Debug-only wrapper around a steal claim, for the `steal` namespace.
|
|
1018
|
+
*
|
|
1019
|
+
* It is selected once, when `decode` is chosen below, so an untraced lock
|
|
1020
|
+
* hands out the bare claim function: no flag test, no extra load, no change
|
|
1021
|
+
* to the claim path. Everything reported is read from outside the claim,
|
|
1022
|
+
* before and after it, so the wrapper cannot perturb the protocol either.
|
|
1023
|
+
*
|
|
1024
|
+
* - `lanes`: in ticket order for `ticket`, ascending for `dekker`.
|
|
1025
|
+
* - `head`: ticket head before and after. `peers` is how many tickets other
|
|
1026
|
+
* consumers took in between; 0 means `head` spans exactly this claim.
|
|
1027
|
+
* - `region`/`home`: `dekker` only; `steal` marks a region off this
|
|
1028
|
+
* consumer's home.
|
|
1029
|
+
* - `lost`: polls since the last claim that saw work and still came back
|
|
1030
|
+
* empty, i.e. a peer got there first.
|
|
1031
|
+
*/
|
|
1032
|
+
const traceStealClaims = (claim, trace) => {
|
|
1033
|
+
let lost = 0;
|
|
1034
|
+
return () => {
|
|
1035
|
+
const before = LastWorker;
|
|
1036
|
+
let head = 0n;
|
|
1037
|
+
let visible = 0;
|
|
1038
|
+
if (stealTicket) {
|
|
1039
|
+
head = Atomics.load(stealTicketView, stealTicketHead64);
|
|
1040
|
+
if (head !== STEAL_TICKET_FAILED) {
|
|
1041
|
+
visible = (a_load(stealTicketTailWord, 0) -
|
|
1042
|
+
Number(head & STEAL_TICKET_LOW_MASK)) >>> 0;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
else {
|
|
1046
|
+
visible = (a_load(hostBits, 0) ^ a_load(workerBits, 0)) | 0;
|
|
1047
|
+
}
|
|
1048
|
+
if (!claim()) {
|
|
1049
|
+
if (visible !== 0)
|
|
1050
|
+
lost++;
|
|
1051
|
+
return false;
|
|
1052
|
+
}
|
|
1053
|
+
const taken = (LastWorker ^ before) | 0;
|
|
1054
|
+
let count = 0;
|
|
1055
|
+
for (let bits = taken; bits !== 0; bits &= bits - 1)
|
|
1056
|
+
count++;
|
|
1057
|
+
let message;
|
|
1058
|
+
if (stealTicket) {
|
|
1059
|
+
const after = Atomics.load(stealTicketView, stealTicketHead64);
|
|
1060
|
+
const lanes = [];
|
|
1061
|
+
for (let i = 0; i < count; i++)
|
|
1062
|
+
lanes.push(stealTicketSlots[i]);
|
|
1063
|
+
message = `claim count=${count} lanes=${lanes.join(",")}` +
|
|
1064
|
+
` head=${head}→${after} peers=${Number(after - head) - count}`;
|
|
1065
|
+
}
|
|
1066
|
+
else {
|
|
1067
|
+
const lanes = [];
|
|
1068
|
+
for (let lane = 0; lane < LockBound.slots; lane++) {
|
|
1069
|
+
if ((taken & (1 << lane)) !== 0)
|
|
1070
|
+
lanes.push(lane);
|
|
1071
|
+
}
|
|
1072
|
+
const region = (lanes[0] / stealRegionLanes) | 0;
|
|
1073
|
+
message = `claim count=${count} lanes=${lanes.join(",")}` +
|
|
1074
|
+
` region=${region} home=${stealHome}` +
|
|
1075
|
+
(region === stealHome ? "" : " steal");
|
|
1076
|
+
}
|
|
1077
|
+
trace(`${message} lost=${lost}`);
|
|
1078
|
+
lost = 0;
|
|
1079
|
+
return true;
|
|
1080
|
+
};
|
|
876
1081
|
};
|
|
877
1082
|
const resolveHost = ({ queue, onResolved, shouldSettle, activeRejectPlaceholder, }) => {
|
|
878
1083
|
const getTask = takeTask({ queue });
|
|
@@ -1090,10 +1295,10 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
1090
1295
|
}
|
|
1091
1296
|
const bit = 1 << id;
|
|
1092
1297
|
const previous = Atomics.and(stealView, stealLiveIndex, ~bit);
|
|
1093
|
-
if (
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1298
|
+
if (stealTicket) {
|
|
1299
|
+
// No durable ownership record exists for a dead claimant. The host must
|
|
1300
|
+
// close the shared pending registry instead of reusing this queue.
|
|
1301
|
+
Atomics.store(stealTicketView, stealTicketHead64, STEAL_TICKET_FAILED);
|
|
1097
1302
|
}
|
|
1098
1303
|
return (previous & bit) !== 0;
|
|
1099
1304
|
};
|
|
@@ -1104,8 +1309,11 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
1104
1309
|
encodeAll,
|
|
1105
1310
|
publish,
|
|
1106
1311
|
flushPending,
|
|
1312
|
+
// Chosen once: an untraced lock gets the claim function itself.
|
|
1107
1313
|
decode: stealEnabled
|
|
1108
|
-
? (
|
|
1314
|
+
? (traceClaim === undefined
|
|
1315
|
+
? (stealTicket ? decodeStealTicket : decodeSteal)
|
|
1316
|
+
: traceStealClaims(stealTicket ? decodeStealTicket : decodeSteal, traceClaim))
|
|
1109
1317
|
: decode,
|
|
1110
1318
|
hasSpace,
|
|
1111
1319
|
resolved,
|