knitting 0.1.63 → 0.1.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +623 -342
- package/knitting.browser.d.ts +3 -1
- package/knitting.browser.js +1 -1
- package/knitting.d.ts +3 -1
- package/knitting.js +2 -1
- package/map.md +0 -6
- package/package.json +10 -5
- package/prebuilds/darwin-arm64-node-127/knitting_buffer_pointer.node +0 -0
- package/prebuilds/darwin-arm64-node-127/knitting_doorbell.node +0 -0
- package/prebuilds/darwin-arm64-node-137/knitting_buffer_pointer.node +0 -0
- package/prebuilds/darwin-arm64-node-137/knitting_doorbell.node +0 -0
- package/prebuilds/darwin-x64-node-127/knitting_buffer_pointer.node +0 -0
- package/prebuilds/darwin-x64-node-127/knitting_doorbell.node +0 -0
- package/prebuilds/darwin-x64-node-137/knitting_buffer_pointer.node +0 -0
- package/prebuilds/darwin-x64-node-137/knitting_doorbell.node +0 -0
- package/prebuilds/linux-x64-node-127/knitting_buffer_pointer.node +0 -0
- package/prebuilds/linux-x64-node-127/knitting_doorbell.node +0 -0
- package/prebuilds/linux-x64-node-137/knitting_buffer_pointer.node +0 -0
- package/prebuilds/linux-x64-node-137/knitting_doorbell.node +0 -0
- package/prebuilds/win32-x64/knitting_windows_shared_memory.dll +0 -0
- package/prebuilds/win32-x64-node-127/knitting_buffer_pointer.node +0 -0
- package/prebuilds/win32-x64-node-127/knitting_doorbell.node +0 -0
- package/prebuilds/win32-x64-node-127/knitting_shared_memory.node +0 -0
- package/prebuilds/win32-x64-node-127/knitting_shm.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_buffer_pointer.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_doorbell.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_shared_memory.node +0 -0
- package/prebuilds/win32-x64-node-137/knitting_shm.node +0 -0
- package/scripts/build-native-addons.ts +5 -0
- package/shared-memory.d.ts +3 -0
- package/shared-memory.js +3 -0
- package/src/api.js +158 -71
- package/src/common/with-resolvers.js +2 -5
- package/src/common/worker-runtime.d.ts +7 -0
- package/src/common/worker-runtime.js +7 -0
- package/src/connections/buffer-reference.d.ts +10 -36
- package/src/connections/buffer-reference.js +15 -170
- package/src/connections/node-addons.d.ts +1 -1
- package/src/connections/node-addons.js +11 -1
- package/src/connections/shared-array-buffer-payload.d.ts +7 -0
- package/src/connections/shared-array-buffer-payload.js +27 -11
- package/src/debug/gate.js +1 -1
- package/src/debug/handle.d.ts +6 -1
- package/src/debug/handle.js +14 -6
- package/src/error.d.ts +9 -0
- package/src/error.js +16 -2
- package/src/knitting_buffer_pointer.cc +57 -2
- package/src/knitting_doorbell.cc +220 -0
- package/src/memory/knitting-body.d.ts +44 -0
- package/src/memory/knitting-body.js +51 -0
- package/src/memory/knitting-buffer-http.d.ts +116 -0
- package/src/memory/knitting-buffer-http.js +255 -0
- package/src/memory/knitting-buffer.d.ts +250 -0
- package/src/memory/knitting-buffer.js +695 -0
- package/src/memory/lazy-region-registry.d.ts +83 -0
- package/src/memory/lazy-region-registry.js +355 -0
- package/src/memory/lock.d.ts +80 -15
- package/src/memory/lock.js +473 -139
- package/src/memory/payloadCodec.d.ts +18 -2
- package/src/memory/payloadCodec.js +340 -76
- package/src/memory/regionRegistry.d.ts +6 -0
- package/src/memory/regionRegistry.js +125 -240
- package/src/memory/shared-buffer-io.d.ts +7 -0
- package/src/memory/shared-buffer-io.js +34 -8
- package/src/permission/protocol.d.ts +1 -0
- package/src/permission/protocol.js +8 -3
- package/src/runtime/deno-doorbell.d.ts +26 -0
- package/src/runtime/deno-doorbell.js +117 -0
- package/src/runtime/dispatcher.d.ts +13 -6
- package/src/runtime/dispatcher.js +101 -63
- package/src/runtime/host-arg-arena.d.ts +3 -0
- package/src/runtime/host-arg-arena.js +16 -0
- package/src/runtime/inline-executor.js +2 -1
- package/src/runtime/node-doorbell.d.ts +14 -0
- package/src/runtime/node-doorbell.js +84 -0
- package/src/runtime/pool.d.ts +30 -15
- package/src/runtime/pool.js +199 -151
- package/src/runtime/process-worker.d.ts +9 -0
- package/src/runtime/process-worker.js +32 -3
- package/src/runtime/tx-queue.d.ts +4 -6
- package/src/runtime/tx-queue.js +63 -48
- package/src/runtime/worker-common.d.ts +7 -0
- package/src/runtime/worker-common.js +28 -2
- package/src/types.d.ts +66 -78
- package/src/worker/loop.js +95 -60
- package/src/worker/rx-queue.d.ts +2 -3
- package/src/worker/rx-queue.js +34 -40
- package/src/worker/safety/index.d.ts +1 -1
- package/src/worker/safety/index.js +1 -1
- package/src/worker/safety/process.d.ts +2 -0
- package/src/worker/safety/process.js +8 -1
- package/src/worker/safety/startup.js +11 -6
- package/src/worker/shared-return.d.ts +9 -0
- package/src/worker/shared-return.js +22 -0
- package/src/worker/task-loader.js +1 -2
- package/src/worker/timers.d.ts +2 -6
- package/src/worker/timers.js +39 -22
- package/unsafe.d.ts +2 -1
- package/unsafe.js +2 -1
package/src/memory/lock.js
CHANGED
|
@@ -9,12 +9,7 @@ import { createSharedArrayBuffer, createWasmSharedArrayBuffer, } from "../common
|
|
|
9
9
|
import { toSharedBufferRegion, } from "../common/shared-buffer-region.js";
|
|
10
10
|
import { probeLockBufferTextCompat, } from "../common/shared-buffer-text.js";
|
|
11
11
|
import { resolvePayloadBufferOptions, } from "./payload-config.js";
|
|
12
|
-
|
|
13
|
-
* TODO: Compose all the instance where the array is passed as argument
|
|
14
|
-
*/
|
|
15
|
-
// const objects replace `enum`s throughout this module: Andromeda's Nova engine
|
|
16
|
-
// panics on `enum`. Value access, duplicate-value members, and type preserved;
|
|
17
|
-
// identical emit on Node/Deno/Bun.
|
|
12
|
+
// Use const objects instead of enums; Andromeda's Nova engine cannot parse enums.
|
|
18
13
|
export const PayloadSignal = {
|
|
19
14
|
UNREACHABLE: 0,
|
|
20
15
|
BigInt: 2,
|
|
@@ -68,6 +63,12 @@ export const PayloadBuffer = {
|
|
|
68
63
|
EnvelopeDynamicHeaderStringExternal: 52,
|
|
69
64
|
NumericArray: 53,
|
|
70
65
|
StaticNumericArray: 54,
|
|
66
|
+
/** Binary payload already stored in the dynamic arena. */
|
|
67
|
+
ArenaBinary: 55,
|
|
68
|
+
/** A returned Uint8Array whose ArrayBuffer ownership was moved to the host. */
|
|
69
|
+
MovedBinary: 56,
|
|
70
|
+
/** A returned ArrayBuffer whose ownership was moved to the host. */
|
|
71
|
+
MovedArrayBuffer: 57,
|
|
71
72
|
};
|
|
72
73
|
// Value -> name lookup (enum reverse-map), for payload-limit error labels.
|
|
73
74
|
// Later keys win on duplicate values (11 -> "String"), as TS enums do.
|
|
@@ -116,6 +117,13 @@ export const addTaskFinalizer = (task, finalizer) => {
|
|
|
116
117
|
}
|
|
117
118
|
};
|
|
118
119
|
};
|
|
120
|
+
/** Take an owner-lifetime transport hold, if the value supports one. */
|
|
121
|
+
export const takePayloadTransportHold = (value) => {
|
|
122
|
+
if (value === null || typeof value !== "object")
|
|
123
|
+
return undefined;
|
|
124
|
+
const finalizer = value[PayloadTransportFinalizer]?.();
|
|
125
|
+
return typeof finalizer === "function" ? finalizer : undefined;
|
|
126
|
+
};
|
|
119
127
|
export const attachPayloadTransportFinalizer = (task, value) => {
|
|
120
128
|
if (task.finalize !== undefined || value === null || typeof value !== "object") {
|
|
121
129
|
return;
|
|
@@ -143,18 +151,19 @@ export const TaskIndex = {
|
|
|
143
151
|
/**
|
|
144
152
|
* Host -> worker request function id (low 16 bits).
|
|
145
153
|
* High 16 bits are reserved for caller metadata on request path.
|
|
146
|
-
*
|
|
154
|
+
* Shares storage with `FlagsToHost`.
|
|
147
155
|
*/
|
|
148
156
|
FunctionID: 0,
|
|
149
157
|
ID: 1,
|
|
150
158
|
Type: 2,
|
|
151
159
|
Start: 3,
|
|
152
|
-
End: 4,
|
|
153
|
-
PayloadLen: 5,
|
|
154
160
|
/**
|
|
155
|
-
*
|
|
156
|
-
*
|
|
161
|
+
* Payload length; bit 31 stores the high bit of a dynamic-region index.
|
|
162
|
+
* Write the length before `tagTaskSlot()` and mask bit 31 when reading it.
|
|
157
163
|
*/
|
|
164
|
+
End: 4,
|
|
165
|
+
PayloadLen: 5,
|
|
166
|
+
/** Low 5 bits hold the dynamic-region index; the remaining bits hold metadata. */
|
|
158
167
|
slotBuffer: 6,
|
|
159
168
|
Size: 8,
|
|
160
169
|
/**
|
|
@@ -248,6 +257,35 @@ export const STEAL_PAYLOAD_ACK_SLOT_OFFSET_U32 = STEAL_ACK_SLOT_OFFSET_U32 + 2;
|
|
|
248
257
|
export const STEAL_LIVE_SLOT_OFFSET_U32 = STEAL_PAYLOAD_ACK_SLOT_OFFSET_U32 + 1;
|
|
249
258
|
/** Host-owned arm word for the return-lock completion doorbell. */
|
|
250
259
|
export const DOORBELL_ARMED_SLOT_OFFSET_U32 = STEAL_LIVE_SLOT_OFFSET_U32 + 1;
|
|
260
|
+
/**
|
|
261
|
+
* Non-wrapping 64-bit claim head for the `ticket` discipline. The producer
|
|
262
|
+
* publishes only the low 32 bits of the tail. A successful head CAS validates
|
|
263
|
+
* the snapshot: with that head unchanged, at most 32 tickets can be pending,
|
|
264
|
+
* so unsigned subtraction recovers the exact distance even across tail wrap.
|
|
265
|
+
* Head and tail occupy different slots' header lines (slots 0 and 1).
|
|
266
|
+
*/
|
|
267
|
+
export const STEAL_TICKET_HEAD_SLOT_OFFSET_U32 = DOORBELL_ARMED_SLOT_OFFSET_U32 + 2; // Keep the 64-bit head aligned.
|
|
268
|
+
/** Negative head permanently closes a failed ticket queue. */
|
|
269
|
+
export const STEAL_TICKET_FAILED = -1n;
|
|
270
|
+
const STEAL_TICKET_LIMIT = (1n << 63n) - 1n;
|
|
271
|
+
const STEAL_TICKET_LOW_MASK = 0xffffffffn;
|
|
272
|
+
/**
|
|
273
|
+
* Publication-order ring. `order[t & 31]` is the slot the producer filled for
|
|
274
|
+
* ticket `t`, so a ticket names a lane without the ticket having to *be* the
|
|
275
|
+
* lane: the producer keeps its free-bit allocation and claims follow publication
|
|
276
|
+
* order. Cell `i` uses the former ACK word; words 14..15
|
|
277
|
+
* are reserved for the head in slot 0 and the tail in slot 1.
|
|
278
|
+
*/
|
|
279
|
+
export const STEAL_TICKET_ORDER_SLOT_OFFSET_U32 = STEAL_ACK_SLOT_OFFSET_U32;
|
|
280
|
+
/** Ring cells, and therefore the wrap mask, follow the slot count. */
|
|
281
|
+
export const STEAL_TICKET_RING_MASK = (LockBound.slots - 1) | 0;
|
|
282
|
+
/** Bounded CAS retries before a claimant re-polls from the worker loop. */
|
|
283
|
+
const STEAL_TICKET_ATTEMPTS = 2;
|
|
284
|
+
// A producer claims ARMED before ringing the host. Keeping SIGNALLED until the
|
|
285
|
+
// host drains coalesces a burst of published frames into one native/IPC wake.
|
|
286
|
+
const DOORBELL_OFF = 0;
|
|
287
|
+
const DOORBELL_ARMED = 1;
|
|
288
|
+
const DOORBELL_SIGNALLED = 2;
|
|
251
289
|
export const HEADER_U32_LENGTH = LockBound.header +
|
|
252
290
|
(HEADER_SLOT_STRIDE_U32 * LockBound.slots);
|
|
253
291
|
export const HEADER_BYTE_LENGTH = HEADER_U32_LENGTH *
|
|
@@ -287,18 +325,56 @@ const makeTaskFrom = (array, at) => {
|
|
|
287
325
|
fillTaskFrom(task, array, at);
|
|
288
326
|
return task;
|
|
289
327
|
};
|
|
290
|
-
// could be inlined
|
|
291
328
|
const settleTask = (task) => {
|
|
292
329
|
if (task[TaskIndex["FlagsToHost"]] === 0) {
|
|
293
330
|
task.resolve(task.value);
|
|
294
331
|
}
|
|
295
332
|
else {
|
|
296
333
|
task.reject(task.value);
|
|
297
|
-
//
|
|
334
|
+
// Clear the rejection marker before the task slot is reused.
|
|
298
335
|
task[TaskIndex["FlagsToHost"]] = 0;
|
|
299
336
|
}
|
|
300
337
|
};
|
|
301
|
-
|
|
338
|
+
/**
|
|
339
|
+
* Claim discipline for stealing consumers.
|
|
340
|
+
*
|
|
341
|
+
* - `dekker`: per-consumer intent words and an O(N) peer survey.
|
|
342
|
+
* - `ticket`: one monotonic counter; a CAS on the head claims lanes in
|
|
343
|
+
* publication order, with no regions and no per-consumer state.
|
|
344
|
+
*/
|
|
345
|
+
export const STEAL_CLAIM_DISCIPLINES = ["dekker", "ticket"];
|
|
346
|
+
/** Disciplines that once existed, so a stale config gets a useful error. */
|
|
347
|
+
const REMOVED_STEAL_CLAIMS = {
|
|
348
|
+
"cas-mask": "cas-mask was removed; use ticket",
|
|
349
|
+
};
|
|
350
|
+
/**
|
|
351
|
+
* Validate a claim discipline coming from an untrusted source — an env var, or
|
|
352
|
+
* a JS caller the type system never saw.
|
|
353
|
+
*
|
|
354
|
+
* This throws rather than falling back, because a silent fallback is worse than
|
|
355
|
+
* a crash here: a stale `cas-mask` setting or a typo would quietly run Dekker,
|
|
356
|
+
* and every benchmark and test built on it would report Dekker's numbers under
|
|
357
|
+
* another name. Both sides of a lock must also agree, so an unrecognised value
|
|
358
|
+
* that resolved differently on host and worker would corrupt the protocol.
|
|
359
|
+
*/
|
|
360
|
+
export const assertStealClaim = (value, source) => {
|
|
361
|
+
if (typeof value === "string" &&
|
|
362
|
+
STEAL_CLAIM_DISCIPLINES.includes(value)) {
|
|
363
|
+
return value;
|
|
364
|
+
}
|
|
365
|
+
const hint = typeof value === "string" && REMOVED_STEAL_CLAIMS[value] !== undefined
|
|
366
|
+
? ` (${REMOVED_STEAL_CLAIMS[value]})`
|
|
367
|
+
: "";
|
|
368
|
+
throw new RangeError(`${source} must be one of ${STEAL_CLAIM_DISCIPLINES.join(", ")}; ` +
|
|
369
|
+
`got ${JSON.stringify(value)}${hint}`);
|
|
370
|
+
};
|
|
371
|
+
/**
|
|
372
|
+
* The discipline chosen when nothing selects one. Host and worker default
|
|
373
|
+
* independently, so this must be a single shared constant: two sides landing on
|
|
374
|
+
* different disciplines would read the same buffer under different protocols.
|
|
375
|
+
*/
|
|
376
|
+
export const DEFAULT_STEAL_CLAIM = "ticket";
|
|
377
|
+
export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload, payloadConfig, payloadSector, textCompat, resultList, toSentList, recycleList, processBoundary, sharedReturn, moveReturn, consumers, consumerId, regionLanes, stealClaim, notifyOnHostPublish, notifyHostPublish, traceClaim, }) => {
|
|
302
378
|
// Layout within `lockSectorRegion`:
|
|
303
379
|
// - hostBits starts at byte 0
|
|
304
380
|
// - workerBits starts at byte 64
|
|
@@ -306,9 +382,8 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
306
382
|
// The remaining two cache lines in the 256-byte sector are reserved for the
|
|
307
383
|
// payload allocator lock (`PAYLOAD_LOCK_*` at bytes 128 and 192).
|
|
308
384
|
//
|
|
309
|
-
//
|
|
310
|
-
//
|
|
311
|
-
// the "return lock" (worker->host responses) still publishes into `hostBits`.
|
|
385
|
+
// encode toggles hostBits; decode/resolveHost toggles workerBits. The return
|
|
386
|
+
// lock therefore publishes responses through hostBits as well.
|
|
312
387
|
const lockSectorRegion = toSharedBufferRegion(LockBoundSector ??
|
|
313
388
|
createWasmSharedArrayBuffer(LOCK_SECTOR_BYTE_LENGTH));
|
|
314
389
|
const LockBoundSAB = lockSectorRegion.sab;
|
|
@@ -335,23 +410,28 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
335
410
|
headers: headersRegion,
|
|
336
411
|
payload: payloadSAB,
|
|
337
412
|
});
|
|
338
|
-
//
|
|
339
|
-
// With N consumers the pending set is A ^ ACK[0] ^ ... ^ ACK[N-1]; every word
|
|
340
|
-
// still has exactly one writer, so no shared writable word is introduced.
|
|
413
|
+
// Work-stealing state. Pending lanes are A ^ ACK[0] ^ ... ^ ACK[N-1].
|
|
341
414
|
const stealConsumers = Math.max(1, (consumers ?? 1) | 0);
|
|
342
415
|
const stealEnabled = stealConsumers > 1;
|
|
343
|
-
//
|
|
344
|
-
// never claims. It still frees payload slots, so it needs an acknowledgement
|
|
345
|
-
// word of its own rather than sharing consumer 0's.
|
|
416
|
+
// The producer encodes and frees payload slots but never claims a region.
|
|
346
417
|
const stealIsProducer = consumerId === undefined;
|
|
347
418
|
const stealId = (consumerId ?? 0) | 0;
|
|
348
419
|
const stealRegionLanes = (regionLanes ?? 8) | 0;
|
|
349
420
|
const stealRegions = (LockBound.slots / stealRegionLanes) | 0;
|
|
421
|
+
// Validate before use: an unrecognised value must not degrade to Dekker.
|
|
422
|
+
const stealClaimResolved = stealClaim === undefined
|
|
423
|
+
? DEFAULT_STEAL_CLAIM
|
|
424
|
+
: assertStealClaim(stealClaim, "stealClaim");
|
|
425
|
+
const stealTicket = stealClaimResolved === "ticket";
|
|
350
426
|
if (stealEnabled) {
|
|
427
|
+
if (stealTicket && headersSlotStride % 2 !== 0) {
|
|
428
|
+
throw new RangeError("ticket headers must have an 8-byte aligned slot stride");
|
|
429
|
+
}
|
|
351
430
|
if (stealRegionLanes < 1 || (stealRegionLanes & (stealRegionLanes - 1)) !== 0) {
|
|
352
431
|
throw new RangeError("regionLanes must be a power of two");
|
|
353
432
|
}
|
|
354
|
-
|
|
433
|
+
// Dekker requires at least one region per consumer.
|
|
434
|
+
if (!stealTicket && stealRegions < stealConsumers) {
|
|
355
435
|
throw new RangeError(`regionLanes=${stealRegionLanes} yields ${stealRegions} regions, ` +
|
|
356
436
|
`too few for ${stealConsumers} consumers`);
|
|
357
437
|
}
|
|
@@ -368,11 +448,41 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
368
448
|
STEAL_WANT_SLOT_OFFSET_U32;
|
|
369
449
|
}
|
|
370
450
|
const stealLiveIndex = LockBound.header + STEAL_LIVE_SLOT_OFFSET_U32;
|
|
451
|
+
/** Head (consumer-CAS) and tail (producer-store) on separate slot lines. */
|
|
452
|
+
const stealTicketHeadIndex = LockBound.header +
|
|
453
|
+
STEAL_TICKET_HEAD_SLOT_OFFSET_U32;
|
|
454
|
+
const stealTicketTailIndex = headersSlotStride + LockBound.header +
|
|
455
|
+
STEAL_TICKET_HEAD_SLOT_OFFSET_U32;
|
|
456
|
+
const stealTicketTailWord = stealEnabled && stealTicket
|
|
457
|
+
? new Int32Array(headersRegion.sab, headersRegion.byteOffset + stealTicketTailIndex * Int32Array.BYTES_PER_ELEMENT, 1)
|
|
458
|
+
: undefined;
|
|
459
|
+
const stealTicketView = stealEnabled && stealTicket
|
|
460
|
+
? new BigInt64Array(headersRegion.sab, headersRegion.byteOffset, headersRegion.byteLength >>> 3)
|
|
461
|
+
: undefined;
|
|
462
|
+
const stealTicketHead64 = stealTicketHeadIndex >>> 1;
|
|
463
|
+
const stealTicketOrderIndex = new Int32Array(LockBound.slots);
|
|
464
|
+
for (let slot = 0; slot < LockBound.slots; slot++) {
|
|
465
|
+
stealTicketOrderIndex[slot] = (slot * headersSlotStride) +
|
|
466
|
+
LockBound.header + STEAL_TICKET_ORDER_SLOT_OFFSET_U32;
|
|
467
|
+
}
|
|
468
|
+
/** A claimant drains at most one region's worth of tickets per CAS. */
|
|
469
|
+
const stealTicketBatch = stealRegionLanes;
|
|
470
|
+
const stealTicketSlots = new Int32Array(stealTicketBatch);
|
|
471
|
+
const stealTicketPublish = stealEnabled && stealIsProducer && stealTicket;
|
|
472
|
+
/** Producer-private shadow of the tail; the producer is its only writer. */
|
|
473
|
+
let stealTicketTail = 0 | 0;
|
|
474
|
+
/** At most 32 tickets can be staged, so equality is safe across uint32 wrap. */
|
|
475
|
+
let stealTicketPublishedTail = 0 | 0;
|
|
371
476
|
const stealAllLiveMask = stealConsumers === 32
|
|
372
477
|
? -1
|
|
373
478
|
: ((1 << stealConsumers) - 1) | 0;
|
|
374
479
|
if (stealEnabled && stealIsProducer) {
|
|
375
480
|
Atomics.store(stealView, stealLiveIndex, stealAllLiveMask);
|
|
481
|
+
// A carpet can outlive one pool on a reused buffer; never inherit owners.
|
|
482
|
+
if (stealTicket) {
|
|
483
|
+
Atomics.store(stealTicketView, stealTicketHead64, 0n);
|
|
484
|
+
Atomics.store(stealTicketTailWord, 0, 0);
|
|
485
|
+
}
|
|
376
486
|
}
|
|
377
487
|
const stealIsLive = (mask, consumer) => (mask & (1 << consumer)) !== 0;
|
|
378
488
|
let promiseHandler;
|
|
@@ -391,6 +501,8 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
391
501
|
lockSector: payloadLockRegion,
|
|
392
502
|
textCompat: resolvedTextCompat,
|
|
393
503
|
processBoundary,
|
|
504
|
+
sharedReturn,
|
|
505
|
+
moveReturn,
|
|
394
506
|
onPromise: (task, isRejected, value) => {
|
|
395
507
|
if ((task[TASK_LOCAL_FLAGS_INDEX] & TASK_LOCAL_PROMISE_TRACKED_FLAG) !==
|
|
396
508
|
0 &&
|
|
@@ -422,9 +534,9 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
422
534
|
const resolved = resultList ?? new RingQueue();
|
|
423
535
|
let deferredCount = 0 | 0;
|
|
424
536
|
let pendingPromiseCount = 0 | 0;
|
|
425
|
-
// Atomics aliases (hot path)
|
|
426
537
|
const a_load = Atomics.load;
|
|
427
538
|
const a_store = Atomics.store;
|
|
539
|
+
const a_compareExchange = Atomics.compareExchange;
|
|
428
540
|
const a_notify = Atomics.notify;
|
|
429
541
|
const shouldNotifyHostPublish = notifyOnHostPublish === true;
|
|
430
542
|
const a_waitAsync = typeof Atomics.waitAsync === "function"
|
|
@@ -432,14 +544,13 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
432
544
|
: undefined;
|
|
433
545
|
// Sender-side cached shadow of the receiver-owned queue word. Under the XSC
|
|
434
546
|
// false-busy-only sender-side staleness property, this may hide newly freed
|
|
435
|
-
// lanes but cannot make a
|
|
547
|
+
// lanes but cannot make a pending lane appear free. Refresh only
|
|
436
548
|
// when the cached free set is exhausted.
|
|
437
549
|
let workerShadow = 0 | 0;
|
|
438
550
|
// Under stealing, workerBits also records retired lanes.
|
|
439
551
|
const refreshWorkerShadow = () => workerShadow = a_load(workerBits, 0) | 0;
|
|
440
552
|
refreshWorkerShadow();
|
|
441
553
|
const ensureSenderStateHasFree = (state) => (~state) !== 0 ? state : (LastLocal ^ refreshWorkerShadow()) | 0;
|
|
442
|
-
// RingQueue method aliases (hot path)
|
|
443
554
|
const toBeSentPush = (task) => toBeSent.push(task);
|
|
444
555
|
const toBeSentShift = () => toBeSent.shiftNoClear();
|
|
445
556
|
const toBeSentUnshift = (task) => toBeSent.unshift(task);
|
|
@@ -476,83 +587,93 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
476
587
|
const encodeManyFrom = (list) => {
|
|
477
588
|
let state = ensureSenderStateHasFree((LastLocal ^ workerShadow) | 0);
|
|
478
589
|
let encoded = 0 | 0;
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
590
|
+
try {
|
|
591
|
+
if (list === toBeSent) {
|
|
592
|
+
while (true) {
|
|
593
|
+
const task = toBeSentShift();
|
|
594
|
+
if (!task)
|
|
595
|
+
break;
|
|
596
|
+
state = ensureSenderStateHasFree(state);
|
|
597
|
+
const bit = encodeWithState(task, state) | 0;
|
|
598
|
+
if (bit === 0) {
|
|
599
|
+
toBeSentUnshift(task);
|
|
600
|
+
break;
|
|
601
|
+
}
|
|
602
|
+
state = (state ^ bit) | 0;
|
|
603
|
+
encoded = (encoded + 1) | 0;
|
|
489
604
|
}
|
|
490
|
-
state = (state ^ bit) | 0;
|
|
491
|
-
encoded = (encoded + 1) | 0;
|
|
492
605
|
}
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
606
|
+
else {
|
|
607
|
+
while (true) {
|
|
608
|
+
const task = list.shiftNoClear();
|
|
609
|
+
if (!task)
|
|
610
|
+
break;
|
|
611
|
+
state = ensureSenderStateHasFree(state);
|
|
612
|
+
const bit = encodeWithState(task, state) | 0;
|
|
613
|
+
if (bit === 0) {
|
|
614
|
+
list.unshift(task);
|
|
615
|
+
break;
|
|
616
|
+
}
|
|
617
|
+
state = (state ^ bit) | 0;
|
|
618
|
+
encoded = (encoded + 1) | 0;
|
|
504
619
|
}
|
|
505
|
-
state = (state ^ bit) | 0;
|
|
506
|
-
encoded = (encoded + 1) | 0;
|
|
507
620
|
}
|
|
621
|
+
return encoded;
|
|
622
|
+
}
|
|
623
|
+
finally {
|
|
624
|
+
flushTicketTail();
|
|
508
625
|
}
|
|
509
|
-
return encoded;
|
|
510
626
|
};
|
|
511
627
|
const encodeManyTrackedFrom = (list) => {
|
|
512
628
|
let state = ensureSenderStateHasFree((LastLocal ^ workerShadow) | 0);
|
|
513
629
|
let encoded = 0 | 0;
|
|
514
630
|
deferredCount = 0 | 0;
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
if (
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
631
|
+
try {
|
|
632
|
+
if (list === toBeSent) {
|
|
633
|
+
while (true) {
|
|
634
|
+
const task = toBeSentShift();
|
|
635
|
+
if (!task)
|
|
636
|
+
break;
|
|
637
|
+
state = ensureSenderStateHasFree(state);
|
|
638
|
+
const bit = encodeWithState(task, state) | 0;
|
|
639
|
+
if (bit === 0) {
|
|
640
|
+
if (isPromisePayloadPending(task)) {
|
|
641
|
+
deferredCount = (deferredCount + 1) | 0;
|
|
642
|
+
trackDeferredTask(task);
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
toBeSentUnshift(task);
|
|
646
|
+
break;
|
|
527
647
|
}
|
|
528
|
-
|
|
529
|
-
|
|
648
|
+
state = (state ^ bit) | 0;
|
|
649
|
+
encoded = (encoded + 1) | 0;
|
|
530
650
|
}
|
|
531
|
-
state = (state ^ bit) | 0;
|
|
532
|
-
encoded = (encoded + 1) | 0;
|
|
533
651
|
}
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
652
|
+
else {
|
|
653
|
+
while (true) {
|
|
654
|
+
const task = list.shiftNoClear();
|
|
655
|
+
if (!task)
|
|
656
|
+
break;
|
|
657
|
+
state = ensureSenderStateHasFree(state);
|
|
658
|
+
const bit = encodeWithState(task, state) | 0;
|
|
659
|
+
if (bit === 0) {
|
|
660
|
+
if (isPromisePayloadPending(task)) {
|
|
661
|
+
deferredCount = (deferredCount + 1) | 0;
|
|
662
|
+
trackDeferredTask(task);
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
list.unshift(task);
|
|
666
|
+
break;
|
|
547
667
|
}
|
|
548
|
-
|
|
549
|
-
|
|
668
|
+
state = (state ^ bit) | 0;
|
|
669
|
+
encoded = (encoded + 1) | 0;
|
|
550
670
|
}
|
|
551
|
-
state = (state ^ bit) | 0;
|
|
552
|
-
encoded = (encoded + 1) | 0;
|
|
553
671
|
}
|
|
672
|
+
return encoded;
|
|
673
|
+
}
|
|
674
|
+
finally {
|
|
675
|
+
flushTicketTail();
|
|
554
676
|
}
|
|
555
|
-
return encoded;
|
|
556
677
|
};
|
|
557
678
|
const encodeAll = () => {
|
|
558
679
|
if (toBeSent.isEmpty)
|
|
@@ -561,11 +682,28 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
561
682
|
deferredCount = 0 | 0;
|
|
562
683
|
return toBeSent.isEmpty;
|
|
563
684
|
};
|
|
685
|
+
const notifyHostPublication = () => {
|
|
686
|
+
if (a_compareExchange(doorbellArmed, 0, DOORBELL_ARMED, DOORBELL_SIGNALLED) === DOORBELL_ARMED) {
|
|
687
|
+
if (notifyHostPublish !== undefined) {
|
|
688
|
+
try {
|
|
689
|
+
notifyHostPublish();
|
|
690
|
+
}
|
|
691
|
+
catch {
|
|
692
|
+
// A callback can be unavailable after forced worker teardown. Restore
|
|
693
|
+
// the arm only when it still belongs to this publication, so a later
|
|
694
|
+
// result can retry without overwriting a host-side disarm/re-arm.
|
|
695
|
+
a_compareExchange(doorbellArmed, 0, DOORBELL_SIGNALLED, DOORBELL_ARMED);
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
else {
|
|
699
|
+
a_notify(hostBits, 0, 1);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
};
|
|
564
703
|
const storeHost = (bit) => {
|
|
565
704
|
a_store(hostBits, 0, LastLocal = (LastLocal ^ bit) | 0);
|
|
566
|
-
if (shouldNotifyHostPublish
|
|
567
|
-
|
|
568
|
-
}
|
|
705
|
+
if (shouldNotifyHostPublish)
|
|
706
|
+
notifyHostPublication();
|
|
569
707
|
};
|
|
570
708
|
const storeWorker = (bit) => a_store(workerBits, 0, LastWorker = (LastWorker ^ bit) | 0);
|
|
571
709
|
const encode = (task, state = (LastLocal ^ workerShadow) | 0) => {
|
|
@@ -576,7 +714,12 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
576
714
|
if (!encodeTaskValue(task, selectedSlotIndex = 31 - clz32(free))) {
|
|
577
715
|
return false;
|
|
578
716
|
}
|
|
579
|
-
|
|
717
|
+
try {
|
|
718
|
+
return encodeAt(task, selectedSlotIndex, selectedSlotBit = 1 << selectedSlotIndex);
|
|
719
|
+
}
|
|
720
|
+
finally {
|
|
721
|
+
flushTicketTail();
|
|
722
|
+
}
|
|
580
723
|
};
|
|
581
724
|
const encodeTracked = (task, state = (LastLocal ^ workerShadow) | 0) => {
|
|
582
725
|
deferredCount = 0 | 0;
|
|
@@ -591,7 +734,12 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
591
734
|
}
|
|
592
735
|
return false;
|
|
593
736
|
}
|
|
594
|
-
|
|
737
|
+
try {
|
|
738
|
+
return encodeAt(task, selectedSlotIndex, selectedSlotBit = 1 << selectedSlotIndex);
|
|
739
|
+
}
|
|
740
|
+
finally {
|
|
741
|
+
flushTicketTail();
|
|
742
|
+
}
|
|
595
743
|
};
|
|
596
744
|
const encodeAt = (task, at, bit) => {
|
|
597
745
|
const off = (at * headersSlotStride) + slotBaseU32;
|
|
@@ -603,13 +751,40 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
603
751
|
headersBuffer[off + 5] = task[5];
|
|
604
752
|
headersBuffer[off + 6] = task[6];
|
|
605
753
|
headersBuffer[off + TASK_LOCAL_FLAGS_INDEX] = 0;
|
|
754
|
+
if (stealTicketPublish) {
|
|
755
|
+
// The tail's seq-cst store publishes the header, ring cell and host bit
|
|
756
|
+
// together. Ticket consumers acquire the tail before reading a lane;
|
|
757
|
+
// unlike region consumers, they never arbitrate through hostBits.
|
|
758
|
+
stealView[stealTicketOrderIndex[stealTicketTail & STEAL_TICKET_RING_MASK]] = at;
|
|
759
|
+
stealTicketTail = (stealTicketTail + 1) | 0;
|
|
760
|
+
if (shouldNotifyHostPublish) {
|
|
761
|
+
a_store(hostBits, 0, LastLocal = (LastLocal ^ bit) | 0);
|
|
762
|
+
}
|
|
763
|
+
else {
|
|
764
|
+
hostBits[0] = LastLocal = (LastLocal ^ bit) | 0;
|
|
765
|
+
}
|
|
766
|
+
// The tail store is deferred so a burst pays one atomic, not one each.
|
|
767
|
+
// Nothing is claimable until it lands, which is what makes the batch
|
|
768
|
+
// atomic; every exit from an encode path must flush, including throws.
|
|
769
|
+
return true;
|
|
770
|
+
}
|
|
606
771
|
storeHost(bit);
|
|
607
772
|
return true;
|
|
608
773
|
};
|
|
609
|
-
const hasSpace = () => (hostBits[0] ^ LastWorker) !== 0;
|
|
610
774
|
/**
|
|
611
|
-
*
|
|
775
|
+
* Publish accumulated tickets. Consumers cannot see any lane of the batch
|
|
776
|
+
* until this store lands, so skipping it strands the slots forever.
|
|
612
777
|
*/
|
|
778
|
+
const flushTicketTail = () => {
|
|
779
|
+
if (stealTicketPublishedTail === stealTicketTail)
|
|
780
|
+
return;
|
|
781
|
+
stealTicketPublishedTail = stealTicketTail;
|
|
782
|
+
a_store(stealTicketTailWord, 0, stealTicketTail);
|
|
783
|
+
// A notifier must not run while its tickets are still invisible.
|
|
784
|
+
if (shouldNotifyHostPublish)
|
|
785
|
+
notifyHostPublication();
|
|
786
|
+
};
|
|
787
|
+
const hasSpace = () => (hostBits[0] ^ LastWorker) !== 0;
|
|
613
788
|
const decode = () => {
|
|
614
789
|
let diff = (a_load(hostBits, 0) ^ LastWorker) | 0;
|
|
615
790
|
if (diff === 0)
|
|
@@ -640,21 +815,21 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
640
815
|
lastTake = last;
|
|
641
816
|
return true;
|
|
642
817
|
};
|
|
643
|
-
/**
|
|
644
|
-
* WORKER SIDE: decode, region-Dekker stealing variant.
|
|
645
|
-
*
|
|
646
|
-
* Claims one whole region of `stealRegionLanes` lanes per handshake, so the
|
|
647
|
-
* StoreLoad barrier is amortised over the region rather than paid per task.
|
|
648
|
-
* Priority is fixed by endpoint id: a junior withdraws for a senior, so at
|
|
649
|
-
* most one consumer ever acknowledges a published generation.
|
|
650
|
-
*
|
|
651
|
-
* The C reference needs an explicit mfence between the intent store and the
|
|
652
|
-
* peer loads. `Atomics.store` is sequentially consistent in JS, so that
|
|
653
|
-
* barrier is implicit here and holds on x86 and ARM alike.
|
|
654
|
-
*/
|
|
655
818
|
const stealLaneMask = (region) => stealRegionLanes === 32
|
|
656
819
|
? -1
|
|
657
820
|
: ((((1 << stealRegionLanes) - 1) << (region * stealRegionLanes)) | 0);
|
|
821
|
+
const stealRegionBits = stealRegionLanes === 32
|
|
822
|
+
? -1
|
|
823
|
+
: ((1 << stealRegionLanes) - 1) | 0;
|
|
824
|
+
/** Find the oldest pending lane in a claimed region. */
|
|
825
|
+
const stealOldestLane = (region, take) => {
|
|
826
|
+
const low = (region * stealRegionLanes) | 0;
|
|
827
|
+
const lanes = ((take >>> low) & stealRegionBits) | 0;
|
|
828
|
+
const predecessors = ((lanes >>> 1) | (lanes << (stealRegionLanes - 1))) & stealRegionBits;
|
|
829
|
+
const oldest = (lanes & ~predecessors) | 0;
|
|
830
|
+
return low +
|
|
831
|
+
(oldest === 0 ? stealRegionLanes - 1 : 31 - clz32(oldest >>> 0));
|
|
832
|
+
};
|
|
658
833
|
const stealJuniorWants = (intent) => {
|
|
659
834
|
const live = a_load(stealView, stealLiveIndex) | 0;
|
|
660
835
|
for (let c = stealId + 1; c < stealConsumers; c++) {
|
|
@@ -667,6 +842,7 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
667
842
|
};
|
|
668
843
|
const stealHome = ((stealRegions * stealId) / stealConsumers) | 0;
|
|
669
844
|
let stealCursor = stealHome;
|
|
845
|
+
/** Decode one region using the Dekker claim protocol. */
|
|
670
846
|
const decodeSteal = () => {
|
|
671
847
|
// workerBits includes retired lanes under stealing.
|
|
672
848
|
const pending = (a_load(hostBits, 0) ^ a_load(workerBits, 0)) | 0;
|
|
@@ -682,6 +858,8 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
682
858
|
}
|
|
683
859
|
}
|
|
684
860
|
const liveBeforeClaim = a_load(stealView, stealLiveIndex) | 0;
|
|
861
|
+
// Prefer regions no live peer currently wants; the handshake below provides
|
|
862
|
+
// the actual mutual exclusion.
|
|
685
863
|
let peerIntent = 0 | 0;
|
|
686
864
|
let seniorIntent = 0 | 0;
|
|
687
865
|
for (let c = 0; c < stealConsumers; c++) {
|
|
@@ -709,59 +887,198 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
709
887
|
return false;
|
|
710
888
|
const intent = (1 << region) | 0;
|
|
711
889
|
a_store(stealView, stealWantIndex[stealId], intent);
|
|
712
|
-
//
|
|
890
|
+
// Atomics.store is sequentially consistent, so it supplies the StoreLoad
|
|
891
|
+
// barrier required by Dekker.
|
|
892
|
+
// A senior claimant has priority.
|
|
713
893
|
let seniorConflict = false;
|
|
714
|
-
let
|
|
715
|
-
|
|
716
|
-
if (c === stealId || !stealIsLive(liveBeforeClaim, c))
|
|
894
|
+
for (let c = 0; c < stealId; c++) {
|
|
895
|
+
if (!stealIsLive(liveBeforeClaim, c))
|
|
717
896
|
continue;
|
|
718
|
-
if ((a_load(stealView, stealWantIndex[c]) & intent)
|
|
719
|
-
continue;
|
|
720
|
-
if (c < stealId)
|
|
897
|
+
if ((a_load(stealView, stealWantIndex[c]) & intent) !== 0) {
|
|
721
898
|
seniorConflict = true;
|
|
722
|
-
|
|
723
|
-
|
|
899
|
+
break;
|
|
900
|
+
}
|
|
724
901
|
}
|
|
725
902
|
if (seniorConflict) {
|
|
726
903
|
a_store(stealView, stealWantIndex[stealId], 0);
|
|
727
904
|
return false;
|
|
728
905
|
}
|
|
906
|
+
let juniorConflict = false;
|
|
907
|
+
for (let c = stealId + 1; c < stealConsumers; c++) {
|
|
908
|
+
if (!stealIsLive(liveBeforeClaim, c))
|
|
909
|
+
continue;
|
|
910
|
+
if ((a_load(stealView, stealWantIndex[c]) & intent) !== 0) {
|
|
911
|
+
juniorConflict = true;
|
|
912
|
+
break;
|
|
913
|
+
}
|
|
914
|
+
}
|
|
729
915
|
if (juniorConflict) {
|
|
730
916
|
while (stealJuniorWants(intent)) { /* junior withdraws */ }
|
|
731
917
|
}
|
|
732
|
-
//
|
|
733
|
-
// until every slot in `take` has been decoded.
|
|
918
|
+
// No control word is read again until all selected slots are decoded.
|
|
734
919
|
const take = ((a_load(hostBits, 0) ^ a_load(workerBits, 0)) &
|
|
735
920
|
stealLaneMask(region)) | 0;
|
|
736
921
|
if (take === 0) {
|
|
737
922
|
a_store(stealView, stealWantIndex[stealId], 0);
|
|
738
923
|
return false;
|
|
739
924
|
}
|
|
740
|
-
// Retire
|
|
925
|
+
// Retire lanes and release the claim even if decoding throws.
|
|
741
926
|
let lanes = take;
|
|
742
927
|
let done = 0 | 0;
|
|
928
|
+
// Decode the claimed lanes in producer order.
|
|
929
|
+
let at = stealOldestLane(region, take);
|
|
930
|
+
const regionLow = (region * stealRegionLanes) | 0;
|
|
931
|
+
const regionHigh = (regionLow + stealRegionLanes - 1) | 0;
|
|
743
932
|
try {
|
|
744
|
-
|
|
745
|
-
const bit = (
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
933
|
+
for (let step = 0; step < stealRegionLanes; step++) {
|
|
934
|
+
const bit = (1 << at) | 0;
|
|
935
|
+
if ((lanes & bit) !== 0) {
|
|
936
|
+
decodeAt(at);
|
|
937
|
+
lanes = (lanes ^ bit) | 0;
|
|
938
|
+
done = (done ^ bit) | 0;
|
|
939
|
+
if (lanes === 0)
|
|
940
|
+
break;
|
|
941
|
+
}
|
|
942
|
+
at = at === regionLow ? regionHigh : at - 1;
|
|
749
943
|
}
|
|
750
944
|
}
|
|
751
945
|
finally {
|
|
752
|
-
// ACK before clearing intent.
|
|
946
|
+
// Publish the ACK before clearing intent.
|
|
753
947
|
LastWorker = (LastWorker ^ done) | 0;
|
|
754
948
|
if (done !== 0)
|
|
755
949
|
Atomics.xor(workerBits, 0, done);
|
|
756
950
|
a_store(stealView, stealWantIndex[stealId], 0);
|
|
757
|
-
// Restart from this consumer's home region.
|
|
758
951
|
stealCursor = stealHome;
|
|
952
|
+
// Maybe in the future
|
|
953
|
+
// stealCursor = (region + 1) % stealRegions;
|
|
759
954
|
}
|
|
760
955
|
return true;
|
|
761
956
|
};
|
|
762
957
|
/**
|
|
763
|
-
*
|
|
958
|
+
* Claim tickets from one monotonic counter.
|
|
959
|
+
*
|
|
960
|
+
* The ticket is the arbiter and the FIFO order at once: a winning CAS owns
|
|
961
|
+
* lanes nobody else can name, so there are no regions, no intent words, no
|
|
962
|
+
* peer survey and no liveness mask. An idle poll costs two loads and no RMW.
|
|
764
963
|
*/
|
|
964
|
+
const decodeStealTicket = () => {
|
|
965
|
+
for (let attempt = 0; attempt < STEAL_TICKET_ATTEMPTS; attempt++) {
|
|
966
|
+
const head = Atomics.load(stealTicketView, stealTicketHead64);
|
|
967
|
+
if (head === STEAL_TICKET_FAILED)
|
|
968
|
+
return false;
|
|
969
|
+
const lowHead = Number(head & STEAL_TICKET_LOW_MASK);
|
|
970
|
+
const available = (a_load(stealTicketTailWord, 0) - lowHead) >>> 0;
|
|
971
|
+
if (available === 0)
|
|
972
|
+
return false;
|
|
973
|
+
// A valid head has at most 32 published tickets ahead of it. Larger
|
|
974
|
+
// distances are stale snapshots; the 64-bit head is still the arbiter.
|
|
975
|
+
if (available > LockBound.slots)
|
|
976
|
+
continue;
|
|
977
|
+
const batch = Math.min(available, stealTicketBatch);
|
|
978
|
+
const next = head + BigInt(batch);
|
|
979
|
+
if (next > STEAL_TICKET_LIMIT) {
|
|
980
|
+
Atomics.store(stealTicketView, stealTicketHead64, STEAL_TICKET_FAILED);
|
|
981
|
+
throw new RangeError("ticket queue exhausted its 64-bit sequence");
|
|
982
|
+
}
|
|
983
|
+
const ringHead = lowHead & STEAL_TICKET_RING_MASK;
|
|
984
|
+
// Read the lane ids *before* the CAS. Cell `head & 31` cannot be refilled
|
|
985
|
+
// while the head still sits at `head` (refilling it needs a free slot,
|
|
986
|
+
// which needs a retire, which needs a claim), but once the CAS moves the
|
|
987
|
+
// head past a ticket the producer may reuse that cell immediately.
|
|
988
|
+
for (let i = 0; i < batch; i++) {
|
|
989
|
+
stealTicketSlots[i] = stealView[stealTicketOrderIndex[(ringHead + i) & STEAL_TICKET_RING_MASK]] & STEAL_TICKET_RING_MASK;
|
|
990
|
+
}
|
|
991
|
+
if (Atomics.compareExchange(stealTicketView, stealTicketHead64, head, next) !== head)
|
|
992
|
+
continue;
|
|
993
|
+
let done = 0 | 0;
|
|
994
|
+
try {
|
|
995
|
+
for (let i = 0; i < batch; i++) {
|
|
996
|
+
const at = stealTicketSlots[i];
|
|
997
|
+
decodeAt(at);
|
|
998
|
+
done = (done ^ (1 << at)) | 0;
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
catch (error) {
|
|
1002
|
+
// Decoding may already have consumed payload state. Replaying the
|
|
1003
|
+
// unfinished batch is unsafe; report the fatal error and stop claims.
|
|
1004
|
+
Atomics.store(stealTicketView, stealTicketHead64, STEAL_TICKET_FAILED);
|
|
1005
|
+
throw error;
|
|
1006
|
+
}
|
|
1007
|
+
finally {
|
|
1008
|
+
LastWorker = (LastWorker ^ done) | 0;
|
|
1009
|
+
if (done !== 0)
|
|
1010
|
+
Atomics.xor(workerBits, 0, done);
|
|
1011
|
+
}
|
|
1012
|
+
return true;
|
|
1013
|
+
}
|
|
1014
|
+
return false;
|
|
1015
|
+
};
|
|
1016
|
+
/**
|
|
1017
|
+
* Debug-only wrapper around a steal claim, for the `steal` namespace.
|
|
1018
|
+
*
|
|
1019
|
+
* It is selected once, when `decode` is chosen below, so an untraced lock
|
|
1020
|
+
* hands out the bare claim function: no flag test, no extra load, no change
|
|
1021
|
+
* to the claim path. Everything reported is read from outside the claim,
|
|
1022
|
+
* before and after it, so the wrapper cannot perturb the protocol either.
|
|
1023
|
+
*
|
|
1024
|
+
* - `lanes`: in ticket order for `ticket`, ascending for `dekker`.
|
|
1025
|
+
* - `head`: ticket head before and after. `peers` is how many tickets other
|
|
1026
|
+
* consumers took in between; 0 means `head` spans exactly this claim.
|
|
1027
|
+
* - `region`/`home`: `dekker` only; `steal` marks a region off this
|
|
1028
|
+
* consumer's home.
|
|
1029
|
+
* - `lost`: polls since the last claim that saw work and still came back
|
|
1030
|
+
* empty, i.e. a peer got there first.
|
|
1031
|
+
*/
|
|
1032
|
+
const traceStealClaims = (claim, trace) => {
|
|
1033
|
+
let lost = 0;
|
|
1034
|
+
return () => {
|
|
1035
|
+
const before = LastWorker;
|
|
1036
|
+
let head = 0n;
|
|
1037
|
+
let visible = 0;
|
|
1038
|
+
if (stealTicket) {
|
|
1039
|
+
head = Atomics.load(stealTicketView, stealTicketHead64);
|
|
1040
|
+
if (head !== STEAL_TICKET_FAILED) {
|
|
1041
|
+
visible = (a_load(stealTicketTailWord, 0) -
|
|
1042
|
+
Number(head & STEAL_TICKET_LOW_MASK)) >>> 0;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
else {
|
|
1046
|
+
visible = (a_load(hostBits, 0) ^ a_load(workerBits, 0)) | 0;
|
|
1047
|
+
}
|
|
1048
|
+
if (!claim()) {
|
|
1049
|
+
if (visible !== 0)
|
|
1050
|
+
lost++;
|
|
1051
|
+
return false;
|
|
1052
|
+
}
|
|
1053
|
+
const taken = (LastWorker ^ before) | 0;
|
|
1054
|
+
let count = 0;
|
|
1055
|
+
for (let bits = taken; bits !== 0; bits &= bits - 1)
|
|
1056
|
+
count++;
|
|
1057
|
+
let message;
|
|
1058
|
+
if (stealTicket) {
|
|
1059
|
+
const after = Atomics.load(stealTicketView, stealTicketHead64);
|
|
1060
|
+
const lanes = [];
|
|
1061
|
+
for (let i = 0; i < count; i++)
|
|
1062
|
+
lanes.push(stealTicketSlots[i]);
|
|
1063
|
+
message = `claim count=${count} lanes=${lanes.join(",")}` +
|
|
1064
|
+
` head=${head}→${after} peers=${Number(after - head) - count}`;
|
|
1065
|
+
}
|
|
1066
|
+
else {
|
|
1067
|
+
const lanes = [];
|
|
1068
|
+
for (let lane = 0; lane < LockBound.slots; lane++) {
|
|
1069
|
+
if ((taken & (1 << lane)) !== 0)
|
|
1070
|
+
lanes.push(lane);
|
|
1071
|
+
}
|
|
1072
|
+
const region = (lanes[0] / stealRegionLanes) | 0;
|
|
1073
|
+
message = `claim count=${count} lanes=${lanes.join(",")}` +
|
|
1074
|
+
` region=${region} home=${stealHome}` +
|
|
1075
|
+
(region === stealHome ? "" : " steal");
|
|
1076
|
+
}
|
|
1077
|
+
trace(`${message} lost=${lost}`);
|
|
1078
|
+
lost = 0;
|
|
1079
|
+
return true;
|
|
1080
|
+
};
|
|
1081
|
+
};
|
|
765
1082
|
const resolveHost = ({ queue, onResolved, shouldSettle, activeRejectPlaceholder, }) => {
|
|
766
1083
|
const getTask = takeTask({ queue });
|
|
767
1084
|
let lastResolved = 32;
|
|
@@ -887,27 +1204,21 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
887
1204
|
return modified;
|
|
888
1205
|
};
|
|
889
1206
|
};
|
|
890
|
-
/**
|
|
891
|
-
* HOST SIDE: wait until the producer changes hostBits.
|
|
892
|
-
*
|
|
893
|
-
* LastWorker is the host's acknowledgement shadow. Passing it as the
|
|
894
|
-
* expected value makes the arm race-free: a publication between the drain
|
|
895
|
-
* and this call returns `not-equal` synchronously instead of being lost.
|
|
896
|
-
*/
|
|
1207
|
+
/** Wait for a producer publication without losing an intervening wakeup. */
|
|
897
1208
|
const waitForHostChange = (timeoutMs) => {
|
|
898
1209
|
if (a_waitAsync === undefined) {
|
|
899
|
-
a_store(doorbellArmed, 0,
|
|
1210
|
+
a_store(doorbellArmed, 0, DOORBELL_OFF);
|
|
900
1211
|
return undefined;
|
|
901
1212
|
}
|
|
902
|
-
a_store(doorbellArmed, 0,
|
|
1213
|
+
a_store(doorbellArmed, 0, DOORBELL_ARMED);
|
|
903
1214
|
try {
|
|
904
1215
|
const wait = a_waitAsync(hostBits, 0, LastWorker | 0, timeoutMs);
|
|
905
1216
|
if (!wait.async)
|
|
906
|
-
a_store(doorbellArmed, 0,
|
|
1217
|
+
a_store(doorbellArmed, 0, DOORBELL_OFF);
|
|
907
1218
|
return wait;
|
|
908
1219
|
}
|
|
909
1220
|
catch {
|
|
910
|
-
a_store(doorbellArmed, 0,
|
|
1221
|
+
a_store(doorbellArmed, 0, DOORBELL_OFF);
|
|
911
1222
|
// Some runtimes reject particular SharedArrayBuffer implementations
|
|
912
1223
|
// (for example a growable or native-backed buffer). Fall back to the
|
|
913
1224
|
// existing dispatcher rather than turning a capability issue into a
|
|
@@ -915,6 +1226,18 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
915
1226
|
return undefined;
|
|
916
1227
|
}
|
|
917
1228
|
};
|
|
1229
|
+
/**
|
|
1230
|
+
* Arm a native host notifier with the same no-lost-wakeup property as
|
|
1231
|
+
* waitAsync: a publication before or during the arm is observed directly,
|
|
1232
|
+
* while one after the observation sees the shared armed bit and rings.
|
|
1233
|
+
*/
|
|
1234
|
+
const armHostNotifier = () => {
|
|
1235
|
+
a_store(doorbellArmed, 0, DOORBELL_ARMED);
|
|
1236
|
+
if (a_load(hostBits, 0) === (LastWorker | 0))
|
|
1237
|
+
return true;
|
|
1238
|
+
a_store(doorbellArmed, 0, DOORBELL_OFF);
|
|
1239
|
+
return false;
|
|
1240
|
+
};
|
|
918
1241
|
const decodeAt = (at) => {
|
|
919
1242
|
const off = (at * headersSlotStride) + slotBaseU32;
|
|
920
1243
|
const recycled = recycleShift();
|
|
@@ -972,6 +1295,11 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
972
1295
|
}
|
|
973
1296
|
const bit = 1 << id;
|
|
974
1297
|
const previous = Atomics.and(stealView, stealLiveIndex, ~bit);
|
|
1298
|
+
if (stealTicket) {
|
|
1299
|
+
// No durable ownership record exists for a dead claimant. The host must
|
|
1300
|
+
// close the shared pending registry instead of reusing this queue.
|
|
1301
|
+
Atomics.store(stealTicketView, stealTicketHead64, STEAL_TICKET_FAILED);
|
|
1302
|
+
}
|
|
975
1303
|
return (previous & bit) !== 0;
|
|
976
1304
|
};
|
|
977
1305
|
return {
|
|
@@ -981,7 +1309,12 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
981
1309
|
encodeAll,
|
|
982
1310
|
publish,
|
|
983
1311
|
flushPending,
|
|
984
|
-
|
|
1312
|
+
// Chosen once: an untraced lock gets the claim function itself.
|
|
1313
|
+
decode: stealEnabled
|
|
1314
|
+
? (traceClaim === undefined
|
|
1315
|
+
? (stealTicket ? decodeStealTicket : decodeSteal)
|
|
1316
|
+
: traceStealClaims(stealTicket ? decodeStealTicket : decodeSteal, traceClaim))
|
|
1317
|
+
: decode,
|
|
985
1318
|
hasSpace,
|
|
986
1319
|
resolved,
|
|
987
1320
|
hostBits,
|
|
@@ -989,8 +1322,9 @@ export const lock2 = ({ headers, headerSlotStrideU32, LockBoundSector, payload,
|
|
|
989
1322
|
recyclecList,
|
|
990
1323
|
resolveHost,
|
|
991
1324
|
waitForHostChange,
|
|
1325
|
+
armHostNotifier,
|
|
992
1326
|
setHostWaiterArmed: (armed) => {
|
|
993
|
-
a_store(doorbellArmed, 0, armed ?
|
|
1327
|
+
a_store(doorbellArmed, 0, armed ? DOORBELL_ARMED : DOORBELL_OFF);
|
|
994
1328
|
},
|
|
995
1329
|
hasPendingFrames: () => toBeSent.size !== 0,
|
|
996
1330
|
getPendingFrameCount: () => toBeSent.size | 0,
|