@typeberry/jam 0.1.3-135961b → 0.1.3-462ca77

package/bootstrap-importer.mjs

@@ -4572,6 +4572,20 @@ const result_Result = {
     },
 };
 
+;// CONCATENATED MODULE: ./packages/core/utils/safe-alloc-uint8array.ts
+// about 2GB, the maximum ArrayBuffer length on Chrome confirmed by several sources:
+// - https://issues.chromium.org/issues/40055619
+// - https://stackoverflow.com/a/72124984
+// - https://onnxruntime.ai/docs/tutorials/web/large-models.html#maximum-size-of-arraybuffer
+const MAX_LENGTH = 2145386496;
+function safe_alloc_uint8array_safeAllocUint8Array(length) {
+    if (length > MAX_LENGTH) {
+        // biome-ignore lint/suspicious/noConsole: can't have a dependency on logger here
+        console.warn(`Trying to allocate ${length} bytes, which is greater than the maximum of ${MAX_LENGTH}.`);
+    }
+    return new Uint8Array(Math.min(MAX_LENGTH, length));
+}
+
 ;// CONCATENATED MODULE: external "node:assert"
 const external_node_assert_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:assert");
 ;// CONCATENATED MODULE: ./packages/core/utils/test.ts
@@ -4815,6 +4829,7 @@ function isResult(x) {
 
 
 
+
 ;// CONCATENATED MODULE: ./packages/core/bytes/bitvec.ts
 
 /**
@@ -4836,7 +4851,7 @@ class BitVec {
      * Create new [`BitVec`] with all values set to `false`.
      */
     static empty(bitLength) {
-        const data = new Uint8Array(Math.ceil(bitLength / 8));
+        const data = safe_alloc_uint8array_safeAllocUint8Array(Math.ceil(bitLength / 8));
         return new BitVec(data, bitLength);
     }
     byteLength;
@@ -5036,7 +5051,7 @@ class bytes_BytesBlob {
     static blobFromParts(v, ...rest) {
         const vArr = v instanceof Uint8Array ? [v] : v;
         const totalLength = vArr.reduce((a, v) => a + v.length, 0) + rest.reduce((a, v) => a + v.length, 0);
-        const buffer = new Uint8Array(totalLength);
+        const buffer = safe_alloc_uint8array_safeAllocUint8Array(totalLength);
         let offset = 0;
         for (const r of vArr) {
             buffer.set(r, offset);
@@ -5109,7 +5124,7 @@ class bytes_Bytes extends bytes_BytesBlob {
     }
     /** Create an empty [`Bytes<X>`] of given length. */
     static zero(len) {
-        return new bytes_Bytes(new Uint8Array(len), len);
+        return new bytes_Bytes(safe_alloc_uint8array_safeAllocUint8Array(len), len);
     }
     // TODO [ToDr] `fill` should have the argments swapped to align with the rest.
     /** Create a [`Bytes<X>`] with all bytes filled with given input number. */
@@ -5695,7 +5710,7 @@ async function ed25519_verify(input) {
         return Promise.resolve([]);
     }
     const dataLength = input.reduce((acc, { message, key, signature }) => acc + key.length + signature.length + message.length + 1, 0);
-    const data = new Uint8Array(dataLength);
+    const data = safe_alloc_uint8array_safeAllocUint8Array(dataLength);
     let offset = 0;
     for (const { key, message, signature } of input) {
         data.set(key.raw, offset);
@@ -5782,7 +5797,7 @@ class allocator_SimpleAllocator {
 /** An allocator that works by allocating larger (continuous) pages of memory. */
 class PageAllocator {
     hashesPerPage;
-    page = new Uint8Array(0);
+    page = safeAllocUint8Array(0);
     currentHash = 0;
     // TODO [ToDr] Benchmark the performance!
     constructor(hashesPerPage) {
@@ -5793,7 +5808,7 @@ class PageAllocator {
     resetPage() {
         const pageSizeBytes = this.hashesPerPage * HASH_SIZE;
         this.currentHash = 0;
-        this.page = new Uint8Array(pageSizeBytes);
+        this.page = safeAllocUint8Array(pageSizeBytes);
     }
     emptyHash() {
         const startIdx = this.currentHash * HASH_SIZE;
@@ -6517,7 +6532,7 @@ function addSizeHints(a, b) {
     };
 }
 const DEFAULT_START_LENGTH = 512; // 512B
-const MAX_LENGTH = 10 * 1024 * 1024; // 10MB
+const encoder_MAX_LENGTH = 10 * 1024 * 1024; // 10MB
 /**
  * JAM encoder.
  */
@@ -6533,7 +6548,7 @@ class encoder_Encoder {
             return new encoder_Encoder(options.destination);
         }
         const startLength = options?.expectedLength ?? DEFAULT_START_LENGTH;
-        const buffer = new ArrayBuffer(Math.min(MAX_LENGTH, startLength), { maxByteLength: MAX_LENGTH });
+        const buffer = new ArrayBuffer(Math.min(encoder_MAX_LENGTH, startLength), { maxByteLength: encoder_MAX_LENGTH });
         const destination = new Uint8Array(buffer);
         return new encoder_Encoder(destination, buffer);
     }
@@ -6866,11 +6881,11 @@ class encoder_Encoder {
     ensureBigEnough(length, options = { silent: false }) {
         debug_check `${length >= 0} Negative length given`;
         const newLength = this.offset + length;
-        if (newLength > MAX_LENGTH) {
+        if (newLength > encoder_MAX_LENGTH) {
             if (options.silent) {
                 return;
             }
-            throw new Error(`The encoded size would reach the maximum of ${MAX_LENGTH}.`);
+            throw new Error(`The encoded size would reach the maximum of ${encoder_MAX_LENGTH}.`);
         }
         if (newLength > this.destination.length) {
             // we can try to resize the underlying buffer
@@ -6878,7 +6893,7 @@ class encoder_Encoder {
                 // make sure we at least double the size of the buffer every time.
                 const minExtend = Math.max(newLength, this.buffer.byteLength << 1);
                 // but we must never exceed the max length.
-                this.buffer.resize(Math.min(MAX_LENGTH, minExtend));
+                this.buffer.resize(Math.min(encoder_MAX_LENGTH, minExtend));
             }
             // and then check again
             if (newLength > this.destination.length) {
@@ -12036,7 +12051,7 @@ class SerializedService {
     getStorage(rawKey) {
         if (Compatibility.isLessThan(GpVersion.V0_6_7)) {
             const SERVICE_ID_BYTES = 4;
-            const serviceIdAndKey = new Uint8Array(SERVICE_ID_BYTES + rawKey.length);
+            const serviceIdAndKey = safe_alloc_uint8array_safeAllocUint8Array(SERVICE_ID_BYTES + rawKey.length);
             serviceIdAndKey.set(numbers_u32AsLeBytes(this.serviceId));
             serviceIdAndKey.set(rawKey.raw, SERVICE_ID_BYTES);
             const key = opaque_asOpaqueType(bytes_BytesBlob.blobFrom(hashBytes(serviceIdAndKey).raw));
@@ -12127,7 +12142,7 @@ class TrieNode {
     raw;
     constructor(
     /** Exactly 512 bits / 64 bytes */
-    raw = new Uint8Array(TRIE_NODE_BYTES)) {
+    raw = safe_alloc_uint8array_safeAllocUint8Array(TRIE_NODE_BYTES)) {
         this.raw = raw;
     }
     /** Returns the type of the node */
@@ -14572,7 +14587,7 @@ class Registers {
     bytes;
     asSigned;
     asUnsigned;
-    constructor(bytes = new Uint8Array(NO_OF_REGISTERS << REGISTER_SIZE_SHIFT)) {
+    constructor(bytes = safe_alloc_uint8array_safeAllocUint8Array(NO_OF_REGISTERS << REGISTER_SIZE_SHIFT)) {
         this.bytes = bytes;
         debug_check `${bytes.length === NO_OF_REGISTERS << REGISTER_SIZE_SHIFT} Invalid size of registers array.`;
         this.asSigned = new BigInt64Array(bytes.buffer, bytes.byteOffset);
@@ -14735,7 +14750,7 @@ class Mask {
         return Math.min(this.lookupTableForward[index] ?? 0, MAX_INSTRUCTION_DISTANCE);
     }
     buildLookupTableForward(mask) {
-        const table = new Uint8Array(mask.bitLength);
+        const table = safe_alloc_uint8array_safeAllocUint8Array(mask.bitLength);
         let lastInstructionOffset = 0;
         for (let i = mask.bitLength - 1; i >= 0; i--) {
             if (mask.isSet(i)) {
@@ -18553,7 +18568,7 @@ class HostCalls {
             const regs = pvmInstance.getRegisters();
             const maybeAddress = regs.getLowerU32(7);
             const maybeLength = regs.getLowerU32(8);
-            const result = new Uint8Array(maybeLength);
+            const result = safe_alloc_uint8array_safeAllocUint8Array(maybeLength);
             const startAddress = tryAsMemoryIndex(maybeAddress);
             const loadResult = memory.loadInto(result, startAddress);
             if (loadResult.isError) {
@@ -19635,6 +19650,15 @@ var ResultValues;
     ResultValues[ResultValues["Ok"] = 0] = "Ok";
     ResultValues[ResultValues["Error"] = 1] = "Error";
 })(ResultValues || (ResultValues = {}));
+/**
+ * Getting a ring commitment is pretty expensive (hundreds of ms),
+ * yet the validators do not always change.
+ * For current benchmarks, we get a huge hit every epoch, hence
+ * to overcome that we cache the results of getting ring commitment.
+ * Note we can also tentatively populate this cache, before we even
+ * reach the epoch change block.
+ */
+const ringCommitmentCache = [];
 // TODO [ToDr] We export the entire object to allow mocking in tests.
 // Ideally we would just export functions and figure out how to mock
 // properly in ESM.
@@ -19650,9 +19674,27 @@ async function verifySeal(bandersnatch, authorKey, signature, payload, encodedUn
     }
     return result_Result.ok(bytes_Bytes.fromBlob(sealResult.subarray(1), hash_HASH_SIZE).asOpaque());
 }
-async function getRingCommitment(bandersnatch, validators) {
-    const keys = bytes_BytesBlob.blobFromParts(validators.map((x) => x.raw)).raw;
-    const commitmentResult = await bandersnatch.getRingCommitment(keys);
+function getRingCommitment(bandersnatch, validators) {
+    const keys = bytes_BytesBlob.blobFromParts(validators.map((x) => x.raw));
+    // We currently compare the large bytes blob, but the number of entries in the cache
+    // must be low. If the cache ever grows larger, we should rather consider hashing the keys.
+    const MAX_CACHE_ENTRIES = 3;
+    const cacheEntry = ringCommitmentCache.find((v) => v.keys.isEqualTo(keys));
+    if (cacheEntry !== undefined) {
+        return cacheEntry.value;
+    }
+    const value = getRingCommitmentNoCache(bandersnatch, keys);
+    ringCommitmentCache.push({
+        keys,
+        value,
+    });
+    if (ringCommitmentCache.length > MAX_CACHE_ENTRIES) {
+        ringCommitmentCache.shift();
+    }
+    return value;
+}
+async function getRingCommitmentNoCache(bandersnatch, keys) {
+    const commitmentResult = await bandersnatch.getRingCommitment(keys.raw);
     if (commitmentResult[RESULT_INDEX] === ResultValues.Error) {
         return result_Result.error(null);
     }
@@ -19778,6 +19820,18 @@ class Safrole {
         }
         return sized_array_FixedSizeArray.new([newRandomnessAcc, ...rest], 4);
     }
+    /**
+     * Pre-populate cache for validator keys, and especially the ring commitment.
+     *
+     * NOTE the function is still doing quite some work, so it should only be used
+     *  once per epoch. The optimisation relies on the fact that the `bandersnatch.getRingCommitment`
+     * call will be cached.
+     */
+    async prepareValidatorKeysForNextEpoch(postOffenders) {
+        const stateEpoch = Math.floor(this.state.timeslot / this.chainSpec.epochLength);
+        const nextEpochStart = (stateEpoch + 1) * this.chainSpec.epochLength;
+        return await this.getValidatorKeys(common_tryAsTimeSlot(nextEpochStart), postOffenders);
+    }
     async getValidatorKeys(timeslot, postOffenders) {
         /**
          * Epoch is not changed so the previous state is returned
@@ -21935,7 +21989,7 @@ class Assign {
         const authorizationQueueStart = regs.get(8);
         // a
         const authManager = getServiceId(regs.get(9));
-        const res = new Uint8Array(hash_HASH_SIZE * AUTHORIZATION_QUEUE_SIZE);
+        const res = safe_alloc_uint8array_safeAllocUint8Array(hash_HASH_SIZE * AUTHORIZATION_QUEUE_SIZE);
         const memoryReadResult = memory.loadInto(res, authorizationQueueStart);
         // error while reading the memory.
         if (memoryReadResult.isError) {
@@ -22023,7 +22077,7 @@ class Bless {
          * https://graypaper.fluffylabs.dev/#/7e6ff6a/368100368100?v=0.6.7
          */
         const autoAccumulateEntries = [];
-        const result = new Uint8Array(tryAsExactBytes(serviceIdAndGasCodec.sizeHint));
+        const result = safe_alloc_uint8array_safeAllocUint8Array(tryAsExactBytes(serviceIdAndGasCodec.sizeHint));
         const decoder = decoder_Decoder.fromBlob(result);
         let memIndex = sourceStart;
         for (let i = 0n; i < numberOfItems; i += 1n) {
@@ -22040,7 +22094,7 @@ class Bless {
             memIndex = numbers_tryAsU64(memIndex + numbers_tryAsU64(decoder.bytesRead()));
         }
         // https://graypaper.fluffylabs.dev/#/7e6ff6a/367200367200?v=0.6.7
-        const res = new Uint8Array(tryAsExactBytes(descriptors_codec.u32.sizeHint) * this.chainSpec.coresCount);
+        const res = safe_alloc_uint8array_safeAllocUint8Array(tryAsExactBytes(descriptors_codec.u32.sizeHint) * this.chainSpec.coresCount);
         const authorizersDecoder = decoder_Decoder.fromBlob(res);
         const memoryReadResult = memory.loadInto(res, authorization);
         if (memoryReadResult.isError) {
@@ -22136,6 +22190,7 @@ class Checkpoint {
 
 
 
+
 const designate_IN_OUT_REG = 7;
 const VALIDATOR_DATA_BYTES = tryAsExactBytes(ValidatorData.Codec.sizeHint);
 /**
@@ -22158,7 +22213,7 @@ class Designate {
     async execute(_gas, regs, memory) {
         // `o`
         const validatorsStart = regs.get(designate_IN_OUT_REG);
-        const res = new Uint8Array(VALIDATOR_DATA_BYTES * this.chainSpec.validatorsCount);
+        const res = safe_alloc_uint8array_safeAllocUint8Array(VALIDATOR_DATA_BYTES * this.chainSpec.validatorsCount);
         const memoryReadResult = memory.loadInto(res, validatorsStart);
         // error while reading the memory.
         if (memoryReadResult.isError) {
@@ -22389,7 +22444,7 @@ class Provide {
         const preimageLength = regs.get(9);
         const length = clampU64ToU32(preimageLength);
         // `i`
-        const preimage = bytes_BytesBlob.blobFrom(new Uint8Array(length));
+        const preimage = bytes_BytesBlob.blobFrom(safe_alloc_uint8array_safeAllocUint8Array(length));
         const memoryReadResult = memory.loadInto(preimage.raw, preimageStart);
         if (memoryReadResult.isError) {
             logger_logger.trace `PROVIDE(${serviceId}, ${preimage.toStringTruncated()}) <- PANIC`;
@@ -22951,6 +23006,7 @@ const codecServiceAccountInfoWithThresholdBalance = descriptors_codec.object({
 
 
 
+
 const decoder = new TextDecoder("utf8");
 /**
  * Log message to the console
@@ -22972,8 +23028,8 @@ class LogHostCall {
         const targetLength = regs.get(9);
         const msgStart = regs.get(10);
         const msgLength = regs.get(11);
-        const target = new Uint8Array(clampU64ToU32(targetLength));
-        const message = new Uint8Array(clampU64ToU32(msgLength));
+        const target = safe_alloc_uint8array_safeAllocUint8Array(clampU64ToU32(targetLength));
+        const message = safe_alloc_uint8array_safeAllocUint8Array(clampU64ToU32(msgLength));
         if (targetStart !== 0n) {
             memory.loadInto(target, targetStart);
         }
@@ -22992,6 +23048,7 @@ class LogHostCall {
 
 
 
+
 const lookup_IN_OUT_REG = 7;
 /**
  * Lookup a preimage.
@@ -23034,7 +23091,7 @@ class Lookup {
         // NOTE [MaSo] this is ok to cast to number, because we are bounded by the
         // valueLength in both cases and valueLength is WC (4,000,000,000) + metadata
         // which is less than 2^32
-        const chunk = preImage === null ? new Uint8Array(0) : preImage.raw.subarray(Number(offset), Number(offset + length));
+        const chunk = preImage === null ? safe_alloc_uint8array_safeAllocUint8Array(0) : preImage.raw.subarray(Number(offset), Number(offset + length));
         const memoryWriteResult = memory.storeFrom(destinationAddress, chunk);
         if (memoryWriteResult.isError) {
             return PvmExecution.Panic;
@@ -23055,6 +23112,7 @@ class Lookup {
 
 
 
+
 const read_IN_OUT_REG = 7;
 /**
  * Read account storage.
@@ -23082,7 +23140,7 @@ class Read {
         const destinationAddress = regs.get(10);
         const storageKeyLengthClamped = clampU64ToU32(storageKeyLength);
         // k
-        const rawKey = bytes_BytesBlob.blobFrom(new Uint8Array(storageKeyLengthClamped));
+        const rawKey = bytes_BytesBlob.blobFrom(safe_alloc_uint8array_safeAllocUint8Array(storageKeyLengthClamped));
         const memoryReadResult = memory.loadInto(rawKey.raw, storageKeyStartAddress);
         if (memoryReadResult.isError) {
             logger_logger.trace `READ(${serviceId}, ${rawKey}) <- PANIC`;
@@ -23100,7 +23158,7 @@ class Read {
         // NOTE [MaSo] this is ok to cast to number, because we are bounded by the
         // valueLength in both cases and valueLength is WC (4,000,000,000) + metadata
         // which is less than 2^32
-        const chunk = value === null ? new Uint8Array(0) : value.raw.subarray(Number(offset), Number(offset + blobLength));
+        const chunk = value === null ? safe_alloc_uint8array_safeAllocUint8Array(0) : value.raw.subarray(Number(offset), Number(offset + blobLength));
         const memoryWriteResult = memory.storeFrom(destinationAddress, chunk);
         if (memoryWriteResult.isError) {
             logger_logger.trace `READ(${serviceId}, ${rawKey}) <- PANIC`;
@@ -23151,7 +23209,7 @@ class Write {
         // v_z
         const valueLength = regs.get(10);
         const storageKeyLengthClamped = clampU64ToU32(storageKeyLength);
-        const rawStorageKey = new Uint8Array(storageKeyLengthClamped);
+        const rawStorageKey = safe_alloc_uint8array_safeAllocUint8Array(storageKeyLengthClamped);
         const keyLoadingResult = memory.loadInto(rawStorageKey, storageKeyStartAddress);
         if (keyLoadingResult.isError) {
             logger_logger.trace `WRITE() <- PANIC`;
@@ -23160,7 +23218,7 @@ class Write {
         // k
         const storageKey = opaque_asOpaqueType(bytes_BytesBlob.blobFrom(rawStorageKey));
         const valueLengthClamped = clampU64ToU32(valueLength);
-        const value = new Uint8Array(valueLengthClamped);
+        const value = safe_alloc_uint8array_safeAllocUint8Array(valueLengthClamped);
         const valueLoadingResult = memory.loadInto(value, valueStart);
         // Note [MaSo] this is ok to return bcs if valueLength is 0, then this panic won't happen
         if (valueLoadingResult.isError) {
@@ -24049,7 +24107,7 @@ function fisherYatesShuffle(arr, entropy) {
 }
 function hashToNumberSequence(entropy, length) {
     const result = new Array(length);
-    const randomBytes = new Uint8Array(ENTROPY_BYTES + 4);
+    const randomBytes = safe_alloc_uint8array_safeAllocUint8Array(ENTROPY_BYTES + 4);
     randomBytes.set(entropy.raw);
     for (let i = 0; i < length; i++) {
         randomBytes.set(numbers_u32AsLeBytes(numbers_tryAsU32(Math.floor(i / 8))), ENTROPY_BYTES);
@@ -25079,6 +25137,7 @@ class OnChain {
     authorization;
     // chapter 13: https://graypaper.fluffylabs.dev/#/68eaa1f/18b60118b601?v=0.6.4
     statistics;
+    isReadyForNextEpoch = Promise.resolve(false);
     constructor(chainSpec, state, blocks, hasher) {
         this.chainSpec = chainSpec;
         this.state = state;
@@ -25097,6 +25156,14 @@ class OnChain {
         this.preimages = new Preimages(state);
         this.authorization = new Authorization(chainSpec, state);
     }
+    /** Pre-populate things worth caching for the next epoch. */
+    async prepareForNextEpoch() {
+        if (await this.isReadyForNextEpoch) {
+            return;
+        }
+        const ready = this.safrole.prepareValidatorKeysForNextEpoch(this.state.disputesRecords.punishSet);
+        this.isReadyForNextEpoch = ready.then((_) => true);
+    }
     async verifySeal(timeSlot, block) {
         const sealState = this.safrole.getSafroleSealState(timeSlot);
         return await this.safroleSeal.verifyHeaderSeal(block.header.view(), sealState);
@@ -25105,6 +25172,10 @@ class OnChain {
         const headerView = block.header.view();
         const header = block.header.materialize();
         const timeSlot = header.timeSlotIndex;
+        // reset the epoch cache state
+        if (headerView.epochMarker.view() !== null) {
+            this.isReadyForNextEpoch = Promise.resolve(false);
+        }
         // safrole seal
         let newEntropyHash;
         if (omitSealVerification) {
@@ -25328,8 +25399,18 @@ class Importer {
         this.stf = new OnChain(spec, state, blocks, hasher);
         this.state = state;
         this.currentHash = currentBestHeaderHash;
+        this.prepareForNextEpoch();
         logger.info `😎 Best time slot: ${state.timeslot} (header hash: ${currentBestHeaderHash})`;
     }
+    /** Do some extra work for preparation for the next epoch. */
+    async prepareForNextEpoch() {
+        try {
+            await this.stf.prepareForNextEpoch();
+        }
+        catch (e) {
+            this.logger.error `Unable to prepare for next epoch: ${e}`;
+        }
+    }
     async importBlock(block, omitSealVerification) {
         const timer = measure("importBlock");
         const timeSlot = extractTimeSlot(block);
@@ -25365,6 +25446,7 @@ class Importer {
                 return importerError(ImporterErrorKind.Verifier, e);
             }
             this.state.updateBackend(state?.backend);
+            this.prepareForNextEpoch();
             this.currentHash = parentHash;
         }
         const timeSlot = block.header.view().timeSlotIndex.materialize();
@@ -25391,6 +25473,7 @@ class Importer {
         // TODO [ToDr] This is a temporary measure. We should rather read
         // the state of a parent block to support forks and create a fresh STF.
         this.state.updateBackend(newState.backend);
+        this.prepareForNextEpoch();
         this.currentHash = headerHash;
         logger.log `${timerState()}`;
         // insert new state and the block to DB.