@typeberry/convert 0.1.3-135961b → 0.1.3-462ca77

package/index.js

@@ -5264,6 +5264,20 @@ const result_Result = {
     },
 };
 
+;// CONCATENATED MODULE: ./packages/core/utils/safe-alloc-uint8array.ts
+// about 2GB, the maximum ArrayBuffer length on Chrome confirmed by several sources:
+// - https://issues.chromium.org/issues/40055619
+// - https://stackoverflow.com/a/72124984
+// - https://onnxruntime.ai/docs/tutorials/web/large-models.html#maximum-size-of-arraybuffer
+const MAX_LENGTH = 2145386496;
+function safe_alloc_uint8array_safeAllocUint8Array(length) {
+    if (length > MAX_LENGTH) {
+        // biome-ignore lint/suspicious/noConsole: can't have a dependency on logger here
+        console.warn(`Trying to allocate ${length} bytes, which is greater than the maximum of ${MAX_LENGTH}.`);
+    }
+    return new Uint8Array(Math.min(MAX_LENGTH, length));
+}
+
 ;// CONCATENATED MODULE: external "node:assert"
 const external_node_assert_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:assert");
 ;// CONCATENATED MODULE: ./packages/core/utils/test.ts
@@ -5507,6 +5521,7 @@ function isResult(x) {
 
 
 
+
 // EXTERNAL MODULE: ./node_modules/minimist/index.js
 var minimist = __nccwpck_require__(595);
 var minimist_default = /*#__PURE__*/__nccwpck_require__.n(minimist);
@@ -5533,7 +5548,7 @@ class bitvec_BitVec {
      * Create new [`BitVec`] with all values set to `false`.
      */
     static empty(bitLength) {
-        const data = new Uint8Array(Math.ceil(bitLength / 8));
+        const data = safe_alloc_uint8array_safeAllocUint8Array(Math.ceil(bitLength / 8));
         return new bitvec_BitVec(data, bitLength);
     }
     byteLength;
@@ -5733,7 +5748,7 @@ class bytes_BytesBlob {
     static blobFromParts(v, ...rest) {
         const vArr = v instanceof Uint8Array ? [v] : v;
         const totalLength = vArr.reduce((a, v) => a + v.length, 0) + rest.reduce((a, v) => a + v.length, 0);
-        const buffer = new Uint8Array(totalLength);
+        const buffer = safe_alloc_uint8array_safeAllocUint8Array(totalLength);
         let offset = 0;
         for (const r of vArr) {
             buffer.set(r, offset);
@@ -5806,7 +5821,7 @@ class bytes_Bytes extends bytes_BytesBlob {
     }
     /** Create an empty [`Bytes<X>`] of given length. */
     static zero(len) {
-        return new bytes_Bytes(new Uint8Array(len), len);
+        return new bytes_Bytes(safe_alloc_uint8array_safeAllocUint8Array(len), len);
     }
     // TODO [ToDr] `fill` should have the argments swapped to align with the rest.
     /** Create a [`Bytes<X>`] with all bytes filled with given input number. */
@@ -6476,7 +6491,7 @@ function addSizeHints(a, b) {
     };
 }
 const DEFAULT_START_LENGTH = 512; // 512B
-const MAX_LENGTH = 10 * 1024 * 1024; // 10MB
+const encoder_MAX_LENGTH = 10 * 1024 * 1024; // 10MB
 /**
  * JAM encoder.
  */
@@ -6492,7 +6507,7 @@ class encoder_Encoder {
             return new encoder_Encoder(options.destination);
         }
         const startLength = options?.expectedLength ?? DEFAULT_START_LENGTH;
-        const buffer = new ArrayBuffer(Math.min(MAX_LENGTH, startLength), { maxByteLength: MAX_LENGTH });
+        const buffer = new ArrayBuffer(Math.min(encoder_MAX_LENGTH, startLength), { maxByteLength: encoder_MAX_LENGTH });
         const destination = new Uint8Array(buffer);
         return new encoder_Encoder(destination, buffer);
     }
@@ -6825,11 +6840,11 @@ class encoder_Encoder {
     ensureBigEnough(length, options = { silent: false }) {
         debug_check `${length >= 0} Negative length given`;
         const newLength = this.offset + length;
-        if (newLength > MAX_LENGTH) {
+        if (newLength > encoder_MAX_LENGTH) {
             if (options.silent) {
                 return;
             }
-            throw new Error(`The encoded size would reach the maximum of ${MAX_LENGTH}.`);
+            throw new Error(`The encoded size would reach the maximum of ${encoder_MAX_LENGTH}.`);
         }
         if (newLength > this.destination.length) {
             // we can try to resize the underlying buffer
@@ -6837,7 +6852,7 @@ class encoder_Encoder {
                 // make sure we at least double the size of the buffer every time.
                 const minExtend = Math.max(newLength, this.buffer.byteLength << 1);
                 // but we must never exceed the max length.
-                this.buffer.resize(Math.min(MAX_LENGTH, minExtend));
+                this.buffer.resize(Math.min(encoder_MAX_LENGTH, minExtend));
             }
             // and then check again
             if (newLength > this.destination.length) {
@@ -8169,7 +8184,7 @@ async function verify(input) {
         return Promise.resolve([]);
     }
     const dataLength = input.reduce((acc, { message, key, signature }) => acc + key.length + signature.length + message.length + 1, 0);
-    const data = new Uint8Array(dataLength);
+    const data = safeAllocUint8Array(dataLength);
     let offset = 0;
     for (const { key, message, signature } of input) {
         data.set(key.raw, offset);
@@ -8256,7 +8271,7 @@ class allocator_SimpleAllocator {
 /** An allocator that works by allocating larger (continuous) pages of memory. */
 class PageAllocator {
     hashesPerPage;
-    page = new Uint8Array(0);
+    page = safeAllocUint8Array(0);
     currentHash = 0;
     // TODO [ToDr] Benchmark the performance!
     constructor(hashesPerPage) {
@@ -8267,7 +8282,7 @@ class PageAllocator {
     resetPage() {
         const pageSizeBytes = this.hashesPerPage * HASH_SIZE;
         this.currentHash = 0;
-        this.page = new Uint8Array(pageSizeBytes);
+        this.page = safeAllocUint8Array(pageSizeBytes);
     }
     emptyHash() {
         const startIdx = this.currentHash * HASH_SIZE;
@@ -14661,7 +14676,7 @@ class SerializedService {
     getStorage(rawKey) {
         if (compatibility_Compatibility.isLessThan(compatibility_GpVersion.V0_6_7)) {
             const SERVICE_ID_BYTES = 4;
-            const serviceIdAndKey = new Uint8Array(SERVICE_ID_BYTES + rawKey.length);
+            const serviceIdAndKey = safe_alloc_uint8array_safeAllocUint8Array(SERVICE_ID_BYTES + rawKey.length);
             serviceIdAndKey.set(numbers_u32AsLeBytes(this.serviceId));
             serviceIdAndKey.set(rawKey.raw, SERVICE_ID_BYTES);
             const key = opaque_asOpaqueType(bytes_BytesBlob.blobFrom(hashBytes(serviceIdAndKey).raw));
@@ -14752,7 +14767,7 @@ class nodes_TrieNode {
     raw;
     constructor(
     /** Exactly 512 bits / 64 bytes */
-    raw = new Uint8Array(nodes_TRIE_NODE_BYTES)) {
+    raw = safe_alloc_uint8array_safeAllocUint8Array(nodes_TRIE_NODE_BYTES)) {
         this.raw = raw;
     }
     /** Returns the type of the node */
@@ -16187,7 +16202,7 @@ class registers_Registers {
     bytes;
     asSigned;
     asUnsigned;
-    constructor(bytes = new Uint8Array(registers_NO_OF_REGISTERS << REGISTER_SIZE_SHIFT)) {
+    constructor(bytes = safeAllocUint8Array(registers_NO_OF_REGISTERS << REGISTER_SIZE_SHIFT)) {
         this.bytes = bytes;
         check `${bytes.length === registers_NO_OF_REGISTERS << REGISTER_SIZE_SHIFT} Invalid size of registers array.`;
         this.asSigned = new BigInt64Array(bytes.buffer, bytes.byteOffset);
@@ -16350,7 +16365,7 @@ class mask_Mask {
         return Math.min(this.lookupTableForward[index] ?? 0, MAX_INSTRUCTION_DISTANCE);
     }
     buildLookupTableForward(mask) {
-        const table = new Uint8Array(mask.bitLength);
+        const table = safeAllocUint8Array(mask.bitLength);
         let lastInstructionOffset = 0;
         for (let i = mask.bitLength - 1; i >= 0; i--) {
             if (mask.isSet(i)) {
@@ -19861,7 +19876,7 @@ class host_calls_HostCalls {
             const regs = pvmInstance.getRegisters();
             const maybeAddress = regs.getLowerU32(7);
             const maybeLength = regs.getLowerU32(8);
-            const result = new Uint8Array(maybeLength);
+            const result = safeAllocUint8Array(maybeLength);
             const startAddress = tryAsMemoryIndex(maybeAddress);
             const loadResult = memory.loadInto(result, startAddress);
             if (loadResult.isError) {
@@ -20785,6 +20800,15 @@ var ResultValues;
     ResultValues[ResultValues["Ok"] = 0] = "Ok";
     ResultValues[ResultValues["Error"] = 1] = "Error";
 })(ResultValues || (ResultValues = {}));
+/**
+ * Getting a ring commitment is pretty expensive (hundreds of ms),
+ * yet the validators do not always change.
+ * For current benchmarks, we get a huge hit every epoch, hence
+ * to overcome that we cache the results of getting ring commitment.
+ * Note we can also tentatively populate this cache, before we even
+ * reach the epoch change block.
+ */
+const ringCommitmentCache = [];
 // TODO [ToDr] We export the entire object to allow mocking in tests.
 // Ideally we would just export functions and figure out how to mock
 // properly in ESM.
@@ -20800,9 +20824,27 @@ async function verifySeal(bandersnatch, authorKey, signature, payload, encodedUn
     }
     return result_Result.ok(bytes_Bytes.fromBlob(sealResult.subarray(1), hash_HASH_SIZE).asOpaque());
 }
-async function getRingCommitment(bandersnatch, validators) {
-    const keys = bytes_BytesBlob.blobFromParts(validators.map((x) => x.raw)).raw;
-    const commitmentResult = await bandersnatch.getRingCommitment(keys);
+function getRingCommitment(bandersnatch, validators) {
+    const keys = bytes_BytesBlob.blobFromParts(validators.map((x) => x.raw));
+    // We currently compare the large bytes blob, but the number of entries in the cache
+    // must be low. If the cache ever grows larger, we should rather consider hashing the keys.
+    const MAX_CACHE_ENTRIES = 3;
+    const cacheEntry = ringCommitmentCache.find((v) => v.keys.isEqualTo(keys));
+    if (cacheEntry !== undefined) {
+        return cacheEntry.value;
+    }
+    const value = getRingCommitmentNoCache(bandersnatch, keys);
+    ringCommitmentCache.push({
+        keys,
+        value,
+    });
+    if (ringCommitmentCache.length > MAX_CACHE_ENTRIES) {
+        ringCommitmentCache.shift();
+    }
+    return value;
+}
+async function getRingCommitmentNoCache(bandersnatch, keys) {
+    const commitmentResult = await bandersnatch.getRingCommitment(keys.raw);
     if (commitmentResult[RESULT_INDEX] === ResultValues.Error) {
         return result_Result.error(null);
     }
@@ -20928,6 +20970,18 @@ class safrole_Safrole {
         }
         return FixedSizeArray.new([newRandomnessAcc, ...rest], 4);
     }
+    /**
+     * Pre-populate cache for validator keys, and especially the ring commitment.
+     *
+     * NOTE the function is still doing quite some work, so it should only be used
+     *  once per epoch. The optimisation relies on the fact that the `bandersnatch.getRingCommitment`
+     * call will be cached.
+     */
+    async prepareValidatorKeysForNextEpoch(postOffenders) {
+        const stateEpoch = Math.floor(this.state.timeslot / this.chainSpec.epochLength);
+        const nextEpochStart = (stateEpoch + 1) * this.chainSpec.epochLength;
+        return await this.getValidatorKeys(tryAsTimeSlot(nextEpochStart), postOffenders);
+    }
     async getValidatorKeys(timeslot, postOffenders) {
         /**
          * Epoch is not changed so the previous state is returned
@@ -23081,7 +23135,7 @@ class Assign {
         const authorizationQueueStart = regs.get(8);
         // a
         const authManager = getServiceId(regs.get(9));
-        const res = new Uint8Array(hash_HASH_SIZE * gp_constants_AUTHORIZATION_QUEUE_SIZE);
+        const res = safe_alloc_uint8array_safeAllocUint8Array(hash_HASH_SIZE * gp_constants_AUTHORIZATION_QUEUE_SIZE);
         const memoryReadResult = memory.loadInto(res, authorizationQueueStart);
         // error while reading the memory.
         if (memoryReadResult.isError) {
@@ -23169,7 +23223,7 @@ class Bless {
          * https://graypaper.fluffylabs.dev/#/7e6ff6a/368100368100?v=0.6.7
          */
         const autoAccumulateEntries = [];
-        const result = new Uint8Array(tryAsExactBytes(serviceIdAndGasCodec.sizeHint));
+        const result = safe_alloc_uint8array_safeAllocUint8Array(tryAsExactBytes(serviceIdAndGasCodec.sizeHint));
         const decoder = decoder_Decoder.fromBlob(result);
         let memIndex = sourceStart;
         for (let i = 0n; i < numberOfItems; i += 1n) {
@@ -23186,7 +23240,7 @@ class Bless {
             memIndex = numbers_tryAsU64(memIndex + numbers_tryAsU64(decoder.bytesRead()));
         }
         // https://graypaper.fluffylabs.dev/#/7e6ff6a/367200367200?v=0.6.7
-        const res = new Uint8Array(tryAsExactBytes(descriptors_codec.u32.sizeHint) * this.chainSpec.coresCount);
+        const res = safe_alloc_uint8array_safeAllocUint8Array(tryAsExactBytes(descriptors_codec.u32.sizeHint) * this.chainSpec.coresCount);
         const authorizersDecoder = decoder_Decoder.fromBlob(res);
         const memoryReadResult = memory.loadInto(res, authorization);
         if (memoryReadResult.isError) {
@@ -23282,6 +23336,7 @@ class Checkpoint {
 
 
 
+
 const designate_IN_OUT_REG = 7;
 const VALIDATOR_DATA_BYTES = tryAsExactBytes(validator_data_ValidatorData.Codec.sizeHint);
 /**
@@ -23304,7 +23359,7 @@ class Designate {
     async execute(_gas, regs, memory) {
         // `o`
         const validatorsStart = regs.get(designate_IN_OUT_REG);
-        const res = new Uint8Array(VALIDATOR_DATA_BYTES * this.chainSpec.validatorsCount);
+        const res = safe_alloc_uint8array_safeAllocUint8Array(VALIDATOR_DATA_BYTES * this.chainSpec.validatorsCount);
         const memoryReadResult = memory.loadInto(res, validatorsStart);
         // error while reading the memory.
         if (memoryReadResult.isError) {
@@ -23535,7 +23590,7 @@ class Provide {
         const preimageLength = regs.get(9);
         const length = utils_clampU64ToU32(preimageLength);
         // `i`
-        const preimage = bytes_BytesBlob.blobFrom(new Uint8Array(length));
+        const preimage = bytes_BytesBlob.blobFrom(safe_alloc_uint8array_safeAllocUint8Array(length));
         const memoryReadResult = memory.loadInto(preimage.raw, preimageStart);
         if (memoryReadResult.isError) {
             logger_logger.trace `PROVIDE(${serviceId}, ${preimage.toStringTruncated()}) <- PANIC`;
@@ -24097,6 +24152,7 @@ const codecServiceAccountInfoWithThresholdBalance = descriptors_codec.object({
 
 
 
+
 const decoder = new TextDecoder("utf8");
 /**
  * Log message to the console
@@ -24118,8 +24174,8 @@ class log_LogHostCall {
         const targetLength = regs.get(9);
         const msgStart = regs.get(10);
         const msgLength = regs.get(11);
-        const target = new Uint8Array(clampU64ToU32(targetLength));
-        const message = new Uint8Array(clampU64ToU32(msgLength));
+        const target = safeAllocUint8Array(clampU64ToU32(targetLength));
+        const message = safeAllocUint8Array(clampU64ToU32(msgLength));
         if (targetStart !== 0n) {
             memory.loadInto(target, targetStart);
         }
@@ -24138,6 +24194,7 @@ class log_LogHostCall {
 
 
 
+
 const lookup_IN_OUT_REG = 7;
 /**
  * Lookup a preimage.
@@ -24180,7 +24237,7 @@ class lookup_Lookup {
         // NOTE [MaSo] this is ok to cast to number, because we are bounded by the
         // valueLength in both cases and valueLength is WC (4,000,000,000) + metadata
         // which is less than 2^32
-        const chunk = preImage === null ? new Uint8Array(0) : preImage.raw.subarray(Number(offset), Number(offset + length));
+        const chunk = preImage === null ? safeAllocUint8Array(0) : preImage.raw.subarray(Number(offset), Number(offset + length));
         const memoryWriteResult = memory.storeFrom(destinationAddress, chunk);
         if (memoryWriteResult.isError) {
             return PvmExecution.Panic;
@@ -24201,6 +24258,7 @@ class lookup_Lookup {
 
 
 
+
 const read_IN_OUT_REG = 7;
 /**
  * Read account storage.
@@ -24228,7 +24286,7 @@ class read_Read {
         const destinationAddress = regs.get(10);
         const storageKeyLengthClamped = clampU64ToU32(storageKeyLength);
         // k
-        const rawKey = BytesBlob.blobFrom(new Uint8Array(storageKeyLengthClamped));
+        const rawKey = BytesBlob.blobFrom(safeAllocUint8Array(storageKeyLengthClamped));
         const memoryReadResult = memory.loadInto(rawKey.raw, storageKeyStartAddress);
         if (memoryReadResult.isError) {
             logger.trace `READ(${serviceId}, ${rawKey}) <- PANIC`;
@@ -24246,7 +24304,7 @@ class read_Read {
         // NOTE [MaSo] this is ok to cast to number, because we are bounded by the
         // valueLength in both cases and valueLength is WC (4,000,000,000) + metadata
         // which is less than 2^32
-        const chunk = value === null ? new Uint8Array(0) : value.raw.subarray(Number(offset), Number(offset + blobLength));
+        const chunk = value === null ? safeAllocUint8Array(0) : value.raw.subarray(Number(offset), Number(offset + blobLength));
         const memoryWriteResult = memory.storeFrom(destinationAddress, chunk);
         if (memoryWriteResult.isError) {
             logger.trace `READ(${serviceId}, ${rawKey}) <- PANIC`;
@@ -24297,7 +24355,7 @@ class write_Write {
         // v_z
         const valueLength = regs.get(10);
         const storageKeyLengthClamped = clampU64ToU32(storageKeyLength);
-        const rawStorageKey = new Uint8Array(storageKeyLengthClamped);
+        const rawStorageKey = safeAllocUint8Array(storageKeyLengthClamped);
         const keyLoadingResult = memory.loadInto(rawStorageKey, storageKeyStartAddress);
         if (keyLoadingResult.isError) {
             logger.trace `WRITE() <- PANIC`;
@@ -24306,7 +24364,7 @@ class write_Write {
         // k
         const storageKey = asOpaqueType(BytesBlob.blobFrom(rawStorageKey));
         const valueLengthClamped = clampU64ToU32(valueLength);
-        const value = new Uint8Array(valueLengthClamped);
+        const value = safeAllocUint8Array(valueLengthClamped);
         const valueLoadingResult = memory.loadInto(value, valueStart);
         // Note [MaSo] this is ok to return bcs if valueLength is 0, then this panic won't happen
         if (valueLoadingResult.isError) {
@@ -25195,7 +25253,7 @@ function shuffling_fisherYatesShuffle(arr, entropy) {
 }
 function hashToNumberSequence(entropy, length) {
     const result = new Array(length);
-    const randomBytes = new Uint8Array(ENTROPY_BYTES + 4);
+    const randomBytes = safeAllocUint8Array(ENTROPY_BYTES + 4);
     randomBytes.set(entropy.raw);
     for (let i = 0; i < length; i++) {
         randomBytes.set(u32AsLeBytes(tryAsU32(Math.floor(i / 8))), ENTROPY_BYTES);
@@ -26225,6 +26283,7 @@ class chain_stf_OnChain {
     authorization;
     // chapter 13: https://graypaper.fluffylabs.dev/#/68eaa1f/18b60118b601?v=0.6.4
     statistics;
+    isReadyForNextEpoch = Promise.resolve(false);
     constructor(chainSpec, state, blocks, hasher) {
         this.chainSpec = chainSpec;
         this.state = state;
@@ -26243,6 +26302,14 @@ class chain_stf_OnChain {
         this.preimages = new Preimages(state);
         this.authorization = new Authorization(chainSpec, state);
     }
+    /** Pre-populate things worth caching for the next epoch. */
+    async prepareForNextEpoch() {
+        if (await this.isReadyForNextEpoch) {
+            return;
+        }
+        const ready = this.safrole.prepareValidatorKeysForNextEpoch(this.state.disputesRecords.punishSet);
+        this.isReadyForNextEpoch = ready.then((_) => true);
+    }
     async verifySeal(timeSlot, block) {
         const sealState = this.safrole.getSafroleSealState(timeSlot);
         return await this.safroleSeal.verifyHeaderSeal(block.header.view(), sealState);
@@ -26251,6 +26318,10 @@ class chain_stf_OnChain {
         const headerView = block.header.view();
         const header = block.header.materialize();
         const timeSlot = header.timeSlotIndex;
+        // reset the epoch cache state
+        if (headerView.epochMarker.view() !== null) {
+            this.isReadyForNextEpoch = Promise.resolve(false);
+        }
         // safrole seal
         let newEntropyHash;
         if (omitSealVerification) {
@@ -26715,6 +26786,7 @@ async function runWorkPackageTest(test, file) {
 
 
 
+
 class MemoryChunkItem {
     static fromJson = {
         address: "number",
@@ -26772,10 +26844,10 @@ async function runPvmTest(testContent) {
         const endPageIndex = tryAsMemoryIndex(startPageIndex + page.length);
         const isWriteable = page["is-writable"];
         if (isWriteable) {
-            memoryBuilder.setWriteablePages(startPageIndex, endPageIndex, new Uint8Array(page.length));
+            memoryBuilder.setWriteablePages(startPageIndex, endPageIndex, safeAllocUint8Array(page.length));
         }
         else {
-            memoryBuilder.setReadablePages(startPageIndex, endPageIndex, new Uint8Array(page.length));
+            memoryBuilder.setReadablePages(startPageIndex, endPageIndex, safeAllocUint8Array(page.length));
         }
     }
     for (const memoryChunk of initialMemory) {
@@ -26827,7 +26899,7 @@ async function runPvmTest(testContent) {
     }, {});
     for (const [pageNumberAsString, memoryChunks] of Object.entries(expectedMemoryByPageNumber)) {
         const pageNumber = tryAsPageNumber(Number(pageNumberAsString));
-        const expectedPage = new Uint8Array(PAGE_SIZE);
+        const expectedPage = safeAllocUint8Array(PAGE_SIZE);
         for (const memoryChunk of memoryChunks) {
             const pageIndex = memoryChunk.address % PAGE_SIZE;
             expectedPage.set(memoryChunk.contents, pageIndex);