@typeberry/lib 0.5.1 → 0.5.2-03e1aed

package/packages/core/erasure-coding/erasure-coding.d.ts

@@ -0,0 +1,142 @@
+import { type PerValidator } from "#@typeberry/block";
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { FixedSizeArray } from "#@typeberry/collections";
+import type { ChainSpec } from "#@typeberry/config";
+/**
+ * Number of main chunks generated by EC. Together with redundancy shards they
+ * create `N_CHUNKS_TOTAL`.
+ *
+ * Any combination of this number of shards generated by EC is sufficient to
+ * recover the original datum.
+ *
+ * https://graypaper.fluffylabs.dev/#/7e6ff6a/3e89013e8901?v=0.6.7
+ */
+export declare const N_CHUNKS_REQUIRED = 342;
+export type N_CHUNKS_REQUIRED = typeof N_CHUNKS_REQUIRED;
+/** Number of redundancy chunks generated by EC. */
+export declare const N_CHUNKS_REDUNDANCY = 681;
+/** Total number of chunks generated by EC. */
+export declare const N_CHUNKS_TOTAL = 1023;
+export type N_CHUNKS_TOTAL = typeof N_CHUNKS_TOTAL;
+/**
+ * The points are 2 bytes length because the encoding function is defined in GF(16)
+ * https://graypaper.fluffylabs.dev/#/7e6ff6a/3e4b013e4b01?v=0.6.7
+ */
+export declare const POINT_LENGTH = 2;
+export type POINT_LENGTH = typeof POINT_LENGTH;
+export declare const PIECE_SIZE = 684;
+export type PIECE_SIZE = typeof PIECE_SIZE;
+export declare function checkConsistency(): void;
+/**
+ * Takes arbitrarily long input data, padds it to multiple of `PIECE_SIZE` and returns
+ * exactly `N_CHUNKS_TOTAL` ec-coded segments.
+ */
+export declare function padAndEncodeData(input: BytesBlob): FixedSizeArray<BytesBlob, 1023>;
+/**
+ * Takes ec-coded chunks and returns the original data trimmed to the expected length.
+ */
+export declare function decodeDataAndTrim(input: FixedSizeArray<[number, BytesBlob], N_CHUNKS_REQUIRED>, expectedLength: number): BytesBlob;
+/**
+ * Takes ec-coded chunks and returns the original data.
+ *
+ * NOTE the chunk size MUST be a multiple of `POINT_LENGTH` and each chunk must
+ * be the same length.
+ *
+ * NOTE the output data may have been padded to match the multiple of `PIECE_SIZE`.
+ */
+export declare function decodeData(input: FixedSizeArray<[number, BytesBlob], N_CHUNKS_REQUIRED>): BytesBlob;
+/**
+ *  Erasure-encoding function. Takes exactly `PIECE_SIZE` data and generate `N_CHUNKS_TOTAL` points.
+ *
+ *  https://graypaper.fluffylabs.dev/#/9a08063/3e4e013e5a01?v=0.6.6
+ */
+export declare function encodePoints(input: Bytes<PIECE_SIZE>): FixedSizeArray<Bytes<POINT_LENGTH>, N_CHUNKS_TOTAL>;
+/**
+ * The function takes exactly `N_CHUNKS_REQUIRED` chunks to recover the original piece.
+ *
+ * NOTE the piece may contain padding, so it should be trimmed externally.
+ */
+export declare function decodePiece(input: FixedSizeArray<[number, Bytes<POINT_LENGTH>], N_CHUNKS_REQUIRED>): Bytes<PIECE_SIZE>;
+/**
+ * `split`: Takes a single BytesBlob and divides it into sequential,
+ * contiguous Bytes of a given size.
+ *
+ * Opposite of `join`.
+ *
+ * Input: [a0, a1, a2, a3, a4, a5], n = 2, k = 3
+ *
+ * Output: [[a0, a1], [a2, a3], [a4, a5]]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3eb4013eb401?v=0.6.6
+ */
+export declare function split<N extends number, K extends number>(input: BytesBlob, n: N, k: K): FixedSizeArray<Bytes<N>, K>;
+/**
+ * `join`: Takes an array of Bytes and concatenates them sequentially
+ * (one after another) into a single BytesBlob.
+ *
+ * The resulting blob has length of `N * K`.
+ *
+ * Opposite of `split`.
+ *
+ * Input: [[a0, a1], [a2, a3], [a4, a5]]
+ *
+ * Output: [a0, a1, a2, a3, a4, a5]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3ed4013ed401?v=0.6.6
+ */
+export declare function join<N extends number, K extends number>(input: FixedSizeArray<Bytes<N>, K>): BytesBlob;
+/**
+ * `unzip`: Reorganizes the data by de-interleaving: it takes every `K-th` byte
+ * for each of output Bytes, where `K` is the number of output Bytes
+ * and `N` is the size of each output Bytes.
+ *
+ * Opposite of `lace`.
+ *
+ * Input: [a0, b0, c0, a1, b1, c1], n = 2, k = 3
+ *
+ * Output: [[a0, a1], [b0, b1], [c0, c1]]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3e06023e0602?v=0.6.6
+ */
+export declare function unzip<N extends number, K extends number>(input: BytesBlob, n: N, k: K): FixedSizeArray<Bytes<N>, K>;
+/**
+ * `lace`: Takes an array of Bytes and interleaves:
+ * it takes the first Byte from each, then the second byte from
+ * each, etc., producing a single BytesBlob.
+ *
+ * Opposite of `unzip`.
+ *
+ * Input: [[a0, a1], [b0, b1], [c0, c1]]
+ *
+ * Output: [a0, b0, c0, a1, b1, c1]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3e2a023e2a02?v=0.6.6
+ */
+export declare function lace<N extends number, K extends number>(input: FixedSizeArray<Bytes<N>, K>): BytesBlob;
+/**
+ * `T`: Transposing function which accepts an array of `K` pieces of data
+ * which each have same lenght of `N` octets and returns an array of `N`
+ * pieces of data which each have length of `K` octets.
+ *
+ * T[[x0,0, x0,1, x0,2, . . . ], [x1,0, x1,1, . . . ], . . . ] ≡
+ * [[x0,0, x1,0, x2,0, . . . ], [x0,1, x1,1, . . . ], . . . ]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3e2e023e2e02?v=0.6.6
+ */
+export declare function transpose<T, N extends number, K extends number>(input: FixedSizeArray<FixedSizeArray<T, K>, N>, k: K): FixedSizeArray<FixedSizeArray<T, N>, K>;
+/**
+ * `C`: Erasure-code chunking function which accepts an arbitrary sized data blob whose
+ *       length divides wholly into `PIECE_SIZE` octets and results in `N_CHUNKS_TOTAL`
+ *       sequences of sequences, each of `POINT_LENGTH * K` octets blobs.
+ *       Where `K` is the number that divides input length by `PIECE_SIZE`.
+ *
+ *       Each element of resulting array is the same length.
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3f15003f1500?v=0.6.6
+ */
+export declare function chunkingFunction(input: BytesBlob): FixedSizeArray<BytesBlob, N_CHUNKS_TOTAL>;
+/** Split each validator's shard into numbered chunks it originally should have got. */
+export declare function shardsToChunks(spec: ChainSpec, shards: PerValidator<BytesBlob>): PerValidator<[number, BytesBlob][]>;
+/** Divide chunks between validators. */
+export declare function chunksToShards(spec: ChainSpec, chunks: FixedSizeArray<BytesBlob, N_CHUNKS_TOTAL>): PerValidator<BytesBlob>;
+//# sourceMappingURL=erasure-coding.d.ts.map

package/packages/core/erasure-coding/erasure-coding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"erasure-coding.d.ts","sourceRoot":"","sources":["../../../../../packages/core/erasure-coding/erasure-coding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAqB,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAmBnD;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,MAAM,MAAM,iBAAiB,GAAG,OAAO,iBAAiB,CAAC;AAEzD,mDAAmD;AACnD,eAAO,MAAM,mBAAmB,MAAM,CAAC;AACvC,8CAA8C;AAC9C,eAAO,MAAM,cAAc,OAAO,CAAC;AACnC,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC;AAQnD;;;GAGG;AACH,eAAO,MAAM,YAAY,IAAI,CAAC;AAC9B,MAAM,MAAM,YAAY,GAAG,OAAO,YAAY,CAAC;AAE/C,eAAO,MAAM,UAAU,MAAM,CAAC;AAC9B,MAAM,MAAM,UAAU,GAAG,OAAO,UAAU,CAAC;AAE3C,wBAAgB,gBAAgB,SAG/B;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,mCAUhD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,iBAAiB,CAAC,EAC7D,cAAc,EAAE,MAAM,GACrB,SAAS,CAEX;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,iBAAiB,CAAC,GAAG,SAAS,CAuBnG;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,cAAc,CAAC,CA8B1G;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,cAAc,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC,EAAE,iBAAiB,CAAC,GACtE,KAAK,CAAC,UAAU,CAAC,CA0CnB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,KAAK,CAAC,CAAC,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAUnH;AAED;;;;;;;;;;;;;GAaG;AAEH,wBAAgB,IAAI,CAAC,CAAC,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAEtG;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,KAAK,CAAC,CAAC,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CASnH;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,IAAI,CAAC,CAAC,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CActG;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,EAC7D,KAAK,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAC9C,CAAC,EAAE,CAAC,GACH,cAAc,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAazC;AAED;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,GAAG,cAAc,CAAC,SAAS,EAAE,cAAc,CAAC,CAkB5F;AAED,uFAAuF;AACvF,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,YAAY,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC,CA+BpH;AAED,wCAAwC;AACxC,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,EACf,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE,cAAc,CAAC,GAChD,YAAY,CAAC,SAAS,CAAC,CAoBzB"}

package/packages/core/erasure-coding/erasure-coding.js

@@ -0,0 +1,353 @@
+import { tryAsPerValidator } from "#@typeberry/block";
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { FixedSizeArray } from "#@typeberry/collections";
+import { reedSolomon } from "@typeberry/native";
+import { check, safeAllocUint8Array } from "#@typeberry/utils";
+/**
+ * `point` - [Y_2]
+ * `chunk` - [Y_2*k]
+ * `shard` - PerValidator<chunk/s>
+ * `piece` - 684 octets
+ * `segment` - 4104 octets = 6 * piece
+ */
+/**
+ * Point size must be multiple of 64 bytes.
+ * (reed-solomon-simd limitation:
+ * https://github.com/ordian/reed-solomon-simd/blob/3c2f87329955b41e07de01c01c8c66e9899f1535/src/lib.rs#L78)
+ */
+const POINT_ALIGNMENT = 64;
+/**
+ * Number of main chunks generated by EC. Together with redundancy shards they
+ * create `N_CHUNKS_TOTAL`.
+ *
+ * Any combination of this number of shards generated by EC is sufficient to
+ * recover the original datum.
+ *
+ * https://graypaper.fluffylabs.dev/#/7e6ff6a/3e89013e8901?v=0.6.7
+ */
+export const N_CHUNKS_REQUIRED = 342;
+/** Number of redundancy chunks generated by EC. */
+export const N_CHUNKS_REDUNDANCY = 681;
+/** Total number of chunks generated by EC. */
+export const N_CHUNKS_TOTAL = 1023;
+/**
+ * reed-solomon-simd requires point size to be multiple of 64 bytes but we need only 2 bytes.
+ * It does not matter what indices are selected, but it has to be n and n + 32
+ */
+const HALF_POINT_SIZE = 32;
+/**
+ * The points are 2 bytes length because the encoding function is defined in GF(16)
+ * https://graypaper.fluffylabs.dev/#/7e6ff6a/3e4b013e4b01?v=0.6.7
+ */
+export const POINT_LENGTH = 2;
+export const PIECE_SIZE = 684;
+export function checkConsistency() {
+    check `${N_CHUNKS_TOTAL === N_CHUNKS_REQUIRED + N_CHUNKS_REDUNDANCY} no of required and redundancy chunks need to match ${N_CHUNKS_TOTAL}`;
+    check `${PIECE_SIZE === N_CHUNKS_REQUIRED * POINT_LENGTH} piece size needs to match ${N_CHUNKS_REQUIRED} * ${POINT_LENGTH}`;
+}
+/**
+ * Takes arbitrarily long input data, padds it to multiple of `PIECE_SIZE` and returns
+ * exactly `N_CHUNKS_TOTAL` ec-coded segments.
+ */
+export function padAndEncodeData(input) {
+    // since we can only work with data that is a multiple of `PIECE_SIZE` we pad it before
+    // passing to `chunkingFunction`.
+    const paddedLength = Math.ceil(input.length / PIECE_SIZE) * PIECE_SIZE;
+    let padded = input;
+    if (input.length !== paddedLength) {
+        padded = BytesBlob.blobFrom(safeAllocUint8Array(paddedLength));
+        padded.raw.set(input.raw, 0);
+    }
+    return chunkingFunction(padded);
+}
+/**
+ * Takes ec-coded chunks and returns the original data trimmed to the expected length.
+ */
+export function decodeDataAndTrim(input, expectedLength) {
+    return BytesBlob.blobFrom(decodeData(input).raw.slice(0, expectedLength));
+}
+/**
+ * Takes ec-coded chunks and returns the original data.
+ *
+ * NOTE the chunk size MUST be a multiple of `POINT_LENGTH` and each chunk must
+ * be the same length.
+ *
+ * NOTE the output data may have been padded to match the multiple of `PIECE_SIZE`.
+ */
+export function decodeData(input) {
+    const pointBytes = input[0][1].length;
+    const points = Math.floor(pointBytes / POINT_LENGTH);
+    check `${points * POINT_LENGTH === pointBytes} Each point length needs to be a multiple of ${POINT_LENGTH}`;
+    check `
+    ${input.every(([_idx, point]) => point.length === pointBytes)},
+    Every piece must have the same length!
+  `;
+    const pieces = FixedSizeArray.fill(() => Bytes.zero(PIECE_SIZE), points);
+    for (let i = 0; i < points; i++) {
+        const start = i * POINT_LENGTH;
+        const pieceChunks = input.map(([index, piece]) => [
+            index,
+            // this is split?
+            Bytes.fromBlob(piece.raw.subarray(start, start + POINT_LENGTH), POINT_LENGTH),
+        ]);
+        pieces[i] = decodePiece(FixedSizeArray.new(pieceChunks, N_CHUNKS_REQUIRED));
+    }
+    return lace(pieces);
+}
+/**
+ *  Erasure-encoding function. Takes exactly `PIECE_SIZE` data and generate `N_CHUNKS_TOTAL` points.
+ *
+ *  https://graypaper.fluffylabs.dev/#/9a08063/3e4e013e5a01?v=0.6.6
+ */
+export function encodePoints(input) {
+    const result = [];
+    const data = safeAllocUint8Array(POINT_ALIGNMENT * N_CHUNKS_REQUIRED);
+    // add original shards to the result
+    for (let i = 0; i < N_CHUNKS_REQUIRED; i++) {
+        const pointStart = POINT_LENGTH * i;
+        result.push(Bytes.fromBlob(input.raw.subarray(pointStart, pointStart + POINT_LENGTH), POINT_LENGTH));
+        // fill array that will be passed to wasm lib
+        for (let j = 0; j < POINT_LENGTH; j++) {
+            data[i * POINT_ALIGNMENT + j * HALF_POINT_SIZE] = input.raw[pointStart + j];
+        }
+    }
+    // encode and add redundancy shards
+    const points = new reedSolomon.ShardsCollection(POINT_ALIGNMENT, data);
+    const encodedResult = reedSolomon.encode(N_CHUNKS_REDUNDANCY, points);
+    const encodedData = encodedResult.take_data();
+    for (let i = 0; i < N_CHUNKS_REDUNDANCY; i++) {
+        const pointIndex = i * POINT_ALIGNMENT;
+        const redundancyPoint = safeAllocUint8Array(POINT_LENGTH);
+        for (let j = 0; j < POINT_LENGTH; j++) {
+            redundancyPoint[j] = encodedData[pointIndex + j * HALF_POINT_SIZE];
+        }
+        result.push(Bytes.fromBlob(redundancyPoint, POINT_LENGTH));
+    }
+    return FixedSizeArray.new(result, N_CHUNKS_TOTAL);
+}
+/**
+ * The function takes exactly `N_CHUNKS_REQUIRED` chunks to recover the original piece.
+ *
+ * NOTE the piece may contain padding, so it should be trimmed externally.
+ */
+export function decodePiece(input) {
+    const result = Bytes.zero(PIECE_SIZE);
+    const data = safeAllocUint8Array(N_CHUNKS_REQUIRED * POINT_ALIGNMENT);
+    const indices = new Uint16Array(input.length);
+    for (let i = 0; i < N_CHUNKS_REQUIRED; i++) {
+        const [index, points] = input[i];
+        const pointStart = i * POINT_ALIGNMENT;
+        for (let j = 0; j < POINT_LENGTH; j++) {
+            data[pointStart + j * HALF_POINT_SIZE] = points.raw[j];
+        }
+        indices[i] = index;
+        if (index < N_CHUNKS_REQUIRED) {
+            // fill original shards in result
+            const pointStartInResult = POINT_LENGTH * index;
+            result.raw.set(points.raw, pointStartInResult);
+        }
+    }
+    const points = new reedSolomon.ShardsCollection(POINT_ALIGNMENT, data, indices);
+    const decodingResult = reedSolomon.decode(N_CHUNKS_REQUIRED, N_CHUNKS_REDUNDANCY, points);
+    const resultIndices = decodingResult.take_indices(); // it has to be called before take_data
+    const resultData = decodingResult.take_data(); // it destroys the result object in rust
+    if (resultIndices === undefined) {
+        throw new Error("indices array in decoded result must exist!");
+    }
+    check `${resultData.length === resultIndices.length * POINT_ALIGNMENT} incorrect length of data or indices!`;
+    for (let i = 0; i < resultIndices.length; i++) {
+        // fill reconstructed shards in result
+        const index = resultIndices[i];
+        const resultIndex = POINT_LENGTH * index;
+        const pointIndex = i * POINT_ALIGNMENT;
+        for (let j = 0; j < POINT_LENGTH; j++) {
+            result.raw[resultIndex + j] = resultData[pointIndex + j * HALF_POINT_SIZE];
+        }
+    }
+    return result;
+}
+/**
+ * `split`: Takes a single BytesBlob and divides it into sequential,
+ * contiguous Bytes of a given size.
+ *
+ * Opposite of `join`.
+ *
+ * Input: [a0, a1, a2, a3, a4, a5], n = 2, k = 3
+ *
+ * Output: [[a0, a1], [a2, a3], [a4, a5]]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3eb4013eb401?v=0.6.6
+ */
+export function split(input, n, k) {
+    check `${n * k === input.length}`;
+    const result = [];
+    for (let i = 0; i < k; i++) {
+        const start = i * n;
+        const bytes = Bytes.zero(n);
+        bytes.raw.set(input.raw.subarray(start, start + n));
+        result[i] = bytes;
+    }
+    return FixedSizeArray.new(result, k);
+}
+/**
+ * `join`: Takes an array of Bytes and concatenates them sequentially
+ * (one after another) into a single BytesBlob.
+ *
+ * The resulting blob has length of `N * K`.
+ *
+ * Opposite of `split`.
+ *
+ * Input: [[a0, a1], [a2, a3], [a4, a5]]
+ *
+ * Output: [a0, a1, a2, a3, a4, a5]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3ed4013ed401?v=0.6.6
+ */
+export function join(input) {
+    return BytesBlob.blobFromParts(input.map((x) => x.raw));
+}
+/**
+ * `unzip`: Reorganizes the data by de-interleaving: it takes every `K-th` byte
+ * for each of output Bytes, where `K` is the number of output Bytes
+ * and `N` is the size of each output Bytes.
+ *
+ * Opposite of `lace`.
+ *
+ * Input: [a0, b0, c0, a1, b1, c1], n = 2, k = 3
+ *
+ * Output: [[a0, a1], [b0, b1], [c0, c1]]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3e06023e0602?v=0.6.6
+ */
+export function unzip(input, n, k) {
+    const result = Array.from({ length: k }, () => Bytes.zero(n));
+    for (let i = 0; i < k; i++) {
+        const entry = result[i].raw;
+        for (let j = 0; j < n; j++) {
+            entry[j] = input.raw[j * k + i];
+        }
+    }
+    return FixedSizeArray.new(result, k);
+}
+/**
+ * `lace`: Takes an array of Bytes and interleaves:
+ * it takes the first Byte from each, then the second byte from
+ * each, etc., producing a single BytesBlob.
+ *
+ * Opposite of `unzip`.
+ *
+ * Input: [[a0, a1], [b0, b1], [c0, c1]]
+ *
+ * Output: [a0, b0, c0, a1, b1, c1]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3e2a023e2a02?v=0.6.6
+ */
+export function lace(input) {
+    const k = input.length;
+    if (k === 0) {
+        return BytesBlob.empty();
+    }
+    const n = input[0].length;
+    const result = BytesBlob.blobFrom(safeAllocUint8Array(k * n));
+    for (let i = 0; i < k; i++) {
+        const entry = input[i].raw;
+        for (let j = 0; j < n; j++) {
+            result.raw[j * k + i] = entry[j];
+        }
+    }
+    return result;
+}
+/**
+ * `T`: Transposing function which accepts an array of `K` pieces of data
+ * which each have same lenght of `N` octets and returns an array of `N`
+ * pieces of data which each have length of `K` octets.
+ *
+ * T[[x0,0, x0,1, x0,2, . . . ], [x1,0, x1,1, . . . ], . . . ] ≡
+ * [[x0,0, x1,0, x2,0, . . . ], [x0,1, x1,1, . . . ], . . . ]
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3e2e023e2e02?v=0.6.6
+ */
+export function transpose(input, k) {
+    const n = input.fixedLength;
+    const columns = [];
+    for (let c = 0; c < k; c++) {
+        const newColumn = [];
+        for (let r = 0; r < n; r++) {
+            const cell = input[r][c];
+            newColumn.push(cell);
+        }
+        columns.push(FixedSizeArray.new(newColumn, n));
+    }
+    return FixedSizeArray.new(columns, k);
+}
+/**
+ * `C`: Erasure-code chunking function which accepts an arbitrary sized data blob whose
+ *       length divides wholly into `PIECE_SIZE` octets and results in `N_CHUNKS_TOTAL`
+ *       sequences of sequences, each of `POINT_LENGTH * K` octets blobs.
+ *       Where `K` is the number that divides input length by `PIECE_SIZE`.
+ *
+ *       Each element of resulting array is the same length.
+ *
+ * https://graypaper.fluffylabs.dev/#/9a08063/3f15003f1500?v=0.6.6
+ */
+export function chunkingFunction(input) {
+    const k = Math.floor(input.length / PIECE_SIZE);
+    check `${k * PIECE_SIZE === input.length} Input length ${input.length} is not divisible by ${PIECE_SIZE}`;
+    // we get a `k` pieces.
+    const pieces = unzip(input, PIECE_SIZE, k);
+    // and each piece get's ec-codec
+    const points = pieces.map((p) => encodePoints(p));
+    const pointsTyped = FixedSizeArray.new(points, k);
+    // next we transpose the array, getting back an array of `N_CHUNKS_TOTAL` elements,
+    // where each element is a `k` points (`Bytes<POINT_LENGTH>`).
+    const transposed = transpose(pointsTyped, N_CHUNKS_TOTAL);
+    // lastly we join each element of that resulting array
+    // we get an array of `N_SHARDS_TOTAL` elements, each of length `POINT_LENGTH * k`.
+    const chunks = transposed.map((c) => join(c));
+    return FixedSizeArray.new(chunks, N_CHUNKS_TOTAL);
+}
+/** Split each validator's shard into numbered chunks it originally should have got. */
+export function shardsToChunks(spec, shards) {
+    const result = [];
+    const shardSize = shards[0].length;
+    check `
+    ${shards.every((s) => s.length === shardSize)}
+    Each shard must be the same length!
+  `;
+    const totalData = shards.map((s) => s.length).reduce((sum, sLength) => sum + sLength, 0);
+    const chunkSize = Math.floor(totalData / N_CHUNKS_TOTAL);
+    const piecesPerChunk = Math.floor(shardSize / chunkSize);
+    let currentChunk = 0;
+    for (const s of shards) {
+        const validatorChunks = [];
+        for (let i = 0; i < piecesPerChunk; i++) {
+            const start = i * chunkSize;
+            const end = start + chunkSize;
+            const chunk = BytesBlob.blobFrom(s.raw.subarray(start, end));
+            // TODO [ToDr] we may possibly have not enough data for some of the chunk here
+            if (chunk.length === chunkSize) {
+                validatorChunks.push([currentChunk, chunk]);
+            }
+            currentChunk = (currentChunk + 1) % N_CHUNKS_TOTAL;
+        }
+        result.push(validatorChunks);
+    }
+    return tryAsPerValidator(result, spec);
+}
+/** Divide chunks between validators. */
+export function chunksToShards(spec, chunks) {
+    const result = [];
+    const allChunks = BytesBlob.blobFromParts(chunks.map((c) => c.raw));
+    const shardSize = allChunks.length / N_CHUNKS_TOTAL;
+    // wrap around the data to have enough
+    const bytesToDrawFrom = BytesBlob.blobFromParts(allChunks.raw, allChunks.raw);
+    const bytesPerValidator = Math.ceil(allChunks.length / spec.validatorsCount);
+    // align number of bytes to the shard length.
+    const alignedBytesPerValidator = Math.ceil(bytesPerValidator / shardSize) * shardSize;
+    for (let i = 0; i < spec.validatorsCount; i++) {
+        const start = i * alignedBytesPerValidator;
+        const end = start + alignedBytesPerValidator;
+        result.push(BytesBlob.blobFrom(bytesToDrawFrom.raw.subarray(start, end)));
+    }
+    return tryAsPerValidator(result, spec);
+}

package/packages/core/erasure-coding/erasure-coding.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=erasure-coding.test.d.ts.map

package/packages/core/erasure-coding/erasure-coding.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"erasure-coding.test.d.ts","sourceRoot":"","sources":["../../../../../packages/core/erasure-coding/erasure-coding.test.ts"],"names":[],"mappings":""}