mca-json 1.0.0

package/src/lib/anvil.ts

@@ -0,0 +1,178 @@
+import { BinaryData } from './binary-data';
+import { Chunk } from './chunk';
+import { Coords2d } from '../types/coords';
+import { inflate } from 'pako';
+import { Nbt } from '../nbt/nbt';
+
+type LocationEntry = {
+    offset: number; // 3 bytes
+    sectorCount: number; // 1 byte
+};
+
+enum CompressionType {
+    GZIP = 1,
+    ZLIB = 2,
+    NONE = 3,
+}
+
+const LOCATION_ENTRIES_PER_FILE = 1024;
+const SECTOR_SIZE = 4096;
+
+/**
+ * Read and write Minecraft Anvil region files.
+ */
+export class Anvil {
+    static fromBuffer(buffer: ArrayBufferLike): Anvil {
+        return new Anvil(buffer);
+    }
+
+    cachedLocationEntries: LocationEntry[] | null = null;
+    data: BinaryData;
+
+    constructor(data: ArrayBufferLike) {
+        this.data = new BinaryData(data);
+    }
+
+    /**
+     * Returns the buffer that holds the Anvil data.
+     */
+    buffer(): ArrayBufferLike {
+        return this.data.buffer();
+    }
+
+    /**
+     * Delete a chunk. The location is either given in X,Z chunk coordinates or
+     * passed in from a Chunk object.
+     */
+    deleteChunk(chunkLocation: Coords2d | Chunk): void {
+        let index;
+
+        if (Array.isArray(chunkLocation)) {
+            index = this.chunkCoordinatesToIndex(chunkLocation);
+        } else {
+            const coords = chunkLocation.chunkCoordinates();
+
+            if (!coords) {
+                throw new Error('Chunk does not have coordinates');
+            }
+
+            index = this.chunkCoordinatesToIndex(coords);
+        }
+
+        // Clear cache before modifying location entries
+        this.cachedLocationEntries = null;
+        this.data.seek(index * 4);
+        this.data.setNByteInteger(0, 3);
+        this.data.setByte(0);
+    }
+
+    /**
+     * Parses all chunks in the Anvil blob. Chunk objects are read in the order
+     * they are stored in the location entry header sector.
+     *
+     * @returns Chunk[] a list of Chunk objects
+     */
+    getAllChunks(): Chunk[] {
+        // Empty files can be written
+        if (this.data.byteLength() === 0) {
+            return [];
+        }
+
+        const chunks = this.getLocationEntries()
+            .filter((x) => x.sectorCount > 0)
+            .map((offset) => {
+                const nbt = Nbt.fromBuffer(this.getChunkData(offset.offset));
+
+                return new Chunk(nbt);
+            });
+
+        return chunks;
+    }
+
+    // Gets a chunk using chunk coordinates.
+    getChunk(chunkCoords: Coords2d): Chunk | undefined {
+        const locationEntries = this.getLocationEntries();
+        const entry = locationEntries[this.chunkCoordinatesToIndex(chunkCoords)];
+
+        if (entry?.sectorCount > 0) {
+            const nbt = Nbt.fromBuffer(this.getChunkData(entry.offset));
+
+            return new Chunk(nbt);
+        }
+
+        return;
+    }
+
+    /**
+     * Convert from chunk coordinates to region-relative chunk coordinates, and
+     * then to an index.
+     */
+    private chunkCoordinatesToIndex(chunkCoords: Coords2d): number {
+        let regionChunkX = chunkCoords[0];
+        let regionChunkZ = chunkCoords[1];
+        const step = 4096 * 4096;
+
+        while (regionChunkX < 0) {
+            regionChunkX += step;
+        }
+
+        while (regionChunkZ < 0) {
+            regionChunkZ += step;
+        }
+
+        return (regionChunkX % 32) + (regionChunkZ % 32) * 32;
+    }
+
+    /**
+     * Extracts data for a chunk present at the given offset in the Anvil blob.
+     * The offset is provided in sectors of SECTOR_SIZE bytes (as in the chunk
+     * location entries from the blob's first header sector).
+     *
+     * @param offset the chunk's offset in number of sectors from the Anvil blob's start.
+     */
+    protected getChunkData(offset?: number): ArrayBufferLike {
+        if (offset !== undefined) {
+            this.data.seek(offset * SECTOR_SIZE);
+        }
+
+        const length = this.data.getUInt();
+        const compressionType = this.data.getByte();
+        const data = this.data.getArrayBuffer(length - 1); // First byte is the compression type
+
+        if (
+            compressionType === CompressionType.ZLIB ||
+            compressionType === CompressionType.GZIP
+        ) {
+            // Could be SharedArrayBuffer, but it won't get used again after
+            // inflation, so it being "shared" does not matter.
+            return inflate(data as ArrayBuffer).buffer;
+        }
+
+        return data;
+    }
+
+    /**
+     * Retrieves the full list of chunk location entries from this Anvil blob.
+     * The pointer will be moved to the beginning of the first header sector at
+     * the start of this method and advanced to the end during reading.
+     */
+    protected getLocationEntries(): LocationEntry[] {
+        if (this.cachedLocationEntries) {
+            return this.cachedLocationEntries;
+        }
+
+        this.data.seek(0);
+        const result: LocationEntry[] = [];
+
+        for (let i = 0; i < LOCATION_ENTRIES_PER_FILE; i += 1) {
+            result.push({
+                offset: this.data.getNByteInteger(3),
+                sectorCount: this.data.getByte(),
+            });
+        }
+
+        this.cachedLocationEntries = result;
+
+        return result;
+    }
+}

package/src/lib/binary-data.ts

@@ -0,0 +1,247 @@
+/**
+ * Create a read/write view of a binary data buffer. Write operations that are
+ * beyond the buffer's current size will increase the buffer's size in chunks.
+ */
+export class BinaryData {
+    private position = 0;
+    private view: DataView;
+
+    constructor(data: ArrayBufferLike) {
+        this.view = new DataView(data);
+    }
+
+    buffer(): ArrayBufferLike {
+        return this.view.buffer;
+    }
+
+    byteLength(): number {
+        return this.view.byteLength;
+    }
+
+    currentPosition(): number {
+        return this.position;
+    }
+
+    getArrayBuffer(length: number) {
+        const data = this.view.buffer.slice(
+            this.position,
+            this.position + length
+        );
+        this.position += length;
+
+        return data;
+    }
+
+    getByte() {
+        const value = this.view.getUint8(this.position);
+        this.position += 1;
+
+        return value;
+    }
+
+    getDouble() {
+        const value = this.view.getFloat64(this.position);
+        this.position += 8;
+
+        return value;
+    }
+
+    getFloat() {
+        const value = this.view.getFloat32(this.position);
+        this.position += 4;
+
+        return value;
+    }
+
+    getInt() {
+        const value = this.view.getInt32(this.position);
+        this.position += 4;
+
+        return value;
+    }
+
+    getInt64(): bigint {
+        const value = this.view.getBigInt64(this.position, true);
+        this.position += 8;
+
+        return value;
+    }
+
+    getInt64LE(): bigint {
+        const value = this.view.getBigInt64(this.position);
+        this.position += 8;
+
+        return value;
+    }
+
+    getNByteInteger(n: number) {
+        let value = 0;
+
+        for (let i = 0; i < n; i += 1) {
+            const b = this.getByte();
+            value = value * 256 + b;
+        }
+
+        return value;
+    }
+
+    getShort() {
+        const value = this.view.getInt16(this.position);
+        this.position += 2;
+
+        return value;
+    }
+
+    getString(len: number) {
+        let s = '';
+
+        for (let i = 0; i < len; i += 1) {
+            const c = this.getByte();
+
+            if (c > 0) {
+                s += String.fromCharCode(c);
+            }
+        }
+
+        return s;
+    }
+
+    getUInt() {
+        const value = this.view.getUint32(this.position);
+        this.position += 4;
+
+        return value;
+    }
+
+    getUInt64(): bigint {
+        const v = this.view.getBigUint64(this.position);
+        this.position += 8;
+
+        return v;
+    }
+
+    getUInt64LE(): bigint {
+        const v = this.view.getBigUint64(this.position, true);
+        this.position += 8;
+
+        return v;
+    }
+
+    getUShort() {
+        const value = this.view.getUint16(this.position);
+        this.position += 2;
+
+        return value;
+    }
+
+    /**
+     * Checks how many bytes are between the current position and the end of
+     * the buffer.
+     */
+    remainingLength() {
+        return this.view.byteLength - this.position;
+    }
+
+    seek(offset: number): void {
+        this.position = offset;
+    }
+
+    setArrayBuffer(value: ArrayBuffer) {
+        this.growToFit(value.byteLength);
+        new Uint8Array(this.view.buffer).set(
+            new Uint8Array(value),
+            this.position
+        );
+        this.position += value.byteLength;
+    }
+
+    setByte(value: number) {
+        this.growToFit(1);
+        this.view.setUint8(this.position, value);
+        this.position += 1;
+    }
+
+    setDouble(value: number) {
+        this.growToFit(8);
+        this.view.setFloat64(this.position, value);
+        this.position += 8;
+    }
+
+    setFloat(value: number) {
+        this.growToFit(4);
+        this.view.setFloat32(this.position, value);
+        this.position += 4;
+    }
+
+    setInt(value: number) {
+        this.growToFit(4);
+        this.view.setInt32(this.position, value);
+        this.position += 4;
+    }
+
+    setInt64(value: bigint) {
+        this.growToFit(8);
+        this.view.setBigInt64(this.position, value);
+        this.position += 8;
+    }
+
+    setInt64LE(value: bigint) {
+        this.growToFit(8);
+        this.view.setBigUint64(this.position, value, true);
+        this.position += 8;
+    }
+
+    setNByteInteger(value: number, size: number) {
+        this.growToFit(size);
+
+        for (let i = size - 1; i >= 0; i -= 1) {
+            this.setByte(Math.floor(value / Math.pow(256, i)) % 256);
+        }
+    }
+
+    setShort(value: number) {
+        this.growToFit(2);
+        this.view.setInt16(this.position, value);
+        this.position += 2;
+    }
+
+    setString(value: string) {
+        this.growToFit(value.length);
+
+        for (let i = 0; i < value.length; i += 1) {
+            this.setByte(value.charCodeAt(i));
+        }
+    }
+
+    setUInt(value: number) {
+        this.growToFit(4);
+        this.view.setUint32(this.position, value);
+        this.position += 4;
+    }
+
+    setUInt64(value: bigint) {
+        this.growToFit(8);
+        this.view.setBigUint64(this.position, value);
+        this.position += 8;
+    }
+
+    setUInt64LE(value: bigint) {
+        this.growToFit(8);
+        this.view.setBigUint64(this.position, value, true);
+        this.position += 8;
+    }
+
+    setUShort(value: number) {
+        this.growToFit(2);
+        this.view.setUint16(this.position, value);
+        this.position += 2;
+    }
+
+    private growToFit(size: number) {
+        while (size > this.remainingLength()) {
+            const newBuffer = new ArrayBuffer(this.view.byteLength * 2);
+            new Uint8Array(newBuffer).set(new Uint8Array(this.view.buffer));
+            this.view = new DataView(newBuffer);
+        }
+    }
+}

package/src/lib/bit-data.ts

@@ -0,0 +1,101 @@
+import debug from 'debug';
+import { NbtLongArray } from '../nbt/nbt-long-array';
+
+const debugLog = debug('bit-data');
+
+// At most, we will have 7 bits to mask
+const MASK = [0, 1, 3, 7, 15, 31, 63, 127];
+
+export class BitData {
+    static fromLongArrayTag(tag: NbtLongArray): BitData {
+        const arrayBuffer = new ArrayBuffer(tag.data.length * 8);
+        const view = new DataView(arrayBuffer);
+        let position = 0;
+
+        for (const value of tag.data) {
+            // Convert back to a byte stream. The source comes from
+            // NbtLongArray.fromBinaryData(). Make sure the endianness is
+            // correct and matches the reader.
+            view.setBigInt64(position, value, true);
+            position += 8;
+        }
+
+        return new BitData(arrayBuffer);
+    }
+
+    private length: number;
+    private partial = 0;
+    private partialCount = 0;
+    private position = 0;
+    private view: DataView;
+
+    constructor(data: ArrayBuffer) {
+        this.view = new DataView(data);
+        this.length = data.byteLength;
+    }
+
+    currentPosition() {
+        return this.position;
+    }
+
+    getBits(n: number): number {
+        debugLog(
+            `getBits(${n}), start, position ${this.position}, bits ${this.partial.toString(2).padStart(this.partialCount, '0').substr(-this.partialCount)}`
+        );
+
+        // Add extra bits to the left
+        while (n > this.partialCount) {
+            if (this.position >= this.length) {
+                throw new Error('Out of data');
+            }
+
+            this.partial |=
+                this.view.getUint8(this.position) << this.partialCount;
+            debugLog(
+                `getBits(${n}), add byte, position ${this.position}, bits ${this.partial.toString(2).padStart(this.partialCount, '0').substr(-this.partialCount)}`
+            );
+            this.position += 1;
+            this.partialCount += 8;
+        }
+
+        // Return bits from the right
+        const result = this.partial & MASK[n];
+        this.partial >>= n;
+        this.partialCount -= n;
+        debugLog(
+            `getBits(${n}), end, position ${this.position}, bits ${this.partial.toString(2).padStart(this.partialCount, '0').substr(-this.partialCount)}, result ${result.toString(2).padStart(n, '0')} (${result})`
+        );
+
+        return result;
+    }
+
+    // The amount of bytes left in the buffer
+    remainingLength() {
+        return this.length - this.position;
+    }
+
+    seek(position: number, partialCount = 0) {
+        this.position = position;
+        this.partial = 0;
+        this.partialCount = partialCount;
+    }
+
+    // FIXME - need to figure out how this is written. Does it update just one byte?
+    // FIXME - I don't see a flush method.
+    setBits(n: number, value: number) {
+        // Add the new bits to the right
+        this.partial <<= n;
+        this.partial |= value;
+        this.partialCount += n;
+
+        while (this.partialCount >= 8) {
+            this.view.setUint8(
+                this.position,
+                this.partial >> (this.partialCount - 8)
+            );
+            this.position += 1;
+            this.partialCount -= 8;
+            this.partial &= MASK[this.partialCount];
+        }
+    }
+}

package/src/lib/block-data.ts

@@ -0,0 +1,180 @@
+import debug from 'debug';
+import { BitData } from './bit-data';
+import { Coords3d } from '../types/coords';
+import { NbtCompound } from '../nbt/nbt-compound';
+import { NbtList } from '../nbt/nbt-list';
+import { NbtLongArray } from '../nbt/nbt-long-array';
+import { NbtString } from '../nbt/nbt-string';
+
+const BLOCKS_PER_CHUNK = 16 * 16 * 16;
+const debugLog = debug('block-data');
+
+function parseBlockData(
+    blockIds: number[],
+    bitData: BitData,
+    paletteMap: Map<number, string>
+) {
+    let bitsRead = 0;
+    let blockIdsRead = 0;
+    const bitsNeededForPalette = Math.max(
+        4,
+        Math.ceil(Math.log2(paletteMap.size || 1))
+    );
+
+    while (bitData.remainingLength() > 0) {
+        while (bitsRead + bitsNeededForPalette <= 64) {
+            const blockId = bitData.getBits(bitsNeededForPalette);
+
+            if (!paletteMap.has(blockId)) {
+                throw new Error(
+                    `Block ID ${blockId}, index ${blockIdsRead} not found in palette when reading index ${blockIdsRead}`
+                );
+            }
+
+            debugLog(
+                `Block ID ${blockId} (${paletteMap.get(blockId)}) at index ${blockIdsRead}`
+            );
+            blockIds.push(blockId);
+            bitsRead += bitsNeededForPalette;
+            blockIdsRead += 1;
+        }
+
+        if (bitsRead < 64) {
+            // These extra bits can have any values. They are not
+            // initialized before writing to disk.
+            bitData.getBits(64 - bitsRead);
+            debugLog(`Skipped ${64 - bitsRead} bits`);
+        }
+
+        bitsRead = 0;
+    }
+
+    return blockIds;
+}
+
+export class BlockData {
+    static fromPaletteBlockStates(
+        dataVersion: number,
+        palette: NbtList<NbtCompound>,
+        blockStates?: NbtLongArray
+    ) {
+        // Convert the palette values to a map
+        const paletteSize = palette.data.length;
+        const paletteMap = new Map<number, string>();
+
+        if (dataVersion < 0) {
+            throw new Error('Data version is required to read block data');
+        }
+
+        for (let i = 0; i < paletteSize; i++) {
+            const name = palette.data[i].findChild<NbtString>('Name');
+
+            if (name) {
+                paletteMap.set(i, name.data);
+            } else {
+                throw new Error(`Palette entry ${i} does not have a name`);
+            }
+        }
+
+        const blockIds: number[] = [];
+
+        if (blockStates) {
+            const bitData = BitData.fromLongArrayTag(blockStates);
+            parseBlockData(blockIds, bitData, paletteMap);
+        }
+
+        // Fill the rest of the blocks with the first element in the palette.
+        // The documentation says that a chunk could be filled with just the
+        // first index in the palette and not have any block data.
+        while (blockIds.length < BLOCKS_PER_CHUNK) {
+            blockIds.push(0);
+        }
+
+        // Safety
+        while (blockIds.length > BLOCKS_PER_CHUNK) {
+            blockIds.pop();
+        }
+
+        return new BlockData(paletteMap, blockIds);
+    }
+
+    constructor(
+        public paletteMap: Map<number, string>,
+        public blockIds: number[]
+    ) {}
+
+    chunkCoordinatesToIndex([x, y, z]: Coords3d) {
+        return (y << 8) | (z << 4) | x;
+    }
+
+    findBlocksByName(name: string): number[] {
+        const number = Array.from(this.paletteMap.keys()).find(
+            (key) => this.paletteMap.get(key) === name
+        );
+        const result = [];
+
+        for (let i = 0; i < this.blockIds.length; i++) {
+            if (this.blockIds[i] === number) {
+                result.push(i);
+            }
+        }
+
+        return result;
+    }
+
+    getBlockByIndex(index: number) {
+        const blockId = this.blockIds[index];
+        const blockName = this.paletteMap.get(blockId);
+
+        if (!blockName) {
+            throw new Error(
+                `Block ID ${blockId} not found in palette when reading index ${index}`
+            );
+        }
+
+        return blockName;
+    }
+
+    getBlockByChunkCoordinates(coordinates: Coords3d) {
+        const index = this.chunkCoordinatesToIndex(coordinates);
+
+        return this.getBlockByIndex(index);
+    }
+
+    indexToChunkCoordinates(index: number): Coords3d {
+        const x = index & 0xf;
+        const y = (index >> 8) & 0xf;
+        const z = (index >> 4) & 0xf;
+
+        return [x, y, z];
+    }
+
+    toArray() {
+        let errors: any[] = [];
+
+        const result = this.blockIds.map((blockId, index) => {
+            const blockName = this.paletteMap.get(blockId);
+
+            if (!blockName) {
+                errors.push({
+                    index,
+                    chunkCoordinates: this.indexToChunkCoordinates(index),
+                    blockId,
+                    paletteLength: this.paletteMap.size,
+                    palette: Array.from(this.paletteMap.values()),
+                });
+            }
+
+            return {
+                chunkCoordinates: this.indexToChunkCoordinates(index),
+                blockName,
+            };
+        });
+
+        if (errors.length) {
+            throw new Error(JSON.stringify(errors));
+        }
+
+        return result;
+    }
+}