@rhyster/wow-casc-dbc 2.6.18 → 2.7.0

package/src/dbd.ts

@@ -0,0 +1,427 @@
+import assert from 'node:assert';
+
+import type WDCReader from './wdc.ts';
+
+interface Manifest {
+    tableHash: string,
+    tableName?: string,
+    db2FileDataID?: number,
+    dbcFileDataID?: number,
+}
+
+interface Column {
+    name: string,
+    type: string,
+    isID: boolean,
+    isInline: boolean,
+    isRelation: boolean,
+    isSigned: boolean,
+    size?: number,
+    arraySize?: number,
+}
+
+type BasicColumnData = number | bigint | string | undefined;
+
+type ColumnData = BasicColumnData | BasicColumnData[];
+
+const PATTERN_COLUMN = /^(int|float|locstring|string)(<[^:]+::[^>]+>)?\s([^\s]+)/;
+const PATTERN_LAYOUT = /^LAYOUT\s(.*)/;
+const PATTERN_FIELD = /^(\$([^$]+)\$)?([^<[]+)(<(u|)(\d+)>)?(\[(\d+)\])?$/;
+
+const castIntegerBySize = (
+    value: number,
+    src: number,
+    srcSigned: boolean,
+    dst: number,
+    dstSigned: boolean,
+): number => {
+    const castBuffer = Buffer.alloc(6);
+
+    if (srcSigned) {
+        castBuffer.writeIntLE(value, 0, src);
+    } else {
+        castBuffer.writeUIntLE(value, 0, src);
+    }
+
+    return dstSigned ? castBuffer.readIntLE(0, dst) : castBuffer.readUIntLE(0, dst);
+};
+
+const castFloat = (value: number, src: number, srcSigned: boolean): number => {
+    const castBuffer = Buffer.alloc(4);
+
+    if (srcSigned) {
+        castBuffer.writeIntLE(value, 0, src);
+    } else {
+        castBuffer.writeUIntLE(value, 0, src);
+    }
+
+    const result = castBuffer.readFloatLE(0);
+    return Math.round(result * 100) / 100;
+};
+
+const castBigInt64 = (value: bigint, srcSigned: boolean, dstSigned: boolean): bigint => {
+    const castBuffer = Buffer.alloc(8);
+
+    if (srcSigned) {
+        castBuffer.writeBigInt64LE(value, 0);
+    } else {
+        castBuffer.writeBigUInt64LE(value, 0);
+    }
+
+    return dstSigned ? castBuffer.readBigInt64LE(0) : castBuffer.readBigUInt64LE(0);
+};
+
+const getCastBuffer = (value: bigint, srcSize: number, dstSize: number): Buffer => {
+    const castBuffer = Buffer.alloc(dstSize);
+    let remain = value;
+
+    // eslint-disable-next-line no-bitwise
+    for (let i = 0; i < srcSize && remain > 0n; i += 1, remain >>= 8n) {
+        const byte = Number(BigInt.asUintN(8, remain));
+        castBuffer.writeUInt8(byte, i);
+    }
+
+    return castBuffer;
+};
+
+export default class DBDParser {
+    public readonly wdc: WDCReader;
+
+    public readonly definitions = new Map<string, string>();
+
+    public columns: Column[] = [];
+
+    private cache = new Map<number, Record<string, ColumnData>>();
+
+    private constructor(wdc: WDCReader) {
+        this.wdc = wdc;
+    }
+
+    private async init(): Promise<void> {
+        const manifestsURL = 'https://raw.githubusercontent.com/wowdev/WoWDBDefs/master/manifest.json';
+        const manifests = await (await fetch(manifestsURL)).json() as Manifest[];
+
+        const tableHashHex = this.wdc.tableHash.toString(16).padStart(8, '0').toLowerCase();
+        const manifest = manifests.find((v) => v.tableHash.toLowerCase() === tableHashHex);
+
+        assert(manifest?.tableName !== undefined, `No manifest found for table hash ${tableHashHex}`);
+
+        const url = `https://raw.githubusercontent.com/wowdev/WoWDBDefs/master/definitions/${manifest.tableName}.dbd`;
+        const text = await (await fetch(url)).text();
+        const lines = text.split('\n').map((v) => v.trim());
+
+        const chunks = lines.reduce<string[][]>((acc, line) => {
+            if (line.length > 0) {
+                acc[acc.length - 1].push(line);
+            } else {
+                acc.push([]);
+            }
+            return acc;
+        }, [[]]).filter((chunk) => chunk.length > 0);
+
+        const columnsChunk = chunks.shift();
+        assert(columnsChunk?.[0] === 'COLUMNS', 'No column definitions found');
+
+        columnsChunk.shift();
+        columnsChunk.forEach((line) => {
+            const match = PATTERN_COLUMN.exec(line);
+            if (match) {
+                const [, type, , name] = match;
+                this.definitions.set(name.replace('?', ''), type);
+            }
+        });
+
+        const layoutHashHex = this.wdc.layoutHash.toString(16).padStart(8, '0').toLowerCase();
+        const versionChunk = chunks.find((chunk) => chunk.find((line) => {
+            const layoutsMatch = PATTERN_LAYOUT.exec(line);
+            const layouts = layoutsMatch?.[1].split(',').map((v) => v.trim().toLowerCase());
+            return layouts?.includes(layoutHashHex);
+        }));
+
+        assert(versionChunk, `No version definition found for layout hash ${layoutHashHex}`);
+
+        versionChunk.forEach((line) => {
+            if (line.startsWith('LAYOUT') || line.startsWith('BUILD') || line.startsWith('COMMENT')) {
+                return;
+            }
+
+            const match = PATTERN_FIELD.exec(line);
+            if (match) {
+                const [
+                    , ,
+                    annotationsText,
+                    name, ,
+                    unsigned,
+                    sizeText, ,
+                    arraySizeText,
+                ] = match;
+                const type = this.definitions.get(name);
+
+                assert(type !== undefined, `No type found for column ${name}`);
+
+                const annotations = annotationsText ? annotationsText.split(',').map((v) => v.trim()) : [];
+                const size = sizeText ? parseInt(sizeText, 10) : undefined;
+                const arraySize = arraySizeText ? parseInt(arraySizeText, 10) : undefined;
+
+                const isID = !!annotations.includes('id');
+                const isInline = !annotations.includes('noninline');
+                const isRelation = !!annotations.includes('relation');
+                const isSigned = !unsigned;
+
+                this.columns.push({
+                    name,
+                    type,
+                    isID,
+                    isInline,
+                    isRelation,
+                    isSigned,
+                    size,
+                    arraySize,
+                });
+            }
+        });
+    }
+
+    static async parse(wdc: WDCReader): Promise<DBDParser> {
+        const parser = new DBDParser(wdc);
+
+        await parser.init();
+
+        return parser;
+    }
+
+    getAllIDs(): number[] {
+        return this.wdc.getAllIDs();
+    }
+
+    getRowData(id: number): Record<string, ColumnData> | undefined {
+        if (this.cache.has(id)) {
+            return structuredClone(this.cache.get(id));
+        }
+
+        const row = this.wdc.getRowData(id);
+        if (!row) {
+            return undefined;
+        }
+
+        const data: Record<string, ColumnData> = {};
+        if (Array.isArray(row)) {
+            let fieldIndex = 0;
+            this.columns.forEach((column) => {
+                if (column.isID) {
+                    data[column.name] = id;
+
+                    if (column.isInline) {
+                        fieldIndex += 1;
+                    }
+                } else if (column.isInline) {
+                    assert(row.length > fieldIndex, `No value found for column ${column.name}`);
+
+                    const cell = row[fieldIndex];
+                    const fieldInfo = this.wdc.fieldsInfo[fieldIndex];
+                    const srcSigned = fieldInfo.storageType === 'bitpackedSigned';
+                    const srcSize = (
+                        fieldInfo.storageType === 'none'
+                        || fieldInfo.storageType === 'bitpacked'
+                        || fieldInfo.storageType === 'bitpackedSigned'
+                    )
+                        ? Math.ceil(fieldInfo.fieldSizeBits / 8)
+                        : 4;
+                    const dstSize = column.size !== undefined
+                        ? Math.ceil(column.size / 8)
+                        : undefined;
+
+                    if (cell.type === 'bitpackedArray') {
+                        data[column.name] = cell.data.map((v) => {
+                            if (column.type === 'float') {
+                                return castFloat(v, srcSize, srcSigned);
+                            }
+                            if (dstSize !== undefined) {
+                                return castIntegerBySize(
+                                    v,
+                                    srcSize,
+                                    srcSigned,
+                                    dstSize,
+                                    column.isSigned,
+                                );
+                            }
+                            return v;
+                        });
+                    } else if (column.type === 'string' || column.type === 'locstring') {
+                        if (cell.data > 0) {
+                            assert(cell.type === 'none', `Invalid data type for string column ${column.name}`);
+                            assert(typeof cell.string === 'string', `Missing string for string column ${column.name}`);
+
+                            data[column.name] = cell.string;
+                        }
+                    } else if (column.type === 'float') {
+                        if (column.arraySize !== undefined) {
+                            const castBuffer = getCastBuffer(
+                                typeof cell.data === 'number' ? BigInt(cell.data) : cell.data,
+                                srcSize,
+                                4 * column.arraySize,
+                            );
+
+                            const values: number[] = [];
+                            for (let i = 0; i < column.arraySize; i += 1) {
+                                const value = castBuffer.readFloatLE(i * 4);
+                                values.push(Math.round(value * 100) / 100);
+                            }
+
+                            data[column.name] = values;
+                        } else {
+                            assert(typeof cell.data === 'number', `Invalid data type for float column ${column.name}`);
+
+                            data[column.name] = castFloat(cell.data, srcSize, srcSigned);
+                        }
+                    } else if (column.type === 'int') {
+                        if (column.arraySize !== undefined) {
+                            assert(dstSize !== undefined, `Missing size for int array column ${column.name}`);
+
+                            const castBuffer = getCastBuffer(
+                                typeof cell.data === 'number' ? BigInt(cell.data) : cell.data,
+                                srcSize,
+                                4 * column.arraySize,
+                            );
+
+                            const values: number[] = [];
+                            if (column.isSigned) {
+                                for (let i = 0; i < column.arraySize; i += 1) {
+                                    const value = castBuffer.readIntLE(i * dstSize, dstSize);
+                                    values.push(value);
+                                }
+                            } else {
+                                for (let i = 0; i < column.arraySize; i += 1) {
+                                    const value = castBuffer.readUIntLE(i * dstSize, dstSize);
+                                    values.push(value);
+                                }
+                            }
+
+                            data[column.name] = values;
+                        } else if (typeof cell.data === 'number') {
+                            data[column.name] = castIntegerBySize(
+                                cell.data,
+                                srcSize,
+                                srcSigned,
+                                dstSize ?? srcSize,
+                                column.isSigned,
+                            );
+                        } else {
+                            assert(column.size === undefined || column.size === 64, `Unexpected size ${column.size?.toString() ?? ''} for column ${column.name}`);
+
+                            if (srcSigned !== column.isSigned) {
+                                data[column.name] = castBigInt64(
+                                    cell.data,
+                                    srcSigned,
+                                    column.isSigned,
+                                );
+                            } else {
+                                data[column.name] = cell.data;
+                            }
+                        }
+                    } else {
+                        throw new Error(`Unsupported column type ${column.type} for column ${column.name}`);
+                    }
+
+                    fieldIndex += 1;
+                } else if (column.isRelation) {
+                    const relation = this.wdc.getRowRelationship(id);
+                    data[column.name] = relation ?? 0;
+                }
+            });
+        } else {
+            const buffer = row.data;
+            let offset = 0;
+            let fieldIndex = 0;
+            this.columns.forEach((column) => {
+                if (column.isID) {
+                    data[column.name] = id;
+
+                    if (column.isInline) {
+                        const currField = this.wdc.fields[fieldIndex];
+                        const size = Math.ceil((column.size ?? (32 - currField.size)) / 8);
+                        offset += size;
+
+                        fieldIndex += 1;
+                    }
+                } else if (column.isInline) {
+                    const values = [];
+
+                    if (column.type === 'string' || column.type === 'locstring') {
+                        const count = column.arraySize ?? 1;
+
+                        for (let i = 0; i < count; i += 1) {
+                            const startOffset = offset;
+                            while (buffer[offset] !== 0x00) {
+                                offset += 1;
+                            }
+
+                            values.push(buffer.toString('utf-8', startOffset, offset));
+                            offset += 1;
+                        }
+
+                        data[column.name] = count > 1 ? values : values[0];
+                    } else {
+                        // note: layout hash won't change when array size changes
+                        // so we try to determine array size based on field structure
+                        const currField = this.wdc.fields[fieldIndex];
+                        const nextField = this.wdc.fields[fieldIndex + 1];
+                        const size = Math.ceil((column.size ?? (32 - currField.size)) / 8);
+
+                        let count;
+                        if (fieldIndex + 1 < this.wdc.fields.length) {
+                            // reading hotfix for normal db2 may have fields with same position
+                            count = Math.max((nextField.position - currField.position) / size, 1);
+                        } else {
+                            // nextPos = byteLength - offset + currPos
+                            count = column.arraySize !== undefined
+                                ? ((buffer.byteLength - offset) / size)
+                                : 1;
+                        }
+
+                        for (let i = 0; i < count; i += 1) {
+                            if (column.type === 'float') {
+                                const value = buffer.readFloatLE(offset);
+                                values.push(Math.round(value * 100) / 100);
+                                offset += 4;
+                            } else if (size > 6) {
+                                assert(size === 8, `Unexpected size ${size.toString()} for column ${column.name}`);
+
+                                const value = column.isSigned
+                                    ? buffer.readBigInt64LE(offset)
+                                    : buffer.readBigUInt64LE(offset);
+
+                                values.push(value);
+                                offset += size;
+                            } else {
+                                const value = column.isSigned
+                                    ? buffer.readIntLE(offset, size)
+                                    : buffer.readUIntLE(offset, size);
+
+                                values.push(value);
+                                offset += size;
+                            }
+                        }
+
+                        data[column.name] = count > 1 ? values : values[0];
+                    }
+
+                    fieldIndex += 1;
+                } else if (column.isRelation) {
+                    const relation = this.wdc.getRowRelationship(id);
+                    data[column.name] = relation ?? 0;
+                }
+            });
+        }
+
+        this.cache.set(id, data);
+
+        return structuredClone(data);
+    }
+}
+
+export type {
+    Column,
+    ColumnData,
+    BasicColumnData,
+};

package/src/fetcher.ts

@@ -0,0 +1,223 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import http from 'node:http';
+import path from 'node:path';
+
+import cliProgress from 'cli-progress';
+
+import Store from './store.ts';
+
+const USER_AGENT = 'node-wow-casc-dbc';
+
+const CACHE_ROOT = path.resolve('cache');
+const CACHE_DIRS = {
+    build: 'builds',
+    indexes: 'indices',
+    data: 'data',
+    dbd: 'dbd',
+};
+
+const CACHE_INTEGRITY_FILE = path.resolve(CACHE_ROOT, 'integrity.json');
+
+const cacheIntegrity = new Store<string, string>(CACHE_INTEGRITY_FILE);
+
+const formatCDNKey = (key: string): string => `${key.substring(0, 2)}/${key.substring(2, 4)}/${key}`;
+
+const requestData = async (
+    url: string,
+    {
+        partialOffset,
+        partialLength,
+        showProgress,
+    }: {
+        partialOffset?: number,
+        partialLength?: number,
+        showProgress?: boolean,
+    } = {},
+): Promise<Buffer> => new Promise((resolve, reject) => {
+    const options = {
+        headers: {
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            'User-Agent': USER_AGENT,
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            Range: partialOffset !== undefined && partialLength !== undefined
+                ? `bytes=${partialOffset.toString()}-${(partialOffset + partialLength - 1).toString()}`
+                : 'bytes=0-',
+        },
+    };
+
+    http.get(url, options, (res) => {
+        if (res.statusCode === 301 || res.statusCode === 302) {
+            if (res.headers.location !== undefined) {
+                requestData(res.headers.location, { partialOffset, partialLength, showProgress })
+                    .then(resolve)
+                    .catch((err: unknown) => {
+                        throw err;
+                    });
+            } else {
+                reject(new Error(`Failed to request ${url}, Status Code: ${res.statusCode.toString()}`));
+            }
+            return;
+        }
+
+        if (res.statusCode === undefined || res.statusCode < 200 || res.statusCode > 302) {
+            reject(new Error(`Failed to request ${url}, Status Code: ${res.statusCode?.toString() ?? 'undefined'}`));
+            return;
+        }
+
+        const lengthText = res.headers['content-length'];
+        const length = lengthText !== undefined ? parseInt(lengthText, 10) : 0;
+        const bar = showProgress === true && !Number.isNaN(length) && length >= 10485760
+            ? new cliProgress.SingleBar({ etaBuffer: 10240 }, cliProgress.Presets.shades_classic)
+            : undefined;
+        bar?.start(length, 0);
+
+        const chunks: Buffer[] = [];
+        res.on('data', (chunk: Buffer) => {
+            bar?.increment(chunk.length);
+            chunks.push(chunk);
+        });
+        res.on('end', () => {
+            bar?.stop();
+            resolve(Buffer.concat(chunks));
+        });
+        res.on('error', (err) => {
+            bar?.stop();
+            reject(err);
+        });
+    })
+        .on('error', reject)
+        .end();
+});
+
+const downloadFile = (
+    prefixes: string[],
+    type: 'data' | 'config',
+    key: string,
+    {
+        partialOffset,
+        partialLength,
+        showProgress,
+        showAttemptFail,
+    }: {
+        partialOffset?: number,
+        partialLength?: number,
+        showProgress?: boolean,
+        showAttemptFail?: boolean,
+    } = {},
+): Promise<Buffer> => {
+    const urls = prefixes.map((prefix) => `${prefix}/${type}/${formatCDNKey(key)}`);
+
+    return urls
+        .reduce(
+            (prev, url, index) => prev
+                .catch((err: unknown) => {
+                    if (showAttemptFail === true && index > 0 && err instanceof Error) {
+                        console.warn(`${new Date().toISOString()} [WARN]:`, err.message);
+                    }
+                    return requestData(url, { partialOffset, partialLength, showProgress });
+                }),
+            Promise.reject<Buffer>(new Error('')),
+        );
+};
+
+const getFileCache = async (file: string): Promise<Buffer | undefined> => {
+    const integrity = await cacheIntegrity.get(file);
+    if (integrity !== undefined) {
+        try {
+            const buffer = await fs.readFile(path.resolve(CACHE_ROOT, file));
+            const hash = crypto.createHash('sha256').update(buffer).digest('hex');
+            if (hash === integrity) {
+                return buffer;
+            }
+        } catch {
+            // ignore
+        }
+    }
+    return undefined;
+};
+
+export const getDataFile = async (
+    prefixes: string[],
+    key: string,
+    type: keyof typeof CACHE_DIRS,
+    buildCKey: string,
+    {
+        name,
+        partialOffset,
+        partialLength,
+        showProgress,
+        showAttemptFail,
+    }: {
+        name?: string,
+        partialOffset?: number,
+        partialLength?: number,
+        showProgress?: boolean,
+        showAttemptFail?: boolean,
+    } = {},
+): Promise<Buffer> => {
+    const dir = type === 'build'
+        ? path.join(CACHE_DIRS[type], buildCKey)
+        : CACHE_DIRS[type];
+    const file = name !== undefined ? path.join(dir, name) : path.join(dir, key);
+    const cacheBuffer = await getFileCache(file);
+
+    if (cacheBuffer) {
+        if (name === undefined && partialOffset !== undefined && partialLength !== undefined) {
+            return cacheBuffer.subarray(partialOffset, partialOffset + partialLength);
+        }
+        return cacheBuffer;
+    }
+
+    const downloadBuffer = await downloadFile(prefixes, 'data', key, {
+        partialOffset, partialLength, showProgress, showAttemptFail,
+    });
+    if ((partialOffset === undefined && partialLength === undefined) || name !== undefined) {
+        await fs.mkdir(path.resolve(CACHE_ROOT, dir), { recursive: true });
+        await fs.writeFile(path.resolve(CACHE_ROOT, file), downloadBuffer);
+
+        const hash = crypto.createHash('sha256').update(downloadBuffer).digest('hex');
+        await cacheIntegrity.set(file, hash);
+    }
+
+    return downloadBuffer;
+};
+
+export const getConfigFile = async (
+    prefixes: string[],
+    key: string,
+    {
+        showProgress, showAttemptFail,
+    }: {
+        showProgress?: boolean, showAttemptFail?: boolean,
+    } = {},
+): Promise<string> => {
+    const downloadBuffer = await downloadFile(prefixes, 'config', key, { showProgress, showAttemptFail });
+    return downloadBuffer.toString('utf-8');
+};
+
+export const getProductVersions = async (
+    region: string,
+    product: string,
+): Promise<string> => {
+    const url = `http://${region}.patch.battle.net:1119/${product}/versions`;
+    const headers = new Headers();
+    headers.set('User-Agent', USER_AGENT);
+
+    const res = await fetch(url, { headers });
+
+    return res.text();
+};
+
+export const getProductCDNs = async (
+    region: string,
+    product: string,
+): Promise<string> => {
+    const url = `http://${region}.patch.battle.net:1119/${product}/cdns`;
+    const headers = new Headers();
+    headers.set('User-Agent', USER_AGENT);
+
+    const res = await fetch(url, { headers });
+
+    return res.text();
+};

package/src/index.ts

@@ -0,0 +1,44 @@
+/* eslint-disable import-x/no-unused-modules */
+
+export { default as ADBReader } from './adb.ts';
+export { default as CASCClient } from './client.ts';
+export { default as DBDParser } from './dbd.ts';
+export { default as WDCReader } from './wdc.ts';
+
+export type { HotfixEntry } from './adb.ts';
+export type {
+    Version,
+    ClientPreloadData,
+    ArchiveIndex,
+    EncodingData,
+    RootData,
+    FileInfo,
+    FileFetchResultFull,
+    FileFetchResultPartial,
+    FileFetchResult,
+    MissingKeyBlock,
+} from './client.ts';
+export type {
+    Column,
+    ColumnData,
+    BasicColumnData,
+} from './dbd.ts';
+export type {
+    FieldStructure,
+    FieldStorageInfo,
+    FieldStorageInfoCompressionNone,
+    FieldStorageInfoCompressionBitpacked,
+    FieldStorageInfoCompressionCommonData,
+    FieldStorageInfoCompressionBitpackedIndexed,
+    FieldStorageInfoCompressionBitpackedIndexedArray,
+    FieldStorageInfoCompressionBitpackedSigned,
+    ParsedField,
+    ParsedFieldNone,
+    ParsedFieldCommonData,
+    ParsedFieldBitpacked,
+    ParsedFieldBitpackedArray,
+    SparseRow,
+    Hotfix,
+    HotfixModify,
+    HotfixDelete,
+} from './wdc.ts';