@loaders.gl/i3s 4.0.0-alpha.5 → 4.0.0-alpha.7

package/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -0,0 +1,108 @@
+import {processOnWorker} from '@loaders.gl/worker-utils';
+import md5 from 'md5';
+import {CompressionWorker} from '@loaders.gl/compression';
+import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+
+/** Element of hash array */
+type HashElement = {
+  /**
+   * File name hash
+   */
+  hash: Buffer;
+  /**
+   * File offset in the archive
+   */
+  offset: number;
+};
+
+/**
+ * Class for handling information about slpk file
+ */
+export class SLPKArchive {
+  slpkArchive: DataView;
+  hashArray: {hash: Buffer; offset: number}[];
+  constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) {
+    this.slpkArchive = new DataView(slpkArchiveBuffer);
+    this.hashArray = this.parseHashFile(hashFile);
+  }
+
+  /**
+   * Reads hash file from buffer and returns it in ready-to-use form
+   * @param hashFile - bufer containing hash file
+   * @returns Array containing file info
+   */
+  private parseHashFile(hashFile: ArrayBuffer): HashElement[] {
+    const hashFileBuffer = Buffer.from(hashFile);
+    const hashArray: HashElement[] = [];
+    for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) {
+      const offsetBuffer = new DataView(
+        hashFileBuffer.buffer.slice(
+          hashFileBuffer.byteOffset + i + 16,
+          hashFileBuffer.byteOffset + i + 24
+        )
+      );
+      const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true);
+      hashArray.push({
+        hash: Buffer.from(
+          hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
+        ),
+        offset
+      });
+    }
+    return hashArray;
+  }
+
+  /**
+   * Returns file with the given path from slpk archive
+   * @param path - path inside the slpk
+   * @param mode - currently only raw mode supported
+   * @returns buffer with ready to use file
+   */
+  async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise<Buffer> {
+    if (mode === 'http') {
+      throw new Error('http mode is not supported');
+    }
+
+    const fileToDecompress = this.getFileBytes(`${path}.gz`);
+
+    if (fileToDecompress) {
+      const decompressedData = await processOnWorker(CompressionWorker, fileToDecompress, {
+        compression: 'gzip',
+        operation: 'decompress',
+        _workerType: 'test',
+        gzip: {}
+      });
+      return decompressedData;
+    }
+    const fileWithoutCompression = this.getFileBytes(path);
+    if (fileWithoutCompression) {
+      return Promise.resolve(Buffer.from(fileWithoutCompression));
+    }
+    throw new Error('No such file in the archieve');
+  }
+
+  /**
+   * Trying to get raw file data by adress
+   * @param path - path inside the archive
+   * @returns buffer with the raw file data
+   */
+  private getFileBytes(path: string): ArrayBuffer | undefined {
+    const nameHash = Buffer.from(md5(path), 'hex');
+    const fileInfo = this.hashArray.find((val) => Buffer.compare(val.hash, nameHash) === 0);
+    if (!fileInfo) {
+      return undefined;
+    }
+
+    const localFileHeader = parseZipLocalFileHeader(
+      this.slpkArchive.byteOffset + fileInfo?.offset,
+      this.slpkArchive
+    );
+
+    const compressedFile = this.slpkArchive.buffer.slice(
+      localFileHeader.fileDataOffset,
+      localFileHeader.fileDataOffset + localFileHeader.compressedSize
+    );
+
+    return compressedFile;
+  }
+}

package/src/lib/parsers/parse-zip/cd-file-header.ts

@@ -0,0 +1,93 @@
+/**
+ * zip central directory file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipCDFileHeader = {
+  /**
+   * Compressed size
+   */
+  compressedSize: number;
+  /**
+   * Uncompressed size
+   */
+  uncompressedSize: number;
+  /**
+   * File name length
+   */
+  fileNameLength: number;
+  /**
+   * File name
+   */
+  fileName: ArrayBuffer;
+  /**
+   * Extra field offset
+   */
+  extraOffset: number;
+  /**
+   * Relative offset of local file header
+   */
+  localHeaderOffset: number;
+};
+
+/**
+ * Parses central directory file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export const parseZipCDFileHeader = (headerOffset: number, buffer: DataView): ZipCDFileHeader => {
+  const offsets = {
+    CD_COMPRESSED_SIZE_OFFSET: 20,
+    CD_UNCOMPRESSED_SIZE_OFFSET: 24,
+    CD_FILE_NAME_LENGTH_OFFSET: 28,
+    CD_EXTRA_FIELD_LENGTH_OFFSET: 30,
+    CD_LOCAL_HEADER_OFFSET_OFFSET: 42,
+    CD_FILE_NAME_OFFSET: 46
+  };
+
+  const compressedSize = buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET, true);
+
+  const uncompressedSize = buffer.getUint32(
+    headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET,
+    true
+  );
+
+  const fileNameLength = buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET, true);
+
+  const fileName = buffer.buffer.slice(
+    headerOffset + offsets.CD_FILE_NAME_OFFSET,
+    headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
+  );
+
+  const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength;
+
+  const oldFormatOffset = buffer.getUint32(
+    headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET,
+    true
+  );
+
+  let fileDataOffset = oldFormatOffset;
+  if (fileDataOffset === 0xffffffff) {
+    let offsetInZip64Data = 4;
+    // looking for info that might be also be in zip64 extra field
+    if (compressedSize === 0xffffffff) {
+      offsetInZip64Data += 8;
+    }
+    if (uncompressedSize === 0xffffffff) {
+      offsetInZip64Data += 8;
+    }
+
+    // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb
+    fileDataOffset = buffer.getUint32(extraOffset + offsetInZip64Data, true); // setting it to the one from zip64
+  }
+  const localHeaderOffset = fileDataOffset;
+
+  return {
+    compressedSize,
+    uncompressedSize,
+    fileNameLength,
+    fileName,
+    extraOffset,
+    localHeaderOffset
+  };
+};

package/src/lib/parsers/parse-zip/local-file-header.ts

@@ -0,0 +1,56 @@
+/**
+ * zip local file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipLocalFileHeader = {
+  /**
+   * File name length
+   */
+  fileNameLength: number;
+  /**
+   * Extra field length
+   */
+  extraFieldLength: number;
+  /**
+   * Offset of the file data
+   */
+  fileDataOffset: number;
+  /**
+   * Compressed size
+   */
+  compressedSize: number;
+};
+
+/**
+ * Parses local file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export const parseZipLocalFileHeader = (
+  headerOffset: number,
+  buffer: DataView
+): ZipLocalFileHeader => {
+  const offsets = {
+    COMPRESSED_SIZE_OFFSET: 18,
+    FILE_NAME_LENGTH_OFFSET: 26,
+    EXTRA_FIELD_LENGTH_OFFSET: 28,
+    FILE_NAME_OFFSET: 30
+  };
+
+  const fileNameLength = buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET, true);
+
+  const extraFieldLength = buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET, true);
+
+  const fileDataOffset =
+    headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + extraFieldLength;
+
+  const compressedSize = buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET, true);
+
+  return {
+    fileNameLength,
+    extraFieldLength,
+    fileDataOffset,
+    compressedSize
+  };
+};

package/src/lib/utils/customizeColors.ts

@@ -0,0 +1,134 @@
+import type {MeshAttribute} from '@loaders.gl/schema';
+import type {COLOR, I3STileOptions, I3STilesetOptions} from '../../types';
+
+import {load} from '@loaders.gl/core';
+import {getAttributeValueType, I3SAttributeLoader} from '../../i3s-attribute-loader';
+import {I3SLoaderOptions} from '../../i3s-loader';
+import {getUrlWithToken} from '../utils/url-utils';
+
+/**
+ * Modify vertex colors array to visualize 3D objects in a attribute driven way
+ * @param colors - vertex colors attribute
+ * @param featureIds - feature Ids attribute
+ * @param tileOptions - tile - related options
+ * @param tilesetOptions - tileset-related options
+ * @param options - loader options
+ * @returns midified colors attribute
+ */
+export async function customizeColors(
+  colors: MeshAttribute,
+  featureIds: MeshAttribute,
+  tileOptions: I3STileOptions,
+  tilesetOptions: I3STilesetOptions,
+  options?: I3SLoaderOptions
+): Promise<MeshAttribute> {
+  if (!options?.i3s?.colorsByAttribute) {
+    return colors;
+  }
+
+  const colorizeAttributeField = tilesetOptions.fields.find(
+    ({name}) => name === options?.i3s?.colorsByAttribute?.attributeName
+  );
+  if (
+    !colorizeAttributeField ||
+    !['esriFieldTypeDouble', 'esriFieldTypeInteger', 'esriFieldTypeSmallInteger'].includes(
+      colorizeAttributeField.type
+    )
+  ) {
+    return colors;
+  }
+
+  const colorizeAttributeData = await loadFeatureAttributeData(
+    colorizeAttributeField.name,
+    tileOptions,
+    tilesetOptions,
+    options
+  );
+  if (!colorizeAttributeData) {
+    return colors;
+  }
+
+  const objectIdField = tilesetOptions.fields.find(({type}) => type === 'esriFieldTypeOID');
+  if (!objectIdField) {
+    return colors;
+  }
+
+  const objectIdAttributeData = await loadFeatureAttributeData(
+    objectIdField.name,
+    tileOptions,
+    tilesetOptions,
+    options
+  );
+  if (!objectIdAttributeData) {
+    return colors;
+  }
+
+  const attributeValuesMap: {[key: number]: COLOR} = {};
+  // @ts-expect-error
+  for (let i = 0; i < objectIdAttributeData[objectIdField.name].length; i++) {
+    // @ts-expect-error
+    attributeValuesMap[objectIdAttributeData[objectIdField.name][i]] = calculateColorForAttribute(
+      // @ts-expect-error
+      colorizeAttributeData[colorizeAttributeField.name][i] as number,
+      options
+    );
+  }
+
+  for (let i = 0; i < featureIds.value.length; i++) {
+    const color = attributeValuesMap[featureIds.value[i]];
+    if (!color) {
+      continue; // eslint-disable-line no-continue
+    }
+    colors.value.set(color, i * 4);
+  }
+
+  return colors;
+}
+
+/**
+ * Calculate rgba color from the attribute value
+ * @param attributeValue - value of the attribute
+ * @param options - loader options
+ * @returns - color array for a specific attribute value
+ */
+function calculateColorForAttribute(attributeValue: number, options?: I3SLoaderOptions): COLOR {
+  if (!options?.i3s?.colorsByAttribute) {
+    return [255, 255, 255, 255];
+  }
+  const {minValue, maxValue, minColor, maxColor} = options.i3s.colorsByAttribute;
+  const rate = (attributeValue - minValue) / (maxValue - minValue);
+  const color: COLOR = [255, 255, 255, 255];
+  for (let i = 0; i < minColor.length; i++) {
+    color[i] = Math.round((maxColor[i] - minColor[i]) * rate + minColor[i]);
+  }
+  return color;
+}
+
+/**
+ * Load feature attribute data from the ArcGIS rest service
+ * @param attributeName - attribute name
+ * @param tileOptions - tile-related options
+ * @param tilesetOptions - tileset-related options
+ * @param options - loader options
+ * @returns - Array-like list of the attribute values
+ */
+async function loadFeatureAttributeData(
+  attributeName: string,
+  {attributeUrls}: I3STileOptions,
+  {attributeStorageInfo}: I3STilesetOptions,
+  options?: I3SLoaderOptions
+): Promise<{[key: string]: string[] | Uint32Array | Uint16Array | Float64Array | null} | null> {
+  const attributeIndex = attributeStorageInfo.findIndex(({name}) => attributeName === name);
+  if (attributeIndex === -1) {
+    return null;
+  }
+  const objectIdAttributeUrl = getUrlWithToken(attributeUrls[attributeIndex], options?.i3s?.token);
+  const attributeType = getAttributeValueType(attributeStorageInfo[attributeIndex]);
+  const objectIdAttributeData = await load(objectIdAttributeUrl, I3SAttributeLoader, {
+    attributeName,
+    attributeType
+  });
+
+  // @ts-expect-error TODO action engine
+  return objectIdAttributeData;
+}