@loaders.gl/i3s 4.0.0-alpha.15 → 4.0.0-alpha.16

package/src/lib/parsers/parse-zip/cd-file-header.ts

@@ -1,4 +1,5 @@
 import {FileProvider} from './file-provider';
+import {ZipSignature} from './search-from-the-end';
 
 /**
  * zip central directory file header info
@@ -9,6 +10,8 @@ export type ZipCDFileHeader = {
   compressedSize: bigint;
   /** Uncompressed size */
   uncompressedSize: bigint;
+  /** Extra field size */
+  extraFieldLength: number;
   /** File name length */
   fileNameLength: number;
   /** File name */
@@ -19,6 +22,17 @@ export type ZipCDFileHeader = {
   localHeaderOffset: bigint;
 };
 
+const offsets = {
+  CD_COMPRESSED_SIZE_OFFSET: 20n,
+  CD_UNCOMPRESSED_SIZE_OFFSET: 24n,
+  CD_FILE_NAME_LENGTH_OFFSET: 28n,
+  CD_EXTRA_FIELD_LENGTH_OFFSET: 30n,
+  CD_LOCAL_HEADER_OFFSET_OFFSET: 42n,
+  CD_FILE_NAME_OFFSET: 46n
+};
+
+export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];
+
 /**
  * Parses central directory file header of zip file
  * @param headerOffset - offset in the archive where header starts
@@ -28,15 +42,14 @@ export type ZipCDFileHeader = {
 export const parseZipCDFileHeader = async (
   headerOffset: bigint,
   buffer: FileProvider
-): Promise<ZipCDFileHeader> => {
-  const offsets = {
-    CD_COMPRESSED_SIZE_OFFSET: 20n,
-    CD_UNCOMPRESSED_SIZE_OFFSET: 24n,
-    CD_FILE_NAME_LENGTH_OFFSET: 28n,
-    CD_EXTRA_FIELD_LENGTH_OFFSET: 30n,
-    CD_LOCAL_HEADER_OFFSET_OFFSET: 42n,
-    CD_FILE_NAME_OFFSET: 46n
-  };
+): Promise<ZipCDFileHeader | null> => {
+  if (
+    Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(
+      Buffer.from(signature)
+    ) !== 0
+  ) {
+    return null;
+  }
 
   let compressedSize = BigInt(
     await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET)
@@ -46,6 +59,10 @@ export const parseZipCDFileHeader = async (
     await buffer.getUint32(headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET)
   );
 
+  const extraFieldLength = await buffer.getUint16(
+    headerOffset + offsets.CD_EXTRA_FIELD_LENGTH_OFFSET
+  );
+
   const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET);
 
   const fileName = new TextDecoder().decode(
@@ -80,6 +97,7 @@ export const parseZipCDFileHeader = async (
   return {
     compressedSize,
     uncompressedSize,
+    extraFieldLength,
     fileNameLength,
     fileName,
     extraOffset,

package/src/lib/parsers/parse-zip/end-of-central-directory.ts

@@ -0,0 +1,78 @@
+import {FileProvider} from './file-provider';
+import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
+
+/**
+ * End of central directory info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipEoCDRecord = {
+  /** Relative offset of local file header */
+  cdStartOffset: bigint;
+  /** Relative offset of local file header */
+  cdRecordsNumber: bigint;
+};
+
+const eoCDSignature: ZipSignature = [0x50, 0x4b, 0x05, 0x06];
+const zip64EoCDLocatorSignature = Buffer.from([0x50, 0x4b, 0x06, 0x07]);
+const zip64EoCDSignature = Buffer.from([0x50, 0x4b, 0x06, 0x06]);
+
+const offsets = {
+  CD_RECORDS_NUMBER_OFFSET: 8n,
+  CD_START_OFFSET_OFFSET: 16n,
+
+  ZIP64_EOCD_START_OFFSET_OFFSET: 8n,
+
+  ZIP64_CD_RECORDS_NUMBER_OFFSET: 24n,
+  ZIP64_CD_START_OFFSET_OFFSET: 48n
+};
+
+/**
+ * Parses end of central directory record of zip file
+ * @param fileProvider - FileProvider instance
+ * @returns Info from the header
+ */
+export const parseEoCDRecord = async (fileProvider: FileProvider): Promise<ZipEoCDRecord> => {
+  const zipEoCDOffset = await searchFromTheEnd(fileProvider, eoCDSignature);
+
+  let cdRecordsNumber = BigInt(
+    await fileProvider.getUint16(zipEoCDOffset + offsets.CD_RECORDS_NUMBER_OFFSET)
+  );
+  let cdStartOffset = BigInt(
+    await fileProvider.getUint32(zipEoCDOffset + offsets.CD_START_OFFSET_OFFSET)
+  );
+
+  if (cdStartOffset === BigInt(0xffffffff) || cdRecordsNumber === BigInt(0xffffffff)) {
+    const zip64EoCDLocatorOffset = zipEoCDOffset - 20n;
+
+    if (
+      Buffer.from(
+        await fileProvider.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n)
+      ).compare(zip64EoCDLocatorSignature) !== 0
+    ) {
+      throw new Error('zip64 EoCD locator not found');
+    }
+    const zip64EoCDOffset = await fileProvider.getBigUint64(
+      zip64EoCDLocatorOffset + offsets.ZIP64_EOCD_START_OFFSET_OFFSET
+    );
+
+    if (
+      Buffer.from(await fileProvider.slice(zip64EoCDOffset, zip64EoCDOffset + 4n)).compare(
+        zip64EoCDSignature
+      ) !== 0
+    ) {
+      throw new Error('zip64 EoCD not found');
+    }
+
+    cdRecordsNumber = await fileProvider.getBigUint64(
+      zip64EoCDOffset + offsets.ZIP64_CD_RECORDS_NUMBER_OFFSET
+    );
+    cdStartOffset = await fileProvider.getBigUint64(
+      zip64EoCDOffset + offsets.ZIP64_CD_START_OFFSET_OFFSET
+    );
+  }
+
+  return {
+    cdRecordsNumber,
+    cdStartOffset
+  };
+};

package/src/lib/parsers/parse-zip/local-file-header.ts

@@ -36,9 +36,9 @@ export const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
 export const parseZipLocalFileHeader = async (
   headerOffset: bigint,
   buffer: FileProvider
-): Promise<ZipLocalFileHeader | undefined> => {
+): Promise<ZipLocalFileHeader | null> => {
   if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(signature) !== 0) {
-    return Promise.resolve(undefined);
+    return null;
   }
 
   const fileNameLength = await buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET);

package/src/types.ts

@@ -2,18 +2,6 @@ import type {Matrix4, Quaternion, Vector3} from '@math.gl/core';
 import type {TypedArray, MeshAttribute, TextureLevel} from '@loaders.gl/schema';
 import {Tile3D, Tileset3D} from '@loaders.gl/tiles';
 
-export enum DATA_TYPE {
-  UInt8 = 'UInt8',
-  UInt16 = 'UInt16',
-  UInt32 = 'UInt32',
-  UInt64 = 'UInt64',
-  Int16 = 'Int16',
-  Int32 = 'Int32',
-  Int64 = 'Int64',
-  Float32 = 'Float32',
-  Float64 = 'Float64'
-}
-
 export type COLOR = [number, number, number, number];
 
 /**
@@ -690,20 +678,24 @@ type Domain = {
  * spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.8/store.cmn.md
  */
 type Store = {
-  id: string | number;
+  id?: string | number;
   profile: string;
   version: number | string;
-  resourcePattern: string[];
-  rootNode: string;
-  extent: number[];
-  indexCRS: string;
-  vertexCRS: string;
-  normalReferenceFrame: string;
-  attributeEncoding: string;
-  textureEncoding: string[];
-  lodType: string;
-  lodModel: string;
+  resourcePattern?: string[];
+  rootNode?: string;
+  extent?: number[];
+  indexCRS?: string;
+  vertexCRS?: string;
+  normalReferenceFrame?: string;
+  lodType?: string;
+  lodModel?: string;
   defaultGeometrySchema: DefaultGeometrySchema;
+  nidEncoding?: string;
+  textureEncoding?: string[];
+  featureEncoding?: string;
+  geometryEncoding?: string;
+  attributeEncoding?: string;
+  indexingScheme?: string;
 };
 /**
  * Spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.8/defaultGeometrySchema.cmn.md
@@ -726,15 +718,15 @@ type DefaultGeometrySchema = {
 export type HeaderAttribute = {
   property: HeaderAttributeProperty.vertexCount | HeaderAttributeProperty.featureCount | string;
   type:
-    | DATA_TYPE.UInt8
-    | DATA_TYPE.UInt16
-    | DATA_TYPE.UInt32
-    | DATA_TYPE.UInt64
-    | DATA_TYPE.Int16
-    | DATA_TYPE.Int32
-    | DATA_TYPE.Int64
-    | DATA_TYPE.Float32
-    | DATA_TYPE.Float64;
+    | 'UInt8'
+    | 'UInt16'
+    | 'UInt32'
+    | 'UInt64'
+    | 'Int16'
+    | 'Int32'
+    | 'Int64'
+    | 'Float32'
+    | 'Float64';
 };
 export enum HeaderAttributeProperty {
   vertexCount = 'vertexCount',
@@ -749,14 +741,7 @@ export type VertexAttribute = {
 };
 export type GeometryAttribute = {
   byteOffset?: number;
-  valueType:
-    | DATA_TYPE.UInt8
-    | DATA_TYPE.UInt16
-    | DATA_TYPE.Int16
-    | DATA_TYPE.Int32
-    | DATA_TYPE.Int64
-    | DATA_TYPE.Float32
-    | DATA_TYPE.Float64;
+  valueType: 'UInt8' | 'UInt16' | 'Int16' | 'Int32' | 'Int64' | 'Float32' | 'Float64';
   valuesPerElement: number;
 };
 export type I3SMeshAttributes = {

package/dist/es5/lib/parsers/parse-slpk/search-from-the-end.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"search-from-the-end.js","names":["searchFromTheEnd","_ref","_asyncToGenerator2","default","_regenerator","mark","_callee","file","target","searchWindow","targetOffset","i","wrap","_callee$","_context","prev","next","getUint8","length","t0","sent","t1","t2","t3","undefined","every","val","index","abrupt","stop","_x","_x2","apply","arguments","exports"],"sources":["../../../../../src/lib/parsers/parse-slpk/search-from-the-end.ts"],"sourcesContent":["import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider';\n\n/** Description of zip signature type */\nexport type ZipSignature = [number, number, number, number];\n\n/**\n * looking for the last occurrence of the provided\n * @param file\n * @param target\n * @returns\n */\nexport const searchFromTheEnd = async (\n  file: FileProvider,\n  target: ZipSignature\n): Promise<bigint> => {\n  const searchWindow = [\n    await file.getUint8(file.length - 1n),\n    await file.getUint8(file.length - 2n),\n    await file.getUint8(file.length - 3n),\n    undefined\n  ];\n\n  let targetOffset = 0n;\n\n  // looking for the last record in the central directory\n  for (let i = file.length - 4n; i > -1; i--) {\n    searchWindow[3] = searchWindow[2];\n    searchWindow[2] = searchWindow[1];\n    searchWindow[1] = searchWindow[0];\n    searchWindow[0] = await file.getUint8(i);\n    if (searchWindow.every((val, index) => val === target[index])) {\n      targetOffset = i;\n      break;\n    }\n  }\n\n  return targetOffset;\n};\n"],"mappings":";;;;;;;;;AAWO,IAAMA,gBAAgB;EAAA,IAAAC,IAAA,OAAAC,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAG,SAAAC,QAC9BC,IAAkB,EAClBC,MAAoB;IAAA,IAAAC,YAAA,EAAAC,YAAA,EAAAC,CAAA;IAAA,OAAAP,YAAA,CAAAD,OAAA,CAAAS,IAAA,UAAAC,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAE,IAAA;QAAA;UAAAF,QAAA,CAAAE,IAAA;UAAA,OAGZT,IAAI,CAACU,QAAQ,CAACV,IAAI,CAACW,MAAM,GAAG,EAAE,CAAC;QAAA;UAAAJ,QAAA,CAAAK,EAAA,GAAAL,QAAA,CAAAM,IAAA;UAAAN,QAAA,CAAAE,IAAA;UAAA,OAC/BT,IAAI,CAACU,QAAQ,CAACV,IAAI,CAACW,MAAM,GAAG,EAAE,CAAC;QAAA;UAAAJ,QAAA,CAAAO,EAAA,GAAAP,QAAA,CAAAM,IAAA;UAAAN,QAAA,CAAAE,IAAA;UAAA,OAC/BT,IAAI,CAACU,QAAQ,CAACV,IAAI,CAACW,MAAM,GAAG,EAAE,CAAC;QAAA;UAAAJ,QAAA,CAAAQ,EAAA,GAAAR,QAAA,CAAAM,IAAA;UAAAN,QAAA,CAAAS,EAAA,GACrCC,SAAS;UAJLf,YAAY,IAAAK,QAAA,CAAAK,EAAA,EAAAL,QAAA,CAAAO,EAAA,EAAAP,QAAA,CAAAQ,EAAA,EAAAR,QAAA,CAAAS,EAAA;UAOdb,YAAY,GAAG,EAAE;UAGZC,CAAC,GAAGJ,IAAI,CAACW,MAAM,GAAG,EAAE;QAAA;UAAA,MAAEP,CAAC,GAAG,CAAC,CAAC;YAAAG,QAAA,CAAAE,IAAA;YAAA;UAAA;UACnCP,YAAY,CAAC,CAAC,CAAC,GAAGA,YAAY,CAAC,CAAC,CAAC;UACjCA,YAAY,CAAC,CAAC,CAAC,GAAGA,YAAY,CAAC,CAAC,CAAC;UACjCA,YAAY,CAAC,CAAC,CAAC,GAAGA,YAAY,CAAC,CAAC,CAAC;UAACK,QAAA,CAAAE,IAAA;UAAA,OACVT,IAAI,CAACU,QAAQ,CAACN,CAAC,CAAC;QAAA;UAAxCF,YAAY,CAAC,CAAC,CAAC,GAAAK,QAAA,CAAAM,IAAA;UAAA,KACXX,YAAY,CAACgB,KAAK,CAAC,UAACC,GAAG,EAAEC,KAAK;YAAA,OAAKD,GAAG,KAAKlB,MAAM,CAACmB,KAAK,CAAC;UAAA,EAAC;YAAAb,QAAA,CAAAE,IAAA;YAAA;UAAA;UAC3DN,YAAY,GAAGC,CAAC;UAAC,OAAAG,QAAA,CAAAc,MAAA;QAAA;UANkBjB,CAAC,EAAE;UAAAG,QAAA,CAAAE,IAAA;UAAA;QAAA;UAAA,OAAAF,QAAA,CAAAc,MAAA,WAWnClB,YAAY;QAAA;QAAA;UAAA,OAAAI,QAAA,CAAAe,IAAA;MAAA;IAAA,GAAAvB,OAAA;EAAA,CACpB;EAAA,gBA1BYN,gBAAgBA,CAAA8B,EAAA,EAAAC,GAAA;IAAA,OAAA9B,IAAA,CAAA+B,KAAA,OAAAC,SAAA;EAAA;AAAA,GA0B5B;AAACC,OAAA,CAAAlC,gBAAA,GAAAA,gBAAA"}

package/dist/esm/lib/parsers/parse-slpk/search-from-the-end.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"search-from-the-end.js","names":["searchFromTheEnd","file","target","searchWindow","getUint8","length","undefined","targetOffset","i","every","val","index"],"sources":["../../../../../src/lib/parsers/parse-slpk/search-from-the-end.ts"],"sourcesContent":["import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider';\n\n/** Description of zip signature type */\nexport type ZipSignature = [number, number, number, number];\n\n/**\n * looking for the last occurrence of the provided\n * @param file\n * @param target\n * @returns\n */\nexport const searchFromTheEnd = async (\n  file: FileProvider,\n  target: ZipSignature\n): Promise<bigint> => {\n  const searchWindow = [\n    await file.getUint8(file.length - 1n),\n    await file.getUint8(file.length - 2n),\n    await file.getUint8(file.length - 3n),\n    undefined\n  ];\n\n  let targetOffset = 0n;\n\n  // looking for the last record in the central directory\n  for (let i = file.length - 4n; i > -1; i--) {\n    searchWindow[3] = searchWindow[2];\n    searchWindow[2] = searchWindow[1];\n    searchWindow[1] = searchWindow[0];\n    searchWindow[0] = await file.getUint8(i);\n    if (searchWindow.every((val, index) => val === target[index])) {\n      targetOffset = i;\n      break;\n    }\n  }\n\n  return targetOffset;\n};\n"],"mappings":"AAWA,OAAO,MAAMA,gBAAgB,GAAG,MAAAA,CAC9BC,IAAkB,EAClBC,MAAoB,KACA;EACpB,MAAMC,YAAY,GAAG,CACnB,MAAMF,IAAI,CAACG,QAAQ,CAACH,IAAI,CAACI,MAAM,GAAG,EAAE,CAAC,EACrC,MAAMJ,IAAI,CAACG,QAAQ,CAACH,IAAI,CAACI,MAAM,GAAG,EAAE,CAAC,EACrC,MAAMJ,IAAI,CAACG,QAAQ,CAACH,IAAI,CAACI,MAAM,GAAG,EAAE,CAAC,EACrCC,SAAS,CACV;EAED,IAAIC,YAAY,GAAG,EAAE;EAGrB,KAAK,IAAIC,CAAC,GAAGP,IAAI,CAACI,MAAM,GAAG,EAAE,EAAEG,CAAC,GAAG,CAAC,CAAC,EAAEA,CAAC,EAAE,EAAE;IAC1CL,YAAY,CAAC,CAAC,CAAC,GAAGA,YAAY,CAAC,CAAC,CAAC;IACjCA,YAAY,CAAC,CAAC,CAAC,GAAGA,YAAY,CAAC,CAAC,CAAC;IACjCA,YAAY,CAAC,CAAC,CAAC,GAAGA,YAAY,CAAC,CAAC,CAAC;IACjCA,YAAY,CAAC,CAAC,CAAC,GAAG,MAAMF,IAAI,CAACG,QAAQ,CAACI,CAAC,CAAC;IACxC,IAAIL,YAAY,CAACM,KAAK,CAAC,CAACC,GAAG,EAAEC,KAAK,KAAKD,GAAG,KAAKR,MAAM,CAACS,KAAK,CAAC,CAAC,EAAE;MAC7DJ,YAAY,GAAGC,CAAC;MAChB;IACF;EACF;EAEA,OAAOD,YAAY;AACrB,CAAC"}

package/dist/lib/parsers/parse-slpk/search-from-the-end.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"search-from-the-end.d.ts","sourceRoot":"","sources":["../../../../src/lib/parsers/parse-slpk/search-from-the-end.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,qDAAqD,CAAC;AAEjF,wCAAwC;AACxC,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAE5D;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,SACrB,YAAY,2BAEjB,QAAQ,MAAM,CAuBhB,CAAC"}