@loaders.gl/zip 4.0.0-alpha.20 → 4.0.0-alpha.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/dist/dist.min.js +5025 -151
  2. package/dist/es5/{parse-zip → file-provider}/data-view-file.js +46 -29
  3. package/dist/es5/file-provider/data-view-file.js.map +1 -0
  4. package/dist/es5/file-provider/file-handle-file.js +234 -0
  5. package/dist/es5/file-provider/file-handle-file.js.map +1 -0
  6. package/dist/es5/file-provider/file-handle.js +101 -0
  7. package/dist/es5/file-provider/file-handle.js.map +1 -0
  8. package/dist/es5/file-provider/file-provider.js +11 -0
  9. package/dist/es5/file-provider/file-provider.js.map +1 -0
  10. package/dist/es5/filesystems/zip-filesystem.js +374 -0
  11. package/dist/es5/filesystems/zip-filesystem.js.map +1 -0
  12. package/dist/es5/hash-file-utility.js +81 -1
  13. package/dist/es5/hash-file-utility.js.map +1 -1
  14. package/dist/es5/index.js +27 -1
  15. package/dist/es5/index.js.map +1 -1
  16. package/dist/es5/parse-zip/cd-file-header.js +58 -16
  17. package/dist/es5/parse-zip/cd-file-header.js.map +1 -1
  18. package/dist/es5/parse-zip/end-of-central-directory.js +10 -12
  19. package/dist/es5/parse-zip/end-of-central-directory.js.map +1 -1
  20. package/dist/es5/parse-zip/local-file-header.js +37 -33
  21. package/dist/es5/parse-zip/local-file-header.js.map +1 -1
  22. package/dist/es5/parse-zip/search-from-the-end.js.map +1 -1
  23. package/dist/es5/zip-loader.js +1 -1
  24. package/dist/esm/{parse-zip → file-provider}/data-view-file.js +1 -0
  25. package/dist/esm/file-provider/data-view-file.js.map +1 -0
  26. package/dist/esm/file-provider/file-handle-file.js +57 -0
  27. package/dist/esm/file-provider/file-handle-file.js.map +1 -0
  28. package/dist/esm/file-provider/file-handle.js +37 -0
  29. package/dist/esm/file-provider/file-handle.js.map +1 -0
  30. package/dist/esm/file-provider/file-provider.js +4 -0
  31. package/dist/esm/file-provider/file-provider.js.map +1 -0
  32. package/dist/esm/filesystems/zip-filesystem.js +99 -0
  33. package/dist/esm/filesystems/zip-filesystem.js.map +1 -0
  34. package/dist/esm/hash-file-utility.js +14 -0
  35. package/dist/esm/hash-file-utility.js.map +1 -1
  36. package/dist/esm/index.js +5 -3
  37. package/dist/esm/index.js.map +1 -1
  38. package/dist/esm/parse-zip/cd-file-header.js +24 -15
  39. package/dist/esm/parse-zip/cd-file-header.js.map +1 -1
  40. package/dist/esm/parse-zip/end-of-central-directory.js +10 -12
  41. package/dist/esm/parse-zip/end-of-central-directory.js.map +1 -1
  42. package/dist/esm/parse-zip/local-file-header.js +16 -15
  43. package/dist/esm/parse-zip/local-file-header.js.map +1 -1
  44. package/dist/esm/parse-zip/search-from-the-end.js.map +1 -1
  45. package/dist/esm/zip-loader.js +1 -1
  46. package/dist/{parse-zip → file-provider}/data-view-file.d.ts +1 -0
  47. package/dist/file-provider/data-view-file.d.ts.map +1 -0
  48. package/dist/{parse-zip → file-provider}/data-view-file.js +2 -0
  49. package/dist/file-provider/file-handle-file.d.ts +53 -0
  50. package/dist/file-provider/file-handle-file.d.ts.map +1 -0
  51. package/dist/file-provider/file-handle-file.js +90 -0
  52. package/dist/file-provider/file-handle.d.ts +40 -0
  53. package/dist/file-provider/file-handle.d.ts.map +1 -0
  54. package/dist/file-provider/file-handle.js +57 -0
  55. package/dist/{parse-zip → file-provider}/file-provider.d.ts +9 -0
  56. package/dist/file-provider/file-provider.d.ts.map +1 -0
  57. package/dist/file-provider/file-provider.js +13 -0
  58. package/dist/filesystems/zip-filesystem.d.ts +45 -0
  59. package/dist/filesystems/zip-filesystem.d.ts.map +1 -0
  60. package/dist/filesystems/zip-filesystem.js +128 -0
  61. package/dist/hash-file-utility.d.ts +7 -0
  62. package/dist/hash-file-utility.d.ts.map +1 -1
  63. package/dist/hash-file-utility.js +24 -1
  64. package/dist/index.d.ts +6 -4
  65. package/dist/index.d.ts.map +1 -1
  66. package/dist/index.js +9 -3
  67. package/dist/parse-zip/cd-file-header.d.ts +6 -1
  68. package/dist/parse-zip/cd-file-header.d.ts.map +1 -1
  69. package/dist/parse-zip/cd-file-header.js +29 -16
  70. package/dist/parse-zip/end-of-central-directory.d.ts +1 -1
  71. package/dist/parse-zip/end-of-central-directory.d.ts.map +1 -1
  72. package/dist/parse-zip/end-of-central-directory.js +11 -12
  73. package/dist/parse-zip/local-file-header.d.ts +3 -1
  74. package/dist/parse-zip/local-file-header.d.ts.map +1 -1
  75. package/dist/parse-zip/local-file-header.js +17 -15
  76. package/dist/parse-zip/search-from-the-end.d.ts +1 -1
  77. package/dist/parse-zip/search-from-the-end.d.ts.map +1 -1
  78. package/package.json +9 -2
  79. package/src/{parse-zip → file-provider}/data-view-file.ts +3 -0
  80. package/src/file-provider/file-handle-file.ts +114 -0
  81. package/src/file-provider/file-handle.ts +73 -0
  82. package/src/{parse-zip → file-provider}/file-provider.ts +16 -0
  83. package/src/filesystems/zip-filesystem.ts +142 -0
  84. package/src/hash-file-utility.ts +22 -0
  85. package/src/index.ts +12 -4
  86. package/src/parse-zip/cd-file-header.ts +33 -25
  87. package/src/parse-zip/end-of-central-directory.ts +12 -19
  88. package/src/parse-zip/local-file-header.ts +22 -22
  89. package/src/parse-zip/search-from-the-end.ts +1 -1
  90. package/dist/es5/parse-zip/data-view-file.js.map +0 -1
  91. package/dist/es5/parse-zip/file-provider.js +0 -2
  92. package/dist/es5/parse-zip/file-provider.js.map +0 -1
  93. package/dist/esm/parse-zip/data-view-file.js.map +0 -1
  94. package/dist/esm/parse-zip/file-provider.js +0 -2
  95. package/dist/esm/parse-zip/file-provider.js.map +0 -1
  96. package/dist/parse-zip/data-view-file.d.ts.map +0 -1
  97. package/dist/parse-zip/file-provider.d.ts.map +0 -1
  98. package/dist/parse-zip/file-provider.js +0 -2
@@ -5,13 +5,12 @@ const search_from_the_end_1 = require("./search-from-the-end");
5
5
  const eoCDSignature = [0x50, 0x4b, 0x05, 0x06];
6
6
  const zip64EoCDLocatorSignature = Buffer.from([0x50, 0x4b, 0x06, 0x07]);
7
7
  const zip64EoCDSignature = Buffer.from([0x50, 0x4b, 0x06, 0x06]);
8
- const offsets = {
9
- CD_RECORDS_NUMBER_OFFSET: 8n,
10
- CD_START_OFFSET_OFFSET: 16n,
11
- ZIP64_EOCD_START_OFFSET_OFFSET: 8n,
12
- ZIP64_CD_RECORDS_NUMBER_OFFSET: 24n,
13
- ZIP64_CD_START_OFFSET_OFFSET: 48n
14
- };
8
+ // offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
9
+ const CD_RECORDS_NUMBER_OFFSET = 8n;
10
+ const CD_START_OFFSET_OFFSET = 16n;
11
+ const ZIP64_EOCD_START_OFFSET_OFFSET = 8n;
12
+ const ZIP64_CD_RECORDS_NUMBER_OFFSET = 24n;
13
+ const ZIP64_CD_START_OFFSET_OFFSET = 48n;
15
14
  /**
16
15
  * Parses end of central directory record of zip file
17
16
  * @param fileProvider - FileProvider instance
@@ -19,19 +18,19 @@ const offsets = {
19
18
  */
20
19
  const parseEoCDRecord = async (fileProvider) => {
21
20
  const zipEoCDOffset = await (0, search_from_the_end_1.searchFromTheEnd)(fileProvider, eoCDSignature);
22
- let cdRecordsNumber = BigInt(await fileProvider.getUint16(zipEoCDOffset + offsets.CD_RECORDS_NUMBER_OFFSET));
23
- let cdStartOffset = BigInt(await fileProvider.getUint32(zipEoCDOffset + offsets.CD_START_OFFSET_OFFSET));
21
+ let cdRecordsNumber = BigInt(await fileProvider.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET));
22
+ let cdStartOffset = BigInt(await fileProvider.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET));
24
23
  if (cdStartOffset === BigInt(0xffffffff) || cdRecordsNumber === BigInt(0xffffffff)) {
25
24
  const zip64EoCDLocatorOffset = zipEoCDOffset - 20n;
26
25
  if (Buffer.from(await fileProvider.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n)).compare(zip64EoCDLocatorSignature) !== 0) {
27
26
  throw new Error('zip64 EoCD locator not found');
28
27
  }
29
- const zip64EoCDOffset = await fileProvider.getBigUint64(zip64EoCDLocatorOffset + offsets.ZIP64_EOCD_START_OFFSET_OFFSET);
28
+ const zip64EoCDOffset = await fileProvider.getBigUint64(zip64EoCDLocatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET);
30
29
  if (Buffer.from(await fileProvider.slice(zip64EoCDOffset, zip64EoCDOffset + 4n)).compare(zip64EoCDSignature) !== 0) {
31
30
  throw new Error('zip64 EoCD not found');
32
31
  }
33
- cdRecordsNumber = await fileProvider.getBigUint64(zip64EoCDOffset + offsets.ZIP64_CD_RECORDS_NUMBER_OFFSET);
34
- cdStartOffset = await fileProvider.getBigUint64(zip64EoCDOffset + offsets.ZIP64_CD_START_OFFSET_OFFSET);
32
+ cdRecordsNumber = await fileProvider.getBigUint64(zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET);
33
+ cdStartOffset = await fileProvider.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET);
35
34
  }
36
35
  return {
37
36
  cdRecordsNumber,
@@ -1,5 +1,5 @@
1
1
  /// <reference types="node" />
2
- import { FileProvider } from './file-provider';
2
+ import { FileProvider } from '../file-provider/file-provider';
3
3
  /**
4
4
  * zip local file header info
5
5
  * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
@@ -15,6 +15,8 @@ export type ZipLocalFileHeader = {
15
15
  fileDataOffset: bigint;
16
16
  /** Compressed size */
17
17
  compressedSize: bigint;
18
+ /** Compression method */
19
+ compressionMethod: number;
18
20
  };
19
21
  export declare const signature: Buffer;
20
22
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"local-file-header.d.ts","sourceRoot":"","sources":["../../src/parse-zip/local-file-header.ts"],"names":[],"mappings":";AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,iBAAiB,CAAC;AAE7C;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,uBAAuB;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAUF,eAAO,MAAM,SAAS,QAAwC,CAAC;AAE/D;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,iBACpB,MAAM,UACZ,YAAY,KACnB,QAAQ,kBAAkB,GAAG,IAAI,CAoDnC,CAAC"}
1
+ {"version":3,"file":"local-file-header.d.ts","sourceRoot":"","sources":["../../src/parse-zip/local-file-header.ts"],"names":[],"mappings":";AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,gCAAgC,CAAC;AAE5D;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,uBAAuB;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,yBAAyB;IACzB,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAUF,eAAO,MAAM,SAAS,QAAwC,CAAC;AAE/D;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,iBACpB,MAAM,UACZ,YAAY,KACnB,QAAQ,kBAAkB,GAAG,IAAI,CAkDnC,CAAC"}
@@ -1,13 +1,13 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.parseZipLocalFileHeader = exports.signature = void 0;
4
- const offsets = {
5
- COMPRESSED_SIZE_OFFSET: 18n,
6
- UNCOMPRESSED_SIZE_OFFSET: 22n,
7
- FILE_NAME_LENGTH_OFFSET: 26n,
8
- EXTRA_FIELD_LENGTH_OFFSET: 28n,
9
- FILE_NAME_OFFSET: 30n
10
- };
4
+ // offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
5
+ const COMPRESSION_METHOD_OFFSET = 8n;
6
+ const COMPRESSED_SIZE_OFFSET = 18n;
7
+ const UNCOMPRESSED_SIZE_OFFSET = 22n;
8
+ const FILE_NAME_LENGTH_OFFSET = 26n;
9
+ const EXTRA_FIELD_LENGTH_OFFSET = 28n;
10
+ const FILE_NAME_OFFSET = 30n;
11
11
  exports.signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
12
12
  /**
13
13
  * Parses local file header of zip file
@@ -19,16 +19,17 @@ const parseZipLocalFileHeader = async (headerOffset, buffer) => {
19
19
  if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(exports.signature) !== 0) {
20
20
  return null;
21
21
  }
22
- const fileNameLength = await buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET);
22
+ const fileNameLength = await buffer.getUint16(headerOffset + FILE_NAME_LENGTH_OFFSET);
23
23
  const fileName = new TextDecoder()
24
- .decode(await buffer.slice(headerOffset + offsets.FILE_NAME_OFFSET, headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength)))
24
+ .decode(await buffer.slice(headerOffset + FILE_NAME_OFFSET, headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength)))
25
25
  .split('\\')
26
26
  .join('/');
27
- const extraFieldLength = await buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET);
28
- let fileDataOffset = headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength);
29
- let compressedSize = BigInt(await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET)); // add zip 64 logic
30
- let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + offsets.UNCOMPRESSED_SIZE_OFFSET)); // add zip 64 logic
31
- const extraOffset = headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength);
27
+ const extraFieldLength = await buffer.getUint16(headerOffset + EXTRA_FIELD_LENGTH_OFFSET);
28
+ let fileDataOffset = headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength);
29
+ const compressionMethod = await buffer.getUint16(headerOffset + COMPRESSION_METHOD_OFFSET);
30
+ let compressedSize = BigInt(await buffer.getUint32(headerOffset + COMPRESSED_SIZE_OFFSET)); // add zip 64 logic
31
+ let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + UNCOMPRESSED_SIZE_OFFSET)); // add zip 64 logic
32
+ const extraOffset = headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength);
32
33
  let offsetInZip64Data = 4n;
33
34
  // looking for info that might be also be in zip64 extra field
34
35
  if (uncompressedSize === BigInt(0xffffffff)) {
@@ -47,7 +48,8 @@ const parseZipLocalFileHeader = async (headerOffset, buffer) => {
47
48
  fileName,
48
49
  extraFieldLength,
49
50
  fileDataOffset,
50
- compressedSize
51
+ compressedSize,
52
+ compressionMethod
51
53
  };
52
54
  };
53
55
  exports.parseZipLocalFileHeader = parseZipLocalFileHeader;
@@ -1,4 +1,4 @@
1
- import { FileProvider } from './file-provider';
1
+ import { FileProvider } from '../file-provider/file-provider';
2
2
  /** Description of zip signature type */
3
3
  export type ZipSignature = [number, number, number, number];
4
4
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"search-from-the-end.d.ts","sourceRoot":"","sources":["../../src/parse-zip/search-from-the-end.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,iBAAiB,CAAC;AAE7C,wCAAwC;AACxC,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAE5D;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,SACrB,YAAY,2BAEjB,QAAQ,MAAM,CAuBhB,CAAC"}
1
+ {"version":3,"file":"search-from-the-end.d.ts","sourceRoot":"","sources":["../../src/parse-zip/search-from-the-end.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,gCAAgC,CAAC;AAE5D,wCAAwC;AACxC,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAE5D;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,SACrB,YAAY,2BAEjB,QAAQ,MAAM,CAuBhB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loaders.gl/zip",
3
- "version": "4.0.0-alpha.20",
3
+ "version": "4.0.0-alpha.22",
4
4
  "description": "Zip Archive Loader",
5
5
  "license": "MIT",
6
6
  "publishConfig": {
@@ -25,12 +25,19 @@
25
25
  "dist",
26
26
  "README.md"
27
27
  ],
28
+ "browser": {
29
+ "fs": false
30
+ },
28
31
  "scripts": {
29
32
  "pre-build": "npm run build-bundle",
30
33
  "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js"
31
34
  },
32
35
  "dependencies": {
36
+ "@loaders.gl/compression": "4.0.0-alpha.22",
33
37
  "jszip": "^3.1.5"
34
38
  },
35
- "gitHead": "ac122e83102657c38207d59c631a5ce4e7aa46bd"
39
+ "peerDependencies": {
40
+ "@loaders.gl/core": "^4.0.0-alpha.21"
41
+ },
42
+ "gitHead": "0da838c506d1275383f2fd3d244d9c72b25397d2"
36
43
  }
@@ -21,6 +21,9 @@ export class DataViewFile implements FileProvider {
21
21
  this.file = file;
22
22
  }
23
23
 
24
+ // eslint-disable-next-line @typescript-eslint/no-empty-function
25
+ async destroy(): Promise<void> {}
26
+
24
27
  /**
25
28
  * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
26
29
  * @param offset The offset, in bytes, from the start of the file where to read the data.
@@ -0,0 +1,114 @@
1
+ import {FileProvider} from '@loaders.gl/zip';
2
+ import {FileHandle} from './file-handle';
3
+
4
+ /**
5
+ * Provides file data using node fs library
6
+ */
7
+ export class FileHandleFile implements FileProvider {
8
+ /**
9
+ * Returns a new copy of FileHandleFile
10
+ * @param path The path to the file in file system
11
+ */
12
+ static async from(path: string): Promise<FileHandleFile> {
13
+ const fileDescriptor = await FileHandle.open(path);
14
+ return new FileHandleFile(fileDescriptor, fileDescriptor.stat.size);
15
+ }
16
+
17
+ /**
18
+ * The FileHandle from which data is provided
19
+ */
20
+ private fileDescriptor: FileHandle;
21
+
22
+ /**
23
+ * The file length in bytes
24
+ */
25
+ private size: bigint;
26
+
27
+ private constructor(fileDescriptor: FileHandle, size: bigint) {
28
+ this.fileDescriptor = fileDescriptor;
29
+ this.size = size;
30
+ }
31
+
32
+ /** Close file */
33
+ async destroy(): Promise<void> {
34
+ await this.fileDescriptor.close();
35
+ }
36
+
37
+ /**
38
+ * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
39
+ * @param offset The offset, in bytes, from the start of the file where to read the data.
40
+ */
41
+ async getUint8(offset: bigint): Promise<number> {
42
+ const val = new Uint8Array(
43
+ (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer
44
+ ).at(0);
45
+ if (val === undefined) {
46
+ throw new Error('something went wrong');
47
+ }
48
+ return val;
49
+ }
50
+
51
+ /**
52
+ * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
53
+ * @param offset The offset, in bytes, from the start of the file where to read the data.
54
+ */
55
+ async getUint16(offset: bigint): Promise<number> {
56
+ const val = new Uint16Array(
57
+ (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer
58
+ ).at(0);
59
+ if (val === undefined) {
60
+ throw new Error('something went wrong');
61
+ }
62
+ return val;
63
+ }
64
+
65
+ /**
66
+ * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
67
+ * @param offset The offset, in bytes, from the start of the file where to read the data.
68
+ */
69
+ async getUint32(offset: bigint): Promise<number> {
70
+ const val = new Uint32Array(
71
+ (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer
72
+ ).at(0);
73
+ if (val === undefined) {
74
+ throw new Error('something went wrong');
75
+ }
76
+ return val;
77
+ }
78
+
79
+ /**
80
+ * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
81
+ * @param offset The offset, in bytes, from the start of the file where to read the data.
82
+ */
83
+ async getBigUint64(offset: bigint): Promise<bigint> {
84
+ const val = new BigInt64Array(
85
+ (await this.fileDescriptor.read(Buffer.alloc(8), 0, 8, offset)).buffer.buffer
86
+ ).at(0);
87
+ if (val === undefined) {
88
+ throw new Error('something went wrong');
89
+ }
90
+ return val;
91
+ }
92
+
93
+ /**
94
+ * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
95
+ * @param startOffsset The offset, in byte, from the start of the file where to start reading the data.
96
+ * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
97
+ */
98
+ async slice(startOffsset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
99
+ const bigLength = endOffset - startOffsset;
100
+ if (bigLength > Number.MAX_SAFE_INTEGER) {
101
+ throw new Error('too big slice');
102
+ }
103
+ const length = Number(bigLength);
104
+ return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffsset)).buffer
105
+ .buffer;
106
+ }
107
+
108
+ /**
109
+ * the length (in bytes) of the data.
110
+ */
111
+ get length(): bigint {
112
+ return this.size;
113
+ }
114
+ }
@@ -0,0 +1,73 @@
1
+ import {read, open, close, stat, BigIntStats} from 'fs';
2
+
3
+ /** file reading result */
4
+ export type FileReadResult = {
5
+ /** amount of the bytes read */
6
+ bytesRead: number;
7
+ /** the buffer filled with data from file*/
8
+ buffer: Buffer;
9
+ };
10
+
11
+ /** Object handling file info */
12
+ export class FileHandle {
13
+ private fileDescriptor: number;
14
+ private stats: BigIntStats;
15
+ private constructor(fileDescriptor: number, stats: BigIntStats) {
16
+ this.fileDescriptor = fileDescriptor;
17
+ this.stats = stats;
18
+ }
19
+ /**
20
+ * Opens a `FileHandle`.
21
+ *
22
+ * @param path path to the file
23
+ * @return Fulfills with a {FileHandle} object.
24
+ */
25
+
26
+ static open = async (path: string): Promise<FileHandle> => {
27
+ const [fd, stats] = await Promise.all([
28
+ new Promise<number>((s) => {
29
+ open(path, undefined, undefined, (_err, fd) => s(fd));
30
+ }),
31
+ new Promise<BigIntStats>((s) => {
32
+ stat(path, {bigint: true}, (_err, stats) => s(stats));
33
+ })
34
+ ]);
35
+ return new FileHandle(fd, stats);
36
+ };
37
+
38
+ /** Close file */
39
+ async close(): Promise<void> {
40
+ return new Promise<void>((resolve) => {
41
+ close(this.fileDescriptor, (_err) => resolve());
42
+ });
43
+ }
44
+
45
+ /**
46
+ * Reads data from the file and stores that in the given buffer.
47
+ *
48
+ * If the file is not modified concurrently, the end-of-file is reached when the
49
+ * number of bytes read is zero.
50
+ * @param buffer A buffer that will be filled with the file data read.
51
+ * @param offset The location in the buffer at which to start filling.
52
+ * @param length The number of bytes to read.
53
+ * @param position The location where to begin reading data from the file. If `null`, data will be read from the current file position, and the position will be updated. If `position` is an
54
+ * integer, the current file position will remain unchanged.
55
+ * @return Fulfills upon success with a FileReadResult object
56
+ */
57
+ read = (
58
+ buffer: Buffer,
59
+ offset: number,
60
+ length: number,
61
+ position: number | bigint
62
+ ): Promise<FileReadResult> => {
63
+ return new Promise((s) => {
64
+ read(this.fileDescriptor, buffer, offset, length, position, (_err, bytesRead, buffer) =>
65
+ s({bytesRead, buffer})
66
+ );
67
+ });
68
+ };
69
+
70
+ get stat(): BigIntStats {
71
+ return this.stats;
72
+ }
73
+ }
@@ -2,6 +2,10 @@
2
2
  * Interface for providing file data
3
3
  */
4
4
  export interface FileProvider {
5
+ /**
6
+ * Cleanup class data
7
+ */
8
+ destroy(): Promise<void>;
5
9
  /**
6
10
  * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
7
11
  * @param offset The offset, in bytes, from the start of the file where to read the data.
@@ -38,3 +42,15 @@ export interface FileProvider {
38
42
  */
39
43
  length: bigint;
40
44
  }
45
+
46
+ /**
47
+ * Check is the object has FileProvider members
48
+ * @param fileProvider - tested object
49
+ */
50
+ export const isFileProvider = (fileProvider: unknown) => {
51
+ return (
52
+ (fileProvider as FileProvider)?.getUint8 &&
53
+ (fileProvider as FileProvider)?.slice &&
54
+ (fileProvider as FileProvider)?.length
55
+ );
56
+ };
@@ -0,0 +1,142 @@
1
+ import {FileSystem, isBrowser} from '@loaders.gl/core';
2
+ import {FileProvider, isFileProvider} from '../file-provider/file-provider';
3
+ import {FileHandleFile} from '../file-provider/file-handle-file';
4
+ import {ZipCDFileHeader, zipCDFileHeaderGenerator} from '../parse-zip/cd-file-header';
5
+ import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
6
+ import {DeflateCompression} from '@loaders.gl/compression';
7
+
8
+ type CompressionHandler = (compressedFile: ArrayBuffer) => Promise<ArrayBuffer>;
9
+ /** Handling different compression types in zip */
10
+ const COMPRESSION_METHODS: {[key: number]: CompressionHandler} = {
11
+ /** No compression */
12
+ 0: async (compressedFile) => compressedFile,
13
+ /** Deflation */
14
+ 8: async (compressedFile) => {
15
+ const compression = new DeflateCompression({raw: true});
16
+ const decompressedData = await compression.decompress(compressedFile);
17
+ return decompressedData;
18
+ }
19
+ };
20
+
21
+ /**
22
+ * FileSystem adapter for a ZIP file
23
+ * Holds FileProvider object that provides random access to archived files
24
+ */
25
+ export class ZipFileSystem implements FileSystem {
26
+ /** FileProvider instance promise */
27
+ protected fileProvider: Promise<FileProvider | null> = Promise.resolve(null);
28
+ public fileName?: string;
29
+
30
+ /**
31
+ * Constructor
32
+ * @param file - instance of FileProvider or file path string
33
+ */
34
+ constructor(file: FileProvider | string) {
35
+ // Try to open file in NodeJS
36
+ if (typeof file === 'string') {
37
+ this.fileName = file;
38
+ if (!isBrowser) {
39
+ this.fileProvider = FileHandleFile.from(file);
40
+ } else {
41
+ throw new Error('Cannot open file for random access in a WEB browser');
42
+ }
43
+ } else if (isFileProvider(file)) {
44
+ this.fileProvider = Promise.resolve(file);
45
+ }
46
+ }
47
+
48
+ /** Clean up resources */
49
+ async destroy() {
50
+ const fileProvider = await this.fileProvider;
51
+ if (fileProvider) {
52
+ await fileProvider.destroy();
53
+ }
54
+ }
55
+
56
+ /**
57
+ * Get file names list from zip archive
58
+ * @returns array of file names
59
+ */
60
+ async readdir(): Promise<string[]> {
61
+ const fileProvider = await this.fileProvider;
62
+ if (!fileProvider) {
63
+ throw new Error('No data detected in the zip archive');
64
+ }
65
+ const fileNames: string[] = [];
66
+ const zipCDIterator = zipCDFileHeaderGenerator(fileProvider);
67
+ for await (const cdHeader of zipCDIterator) {
68
+ fileNames.push(cdHeader.fileName);
69
+ }
70
+ return fileNames;
71
+ }
72
+
73
+ /**
74
+ * Get file metadata
75
+ * @param filename - name of a file
76
+ * @returns central directory data
77
+ */
78
+ async stat(filename: string): Promise<ZipCDFileHeader & {size: number}> {
79
+ const cdFileHeader = await this.getCDFileHeader(filename);
80
+ return {...cdFileHeader, size: Number(cdFileHeader.uncompressedSize)};
81
+ }
82
+
83
+ /**
84
+ * Implementation of fetch against this file system
85
+ * @param filename - name of a file
86
+ * @returns - Response with file data
87
+ */
88
+ async fetch(filename: string): Promise<Response> {
89
+ const fileProvider = await this.fileProvider;
90
+ if (!fileProvider) {
91
+ throw new Error('No data detected in the zip archive');
92
+ }
93
+ const cdFileHeader = await this.getCDFileHeader(filename);
94
+ const localFileHeader = await parseZipLocalFileHeader(
95
+ cdFileHeader.localHeaderOffset,
96
+ fileProvider
97
+ );
98
+ if (!localFileHeader) {
99
+ throw new Error('Local file header has not been found in the zip archive`');
100
+ }
101
+
102
+ const compressionHandler = COMPRESSION_METHODS[localFileHeader.compressionMethod.toString()];
103
+ if (!compressionHandler) {
104
+ throw Error('Only Deflation compression is supported');
105
+ }
106
+
107
+ const compressedFile = await fileProvider.slice(
108
+ localFileHeader.fileDataOffset,
109
+ localFileHeader.fileDataOffset + localFileHeader.compressedSize
110
+ );
111
+
112
+ const uncompressedFile = await compressionHandler(compressedFile);
113
+
114
+ const response = new Response(uncompressedFile);
115
+ Object.defineProperty(response, 'url', {value: `${this.fileName || ''}/${filename}`});
116
+ return response;
117
+ }
118
+
119
+ /**
120
+ * Get central directory file header
121
+ * @param filename - name of a file
122
+ * @returns central directory file header
123
+ */
124
+ private async getCDFileHeader(filename: string): Promise<ZipCDFileHeader> {
125
+ const fileProvider = await this.fileProvider;
126
+ if (!fileProvider) {
127
+ throw new Error('No data detected in the zip archive');
128
+ }
129
+ const zipCDIterator = zipCDFileHeaderGenerator(fileProvider);
130
+ let result: ZipCDFileHeader | null = null;
131
+ for await (const cdHeader of zipCDIterator) {
132
+ if (cdHeader.fileName === filename) {
133
+ result = cdHeader;
134
+ break;
135
+ }
136
+ }
137
+ if (!result) {
138
+ throw new Error('File has not been found in the zip archive');
139
+ }
140
+ return result;
141
+ }
142
+ }
@@ -1,3 +1,7 @@
1
+ import md5 from 'md5';
2
+ import {zipCDFileHeaderGenerator} from './parse-zip/cd-file-header';
3
+ import {FileProvider} from './file-provider/file-provider';
4
+
1
5
  /** Element of hash array */
2
6
  export type HashElement = {
3
7
  /** File name hash */
@@ -77,3 +81,21 @@ export const findBin = (
77
81
  }
78
82
  return undefined;
79
83
  };
84
+
85
+ /**
86
+ * generates hash info from central directory
87
+ * @param fileProvider - provider of the archive
88
+ * @returns ready to use hash info
89
+ */
90
+ export const generateHashInfo = async (fileProvider: FileProvider): Promise<HashElement[]> => {
91
+ const zipCDIterator = zipCDFileHeaderGenerator(fileProvider);
92
+ const hashInfo: HashElement[] = [];
93
+ for await (const cdHeader of zipCDIterator) {
94
+ hashInfo.push({
95
+ hash: Buffer.from(md5(cdHeader.fileName.split('\\').join('/').toLocaleLowerCase()), 'hex'),
96
+ offset: cdHeader.localHeaderOffset
97
+ });
98
+ }
99
+ hashInfo.sort((a, b) => compareHashes(a.hash, b.hash));
100
+ return hashInfo;
101
+ };
package/src/index.ts CHANGED
@@ -2,15 +2,23 @@ export {ZipLoader} from './zip-loader';
2
2
  export {ZipWriter} from './zip-writer';
3
3
  export {default as TarBuilder} from './tar-builder';
4
4
 
5
- export type {FileProvider} from './parse-zip/file-provider';
6
- export {parseZipCDFileHeader, signature as cdSignature} from './parse-zip/cd-file-header';
5
+ export type {FileProvider} from './file-provider/file-provider';
6
+ export {FileHandleFile} from './file-provider/file-handle-file';
7
+ export {DataViewFile} from './file-provider/data-view-file';
8
+
9
+ export {
10
+ parseZipCDFileHeader,
11
+ zipCDFileHeaderGenerator,
12
+ signature as cdSignature
13
+ } from './parse-zip/cd-file-header';
7
14
  export {
8
15
  parseZipLocalFileHeader,
9
16
  signature as localHeaderSignature
10
17
  } from './parse-zip/local-file-header';
11
18
  export {parseEoCDRecord} from './parse-zip/end-of-central-directory';
12
19
  export {searchFromTheEnd} from './parse-zip/search-from-the-end';
13
- export {DataViewFile} from './parse-zip/data-view-file';
14
20
 
15
21
  export type {HashElement} from './hash-file-utility';
16
- export {compareHashes, parseHashFile, findBin} from './hash-file-utility';
22
+ export {compareHashes, parseHashFile, findBin, generateHashInfo} from './hash-file-utility';
23
+
24
+ export {ZipFileSystem} from './filesystems/zip-filesystem';