@loaders.gl/zip 4.4.0-alpha.1 → 4.4.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.dev.js +275 -248
- package/dist/dist.min.js +2 -2
- package/dist/filesystems/IndexedArchive.d.ts +4 -4
- package/dist/filesystems/IndexedArchive.d.ts.map +1 -1
- package/dist/filesystems/IndexedArchive.js +6 -5
- package/dist/filesystems/IndexedArchive.js.map +1 -0
- package/dist/filesystems/zip-filesystem.d.ts +7 -6
- package/dist/filesystems/zip-filesystem.d.ts.map +1 -1
- package/dist/filesystems/zip-filesystem.js +25 -22
- package/dist/filesystems/zip-filesystem.js.map +1 -0
- package/dist/hash-file-utility.d.ts +3 -3
- package/dist/hash-file-utility.d.ts.map +1 -1
- package/dist/hash-file-utility.js +2 -1
- package/dist/hash-file-utility.js.map +1 -0
- package/dist/index.cjs +181 -82
- package/dist/index.cjs.map +4 -4
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/tar/header.js +1 -0
- package/dist/lib/tar/header.js.map +1 -0
- package/dist/lib/tar/tar.js +1 -0
- package/dist/lib/tar/tar.js.map +1 -0
- package/dist/lib/tar/types.js +1 -0
- package/dist/lib/tar/types.js.map +1 -0
- package/dist/lib/tar/utils.js +1 -0
- package/dist/lib/tar/utils.js.map +1 -0
- package/dist/parse-zip/cd-file-header.d.ts +4 -4
- package/dist/parse-zip/cd-file-header.d.ts.map +1 -1
- package/dist/parse-zip/cd-file-header.js +9 -6
- package/dist/parse-zip/cd-file-header.js.map +1 -0
- package/dist/parse-zip/end-of-central-directory.d.ts +3 -3
- package/dist/parse-zip/end-of-central-directory.d.ts.map +1 -1
- package/dist/parse-zip/end-of-central-directory.js +13 -11
- package/dist/parse-zip/end-of-central-directory.js.map +1 -0
- package/dist/parse-zip/local-file-header.d.ts +2 -2
- package/dist/parse-zip/local-file-header.d.ts.map +1 -1
- package/dist/parse-zip/local-file-header.js +5 -3
- package/dist/parse-zip/local-file-header.js.map +1 -0
- package/dist/parse-zip/readable-file-utils.d.ts +34 -0
- package/dist/parse-zip/readable-file-utils.d.ts.map +1 -0
- package/dist/parse-zip/readable-file-utils.js +111 -0
- package/dist/parse-zip/readable-file-utils.js.map +1 -0
- package/dist/parse-zip/search-from-the-end.d.ts +2 -2
- package/dist/parse-zip/search-from-the-end.d.ts.map +1 -1
- package/dist/parse-zip/search-from-the-end.js +7 -8
- package/dist/parse-zip/search-from-the-end.js.map +1 -0
- package/dist/parse-zip/zip-composition.d.ts.map +1 -1
- package/dist/parse-zip/zip-composition.js +16 -8
- package/dist/parse-zip/zip-composition.js.map +1 -0
- package/dist/parse-zip/zip64-info-generation.js +1 -0
- package/dist/parse-zip/zip64-info-generation.js.map +1 -0
- package/dist/tar-builder.js +1 -0
- package/dist/tar-builder.js.map +1 -0
- package/dist/zip-loader.js +2 -1
- package/dist/zip-loader.js.map +1 -0
- package/dist/zip-writer.js +3 -2
- package/dist/zip-writer.js.map +1 -0
- package/package.json +6 -6
- package/src/filesystems/IndexedArchive.ts +6 -10
- package/src/filesystems/zip-filesystem.ts +26 -28
- package/src/hash-file-utility.ts +4 -7
- package/src/index.ts +5 -0
- package/src/parse-zip/cd-file-header.ts +18 -16
- package/src/parse-zip/end-of-central-directory.ts +16 -17
- package/src/parse-zip/local-file-header.ts +8 -9
- package/src/parse-zip/readable-file-utils.ts +134 -0
- package/src/parse-zip/search-from-the-end.ts +8 -10
- package/src/parse-zip/zip-composition.ts +25 -18
- package/src/zip-writer.ts +1 -1
package/dist/zip-writer.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
import JSZip from 'jszip';
|
|
5
5
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
6
|
-
const VERSION = typeof "4.4.0-alpha.
|
|
6
|
+
const VERSION = typeof "4.4.0-alpha.9" !== 'undefined' ? "4.4.0-alpha.9" : 'latest';
|
|
7
7
|
/**
|
|
8
8
|
* Zip exporter
|
|
9
9
|
*/
|
|
@@ -39,7 +39,8 @@ async function encodeZipAsync(fileMap, options = {}) {
|
|
|
39
39
|
zipOptions.onUpdate);
|
|
40
40
|
}
|
|
41
41
|
catch (error) {
|
|
42
|
-
options.log
|
|
42
|
+
options.core?.log?.error(`Unable to encode zip archive: ${error}`);
|
|
43
43
|
throw error;
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
|
+
//# sourceMappingURL=zip-writer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"zip-writer.js","sourceRoot":"","sources":["../src/zip-writer.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,KAA8B,MAAM,OAAO,CAAC;AAEnD,qDAAqD;AACrD,MAAM,OAAO,GAAG,sBAAkB,KAAK,WAAW,CAAC,CAAC,iBAAa,CAAC,CAAC,QAAQ,CAAC;AAU5E;;GAEG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,IAAI,EAAE,aAAa;IACnB,EAAE,EAAE,KAAK;IACT,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,OAAO;IAChB,UAAU,EAAE,CAAC,KAAK,CAAC;IACnB,QAAQ,EAAE,SAAS;IACnB,SAAS,EAAE,CAAC,iBAAiB,CAAC;IAC9B,OAAO,EAAE;QACP,GAAG,EAAE;YACH,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;SACnB;QACD,KAAK,EAAE,EAAE;KACV;IACD,MAAM,EAAE,cAAc;CACoE,CAAC;AAE7F,KAAK,UAAU,cAAc,CAC3B,OAAoC,EACpC,UAA4B,EAAE;IAE9B,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,uBAAuB;IACvB,KAAK,MAAM,WAAW,IAAI,OAAO,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;QAEzC,8EAA8E;QAC9E,sEAAsE;QACtE,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,UAAU,GAAG,EAAC,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,EAAC,CAAC;IAC/D,MAAM,YAAY,GAA0B,EAAC,GAAG,SAAS,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,EAAC,CAAC;IAE5F,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,aAAa,CAC9B,EAAC,GAAG,YAAY,EAAE,IAAI,EAAE,aAAa,EAAC,EAAE,0BAA0B;QAClE,UAAU,CAAC,QAAQ,CACpB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,iCAAiC,KAAK,EAAE,CAAC,CAAC;QACnE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/zip",
|
|
3
|
-
"version": "4.4.0-alpha.
|
|
3
|
+
"version": "4.4.0-alpha.9",
|
|
4
4
|
"description": "Zip Archive Loader",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -39,14 +39,14 @@
|
|
|
39
39
|
"build-bundle-dev": "ocular-bundle ./bundle.ts --env=dev --output=dist/dist.dev.js"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@loaders.gl/compression": "4.4.0-alpha.
|
|
43
|
-
"@loaders.gl/crypto": "4.4.0-alpha.
|
|
44
|
-
"@loaders.gl/loader-utils": "4.4.0-alpha.
|
|
42
|
+
"@loaders.gl/compression": "4.4.0-alpha.9",
|
|
43
|
+
"@loaders.gl/crypto": "4.4.0-alpha.9",
|
|
44
|
+
"@loaders.gl/loader-utils": "4.4.0-alpha.9",
|
|
45
45
|
"jszip": "^3.1.5",
|
|
46
46
|
"md5": "^2.3.0"
|
|
47
47
|
},
|
|
48
48
|
"peerDependencies": {
|
|
49
|
-
"@loaders.gl/core": "4.4.0-alpha.
|
|
49
|
+
"@loaders.gl/core": "4.4.0-alpha.1"
|
|
50
50
|
},
|
|
51
|
-
"gitHead": "
|
|
51
|
+
"gitHead": "e9e6710379718c7663e97eba868c76e15de4cb84"
|
|
52
52
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
2
2
|
import {ZipFileSystem} from './zip-filesystem';
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -6,21 +6,17 @@ import {ZipFileSystem} from './zip-filesystem';
|
|
|
6
6
|
* a hash file inside that allows to increase reading speed
|
|
7
7
|
*/
|
|
8
8
|
export abstract class IndexedArchive {
|
|
9
|
-
public
|
|
9
|
+
public file: ReadableFile;
|
|
10
10
|
public fileName?: string;
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Constructor
|
|
14
|
-
* @param fileProvider -
|
|
14
|
+
* @param fileProvider - readable file instance for random access
|
|
15
15
|
* @param hashTable - pre-loaded hashTable. If presented, getFile will skip reading the hash file
|
|
16
16
|
* @param fileName - name of the archive. It is used to add to an URL of a loader context
|
|
17
17
|
*/
|
|
18
|
-
constructor(
|
|
19
|
-
|
|
20
|
-
hashTable?: Record<string, bigint>,
|
|
21
|
-
fileName?: string
|
|
22
|
-
) {
|
|
23
|
-
this.fileProvider = fileProvider;
|
|
18
|
+
constructor(file: ReadableFile, hashTable?: Record<string, bigint>, fileName?: string) {
|
|
19
|
+
this.file = file;
|
|
24
20
|
this.fileName = fileName;
|
|
25
21
|
}
|
|
26
22
|
|
|
@@ -37,7 +33,7 @@ export abstract class IndexedArchive {
|
|
|
37
33
|
* @returns
|
|
38
34
|
*/
|
|
39
35
|
protected async getFileWithoutHash(filename: string): Promise<ArrayBuffer> {
|
|
40
|
-
const zipFS = new ZipFileSystem(this.
|
|
36
|
+
const zipFS = new ZipFileSystem(this.file);
|
|
41
37
|
const response = await zipFS.fetch(filename);
|
|
42
38
|
return await response.arrayBuffer();
|
|
43
39
|
}
|
|
@@ -2,17 +2,13 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
isBrowser,
|
|
8
|
-
FileProviderInterface,
|
|
9
|
-
isFileProvider,
|
|
10
|
-
FileHandleFile
|
|
11
|
-
} from '@loaders.gl/loader-utils';
|
|
5
|
+
import {FileSystem, isBrowser, BlobFile, NodeFile} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
12
7
|
import {ZipCDFileHeader, makeZipCDHeaderIterator} from '../parse-zip/cd-file-header';
|
|
13
8
|
import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
|
|
14
9
|
import {DeflateCompression} from '@loaders.gl/compression';
|
|
15
10
|
import {IndexedArchive} from './IndexedArchive';
|
|
11
|
+
import {readRange} from '../parse-zip/readable-file-utils';
|
|
16
12
|
|
|
17
13
|
export type CompressionHandler = (compressedFile: ArrayBuffer) => Promise<ArrayBuffer>;
|
|
18
14
|
/** Handling different compression types in zip */
|
|
@@ -29,40 +25,41 @@ export const ZIP_COMPRESSION_HANDLERS: {[key: number]: CompressionHandler} = {
|
|
|
29
25
|
|
|
30
26
|
/**
|
|
31
27
|
* FileSystem adapter for a ZIP file
|
|
32
|
-
* Holds
|
|
28
|
+
* Holds a ReadableFile object that provides random access to archived files
|
|
33
29
|
*/
|
|
34
30
|
export class ZipFileSystem implements FileSystem {
|
|
35
|
-
/**
|
|
36
|
-
public
|
|
31
|
+
/** File instance */
|
|
32
|
+
public file: ReadableFile | null = null;
|
|
37
33
|
public fileName?: string;
|
|
38
34
|
public archive: IndexedArchive | null = null;
|
|
39
35
|
|
|
40
36
|
/**
|
|
41
37
|
* Constructor
|
|
42
|
-
* @param file - instance of
|
|
38
|
+
* @param file - instance of ReadableFile or file path string
|
|
43
39
|
*/
|
|
44
|
-
constructor(file:
|
|
40
|
+
constructor(file: ReadableFile | IndexedArchive | string | Blob | ArrayBuffer) {
|
|
45
41
|
// Try to open file in NodeJS
|
|
46
42
|
if (typeof file === 'string') {
|
|
47
43
|
this.fileName = file;
|
|
48
|
-
if (
|
|
49
|
-
|
|
50
|
-
} else {
|
|
51
|
-
throw new Error('Cannot open file for random access in a WEB browser');
|
|
44
|
+
if (isBrowser) {
|
|
45
|
+
throw new Error('ZipFileSystem cannot open file paths in browser environments');
|
|
52
46
|
}
|
|
47
|
+
this.file = new NodeFile(file);
|
|
48
|
+
} else if (file instanceof Blob || file instanceof ArrayBuffer) {
|
|
49
|
+
this.file = new BlobFile(file);
|
|
53
50
|
} else if (file instanceof IndexedArchive) {
|
|
54
|
-
this.
|
|
51
|
+
this.file = file.file;
|
|
55
52
|
this.archive = file;
|
|
56
53
|
this.fileName = file.fileName;
|
|
57
|
-
} else
|
|
58
|
-
this.
|
|
54
|
+
} else {
|
|
55
|
+
this.file = file;
|
|
59
56
|
}
|
|
60
57
|
}
|
|
61
58
|
|
|
62
59
|
/** Clean up resources */
|
|
63
60
|
async destroy() {
|
|
64
|
-
if (this.
|
|
65
|
-
await this.
|
|
61
|
+
if (this.file) {
|
|
62
|
+
await this.file.close();
|
|
66
63
|
}
|
|
67
64
|
}
|
|
68
65
|
|
|
@@ -71,11 +68,11 @@ export class ZipFileSystem implements FileSystem {
|
|
|
71
68
|
* @returns array of file names
|
|
72
69
|
*/
|
|
73
70
|
async readdir(): Promise<string[]> {
|
|
74
|
-
if (!this.
|
|
71
|
+
if (!this.file) {
|
|
75
72
|
throw new Error('No data detected in the zip archive');
|
|
76
73
|
}
|
|
77
74
|
const fileNames: string[] = [];
|
|
78
|
-
const zipCDIterator = makeZipCDHeaderIterator(this.
|
|
75
|
+
const zipCDIterator = makeZipCDHeaderIterator(this.file);
|
|
79
76
|
for await (const cdHeader of zipCDIterator) {
|
|
80
77
|
fileNames.push(cdHeader.fileName);
|
|
81
78
|
}
|
|
@@ -106,13 +103,13 @@ export class ZipFileSystem implements FileSystem {
|
|
|
106
103
|
if (this.archive) {
|
|
107
104
|
uncompressedFile = await this.archive.getFile(filename, 'http');
|
|
108
105
|
} else {
|
|
109
|
-
if (!this.
|
|
106
|
+
if (!this.file) {
|
|
110
107
|
throw new Error('No data detected in the zip archive');
|
|
111
108
|
}
|
|
112
109
|
const cdFileHeader = await this.getCDFileHeader(filename);
|
|
113
110
|
const localFileHeader = await parseZipLocalFileHeader(
|
|
114
111
|
cdFileHeader.localHeaderOffset,
|
|
115
|
-
this.
|
|
112
|
+
this.file
|
|
116
113
|
);
|
|
117
114
|
if (!localFileHeader) {
|
|
118
115
|
throw new Error('Local file header has not been found in the zip archive`');
|
|
@@ -124,7 +121,8 @@ export class ZipFileSystem implements FileSystem {
|
|
|
124
121
|
throw Error('Only Deflation compression is supported');
|
|
125
122
|
}
|
|
126
123
|
|
|
127
|
-
const compressedFile = await
|
|
124
|
+
const compressedFile = await readRange(
|
|
125
|
+
this.file,
|
|
128
126
|
localFileHeader.fileDataOffset,
|
|
129
127
|
localFileHeader.fileDataOffset + localFileHeader.compressedSize
|
|
130
128
|
);
|
|
@@ -145,10 +143,10 @@ export class ZipFileSystem implements FileSystem {
|
|
|
145
143
|
* @returns central directory file header
|
|
146
144
|
*/
|
|
147
145
|
private async getCDFileHeader(filename: string): Promise<ZipCDFileHeader> {
|
|
148
|
-
if (!this.
|
|
146
|
+
if (!this.file) {
|
|
149
147
|
throw new Error('No data detected in the zip archive');
|
|
150
148
|
}
|
|
151
|
-
const zipCDIterator = makeZipCDHeaderIterator(this.
|
|
149
|
+
const zipCDIterator = makeZipCDHeaderIterator(this.file);
|
|
152
150
|
let result: ZipCDFileHeader | null = null;
|
|
153
151
|
for await (const cdHeader of zipCDIterator) {
|
|
154
152
|
if (cdHeader.fileName === filename) {
|
package/src/hash-file-utility.ts
CHANGED
|
@@ -3,11 +3,8 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
import {MD5Hash} from '@loaders.gl/crypto';
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
concatenateArrayBuffers,
|
|
9
|
-
concatenateArrayBuffersFromArray
|
|
10
|
-
} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {concatenateArrayBuffers, concatenateArrayBuffersFromArray} from '@loaders.gl/loader-utils';
|
|
7
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
11
8
|
import {ZipCDFileHeader, makeZipCDHeaderIterator} from './parse-zip/cd-file-header';
|
|
12
9
|
|
|
13
10
|
/**
|
|
@@ -38,11 +35,11 @@ function bufferToHex(buffer: ArrayBuffer, start: number, length: number): string
|
|
|
38
35
|
|
|
39
36
|
/**
|
|
40
37
|
* generates hash info from zip files "central directory"
|
|
41
|
-
* @param fileProvider -
|
|
38
|
+
* @param fileProvider - readable archive source
|
|
42
39
|
* @returns ready to use hash info
|
|
43
40
|
*/
|
|
44
41
|
export async function makeHashTableFromZipHeaders(
|
|
45
|
-
fileProvider:
|
|
42
|
+
fileProvider: ReadableFile
|
|
46
43
|
): Promise<Record<string, bigint>> {
|
|
47
44
|
const zipCDIterator = makeZipCDHeaderIterator(fileProvider);
|
|
48
45
|
return getHashTable(zipCDIterator);
|
package/src/index.ts
CHANGED
|
@@ -19,6 +19,11 @@ export {
|
|
|
19
19
|
} from './parse-zip/local-file-header';
|
|
20
20
|
export {parseEoCDRecord} from './parse-zip/end-of-central-directory';
|
|
21
21
|
export {searchFromTheEnd} from './parse-zip/search-from-the-end';
|
|
22
|
+
export {
|
|
23
|
+
readRange,
|
|
24
|
+
getReadableFileSize,
|
|
25
|
+
DataViewReadableFile
|
|
26
|
+
} from './parse-zip/readable-file-utils';
|
|
22
27
|
export {addOneFile, createZip} from './parse-zip/zip-composition';
|
|
23
28
|
|
|
24
29
|
// export type {HashElement} from './hash-file-utility';
|
|
@@ -2,15 +2,17 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
FileProviderInterface,
|
|
8
|
-
compareArrayBuffers,
|
|
9
|
-
concatenateArrayBuffers
|
|
10
|
-
} from '@loaders.gl/loader-utils';
|
|
5
|
+
import {compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
11
7
|
import {parseEoCDRecord} from './end-of-central-directory';
|
|
12
8
|
import {ZipSignature} from './search-from-the-end';
|
|
13
9
|
import {createZip64Info, setFieldToNumber} from './zip64-info-generation';
|
|
10
|
+
import {
|
|
11
|
+
DataViewReadableFile,
|
|
12
|
+
getReadableFileSize,
|
|
13
|
+
readDataView,
|
|
14
|
+
readRange
|
|
15
|
+
} from './readable-file-utils';
|
|
14
16
|
|
|
15
17
|
/**
|
|
16
18
|
* zip central directory file header info
|
|
@@ -66,14 +68,13 @@ export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x01, 0x02]);
|
|
|
66
68
|
*/
|
|
67
69
|
export const parseZipCDFileHeader = async (
|
|
68
70
|
headerOffset: bigint,
|
|
69
|
-
file:
|
|
71
|
+
file: ReadableFile
|
|
70
72
|
): Promise<ZipCDFileHeader | null> => {
|
|
71
|
-
|
|
73
|
+
const fileLength = await getReadableFileSize(file);
|
|
74
|
+
if (headerOffset >= fileLength) {
|
|
72
75
|
return null;
|
|
73
76
|
}
|
|
74
|
-
const mainHeader =
|
|
75
|
-
await file.slice(headerOffset, headerOffset + CD_FILE_NAME_OFFSET)
|
|
76
|
-
);
|
|
77
|
+
const mainHeader = await readDataView(file, headerOffset, headerOffset + CD_FILE_NAME_OFFSET);
|
|
77
78
|
|
|
78
79
|
const magicBytes = mainHeader.buffer.slice(0, 4);
|
|
79
80
|
if (!compareArrayBuffers(magicBytes, signature.buffer)) {
|
|
@@ -86,7 +87,8 @@ export const parseZipCDFileHeader = async (
|
|
|
86
87
|
const startDisk = BigInt(mainHeader.getUint16(CD_START_DISK_OFFSET, true));
|
|
87
88
|
const fileNameLength = mainHeader.getUint16(CD_FILE_NAME_LENGTH_OFFSET, true);
|
|
88
89
|
|
|
89
|
-
const additionalHeader = await
|
|
90
|
+
const additionalHeader = await readRange(
|
|
91
|
+
file,
|
|
90
92
|
headerOffset + CD_FILE_NAME_OFFSET,
|
|
91
93
|
headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength)
|
|
92
94
|
);
|
|
@@ -124,14 +126,14 @@ export const parseZipCDFileHeader = async (
|
|
|
124
126
|
|
|
125
127
|
/**
|
|
126
128
|
* Create iterator over files of zip archive
|
|
127
|
-
* @param fileProvider - file
|
|
129
|
+
* @param fileProvider - readable file that provides random access to the file
|
|
128
130
|
*/
|
|
129
131
|
export async function* makeZipCDHeaderIterator(
|
|
130
|
-
fileProvider:
|
|
132
|
+
fileProvider: ReadableFile
|
|
131
133
|
): AsyncIterable<ZipCDFileHeader> {
|
|
132
134
|
const {cdStartOffset, cdByteSize} = await parseEoCDRecord(fileProvider);
|
|
133
|
-
const centralDirectory = new
|
|
134
|
-
new DataView(await fileProvider
|
|
135
|
+
const centralDirectory = new DataViewReadableFile(
|
|
136
|
+
new DataView(await readRange(fileProvider, cdStartOffset, cdStartOffset + cdByteSize))
|
|
135
137
|
);
|
|
136
138
|
let cdHeader = await parseZipCDFileHeader(0n, centralDirectory);
|
|
137
139
|
while (cdHeader) {
|
|
@@ -2,12 +2,10 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
compareArrayBuffers,
|
|
8
|
-
concatenateArrayBuffers
|
|
9
|
-
} from '@loaders.gl/loader-utils';
|
|
5
|
+
import {compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
10
7
|
import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
|
|
8
|
+
import {readBigUint64, readUint16, readUint32, readRange} from './readable-file-utils';
|
|
11
9
|
import {setFieldToNumber} from './zip64-info-generation';
|
|
12
10
|
|
|
13
11
|
/**
|
|
@@ -65,33 +63,34 @@ const ZIP64_COMMENT_OFFSET = 56n;
|
|
|
65
63
|
|
|
66
64
|
/**
|
|
67
65
|
* Parses end of central directory record of zip file
|
|
68
|
-
* @param file -
|
|
66
|
+
* @param file - ReadableFile instance
|
|
69
67
|
* @returns Info from the header
|
|
70
68
|
*/
|
|
71
|
-
export const parseEoCDRecord = async (file:
|
|
69
|
+
export const parseEoCDRecord = async (file: ReadableFile): Promise<ZipEoCDRecord> => {
|
|
72
70
|
const zipEoCDOffset = await searchFromTheEnd(file, eoCDSignature);
|
|
73
71
|
|
|
74
|
-
let cdRecordsNumber = BigInt(await file
|
|
75
|
-
let cdByteSize = BigInt(await file
|
|
76
|
-
let cdStartOffset = BigInt(await file
|
|
72
|
+
let cdRecordsNumber = BigInt(await readUint16(file, zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET));
|
|
73
|
+
let cdByteSize = BigInt(await readUint32(file, zipEoCDOffset + CD_CD_BYTE_SIZE_OFFSET));
|
|
74
|
+
let cdStartOffset = BigInt(await readUint32(file, zipEoCDOffset + CD_START_OFFSET_OFFSET));
|
|
77
75
|
|
|
78
76
|
let zip64EoCDLocatorOffset = zipEoCDOffset - 20n;
|
|
79
77
|
let zip64EoCDOffset = 0n;
|
|
80
78
|
|
|
81
|
-
const magicBytes = await file
|
|
82
|
-
if (compareArrayBuffers(magicBytes, zip64EoCDLocatorSignature)) {
|
|
83
|
-
zip64EoCDOffset = await
|
|
79
|
+
const magicBytes = await readRange(file, zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n);
|
|
80
|
+
if (compareArrayBuffers(magicBytes, zip64EoCDLocatorSignature.buffer)) {
|
|
81
|
+
zip64EoCDOffset = await readBigUint64(
|
|
82
|
+
file,
|
|
84
83
|
zip64EoCDLocatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET
|
|
85
84
|
);
|
|
86
85
|
|
|
87
|
-
const endOfCDMagicBytes = await file
|
|
86
|
+
const endOfCDMagicBytes = await readRange(file, zip64EoCDOffset, zip64EoCDOffset + 4n);
|
|
88
87
|
if (!compareArrayBuffers(endOfCDMagicBytes, zip64EoCDSignature.buffer)) {
|
|
89
88
|
throw new Error('zip64 EoCD not found');
|
|
90
89
|
}
|
|
91
90
|
|
|
92
|
-
cdRecordsNumber = await file
|
|
93
|
-
cdByteSize = await file
|
|
94
|
-
cdStartOffset = await file
|
|
91
|
+
cdRecordsNumber = await readBigUint64(file, zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET);
|
|
92
|
+
cdByteSize = await readBigUint64(file, zip64EoCDOffset + ZIP64_CD_CD_BYTE_SIZE_OFFSET);
|
|
93
|
+
cdStartOffset = await readBigUint64(file, zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET);
|
|
95
94
|
} else {
|
|
96
95
|
zip64EoCDLocatorOffset = 0n;
|
|
97
96
|
}
|
|
@@ -2,13 +2,11 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
compareArrayBuffers,
|
|
8
|
-
concatenateArrayBuffers
|
|
9
|
-
} from '@loaders.gl/loader-utils';
|
|
5
|
+
import {compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
10
7
|
import {ZipSignature} from './search-from-the-end';
|
|
11
8
|
import {createZip64Info, setFieldToNumber} from './zip64-info-generation';
|
|
9
|
+
import {readDataView, readRange} from './readable-file-utils';
|
|
12
10
|
|
|
13
11
|
/**
|
|
14
12
|
* zip local file header info
|
|
@@ -47,12 +45,12 @@ export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x03, 0x04]);
|
|
|
47
45
|
*/
|
|
48
46
|
export const parseZipLocalFileHeader = async (
|
|
49
47
|
headerOffset: bigint,
|
|
50
|
-
file:
|
|
48
|
+
file: ReadableFile
|
|
51
49
|
): Promise<ZipLocalFileHeader | null> => {
|
|
52
|
-
const mainHeader =
|
|
50
|
+
const mainHeader = await readDataView(file, headerOffset, headerOffset + FILE_NAME_OFFSET);
|
|
53
51
|
|
|
54
52
|
const magicBytes = mainHeader.buffer.slice(0, 4);
|
|
55
|
-
if (!compareArrayBuffers(magicBytes, signature)) {
|
|
53
|
+
if (!compareArrayBuffers(magicBytes, signature.buffer)) {
|
|
56
54
|
return null;
|
|
57
55
|
}
|
|
58
56
|
|
|
@@ -60,7 +58,8 @@ export const parseZipLocalFileHeader = async (
|
|
|
60
58
|
|
|
61
59
|
const extraFieldLength = mainHeader.getUint16(EXTRA_FIELD_LENGTH_OFFSET, true);
|
|
62
60
|
|
|
63
|
-
const additionalHeader = await
|
|
61
|
+
const additionalHeader = await readRange(
|
|
62
|
+
file,
|
|
64
63
|
headerOffset + FILE_NAME_OFFSET,
|
|
65
64
|
headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength)
|
|
66
65
|
);
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {copyToArrayBuffer, type ReadableFile, type Stat} from '@loaders.gl/loader-utils';
|
|
6
|
+
|
|
7
|
+
function toBigInt(value: number | bigint): bigint {
|
|
8
|
+
return typeof value === 'bigint' ? value : BigInt(value);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function toNumber(value: number | bigint): number {
|
|
12
|
+
const numberValue = Number(value);
|
|
13
|
+
if (!Number.isFinite(numberValue)) {
|
|
14
|
+
throw new Error('Offset is out of bounds');
|
|
15
|
+
}
|
|
16
|
+
return numberValue;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function normalizeOffset(offset: number, size: number): number {
|
|
20
|
+
if (offset < 0) {
|
|
21
|
+
return Math.max(size + offset, 0);
|
|
22
|
+
}
|
|
23
|
+
return Math.min(offset, size);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Read a byte range from a readable file.
|
|
28
|
+
* @param file readable file handle
|
|
29
|
+
* @param start inclusive start offset
|
|
30
|
+
* @param end exclusive end offset
|
|
31
|
+
* @returns requested slice
|
|
32
|
+
*/
|
|
33
|
+
export async function readRange(
|
|
34
|
+
file: ReadableFile,
|
|
35
|
+
start: number | bigint,
|
|
36
|
+
end: number | bigint
|
|
37
|
+
): Promise<ArrayBuffer> {
|
|
38
|
+
const startOffset = toBigInt(start);
|
|
39
|
+
const endOffset = toBigInt(end);
|
|
40
|
+
const length = endOffset - startOffset;
|
|
41
|
+
if (length < 0) {
|
|
42
|
+
throw new Error('Invalid range requested');
|
|
43
|
+
}
|
|
44
|
+
return await file.read(startOffset, toNumber(length));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function readDataView(
|
|
48
|
+
file: ReadableFile,
|
|
49
|
+
start: number | bigint,
|
|
50
|
+
end: number | bigint
|
|
51
|
+
): Promise<DataView> {
|
|
52
|
+
const arrayBuffer = await readRange(file, start, end);
|
|
53
|
+
return new DataView(arrayBuffer);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export async function readUint8(file: ReadableFile, offset: number | bigint): Promise<number> {
|
|
57
|
+
const dataView = await readDataView(file, offset, toBigInt(offset) + 1n);
|
|
58
|
+
return dataView.getUint8(0);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function readUint16(file: ReadableFile, offset: number | bigint): Promise<number> {
|
|
62
|
+
const dataView = await readDataView(file, offset, toBigInt(offset) + 2n);
|
|
63
|
+
return dataView.getUint16(0, true);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export async function readUint32(file: ReadableFile, offset: number | bigint): Promise<number> {
|
|
67
|
+
const dataView = await readDataView(file, offset, toBigInt(offset) + 4n);
|
|
68
|
+
return dataView.getUint32(0, true);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export async function readBigUint64(file: ReadableFile, offset: number | bigint): Promise<bigint> {
|
|
72
|
+
const dataView = await readDataView(file, offset, toBigInt(offset) + 8n);
|
|
73
|
+
return dataView.getBigUint64(0, true);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Resolve the size of a readable file.
|
|
78
|
+
* @param file readable file handle
|
|
79
|
+
* @returns file size as bigint
|
|
80
|
+
*/
|
|
81
|
+
export async function getReadableFileSize(file: ReadableFile): Promise<bigint> {
|
|
82
|
+
if (file.bigsize > 0n) {
|
|
83
|
+
return file.bigsize;
|
|
84
|
+
}
|
|
85
|
+
if (file.size > 0) {
|
|
86
|
+
return BigInt(file.size);
|
|
87
|
+
}
|
|
88
|
+
if (file.stat) {
|
|
89
|
+
const stats: Stat = await file.stat();
|
|
90
|
+
if (stats?.bigsize !== undefined) {
|
|
91
|
+
return stats.bigsize;
|
|
92
|
+
}
|
|
93
|
+
if (stats?.size !== undefined) {
|
|
94
|
+
return BigInt(stats.size);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return 0n;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Minimal readable file backed by a DataView.
|
|
102
|
+
*/
|
|
103
|
+
export class DataViewReadableFile implements ReadableFile {
|
|
104
|
+
readonly handle: DataView;
|
|
105
|
+
readonly size: number;
|
|
106
|
+
readonly bigsize: bigint;
|
|
107
|
+
readonly url: string;
|
|
108
|
+
|
|
109
|
+
constructor(dataView: DataView, url: string = '') {
|
|
110
|
+
this.handle = dataView;
|
|
111
|
+
this.size = dataView.byteLength;
|
|
112
|
+
this.bigsize = BigInt(dataView.byteLength);
|
|
113
|
+
this.url = url;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async close(): Promise<void> {}
|
|
117
|
+
|
|
118
|
+
async stat(): Promise<Stat> {
|
|
119
|
+
return {size: this.size, bigsize: this.bigsize, isDirectory: false};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async read(start: number | bigint = 0, length?: number): Promise<ArrayBuffer> {
|
|
123
|
+
const offset = toNumber(start);
|
|
124
|
+
const end = length ? offset + length : this.size;
|
|
125
|
+
const normalizedStart = normalizeOffset(offset, this.size);
|
|
126
|
+
const normalizedEnd = normalizeOffset(end, this.size);
|
|
127
|
+
const clampedEnd = Math.max(normalizedEnd, normalizedStart);
|
|
128
|
+
const lengthToRead = clampedEnd - normalizedStart;
|
|
129
|
+
if (lengthToRead <= 0) {
|
|
130
|
+
return new ArrayBuffer(0);
|
|
131
|
+
}
|
|
132
|
+
return copyToArrayBuffer(this.handle.buffer, normalizedStart, lengthToRead);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {getReadableFileSize, readRange} from './readable-file-utils';
|
|
6
7
|
|
|
7
8
|
/** Description of zip signature type */
|
|
8
9
|
export type ZipSignature = Uint8Array;
|
|
@@ -16,25 +17,22 @@ const buffLength = 1024;
|
|
|
16
17
|
* @returns
|
|
17
18
|
*/
|
|
18
19
|
export const searchFromTheEnd = async (
|
|
19
|
-
file:
|
|
20
|
+
file: ReadableFile,
|
|
20
21
|
target: ZipSignature
|
|
21
22
|
): Promise<bigint> => {
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
await file.getUint8(file.length - 3n),
|
|
26
|
-
undefined
|
|
27
|
-
];
|
|
23
|
+
const fileLength = await getReadableFileSize(file);
|
|
24
|
+
const lastBytes = new Uint8Array(await readRange(file, fileLength - 3n, fileLength + 1n));
|
|
25
|
+
const searchWindow = [lastBytes[3], lastBytes[2], lastBytes[1], undefined];
|
|
28
26
|
|
|
29
27
|
let targetOffset = -1;
|
|
30
28
|
|
|
31
29
|
// looking for the last record in the central directory
|
|
32
|
-
let point =
|
|
30
|
+
let point = fileLength - 4n;
|
|
33
31
|
do {
|
|
34
32
|
const prevPoint = point;
|
|
35
33
|
point -= BigInt(buffLength);
|
|
36
34
|
point = point >= 0n ? point : 0n;
|
|
37
|
-
const buff = new Uint8Array(await file
|
|
35
|
+
const buff = new Uint8Array(await readRange(file, point, prevPoint));
|
|
38
36
|
for (let i = buff.length - 1; i > -1; i--) {
|
|
39
37
|
searchWindow[3] = searchWindow[2];
|
|
40
38
|
searchWindow[2] = searchWindow[1];
|