@loaders.gl/i3s 4.0.0-alpha.14 → 4.0.0-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +139 -102
- package/dist/es5/arcgis-webscene-loader.js +1 -1
- package/dist/es5/i3s-attribute-loader.js +1 -1
- package/dist/es5/i3s-building-scene-layer-loader.js +1 -1
- package/dist/es5/i3s-content-loader.js +1 -1
- package/dist/es5/i3s-loader.js +1 -1
- package/dist/es5/i3s-node-page-loader.js +1 -1
- package/dist/es5/i3s-slpk-loader.js +30 -2
- package/dist/es5/i3s-slpk-loader.js.map +1 -1
- package/dist/es5/index.js +7 -7
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/parsers/parse-slpk/parse-slpk.js +30 -69
- package/dist/es5/lib/parsers/parse-slpk/parse-slpk.js.map +1 -1
- package/dist/es5/lib/parsers/parse-slpk/search-from-the-end.js +69 -0
- package/dist/es5/lib/parsers/parse-slpk/search-from-the-end.js.map +1 -0
- package/dist/es5/lib/parsers/parse-slpk/slpk-archieve.js +5 -6
- package/dist/es5/lib/parsers/parse-slpk/slpk-archieve.js.map +1 -1
- package/dist/es5/lib/parsers/parse-zip/cd-file-header.js +53 -35
- package/dist/es5/lib/parsers/parse-zip/cd-file-header.js.map +1 -1
- package/dist/es5/lib/parsers/parse-zip/data-view-file-provider.js +129 -0
- package/dist/es5/lib/parsers/parse-zip/data-view-file-provider.js.map +1 -0
- package/dist/es5/lib/parsers/parse-zip/file-provider.js.map +1 -1
- package/dist/es5/lib/parsers/parse-zip/local-file-header.js +54 -13
- package/dist/es5/lib/parsers/parse-zip/local-file-header.js.map +1 -1
- package/dist/esm/arcgis-webscene-loader.js +1 -1
- package/dist/esm/i3s-attribute-loader.js +1 -1
- package/dist/esm/i3s-building-scene-layer-loader.js +1 -1
- package/dist/esm/i3s-content-loader.js +1 -1
- package/dist/esm/i3s-loader.js +1 -1
- package/dist/esm/i3s-node-page-loader.js +1 -1
- package/dist/esm/i3s-slpk-loader.js +8 -2
- package/dist/esm/i3s-slpk-loader.js.map +1 -1
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/parsers/parse-slpk/parse-slpk.js +7 -25
- package/dist/esm/lib/parsers/parse-slpk/parse-slpk.js.map +1 -1
- package/dist/esm/lib/parsers/parse-slpk/search-from-the-end.js +16 -0
- package/dist/esm/lib/parsers/parse-slpk/search-from-the-end.js.map +1 -0
- package/dist/esm/lib/parsers/parse-slpk/slpk-archieve.js +5 -6
- package/dist/esm/lib/parsers/parse-slpk/slpk-archieve.js.map +1 -1
- package/dist/esm/lib/parsers/parse-zip/cd-file-header.js +22 -20
- package/dist/esm/lib/parsers/parse-zip/cd-file-header.js.map +1 -1
- package/dist/esm/lib/parsers/parse-zip/data-view-file-provider.js +32 -0
- package/dist/esm/lib/parsers/parse-zip/data-view-file-provider.js.map +1 -0
- package/dist/esm/lib/parsers/parse-zip/file-provider.js.map +1 -1
- package/dist/esm/lib/parsers/parse-zip/local-file-header.js +24 -9
- package/dist/esm/lib/parsers/parse-zip/local-file-header.js.map +1 -1
- package/dist/i3s-content-worker-node.js +2 -2
- package/dist/i3s-content-worker.js +8 -8
- package/dist/i3s-slpk-loader.d.ts +3 -0
- package/dist/i3s-slpk-loader.d.ts.map +1 -1
- package/dist/i3s-slpk-loader.js +11 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/lib/parsers/parse-slpk/parse-slpk.d.ts +8 -3
- package/dist/lib/parsers/parse-slpk/parse-slpk.d.ts.map +1 -1
- package/dist/lib/parsers/parse-slpk/parse-slpk.js +10 -33
- package/dist/lib/parsers/parse-slpk/search-from-the-end.d.ts +11 -0
- package/dist/lib/parsers/parse-slpk/search-from-the-end.d.ts.map +1 -0
- package/dist/lib/parsers/parse-slpk/search-from-the-end.js +31 -0
- package/dist/lib/parsers/parse-slpk/slpk-archieve.d.ts +4 -3
- package/dist/lib/parsers/parse-slpk/slpk-archieve.d.ts.map +1 -1
- package/dist/lib/parsers/parse-slpk/slpk-archieve.js +5 -6
- package/dist/lib/parsers/parse-zip/cd-file-header.d.ts +5 -5
- package/dist/lib/parsers/parse-zip/cd-file-header.d.ts.map +1 -1
- package/dist/lib/parsers/parse-zip/cd-file-header.js +23 -22
- package/dist/lib/parsers/parse-zip/{buffer-file-provider.d.ts → data-view-file-provider.d.ts} +15 -16
- package/dist/lib/parsers/parse-zip/data-view-file-provider.d.ts.map +1 -0
- package/dist/lib/parsers/parse-zip/{buffer-file-provider.js → data-view-file-provider.js} +28 -14
- package/dist/lib/parsers/parse-zip/file-provider.d.ts +10 -5
- package/dist/lib/parsers/parse-zip/file-provider.d.ts.map +1 -1
- package/dist/lib/parsers/parse-zip/local-file-header.d.ts +5 -3
- package/dist/lib/parsers/parse-zip/local-file-header.d.ts.map +1 -1
- package/dist/lib/parsers/parse-zip/local-file-header.js +29 -10
- package/package.json +9 -9
- package/src/i3s-slpk-loader.ts +19 -1
- package/src/index.ts +2 -1
- package/src/lib/parsers/parse-slpk/parse-slpk.ts +11 -40
- package/src/lib/parsers/parse-slpk/search-from-the-end.ts +38 -0
- package/src/lib/parsers/parse-slpk/slpk-archieve.ts +9 -12
- package/src/lib/parsers/parse-zip/cd-file-header.ts +31 -29
- package/src/lib/parsers/parse-zip/data-view-file-provider.ts +69 -0
- package/src/lib/parsers/parse-zip/file-provider.ts +11 -5
- package/src/lib/parsers/parse-zip/local-file-header.ts +43 -17
- package/dist/es5/lib/parsers/parse-zip/buffer-file-provider.js +0 -46
- package/dist/es5/lib/parsers/parse-zip/buffer-file-provider.js.map +0 -1
- package/dist/esm/lib/parsers/parse-zip/buffer-file-provider.js +0 -23
- package/dist/esm/lib/parsers/parse-zip/buffer-file-provider.js.map +0 -1
- package/dist/lib/parsers/parse-zip/buffer-file-provider.d.ts.map +0 -1
- package/src/lib/parsers/parse-zip/buffer-file-provider.ts +0 -55
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
1
2
|
import { FileProvider } from './file-provider';
|
|
2
3
|
/**
|
|
3
4
|
* zip local file header info
|
|
@@ -11,15 +12,16 @@ export type ZipLocalFileHeader = {
|
|
|
11
12
|
/** Extra field length */
|
|
12
13
|
extraFieldLength: number;
|
|
13
14
|
/** Offset of the file data */
|
|
14
|
-
fileDataOffset:
|
|
15
|
+
fileDataOffset: bigint;
|
|
15
16
|
/** Compressed size */
|
|
16
|
-
compressedSize:
|
|
17
|
+
compressedSize: bigint;
|
|
17
18
|
};
|
|
19
|
+
export declare const signature: Buffer;
|
|
18
20
|
/**
|
|
19
21
|
* Parses local file header of zip file
|
|
20
22
|
* @param headerOffset - offset in the archive where header starts
|
|
21
23
|
* @param buffer - buffer containing whole array
|
|
22
24
|
* @returns Info from the header
|
|
23
25
|
*/
|
|
24
|
-
export declare const parseZipLocalFileHeader: (headerOffset:
|
|
26
|
+
export declare const parseZipLocalFileHeader: (headerOffset: bigint, buffer: FileProvider) => Promise<ZipLocalFileHeader | undefined>;
|
|
25
27
|
//# sourceMappingURL=local-file-header.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-file-header.d.ts","sourceRoot":"","sources":["../../../../src/lib/parsers/parse-zip/local-file-header.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,iBAAiB,CAAC;AAE7C;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,uBAAuB;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;
|
|
1
|
+
{"version":3,"file":"local-file-header.d.ts","sourceRoot":"","sources":["../../../../src/lib/parsers/parse-zip/local-file-header.ts"],"names":[],"mappings":";AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,iBAAiB,CAAC;AAE7C;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,uBAAuB;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAUF,eAAO,MAAM,SAAS,QAAwC,CAAC;AAE/D;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,iBACpB,MAAM,UACZ,YAAY,KACnB,QAAQ,kBAAkB,GAAG,SAAS,CAoDxC,CAAC"}
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.parseZipLocalFileHeader = void 0;
|
|
3
|
+
exports.parseZipLocalFileHeader = exports.signature = void 0;
|
|
4
4
|
const offsets = {
|
|
5
|
-
COMPRESSED_SIZE_OFFSET:
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
COMPRESSED_SIZE_OFFSET: 18n,
|
|
6
|
+
UNCOMPRESSED_SIZE_OFFSET: 22n,
|
|
7
|
+
FILE_NAME_LENGTH_OFFSET: 26n,
|
|
8
|
+
EXTRA_FIELD_LENGTH_OFFSET: 28n,
|
|
9
|
+
FILE_NAME_OFFSET: 30n
|
|
9
10
|
};
|
|
10
|
-
|
|
11
|
+
exports.signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
|
|
11
12
|
/**
|
|
12
13
|
* Parses local file header of zip file
|
|
13
14
|
* @param headerOffset - offset in the archive where header starts
|
|
@@ -15,14 +16,32 @@ const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
|
|
|
15
16
|
* @returns Info from the header
|
|
16
17
|
*/
|
|
17
18
|
const parseZipLocalFileHeader = async (headerOffset, buffer) => {
|
|
18
|
-
if (Buffer.from(await buffer.slice(headerOffset, headerOffset +
|
|
19
|
+
if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(exports.signature) !== 0) {
|
|
19
20
|
return Promise.resolve(undefined);
|
|
20
21
|
}
|
|
21
22
|
const fileNameLength = await buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET);
|
|
22
|
-
const fileName = new TextDecoder()
|
|
23
|
+
const fileName = new TextDecoder()
|
|
24
|
+
.decode(await buffer.slice(headerOffset + offsets.FILE_NAME_OFFSET, headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength)))
|
|
25
|
+
.split('\\')
|
|
26
|
+
.join('/');
|
|
23
27
|
const extraFieldLength = await buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET);
|
|
24
|
-
|
|
25
|
-
|
|
28
|
+
let fileDataOffset = headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength);
|
|
29
|
+
let compressedSize = BigInt(await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET)); // add zip 64 logic
|
|
30
|
+
let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + offsets.UNCOMPRESSED_SIZE_OFFSET)); // add zip 64 logic
|
|
31
|
+
const extraOffset = headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength);
|
|
32
|
+
let offsetInZip64Data = 4n;
|
|
33
|
+
// looking for info that might be also be in zip64 extra field
|
|
34
|
+
if (uncompressedSize === BigInt(0xffffffff)) {
|
|
35
|
+
uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
|
|
36
|
+
offsetInZip64Data += 8n;
|
|
37
|
+
}
|
|
38
|
+
if (compressedSize === BigInt(0xffffffff)) {
|
|
39
|
+
compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
|
|
40
|
+
offsetInZip64Data += 8n;
|
|
41
|
+
}
|
|
42
|
+
if (fileDataOffset === BigInt(0xffffffff)) {
|
|
43
|
+
fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
|
|
44
|
+
}
|
|
26
45
|
return {
|
|
27
46
|
fileNameLength,
|
|
28
47
|
fileName,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/i3s",
|
|
3
|
-
"version": "4.0.0-alpha.
|
|
3
|
+
"version": "4.0.0-alpha.15",
|
|
4
4
|
"description": "i3s .",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -33,13 +33,13 @@
|
|
|
33
33
|
"build-worker-node": "esbuild src/workers/i3s-content-worker-node.ts --outfile=dist/i3s-content-worker-node.js --platform=node --target=node16 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@loaders.gl/compression": "4.0.0-alpha.
|
|
37
|
-
"@loaders.gl/draco": "4.0.0-alpha.
|
|
38
|
-
"@loaders.gl/images": "4.0.0-alpha.
|
|
39
|
-
"@loaders.gl/loader-utils": "4.0.0-alpha.
|
|
40
|
-
"@loaders.gl/schema": "4.0.0-alpha.
|
|
41
|
-
"@loaders.gl/textures": "4.0.0-alpha.
|
|
42
|
-
"@loaders.gl/tiles": "4.0.0-alpha.
|
|
36
|
+
"@loaders.gl/compression": "4.0.0-alpha.15",
|
|
37
|
+
"@loaders.gl/draco": "4.0.0-alpha.15",
|
|
38
|
+
"@loaders.gl/images": "4.0.0-alpha.15",
|
|
39
|
+
"@loaders.gl/loader-utils": "4.0.0-alpha.15",
|
|
40
|
+
"@loaders.gl/schema": "4.0.0-alpha.15",
|
|
41
|
+
"@loaders.gl/textures": "4.0.0-alpha.15",
|
|
42
|
+
"@loaders.gl/tiles": "4.0.0-alpha.15",
|
|
43
43
|
"@luma.gl/constants": "^8.5.4",
|
|
44
44
|
"@math.gl/core": "^3.5.1",
|
|
45
45
|
"@math.gl/culling": "^3.5.1",
|
|
@@ -49,5 +49,5 @@
|
|
|
49
49
|
"peerDependencies": {
|
|
50
50
|
"@loaders.gl/core": "^4.0.0-alpha.8"
|
|
51
51
|
},
|
|
52
|
-
"gitHead": "
|
|
52
|
+
"gitHead": "47467ab23c65917e4642730e99b5d005af333c76"
|
|
53
53
|
}
|
package/src/i3s-slpk-loader.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
|
|
2
|
-
import {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';
|
|
2
|
+
import {parseSLPK as parseSLPKFromProvider} from './lib/parsers/parse-slpk/parse-slpk';
|
|
3
|
+
import {DataViewFileProvider} from './lib/parsers/parse-zip/data-view-file-provider';
|
|
3
4
|
|
|
4
5
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
5
6
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
6
7
|
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
7
8
|
|
|
9
|
+
/** options to load data from SLPK */
|
|
8
10
|
export type SLPKLoaderOptions = LoaderOptions & {
|
|
9
11
|
slpk?: {
|
|
12
|
+
/** path inside the slpk archive */
|
|
10
13
|
path?: string;
|
|
14
|
+
/** mode of the path */
|
|
11
15
|
pathMode?: 'http' | 'raw';
|
|
12
16
|
};
|
|
13
17
|
};
|
|
@@ -25,3 +29,17 @@ export const SLPKLoader: LoaderWithParser<Buffer, never, SLPKLoaderOptions> = {
|
|
|
25
29
|
extensions: ['slpk'],
|
|
26
30
|
options: {}
|
|
27
31
|
};
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* returns a single file from the slpk archive
|
|
35
|
+
* @param data slpk archive data
|
|
36
|
+
* @param options options
|
|
37
|
+
* @returns requested file
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
|
|
41
|
+
return (await parseSLPKFromProvider(new DataViewFileProvider(new DataView(data)))).getFile(
|
|
42
|
+
options.slpk?.path ?? '',
|
|
43
|
+
options.slpk?.pathMode
|
|
44
|
+
);
|
|
45
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -33,6 +33,7 @@ export type {
|
|
|
33
33
|
OperationalLayer,
|
|
34
34
|
TextureSetDefinitionFormats
|
|
35
35
|
} from './types';
|
|
36
|
+
export type {FileProvider} from './lib/parsers/parse-zip/file-provider';
|
|
36
37
|
|
|
37
38
|
export {COORDINATE_SYSTEM} from './lib/parsers/constants';
|
|
38
39
|
|
|
@@ -44,4 +45,4 @@ export {I3SBuildingSceneLayerLoader} from './i3s-building-scene-layer-loader';
|
|
|
44
45
|
export {I3SNodePageLoader} from './i3s-node-page-loader';
|
|
45
46
|
export {ArcGisWebSceneLoader} from './arcgis-webscene-loader';
|
|
46
47
|
export {parseZipLocalFileHeader} from './lib/parsers/parse-zip/local-file-header';
|
|
47
|
-
export {
|
|
48
|
+
export {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';
|
|
@@ -1,45 +1,19 @@
|
|
|
1
|
-
import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader';
|
|
2
|
-
import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
|
|
3
1
|
import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
|
|
2
|
+
import {FileProvider} from '../parse-zip/file-provider';
|
|
4
3
|
import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
|
|
4
|
+
import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
|
|
5
5
|
import {SLPKArchive} from './slpk-archieve';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
* @param
|
|
10
|
-
* @
|
|
11
|
-
* @returns one byte from the provided buffer at the provided position
|
|
8
|
+
* Creates slpk file handler from raw file
|
|
9
|
+
* @param fileProvider raw file data
|
|
10
|
+
* @returns slpk file handler
|
|
12
11
|
*/
|
|
13
|
-
const getByteAt = (offset: number, buffer: DataView): number => {
|
|
14
|
-
return buffer.getUint8(buffer.byteOffset + offset);
|
|
15
|
-
};
|
|
16
|
-
|
|
17
|
-
export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
|
|
18
|
-
const archive = new DataView(data);
|
|
19
|
-
const cdFileHeaderSignature = [80, 75, 1, 2];
|
|
20
12
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
getByteAt(archive.byteLength - 2, archive),
|
|
24
|
-
getByteAt(archive.byteLength - 3, archive),
|
|
25
|
-
undefined
|
|
26
|
-
];
|
|
13
|
+
export const parseSLPK = async (fileProvider: FileProvider): Promise<SLPKArchive> => {
|
|
14
|
+
const cdFileHeaderSignature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];
|
|
27
15
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
// looking for the last record in the central directory
|
|
31
|
-
for (let i = archive.byteLength - 4; i > -1; i--) {
|
|
32
|
-
searchWindow[3] = searchWindow[2];
|
|
33
|
-
searchWindow[2] = searchWindow[1];
|
|
34
|
-
searchWindow[1] = searchWindow[0];
|
|
35
|
-
searchWindow[0] = getByteAt(i, archive);
|
|
36
|
-
if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) {
|
|
37
|
-
hashCDOffset = i;
|
|
38
|
-
break;
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
const fileProvider = new DataViewFileProvider(archive);
|
|
16
|
+
const hashCDOffset = await searchFromTheEnd(fileProvider, cdFileHeaderSignature);
|
|
43
17
|
|
|
44
18
|
const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
|
|
45
19
|
|
|
@@ -56,7 +30,7 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions =
|
|
|
56
30
|
}
|
|
57
31
|
|
|
58
32
|
const fileDataOffset = localFileHeader.fileDataOffset;
|
|
59
|
-
const hashFile =
|
|
33
|
+
const hashFile = await fileProvider.slice(
|
|
60
34
|
fileDataOffset,
|
|
61
35
|
fileDataOffset + localFileHeader.compressedSize
|
|
62
36
|
);
|
|
@@ -65,8 +39,5 @@ export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions =
|
|
|
65
39
|
throw new Error('No hash file in slpk');
|
|
66
40
|
}
|
|
67
41
|
|
|
68
|
-
return
|
|
69
|
-
|
|
70
|
-
options.slpk?.pathMode
|
|
71
|
-
);
|
|
72
|
-
}
|
|
42
|
+
return new SLPKArchive(fileProvider, hashFile);
|
|
43
|
+
};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import {FileProvider} from 'modules/i3s/src/lib/parsers/parse-zip/file-provider';
|
|
2
|
+
|
|
3
|
+
/** Description of zip signature type */
|
|
4
|
+
export type ZipSignature = [number, number, number, number];
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* looking for the last occurrence of the provided
|
|
8
|
+
* @param file
|
|
9
|
+
* @param target
|
|
10
|
+
* @returns
|
|
11
|
+
*/
|
|
12
|
+
export const searchFromTheEnd = async (
|
|
13
|
+
file: FileProvider,
|
|
14
|
+
target: ZipSignature
|
|
15
|
+
): Promise<bigint> => {
|
|
16
|
+
const searchWindow = [
|
|
17
|
+
await file.getUint8(file.length - 1n),
|
|
18
|
+
await file.getUint8(file.length - 2n),
|
|
19
|
+
await file.getUint8(file.length - 3n),
|
|
20
|
+
undefined
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
let targetOffset = 0n;
|
|
24
|
+
|
|
25
|
+
// looking for the last record in the central directory
|
|
26
|
+
for (let i = file.length - 4n; i > -1; i--) {
|
|
27
|
+
searchWindow[3] = searchWindow[2];
|
|
28
|
+
searchWindow[2] = searchWindow[1];
|
|
29
|
+
searchWindow[1] = searchWindow[0];
|
|
30
|
+
searchWindow[0] = await file.getUint8(i);
|
|
31
|
+
if (searchWindow.every((val, index) => val === target[index])) {
|
|
32
|
+
targetOffset = i;
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
return targetOffset;
|
|
38
|
+
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import md5 from 'md5';
|
|
2
2
|
import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
|
|
3
|
-
import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
|
|
4
3
|
import {GZipCompression} from '@loaders.gl/compression';
|
|
4
|
+
import {FileProvider} from '../parse-zip/file-provider';
|
|
5
5
|
|
|
6
6
|
/** Element of hash array */
|
|
7
7
|
type HashElement = {
|
|
@@ -12,7 +12,7 @@ type HashElement = {
|
|
|
12
12
|
/**
|
|
13
13
|
* File offset in the archive
|
|
14
14
|
*/
|
|
15
|
-
offset:
|
|
15
|
+
offset: bigint;
|
|
16
16
|
};
|
|
17
17
|
|
|
18
18
|
/** Description of real paths for different file types */
|
|
@@ -55,10 +55,10 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
|
|
|
55
55
|
* Class for handling information about slpk file
|
|
56
56
|
*/
|
|
57
57
|
export class SLPKArchive {
|
|
58
|
-
slpkArchive:
|
|
59
|
-
hashArray: {hash: Buffer; offset:
|
|
60
|
-
constructor(
|
|
61
|
-
this.slpkArchive =
|
|
58
|
+
slpkArchive: FileProvider;
|
|
59
|
+
hashArray: {hash: Buffer; offset: bigint}[];
|
|
60
|
+
constructor(slpkArchive: FileProvider, hashFile: ArrayBuffer) {
|
|
61
|
+
this.slpkArchive = slpkArchive;
|
|
62
62
|
this.hashArray = this.parseHashFile(hashFile);
|
|
63
63
|
}
|
|
64
64
|
|
|
@@ -77,7 +77,7 @@ export class SLPKArchive {
|
|
|
77
77
|
hashFileBuffer.byteOffset + i + 24
|
|
78
78
|
)
|
|
79
79
|
);
|
|
80
|
-
const offset = offsetBuffer.
|
|
80
|
+
const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true);
|
|
81
81
|
hashArray.push({
|
|
82
82
|
hash: Buffer.from(
|
|
83
83
|
hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
|
|
@@ -155,15 +155,12 @@ export class SLPKArchive {
|
|
|
155
155
|
return undefined;
|
|
156
156
|
}
|
|
157
157
|
|
|
158
|
-
const localFileHeader = await parseZipLocalFileHeader(
|
|
159
|
-
this.slpkArchive.byteOffset + fileInfo?.offset,
|
|
160
|
-
new DataViewFileProvider(this.slpkArchive)
|
|
161
|
-
);
|
|
158
|
+
const localFileHeader = await parseZipLocalFileHeader(fileInfo?.offset, this.slpkArchive);
|
|
162
159
|
if (!localFileHeader) {
|
|
163
160
|
return undefined;
|
|
164
161
|
}
|
|
165
162
|
|
|
166
|
-
const compressedFile = this.slpkArchive.
|
|
163
|
+
const compressedFile = this.slpkArchive.slice(
|
|
167
164
|
localFileHeader.fileDataOffset,
|
|
168
165
|
localFileHeader.fileDataOffset + localFileHeader.compressedSize
|
|
169
166
|
);
|
|
@@ -6,17 +6,17 @@ import {FileProvider} from './file-provider';
|
|
|
6
6
|
*/
|
|
7
7
|
export type ZipCDFileHeader = {
|
|
8
8
|
/** Compressed size */
|
|
9
|
-
compressedSize:
|
|
9
|
+
compressedSize: bigint;
|
|
10
10
|
/** Uncompressed size */
|
|
11
|
-
uncompressedSize:
|
|
11
|
+
uncompressedSize: bigint;
|
|
12
12
|
/** File name length */
|
|
13
13
|
fileNameLength: number;
|
|
14
14
|
/** File name */
|
|
15
15
|
fileName: string;
|
|
16
16
|
/** Extra field offset */
|
|
17
|
-
extraOffset:
|
|
17
|
+
extraOffset: bigint;
|
|
18
18
|
/** Relative offset of local file header */
|
|
19
|
-
localHeaderOffset:
|
|
19
|
+
localHeaderOffset: bigint;
|
|
20
20
|
};
|
|
21
21
|
|
|
22
22
|
/**
|
|
@@ -26,22 +26,24 @@ export type ZipCDFileHeader = {
|
|
|
26
26
|
* @returns Info from the header
|
|
27
27
|
*/
|
|
28
28
|
export const parseZipCDFileHeader = async (
|
|
29
|
-
headerOffset:
|
|
29
|
+
headerOffset: bigint,
|
|
30
30
|
buffer: FileProvider
|
|
31
31
|
): Promise<ZipCDFileHeader> => {
|
|
32
32
|
const offsets = {
|
|
33
|
-
CD_COMPRESSED_SIZE_OFFSET:
|
|
34
|
-
CD_UNCOMPRESSED_SIZE_OFFSET:
|
|
35
|
-
CD_FILE_NAME_LENGTH_OFFSET:
|
|
36
|
-
CD_EXTRA_FIELD_LENGTH_OFFSET:
|
|
37
|
-
CD_LOCAL_HEADER_OFFSET_OFFSET:
|
|
38
|
-
CD_FILE_NAME_OFFSET:
|
|
33
|
+
CD_COMPRESSED_SIZE_OFFSET: 20n,
|
|
34
|
+
CD_UNCOMPRESSED_SIZE_OFFSET: 24n,
|
|
35
|
+
CD_FILE_NAME_LENGTH_OFFSET: 28n,
|
|
36
|
+
CD_EXTRA_FIELD_LENGTH_OFFSET: 30n,
|
|
37
|
+
CD_LOCAL_HEADER_OFFSET_OFFSET: 42n,
|
|
38
|
+
CD_FILE_NAME_OFFSET: 46n
|
|
39
39
|
};
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
let compressedSize = BigInt(
|
|
42
|
+
await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET)
|
|
43
|
+
);
|
|
42
44
|
|
|
43
|
-
|
|
44
|
-
headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET
|
|
45
|
+
let uncompressedSize = BigInt(
|
|
46
|
+
await buffer.getUint32(headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET)
|
|
45
47
|
);
|
|
46
48
|
|
|
47
49
|
const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET);
|
|
@@ -49,29 +51,29 @@ export const parseZipCDFileHeader = async (
|
|
|
49
51
|
const fileName = new TextDecoder().decode(
|
|
50
52
|
await buffer.slice(
|
|
51
53
|
headerOffset + offsets.CD_FILE_NAME_OFFSET,
|
|
52
|
-
headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
|
|
54
|
+
headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
|
|
53
55
|
)
|
|
54
56
|
);
|
|
55
57
|
|
|
56
|
-
const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength;
|
|
58
|
+
const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + BigInt(fileNameLength);
|
|
57
59
|
|
|
58
60
|
const oldFormatOffset = await buffer.getUint32(
|
|
59
61
|
headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET
|
|
60
62
|
);
|
|
61
63
|
|
|
62
|
-
let fileDataOffset = oldFormatOffset;
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
fileDataOffset = await buffer.
|
|
64
|
+
let fileDataOffset = BigInt(oldFormatOffset);
|
|
65
|
+
let offsetInZip64Data = 4n;
|
|
66
|
+
// looking for info that might be also be in zip64 extra field
|
|
67
|
+
if (uncompressedSize === BigInt(0xffffffff)) {
|
|
68
|
+
uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
|
|
69
|
+
offsetInZip64Data += 8n;
|
|
70
|
+
}
|
|
71
|
+
if (compressedSize === BigInt(0xffffffff)) {
|
|
72
|
+
compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
|
|
73
|
+
offsetInZip64Data += 8n;
|
|
74
|
+
}
|
|
75
|
+
if (fileDataOffset === BigInt(0xffffffff)) {
|
|
76
|
+
fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
|
|
75
77
|
}
|
|
76
78
|
const localHeaderOffset = fileDataOffset;
|
|
77
79
|
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import {FileProvider} from './file-provider';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Checks if bigint can be converted to number and convert it if possible
|
|
5
|
+
* @param bigint bigint to be converted
|
|
6
|
+
* @returns number
|
|
7
|
+
*/
|
|
8
|
+
const toNumber = (bigint: bigint) => {
|
|
9
|
+
if (bigint > Number.MAX_SAFE_INTEGER) {
|
|
10
|
+
throw new Error('Offset is out of bounds');
|
|
11
|
+
}
|
|
12
|
+
return Number(bigint);
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
/** Provides file data using DataView */
|
|
16
|
+
export class DataViewFileProvider implements FileProvider {
|
|
17
|
+
/** The DataView from which data is provided */
|
|
18
|
+
private file: DataView;
|
|
19
|
+
|
|
20
|
+
constructor(file: DataView) {
|
|
21
|
+
this.file = file;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
|
|
26
|
+
* @param offset The offset, in bytes, from the start of the file where to read the data.
|
|
27
|
+
*/
|
|
28
|
+
async getUint8(offset: bigint): Promise<number> {
|
|
29
|
+
return this.file.getUint8(toNumber(offset));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Gets an unsigned 16-bit intege at the specified byte offset from the start of the file.
|
|
34
|
+
* @param offset The offset, in bytes, from the start of the file where to read the data.
|
|
35
|
+
*/
|
|
36
|
+
async getUint16(offset: bigint): Promise<number> {
|
|
37
|
+
return this.file.getUint16(toNumber(offset), true);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
|
|
42
|
+
* @param offset The offset, in bytes, from the start of the file where to read the data.
|
|
43
|
+
*/
|
|
44
|
+
async getUint32(offset: bigint): Promise<number> {
|
|
45
|
+
return this.file.getUint32(toNumber(offset), true);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Gets an unsigned 64-bit integer at the specified byte offset from the start of the file.
|
|
50
|
+
* @param offset The offset, in bytes, from the start of the file where to read the data.
|
|
51
|
+
*/
|
|
52
|
+
async getBigUint64(offset: bigint): Promise<bigint> {
|
|
53
|
+
return this.file.getBigUint64(toNumber(offset), true);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
|
|
58
|
+
* @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
|
|
59
|
+
* @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
|
|
60
|
+
*/
|
|
61
|
+
async slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
|
|
62
|
+
return this.file.buffer.slice(toNumber(startOffset), toNumber(endOffset));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** the length (in bytes) of the data. */
|
|
66
|
+
get length() {
|
|
67
|
+
return BigInt(this.file.byteLength);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -6,29 +6,35 @@ export interface FileProvider {
|
|
|
6
6
|
* Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
|
|
7
7
|
* @param offset The offset, in bytes, from the start of the file where to read the data.
|
|
8
8
|
*/
|
|
9
|
-
getUint8(offset:
|
|
9
|
+
getUint8(offset: bigint): Promise<number>;
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
|
|
13
13
|
* @param offset The offset, in bytes, from the start of the file where to read the data.
|
|
14
14
|
*/
|
|
15
|
-
getUint16(offset:
|
|
15
|
+
getUint16(offset: bigint): Promise<number>;
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
18
|
* Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
|
|
19
19
|
* @param offset The offset, in bytes, from the file of the view where to read the data.
|
|
20
20
|
*/
|
|
21
|
-
getUint32(offset:
|
|
21
|
+
getUint32(offset: bigint): Promise<number>;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
|
|
25
|
+
* @param offset The offset, in byte, from the file of the view where to read the data.
|
|
26
|
+
*/
|
|
27
|
+
getBigUint64(offset: bigint): Promise<bigint>;
|
|
22
28
|
|
|
23
29
|
/**
|
|
24
30
|
* returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
|
|
25
31
|
* @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
|
|
26
32
|
* @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
|
|
27
33
|
*/
|
|
28
|
-
slice(startOffset:
|
|
34
|
+
slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer>;
|
|
29
35
|
|
|
30
36
|
/**
|
|
31
37
|
* the length (in bytes) of the data.
|
|
32
38
|
*/
|
|
33
|
-
length:
|
|
39
|
+
length: bigint;
|
|
34
40
|
}
|
|
@@ -12,19 +12,20 @@ export type ZipLocalFileHeader = {
|
|
|
12
12
|
/** Extra field length */
|
|
13
13
|
extraFieldLength: number;
|
|
14
14
|
/** Offset of the file data */
|
|
15
|
-
fileDataOffset:
|
|
15
|
+
fileDataOffset: bigint;
|
|
16
16
|
/** Compressed size */
|
|
17
|
-
compressedSize:
|
|
17
|
+
compressedSize: bigint;
|
|
18
18
|
};
|
|
19
19
|
|
|
20
20
|
const offsets = {
|
|
21
|
-
COMPRESSED_SIZE_OFFSET:
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
21
|
+
COMPRESSED_SIZE_OFFSET: 18n,
|
|
22
|
+
UNCOMPRESSED_SIZE_OFFSET: 22n,
|
|
23
|
+
FILE_NAME_LENGTH_OFFSET: 26n,
|
|
24
|
+
EXTRA_FIELD_LENGTH_OFFSET: 28n,
|
|
25
|
+
FILE_NAME_OFFSET: 30n
|
|
25
26
|
};
|
|
26
27
|
|
|
27
|
-
const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
|
|
28
|
+
export const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
|
|
28
29
|
|
|
29
30
|
/**
|
|
30
31
|
* Parses local file header of zip file
|
|
@@ -33,27 +34,52 @@ const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
|
|
|
33
34
|
* @returns Info from the header
|
|
34
35
|
*/
|
|
35
36
|
export const parseZipLocalFileHeader = async (
|
|
36
|
-
headerOffset:
|
|
37
|
+
headerOffset: bigint,
|
|
37
38
|
buffer: FileProvider
|
|
38
39
|
): Promise<ZipLocalFileHeader | undefined> => {
|
|
39
|
-
if (Buffer.from(await buffer.slice(headerOffset, headerOffset +
|
|
40
|
+
if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(signature) !== 0) {
|
|
40
41
|
return Promise.resolve(undefined);
|
|
41
42
|
}
|
|
42
43
|
|
|
43
44
|
const fileNameLength = await buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET);
|
|
44
45
|
|
|
45
|
-
const fileName = new TextDecoder()
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
const fileName = new TextDecoder()
|
|
47
|
+
.decode(
|
|
48
|
+
await buffer.slice(
|
|
49
|
+
headerOffset + offsets.FILE_NAME_OFFSET,
|
|
50
|
+
headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength)
|
|
51
|
+
)
|
|
49
52
|
)
|
|
50
|
-
|
|
53
|
+
.split('\\')
|
|
54
|
+
.join('/');
|
|
51
55
|
const extraFieldLength = await buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET);
|
|
52
56
|
|
|
53
|
-
|
|
54
|
-
headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + extraFieldLength;
|
|
57
|
+
let fileDataOffset =
|
|
58
|
+
headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength);
|
|
55
59
|
|
|
56
|
-
|
|
60
|
+
let compressedSize = BigInt(
|
|
61
|
+
await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET)
|
|
62
|
+
); // add zip 64 logic
|
|
63
|
+
|
|
64
|
+
let uncompressedSize = BigInt(
|
|
65
|
+
await buffer.getUint32(headerOffset + offsets.UNCOMPRESSED_SIZE_OFFSET)
|
|
66
|
+
); // add zip 64 logic
|
|
67
|
+
|
|
68
|
+
const extraOffset = headerOffset + offsets.FILE_NAME_OFFSET + BigInt(fileNameLength);
|
|
69
|
+
|
|
70
|
+
let offsetInZip64Data = 4n;
|
|
71
|
+
// looking for info that might be also be in zip64 extra field
|
|
72
|
+
if (uncompressedSize === BigInt(0xffffffff)) {
|
|
73
|
+
uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
|
|
74
|
+
offsetInZip64Data += 8n;
|
|
75
|
+
}
|
|
76
|
+
if (compressedSize === BigInt(0xffffffff)) {
|
|
77
|
+
compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
|
|
78
|
+
offsetInZip64Data += 8n;
|
|
79
|
+
}
|
|
80
|
+
if (fileDataOffset === BigInt(0xffffffff)) {
|
|
81
|
+
fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
|
|
82
|
+
}
|
|
57
83
|
|
|
58
84
|
return {
|
|
59
85
|
fileNameLength,
|