@loaders.gl/zip 4.1.0-alpha.9 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.dev.js +575 -24
- package/dist/hash-file-utility.d.ts +15 -2
- package/dist/hash-file-utility.d.ts.map +1 -1
- package/dist/hash-file-utility.js +16 -8
- package/dist/hash-file-utility.js.map +1 -1
- package/dist/index.cjs +260 -9
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/parse-zip/end-of-central-directory.d.ts +17 -1
- package/dist/parse-zip/end-of-central-directory.d.ts.map +1 -1
- package/dist/parse-zip/end-of-central-directory.js +121 -2
- package/dist/parse-zip/end-of-central-directory.js.map +1 -1
- package/dist/parse-zip/zip-composition.d.ts +38 -0
- package/dist/parse-zip/zip-composition.d.ts.map +1 -0
- package/dist/parse-zip/zip-composition.js +114 -0
- package/dist/parse-zip/zip-composition.js.map +1 -0
- package/dist/parse-zip/zip64-info-generation.js +2 -2
- package/dist/parse-zip/zip64-info-generation.js.map +1 -1
- package/dist/zip-loader.js +1 -1
- package/dist/zip-loader.js.map +1 -1
- package/dist/zip-writer.js +1 -1
- package/dist/zip-writer.js.map +1 -1
- package/package.json +7 -7
- package/src/hash-file-utility.ts +44 -9
- package/src/index.ts +1 -1
- package/src/parse-zip/end-of-central-directory.ts +241 -3
- package/src/parse-zip/zip-composition.ts +234 -0
- package/src/parse-zip/zip64-info-generation.ts +3 -3
- package/dist/parse-zip/zip-compozition.d.ts +0 -8
- package/dist/parse-zip/zip-compozition.d.ts.map +0 -1
- package/dist/parse-zip/zip-compozition.js +0 -43
- package/dist/parse-zip/zip-compozition.js.map +0 -1
- package/src/parse-zip/zip-compozition.ts +0 -113
package/src/index.ts
CHANGED
|
@@ -19,7 +19,7 @@ export {
|
|
|
19
19
|
} from './parse-zip/local-file-header';
|
|
20
20
|
export {parseEoCDRecord} from './parse-zip/end-of-central-directory';
|
|
21
21
|
export {searchFromTheEnd} from './parse-zip/search-from-the-end';
|
|
22
|
-
export {addOneFile} from './parse-zip/zip-
|
|
22
|
+
export {addOneFile, createZip} from './parse-zip/zip-composition';
|
|
23
23
|
|
|
24
24
|
// export type {HashElement} from './hash-file-utility';
|
|
25
25
|
export {parseHashTable, makeHashTableFromZipHeaders, composeHashFile} from './hash-file-utility';
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
|
|
5
|
+
import {FileProvider, compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
|
|
6
6
|
import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
|
|
7
7
|
import {setFieldToNumber} from './zip64-info-generation';
|
|
8
8
|
|
|
@@ -29,6 +29,17 @@ export type ZipEoCDRecordOffsets = {
|
|
|
29
29
|
zip64EoCDLocatorOffset?: bigint;
|
|
30
30
|
};
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Data to generate End of central directory record
|
|
34
|
+
* according to https://en.wikipedia.org/wiki/ZIP_(file_format)
|
|
35
|
+
*/
|
|
36
|
+
export type ZipEoCDGenerationOptions = {
|
|
37
|
+
recordsNumber: number;
|
|
38
|
+
cdSize: number;
|
|
39
|
+
cdOffset: bigint;
|
|
40
|
+
eoCDStart: bigint;
|
|
41
|
+
};
|
|
42
|
+
|
|
32
43
|
const eoCDSignature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x05, 0x06]);
|
|
33
44
|
const zip64EoCDLocatorSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x07]);
|
|
34
45
|
const zip64EoCDSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x06]);
|
|
@@ -38,11 +49,13 @@ const CD_RECORDS_NUMBER_OFFSET = 8n;
|
|
|
38
49
|
const CD_RECORDS_NUMBER_ON_DISC_OFFSET = 10n;
|
|
39
50
|
const CD_CD_BYTE_SIZE_OFFSET = 12n;
|
|
40
51
|
const CD_START_OFFSET_OFFSET = 16n;
|
|
52
|
+
const CD_COMMENT_OFFSET = 22n;
|
|
41
53
|
const ZIP64_EOCD_START_OFFSET_OFFSET = 8n;
|
|
42
54
|
const ZIP64_CD_RECORDS_NUMBER_OFFSET = 24n;
|
|
43
55
|
const ZIP64_CD_RECORDS_NUMBER_ON_DISC_OFFSET = 32n;
|
|
44
56
|
const ZIP64_CD_CD_BYTE_SIZE_OFFSET = 40n;
|
|
45
57
|
const ZIP64_CD_START_OFFSET_OFFSET = 48n;
|
|
58
|
+
const ZIP64_COMMENT_OFFSET = 56n;
|
|
46
59
|
|
|
47
60
|
/**
|
|
48
61
|
* Parses end of central directory record of zip file
|
|
@@ -94,13 +107,13 @@ export const parseEoCDRecord = async (file: FileProvider): Promise<ZipEoCDRecord
|
|
|
94
107
|
* @param eocdStartOffset EoCD start offset to be updated
|
|
95
108
|
* @returns new EoCD header
|
|
96
109
|
*/
|
|
97
|
-
export
|
|
110
|
+
export function updateEoCD(
|
|
98
111
|
eocdBody: ArrayBuffer,
|
|
99
112
|
oldEoCDOffsets: ZipEoCDRecordOffsets,
|
|
100
113
|
newCDStartOffset: bigint,
|
|
101
114
|
eocdStartOffset: bigint,
|
|
102
115
|
newCDRecordsNumber: bigint
|
|
103
|
-
):
|
|
116
|
+
): Uint8Array {
|
|
104
117
|
const eocd = new DataView(eocdBody);
|
|
105
118
|
|
|
106
119
|
const classicEoCDOffset = oldEoCDOffsets.zip64EoCDOffset
|
|
@@ -152,3 +165,228 @@ export async function updateEoCD(
|
|
|
152
165
|
|
|
153
166
|
return new Uint8Array(eocd.buffer);
|
|
154
167
|
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* generates EoCD record
|
|
171
|
+
* @param options data to generate EoCD record
|
|
172
|
+
* @returns ArrayBuffer with EoCD record
|
|
173
|
+
*/
|
|
174
|
+
export function generateEoCD(options: ZipEoCDGenerationOptions): ArrayBuffer {
|
|
175
|
+
const header = new DataView(new ArrayBuffer(Number(CD_COMMENT_OFFSET)));
|
|
176
|
+
|
|
177
|
+
for (const field of EOCD_FIELDS) {
|
|
178
|
+
setFieldToNumber(
|
|
179
|
+
header,
|
|
180
|
+
field.size,
|
|
181
|
+
field.offset,
|
|
182
|
+
options[field.name ?? ''] ?? field.default ?? 0
|
|
183
|
+
);
|
|
184
|
+
}
|
|
185
|
+
const locator = generateZip64InfoLocator(options);
|
|
186
|
+
|
|
187
|
+
const zip64Record = generateZip64Info(options);
|
|
188
|
+
|
|
189
|
+
return concatenateArrayBuffers(zip64Record, locator, header.buffer);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** standart EoCD fields */
|
|
193
|
+
const EOCD_FIELDS = [
|
|
194
|
+
// End of central directory signature = 0x06054b50
|
|
195
|
+
{
|
|
196
|
+
offset: 0,
|
|
197
|
+
size: 4,
|
|
198
|
+
default: new DataView(eoCDSignature.buffer).getUint32(0, true)
|
|
199
|
+
},
|
|
200
|
+
|
|
201
|
+
// Number of this disk (or 0xffff for ZIP64)
|
|
202
|
+
{
|
|
203
|
+
offset: 4,
|
|
204
|
+
size: 2,
|
|
205
|
+
default: 0
|
|
206
|
+
},
|
|
207
|
+
|
|
208
|
+
// Disk where central directory starts (or 0xffff for ZIP64)
|
|
209
|
+
{
|
|
210
|
+
offset: 6,
|
|
211
|
+
size: 2,
|
|
212
|
+
default: 0
|
|
213
|
+
},
|
|
214
|
+
|
|
215
|
+
// Number of central directory records on this disk (or 0xffff for ZIP64)
|
|
216
|
+
{
|
|
217
|
+
offset: 8,
|
|
218
|
+
size: 2,
|
|
219
|
+
name: 'recordsNumber'
|
|
220
|
+
},
|
|
221
|
+
|
|
222
|
+
// Total number of central directory records (or 0xffff for ZIP64)
|
|
223
|
+
{
|
|
224
|
+
offset: 10,
|
|
225
|
+
size: 2,
|
|
226
|
+
name: 'recordsNumber'
|
|
227
|
+
},
|
|
228
|
+
|
|
229
|
+
// Size of central directory (bytes) (or 0xffffffff for ZIP64)
|
|
230
|
+
{
|
|
231
|
+
offset: 12,
|
|
232
|
+
size: 4,
|
|
233
|
+
name: 'cdSize'
|
|
234
|
+
},
|
|
235
|
+
|
|
236
|
+
// Offset of start of central directory, relative to start of archive (or 0xffffffff for ZIP64)
|
|
237
|
+
{
|
|
238
|
+
offset: 16,
|
|
239
|
+
size: 4,
|
|
240
|
+
name: 'cdOffset'
|
|
241
|
+
},
|
|
242
|
+
|
|
243
|
+
// Comment length (n)
|
|
244
|
+
{
|
|
245
|
+
offset: 20,
|
|
246
|
+
size: 2,
|
|
247
|
+
default: 0
|
|
248
|
+
}
|
|
249
|
+
];
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* generates eocd zip64 record
|
|
253
|
+
* @param options data to generate eocd zip64 record
|
|
254
|
+
* @returns buffer with eocd zip64 record
|
|
255
|
+
*/
|
|
256
|
+
function generateZip64Info(options: ZipEoCDGenerationOptions): ArrayBuffer {
|
|
257
|
+
const record = new DataView(new ArrayBuffer(Number(ZIP64_COMMENT_OFFSET)));
|
|
258
|
+
for (const field of ZIP64_EOCD_FIELDS) {
|
|
259
|
+
setFieldToNumber(
|
|
260
|
+
record,
|
|
261
|
+
field.size,
|
|
262
|
+
field.offset,
|
|
263
|
+
options[field.name ?? ''] ?? field.default ?? 0
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
return record.buffer;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* generates eocd zip64 record locator
|
|
272
|
+
* @param options data to generate eocd zip64 record
|
|
273
|
+
* @returns buffer with eocd zip64 record
|
|
274
|
+
*/
|
|
275
|
+
function generateZip64InfoLocator(options: ZipEoCDGenerationOptions): ArrayBuffer {
|
|
276
|
+
const locator = new DataView(new ArrayBuffer(Number(20)));
|
|
277
|
+
|
|
278
|
+
for (const field of ZIP64_EOCD_LOCATOR_FIELDS) {
|
|
279
|
+
setFieldToNumber(
|
|
280
|
+
locator,
|
|
281
|
+
field.size,
|
|
282
|
+
field.offset,
|
|
283
|
+
options[field.name ?? ''] ?? field.default ?? 0
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return locator.buffer;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** zip64 EoCD record locater fields */
|
|
291
|
+
const ZIP64_EOCD_LOCATOR_FIELDS = [
|
|
292
|
+
// zip64 end of central dir locator signature
|
|
293
|
+
{
|
|
294
|
+
offset: 0,
|
|
295
|
+
size: 4,
|
|
296
|
+
default: new DataView(zip64EoCDLocatorSignature.buffer).getUint32(0, true)
|
|
297
|
+
},
|
|
298
|
+
|
|
299
|
+
// number of the disk with the start of the zip64 end of
|
|
300
|
+
{
|
|
301
|
+
offset: 4,
|
|
302
|
+
size: 4,
|
|
303
|
+
default: 0
|
|
304
|
+
},
|
|
305
|
+
|
|
306
|
+
// start of the zip64 end of central directory
|
|
307
|
+
{
|
|
308
|
+
offset: 8,
|
|
309
|
+
size: 8,
|
|
310
|
+
name: 'eoCDStart'
|
|
311
|
+
},
|
|
312
|
+
|
|
313
|
+
// total number of disks
|
|
314
|
+
{
|
|
315
|
+
offset: 16,
|
|
316
|
+
size: 4,
|
|
317
|
+
default: 1
|
|
318
|
+
}
|
|
319
|
+
];
|
|
320
|
+
|
|
321
|
+
/** zip64 EoCD recodrd fields */
|
|
322
|
+
const ZIP64_EOCD_FIELDS = [
|
|
323
|
+
// End of central directory signature = 0x06064b50
|
|
324
|
+
{
|
|
325
|
+
offset: 0,
|
|
326
|
+
size: 4,
|
|
327
|
+
default: new DataView(zip64EoCDSignature.buffer).getUint32(0, true)
|
|
328
|
+
},
|
|
329
|
+
|
|
330
|
+
// Size of the EOCD64 minus 12
|
|
331
|
+
{
|
|
332
|
+
offset: 4,
|
|
333
|
+
size: 8,
|
|
334
|
+
default: 44
|
|
335
|
+
},
|
|
336
|
+
|
|
337
|
+
// Version made by
|
|
338
|
+
{
|
|
339
|
+
offset: 12,
|
|
340
|
+
size: 2,
|
|
341
|
+
default: 45
|
|
342
|
+
},
|
|
343
|
+
|
|
344
|
+
// Version needed to extract (minimum)
|
|
345
|
+
{
|
|
346
|
+
offset: 14,
|
|
347
|
+
size: 2,
|
|
348
|
+
default: 45
|
|
349
|
+
},
|
|
350
|
+
|
|
351
|
+
// Number of this disk
|
|
352
|
+
{
|
|
353
|
+
offset: 16,
|
|
354
|
+
size: 4,
|
|
355
|
+
default: 0
|
|
356
|
+
},
|
|
357
|
+
|
|
358
|
+
// Disk where central directory starts
|
|
359
|
+
{
|
|
360
|
+
offset: 20,
|
|
361
|
+
size: 4,
|
|
362
|
+
default: 0
|
|
363
|
+
},
|
|
364
|
+
|
|
365
|
+
// Number of central directory records on this disk
|
|
366
|
+
{
|
|
367
|
+
offset: 24,
|
|
368
|
+
size: 8,
|
|
369
|
+
name: 'recordsNumber'
|
|
370
|
+
},
|
|
371
|
+
|
|
372
|
+
// Total number of central directory records
|
|
373
|
+
{
|
|
374
|
+
offset: 32,
|
|
375
|
+
size: 8,
|
|
376
|
+
name: 'recordsNumber'
|
|
377
|
+
},
|
|
378
|
+
|
|
379
|
+
// Size of central directory (bytes)
|
|
380
|
+
{
|
|
381
|
+
offset: 40,
|
|
382
|
+
size: 8,
|
|
383
|
+
name: 'cdSize'
|
|
384
|
+
},
|
|
385
|
+
|
|
386
|
+
// Offset of start of central directory, relative to start of archive
|
|
387
|
+
{
|
|
388
|
+
offset: 48,
|
|
389
|
+
size: 8,
|
|
390
|
+
name: 'cdOffset'
|
|
391
|
+
}
|
|
392
|
+
];
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import {
|
|
2
|
+
FileHandleFile,
|
|
3
|
+
concatenateArrayBuffers,
|
|
4
|
+
path,
|
|
5
|
+
NodeFilesystem,
|
|
6
|
+
NodeFile
|
|
7
|
+
} from '@loaders.gl/loader-utils';
|
|
8
|
+
import {ZipEoCDRecord, generateEoCD, parseEoCDRecord, updateEoCD} from './end-of-central-directory';
|
|
9
|
+
import {CRC32Hash} from '@loaders.gl/crypto';
|
|
10
|
+
import {generateLocalHeader} from './local-file-header';
|
|
11
|
+
import {generateCDHeader} from './cd-file-header';
|
|
12
|
+
import {fetchFile} from '@loaders.gl/core';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* cut off CD and EoCD records from zip file
|
|
16
|
+
* @param provider zip file
|
|
17
|
+
* @returns tuple with three values: CD, EoCD record, EoCD information
|
|
18
|
+
*/
|
|
19
|
+
async function cutTheTailOff(
|
|
20
|
+
provider: FileHandleFile
|
|
21
|
+
): Promise<[ArrayBuffer, ArrayBuffer, ZipEoCDRecord]> {
|
|
22
|
+
// define where the body ends
|
|
23
|
+
const oldEoCDinfo = await parseEoCDRecord(provider);
|
|
24
|
+
const oldCDStartOffset = oldEoCDinfo.cdStartOffset;
|
|
25
|
+
|
|
26
|
+
// define cd length
|
|
27
|
+
const oldCDLength = Number(
|
|
28
|
+
oldEoCDinfo.offsets.zip64EoCDOffset
|
|
29
|
+
? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset
|
|
30
|
+
: oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
// cut off everything except of archieve body
|
|
34
|
+
const zipEnding = await provider.slice(oldCDStartOffset, provider.length);
|
|
35
|
+
await provider.truncate(Number(oldCDStartOffset));
|
|
36
|
+
|
|
37
|
+
// divide cd body and eocd record
|
|
38
|
+
const oldCDBody = zipEnding.slice(0, oldCDLength);
|
|
39
|
+
const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);
|
|
40
|
+
|
|
41
|
+
return [oldCDBody, eocdBody, oldEoCDinfo];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* generates CD and local headers for the file
|
|
46
|
+
* @param fileName name of the file
|
|
47
|
+
* @param fileToAdd buffer with the file
|
|
48
|
+
* @param localFileHeaderOffset offset of the file local header
|
|
49
|
+
* @returns tuple with two values: local header and file body, cd header
|
|
50
|
+
*/
|
|
51
|
+
async function generateFileHeaders(
|
|
52
|
+
fileName: string,
|
|
53
|
+
fileToAdd: ArrayBuffer,
|
|
54
|
+
localFileHeaderOffset: bigint
|
|
55
|
+
): Promise<[Uint8Array, Uint8Array]> {
|
|
56
|
+
// generating CRC32 of the content
|
|
57
|
+
const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);
|
|
58
|
+
|
|
59
|
+
// generate local header for the file
|
|
60
|
+
const newFileLocalHeader = generateLocalHeader({
|
|
61
|
+
crc32: newFileCRC322,
|
|
62
|
+
fileName,
|
|
63
|
+
length: fileToAdd.byteLength
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// generate hash file cd header
|
|
67
|
+
const newFileCDHeader = generateCDHeader({
|
|
68
|
+
crc32: newFileCRC322,
|
|
69
|
+
fileName,
|
|
70
|
+
offset: localFileHeaderOffset,
|
|
71
|
+
length: fileToAdd.byteLength
|
|
72
|
+
});
|
|
73
|
+
return [
|
|
74
|
+
new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)),
|
|
75
|
+
new Uint8Array(newFileCDHeader)
|
|
76
|
+
];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* adds one file in the end of the archieve
|
|
81
|
+
* @param zipUrl path to the file
|
|
82
|
+
* @param fileToAdd new file body
|
|
83
|
+
* @param fileName new file name
|
|
84
|
+
*/
|
|
85
|
+
export async function addOneFile(zipUrl: string, fileToAdd: ArrayBuffer, fileName: string) {
|
|
86
|
+
// init file handler
|
|
87
|
+
const provider = new FileHandleFile(zipUrl, true);
|
|
88
|
+
|
|
89
|
+
const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);
|
|
90
|
+
|
|
91
|
+
// remember the new file local header start offset
|
|
92
|
+
const newFileOffset = provider.length;
|
|
93
|
+
|
|
94
|
+
const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);
|
|
95
|
+
|
|
96
|
+
// write down the file local header
|
|
97
|
+
await provider.append(localPart);
|
|
98
|
+
|
|
99
|
+
// add the file CD header to the CD
|
|
100
|
+
const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);
|
|
101
|
+
|
|
102
|
+
// remember the CD start offset
|
|
103
|
+
const newCDStartOffset = provider.length;
|
|
104
|
+
|
|
105
|
+
// write down new CD
|
|
106
|
+
await provider.append(new Uint8Array(newCDBody));
|
|
107
|
+
|
|
108
|
+
// remember where eocd starts
|
|
109
|
+
const eocdOffset = provider.length;
|
|
110
|
+
|
|
111
|
+
await provider.append(
|
|
112
|
+
updateEoCD(
|
|
113
|
+
eocdBody,
|
|
114
|
+
oldEoCDinfo.offsets,
|
|
115
|
+
newCDStartOffset,
|
|
116
|
+
eocdOffset,
|
|
117
|
+
oldEoCDinfo.cdRecordsNumber + 1n
|
|
118
|
+
)
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* creates zip archive with no compression
|
|
124
|
+
* @note This is a node specific function that works on files
|
|
125
|
+
* @param inputPath path where files for the achive are stored
|
|
126
|
+
* @param outputPath path where zip archive will be placed
|
|
127
|
+
*/
|
|
128
|
+
export async function createZip(
|
|
129
|
+
inputPath: string,
|
|
130
|
+
outputPath: string,
|
|
131
|
+
createAdditionalData?: (
|
|
132
|
+
fileList: {fileName: string; localHeaderOffset: bigint}[]
|
|
133
|
+
) => Promise<{path: string; file: ArrayBuffer}>
|
|
134
|
+
) {
|
|
135
|
+
const fileIterator = getFileIterator(inputPath);
|
|
136
|
+
|
|
137
|
+
const resFile = new NodeFile(outputPath, 'w');
|
|
138
|
+
const fileList: {fileName: string; localHeaderOffset: bigint}[] = [];
|
|
139
|
+
|
|
140
|
+
const cdArray: ArrayBuffer[] = [];
|
|
141
|
+
for await (const file of fileIterator) {
|
|
142
|
+
await addFile(file, resFile, cdArray, fileList);
|
|
143
|
+
}
|
|
144
|
+
if (createAdditionalData) {
|
|
145
|
+
const additionaldata = await createAdditionalData(fileList);
|
|
146
|
+
await addFile(additionaldata, resFile, cdArray);
|
|
147
|
+
}
|
|
148
|
+
const cdOffset = (await resFile.stat()).bigsize;
|
|
149
|
+
const cd = concatenateArrayBuffers(...cdArray);
|
|
150
|
+
await resFile.append(new Uint8Array(cd));
|
|
151
|
+
const eoCDStart = (await resFile.stat()).bigsize;
|
|
152
|
+
await resFile.append(
|
|
153
|
+
new Uint8Array(
|
|
154
|
+
generateEoCD({recordsNumber: cdArray.length, cdSize: cd.byteLength, cdOffset, eoCDStart})
|
|
155
|
+
)
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Adds file to zip parts
|
|
161
|
+
* @param file file to add
|
|
162
|
+
* @param resFile zip file body
|
|
163
|
+
* @param cdArray zip file central directory
|
|
164
|
+
* @param fileList list of file offsets
|
|
165
|
+
*/
|
|
166
|
+
async function addFile(
|
|
167
|
+
file: {path: string; file: ArrayBuffer},
|
|
168
|
+
resFile: NodeFile,
|
|
169
|
+
cdArray: ArrayBuffer[],
|
|
170
|
+
fileList?: {fileName: string; localHeaderOffset: bigint}[]
|
|
171
|
+
) {
|
|
172
|
+
const size = (await resFile.stat()).bigsize;
|
|
173
|
+
fileList?.push({fileName: file.path, localHeaderOffset: size});
|
|
174
|
+
const [localPart, cdHeaderPart] = await generateFileHeaders(file.path, file.file, size);
|
|
175
|
+
await resFile.append(localPart);
|
|
176
|
+
cdArray.push(cdHeaderPart);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* creates iterator providing buffer with file content and path to every file in the input folder
|
|
181
|
+
* @param inputPath path to the input folder
|
|
182
|
+
* @returns iterator
|
|
183
|
+
*/
|
|
184
|
+
export function getFileIterator(
|
|
185
|
+
inputPath: string
|
|
186
|
+
): AsyncIterable<{path: string; file: ArrayBuffer}> {
|
|
187
|
+
async function* iterable() {
|
|
188
|
+
const fileList = await getAllFiles(inputPath);
|
|
189
|
+
for (const filePath of fileList) {
|
|
190
|
+
const file = await (await fetchFile(path.join(inputPath, filePath))).arrayBuffer();
|
|
191
|
+
yield {path: filePath, file};
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return iterable();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* creates a list of relative paths to all files in the provided folder
|
|
199
|
+
* @param basePath path of the root folder
|
|
200
|
+
* @param subfolder relative path from the root folder.
|
|
201
|
+
* @returns list of paths
|
|
202
|
+
*/
|
|
203
|
+
export async function getAllFiles(
|
|
204
|
+
basePath: string,
|
|
205
|
+
subfolder: string = '',
|
|
206
|
+
fsPassed?: NodeFilesystem
|
|
207
|
+
): Promise<string[]> {
|
|
208
|
+
const fs = fsPassed ? fsPassed : new NodeFilesystem({});
|
|
209
|
+
const files = await fs.readdir(pathJoin(basePath, subfolder));
|
|
210
|
+
|
|
211
|
+
const arrayOfFiles: string[] = [];
|
|
212
|
+
|
|
213
|
+
for (const file of files) {
|
|
214
|
+
const fullPath = pathJoin(basePath, subfolder, file);
|
|
215
|
+
if ((await fs.stat(fullPath)).isDirectory) {
|
|
216
|
+
const files = await getAllFiles(basePath, pathJoin(subfolder, file));
|
|
217
|
+
arrayOfFiles.push(...files);
|
|
218
|
+
} else {
|
|
219
|
+
arrayOfFiles.push(pathJoin(subfolder, file));
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
return arrayOfFiles;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* removes empty parts from path array and joins it
|
|
228
|
+
* @param paths paths to join
|
|
229
|
+
* @returns joined path
|
|
230
|
+
*/
|
|
231
|
+
function pathJoin(...paths: string[]): string {
|
|
232
|
+
const resPaths: string[] = paths.filter((val) => val.length);
|
|
233
|
+
return path.join(...resPaths);
|
|
234
|
+
}
|
|
@@ -25,7 +25,7 @@ export function createZip64Info(options: Zip64Options): ArrayBuffer {
|
|
|
25
25
|
|
|
26
26
|
for (const field of ZIP64_FIELDS) {
|
|
27
27
|
if (!optionsToUse[field.name ?? ''] && !field.default) {
|
|
28
|
-
continue;
|
|
28
|
+
continue; // eslint-disable-line no-continue
|
|
29
29
|
}
|
|
30
30
|
const newValue = new DataView(new ArrayBuffer(field.size));
|
|
31
31
|
NUMBER_SETTERS[field.size](newValue, 0, optionsToUse[field.name ?? ''] ?? field.default);
|
|
@@ -62,10 +62,10 @@ export function setFieldToNumber(
|
|
|
62
62
|
/** functions to write values into buffer according to the bytes amount */
|
|
63
63
|
const NUMBER_SETTERS: {[key: number]: NumberSetter} = {
|
|
64
64
|
2: (header, offset, value) => {
|
|
65
|
-
header.setUint16(offset, Number(value), true);
|
|
65
|
+
header.setUint16(offset, Number(value > 0xffff ? 0xffff : value), true);
|
|
66
66
|
},
|
|
67
67
|
4: (header, offset, value) => {
|
|
68
|
-
header.setUint32(offset, Number(value), true);
|
|
68
|
+
header.setUint32(offset, Number(value > 0xffffffff ? 0xffffffff : value), true);
|
|
69
69
|
},
|
|
70
70
|
8: (header, offset, value) => {
|
|
71
71
|
header.setBigUint64(offset, BigInt(value), true);
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* adds one file in the end of the archieve
|
|
3
|
-
* @param zipUrl path to the file
|
|
4
|
-
* @param fileToAdd new file body
|
|
5
|
-
* @param fileName new file name
|
|
6
|
-
*/
|
|
7
|
-
export declare function addOneFile(zipUrl: string, fileToAdd: ArrayBuffer, fileName: string): Promise<void>;
|
|
8
|
-
//# sourceMappingURL=zip-compozition.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"zip-compozition.d.ts","sourceRoot":"","sources":["../../src/parse-zip/zip-compozition.ts"],"names":[],"mappings":"AAuEA;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,iBAmCxF"}
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import { FileHandleFile, concatenateArrayBuffers } from '@loaders.gl/loader-utils';
|
|
2
|
-
import { parseEoCDRecord, updateEoCD } from "./end-of-central-directory.js";
|
|
3
|
-
import { CRC32Hash } from '@loaders.gl/crypto';
|
|
4
|
-
import { generateLocalHeader } from "./local-file-header.js";
|
|
5
|
-
import { generateCDHeader } from "./cd-file-header.js";
|
|
6
|
-
async function cutTheTailOff(provider) {
|
|
7
|
-
const oldEoCDinfo = await parseEoCDRecord(provider);
|
|
8
|
-
const oldCDStartOffset = oldEoCDinfo.cdStartOffset;
|
|
9
|
-
const oldCDLength = Number(oldEoCDinfo.offsets.zip64EoCDOffset ? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset : oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset);
|
|
10
|
-
const zipEnding = await provider.slice(oldCDStartOffset, provider.length);
|
|
11
|
-
await provider.truncate(Number(oldCDStartOffset));
|
|
12
|
-
const oldCDBody = zipEnding.slice(0, oldCDLength);
|
|
13
|
-
const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);
|
|
14
|
-
return [oldCDBody, eocdBody, oldEoCDinfo];
|
|
15
|
-
}
|
|
16
|
-
async function generateFileHeaders(fileName, fileToAdd, localFileHeaderOffset) {
|
|
17
|
-
const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);
|
|
18
|
-
const newFileLocalHeader = generateLocalHeader({
|
|
19
|
-
crc32: newFileCRC322,
|
|
20
|
-
fileName,
|
|
21
|
-
length: fileToAdd.byteLength
|
|
22
|
-
});
|
|
23
|
-
const newFileCDHeader = generateCDHeader({
|
|
24
|
-
crc32: newFileCRC322,
|
|
25
|
-
fileName,
|
|
26
|
-
offset: localFileHeaderOffset,
|
|
27
|
-
length: fileToAdd.byteLength
|
|
28
|
-
});
|
|
29
|
-
return [new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)), new Uint8Array(newFileCDHeader)];
|
|
30
|
-
}
|
|
31
|
-
export async function addOneFile(zipUrl, fileToAdd, fileName) {
|
|
32
|
-
const provider = new FileHandleFile(zipUrl, true);
|
|
33
|
-
const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);
|
|
34
|
-
const newFileOffset = provider.length;
|
|
35
|
-
const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);
|
|
36
|
-
await provider.append(localPart);
|
|
37
|
-
const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);
|
|
38
|
-
const newCDStartOffset = provider.length;
|
|
39
|
-
await provider.append(new Uint8Array(newCDBody));
|
|
40
|
-
const eocdOffset = provider.length;
|
|
41
|
-
await provider.append(await updateEoCD(eocdBody, oldEoCDinfo.offsets, newCDStartOffset, eocdOffset, oldEoCDinfo.cdRecordsNumber + 1n));
|
|
42
|
-
}
|
|
43
|
-
//# sourceMappingURL=zip-compozition.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"zip-compozition.js","names":["FileHandleFile","concatenateArrayBuffers","parseEoCDRecord","updateEoCD","CRC32Hash","generateLocalHeader","generateCDHeader","cutTheTailOff","provider","oldEoCDinfo","oldCDStartOffset","cdStartOffset","oldCDLength","Number","offsets","zip64EoCDOffset","zipEoCDOffset","zipEnding","slice","length","truncate","oldCDBody","eocdBody","byteLength","generateFileHeaders","fileName","fileToAdd","localFileHeaderOffset","newFileCRC322","parseInt","hash","newFileLocalHeader","crc32","newFileCDHeader","offset","Uint8Array","addOneFile","zipUrl","newFileOffset","localPart","cdHeaderPart","append","newCDBody","newCDStartOffset","eocdOffset","cdRecordsNumber"],"sources":["../../src/parse-zip/zip-compozition.ts"],"sourcesContent":["import {FileHandleFile, concatenateArrayBuffers} from '@loaders.gl/loader-utils';\nimport {ZipEoCDRecord, parseEoCDRecord, updateEoCD} from './end-of-central-directory';\nimport {CRC32Hash} from '@loaders.gl/crypto';\nimport {generateLocalHeader} from './local-file-header';\nimport {generateCDHeader} from './cd-file-header';\n\n/**\n * cut off CD and EoCD records from zip file\n * @param provider zip file\n * @returns tuple with three values: CD, EoCD record, EoCD information\n */\nasync function cutTheTailOff(\n provider: FileHandleFile\n): Promise<[ArrayBuffer, ArrayBuffer, ZipEoCDRecord]> {\n // define where the body ends\n const oldEoCDinfo = await parseEoCDRecord(provider);\n const oldCDStartOffset = oldEoCDinfo.cdStartOffset;\n\n // define cd length\n const oldCDLength = Number(\n oldEoCDinfo.offsets.zip64EoCDOffset\n ? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset\n : oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset\n );\n\n // cut off everything except of archieve body\n const zipEnding = await provider.slice(oldCDStartOffset, provider.length);\n await provider.truncate(Number(oldCDStartOffset));\n\n // divide cd body and eocd record\n const oldCDBody = zipEnding.slice(0, oldCDLength);\n const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);\n\n return [oldCDBody, eocdBody, oldEoCDinfo];\n}\n\n/**\n * generates CD and local headers for the file\n * @param fileName name of the file\n * @param fileToAdd buffer with the file\n * @param localFileHeaderOffset offset of the file local header\n * @returns tuple with two values: local header and file body, cd header\n */\nasync function generateFileHeaders(\n fileName: string,\n fileToAdd: ArrayBuffer,\n localFileHeaderOffset: bigint\n): Promise<[Uint8Array, Uint8Array]> {\n // generating CRC32 of the content\n const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);\n\n // generate local header for the file\n const newFileLocalHeader = generateLocalHeader({\n crc32: newFileCRC322,\n fileName,\n length: fileToAdd.byteLength\n });\n\n // generate hash file cd header\n const newFileCDHeader = generateCDHeader({\n crc32: newFileCRC322,\n fileName,\n offset: localFileHeaderOffset,\n length: fileToAdd.byteLength\n });\n return [\n new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)),\n new Uint8Array(newFileCDHeader)\n ];\n}\n\n/**\n * adds one file in the end of the archieve\n * @param zipUrl path to the file\n * @param fileToAdd new file body\n * @param fileName new file name\n */\nexport async function addOneFile(zipUrl: string, fileToAdd: ArrayBuffer, fileName: string) {\n // init file handler\n const provider = new FileHandleFile(zipUrl, true);\n\n const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);\n\n // remember the new file local header start offset\n const newFileOffset = provider.length;\n\n const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);\n\n // write down the file local header\n await provider.append(localPart);\n\n // add the file CD header to the CD\n const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);\n\n // remember the CD start offset\n const newCDStartOffset = provider.length;\n\n // write down new CD\n await provider.append(new Uint8Array(newCDBody));\n\n // remember where eocd starts\n const eocdOffset = provider.length;\n\n await provider.append(\n await updateEoCD(\n eocdBody,\n oldEoCDinfo.offsets,\n newCDStartOffset,\n eocdOffset,\n oldEoCDinfo.cdRecordsNumber + 1n\n )\n );\n}\n"],"mappings":"AAAA,SAAQA,cAAc,EAAEC,uBAAuB,QAAO,0BAA0B;AAAC,SAC1DC,eAAe,EAAEC,UAAU;AAClD,SAAQC,SAAS,QAAO,oBAAoB;AAAC,SACrCC,mBAAmB;AAAA,SACnBC,gBAAgB;AAOxB,eAAeC,aAAaA,CAC1BC,QAAwB,EAC4B;EAEpD,MAAMC,WAAW,GAAG,MAAMP,eAAe,CAACM,QAAQ,CAAC;EACnD,MAAME,gBAAgB,GAAGD,WAAW,CAACE,aAAa;EAGlD,MAAMC,WAAW,GAAGC,MAAM,CACxBJ,WAAW,CAACK,OAAO,CAACC,eAAe,GAC/BN,WAAW,CAACK,OAAO,CAACC,eAAe,GAAGL,gBAAgB,GACtDD,WAAW,CAACK,OAAO,CAACE,aAAa,GAAGN,gBAC1C,CAAC;EAGD,MAAMO,SAAS,GAAG,MAAMT,QAAQ,CAACU,KAAK,CAACR,gBAAgB,EAAEF,QAAQ,CAACW,MAAM,CAAC;EACzE,MAAMX,QAAQ,CAACY,QAAQ,CAACP,MAAM,CAACH,gBAAgB,CAAC,CAAC;EAGjD,MAAMW,SAAS,GAAGJ,SAAS,CAACC,KAAK,CAAC,CAAC,EAAEN,WAAW,CAAC;EACjD,MAAMU,QAAQ,GAAGL,SAAS,CAACC,KAAK,CAACN,WAAW,EAAEK,SAAS,CAACM,UAAU,CAAC;EAEnE,OAAO,CAACF,SAAS,EAAEC,QAAQ,EAAEb,WAAW,CAAC;AAC3C;AASA,eAAee,mBAAmBA,CAChCC,QAAgB,EAChBC,SAAsB,EACtBC,qBAA6B,EACM;EAEnC,MAAMC,aAAa,GAAGC,QAAQ,CAAC,MAAM,IAAIzB,SAAS,CAAC,CAAC,CAAC0B,IAAI,CAACJ,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;EAGhF,MAAMK,kBAAkB,GAAG1B,mBAAmB,CAAC;IAC7C2B,KAAK,EAAEJ,aAAa;IACpBH,QAAQ;IACRN,MAAM,EAAEO,SAAS,CAACH;EACpB,CAAC,CAAC;EAGF,MAAMU,eAAe,GAAG3B,gBAAgB,CAAC;IACvC0B,KAAK,EAAEJ,aAAa;IACpBH,QAAQ;IACRS,MAAM,EAAEP,qBAAqB;IAC7BR,MAAM,EAAEO,SAAS,CAACH;EACpB,CAAC,CAAC;EACF,OAAO,CACL,IAAIY,UAAU,CAAClC,uBAAuB,CAAC8B,kBAAkB,EAAEL,SAAS,CAAC,CAAC,EACtE,IAAIS,UAAU,CAACF,eAAe,CAAC,CAChC;AACH;AAQA,OAAO,eAAeG,UAAUA,CAACC,MAAc,EAAEX,SAAsB,EAAED,QAAgB,EAAE;EAEzF,MAAMjB,QAAQ,GAAG,IAAIR,cAAc,CAACqC,MAAM,EAAE,IAAI,CAAC;EAEjD,MAAM,CAAChB,SAAS,EAAEC,QAAQ,EAAEb,WAAW,CAAC,GAAG,MAAMF,aAAa,CAACC,QAAQ,CAAC;EAGxE,MAAM8B,aAAa,GAAG9B,QAAQ,CAACW,MAAM;EAErC,MAAM,CAACoB,SAAS,EAAEC,YAAY,CAAC,GAAG,MAAMhB,mBAAmB,CAACC,QAAQ,EAAEC,SAAS,EAAEY,aAAa,CAAC;EAG/F,MAAM9B,QAAQ,CAACiC,MAAM,CAACF,SAAS,CAAC;EAGhC,MAAMG,SAAS,GAAGzC,uBAAuB,CAACoB,SAAS,EAAEmB,YAAY,CAAC;EAGlE,MAAMG,gBAAgB,GAAGnC,QAAQ,CAACW,MAAM;EAGxC,MAAMX,QAAQ,CAACiC,MAAM,CAAC,IAAIN,UAAU,CAACO,SAAS,CAAC,CAAC;EAGhD,MAAME,UAAU,GAAGpC,QAAQ,CAACW,MAAM;EAElC,MAAMX,QAAQ,CAACiC,MAAM,CACnB,MAAMtC,UAAU,CACdmB,QAAQ,EACRb,WAAW,CAACK,OAAO,EACnB6B,gBAAgB,EAChBC,UAAU,EACVnC,WAAW,CAACoC,eAAe,GAAG,EAChC,CACF,CAAC;AACH"}
|