@loaders.gl/archives 0.0.0-stage → 5.0.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/dist.dev.js +5101 -0
- package/dist/dist.min.js +9 -0
- package/dist/index.cjs +2982 -0
- package/dist/index.cjs.map +6 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/tar/index.cjs +576 -0
- package/dist/tar/index.cjs.map +6 -0
- package/dist/tar/index.d.ts +8 -0
- package/dist/tar/index.d.ts.map +1 -0
- package/dist/tar/index.js +9 -0
- package/dist/tar/index.js.map +1 -0
- package/dist/tar/lib/header.d.ts +9 -0
- package/dist/tar/lib/header.d.ts.map +1 -0
- package/dist/tar/lib/header.js +76 -0
- package/dist/tar/lib/header.js.map +1 -0
- package/dist/tar/lib/tar.d.ts +29 -0
- package/dist/tar/lib/tar.d.ts.map +1 -0
- package/dist/tar/lib/tar.js +136 -0
- package/dist/tar/lib/tar.js.map +1 -0
- package/dist/tar/lib/types.d.ts +48 -0
- package/dist/tar/lib/types.d.ts.map +1 -0
- package/dist/tar/lib/types.js +5 -0
- package/dist/tar/lib/types.js.map +1 -0
- package/dist/tar/lib/utils.d.ts +23 -0
- package/dist/tar/lib/utils.d.ts.map +1 -0
- package/dist/tar/lib/utils.js +52 -0
- package/dist/tar/lib/utils.js.map +1 -0
- package/dist/tar/tar-archive.d.ts +4 -0
- package/dist/tar/tar-archive.d.ts.map +1 -0
- package/dist/tar/tar-archive.js +97 -0
- package/dist/tar/tar-archive.js.map +1 -0
- package/dist/tar/tar-builder.d.ts +28 -0
- package/dist/tar/tar-builder.d.ts.map +1 -0
- package/dist/tar/tar-builder.js +38 -0
- package/dist/tar/tar-builder.js.map +1 -0
- package/dist/tar/tar-format.d.ts +12 -0
- package/dist/tar/tar-format.d.ts.map +1 -0
- package/dist/tar/tar-format.js +15 -0
- package/dist/tar/tar-format.js.map +1 -0
- package/dist/tar/tar-loader-types.d.ts +15 -0
- package/dist/tar/tar-loader-types.d.ts.map +1 -0
- package/dist/tar/tar-loader-types.js +20 -0
- package/dist/tar/tar-loader-types.js.map +1 -0
- package/dist/tar/tar-loader.cjs +285 -0
- package/dist/tar/tar-loader.cjs.map +6 -0
- package/dist/tar/tar-loader.d.ts +3 -0
- package/dist/tar/tar-loader.d.ts.map +1 -0
- package/dist/tar/tar-loader.js +12 -0
- package/dist/tar/tar-loader.js.map +1 -0
- package/dist/tar/tar-writer.cjs +114 -0
- package/dist/tar/tar-writer.cjs.map +6 -0
- package/dist/tar/tar-writer.d.ts +20 -0
- package/dist/tar/tar-writer.d.ts.map +1 -0
- package/dist/tar/tar-writer.js +94 -0
- package/dist/tar/tar-writer.js.map +1 -0
- package/dist/zip/filesystems/IndexedArchive.d.ts +40 -0
- package/dist/zip/filesystems/IndexedArchive.d.ts.map +1 -0
- package/dist/zip/filesystems/IndexedArchive.js +99 -0
- package/dist/zip/filesystems/IndexedArchive.js.map +1 -0
- package/dist/zip/filesystems/zip-filesystem.d.ts +53 -0
- package/dist/zip/filesystems/zip-filesystem.d.ts.map +1 -0
- package/dist/zip/filesystems/zip-filesystem.js +231 -0
- package/dist/zip/filesystems/zip-filesystem.js.map +1 -0
- package/dist/zip/hash-file-utility.d.ts +20 -0
- package/dist/zip/hash-file-utility.d.ts.map +1 -0
- package/dist/zip/hash-file-utility.js +83 -0
- package/dist/zip/hash-file-utility.js.map +1 -0
- package/dist/zip/index.cjs +4614 -0
- package/dist/zip/index.cjs.map +6 -0
- package/dist/zip/index.d.ts +18 -0
- package/dist/zip/index.d.ts.map +1 -0
- package/dist/zip/index.js +14 -0
- package/dist/zip/index.js.map +1 -0
- package/dist/zip/indexed-zip-writer.cjs +1211 -0
- package/dist/zip/indexed-zip-writer.cjs.map +6 -0
- package/dist/zip/indexed-zip-writer.d.ts +46 -0
- package/dist/zip/indexed-zip-writer.d.ts.map +1 -0
- package/dist/zip/indexed-zip-writer.js +229 -0
- package/dist/zip/indexed-zip-writer.js.map +1 -0
- package/dist/zip/lib/zip-composition.d.ts +37 -0
- package/dist/zip/lib/zip-composition.d.ts.map +1 -0
- package/dist/zip/lib/zip-composition.js +157 -0
- package/dist/zip/lib/zip-composition.js.map +1 -0
- package/dist/zip/lib/zip-entry-metadata.d.ts +34 -0
- package/dist/zip/lib/zip-entry-metadata.d.ts.map +1 -0
- package/dist/zip/lib/zip-entry-metadata.js +132 -0
- package/dist/zip/lib/zip-entry-metadata.js.map +1 -0
- package/dist/zip/lib/zip-serialization.d.ts +36 -0
- package/dist/zip/lib/zip-serialization.d.ts.map +1 -0
- package/dist/zip/lib/zip-serialization.js +148 -0
- package/dist/zip/lib/zip-serialization.js.map +1 -0
- package/dist/zip/lib/zip64-extra-field.d.ts +20 -0
- package/dist/zip/lib/zip64-extra-field.d.ts.map +1 -0
- package/dist/zip/lib/zip64-extra-field.js +53 -0
- package/dist/zip/lib/zip64-extra-field.js.map +1 -0
- package/dist/zip/zip-archive-types.d.ts +56 -0
- package/dist/zip/zip-archive-types.d.ts.map +1 -0
- package/dist/zip/zip-archive-types.js +5 -0
- package/dist/zip/zip-archive-types.js.map +1 -0
- package/dist/zip/zip-archive.d.ts +101 -0
- package/dist/zip/zip-archive.d.ts.map +1 -0
- package/dist/zip/zip-archive.js +593 -0
- package/dist/zip/zip-archive.js.map +1 -0
- package/dist/zip/zip-encode-types.d.ts +41 -0
- package/dist/zip/zip-encode-types.d.ts.map +1 -0
- package/dist/zip/zip-encode-types.js +5 -0
- package/dist/zip/zip-encode-types.js.map +1 -0
- package/dist/zip/zip-encode.cjs +1154 -0
- package/dist/zip/zip-encode.cjs.map +6 -0
- package/dist/zip/zip-encode.d.ts +15 -0
- package/dist/zip/zip-encode.d.ts.map +1 -0
- package/dist/zip/zip-encode.js +312 -0
- package/dist/zip/zip-encode.js.map +1 -0
- package/dist/zip/zip-format.d.ts +14 -0
- package/dist/zip/zip-format.d.ts.map +1 -0
- package/dist/zip/zip-format.js +17 -0
- package/dist/zip/zip-format.js.map +1 -0
- package/dist/zip/zip-loader-types.d.ts +15 -0
- package/dist/zip/zip-loader-types.d.ts.map +1 -0
- package/dist/zip/zip-loader-types.js +20 -0
- package/dist/zip/zip-loader-types.js.map +1 -0
- package/dist/zip/zip-loader.cjs +2681 -0
- package/dist/zip/zip-loader.cjs.map +6 -0
- package/dist/zip/zip-loader.d.ts +3 -0
- package/dist/zip/zip-loader.d.ts.map +1 -0
- package/dist/zip/zip-loader.js +38 -0
- package/dist/zip/zip-loader.js.map +1 -0
- package/dist/zip/zip-writer.cjs +1208 -0
- package/dist/zip/zip-writer.cjs.map +6 -0
- package/dist/zip/zip-writer.d.ts +40 -0
- package/dist/zip/zip-writer.d.ts.map +1 -0
- package/dist/zip/zip-writer.js +48 -0
- package/dist/zip/zip-writer.js.map +1 -0
- package/package.json +88 -4
- package/src/index.ts +14 -0
- package/src/tar/index.ts +11 -0
- package/src/tar/lib/header.ts +82 -0
- package/src/tar/lib/tar.ts +160 -0
- package/src/tar/lib/types.ts +51 -0
- package/src/tar/lib/utils.ts +55 -0
- package/src/tar/tar-archive.ts +98 -0
- package/src/tar/tar-builder.ts +47 -0
- package/src/tar/tar-format.ts +17 -0
- package/src/tar/tar-loader-types.ts +36 -0
- package/src/tar/tar-loader.ts +17 -0
- package/src/tar/tar-writer.ts +102 -0
- package/src/zip/filesystems/IndexedArchive.ts +110 -0
- package/src/zip/filesystems/zip-filesystem.ts +244 -0
- package/src/zip/hash-file-utility.ts +109 -0
- package/src/zip/index.ts +38 -0
- package/src/zip/indexed-zip-writer.ts +345 -0
- package/src/zip/lib/zip-composition.ts +243 -0
- package/src/zip/lib/zip-entry-metadata.ts +157 -0
- package/src/zip/lib/zip-serialization.ts +206 -0
- package/src/zip/lib/zip64-extra-field.ts +84 -0
- package/src/zip/zip-archive-types.ts +64 -0
- package/src/zip/zip-archive.ts +733 -0
- package/src/zip/zip-encode-types.ts +53 -0
- package/src/zip/zip-encode.ts +370 -0
- package/src/zip/zip-format.ts +19 -0
- package/src/zip/zip-loader-types.ts +36 -0
- package/src/zip/zip-loader.ts +43 -0
- package/src/zip/zip-writer.ts +73 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
/** One binary input chunk; typed-array byte offsets and lengths are respected. */
|
|
6
|
+
export type ZipByteChunk = ArrayBuffer | ArrayBufferView;
|
|
7
|
+
|
|
8
|
+
/** Immutable bytes, a Blob, or a lazily consumed byte stream. */
|
|
9
|
+
export type ZipByteSource =
|
|
10
|
+
| ZipByteChunk
|
|
11
|
+
| Blob
|
|
12
|
+
| ReadableStream<ZipByteChunk>
|
|
13
|
+
| AsyncIterable<ZipByteChunk>
|
|
14
|
+
| Iterable<ZipByteChunk>;
|
|
15
|
+
|
|
16
|
+
/** Optional metadata for one named entry. */
|
|
17
|
+
export interface ZipEncodeEntry {
|
|
18
|
+
/** Immutable entry bytes or a byte stream consumed once. */
|
|
19
|
+
readonly data: ZipByteSource;
|
|
20
|
+
/** Compression override; directories always use STORE. */
|
|
21
|
+
readonly compression?: 'store' | 'deflate';
|
|
22
|
+
/** UTC modification time; the default is 1980-01-01. */
|
|
23
|
+
readonly date?: Date;
|
|
24
|
+
/** UTF-8 entry comment. */
|
|
25
|
+
readonly comment?: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Named canonical relative paths; directories end in a slash and have empty payloads. */
|
|
29
|
+
export type ZipEncodeFiles = Readonly<Record<string, ZipByteSource | ZipEncodeEntry>>;
|
|
30
|
+
|
|
31
|
+
/** Independent output/input/metadata budgets; these do not cap total application heap. */
|
|
32
|
+
export interface ZipEncodeOptions {
|
|
33
|
+
/** Default compression; STORE when omitted. */
|
|
34
|
+
readonly compression?: 'store' | 'deflate';
|
|
35
|
+
/** ZIP64 policy; auto uses ZIP64 for unknown compressed sizes and overflowing fields. */
|
|
36
|
+
readonly zip64?: 'auto' | 'always' | 'never';
|
|
37
|
+
/** UTC modification time, fixed at 1980-01-01 when omitted. */
|
|
38
|
+
readonly date?: Date;
|
|
39
|
+
/** CP437 archive comment; unrepresentable characters reject. */
|
|
40
|
+
readonly comment?: string;
|
|
41
|
+
/** Maximum complete archive bytes, including metadata; default 1 TiB. */
|
|
42
|
+
readonly maxArchiveBytes?: bigint;
|
|
43
|
+
/** Maximum uncompressed bytes per entry; default 1 TiB. */
|
|
44
|
+
readonly maxEntryBytes?: bigint;
|
|
45
|
+
/** Maximum retained central-directory bytes; default 32 MiB. */
|
|
46
|
+
readonly maxDirectoryBytes?: number;
|
|
47
|
+
/** Maximum named entries; default 100000. */
|
|
48
|
+
readonly maxEntries?: number;
|
|
49
|
+
/** Maximum input/output chunk bytes; default 64 KiB. */
|
|
50
|
+
readonly chunkBytes?: number;
|
|
51
|
+
/** Cancellation during source reads and between output pulls. */
|
|
52
|
+
readonly signal?: AbortSignal;
|
|
53
|
+
}
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {DeflateCompressor} from '@loaders.gl/compression/deflate-compressor';
|
|
6
|
+
import {CRC32Hash} from '@loaders.gl/crypto';
|
|
7
|
+
import {decodeZipText, encodeUtf8Text, validateZipPath} from './lib/zip-entry-metadata';
|
|
8
|
+
import {
|
|
9
|
+
encodeZipLocalHeader,
|
|
10
|
+
encodeZipCentralHeader,
|
|
11
|
+
encodeZipDescriptor,
|
|
12
|
+
encodeZipEndRecords,
|
|
13
|
+
encodeZipTimestamp,
|
|
14
|
+
type ZipHeaderEntry
|
|
15
|
+
} from './lib/zip-serialization';
|
|
16
|
+
import type {
|
|
17
|
+
ZipByteChunk,
|
|
18
|
+
ZipByteSource,
|
|
19
|
+
ZipEncodeEntry,
|
|
20
|
+
ZipEncodeFiles,
|
|
21
|
+
ZipEncodeOptions
|
|
22
|
+
} from './zip-encode-types';
|
|
23
|
+
|
|
24
|
+
/** Captured, validated settings shared by both encoders. */
|
|
25
|
+
const DEFAULT_OPTIONS = {
|
|
26
|
+
compression: 'store',
|
|
27
|
+
zip64: 'auto',
|
|
28
|
+
maxArchiveBytes: 1024n ** 4n,
|
|
29
|
+
maxEntryBytes: 1024n ** 4n,
|
|
30
|
+
maxDirectoryBytes: 32 * 1024 * 1024,
|
|
31
|
+
maxEntries: 100000,
|
|
32
|
+
chunkBytes: 64 * 1024
|
|
33
|
+
} as const;
|
|
34
|
+
|
|
35
|
+
/** Immutable metadata captured before any source reads. */
|
|
36
|
+
interface PreparedEntry {
|
|
37
|
+
/** Caller-owned source, consumed once. */
|
|
38
|
+
readonly source: ZipByteSource;
|
|
39
|
+
/** Captured byte length for buffers/Blobs; streams have unknown length. */
|
|
40
|
+
readonly size?: bigint;
|
|
41
|
+
/** Header fields before final CRC and compressed size are known. */
|
|
42
|
+
readonly header: ZipHeaderEntry;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Encodes a deterministic ZIP from named buffers, Blobs, or byte streams.
|
|
47
|
+
* Returns one buffer, so callers needing bounded output storage should use encodeZipInBatches.
|
|
48
|
+
* Source bytes must remain immutable. Cancellation/failure discards this encoder's partial output.
|
|
49
|
+
*/
|
|
50
|
+
export async function encodeZip(
|
|
51
|
+
files: ZipEncodeFiles,
|
|
52
|
+
options: ZipEncodeOptions = {}
|
|
53
|
+
): Promise<ArrayBuffer> {
|
|
54
|
+
const chunks: Uint8Array<ArrayBuffer>[] = [];
|
|
55
|
+
let size = 0;
|
|
56
|
+
for await (const chunk of encodeZipInBatches(files, options)) {
|
|
57
|
+
if (size + chunk.length > 0xffffffff)
|
|
58
|
+
throw new RangeError('ZIP buffer capacity exceeded; use encodeZipInBatches');
|
|
59
|
+
chunks.push(chunk);
|
|
60
|
+
size += chunk.length;
|
|
61
|
+
}
|
|
62
|
+
const output = new Uint8Array(size);
|
|
63
|
+
let offset = 0;
|
|
64
|
+
for (const chunk of chunks) {
|
|
65
|
+
output.set(chunk, offset);
|
|
66
|
+
offset += chunk.length;
|
|
67
|
+
}
|
|
68
|
+
return output.buffer;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Streams independent ZIP chunks with backpressure and signed data descriptors.
|
|
73
|
+
* Metadata is retained within maxDirectoryBytes; no complete payload/archive is allocated.
|
|
74
|
+
* Auto ZIP64 reserves 64-bit sizes for streams and DEFLATE, whose encoded size is unknown.
|
|
75
|
+
* Consumers own discarding partial output on cancellation/failure and finalizing storage.
|
|
76
|
+
*/
|
|
77
|
+
export async function* encodeZipInBatches(
|
|
78
|
+
files: ZipEncodeFiles,
|
|
79
|
+
options: ZipEncodeOptions = {}
|
|
80
|
+
): AsyncIterable<Uint8Array<ArrayBuffer>> {
|
|
81
|
+
const settings = {
|
|
82
|
+
compression: options.compression ?? DEFAULT_OPTIONS.compression,
|
|
83
|
+
zip64: options.zip64 ?? DEFAULT_OPTIONS.zip64,
|
|
84
|
+
maxArchiveBytes: options.maxArchiveBytes ?? DEFAULT_OPTIONS.maxArchiveBytes,
|
|
85
|
+
maxEntryBytes: options.maxEntryBytes ?? DEFAULT_OPTIONS.maxEntryBytes,
|
|
86
|
+
maxDirectoryBytes: options.maxDirectoryBytes ?? DEFAULT_OPTIONS.maxDirectoryBytes,
|
|
87
|
+
maxEntries: options.maxEntries ?? DEFAULT_OPTIONS.maxEntries,
|
|
88
|
+
chunkBytes: options.chunkBytes ?? DEFAULT_OPTIONS.chunkBytes
|
|
89
|
+
};
|
|
90
|
+
validateSettings(settings);
|
|
91
|
+
const signal = options.signal;
|
|
92
|
+
signal?.throwIfAborted();
|
|
93
|
+
const comment = encodeArchiveComment(options.comment ?? '');
|
|
94
|
+
const date = options.date ?? new Date('1980-01-01T00:00:00Z');
|
|
95
|
+
encodeZipTimestamp(date);
|
|
96
|
+
const paths = Object.keys(files).sort();
|
|
97
|
+
if (paths.length > settings.maxEntries) throw new RangeError('ZIP entry count limit exceeded');
|
|
98
|
+
if (settings.zip64 === 'never' && paths.length > 0xffff)
|
|
99
|
+
throw new RangeError('ZIP32 entry count capacity exceeded');
|
|
100
|
+
let minimumDirectoryBytes = 0;
|
|
101
|
+
const entries: PreparedEntry[] = paths.map(path => {
|
|
102
|
+
validateZipPath(path);
|
|
103
|
+
const value = files[path];
|
|
104
|
+
const entry: ZipEncodeEntry =
|
|
105
|
+
value && typeof value === 'object' && 'data' in value
|
|
106
|
+
? (value as ZipEncodeEntry)
|
|
107
|
+
: {data: value as ZipByteSource};
|
|
108
|
+
const size = getSourceSize(entry.data);
|
|
109
|
+
const directory = path.endsWith('/');
|
|
110
|
+
const compression = entry.compression ?? settings.compression;
|
|
111
|
+
if (compression !== 'store' && compression !== 'deflate')
|
|
112
|
+
throw new TypeError('Unsupported ZIP compression');
|
|
113
|
+
if (size !== undefined && (size > settings.maxEntryBytes || (directory && size !== 0n)))
|
|
114
|
+
throw new RangeError('ZIP entry byte limit or nonempty directory');
|
|
115
|
+
if (path.length > 0xffff || (entry.comment?.length ?? 0) > 0xffff)
|
|
116
|
+
throw new RangeError('ZIP name/comment exceeds header capacity');
|
|
117
|
+
if (entry.comment !== undefined && typeof entry.comment !== 'string')
|
|
118
|
+
throw new TypeError('ZIP entry comment must be a string');
|
|
119
|
+
const nameBytes = encodeUtf8Text(path);
|
|
120
|
+
const commentBytes = encodeUtf8Text(entry.comment ?? '');
|
|
121
|
+
encodeZipTimestamp(entry.date ?? date);
|
|
122
|
+
const timestamp = new Date((entry.date ?? date).getTime());
|
|
123
|
+
const zip64 =
|
|
124
|
+
settings.zip64 === 'always' ||
|
|
125
|
+
(settings.zip64 === 'auto' &&
|
|
126
|
+
(size === undefined || size >= 0xffffffffn || (!directory && compression === 'deflate')));
|
|
127
|
+
const header: ZipHeaderEntry = {
|
|
128
|
+
nameBytes,
|
|
129
|
+
commentBytes,
|
|
130
|
+
date: timestamp,
|
|
131
|
+
method: directory || compression === 'store' ? 0 : 8,
|
|
132
|
+
flags: 0x808,
|
|
133
|
+
crc32: 0,
|
|
134
|
+
size: size ?? 0n,
|
|
135
|
+
compressedSize: 0n,
|
|
136
|
+
localHeaderOffset: 0n,
|
|
137
|
+
zip64,
|
|
138
|
+
directory
|
|
139
|
+
};
|
|
140
|
+
// Validate name/comment capacities and ZIP32 fields before any source is consumed.
|
|
141
|
+
encodeZipCentralHeader(header);
|
|
142
|
+
minimumDirectoryBytes += 46 + nameBytes.length + commentBytes.length + (zip64 ? 28 : 0);
|
|
143
|
+
if (minimumDirectoryBytes > settings.maxDirectoryBytes)
|
|
144
|
+
throw new RangeError('ZIP directory byte limit exceeded');
|
|
145
|
+
return {source: entry.data, size, header};
|
|
146
|
+
});
|
|
147
|
+
let outputBytes = 0n;
|
|
148
|
+
let directoryBytes = 0;
|
|
149
|
+
let usedZip64 = settings.zip64 === 'always';
|
|
150
|
+
const directory: Uint8Array<ArrayBuffer>[] = [];
|
|
151
|
+
/** Copies bounded independent output chunks; transferred buffers cannot detach retained metadata. */
|
|
152
|
+
function* emit(bytes: Uint8Array): Iterable<Uint8Array<ArrayBuffer>> {
|
|
153
|
+
for (let offset = 0; offset < bytes.length; offset += settings.chunkBytes) {
|
|
154
|
+
signal?.throwIfAborted();
|
|
155
|
+
const chunk = bytes.slice(offset, offset + settings.chunkBytes);
|
|
156
|
+
if (outputBytes + BigInt(chunk.length) > settings.maxArchiveBytes)
|
|
157
|
+
throw new RangeError('ZIP archive output byte limit exceeded');
|
|
158
|
+
outputBytes += BigInt(chunk.length);
|
|
159
|
+
yield chunk;
|
|
160
|
+
signal?.throwIfAborted();
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
for (const entry of entries) {
|
|
164
|
+
signal?.throwIfAborted();
|
|
165
|
+
const zip64 = entry.header.zip64 || (settings.zip64 === 'auto' && outputBytes >= 0xffffffffn);
|
|
166
|
+
const header = {...entry.header, localHeaderOffset: outputBytes, zip64};
|
|
167
|
+
usedZip64 ||= zip64;
|
|
168
|
+
yield* emit(encodeZipLocalHeader(header));
|
|
169
|
+
let size = 0n;
|
|
170
|
+
let compressedSize = 0n;
|
|
171
|
+
let checksum = 0;
|
|
172
|
+
/** Applies entry limits to each input block before CRC/compression. */
|
|
173
|
+
async function* measuredInput(): AsyncIterable<ArrayBuffer> {
|
|
174
|
+
for await (const block of readSource(entry.source, settings.chunkBytes, signal)) {
|
|
175
|
+
size += BigInt(block.byteLength);
|
|
176
|
+
if (size > settings.maxEntryBytes || (header.directory && size !== 0n))
|
|
177
|
+
throw new RangeError('ZIP entry byte limit or nonempty directory');
|
|
178
|
+
if (!zip64 && size >= 0xffffffffn) throw new RangeError('ZIP32 entry capacity exceeded');
|
|
179
|
+
yield block;
|
|
180
|
+
}
|
|
181
|
+
if (entry.size !== undefined && size !== entry.size)
|
|
182
|
+
throw new TypeError('ZIP source size changed during encoding');
|
|
183
|
+
}
|
|
184
|
+
const hash = new CRC32Hash({
|
|
185
|
+
crypto: {
|
|
186
|
+
/** Captures the CRC after all decoded input bytes have been consumed. */
|
|
187
|
+
onEnd: (result: {/** Hex-encoded CRC digest. */ hash: string}) => {
|
|
188
|
+
checksum = Number.parseInt(result.hash, 16);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
});
|
|
192
|
+
const input = hash.hashBatches(measuredInput(), 'hex');
|
|
193
|
+
const payload =
|
|
194
|
+
header.method === 8 ? new DeflateCompressor({raw: true}).compressBatches(input) : input;
|
|
195
|
+
for await (const block of payload) {
|
|
196
|
+
signal?.throwIfAborted();
|
|
197
|
+
compressedSize += BigInt(block.byteLength);
|
|
198
|
+
if (!zip64 && compressedSize >= 0xffffffffn)
|
|
199
|
+
throw new RangeError('ZIP32 compressed entry capacity exceeded');
|
|
200
|
+
yield* emit(new Uint8Array(block));
|
|
201
|
+
}
|
|
202
|
+
const finalHeader = {...header, size, compressedSize, crc32: checksum};
|
|
203
|
+
yield* emit(encodeZipDescriptor(finalHeader));
|
|
204
|
+
const centralHeader = encodeZipCentralHeader(finalHeader);
|
|
205
|
+
directoryBytes += centralHeader.length;
|
|
206
|
+
if (directoryBytes > settings.maxDirectoryBytes)
|
|
207
|
+
throw new RangeError('ZIP directory byte limit exceeded');
|
|
208
|
+
directory.push(centralHeader);
|
|
209
|
+
}
|
|
210
|
+
const directoryOffset = outputBytes;
|
|
211
|
+
const zip64 =
|
|
212
|
+
usedZip64 ||
|
|
213
|
+
(settings.zip64 === 'auto' &&
|
|
214
|
+
(paths.length >= 0xffff || directoryBytes >= 0xffffffff || directoryOffset >= 0xffffffffn));
|
|
215
|
+
const footer = encodeZipEndRecords(
|
|
216
|
+
BigInt(paths.length),
|
|
217
|
+
BigInt(directoryBytes),
|
|
218
|
+
directoryOffset,
|
|
219
|
+
zip64,
|
|
220
|
+
comment
|
|
221
|
+
);
|
|
222
|
+
for (const header of directory) yield* emit(header);
|
|
223
|
+
yield* emit(footer);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Rejects invalid runtime option values, including JavaScript callers bypassing the types. */
|
|
227
|
+
function validateSettings(
|
|
228
|
+
settings: Required<Omit<ZipEncodeOptions, 'date' | 'comment' | 'signal'>>
|
|
229
|
+
): void {
|
|
230
|
+
if (settings.compression !== 'store' && settings.compression !== 'deflate')
|
|
231
|
+
throw new TypeError('Unsupported ZIP compression');
|
|
232
|
+
if (!['auto', 'always', 'never'].includes(settings.zip64))
|
|
233
|
+
throw new TypeError('Invalid ZIP64 policy');
|
|
234
|
+
for (const [name, value] of Object.entries(settings)) {
|
|
235
|
+
if (name === 'compression' || name === 'zip64') continue;
|
|
236
|
+
const bigintLimit = name === 'maxArchiveBytes' || name === 'maxEntryBytes';
|
|
237
|
+
if (
|
|
238
|
+
bigintLimit
|
|
239
|
+
? typeof value !== 'bigint' || value < 0n
|
|
240
|
+
: typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1
|
|
241
|
+
)
|
|
242
|
+
throw new RangeError(`Invalid ZIP ${name} limit`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Returns native buffer/Blob sizes; iterator inputs deliberately remain unknown. */
|
|
247
|
+
function getSourceSize(source: ZipByteSource): bigint | undefined {
|
|
248
|
+
if (source instanceof ArrayBuffer || ArrayBuffer.isView(source)) return BigInt(source.byteLength);
|
|
249
|
+
if (source instanceof Blob)
|
|
250
|
+
return BigInt(Object.getOwnPropertyDescriptor(Blob.prototype, 'size')!.get!.call(source));
|
|
251
|
+
if (
|
|
252
|
+
!source ||
|
|
253
|
+
typeof source !== 'object' ||
|
|
254
|
+
(!(source instanceof ReadableStream) &&
|
|
255
|
+
!(Symbol.asyncIterator in source) &&
|
|
256
|
+
!(Symbol.iterator in source))
|
|
257
|
+
)
|
|
258
|
+
throw new TypeError('ZIP source must be bytes, a Blob or a byte stream');
|
|
259
|
+
return undefined;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Copies input slices in bounded blocks, respecting byte offsets and closing interrupted streams. */
|
|
263
|
+
async function* readSource(
|
|
264
|
+
source: ZipByteSource,
|
|
265
|
+
chunkBytes: number,
|
|
266
|
+
signal?: AbortSignal
|
|
267
|
+
): AsyncIterable<ArrayBuffer> {
|
|
268
|
+
if (source instanceof Blob) {
|
|
269
|
+
const size = Number(getSourceSize(source)!);
|
|
270
|
+
for (let offset = 0; offset < size; offset += chunkBytes) {
|
|
271
|
+
signal?.throwIfAborted();
|
|
272
|
+
const blob = Blob.prototype.slice.call(source, offset, Math.min(size, offset + chunkBytes));
|
|
273
|
+
const block = await awaitWithSignal(Blob.prototype.arrayBuffer.call(blob), signal);
|
|
274
|
+
signal?.throwIfAborted();
|
|
275
|
+
if (block.byteLength !== Math.min(chunkBytes, size - offset))
|
|
276
|
+
throw new TypeError('ZIP Blob returned a truncated block');
|
|
277
|
+
yield block;
|
|
278
|
+
}
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
if (source instanceof ArrayBuffer || ArrayBuffer.isView(source)) {
|
|
282
|
+
yield* splitByteChunk(source, chunkBytes, signal);
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
const reader = source instanceof ReadableStream ? source.getReader() : undefined;
|
|
286
|
+
const iterator = reader
|
|
287
|
+
? undefined
|
|
288
|
+
: Symbol.asyncIterator in source
|
|
289
|
+
? source[Symbol.asyncIterator]()
|
|
290
|
+
: (source as Iterable<ZipByteChunk>)[Symbol.iterator]();
|
|
291
|
+
let done = false;
|
|
292
|
+
try {
|
|
293
|
+
while (!done) {
|
|
294
|
+
signal?.throwIfAborted();
|
|
295
|
+
const result = await awaitWithSignal(
|
|
296
|
+
reader ? reader.read() : Promise.resolve(iterator!.next()),
|
|
297
|
+
signal
|
|
298
|
+
);
|
|
299
|
+
done = Boolean(result.done);
|
|
300
|
+
if (!done) yield* splitByteChunk(result.value!, chunkBytes, signal);
|
|
301
|
+
}
|
|
302
|
+
} finally {
|
|
303
|
+
if (reader) {
|
|
304
|
+
if (!done) void reader.cancel(signal?.reason).catch(() => {});
|
|
305
|
+
reader.releaseLock();
|
|
306
|
+
} else if (!done && iterator?.return) {
|
|
307
|
+
void Promise.resolve()
|
|
308
|
+
.then(() => iterator.return!())
|
|
309
|
+
.catch(() => {});
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/** Validates binary chunks and copies views before the source or consumer can reuse them. */
|
|
315
|
+
function* splitByteChunk(
|
|
316
|
+
chunk: ZipByteChunk,
|
|
317
|
+
chunkBytes: number,
|
|
318
|
+
signal?: AbortSignal
|
|
319
|
+
): Iterable<ArrayBuffer> {
|
|
320
|
+
if (!(chunk instanceof ArrayBuffer) && !ArrayBuffer.isView(chunk))
|
|
321
|
+
throw new TypeError('ZIP stream chunks must be binary');
|
|
322
|
+
const bytes =
|
|
323
|
+
chunk instanceof ArrayBuffer
|
|
324
|
+
? new Uint8Array(chunk)
|
|
325
|
+
: new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
|
326
|
+
const length = bytes.length;
|
|
327
|
+
for (let offset = 0; offset < length; offset += chunkBytes) {
|
|
328
|
+
signal?.throwIfAborted();
|
|
329
|
+
if (bytes.byteLength !== length) throw new TypeError('ZIP source size changed during encoding');
|
|
330
|
+
yield bytes.slice(offset, offset + chunkBytes).buffer;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Observes cancellation while an asynchronous source pull is pending, without starting another pull. */
|
|
335
|
+
function awaitWithSignal<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
336
|
+
if (!signal) return promise;
|
|
337
|
+
return new Promise((resolve, reject) => {
|
|
338
|
+
/** Removes the listener and settles this one source operation. */
|
|
339
|
+
const finish = (settle: () => void) => {
|
|
340
|
+
signal.removeEventListener('abort', abort);
|
|
341
|
+
settle();
|
|
342
|
+
};
|
|
343
|
+
/** Rejects immediately; source cleanup is owned by readSource. */
|
|
344
|
+
const abort = () => finish(() => reject(signal.reason));
|
|
345
|
+
signal.addEventListener('abort', abort, {once: true});
|
|
346
|
+
promise.then(
|
|
347
|
+
value => finish(() => resolve(value)),
|
|
348
|
+
error => finish(() => reject(error))
|
|
349
|
+
);
|
|
350
|
+
if (signal.aborted) abort();
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Encodes archive comments in CP437, which has no archive-level UTF-8 flag. */
|
|
355
|
+
function encodeArchiveComment(comment: string): Uint8Array<ArrayBuffer> {
|
|
356
|
+
if (typeof comment !== 'string') throw new TypeError('ZIP archive comment must be a string');
|
|
357
|
+
const codepoints = [...comment];
|
|
358
|
+
if (codepoints.length > 0xffff) throw new RangeError('ZIP archive comment exceeds 65535 bytes');
|
|
359
|
+
const alphabet = decodeZipText(
|
|
360
|
+
Uint8Array.from({length: 256}, (_, index) => index),
|
|
361
|
+
false
|
|
362
|
+
);
|
|
363
|
+
const bytes = new Uint8Array(codepoints.length);
|
|
364
|
+
codepoints.forEach((character, index) => {
|
|
365
|
+
const position = alphabet.indexOf(character);
|
|
366
|
+
if (position < 0) throw new TypeError('ZIP archive comment is not representable in CP437');
|
|
367
|
+
bytes[index] = position;
|
|
368
|
+
});
|
|
369
|
+
return bytes;
|
|
370
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Format} from '@loaders.gl/core/loader-utils';
|
|
6
|
+
|
|
7
|
+
/** ZIP archive format. */
|
|
8
|
+
export const ZipFormat = {
|
|
9
|
+
name: 'Zip Archive',
|
|
10
|
+
id: 'zip',
|
|
11
|
+
module: 'archives',
|
|
12
|
+
encoding: 'zip',
|
|
13
|
+
format: 'zip',
|
|
14
|
+
extensions: ['zip'],
|
|
15
|
+
mimeTypes: ['application/zip'],
|
|
16
|
+
category: 'archive',
|
|
17
|
+
binary: true,
|
|
18
|
+
tests: ['PK']
|
|
19
|
+
} as const satisfies Format;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Loader, LoaderOptions} from '@loaders.gl/core/loader-utils';
|
|
6
|
+
import type {ZipArchiveOptions} from './zip-archive-types';
|
|
7
|
+
import {ZipFormat} from './zip-format';
|
|
8
|
+
|
|
9
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
10
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
11
|
+
|
|
12
|
+
/** Complete decoded files, excluding directory entries. */
|
|
13
|
+
export type ZipFileMap = Record<string, ArrayBuffer>;
|
|
14
|
+
|
|
15
|
+
/** Native ZIP validation and complete file-map allocation controls. */
|
|
16
|
+
export type ZipLoaderOptions = LoaderOptions & {
|
|
17
|
+
/** Container limits and cancellation, shared with openZipArchive. */
|
|
18
|
+
zip?: ZipArchiveOptions & {
|
|
19
|
+
/** Maximum sum of decoded file bytes retained in the map; default 64 MiB. */
|
|
20
|
+
readonly maxTotalDecodedBytes?: bigint;
|
|
21
|
+
};
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/** Metadata-only ZIP loader; complete reads validate sizes and CRCs. */
|
|
25
|
+
export const ZipLoader = {
|
|
26
|
+
...ZipFormat,
|
|
27
|
+
version: VERSION,
|
|
28
|
+
dataType: null as unknown as ZipFileMap,
|
|
29
|
+
batchType: null as unknown as never,
|
|
30
|
+
options: {zip: {maxTotalDecodedBytes: 64n * 1024n * 1024n}},
|
|
31
|
+
/** Loads the parser implementation through its installed package subpath. */
|
|
32
|
+
async preload() {
|
|
33
|
+
const {ZipLoaderWithParser} = await import('@loaders.gl/archives/zip/zip-loader');
|
|
34
|
+
return ZipLoaderWithParser;
|
|
35
|
+
}
|
|
36
|
+
} as const satisfies Loader<ZipFileMap, never, ZipLoaderOptions>;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {LoaderWithParser} from '@loaders.gl/core/loader-utils';
|
|
6
|
+
import {ZipLoader} from './zip-loader-types';
|
|
7
|
+
import type {ZipFileMap, ZipLoaderOptions} from './zip-loader-types';
|
|
8
|
+
import type {ZipArchiveInput, ZipEntry} from './zip-archive-types';
|
|
9
|
+
import {openZipArchive} from './zip-archive';
|
|
10
|
+
|
|
11
|
+
const {preload: _preload, ...metadata} = ZipLoader;
|
|
12
|
+
|
|
13
|
+
/** Parser-bearing ZIP loader, available through @loaders.gl/archives/zip/zip-loader. */
|
|
14
|
+
export const ZipLoaderWithParser = {
|
|
15
|
+
...metadata,
|
|
16
|
+
parse: parseZip
|
|
17
|
+
} as const satisfies LoaderWithParser<ZipFileMap, never, ZipLoaderOptions>;
|
|
18
|
+
|
|
19
|
+
/** Validates retained output size before decoding files, and closes on every path. */
|
|
20
|
+
async function parseZip(
|
|
21
|
+
input: ZipArchiveInput,
|
|
22
|
+
options: ZipLoaderOptions = {}
|
|
23
|
+
): Promise<ZipFileMap> {
|
|
24
|
+
const maximum = options.zip?.maxTotalDecodedBytes ?? ZipLoader.options.zip.maxTotalDecodedBytes;
|
|
25
|
+
if (typeof maximum !== 'bigint' || maximum < 0n)
|
|
26
|
+
throw new RangeError('Invalid ZIP total decoded-byte limit');
|
|
27
|
+
const archive = await openZipArchive(input, options.zip);
|
|
28
|
+
try {
|
|
29
|
+
const entries: ZipEntry[] = [];
|
|
30
|
+
let totalBytes = 0n;
|
|
31
|
+
for await (const entry of archive.entries()) {
|
|
32
|
+
if (entry.directory) continue;
|
|
33
|
+
totalBytes += entry.size;
|
|
34
|
+
if (totalBytes > maximum) throw new RangeError('ZIP total decoded-byte limit exceeded');
|
|
35
|
+
entries.push(entry);
|
|
36
|
+
}
|
|
37
|
+
const files: ZipFileMap = Object.create(null);
|
|
38
|
+
for (const entry of entries) files[entry.path] = await archive.read(entry);
|
|
39
|
+
return files;
|
|
40
|
+
} finally {
|
|
41
|
+
await archive.close();
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/core/loader-utils';
|
|
6
|
+
import {ZipFormat} from './zip-format';
|
|
7
|
+
import {encodeZip} from './zip-encode';
|
|
8
|
+
import type {
|
|
9
|
+
ZipEncodeFiles,
|
|
10
|
+
ZipEncodeOptions,
|
|
11
|
+
ZipByteSource,
|
|
12
|
+
ZipEncodeEntry
|
|
13
|
+
} from './zip-encode-types';
|
|
14
|
+
|
|
15
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
16
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
17
|
+
|
|
18
|
+
/** Named binary sources or UTF-8 strings accepted by the buffered writer. */
|
|
19
|
+
export type ZipWriterFiles = Readonly<Record<string, string | ZipByteSource | ZipEncodeEntry>>;
|
|
20
|
+
|
|
21
|
+
/** Native encoding controls and optional directory/completion notifications. */
|
|
22
|
+
export type ZipWriterOptions = WriterOptions & {
|
|
23
|
+
/** Native ZIP options; no third-party passthrough is supported. */
|
|
24
|
+
zip?: ZipEncodeOptions & {
|
|
25
|
+
/** Reports 0 before encoding and 100 after successful completion. */
|
|
26
|
+
readonly onUpdate?: (metadata: {percent: number}) => void;
|
|
27
|
+
/** Emits parent directory entries for nested file keys; default false. */
|
|
28
|
+
readonly createFolders?: boolean;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/** Buffered native ZIP32/ZIP64 writer with STORE as the default compression. */
|
|
33
|
+
export const ZipWriter = {
|
|
34
|
+
...ZipFormat,
|
|
35
|
+
version: VERSION,
|
|
36
|
+
options: {zip: {createFolders: false}},
|
|
37
|
+
encode: encodeZipFiles
|
|
38
|
+
} as const satisfies WriterWithEncoder<ZipWriterFiles, never, ZipWriterOptions>;
|
|
39
|
+
|
|
40
|
+
/** Converts string values and directory options before delegating to the native encoder. */
|
|
41
|
+
async function encodeZipFiles(
|
|
42
|
+
input: ZipWriterFiles,
|
|
43
|
+
options: ZipWriterOptions = {}
|
|
44
|
+
): Promise<ArrayBuffer> {
|
|
45
|
+
options.zip?.signal?.throwIfAborted();
|
|
46
|
+
if ('jszip' in options) throw new TypeError('jszip options were removed; use typed zip options');
|
|
47
|
+
const files: Record<string, ZipEncodeFiles[string]> = Object.create(null);
|
|
48
|
+
const maximum = options.zip?.maxEntries ?? 100000;
|
|
49
|
+
if (!Number.isSafeInteger(maximum) || maximum < 1)
|
|
50
|
+
throw new RangeError('Invalid ZIP entry limit');
|
|
51
|
+
let count = 0;
|
|
52
|
+
/** Adds a unique resource without exceeding the native entry budget during folder expansion. */
|
|
53
|
+
const addFile = (path: string, data: ZipEncodeFiles[string]): void => {
|
|
54
|
+
if (!Object.hasOwn(files, path) && ++count > maximum)
|
|
55
|
+
throw new RangeError('ZIP entry limit exceeded');
|
|
56
|
+
files[path] = data;
|
|
57
|
+
};
|
|
58
|
+
for (const [path, data] of Object.entries(input)) {
|
|
59
|
+
if (path.endsWith('/') || options.zip?.createFolders) {
|
|
60
|
+
let offset = path.indexOf('/');
|
|
61
|
+
while (offset !== -1) {
|
|
62
|
+
const directory = path.slice(0, offset + 1);
|
|
63
|
+
if (!Object.hasOwn(input, directory)) addFile(directory, new Uint8Array(0));
|
|
64
|
+
offset = path.indexOf('/', offset + 1);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
addFile(path, typeof data === 'string' ? new TextEncoder().encode(data) : data);
|
|
68
|
+
}
|
|
69
|
+
options.zip?.onUpdate?.({percent: 0});
|
|
70
|
+
const bytes = await encodeZip(files, options.zip);
|
|
71
|
+
options.zip?.onUpdate?.({percent: 100});
|
|
72
|
+
return bytes;
|
|
73
|
+
}
|