@versatiles/container 1.5.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/dist/index.d.ts +20 -12
- package/dist/index.js +92 -103
- package/dist/lib/decompress.d.ts +8 -7
- package/dist/lib/decompress.js +15 -8
- package/dist/lib/interfaces.d.ts +11 -12
- package/dist/lib/reader_file.d.ts +3 -4
- package/dist/lib/reader_file.js +10 -12
- package/dist/lib/reader_http.d.ts +4 -0
- package/dist/lib/reader_http.js +33 -12
- package/package.json +11 -3
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
[](https://www.npmjs.com/package/@versatiles/container)
|
|
3
3
|
[](https://codecov.io/gh/versatiles-org/node-versatiles-container)
|
|
4
4
|
[](https://github.com/versatiles-org/node-versatiles-container/actions/workflows/ci.yml)
|
|
5
|
-
[](LICENSE)
|
|
6
6
|
|
|
7
7
|
A client library for [VersaTiles containers](https://github.com/versatiles-org/versatiles-spec).
|
|
8
8
|
|
|
@@ -10,6 +10,8 @@ A client library for [VersaTiles containers](https://github.com/versatiles-org/v
|
|
|
10
10
|
|
|
11
11
|
`npm i @versatiles/container`
|
|
12
12
|
|
|
13
|
+
Requires Node.js 22.15 or newer, since Zstandard decompression uses the built-in `node:zlib` zstd support.
|
|
14
|
+
|
|
13
15
|
# Usage Examples
|
|
14
16
|
|
|
15
17
|
## Basic: read a tile and write to disk
|
|
@@ -23,7 +25,7 @@ const header = await container.getHeader();
|
|
|
23
25
|
const tile = await container.getTileUncompressed(z, x, y);
|
|
24
26
|
fs.writeFileSync("tile." + header.tileFormat, tile);
|
|
25
27
|
|
|
26
|
-
//
|
|
28
|
+
// release the file descriptor (or pooled HTTP connections) when done
|
|
27
29
|
await container.close();
|
|
28
30
|
```
|
|
29
31
|
|
|
@@ -51,6 +53,20 @@ if (metadata) {
|
|
|
51
53
|
await container.close();
|
|
52
54
|
```
|
|
53
55
|
|
|
56
|
+
## Options
|
|
57
|
+
|
|
58
|
+
Both options are optional:
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
const container = new Container("https://example.org/planet.versatiles", {
|
|
62
|
+
tms: true, // read y=0 as the southernmost row (TMS ordering); default false
|
|
63
|
+
timeout: 30000, // idle timeout in ms for HTTP(S) sources; default 10000
|
|
64
|
+
});
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`timeout` applies while waiting for response headers and between body chunks, so a slow but
|
|
68
|
+
progressing download is not aborted. It has no effect on local files or on a custom reader.
|
|
69
|
+
|
|
54
70
|
## Handle missing tiles
|
|
55
71
|
|
|
56
72
|
Coordinates must lie within the zoom level's grid (`0 <= x, y < 2 ** z`), otherwise `getTile` throws a `RangeError`. For a valid coordinate that simply has no data, the container returns `null`:
|
package/dist/index.d.ts
CHANGED
|
@@ -12,9 +12,8 @@
|
|
|
12
12
|
* from the module's `interfaces.js` file to ensure type safety and consistency.
|
|
13
13
|
*
|
|
14
14
|
* Exported Constants:
|
|
15
|
-
* - `FORMATS`: Supported
|
|
16
|
-
*
|
|
17
|
-
* - `COMPRESSIONS`: Available compression types, including 'raw' (no compression), 'gzip', and 'br' (Brotli).
|
|
15
|
+
* - `FORMATS`: Supported tile formats, including image formats (e.g., PNG, JPG) and vector formats (e.g., PBF).
|
|
16
|
+
* - `COMPRESSIONS`: Available compression types, including 'raw' (no compression), 'gzip', 'br' (Brotli) and 'zstd' (Zstandard).
|
|
18
17
|
* - `MIMETYPES`: Maps supported tile formats to their respective MIME types for handling and delivery.
|
|
19
18
|
*
|
|
20
19
|
* Exported Types:
|
|
@@ -46,8 +45,10 @@ export declare class Container {
|
|
|
46
45
|
*
|
|
47
46
|
* @param source - The data source for the tiles. This can be a URL starting with `http://` or `https://`,
|
|
48
47
|
* a path to a local file, or a custom `Reader` function that reads data chunks based on offset and length.
|
|
49
|
-
* @param options - Optional settings that configure tile handling.
|
|
48
|
+
* @param options - Optional settings that configure tile handling. See {@link OpenOptions}; `timeout`
|
|
49
|
+
* only applies when `source` is an HTTP(S) URL.
|
|
50
50
|
* @throws Will throw an error if the provided source is neither a string nor a Reader function.
|
|
51
|
+
* @throws Will throw an error if `source` is a path to a file that cannot be opened.
|
|
51
52
|
*/
|
|
52
53
|
constructor(source: Reader | string, options?: OpenOptions);
|
|
53
54
|
/**
|
|
@@ -55,14 +56,21 @@ export declare class Container {
|
|
|
55
56
|
*
|
|
56
57
|
* @returns A promise that resolves with the header object.
|
|
57
58
|
* @throws Will throw an error if the container does not start with expected magic bytes indicating a valid format.
|
|
59
|
+
* @throws Will throw an error if the container is a `versatiles_v01` file. That format is obsolete and
|
|
60
|
+
* no longer readable by this library; convert such containers to v02 first.
|
|
61
|
+
* @throws Will throw an error if `tile_format` or `precompression` holds a value this library does not support.
|
|
62
|
+
* Such a container is rejected rather than read with a substituted default, as required by the
|
|
63
|
+
* [container spec](https://github.com/versatiles-org/versatiles-spec/blob/main/v02/readme.md#2-general-guidelines).
|
|
58
64
|
*/
|
|
59
65
|
getHeader(): Promise<Header>;
|
|
60
66
|
/**
|
|
61
67
|
* Asynchronously retrieves the metadata associated with the `.versatiles` container.
|
|
62
|
-
* Metadata
|
|
63
|
-
*
|
|
68
|
+
* Metadata is stored as a string, usually TileJSON, and typically includes information
|
|
69
|
+
* about `vector_layers` for vector tiles.
|
|
70
|
+
* If the container does not include metadata, this method returns `undefined`.
|
|
64
71
|
*
|
|
65
|
-
* @returns A promise that resolves with
|
|
72
|
+
* @returns A promise that resolves with the metadata string, or `undefined` if the
|
|
73
|
+
* container holds no metadata.
|
|
66
74
|
*/
|
|
67
75
|
getMetadata(): Promise<string | undefined>;
|
|
68
76
|
/**
|
|
@@ -107,11 +115,11 @@ export declare class Container {
|
|
|
107
115
|
*/
|
|
108
116
|
protected getTileIndex(block: Block): Promise<TileIndex>;
|
|
109
117
|
/**
|
|
110
|
-
* Releases any resources held by the underlying reader
|
|
111
|
-
*
|
|
112
|
-
* (
|
|
113
|
-
*
|
|
114
|
-
*
|
|
118
|
+
* Releases any resources held by the underlying reader: the open file descriptor
|
|
119
|
+
* when reading from a local file, or the pooled keep-alive sockets when reading
|
|
120
|
+
* over HTTP(S). A custom `Reader` function without a `close` method is left
|
|
121
|
+
* untouched. After calling `close`, no further methods should be called on this
|
|
122
|
+
* container.
|
|
115
123
|
*
|
|
116
124
|
* @returns A promise that resolves once the reader has been closed.
|
|
117
125
|
*/
|
package/dist/index.js
CHANGED
|
@@ -12,9 +12,8 @@
|
|
|
12
12
|
* from the module's `interfaces.js` file to ensure type safety and consistency.
|
|
13
13
|
*
|
|
14
14
|
* Exported Constants:
|
|
15
|
-
* - `FORMATS`: Supported
|
|
16
|
-
*
|
|
17
|
-
* - `COMPRESSIONS`: Available compression types, including 'raw' (no compression), 'gzip', and 'br' (Brotli).
|
|
15
|
+
* - `FORMATS`: Supported tile formats, including image formats (e.g., PNG, JPG) and vector formats (e.g., PBF).
|
|
16
|
+
* - `COMPRESSIONS`: Available compression types, including 'raw' (no compression), 'gzip', 'br' (Brotli) and 'zstd' (Zstandard).
|
|
18
17
|
* - `MIMETYPES`: Maps supported tile formats to their respective MIME types for handling and delivery.
|
|
19
18
|
*
|
|
20
19
|
* Exported Types:
|
|
@@ -36,48 +35,45 @@
|
|
|
36
35
|
import HttpReader from './lib/reader_http.js';
|
|
37
36
|
import FileReader from './lib/reader_file.js';
|
|
38
37
|
import { decompress } from './lib/decompress.js';
|
|
39
|
-
const FORMATS =
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
],
|
|
79
|
-
};
|
|
80
|
-
const COMPRESSIONS = ['raw', 'gzip', 'br'];
|
|
38
|
+
const FORMATS = [
|
|
39
|
+
'bin',
|
|
40
|
+
null,
|
|
41
|
+
null,
|
|
42
|
+
null,
|
|
43
|
+
null,
|
|
44
|
+
null,
|
|
45
|
+
null,
|
|
46
|
+
null,
|
|
47
|
+
null,
|
|
48
|
+
null,
|
|
49
|
+
null,
|
|
50
|
+
null,
|
|
51
|
+
null,
|
|
52
|
+
null,
|
|
53
|
+
null,
|
|
54
|
+
null,
|
|
55
|
+
'png',
|
|
56
|
+
'jpg',
|
|
57
|
+
'webp',
|
|
58
|
+
'avif',
|
|
59
|
+
'svg',
|
|
60
|
+
null,
|
|
61
|
+
null,
|
|
62
|
+
null,
|
|
63
|
+
null,
|
|
64
|
+
null,
|
|
65
|
+
null,
|
|
66
|
+
null,
|
|
67
|
+
null,
|
|
68
|
+
null,
|
|
69
|
+
null,
|
|
70
|
+
null,
|
|
71
|
+
'pbf',
|
|
72
|
+
'geojson',
|
|
73
|
+
'topojson',
|
|
74
|
+
'json',
|
|
75
|
+
];
|
|
76
|
+
const COMPRESSIONS = ['raw', 'gzip', 'br', 'zstd'];
|
|
81
77
|
const MIMETYPES = {
|
|
82
78
|
avif: 'image/avif',
|
|
83
79
|
bin: 'application/octet-stream',
|
|
@@ -107,15 +103,17 @@ export class Container {
|
|
|
107
103
|
*
|
|
108
104
|
* @param source - The data source for the tiles. This can be a URL starting with `http://` or `https://`,
|
|
109
105
|
* a path to a local file, or a custom `Reader` function that reads data chunks based on offset and length.
|
|
110
|
-
* @param options - Optional settings that configure tile handling.
|
|
106
|
+
* @param options - Optional settings that configure tile handling. See {@link OpenOptions}; `timeout`
|
|
107
|
+
* only applies when `source` is an HTTP(S) URL.
|
|
111
108
|
* @throws Will throw an error if the provided source is neither a string nor a Reader function.
|
|
109
|
+
* @throws Will throw an error if `source` is a path to a file that cannot be opened.
|
|
112
110
|
*/
|
|
113
111
|
constructor(source, options) {
|
|
114
112
|
if (options)
|
|
115
113
|
Object.assign(this.#options, options);
|
|
116
114
|
if (typeof source === 'string') {
|
|
117
115
|
if (source.startsWith('https://') || source.startsWith('http://')) {
|
|
118
|
-
this.#reader = HttpReader(source);
|
|
116
|
+
this.#reader = HttpReader(source, this.#options.timeout);
|
|
119
117
|
}
|
|
120
118
|
else {
|
|
121
119
|
this.#reader = FileReader(source);
|
|
@@ -133,6 +131,11 @@ export class Container {
|
|
|
133
131
|
*
|
|
134
132
|
* @returns A promise that resolves with the header object.
|
|
135
133
|
* @throws Will throw an error if the container does not start with expected magic bytes indicating a valid format.
|
|
134
|
+
* @throws Will throw an error if the container is a `versatiles_v01` file. That format is obsolete and
|
|
135
|
+
* no longer readable by this library; convert such containers to v02 first.
|
|
136
|
+
* @throws Will throw an error if `tile_format` or `precompression` holds a value this library does not support.
|
|
137
|
+
* Such a container is rejected rather than read with a substituted default, as required by the
|
|
138
|
+
* [container spec](https://github.com/versatiles-org/versatiles-spec/blob/main/v02/readme.md#2-general-guidelines).
|
|
136
139
|
*/
|
|
137
140
|
async getHeader() {
|
|
138
141
|
// deliver if known
|
|
@@ -140,41 +143,38 @@ export class Container {
|
|
|
140
143
|
return this.#header;
|
|
141
144
|
const data = await this.read(0, 66);
|
|
142
145
|
// check magic bytes
|
|
143
|
-
|
|
146
|
+
const magic = data.toString('utf8', 0, 14);
|
|
147
|
+
if (magic === 'versatiles_v01') {
|
|
148
|
+
throw new Error('versatiles_v01 containers are no longer supported. Convert it to v02 with: versatiles convert <input> <output>');
|
|
149
|
+
}
|
|
150
|
+
if (magic !== 'versatiles_v02') {
|
|
144
151
|
throw new Error('Invalid Container');
|
|
145
152
|
}
|
|
146
|
-
const magic = data.toString('utf8', 0, 14);
|
|
147
153
|
const version = data.toString('utf8', 11, 14);
|
|
148
|
-
const
|
|
154
|
+
const tileFormatValue = data.readUInt8(14);
|
|
155
|
+
const tileFormat = FORMATS[tileFormatValue] ?? null;
|
|
156
|
+
if (tileFormat === null) {
|
|
157
|
+
throw new Error(`Unsupported tile_format value ${tileFormatValue} in ${version} container header`);
|
|
158
|
+
}
|
|
149
159
|
const tileMime = MIMETYPES[tileFormat];
|
|
150
|
-
const
|
|
160
|
+
const compressionValue = data.readUInt8(15);
|
|
161
|
+
const tileCompression = COMPRESSIONS[compressionValue];
|
|
162
|
+
if (tileCompression === undefined) {
|
|
163
|
+
throw new Error(`Unsupported precompression value ${compressionValue} in ${version} container header`);
|
|
164
|
+
}
|
|
151
165
|
const zoomMin = data.readUInt8(16);
|
|
152
166
|
const zoomMax = data.readUInt8(17);
|
|
153
|
-
let bbox;
|
|
154
167
|
const metaOffset = Number(data.readBigUInt64BE(34));
|
|
155
168
|
const metaLength = Number(data.readBigUInt64BE(42));
|
|
156
169
|
const blockIndexOffset = Number(data.readBigUInt64BE(50));
|
|
157
170
|
const blockIndexLength = Number(data.readBigUInt64BE(58));
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
];
|
|
166
|
-
break;
|
|
167
|
-
case 'v02':
|
|
168
|
-
bbox = [
|
|
169
|
-
data.readInt32BE(18) / 1e7,
|
|
170
|
-
data.readInt32BE(22) / 1e7,
|
|
171
|
-
data.readInt32BE(26) / 1e7,
|
|
172
|
-
data.readInt32BE(30) / 1e7,
|
|
173
|
-
];
|
|
174
|
-
break;
|
|
175
|
-
default:
|
|
176
|
-
throw new Error('Invalid Container');
|
|
177
|
-
}
|
|
171
|
+
// bbox coordinates are stored as int32, scaled by 1e7
|
|
172
|
+
const bbox = [
|
|
173
|
+
data.readInt32BE(18) / 1e7,
|
|
174
|
+
data.readInt32BE(22) / 1e7,
|
|
175
|
+
data.readInt32BE(26) / 1e7,
|
|
176
|
+
data.readInt32BE(30) / 1e7,
|
|
177
|
+
];
|
|
178
178
|
this.#header = {
|
|
179
179
|
magic,
|
|
180
180
|
version,
|
|
@@ -193,10 +193,12 @@ export class Container {
|
|
|
193
193
|
}
|
|
194
194
|
/**
|
|
195
195
|
* Asynchronously retrieves the metadata associated with the `.versatiles` container.
|
|
196
|
-
* Metadata
|
|
197
|
-
*
|
|
196
|
+
* Metadata is stored as a string, usually TileJSON, and typically includes information
|
|
197
|
+
* about `vector_layers` for vector tiles.
|
|
198
|
+
* If the container does not include metadata, this method returns `undefined`.
|
|
198
199
|
*
|
|
199
|
-
* @returns A promise that resolves with
|
|
200
|
+
* @returns A promise that resolves with the metadata string, or `undefined` if the
|
|
201
|
+
* container holds no metadata.
|
|
200
202
|
*/
|
|
201
203
|
async getMetadata() {
|
|
202
204
|
if (this.#metadata !== undefined)
|
|
@@ -263,8 +265,8 @@ export class Container {
|
|
|
263
265
|
const j = (ty - block.rowMin) * (block.colMax - block.colMin + 1) + (tx - block.colMin);
|
|
264
266
|
// get tile index
|
|
265
267
|
const tileIndex = await this.getTileIndex(block);
|
|
266
|
-
const offset = tileIndex.offsets[j];
|
|
267
|
-
const length = tileIndex.lengths[j];
|
|
268
|
+
const offset = tileIndex.offsets[j] ?? 0;
|
|
269
|
+
const length = tileIndex.lengths[j] ?? 0;
|
|
268
270
|
if (length === 0)
|
|
269
271
|
return null;
|
|
270
272
|
return this.read(offset, length);
|
|
@@ -303,26 +305,13 @@ export class Container {
|
|
|
303
305
|
data = await decompress(data, 'br');
|
|
304
306
|
// read index from buffer
|
|
305
307
|
const blocks = [];
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
addBlock(slice, 0n, slice.readBigUInt64BE(13), slice.readBigUInt64BE(21));
|
|
314
|
-
}
|
|
315
|
-
break;
|
|
316
|
-
case 'v02':
|
|
317
|
-
// check block index length
|
|
318
|
-
if (data.length % 33 !== 0)
|
|
319
|
-
throw new Error('invalid block index');
|
|
320
|
-
for (let i = 0; i < data.length; i += 33) {
|
|
321
|
-
const slice = data.subarray(i, i + 33);
|
|
322
|
-
addBlock(slice, slice.readBigUInt64BE(13), slice.readBigUInt64BE(13) + slice.readBigUInt64BE(21), // block_offset + tile_blobs_length
|
|
323
|
-
slice.readUInt32BE(29));
|
|
324
|
-
}
|
|
325
|
-
break;
|
|
308
|
+
// check block index length
|
|
309
|
+
if (data.length % 33 !== 0)
|
|
310
|
+
throw new Error('invalid block index');
|
|
311
|
+
for (let i = 0; i < data.length; i += 33) {
|
|
312
|
+
const slice = data.subarray(i, i + 33);
|
|
313
|
+
addBlock(slice, slice.readBigUInt64BE(13), slice.readBigUInt64BE(13) + slice.readBigUInt64BE(21), // block_offset + tile_blobs_length
|
|
314
|
+
slice.readUInt32BE(29));
|
|
326
315
|
}
|
|
327
316
|
// build map
|
|
328
317
|
this.#blockIndex = new Map(blocks.map((b) => [`${b.level},${b.column},${b.row}`, b]));
|
|
@@ -338,7 +327,7 @@ export class Container {
|
|
|
338
327
|
rowMax: buffer.readUInt8(12),
|
|
339
328
|
blockOffset: Number(blockOffset),
|
|
340
329
|
tileIndexOffset: Number(tileIndexOffset),
|
|
341
|
-
tileIndexLength
|
|
330
|
+
tileIndexLength,
|
|
342
331
|
tileCount: 0,
|
|
343
332
|
};
|
|
344
333
|
block.tileCount = (block.colMax - block.colMin + 1) * (block.rowMax - block.rowMin + 1);
|
|
@@ -379,11 +368,11 @@ export class Container {
|
|
|
379
368
|
return block.tileIndex;
|
|
380
369
|
}
|
|
381
370
|
/**
|
|
382
|
-
* Releases any resources held by the underlying reader
|
|
383
|
-
*
|
|
384
|
-
* (
|
|
385
|
-
*
|
|
386
|
-
*
|
|
371
|
+
* Releases any resources held by the underlying reader: the open file descriptor
|
|
372
|
+
* when reading from a local file, or the pooled keep-alive sockets when reading
|
|
373
|
+
* over HTTP(S). A custom `Reader` function without a `close` method is left
|
|
374
|
+
* untouched. After calling `close`, no further methods should be called on this
|
|
375
|
+
* container.
|
|
387
376
|
*
|
|
388
377
|
* @returns A promise that resolves once the reader has been closed.
|
|
389
378
|
*/
|
package/dist/lib/decompress.d.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import type { Compression } from './interfaces.js';
|
|
2
2
|
/**
|
|
3
|
-
* Decompresses a buffer using the specified compression algorithm. Currently supports 'br' for Brotli
|
|
4
|
-
*
|
|
3
|
+
* Decompresses a buffer using the specified compression algorithm. Currently supports 'br' for Brotli, 'gzip' for GZIP
|
|
4
|
+
* and 'zstd' for Zstandard. For 'raw' the buffer is passed through unchanged. Any other value is rejected rather than passed through, so that
|
|
5
|
+
* data the library cannot decompress is never handed back mislabelled as uncompressed.
|
|
5
6
|
*
|
|
6
7
|
* @param {Buffer} buffer - The buffer to be decompressed.
|
|
7
|
-
* @param {Compression} compression - The compression algorithm to use. Supported values are 'br' for Brotli and '
|
|
8
|
-
* @returns {Promise<Buffer>} A promise that, when resolved, provides the decompressed buffer. If
|
|
9
|
-
* the promise will
|
|
10
|
-
* @throws {Error} Throws an error if the
|
|
11
|
-
* the compression algorithm that was attempted.
|
|
8
|
+
* @param {Compression} compression - The compression algorithm to use. Supported values are 'br' for Brotli, 'gzip' for GZIP, 'zstd' for Zstandard and 'raw' for uncompressed data.
|
|
9
|
+
* @returns {Promise<Buffer>} A promise that, when resolved, provides the decompressed buffer. If decompression fails,
|
|
10
|
+
* the promise will be rejected with an error message.
|
|
11
|
+
* @throws {Error} Throws an error if the compression algorithm is unsupported, or if the decompression process encounters
|
|
12
|
+
* an error. The error includes the buffer's length and the compression algorithm that was attempted.
|
|
12
13
|
*/
|
|
13
14
|
export declare function decompress(buffer: Buffer, compression: Compression): Promise<Buffer>;
|
package/dist/lib/decompress.js
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import zlib from 'zlib';
|
|
2
2
|
/**
|
|
3
|
-
* Decompresses a buffer using the specified compression algorithm. Currently supports 'br' for Brotli
|
|
4
|
-
*
|
|
3
|
+
* Decompresses a buffer using the specified compression algorithm. Currently supports 'br' for Brotli, 'gzip' for GZIP
|
|
4
|
+
* and 'zstd' for Zstandard. For 'raw' the buffer is passed through unchanged. Any other value is rejected rather than passed through, so that
|
|
5
|
+
* data the library cannot decompress is never handed back mislabelled as uncompressed.
|
|
5
6
|
*
|
|
6
7
|
* @param {Buffer} buffer - The buffer to be decompressed.
|
|
7
|
-
* @param {Compression} compression - The compression algorithm to use. Supported values are 'br' for Brotli and '
|
|
8
|
-
* @returns {Promise<Buffer>} A promise that, when resolved, provides the decompressed buffer. If
|
|
9
|
-
* the promise will
|
|
10
|
-
* @throws {Error} Throws an error if the
|
|
11
|
-
* the compression algorithm that was attempted.
|
|
8
|
+
* @param {Compression} compression - The compression algorithm to use. Supported values are 'br' for Brotli, 'gzip' for GZIP, 'zstd' for Zstandard and 'raw' for uncompressed data.
|
|
9
|
+
* @returns {Promise<Buffer>} A promise that, when resolved, provides the decompressed buffer. If decompression fails,
|
|
10
|
+
* the promise will be rejected with an error message.
|
|
11
|
+
* @throws {Error} Throws an error if the compression algorithm is unsupported, or if the decompression process encounters
|
|
12
|
+
* an error. The error includes the buffer's length and the compression algorithm that was attempted.
|
|
12
13
|
*/
|
|
13
14
|
export async function decompress(buffer, compression) {
|
|
14
15
|
return new Promise((resolve, reject) => {
|
|
@@ -19,9 +20,15 @@ export async function decompress(buffer, compression) {
|
|
|
19
20
|
case 'gzip':
|
|
20
21
|
zlib.gunzip(buffer, handle);
|
|
21
22
|
break;
|
|
22
|
-
|
|
23
|
+
case 'zstd':
|
|
24
|
+
zlib.zstdDecompress(buffer, handle);
|
|
25
|
+
break;
|
|
26
|
+
case 'raw':
|
|
23
27
|
resolve(buffer);
|
|
24
28
|
break;
|
|
29
|
+
default:
|
|
30
|
+
reject(new Error(`Can not decompress buffer (length=${buffer.length}): unsupported compression "${String(compression)}"`));
|
|
31
|
+
break;
|
|
25
32
|
}
|
|
26
33
|
/**
|
|
27
34
|
* Handles the result of the decompression operation. If decompression is successful, the promise is resolved
|
package/dist/lib/interfaces.d.ts
CHANGED
|
@@ -1,20 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Supported compression.
|
|
3
|
-
* `
|
|
3
|
+
* `'raw'` signifies that the data is uncompressed.
|
|
4
4
|
*/
|
|
5
|
-
export type Compression = 'br' | 'gzip' | 'raw';
|
|
5
|
+
export type Compression = 'br' | 'gzip' | 'raw' | 'zstd';
|
|
6
6
|
/**
|
|
7
7
|
* Supported tile formats.
|
|
8
8
|
*/
|
|
9
9
|
export type Format = 'avif' | 'bin' | 'geojson' | 'jpg' | 'json' | 'pbf' | 'png' | 'svg' | 'topojson' | 'webp';
|
|
10
|
-
/**
|
|
11
|
-
* Type definition for a function that decompresses data.
|
|
12
|
-
*
|
|
13
|
-
* @param {Buffer} data - The compressed data.
|
|
14
|
-
* @param {Compression} compression - The type of compression used.
|
|
15
|
-
* @returns {Promise<Buffer>} - The decompressed data.
|
|
16
|
-
*/
|
|
17
|
-
export type Decompressor = (data: Buffer, compression: Compression) => Promise<Buffer>;
|
|
18
10
|
/**
|
|
19
11
|
* Type definition for reading content from a VersaTiles container.
|
|
20
12
|
*
|
|
@@ -120,8 +112,15 @@ export interface TileIndex {
|
|
|
120
112
|
}
|
|
121
113
|
/**
|
|
122
114
|
* Interface for defining the options available for reading a container.
|
|
123
|
-
*
|
|
115
|
+
*
|
|
116
|
+
* All properties are optional; omitted ones fall back to the documented default.
|
|
117
|
+
*
|
|
118
|
+
* @property {boolean} tms - If set to true, uses the [TMS (Tile Map Service) tile ordering](https://wiki.openstreetmap.org/wiki/TMS) where y=0 is the southernmost point. Defaults to `false`.
|
|
119
|
+
* @property {number} timeout - Idle timeout in milliseconds for HTTP(S) sources, applied both while
|
|
120
|
+
* waiting for the response headers and between body chunks. Has no effect on local files or on a
|
|
121
|
+
* custom {@link Reader}, which manages its own timeouts. Defaults to 10000.
|
|
124
122
|
*/
|
|
125
123
|
export interface OpenOptions {
|
|
126
|
-
tms
|
|
124
|
+
tms?: boolean;
|
|
125
|
+
timeout?: number;
|
|
127
126
|
}
|
|
@@ -11,9 +11,8 @@ import type { Reader } from './interfaces.js';
|
|
|
11
11
|
* @returns {Reader} A reader function that when called, returns a promise. This promise resolves with a
|
|
12
12
|
* Buffer containing the read bytes from the file. The function reads `length` bytes of data
|
|
13
13
|
* from the file starting at `position`. If an error occurs during the read operation,
|
|
14
|
-
* the promise is rejected with the error.
|
|
15
|
-
* the
|
|
16
|
-
*
|
|
17
|
-
* that could be read.
|
|
14
|
+
* the promise is rejected with the error. Reads are bounds-checked against the file size
|
|
15
|
+
* captured when the reader was created: a range that ends beyond the end of the file is
|
|
16
|
+
* rejected with a `RangeError` rather than returning a short buffer.
|
|
18
17
|
*/
|
|
19
18
|
export default function getFileReader(filename: string): Reader;
|
package/dist/lib/reader_file.js
CHANGED
|
@@ -11,10 +11,9 @@ import fs from 'node:fs';
|
|
|
11
11
|
* @returns {Reader} A reader function that when called, returns a promise. This promise resolves with a
|
|
12
12
|
* Buffer containing the read bytes from the file. The function reads `length` bytes of data
|
|
13
13
|
* from the file starting at `position`. If an error occurs during the read operation,
|
|
14
|
-
* the promise is rejected with the error.
|
|
15
|
-
* the
|
|
16
|
-
*
|
|
17
|
-
* that could be read.
|
|
14
|
+
* the promise is rejected with the error. Reads are bounds-checked against the file size
|
|
15
|
+
* captured when the reader was created: a range that ends beyond the end of the file is
|
|
16
|
+
* rejected with a `RangeError` rather than returning a short buffer.
|
|
18
17
|
*/
|
|
19
18
|
export default function getFileReader(filename) {
|
|
20
19
|
const fd = fs.openSync(filename, 'r');
|
|
@@ -26,16 +25,15 @@ export default function getFileReader(filename) {
|
|
|
26
25
|
* that are not being concurrently modified, as it relies on consistent file state to
|
|
27
26
|
* function correctly.
|
|
28
27
|
*
|
|
29
|
-
* @param {number} position - The starting position in the file to read from.
|
|
30
|
-
* is beyond the end of the file, the function will resolve to an
|
|
31
|
-
* empty buffer.
|
|
28
|
+
* @param {number} position - The starting position in the file to read from. Must be non-negative.
|
|
32
29
|
* @param {number} length - The number of bytes to read from the file starting at `position`.
|
|
33
|
-
*
|
|
34
|
-
* will only include the bytes that could be read up to the end of the file.
|
|
30
|
+
* Must be non-negative.
|
|
35
31
|
* @returns {Promise<Buffer>} A promise that resolves with the buffer containing the read bytes. If
|
|
36
|
-
* an error occurs during the read, such as
|
|
37
|
-
*
|
|
38
|
-
*
|
|
32
|
+
* an error occurs during the read, such as the file descriptor becoming
|
|
33
|
+
* invalid due to the file being closed, the promise will be rejected with
|
|
34
|
+
* an error.
|
|
35
|
+
* @throws {RangeError} If `position` or `length` is negative, or if `position + length` exceeds the
|
|
36
|
+
* size of the file.
|
|
39
37
|
*/
|
|
40
38
|
const read = async function read(position, length) {
|
|
41
39
|
if (position < 0) {
|
|
@@ -4,6 +4,10 @@ import type { Reader } from './interfaces.js';
|
|
|
4
4
|
* to read data chunks in an HTTP GET request. This is particularly useful for
|
|
5
5
|
* operations such as streaming or handling large data in segments.
|
|
6
6
|
*
|
|
7
|
+
* The returned reader keeps HTTP connections alive between reads, which matters because
|
|
8
|
+
* reading a container issues many small range requests. Call its `close` method when done
|
|
9
|
+
* to release those pooled sockets.
|
|
10
|
+
*
|
|
7
11
|
* @param url - The URL from which data will be read.
|
|
8
12
|
* @param timeout - Idle timeout in milliseconds applied both while waiting for the
|
|
9
13
|
* response headers and between body chunks. Defaults to {@link DEFAULT_TIMEOUT}.
|
package/dist/lib/reader_http.js
CHANGED
|
@@ -5,14 +5,18 @@ const DEFAULT_TIMEOUT = 10000;
|
|
|
5
5
|
* A collection mapping protocol names to their respective `ClientInfo`.
|
|
6
6
|
*/
|
|
7
7
|
const clients = {
|
|
8
|
-
https: { client: https,
|
|
9
|
-
http: { client: http,
|
|
8
|
+
https: { client: https, createAgent: () => new https.Agent({ keepAlive: true }) },
|
|
9
|
+
http: { client: http, createAgent: () => new http.Agent({ keepAlive: true }) },
|
|
10
10
|
};
|
|
11
11
|
/**
|
|
12
12
|
* Creates a function capable of reading data from a specified URL, which can be used
|
|
13
13
|
* to read data chunks in an HTTP GET request. This is particularly useful for
|
|
14
14
|
* operations such as streaming or handling large data in segments.
|
|
15
15
|
*
|
|
16
|
+
* The returned reader keeps HTTP connections alive between reads, which matters because
|
|
17
|
+
* reading a container issues many small range requests. Call its `close` method when done
|
|
18
|
+
* to release those pooled sockets.
|
|
19
|
+
*
|
|
16
20
|
* @param url - The URL from which data will be read.
|
|
17
21
|
* @param timeout - Idle timeout in milliseconds applied both while waiting for the
|
|
18
22
|
* response headers and between body chunks. Defaults to {@link DEFAULT_TIMEOUT}.
|
|
@@ -20,9 +24,12 @@ const clients = {
|
|
|
20
24
|
*/
|
|
21
25
|
export default function getHTTPReader(url, timeout = DEFAULT_TIMEOUT) {
|
|
22
26
|
const protocol = new URL(url).protocol.slice(0, -1);
|
|
23
|
-
|
|
27
|
+
const clientInfo = clients[protocol];
|
|
28
|
+
if (clientInfo == null) {
|
|
24
29
|
throw new Error(`Unsupported protocol: ${protocol}`);
|
|
25
30
|
}
|
|
31
|
+
const { client, createAgent } = clientInfo;
|
|
32
|
+
const agent = createAgent();
|
|
26
33
|
/**
|
|
27
34
|
* Asynchronously reads a data chunk from the provided URL based on the specified range.
|
|
28
35
|
*
|
|
@@ -32,7 +39,7 @@ export default function getHTTPReader(url, timeout = DEFAULT_TIMEOUT) {
|
|
|
32
39
|
* If the request fails or the server responds with a non-successful status code,
|
|
33
40
|
* the promise is rejected with an error.
|
|
34
41
|
*/
|
|
35
|
-
|
|
42
|
+
const read = async function read(position, length) {
|
|
36
43
|
if (position < 0) {
|
|
37
44
|
throw new RangeError(`Invalid read position: ${position}. The read position must be a non-negative integer.`);
|
|
38
45
|
}
|
|
@@ -40,7 +47,7 @@ export default function getHTTPReader(url, timeout = DEFAULT_TIMEOUT) {
|
|
|
40
47
|
throw new RangeError(`Invalid read length: ${length}. The read length must be a non-negative integer.`);
|
|
41
48
|
}
|
|
42
49
|
const headers = {
|
|
43
|
-
'user-agent': 'Mozilla/5.0 (compatible; versatiles; +https://www.npmjs.com/package/
|
|
50
|
+
'user-agent': 'Mozilla/5.0 (compatible; versatiles; +https://www.npmjs.com/package/@versatiles/container)',
|
|
44
51
|
range: `bytes=${position}-${position + length - 1}`,
|
|
45
52
|
};
|
|
46
53
|
/**
|
|
@@ -52,10 +59,10 @@ export default function getHTTPReader(url, timeout = DEFAULT_TIMEOUT) {
|
|
|
52
59
|
req.destroy();
|
|
53
60
|
reject(new Error('Request timed out'));
|
|
54
61
|
}, timeout);
|
|
55
|
-
const req =
|
|
62
|
+
const req = client
|
|
56
63
|
.request(url, {
|
|
57
64
|
method: 'GET',
|
|
58
|
-
agent
|
|
65
|
+
agent,
|
|
59
66
|
headers,
|
|
60
67
|
timeout,
|
|
61
68
|
})
|
|
@@ -89,12 +96,17 @@ export default function getHTTPReader(url, timeout = DEFAULT_TIMEOUT) {
|
|
|
89
96
|
const parts = /^bytes (\d+)-(\d+)\/(\d+)/i.exec(contentRange);
|
|
90
97
|
if (parts == null)
|
|
91
98
|
fail(new Error('"content-range" in response header is malformed'));
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
99
|
+
// The pattern matches digits only, so these are finite numbers whenever it
|
|
100
|
+
// matched; Number() of a missing group would be NaN, which fails the same checks.
|
|
101
|
+
const returnedOffset = Number(parts[1]);
|
|
102
|
+
const returnedLast = Number(parts[2]);
|
|
103
|
+
const totalSize = Number(parts[3]);
|
|
104
|
+
if (position !== returnedOffset)
|
|
105
|
+
fail(new Error(`requested position (${position}) and returned offset (${returnedOffset}) must be equal`));
|
|
106
|
+
if (position + length > totalSize) {
|
|
107
|
+
fail(new RangeError(`Read range out of bounds: The requested range ends at position ${position + length}, which exceeds the file's limit of ${totalSize} bytes.`));
|
|
96
108
|
}
|
|
97
|
-
const returnedLength =
|
|
109
|
+
const returnedLength = returnedLast + 1 - position;
|
|
98
110
|
if (length !== returnedLength) {
|
|
99
111
|
fail(new Error(`Returned length (${returnedLength}) is not requested length (${length}).`));
|
|
100
112
|
}
|
|
@@ -130,4 +142,13 @@ export default function getHTTPReader(url, timeout = DEFAULT_TIMEOUT) {
|
|
|
130
142
|
});
|
|
131
143
|
return body;
|
|
132
144
|
};
|
|
145
|
+
/**
|
|
146
|
+
* Destroys this reader's connection agent, closing any sockets it is holding
|
|
147
|
+
* open for keep-alive. After this resolves, the reader must not be called again.
|
|
148
|
+
*/
|
|
149
|
+
read.close = function close() {
|
|
150
|
+
agent.destroy();
|
|
151
|
+
return Promise.resolve();
|
|
152
|
+
};
|
|
153
|
+
return read;
|
|
133
154
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@versatiles/container",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "A client library for reading VersaTiles containers",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"build": "rm -rf dist && tsc -p tsconfig.build.json && npm run doc-graph",
|
|
@@ -28,11 +28,19 @@
|
|
|
28
28
|
"homepage": "https://github.com/versatiles-org/node-versatiles-container",
|
|
29
29
|
"main": "dist/index.js",
|
|
30
30
|
"types": "dist/index.d.ts",
|
|
31
|
+
"exports": {
|
|
32
|
+
".": {
|
|
33
|
+
"types": "./dist/index.d.ts",
|
|
34
|
+
"default": "./dist/index.js"
|
|
35
|
+
},
|
|
36
|
+
"./package.json": "./package.json"
|
|
37
|
+
},
|
|
31
38
|
"type": "module",
|
|
39
|
+
"sideEffects": false,
|
|
32
40
|
"author": "yetzt <node@yetzt.me>, Michael Kreil <versatiles@michael-kreil.de>",
|
|
33
41
|
"license": "Unlicense",
|
|
34
42
|
"engines": {
|
|
35
|
-
"node": ">=
|
|
43
|
+
"node": ">= 22.15"
|
|
36
44
|
},
|
|
37
45
|
"keywords": [
|
|
38
46
|
"cloudtiles",
|
|
@@ -57,7 +65,7 @@
|
|
|
57
65
|
"@types/node": "^26.2.0",
|
|
58
66
|
"@typescript-eslint/eslint-plugin": "^8.67.0",
|
|
59
67
|
"@typescript-eslint/parser": "^8.67.0",
|
|
60
|
-
"@versatiles/release-tool": "^2.9.
|
|
68
|
+
"@versatiles/release-tool": "^2.9.1",
|
|
61
69
|
"@vitest/coverage-v8": "^4.1.11",
|
|
62
70
|
"eslint": "^10.8.1",
|
|
63
71
|
"eslint-config-prettier": "^10.1.8",
|