@ferrybig/cgeodata-reader 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +313 -2
- package/binary-format.md +2620 -0
- package/dist/cache.d.ts +24 -0
- package/dist/cache.js +60 -0
- package/dist/cellBounds.d.ts +19 -0
- package/dist/cellBounds.js +36 -0
- package/dist/chunk.d.ts +35 -0
- package/dist/chunk.js +277 -0
- package/dist/chunkDecoder.d.ts +39 -0
- package/dist/chunkDecoder.js +116 -0
- package/dist/chunkDecoderWasm.d.ts +2 -0
- package/dist/chunkDecoderWasm.js +4 -0
- package/dist/cli/buildLocateIndex.d.ts +2 -0
- package/dist/cli/buildLocateIndex.js +46 -0
- package/dist/cli/buildSearchIndex.d.ts +2 -0
- package/dist/cli/buildSearchIndex.js +90 -0
- package/dist/cli/chunk.d.ts +2 -0
- package/dist/cli/chunk.js +67 -0
- package/dist/cli/info.d.ts +2 -0
- package/dist/cli/info.js +87 -0
- package/dist/cli/list.d.ts +2 -0
- package/dist/cli/list.js +71 -0
- package/dist/cli/locate.d.ts +2 -0
- package/dist/cli/locate.js +64 -0
- package/dist/cli/lookup.d.ts +2 -0
- package/dist/cli/lookup.js +128 -0
- package/dist/cli/search.d.ts +2 -0
- package/dist/cli/search.js +119 -0
- package/dist/cli/shared.d.ts +50 -0
- package/dist/cli/shared.js +140 -0
- package/dist/cli/show.d.ts +2 -0
- package/dist/cli/show.js +38 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +107 -0
- package/dist/container.d.ts +17 -0
- package/dist/container.js +68 -0
- package/dist/directory.d.ts +21 -0
- package/dist/directory.js +31 -0
- package/dist/errors.d.ts +52 -0
- package/dist/errors.js +56 -0
- package/dist/format.d.ts +22 -0
- package/dist/format.js +23 -0
- package/dist/header.d.ts +30 -0
- package/dist/header.js +69 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.js +18 -0
- package/dist/indexes.d.ts +11 -0
- package/dist/indexes.js +12 -0
- package/dist/inflate.d.ts +21 -0
- package/dist/inflate.js +61 -0
- package/dist/locateIndex.d.ts +142 -0
- package/dist/locateIndex.js +561 -0
- package/dist/morton.d.ts +28 -0
- package/dist/morton.js +51 -0
- package/dist/node.d.ts +65 -0
- package/dist/node.js +137 -0
- package/dist/placeTable.d.ts +69 -0
- package/dist/placeTable.js +177 -0
- package/dist/rangeChunk.d.ts +46 -0
- package/dist/rangeChunk.js +401 -0
- package/dist/rangeDecoder.d.ts +17 -0
- package/dist/rangeDecoder.js +129 -0
- package/dist/rangeDecoderWasm.d.ts +2 -0
- package/dist/rangeDecoderWasm.js +4 -0
- package/dist/reader.d.ts +534 -0
- package/dist/reader.js +1132 -0
- package/dist/relevance.d.ts +184 -0
- package/dist/relevance.js +266 -0
- package/dist/searchIndex.d.ts +148 -0
- package/dist/searchIndex.js +786 -0
- package/dist/source.d.ts +49 -0
- package/dist/source.js +50 -0
- package/dist/turns.d.ts +19 -0
- package/dist/turns.js +40 -0
- package/dist/varint.d.ts +52 -0
- package/dist/varint.js +137 -0
- package/package.json +76 -5
package/dist/errors.js
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Errors the reader throws because of the data it was given, as opposed to
|
|
3
|
+
* how it was called: a caller's invalid argument (a latitude out of range,
|
|
4
|
+
* a place id this file doesn't have) is a plain `RangeError`, and anything
|
|
5
|
+
* else is a bug.
|
|
6
|
+
*
|
|
7
|
+
* ```ts
|
|
8
|
+
* try {
|
|
9
|
+
* reader = CGeoDataReader.open(path);
|
|
10
|
+
* } catch (error) {
|
|
11
|
+
* if (error instanceof CGeoDataVersionError) {
|
|
12
|
+
* // written by a newer compressor: upgrade the reader
|
|
13
|
+
* } else if (error instanceof CGeoDataFormatError) {
|
|
14
|
+
* // truncated or corrupt download: fetch the file again
|
|
15
|
+
* } else throw error;
|
|
16
|
+
* }
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export class CGeoDataError extends Error {
|
|
20
|
+
/** "CGeoDataError", and each subclass its own name. */
|
|
21
|
+
name = "CGeoDataError";
|
|
22
|
+
}
|
|
23
|
+
/** A `.bin` file, or a search or locate index, is malformed or truncated:
|
|
24
|
+
* it breaks a rule of binary-format.md, so nothing read from it can be
|
|
25
|
+
* trusted. */
|
|
26
|
+
export class CGeoDataFormatError extends CGeoDataError {
|
|
27
|
+
/** "CGeoDataFormatError" */
|
|
28
|
+
name = "CGeoDataFormatError";
|
|
29
|
+
}
|
|
30
|
+
/** The file is a CGEODATA file (or index) of a format version this reader
|
|
31
|
+
* doesn't know -- most likely written by a newer compressor. */
|
|
32
|
+
export class CGeoDataVersionError extends CGeoDataFormatError {
|
|
33
|
+
/** "CGeoDataVersionError" */
|
|
34
|
+
name = "CGeoDataVersionError";
|
|
35
|
+
/** The version number the file declares. */
|
|
36
|
+
version;
|
|
37
|
+
constructor(message, version) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.version = version;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** The file is valid, but uses something this reader doesn't support: a
|
|
43
|
+
* `max_depth` past 16 (see `MAX_SUPPORTED_DEPTH`). Rebuild it with a
|
|
44
|
+
* bigger chunk grid instead of deeper chunks. */
|
|
45
|
+
export class CGeoDataUnsupportedError extends CGeoDataFormatError {
|
|
46
|
+
/** "CGeoDataUnsupportedError" */
|
|
47
|
+
name = "CGeoDataUnsupportedError";
|
|
48
|
+
}
|
|
49
|
+
/** A search or locate index is well formed, but was built from a different
|
|
50
|
+
* `.bin` file than the one it's being used with (or an older build of the
|
|
51
|
+
* same one). Rebuild the index, or delete it to work without one. */
|
|
52
|
+
export class CGeoDataIndexMismatchError extends CGeoDataError {
|
|
53
|
+
/** "CGeoDataIndexMismatchError" */
|
|
54
|
+
name = "CGeoDataIndexMismatchError";
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=errors.js.map
|
package/dist/format.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* binary-format.md at the byte level: the codecs and parsers the reader is
|
|
3
|
+
* built from, for tools that write or inspect CGEODATA files (the
|
|
4
|
+
* compressor uses them). They follow the format, not a stable API, and
|
|
5
|
+
* may change when it does; `CGeoDataReader` is the interface for reading.
|
|
6
|
+
*
|
|
7
|
+
* @module @ferrybig/cgeodata-reader/format
|
|
8
|
+
*/
|
|
9
|
+
export { type BoundingBox, cellBoundingBox, chunkBoundingBox, } from "./cellBounds.ts";
|
|
10
|
+
export { type ChunkRecords, MAX_SUPPORTED_DEPTH, parseChunk, } from "./chunk.ts";
|
|
11
|
+
export { readHeaderAndDirectory } from "./container.ts";
|
|
12
|
+
export { type CGeoDataDirectory, parseDirectory } from "./directory.ts";
|
|
13
|
+
export { type CGeoDataHeader, parseHeader } from "./header.ts";
|
|
14
|
+
export { type Inflater, inflateBlock, inflateCeiling, streamInflater, } from "./inflate.ts";
|
|
15
|
+
export { decodeLocateEntry, encodeLocateEntry, type LocateCell, type LocateGrid, locateEntryWidth, } from "./locateIndex.ts";
|
|
16
|
+
export { lowBits, mortonDecode, mortonEncode, shiftRight } from "./morton.ts";
|
|
17
|
+
export { type PlaceTableInfo, parsePlaceTable, type ResolvedPlace, readNameBlock, readNameEntry, resolveIdentifier, } from "./placeTable.ts";
|
|
18
|
+
export { decodeRangeCodedRecords, encodeRangeCodedRecords, type RangeCodedRecords, } from "./rangeChunk.ts";
|
|
19
|
+
export { containsTokenRun, EXACT_SPELLING_BONUS, foldDiacritics, LEVEL_BONUS, type Needle, nameMatchQuality, placeMatchQuality, prepareNeedle, QUALITY_EXACT, QUALITY_MAX, QUALITY_SUBSTRING, QUALITY_WHOLE_WORD, tokenize, tokenizeFolded, } from "./relevance.ts";
|
|
20
|
+
export { fnv1a32, foldSymbols, trigramListIndex, trigramListIndices, } from "./searchIndex.ts";
|
|
21
|
+
export { toSource } from "./source.ts";
|
|
22
|
+
export { IncompleteBufferError, readDeltaVarintList, readDeltaVarints32Into, readDeltaVarintsInto, readVarint, readVarintList, type VarintListResult, type VarintResult, writeVarint, } from "./varint.ts";
|
package/dist/format.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* binary-format.md at the byte level: the codecs and parsers the reader is
|
|
3
|
+
* built from, for tools that write or inspect CGEODATA files (the
|
|
4
|
+
* compressor uses them). They follow the format, not a stable API, and
|
|
5
|
+
* may change when it does; `CGeoDataReader` is the interface for reading.
|
|
6
|
+
*
|
|
7
|
+
* @module @ferrybig/cgeodata-reader/format
|
|
8
|
+
*/
|
|
9
|
+
export { cellBoundingBox, chunkBoundingBox, } from "./cellBounds.js";
|
|
10
|
+
export { MAX_SUPPORTED_DEPTH, parseChunk, } from "./chunk.js";
|
|
11
|
+
export { readHeaderAndDirectory } from "./container.js";
|
|
12
|
+
export { parseDirectory } from "./directory.js";
|
|
13
|
+
export { parseHeader } from "./header.js";
|
|
14
|
+
export { inflateBlock, inflateCeiling, streamInflater, } from "./inflate.js";
|
|
15
|
+
export { decodeLocateEntry, encodeLocateEntry, locateEntryWidth, } from "./locateIndex.js";
|
|
16
|
+
export { lowBits, mortonDecode, mortonEncode, shiftRight } from "./morton.js";
|
|
17
|
+
export { parsePlaceTable, readNameBlock, readNameEntry, resolveIdentifier, } from "./placeTable.js";
|
|
18
|
+
export { decodeRangeCodedRecords, encodeRangeCodedRecords, } from "./rangeChunk.js";
|
|
19
|
+
export { containsTokenRun, EXACT_SPELLING_BONUS, foldDiacritics, LEVEL_BONUS, nameMatchQuality, placeMatchQuality, prepareNeedle, QUALITY_EXACT, QUALITY_MAX, QUALITY_SUBSTRING, QUALITY_WHOLE_WORD, tokenize, tokenizeFolded, } from "./relevance.js";
|
|
20
|
+
export { fnv1a32, foldSymbols, trigramListIndex, trigramListIndices, } from "./searchIndex.js";
|
|
21
|
+
export { toSource } from "./source.js";
|
|
22
|
+
export { IncompleteBufferError, readDeltaVarintList, readDeltaVarints32Into, readDeltaVarintsInto, readVarint, readVarintList, writeVarint, } from "./varint.js";
|
|
23
|
+
//# sourceMappingURL=format.js.map
|
package/dist/header.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/** A file's header (Section 5). */
|
|
2
|
+
export interface CGeoDataHeader {
|
|
3
|
+
/** Format version: 2 (range-coded chunks, Section 8.7), 1 (zlib
|
|
4
|
+
* chunks), or 0 for files written before chunks dropped their codes
|
|
5
|
+
* (Section 8.6). This reader reads all three. */
|
|
6
|
+
version: number;
|
|
7
|
+
/** The data's attribution notice, as given when the file was built.
|
|
8
|
+
* Show it wherever results are shown. */
|
|
9
|
+
attribution: string;
|
|
10
|
+
/** The chunk grid is 2^widthExp chunks wide... */
|
|
11
|
+
widthExp: number;
|
|
12
|
+
/** ...and 2^heightExp chunks high (Section 6.1). */
|
|
13
|
+
heightExp: number;
|
|
14
|
+
/** How many times a chunk's quadtree can split (Section 8.1): a chunk
|
|
15
|
+
* is at most 2^maxDepth cells across. */
|
|
16
|
+
maxDepth: number;
|
|
17
|
+
/** 2^keyframeExp names per name block (Section 7.3); 0 for one block. */
|
|
18
|
+
keyframeExp: number;
|
|
19
|
+
/** W = 2^widthExp */
|
|
20
|
+
width: number;
|
|
21
|
+
/** H = 2^heightExp */
|
|
22
|
+
height: number;
|
|
23
|
+
/** Byte length of the header, i.e. where the directory begins. */
|
|
24
|
+
headerLength: number;
|
|
25
|
+
}
|
|
26
|
+
/** Parses Section 5's header from a buffer that starts at the file's first
|
|
27
|
+
* byte. Throws IncompleteBufferError if `buf` is too short to tell (the
|
|
28
|
+
* caller should retry with more bytes), or a plain Error for a genuine
|
|
29
|
+
* format problem. */
|
|
30
|
+
export declare function parseHeader(buf: Uint8Array): CGeoDataHeader;
|
package/dist/header.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { CGeoDataFormatError, CGeoDataVersionError } from "./errors.js";
|
|
2
|
+
import { IncompleteBufferError, readVarint } from "./varint.js";
|
|
3
|
+
const utf8 = new TextDecoder();
|
|
4
|
+
// "CGEODATA"
|
|
5
|
+
const SIGNATURE = Uint8Array.from([
|
|
6
|
+
0x43, 0x47, 0x45, 0x4f, 0x44, 0x41, 0x54, 0x41,
|
|
7
|
+
]);
|
|
8
|
+
/** Parses Section 5's header from a buffer that starts at the file's first
|
|
9
|
+
* byte. Throws IncompleteBufferError if `buf` is too short to tell (the
|
|
10
|
+
* caller should retry with more bytes), or a plain Error for a genuine
|
|
11
|
+
* format problem. */
|
|
12
|
+
export function parseHeader(buf) {
|
|
13
|
+
if (buf.length < SIGNATURE.length) {
|
|
14
|
+
throw new IncompleteBufferError("buffer too short for the CGEODATA signature");
|
|
15
|
+
}
|
|
16
|
+
for (let i = 0; i < SIGNATURE.length; i++) {
|
|
17
|
+
if (buf[i] !== SIGNATURE[i]) {
|
|
18
|
+
throw new CGeoDataFormatError("not a CGEODATA file: signature mismatch");
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
let offset = SIGNATURE.length;
|
|
22
|
+
const version = readVarint(buf, offset);
|
|
23
|
+
offset += version.bytesRead;
|
|
24
|
+
if (version.value > 2) {
|
|
25
|
+
throw new CGeoDataVersionError(`unsupported CGEODATA version ${version.value}`, version.value);
|
|
26
|
+
}
|
|
27
|
+
const attrLen = readVarint(buf, offset);
|
|
28
|
+
offset += attrLen.bytesRead;
|
|
29
|
+
const attrEnd = offset + attrLen.value;
|
|
30
|
+
if (attrEnd + 4 > buf.length) {
|
|
31
|
+
throw new IncompleteBufferError("buffer too short for attribution + fixed header fields");
|
|
32
|
+
}
|
|
33
|
+
const attribution = utf8.decode(buf.subarray(offset, attrEnd));
|
|
34
|
+
offset = attrEnd;
|
|
35
|
+
const widthExp = buf[offset];
|
|
36
|
+
const heightExp = buf[offset + 1];
|
|
37
|
+
const maxDepth = buf[offset + 2];
|
|
38
|
+
const keyframeExp = buf[offset + 3];
|
|
39
|
+
offset += 4;
|
|
40
|
+
if (maxDepth > 26) {
|
|
41
|
+
throw new CGeoDataFormatError(`max_depth ${maxDepth} exceeds the maximum of 26`);
|
|
42
|
+
}
|
|
43
|
+
if (widthExp > 15) {
|
|
44
|
+
throw new CGeoDataFormatError(`width_exp ${widthExp} exceeds the maximum of 15`);
|
|
45
|
+
}
|
|
46
|
+
if (heightExp > 15) {
|
|
47
|
+
throw new CGeoDataFormatError(`height_exp ${heightExp} exceeds the maximum of 15`);
|
|
48
|
+
}
|
|
49
|
+
// The only other u8 of the header, and the only one that was left
|
|
50
|
+
// unchecked: a place count is below 2^53 (Section 3.2), so a block of
|
|
51
|
+
// 2^53 names or more can never be anything but malformed -- and an
|
|
52
|
+
// unchecked value renders as nonsense ("keyframe 255 (5.8e+76 names per
|
|
53
|
+
// block)" from `cgeodata info`).
|
|
54
|
+
if (keyframeExp > 52) {
|
|
55
|
+
throw new CGeoDataFormatError(`keyframe_exp ${keyframeExp} exceeds the maximum of 52`);
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
version: version.value,
|
|
59
|
+
attribution,
|
|
60
|
+
widthExp,
|
|
61
|
+
heightExp,
|
|
62
|
+
maxDepth,
|
|
63
|
+
keyframeExp,
|
|
64
|
+
width: 2 ** widthExp,
|
|
65
|
+
height: 2 ** heightExp,
|
|
66
|
+
headerLength: offset,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=header.js.map
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CGEODATA reader: resolves a latitude/longitude to the country, region
|
|
3
|
+
* and city containing it, from a CGEODATA file (see binary-format.md).
|
|
4
|
+
*
|
|
5
|
+
* This entry point is platform-neutral. On Node.js, open files with
|
|
6
|
+
* `openFile()` from `./node`; anywhere else, hand `CGeoDataReader.open()`
|
|
7
|
+
* the file's bytes or a `Blob`. Search and locate indexes, and their
|
|
8
|
+
* builders, are in `./indexes`; the format's low-level codecs, for tools
|
|
9
|
+
* that write or inspect files, in `./format`.
|
|
10
|
+
*
|
|
11
|
+
* @module @ferrybig/cgeodata-reader
|
|
12
|
+
*/
|
|
13
|
+
export { type BoundingBox, boundingBoxContains } from "./cellBounds.ts";
|
|
14
|
+
export type { ChunkRecords } from "./chunk.ts";
|
|
15
|
+
export type { CGeoDataDirectory } from "./directory.ts";
|
|
16
|
+
export { CGeoDataError, CGeoDataFormatError, CGeoDataIndexMismatchError, CGeoDataUnsupportedError, CGeoDataVersionError, } from "./errors.ts";
|
|
17
|
+
export type { CGeoDataHeader } from "./header.ts";
|
|
18
|
+
export { type Inflater, streamInflater } from "./inflate.ts";
|
|
19
|
+
export { CGeoDataReader, type ChunkRecord, DEFAULT_CACHE_BYTES, type IndexOption, type Location, type OpenOptions, type Place, type PlaceCounts, type PlaceFilter, type PlaceIdResult, type PlaceIds, type PlaceLevel, type PlaceRef, type QueryResult, type SearchOptions, } from "./reader.ts";
|
|
20
|
+
export { BlobSource, type ByteSource, BytesSource, type SourceInput, } from "./source.ts";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CGEODATA reader: resolves a latitude/longitude to the country, region
|
|
3
|
+
* and city containing it, from a CGEODATA file (see binary-format.md).
|
|
4
|
+
*
|
|
5
|
+
* This entry point is platform-neutral. On Node.js, open files with
|
|
6
|
+
* `openFile()` from `./node`; anywhere else, hand `CGeoDataReader.open()`
|
|
7
|
+
* the file's bytes or a `Blob`. Search and locate indexes, and their
|
|
8
|
+
* builders, are in `./indexes`; the format's low-level codecs, for tools
|
|
9
|
+
* that write or inspect files, in `./format`.
|
|
10
|
+
*
|
|
11
|
+
* @module @ferrybig/cgeodata-reader
|
|
12
|
+
*/
|
|
13
|
+
export { boundingBoxContains } from "./cellBounds.js";
|
|
14
|
+
export { CGeoDataError, CGeoDataFormatError, CGeoDataIndexMismatchError, CGeoDataUnsupportedError, CGeoDataVersionError, } from "./errors.js";
|
|
15
|
+
export { streamInflater } from "./inflate.js";
|
|
16
|
+
export { CGeoDataReader, DEFAULT_CACHE_BYTES, } from "./reader.js";
|
|
17
|
+
export { BlobSource, BytesSource, } from "./source.js";
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The sidecar indexes (binary-format.md Appendices B and C): building
|
|
3
|
+
* them from a `.bin` file, and opening them yourself -- to share one
|
|
4
|
+
* index between several readers, say. A reader opened with `openFile()`
|
|
5
|
+
* already finds the ones next to its file; elsewhere,
|
|
6
|
+
* `OpenOptions.searchIndex` / `locateIndex` take an index's bytes.
|
|
7
|
+
*
|
|
8
|
+
* @module @ferrybig/cgeodata-reader/indexes
|
|
9
|
+
*/
|
|
10
|
+
export { type BuildLocateIndexOptions, buildLocateIndex, LocateIndex, } from "./locateIndex.ts";
|
|
11
|
+
export { buildSearchIndex, DEFAULT_SEARCH_INDEX_OPTIONS, SearchIndex, type SearchIndexOptions, } from "./searchIndex.ts";
|
package/dist/indexes.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The sidecar indexes (binary-format.md Appendices B and C): building
|
|
3
|
+
* them from a `.bin` file, and opening them yourself -- to share one
|
|
4
|
+
* index between several readers, say. A reader opened with `openFile()`
|
|
5
|
+
* already finds the ones next to its file; elsewhere,
|
|
6
|
+
* `OpenOptions.searchIndex` / `locateIndex` take an index's bytes.
|
|
7
|
+
*
|
|
8
|
+
* @module @ferrybig/cgeodata-reader/indexes
|
|
9
|
+
*/
|
|
10
|
+
export { buildLocateIndex, LocateIndex, } from "./locateIndex.js";
|
|
11
|
+
export { buildSearchIndex, DEFAULT_SEARCH_INDEX_OPTIONS, SearchIndex, } from "./searchIndex.js";
|
|
12
|
+
//# sourceMappingURL=indexes.js.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decompresses one zlib stream (RFC 1950, binary-format.md Section 4),
|
|
3
|
+
* refusing to produce more than `maxOutputLength` bytes. May answer
|
|
4
|
+
* synchronously: Node's inflater does for small blocks, where handing the
|
|
5
|
+
* work to a thread costs more than doing it.
|
|
6
|
+
*/
|
|
7
|
+
export type Inflater = (compressed: Uint8Array, maxOutputLength: number) => Uint8Array | Promise<Uint8Array>;
|
|
8
|
+
/** The most a block of `compressedLength` bytes may inflate to. */
|
|
9
|
+
export declare function inflateCeiling(compressedLength: number): number;
|
|
10
|
+
/**
|
|
11
|
+
* The platform-neutral inflater: the web's `DecompressionStream`, which
|
|
12
|
+
* browsers, Deno, Bun and Node.js all have. On Node.js, `./node.ts`'s
|
|
13
|
+
* zlib inflater is several times faster for small blocks.
|
|
14
|
+
*/
|
|
15
|
+
export declare const streamInflater: Inflater;
|
|
16
|
+
/** Inflates one compressed block (Section 4) with `inflater`, under the
|
|
17
|
+
* size ceiling above. Anything the inflater rejects -- a corrupt stream,
|
|
18
|
+
* or one that would inflate past the ceiling -- is the file's fault, so
|
|
19
|
+
* it surfaces as a `CGeoDataFormatError` with the inflater's own error as
|
|
20
|
+
* its `cause`. */
|
|
21
|
+
export declare function inflateBlock(inflater: Inflater, compressed: Uint8Array): Promise<Uint8Array>;
|
package/dist/inflate.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { CGeoDataFormatError } from "./errors.js";
|
|
2
|
+
// Section 11: readers MUST bound the inflated size of every compressed
|
|
3
|
+
// block, to defend against decompression bombs. zlib's documented
|
|
4
|
+
// worst-case expansion ratio is on the order of 1032:1; we round up and add
|
|
5
|
+
// a floor so tiny legitimate blocks are never starved.
|
|
6
|
+
const MAX_EXPANSION_RATIO = 1100;
|
|
7
|
+
const MIN_OUTPUT_CEILING = 1 << 20; // 1 MiB
|
|
8
|
+
/** The most a block of `compressedLength` bytes may inflate to. */
|
|
9
|
+
export function inflateCeiling(compressedLength) {
|
|
10
|
+
return Math.max(MIN_OUTPUT_CEILING, compressedLength * MAX_EXPANSION_RATIO);
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* The platform-neutral inflater: the web's `DecompressionStream`, which
|
|
14
|
+
* browsers, Deno, Bun and Node.js all have. On Node.js, `./node.ts`'s
|
|
15
|
+
* zlib inflater is several times faster for small blocks.
|
|
16
|
+
*/
|
|
17
|
+
export const streamInflater = async (compressed, maxOutputLength) => {
|
|
18
|
+
const stream = new DecompressionStream("deflate");
|
|
19
|
+
const writer = stream.writable.getWriter();
|
|
20
|
+
// Not awaited: the readable side below is what drains it, and a corrupt
|
|
21
|
+
// stream surfaces there (writer.closed's rejection is the same error).
|
|
22
|
+
writer.write(compressed).catch(() => { });
|
|
23
|
+
writer.close().catch(() => { });
|
|
24
|
+
const reader = stream.readable.getReader();
|
|
25
|
+
const parts = [];
|
|
26
|
+
let total = 0;
|
|
27
|
+
for (;;) {
|
|
28
|
+
const { done, value } = await reader.read();
|
|
29
|
+
if (done)
|
|
30
|
+
break;
|
|
31
|
+
total += value.length;
|
|
32
|
+
if (total > maxOutputLength) {
|
|
33
|
+
await reader.cancel();
|
|
34
|
+
throw new RangeError(`inflates past ${maxOutputLength} bytes, the limit for its size`);
|
|
35
|
+
}
|
|
36
|
+
parts.push(value);
|
|
37
|
+
}
|
|
38
|
+
if (parts.length === 1)
|
|
39
|
+
return parts[0];
|
|
40
|
+
const out = new Uint8Array(total);
|
|
41
|
+
let offset = 0;
|
|
42
|
+
for (const part of parts) {
|
|
43
|
+
out.set(part, offset);
|
|
44
|
+
offset += part.length;
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
};
|
|
48
|
+
/** Inflates one compressed block (Section 4) with `inflater`, under the
|
|
49
|
+
* size ceiling above. Anything the inflater rejects -- a corrupt stream,
|
|
50
|
+
* or one that would inflate past the ceiling -- is the file's fault, so
|
|
51
|
+
* it surfaces as a `CGeoDataFormatError` with the inflater's own error as
|
|
52
|
+
* its `cause`. */
|
|
53
|
+
export async function inflateBlock(inflater, compressed) {
|
|
54
|
+
try {
|
|
55
|
+
return await inflater(compressed, inflateCeiling(compressed.length));
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
throw new CGeoDataFormatError(`compressed block does not inflate: ${error.message}`, { cause: error });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=inflate.js.map
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Place location index (Appendix C of binary-format.md): a sidecar file
|
|
3
|
+
* that stores one representative cell per place, so that
|
|
4
|
+
* `CGeoDataReader.locate()` is a single read of a few bytes instead of a
|
|
5
|
+
* scan through every chunk.
|
|
6
|
+
*
|
|
7
|
+
* The file is CGEODATA kind 2: the 8-byte "CGEODATA" signature and a
|
|
8
|
+
* `version` varint (kind × 16 + version = 33), then an FNV-1a 64-bit
|
|
9
|
+
* checksum over the source's header, directory and place table, the
|
|
10
|
+
* entry width, the place count, one fixed-width entry per place id, and
|
|
11
|
+
* one area byte per place id. Entries are raw, so an entry is found by
|
|
12
|
+
* multiplying its id by the entry width.
|
|
13
|
+
*
|
|
14
|
+
* The representative cell and area of a place are defined in Section
|
|
15
|
+
* C.3, and `locate()` without an index computes exactly the same ones by
|
|
16
|
+
* scanning (`computeLocateCell`). The index is only a precomputed copy of
|
|
17
|
+
* them.
|
|
18
|
+
*/
|
|
19
|
+
import type { CGeoDataHeader } from "./header.ts";
|
|
20
|
+
import type { CGeoDataReader } from "./reader.ts";
|
|
21
|
+
import { type SourceInput } from "./source.ts";
|
|
22
|
+
/** The header fields a locate entry's layout depends on. */
|
|
23
|
+
export type LocateGrid = Pick<CGeoDataHeader, "widthExp" | "heightExp" | "maxDepth">;
|
|
24
|
+
/** One cell as an index entry names it (Section C.2): a cell of a chunk,
|
|
25
|
+
* addressed like a chunk record (Section 8.4) -- `code` is the stored
|
|
26
|
+
* `2 × max_depth`-bit Morton code, `depth` the cell's depth. */
|
|
27
|
+
export interface LocateCell {
|
|
28
|
+
/** The chunk the cell is in (Section 6.1). */
|
|
29
|
+
chunkIndex: number;
|
|
30
|
+
/** The cell's stored Morton code within its chunk (Section 8.1). */
|
|
31
|
+
code: number;
|
|
32
|
+
/** The cell's depth in the chunk's quadtree. */
|
|
33
|
+
depth: number;
|
|
34
|
+
}
|
|
35
|
+
/** `ceil((width_exp + height_exp + 2 × max_depth + 1) / 8)`: enough bytes
|
|
36
|
+
* for a depth-`max_depth` cell plus its marker bit. */
|
|
37
|
+
export declare function locateEntryWidth(grid: LocateGrid): number;
|
|
38
|
+
/** The entry for `cell`, or `0` ("no geometry") for null. */
|
|
39
|
+
export declare function encodeLocateEntry(cell: LocateCell | null, grid: LocateGrid): number;
|
|
40
|
+
/** Inverse of `encodeLocateEntry`: null for `0`, and an error for a value
|
|
41
|
+
* whose marker bit doesn't sit at a valid depth. */
|
|
42
|
+
export declare function decodeLocateEntry(value: number, grid: LocateGrid): LocateCell | null;
|
|
43
|
+
/**
|
|
44
|
+
* Record cells in global full-precision cell coordinates -- Section 9's
|
|
45
|
+
* `gx`/`gy` grid, `W × 2^D` columns by `H × 2^D` rows. Cell `i` covers
|
|
46
|
+
* columns `gx[i] .. gx[i] + 2^(D - depth[i]) - 1` and the same run of
|
|
47
|
+
* rows from `gy[i]`.
|
|
48
|
+
*/
|
|
49
|
+
export interface CellList {
|
|
50
|
+
gx: Uint32Array;
|
|
51
|
+
gy: Uint32Array;
|
|
52
|
+
depth: Uint8Array;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* The representative cell of Section C.3 for the cells `start .. end - 1`
|
|
56
|
+
* of `cells`: the full-precision cell nearest to the per-axis weighted
|
|
57
|
+
* median of the cells' centres, among the cells themselves. Returned in
|
|
58
|
+
* global full-precision coordinates; undefined for an empty range.
|
|
59
|
+
*/
|
|
60
|
+
export declare function representativeCell(cells: CellList, start: number, end: number, header: CGeoDataHeader): {
|
|
61
|
+
gx: number;
|
|
62
|
+
gy: number;
|
|
63
|
+
} | undefined;
|
|
64
|
+
/** A place's representative cell and area byte (Section C.3). */
|
|
65
|
+
export interface PlaceLocation {
|
|
66
|
+
/** The representative cell. */
|
|
67
|
+
cell: LocateCell;
|
|
68
|
+
/** The area byte; never 0, since a place with a cell has an area. */
|
|
69
|
+
area: number;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Section C.3 for one place, straight from the chunks: collects the cells
|
|
73
|
+
* of every record attributed to `id` or a place under it, then picks the
|
|
74
|
+
* representative cell and totals the area. This is what `locate()` does
|
|
75
|
+
* without an index, and it reads every non-empty chunk.
|
|
76
|
+
*/
|
|
77
|
+
export declare function computeLocateCell(reader: CGeoDataReader, id: number): Promise<PlaceLocation | null>;
|
|
78
|
+
/** Tuning for `buildLocateIndex`; the output doesn't depend on it. */
|
|
79
|
+
export interface BuildLocateIndexOptions {
|
|
80
|
+
/**
|
|
81
|
+
* Roughly how many records to hold in memory at once. Countries are
|
|
82
|
+
* processed in batches whose records add up to about this many (a single
|
|
83
|
+
* country bigger than this is its own batch), and every chunk a batch
|
|
84
|
+
* touches is decoded once per batch. At ~9 bytes a record plus sort
|
|
85
|
+
* buffers, the default keeps the builder under a few hundred MB.
|
|
86
|
+
*/
|
|
87
|
+
batchRecords?: number;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Builds a locate index (Section C.4) for every place in `reader` and
|
|
91
|
+
* returns the file's bytes.
|
|
92
|
+
*
|
|
93
|
+
* Every place's cells are gathered from the chunks country by country.
|
|
94
|
+
* Within a batch the places are laid out in depth-first order -- a
|
|
95
|
+
* country, then each of its regions followed by that region's cities --
|
|
96
|
+
* so the records of any place and everything under it are one contiguous
|
|
97
|
+
* run, and each place is one call to `representativeCell`.
|
|
98
|
+
*/
|
|
99
|
+
export declare function buildLocateIndex(reader: CGeoDataReader, options?: BuildLocateIndexOptions): Promise<Uint8Array>;
|
|
100
|
+
/**
|
|
101
|
+
* An open locate index file (Section C.4). Opening reads only the header,
|
|
102
|
+
* and each `entry()` or `area()` reads just that entry's bytes; only
|
|
103
|
+
* `areas()`, which a search ranked by area needs, keeps anything (one
|
|
104
|
+
* byte per place), so an idle index otherwise costs an open file handle
|
|
105
|
+
* and no memory.
|
|
106
|
+
*
|
|
107
|
+
* Opening does NOT verify the source checksum -- `CGeoDataReader`'s
|
|
108
|
+
* `useLocateIndex()` does that against the file it's attached to.
|
|
109
|
+
*/
|
|
110
|
+
export declare class LocateIndex {
|
|
111
|
+
private readonly source;
|
|
112
|
+
/** The index file's name, when its source has one (`ByteSource.name`). */
|
|
113
|
+
readonly name?: string;
|
|
114
|
+
private readonly entriesStart;
|
|
115
|
+
private readonly areasStart;
|
|
116
|
+
private areasPromise;
|
|
117
|
+
/** Checksum of the header, directory and place table it was built
|
|
118
|
+
* from (Section C.4); a reader only accepts the index when it matches
|
|
119
|
+
* its own. */
|
|
120
|
+
readonly sourceChecksum: bigint;
|
|
121
|
+
/** Bytes per entry (Section C.2). */
|
|
122
|
+
readonly entryWidth: number;
|
|
123
|
+
/** How many places it has an entry for. */
|
|
124
|
+
readonly placeCount: number;
|
|
125
|
+
private constructor();
|
|
126
|
+
/** Opens a locate index file (read on demand, like a `.bin`).
|
|
127
|
+
* Validates signature, kind, version, entry width and length. Takes
|
|
128
|
+
* ownership of a `ByteSource`: `close()` closes it, as does a failed
|
|
129
|
+
* open. */
|
|
130
|
+
static open(input: SourceInput): Promise<LocateIndex>;
|
|
131
|
+
private static parseHeader;
|
|
132
|
+
/** Closes the index's source. */
|
|
133
|
+
close(): Promise<void>;
|
|
134
|
+
/** Place `id`'s raw entry (Section C.2); `0` means no geometry. */
|
|
135
|
+
entry(id: number): Promise<number>;
|
|
136
|
+
/** Place `id`'s area byte (Section C.3); `0` means no area. */
|
|
137
|
+
area(id: number): Promise<number>;
|
|
138
|
+
/** Every place's area byte, by id: one read of `placeCount` bytes, the
|
|
139
|
+
* first time, then kept -- what ranking a search by area needs. */
|
|
140
|
+
areas(): Promise<Uint8Array>;
|
|
141
|
+
private checkId;
|
|
142
|
+
}
|