@ferrybig/cgeodata-reader 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +313 -2
- package/binary-format.md +2366 -0
- package/dist/cache.d.ts +24 -0
- package/dist/cache.js +60 -0
- package/dist/cellBounds.d.ts +19 -0
- package/dist/cellBounds.js +36 -0
- package/dist/chunk.d.ts +34 -0
- package/dist/chunk.js +269 -0
- package/dist/chunkDecoder.d.ts +39 -0
- package/dist/chunkDecoder.js +116 -0
- package/dist/chunkDecoderWasm.d.ts +2 -0
- package/dist/chunkDecoderWasm.js +4 -0
- package/dist/cli/buildLocateIndex.d.ts +2 -0
- package/dist/cli/buildLocateIndex.js +46 -0
- package/dist/cli/buildSearchIndex.d.ts +2 -0
- package/dist/cli/buildSearchIndex.js +90 -0
- package/dist/cli/chunk.d.ts +2 -0
- package/dist/cli/chunk.js +67 -0
- package/dist/cli/info.d.ts +2 -0
- package/dist/cli/info.js +87 -0
- package/dist/cli/list.d.ts +2 -0
- package/dist/cli/list.js +71 -0
- package/dist/cli/locate.d.ts +2 -0
- package/dist/cli/locate.js +64 -0
- package/dist/cli/lookup.d.ts +2 -0
- package/dist/cli/lookup.js +128 -0
- package/dist/cli/search.d.ts +2 -0
- package/dist/cli/search.js +119 -0
- package/dist/cli/shared.d.ts +50 -0
- package/dist/cli/shared.js +140 -0
- package/dist/cli/show.d.ts +2 -0
- package/dist/cli/show.js +38 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +107 -0
- package/dist/container.d.ts +17 -0
- package/dist/container.js +68 -0
- package/dist/directory.d.ts +21 -0
- package/dist/directory.js +31 -0
- package/dist/errors.d.ts +52 -0
- package/dist/errors.js +56 -0
- package/dist/format.d.ts +21 -0
- package/dist/format.js +22 -0
- package/dist/header.d.ts +29 -0
- package/dist/header.js +69 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.js +18 -0
- package/dist/indexes.d.ts +11 -0
- package/dist/indexes.js +12 -0
- package/dist/inflate.d.ts +21 -0
- package/dist/inflate.js +61 -0
- package/dist/locateIndex.d.ts +142 -0
- package/dist/locateIndex.js +561 -0
- package/dist/morton.d.ts +28 -0
- package/dist/morton.js +51 -0
- package/dist/node.d.ts +65 -0
- package/dist/node.js +137 -0
- package/dist/placeTable.d.ts +69 -0
- package/dist/placeTable.js +177 -0
- package/dist/reader.d.ts +534 -0
- package/dist/reader.js +1128 -0
- package/dist/relevance.d.ts +184 -0
- package/dist/relevance.js +266 -0
- package/dist/searchIndex.d.ts +148 -0
- package/dist/searchIndex.js +786 -0
- package/dist/source.d.ts +49 -0
- package/dist/source.js +50 -0
- package/dist/turns.d.ts +19 -0
- package/dist/turns.js +40 -0
- package/dist/varint.d.ts +52 -0
- package/dist/varint.js +137 -0
- package/package.json +76 -5
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { type ParseArgsConfig } from "node:util";
|
|
2
|
+
import type { BoundingBox } from "../cellBounds.ts";
|
|
3
|
+
import { FileSource } from "../node.ts";
|
|
4
|
+
import { CGeoDataReader, type Place, type PlaceLevel } from "../reader.ts";
|
|
5
|
+
/** grep's convention: 0 = success/matched, 1 = ran fine but found nothing
|
|
6
|
+
* (search only), 2 = usage or runtime error. */
|
|
7
|
+
export declare const EXIT_OK = 0;
|
|
8
|
+
export declare const EXIT_NO_MATCH = 1;
|
|
9
|
+
export declare const EXIT_ERROR = 2;
|
|
10
|
+
/** A usage problem: reported with the command's usage text, exit 2. */
|
|
11
|
+
export declare class UsageError extends Error {
|
|
12
|
+
}
|
|
13
|
+
export interface Command {
|
|
14
|
+
name: string;
|
|
15
|
+
summary: string;
|
|
16
|
+
usage: string;
|
|
17
|
+
run(argv: string[]): Promise<number> | number;
|
|
18
|
+
}
|
|
19
|
+
type Options = NonNullable<ParseArgsConfig["options"]>;
|
|
20
|
+
export declare function parse<O extends Options>(argv: string[], options: O): {
|
|
21
|
+
values: O extends import("util").ParseArgsOptionsConfig ? { -readonly [LongOption in keyof O]?: { [LongOption_2 in keyof O]: O[LongOption_2]["multiple"] extends infer T ? T extends O[LongOption_2]["multiple"] ? T extends false ? O[LongOption_2]["type"] extends "string" ? string : O[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : T extends true ? (O[LongOption_2]["type"] extends "string" ? string : O[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean)[] : O[LongOption_2]["type"] extends "string" ? string : O[LongOption_2]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption]; } & { [LongOption_1 in keyof O as O[LongOption_1]["default"] extends {} ? LongOption_1 : never]: { [LongOption_3 in keyof O]: O[LongOption_3]["multiple"] extends infer T ? T extends O[LongOption_3]["multiple"] ? T extends false ? O[LongOption_3]["type"] extends "string" ? string : O[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : T extends true ? (O[LongOption_3]["type"] extends "string" ? string : O[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean)[] : O[LongOption_3]["type"] extends "string" ? string : O[LongOption_3]["type"] extends "boolean" ? boolean : string | boolean : never : never; }[LongOption_1]; } extends infer P ? { [K in keyof P]: P[K]; } : never : {};
|
|
22
|
+
positionals: string[];
|
|
23
|
+
};
|
|
24
|
+
/** "-" reads the whole file from stdin; the reader needs random access,
|
|
25
|
+
* so that means buffering it, which is fine at the sizes these files come
|
|
26
|
+
* in (the worldwide build is under 10MB). */
|
|
27
|
+
export declare function openReader(file: string | undefined): Promise<CGeoDataReader>;
|
|
28
|
+
/** `--index <file>`: attaches that index file instead of the one next to
|
|
29
|
+
* the .bin, naming it in any error. */
|
|
30
|
+
export declare function useIndexFile(use: (index: FileSource) => Promise<void>, path: string): Promise<void>;
|
|
31
|
+
export declare function parseLevel(text: string | undefined): PlaceLevel | undefined;
|
|
32
|
+
/** `--area-tie-break auto|on|off`, as `SearchOptions.areaTieBreak`. */
|
|
33
|
+
export declare function parseAreaTieBreak(text: string | undefined): "auto" | boolean;
|
|
34
|
+
export declare function parseId(text: string, what: string): number;
|
|
35
|
+
/** `--country` takes an id or a name; a name must match exactly one
|
|
36
|
+
* country, case-insensitively, on any of its names. */
|
|
37
|
+
export declare function resolveCountry(reader: CGeoDataReader, text: string): Promise<number>;
|
|
38
|
+
export declare function names(list: string[] | null | undefined): string;
|
|
39
|
+
/** One place per line: id, level, its own names, region, country -- `-`
|
|
40
|
+
* where a level doesn't apply. Fixed columns so `cut`/`awk` work
|
|
41
|
+
* regardless of level. */
|
|
42
|
+
export declare function formatPlaceLine(place: Place): string;
|
|
43
|
+
/** The block form used by `show` and `lookup`. */
|
|
44
|
+
export declare function formatPlaceBlock(place: Place): string[];
|
|
45
|
+
/** Decimal places that meaningfully represent a coordinate given the cell
|
|
46
|
+
* it came out of. Shared by `locate` and `search --locate`, which have to
|
|
47
|
+
* print the same coordinate the same way. */
|
|
48
|
+
export declare function coordPrecision(box: BoundingBox): number;
|
|
49
|
+
export declare function writeLine(line: string): void;
|
|
50
|
+
export {};
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { parseArgs } from "node:util";
|
|
3
|
+
import { FileSource, nodeInflater, openFile } from "../node.js";
|
|
4
|
+
import { CGeoDataReader } from "../reader.js";
|
|
5
|
+
/** grep's convention: 0 = success/matched, 1 = ran fine but found nothing
|
|
6
|
+
* (search only), 2 = usage or runtime error. */
|
|
7
|
+
export const EXIT_OK = 0;
|
|
8
|
+
export const EXIT_NO_MATCH = 1;
|
|
9
|
+
export const EXIT_ERROR = 2;
|
|
10
|
+
/** A usage problem: reported with the command's usage text, exit 2. */
|
|
11
|
+
export class UsageError extends Error {
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* parseArgs with every option unknown to the command rejected, and
|
|
15
|
+
* negative numbers ("-10,10", a perfectly good latitude) allowed as
|
|
16
|
+
* positionals -- parseArgs would otherwise read that as short options
|
|
17
|
+
* `-1 -0 ...`. Such tokens are hidden behind a NUL prefix for the parse
|
|
18
|
+
* and restored afterwards; nothing else can legitimately start with NUL.
|
|
19
|
+
*
|
|
20
|
+
* The leading digit may come after a decimal point: "-.5,10" is as good a
|
|
21
|
+
* coordinate as "-0.5,10", and matching only `-\d` rejected it with
|
|
22
|
+
* "Unknown option '-.'".
|
|
23
|
+
*/
|
|
24
|
+
const NEGATIVE_NUMBER = /^-(?:\d|\.\d)/;
|
|
25
|
+
export function parse(argv, options) {
|
|
26
|
+
const shielded = argv.map((arg) => NEGATIVE_NUMBER.test(arg) ? `\0${arg}` : arg);
|
|
27
|
+
try {
|
|
28
|
+
const { values, positionals } = parseArgs({
|
|
29
|
+
args: shielded,
|
|
30
|
+
options,
|
|
31
|
+
allowPositionals: true,
|
|
32
|
+
strict: true,
|
|
33
|
+
});
|
|
34
|
+
return {
|
|
35
|
+
values,
|
|
36
|
+
positionals: positionals.map((p) => p.startsWith("\0") ? p.slice(1) : p),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
catch (err) {
|
|
40
|
+
throw new UsageError(err instanceof Error
|
|
41
|
+
? err.message.replace(/\. To specify.*$/s, "")
|
|
42
|
+
: String(err));
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** "-" reads the whole file from stdin; the reader needs random access,
|
|
46
|
+
* so that means buffering it, which is fine at the sizes these files come
|
|
47
|
+
* in (the worldwide build is under 10MB). */
|
|
48
|
+
export async function openReader(file) {
|
|
49
|
+
if (file === undefined)
|
|
50
|
+
throw new UsageError("missing <file.bin>");
|
|
51
|
+
return file === "-"
|
|
52
|
+
? CGeoDataReader.open(readFileSync(0), { inflater: nodeInflater })
|
|
53
|
+
: openFile(file);
|
|
54
|
+
}
|
|
55
|
+
/** `--index <file>`: attaches that index file instead of the one next to
|
|
56
|
+
* the .bin, naming it in any error. */
|
|
57
|
+
export async function useIndexFile(use, path) {
|
|
58
|
+
try {
|
|
59
|
+
await use(await FileSource.open(path));
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
throw new Error(`--index ${path}: ${error.message}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
export function parseLevel(text) {
|
|
66
|
+
if (text === undefined)
|
|
67
|
+
return undefined;
|
|
68
|
+
if (text === "country" || text === "region" || text === "city")
|
|
69
|
+
return text;
|
|
70
|
+
throw new UsageError(`--level must be country, region or city, not ${JSON.stringify(text)}`);
|
|
71
|
+
}
|
|
72
|
+
/** `--area-tie-break auto|on|off`, as `SearchOptions.areaTieBreak`. */
|
|
73
|
+
export function parseAreaTieBreak(text) {
|
|
74
|
+
if (text === undefined || text === "auto")
|
|
75
|
+
return "auto";
|
|
76
|
+
if (text === "on")
|
|
77
|
+
return true;
|
|
78
|
+
if (text === "off")
|
|
79
|
+
return false;
|
|
80
|
+
throw new UsageError(`--area-tie-break must be auto, on or off, not ${JSON.stringify(text)}`);
|
|
81
|
+
}
|
|
82
|
+
export function parseId(text, what) {
|
|
83
|
+
const id = Number(text);
|
|
84
|
+
if (!/^\d+$/.test(text) || !Number.isSafeInteger(id))
|
|
85
|
+
throw new UsageError(`${what} ${JSON.stringify(text)} is not a place id`);
|
|
86
|
+
return id;
|
|
87
|
+
}
|
|
88
|
+
/** `--country` takes an id or a name; a name must match exactly one
|
|
89
|
+
* country, case-insensitively, on any of its names. */
|
|
90
|
+
export async function resolveCountry(reader, text) {
|
|
91
|
+
if (/^\d+$/.test(text))
|
|
92
|
+
return parseId(text, "--country");
|
|
93
|
+
const wanted = text.toLowerCase();
|
|
94
|
+
const matches = [];
|
|
95
|
+
for await (const c of reader.places({ level: "country" })) {
|
|
96
|
+
if (c.names.some((n) => n.toLowerCase() === wanted))
|
|
97
|
+
matches.push(c);
|
|
98
|
+
}
|
|
99
|
+
if (matches.length === 1)
|
|
100
|
+
return matches[0].id;
|
|
101
|
+
if (matches.length === 0)
|
|
102
|
+
throw new UsageError(`no country is named ${JSON.stringify(text)}`);
|
|
103
|
+
throw new UsageError(`${JSON.stringify(text)} names ${matches.length} countries: ${matches.map((m) => `${m.id} (${m.names.join(" | ")})`).join(", ")}`);
|
|
104
|
+
}
|
|
105
|
+
export function names(list) {
|
|
106
|
+
return list ? list.join(" | ") : "-";
|
|
107
|
+
}
|
|
108
|
+
/** One place per line: id, level, its own names, region, country -- `-`
|
|
109
|
+
* where a level doesn't apply. Fixed columns so `cut`/`awk` work
|
|
110
|
+
* regardless of level. */
|
|
111
|
+
export function formatPlaceLine(place) {
|
|
112
|
+
const region = place.level === "city" ? place.region?.names : null;
|
|
113
|
+
const country = place.level === "country" ? null : place.country.names;
|
|
114
|
+
return [
|
|
115
|
+
place.id,
|
|
116
|
+
place.level,
|
|
117
|
+
names(place.names),
|
|
118
|
+
names(region),
|
|
119
|
+
names(country),
|
|
120
|
+
].join("\t");
|
|
121
|
+
}
|
|
122
|
+
/** The block form used by `show` and `lookup`. */
|
|
123
|
+
export function formatPlaceBlock(place) {
|
|
124
|
+
return [
|
|
125
|
+
` country ${names(place.country.names)} [${place.country.id}]`,
|
|
126
|
+
` region ${place.region ? `${names(place.region.names)} [${place.region.id}]` : "-"}`,
|
|
127
|
+
` city ${place.city ? `${names(place.city.names)} [${place.city.id}]` : "-"}`,
|
|
128
|
+
];
|
|
129
|
+
}
|
|
130
|
+
/** Decimal places that meaningfully represent a coordinate given the cell
|
|
131
|
+
* it came out of. Shared by `locate` and `search --locate`, which have to
|
|
132
|
+
* print the same coordinate the same way. */
|
|
133
|
+
export function coordPrecision(box) {
|
|
134
|
+
const cellSize = Math.min(box.maxLat - box.minLat, box.maxLon - box.minLon);
|
|
135
|
+
return Math.max(0, Math.round(-Math.log10(cellSize) + 1));
|
|
136
|
+
}
|
|
137
|
+
export function writeLine(line) {
|
|
138
|
+
process.stdout.write(line + "\n");
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=shared.js.map
|
package/dist/cli/show.js
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { formatPlaceBlock, openReader, parse, parseId, UsageError, writeLine, } from "./shared.js";
|
|
2
|
+
export const show = {
|
|
3
|
+
name: "show",
|
|
4
|
+
summary: "resolve place ids to names and parents",
|
|
5
|
+
usage: `usage: cgeodata show <file.bin> <id> [<id> ...] [--json]
|
|
6
|
+
|
|
7
|
+
Resolves each place id (as printed by list, search, lookup or chunk) to
|
|
8
|
+
its level, names and ancestors.
|
|
9
|
+
|
|
10
|
+
--json one JSON object per id instead of the readable form`,
|
|
11
|
+
async run(argv) {
|
|
12
|
+
const { values, positionals } = parse(argv, {
|
|
13
|
+
json: { type: "boolean", default: false },
|
|
14
|
+
});
|
|
15
|
+
const [file, ...idArgs] = positionals;
|
|
16
|
+
if (idArgs.length === 0)
|
|
17
|
+
throw new UsageError("missing <id>");
|
|
18
|
+
const ids = idArgs.map((arg) => parseId(arg, "id"));
|
|
19
|
+
const reader = await openReader(file);
|
|
20
|
+
try {
|
|
21
|
+
for (const id of ids) {
|
|
22
|
+
const place = await reader.place(id);
|
|
23
|
+
if (values.json) {
|
|
24
|
+
writeLine(JSON.stringify(place));
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
writeLine(`${place.id}\t${place.level}`);
|
|
28
|
+
for (const line of formatPlaceBlock(place))
|
|
29
|
+
writeLine(line);
|
|
30
|
+
}
|
|
31
|
+
return 0;
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
await reader.close();
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
//# sourceMappingURL=show.js.map
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { buildLocateIndexCmd } from "./cli/buildLocateIndex.js";
|
|
4
|
+
import { buildSearchIndexCmd } from "./cli/buildSearchIndex.js";
|
|
5
|
+
import { chunk } from "./cli/chunk.js";
|
|
6
|
+
import { info } from "./cli/info.js";
|
|
7
|
+
import { list } from "./cli/list.js";
|
|
8
|
+
import { locate } from "./cli/locate.js";
|
|
9
|
+
import { lookup } from "./cli/lookup.js";
|
|
10
|
+
import { search } from "./cli/search.js";
|
|
11
|
+
import { EXIT_ERROR, UsageError } from "./cli/shared.js";
|
|
12
|
+
import { show } from "./cli/show.js";
|
|
13
|
+
const COMMANDS = [
|
|
14
|
+
info,
|
|
15
|
+
lookup,
|
|
16
|
+
list,
|
|
17
|
+
search,
|
|
18
|
+
buildSearchIndexCmd,
|
|
19
|
+
show,
|
|
20
|
+
chunk,
|
|
21
|
+
locate,
|
|
22
|
+
buildLocateIndexCmd,
|
|
23
|
+
];
|
|
24
|
+
// Wide enough for the longest name there actually is: a fixed 8 left
|
|
25
|
+
// `build-search-index` and `build-locate-index` hanging ten characters
|
|
26
|
+
// into their own summaries.
|
|
27
|
+
const NAME_WIDTH = Math.max(...COMMANDS.map((c) => c.name.length));
|
|
28
|
+
const USAGE = `usage: cgeodata <command> <file.bin> [...]
|
|
29
|
+
cgeodata help [<command>]
|
|
30
|
+
|
|
31
|
+
Reads a CGEODATA file. <file.bin> may be "-" to read the file from stdin.
|
|
32
|
+
|
|
33
|
+
commands:
|
|
34
|
+
${COMMANDS.map((c) => ` ${c.name.padEnd(NAME_WIDTH)} ${c.summary}`).join("\n")}
|
|
35
|
+
|
|
36
|
+
Options:
|
|
37
|
+
-h, --help show this help
|
|
38
|
+
-v, --version print the package version
|
|
39
|
+
|
|
40
|
+
Exit status is 0 on success, 1 when a search matched nothing, 2 on a
|
|
41
|
+
usage or runtime error.`;
|
|
42
|
+
function findCommand(name) {
|
|
43
|
+
return COMMANDS.find((c) => c.name === name);
|
|
44
|
+
}
|
|
45
|
+
// `../package.json` is the package's own from both `src/` and `dist/`.
|
|
46
|
+
function packageVersion() {
|
|
47
|
+
const manifest = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
|
|
48
|
+
return manifest.version;
|
|
49
|
+
}
|
|
50
|
+
async function main(argv) {
|
|
51
|
+
const [name, ...rest] = argv;
|
|
52
|
+
if (name === "--version" || name === "-v") {
|
|
53
|
+
process.stdout.write(`${packageVersion()}\n`);
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
if (name === undefined ||
|
|
57
|
+
name === "help" ||
|
|
58
|
+
name === "--help" ||
|
|
59
|
+
name === "-h") {
|
|
60
|
+
const topic = rest[0] === undefined ? undefined : findCommand(rest[0]);
|
|
61
|
+
if (rest[0] !== undefined && !topic) {
|
|
62
|
+
process.stderr.write(`unknown command ${JSON.stringify(rest[0])}\n${USAGE}\n`);
|
|
63
|
+
return EXIT_ERROR;
|
|
64
|
+
}
|
|
65
|
+
const text = topic ? topic.usage : USAGE;
|
|
66
|
+
if (name === undefined) {
|
|
67
|
+
process.stderr.write(`${text}\n`);
|
|
68
|
+
return EXIT_ERROR;
|
|
69
|
+
}
|
|
70
|
+
process.stdout.write(`${text}\n`);
|
|
71
|
+
return 0;
|
|
72
|
+
}
|
|
73
|
+
const command = findCommand(name);
|
|
74
|
+
if (!command) {
|
|
75
|
+
process.stderr.write(`unknown command ${JSON.stringify(name)}\n${USAGE}\n`);
|
|
76
|
+
return EXIT_ERROR;
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
return await command.run(rest);
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
if (err instanceof UsageError) {
|
|
83
|
+
process.stderr.write(`${err.message}\n${command.usage}\n`);
|
|
84
|
+
return EXIT_ERROR;
|
|
85
|
+
}
|
|
86
|
+
throw err;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
// `cgeodata list world.bin | head` closes our stdout early; that's the
|
|
90
|
+
// reader's business, not an error worth a stack trace.
|
|
91
|
+
process.stdout.on("error", (err) => {
|
|
92
|
+
if (err.code === "EPIPE")
|
|
93
|
+
process.exit(0);
|
|
94
|
+
// Throwing from an 'error' listener raises an uncaughtException, so a
|
|
95
|
+
// plain write failure (`cgeodata list world.bin > /dev/full`, ENOSPC)
|
|
96
|
+
// would print a stack trace -- the very thing this handler exists to
|
|
97
|
+
// avoid. Report it the way main()'s own failure path does.
|
|
98
|
+
process.stderr.write(`${err.message}\n`);
|
|
99
|
+
process.exit(EXIT_ERROR);
|
|
100
|
+
});
|
|
101
|
+
main(process.argv.slice(2)).then((code) => {
|
|
102
|
+
process.exitCode = code;
|
|
103
|
+
}, (err) => {
|
|
104
|
+
process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
105
|
+
process.exitCode = EXIT_ERROR;
|
|
106
|
+
});
|
|
107
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type CGeoDataDirectory } from "./directory.ts";
|
|
2
|
+
import { type CGeoDataHeader } from "./header.ts";
|
|
3
|
+
import type { ByteSource } from "./source.ts";
|
|
4
|
+
/**
|
|
5
|
+
* Reads the header and directory (Sections 5-6), which per the spec are
|
|
6
|
+
* "small and expected to be read once and kept in memory". Since the
|
|
7
|
+
* directory's size depends on the header (W x H entries) and the header's
|
|
8
|
+
* size depends on the attribution string length, this starts with a small
|
|
9
|
+
* probe read and grows it until both parse cleanly -- never reading more of
|
|
10
|
+
* the file than these two small, fixed-at-load-time sections actually need.
|
|
11
|
+
*/
|
|
12
|
+
export declare function readHeaderAndDirectory(source: ByteSource): Promise<{
|
|
13
|
+
/** The parsed header (Section 5). */
|
|
14
|
+
header: CGeoDataHeader;
|
|
15
|
+
/** The parsed directory (Section 6). */
|
|
16
|
+
directory: CGeoDataDirectory;
|
|
17
|
+
}>;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { parseDirectory } from "./directory.js";
|
|
2
|
+
import { CGeoDataFormatError } from "./errors.js";
|
|
3
|
+
import { parseHeader } from "./header.js";
|
|
4
|
+
import { IncompleteBufferError } from "./varint.js";
|
|
5
|
+
/**
|
|
6
|
+
* Reads the header and directory (Sections 5-6), which per the spec are
|
|
7
|
+
* "small and expected to be read once and kept in memory". Since the
|
|
8
|
+
* directory's size depends on the header (W x H entries) and the header's
|
|
9
|
+
* size depends on the attribution string length, this starts with a small
|
|
10
|
+
* probe read and grows it until both parse cleanly -- never reading more of
|
|
11
|
+
* the file than these two small, fixed-at-load-time sections actually need.
|
|
12
|
+
*/
|
|
13
|
+
export async function readHeaderAndDirectory(source) {
|
|
14
|
+
let size = Math.min(1024, source.size);
|
|
15
|
+
for (;;) {
|
|
16
|
+
const buf = await source.read(0, size);
|
|
17
|
+
try {
|
|
18
|
+
const header = parseHeader(buf);
|
|
19
|
+
checkDirectoryFits(header, source.size);
|
|
20
|
+
const directory = parseDirectory(buf, header.headerLength, header.width, header.height);
|
|
21
|
+
checkSectionsWithinFile(directory, source.size);
|
|
22
|
+
return { header, directory };
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
if (err instanceof IncompleteBufferError && size < source.size) {
|
|
26
|
+
size = Math.min(size * 4, source.size);
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
throw err;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* The cheap half of Section 11's "chunk offsets and lengths MUST lie
|
|
35
|
+
* within the file", checkable from the header alone: the directory is
|
|
36
|
+
* `place_table_length` plus one varint per chunk, every varint at least
|
|
37
|
+
* one byte, so a header claiming more chunks than the file has bytes
|
|
38
|
+
* left cannot be well formed.
|
|
39
|
+
*
|
|
40
|
+
* Checking this *before* parsing matters: `width_exp` and `height_exp` of
|
|
41
|
+
* 15 are both legal (Section 5) and claim 2^30 chunks, so without this a
|
|
42
|
+
* ~1 MiB crafted file sends the growing read above through a dozen passes
|
|
43
|
+
* that each allocate a 2^30-entry array -- ~1 s of CPU and a few hundred
|
|
44
|
+
* MB of RSS -- before finally failing on a short varint.
|
|
45
|
+
*/
|
|
46
|
+
function checkDirectoryFits(header, fileSize) {
|
|
47
|
+
const chunkCount = header.width * header.height;
|
|
48
|
+
const afterHeader = fileSize - header.headerLength;
|
|
49
|
+
if (chunkCount + 1 > afterHeader) {
|
|
50
|
+
throw new CGeoDataFormatError(`header's ${header.width}x${header.height} grid needs a directory of at least ${chunkCount + 1} bytes, but only ${afterHeader} follow the header in this ${fileSize}-byte file`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** The rest of Section 11's bounds check. Chunk offsets are the running
|
|
54
|
+
* sum of the lengths from the place table's end, so the last chunk's end
|
|
55
|
+
* bounds every chunk's. */
|
|
56
|
+
function checkSectionsWithinFile(directory, fileSize) {
|
|
57
|
+
const { placeTableStart, placeTableLength, chunkOffsets, chunkLengths } = directory;
|
|
58
|
+
const placeTableEnd = placeTableStart + placeTableLength;
|
|
59
|
+
if (placeTableEnd > fileSize) {
|
|
60
|
+
throw new CGeoDataFormatError(`place table runs to byte ${placeTableEnd}, past the end of the ${fileSize}-byte file`);
|
|
61
|
+
}
|
|
62
|
+
const last = chunkLengths.length - 1;
|
|
63
|
+
const chunksEnd = last < 0 ? placeTableEnd : chunkOffsets[last] + chunkLengths[last];
|
|
64
|
+
if (chunksEnd > fileSize) {
|
|
65
|
+
throw new CGeoDataFormatError(`chunks run to byte ${chunksEnd}, past the end of the ${fileSize}-byte file`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=container.js.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** The directory (Section 6): where the place table and each chunk are. */
|
|
2
|
+
export interface CGeoDataDirectory {
|
|
3
|
+
/** Absolute file offset of the place table (Section 7). */
|
|
4
|
+
placeTableStart: number;
|
|
5
|
+
/** Byte length of the place table. */
|
|
6
|
+
placeTableLength: number;
|
|
7
|
+
/** Entry i is the compressed byte length of chunk i (0 = empty). */
|
|
8
|
+
chunkLengths: number[];
|
|
9
|
+
/** Entry i is the absolute file offset of chunk i (meaningless when
|
|
10
|
+
* chunkLengths[i] === 0). Precomputed per Section 6.3. */
|
|
11
|
+
chunkOffsets: number[];
|
|
12
|
+
/** Byte length of the directory itself. */
|
|
13
|
+
directoryLength: number;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Parses Section 6. `buf` MUST start at the file's absolute byte 0 (not a
|
|
17
|
+
* sub-slice), since offsets computed here -- and returned in
|
|
18
|
+
* `placeTableStart`/`chunkOffsets` -- are absolute file offsets. `offset` is
|
|
19
|
+
* the header length, i.e. where the directory begins.
|
|
20
|
+
*/
|
|
21
|
+
export declare function parseDirectory(buf: Uint8Array, offset: number, width: number, height: number): CGeoDataDirectory;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { readVarint, readVarintList } from "./varint.js";
|
|
2
|
+
/**
|
|
3
|
+
* Parses Section 6. `buf` MUST start at the file's absolute byte 0 (not a
|
|
4
|
+
* sub-slice), since offsets computed here -- and returned in
|
|
5
|
+
* `placeTableStart`/`chunkOffsets` -- are absolute file offsets. `offset` is
|
|
6
|
+
* the header length, i.e. where the directory begins.
|
|
7
|
+
*/
|
|
8
|
+
export function parseDirectory(buf, offset, width, height) {
|
|
9
|
+
const start = offset;
|
|
10
|
+
const { value: placeTableLength, bytesRead: b1 } = readVarint(buf, offset);
|
|
11
|
+
offset += b1;
|
|
12
|
+
const count = width * height;
|
|
13
|
+
const { values: chunkLengths, bytesRead: b2 } = readVarintList(buf, offset, count);
|
|
14
|
+
offset += b2;
|
|
15
|
+
const placeTableStart = offset;
|
|
16
|
+
const chunksStart = placeTableStart + placeTableLength;
|
|
17
|
+
const chunkOffsets = new Array(count);
|
|
18
|
+
let running = chunksStart;
|
|
19
|
+
for (let i = 0; i < count; i++) {
|
|
20
|
+
chunkOffsets[i] = running;
|
|
21
|
+
running += chunkLengths[i];
|
|
22
|
+
}
|
|
23
|
+
return {
|
|
24
|
+
placeTableStart,
|
|
25
|
+
placeTableLength,
|
|
26
|
+
chunkLengths,
|
|
27
|
+
chunkOffsets,
|
|
28
|
+
directoryLength: offset - start,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=directory.js.map
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Errors the reader throws because of the data it was given, as opposed to
|
|
3
|
+
* how it was called: a caller's invalid argument (a latitude out of range,
|
|
4
|
+
* a place id this file doesn't have) is a plain `RangeError`, and anything
|
|
5
|
+
* else is a bug.
|
|
6
|
+
*
|
|
7
|
+
* ```ts
|
|
8
|
+
* try {
|
|
9
|
+
* reader = CGeoDataReader.open(path);
|
|
10
|
+
* } catch (error) {
|
|
11
|
+
* if (error instanceof CGeoDataVersionError) {
|
|
12
|
+
* // written by a newer compressor: upgrade the reader
|
|
13
|
+
* } else if (error instanceof CGeoDataFormatError) {
|
|
14
|
+
* // truncated or corrupt download: fetch the file again
|
|
15
|
+
* } else throw error;
|
|
16
|
+
* }
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export declare class CGeoDataError extends Error {
|
|
20
|
+
/** "CGeoDataError", and each subclass its own name. */
|
|
21
|
+
name: string;
|
|
22
|
+
}
|
|
23
|
+
/** A `.bin` file, or a search or locate index, is malformed or truncated:
|
|
24
|
+
* it breaks a rule of binary-format.md, so nothing read from it can be
|
|
25
|
+
* trusted. */
|
|
26
|
+
export declare class CGeoDataFormatError extends CGeoDataError {
|
|
27
|
+
/** "CGeoDataFormatError" */
|
|
28
|
+
name: string;
|
|
29
|
+
}
|
|
30
|
+
/** The file is a CGEODATA file (or index) of a format version this reader
|
|
31
|
+
* doesn't know -- most likely written by a newer compressor. */
|
|
32
|
+
export declare class CGeoDataVersionError extends CGeoDataFormatError {
|
|
33
|
+
/** "CGeoDataVersionError" */
|
|
34
|
+
name: string;
|
|
35
|
+
/** The version number the file declares. */
|
|
36
|
+
readonly version: number;
|
|
37
|
+
constructor(message: string, version: number);
|
|
38
|
+
}
|
|
39
|
+
/** The file is valid, but uses something this reader doesn't support: a
|
|
40
|
+
* `max_depth` past 16 (see `MAX_SUPPORTED_DEPTH`). Rebuild it with a
|
|
41
|
+
* bigger chunk grid instead of deeper chunks. */
|
|
42
|
+
export declare class CGeoDataUnsupportedError extends CGeoDataFormatError {
|
|
43
|
+
/** "CGeoDataUnsupportedError" */
|
|
44
|
+
name: string;
|
|
45
|
+
}
|
|
46
|
+
/** A search or locate index is well formed, but was built from a different
|
|
47
|
+
* `.bin` file than the one it's being used with (or an older build of the
|
|
48
|
+
* same one). Rebuild the index, or delete it to work without one. */
|
|
49
|
+
export declare class CGeoDataIndexMismatchError extends CGeoDataError {
|
|
50
|
+
/** "CGeoDataIndexMismatchError" */
|
|
51
|
+
name: string;
|
|
52
|
+
}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Errors the reader throws because of the data it was given, as opposed to
|
|
3
|
+
* how it was called: a caller's invalid argument (a latitude out of range,
|
|
4
|
+
* a place id this file doesn't have) is a plain `RangeError`, and anything
|
|
5
|
+
* else is a bug.
|
|
6
|
+
*
|
|
7
|
+
* ```ts
|
|
8
|
+
* try {
|
|
9
|
+
* reader = CGeoDataReader.open(path);
|
|
10
|
+
* } catch (error) {
|
|
11
|
+
* if (error instanceof CGeoDataVersionError) {
|
|
12
|
+
* // written by a newer compressor: upgrade the reader
|
|
13
|
+
* } else if (error instanceof CGeoDataFormatError) {
|
|
14
|
+
* // truncated or corrupt download: fetch the file again
|
|
15
|
+
* } else throw error;
|
|
16
|
+
* }
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export class CGeoDataError extends Error {
|
|
20
|
+
/** "CGeoDataError", and each subclass its own name. */
|
|
21
|
+
name = "CGeoDataError";
|
|
22
|
+
}
|
|
23
|
+
/** A `.bin` file, or a search or locate index, is malformed or truncated:
|
|
24
|
+
* it breaks a rule of binary-format.md, so nothing read from it can be
|
|
25
|
+
* trusted. */
|
|
26
|
+
export class CGeoDataFormatError extends CGeoDataError {
|
|
27
|
+
/** "CGeoDataFormatError" */
|
|
28
|
+
name = "CGeoDataFormatError";
|
|
29
|
+
}
|
|
30
|
+
/** The file is a CGEODATA file (or index) of a format version this reader
|
|
31
|
+
* doesn't know -- most likely written by a newer compressor. */
|
|
32
|
+
export class CGeoDataVersionError extends CGeoDataFormatError {
|
|
33
|
+
/** "CGeoDataVersionError" */
|
|
34
|
+
name = "CGeoDataVersionError";
|
|
35
|
+
/** The version number the file declares. */
|
|
36
|
+
version;
|
|
37
|
+
constructor(message, version) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.version = version;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** The file is valid, but uses something this reader doesn't support: a
|
|
43
|
+
* `max_depth` past 16 (see `MAX_SUPPORTED_DEPTH`). Rebuild it with a
|
|
44
|
+
* bigger chunk grid instead of deeper chunks. */
|
|
45
|
+
export class CGeoDataUnsupportedError extends CGeoDataFormatError {
|
|
46
|
+
/** "CGeoDataUnsupportedError" */
|
|
47
|
+
name = "CGeoDataUnsupportedError";
|
|
48
|
+
}
|
|
49
|
+
/** A search or locate index is well formed, but was built from a different
|
|
50
|
+
* `.bin` file than the one it's being used with (or an older build of the
|
|
51
|
+
* same one). Rebuild the index, or delete it to work without one. */
|
|
52
|
+
export class CGeoDataIndexMismatchError extends CGeoDataError {
|
|
53
|
+
/** "CGeoDataIndexMismatchError" */
|
|
54
|
+
name = "CGeoDataIndexMismatchError";
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=errors.js.map
|
package/dist/format.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* binary-format.md at the byte level: the codecs and parsers the reader is
|
|
3
|
+
* built from, for tools that write or inspect CGEODATA files (the
|
|
4
|
+
* compressor uses them). They follow the format, not a stable API, and
|
|
5
|
+
* may change when it does; `CGeoDataReader` is the interface for reading.
|
|
6
|
+
*
|
|
7
|
+
* @module @ferrybig/cgeodata-reader/format
|
|
8
|
+
*/
|
|
9
|
+
export { type BoundingBox, cellBoundingBox, chunkBoundingBox, } from "./cellBounds.ts";
|
|
10
|
+
export { type ChunkRecords, MAX_SUPPORTED_DEPTH, parseChunk, } from "./chunk.ts";
|
|
11
|
+
export { readHeaderAndDirectory } from "./container.ts";
|
|
12
|
+
export { type CGeoDataDirectory, parseDirectory } from "./directory.ts";
|
|
13
|
+
export { type CGeoDataHeader, parseHeader } from "./header.ts";
|
|
14
|
+
export { type Inflater, inflateBlock, inflateCeiling, streamInflater, } from "./inflate.ts";
|
|
15
|
+
export { decodeLocateEntry, encodeLocateEntry, type LocateCell, type LocateGrid, locateEntryWidth, } from "./locateIndex.ts";
|
|
16
|
+
export { lowBits, mortonDecode, mortonEncode, shiftRight } from "./morton.ts";
|
|
17
|
+
export { type PlaceTableInfo, parsePlaceTable, type ResolvedPlace, readNameBlock, readNameEntry, resolveIdentifier, } from "./placeTable.ts";
|
|
18
|
+
export { containsTokenRun, EXACT_SPELLING_BONUS, foldDiacritics, LEVEL_BONUS, type Needle, nameMatchQuality, placeMatchQuality, prepareNeedle, QUALITY_EXACT, QUALITY_MAX, QUALITY_SUBSTRING, QUALITY_WHOLE_WORD, tokenize, tokenizeFolded, } from "./relevance.ts";
|
|
19
|
+
export { fnv1a32, foldSymbols, trigramListIndex, trigramListIndices, } from "./searchIndex.ts";
|
|
20
|
+
export { toSource } from "./source.ts";
|
|
21
|
+
export { IncompleteBufferError, readDeltaVarintList, readDeltaVarints32Into, readDeltaVarintsInto, readVarint, readVarintList, type VarintListResult, type VarintResult, writeVarint, } from "./varint.ts";
|
package/dist/format.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* binary-format.md at the byte level: the codecs and parsers the reader is
|
|
3
|
+
* built from, for tools that write or inspect CGEODATA files (the
|
|
4
|
+
* compressor uses them). They follow the format, not a stable API, and
|
|
5
|
+
* may change when it does; `CGeoDataReader` is the interface for reading.
|
|
6
|
+
*
|
|
7
|
+
* @module @ferrybig/cgeodata-reader/format
|
|
8
|
+
*/
|
|
9
|
+
export { cellBoundingBox, chunkBoundingBox, } from "./cellBounds.js";
|
|
10
|
+
export { MAX_SUPPORTED_DEPTH, parseChunk, } from "./chunk.js";
|
|
11
|
+
export { readHeaderAndDirectory } from "./container.js";
|
|
12
|
+
export { parseDirectory } from "./directory.js";
|
|
13
|
+
export { parseHeader } from "./header.js";
|
|
14
|
+
export { inflateBlock, inflateCeiling, streamInflater, } from "./inflate.js";
|
|
15
|
+
export { decodeLocateEntry, encodeLocateEntry, locateEntryWidth, } from "./locateIndex.js";
|
|
16
|
+
export { lowBits, mortonDecode, mortonEncode, shiftRight } from "./morton.js";
|
|
17
|
+
export { parsePlaceTable, readNameBlock, readNameEntry, resolveIdentifier, } from "./placeTable.js";
|
|
18
|
+
export { containsTokenRun, EXACT_SPELLING_BONUS, foldDiacritics, LEVEL_BONUS, nameMatchQuality, placeMatchQuality, prepareNeedle, QUALITY_EXACT, QUALITY_MAX, QUALITY_SUBSTRING, QUALITY_WHOLE_WORD, tokenize, tokenizeFolded, } from "./relevance.js";
|
|
19
|
+
export { fnv1a32, foldSymbols, trigramListIndex, trigramListIndices, } from "./searchIndex.js";
|
|
20
|
+
export { toSource } from "./source.js";
|
|
21
|
+
export { IncompleteBufferError, readDeltaVarintList, readDeltaVarints32Into, readDeltaVarintsInto, readVarint, readVarintList, writeVarint, } from "./varint.js";
|
|
22
|
+
//# sourceMappingURL=format.js.map
|