@ferrybig/cgeodata-reader 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +313 -2
- package/binary-format.md +2366 -0
- package/dist/cache.d.ts +24 -0
- package/dist/cache.js +60 -0
- package/dist/cellBounds.d.ts +19 -0
- package/dist/cellBounds.js +36 -0
- package/dist/chunk.d.ts +34 -0
- package/dist/chunk.js +269 -0
- package/dist/chunkDecoder.d.ts +39 -0
- package/dist/chunkDecoder.js +116 -0
- package/dist/chunkDecoderWasm.d.ts +2 -0
- package/dist/chunkDecoderWasm.js +4 -0
- package/dist/cli/buildLocateIndex.d.ts +2 -0
- package/dist/cli/buildLocateIndex.js +46 -0
- package/dist/cli/buildSearchIndex.d.ts +2 -0
- package/dist/cli/buildSearchIndex.js +90 -0
- package/dist/cli/chunk.d.ts +2 -0
- package/dist/cli/chunk.js +67 -0
- package/dist/cli/info.d.ts +2 -0
- package/dist/cli/info.js +87 -0
- package/dist/cli/list.d.ts +2 -0
- package/dist/cli/list.js +71 -0
- package/dist/cli/locate.d.ts +2 -0
- package/dist/cli/locate.js +64 -0
- package/dist/cli/lookup.d.ts +2 -0
- package/dist/cli/lookup.js +128 -0
- package/dist/cli/search.d.ts +2 -0
- package/dist/cli/search.js +119 -0
- package/dist/cli/shared.d.ts +50 -0
- package/dist/cli/shared.js +140 -0
- package/dist/cli/show.d.ts +2 -0
- package/dist/cli/show.js +38 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +107 -0
- package/dist/container.d.ts +17 -0
- package/dist/container.js +68 -0
- package/dist/directory.d.ts +21 -0
- package/dist/directory.js +31 -0
- package/dist/errors.d.ts +52 -0
- package/dist/errors.js +56 -0
- package/dist/format.d.ts +21 -0
- package/dist/format.js +22 -0
- package/dist/header.d.ts +29 -0
- package/dist/header.js +69 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.js +18 -0
- package/dist/indexes.d.ts +11 -0
- package/dist/indexes.js +12 -0
- package/dist/inflate.d.ts +21 -0
- package/dist/inflate.js +61 -0
- package/dist/locateIndex.d.ts +142 -0
- package/dist/locateIndex.js +561 -0
- package/dist/morton.d.ts +28 -0
- package/dist/morton.js +51 -0
- package/dist/node.d.ts +65 -0
- package/dist/node.js +137 -0
- package/dist/placeTable.d.ts +69 -0
- package/dist/placeTable.js +177 -0
- package/dist/reader.d.ts +534 -0
- package/dist/reader.js +1128 -0
- package/dist/relevance.d.ts +184 -0
- package/dist/relevance.js +266 -0
- package/dist/searchIndex.d.ts +148 -0
- package/dist/searchIndex.js +786 -0
- package/dist/source.d.ts +49 -0
- package/dist/source.js +50 -0
- package/dist/turns.d.ts +19 -0
- package/dist/turns.js +40 -0
- package/dist/varint.d.ts +52 -0
- package/dist/varint.js +137 -0
- package/package.json +76 -5
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { writeFileSync } from "node:fs";
|
|
2
|
+
import { buildSearchIndex, DEFAULT_SEARCH_INDEX_OPTIONS, } from "../searchIndex.js";
|
|
3
|
+
import { EXIT_OK, openReader, parse, parseAreaTieBreak, UsageError, useIndexFile, } from "./shared.js";
|
|
4
|
+
const { hashExp, topCount, topBlockExp } = DEFAULT_SEARCH_INDEX_OPTIONS;
|
|
5
|
+
export const buildSearchIndexCmd = {
|
|
6
|
+
name: "build-search-index",
|
|
7
|
+
summary: "build a trigram search index for a .bin file",
|
|
8
|
+
usage: `usage: cgeodata build-search-index <file.bin> [--output <index.bin>] [--hash-exp N] [--top-count N] [--top-block-exp N] [--area-tie-break auto|on|off] [--locate-index <index.bin>]
|
|
9
|
+
|
|
10
|
+
Builds a trigram search index (binary-format.md Appendix B) over every
|
|
11
|
+
place in <file.bin> and writes it to <index.bin> (or stdout when
|
|
12
|
+
--output is omitted). Written next to <file.bin> as
|
|
13
|
+
<file>.search-index.bin, \`cgeodata search\` (and any CGeoDataReader
|
|
14
|
+
opened from that path) uses it automatically; elsewhere, pass it with
|
|
15
|
+
\`search --index\`. Queries of 3+ characters then only compare the
|
|
16
|
+
places whose names contain all of their trigrams, and sorted searches of
|
|
17
|
+
1-2 characters with --limit up to the top count are answered from a
|
|
18
|
+
precomputed table.
|
|
19
|
+
|
|
20
|
+
That table is ranked like \`search --sort --area-tie-break\`, and only
|
|
21
|
+
answers searches ranked the same way: by area when a locate index is
|
|
22
|
+
next to <file.bin> (build it first, with build-locate-index), otherwise
|
|
23
|
+
plainly.
|
|
24
|
+
|
|
25
|
+
--output F write to F instead of stdout
|
|
26
|
+
--hash-exp N 2^N shared lists for non-Latin trigrams (default ${hashExp})
|
|
27
|
+
--top-count N places per 1-2 character query; 0 leaves the
|
|
28
|
+
table out (default ${topCount})
|
|
29
|
+
--top-block-exp N 2^N table entries per block (default ${topBlockExp})
|
|
30
|
+
--area-tie-break M rank the table by area (on; needs a locate index),
|
|
31
|
+
plainly (off), or by area if there is a locate
|
|
32
|
+
index (auto, the default)
|
|
33
|
+
--locate-index F take areas from this locate index instead of
|
|
34
|
+
<file>'s default <file>.locate-index.bin`,
|
|
35
|
+
async run(argv) {
|
|
36
|
+
const { values, positionals } = parse(argv, {
|
|
37
|
+
output: { type: "string" },
|
|
38
|
+
"hash-exp": { type: "string" },
|
|
39
|
+
"top-count": { type: "string" },
|
|
40
|
+
"top-block-exp": { type: "string" },
|
|
41
|
+
"area-tie-break": { type: "string" },
|
|
42
|
+
"locate-index": { type: "string" },
|
|
43
|
+
});
|
|
44
|
+
const [file, ...extra] = positionals;
|
|
45
|
+
if (extra.length > 0)
|
|
46
|
+
throw new UsageError(`unexpected argument ${JSON.stringify(extra[0])}`);
|
|
47
|
+
const options = {};
|
|
48
|
+
for (const [flag, key] of [
|
|
49
|
+
["hash-exp", "hashExp"],
|
|
50
|
+
["top-count", "topCount"],
|
|
51
|
+
["top-block-exp", "topBlockExp"],
|
|
52
|
+
]) {
|
|
53
|
+
const text = values[flag];
|
|
54
|
+
if (text === undefined)
|
|
55
|
+
continue;
|
|
56
|
+
const value = Number(text);
|
|
57
|
+
if (!(Number.isInteger(value) && value >= 0)) {
|
|
58
|
+
throw new UsageError(`--${flag} must be a non-negative integer, not ${JSON.stringify(text)}`);
|
|
59
|
+
}
|
|
60
|
+
options[key] = value;
|
|
61
|
+
}
|
|
62
|
+
options.areaTieBreak = parseAreaTieBreak(values["area-tie-break"]);
|
|
63
|
+
const reader = await openReader(file);
|
|
64
|
+
const { countries, regions, cities } = reader.placeCounts;
|
|
65
|
+
const t0 = performance.now();
|
|
66
|
+
let bytes;
|
|
67
|
+
try {
|
|
68
|
+
if (values["locate-index"] !== undefined)
|
|
69
|
+
await useIndexFile((index) => reader.useLocateIndex(index), values["locate-index"]);
|
|
70
|
+
const byArea = options.areaTieBreak !== false && (await reader.hasLocateIndex());
|
|
71
|
+
process.stderr.write(`building search index over ${countries + regions + cities} places, ranked ${byArea ? "by area" : "plainly"}...\n`);
|
|
72
|
+
bytes = await buildSearchIndex(reader, options);
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
throw error instanceof RangeError ? new UsageError(error.message) : error;
|
|
76
|
+
}
|
|
77
|
+
finally {
|
|
78
|
+
await reader.close();
|
|
79
|
+
}
|
|
80
|
+
process.stderr.write(`done in ${((performance.now() - t0) / 1000).toFixed(1)} s, ${(bytes.length / 1024 / 1024).toFixed(1)} MiB\n`);
|
|
81
|
+
if (values.output !== undefined) {
|
|
82
|
+
writeFileSync(values.output, bytes);
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
process.stdout.write(bytes);
|
|
86
|
+
}
|
|
87
|
+
return EXIT_OK;
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
//# sourceMappingURL=buildSearchIndex.js.map
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { names, openReader, parse, parseId, UsageError, writeLine, } from "./shared.js";
|
|
2
|
+
export const chunk = {
|
|
3
|
+
name: "chunk",
|
|
4
|
+
summary: "dump one chunk's records with their cells and places",
|
|
5
|
+
usage: `usage: cgeodata chunk <file.bin> <index> [--json]
|
|
6
|
+
|
|
7
|
+
Dumps the records of chunk <index> (the number \`lookup\` prints): first
|
|
8
|
+
a header with the chunk's extent, compressed size, palette size and
|
|
9
|
+
record count, then one record per line as tab-separated columns: code,
|
|
10
|
+
depth, cell (lat min..max, lon min..max), place id, level, names,
|
|
11
|
+
region, country. A record that resolves to no place has "-" columns.
|
|
12
|
+
An empty chunk prints just the header.
|
|
13
|
+
|
|
14
|
+
--json the header as one JSON object, then one object per record`,
|
|
15
|
+
async run(argv) {
|
|
16
|
+
const { values, positionals } = parse(argv, {
|
|
17
|
+
json: { type: "boolean", default: false },
|
|
18
|
+
});
|
|
19
|
+
const [file, indexArg, ...extra] = positionals;
|
|
20
|
+
if (indexArg === undefined)
|
|
21
|
+
throw new UsageError("missing <index>");
|
|
22
|
+
if (extra.length > 0)
|
|
23
|
+
throw new UsageError(`unexpected argument ${JSON.stringify(extra[0])}`);
|
|
24
|
+
const index = parseId(indexArg, "index");
|
|
25
|
+
const reader = await openReader(file);
|
|
26
|
+
try {
|
|
27
|
+
const records = await reader.chunk(index); // also range-checks the index
|
|
28
|
+
const box = reader.chunkBoundingBox(index);
|
|
29
|
+
const { width } = reader.header;
|
|
30
|
+
const header = {
|
|
31
|
+
index,
|
|
32
|
+
col: index % width,
|
|
33
|
+
row: Math.floor(index / width),
|
|
34
|
+
boundingBox: box,
|
|
35
|
+
compressedBytes: reader.directory.chunkLengths[index],
|
|
36
|
+
paletteSize: records?.palette.length ?? 0,
|
|
37
|
+
recordCount: records?.codes.length ?? 0,
|
|
38
|
+
};
|
|
39
|
+
if (values.json) {
|
|
40
|
+
writeLine(JSON.stringify(header));
|
|
41
|
+
for await (const record of reader.chunkRecords(index))
|
|
42
|
+
writeLine(JSON.stringify(record));
|
|
43
|
+
return 0;
|
|
44
|
+
}
|
|
45
|
+
writeLine(`chunk ${index} (col ${header.col}, row ${header.row}) lat ${box.minLat}..${box.maxLat} lon ${box.minLon}..${box.maxLon} ` +
|
|
46
|
+
`${header.compressedBytes} bytes, ${header.paletteSize} palette entries, ${header.recordCount} records`);
|
|
47
|
+
for await (const { code, depth, boundingBox: cell, place, } of reader.chunkRecords(index)) {
|
|
48
|
+
const where = `lat ${cell.minLat}..${cell.maxLat}, lon ${cell.minLon}..${cell.maxLon}`;
|
|
49
|
+
const who = place
|
|
50
|
+
? [
|
|
51
|
+
place.id,
|
|
52
|
+
place.level,
|
|
53
|
+
names(place.names),
|
|
54
|
+
names(place.level === "city" ? place.region?.names : null),
|
|
55
|
+
names(place.level === "country" ? null : place.country.names),
|
|
56
|
+
]
|
|
57
|
+
: ["-", "-", "-", "-", "-"];
|
|
58
|
+
writeLine([code, depth, where, ...who].join("\t"));
|
|
59
|
+
}
|
|
60
|
+
return 0;
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
await reader.close();
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
//# sourceMappingURL=chunk.js.map
|
package/dist/cli/info.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { openReader, parse, writeLine } from "./shared.js";
|
|
2
|
+
export const info = {
|
|
3
|
+
name: "info",
|
|
4
|
+
summary: "describe a file: header, place counts, chunk statistics",
|
|
5
|
+
usage: `usage: cgeodata info <file.bin> [--top N] [--json]
|
|
6
|
+
|
|
7
|
+
Prints the header fields, how many countries/regions/cities the place
|
|
8
|
+
table holds, how the bytes split between place table and chunks, and the
|
|
9
|
+
N (default 10) largest chunks by compressed size -- as indices that paste
|
|
10
|
+
straight into the compressor's --only-chunks.
|
|
11
|
+
|
|
12
|
+
--top N how many of the largest chunks to list (0 for none)
|
|
13
|
+
--json one JSON object instead of the readable form`,
|
|
14
|
+
async run(argv) {
|
|
15
|
+
const { values, positionals } = parse(argv, {
|
|
16
|
+
top: { type: "string", default: "10" },
|
|
17
|
+
json: { type: "boolean", default: false },
|
|
18
|
+
});
|
|
19
|
+
const top = Number(values.top);
|
|
20
|
+
if (!Number.isInteger(top) || top < 0)
|
|
21
|
+
throw new Error(`--top must be a non-negative integer, not ${JSON.stringify(values.top)}`);
|
|
22
|
+
const reader = await openReader(positionals[0]);
|
|
23
|
+
try {
|
|
24
|
+
const { header, directory } = reader;
|
|
25
|
+
const chunkLengths = directory.chunkLengths;
|
|
26
|
+
const nonEmpty = chunkLengths.filter((len) => len > 0).length;
|
|
27
|
+
const chunkBytes = chunkLengths.reduce((a, b) => a + b, 0);
|
|
28
|
+
const fileSize = header.headerLength +
|
|
29
|
+
directory.directoryLength +
|
|
30
|
+
directory.placeTableLength +
|
|
31
|
+
chunkBytes;
|
|
32
|
+
const largest = chunkLengths
|
|
33
|
+
.map((bytes, index) => ({ index, bytes }))
|
|
34
|
+
.filter((c) => c.bytes > 0)
|
|
35
|
+
.sort((a, b) => b.bytes - a.bytes || a.index - b.index)
|
|
36
|
+
.slice(0, top);
|
|
37
|
+
const report = {
|
|
38
|
+
version: header.version,
|
|
39
|
+
attribution: header.attribution,
|
|
40
|
+
grid: {
|
|
41
|
+
width: header.width,
|
|
42
|
+
height: header.height,
|
|
43
|
+
widthExp: header.widthExp,
|
|
44
|
+
heightExp: header.heightExp,
|
|
45
|
+
},
|
|
46
|
+
maxDepth: header.maxDepth,
|
|
47
|
+
keyframeExp: header.keyframeExp,
|
|
48
|
+
places: reader.placeCounts,
|
|
49
|
+
bytes: {
|
|
50
|
+
total: fileSize,
|
|
51
|
+
header: header.headerLength,
|
|
52
|
+
directory: directory.directoryLength,
|
|
53
|
+
placeTable: directory.placeTableLength,
|
|
54
|
+
chunks: chunkBytes,
|
|
55
|
+
},
|
|
56
|
+
chunks: { total: chunkLengths.length, nonEmpty, largest },
|
|
57
|
+
};
|
|
58
|
+
if (values.json) {
|
|
59
|
+
writeLine(JSON.stringify(report));
|
|
60
|
+
return 0;
|
|
61
|
+
}
|
|
62
|
+
const { places, bytes, grid } = report;
|
|
63
|
+
const pct = (n) => `${((100 * n) / bytes.total).toFixed(1)}%`;
|
|
64
|
+
const lines = [
|
|
65
|
+
`version ${report.version}`,
|
|
66
|
+
`attribution ${report.attribution}`,
|
|
67
|
+
`grid ${grid.width}x${grid.height} chunks (width_exp ${grid.widthExp}, height_exp ${grid.heightExp})`,
|
|
68
|
+
`max depth ${report.maxDepth} (${grid.width * 2 ** report.maxDepth}x${grid.height * 2 ** report.maxDepth} finest cells worldwide)`,
|
|
69
|
+
`keyframe ${report.keyframeExp} (${2 ** report.keyframeExp} names per block)`,
|
|
70
|
+
`places ${places.countries} countries, ${places.regions} regions, ${places.cities} cities`,
|
|
71
|
+
`bytes ${bytes.total} total: header ${bytes.header}, directory ${bytes.directory}, place table ${bytes.placeTable} (${pct(bytes.placeTable)}), chunks ${bytes.chunks} (${pct(bytes.chunks)})`,
|
|
72
|
+
`chunks ${nonEmpty} of ${chunkLengths.length} non-empty`,
|
|
73
|
+
];
|
|
74
|
+
if (largest.length > 0) {
|
|
75
|
+
lines.push(`largest ${largest.map((c) => `${c.index} (${c.bytes})`).join(", ")}`);
|
|
76
|
+
lines.push(` --only-chunks ${largest.map((c) => c.index).join(",")}`);
|
|
77
|
+
}
|
|
78
|
+
for (const line of lines)
|
|
79
|
+
writeLine(line);
|
|
80
|
+
return 0;
|
|
81
|
+
}
|
|
82
|
+
finally {
|
|
83
|
+
await reader.close();
|
|
84
|
+
}
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
//# sourceMappingURL=info.js.map
|
package/dist/cli/list.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { formatPlaceLine, names, openReader, parse, parseLevel, resolveCountry, UsageError, writeLine, } from "./shared.js";
|
|
2
|
+
const DEPTH = { country: 0, region: 1, city: 2 };
|
|
3
|
+
async function printTree(reader, place, deepest, json) {
|
|
4
|
+
const depth = DEPTH[place.level];
|
|
5
|
+
writeLine(json
|
|
6
|
+
? JSON.stringify(place)
|
|
7
|
+
: `${" ".repeat(depth)}${place.id}\t${names(place.names)}`);
|
|
8
|
+
if (depth >= DEPTH[deepest])
|
|
9
|
+
return;
|
|
10
|
+
const children = place.level === "country"
|
|
11
|
+
? reader.places({ level: "region", country: place.id })
|
|
12
|
+
: reader.places({ level: "city", region: place.id });
|
|
13
|
+
for await (const child of children)
|
|
14
|
+
await printTree(reader, child, deepest, json);
|
|
15
|
+
}
|
|
16
|
+
export const list = {
|
|
17
|
+
name: "list",
|
|
18
|
+
summary: "list places: countries by default, or a level, scoped to a country",
|
|
19
|
+
usage: `usage: cgeodata list <file.bin> [--level country|region|city] [--country <id|name>] [--childless] [--tree] [--json]
|
|
20
|
+
|
|
21
|
+
Lists places one per line as tab-separated columns: id, level, names,
|
|
22
|
+
region, country. Alternative names within a column are joined by " | ";
|
|
23
|
+
a column that doesn't apply is "-".
|
|
24
|
+
|
|
25
|
+
--level L which level to list (default: country)
|
|
26
|
+
--country C only places inside this country, given as an id or as one
|
|
27
|
+
of its names (case-insensitive, must be unambiguous)
|
|
28
|
+
--childless only places with nothing directly under them: regions with
|
|
29
|
+
no city mapped to them (with --level region), countries
|
|
30
|
+
with no region. Read from the place table's counts
|
|
31
|
+
--tree print the hierarchy indented instead of flat, down to
|
|
32
|
+
--level (default with --tree: city); lines are "id<TAB>names"
|
|
33
|
+
--json one JSON object per place instead of the text form`,
|
|
34
|
+
async run(argv) {
|
|
35
|
+
const { values, positionals } = parse(argv, {
|
|
36
|
+
level: { type: "string" },
|
|
37
|
+
country: { type: "string" },
|
|
38
|
+
childless: { type: "boolean", default: false },
|
|
39
|
+
tree: { type: "boolean", default: false },
|
|
40
|
+
json: { type: "boolean", default: false },
|
|
41
|
+
});
|
|
42
|
+
const level = parseLevel(values.level) ?? (values.tree ? "city" : "country");
|
|
43
|
+
if (values.childless && values.tree)
|
|
44
|
+
throw new UsageError("--childless cannot be combined with --tree");
|
|
45
|
+
const reader = await openReader(positionals[0]);
|
|
46
|
+
try {
|
|
47
|
+
const country = values.country === undefined
|
|
48
|
+
? undefined
|
|
49
|
+
: await resolveCountry(reader, values.country);
|
|
50
|
+
if (values.tree) {
|
|
51
|
+
const roots = country === undefined
|
|
52
|
+
? reader.places({ level: "country" })
|
|
53
|
+
: [await reader.place(country)];
|
|
54
|
+
for await (const root of roots)
|
|
55
|
+
await printTree(reader, root, level, values.json);
|
|
56
|
+
return 0;
|
|
57
|
+
}
|
|
58
|
+
const filter = country === undefined ? { level } : { level, country };
|
|
59
|
+
for await (const place of reader.places(filter)) {
|
|
60
|
+
if (values.childless && reader.childCount(place.id) > 0)
|
|
61
|
+
continue;
|
|
62
|
+
writeLine(values.json ? JSON.stringify(place) : formatPlaceLine(place));
|
|
63
|
+
}
|
|
64
|
+
return 0;
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
await reader.close();
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
//# sourceMappingURL=list.js.map
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { coordPrecision, openReader, parse, parseId, UsageError, useIndexFile, writeLine, } from "./shared.js";
|
|
2
|
+
export const locate = {
|
|
3
|
+
name: "locate",
|
|
4
|
+
summary: "find a representative lat/lon for a place id",
|
|
5
|
+
usage: `usage: cgeodata locate <file.bin> <id> [<id> ...] [--index <index.bin> | --no-index] [--json]
|
|
6
|
+
|
|
7
|
+
Prints a representative lat, lon for each place id: the centre of a cell
|
|
8
|
+
near the middle of the place's area (its own cells plus those of every
|
|
9
|
+
place under it), and always inside that area, so looking the point up
|
|
10
|
+
gives the place back. Printed with as many decimals as that cell's size
|
|
11
|
+
warrants.
|
|
12
|
+
|
|
13
|
+
With a locate index (build-locate-index) this reads one entry per id;
|
|
14
|
+
without one it scans every chunk, which takes seconds on a worldwide
|
|
15
|
+
file. Same result either way.
|
|
16
|
+
|
|
17
|
+
--index F use this locate index instead of <file>'s default
|
|
18
|
+
<file>.locate-index.bin (used automatically when it exists)
|
|
19
|
+
--no-index scan the chunks even if a locate index exists -- the
|
|
20
|
+
baseline an index is measured against
|
|
21
|
+
--json one JSON object per id instead: {"id", "lat", "lon",
|
|
22
|
+
"area", "boundingBox"}, area being the place's size in
|
|
23
|
+
square degrees (null when it has no geometry)`,
|
|
24
|
+
async run(argv) {
|
|
25
|
+
const { values, positionals } = parse(argv, {
|
|
26
|
+
index: { type: "string" },
|
|
27
|
+
"no-index": { type: "boolean", default: false },
|
|
28
|
+
json: { type: "boolean", default: false },
|
|
29
|
+
});
|
|
30
|
+
const [file, ...idArgs] = positionals;
|
|
31
|
+
if (idArgs.length === 0)
|
|
32
|
+
throw new UsageError("missing <id>");
|
|
33
|
+
if (values.index !== undefined && values["no-index"])
|
|
34
|
+
throw new UsageError("--index and --no-index can't be combined");
|
|
35
|
+
const ids = idArgs.map((arg) => parseId(arg, "id"));
|
|
36
|
+
const reader = await openReader(file);
|
|
37
|
+
try {
|
|
38
|
+
if (values["no-index"])
|
|
39
|
+
await reader.useLocateIndex(undefined);
|
|
40
|
+
if (values.index !== undefined)
|
|
41
|
+
await useIndexFile((index) => reader.useLocateIndex(index), values.index);
|
|
42
|
+
let anyMissing = false;
|
|
43
|
+
for (const id of ids) {
|
|
44
|
+
const pos = await reader.locate(id);
|
|
45
|
+
if (values.json)
|
|
46
|
+
writeLine(JSON.stringify(pos ? { id, ...pos } : null));
|
|
47
|
+
if (!pos) {
|
|
48
|
+
process.stderr.write(`${id}: no geometry found\n`);
|
|
49
|
+
anyMissing = true;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (values.json)
|
|
53
|
+
continue;
|
|
54
|
+
const p = coordPrecision(pos.boundingBox);
|
|
55
|
+
writeLine(`${pos.lat.toFixed(p)}, ${pos.lon.toFixed(p)}`);
|
|
56
|
+
}
|
|
57
|
+
return anyMissing ? 1 : 0;
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
await reader.close();
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
//# sourceMappingURL=locate.js.map
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { createInterface } from "node:readline";
|
|
2
|
+
import { names, openReader, parse, UsageError, writeLine, } from "./shared.js";
|
|
3
|
+
function parseCoordinate(text, what) {
|
|
4
|
+
const trimmed = text.trim();
|
|
5
|
+
// `Number("")` is 0, not NaN, so an emptiness check has to come first:
|
|
6
|
+
// otherwise a truncated "52.3," parses as (52.3, 0) and reports a
|
|
7
|
+
// confident, wrong place off the coast of England instead of an error.
|
|
8
|
+
const value = trimmed === "" ? Number.NaN : Number(trimmed);
|
|
9
|
+
if (!Number.isFinite(value))
|
|
10
|
+
throw new Error(`${what} ${JSON.stringify(text)} is not a number`);
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
/** Accepts "lat,lon" as one field or lat and lon as two; anything after
|
|
14
|
+
* the coordinates is the tag. */
|
|
15
|
+
function parsePoint(line) {
|
|
16
|
+
const fields = line.split("\t");
|
|
17
|
+
const first = fields[0] ?? "";
|
|
18
|
+
if (first.includes(",")) {
|
|
19
|
+
const [lat = "", lon = ""] = first.split(",", 2);
|
|
20
|
+
return {
|
|
21
|
+
lat: parseCoordinate(lat, "latitude"),
|
|
22
|
+
lon: parseCoordinate(lon, "longitude"),
|
|
23
|
+
tag: fields.slice(1).join("\t"),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
if (fields.length < 2)
|
|
27
|
+
throw new Error(`expected "lat,lon" or "lat<TAB>lon", got ${JSON.stringify(line)}`);
|
|
28
|
+
return {
|
|
29
|
+
lat: parseCoordinate(first, "latitude"),
|
|
30
|
+
lon: parseCoordinate(fields[1] ?? "", "longitude"),
|
|
31
|
+
tag: fields.slice(2).join("\t"),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function formatReadable(point, result, reader) {
|
|
35
|
+
const { boundingBox: box, chunkIndex } = result;
|
|
36
|
+
const col = chunkIndex % reader.header.width;
|
|
37
|
+
const row = Math.floor(chunkIndex / reader.header.width);
|
|
38
|
+
const head = point.tag
|
|
39
|
+
? `${point.lat},${point.lon}\t${point.tag}`
|
|
40
|
+
: `${point.lat},${point.lon}`;
|
|
41
|
+
return [
|
|
42
|
+
head,
|
|
43
|
+
` country ${names(result.country?.names)}`,
|
|
44
|
+
` region ${names(result.region?.names)}`,
|
|
45
|
+
` city ${names(result.city?.names)}`,
|
|
46
|
+
` cell lat ${box.minLat}..${box.maxLat} lon ${box.minLon}..${box.maxLon}`,
|
|
47
|
+
` chunk ${chunkIndex} (col ${col}, row ${row} of ${reader.header.width}x${reader.header.height})`,
|
|
48
|
+
].join("\n");
|
|
49
|
+
}
|
|
50
|
+
function formatJson(point, result) {
|
|
51
|
+
return JSON.stringify({
|
|
52
|
+
lat: point.lat,
|
|
53
|
+
lon: point.lon,
|
|
54
|
+
...(point.tag ? { tag: point.tag } : {}),
|
|
55
|
+
country: result.country,
|
|
56
|
+
region: result.region,
|
|
57
|
+
city: result.city,
|
|
58
|
+
boundingBox: result.boundingBox,
|
|
59
|
+
chunkIndex: result.chunkIndex,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
export const lookup = {
|
|
63
|
+
name: "lookup",
|
|
64
|
+
summary: "resolve points to country/region/city",
|
|
65
|
+
usage: `usage: cgeodata lookup <file.bin> <lat,lon> [<lat,lon> ...] [--json]
|
|
66
|
+
cgeodata lookup <file.bin> [--json] < points.txt
|
|
67
|
+
|
|
68
|
+
Resolves each point to its country/region/city, the lat/lon cell the
|
|
69
|
+
answer is valid for, and the chunk it was read from. With no points on
|
|
70
|
+
the command line they are read from stdin, one per line: "lat,lon" or
|
|
71
|
+
"lat<TAB>lon"; anything after that (tab-separated) is echoed back on the
|
|
72
|
+
output unchanged, so a batch can carry an id or name through. Blank lines
|
|
73
|
+
and lines starting with # are skipped; a malformed line is reported on
|
|
74
|
+
stderr and skipped rather than aborting the batch.
|
|
75
|
+
|
|
76
|
+
--json one JSON object per point (the query result plus the input
|
|
77
|
+
lat/lon and any echoed text) instead of the readable form.`,
|
|
78
|
+
async run(argv) {
|
|
79
|
+
const { values, positionals } = parse(argv, {
|
|
80
|
+
json: { type: "boolean", default: false },
|
|
81
|
+
});
|
|
82
|
+
const [file, ...pointArgs] = positionals;
|
|
83
|
+
const fromStdin = pointArgs.length === 0;
|
|
84
|
+
if (fromStdin && file === "-")
|
|
85
|
+
throw new UsageError("stdin can't be both the file and the list of points");
|
|
86
|
+
const reader = await openReader(file);
|
|
87
|
+
const emit = async (point) => {
|
|
88
|
+
const result = await reader.query(point.lat, point.lon);
|
|
89
|
+
writeLine(values.json
|
|
90
|
+
? formatJson(point, result)
|
|
91
|
+
: formatReadable(point, result, reader));
|
|
92
|
+
};
|
|
93
|
+
try {
|
|
94
|
+
if (!fromStdin) {
|
|
95
|
+
for (const arg of pointArgs)
|
|
96
|
+
await emit(parsePoint(arg));
|
|
97
|
+
return 0;
|
|
98
|
+
}
|
|
99
|
+
// Batch mode: a bad line is somebody's data problem, not a reason to
|
|
100
|
+
// throw away the rest of a long run, so report it and keep going.
|
|
101
|
+
let lineNumber = 0;
|
|
102
|
+
let skipped = 0;
|
|
103
|
+
for await (const raw of createInterface({
|
|
104
|
+
input: process.stdin,
|
|
105
|
+
crlfDelay: Infinity,
|
|
106
|
+
})) {
|
|
107
|
+
lineNumber++;
|
|
108
|
+
const line = raw.trim();
|
|
109
|
+
if (line === "" || line.startsWith("#"))
|
|
110
|
+
continue;
|
|
111
|
+
try {
|
|
112
|
+
await emit(parsePoint(line));
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
skipped++;
|
|
116
|
+
process.stderr.write(`line ${lineNumber}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (skipped > 0)
|
|
120
|
+
process.stderr.write(`skipped ${skipped} malformed line${skipped === 1 ? "" : "s"}\n`);
|
|
121
|
+
return 0;
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
await reader.close();
|
|
125
|
+
}
|
|
126
|
+
},
|
|
127
|
+
};
|
|
128
|
+
//# sourceMappingURL=lookup.js.map
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { coordPrecision, EXIT_NO_MATCH, EXIT_OK, formatPlaceLine, openReader, parse, parseAreaTieBreak, parseLevel, resolveCountry, UsageError, useIndexFile, writeLine, } from "./shared.js";
|
|
2
|
+
export const search = {
|
|
3
|
+
name: "search",
|
|
4
|
+
summary: "find places whose name contains a text",
|
|
5
|
+
usage: `usage: cgeodata search <file.bin> <text> [--level country|region|city] [--country <id|name>] [--limit N] [--sort [--area-tie-break auto|on|off]] [--locate] [--index <index.bin> | --no-index] [--json]
|
|
6
|
+
|
|
7
|
+
Prints every place (country, region or city) with a name containing
|
|
8
|
+
<text>, case- and diacritic-insensitively, in the same tab-separated
|
|
9
|
+
form as \`list\`. A <text> with commas also reads as a place followed by
|
|
10
|
+
places it lies in: "Paris, Texas" finds places named that and every
|
|
11
|
+
Paris whose region or country matches "Texas".
|
|
12
|
+
|
|
13
|
+
Exits 0 when something matched, 1 when nothing did, 2 on an error --
|
|
14
|
+
grep's convention.
|
|
15
|
+
|
|
16
|
+
--level L only places of this level
|
|
17
|
+
--country C only places inside this country (id or unambiguous name)
|
|
18
|
+
--limit N stop after N matches
|
|
19
|
+
--sort rank matches by relevance instead of id order: an exact
|
|
20
|
+
name match beats a match on a whole word of the name,
|
|
21
|
+
which beats a plain substring match; among equal
|
|
22
|
+
matches, a country comes first. Without --sort, --limit
|
|
23
|
+
stops the scan as soon as it has enough matches; with
|
|
24
|
+
--sort, the whole file has to be scored first, since the
|
|
25
|
+
best match isn't necessarily the first one found.
|
|
26
|
+
--area-tie-break M
|
|
27
|
+
with --sort, how to order equal matches after countries:
|
|
28
|
+
on = larger places first (needs a locate index), off =
|
|
29
|
+
city before region, then id order; auto (the default) =
|
|
30
|
+
on when a locate index is next to <file>
|
|
31
|
+
--locate append a lat,lon coordinate to each result, as
|
|
32
|
+
\`locate\` prints it (one read per match with a locate
|
|
33
|
+
index next to <file>, a scan of every chunk per match
|
|
34
|
+
without one -- then use --limit to keep it fast)
|
|
35
|
+
--index F use this trigram search index (built with
|
|
36
|
+
build-search-index) to skip most of the place table,
|
|
37
|
+
instead of <file>'s default <file>.search-index.bin
|
|
38
|
+
(used automatically when it exists). Queries too short
|
|
39
|
+
to contain a trigram still compare every place.
|
|
40
|
+
--no-index compare every place even if a search index exists --
|
|
41
|
+
the baseline an index is measured against
|
|
42
|
+
--json one JSON object per match instead of the text form`,
|
|
43
|
+
async run(argv) {
|
|
44
|
+
const { values, positionals } = parse(argv, {
|
|
45
|
+
level: { type: "string" },
|
|
46
|
+
country: { type: "string" },
|
|
47
|
+
limit: { type: "string" },
|
|
48
|
+
sort: { type: "boolean", default: false },
|
|
49
|
+
"area-tie-break": { type: "string" },
|
|
50
|
+
locate: { type: "boolean", default: false },
|
|
51
|
+
index: { type: "string" },
|
|
52
|
+
"no-index": { type: "boolean", default: false },
|
|
53
|
+
json: { type: "boolean", default: false },
|
|
54
|
+
});
|
|
55
|
+
const [file, text, ...extra] = positionals;
|
|
56
|
+
if (text === undefined)
|
|
57
|
+
throw new UsageError("missing <text> to search for");
|
|
58
|
+
if (extra.length > 0)
|
|
59
|
+
throw new UsageError(`unexpected argument ${JSON.stringify(extra[0])}`);
|
|
60
|
+
if (values.index !== undefined && values["no-index"])
|
|
61
|
+
throw new UsageError("--index and --no-index can't be combined");
|
|
62
|
+
const level = parseLevel(values.level);
|
|
63
|
+
const areaTieBreak = parseAreaTieBreak(values["area-tie-break"]);
|
|
64
|
+
const limit = values.limit === undefined ? undefined : Number(values.limit);
|
|
65
|
+
if (limit !== undefined && !(Number.isInteger(limit) && limit >= 0)) {
|
|
66
|
+
throw new UsageError(`--limit must be a non-negative integer, not ${JSON.stringify(values.limit)}`);
|
|
67
|
+
}
|
|
68
|
+
const reader = await openReader(file);
|
|
69
|
+
try {
|
|
70
|
+
const country = values.country === undefined
|
|
71
|
+
? undefined
|
|
72
|
+
: await resolveCountry(reader, values.country);
|
|
73
|
+
const searchOptions = {
|
|
74
|
+
...(level !== undefined ? { level } : {}),
|
|
75
|
+
...(country !== undefined ? { country } : {}),
|
|
76
|
+
...(limit !== undefined ? { limit } : {}),
|
|
77
|
+
sort: values.sort,
|
|
78
|
+
areaTieBreak,
|
|
79
|
+
};
|
|
80
|
+
if (values["no-index"])
|
|
81
|
+
await reader.useSearchIndex(undefined);
|
|
82
|
+
if (values.index !== undefined)
|
|
83
|
+
await useIndexFile((index) => reader.useSearchIndex(index), values.index);
|
|
84
|
+
let matched = 0;
|
|
85
|
+
for await (const place of reader.search(text, searchOptions)) {
|
|
86
|
+
matched++;
|
|
87
|
+
if (values.json) {
|
|
88
|
+
if (values.locate) {
|
|
89
|
+
const pos = await reader.locate(place.id);
|
|
90
|
+
writeLine(JSON.stringify(pos
|
|
91
|
+
? { ...place, lat: pos.lat, lon: pos.lon, area: pos.area }
|
|
92
|
+
: place));
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
writeLine(JSON.stringify(place));
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
else if (values.locate) {
|
|
99
|
+
const pos = await reader.locate(place.id);
|
|
100
|
+
if (pos) {
|
|
101
|
+
const p = coordPrecision(pos.boundingBox);
|
|
102
|
+
writeLine(`${formatPlaceLine(place)}\t${pos.lat.toFixed(p)},${pos.lon.toFixed(p)}`);
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
writeLine(`${formatPlaceLine(place)}\t-`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
writeLine(formatPlaceLine(place));
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return matched > 0 ? EXIT_OK : EXIT_NO_MATCH;
|
|
113
|
+
}
|
|
114
|
+
finally {
|
|
115
|
+
await reader.close();
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
//# sourceMappingURL=search.js.map
|