@nshiab/simple-data-analysis-core 2.0.2 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/class/SimpleTable.d.ts +50 -32
- package/esm/class/SimpleTable.d.ts.map +1 -1
- package/esm/class/SimpleTable.js +46 -32
- package/esm/helpers/loadOsmFile.d.ts +1 -3
- package/esm/helpers/loadOsmFile.d.ts.map +1 -1
- package/esm/helpers/loadOsmFile.js +3 -11
- package/esm/helpers/osmCache.d.ts +10 -13
- package/esm/helpers/osmCache.d.ts.map +1 -1
- package/esm/helpers/osmCache.js +177 -32
- package/esm/helpers/osmFiles.d.ts +9 -0
- package/esm/helpers/osmFiles.d.ts.map +1 -1
- package/esm/helpers/osmFiles.js +234 -23
- package/esm/methods/loadGeoData.d.ts.map +1 -1
- package/esm/methods/loadGeoData.js +4 -39
- package/esm/methods/loadOpenStreetMap.d.ts +24 -0
- package/esm/methods/loadOpenStreetMap.d.ts.map +1 -0
- package/esm/methods/loadOpenStreetMap.js +368 -0
- package/llm.md +79 -46
- package/llms.txt +3 -0
- package/package.json +1 -1
- package/script/class/SimpleTable.d.ts +50 -32
- package/script/class/SimpleTable.d.ts.map +1 -1
- package/script/class/SimpleTable.js +46 -32
- package/script/helpers/loadOsmFile.d.ts +1 -3
- package/script/helpers/loadOsmFile.d.ts.map +1 -1
- package/script/helpers/loadOsmFile.js +3 -11
- package/script/helpers/osmCache.d.ts +10 -13
- package/script/helpers/osmCache.d.ts.map +1 -1
- package/script/helpers/osmCache.js +177 -65
- package/script/helpers/osmFiles.d.ts +9 -0
- package/script/helpers/osmFiles.d.ts.map +1 -1
- package/script/helpers/osmFiles.js +237 -23
- package/script/methods/loadGeoData.d.ts.map +1 -1
- package/script/methods/loadGeoData.js +3 -38
- package/script/methods/loadOpenStreetMap.d.ts +24 -0
- package/script/methods/loadOpenStreetMap.d.ts.map +1 -0
- package/script/methods/loadOpenStreetMap.js +375 -0
- package/esm/methods/loadOSM.d.ts +0 -17
- package/esm/methods/loadOSM.d.ts.map +0 -1
- package/esm/methods/loadOSM.js +0 -194
- package/script/methods/loadOSM.d.ts +0 -17
- package/script/methods/loadOSM.d.ts.map +0 -1
- package/script/methods/loadOSM.js +0 -201
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.default = loadOpenStreetMap;
|
|
7
|
+
exports.generateOverpassQuery = generateOverpassQuery;
|
|
8
|
+
const node_fs_1 = require("node:fs");
|
|
9
|
+
const node_path_1 = require("node:path");
|
|
10
|
+
const SDAError_js_1 = __importDefault(require("../class/SDAError.js"));
|
|
11
|
+
const loadOsmFile_js_1 = __importDefault(require("../helpers/loadOsmFile.js"));
|
|
12
|
+
const osmCache_js_1 = require("../helpers/osmCache.js");
|
|
13
|
+
const osmFiles_js_1 = require("../helpers/osmFiles.js");
|
|
14
|
+
const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
|
|
15
|
+
const prettyDuration_js_1 = __importDefault(require("../helpers/prettyDuration.js"));
|
|
16
|
+
const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
|
|
17
|
+
const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
|
|
18
|
+
const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
|
|
19
|
+
const DEFAULT_OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter";
|
|
20
|
+
const DEFAULT_REMOTE_FILE_TIMEOUT = 30 * 60;
|
|
21
|
+
const QUERY_GENERATOR_VERSION = "overpass-ql-v1";
|
|
22
|
+
function loadOpenStreetMap(table, source, options = {}) {
|
|
23
|
+
if (typeof source === "string") {
|
|
24
|
+
queueOpenStreetMapFileLoad(table, source, options);
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
queueOverpassLoad(table, source, options);
|
|
28
|
+
}
|
|
29
|
+
function queueOpenStreetMapFileLoad(table, file, options) {
|
|
30
|
+
const suffix = (0, osmFiles_js_1.getOsmFileSuffix)(file);
|
|
31
|
+
if (suffix === null) {
|
|
32
|
+
throw new Error("loadOpenStreetMap() file paths and URLs must end with .osm or .osm.pbf.");
|
|
33
|
+
}
|
|
34
|
+
options = structuredClone(options);
|
|
35
|
+
assertOptions(options, false);
|
|
36
|
+
let remoteUrl;
|
|
37
|
+
if ((0, osmFiles_js_1.isRemoteOsmUrl)(file)) {
|
|
38
|
+
remoteUrl = (0, osmFiles_js_1.normalizeRemoteOsmUrl)(file);
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
try {
|
|
42
|
+
const url = new URL(file);
|
|
43
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
44
|
+
throw new Error("loadOpenStreetMap() URL sources must use HTTP or HTTPS.");
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
if (error instanceof Error && error.message.includes("must use HTTP")) {
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (options.retries !== undefined || options.retryDelay !== undefined) {
|
|
53
|
+
throw new Error("loadOpenStreetMap() options.retries and options.retryDelay are network-only and cannot be used with a local path.");
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
const parameters = { file, options };
|
|
57
|
+
(0, queueOp_js_1.default)(table, {
|
|
58
|
+
kind: "barrier",
|
|
59
|
+
method: "loadOpenStreetMap()",
|
|
60
|
+
parameters,
|
|
61
|
+
execute: () => executeOpenStreetMapFileLoad(table, remoteUrl ?? file, suffix, options, parameters),
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
async function executeOpenStreetMapFileLoad(table, file, suffix, options, parameters) {
|
|
65
|
+
const started = Date.now();
|
|
66
|
+
const verbose = options.verbose === true || table.sdb.cacheVerbose;
|
|
67
|
+
const remote = (0, osmFiles_js_1.isRemoteOsmUrl)(file);
|
|
68
|
+
const displayedSource = remote ? (0, osmFiles_js_1.sanitizeOsmUrl)(file) : (0, node_path_1.resolve)(file);
|
|
69
|
+
verbose && console.log(`loadOpenStreetMap(): loading ${displayedSource}`);
|
|
70
|
+
try {
|
|
71
|
+
if (remote) {
|
|
72
|
+
const cacheId = (0, osmCache_js_1.createOsmCacheId)({ kind: "remote", url: file });
|
|
73
|
+
await (0, osmCache_js_1.useProcessedOsmCache)(table, {
|
|
74
|
+
id: cacheId,
|
|
75
|
+
enabled: options.cache ?? true,
|
|
76
|
+
ttl: options.ttl,
|
|
77
|
+
verbose,
|
|
78
|
+
parameters,
|
|
79
|
+
compute: async () => {
|
|
80
|
+
const temporaryFile = await (0, osmFiles_js_1.downloadOsmToTemporaryFile)(file, {
|
|
81
|
+
suffix,
|
|
82
|
+
prefix: cacheId,
|
|
83
|
+
request: { headers: { "accept-encoding": "gzip" } },
|
|
84
|
+
retries: options.retries,
|
|
85
|
+
retryDelay: options.retryDelay,
|
|
86
|
+
timeout: DEFAULT_REMOTE_FILE_TIMEOUT,
|
|
87
|
+
verbose,
|
|
88
|
+
});
|
|
89
|
+
try {
|
|
90
|
+
await processOsmFile(table, temporaryFile, parameters, verbose);
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
(0, node_fs_1.rmSync)(temporaryFile, { force: true });
|
|
94
|
+
verbose && console.log("Removed the temporary raw OSM download.");
|
|
95
|
+
}
|
|
96
|
+
},
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
verbose &&
|
|
101
|
+
console.log("Resolving and fingerprinting the local OSM file...");
|
|
102
|
+
const fingerprintStarted = Date.now();
|
|
103
|
+
const resolvedPath = (0, node_fs_1.realpathSync)((0, node_path_1.resolve)(file));
|
|
104
|
+
const stat = (0, node_fs_1.statSync)(resolvedPath, { bigint: true });
|
|
105
|
+
if (!stat.isFile()) {
|
|
106
|
+
throw new Error(`The OpenStreetMap source is not a file: ${resolvedPath}`);
|
|
107
|
+
}
|
|
108
|
+
verbose && console.log(`Resolved ${(0, osmFiles_js_1.formatBytes)(Number(stat.size))} and fingerprinted it in ${(0, prettyDuration_js_1.default)(fingerprintStarted)}.`);
|
|
109
|
+
const cacheId = (0, osmCache_js_1.createOsmCacheId)({
|
|
110
|
+
kind: "local",
|
|
111
|
+
path: resolvedPath,
|
|
112
|
+
size: stat.size.toString(),
|
|
113
|
+
mtimeNs: stat.mtimeNs.toString(),
|
|
114
|
+
});
|
|
115
|
+
await (0, osmCache_js_1.useProcessedOsmCache)(table, {
|
|
116
|
+
id: cacheId,
|
|
117
|
+
enabled: options.cache ?? true,
|
|
118
|
+
ttl: options.ttl,
|
|
119
|
+
verbose,
|
|
120
|
+
parameters,
|
|
121
|
+
compute: () => processOsmFile(table, resolvedPath, parameters, verbose),
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
verbose && console.log(`loadOpenStreetMap(): completed in ${(0, prettyDuration_js_1.default)(started)}.`);
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
if (error instanceof SDAError_js_1.default)
|
|
128
|
+
throw error;
|
|
129
|
+
throw new SDAError_js_1.default({
|
|
130
|
+
method: "loadOpenStreetMap()",
|
|
131
|
+
parameters,
|
|
132
|
+
query: "",
|
|
133
|
+
cause: error,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
function queueOverpassLoad(table, bbox, options) {
|
|
138
|
+
bbox = normalizeBoundingBox(bbox);
|
|
139
|
+
options = structuredClone(options);
|
|
140
|
+
assertOptions(options, true);
|
|
141
|
+
const endpoint = normalizeEndpoint(options.endpoint ?? DEFAULT_OVERPASS_ENDPOINT);
|
|
142
|
+
const filters = normalizeFilters(options.filters);
|
|
143
|
+
const query = generateOverpassQuery(bbox, filters, options.timeout);
|
|
144
|
+
const parameters = { bbox, options };
|
|
145
|
+
(0, queueOp_js_1.default)(table, {
|
|
146
|
+
kind: "barrier",
|
|
147
|
+
method: "loadOpenStreetMap()",
|
|
148
|
+
parameters,
|
|
149
|
+
execute: () => executeOverpassLoad(table, options, endpoint, query, parameters),
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
async function executeOverpassLoad(table, options, endpoint, query, parameters) {
|
|
153
|
+
const started = Date.now();
|
|
154
|
+
const verbose = options.verbose === true || table.sdb.cacheVerbose;
|
|
155
|
+
verbose && console.log(`loadOpenStreetMap(): querying ${(0, osmFiles_js_1.sanitizeOsmUrl)(endpoint)}`);
|
|
156
|
+
try {
|
|
157
|
+
const cacheId = (0, osmCache_js_1.createOsmCacheId)({
|
|
158
|
+
kind: "overpass",
|
|
159
|
+
endpoint,
|
|
160
|
+
query,
|
|
161
|
+
queryGeneratorVersion: QUERY_GENERATOR_VERSION,
|
|
162
|
+
});
|
|
163
|
+
await (0, osmCache_js_1.useProcessedOsmCache)(table, {
|
|
164
|
+
id: cacheId,
|
|
165
|
+
enabled: options.cache ?? true,
|
|
166
|
+
ttl: options.ttl,
|
|
167
|
+
verbose,
|
|
168
|
+
parameters,
|
|
169
|
+
compute: async () => {
|
|
170
|
+
const body = new URLSearchParams({ data: query });
|
|
171
|
+
const temporaryFile = await (0, osmFiles_js_1.downloadOsmToTemporaryFile)(endpoint, {
|
|
172
|
+
suffix: ".osm",
|
|
173
|
+
prefix: cacheId,
|
|
174
|
+
request: {
|
|
175
|
+
method: "POST",
|
|
176
|
+
headers: {
|
|
177
|
+
"accept-encoding": "gzip",
|
|
178
|
+
"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
|
|
179
|
+
"user-agent": "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
|
|
180
|
+
},
|
|
181
|
+
body,
|
|
182
|
+
},
|
|
183
|
+
retries: options.retries,
|
|
184
|
+
retryDelay: options.retryDelay,
|
|
185
|
+
timeout: options.timeout,
|
|
186
|
+
overpass: true,
|
|
187
|
+
verbose,
|
|
188
|
+
});
|
|
189
|
+
try {
|
|
190
|
+
try {
|
|
191
|
+
await assertSuccessfulOverpassResponse(temporaryFile);
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
const reason = error instanceof Error
|
|
195
|
+
? error.message
|
|
196
|
+
: String(error);
|
|
197
|
+
throw new Error(`${reason}\nNot retrying because the Overpass response was well-formed and reported a query error.\nNo invalid cached data was returned.`, { cause: error });
|
|
198
|
+
}
|
|
199
|
+
await processOsmFile(table, temporaryFile, parameters, verbose);
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
(0, node_fs_1.rmSync)(temporaryFile, { force: true });
|
|
203
|
+
verbose &&
|
|
204
|
+
console.log("Removed the temporary raw Overpass download.");
|
|
205
|
+
}
|
|
206
|
+
},
|
|
207
|
+
});
|
|
208
|
+
verbose && console.log(`loadOpenStreetMap(): completed in ${(0, prettyDuration_js_1.default)(started)}.`);
|
|
209
|
+
}
|
|
210
|
+
catch (error) {
|
|
211
|
+
if (error instanceof SDAError_js_1.default)
|
|
212
|
+
throw error;
|
|
213
|
+
throw new SDAError_js_1.default({
|
|
214
|
+
method: "loadOpenStreetMap()",
|
|
215
|
+
parameters,
|
|
216
|
+
query,
|
|
217
|
+
cause: error,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
async function processOsmFile(table, file, parameters, verbose) {
|
|
222
|
+
verbose && console.log("Processing OSM data with Osmium...");
|
|
223
|
+
const started = Date.now();
|
|
224
|
+
try {
|
|
225
|
+
await (0, loadOsmFile_js_1.default)(table, file, {
|
|
226
|
+
method: "loadOpenStreetMap()",
|
|
227
|
+
parameters,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
const cause = error instanceof SDAError_js_1.default ? error.cause : error;
|
|
232
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
233
|
+
throw new Error(`${reason}\nNot retrying because Osmium or a downstream DuckDB operation rejected the source data.\nNo invalid cached data was returned.`, { cause: error });
|
|
234
|
+
}
|
|
235
|
+
if (verbose) {
|
|
236
|
+
const data = await (0, queryDB_js_1.default)(table, `SELECT count(*)::DOUBLE AS count FROM ${(0, quoteIdentifier_js_1.default)(table.name)};`, (0, mergeOptions_js_1.default)(table, {
|
|
237
|
+
table: table.name,
|
|
238
|
+
method: "loadOpenStreetMap()",
|
|
239
|
+
parameters,
|
|
240
|
+
returnData: true,
|
|
241
|
+
}));
|
|
242
|
+
const count = Number(data?.[0]?.count ?? 0);
|
|
243
|
+
console.log(`Processed ${count.toLocaleString("en-US")} rows in ${(0, prettyDuration_js_1.default)(started)}.`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
async function assertSuccessfulOverpassResponse(file) {
|
|
247
|
+
const contents = await Deno.readTextFile(file);
|
|
248
|
+
const match = /<(remark|error)\b[^>]*>([\s\S]*?)(?:<\/\1\s*>|$)/i.exec(contents);
|
|
249
|
+
if (match === null)
|
|
250
|
+
return;
|
|
251
|
+
const message = match[2]
|
|
252
|
+
.replace(/<[^>]*>/g, " ")
|
|
253
|
+
.replace(/\s+/g, " ")
|
|
254
|
+
.trim()
|
|
255
|
+
.slice(0, 500);
|
|
256
|
+
throw new Error(message.length === 0
|
|
257
|
+
? "The Overpass endpoint returned an error response."
|
|
258
|
+
: `The Overpass endpoint returned an error response: ${message}`);
|
|
259
|
+
}
|
|
260
|
+
function generateOverpassQuery(bbox, filters, timeout) {
|
|
261
|
+
const box = `${bbox.south},${bbox.west},${bbox.north},${bbox.east}`;
|
|
262
|
+
const selections = filters.map((filter) => `nwr${filter}(${box});`);
|
|
263
|
+
const selection = selections.length === 1
|
|
264
|
+
? selections[0]
|
|
265
|
+
: `(${selections.join("")});`;
|
|
266
|
+
return `[out:xml]${timeout === undefined ? "" : `[timeout:${timeout}]`};${selection}(._;>;);out body;`;
|
|
267
|
+
}
|
|
268
|
+
function normalizeBoundingBox(bbox) {
|
|
269
|
+
const normalized = structuredClone(bbox);
|
|
270
|
+
for (const [name, value] of Object.entries(normalized)) {
|
|
271
|
+
if (!Number.isFinite(value)) {
|
|
272
|
+
throw new Error(`loadOpenStreetMap() bbox.${name} must be a finite number.`);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
if (normalized.west < -180 || normalized.west > 180) {
|
|
276
|
+
throw new Error("loadOpenStreetMap() bbox.west must be between -180 and 180.");
|
|
277
|
+
}
|
|
278
|
+
if (normalized.east < -180 || normalized.east > 180) {
|
|
279
|
+
throw new Error("loadOpenStreetMap() bbox.east must be between -180 and 180.");
|
|
280
|
+
}
|
|
281
|
+
if (normalized.south < -90 || normalized.south > 90) {
|
|
282
|
+
throw new Error("loadOpenStreetMap() bbox.south must be between -90 and 90.");
|
|
283
|
+
}
|
|
284
|
+
if (normalized.north < -90 || normalized.north > 90) {
|
|
285
|
+
throw new Error("loadOpenStreetMap() bbox.north must be between -90 and 90.");
|
|
286
|
+
}
|
|
287
|
+
if (normalized.west >= normalized.east) {
|
|
288
|
+
throw new Error("loadOpenStreetMap() bbox.west must be less than bbox.east.");
|
|
289
|
+
}
|
|
290
|
+
if (normalized.south >= normalized.north) {
|
|
291
|
+
throw new Error("loadOpenStreetMap() bbox.south must be less than bbox.north.");
|
|
292
|
+
}
|
|
293
|
+
normalized.west = Object.is(normalized.west, -0) ? 0 : normalized.west;
|
|
294
|
+
normalized.south = Object.is(normalized.south, -0) ? 0 : normalized.south;
|
|
295
|
+
normalized.east = Object.is(normalized.east, -0) ? 0 : normalized.east;
|
|
296
|
+
normalized.north = Object.is(normalized.north, -0) ? 0 : normalized.north;
|
|
297
|
+
return normalized;
|
|
298
|
+
}
|
|
299
|
+
function normalizeFilters(filters) {
|
|
300
|
+
if (typeof filters === "string") {
|
|
301
|
+
const filter = filters.trim();
|
|
302
|
+
if (filter.length === 0) {
|
|
303
|
+
throw new Error("loadOpenStreetMap() filters must contain at least one non-empty filter fragment.");
|
|
304
|
+
}
|
|
305
|
+
return [filter];
|
|
306
|
+
}
|
|
307
|
+
if (!Array.isArray(filters)) {
|
|
308
|
+
throw new TypeError("loadOpenStreetMap() options.filters is required.");
|
|
309
|
+
}
|
|
310
|
+
const tuples = isFilterTuple(filters) ? [filters] : filters;
|
|
311
|
+
const normalized = tuples.map(([key, value]) => {
|
|
312
|
+
if (typeof key !== "string" || typeof value !== "string") {
|
|
313
|
+
throw new TypeError("loadOpenStreetMap() filter tuples must contain two strings.");
|
|
314
|
+
}
|
|
315
|
+
if (key.length === 0) {
|
|
316
|
+
throw new Error("loadOpenStreetMap() filter keys must not be empty.");
|
|
317
|
+
}
|
|
318
|
+
return `[${JSON.stringify(key)}=${JSON.stringify(value)}]`;
|
|
319
|
+
});
|
|
320
|
+
if (normalized.length === 0) {
|
|
321
|
+
throw new Error("loadOpenStreetMap() filters must contain at least one filter tuple.");
|
|
322
|
+
}
|
|
323
|
+
return normalized;
|
|
324
|
+
}
|
|
325
|
+
function isFilterTuple(filters) {
|
|
326
|
+
return filters.length === 2 && typeof filters[0] === "string" &&
|
|
327
|
+
typeof filters[1] === "string";
|
|
328
|
+
}
|
|
329
|
+
function normalizeEndpoint(endpoint) {
|
|
330
|
+
try {
|
|
331
|
+
return (0, osmFiles_js_1.normalizeRemoteOsmUrl)(endpoint);
|
|
332
|
+
}
|
|
333
|
+
catch {
|
|
334
|
+
throw new Error("loadOpenStreetMap() endpoint must be a valid HTTP or HTTPS URL.");
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
function assertOptions(options, overpass) {
|
|
338
|
+
const allowed = new Set([
|
|
339
|
+
"cache",
|
|
340
|
+
"ttl",
|
|
341
|
+
"retries",
|
|
342
|
+
"retryDelay",
|
|
343
|
+
"verbose",
|
|
344
|
+
...(overpass ? ["filters", "endpoint", "timeout"] : []),
|
|
345
|
+
]);
|
|
346
|
+
const unsupported = Object.keys(options).filter((key) => !allowed.has(key));
|
|
347
|
+
if (unsupported.length > 0) {
|
|
348
|
+
throw new Error(`loadOpenStreetMap() does not support ${unsupported.map((key) => `options.${key}`).join(", ")} for this source. Apply filtering and projection with normal table methods after loading.`);
|
|
349
|
+
}
|
|
350
|
+
if (options.cache !== undefined && typeof options.cache !== "boolean") {
|
|
351
|
+
throw new TypeError("loadOpenStreetMap() options.cache must be a boolean.");
|
|
352
|
+
}
|
|
353
|
+
if (options.verbose !== undefined && typeof options.verbose !== "boolean") {
|
|
354
|
+
throw new TypeError("loadOpenStreetMap() options.verbose must be a boolean.");
|
|
355
|
+
}
|
|
356
|
+
if (options.ttl !== undefined &&
|
|
357
|
+
(!Number.isFinite(options.ttl) || options.ttl < 0)) {
|
|
358
|
+
throw new Error("loadOpenStreetMap() options.ttl must be a finite, non-negative number of seconds.");
|
|
359
|
+
}
|
|
360
|
+
if (options.cache === false && options.ttl !== undefined) {
|
|
361
|
+
throw new Error("loadOpenStreetMap() options.cache cannot be false when options.ttl is set.");
|
|
362
|
+
}
|
|
363
|
+
if (options.retries !== undefined &&
|
|
364
|
+
(!Number.isInteger(options.retries) || options.retries < 0)) {
|
|
365
|
+
throw new Error("loadOpenStreetMap() options.retries must be a non-negative integer.");
|
|
366
|
+
}
|
|
367
|
+
if (options.retryDelay !== undefined &&
|
|
368
|
+
(!Number.isFinite(options.retryDelay) || options.retryDelay < 0)) {
|
|
369
|
+
throw new Error("loadOpenStreetMap() options.retryDelay must be a finite, non-negative number of seconds.");
|
|
370
|
+
}
|
|
371
|
+
if (overpass && options.timeout !== undefined &&
|
|
372
|
+
(!Number.isInteger(options.timeout) || options.timeout <= 0)) {
|
|
373
|
+
throw new Error("loadOpenStreetMap() options.timeout must be a positive integer in seconds.");
|
|
374
|
+
}
|
|
375
|
+
}
|
package/esm/methods/loadOSM.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import type SimpleTable from "../class/SimpleTable.js";
|
|
2
|
-
type BoundingBox = {
|
|
3
|
-
west: number;
|
|
4
|
-
south: number;
|
|
5
|
-
east: number;
|
|
6
|
-
north: number;
|
|
7
|
-
};
|
|
8
|
-
type LoadOsmOptions = {
|
|
9
|
-
filters: string | [string, string] | [string, string][];
|
|
10
|
-
endpoint?: string;
|
|
11
|
-
timeout?: number;
|
|
12
|
-
cache?: boolean;
|
|
13
|
-
};
|
|
14
|
-
export default function loadOSM(table: SimpleTable, bbox: BoundingBox, options: LoadOsmOptions): void;
|
|
15
|
-
export declare function generateOverpassQuery(bbox: BoundingBox, filters: string[], timeout?: number): string;
|
|
16
|
-
export {};
|
|
17
|
-
//# sourceMappingURL=loadOSM.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"loadOSM.d.ts","sourceRoot":"","sources":["../../src/methods/loadOSM.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AASvD,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,cAAc,GAAG;IACpB,OAAO,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,cAAc,GACtB,IAAI,CAkBN;AAgHD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE,MAAM,GACf,MAAM,CASR"}
|
package/esm/methods/loadOSM.js
DELETED
|
@@ -1,194 +0,0 @@
|
|
|
1
|
-
import { createReadStream, rmSync } from "node:fs";
|
|
2
|
-
import SDAError from "../class/SDAError.js";
|
|
3
|
-
import loadOsmFile from "../helpers/loadOsmFile.js";
|
|
4
|
-
import { createOsmCacheId, useProcessedOsmCache } from "../helpers/osmCache.js";
|
|
5
|
-
import { downloadOsmToTemporaryFile } from "../helpers/osmFiles.js";
|
|
6
|
-
import queueOp from "../helpers/queueOp.js";
|
|
7
|
-
const DEFAULT_OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter";
|
|
8
|
-
const QUERY_GENERATOR_VERSION = "overpass-ql-v1";
|
|
9
|
-
export default function loadOSM(table, bbox, options) {
|
|
10
|
-
bbox = normalizeBoundingBox(bbox);
|
|
11
|
-
options = structuredClone(options);
|
|
12
|
-
const endpoint = normalizeEndpoint(options.endpoint ?? DEFAULT_OVERPASS_ENDPOINT);
|
|
13
|
-
const filters = normalizeFilters(options.filters);
|
|
14
|
-
assertTimeout(options.timeout);
|
|
15
|
-
const query = generateOverpassQuery(bbox, filters, options.timeout);
|
|
16
|
-
const parameters = { bbox, options };
|
|
17
|
-
queueOp(table, {
|
|
18
|
-
kind: "barrier",
|
|
19
|
-
method: "loadOSM()",
|
|
20
|
-
parameters,
|
|
21
|
-
execute: () => executeLoadOSM(table, bbox, options, endpoint, query, parameters),
|
|
22
|
-
});
|
|
23
|
-
}
|
|
24
|
-
async function executeLoadOSM(table, bbox, options, endpoint, query, parameters) {
|
|
25
|
-
try {
|
|
26
|
-
const cacheEnabled = options.cache ?? true;
|
|
27
|
-
const cacheId = createOsmCacheId({
|
|
28
|
-
endpoint,
|
|
29
|
-
bbox,
|
|
30
|
-
query,
|
|
31
|
-
queryGeneratorVersion: QUERY_GENERATOR_VERSION,
|
|
32
|
-
});
|
|
33
|
-
await useProcessedOsmCache(table, cacheId, cacheEnabled, parameters, async () => {
|
|
34
|
-
const timeoutSignal = options.timeout === undefined
|
|
35
|
-
? undefined
|
|
36
|
-
: AbortSignal.timeout(options.timeout * 1000);
|
|
37
|
-
const body = new URLSearchParams({ data: query });
|
|
38
|
-
const temporaryFile = await downloadOsmToTemporaryFile(endpoint, {
|
|
39
|
-
suffix: ".osm",
|
|
40
|
-
prefix: cacheId,
|
|
41
|
-
request: {
|
|
42
|
-
method: "POST",
|
|
43
|
-
headers: {
|
|
44
|
-
"accept-encoding": "gzip",
|
|
45
|
-
"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
|
|
46
|
-
"user-agent": "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
|
|
47
|
-
},
|
|
48
|
-
body,
|
|
49
|
-
signal: timeoutSignal,
|
|
50
|
-
},
|
|
51
|
-
});
|
|
52
|
-
try {
|
|
53
|
-
await assertSuccessfulOverpassResponse(temporaryFile);
|
|
54
|
-
await loadOsmFile(table, temporaryFile, {
|
|
55
|
-
method: "loadOSM()",
|
|
56
|
-
parameters,
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
finally {
|
|
60
|
-
rmSync(temporaryFile, { force: true });
|
|
61
|
-
}
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
catch (error) {
|
|
65
|
-
if (error instanceof SDAError) {
|
|
66
|
-
throw error;
|
|
67
|
-
}
|
|
68
|
-
throw new SDAError({
|
|
69
|
-
method: "loadOSM()",
|
|
70
|
-
parameters,
|
|
71
|
-
query,
|
|
72
|
-
cause: error,
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
async function assertSuccessfulOverpassResponse(file) {
|
|
77
|
-
const stream = createReadStream(file, { encoding: "utf-8" });
|
|
78
|
-
let buffer = "";
|
|
79
|
-
let errorTag;
|
|
80
|
-
for await (const chunk of stream) {
|
|
81
|
-
buffer += String(chunk);
|
|
82
|
-
if (errorTag === undefined) {
|
|
83
|
-
const openingTag = /<(remark|error)\b[^>]*>/i.exec(buffer);
|
|
84
|
-
if (openingTag === null) {
|
|
85
|
-
buffer = buffer.slice(-4096);
|
|
86
|
-
continue;
|
|
87
|
-
}
|
|
88
|
-
errorTag = openingTag[1].toLowerCase() === "remark" ? "remark" : "error";
|
|
89
|
-
buffer = buffer.slice(openingTag.index + openingTag[0].length);
|
|
90
|
-
}
|
|
91
|
-
const closingTag = new RegExp(`</${errorTag}\\s*>`, "i").exec(buffer);
|
|
92
|
-
if (closingTag !== null) {
|
|
93
|
-
throwOverpassResponseError(buffer.slice(0, closingTag.index));
|
|
94
|
-
}
|
|
95
|
-
if (buffer.length >= 4096) {
|
|
96
|
-
throwOverpassResponseError(buffer.slice(0, 4096));
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
if (errorTag !== undefined) {
|
|
100
|
-
throwOverpassResponseError(buffer);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
function throwOverpassResponseError(contents) {
|
|
104
|
-
const message = contents
|
|
105
|
-
.replace(/<[^>]*>/g, " ")
|
|
106
|
-
.replace(/\s+/g, " ")
|
|
107
|
-
.trim()
|
|
108
|
-
.slice(0, 500);
|
|
109
|
-
throw new Error(message.length === 0
|
|
110
|
-
? "The Overpass endpoint returned an error response."
|
|
111
|
-
: `The Overpass endpoint returned an error response: ${message}`);
|
|
112
|
-
}
|
|
113
|
-
export function generateOverpassQuery(bbox, filters, timeout) {
|
|
114
|
-
const box = `${bbox.south},${bbox.west},${bbox.north},${bbox.east}`;
|
|
115
|
-
const selections = filters.map((filter) => `nwr${filter}(${box});`);
|
|
116
|
-
const selection = selections.length === 1
|
|
117
|
-
? selections[0]
|
|
118
|
-
: `(${selections.join("")});`;
|
|
119
|
-
return `[out:xml]${timeout === undefined ? "" : `[timeout:${timeout}]`};${selection}(._;>;);out body;`;
|
|
120
|
-
}
|
|
121
|
-
function normalizeBoundingBox(bbox) {
|
|
122
|
-
const normalized = structuredClone(bbox);
|
|
123
|
-
for (const [name, value] of Object.entries(normalized)) {
|
|
124
|
-
if (!Number.isFinite(value)) {
|
|
125
|
-
throw new Error(`loadOSM() bbox.${name} must be a finite number.`);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
if (normalized.west < -180 || normalized.west > 180) {
|
|
129
|
-
throw new Error("loadOSM() bbox.west must be between -180 and 180.");
|
|
130
|
-
}
|
|
131
|
-
if (normalized.east < -180 || normalized.east > 180) {
|
|
132
|
-
throw new Error("loadOSM() bbox.east must be between -180 and 180.");
|
|
133
|
-
}
|
|
134
|
-
if (normalized.south < -90 || normalized.south > 90) {
|
|
135
|
-
throw new Error("loadOSM() bbox.south must be between -90 and 90.");
|
|
136
|
-
}
|
|
137
|
-
if (normalized.north < -90 || normalized.north > 90) {
|
|
138
|
-
throw new Error("loadOSM() bbox.north must be between -90 and 90.");
|
|
139
|
-
}
|
|
140
|
-
if (normalized.west >= normalized.east) {
|
|
141
|
-
throw new Error("loadOSM() bbox.west must be less than bbox.east.");
|
|
142
|
-
}
|
|
143
|
-
if (normalized.south >= normalized.north) {
|
|
144
|
-
throw new Error("loadOSM() bbox.south must be less than bbox.north.");
|
|
145
|
-
}
|
|
146
|
-
normalized.west = Object.is(normalized.west, -0) ? 0 : normalized.west;
|
|
147
|
-
normalized.south = Object.is(normalized.south, -0) ? 0 : normalized.south;
|
|
148
|
-
normalized.east = Object.is(normalized.east, -0) ? 0 : normalized.east;
|
|
149
|
-
normalized.north = Object.is(normalized.north, -0) ? 0 : normalized.north;
|
|
150
|
-
return normalized;
|
|
151
|
-
}
|
|
152
|
-
function normalizeFilters(filters) {
|
|
153
|
-
if (typeof filters === "string") {
|
|
154
|
-
const filter = filters.trim();
|
|
155
|
-
if (filter.length === 0) {
|
|
156
|
-
throw new Error("loadOSM() filters must contain at least one non-empty filter fragment.");
|
|
157
|
-
}
|
|
158
|
-
return [filter];
|
|
159
|
-
}
|
|
160
|
-
const tuples = isFilterTuple(filters) ? [filters] : filters;
|
|
161
|
-
const normalized = tuples.map(([key, value]) => {
|
|
162
|
-
if (key.length === 0) {
|
|
163
|
-
throw new Error("loadOSM() filter keys must not be empty.");
|
|
164
|
-
}
|
|
165
|
-
return `[${JSON.stringify(key)}=${JSON.stringify(value)}]`;
|
|
166
|
-
});
|
|
167
|
-
if (normalized.length === 0) {
|
|
168
|
-
throw new Error("loadOSM() filters must contain at least one filter tuple.");
|
|
169
|
-
}
|
|
170
|
-
return normalized;
|
|
171
|
-
}
|
|
172
|
-
function isFilterTuple(filters) {
|
|
173
|
-
return filters.length === 2 && typeof filters[0] === "string" &&
|
|
174
|
-
typeof filters[1] === "string";
|
|
175
|
-
}
|
|
176
|
-
function normalizeEndpoint(endpoint) {
|
|
177
|
-
let url;
|
|
178
|
-
try {
|
|
179
|
-
url = new URL(endpoint);
|
|
180
|
-
}
|
|
181
|
-
catch {
|
|
182
|
-
throw new Error("loadOSM() endpoint must be a valid HTTP or HTTPS URL.");
|
|
183
|
-
}
|
|
184
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
185
|
-
throw new Error("loadOSM() endpoint must be an HTTP or HTTPS URL.");
|
|
186
|
-
}
|
|
187
|
-
return url.toString();
|
|
188
|
-
}
|
|
189
|
-
function assertTimeout(timeout) {
|
|
190
|
-
if (timeout !== undefined &&
|
|
191
|
-
(!Number.isInteger(timeout) || timeout <= 0)) {
|
|
192
|
-
throw new Error("loadOSM() timeout must be a positive integer in seconds.");
|
|
193
|
-
}
|
|
194
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import type SimpleTable from "../class/SimpleTable.js";
|
|
2
|
-
type BoundingBox = {
|
|
3
|
-
west: number;
|
|
4
|
-
south: number;
|
|
5
|
-
east: number;
|
|
6
|
-
north: number;
|
|
7
|
-
};
|
|
8
|
-
type LoadOsmOptions = {
|
|
9
|
-
filters: string | [string, string] | [string, string][];
|
|
10
|
-
endpoint?: string;
|
|
11
|
-
timeout?: number;
|
|
12
|
-
cache?: boolean;
|
|
13
|
-
};
|
|
14
|
-
export default function loadOSM(table: SimpleTable, bbox: BoundingBox, options: LoadOsmOptions): void;
|
|
15
|
-
export declare function generateOverpassQuery(bbox: BoundingBox, filters: string[], timeout?: number): string;
|
|
16
|
-
export {};
|
|
17
|
-
//# sourceMappingURL=loadOSM.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"loadOSM.d.ts","sourceRoot":"","sources":["../../src/methods/loadOSM.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AASvD,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,cAAc,GAAG;IACpB,OAAO,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,cAAc,GACtB,IAAI,CAkBN;AAgHD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE,MAAM,GACf,MAAM,CASR"}
|