@nshiab/simple-data-analysis-core 2.0.2 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/class/SimpleTable.d.ts +50 -32
- package/esm/class/SimpleTable.d.ts.map +1 -1
- package/esm/class/SimpleTable.js +46 -32
- package/esm/helpers/loadOsmFile.d.ts +1 -3
- package/esm/helpers/loadOsmFile.d.ts.map +1 -1
- package/esm/helpers/loadOsmFile.js +3 -11
- package/esm/helpers/osmCache.d.ts +10 -13
- package/esm/helpers/osmCache.d.ts.map +1 -1
- package/esm/helpers/osmCache.js +177 -32
- package/esm/helpers/osmFiles.d.ts +9 -0
- package/esm/helpers/osmFiles.d.ts.map +1 -1
- package/esm/helpers/osmFiles.js +234 -23
- package/esm/methods/loadGeoData.d.ts.map +1 -1
- package/esm/methods/loadGeoData.js +4 -39
- package/esm/methods/loadOpenStreetMap.d.ts +24 -0
- package/esm/methods/loadOpenStreetMap.d.ts.map +1 -0
- package/esm/methods/loadOpenStreetMap.js +368 -0
- package/llm.md +79 -46
- package/llms.txt +3 -0
- package/package.json +1 -1
- package/script/class/SimpleTable.d.ts +50 -32
- package/script/class/SimpleTable.d.ts.map +1 -1
- package/script/class/SimpleTable.js +46 -32
- package/script/helpers/loadOsmFile.d.ts +1 -3
- package/script/helpers/loadOsmFile.d.ts.map +1 -1
- package/script/helpers/loadOsmFile.js +3 -11
- package/script/helpers/osmCache.d.ts +10 -13
- package/script/helpers/osmCache.d.ts.map +1 -1
- package/script/helpers/osmCache.js +177 -65
- package/script/helpers/osmFiles.d.ts +9 -0
- package/script/helpers/osmFiles.d.ts.map +1 -1
- package/script/helpers/osmFiles.js +237 -23
- package/script/methods/loadGeoData.d.ts.map +1 -1
- package/script/methods/loadGeoData.js +3 -38
- package/script/methods/loadOpenStreetMap.d.ts +24 -0
- package/script/methods/loadOpenStreetMap.d.ts.map +1 -0
- package/script/methods/loadOpenStreetMap.js +375 -0
- package/esm/methods/loadOSM.d.ts +0 -17
- package/esm/methods/loadOSM.d.ts.map +0 -1
- package/esm/methods/loadOSM.js +0 -194
- package/script/methods/loadOSM.d.ts +0 -17
- package/script/methods/loadOSM.d.ts.map +0 -1
- package/script/methods/loadOSM.js +0 -201
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
import { realpathSync, rmSync, statSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import SDAError from "../class/SDAError.js";
|
|
4
|
+
import loadOsmFile from "../helpers/loadOsmFile.js";
|
|
5
|
+
import { createOsmCacheId, useProcessedOsmCache } from "../helpers/osmCache.js";
|
|
6
|
+
import { downloadOsmToTemporaryFile, formatBytes, getOsmFileSuffix, isRemoteOsmUrl, normalizeRemoteOsmUrl, sanitizeOsmUrl, } from "../helpers/osmFiles.js";
|
|
7
|
+
import mergeOptions from "../helpers/mergeOptions.js";
|
|
8
|
+
import prettyDuration from "../helpers/prettyDuration.js";
|
|
9
|
+
import queryDB from "../helpers/queryDB.js";
|
|
10
|
+
import quoteIdentifier from "../helpers/quoteIdentifier.js";
|
|
11
|
+
import queueOp from "../helpers/queueOp.js";
|
|
12
|
+
const DEFAULT_OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter";
|
|
13
|
+
const DEFAULT_REMOTE_FILE_TIMEOUT = 30 * 60;
|
|
14
|
+
const QUERY_GENERATOR_VERSION = "overpass-ql-v1";
|
|
15
|
+
export default function loadOpenStreetMap(table, source, options = {}) {
|
|
16
|
+
if (typeof source === "string") {
|
|
17
|
+
queueOpenStreetMapFileLoad(table, source, options);
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
queueOverpassLoad(table, source, options);
|
|
21
|
+
}
|
|
22
|
+
function queueOpenStreetMapFileLoad(table, file, options) {
|
|
23
|
+
const suffix = getOsmFileSuffix(file);
|
|
24
|
+
if (suffix === null) {
|
|
25
|
+
throw new Error("loadOpenStreetMap() file paths and URLs must end with .osm or .osm.pbf.");
|
|
26
|
+
}
|
|
27
|
+
options = structuredClone(options);
|
|
28
|
+
assertOptions(options, false);
|
|
29
|
+
let remoteUrl;
|
|
30
|
+
if (isRemoteOsmUrl(file)) {
|
|
31
|
+
remoteUrl = normalizeRemoteOsmUrl(file);
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
try {
|
|
35
|
+
const url = new URL(file);
|
|
36
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
37
|
+
throw new Error("loadOpenStreetMap() URL sources must use HTTP or HTTPS.");
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
if (error instanceof Error && error.message.includes("must use HTTP")) {
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (options.retries !== undefined || options.retryDelay !== undefined) {
|
|
46
|
+
throw new Error("loadOpenStreetMap() options.retries and options.retryDelay are network-only and cannot be used with a local path.");
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const parameters = { file, options };
|
|
50
|
+
queueOp(table, {
|
|
51
|
+
kind: "barrier",
|
|
52
|
+
method: "loadOpenStreetMap()",
|
|
53
|
+
parameters,
|
|
54
|
+
execute: () => executeOpenStreetMapFileLoad(table, remoteUrl ?? file, suffix, options, parameters),
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
async function executeOpenStreetMapFileLoad(table, file, suffix, options, parameters) {
|
|
58
|
+
const started = Date.now();
|
|
59
|
+
const verbose = options.verbose === true || table.sdb.cacheVerbose;
|
|
60
|
+
const remote = isRemoteOsmUrl(file);
|
|
61
|
+
const displayedSource = remote ? sanitizeOsmUrl(file) : resolve(file);
|
|
62
|
+
verbose && console.log(`loadOpenStreetMap(): loading ${displayedSource}`);
|
|
63
|
+
try {
|
|
64
|
+
if (remote) {
|
|
65
|
+
const cacheId = createOsmCacheId({ kind: "remote", url: file });
|
|
66
|
+
await useProcessedOsmCache(table, {
|
|
67
|
+
id: cacheId,
|
|
68
|
+
enabled: options.cache ?? true,
|
|
69
|
+
ttl: options.ttl,
|
|
70
|
+
verbose,
|
|
71
|
+
parameters,
|
|
72
|
+
compute: async () => {
|
|
73
|
+
const temporaryFile = await downloadOsmToTemporaryFile(file, {
|
|
74
|
+
suffix,
|
|
75
|
+
prefix: cacheId,
|
|
76
|
+
request: { headers: { "accept-encoding": "gzip" } },
|
|
77
|
+
retries: options.retries,
|
|
78
|
+
retryDelay: options.retryDelay,
|
|
79
|
+
timeout: DEFAULT_REMOTE_FILE_TIMEOUT,
|
|
80
|
+
verbose,
|
|
81
|
+
});
|
|
82
|
+
try {
|
|
83
|
+
await processOsmFile(table, temporaryFile, parameters, verbose);
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
rmSync(temporaryFile, { force: true });
|
|
87
|
+
verbose && console.log("Removed the temporary raw OSM download.");
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
verbose &&
|
|
94
|
+
console.log("Resolving and fingerprinting the local OSM file...");
|
|
95
|
+
const fingerprintStarted = Date.now();
|
|
96
|
+
const resolvedPath = realpathSync(resolve(file));
|
|
97
|
+
const stat = statSync(resolvedPath, { bigint: true });
|
|
98
|
+
if (!stat.isFile()) {
|
|
99
|
+
throw new Error(`The OpenStreetMap source is not a file: ${resolvedPath}`);
|
|
100
|
+
}
|
|
101
|
+
verbose && console.log(`Resolved ${formatBytes(Number(stat.size))} and fingerprinted it in ${prettyDuration(fingerprintStarted)}.`);
|
|
102
|
+
const cacheId = createOsmCacheId({
|
|
103
|
+
kind: "local",
|
|
104
|
+
path: resolvedPath,
|
|
105
|
+
size: stat.size.toString(),
|
|
106
|
+
mtimeNs: stat.mtimeNs.toString(),
|
|
107
|
+
});
|
|
108
|
+
await useProcessedOsmCache(table, {
|
|
109
|
+
id: cacheId,
|
|
110
|
+
enabled: options.cache ?? true,
|
|
111
|
+
ttl: options.ttl,
|
|
112
|
+
verbose,
|
|
113
|
+
parameters,
|
|
114
|
+
compute: () => processOsmFile(table, resolvedPath, parameters, verbose),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
verbose && console.log(`loadOpenStreetMap(): completed in ${prettyDuration(started)}.`);
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
if (error instanceof SDAError)
|
|
121
|
+
throw error;
|
|
122
|
+
throw new SDAError({
|
|
123
|
+
method: "loadOpenStreetMap()",
|
|
124
|
+
parameters,
|
|
125
|
+
query: "",
|
|
126
|
+
cause: error,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
function queueOverpassLoad(table, bbox, options) {
|
|
131
|
+
bbox = normalizeBoundingBox(bbox);
|
|
132
|
+
options = structuredClone(options);
|
|
133
|
+
assertOptions(options, true);
|
|
134
|
+
const endpoint = normalizeEndpoint(options.endpoint ?? DEFAULT_OVERPASS_ENDPOINT);
|
|
135
|
+
const filters = normalizeFilters(options.filters);
|
|
136
|
+
const query = generateOverpassQuery(bbox, filters, options.timeout);
|
|
137
|
+
const parameters = { bbox, options };
|
|
138
|
+
queueOp(table, {
|
|
139
|
+
kind: "barrier",
|
|
140
|
+
method: "loadOpenStreetMap()",
|
|
141
|
+
parameters,
|
|
142
|
+
execute: () => executeOverpassLoad(table, options, endpoint, query, parameters),
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
async function executeOverpassLoad(table, options, endpoint, query, parameters) {
|
|
146
|
+
const started = Date.now();
|
|
147
|
+
const verbose = options.verbose === true || table.sdb.cacheVerbose;
|
|
148
|
+
verbose && console.log(`loadOpenStreetMap(): querying ${sanitizeOsmUrl(endpoint)}`);
|
|
149
|
+
try {
|
|
150
|
+
const cacheId = createOsmCacheId({
|
|
151
|
+
kind: "overpass",
|
|
152
|
+
endpoint,
|
|
153
|
+
query,
|
|
154
|
+
queryGeneratorVersion: QUERY_GENERATOR_VERSION,
|
|
155
|
+
});
|
|
156
|
+
await useProcessedOsmCache(table, {
|
|
157
|
+
id: cacheId,
|
|
158
|
+
enabled: options.cache ?? true,
|
|
159
|
+
ttl: options.ttl,
|
|
160
|
+
verbose,
|
|
161
|
+
parameters,
|
|
162
|
+
compute: async () => {
|
|
163
|
+
const body = new URLSearchParams({ data: query });
|
|
164
|
+
const temporaryFile = await downloadOsmToTemporaryFile(endpoint, {
|
|
165
|
+
suffix: ".osm",
|
|
166
|
+
prefix: cacheId,
|
|
167
|
+
request: {
|
|
168
|
+
method: "POST",
|
|
169
|
+
headers: {
|
|
170
|
+
"accept-encoding": "gzip",
|
|
171
|
+
"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
|
|
172
|
+
"user-agent": "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
|
|
173
|
+
},
|
|
174
|
+
body,
|
|
175
|
+
},
|
|
176
|
+
retries: options.retries,
|
|
177
|
+
retryDelay: options.retryDelay,
|
|
178
|
+
timeout: options.timeout,
|
|
179
|
+
overpass: true,
|
|
180
|
+
verbose,
|
|
181
|
+
});
|
|
182
|
+
try {
|
|
183
|
+
try {
|
|
184
|
+
await assertSuccessfulOverpassResponse(temporaryFile);
|
|
185
|
+
}
|
|
186
|
+
catch (error) {
|
|
187
|
+
const reason = error instanceof Error
|
|
188
|
+
? error.message
|
|
189
|
+
: String(error);
|
|
190
|
+
throw new Error(`${reason}\nNot retrying because the Overpass response was well-formed and reported a query error.\nNo invalid cached data was returned.`, { cause: error });
|
|
191
|
+
}
|
|
192
|
+
await processOsmFile(table, temporaryFile, parameters, verbose);
|
|
193
|
+
}
|
|
194
|
+
finally {
|
|
195
|
+
rmSync(temporaryFile, { force: true });
|
|
196
|
+
verbose &&
|
|
197
|
+
console.log("Removed the temporary raw Overpass download.");
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
});
|
|
201
|
+
verbose && console.log(`loadOpenStreetMap(): completed in ${prettyDuration(started)}.`);
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
if (error instanceof SDAError)
|
|
205
|
+
throw error;
|
|
206
|
+
throw new SDAError({
|
|
207
|
+
method: "loadOpenStreetMap()",
|
|
208
|
+
parameters,
|
|
209
|
+
query,
|
|
210
|
+
cause: error,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
async function processOsmFile(table, file, parameters, verbose) {
|
|
215
|
+
verbose && console.log("Processing OSM data with Osmium...");
|
|
216
|
+
const started = Date.now();
|
|
217
|
+
try {
|
|
218
|
+
await loadOsmFile(table, file, {
|
|
219
|
+
method: "loadOpenStreetMap()",
|
|
220
|
+
parameters,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
catch (error) {
|
|
224
|
+
const cause = error instanceof SDAError ? error.cause : error;
|
|
225
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
226
|
+
throw new Error(`${reason}\nNot retrying because Osmium or a downstream DuckDB operation rejected the source data.\nNo invalid cached data was returned.`, { cause: error });
|
|
227
|
+
}
|
|
228
|
+
if (verbose) {
|
|
229
|
+
const data = await queryDB(table, `SELECT count(*)::DOUBLE AS count FROM ${quoteIdentifier(table.name)};`, mergeOptions(table, {
|
|
230
|
+
table: table.name,
|
|
231
|
+
method: "loadOpenStreetMap()",
|
|
232
|
+
parameters,
|
|
233
|
+
returnData: true,
|
|
234
|
+
}));
|
|
235
|
+
const count = Number(data?.[0]?.count ?? 0);
|
|
236
|
+
console.log(`Processed ${count.toLocaleString("en-US")} rows in ${prettyDuration(started)}.`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
async function assertSuccessfulOverpassResponse(file) {
|
|
240
|
+
const contents = await Deno.readTextFile(file);
|
|
241
|
+
const match = /<(remark|error)\b[^>]*>([\s\S]*?)(?:<\/\1\s*>|$)/i.exec(contents);
|
|
242
|
+
if (match === null)
|
|
243
|
+
return;
|
|
244
|
+
const message = match[2]
|
|
245
|
+
.replace(/<[^>]*>/g, " ")
|
|
246
|
+
.replace(/\s+/g, " ")
|
|
247
|
+
.trim()
|
|
248
|
+
.slice(0, 500);
|
|
249
|
+
throw new Error(message.length === 0
|
|
250
|
+
? "The Overpass endpoint returned an error response."
|
|
251
|
+
: `The Overpass endpoint returned an error response: ${message}`);
|
|
252
|
+
}
|
|
253
|
+
export function generateOverpassQuery(bbox, filters, timeout) {
|
|
254
|
+
const box = `${bbox.south},${bbox.west},${bbox.north},${bbox.east}`;
|
|
255
|
+
const selections = filters.map((filter) => `nwr${filter}(${box});`);
|
|
256
|
+
const selection = selections.length === 1
|
|
257
|
+
? selections[0]
|
|
258
|
+
: `(${selections.join("")});`;
|
|
259
|
+
return `[out:xml]${timeout === undefined ? "" : `[timeout:${timeout}]`};${selection}(._;>;);out body;`;
|
|
260
|
+
}
|
|
261
|
+
function normalizeBoundingBox(bbox) {
|
|
262
|
+
const normalized = structuredClone(bbox);
|
|
263
|
+
for (const [name, value] of Object.entries(normalized)) {
|
|
264
|
+
if (!Number.isFinite(value)) {
|
|
265
|
+
throw new Error(`loadOpenStreetMap() bbox.${name} must be a finite number.`);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
if (normalized.west < -180 || normalized.west > 180) {
|
|
269
|
+
throw new Error("loadOpenStreetMap() bbox.west must be between -180 and 180.");
|
|
270
|
+
}
|
|
271
|
+
if (normalized.east < -180 || normalized.east > 180) {
|
|
272
|
+
throw new Error("loadOpenStreetMap() bbox.east must be between -180 and 180.");
|
|
273
|
+
}
|
|
274
|
+
if (normalized.south < -90 || normalized.south > 90) {
|
|
275
|
+
throw new Error("loadOpenStreetMap() bbox.south must be between -90 and 90.");
|
|
276
|
+
}
|
|
277
|
+
if (normalized.north < -90 || normalized.north > 90) {
|
|
278
|
+
throw new Error("loadOpenStreetMap() bbox.north must be between -90 and 90.");
|
|
279
|
+
}
|
|
280
|
+
if (normalized.west >= normalized.east) {
|
|
281
|
+
throw new Error("loadOpenStreetMap() bbox.west must be less than bbox.east.");
|
|
282
|
+
}
|
|
283
|
+
if (normalized.south >= normalized.north) {
|
|
284
|
+
throw new Error("loadOpenStreetMap() bbox.south must be less than bbox.north.");
|
|
285
|
+
}
|
|
286
|
+
normalized.west = Object.is(normalized.west, -0) ? 0 : normalized.west;
|
|
287
|
+
normalized.south = Object.is(normalized.south, -0) ? 0 : normalized.south;
|
|
288
|
+
normalized.east = Object.is(normalized.east, -0) ? 0 : normalized.east;
|
|
289
|
+
normalized.north = Object.is(normalized.north, -0) ? 0 : normalized.north;
|
|
290
|
+
return normalized;
|
|
291
|
+
}
|
|
292
|
+
function normalizeFilters(filters) {
|
|
293
|
+
if (typeof filters === "string") {
|
|
294
|
+
const filter = filters.trim();
|
|
295
|
+
if (filter.length === 0) {
|
|
296
|
+
throw new Error("loadOpenStreetMap() filters must contain at least one non-empty filter fragment.");
|
|
297
|
+
}
|
|
298
|
+
return [filter];
|
|
299
|
+
}
|
|
300
|
+
if (!Array.isArray(filters)) {
|
|
301
|
+
throw new TypeError("loadOpenStreetMap() options.filters is required.");
|
|
302
|
+
}
|
|
303
|
+
const tuples = isFilterTuple(filters) ? [filters] : filters;
|
|
304
|
+
const normalized = tuples.map(([key, value]) => {
|
|
305
|
+
if (typeof key !== "string" || typeof value !== "string") {
|
|
306
|
+
throw new TypeError("loadOpenStreetMap() filter tuples must contain two strings.");
|
|
307
|
+
}
|
|
308
|
+
if (key.length === 0) {
|
|
309
|
+
throw new Error("loadOpenStreetMap() filter keys must not be empty.");
|
|
310
|
+
}
|
|
311
|
+
return `[${JSON.stringify(key)}=${JSON.stringify(value)}]`;
|
|
312
|
+
});
|
|
313
|
+
if (normalized.length === 0) {
|
|
314
|
+
throw new Error("loadOpenStreetMap() filters must contain at least one filter tuple.");
|
|
315
|
+
}
|
|
316
|
+
return normalized;
|
|
317
|
+
}
|
|
318
|
+
function isFilterTuple(filters) {
|
|
319
|
+
return filters.length === 2 && typeof filters[0] === "string" &&
|
|
320
|
+
typeof filters[1] === "string";
|
|
321
|
+
}
|
|
322
|
+
function normalizeEndpoint(endpoint) {
|
|
323
|
+
try {
|
|
324
|
+
return normalizeRemoteOsmUrl(endpoint);
|
|
325
|
+
}
|
|
326
|
+
catch {
|
|
327
|
+
throw new Error("loadOpenStreetMap() endpoint must be a valid HTTP or HTTPS URL.");
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
function assertOptions(options, overpass) {
|
|
331
|
+
const allowed = new Set([
|
|
332
|
+
"cache",
|
|
333
|
+
"ttl",
|
|
334
|
+
"retries",
|
|
335
|
+
"retryDelay",
|
|
336
|
+
"verbose",
|
|
337
|
+
...(overpass ? ["filters", "endpoint", "timeout"] : []),
|
|
338
|
+
]);
|
|
339
|
+
const unsupported = Object.keys(options).filter((key) => !allowed.has(key));
|
|
340
|
+
if (unsupported.length > 0) {
|
|
341
|
+
throw new Error(`loadOpenStreetMap() does not support ${unsupported.map((key) => `options.${key}`).join(", ")} for this source. Apply filtering and projection with normal table methods after loading.`);
|
|
342
|
+
}
|
|
343
|
+
if (options.cache !== undefined && typeof options.cache !== "boolean") {
|
|
344
|
+
throw new TypeError("loadOpenStreetMap() options.cache must be a boolean.");
|
|
345
|
+
}
|
|
346
|
+
if (options.verbose !== undefined && typeof options.verbose !== "boolean") {
|
|
347
|
+
throw new TypeError("loadOpenStreetMap() options.verbose must be a boolean.");
|
|
348
|
+
}
|
|
349
|
+
if (options.ttl !== undefined &&
|
|
350
|
+
(!Number.isFinite(options.ttl) || options.ttl < 0)) {
|
|
351
|
+
throw new Error("loadOpenStreetMap() options.ttl must be a finite, non-negative number of seconds.");
|
|
352
|
+
}
|
|
353
|
+
if (options.cache === false && options.ttl !== undefined) {
|
|
354
|
+
throw new Error("loadOpenStreetMap() options.cache cannot be false when options.ttl is set.");
|
|
355
|
+
}
|
|
356
|
+
if (options.retries !== undefined &&
|
|
357
|
+
(!Number.isInteger(options.retries) || options.retries < 0)) {
|
|
358
|
+
throw new Error("loadOpenStreetMap() options.retries must be a non-negative integer.");
|
|
359
|
+
}
|
|
360
|
+
if (options.retryDelay !== undefined &&
|
|
361
|
+
(!Number.isFinite(options.retryDelay) || options.retryDelay < 0)) {
|
|
362
|
+
throw new Error("loadOpenStreetMap() options.retryDelay must be a finite, non-negative number of seconds.");
|
|
363
|
+
}
|
|
364
|
+
if (overpass && options.timeout !== undefined &&
|
|
365
|
+
(!Number.isInteger(options.timeout) || options.timeout <= 0)) {
|
|
366
|
+
throw new Error("loadOpenStreetMap() options.timeout must be a positive integer in seconds.");
|
|
367
|
+
}
|
|
368
|
+
}
|
package/llm.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# The Simple Data Analysis (Core) Library
|
|
2
2
|
|
|
3
|
+
- Package: `@nshiab/simple-data-analysis-core`
|
|
4
|
+
- Version: `2.0.4`
|
|
5
|
+
|
|
3
6
|
To install the library with Deno, use:
|
|
4
7
|
|
|
5
8
|
```bash
|
|
@@ -948,8 +951,9 @@ loadStatCanData(pid: string, options?: { lang?: "en" | "fr"; cache?: boolean; tt
|
|
|
948
951
|
- **`options`**: Optional retrieval and cache settings.
|
|
949
952
|
- **`options.lang`**: The language of the table data. Defaults to `"en"`.
|
|
950
953
|
- **`options.cache`**: Whether to read and write the cache. Defaults to `true`.
|
|
951
|
-
- **`options.ttl`**: Cache
|
|
952
|
-
|
|
954
|
+
- **`options.ttl`**: Cache lifetime in seconds. Omit for no expiration, use `0`
|
|
955
|
+
to refresh the matching cache entry immediately, or provide a positive value
|
|
956
|
+
to refresh once the entry reaches that age.
|
|
953
957
|
|
|
954
958
|
##### Returns
|
|
955
959
|
|
|
@@ -977,9 +981,7 @@ await table
|
|
|
977
981
|
|
|
978
982
|
#### `loadGeoData`
|
|
979
983
|
|
|
980
|
-
Loads geospatial data from an external file or URL into the table.
|
|
981
|
-
`.osm` and `.osm.pbf` files are loaded through DuckDB's Osmium community
|
|
982
|
-
extension.
|
|
984
|
+
Loads geospatial data from an external file or URL into the table.
|
|
983
985
|
|
|
984
986
|
This method queues the operation; it runs when an async observer method (like
|
|
985
987
|
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
@@ -992,8 +994,8 @@ loadGeoData(file: string, options?: { toEPSG4326?: boolean; columns?: string[];
|
|
|
992
994
|
|
|
993
995
|
##### Parameters
|
|
994
996
|
|
|
995
|
-
- **`file`**: The
|
|
996
|
-
|
|
997
|
+
- **`file`**: The path or URL of the external file containing the geospatial
|
|
998
|
+
data.
|
|
997
999
|
- **`options`**: An optional object with configuration options:
|
|
998
1000
|
- **`options.toEPSG4326`**: If `true`, the method will attempt to reproject the
|
|
999
1001
|
data to EPSG:4326 (WGS84).
|
|
@@ -1004,8 +1006,8 @@ loadGeoData(file: string, options?: { toEPSG4326?: boolean; columns?: string[];
|
|
|
1004
1006
|
keyword, to filter source rows before materialization and reprojection. Uses
|
|
1005
1007
|
the same syntax as `filter()`, including JavaScript operators. Can reference
|
|
1006
1008
|
source columns excluded from `columns`. Geometry conditions use the source
|
|
1007
|
-
coordinate system
|
|
1008
|
-
|
|
1009
|
+
coordinate system. Defaults to no filtering; an empty string behaves the same
|
|
1010
|
+
as omitting this option.
|
|
1009
1011
|
|
|
1010
1012
|
##### Returns
|
|
1011
1013
|
|
|
@@ -1043,11 +1045,6 @@ await table.loadGeoData("./some-data/some-data.shp", { toEPSG4326: true })
|
|
|
1043
1045
|
await table.loadGeoData("./some-data.shp.zip", { toEPSG4326: true }).log();
|
|
1044
1046
|
```
|
|
1045
1047
|
|
|
1046
|
-
```ts
|
|
1047
|
-
// Load OpenStreetMap XML or PBF data
|
|
1048
|
-
await table.loadGeoData("./montreal.osm.pbf").log();
|
|
1049
|
-
```
|
|
1050
|
-
|
|
1051
1048
|
```ts
|
|
1052
1049
|
// Filter on a source property without retaining it in the table
|
|
1053
1050
|
await table
|
|
@@ -1058,22 +1055,29 @@ await table
|
|
|
1058
1055
|
.log();
|
|
1059
1056
|
```
|
|
1060
1057
|
|
|
1061
|
-
#### `
|
|
1058
|
+
#### `loadOpenStreetMap`
|
|
1062
1059
|
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1060
|
+
Loads OpenStreetMap data into the table from a local `.osm` or `.osm.pbf` file,
|
|
1061
|
+
a remote file URL, or an Overpass bounding-box query. Pass a path or URL string
|
|
1062
|
+
to load an existing file, or pass a bounding box with `filters` to download
|
|
1063
|
+
matching features. The method queues the load; it runs when an async observer
|
|
1064
|
+
method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1066
1065
|
|
|
1067
1066
|
DuckDB's
|
|
1068
1067
|
[Osmium community extension](https://duckdb.org/community_extensions/extensions/osmium)
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1068
|
+
materializes the complete result and reconstructs `geom` as EPSG:4326. By
|
|
1069
|
+
default, that canonical result is cached as GeoParquet in `.sda-cache/osm`.
|
|
1070
|
+
Distinct paths, local file fingerprints, normalized URLs, endpoints, queries,
|
|
1071
|
+
and bounding boxes retain independent entries. The cache has no expiration
|
|
1072
|
+
unless `ttl` is set and can be cleared by removing `.sda-cache/osm`. Immutable
|
|
1073
|
+
URLs can safely use the default indefinite cache; mutable URLs should set a TTL.
|
|
1074
|
+
|
|
1075
|
+
Bounding-box filters use the standard
|
|
1072
1076
|
[Overpass QL filter syntax](https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Filters).
|
|
1073
1077
|
Equality filters are passed as `[key, value]` tuples and serialized by the
|
|
1074
1078
|
method. A raw filter fragment string can be used for advanced Overpass filters.
|
|
1075
1079
|
|
|
1076
|
-
The default endpoint is a shared public service. Follow the
|
|
1080
|
+
The default Overpass endpoint is a shared public service. Follow the
|
|
1077
1081
|
[Overpass public-instance usage guidelines](https://dev.overpass-api.de/overpass-doc/en/preface/commons.html),
|
|
1078
1082
|
and configure another endpoint or run your own instance for high-volume usage.
|
|
1079
1083
|
|
|
@@ -1087,33 +1091,48 @@ share-alike requirements.
|
|
|
1087
1091
|
##### Signature
|
|
1088
1092
|
|
|
1089
1093
|
```typescript
|
|
1090
|
-
|
|
1094
|
+
loadOpenStreetMap(source: string | { west: number; south: number; east: number; north: number }, options?: { filters?: string | [string, string] | [string, string][]; endpoint?: string; timeout?: number; cache?: boolean; ttl?: number; retries?: number; retryDelay?: number; verbose?: boolean }): this;
|
|
1091
1095
|
```
|
|
1092
1096
|
|
|
1093
1097
|
##### Parameters
|
|
1094
1098
|
|
|
1095
|
-
- **`
|
|
1096
|
-
|
|
1099
|
+
- **`source`**: The local path or remote URL of an existing `.osm` or `.osm.pbf`
|
|
1100
|
+
file, or a bounding box to query through Overpass.
|
|
1101
|
+
- **`source.west`**: The western longitude, between -180 and 180 and less than
|
|
1097
1102
|
`east`.
|
|
1098
|
-
- **`
|
|
1103
|
+
- **`source.south`**: The southern latitude, between -90 and 90 and less than
|
|
1099
1104
|
`north`.
|
|
1100
|
-
- **`
|
|
1101
|
-
`west`.
|
|
1102
|
-
- **`
|
|
1105
|
+
- **`source.east`**: The eastern longitude, between -180 and 180 and greater
|
|
1106
|
+
than `west`.
|
|
1107
|
+
- **`source.north`**: The northern latitude, between -90 and 90 and greater than
|
|
1103
1108
|
`south`.
|
|
1104
|
-
- **`options`**:
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1109
|
+
- **`options`**: Loading options. When `source` is a bounding box,
|
|
1110
|
+
`options.filters` must be provided.
|
|
1111
|
+
- **`options.filters`**: For a bounding-box query, one `[key, value]` tuple or
|
|
1112
|
+
an array of tuples. Array entries are combined as a union. A raw Overpass QL
|
|
1113
|
+
filter fragment string is also accepted.
|
|
1108
1114
|
- **`options.endpoint`**: The Overpass interpreter endpoint. Defaults to
|
|
1109
1115
|
`https://overpass-api.de/api/interpreter`.
|
|
1110
1116
|
- **`options.timeout`**: A positive integer timeout in seconds, applied to both
|
|
1111
1117
|
the Overpass query and HTTP request. If omitted, the endpoint's default query
|
|
1112
1118
|
timeout applies and no HTTP request timeout is set.
|
|
1113
|
-
- **`options.cache`**:
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
cache
|
|
1119
|
+
- **`options.cache`**: Whether to read and write the processed GeoParquet cache.
|
|
1120
|
+
Defaults to `true`.
|
|
1121
|
+
- **`options.ttl`**: Cache lifetime in seconds. Omit for no expiration, use `0`
|
|
1122
|
+
to refresh the matching cache entry immediately, or provide a positive value
|
|
1123
|
+
to refresh once the processed entry reaches that age. Cannot be combined with
|
|
1124
|
+
`cache: false`.
|
|
1125
|
+
- **`options.retries`**: Additional network retries after the initial request.
|
|
1126
|
+
Defaults to `3`, for up to four attempts. Network-only; passing it for a local
|
|
1127
|
+
path throws an error.
|
|
1128
|
+
- **`options.retryDelay`**: Base retry delay in seconds. Defaults to `5`,
|
|
1129
|
+
producing deterministic delays of 5, 10, and 20 seconds. A longer valid
|
|
1130
|
+
`Retry-After` value takes precedence. Network-only; passing it for a local
|
|
1131
|
+
path throws an error.
|
|
1132
|
+
- **`options.verbose`**: Whether to log OpenStreetMap cache, network, Osmium,
|
|
1133
|
+
cleanup, and timing lifecycle details. Defaults to `false`.
|
|
1134
|
+
`SimpleDB({ cacheVerbose: true })` also enables these messages and is not
|
|
1135
|
+
overridden by `verbose: false`.
|
|
1117
1136
|
|
|
1118
1137
|
##### Returns
|
|
1119
1138
|
|
|
@@ -1122,9 +1141,23 @@ The table, so methods can be chained.
|
|
|
1122
1141
|
##### Examples
|
|
1123
1142
|
|
|
1124
1143
|
```ts
|
|
1125
|
-
// Load
|
|
1126
|
-
|
|
1127
|
-
|
|
1144
|
+
// Load an existing local OpenStreetMap PBF file
|
|
1145
|
+
await table.loadOpenStreetMap("./montreal.osm.pbf").log();
|
|
1146
|
+
```
|
|
1147
|
+
|
|
1148
|
+
```ts
|
|
1149
|
+
// Filter and project after the complete canonical source has loaded
|
|
1150
|
+
await table
|
|
1151
|
+
.loadOpenStreetMap("./montreal.osm.pbf", { verbose: true })
|
|
1152
|
+
.filter("tags['amenity'] = 'school'")
|
|
1153
|
+
.selectColumns(["id", "tags", "geom"])
|
|
1154
|
+
.log();
|
|
1155
|
+
```
|
|
1156
|
+
|
|
1157
|
+
```ts
|
|
1158
|
+
// Download features matching one equality filter within a bounding box
|
|
1159
|
+
await table
|
|
1160
|
+
.loadOpenStreetMap(
|
|
1128
1161
|
{ west: -73.587799, south: 45.445078, east: -73.552265, north: 45.471086 },
|
|
1129
1162
|
{ filters: ["amenity", "school"] },
|
|
1130
1163
|
)
|
|
@@ -1132,8 +1165,8 @@ const schools = await table
|
|
|
1132
1165
|
```
|
|
1133
1166
|
|
|
1134
1167
|
```ts
|
|
1135
|
-
//
|
|
1136
|
-
await table.
|
|
1168
|
+
// Download features matching either equality filter
|
|
1169
|
+
await table.loadOpenStreetMap(
|
|
1137
1170
|
{ west: -73.587799, south: 45.445078, east: -73.552265, north: 45.471086 },
|
|
1138
1171
|
{
|
|
1139
1172
|
filters: [["amenity", "school"], ["amenity", "college"]],
|
|
@@ -1143,7 +1176,7 @@ await table.loadOSM(
|
|
|
1143
1176
|
|
|
1144
1177
|
```ts
|
|
1145
1178
|
// Use a raw Overpass filter fragment for an advanced filter
|
|
1146
|
-
await table.
|
|
1179
|
+
await table.loadOpenStreetMap(
|
|
1147
1180
|
{ west: -73.587799, south: 45.445078, east: -73.552265, north: 45.471086 },
|
|
1148
1181
|
{ filters: `["amenity"~"school|college"]` },
|
|
1149
1182
|
).log();
|
|
@@ -7511,9 +7544,9 @@ async cache(compute: (table: this) => void | Promise<void>, options?: { inputs?:
|
|
|
7511
7544
|
cache. Functions and class constructors are compared by source. `SimpleTable`
|
|
7512
7545
|
dependencies read by `compute` are tracked automatically, and the table being
|
|
7513
7546
|
cached is already tracked, so neither needs to be included here.
|
|
7514
|
-
- **`options.ttl`**:
|
|
7515
|
-
|
|
7516
|
-
refresh the
|
|
7547
|
+
- **`options.ttl`**: Cache lifetime in seconds. Omit for no expiration, use `0`
|
|
7548
|
+
to refresh the matching cache entry immediately, or provide a positive value
|
|
7549
|
+
to refresh once the entry reaches that age. The cache is also invalidated when
|
|
7517
7550
|
the `compute` function, the table, or an input changes.
|
|
7518
7551
|
|
|
7519
7552
|
##### Returns
|
package/llms.txt
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
> Lightweight DuckDB-powered TypeScript library for tabular, SQL, CSV, Parquet, and geospatial data analysis on Deno, Node.js, and Bun.
|
|
4
4
|
|
|
5
|
+
- Package: `@nshiab/simple-data-analysis-core`
|
|
6
|
+
- Version: `2.0.4`
|
|
7
|
+
|
|
5
8
|
Install with Deno: `deno add jsr:@nshiab/simple-data-analysis-core`
|
|
6
9
|
|
|
7
10
|
Install with Node.js: `npm i @nshiab/simple-data-analysis-core`
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nshiab/simple-data-analysis-core",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.4",
|
|
4
4
|
"description": "Lightweight DuckDB-powered TypeScript library for tabular, SQL, CSV, Parquet, and geospatial data analysis on Deno, Node.js, and Bun.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"data-analysis",
|