@nshiab/simple-data-analysis-core 2.0.2 → 2.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/esm/class/SimpleTable.d.ts +50 -32
  2. package/esm/class/SimpleTable.d.ts.map +1 -1
  3. package/esm/class/SimpleTable.js +46 -32
  4. package/esm/helpers/loadOsmFile.d.ts +1 -3
  5. package/esm/helpers/loadOsmFile.d.ts.map +1 -1
  6. package/esm/helpers/loadOsmFile.js +3 -11
  7. package/esm/helpers/osmCache.d.ts +10 -13
  8. package/esm/helpers/osmCache.d.ts.map +1 -1
  9. package/esm/helpers/osmCache.js +177 -32
  10. package/esm/helpers/osmFiles.d.ts +9 -0
  11. package/esm/helpers/osmFiles.d.ts.map +1 -1
  12. package/esm/helpers/osmFiles.js +234 -23
  13. package/esm/methods/loadGeoData.d.ts.map +1 -1
  14. package/esm/methods/loadGeoData.js +4 -39
  15. package/esm/methods/loadOpenStreetMap.d.ts +24 -0
  16. package/esm/methods/loadOpenStreetMap.d.ts.map +1 -0
  17. package/esm/methods/loadOpenStreetMap.js +368 -0
  18. package/llm.md +79 -46
  19. package/llms.txt +3 -0
  20. package/package.json +1 -1
  21. package/script/class/SimpleTable.d.ts +50 -32
  22. package/script/class/SimpleTable.d.ts.map +1 -1
  23. package/script/class/SimpleTable.js +46 -32
  24. package/script/helpers/loadOsmFile.d.ts +1 -3
  25. package/script/helpers/loadOsmFile.d.ts.map +1 -1
  26. package/script/helpers/loadOsmFile.js +3 -11
  27. package/script/helpers/osmCache.d.ts +10 -13
  28. package/script/helpers/osmCache.d.ts.map +1 -1
  29. package/script/helpers/osmCache.js +177 -65
  30. package/script/helpers/osmFiles.d.ts +9 -0
  31. package/script/helpers/osmFiles.d.ts.map +1 -1
  32. package/script/helpers/osmFiles.js +237 -23
  33. package/script/methods/loadGeoData.d.ts.map +1 -1
  34. package/script/methods/loadGeoData.js +3 -38
  35. package/script/methods/loadOpenStreetMap.d.ts +24 -0
  36. package/script/methods/loadOpenStreetMap.d.ts.map +1 -0
  37. package/script/methods/loadOpenStreetMap.js +375 -0
  38. package/esm/methods/loadOSM.d.ts +0 -17
  39. package/esm/methods/loadOSM.d.ts.map +0 -1
  40. package/esm/methods/loadOSM.js +0 -194
  41. package/script/methods/loadOSM.d.ts +0 -17
  42. package/script/methods/loadOSM.d.ts.map +0 -1
  43. package/script/methods/loadOSM.js +0 -201
@@ -0,0 +1,368 @@
1
+ import { realpathSync, rmSync, statSync } from "node:fs";
2
+ import { resolve } from "node:path";
3
+ import SDAError from "../class/SDAError.js";
4
+ import loadOsmFile from "../helpers/loadOsmFile.js";
5
+ import { createOsmCacheId, useProcessedOsmCache } from "../helpers/osmCache.js";
6
+ import { downloadOsmToTemporaryFile, formatBytes, getOsmFileSuffix, isRemoteOsmUrl, normalizeRemoteOsmUrl, sanitizeOsmUrl, } from "../helpers/osmFiles.js";
7
+ import mergeOptions from "../helpers/mergeOptions.js";
8
+ import prettyDuration from "../helpers/prettyDuration.js";
9
+ import queryDB from "../helpers/queryDB.js";
10
+ import quoteIdentifier from "../helpers/quoteIdentifier.js";
11
+ import queueOp from "../helpers/queueOp.js";
12
+ const DEFAULT_OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter";
13
+ const DEFAULT_REMOTE_FILE_TIMEOUT = 30 * 60;
14
+ const QUERY_GENERATOR_VERSION = "overpass-ql-v1";
15
+ export default function loadOpenStreetMap(table, source, options = {}) {
16
+ if (typeof source === "string") {
17
+ queueOpenStreetMapFileLoad(table, source, options);
18
+ return;
19
+ }
20
+ queueOverpassLoad(table, source, options);
21
+ }
22
+ function queueOpenStreetMapFileLoad(table, file, options) {
23
+ const suffix = getOsmFileSuffix(file);
24
+ if (suffix === null) {
25
+ throw new Error("loadOpenStreetMap() file paths and URLs must end with .osm or .osm.pbf.");
26
+ }
27
+ options = structuredClone(options);
28
+ assertOptions(options, false);
29
+ let remoteUrl;
30
+ if (isRemoteOsmUrl(file)) {
31
+ remoteUrl = normalizeRemoteOsmUrl(file);
32
+ }
33
+ else {
34
+ try {
35
+ const url = new URL(file);
36
+ if (url.protocol !== "http:" && url.protocol !== "https:") {
37
+ throw new Error("loadOpenStreetMap() URL sources must use HTTP or HTTPS.");
38
+ }
39
+ }
40
+ catch (error) {
41
+ if (error instanceof Error && error.message.includes("must use HTTP")) {
42
+ throw error;
43
+ }
44
+ }
45
+ if (options.retries !== undefined || options.retryDelay !== undefined) {
46
+ throw new Error("loadOpenStreetMap() options.retries and options.retryDelay are network-only and cannot be used with a local path.");
47
+ }
48
+ }
49
+ const parameters = { file, options };
50
+ queueOp(table, {
51
+ kind: "barrier",
52
+ method: "loadOpenStreetMap()",
53
+ parameters,
54
+ execute: () => executeOpenStreetMapFileLoad(table, remoteUrl ?? file, suffix, options, parameters),
55
+ });
56
+ }
57
+ async function executeOpenStreetMapFileLoad(table, file, suffix, options, parameters) {
58
+ const started = Date.now();
59
+ const verbose = options.verbose === true || table.sdb.cacheVerbose;
60
+ const remote = isRemoteOsmUrl(file);
61
+ const displayedSource = remote ? sanitizeOsmUrl(file) : resolve(file);
62
+ verbose && console.log(`loadOpenStreetMap(): loading ${displayedSource}`);
63
+ try {
64
+ if (remote) {
65
+ const cacheId = createOsmCacheId({ kind: "remote", url: file });
66
+ await useProcessedOsmCache(table, {
67
+ id: cacheId,
68
+ enabled: options.cache ?? true,
69
+ ttl: options.ttl,
70
+ verbose,
71
+ parameters,
72
+ compute: async () => {
73
+ const temporaryFile = await downloadOsmToTemporaryFile(file, {
74
+ suffix,
75
+ prefix: cacheId,
76
+ request: { headers: { "accept-encoding": "gzip" } },
77
+ retries: options.retries,
78
+ retryDelay: options.retryDelay,
79
+ timeout: DEFAULT_REMOTE_FILE_TIMEOUT,
80
+ verbose,
81
+ });
82
+ try {
83
+ await processOsmFile(table, temporaryFile, parameters, verbose);
84
+ }
85
+ finally {
86
+ rmSync(temporaryFile, { force: true });
87
+ verbose && console.log("Removed the temporary raw OSM download.");
88
+ }
89
+ },
90
+ });
91
+ }
92
+ else {
93
+ verbose &&
94
+ console.log("Resolving and fingerprinting the local OSM file...");
95
+ const fingerprintStarted = Date.now();
96
+ const resolvedPath = realpathSync(resolve(file));
97
+ const stat = statSync(resolvedPath, { bigint: true });
98
+ if (!stat.isFile()) {
99
+ throw new Error(`The OpenStreetMap source is not a file: ${resolvedPath}`);
100
+ }
101
+ verbose && console.log(`Resolved ${formatBytes(Number(stat.size))} and fingerprinted it in ${prettyDuration(fingerprintStarted)}.`);
102
+ const cacheId = createOsmCacheId({
103
+ kind: "local",
104
+ path: resolvedPath,
105
+ size: stat.size.toString(),
106
+ mtimeNs: stat.mtimeNs.toString(),
107
+ });
108
+ await useProcessedOsmCache(table, {
109
+ id: cacheId,
110
+ enabled: options.cache ?? true,
111
+ ttl: options.ttl,
112
+ verbose,
113
+ parameters,
114
+ compute: () => processOsmFile(table, resolvedPath, parameters, verbose),
115
+ });
116
+ }
117
+ verbose && console.log(`loadOpenStreetMap(): completed in ${prettyDuration(started)}.`);
118
+ }
119
+ catch (error) {
120
+ if (error instanceof SDAError)
121
+ throw error;
122
+ throw new SDAError({
123
+ method: "loadOpenStreetMap()",
124
+ parameters,
125
+ query: "",
126
+ cause: error,
127
+ });
128
+ }
129
+ }
130
+ function queueOverpassLoad(table, bbox, options) {
131
+ bbox = normalizeBoundingBox(bbox);
132
+ options = structuredClone(options);
133
+ assertOptions(options, true);
134
+ const endpoint = normalizeEndpoint(options.endpoint ?? DEFAULT_OVERPASS_ENDPOINT);
135
+ const filters = normalizeFilters(options.filters);
136
+ const query = generateOverpassQuery(bbox, filters, options.timeout);
137
+ const parameters = { bbox, options };
138
+ queueOp(table, {
139
+ kind: "barrier",
140
+ method: "loadOpenStreetMap()",
141
+ parameters,
142
+ execute: () => executeOverpassLoad(table, options, endpoint, query, parameters),
143
+ });
144
+ }
145
+ async function executeOverpassLoad(table, options, endpoint, query, parameters) {
146
+ const started = Date.now();
147
+ const verbose = options.verbose === true || table.sdb.cacheVerbose;
148
+ verbose && console.log(`loadOpenStreetMap(): querying ${sanitizeOsmUrl(endpoint)}`);
149
+ try {
150
+ const cacheId = createOsmCacheId({
151
+ kind: "overpass",
152
+ endpoint,
153
+ query,
154
+ queryGeneratorVersion: QUERY_GENERATOR_VERSION,
155
+ });
156
+ await useProcessedOsmCache(table, {
157
+ id: cacheId,
158
+ enabled: options.cache ?? true,
159
+ ttl: options.ttl,
160
+ verbose,
161
+ parameters,
162
+ compute: async () => {
163
+ const body = new URLSearchParams({ data: query });
164
+ const temporaryFile = await downloadOsmToTemporaryFile(endpoint, {
165
+ suffix: ".osm",
166
+ prefix: cacheId,
167
+ request: {
168
+ method: "POST",
169
+ headers: {
170
+ "accept-encoding": "gzip",
171
+ "content-type": "application/x-www-form-urlencoded;charset=UTF-8",
172
+ "user-agent": "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
173
+ },
174
+ body,
175
+ },
176
+ retries: options.retries,
177
+ retryDelay: options.retryDelay,
178
+ timeout: options.timeout,
179
+ overpass: true,
180
+ verbose,
181
+ });
182
+ try {
183
+ try {
184
+ await assertSuccessfulOverpassResponse(temporaryFile);
185
+ }
186
+ catch (error) {
187
+ const reason = error instanceof Error
188
+ ? error.message
189
+ : String(error);
190
+ throw new Error(`${reason}\nNot retrying because the Overpass response was well-formed and reported a query error.\nNo invalid cached data was returned.`, { cause: error });
191
+ }
192
+ await processOsmFile(table, temporaryFile, parameters, verbose);
193
+ }
194
+ finally {
195
+ rmSync(temporaryFile, { force: true });
196
+ verbose &&
197
+ console.log("Removed the temporary raw Overpass download.");
198
+ }
199
+ },
200
+ });
201
+ verbose && console.log(`loadOpenStreetMap(): completed in ${prettyDuration(started)}.`);
202
+ }
203
+ catch (error) {
204
+ if (error instanceof SDAError)
205
+ throw error;
206
+ throw new SDAError({
207
+ method: "loadOpenStreetMap()",
208
+ parameters,
209
+ query,
210
+ cause: error,
211
+ });
212
+ }
213
+ }
214
+ async function processOsmFile(table, file, parameters, verbose) {
215
+ verbose && console.log("Processing OSM data with Osmium...");
216
+ const started = Date.now();
217
+ try {
218
+ await loadOsmFile(table, file, {
219
+ method: "loadOpenStreetMap()",
220
+ parameters,
221
+ });
222
+ }
223
+ catch (error) {
224
+ const cause = error instanceof SDAError ? error.cause : error;
225
+ const reason = cause instanceof Error ? cause.message : String(cause);
226
+ throw new Error(`${reason}\nNot retrying because Osmium or a downstream DuckDB operation rejected the source data.\nNo invalid cached data was returned.`, { cause: error });
227
+ }
228
+ if (verbose) {
229
+ const data = await queryDB(table, `SELECT count(*)::DOUBLE AS count FROM ${quoteIdentifier(table.name)};`, mergeOptions(table, {
230
+ table: table.name,
231
+ method: "loadOpenStreetMap()",
232
+ parameters,
233
+ returnData: true,
234
+ }));
235
+ const count = Number(data?.[0]?.count ?? 0);
236
+ console.log(`Processed ${count.toLocaleString("en-US")} rows in ${prettyDuration(started)}.`);
237
+ }
238
+ }
239
+ async function assertSuccessfulOverpassResponse(file) {
240
+ const contents = await Deno.readTextFile(file);
241
+ const match = /<(remark|error)\b[^>]*>([\s\S]*?)(?:<\/\1\s*>|$)/i.exec(contents);
242
+ if (match === null)
243
+ return;
244
+ const message = match[2]
245
+ .replace(/<[^>]*>/g, " ")
246
+ .replace(/\s+/g, " ")
247
+ .trim()
248
+ .slice(0, 500);
249
+ throw new Error(message.length === 0
250
+ ? "The Overpass endpoint returned an error response."
251
+ : `The Overpass endpoint returned an error response: ${message}`);
252
+ }
253
+ export function generateOverpassQuery(bbox, filters, timeout) {
254
+ const box = `${bbox.south},${bbox.west},${bbox.north},${bbox.east}`;
255
+ const selections = filters.map((filter) => `nwr${filter}(${box});`);
256
+ const selection = selections.length === 1
257
+ ? selections[0]
258
+ : `(${selections.join("")});`;
259
+ return `[out:xml]${timeout === undefined ? "" : `[timeout:${timeout}]`};${selection}(._;>;);out body;`;
260
+ }
261
+ function normalizeBoundingBox(bbox) {
262
+ const normalized = structuredClone(bbox);
263
+ for (const [name, value] of Object.entries(normalized)) {
264
+ if (!Number.isFinite(value)) {
265
+ throw new Error(`loadOpenStreetMap() bbox.${name} must be a finite number.`);
266
+ }
267
+ }
268
+ if (normalized.west < -180 || normalized.west > 180) {
269
+ throw new Error("loadOpenStreetMap() bbox.west must be between -180 and 180.");
270
+ }
271
+ if (normalized.east < -180 || normalized.east > 180) {
272
+ throw new Error("loadOpenStreetMap() bbox.east must be between -180 and 180.");
273
+ }
274
+ if (normalized.south < -90 || normalized.south > 90) {
275
+ throw new Error("loadOpenStreetMap() bbox.south must be between -90 and 90.");
276
+ }
277
+ if (normalized.north < -90 || normalized.north > 90) {
278
+ throw new Error("loadOpenStreetMap() bbox.north must be between -90 and 90.");
279
+ }
280
+ if (normalized.west >= normalized.east) {
281
+ throw new Error("loadOpenStreetMap() bbox.west must be less than bbox.east.");
282
+ }
283
+ if (normalized.south >= normalized.north) {
284
+ throw new Error("loadOpenStreetMap() bbox.south must be less than bbox.north.");
285
+ }
286
+ normalized.west = Object.is(normalized.west, -0) ? 0 : normalized.west;
287
+ normalized.south = Object.is(normalized.south, -0) ? 0 : normalized.south;
288
+ normalized.east = Object.is(normalized.east, -0) ? 0 : normalized.east;
289
+ normalized.north = Object.is(normalized.north, -0) ? 0 : normalized.north;
290
+ return normalized;
291
+ }
292
+ function normalizeFilters(filters) {
293
+ if (typeof filters === "string") {
294
+ const filter = filters.trim();
295
+ if (filter.length === 0) {
296
+ throw new Error("loadOpenStreetMap() filters must contain at least one non-empty filter fragment.");
297
+ }
298
+ return [filter];
299
+ }
300
+ if (!Array.isArray(filters)) {
301
+ throw new TypeError("loadOpenStreetMap() options.filters is required.");
302
+ }
303
+ const tuples = isFilterTuple(filters) ? [filters] : filters;
304
+ const normalized = tuples.map(([key, value]) => {
305
+ if (typeof key !== "string" || typeof value !== "string") {
306
+ throw new TypeError("loadOpenStreetMap() filter tuples must contain two strings.");
307
+ }
308
+ if (key.length === 0) {
309
+ throw new Error("loadOpenStreetMap() filter keys must not be empty.");
310
+ }
311
+ return `[${JSON.stringify(key)}=${JSON.stringify(value)}]`;
312
+ });
313
+ if (normalized.length === 0) {
314
+ throw new Error("loadOpenStreetMap() filters must contain at least one filter tuple.");
315
+ }
316
+ return normalized;
317
+ }
318
+ function isFilterTuple(filters) {
319
+ return filters.length === 2 && typeof filters[0] === "string" &&
320
+ typeof filters[1] === "string";
321
+ }
322
+ function normalizeEndpoint(endpoint) {
323
+ try {
324
+ return normalizeRemoteOsmUrl(endpoint);
325
+ }
326
+ catch {
327
+ throw new Error("loadOpenStreetMap() endpoint must be a valid HTTP or HTTPS URL.");
328
+ }
329
+ }
330
+ function assertOptions(options, overpass) {
331
+ const allowed = new Set([
332
+ "cache",
333
+ "ttl",
334
+ "retries",
335
+ "retryDelay",
336
+ "verbose",
337
+ ...(overpass ? ["filters", "endpoint", "timeout"] : []),
338
+ ]);
339
+ const unsupported = Object.keys(options).filter((key) => !allowed.has(key));
340
+ if (unsupported.length > 0) {
341
+ throw new Error(`loadOpenStreetMap() does not support ${unsupported.map((key) => `options.${key}`).join(", ")} for this source. Apply filtering and projection with normal table methods after loading.`);
342
+ }
343
+ if (options.cache !== undefined && typeof options.cache !== "boolean") {
344
+ throw new TypeError("loadOpenStreetMap() options.cache must be a boolean.");
345
+ }
346
+ if (options.verbose !== undefined && typeof options.verbose !== "boolean") {
347
+ throw new TypeError("loadOpenStreetMap() options.verbose must be a boolean.");
348
+ }
349
+ if (options.ttl !== undefined &&
350
+ (!Number.isFinite(options.ttl) || options.ttl < 0)) {
351
+ throw new Error("loadOpenStreetMap() options.ttl must be a finite, non-negative number of seconds.");
352
+ }
353
+ if (options.cache === false && options.ttl !== undefined) {
354
+ throw new Error("loadOpenStreetMap() options.cache cannot be false when options.ttl is set.");
355
+ }
356
+ if (options.retries !== undefined &&
357
+ (!Number.isInteger(options.retries) || options.retries < 0)) {
358
+ throw new Error("loadOpenStreetMap() options.retries must be a non-negative integer.");
359
+ }
360
+ if (options.retryDelay !== undefined &&
361
+ (!Number.isFinite(options.retryDelay) || options.retryDelay < 0)) {
362
+ throw new Error("loadOpenStreetMap() options.retryDelay must be a finite, non-negative number of seconds.");
363
+ }
364
+ if (overpass && options.timeout !== undefined &&
365
+ (!Number.isInteger(options.timeout) || options.timeout <= 0)) {
366
+ throw new Error("loadOpenStreetMap() options.timeout must be a positive integer in seconds.");
367
+ }
368
+ }
package/llm.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # The Simple Data Analysis (Core) Library
2
2
 
3
+ - Package: `@nshiab/simple-data-analysis-core`
4
+ - Version: `2.0.4`
5
+
3
6
  To install the library with Deno, use:
4
7
 
5
8
  ```bash
@@ -948,8 +951,9 @@ loadStatCanData(pid: string, options?: { lang?: "en" | "fr"; cache?: boolean; tt
948
951
  - **`options`**: Optional retrieval and cache settings.
949
952
  - **`options.lang`**: The language of the table data. Defaults to `"en"`.
950
953
  - **`options.cache`**: Whether to read and write the cache. Defaults to `true`.
951
- - **`options.ttl`**: Cache time to live in seconds. By default, cached data does
952
- not expire. Use `0` to refresh and replace the cache entry.
954
+ - **`options.ttl`**: Cache lifetime in seconds. Omit for no expiration, use `0`
955
+ to refresh the matching cache entry immediately, or provide a positive value
956
+ to refresh once the entry reaches that age.
953
957
 
954
958
  ##### Returns
955
959
 
@@ -977,9 +981,7 @@ await table
977
981
 
978
982
  #### `loadGeoData`
979
983
 
980
- Loads geospatial data from an external file or URL into the table. OpenStreetMap
981
- `.osm` and `.osm.pbf` files are loaded through DuckDB's Osmium community
982
- extension.
984
+ Loads geospatial data from an external file or URL into the table.
983
985
 
984
986
  This method queues the operation; it runs when an async observer method (like
985
987
  `getData()` or `log()`) is awaited, or when `run()` is called.
@@ -992,8 +994,8 @@ loadGeoData(file: string, options?: { toEPSG4326?: boolean; columns?: string[];
992
994
 
993
995
  ##### Parameters
994
996
 
995
- - **`file`**: The URL or absolute path to the external file containing the
996
- geospatial data.
997
+ - **`file`**: The path or URL of the external file containing the geospatial
998
+ data.
997
999
  - **`options`**: An optional object with configuration options:
998
1000
  - **`options.toEPSG4326`**: If `true`, the method will attempt to reproject the
999
1001
  data to EPSG:4326 (WGS84).
@@ -1004,8 +1006,8 @@ loadGeoData(file: string, options?: { toEPSG4326?: boolean; columns?: string[];
1004
1006
  keyword, to filter source rows before materialization and reprojection. Uses
1005
1007
  the same syntax as `filter()`, including JavaScript operators. Can reference
1006
1008
  source columns excluded from `columns`. Geometry conditions use the source
1007
- coordinate system; OSM geometry is available as `geom` in EPSG:4326. Defaults
1008
- to no filtering; an empty string behaves the same as omitting this option.
1009
+ coordinate system. Defaults to no filtering; an empty string behaves the same
1010
+ as omitting this option.
1009
1011
 
1010
1012
  ##### Returns
1011
1013
 
@@ -1043,11 +1045,6 @@ await table.loadGeoData("./some-data/some-data.shp", { toEPSG4326: true })
1043
1045
  await table.loadGeoData("./some-data.shp.zip", { toEPSG4326: true }).log();
1044
1046
  ```
1045
1047
 
1046
- ```ts
1047
- // Load OpenStreetMap XML or PBF data
1048
- await table.loadGeoData("./montreal.osm.pbf").log();
1049
- ```
1050
-
1051
1048
  ```ts
1052
1049
  // Filter on a source property without retaining it in the table
1053
1050
  await table
@@ -1058,22 +1055,29 @@ await table
1058
1055
  .log();
1059
1056
  ```
1060
1057
 
1061
- #### `loadOSM`
1058
+ #### `loadOpenStreetMap`
1062
1059
 
1063
- Downloads OpenStreetMap data and loads it as a geospatial table. The method
1064
- queues the download and load; they run when an async observer method (like
1065
- `getData()` or `log()`) is awaited, or when `run()` is called.
1060
+ Loads OpenStreetMap data into the table from a local `.osm` or `.osm.pbf` file,
1061
+ a remote file URL, or an Overpass bounding-box query. Pass a path or URL string
1062
+ to load an existing file, or pass a bounding box with `filters` to download
1063
+ matching features. The method queues the load; it runs when an async observer
1064
+ method (like `getData()` or `log()`) is awaited, or when `run()` is called.
1066
1065
 
1067
1066
  DuckDB's
1068
1067
  [Osmium community extension](https://duckdb.org/community_extensions/extensions/osmium)
1069
- reconstructs the geometries, which are stored with the EPSG:4326 projection.
1070
-
1071
- Filters use the standard
1068
+ materializes the complete result and reconstructs `geom` as EPSG:4326. By
1069
+ default, that canonical result is cached as GeoParquet in `.sda-cache/osm`.
1070
+ Distinct paths, local file fingerprints, normalized URLs, endpoints, queries,
1071
+ and bounding boxes retain independent entries. The cache has no expiration
1072
+ unless `ttl` is set and can be cleared by removing `.sda-cache/osm`. Immutable
1073
+ URLs can safely use the default indefinite cache; mutable URLs should set a TTL.
1074
+
1075
+ Bounding-box filters use the standard
1072
1076
  [Overpass QL filter syntax](https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Filters).
1073
1077
  Equality filters are passed as `[key, value]` tuples and serialized by the
1074
1078
  method. A raw filter fragment string can be used for advanced Overpass filters.
1075
1079
 
1076
- The default endpoint is a shared public service. Follow the
1080
+ The default Overpass endpoint is a shared public service. Follow the
1077
1081
  [Overpass public-instance usage guidelines](https://dev.overpass-api.de/overpass-doc/en/preface/commons.html),
1078
1082
  and configure another endpoint or run your own instance for high-volume usage.
1079
1083
 
@@ -1087,33 +1091,48 @@ share-alike requirements.
1087
1091
  ##### Signature
1088
1092
 
1089
1093
  ```typescript
1090
- loadOSM(bbox: { west: number; south: number; east: number; north: number }, options: { filters: string | [string, string] | [string, string][]; endpoint?: string; timeout?: number; cache?: boolean }): this;
1094
+ loadOpenStreetMap(source: string | { west: number; south: number; east: number; north: number }, options?: { filters?: string | [string, string] | [string, string][]; endpoint?: string; timeout?: number; cache?: boolean; ttl?: number; retries?: number; retryDelay?: number; verbose?: boolean }): this;
1091
1095
  ```
1092
1096
 
1093
1097
  ##### Parameters
1094
1098
 
1095
- - **`bbox`**: The bounding box to query.
1096
- - **`bbox.west`**: The western longitude, between -180 and 180 and less than
1099
+ - **`source`**: The local path or remote URL of an existing `.osm` or `.osm.pbf`
1100
+ file, or a bounding box to query through Overpass.
1101
+ - **`source.west`**: The western longitude, between -180 and 180 and less than
1097
1102
  `east`.
1098
- - **`bbox.south`**: The southern latitude, between -90 and 90 and less than
1103
+ - **`source.south`**: The southern latitude, between -90 and 90 and less than
1099
1104
  `north`.
1100
- - **`bbox.east`**: The eastern longitude, between -180 and 180 and greater than
1101
- `west`.
1102
- - **`bbox.north`**: The northern latitude, between -90 and 90 and greater than
1105
+ - **`source.east`**: The eastern longitude, between -180 and 180 and greater
1106
+ than `west`.
1107
+ - **`source.north`**: The northern latitude, between -90 and 90 and greater than
1103
1108
  `south`.
1104
- - **`options`**: Overpass request options.
1105
- - **`options.filters`**: One `[key, value]` tuple or an array of tuples. Array
1106
- entries are combined as a union. A raw Overpass QL filter fragment string is
1107
- also accepted.
1109
+ - **`options`**: Loading options. When `source` is a bounding box,
1110
+ `options.filters` must be provided.
1111
+ - **`options.filters`**: For a bounding-box query, one `[key, value]` tuple or
1112
+ an array of tuples. Array entries are combined as a union. A raw Overpass QL
1113
+ filter fragment string is also accepted.
1108
1114
  - **`options.endpoint`**: The Overpass interpreter endpoint. Defaults to
1109
1115
  `https://overpass-api.de/api/interpreter`.
1110
1116
  - **`options.timeout`**: A positive integer timeout in seconds, applied to both
1111
1117
  the Overpass query and HTTP request. If omitted, the endpoint's default query
1112
1118
  timeout applies and no HTTP request timeout is set.
1113
- - **`options.cache`**: If `true`, reads and writes the processed GeoParquet
1114
- cache in `.sda-cache/osm`, which remains available until that directory is
1115
- removed. If `false`, always requests fresh data and does not read or write the
1116
- cache. Defaults to `true`.
1119
+ - **`options.cache`**: Whether to read and write the processed GeoParquet cache.
1120
+ Defaults to `true`.
1121
+ - **`options.ttl`**: Cache lifetime in seconds. Omit for no expiration, use `0`
1122
+ to refresh the matching cache entry immediately, or provide a positive value
1123
+ to refresh once the processed entry reaches that age. Cannot be combined with
1124
+ `cache: false`.
1125
+ - **`options.retries`**: Additional network retries after the initial request.
1126
+ Defaults to `3`, for up to four attempts. Network-only; passing it for a local
1127
+ path throws an error.
1128
+ - **`options.retryDelay`**: Base retry delay in seconds. Defaults to `5`,
1129
+ producing deterministic delays of 5, 10, and 20 seconds. A longer valid
1130
+ `Retry-After` value takes precedence. Network-only; passing it for a local
1131
+ path throws an error.
1132
+ - **`options.verbose`**: Whether to log OpenStreetMap cache, network, Osmium,
1133
+ cleanup, and timing lifecycle details. Defaults to `false`.
1134
+ `SimpleDB({ cacheVerbose: true })` also enables these messages and is not
1135
+ overridden by `verbose: false`.
1117
1136
 
1118
1137
  ##### Returns
1119
1138
 
@@ -1122,9 +1141,23 @@ The table, so methods can be chained.
1122
1141
  ##### Examples
1123
1142
 
1124
1143
  ```ts
1125
- // Load features matching one equality filter
1126
- const schools = await table
1127
- .loadOSM(
1144
+ // Load an existing local OpenStreetMap PBF file
1145
+ await table.loadOpenStreetMap("./montreal.osm.pbf").log();
1146
+ ```
1147
+
1148
+ ```ts
1149
+ // Filter and project after the complete canonical source has loaded
1150
+ await table
1151
+ .loadOpenStreetMap("./montreal.osm.pbf", { verbose: true })
1152
+ .filter("tags['amenity'] = 'school'")
1153
+ .selectColumns(["id", "tags", "geom"])
1154
+ .log();
1155
+ ```
1156
+
1157
+ ```ts
1158
+ // Download features matching one equality filter within a bounding box
1159
+ await table
1160
+ .loadOpenStreetMap(
1128
1161
  { west: -73.587799, south: 45.445078, east: -73.552265, north: 45.471086 },
1129
1162
  { filters: ["amenity", "school"] },
1130
1163
  )
@@ -1132,8 +1165,8 @@ const schools = await table
1132
1165
  ```
1133
1166
 
1134
1167
  ```ts
1135
- // Load features matching either equality filter
1136
- await table.loadOSM(
1168
+ // Download features matching either equality filter
1169
+ await table.loadOpenStreetMap(
1137
1170
  { west: -73.587799, south: 45.445078, east: -73.552265, north: 45.471086 },
1138
1171
  {
1139
1172
  filters: [["amenity", "school"], ["amenity", "college"]],
@@ -1143,7 +1176,7 @@ await table.loadOSM(
1143
1176
 
1144
1177
  ```ts
1145
1178
  // Use a raw Overpass filter fragment for an advanced filter
1146
- await table.loadOSM(
1179
+ await table.loadOpenStreetMap(
1147
1180
  { west: -73.587799, south: 45.445078, east: -73.552265, north: 45.471086 },
1148
1181
  { filters: `["amenity"~"school|college"]` },
1149
1182
  ).log();
@@ -7511,9 +7544,9 @@ async cache(compute: (table: this) => void | Promise<void>, options?: { inputs?:
7511
7544
  cache. Functions and class constructors are compared by source. `SimpleTable`
7512
7545
  dependencies read by `compute` are tracked automatically, and the table being
7513
7546
  cached is already tracked, so neither needs to be included here.
7514
- - **`options.ttl`**: Time to live (in seconds). If the data in the cache is
7515
- older than this duration, the `compute` function will be executed again to
7516
- refresh the cache. By default, there is no TTL; the cache is invalidated when
7547
+ - **`options.ttl`**: Cache lifetime in seconds. Omit for no expiration, use `0`
7548
+ to refresh the matching cache entry immediately, or provide a positive value
7549
+ to refresh once the entry reaches that age. The cache is also invalidated when
7517
7550
  the `compute` function, the table, or an input changes.
7518
7551
 
7519
7552
  ##### Returns
package/llms.txt CHANGED
@@ -2,6 +2,9 @@
2
2
 
3
3
  > Lightweight DuckDB-powered TypeScript library for tabular, SQL, CSV, Parquet, and geospatial data analysis on Deno, Node.js, and Bun.
4
4
 
5
+ - Package: `@nshiab/simple-data-analysis-core`
6
+ - Version: `2.0.4`
7
+
5
8
  Install with Deno: `deno add jsr:@nshiab/simple-data-analysis-core`
6
9
 
7
10
  Install with Node.js: `npm i @nshiab/simple-data-analysis-core`
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nshiab/simple-data-analysis-core",
3
- "version": "2.0.2",
3
+ "version": "2.0.4",
4
4
  "description": "Lightweight DuckDB-powered TypeScript library for tabular, SQL, CSV, Parquet, and geospatial data analysis on Deno, Node.js, and Bun.",
5
5
  "keywords": [
6
6
  "data-analysis",