@nshiab/simple-data-analysis-core 2.0.2 → 2.0.4

package/script/helpers/osmCache.js

@@ -1,81 +1,193 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createOsmCacheId = createOsmCacheId;
 exports.useProcessedOsmCache = useProcessedOsmCache;
-const fileCache_js_1 = __importStar(require("./fileCache.js"));
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_fs_1 = require("node:fs");
+const cacheDiagnostics_js_1 = require("./cacheDiagnostics.js");
+const fileCache_js_1 = require("./fileCache.js");
+const osmFiles_js_1 = require("./osmFiles.js");
 const mergeOptions_js_1 = __importDefault(require("./mergeOptions.js"));
 const parseValue_js_1 = __importDefault(require("./parseValue.js"));
+const prettyDuration_js_1 = __importDefault(require("./prettyDuration.js"));
 const queryDB_js_1 = __importDefault(require("./queryDB.js"));
 const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
 const CACHE_DIRECTORY = ".sda-cache/osm";
-const OSM_CACHE_FORMAT_VERSION = "osm-geoparquet-v1";
+const OSM_CACHE_FORMAT_VERSION = "osm-geoparquet-v2";
 function createOsmCacheId(identity) {
-    return (0, fileCache_js_1.createFileCacheId)("osm", identity);
+    return (0, fileCache_js_1.createFileCacheId)("osm", [OSM_CACHE_FORMAT_VERSION, identity]);
+}
+async function useProcessedOsmCache(table, options) {
+    if (!options.enabled) {
+        options.verbose && console.log("Processed cache disabled. Fresh data will be used without reading or writing cache entries.");
+        await options.compute();
+        return;
+    }
+    const sourcesFile = `${CACHE_DIRECTORY}/sources.json`;
+    const sources = readSources(sourcesFile);
+    const cached = sources[options.id];
+    if (cached !== undefined) {
+        if (!isOsmCacheSource(cached)) {
+            delete sources[options.id];
+            writeSources(sourcesFile, sources);
+            console.log("The matching processed cache entry was incomplete and was removed. Fresh data is being requested.");
+        }
+        else {
+            const now = Date.now();
+            if (options.ttl !== undefined &&
+                (0, cacheDiagnostics_js_1.cacheEntryExpired)(cached.creation, options.ttl, now)) {
+                const age = now - cached.creation;
+                removeEntry(options.id, cached, sources, sourcesFile);
+                console.log(`Processed cache entry expired after ${(0, prettyDuration_js_1.default)(0, { end: age })}; ` +
+                    `the configured TTL is ${(0, prettyDuration_js_1.default)(0, { end: options.ttl * 1000 })}.\n` +
+                    "The matching cache file was removed. Fresh data is being requested.");
+            }
+            else if (await loadEntry(table, cached, sources, sourcesFile, options)) {
+                return;
+            }
+        }
+    }
+    else {
+        options.verbose &&
+            console.log("No processed cache entry matched this source.");
+    }
+    const started = Date.now();
+    await options.compute();
+    const computeDuration = Date.now() - started;
+    await writeEntry(table, sources, sourcesFile, options, computeDuration);
 }
-async function useProcessedOsmCache(table, id, enabled, parameters, compute) {
-    await (0, fileCache_js_1.default)(table, {
-        directory: CACHE_DIRECTORY,
-        enabled,
-        extension: "geoparquet",
+async function loadEntry(table, cached, sources, sourcesFile, options) {
+    if (cached.formatVersion !== OSM_CACHE_FORMAT_VERSION ||
+        typeof cached.file !== "string" ||
+        !(0, node_fs_1.existsSync)(cached.file)) {
+        removeEntry(options.id, cached, sources, sourcesFile);
+        console.log("The matching processed cache entry was incomplete or missing and was removed. Fresh data is being requested.");
+        return false;
+    }
+    const size = (0, node_fs_1.statSync)(cached.file).size;
+    options.verbose && console.log(`Processed cache hit.\nLoading ${(0, osmFiles_js_1.formatBytes)(size)} from the GeoParquet cache; Osmium will not run.`);
+    const started = Date.now();
+    try {
+        await (0, queryDB_js_1.default)(table, `INSTALL spatial;
+      LOAD spatial;
+      SET geometry_always_xy = true;
+      CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(table.name)} AS
+      SELECT * FROM read_parquet(${(0, parseValue_js_1.default)(cached.file)});`, (0, mergeOptions_js_1.default)(table, {
+            table: table.name,
+            method: "loadOpenStreetMap()",
+            parameters: options.parameters,
+        }));
+        table.sdb.spatialLoaded = true;
+        if (options.verbose) {
+            const rows = await getRowCount(table, options.parameters);
+            console.log(`Loaded ${rows.toLocaleString("en-US")} rows in ${(0, prettyDuration_js_1.default)(started)}.`);
+            if (cached.duration !== undefined) {
+                table.sdb.cacheTimeSaved += cached.duration - (Date.now() - started);
+            }
+        }
+        return true;
+    }
+    catch {
+        removeEntry(options.id, cached, sources, sourcesFile);
+        console.log("The matching processed cache entry was corrupt or unusable and was removed. Fresh data is being requested.");
+        return false;
+    }
+}
+async function writeEntry(table, sources, sourcesFile, options, computeDuration) {
+    (0, node_fs_1.mkdirSync)(CACHE_DIRECTORY, { recursive: true });
+    const file = `${CACHE_DIRECTORY}/${options.id}.geoparquet`;
+    const temporaryFile = `${file}.${node_crypto_1.default.randomUUID()}.tmp.geoparquet`;
+    options.verbose && console.log("Writing processed GeoParquet cache...");
+    const started = Date.now();
+    try {
+        await (0, queryDB_js_1.default)(table, `COPY ${(0, quoteIdentifier_js_1.default)(table.name)} TO ${(0, parseValue_js_1.default)(temporaryFile)}
+      WITH (FORMAT PARQUET);`, (0, mergeOptions_js_1.default)(table, {
+            table: table.name,
+            method: "loadOpenStreetMap()",
+            parameters: options.parameters,
+        }));
+        (0, node_fs_1.renameSync)(temporaryFile, file);
+    }
+    catch (error) {
+        (0, node_fs_1.rmSync)(temporaryFile, { force: true });
+        throw error;
+    }
+    const creation = Date.now();
+    sources[options.id] = {
+        file,
+        creation,
+        duration: computeDuration,
         formatVersion: OSM_CACHE_FORMAT_VERSION,
-        id,
-        method: "loadOSM()",
-        load: async (file) => {
-            await (0, queryDB_js_1.default)(table, `INSTALL spatial;
-        LOAD spatial;
-        SET geometry_always_xy = true;
-        CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(table.name)} AS
-        SELECT * FROM read_parquet(${(0, parseValue_js_1.default)(file)});`, (0, mergeOptions_js_1.default)(table, {
-                table: table.name,
-                method: "loadOSM()",
-                parameters,
-            }));
-            table.sdb.spatialLoaded = true;
-        },
-        compute,
-        write: async (file) => {
-            await (0, queryDB_js_1.default)(table, `COPY ${(0, quoteIdentifier_js_1.default)(table.name)} TO ${(0, parseValue_js_1.default)(file)}
-        WITH (FORMAT PARQUET);`, (0, mergeOptions_js_1.default)(table, {
-                table: table.name,
-                method: "loadOSM()",
-                parameters,
-            }));
-        },
-    });
+    };
+    try {
+        writeSources(sourcesFile, sources);
+    }
+    catch (error) {
+        (0, node_fs_1.rmSync)(file, { force: true });
+        delete sources[options.id];
+        throw error;
+    }
+    if (options.verbose) {
+        const size = (0, node_fs_1.statSync)(file).size;
+        console.log(`Wrote ${(0, osmFiles_js_1.formatBytes)(size)} in ${(0, prettyDuration_js_1.default)(started)}.`);
+        table.sdb.cacheTimeWriting += Date.now() - started;
+    }
+}
+async function getRowCount(table, parameters) {
+    const data = await (0, queryDB_js_1.default)(table, `SELECT count(*)::DOUBLE AS count FROM ${(0, quoteIdentifier_js_1.default)(table.name)};`, (0, mergeOptions_js_1.default)(table, {
+        table: table.name,
+        method: "loadOpenStreetMap()",
+        parameters,
+        returnData: true,
+    }));
+    const count = data?.[0]?.count;
+    return typeof count === "number" ? count : Number(count ?? 0);
+}
+function removeEntry(id, source, sources, sourcesFile) {
+    if (typeof source.file === "string")
+        (0, node_fs_1.rmSync)(source.file, { force: true });
+    delete sources[id];
+    writeSources(sourcesFile, sources);
+}
+function readSources(sourcesFile) {
+    if (!(0, node_fs_1.existsSync)(sourcesFile))
+        return {};
+    try {
+        const value = JSON.parse((0, node_fs_1.readFileSync)(sourcesFile, "utf-8"));
+        return typeof value === "object" && value !== null
+            ? value
+            : {};
+    }
+    catch {
+        (0, node_fs_1.rmSync)(sourcesFile, { force: true });
+        console.log("The OpenStreetMap cache index was corrupt and was removed. Fresh data is being requested.");
+        return {};
+    }
+}
+function isOsmCacheSource(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const source = value;
+    return typeof source.file === "string" &&
+        typeof source.creation === "number" &&
+        Number.isFinite(source.creation) &&
+        typeof source.formatVersion === "string" &&
+        (source.duration === undefined ||
+            (typeof source.duration === "number" &&
+                Number.isFinite(source.duration)));
+}
+function writeSources(sourcesFile, sources) {
+    (0, node_fs_1.mkdirSync)(CACHE_DIRECTORY, { recursive: true });
+    const temporaryFile = `${sourcesFile}.${node_crypto_1.default.randomUUID()}.tmp`;
+    try {
+        (0, node_fs_1.writeFileSync)(temporaryFile, JSON.stringify(sources));
+        (0, node_fs_1.renameSync)(temporaryFile, sourcesFile);
+    }
+    catch (error) {
+        (0, node_fs_1.rmSync)(temporaryFile, { force: true });
+        throw error;
+    }
 }

package/script/helpers/osmFiles.d.ts

@@ -1,9 +1,18 @@
 export type OsmFileSuffix = ".osm" | ".osm.pbf";
 export declare function getOsmFileSuffix(file: string): OsmFileSuffix | null;
 export declare function isRemoteOsmUrl(file: string): boolean;
+export declare function normalizeRemoteOsmUrl(file: string): string;
+export declare function sanitizeOsmUrl(source: string): string;
 export declare function downloadOsmToTemporaryFile(url: string, options: {
     suffix: OsmFileSuffix;
     prefix?: string;
     request?: RequestInit;
+    retries?: number;
+    retryDelay?: number;
+    timeout?: number;
+    overpass?: boolean;
+    verbose?: boolean;
+    wait?: (milliseconds: number) => Promise<void>;
 }): Promise<string>;
+export declare function formatBytes(bytes: number): string;
 //# sourceMappingURL=osmFiles.d.ts.map

package/script/helpers/osmFiles.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"osmFiles.d.ts","sourceRoot":"","sources":["../../src/helpers/osmFiles.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,UAAU,CAAC;AAEhD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAUnE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAOpD;AAED,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;IACP,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB,GACA,OAAO,CAAC,MAAM,CAAC,CAyCjB"}
+{"version":3,"file":"osmFiles.d.ts","sourceRoot":"","sources":["../../src/helpers/osmFiles.ts"],"names":[],"mappings":"AAOA,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,UAAU,CAAC;AAShD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAMnE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAOpD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAe1D;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAmBrD;AAED,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;IACP,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAChD,GACA,OAAO,CAAC,MAAM,CAAC,CAmHjB;AAgMD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAWjD"}

package/script/helpers/osmFiles.js

@@ -5,21 +5,23 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getOsmFileSuffix = getOsmFileSuffix;
 exports.isRemoteOsmUrl = isRemoteOsmUrl;
+exports.normalizeRemoteOsmUrl = normalizeRemoteOsmUrl;
+exports.sanitizeOsmUrl = sanitizeOsmUrl;
 exports.downloadOsmToTemporaryFile = downloadOsmToTemporaryFile;
+exports.formatBytes = formatBytes;
 const node_crypto_1 = __importDefault(require("node:crypto"));
 const node_fs_1 = require("node:fs");
 const promises_1 = require("node:fs/promises");
-const promises_2 = require("node:stream/promises");
 const node_stream_1 = require("node:stream");
+const promises_2 = require("node:stream/promises");
+const prettyDuration_js_1 = __importDefault(require("./prettyDuration.js"));
 function getOsmFileSuffix(file) {
     const comparable = isRemoteOsmUrl(file) ? new URL(file).pathname : file;
     const lower = comparable.toLowerCase();
-    if (lower.endsWith(".osm.pbf")) {
+    if (lower.endsWith(".osm.pbf"))
         return ".osm.pbf";
-    }
-    if (lower.endsWith(".osm")) {
+    if (lower.endsWith(".osm"))
         return ".osm";
-    }
     return null;
 }
 function isRemoteOsmUrl(file) {
@@ -31,30 +33,242 @@ function isRemoteOsmUrl(file) {
         return false;
     }
 }
-async function downloadOsmToTemporaryFile(url, options) {
-    const temporaryDirectory = ".sda-cache/tmp";
-    (0, node_fs_1.mkdirSync)(temporaryDirectory, { recursive: true });
-    const prefix = options.prefix === undefined ? "remote" : options.prefix;
-    const temporaryFile = `${temporaryDirectory}/${prefix}.${node_crypto_1.default.randomUUID()}${options.suffix}`;
-    const response = await fetch(url, options.request);
-    if (!response.ok) {
-        const responseText = (await response.text()).slice(0, 500).trim();
-        throw new Error(`Could not download OSM data from ${JSON.stringify(url)}: HTTP ${response.status} ${response.statusText}${responseText.length > 0 ? `\n${responseText}` : ""}`);
+function normalizeRemoteOsmUrl(file) {
+    let url;
+    try {
+        url = new URL(file);
+    }
+    catch {
+        throw new Error("loadOpenStreetMap() remote sources must be valid HTTP or HTTPS URLs.");
     }
-    if (response.body === null) {
-        throw new Error(`Could not download OSM data from ${JSON.stringify(url)}: the response body is empty.`);
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        throw new Error("loadOpenStreetMap() remote sources must use an HTTP or HTTPS URL.");
     }
+    return url.href;
+}
+function sanitizeOsmUrl(source) {
+    let url;
     try {
-        await (0, promises_2.pipeline)(node_stream_1.Readable.fromWeb(response.body), (0, node_fs_1.createWriteStream)(temporaryFile, { flags: "wx" }));
+        url = new URL(source);
+    }
+    catch {
+        return source;
+    }
+    url.username = "";
+    url.password = "";
+    for (const key of [...url.searchParams.keys()]) {
+        if (/(?:api[-_]?key|auth|credential|password|secret|signature|token)/i.test(key)) {
+            url.searchParams.set(key, "[REDACTED]");
+        }
     }
-    catch (error) {
+    return url.href;
+}
+async function downloadOsmToTemporaryFile(url, options) {
+    const temporaryDirectory = ".sda-cache/tmp";
+    (0, node_fs_1.mkdirSync)(temporaryDirectory, { recursive: true });
+    const prefix = options.prefix === undefined ? "remote" : options.prefix;
+    const retries = options.retries ?? 3;
+    const retryDelay = options.retryDelay ?? 5;
+    const totalAttempts = retries + 1;
+    const sanitizedSource = sanitizeOsmUrl(url);
+    const wait = options.wait ?? waitFor;
+    let totalDelay = 0;
+    let lastFailure;
+    for (let attempt = 1; attempt <= totalAttempts; attempt++) {
+        const temporaryFile = `${temporaryDirectory}/${prefix}.${node_crypto_1.default.randomUUID()}${options.suffix}`;
+        const start = Date.now();
+        options.verbose && console.log(`${options.overpass ? "Querying Overpass" : "Downloading OSM data"} (attempt ${attempt} of ${totalAttempts})...`);
         try {
-            await (0, promises_1.rm)(temporaryFile, { force: true });
+            const response = await fetchWithTimeout(url, options.request, options.timeout);
+            if (!response.ok) {
+                const failure = classifyHttpFailure(response, options.overpass ?? false);
+                if (!failure.retryable) {
+                    const responseText = (await response.text()).slice(0, 500).trim();
+                    if (responseText.length > 0) {
+                        failure.reason += ` Response: ${responseText}`;
+                    }
+                }
+                else {
+                    await response.body?.cancel();
+                }
+                throw failure;
+            }
+            if (response.body === null) {
+                throw {
+                    reason: "The response body was empty.",
+                    retryable: true,
+                    explanation: "an empty response can be caused by an interrupted transfer",
+                };
+            }
+            await (0, promises_2.pipeline)(node_stream_1.Readable.fromWeb(response.body), (0, node_fs_1.createWriteStream)(temporaryFile, { flags: "wx" }));
+            if (options.verbose) {
+                const size = (await Deno.stat(temporaryFile)).size;
+                console.log(`${options.overpass ? "Received" : "Downloaded"} ${formatBytes(size)} in ${(0, prettyDuration_js_1.default)(start)}.`);
+            }
+            return temporaryFile;
         }
-        catch {
-            // Preserve the download error.
+        catch (error) {
+            await (0, promises_1.rm)(temporaryFile, { force: true }).catch(() => undefined);
+            lastFailure = isRetryFailure(error)
+                ? error
+                : classifyFetchFailure(error, options.timeout);
+            const canRetry = lastFailure.retryable && attempt < totalAttempts;
+            logAttemptFailure(sanitizedSource, attempt, totalAttempts, lastFailure, canRetry);
+            if (!canRetry) {
+                throw new Error(finalRequestError(attempt, retries, totalDelay, lastFailure), { cause: error });
+            }
+            const calculatedDelay = retryDelay * 2 ** (attempt - 1);
+            const delay = Math.max(calculatedDelay, lastFailure.retryAfterSeconds ?? 0);
+            logRetryDelay(delay, calculatedDelay, lastFailure.retryAfterSeconds, attempt, retries, retryDelay);
+            totalDelay += delay;
+            await wait(delay * 1000);
         }
-        throw error;
     }
-    return temporaryFile;
+    throw new Error(finalRequestError(totalAttempts, retries, totalDelay, lastFailure ?? {
+        reason: "Unknown request failure.",
+        retryable: false,
+        explanation: "the failure could not be classified",
+    }));
+}
+async function fetchWithTimeout(url, request, timeout) {
+    const timeoutSignal = timeout === undefined
+        ? undefined
+        : AbortSignal.timeout(timeout * 1000);
+    const requestSignal = request?.signal ?? undefined;
+    const signal = timeoutSignal === undefined
+        ? requestSignal
+        : requestSignal === undefined
+            ? timeoutSignal
+            : AbortSignal.any([timeoutSignal, requestSignal]);
+    return await fetch(url, { ...request, signal });
+}
+function classifyHttpFailure(response, overpass) {
+    const status = `HTTP ${response.status}${response.statusText.length > 0 ? ` ${response.statusText}` : ""}`;
+    const retryAfterSeconds = parseRetryAfter(response.headers.get("retry-after"));
+    if (response.status === 408) {
+        return {
+            reason: status,
+            retryable: true,
+            explanation: "HTTP 408 indicates a transient request timeout",
+            retryAfterSeconds,
+        };
+    }
+    if (response.status === 429) {
+        return {
+            reason: status,
+            retryable: true,
+            explanation: "HTTP 429 indicates temporary rate limiting",
+            retryAfterSeconds,
+        };
+    }
+    if (response.status >= 500 && response.status <= 599) {
+        return {
+            reason: status,
+            retryable: true,
+            explanation: "it is a server error",
+            retryAfterSeconds,
+        };
+    }
+    if (overpass && response.status === 406) {
+        return {
+            reason: status,
+            retryable: true,
+            explanation: "Overpass public instances use HTTP 406 for temporary resource limits",
+            retryAfterSeconds,
+        };
+    }
+    return {
+        reason: status,
+        retryable: false,
+        explanation: `HTTP ${response.status} is not considered a transient failure`,
+    };
+}
+function classifyFetchFailure(error, timeout) {
+    const message = error instanceof Error ? error.message : String(error);
+    const timedOut = error instanceof DOMException &&
+        (error.name === "TimeoutError" || error.name === "AbortError") &&
+        timeout !== undefined;
+    return {
+        reason: timedOut
+            ? `The request timed out after ${timeout} seconds.`
+            : message,
+        retryable: true,
+        explanation: timedOut
+            ? "a configured request timeout is retryable"
+            : "fetch and transfer errors are treated as transient network failures",
+    };
+}
+function isRetryFailure(error) {
+    if (typeof error !== "object" || error === null)
+        return false;
+    const candidate = error;
+    return typeof candidate.reason === "string" &&
+        typeof candidate.retryable === "boolean" &&
+        typeof candidate.explanation === "string";
+}
+function parseRetryAfter(value) {
+    if (value === null)
+        return undefined;
+    const trimmed = value.trim();
+    if (/^\d+$/.test(trimmed))
+        return Number(trimmed);
+    const date = Date.parse(trimmed);
+    if (Number.isNaN(date))
+        return undefined;
+    return Math.max(0, Math.ceil((date - Date.now()) / 1000));
+}
+function logAttemptFailure(source, attempt, totalAttempts, failure, canRetry) {
+    console.log(`loadOpenStreetMap(): request attempt ${attempt} of ${totalAttempts} failed.\n` +
+        `Source: ${source}\n` +
+        `Reason: ${failure.reason}\n` +
+        `${canRetry
+            ? `This failure is retryable because ${failure.explanation}.`
+            : `Not retrying because ${failure.retryable
+                ? totalAttempts === 1
+                    ? "no retries were configured"
+                    : "the configured retries have been exhausted"
+                : failure.explanation}.`}`);
+}
+function logRetryDelay(delay, calculatedDelay, retryAfter, attempt, retries, retryDelay) {
+    if (retryAfter !== undefined && retryAfter > calculatedDelay) {
+        console.log(`Retrying in ${formatSeconds(delay)} because the server returned Retry-After: ${retryAfter}.\n` +
+            `The calculated retry delay was ${formatSeconds(calculatedDelay)}; the longer server-requested delay is being used.`);
+        return;
+    }
+    console.log(`Retrying in ${formatSeconds(delay)} (retry ${attempt} of ${retries}; configured base delay: ${formatSeconds(retryDelay)}).`);
+}
+function finalRequestError(attempts, retries, totalDelay, failure) {
+    const exhausted = failure.retryable && attempts === retries + 1;
+    return `loadOpenStreetMap(): request failed after ${attempts} ${attempts === 1 ? "attempt" : "attempts"}.\n${exhausted
+        ? retries === 0
+            ? "No retries were configured."
+            : `All ${retries} configured ${retries === 1 ? "retry was" : "retries were"} exhausted.`
+        : "No further retry was attempted because the failure is nonretryable."}\nTotal retry delay: ${formatSeconds(totalDelay)}.\n` +
+        `Last failure: ${failure.reason}\n` +
+        `No further retry occurred because ${exhausted
+            ? retries === 0
+                ? "no retries were configured"
+                : "the configured retries were exhausted"
+            : failure.explanation}.\n` +
+        "No invalid cached data was returned.";
+}
+function formatSeconds(seconds) {
+    return `${seconds} ${seconds === 1 ? "second" : "seconds"}`;
+}
+function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} ${bytes === 1 ? "byte" : "bytes"}`;
+    const units = ["KB", "MB", "GB", "TB"];
+    let value = bytes;
+    let unit = units[0];
+    for (const candidate of units) {
+        value /= 1024;
+        unit = candidate;
+        if (value < 1024)
+            break;
+    }
+    return `${value.toFixed(1)} ${unit}`;
+}
+function waitFor(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
 }

package/script/methods/loadGeoData.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/loadGeoData.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAWvD,KAAK,kBAAkB,GAAG;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,kBAAuB,QAYjC"}
+{"version":3,"file":"loadGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/loadGeoData.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,KAAK,kBAAkB,GAAG;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,kBAAuB,QAiBjC"}

package/script/methods/loadGeoData.js

@@ -11,12 +11,12 @@ const parseValue_js_1 = __importDefault(require("../helpers/parseValue.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const removeColumnsNow_js_1 = __importDefault(require("../helpers/removeColumnsNow.js"));
-const node_fs_1 = require("node:fs");
-const SDAError_js_1 = __importDefault(require("../class/SDAError.js"));
-const loadOsmFile_js_1 = __importDefault(require("../helpers/loadOsmFile.js"));
 const cacheTableDependencies_js_1 = require("../helpers/cacheTableDependencies.js");
 const osmFiles_js_1 = require("../helpers/osmFiles.js");
 function loadGeoData(simpleTable, file, options = {}) {
+    if ((0, osmFiles_js_1.getOsmFileSuffix)(file) !== null) {
+        throw new Error("loadGeoData() does not support .osm or .osm.pbf files. Use loadOpenStreetMap() instead.");
+    }
     options = structuredClone(options);
     if (options.conditions !== undefined) {
         (0, cacheTableDependencies_js_1.recordCacheTableReferences)(simpleTable, [options.conditions]);
@@ -29,41 +29,6 @@ function loadGeoData(simpleTable, file, options = {}) {
     });
 }
 async function executeLoadGeoData(simpleTable, file, options) {
-    const osmSuffix = (0, osmFiles_js_1.getOsmFileSuffix)(file);
-    if (osmSuffix !== null) {
-        let temporaryFile;
-        try {
-            const localFile = (0, osmFiles_js_1.isRemoteOsmUrl)(file)
-                ? temporaryFile = await (0, osmFiles_js_1.downloadOsmToTemporaryFile)(file, {
-                    suffix: osmSuffix,
-                    request: { headers: { "accept-encoding": "gzip" } },
-                })
-                : file;
-            await (0, loadOsmFile_js_1.default)(simpleTable, localFile, {
-                method: "loadGeoData()",
-                parameters: { file, options },
-                columns: options.columns,
-                conditions: options.conditions,
-            });
-            return;
-        }
-        catch (error) {
-            if (error instanceof SDAError_js_1.default) {
-                throw error;
-            }
-            throw new SDAError_js_1.default({
-                method: "loadGeoData()",
-                parameters: { file, options },
-                query: "",
-                cause: error,
-            });
-        }
-        finally {
-            if (temporaryFile !== undefined) {
-                (0, node_fs_1.rmSync)(temporaryFile, { force: true });
-            }
-        }
-    }
     const fileExtension = (0, getExtension_js_1.default)(file);
     const selectColumns = options.columns !== undefined &&
         options.columns.length > 0

package/script/methods/loadOpenStreetMap.d.ts

@@ -0,0 +1,24 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type BoundingBox = {
+    west: number;
+    south: number;
+    east: number;
+    north: number;
+};
+type CommonOptions = {
+    cache?: boolean;
+    ttl?: number;
+    retries?: number;
+    retryDelay?: number;
+    verbose?: boolean;
+};
+type LoadOpenStreetMapFileOptions = CommonOptions;
+type LoadOpenStreetMapOverpassOptions = CommonOptions & {
+    filters: string | [string, string] | [string, string][];
+    endpoint?: string;
+    timeout?: number;
+};
+export default function loadOpenStreetMap(table: SimpleTable, source: string | BoundingBox, options?: LoadOpenStreetMapFileOptions | LoadOpenStreetMapOverpassOptions): void;
+export declare function generateOverpassQuery(bbox: BoundingBox, filters: string[], timeout?: number): string;
+export {};
+//# sourceMappingURL=loadOpenStreetMap.d.ts.map

package/script/methods/loadOpenStreetMap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadOpenStreetMap.d.ts","sourceRoot":"","sources":["../../src/methods/loadOpenStreetMap.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAqBvD,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,KAAK,4BAA4B,GAAG,aAAa,CAAC;AAElD,KAAK,gCAAgC,GAAG,aAAa,GAAG;IACtD,OAAO,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,GAAG,WAAW,EAC5B,OAAO,GACH,4BAA4B,GAC5B,gCAAqC,GACxC,IAAI,CAcN;AA+SD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE,MAAM,GACf,MAAM,CASR"}