@nshiab/simple-data-analysis-core 2.0.2 → 2.0.4

package/esm/helpers/osmCache.js

@@ -1,41 +1,186 @@
-import fileCache, { createFileCacheId } from "./fileCache.js";
+import crypto from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs";
+import { cacheEntryExpired } from "./cacheDiagnostics.js";
+import { createFileCacheId } from "./fileCache.js";
+import { formatBytes } from "./osmFiles.js";
 import mergeOptions from "./mergeOptions.js";
 import parseValue from "./parseValue.js";
+import prettyDuration from "./prettyDuration.js";
 import queryDB from "./queryDB.js";
 import quoteIdentifier from "./quoteIdentifier.js";
 const CACHE_DIRECTORY = ".sda-cache/osm";
-const OSM_CACHE_FORMAT_VERSION = "osm-geoparquet-v1";
+const OSM_CACHE_FORMAT_VERSION = "osm-geoparquet-v2";
 export function createOsmCacheId(identity) {
-    return createFileCacheId("osm", identity);
+    return createFileCacheId("osm", [OSM_CACHE_FORMAT_VERSION, identity]);
 }
-export async function useProcessedOsmCache(table, id, enabled, parameters, compute) {
-    await fileCache(table, {
-        directory: CACHE_DIRECTORY,
-        enabled,
-        extension: "geoparquet",
+export async function useProcessedOsmCache(table, options) {
+    if (!options.enabled) {
+        options.verbose && console.log("Processed cache disabled. Fresh data will be used without reading or writing cache entries.");
+        await options.compute();
+        return;
+    }
+    const sourcesFile = `${CACHE_DIRECTORY}/sources.json`;
+    const sources = readSources(sourcesFile);
+    const cached = sources[options.id];
+    if (cached !== undefined) {
+        if (!isOsmCacheSource(cached)) {
+            delete sources[options.id];
+            writeSources(sourcesFile, sources);
+            console.log("The matching processed cache entry was incomplete and was removed. Fresh data is being requested.");
+        }
+        else {
+            const now = Date.now();
+            if (options.ttl !== undefined &&
+                cacheEntryExpired(cached.creation, options.ttl, now)) {
+                const age = now - cached.creation;
+                removeEntry(options.id, cached, sources, sourcesFile);
+                console.log(`Processed cache entry expired after ${prettyDuration(0, { end: age })}; ` +
+                    `the configured TTL is ${prettyDuration(0, { end: options.ttl * 1000 })}.\n` +
+                    "The matching cache file was removed. Fresh data is being requested.");
+            }
+            else if (await loadEntry(table, cached, sources, sourcesFile, options)) {
+                return;
+            }
+        }
+    }
+    else {
+        options.verbose &&
+            console.log("No processed cache entry matched this source.");
+    }
+    const started = Date.now();
+    await options.compute();
+    const computeDuration = Date.now() - started;
+    await writeEntry(table, sources, sourcesFile, options, computeDuration);
+}
+async function loadEntry(table, cached, sources, sourcesFile, options) {
+    if (cached.formatVersion !== OSM_CACHE_FORMAT_VERSION ||
+        typeof cached.file !== "string" ||
+        !existsSync(cached.file)) {
+        removeEntry(options.id, cached, sources, sourcesFile);
+        console.log("The matching processed cache entry was incomplete or missing and was removed. Fresh data is being requested.");
+        return false;
+    }
+    const size = statSync(cached.file).size;
+    options.verbose && console.log(`Processed cache hit.\nLoading ${formatBytes(size)} from the GeoParquet cache; Osmium will not run.`);
+    const started = Date.now();
+    try {
+        await queryDB(table, `INSTALL spatial;
+      LOAD spatial;
+      SET geometry_always_xy = true;
+      CREATE OR REPLACE TABLE ${quoteIdentifier(table.name)} AS
+      SELECT * FROM read_parquet(${parseValue(cached.file)});`, mergeOptions(table, {
+            table: table.name,
+            method: "loadOpenStreetMap()",
+            parameters: options.parameters,
+        }));
+        table.sdb.spatialLoaded = true;
+        if (options.verbose) {
+            const rows = await getRowCount(table, options.parameters);
+            console.log(`Loaded ${rows.toLocaleString("en-US")} rows in ${prettyDuration(started)}.`);
+            if (cached.duration !== undefined) {
+                table.sdb.cacheTimeSaved += cached.duration - (Date.now() - started);
+            }
+        }
+        return true;
+    }
+    catch {
+        removeEntry(options.id, cached, sources, sourcesFile);
+        console.log("The matching processed cache entry was corrupt or unusable and was removed. Fresh data is being requested.");
+        return false;
+    }
+}
+async function writeEntry(table, sources, sourcesFile, options, computeDuration) {
+    mkdirSync(CACHE_DIRECTORY, { recursive: true });
+    const file = `${CACHE_DIRECTORY}/${options.id}.geoparquet`;
+    const temporaryFile = `${file}.${crypto.randomUUID()}.tmp.geoparquet`;
+    options.verbose && console.log("Writing processed GeoParquet cache...");
+    const started = Date.now();
+    try {
+        await queryDB(table, `COPY ${quoteIdentifier(table.name)} TO ${parseValue(temporaryFile)}
+      WITH (FORMAT PARQUET);`, mergeOptions(table, {
+            table: table.name,
+            method: "loadOpenStreetMap()",
+            parameters: options.parameters,
+        }));
+        renameSync(temporaryFile, file);
+    }
+    catch (error) {
+        rmSync(temporaryFile, { force: true });
+        throw error;
+    }
+    const creation = Date.now();
+    sources[options.id] = {
+        file,
+        creation,
+        duration: computeDuration,
         formatVersion: OSM_CACHE_FORMAT_VERSION,
-        id,
-        method: "loadOSM()",
-        load: async (file) => {
-            await queryDB(table, `INSTALL spatial;
-        LOAD spatial;
-        SET geometry_always_xy = true;
-        CREATE OR REPLACE TABLE ${quoteIdentifier(table.name)} AS
-        SELECT * FROM read_parquet(${parseValue(file)});`, mergeOptions(table, {
-                table: table.name,
-                method: "loadOSM()",
-                parameters,
-            }));
-            table.sdb.spatialLoaded = true;
-        },
-        compute,
-        write: async (file) => {
-            await queryDB(table, `COPY ${quoteIdentifier(table.name)} TO ${parseValue(file)}
-        WITH (FORMAT PARQUET);`, mergeOptions(table, {
-                table: table.name,
-                method: "loadOSM()",
-                parameters,
-            }));
-        },
-    });
+    };
+    try {
+        writeSources(sourcesFile, sources);
+    }
+    catch (error) {
+        rmSync(file, { force: true });
+        delete sources[options.id];
+        throw error;
+    }
+    if (options.verbose) {
+        const size = statSync(file).size;
+        console.log(`Wrote ${formatBytes(size)} in ${prettyDuration(started)}.`);
+        table.sdb.cacheTimeWriting += Date.now() - started;
+    }
+}
+async function getRowCount(table, parameters) {
+    const data = await queryDB(table, `SELECT count(*)::DOUBLE AS count FROM ${quoteIdentifier(table.name)};`, mergeOptions(table, {
+        table: table.name,
+        method: "loadOpenStreetMap()",
+        parameters,
+        returnData: true,
+    }));
+    const count = data?.[0]?.count;
+    return typeof count === "number" ? count : Number(count ?? 0);
+}
+function removeEntry(id, source, sources, sourcesFile) {
+    if (typeof source.file === "string")
+        rmSync(source.file, { force: true });
+    delete sources[id];
+    writeSources(sourcesFile, sources);
+}
+function readSources(sourcesFile) {
+    if (!existsSync(sourcesFile))
+        return {};
+    try {
+        const value = JSON.parse(readFileSync(sourcesFile, "utf-8"));
+        return typeof value === "object" && value !== null
+            ? value
+            : {};
+    }
+    catch {
+        rmSync(sourcesFile, { force: true });
+        console.log("The OpenStreetMap cache index was corrupt and was removed. Fresh data is being requested.");
+        return {};
+    }
+}
+function isOsmCacheSource(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const source = value;
+    return typeof source.file === "string" &&
+        typeof source.creation === "number" &&
+        Number.isFinite(source.creation) &&
+        typeof source.formatVersion === "string" &&
+        (source.duration === undefined ||
+            (typeof source.duration === "number" &&
+                Number.isFinite(source.duration)));
+}
+function writeSources(sourcesFile, sources) {
+    mkdirSync(CACHE_DIRECTORY, { recursive: true });
+    const temporaryFile = `${sourcesFile}.${crypto.randomUUID()}.tmp`;
+    try {
+        writeFileSync(temporaryFile, JSON.stringify(sources));
+        renameSync(temporaryFile, sourcesFile);
+    }
+    catch (error) {
+        rmSync(temporaryFile, { force: true });
+        throw error;
+    }
 }

package/esm/helpers/osmFiles.d.ts

@@ -1,9 +1,18 @@
 export type OsmFileSuffix = ".osm" | ".osm.pbf";
 export declare function getOsmFileSuffix(file: string): OsmFileSuffix | null;
 export declare function isRemoteOsmUrl(file: string): boolean;
+export declare function normalizeRemoteOsmUrl(file: string): string;
+export declare function sanitizeOsmUrl(source: string): string;
 export declare function downloadOsmToTemporaryFile(url: string, options: {
     suffix: OsmFileSuffix;
     prefix?: string;
     request?: RequestInit;
+    retries?: number;
+    retryDelay?: number;
+    timeout?: number;
+    overpass?: boolean;
+    verbose?: boolean;
+    wait?: (milliseconds: number) => Promise<void>;
 }): Promise<string>;
+export declare function formatBytes(bytes: number): string;
 //# sourceMappingURL=osmFiles.d.ts.map

package/esm/helpers/osmFiles.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"osmFiles.d.ts","sourceRoot":"","sources":["../../src/helpers/osmFiles.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,UAAU,CAAC;AAEhD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAUnE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAOpD;AAED,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;IACP,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB,GACA,OAAO,CAAC,MAAM,CAAC,CAyCjB"}
+{"version":3,"file":"osmFiles.d.ts","sourceRoot":"","sources":["../../src/helpers/osmFiles.ts"],"names":[],"mappings":"AAOA,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,UAAU,CAAC;AAShD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAMnE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAOpD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAe1D;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAmBrD;AAED,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;IACP,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAChD,GACA,OAAO,CAAC,MAAM,CAAC,CAmHjB;AAgMD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAWjD"}

package/esm/helpers/osmFiles.js

@@ -1,17 +1,16 @@
 import crypto from "node:crypto";
 import { createWriteStream, mkdirSync } from "node:fs";
 import { rm } from "node:fs/promises";
-import { pipeline } from "node:stream/promises";
 import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import prettyDuration from "./prettyDuration.js";
 export function getOsmFileSuffix(file) {
     const comparable = isRemoteOsmUrl(file) ? new URL(file).pathname : file;
     const lower = comparable.toLowerCase();
-    if (lower.endsWith(".osm.pbf")) {
+    if (lower.endsWith(".osm.pbf"))
         return ".osm.pbf";
-    }
-    if (lower.endsWith(".osm")) {
+    if (lower.endsWith(".osm"))
         return ".osm";
-    }
     return null;
 }
 export function isRemoteOsmUrl(file) {
@@ -23,30 +22,242 @@ export function isRemoteOsmUrl(file) {
         return false;
     }
 }
-export async function downloadOsmToTemporaryFile(url, options) {
-    const temporaryDirectory = ".sda-cache/tmp";
-    mkdirSync(temporaryDirectory, { recursive: true });
-    const prefix = options.prefix === undefined ? "remote" : options.prefix;
-    const temporaryFile = `${temporaryDirectory}/${prefix}.${crypto.randomUUID()}${options.suffix}`;
-    const response = await fetch(url, options.request);
-    if (!response.ok) {
-        const responseText = (await response.text()).slice(0, 500).trim();
-        throw new Error(`Could not download OSM data from ${JSON.stringify(url)}: HTTP ${response.status} ${response.statusText}${responseText.length > 0 ? `\n${responseText}` : ""}`);
+export function normalizeRemoteOsmUrl(file) {
+    let url;
+    try {
+        url = new URL(file);
+    }
+    catch {
+        throw new Error("loadOpenStreetMap() remote sources must be valid HTTP or HTTPS URLs.");
     }
-    if (response.body === null) {
-        throw new Error(`Could not download OSM data from ${JSON.stringify(url)}: the response body is empty.`);
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        throw new Error("loadOpenStreetMap() remote sources must use an HTTP or HTTPS URL.");
     }
+    return url.href;
+}
+export function sanitizeOsmUrl(source) {
+    let url;
     try {
-        await pipeline(Readable.fromWeb(response.body), createWriteStream(temporaryFile, { flags: "wx" }));
+        url = new URL(source);
+    }
+    catch {
+        return source;
+    }
+    url.username = "";
+    url.password = "";
+    for (const key of [...url.searchParams.keys()]) {
+        if (/(?:api[-_]?key|auth|credential|password|secret|signature|token)/i.test(key)) {
+            url.searchParams.set(key, "[REDACTED]");
+        }
     }
-    catch (error) {
+    return url.href;
+}
+export async function downloadOsmToTemporaryFile(url, options) {
+    const temporaryDirectory = ".sda-cache/tmp";
+    mkdirSync(temporaryDirectory, { recursive: true });
+    const prefix = options.prefix === undefined ? "remote" : options.prefix;
+    const retries = options.retries ?? 3;
+    const retryDelay = options.retryDelay ?? 5;
+    const totalAttempts = retries + 1;
+    const sanitizedSource = sanitizeOsmUrl(url);
+    const wait = options.wait ?? waitFor;
+    let totalDelay = 0;
+    let lastFailure;
+    for (let attempt = 1; attempt <= totalAttempts; attempt++) {
+        const temporaryFile = `${temporaryDirectory}/${prefix}.${crypto.randomUUID()}${options.suffix}`;
+        const start = Date.now();
+        options.verbose && console.log(`${options.overpass ? "Querying Overpass" : "Downloading OSM data"} (attempt ${attempt} of ${totalAttempts})...`);
         try {
-            await rm(temporaryFile, { force: true });
+            const response = await fetchWithTimeout(url, options.request, options.timeout);
+            if (!response.ok) {
+                const failure = classifyHttpFailure(response, options.overpass ?? false);
+                if (!failure.retryable) {
+                    const responseText = (await response.text()).slice(0, 500).trim();
+                    if (responseText.length > 0) {
+                        failure.reason += ` Response: ${responseText}`;
+                    }
+                }
+                else {
+                    await response.body?.cancel();
+                }
+                throw failure;
+            }
+            if (response.body === null) {
+                throw {
+                    reason: "The response body was empty.",
+                    retryable: true,
+                    explanation: "an empty response can be caused by an interrupted transfer",
+                };
+            }
+            await pipeline(Readable.fromWeb(response.body), createWriteStream(temporaryFile, { flags: "wx" }));
+            if (options.verbose) {
+                const size = (await Deno.stat(temporaryFile)).size;
+                console.log(`${options.overpass ? "Received" : "Downloaded"} ${formatBytes(size)} in ${prettyDuration(start)}.`);
+            }
+            return temporaryFile;
         }
-        catch {
-            // Preserve the download error.
+        catch (error) {
+            await rm(temporaryFile, { force: true }).catch(() => undefined);
+            lastFailure = isRetryFailure(error)
+                ? error
+                : classifyFetchFailure(error, options.timeout);
+            const canRetry = lastFailure.retryable && attempt < totalAttempts;
+            logAttemptFailure(sanitizedSource, attempt, totalAttempts, lastFailure, canRetry);
+            if (!canRetry) {
+                throw new Error(finalRequestError(attempt, retries, totalDelay, lastFailure), { cause: error });
+            }
+            const calculatedDelay = retryDelay * 2 ** (attempt - 1);
+            const delay = Math.max(calculatedDelay, lastFailure.retryAfterSeconds ?? 0);
+            logRetryDelay(delay, calculatedDelay, lastFailure.retryAfterSeconds, attempt, retries, retryDelay);
+            totalDelay += delay;
+            await wait(delay * 1000);
         }
-        throw error;
     }
-    return temporaryFile;
+    throw new Error(finalRequestError(totalAttempts, retries, totalDelay, lastFailure ?? {
+        reason: "Unknown request failure.",
+        retryable: false,
+        explanation: "the failure could not be classified",
+    }));
+}
+async function fetchWithTimeout(url, request, timeout) {
+    const timeoutSignal = timeout === undefined
+        ? undefined
+        : AbortSignal.timeout(timeout * 1000);
+    const requestSignal = request?.signal ?? undefined;
+    const signal = timeoutSignal === undefined
+        ? requestSignal
+        : requestSignal === undefined
+            ? timeoutSignal
+            : AbortSignal.any([timeoutSignal, requestSignal]);
+    return await fetch(url, { ...request, signal });
+}
+function classifyHttpFailure(response, overpass) {
+    const status = `HTTP ${response.status}${response.statusText.length > 0 ? ` ${response.statusText}` : ""}`;
+    const retryAfterSeconds = parseRetryAfter(response.headers.get("retry-after"));
+    if (response.status === 408) {
+        return {
+            reason: status,
+            retryable: true,
+            explanation: "HTTP 408 indicates a transient request timeout",
+            retryAfterSeconds,
+        };
+    }
+    if (response.status === 429) {
+        return {
+            reason: status,
+            retryable: true,
+            explanation: "HTTP 429 indicates temporary rate limiting",
+            retryAfterSeconds,
+        };
+    }
+    if (response.status >= 500 && response.status <= 599) {
+        return {
+            reason: status,
+            retryable: true,
+            explanation: "it is a server error",
+            retryAfterSeconds,
+        };
+    }
+    if (overpass && response.status === 406) {
+        return {
+            reason: status,
+            retryable: true,
+            explanation: "Overpass public instances use HTTP 406 for temporary resource limits",
+            retryAfterSeconds,
+        };
+    }
+    return {
+        reason: status,
+        retryable: false,
+        explanation: `HTTP ${response.status} is not considered a transient failure`,
+    };
+}
+function classifyFetchFailure(error, timeout) {
+    const message = error instanceof Error ? error.message : String(error);
+    const timedOut = error instanceof DOMException &&
+        (error.name === "TimeoutError" || error.name === "AbortError") &&
+        timeout !== undefined;
+    return {
+        reason: timedOut
+            ? `The request timed out after ${timeout} seconds.`
+            : message,
+        retryable: true,
+        explanation: timedOut
+            ? "a configured request timeout is retryable"
+            : "fetch and transfer errors are treated as transient network failures",
+    };
+}
+function isRetryFailure(error) {
+    if (typeof error !== "object" || error === null)
+        return false;
+    const candidate = error;
+    return typeof candidate.reason === "string" &&
+        typeof candidate.retryable === "boolean" &&
+        typeof candidate.explanation === "string";
+}
+function parseRetryAfter(value) {
+    if (value === null)
+        return undefined;
+    const trimmed = value.trim();
+    if (/^\d+$/.test(trimmed))
+        return Number(trimmed);
+    const date = Date.parse(trimmed);
+    if (Number.isNaN(date))
+        return undefined;
+    return Math.max(0, Math.ceil((date - Date.now()) / 1000));
+}
+function logAttemptFailure(source, attempt, totalAttempts, failure, canRetry) {
+    console.log(`loadOpenStreetMap(): request attempt ${attempt} of ${totalAttempts} failed.\n` +
+        `Source: ${source}\n` +
+        `Reason: ${failure.reason}\n` +
+        `${canRetry
+            ? `This failure is retryable because ${failure.explanation}.`
+            : `Not retrying because ${failure.retryable
+                ? totalAttempts === 1
+                    ? "no retries were configured"
+                    : "the configured retries have been exhausted"
+                : failure.explanation}.`}`);
+}
+function logRetryDelay(delay, calculatedDelay, retryAfter, attempt, retries, retryDelay) {
+    if (retryAfter !== undefined && retryAfter > calculatedDelay) {
+        console.log(`Retrying in ${formatSeconds(delay)} because the server returned Retry-After: ${retryAfter}.\n` +
+            `The calculated retry delay was ${formatSeconds(calculatedDelay)}; the longer server-requested delay is being used.`);
+        return;
+    }
+    console.log(`Retrying in ${formatSeconds(delay)} (retry ${attempt} of ${retries}; configured base delay: ${formatSeconds(retryDelay)}).`);
+}
+function finalRequestError(attempts, retries, totalDelay, failure) {
+    const exhausted = failure.retryable && attempts === retries + 1;
+    return `loadOpenStreetMap(): request failed after ${attempts} ${attempts === 1 ? "attempt" : "attempts"}.\n${exhausted
+        ? retries === 0
+            ? "No retries were configured."
+            : `All ${retries} configured ${retries === 1 ? "retry was" : "retries were"} exhausted.`
+        : "No further retry was attempted because the failure is nonretryable."}\nTotal retry delay: ${formatSeconds(totalDelay)}.\n` +
+        `Last failure: ${failure.reason}\n` +
+        `No further retry occurred because ${exhausted
+            ? retries === 0
+                ? "no retries were configured"
+                : "the configured retries were exhausted"
+            : failure.explanation}.\n` +
+        "No invalid cached data was returned.";
+}
+function formatSeconds(seconds) {
+    return `${seconds} ${seconds === 1 ? "second" : "seconds"}`;
+}
+export function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} ${bytes === 1 ? "byte" : "bytes"}`;
+    const units = ["KB", "MB", "GB", "TB"];
+    let value = bytes;
+    let unit = units[0];
+    for (const candidate of units) {
+        value /= 1024;
+        unit = candidate;
+        if (value < 1024)
+            break;
+    }
+    return `${value.toFixed(1)} ${unit}`;
+}
+function waitFor(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
 }

package/esm/methods/loadGeoData.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/loadGeoData.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAWvD,KAAK,kBAAkB,GAAG;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,kBAAuB,QAYjC"}
+{"version":3,"file":"loadGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/loadGeoData.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,KAAK,kBAAkB,GAAG;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,kBAAuB,QAiBjC"}

package/esm/methods/loadGeoData.js

@@ -5,12 +5,12 @@ import parseValue from "../helpers/parseValue.js";
 import queryDB from "../helpers/queryDB.js";
 import queueOp from "../helpers/queueOp.js";
 import removeColumnsNow from "../helpers/removeColumnsNow.js";
-import { rmSync } from "node:fs";
-import SDAError from "../class/SDAError.js";
-import loadOsmFile from "../helpers/loadOsmFile.js";
 import { recordCacheTableReferences } from "../helpers/cacheTableDependencies.js";
-import { downloadOsmToTemporaryFile, getOsmFileSuffix, isRemoteOsmUrl, } from "../helpers/osmFiles.js";
+import { getOsmFileSuffix } from "../helpers/osmFiles.js";
 export default function loadGeoData(simpleTable, file, options = {}) {
+    if (getOsmFileSuffix(file) !== null) {
+        throw new Error("loadGeoData() does not support .osm or .osm.pbf files. Use loadOpenStreetMap() instead.");
+    }
     options = structuredClone(options);
     if (options.conditions !== undefined) {
         recordCacheTableReferences(simpleTable, [options.conditions]);
@@ -23,41 +23,6 @@ export default function loadGeoData(simpleTable, file, options = {}) {
     });
 }
 async function executeLoadGeoData(simpleTable, file, options) {
-    const osmSuffix = getOsmFileSuffix(file);
-    if (osmSuffix !== null) {
-        let temporaryFile;
-        try {
-            const localFile = isRemoteOsmUrl(file)
-                ? temporaryFile = await downloadOsmToTemporaryFile(file, {
-                    suffix: osmSuffix,
-                    request: { headers: { "accept-encoding": "gzip" } },
-                })
-                : file;
-            await loadOsmFile(simpleTable, localFile, {
-                method: "loadGeoData()",
-                parameters: { file, options },
-                columns: options.columns,
-                conditions: options.conditions,
-            });
-            return;
-        }
-        catch (error) {
-            if (error instanceof SDAError) {
-                throw error;
-            }
-            throw new SDAError({
-                method: "loadGeoData()",
-                parameters: { file, options },
-                query: "",
-                cause: error,
-            });
-        }
-        finally {
-            if (temporaryFile !== undefined) {
-                rmSync(temporaryFile, { force: true });
-            }
-        }
-    }
     const fileExtension = getExtension(file);
     const selectColumns = options.columns !== undefined &&
         options.columns.length > 0

package/esm/methods/loadOpenStreetMap.d.ts

@@ -0,0 +1,24 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type BoundingBox = {
+    west: number;
+    south: number;
+    east: number;
+    north: number;
+};
+type CommonOptions = {
+    cache?: boolean;
+    ttl?: number;
+    retries?: number;
+    retryDelay?: number;
+    verbose?: boolean;
+};
+type LoadOpenStreetMapFileOptions = CommonOptions;
+type LoadOpenStreetMapOverpassOptions = CommonOptions & {
+    filters: string | [string, string] | [string, string][];
+    endpoint?: string;
+    timeout?: number;
+};
+export default function loadOpenStreetMap(table: SimpleTable, source: string | BoundingBox, options?: LoadOpenStreetMapFileOptions | LoadOpenStreetMapOverpassOptions): void;
+export declare function generateOverpassQuery(bbox: BoundingBox, filters: string[], timeout?: number): string;
+export {};
+//# sourceMappingURL=loadOpenStreetMap.d.ts.map

package/esm/methods/loadOpenStreetMap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadOpenStreetMap.d.ts","sourceRoot":"","sources":["../../src/methods/loadOpenStreetMap.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAqBvD,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,KAAK,4BAA4B,GAAG,aAAa,CAAC;AAElD,KAAK,gCAAgC,GAAG,aAAa,GAAG;IACtD,OAAO,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,GAAG,WAAW,EAC5B,OAAO,GACH,4BAA4B,GAC5B,gCAAqC,GACxC,IAAI,CAcN;AA+SD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE,MAAM,GACf,MAAM,CASR"}