@nshiab/simple-data-analysis-core 1.0.1 → 2.0.0

package/script/methods/loadGeoData.js

@@ -0,0 +1,110 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadGeoData;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const getExtension_js_1 = __importDefault(require("../helpers/getExtension.js"));
+const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
+const parseValue_js_1 = __importDefault(require("../helpers/parseValue.js"));
+const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const removeColumnsNow_js_1 = __importDefault(require("../helpers/removeColumnsNow.js"));
+const node_fs_1 = require("node:fs");
+const SDAError_js_1 = __importDefault(require("../class/SDAError.js"));
+const loadOsmFile_js_1 = __importDefault(require("../helpers/loadOsmFile.js"));
+const cacheTableDependencies_js_1 = require("../helpers/cacheTableDependencies.js");
+const osmFiles_js_1 = require("../helpers/osmFiles.js");
+function loadGeoData(simpleTable, file, options = {}) {
+    options = structuredClone(options);
+    if (options.conditions !== undefined) {
+        (0, cacheTableDependencies_js_1.recordCacheTableReferences)(simpleTable, [options.conditions]);
+    }
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "barrier",
+        method: "loadGeoData()",
+        parameters: { file, options },
+        execute: () => executeLoadGeoData(simpleTable, file, options),
+    });
+}
+async function executeLoadGeoData(simpleTable, file, options) {
+    const osmSuffix = (0, osmFiles_js_1.getOsmFileSuffix)(file);
+    if (osmSuffix !== null) {
+        let temporaryFile;
+        try {
+            const localFile = (0, osmFiles_js_1.isRemoteOsmUrl)(file)
+                ? temporaryFile = await (0, osmFiles_js_1.downloadOsmToTemporaryFile)(file, {
+                    suffix: osmSuffix,
+                    request: { headers: { "accept-encoding": "gzip" } },
+                })
+                : file;
+            await (0, loadOsmFile_js_1.default)(simpleTable, localFile, {
+                method: "loadGeoData()",
+                parameters: { file, options },
+                columns: options.columns,
+                conditions: options.conditions,
+            });
+            return;
+        }
+        catch (error) {
+            if (error instanceof SDAError_js_1.default) {
+                throw error;
+            }
+            throw new SDAError_js_1.default({
+                method: "loadGeoData()",
+                parameters: { file, options },
+                query: "",
+                cause: error,
+            });
+        }
+        finally {
+            if (temporaryFile !== undefined) {
+                (0, node_fs_1.rmSync)(temporaryFile, { force: true });
+            }
+        }
+    }
+    const fileExtension = (0, getExtension_js_1.default)(file);
+    const selectColumns = options.columns !== undefined &&
+        options.columns.length > 0
+        ? options.columns.map(quoteIdentifier_js_1.default).join(", ")
+        : "*";
+    const spatialSetup = simpleTable.sdb.spatialLoaded
+        ? ""
+        : "INSTALL spatial; LOAD spatial; SET geometry_always_xy = true;";
+    const conditions = options.conditions ? ` WHERE ${options.conditions}` : "";
+    if (fileExtension === "geoparquet" || fileExtension === "parquet") {
+        await (0, queryDB_js_1.default)(simpleTable, `${spatialSetup}${file.toLowerCase().includes("http") ? " INSTALL https; LOAD https;" : ""}
+      CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)} AS SELECT ${selectColumns} FROM read_parquet(${(0, parseValue_js_1.default)(file)})${conditions};`, (0, mergeOptions_js_1.default)(simpleTable, {
+            table: simpleTable.name,
+            method: "loadGeoData()",
+            parameters: { file, options },
+        }));
+    }
+    else {
+        await (0, queryDB_js_1.default)(simpleTable, `${spatialSetup}${file.toLowerCase().includes("http") ? " INSTALL https; LOAD https;" : ""}
+      CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)} AS SELECT ${selectColumns} FROM ST_Read(${(0, parseValue_js_1.default)(file)})${conditions};`, (0, mergeOptions_js_1.default)(simpleTable, {
+            table: simpleTable.name,
+            method: "loadGeoData()",
+            parameters: { file, options },
+        }));
+    }
+    simpleTable.sdb.spatialLoaded = true;
+    const mayIncludeOgcFid = options.columns === undefined ||
+        options.columns.length === 0 || options.columns.includes("OGC_FID");
+    if (mayIncludeOgcFid && await simpleTable.hasColumn("OGC_FID")) {
+        await (0, removeColumnsNow_js_1.default)(simpleTable, ["OGC_FID"], "loadGeoData()");
+    }
+    if (options.toEPSG4326) {
+        const geoType = await simpleTable.getProjection("geom");
+        if (geoType !== "GEOMETRY('EPSG:4326')") {
+            // The reprojection runs directly (not with the sync reproject builder,
+            // which would queue for the next flush).
+            await (0, queryDB_js_1.default)(simpleTable, `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)} AS SELECT * REPLACE (ST_Transform("geom", 'EPSG:4326')::GEOMETRY('EPSG:4326') AS "geom") FROM ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}`, (0, mergeOptions_js_1.default)(simpleTable, {
+                table: simpleTable.name,
+                method: "loadGeoData()",
+                parameters: { file, options },
+            }));
+        }
+    }
+}

package/script/methods/loadOSM.d.ts

@@ -0,0 +1,17 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type BoundingBox = {
+    west: number;
+    south: number;
+    east: number;
+    north: number;
+};
+type LoadOsmOptions = {
+    filters: string | [string, string] | [string, string][];
+    endpoint?: string;
+    timeout?: number;
+    cache?: boolean;
+};
+export default function loadOSM(table: SimpleTable, bbox: BoundingBox, options: LoadOsmOptions): void;
+export declare function generateOverpassQuery(bbox: BoundingBox, filters: string[], timeout?: number): string;
+export {};
+//# sourceMappingURL=loadOSM.d.ts.map

package/script/methods/loadOSM.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadOSM.d.ts","sourceRoot":"","sources":["../../src/methods/loadOSM.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AASvD,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,cAAc,GAAG;IACpB,OAAO,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,cAAc,GACtB,IAAI,CAkBN;AAgHD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE,MAAM,GACf,MAAM,CASR"}

package/script/methods/loadOSM.js

@@ -0,0 +1,201 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadOSM;
+exports.generateOverpassQuery = generateOverpassQuery;
+const node_fs_1 = require("node:fs");
+const SDAError_js_1 = __importDefault(require("../class/SDAError.js"));
+const loadOsmFile_js_1 = __importDefault(require("../helpers/loadOsmFile.js"));
+const osmCache_js_1 = require("../helpers/osmCache.js");
+const osmFiles_js_1 = require("../helpers/osmFiles.js");
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const DEFAULT_OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter";
+const QUERY_GENERATOR_VERSION = "overpass-ql-v1";
+function loadOSM(table, bbox, options) {
+    bbox = normalizeBoundingBox(bbox);
+    options = structuredClone(options);
+    const endpoint = normalizeEndpoint(options.endpoint ?? DEFAULT_OVERPASS_ENDPOINT);
+    const filters = normalizeFilters(options.filters);
+    assertTimeout(options.timeout);
+    const query = generateOverpassQuery(bbox, filters, options.timeout);
+    const parameters = { bbox, options };
+    (0, queueOp_js_1.default)(table, {
+        kind: "barrier",
+        method: "loadOSM()",
+        parameters,
+        execute: () => executeLoadOSM(table, bbox, options, endpoint, query, parameters),
+    });
+}
+async function executeLoadOSM(table, bbox, options, endpoint, query, parameters) {
+    try {
+        const cacheEnabled = options.cache ?? true;
+        const cacheId = (0, osmCache_js_1.createOsmCacheId)({
+            endpoint,
+            bbox,
+            query,
+            queryGeneratorVersion: QUERY_GENERATOR_VERSION,
+        });
+        await (0, osmCache_js_1.useProcessedOsmCache)(table, cacheId, cacheEnabled, parameters, async () => {
+            const timeoutSignal = options.timeout === undefined
+                ? undefined
+                : AbortSignal.timeout(options.timeout * 1000);
+            const body = new URLSearchParams({ data: query });
+            const temporaryFile = await (0, osmFiles_js_1.downloadOsmToTemporaryFile)(endpoint, {
+                suffix: ".osm",
+                prefix: cacheId,
+                request: {
+                    method: "POST",
+                    headers: {
+                        "accept-encoding": "gzip",
+                        "content-type": "application/x-www-form-urlencoded;charset=UTF-8",
+                        "user-agent": "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
+                    },
+                    body,
+                    signal: timeoutSignal,
+                },
+            });
+            try {
+                await assertSuccessfulOverpassResponse(temporaryFile);
+                await (0, loadOsmFile_js_1.default)(table, temporaryFile, {
+                    method: "loadOSM()",
+                    parameters,
+                });
+            }
+            finally {
+                (0, node_fs_1.rmSync)(temporaryFile, { force: true });
+            }
+        });
+    }
+    catch (error) {
+        if (error instanceof SDAError_js_1.default) {
+            throw error;
+        }
+        throw new SDAError_js_1.default({
+            method: "loadOSM()",
+            parameters,
+            query,
+            cause: error,
+        });
+    }
+}
+async function assertSuccessfulOverpassResponse(file) {
+    const stream = (0, node_fs_1.createReadStream)(file, { encoding: "utf-8" });
+    let buffer = "";
+    let errorTag;
+    for await (const chunk of stream) {
+        buffer += String(chunk);
+        if (errorTag === undefined) {
+            const openingTag = /<(remark|error)\b[^>]*>/i.exec(buffer);
+            if (openingTag === null) {
+                buffer = buffer.slice(-4096);
+                continue;
+            }
+            errorTag = openingTag[1].toLowerCase() === "remark" ? "remark" : "error";
+            buffer = buffer.slice(openingTag.index + openingTag[0].length);
+        }
+        const closingTag = new RegExp(`</${errorTag}\\s*>`, "i").exec(buffer);
+        if (closingTag !== null) {
+            throwOverpassResponseError(buffer.slice(0, closingTag.index));
+        }
+        if (buffer.length >= 4096) {
+            throwOverpassResponseError(buffer.slice(0, 4096));
+        }
+    }
+    if (errorTag !== undefined) {
+        throwOverpassResponseError(buffer);
+    }
+}
+function throwOverpassResponseError(contents) {
+    const message = contents
+        .replace(/<[^>]*>/g, " ")
+        .replace(/\s+/g, " ")
+        .trim()
+        .slice(0, 500);
+    throw new Error(message.length === 0
+        ? "The Overpass endpoint returned an error response."
+        : `The Overpass endpoint returned an error response: ${message}`);
+}
+function generateOverpassQuery(bbox, filters, timeout) {
+    const box = `${bbox.south},${bbox.west},${bbox.north},${bbox.east}`;
+    const selections = filters.map((filter) => `nwr${filter}(${box});`);
+    const selection = selections.length === 1
+        ? selections[0]
+        : `(${selections.join("")});`;
+    return `[out:xml]${timeout === undefined ? "" : `[timeout:${timeout}]`};${selection}(._;>;);out body;`;
+}
+function normalizeBoundingBox(bbox) {
+    const normalized = structuredClone(bbox);
+    for (const [name, value] of Object.entries(normalized)) {
+        if (!Number.isFinite(value)) {
+            throw new Error(`loadOSM() bbox.${name} must be a finite number.`);
+        }
+    }
+    if (normalized.west < -180 || normalized.west > 180) {
+        throw new Error("loadOSM() bbox.west must be between -180 and 180.");
+    }
+    if (normalized.east < -180 || normalized.east > 180) {
+        throw new Error("loadOSM() bbox.east must be between -180 and 180.");
+    }
+    if (normalized.south < -90 || normalized.south > 90) {
+        throw new Error("loadOSM() bbox.south must be between -90 and 90.");
+    }
+    if (normalized.north < -90 || normalized.north > 90) {
+        throw new Error("loadOSM() bbox.north must be between -90 and 90.");
+    }
+    if (normalized.west >= normalized.east) {
+        throw new Error("loadOSM() bbox.west must be less than bbox.east.");
+    }
+    if (normalized.south >= normalized.north) {
+        throw new Error("loadOSM() bbox.south must be less than bbox.north.");
+    }
+    normalized.west = Object.is(normalized.west, -0) ? 0 : normalized.west;
+    normalized.south = Object.is(normalized.south, -0) ? 0 : normalized.south;
+    normalized.east = Object.is(normalized.east, -0) ? 0 : normalized.east;
+    normalized.north = Object.is(normalized.north, -0) ? 0 : normalized.north;
+    return normalized;
+}
+function normalizeFilters(filters) {
+    if (typeof filters === "string") {
+        const filter = filters.trim();
+        if (filter.length === 0) {
+            throw new Error("loadOSM() filters must contain at least one non-empty filter fragment.");
+        }
+        return [filter];
+    }
+    const tuples = isFilterTuple(filters) ? [filters] : filters;
+    const normalized = tuples.map(([key, value]) => {
+        if (key.length === 0) {
+            throw new Error("loadOSM() filter keys must not be empty.");
+        }
+        return `[${JSON.stringify(key)}=${JSON.stringify(value)}]`;
+    });
+    if (normalized.length === 0) {
+        throw new Error("loadOSM() filters must contain at least one filter tuple.");
+    }
+    return normalized;
+}
+function isFilterTuple(filters) {
+    return filters.length === 2 && typeof filters[0] === "string" &&
+        typeof filters[1] === "string";
+}
+function normalizeEndpoint(endpoint) {
+    let url;
+    try {
+        url = new URL(endpoint);
+    }
+    catch {
+        throw new Error("loadOSM() endpoint must be a valid HTTP or HTTPS URL.");
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        throw new Error("loadOSM() endpoint must be an HTTP or HTTPS URL.");
+    }
+    return url.toString();
+}
+function assertTimeout(timeout) {
+    if (timeout !== undefined &&
+        (!Number.isInteger(timeout) || timeout <= 0)) {
+        throw new Error("loadOSM() timeout must be a positive integer in seconds.");
+    }
+}

package/script/methods/loadSample.d.ts

@@ -5,5 +5,5 @@ import type SimpleTable from "../class/SimpleTable.js";
  * @param table - The SimpleTable instance to load the data into.
  * @param sample - The name of the sample to load.
  */
-export default function loadSample(table: SimpleTable, sample: "fires" | "recipes" | "temperatures" | "temperaturesCities" | "canada" | "firesGeo"): Promise<SimpleTable>;
+export default function loadSample(table: SimpleTable, sample: "fires" | "recipes" | "temperatures" | "temperaturesCities" | "canada" | "firesGeo"): SimpleTable;
 //# sourceMappingURL=loadSample.d.ts.map

package/script/methods/loadSample.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadSample.d.ts","sourceRoot":"","sources":["../../src/methods/loadSample.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD;;;;;GAKG;AACH,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EACF,OAAO,GACP,SAAS,GACT,cAAc,GACd,oBAAoB,GACpB,QAAQ,GACR,UAAU,GACb,OAAO,CAAC,WAAW,CAAC,CA6CtB"}
+{"version":3,"file":"loadSample.d.ts","sourceRoot":"","sources":["../../src/methods/loadSample.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,CAAC,OAAO,UAAU,UAAU,CAChC,KAAK,EAAE,WAAW,EAClB,MAAM,EACF,OAAO,GACP,SAAS,GACT,cAAc,GACd,oBAAoB,GACpB,QAAQ,GACR,UAAU,GACb,WAAW,CAoDb"}

package/script/methods/loadSample.js

@@ -7,7 +7,7 @@ exports.default = loadSample;
  * @param table - The SimpleTable instance to load the data into.
  * @param sample - The name of the sample to load.
  */
-async function loadSample(table, sample) {
+function loadSample(table, sample) {
     const samples = {
         fires: {
             url: "https://raw.githubusercontent.com/nshiab/simple-data-analysis-core/refs/heads/main/test/geodata/files/firesCanada2023.csv",
@@ -35,13 +35,14 @@ async function loadSample(table, sample) {
         },
     };
     const sampleToLoad = samples[sample];
+    // This validation doesn't need the database, so it stays at call time.
     if (!sampleToLoad) {
-        throw new Error(`Unknown sample: ${sample}`);
+        throw new Error(`loadSample() does not recognize sample ${JSON.stringify(sample)}. Available samples: ${Object.keys(samples).map((name) => JSON.stringify(name)).join(", ")}.`);
     }
     if (sampleToLoad.geo) {
-        return await table.loadGeoData(sampleToLoad.url);
+        return table.loadGeoData(sampleToLoad.url);
     }
     else {
-        return await table.loadData(sampleToLoad.url);
+        return table.loadData(sampleToLoad.url);
     }
 }

package/script/methods/loadStatCanData.d.ts

@@ -0,0 +1,10 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type LoadStatCanOptions = {
+    lang?: "en" | "fr";
+    cache?: boolean;
+    ttl?: number;
+};
+export default function loadStatCanData(table: SimpleTable, pid: string, options?: LoadStatCanOptions): void;
+export declare function normalizeStatCanPid(pid: string): string;
+export {};
+//# sourceMappingURL=loadStatCanData.d.ts.map

package/script/methods/loadStatCanData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadStatCanData.d.ts","sourceRoot":"","sources":["../../src/methods/loadStatCanData.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAYvD,KAAK,kBAAkB,GAAG;IACxB,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAYF,MAAM,CAAC,OAAO,UAAU,eAAe,CACrC,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,kBAAuB,GAC/B,IAAI,CAcN;AAsID,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAWvD"}

package/script/methods/loadStatCanData.js

@@ -0,0 +1,137 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadStatCanData;
+exports.normalizeStatCanPid = normalizeStatCanPid;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_fs_1 = require("node:fs");
+const promises_1 = require("node:stream/promises");
+const node_stream_1 = require("node:stream");
+const SDAError_js_1 = __importDefault(require("../class/SDAError.js"));
+const extractZipEntryToFile_js_1 = __importDefault(require("../helpers/extractZipEntryToFile.js"));
+const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
+const loadData_js_1 = require("./loadData.js");
+const statCanCache_js_1 = require("../helpers/statCanCache.js");
+const DOWNLOAD_ENDPOINT = "https://www150.statcan.gc.ca/t1/wds/rest/getFullTableDownloadCSV";
+// Adapted from getStatCanTable() in @nshiab/journalism-web-scraping.
+function loadStatCanData(table, pid, options = {}) {
+    pid = normalizeStatCanPid(pid);
+    options = structuredClone(options);
+    assertOptions(options);
+    const lang = options.lang ?? "en";
+    const parameters = { pid, options };
+    (0, queueOp_js_1.default)(table, {
+        kind: "barrier",
+        method: "loadStatCanData()",
+        parameters,
+        execute: () => executeLoadStatCanData(table, pid, lang, options, parameters),
+    });
+}
+async function executeLoadStatCanData(table, pid, lang, options, parameters) {
+    const cacheEnabled = options.cache ?? true;
+    const cacheId = (0, statCanCache_js_1.createStatCanCacheId)(pid, lang);
+    try {
+        await (0, statCanCache_js_1.useStatCanCache)(table, cacheId, cacheEnabled, options.ttl, parameters, async () => {
+            const downloadUrl = await getDownloadUrl(pid, lang);
+            const temporaryDirectory = ".sda-cache/tmp";
+            (0, node_fs_1.mkdirSync)(temporaryDirectory, { recursive: true });
+            const prefix = `${cacheId}.${node_crypto_1.default.randomUUID()}`;
+            const archive = `${temporaryDirectory}/${prefix}.zip`;
+            const csv = `${temporaryDirectory}/${prefix}.csv`;
+            try {
+                await downloadToFile(downloadUrl, archive);
+                await (0, extractZipEntryToFile_js_1.default)(archive, `${pid}.csv`, csv);
+                await (0, queryDB_js_1.default)(table, (0, loadData_js_1.loadDataQuery)(table.name, [csv], { fileType: "csv" }), (0, mergeOptions_js_1.default)(table, {
+                    table: table.name,
+                    method: "loadStatCanData()",
+                    parameters,
+                }));
+            }
+            finally {
+                (0, node_fs_1.rmSync)(archive, { force: true });
+                (0, node_fs_1.rmSync)(csv, { force: true });
+            }
+        });
+    }
+    catch (error) {
+        if (error instanceof SDAError_js_1.default) {
+            throw error;
+        }
+        throw new SDAError_js_1.default({
+            method: "loadStatCanData()",
+            parameters,
+            query: "",
+            cause: error,
+        });
+    }
+}
+async function getDownloadUrl(pid, lang) {
+    const url = `${DOWNLOAD_ENDPOINT}/${pid}/${lang}`;
+    const response = await fetch(url, {
+        headers: {
+            "user-agent": "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
+        },
+    });
+    if (!response.ok) {
+        throw await httpError("retrieve the Statistics Canada download URL", url, response);
+    }
+    const result = await response.json();
+    if (result.status !== "SUCCESS" || typeof result.object !== "string") {
+        throw new Error(`Statistics Canada did not return a CSV download URL for PID ${pid}.`);
+    }
+    const downloadUrl = new URL(result.object);
+    if (downloadUrl.protocol !== "http:" && downloadUrl.protocol !== "https:") {
+        throw new Error(`Statistics Canada returned an invalid CSV download URL for PID ${pid}.`);
+    }
+    return downloadUrl.href;
+}
+async function downloadToFile(url, file) {
+    const response = await fetch(url, {
+        headers: {
+            "user-agent": "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
+        },
+    });
+    if (!response.ok) {
+        throw await httpError("download Statistics Canada data", url, response);
+    }
+    if (response.body === null) {
+        throw new Error(`Could not download Statistics Canada data from ${JSON.stringify(url)}: the response body is empty.`);
+    }
+    try {
+        await (0, promises_1.pipeline)(node_stream_1.Readable.fromWeb(response.body), (0, node_fs_1.createWriteStream)(file, { flags: "wx" }));
+    }
+    catch (error) {
+        (0, node_fs_1.rmSync)(file, { force: true });
+        throw error;
+    }
+}
+async function httpError(action, url, response) {
+    const responseText = (await response.text()).slice(0, 500).trim();
+    return new Error(`Could not ${action} from ${JSON.stringify(url)}: HTTP ${response.status} ${response.statusText}${responseText.length > 0 ? `\n${responseText}` : ""}`);
+}
+function normalizeStatCanPid(pid) {
+    if (typeof pid !== "string") {
+        throw new TypeError("loadStatCanData() pid must be a string.");
+    }
+    const normalized = pid.replaceAll("-", "");
+    if (!/^\d{8}(?:\d{2})?$/.test(normalized)) {
+        throw new Error('loadStatCanData() pid must contain 8 or 10 digits, optionally formatted like "36-10-0402-01".');
+    }
+    return normalized.slice(0, 8);
+}
+function assertOptions(options) {
+    if (options.lang !== undefined && options.lang !== "en" && options.lang !== "fr") {
+        throw new Error('loadStatCanData() options.lang must be "en" or "fr".');
+    }
+    if (options.cache !== undefined && typeof options.cache !== "boolean") {
+        throw new TypeError("loadStatCanData() options.cache must be a boolean.");
+    }
+    if (options.ttl !== undefined &&
+        (!Number.isFinite(options.ttl) || options.ttl < 0)) {
+        throw new Error("loadStatCanData() options.ttl must be a finite, non-negative number of seconds.");
+    }
+}

package/script/methods/longer.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function longer(simpleTable: SimpleTable, columns: string[], namesTo: string, valuesTo: string): void;
+//# sourceMappingURL=longer.d.ts.map

package/script/methods/longer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"longer.d.ts","sourceRoot":"","sources":["../../src/methods/longer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,MAAM,CAC5B,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,QAuBjB"}

package/script/methods/longer.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = longer;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const assertNewColumns_js_1 = __importDefault(require("../helpers/assertNewColumns.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+function longer(simpleTable, columns, namesTo, valuesTo) {
+    columns = [...columns];
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "fusable",
+        method: "longer()",
+        parameters: { columns, namesTo, valuesTo },
+        needsSchema: true,
+        buildSelect: (input, schema) => {
+            // The pivoted columns are consumed by UNPIVOT and don't appear in the
+            // output, so namesTo/valuesTo may reuse one of those names.
+            (0, assertNewColumns_js_1.default)(schema, [namesTo, valuesTo], "longer()", columns);
+            return `SELECT * FROM (
+            FROM ${input} UNPIVOT INCLUDE NULLS (
+            ${(0, quoteIdentifier_js_1.default)(valuesTo)}
+            for ${(0, quoteIdentifier_js_1.default)(namesTo)} in (${columns.map((d) => `${(0, quoteIdentifier_js_1.default)(d)}`).join(", ")})
+            )
+        )`;
+        },
+    });
+}

package/script/methods/lower.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function lower(simpleTable: SimpleTable, columns: string | string[]): void;
+//# sourceMappingURL=lower.d.ts.map

package/script/methods/lower.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lower.d.ts","sourceRoot":"","sources":["../../src/methods/lower.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,KAAK,CAC3B,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,QAc3B"}

package/script/methods/lower.js

@@ -0,0 +1,20 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = lower;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const stringToArray_js_1 = __importDefault(require("../helpers/stringToArray.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+function lower(simpleTable, columns) {
+    const cols = (0, stringToArray_js_1.default)(columns);
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "fusable",
+        method: "lower()",
+        parameters: { columns },
+        needsSchema: false,
+        buildSelect: (input) => `SELECT * REPLACE (${cols.map((c) => `LOWER(${(0, quoteIdentifier_js_1.default)(c)}) AS ${(0, quoteIdentifier_js_1.default)(c)}`)
+            .join(", ")}) FROM ${input}`,
+    });
+}

package/script/methods/nest.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function nest(simpleTable: SimpleTable, column: string, separator: string, by: string | string[]): void;
+//# sourceMappingURL=nest.d.ts.map

package/script/methods/nest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nest.d.ts","sourceRoot":"","sources":["../../src/methods/nest.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,QAyBtB"}

package/script/methods/nest.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = nest;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const stringToArray_js_1 = __importDefault(require("../helpers/stringToArray.js"));
+function nest(simpleTable, column, separator, by) {
+    by = Array.isArray(by) ? [...by] : by;
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "fusable",
+        method: "nest()",
+        parameters: { column, separator, by },
+        values: [separator],
+        needsSchema: false,
+        buildSelect: (input) => {
+            const cats = (0, stringToArray_js_1.default)(by);
+            const groupBy = cats.map((d) => `${(0, quoteIdentifier_js_1.default)(d)}`).join(", ");
+            const selectColumns = `${groupBy}, `;
+            const orderBy = `\nORDER BY ${cats.map((d) => `${(0, quoteIdentifier_js_1.default)(d)} ASC`).join(", ")}`;
+            return `SELECT
+  ${selectColumns}STRING_AGG(${(0, quoteIdentifier_js_1.default)(column)}, ?) AS ${(0, quoteIdentifier_js_1.default)(column)}
+FROM ${input}
+GROUP BY ${groupBy}${orderBy}`;
+        },
+    });
+}

package/script/methods/normalize.d.ts

@@ -0,0 +1,7 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function normalize(simpleTable: SimpleTable, column: string, newColumn: string, options?: {
+    by?: string | string[];
+    decimals?: number;
+    range?: [number, number];
+}): void;
+//# sourceMappingURL=normalize.d.ts.map

package/script/methods/normalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../src/methods/normalize.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrB,QAkDP"}