@nshiab/simple-data-analysis-core 1.0.0 → 2.0.0

package/script/helpers/index.js

@@ -1,4 +1,9 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stringToArray = exports.quoteIdentifier = exports.queueAsyncBarrier = exports.queryDB = exports.parseValue = exports.mergeOptions = exports.createDirectory = void 0;
 /**
  * Helper utilities for Simple Data Analysis Core.
  *
@@ -13,11 +18,7 @@
  * const arr = stringToArray("single");
  * ```
  */
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.stringToArray = exports.queryDB = exports.parseValue = exports.mergeOptions = exports.createDirectory = void 0;
+require("../_dnt.polyfills.js");
 var createDirectory_js_1 = require("./createDirectory.js");
 Object.defineProperty(exports, "createDirectory", { enumerable: true, get: function () { return __importDefault(createDirectory_js_1).default; } });
 var mergeOptions_js_1 = require("./mergeOptions.js");
@@ -26,5 +27,9 @@ var parseValue_js_1 = require("./parseValue.js");
 Object.defineProperty(exports, "parseValue", { enumerable: true, get: function () { return __importDefault(parseValue_js_1).default; } });
 var queryDB_js_1 = require("./queryDB.js");
 Object.defineProperty(exports, "queryDB", { enumerable: true, get: function () { return __importDefault(queryDB_js_1).default; } });
+var queueAsyncBarrier_js_1 = require("./queueAsyncBarrier.js");
+Object.defineProperty(exports, "queueAsyncBarrier", { enumerable: true, get: function () { return __importDefault(queueAsyncBarrier_js_1).default; } });
+var quoteIdentifier_js_1 = require("./quoteIdentifier.js");
+Object.defineProperty(exports, "quoteIdentifier", { enumerable: true, get: function () { return __importDefault(quoteIdentifier_js_1).default; } });
 var stringToArray_js_1 = require("./stringToArray.js");
 Object.defineProperty(exports, "stringToArray", { enumerable: true, get: function () { return __importDefault(stringToArray_js_1).default; } });

package/script/helpers/indexDefinitions.d.ts

@@ -0,0 +1,30 @@
+/** The metadata required to identify and recreate a DuckDB HNSW index. */
+export type VssIndexDefinition = {
+    kind: "vss";
+    name: string;
+    column: string;
+    options: {
+        efConstruction?: number;
+        efSearch?: number;
+        M?: number;
+    };
+};
+/** The metadata required to identify and recreate a DuckDB full-text index. */
+export type FtsIndexDefinition = {
+    kind: "fts";
+    name: string;
+    idColumn: string;
+    textColumn: string;
+    options: {
+        stemmer?: string;
+        stopwords?: string;
+        ignore?: string;
+        stripAccents?: boolean;
+        lower?: boolean;
+    };
+};
+/** A complete index definition stored by a SimpleTable. */
+export type IndexDefinition = VssIndexDefinition | FtsIndexDefinition;
+export type IndexCacheStrategy = "persist" | "rebuild";
+export declare function getIndexCacheStrategy(index: IndexDefinition): IndexCacheStrategy;
+//# sourceMappingURL=indexDefinitions.d.ts.map

package/script/helpers/indexDefinitions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexDefinitions.d.ts","sourceRoot":"","sources":["../../src/helpers/indexDefinitions.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,KAAK,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE;QACP,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,CAAC,CAAC,EAAE,MAAM,CAAC;KACZ,CAAC;CACH,CAAC;AAEF,+EAA+E;AAC/E,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,KAAK,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE;QACP,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;CACH,CAAC;AAEF,2DAA2D;AAC3D,MAAM,MAAM,eAAe,GAAG,kBAAkB,GAAG,kBAAkB,CAAC;AAEtE,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,SAAS,CAAC;AAUvD,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,eAAe,GACrB,kBAAkB,CAEpB"}

package/script/helpers/indexDefinitions.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getIndexCacheStrategy = getIndexCacheStrategy;
+const INDEX_CACHE_STRATEGIES = {
+    fts: "persist",
+    vss: "rebuild",
+};
+function getIndexCacheStrategy(index) {
+    return INDEX_CACHE_STRATEGIES[index.kind];
+}

package/script/helpers/indexQueries.d.ts

@@ -0,0 +1,3 @@
+import type { IndexDefinition } from "./indexDefinitions.js";
+export default function buildCreateIndexQuery(table: string, index: IndexDefinition): string;
+//# sourceMappingURL=indexQueries.d.ts.map

package/script/helpers/indexQueries.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexQueries.d.ts","sourceRoot":"","sources":["../../src/helpers/indexQueries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAI7D,MAAM,CAAC,OAAO,UAAU,qBAAqB,CAC3C,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,eAAe,GACrB,MAAM,CASR"}

package/script/helpers/indexQueries.js

@@ -0,0 +1,42 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildCreateIndexQuery;
+const parseValue_js_1 = __importDefault(require("./parseValue.js"));
+const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
+function buildCreateIndexQuery(table, index) {
+    switch (index.kind) {
+        case "fts":
+            return buildFtsIndexQuery(table, index);
+        case "vss":
+            return buildVssIndexQuery(table, index);
+        default:
+            return assertNever(index);
+    }
+}
+function assertNever(value) {
+    throw new Error(`Unsupported index definition: ${JSON.stringify(value)}`);
+}
+function buildVssIndexQuery(table, index) {
+    const withOptions = ["metric = 'cosine'"];
+    if (index.options.efConstruction !== undefined) {
+        withOptions.push(`ef_construction = ${index.options.efConstruction}`);
+    }
+    if (index.options.efSearch !== undefined) {
+        withOptions.push(`ef_search = ${index.options.efSearch}`);
+    }
+    if (index.options.M !== undefined) {
+        withOptions.push(`M = ${index.options.M}`);
+    }
+    return `CREATE INDEX ${(0, quoteIdentifier_js_1.default)(index.name)} ON ${(0, quoteIdentifier_js_1.default)(table)}
+    USING HNSW (${(0, quoteIdentifier_js_1.default)(index.column)})
+    WITH (${withOptions.join(", ")});`;
+}
+function buildFtsIndexQuery(table, index) {
+    const options = index.options;
+    return `PRAGMA create_fts_index(${(0, quoteIdentifier_js_1.default)(table)}, ${(0, quoteIdentifier_js_1.default)(index.idColumn)}, ${(0, quoteIdentifier_js_1.default)(index.textColumn)}${options.stemmer ? `, stemmer = ${(0, parseValue_js_1.default)(options.stemmer)}` : ""}${options.stopwords ? `, stopwords = ${(0, parseValue_js_1.default)(options.stopwords)}` : ""}${options.ignore ? `, ignore = ${(0, parseValue_js_1.default)(options.ignore)}` : ""}${options.stripAccents !== undefined
+        ? `, strip_accents = ${options.stripAccents ? 1 : 0}`
+        : ""}${options.lower !== undefined ? `, lower = ${options.lower ? 1 : 0}` : ""});`;
+}

package/script/helpers/keepNumericalColumns.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"keepNumericalColumns.d.ts","sourceRoot":"","sources":["../../src/helpers/keepNumericalColumns.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,oBAAoB,CAAC,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,YAc5E"}
+{"version":3,"file":"keepNumericalColumns.d.ts","sourceRoot":"","sources":["../../src/helpers/keepNumericalColumns.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,oBAAoB,CAAC,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,YAa5E"}

package/script/helpers/keepNumericalColumns.js

@@ -4,13 +4,12 @@ exports.default = keepNumericalColumns;
 function keepNumericalColumns(types) {
     const columns = [];
     for (const col of Object.keys(types)) {
-        if (["FLOAT", "DOUBLE", "DECIMAL"].includes(types[col]) ||
-            types[col].includes("INT")) {
+        const type = types[col];
+        if (["FLOAT", "DOUBLE", "REAL"].includes(type) ||
+            type.startsWith("DECIMAL") ||
+            type.includes("INT")) {
             columns.push(col);
         }
     }
-    if (columns.length === 0) {
-        throw new Error("No numerical columns");
-    }
     return columns;
 }

package/script/helpers/loadOsmFile.d.ts

@@ -0,0 +1,10 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function loadOsmFile(table: SimpleTable, file: string, context: {
+    method: "loadGeoData()" | "loadOSM()";
+    parameters: {
+        [key: string]: unknown;
+    };
+    columns?: string[];
+    conditions?: string;
+}): Promise<void>;
+//# sourceMappingURL=loadOsmFile.d.ts.map

package/script/helpers/loadOsmFile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadOsmFile.d.ts","sourceRoot":"","sources":["../../src/helpers/loadOsmFile.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,WAAW,CACvC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE;IACP,MAAM,EAAE,eAAe,GAAG,WAAW,CAAC;IACtC,UAAU,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GACA,OAAO,CAAC,IAAI,CAAC,CA2Bf"}

package/script/helpers/loadOsmFile.js

@@ -0,0 +1,34 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadOsmFile;
+const mergeOptions_js_1 = __importDefault(require("./mergeOptions.js"));
+const parseValue_js_1 = __importDefault(require("./parseValue.js"));
+const queryDB_js_1 = __importDefault(require("./queryDB.js"));
+const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
+async function loadOsmFile(table, file, context) {
+    const selectColumns = context.columns !== undefined &&
+        context.columns.length > 0
+        ? context.columns.map(quoteIdentifier_js_1.default).join(", ")
+        : "* EXCLUDE (geometry)";
+    const conditions = context.conditions ? ` WHERE ${context.conditions}` : "";
+    // Expose geom before filtering, even when it is not selected for output.
+    await (0, queryDB_js_1.default)(table, `INSTALL spatial;
+    LOAD spatial;
+    INSTALL osmium FROM community;
+    LOAD osmium;
+    SET geometry_always_xy = true;
+    CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(table.name)} AS
+    SELECT ${selectColumns}
+    FROM (
+      SELECT *, geometry::GEOMETRY('EPSG:4326') AS geom
+      FROM osmium_read(${(0, parseValue_js_1.default)(file)})
+    )${conditions};`, (0, mergeOptions_js_1.default)(table, {
+        table: table.name,
+        method: context.method,
+        parameters: context.parameters,
+    }));
+    table.sdb.spatialLoaded = true;
+}

package/script/helpers/logData.d.ts

@@ -1,6 +1,6 @@
 export default function logData(types: {
     [key: string]: string;
 } | null, data: {
-    [key: string]: string | number | boolean | Date | null;
-}[] | null, nbCharactersToLog?: number): void;
+    [key: string]: unknown;
+}[] | null, charsToLog?: number): void;
 //# sourceMappingURL=logData.d.ts.map

package/script/helpers/logData.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"logData.d.ts","sourceRoot":"","sources":["../../src/helpers/logData.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAAG,IAAI,EACvC,IAAI,EACA;IACA,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD,EAAE,GACD,IAAI,EACR,iBAAiB,CAAC,EAAE,MAAM,QA4D3B"}
+{"version":3,"file":"logData.d.ts","sourceRoot":"","sources":["../../src/helpers/logData.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAAG,IAAI,EACvC,IAAI,EACA;IACA,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,EAAE,GACD,IAAI,EACR,UAAU,CAAC,EAAE,MAAM,QA4DpB"}

package/script/helpers/logData.js

@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = logData;
 const printTable_js_1 = __importDefault(require("./printTable.js"));
-function logData(types, data, nbCharactersToLog) {
+function logData(types, data, charsToLog) {
     if (data === null) {
         console.log("Data is null");
     }
@@ -19,10 +19,10 @@ function logData(types, data, nbCharactersToLog) {
             for (let i = 0; i < data.length; i++) {
                 const newItem = {};
                 for (const key of keys) {
-                    if (typeof nbCharactersToLog === "number" &&
+                    if (typeof charsToLog === "number" &&
                         typeof data[i][key] === "string" &&
-                        data[i][key].length > nbCharactersToLog) {
-                        newItem[key] = data[i][key].slice(0, nbCharactersToLog) + "...";
+                        data[i][key].length > charsToLog) {
+                        newItem[key] = data[i][key].slice(0, charsToLog) + "...";
                     }
                     else {
                         newItem[key] = data[i][key];
@@ -49,8 +49,8 @@ function logData(types, data, nbCharactersToLog) {
                 }
             }
             (0, printTable_js_1.default)(dataToBeLogged, {
-                ...(typeof nbCharactersToLog === "number"
-                    ? { maxColumnWidth: nbCharactersToLog }
+                ...(typeof charsToLog === "number"
+                    ? { maxColumnWidth: charsToLog }
                     : {}),
                 ...(hasTypesRow ? { typesRowIndex: 0 } : {}),
             });

package/script/helpers/mergeOptions.d.ts

@@ -1,28 +1,60 @@
-import type Simple from "../class/Simple.js";
-export default function mergeOptions(simple: Simple, options: {
+import type { DuckDBValue as NativeDuckDBValue } from "@duckdb/node-api";
+/** @internal DuckDB owns this type; retain its exact binding contract without re-exporting it. */
+type DuckDBValue = NativeDuckDBValue;
+/**
+ * Combines a table or database's logging defaults with query-specific options.
+ * Use this helper with `queryDB()` when implementing an extension.
+ *
+ * @param simple - The logging defaults from a table or database.
+ * @param simple.rowsToLog - The default number of rows to display.
+ * @param simple.charsToLog - The default maximum characters to display per cell.
+ * @param options - Query-specific settings, which take precedence over defaults.
+ * @param options.table - The table name used for diagnostics, or `null`.
+ * @param options.method - The calling method used for diagnostics, or `null`.
+ * @param options.parameters - Arguments included in error diagnostics, or `null`.
+ * @param options.rowsToLog - Overrides the default number of displayed rows.
+ * @param options.returnData - Whether to return converted rows. Defaults to false.
+ * @param options.values - DuckDB-native values bound to SQL placeholders, in order.
+ * @param options.noClean - Whether to preserve the SQL text without cleaning it.
+ * @returns The merged query options without modifying either argument.
+ *
+ * @example
+ * ```ts
+ * import { mergeOptions } from "@nshiab/simple-data-analysis-core/helpers";
+ *
+ * const options = mergeOptions(sdb, {
+ *   table: null,
+ *   method: "example()",
+ *   parameters: null,
+ *   returnData: true,
+ * });
+ * console.log(options.rowsToLog, options.returnData);
+ * ```
+ */
+export default function mergeOptions(simple: {
+    rowsToLog: number;
+    charsToLog: number | undefined;
+}, options: {
     table: string | null;
     method: string | null;
     parameters: {
         [key: string]: unknown;
     } | null;
-    nbRowsToLog?: number;
-    returnDataFrom?: "query" | "none";
-    debug?: boolean;
-    types?: {
-        [key: string]: string;
-    };
+    rowsToLog?: number;
+    returnData?: boolean;
+    values?: DuckDBValue[];
+    noClean?: boolean;
 }): {
     table: string | null;
     method: string | null;
     parameters: {
         [key: string]: unknown;
     } | null;
-    nbRowsToLog: number;
-    nbCharactersToLog: number | undefined;
-    returnDataFrom: "query" | "none";
-    debug: boolean;
-    types?: {
-        [key: string]: string;
-    };
+    rowsToLog: number;
+    charsToLog: number | undefined;
+    returnData: boolean;
+    values?: DuckDBValue[];
+    noClean?: boolean;
 };
+export {};
 //# sourceMappingURL=mergeOptions.d.ts.map

package/script/helpers/mergeOptions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mergeOptions.d.ts","sourceRoot":"","sources":["../../src/helpers/mergeOptions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,oBAAoB,CAAC;AAE7C,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;IACP,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAClC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;CACnC,GACA;IACD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,cAAc,EAAE,OAAO,GAAG,MAAM,CAAC;IACjC,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;CACnC,CAWA"}
+{"version":3,"file":"mergeOptions.d.ts","sourceRoot":"","sources":["../../src/helpers/mergeOptions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,IAAI,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEzE,kGAAkG;AAClG,KAAK,WAAW,GAAG,iBAAiB,CAAC;AAErC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,EAC7D,OAAO,EAAE;IACP,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GACA;IACD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAWA"}

package/script/helpers/mergeOptions.js

@@ -1,15 +1,45 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = mergeOptions;
+/**
+ * Combines a table or database's logging defaults with query-specific options.
+ * Use this helper with `queryDB()` when implementing an extension.
+ *
+ * @param simple - The logging defaults from a table or database.
+ * @param simple.rowsToLog - The default number of rows to display.
+ * @param simple.charsToLog - The default maximum characters to display per cell.
+ * @param options - Query-specific settings, which take precedence over defaults.
+ * @param options.table - The table name used for diagnostics, or `null`.
+ * @param options.method - The calling method used for diagnostics, or `null`.
+ * @param options.parameters - Arguments included in error diagnostics, or `null`.
+ * @param options.rowsToLog - Overrides the default number of displayed rows.
+ * @param options.returnData - Whether to return converted rows. Defaults to false.
+ * @param options.values - DuckDB-native values bound to SQL placeholders, in order.
+ * @param options.noClean - Whether to preserve the SQL text without cleaning it.
+ * @returns The merged query options without modifying either argument.
+ *
+ * @example
+ * ```ts
+ * import { mergeOptions } from "@nshiab/simple-data-analysis-core/helpers";
+ *
+ * const options = mergeOptions(sdb, {
+ *   table: null,
+ *   method: "example()",
+ *   parameters: null,
+ *   returnData: true,
+ * });
+ * console.log(options.rowsToLog, options.returnData);
+ * ```
+ */
 function mergeOptions(simple, options) {
     return {
         table: options.table,
         method: options.method,
         parameters: options.parameters,
-        nbRowsToLog: options.nbRowsToLog ?? simple.nbRowsToLog,
-        nbCharactersToLog: simple.nbCharactersToLog,
-        returnDataFrom: options.returnDataFrom ?? "none",
-        debug: options.debug ?? simple.debug,
-        types: options.types,
+        rowsToLog: options.rowsToLog ?? simple.rowsToLog,
+        charsToLog: simple.charsToLog,
+        returnData: options.returnData ?? false,
+        values: options.values,
+        noClean: options.noClean,
     };
 }

package/script/helpers/observeQuery.d.ts

@@ -0,0 +1,9 @@
+import type { DuckDBConnection, DuckDBValue } from "@duckdb/node-api";
+/**
+ * Logs optional SQL diagnostics without changing the statement that will run.
+ */
+export default function observeQuery(connection: DuckDBConnection, query: string, values: DuckDBValue[], options: {
+    logSQL: boolean;
+    explainSQL: boolean;
+}): Promise<void>;
+//# sourceMappingURL=observeQuery.d.ts.map

package/script/helpers/observeQuery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observeQuery.d.ts","sourceRoot":"","sources":["../../src/helpers/observeQuery.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAatE;;GAEG;AACH,wBAA8B,YAAY,CACxC,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,EAAE,EACrB,OAAO,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,OAAO,CAAA;CAAE,GAChD,OAAO,CAAC,IAAI,CAAC,CA2Bf"}

package/script/helpers/observeQuery.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = observeQuery;
+const explainableStatements = new Set([
+    "CREATE",
+    "DELETE",
+    "FROM",
+    "INSERT",
+    "MERGE",
+    "SELECT",
+    "UPDATE",
+    "WITH",
+]);
+/**
+ * Logs optional SQL diagnostics without changing the statement that will run.
+ */
+async function observeQuery(connection, query, values, options) {
+    if (options.explainSQL && await canExplain(connection, query)) {
+        try {
+            const explain = `EXPLAIN ${query}`;
+            const result = values.length === 0
+                ? await connection.run(explain)
+                : await connection.run(explain, values);
+            const rows = await result.getRowsJS();
+            const plan = rows.map((row) => String(row[1] ?? row[0] ?? "")).join("\n");
+            console.log(`EXPLAIN ${query}\n${plan}`);
+        }
+        catch (error) {
+            console.warn(`SDA: Could not explain the following SQL; executing it normally.\n${query}\n${String(error)}`);
+        }
+    }
+    // Keep this immediately before connection.run()/stream() so the last SQL
+    // diagnostic identifies the statement responsible for an execution error.
+    if (options.logSQL) {
+        console.log(query);
+        if (values.length > 0) {
+            console.log("Bound values:", values);
+        }
+    }
+}
+async function canExplain(connection, query) {
+    const keyword = query.trimStart().match(/^([A-Za-z]+)/)?.[1].toUpperCase();
+    if (keyword === undefined || !explainableStatements.has(keyword)) {
+        return false;
+    }
+    try {
+        return (await connection.extractStatements(query)).count === 1;
+    }
+    catch {
+        return false;
+    }
+}

package/script/helpers/osmCache.d.ts

@@ -0,0 +1,16 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export declare function createOsmCacheId(identity: {
+    endpoint: string;
+    bbox: {
+        west: number;
+        south: number;
+        east: number;
+        north: number;
+    };
+    query: string;
+    queryGeneratorVersion: string;
+}): string;
+export declare function useProcessedOsmCache(table: SimpleTable, id: string, enabled: boolean, parameters: {
+    [key: string]: unknown;
+}, compute: () => Promise<void>): Promise<void>;
+//# sourceMappingURL=osmCache.d.ts.map

package/script/helpers/osmCache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"osmCache.d.ts","sourceRoot":"","sources":["../../src/helpers/osmCache.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAUvD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACnE,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB,EAAE,MAAM,CAAC;CAC/B,GAAG,MAAM,CAET;AAED,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,OAAO,EAChB,UAAU,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EACtC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAC3B,OAAO,CAAC,IAAI,CAAC,CAsCf"}

package/script/helpers/osmCache.js

@@ -0,0 +1,81 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createOsmCacheId = createOsmCacheId;
+exports.useProcessedOsmCache = useProcessedOsmCache;
+const fileCache_js_1 = __importStar(require("./fileCache.js"));
+const mergeOptions_js_1 = __importDefault(require("./mergeOptions.js"));
+const parseValue_js_1 = __importDefault(require("./parseValue.js"));
+const queryDB_js_1 = __importDefault(require("./queryDB.js"));
+const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
+const CACHE_DIRECTORY = ".sda-cache/osm";
+const OSM_CACHE_FORMAT_VERSION = "osm-geoparquet-v1";
+function createOsmCacheId(identity) {
+    return (0, fileCache_js_1.createFileCacheId)("osm", identity);
+}
+async function useProcessedOsmCache(table, id, enabled, parameters, compute) {
+    await (0, fileCache_js_1.default)(table, {
+        directory: CACHE_DIRECTORY,
+        enabled,
+        extension: "geoparquet",
+        formatVersion: OSM_CACHE_FORMAT_VERSION,
+        id,
+        method: "loadOSM()",
+        load: async (file) => {
+            await (0, queryDB_js_1.default)(table, `INSTALL spatial;
+        LOAD spatial;
+        SET geometry_always_xy = true;
+        CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(table.name)} AS
+        SELECT * FROM read_parquet(${(0, parseValue_js_1.default)(file)});`, (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "loadOSM()",
+                parameters,
+            }));
+            table.sdb.spatialLoaded = true;
+        },
+        compute,
+        write: async (file) => {
+            await (0, queryDB_js_1.default)(table, `COPY ${(0, quoteIdentifier_js_1.default)(table.name)} TO ${(0, parseValue_js_1.default)(file)}
+        WITH (FORMAT PARQUET);`, (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "loadOSM()",
+                parameters,
+            }));
+        },
+    });
+}

package/script/helpers/osmFiles.d.ts

@@ -0,0 +1,9 @@
+export type OsmFileSuffix = ".osm" | ".osm.pbf";
+export declare function getOsmFileSuffix(file: string): OsmFileSuffix | null;
+export declare function isRemoteOsmUrl(file: string): boolean;
+export declare function downloadOsmToTemporaryFile(url: string, options: {
+    suffix: OsmFileSuffix;
+    prefix?: string;
+    request?: RequestInit;
+}): Promise<string>;
+//# sourceMappingURL=osmFiles.d.ts.map

package/script/helpers/osmFiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"osmFiles.d.ts","sourceRoot":"","sources":["../../src/helpers/osmFiles.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,UAAU,CAAC;AAEhD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAUnE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAOpD;AAED,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;IACP,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB,GACA,OAAO,CAAC,MAAM,CAAC,CAyCjB"}

package/script/helpers/osmFiles.js

@@ -0,0 +1,60 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getOsmFileSuffix = getOsmFileSuffix;
+exports.isRemoteOsmUrl = isRemoteOsmUrl;
+exports.downloadOsmToTemporaryFile = downloadOsmToTemporaryFile;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_fs_1 = require("node:fs");
+const promises_1 = require("node:fs/promises");
+const promises_2 = require("node:stream/promises");
+const node_stream_1 = require("node:stream");
+function getOsmFileSuffix(file) {
+    const comparable = isRemoteOsmUrl(file) ? new URL(file).pathname : file;
+    const lower = comparable.toLowerCase();
+    if (lower.endsWith(".osm.pbf")) {
+        return ".osm.pbf";
+    }
+    if (lower.endsWith(".osm")) {
+        return ".osm";
+    }
+    return null;
+}
+function isRemoteOsmUrl(file) {
+    try {
+        const protocol = new URL(file).protocol;
+        return protocol === "http:" || protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+}
+async function downloadOsmToTemporaryFile(url, options) {
+    const temporaryDirectory = ".sda-cache/tmp";
+    (0, node_fs_1.mkdirSync)(temporaryDirectory, { recursive: true });
+    const prefix = options.prefix === undefined ? "remote" : options.prefix;
+    const temporaryFile = `${temporaryDirectory}/${prefix}.${node_crypto_1.default.randomUUID()}${options.suffix}`;
+    const response = await fetch(url, options.request);
+    if (!response.ok) {
+        const responseText = (await response.text()).slice(0, 500).trim();
+        throw new Error(`Could not download OSM data from ${JSON.stringify(url)}: HTTP ${response.status} ${response.statusText}${responseText.length > 0 ? `\n${responseText}` : ""}`);
+    }
+    if (response.body === null) {
+        throw new Error(`Could not download OSM data from ${JSON.stringify(url)}: the response body is empty.`);
+    }
+    try {
+        await (0, promises_2.pipeline)(node_stream_1.Readable.fromWeb(response.body), (0, node_fs_1.createWriteStream)(temporaryFile, { flags: "wx" }));
+    }
+    catch (error) {
+        try {
+            await (0, promises_1.rm)(temporaryFile, { force: true });
+        }
+        catch {
+            // Preserve the download error.
+        }
+        throw error;
+    }
+    return temporaryFile;
+}

package/script/helpers/parseDuckDBType.d.ts

@@ -1,2 +1,2 @@
-export default function parseDuckDBType(type: string): import("@duckdb/node-api").DuckDBIntegerType | import("@duckdb/node-api").DuckDBBigIntType | import("@duckdb/node-api").DuckDBDoubleType | import("@duckdb/node-api").DuckDBVarCharType | import("@duckdb/node-api").DuckDBTimestampType | import("@duckdb/node-api").DuckDBTimestampTZType | import("@duckdb/node-api").DuckDBDateType | import("@duckdb/node-api").DuckDBTimeType | import("@duckdb/node-api").DuckDBBooleanType | import("@duckdb/node-api").DuckDBArrayType;
+export default function parseDuckDBType(type: string): import("@duckdb/node-api").DuckDBBooleanType | import("@duckdb/node-api").DuckDBIntegerType | import("@duckdb/node-api").DuckDBBigIntType | import("@duckdb/node-api").DuckDBDoubleType | import("@duckdb/node-api").DuckDBTimestampType | import("@duckdb/node-api").DuckDBDateType | import("@duckdb/node-api").DuckDBTimeType | import("@duckdb/node-api").DuckDBVarCharType | import("@duckdb/node-api").DuckDBArrayType | import("@duckdb/node-api").DuckDBTimestampTZType;
 //# sourceMappingURL=parseDuckDBType.d.ts.map