@nshiab/simple-data-analysis-core 1.0.0 → 2.0.0

package/script/methods/area.js

@@ -0,0 +1,31 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = area;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const assertNewColumns_js_1 = __importDefault(require("../helpers/assertNewColumns.js"));
+const findGeoColumnFromSchema_js_1 = __importDefault(require("../helpers/findGeoColumnFromSchema.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+function area(simpleTable, newColumn, options = {}) {
+    options = structuredClone(options);
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "fusable",
+        method: "area()",
+        parameters: { newColumn, options },
+        needsSchema: true,
+        needsSpatial: true,
+        buildSelect: (input, types) => {
+            (0, assertNewColumns_js_1.default)(types, [newColumn], "area()");
+            const column = typeof options.column === "string"
+                ? options.column
+                : (0, findGeoColumnFromSchema_js_1.default)(types, "area()", simpleTable.name);
+            let expression = `ST_Area_Spheroid(${(0, quoteIdentifier_js_1.default)(column)}) ${options.unit === "km2" ? "/ 1000000" : ""}`;
+            if (typeof options.decimals === "number") {
+                expression = `ROUND(${expression}, ${options.decimals})`;
+            }
+            return `SELECT *, CAST(${expression} AS DOUBLE) AS ${(0, quoteIdentifier_js_1.default)(newColumn)} FROM ${input}`;
+        },
+    });
+}

package/script/methods/bins.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function bins(simpleTable: SimpleTable, column: string, interval: number, newColumn: string, options?: {
+    startValue?: number;
+}): void;
+//# sourceMappingURL=bins.d.ts.map

package/script/methods/bins.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bins.d.ts","sourceRoot":"","sources":["../../src/methods/bins.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,QAiCP"}

package/script/methods/bins.js

@@ -0,0 +1,93 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = bins;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const assertNewColumns_js_1 = __importDefault(require("../helpers/assertNewColumns.js"));
+const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
+const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+function bins(simpleTable, column, interval, newColumn, options = {}) {
+    // This validation does not need the database, so it stays at call time.
+    if (!Number.isFinite(interval) || interval <= 0) {
+        throw new Error("bins() interval must be a finite number greater than 0.");
+    }
+    // The intervals depend on the minimum and maximum values of the data, so
+    // bins can't be expressed as a single SELECT over its input: it executes
+    // as a barrier.
+    options = structuredClone(options);
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "barrier",
+        method: "bins()",
+        parameters: { column, interval, newColumn, options },
+        execute: async () => {
+            await (0, queryDB_js_1.default)(simpleTable, await binsQuery(simpleTable, column, interval, newColumn, options), (0, mergeOptions_js_1.default)(simpleTable, {
+                table: simpleTable.name,
+                method: "bins()",
+                parameters: {
+                    column,
+                    interval,
+                    newColumn,
+                    options,
+                },
+            }));
+        },
+    });
+}
+async function binsQuery(SimpleTable, column, interval, newColumn, options = {}) {
+    // A SELECT *, expr AS col colliding with an existing column would be
+    // silently renamed by DuckDB (col -> col_1) instead of erroring, unlike
+    // the ALTER TABLE ADD this used to run.
+    (0, assertNewColumns_js_1.default)(await SimpleTable.getTypes(), [newColumn], "bins()");
+    // The minimum and maximum are computed in one scan instead of one
+    // getMin/getMax query each.
+    const minMax = await (0, queryDB_js_1.default)(SimpleTable, `SELECT MIN(${(0, quoteIdentifier_js_1.default)(column)}) AS "min", MAX(${(0, quoteIdentifier_js_1.default)(column)}) AS "max" FROM ${(0, quoteIdentifier_js_1.default)(SimpleTable.name)}`, (0, mergeOptions_js_1.default)(SimpleTable, {
+        table: SimpleTable.name,
+        method: "bins()",
+        parameters: { column, interval, newColumn, options },
+        returnData: true,
+    }));
+    const minValue = minMax?.[0]?.min;
+    if (typeof minValue !== "number") {
+        throw new Error(`minValue of ${column} is not a number`);
+    }
+    let startValue;
+    if (typeof options.startValue === "number") {
+        if (options.startValue > minValue) {
+            throw new Error(`startValue ${options.startValue} can't be greater than minValue ${minValue}`);
+        }
+        startValue = options.startValue;
+    }
+    else {
+        startValue = minValue;
+    }
+    const maxValue = minMax?.[0]?.max;
+    if (typeof maxValue !== "number") {
+        throw new Error(`maxValue of ${column} is not a number`);
+    }
+    const endValue = maxValue;
+    let increment = 1;
+    let decimals = 0;
+    const intervalAsString = interval.toString();
+    const decimalIndex = intervalAsString.indexOf(".");
+    if (decimalIndex > 0) {
+        decimals = intervalAsString.substring(decimalIndex + 1).length;
+        increment = 1.0 / (10.0 * decimals);
+    }
+    const intervals = [];
+    for (let i = startValue; i <= endValue; i += interval) {
+        const start = i;
+        const end = (i + interval - increment).toFixed(decimals);
+        intervals.push(`WHEN ${(0, quoteIdentifier_js_1.default)(column)} >= ${start} AND ${(0, quoteIdentifier_js_1.default)(column)} <= ${end} THEN '[${start}-${end}]'`);
+    }
+    // A single rewrite, so the table is scanned once instead of once for the
+    // ALTER and once for the UPDATE.
+    const query = `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(SimpleTable.name)} AS
+    SELECT *, CASE
+    ${intervals.join("\n")}
+    END AS ${(0, quoteIdentifier_js_1.default)(newColumn)}
+    FROM ${(0, quoteIdentifier_js_1.default)(SimpleTable.name)}`;
+    return query;
+}

package/script/methods/bm25.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function bm25(simpleTable: SimpleTable, text: string, columnId: string, columnText: string, nbResults: number, options?: {
+export default function bm25(simpleTable: SimpleTable, text: string, idColumn: string, textColumn: string, count: number, options?: {
     stemmer?: "arabic" | "basque" | "catalan" | "danish" | "dutch" | "english" | "finnish" | "french" | "german" | "greek" | "hindi" | "hungarian" | "indonesian" | "irish" | "italian" | "lithuanian" | "nepali" | "norwegian" | "porter" | "portuguese" | "romanian" | "russian" | "serbian" | "spanish" | "swedish" | "tamil" | "turkish" | "none";
     stopwords?: string;
     ignore?: string;
@@ -13,5 +13,5 @@ export default function bm25(simpleTable: SimpleTable, text: string, columnId: s
     conjunctive?: boolean;
     outputTable?: string;
     verbose?: boolean;
-}): Promise<SimpleTable>;
+}): SimpleTable;
 //# sourceMappingURL=bm25.d.ts.map

package/script/methods/bm25.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/methods/bm25.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,wBA+DP"}
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/methods/bm25.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IACP,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,GACL,WAAW,CA4Bb"}

package/script/methods/bm25.js

@@ -4,13 +4,33 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = bm25;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const camelCase_js_1 = __importDefault(require("../helpers/camelCase.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
-async function bm25(simpleTable, text, columnId, columnText, nbResults, options = {}) {
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const createFtsIndex_js_1 = require("./createFtsIndex.js");
+function bm25(simpleTable, text, idColumn, textColumn, count, options = {}) {
+    options = structuredClone(options);
     // This uses the fts extension
     // https://duckdb.org/docs/stable/core_extensions/full_text_search
-    await simpleTable.createFtsIndex(columnId, columnText, {
+    // The output table instance is created at call time so it can be returned
+    // synchronously and chained on right away.
+    const outputTable = typeof options.outputTable === "string"
+        ? simpleTable.sdb.newTable(options.outputTable)
+        : simpleTable;
+    (0, queueOp_js_1.default)(outputTable, {
+        kind: "barrier",
+        method: "bm25()",
+        parameters: { text, idColumn, textColumn, count, options },
+        execute: () => executeBm25(simpleTable, outputTable, text, idColumn, textColumn, count, options),
+    });
+    return outputTable;
+}
+async function executeBm25(simpleTable, outputTable, text, idColumn, textColumn, count, options) {
+    // The index creation runs directly (not with the sync createFtsIndex
+    // builder, which would queue for the next flush).
+    await (0, createFtsIndex_js_1.executeCreateFtsIndex)(simpleTable, idColumn, textColumn, {
         stemmer: options.stemmer,
         stopwords: options.stopwords,
         ignore: options.ignore,
@@ -23,16 +43,16 @@ async function bm25(simpleTable, text, columnId, columnText, nbResults, options
         ? ` AND score >= ${options.minScore}`
         : "";
     const selectClause = typeof options.scoreColumn === "string"
-        ? `* EXCLUDE(score), score AS "${options.scoreColumn}"`
+        ? `* EXCLUDE(score), score AS ${(0, quoteIdentifier_js_1.default)(options.scoreColumn)}`
         : `* EXCLUDE(score)`;
-    await (0, queryDB_js_1.default)(simpleTable, `CREATE OR REPLACE TABLE "${options.outputTable ?? simpleTable.name}" AS SELECT ${selectClause} FROM (SELECT *, fts_main_${(0, camelCase_js_1.default)(simpleTable.name)}.match_bm25(${columnId}, '${text.replace(/'/g, "''")}'${typeof options.k === "number" ? `, k := ${options.k}` : ""}${typeof options.b === "number" ? `, b := ${options.b}` : ""}${options.conjunctive === true ? `, conjunctive := 1` : ""}) AS score FROM "${simpleTable.name}") sq WHERE score NOT NULL${minScoreCondition} ORDER BY score DESC LIMIT ${nbResults};`, (0, mergeOptions_js_1.default)(simpleTable, {
+    await (0, queryDB_js_1.default)(simpleTable, `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(outputTable.name)} AS SELECT ${selectClause} FROM (SELECT *, ${(0, quoteIdentifier_js_1.default)(`fts_main_${(0, camelCase_js_1.default)(simpleTable.name)}`)}.match_bm25(${(0, quoteIdentifier_js_1.default)(idColumn)}, ?${typeof options.k === "number" ? `, k := ${options.k}` : ""}${typeof options.b === "number" ? `, b := ${options.b}` : ""}${options.conjunctive === true ? `, conjunctive := 1` : ""}) AS score FROM ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}) sq WHERE score NOT NULL${minScoreCondition} ORDER BY score DESC LIMIT ${count};`, (0, mergeOptions_js_1.default)(simpleTable, {
         table: simpleTable.name,
         method: "bm25",
         parameters: {
             text,
-            columnId,
-            columnText,
-            nbResults,
+            idColumn,
+            textColumn,
+            count,
             minScore: options.minScore,
             scoreColumn: options.scoreColumn,
             k: options.k,
@@ -44,11 +64,6 @@ async function bm25(simpleTable, text, columnId, columnText, nbResults, options
             lower: options.lower,
             table: options.outputTable ?? simpleTable.name,
         },
+        values: [text],
     }));
-    if (typeof options.outputTable === "string") {
-        return simpleTable.sdb.newTable(options.outputTable);
-    }
-    else {
-        return simpleTable;
-    }
 }

package/script/methods/buffer.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function buffer(simpleTable: SimpleTable, newColumn: string, distance: number, options?: {
+    column?: string;
+}): void;
+//# sourceMappingURL=buffer.d.ts.map

package/script/methods/buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buffer.d.ts","sourceRoot":"","sources":["../../src/methods/buffer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,MAAM,CAC5B,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,QA2BlC"}

package/script/methods/buffer.js

@@ -0,0 +1,30 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buffer;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const findGeoColumnFromSchema_js_1 = __importDefault(require("../helpers/findGeoColumnFromSchema.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+function buffer(simpleTable, newColumn, distance, options = {}) {
+    options = structuredClone(options);
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "fusable",
+        method: "buffer()",
+        parameters: { newColumn, distance, options },
+        needsSchema: true,
+        needsSpatial: true,
+        buildSelect: (input, types) => {
+            const column = typeof options.column === "string"
+                ? options.column
+                : (0, findGeoColumnFromSchema_js_1.default)(types, "buffer()", simpleTable.name);
+            // The schema type carries the projection (e.g. GEOMETRY('EPSG:4326')),
+            // so the cast keeps it on the new geometries.
+            const expression = `ST_Buffer(${(0, quoteIdentifier_js_1.default)(column)}, ${distance})::${types[column]}`;
+            return Object.keys(types).includes(newColumn)
+                ? `SELECT * REPLACE (${expression} AS ${(0, quoteIdentifier_js_1.default)(newColumn)}) FROM ${input}`
+                : `SELECT *, ${expression} AS ${(0, quoteIdentifier_js_1.default)(newColumn)} FROM ${input}`;
+        },
+    });
+}

package/script/methods/cache.d.ts

@@ -1,6 +1,7 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function cache(table: SimpleTable, run: () => Promise<void>, options?: {
+export default function cache<Table extends SimpleTable>(table: Table, compute: (table: Table) => void | Promise<void>, options?: {
     ttl?: number;
+    inputs?: readonly unknown[];
     verbose?: boolean;
 }): Promise<void>;
 //# sourceMappingURL=cache.d.ts.map

package/script/methods/cache.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/methods/cache.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAcvD,wBAA8B,KAAK,CACjC,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EACxB,OAAO,GAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,iBA2HlD"}
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/methods/cache.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AA2DvD,wBAA8B,KAAK,CAAC,KAAK,SAAS,WAAW,EAC3D,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC/C,OAAO,GAAE;IACP,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,iBAwLP"}