@nshiab/simple-data-analysis 5.19.8

package/script/deps/jsr.io/@nshiab/simple-data-analysis-core/0.0.17/src/methods/zScoreQuery.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = zScoreQuery;
+const stringToArray_js_1 = __importDefault(require("../helpers/stringToArray.js"));
+function zScoreQuery(table, column, newColumn, options = {}) {
+    const categories = options.categories
+        ? (0, stringToArray_js_1.default)(options.categories)
+        : [];
+    const partition = categories.length > 0
+        ? `PARTITION BY ${categories.map((d) => `"${d}"`).join(", ")}`
+        : "";
+    const tempQuery = `("${column}"-AVG("${column}") OVER(${partition}))
+            /
+            STDDEV_POP("${column}") OVER(${partition})`;
+    const query = `
+    CREATE OR REPLACE TABLE "${table}" AS
+    SELECT *, (
+        ${typeof options.decimals === "number"
+        ? `ROUND(${tempQuery}, ${options.decimals})`
+        : tempQuery}
+        
+        ) AS ${newColumn},
+    FROM "${table}"
+    `;
+    return query;
+}

package/script/helpers/getRRFRanking.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Executes Reciprocal Rank Fusion and returns only the sorted document IDs.
+ * @param results - An array of string arrays (e.g., [[id1, id2], [id2, id3]])
+ * @param k - The smoothing constant (default 60)
+ * @returns A flat array of document IDs sorted by their fused rank.
+ */
+export default function getRRFRanking(results: string[][], k?: number): string[];
+//# sourceMappingURL=getRRFRanking.d.ts.map

package/script/helpers/getRRFRanking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"getRRFRanking.d.ts","sourceRoot":"","sources":["../../src/helpers/getRRFRanking.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,CAAC,GAAE,MAAW,GACb,MAAM,EAAE,CAiBV"}

package/script/helpers/getRRFRanking.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = getRRFRanking;
+/**
+ * Executes Reciprocal Rank Fusion and returns only the sorted document IDs.
+ * @param results - An array of string arrays (e.g., [[id1, id2], [id2, id3]])
+ * @param k - The smoothing constant (default 60)
+ * @returns A flat array of document IDs sorted by their fused rank.
+ */
+function getRRFRanking(results, k = 60) {
+    const scoreMap = new Map();
+    results.forEach((resultSet) => {
+        resultSet.forEach((id, index) => {
+            const rank = index + 1;
+            const currentScore = scoreMap.get(id) || 0;
+            // Add the RRF score to the existing total for this ID
+            scoreMap.set(id, currentScore + (1 / (k + rank)));
+        });
+    });
+    // Sort by score descending and return only the IDs
+    return Array.from(scoreMap.entries())
+        .sort((a, b) => b[1] - a[1])
+        .map(([id]) => id);
+}

package/script/helpers/sleep.d.ts

@@ -0,0 +1,5 @@
+export default function sleep(ms: number, options?: {
+    start?: Date;
+    log?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=sleep.d.ts.map

package/script/helpers/sleep.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sleep.d.ts","sourceRoot":"","sources":["../../src/helpers/sleep.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,KAAK,CAC3B,EAAE,EAAE,MAAM,EACV,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,IAAI,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5C,OAAO,CAAC,IAAI,CAAC,CA0Bf"}

package/script/helpers/sleep.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = sleep;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-format/1.1.5/src/index.js");
+function sleep(ms, options = {}) {
+    return new Promise((resolve) => {
+        if (options.start) {
+            const end = new Date();
+            const duration = end.getTime() - options.start.getTime();
+            const remaining = ms - duration;
+            if (remaining > 0) {
+                if (options.log) {
+                    console.log(`\nSleeping for ${(0, index_js_1.prettyDuration)(0, { end: remaining })}...`);
+                }
+                return setTimeout(resolve, remaining);
+            }
+            else {
+                if (options.log) {
+                    console.log(`\nNo need to sleep, already took ${(0, index_js_1.prettyDuration)(0, { end: duration })}.`);
+                }
+                return resolve();
+            }
+        }
+        return setTimeout(resolve, ms);
+    });
+}

package/script/helpers/tryAI.d.ts

@@ -0,0 +1,36 @@
+import type { Ollama } from "ollama";
+export default function tryAI(i: number, batchSize: number, rows: {
+    [key: string]: string | number | boolean | Date | null;
+}[], column: string, newColumns: string[], prompt: string, options?: {
+    batchSize?: number;
+    concurrent?: number;
+    cache?: boolean;
+    test?: (result: {
+        [key: string]: unknown;
+    }) => void;
+    retry?: number;
+    model?: string;
+    temperature?: number;
+    apiKey?: string;
+    vertex?: boolean;
+    project?: string;
+    location?: string;
+    ollama?: boolean | Ollama;
+    verbose?: boolean;
+    rateLimitPerMinute?: number;
+    clean?: (response: unknown) => unknown;
+    contextWindow?: number;
+    thinkingBudget?: number;
+    thinkingLevel?: "minimal" | "low" | "medium" | "high";
+    safetyEnabled?: boolean;
+    webSearch?: boolean;
+    extraInstructions?: string;
+    schemaJson?: unknown;
+    metrics?: {
+        totalCost: number;
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        totalRequests: number;
+    };
+}): Promise<void>;
+//# sourceMappingURL=tryAI.d.ts.map

package/script/helpers/tryAI.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tryAI.d.ts","sourceRoot":"","sources":["../../src/helpers/tryAI.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGrC,wBAA8B,KAAK,CACjC,CAAC,EAAE,MAAM,EACT,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE;IACJ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD,EAAE,EACH,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAAE,EACpB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,CACN,QAAQ,EAAE,OAAO,KACd,OAAO,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACE,iBAgHP"}

package/script/helpers/tryAI.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = tryAI;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/1.2.18/src/index.js");
+const zod_1 = require("zod");
+async function tryAI(i, batchSize, rows, column, newColumns, prompt, options = {}) {
+    const batch = rows.slice(i, i + batchSize);
+    let schemaJson;
+    if (options.schemaJson) {
+        schemaJson = options.schemaJson;
+    }
+    else {
+        const objectSchema = {};
+        for (const newColumn of newColumns) {
+            objectSchema[newColumn] = (0, zod_1.string)();
+        }
+        schemaJson = (0, zod_1.toJSONSchema)((0, zod_1.array)((0, zod_1.object)(objectSchema)));
+    }
+    const systemPrompt = `You will be provided with a JSON array of ${batch.length} string items. You must return a JSON array containing exactly ${batch.length} objects, in the same corresponding order.`;
+    const fullPrompt = `${prompt}\n\nHere are the ${column} values as a JSON array:\n${JSON.stringify(batch.map((d) => d[column]), null, 2)}\n\n${options.extraInstructions ? `\n${options.extraInstructions}` : ""}`;
+    const retry = options.retry ?? 1;
+    let testPassed = false;
+    let iterations = 1;
+    let newValues;
+    while (!testPassed && iterations <= retry) {
+        try {
+            newValues = await (0, index_js_1.askAI)(`${fullPrompt}${iterations > 1
+                ? `\nThis is your attempt #${iterations}. So get it right by following my instructions closely!`
+                : ""}`, {
+                ...options,
+                includeThoughts: options.verbose ? true : false,
+                systemPrompt: systemPrompt,
+                schemaJson,
+                test: (response) => {
+                    if (!Array.isArray(response)) {
+                        throw new Error(`The AI returned a non-array value: ${JSON.stringify(response)}`);
+                    }
+                    if (response.length !== batch.length) {
+                        throw new Error(`The AI returned ${response.length} values, but the batch size is ${batch.length}.`);
+                    }
+                    for (const item of response) {
+                        if (typeof item !== "object" || item === null) {
+                            throw new Error(`The AI did not return an object: ${JSON.stringify(item)}`);
+                        }
+                        for (const newColumn of newColumns) {
+                            if (!(newColumn in item)) {
+                                throw new Error(`The AI's response is missing the key '${newColumn}': ${JSON.stringify(item)}`);
+                            }
+                        }
+                    }
+                    if (options.test) {
+                        for (const item of response) {
+                            options.test(item);
+                        }
+                    }
+                },
+            });
+            testPassed = true;
+        }
+        catch (e) {
+            if (iterations < retry) {
+                console.log(`${console.log(e)}\nRetrying... (${iterations}/${retry})`);
+                iterations++;
+            }
+            else {
+                console.log(`${console.log(e)}\nNo more retries left. (${iterations}/${retry}).`);
+                throw e;
+            }
+        }
+    }
+    // Types could be improved
+    for (let j = 0; j <
+        newValues.length; j++) {
+        for (const newColumn of newColumns) {
+            rows[i + j][newColumn] = newValues[j][newColumn];
+        }
+    }
+}

package/script/helpers/tryEmbedding.d.ts

@@ -0,0 +1,15 @@
+import type { Ollama } from "ollama";
+export default function tryEmbedding(i: number, rows: {
+    [key: string]: string | number | boolean | Date | null;
+}[], text: string, newColumn: string, options?: {
+    cache?: boolean;
+    model?: string;
+    apiKey?: string;
+    vertex?: boolean;
+    project?: string;
+    location?: string;
+    ollama?: boolean | Ollama;
+    verbose?: boolean;
+    contextWindow?: number;
+}): Promise<number>;
+//# sourceMappingURL=tryEmbedding.d.ts.map

package/script/helpers/tryEmbedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tryEmbedding.d.ts","sourceRoot":"","sources":["../../src/helpers/tryEmbedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,wBAA8B,YAAY,CACxC,CAAC,EAAE,MAAM,EACT,IAAI,EAAE;IACJ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD,EAAE,EACH,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACnB,mBAOP"}

package/script/helpers/tryEmbedding.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = tryEmbedding;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/1.2.18/src/index.js");
+async function tryEmbedding(i, rows, text, newColumn, options = {}) {
+    // Should be improved...
+    return rows[i][newColumn] = await (0, index_js_1.getEmbedding)(text, options);
+}

package/script/index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * @module
+ *
+ * The Simple Data Analysis Library
+ *
+ * To install the library with Deno, use:
+ * ```bash
+ * deno add jsr:@nshiab/simple-data-analysis
+ * ```
+ *
+ * To install the library with Node.js, use:
+ * ```bash
+ * npx jsr add @nshiab/simple-data-analysis
+ * ```
+ *
+ * To start, create a SimpleDB instance and then a SimpleTable from this instance:
+ * ```ts
+ * import { SimpleDB } from "@nshiab/simple-data-analysis";
+ *
+ * const sdb = new SimpleDB();
+ * const table = sdb.newTable("myTable"); // This returns a SimpleTable instance
+ * await table.loadData("path/to/your/data.csv");
+ *
+ * // You can now perform various data analysis operations on the table.
+ *
+ * await sdb.done(); // Ensure to call done when you're finished.
+ * ```
+ */
+export { default as SimpleDB } from "./class/SimpleDB.js";
+export { default as SimpleTable } from "./class/SimpleTable.js";
+//# sourceMappingURL=index.d.ts.map

package/script/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,OAAO,IAAI,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,wBAAwB,CAAC"}

package/script/index.js

@@ -0,0 +1,38 @@
+"use strict";
+/**
+ * @module
+ *
+ * The Simple Data Analysis Library
+ *
+ * To install the library with Deno, use:
+ * ```bash
+ * deno add jsr:@nshiab/simple-data-analysis
+ * ```
+ *
+ * To install the library with Node.js, use:
+ * ```bash
+ * npx jsr add @nshiab/simple-data-analysis
+ * ```
+ *
+ * To start, create a SimpleDB instance and then a SimpleTable from this instance:
+ * ```ts
+ * import { SimpleDB } from "@nshiab/simple-data-analysis";
+ *
+ * const sdb = new SimpleDB();
+ * const table = sdb.newTable("myTable"); // This returns a SimpleTable instance
+ * await table.loadData("path/to/your/data.csv");
+ *
+ * // You can now perform various data analysis operations on the table.
+ *
+ * await sdb.done(); // Ensure to call done when you're finished.
+ * ```
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SimpleTable = exports.SimpleDB = void 0;
+var SimpleDB_js_1 = require("./class/SimpleDB.js");
+Object.defineProperty(exports, "SimpleDB", { enumerable: true, get: function () { return __importDefault(SimpleDB_js_1).default; } });
+var SimpleTable_js_1 = require("./class/SimpleTable.js");
+Object.defineProperty(exports, "SimpleTable", { enumerable: true, get: function () { return __importDefault(SimpleTable_js_1).default; } });

package/script/methods/aiEmbeddings.d.ts

@@ -0,0 +1,21 @@
+import type { SimpleTable } from "../index.js";
+import type { Ollama } from "ollama";
+export default function aiEmbeddings(simpleTable: SimpleTable, column: string, newColumn: string, options?: {
+    createIndex?: boolean;
+    overwriteIndex?: boolean;
+    concurrent?: number;
+    cache?: boolean;
+    model?: string;
+    apiKey?: string;
+    vertex?: boolean;
+    project?: string;
+    location?: string;
+    ollama?: boolean | Ollama;
+    verbose?: boolean;
+    rateLimitPerMinute?: number;
+    contextWindow?: number;
+    efConstruction?: number;
+    efSearch?: number;
+    M?: number;
+}): Promise<void>;
+//# sourceMappingURL=aiEmbeddings.d.ts.map

package/script/methods/aiEmbeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aiEmbeddings.d.ts","sourceRoot":"","sources":["../../src/methods/aiEmbeddings.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,wBAA8B,YAAY,CACxC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,CAAC,EAAE,MAAM,CAAC;CACP,iBAsEP"}

package/script/methods/aiEmbeddings.js

@@ -0,0 +1,56 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = aiEmbeddings;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-format/1.1.5/src/index.js");
+const sleep_js_1 = __importDefault(require("../helpers/sleep.js"));
+const tryEmbedding_js_1 = __importDefault(require("../helpers/tryEmbedding.js"));
+async function aiEmbeddings(simpleTable, column, newColumn, options = {}) {
+    await simpleTable.updateWithJS(async (rows) => {
+        if (options.verbose) {
+            console.log("\naiEmbeddings()");
+        }
+        const concurrent = options.concurrent ?? 1;
+        let requests = [];
+        for (let i = 0; i < rows.length; i++) {
+            if (options.verbose) {
+                console.log(`\nProcessing row ${i + 1} of ${rows.length}... (${(0, index_js_1.formatNumber)((i + 1) / rows.length * 100, {
+                    significantDigits: 3,
+                    suffix: "%",
+                })})`);
+            }
+            if (requests.length < concurrent) {
+                const text = rows[i][column];
+                if (typeof text !== "string") {
+                    throw new Error(`The column "${column}" must be a string. Found ${text} instead.`);
+                }
+                requests.push((0, tryEmbedding_js_1.default)(i, rows, text, newColumn, options));
+            }
+            if (requests.length === concurrent || i + 1 >= rows.length) {
+                const start = new Date();
+                await Promise.all(requests);
+                const end = new Date();
+                const duration = end.getTime() - start.getTime();
+                // If duration is less than 10ms per request, it should means data comes from cache and we don't need to wait
+                if (typeof options.rateLimitPerMinute === "number" &&
+                    duration > 10 * requests.length && i + 1 < rows.length) {
+                    const delay = Math.round((60 / (options.rateLimitPerMinute / concurrent)) * 1000);
+                    await (0, sleep_js_1.default)(delay, { start, log: options.verbose });
+                }
+                requests = [];
+            }
+        }
+        return rows;
+    });
+    if (options.createIndex) {
+        await simpleTable.createVssIndex(newColumn, {
+            overwrite: options.overwriteIndex,
+            verbose: options.verbose,
+            efConstruction: options.efConstruction,
+            efSearch: options.efSearch,
+            M: options.M,
+        });
+    }
+}

package/script/methods/aiQuery.d.ts

@@ -0,0 +1,21 @@
+import type { SimpleTable } from "../index.js";
+import type { Ollama } from "ollama";
+export default function aiQuery(simpleTable: SimpleTable, prompt: string, options?: {
+    extraInstructions?: string;
+    cache?: boolean;
+    model?: string;
+    apiKey?: string;
+    vertex?: boolean;
+    project?: string;
+    includeThoughts?: boolean;
+    location?: string;
+    ollama?: boolean | Ollama;
+    contextWindow?: number;
+    thinkingBudget?: number;
+    thinkingLevel?: "minimal" | "low" | "medium" | "high";
+    temperature?: number;
+    safetyEnabled?: boolean;
+    outputTable?: string;
+    verbose?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=aiQuery.d.ts.map

package/script/methods/aiQuery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aiQuery.d.ts","sourceRoot":"","sources":["../../src/methods/aiQuery.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGrC,wBAA8B,OAAO,CACnC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,iBA8CP"}

package/script/methods/aiQuery.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = aiQuery;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/1.2.18/src/index.js");
+const zod_1 = require("zod");
+async function aiQuery(simpleTable, prompt, options = {}) {
+    const tableName = options.outputTable ?? simpleTable.name;
+    const p = `I have a SQL table named "${simpleTable.name}". The data is already in it with these columns:\n\n${JSON.stringify(await simpleTable.getTypes(), null, 2)}\n\nI want you to give me a SQL query to do this:\n- ${prompt}\nThe query must ${options.outputTable
+        ? `create a new table named "${tableName}" with the results`
+        : `replace the existing "${simpleTable.name}" table`}. This means the query must start with 'CREATE OR REPLACE TABLE "${tableName}"...'. Return just the query, nothing else.${options.extraInstructions ? `\n${options.extraInstructions}` : ""}`;
+    if (options.verbose) {
+        console.log("\naiQuery()");
+    }
+    const schemaJson = (0, zod_1.toJSONSchema)((0, zod_1.object)({
+        query: (0, zod_1.string)(),
+    }));
+    // Types could be improved
+    const answer = await (0, index_js_1.askAI)(p, {
+        cache: options.cache,
+        model: options.model,
+        apiKey: options.apiKey,
+        vertex: options.vertex,
+        project: options.project,
+        location: options.location,
+        ollama: options.ollama,
+        contextWindow: options.contextWindow,
+        thinkingBudget: options.thinkingBudget,
+        thinkingLevel: options.thinkingLevel,
+        temperature: options.temperature,
+        safetyEnabled: options.safetyEnabled,
+        verbose: options.verbose,
+        includeThoughts: options.includeThoughts,
+        schemaJson,
+    });
+    const query = answer.query;
+    await simpleTable.sdb.customQuery(query);
+}

package/script/methods/aiRAG.d.ts

@@ -0,0 +1,49 @@
+import type SimpleTable from "../class/SimpleTable.js";
+import type { Ollama } from "ollama";
+export default function aiRAG(table: SimpleTable, query: string, columnId: string, columnText: string, nbResults: number, options?: {
+    cache?: boolean;
+    verbose?: boolean;
+    includeThoughts?: boolean;
+    systemPrompt?: string;
+    modelContextWindow?: number;
+    embeddingsModelContextWindow?: number;
+    createIndex?: boolean;
+    thinkingBudget?: number;
+    thinkingLevel?: "minimal" | "low" | "medium" | "high";
+    webSearch?: boolean;
+    safetyEnabled?: boolean;
+    model?: string;
+    temperature?: number;
+    apiKey?: string;
+    vertex?: boolean;
+    project?: string;
+    location?: string;
+    ollama?: boolean | Ollama;
+    metrics?: {
+        totalCost: number;
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        totalRequests: number;
+    };
+    embeddingsModel?: string;
+    ollamaEmbeddings?: boolean;
+    embeddingsConcurrent?: number;
+    stemmer?: "arabic" | "basque" | "catalan" | "danish" | "dutch" | "english" | "finnish" | "french" | "german" | "greek" | "hindi" | "hungarian" | "indonesian" | "irish" | "italian" | "lithuanian" | "nepali" | "norwegian" | "porter" | "portuguese" | "romanian" | "russian" | "serbian" | "spanish" | "swedish" | "tamil" | "turkish" | "none";
+    stopwords?: string;
+    ignore?: string;
+    stripAccents?: boolean;
+    lower?: boolean;
+    k?: number;
+    b?: number;
+    conjunctive?: boolean;
+    bm25?: boolean;
+    bm25MinScore?: number;
+    bm25ScoreColumn?: string;
+    vectorSearch?: boolean;
+    vectorMinSimilarity?: number;
+    vectorSimilarityColumn?: string;
+    efConstruction?: number;
+    efSearch?: number;
+    M?: number;
+}): Promise<string>;
+//# sourceMappingURL=aiRAG.d.ts.map

package/script/methods/aiRAG.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aiRAG.d.ts","sourceRoot":"","sources":["../../src/methods/aiRAG.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,wBAA8B,KAAK,CACjC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,CAAC,EAAE,MAAM,CAAC;CACP,mBAuJP"}

package/script/methods/aiRAG.js

@@ -0,0 +1,111 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = aiRAG;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/1.2.18/src/index.js");
+const index_js_2 = require("../deps/jsr.io/@nshiab/journalism-format/1.1.5/src/index.js");
+const hybridSearch_js_1 = __importDefault(require("./hybridSearch.js"));
+async function aiRAG(table, query, columnId, columnText, nbResults, options = {}) {
+    const times = {
+        start: Date.now(),
+        embeddingStart: 0,
+        embeddingEnd: 0,
+        vectorSearchStart: 0,
+        vectorSearchEnd: 0,
+        bm25Start: 0,
+        bm25End: 0,
+        llmStart: 0,
+        llmEnd: 0,
+    };
+    // Perform hybrid search (vector similarity + BM25 with RRF fusion)
+    const searchResultsTable = await (0, hybridSearch_js_1.default)(table, query, columnId, columnText, nbResults, {
+        cache: options.cache,
+        verbose: options.verbose,
+        embeddingsModelContextWindow: options.embeddingsModelContextWindow,
+        createIndex: options.createIndex,
+        embeddingsModel: options.embeddingsModel,
+        ollamaEmbeddings: options.ollamaEmbeddings,
+        embeddingsConcurrent: options.embeddingsConcurrent,
+        stemmer: options.stemmer,
+        stopwords: options.stopwords,
+        ignore: options.ignore,
+        stripAccents: options.stripAccents,
+        lower: options.lower,
+        k: options.k,
+        b: options.b,
+        conjunctive: options.conjunctive,
+        bm25: options.bm25,
+        bm25MinScore: options.bm25MinScore,
+        bm25ScoreColumn: options.bm25ScoreColumn,
+        vectorSearch: options.vectorSearch,
+        vectorMinSimilarity: options.vectorMinSimilarity,
+        vectorSimilarityColumn: options.vectorSimilarityColumn,
+        outputTable: `${table.name}_rag_search_results`,
+        efConstruction: options.efConstruction,
+        efSearch: options.efSearch,
+        M: options.M,
+        times: times,
+    });
+    await searchResultsTable.selectColumns([columnId, columnText]);
+    // Get the retrieved data
+    const retrievedData = await searchResultsTable.getData();
+    // Clean up the temporary table
+    await searchResultsTable.removeTable();
+    if (options.verbose) {
+        times.llmStart = Date.now();
+    }
+    const response = await (0, index_js_1.askAI)(`Answer the following:
+- ${query}
+
+Base your answer only on the following data:\n
+${retrievedData.map((entry) => `${columnId}: ${entry[columnId]}\n\n${columnText}:\n\n${entry[columnText]}`).join("\n\n-----\n\n")}`, {
+        systemPrompt: options.systemPrompt ??
+            `You are a focused research assistant. Your goal is to answer the user's question using ONLY the provided data.
+
+Rules of Engagement:
+- Directness: Start answers directly (e.g., "The documentation indicates..." or "I found..."). Avoid meta-talk about the process.
+- Strict Relevance: Only include information that directly addresses the user's query. If an entry is irrelevant, ignore it entirely.
+- Ambiguity & Contradiction: If sources conflict, present both views clearly. If the data is incomplete, state what is known and explicitly identify what is missing.
+- Groundedness: If the provided data does not contain the answer, your only response must be: "I do not have data to answer this question." Do not use outside knowledge.`,
+        cache: options.cache,
+        verbose: options.verbose,
+        includeThoughts: options.includeThoughts,
+        contextWindow: options.modelContextWindow,
+        thinkingBudget: options.thinkingBudget,
+        thinkingLevel: options.thinkingLevel,
+        webSearch: options.webSearch,
+        safetyEnabled: options.safetyEnabled,
+        model: options.model,
+        temperature: options.temperature,
+        apiKey: options.apiKey,
+        vertex: options.vertex,
+        project: options.project,
+        location: options.location,
+        ollama: options.ollama,
+        metrics: options.metrics,
+    });
+    if (options.verbose) {
+        times.llmEnd = Date.now();
+        const enableVectorSearch = options.vectorSearch !== false;
+        const enableBm25 = options.bm25 !== false;
+        const parallelLabel = enableVectorSearch && enableBm25 ? " (parallel)" : "";
+        const logParts = [`\nRAG process times:`];
+        if (enableVectorSearch) {
+            logParts.push(`- Embedding: ${(0, index_js_2.prettyDuration)(times.embeddingStart, { end: times.embeddingEnd })}`);
+        }
+        logParts.push(enableVectorSearch
+            ? `- Vector Search${parallelLabel}: ${(0, index_js_2.prettyDuration)(times.vectorSearchStart, {
+                end: times.vectorSearchEnd,
+            })}`
+            : `- Vector Search: disabled`);
+        logParts.push(enableBm25
+            ? `- BM25 Search${parallelLabel}: ${(0, index_js_2.prettyDuration)(times.bm25Start, { end: times.bm25End })}`
+            : `- BM25 Search: disabled`);
+        logParts.push(`- LLM: ${(0, index_js_2.prettyDuration)(times.llmStart, { end: times.llmEnd })}`);
+        logParts.push(`- Total: ${(0, index_js_2.prettyDuration)(times.start)} (verbose adds overhead)`);
+        console.log(logParts.join("\n") + "\n");
+    }
+    return response;
+}