@nshiab/simple-data-analysis 5.19.8

package/script/methods/aiRowByRow.d.ts

@@ -0,0 +1,35 @@
+import type { SimpleTable } from "../index.js";
+import type { Ollama } from "ollama";
+export default function aiRowByRow(simpleTable: SimpleTable, column: string, newColumn: string | string[], prompt: string, options?: {
+    batchSize?: number;
+    concurrent?: number;
+    cache?: boolean;
+    test?: (result: {
+        [key: string]: unknown;
+    }) => void;
+    retry?: number;
+    model?: string;
+    temperature?: number;
+    apiKey?: string;
+    vertex?: boolean;
+    project?: string;
+    location?: string;
+    ollama?: boolean | Ollama;
+    verbose?: boolean;
+    rateLimitPerMinute?: number;
+    clean?: (response: unknown) => unknown;
+    contextWindow?: number;
+    thinkingBudget?: number;
+    thinkingLevel?: "minimal" | "low" | "medium" | "high";
+    safetyEnabled?: boolean;
+    webSearch?: boolean;
+    extraInstructions?: string;
+    schemaJson?: unknown;
+    metrics?: {
+        totalCost: number;
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        totalRequests: number;
+    };
+}): Promise<void>;
+//# sourceMappingURL=aiRowByRow.d.ts.map

package/script/methods/aiRowByRow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aiRowByRow.d.ts","sourceRoot":"","sources":["../../src/methods/aiRowByRow.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGrC,wBAA8B,UAAU,CACtC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,CACN,QAAQ,EAAE,OAAO,KACd,OAAO,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACE,iBA2EP"}

package/script/methods/aiRowByRow.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = aiRowByRow;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-format/1.1.5/src/index.js");
+const sleep_js_1 = __importDefault(require("../helpers/sleep.js"));
+const tryAI_js_1 = __importDefault(require("../helpers/tryAI.js"));
+const index_js_2 = require("../deps/jsr.io/@nshiab/simple-data-analysis-core/0.0.17/src/helpers/index.js");
+async function aiRowByRow(simpleTable, column, newColumn, prompt, options = {}) {
+    const newColumns = (0, index_js_2.stringToArray)(newColumn);
+    await simpleTable.updateWithJS(async (rows) => {
+        if (options.verbose) {
+            console.log("\naiRowByRow()");
+        }
+        const batchSize = options.batchSize ?? 1;
+        const concurrent = options.concurrent ?? 1;
+        let requests = [];
+        let requestsNb = 1;
+        for (let i = 0; i < rows.length; i += batchSize) {
+            if (options.verbose) {
+                console.log(`\nRequest ${requestsNb} - Processing rows ${i + 1} to ${Math.min(i + batchSize, rows.length)}... (${(0, index_js_1.formatNumber)((Math.min(i + batchSize, rows.length)) / rows.length * 100, {
+                    significantDigits: 3,
+                    suffix: "%",
+                })})`);
+                requestsNb++;
+            }
+            if (requests.length < concurrent) {
+                requests.push((0, tryAI_js_1.default)(i, batchSize, rows, column, newColumns, prompt, options));
+            }
+            if (requests.length === concurrent || i + batchSize >= rows.length) {
+                const start = new Date();
+                await Promise.all(requests);
+                const end = new Date();
+                const duration = end.getTime() - start.getTime();
+                // If duration is less than 10ms per request, it should means data comes from cache and we don't need to wait
+                if (typeof options.rateLimitPerMinute === "number" &&
+                    duration > 10 * requests.length && i + batchSize < rows.length) {
+                    const delay = Math.round((60 / (options.rateLimitPerMinute / concurrent)) * 1000);
+                    await (0, sleep_js_1.default)(delay, { start, log: options.verbose });
+                }
+                requests = [];
+            }
+        }
+        return rows;
+    });
+}

package/script/methods/aiRowByRowPool.d.ts

@@ -0,0 +1,36 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function aiRowByRowPool(table: SimpleTable, column: string, newColumn: string | string[], errorColumn: string, prompt: string, poolSize: number, options: {
+    cache?: boolean;
+    batchSize?: number;
+    logProgress?: boolean;
+    verbose?: boolean;
+    includeThoughts?: boolean;
+    test?: (result: {
+        [key: string]: unknown;
+    }) => void;
+    retry?: number;
+    retryCheck?: (error: unknown) => Promise<boolean> | boolean;
+    extraInstructions?: string;
+    minRequestDurationMs?: number;
+    clean?: (response: unknown) => unknown;
+    contextWindow?: number;
+    thinkingBudget?: number;
+    thinkingLevel?: "minimal" | "low" | "medium" | "high";
+    webSearch?: boolean;
+    safetyEnabled?: boolean;
+    schemaJson?: unknown;
+    model?: string;
+    temperature?: number;
+    apiKey?: string;
+    vertex?: boolean;
+    project?: string;
+    location?: string;
+    ollama?: boolean | any;
+    metrics?: {
+        totalCost: number;
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        totalRequests: number;
+    };
+}): Promise<void>;
+//# sourceMappingURL=aiRowByRowPool.d.ts.map

package/script/methods/aiRowByRowPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aiRowByRowPool.d.ts","sourceRoot":"","sources":["../../src/methods/aiRowByRowPool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,wBAA8B,cAAc,CAC1C,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC5D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,KAAK,CAAC,EAAE,CACN,QAAQ,EAAE,OAAO,KACd,OAAO,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,MAAM,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC;IACvB,OAAO,CAAC,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACH,iBA4KF"}

package/script/methods/aiRowByRowPool.js

@@ -0,0 +1,140 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = aiRowByRowPool;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/1.2.18/src/index.js");
+const zod_1 = require("zod");
+async function aiRowByRowPool(table, column, newColumn, errorColumn, prompt, poolSize, options) {
+    const newColumns = Array.isArray(newColumn) ? newColumn : [newColumn];
+    await table.updateWithJS(async (rows) => {
+        if (options.verbose) {
+            console.log("\naiRowByRowPool()");
+        }
+        const batches = [];
+        for (let i = 0; i < rows.length; i += options.batchSize ?? 1) {
+            batches.push(rows.slice(i, i + (options.batchSize ?? 1)));
+        }
+        const requests = batches.map((batch) => {
+            let schemaJson;
+            if (options.schemaJson) {
+                schemaJson = options.schemaJson;
+            }
+            else {
+                const objectSchema = {};
+                for (const newColumn of newColumns) {
+                    objectSchema[newColumn] = (0, zod_1.string)();
+                }
+                schemaJson = (0, zod_1.toJSONSchema)((0, zod_1.array)((0, zod_1.object)(objectSchema)));
+            }
+            const systemPrompt = `You will be provided with a JSON array of ${batch.length} string items. You must return a JSON array containing exactly ${batch.length} objects, in the same corresponding order.`;
+            const batchPrompt = `${prompt}\n\nHere are the ${column} values as a JSON array:\n${JSON.stringify(batch.map((d) => d[column]), null, 2)}\n\n${options.extraInstructions ? `\n${options.extraInstructions}` : ""}`;
+            return {
+                prompt: batchPrompt,
+                options: {
+                    systemPrompt: systemPrompt,
+                    cache: options.cache,
+                    schemaJson,
+                    webSearch: options.webSearch,
+                    clean: options.clean,
+                    model: options.model,
+                    contextWindow: options.contextWindow,
+                    thinkingBudget: options.thinkingBudget,
+                    thinkingLevel: options.thinkingLevel,
+                    metrics: options.metrics,
+                    verbose: options.verbose,
+                    includeThoughts: options.includeThoughts,
+                    temperature: options.temperature,
+                    safetyEnabled: options.safetyEnabled,
+                    apiKey: options.apiKey,
+                    vertex: options.vertex,
+                    project: options.project,
+                    location: options.location,
+                    ollama: options.ollama,
+                    test: (response) => {
+                        if (!Array.isArray(response)) {
+                            throw new Error(`The AI returned a non-array value: ${JSON.stringify(response)}`);
+                        }
+                        if (response.length !== batch.length) {
+                            throw new Error(`The AI returned ${response.length} values, but the batch size is ${batch.length}.`);
+                        }
+                        for (const item of response) {
+                            if (typeof item !== "object" || item === null) {
+                                throw new Error(`The AI did not return an object: ${JSON.stringify(item)}`);
+                            }
+                            for (const newColumn of newColumns) {
+                                if (!(newColumn in item)) {
+                                    throw new Error(`The AI's response is missing the key '${newColumn}': ${JSON.stringify(item)}`);
+                                }
+                            }
+                        }
+                        if (options.test) {
+                            for (const item of response) {
+                                options.test(item);
+                            }
+                        }
+                    },
+                },
+            };
+        });
+        const { results, errors } = await (0, index_js_1.askAIPool)(requests, poolSize, {
+            retry: options.retry,
+            retryCheck: options.retryCheck,
+            minRequestDurationMs: options.minRequestDurationMs,
+            logProgress: options.logProgress,
+            metrics: options.metrics,
+        });
+        const resultsFlattened = results.map((d) => ({
+            index: d.index,
+            result: d.result,
+            error: null,
+        })).flat();
+        const errorsFlattened = errors.map((d) => ({
+            index: d.index,
+            result: null,
+            error: d.error instanceof Error && d.error.message
+                ? d.error.message
+                : String(d.error),
+        })).flat();
+        const resultsAndErrors = [...resultsFlattened, ...errorsFlattened].sort((a, b) => a.index - b.index);
+        const newRows = [];
+        for (let i = 0; i < batches.length; i++) {
+            const resultOrError = resultsAndErrors[i];
+            if (resultOrError.result) {
+                if (Array.isArray(resultOrError.result)) {
+                    for (let j = 0; j < resultOrError.result.length; j++) {
+                        newRows.push({
+                            ...batches[i][j],
+                            ...resultOrError.result[j],
+                            [errorColumn]: null,
+                        });
+                    }
+                }
+                else if (typeof resultOrError.result === "object" &&
+                    resultOrError.result !== null) {
+                    newRows.push({
+                        ...batches[i][0],
+                        ...resultOrError.result,
+                        [errorColumn]: null,
+                    });
+                }
+                else {
+                    throw new Error(`Unexpected result format: ${JSON.stringify(resultOrError.result)}`);
+                }
+            }
+            else if (resultOrError.error) {
+                for (const row of batches[i]) {
+                    const obj = {
+                        [errorColumn]: resultOrError.error,
+                    };
+                    for (const newColumn of newColumns) {
+                        obj[newColumn] = null;
+                    }
+                    newRows.push({
+                        ...row,
+                        ...obj,
+                    });
+                }
+            }
+        }
+        return newRows;
+    });
+}

package/script/methods/aiVectorSimilarity.d.ts

@@ -0,0 +1,14 @@
+import type { SimpleTable } from "../index.js";
+export default function aiVectorSimilarity(simpleTable: SimpleTable, text: string, column: string, nbResults: number, options?: {
+    cache?: boolean;
+    createIndex?: boolean;
+    overwriteIndex?: boolean;
+    outputTable?: string;
+    verbose?: boolean;
+    efConstruction?: number;
+    efSearch?: number;
+    M?: number;
+    minSimilarity?: number;
+    similarityColumn?: string;
+}): Promise<SimpleTable>;
+//# sourceMappingURL=aiVectorSimilarity.d.ts.map

package/script/methods/aiVectorSimilarity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aiVectorSimilarity.d.ts","sourceRoot":"","sources":["../../src/methods/aiVectorSimilarity.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAM/C,wBAA8B,kBAAkB,CAC9C,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACtB,wBAsEP"}

package/script/methods/aiVectorSimilarity.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = aiVectorSimilarity;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/1.2.18/src/index.js");
+const index_js_2 = require("../deps/jsr.io/@nshiab/simple-data-analysis-core/0.0.17/src/helpers/index.js");
+async function aiVectorSimilarity(simpleTable, text, column, nbResults, options = {}) {
+    const textEmbedding = await (0, index_js_1.getEmbedding)(text, options);
+    const types = await simpleTable.getTypes();
+    if (types[column] !== `FLOAT[${textEmbedding.length}]`) {
+        await simpleTable.sdb.customQuery(`ALTER TABLE "${simpleTable.name}" ADD COLUMN "${column}_fixed_floatType" FLOAT[${textEmbedding.length}];
+      UPDATE "${simpleTable.name}" SET "${column}_fixed_floatType" = "${column}"::FLOAT[${textEmbedding.length}];
+      ALTER TABLE "${simpleTable.name}" DROP COLUMN "${column}";
+      ALTER TABLE "${simpleTable.name}" RENAME COLUMN "${column}_fixed_floatType" TO "${column}";`);
+    }
+    if (options.createIndex) {
+        await simpleTable.createVssIndex(column, {
+            overwrite: options.overwriteIndex,
+            verbose: options.verbose,
+            efConstruction: options.efConstruction,
+            efSearch: options.efSearch,
+            M: options.M,
+        });
+    }
+    const targetVector = `${JSON.stringify(textEmbedding)}::FLOAT[${textEmbedding.length}]`;
+    const columnVector = `"${column}"::FLOAT[${textEmbedding.length}]`;
+    const distanceFunction = `array_cosine_distance(${columnVector}, ${targetVector})`;
+    const thresholdClause = options.minSimilarity !== undefined
+        ? `WHERE ${distanceFunction} <= ${1 - options.minSimilarity}`
+        : "";
+    // Conditionally build the SELECT statement to include the similarity math
+    const selectClause = options.similarityColumn
+        ? `*, (1 - ${distanceFunction}) AS "${options.similarityColumn}"`
+        : `*`;
+    await (0, index_js_2.queryDB)(simpleTable, `INSTALL vss; LOAD vss;
+    CREATE OR REPLACE TABLE "${options.outputTable ?? simpleTable.name}" AS 
+    SELECT ${selectClause} FROM "${simpleTable.name}" 
+    ${thresholdClause}
+    ORDER BY ${distanceFunction} ASC
+    LIMIT ${nbResults};`, (0, index_js_2.mergeOptions)(simpleTable, {
+        table: simpleTable.name,
+        method: "aiVectorSimilarity()",
+        parameters: {
+            text,
+            column,
+            nbResults,
+            minSimilarity: options.minSimilarity,
+            similarityColumn: options.similarityColumn, // Include in merged options log
+            table: options.outputTable ?? simpleTable.name,
+        },
+    }));
+    if (typeof options.outputTable === "string") {
+        return simpleTable.sdb.newTable(options.outputTable, structuredClone(simpleTable.projections));
+    }
+    else {
+        return simpleTable;
+    }
+}

package/script/methods/hybridSearch.d.ts

@@ -0,0 +1,38 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function hybridSearch(table: SimpleTable, query: string, columnId: string, columnText: string, nbResults: number, options?: {
+    cache?: boolean;
+    verbose?: boolean;
+    embeddingsModelContextWindow?: number;
+    createIndex?: boolean;
+    embeddingsModel?: string;
+    ollamaEmbeddings?: boolean;
+    embeddingsConcurrent?: number;
+    stemmer?: "arabic" | "basque" | "catalan" | "danish" | "dutch" | "english" | "finnish" | "french" | "german" | "greek" | "hindi" | "hungarian" | "indonesian" | "irish" | "italian" | "lithuanian" | "nepali" | "norwegian" | "porter" | "portuguese" | "romanian" | "russian" | "serbian" | "spanish" | "swedish" | "tamil" | "turkish" | "none";
+    stopwords?: string;
+    ignore?: string;
+    stripAccents?: boolean;
+    lower?: boolean;
+    k?: number;
+    b?: number;
+    conjunctive?: boolean;
+    bm25?: boolean;
+    bm25MinScore?: number;
+    bm25ScoreColumn?: string;
+    vectorSearch?: boolean;
+    vectorMinSimilarity?: number;
+    vectorSimilarityColumn?: string;
+    outputTable?: string;
+    efConstruction?: number;
+    efSearch?: number;
+    M?: number;
+    times?: {
+        start?: number;
+        embeddingStart?: number;
+        embeddingEnd?: number;
+        vectorSearchStart?: number;
+        vectorSearchEnd?: number;
+        bm25Start?: number;
+        bm25End?: number;
+    };
+}): Promise<SimpleTable>;
+//# sourceMappingURL=hybridSearch.d.ts.map

package/script/methods/hybridSearch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybridSearch.d.ts","sourceRoot":"","sources":["../../src/methods/hybridSearch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,wBAA8B,YAAY,CACxC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE;QACN,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACE,GACL,OAAO,CAAC,WAAW,CAAC,CA8VtB"}

package/script/methods/hybridSearch.js

@@ -0,0 +1,241 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = hybridSearch;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-format/1.1.5/src/index.js");
+const getRRFRanking_js_1 = __importDefault(require("../helpers/getRRFRanking.js"));
+const index_js_2 = require("../deps/jsr.io/@nshiab/simple-data-analysis-core/0.0.17/src/helpers/index.js");
+async function hybridSearch(table, query, columnId, columnText, nbResults, options = {}) {
+    const enableVectorSearch = options.vectorSearch !== false;
+    const enableBm25 = options.bm25 !== false;
+    if (!enableVectorSearch && !enableBm25) {
+        throw new Error("At least one search method must be enabled. Set vectorSearch=true or bm25=true.");
+    }
+    const times = options.times ?? {
+        start: Date.now(),
+        embeddingStart: 0,
+        embeddingEnd: 0,
+        vectorSearchStart: 0,
+        vectorSearchEnd: 0,
+        bm25Start: 0,
+        bm25End: 0,
+    };
+    const embeddingColumn = `${columnText}_embeddings`;
+    // Only generate embeddings if vector search is enabled
+    if (enableVectorSearch) {
+        if (options.verbose) {
+            times.embeddingStart = Date.now();
+        }
+        if (await table.hasColumn(embeddingColumn)) {
+            if (options.verbose) {
+                console.log(`"${embeddingColumn}" in table "${table.name}" already exist. Reusing embeddings...`);
+            }
+        }
+        else {
+            const previousCacheVerbose = table.sdb.cacheVerbose;
+            if (options.verbose) {
+                console.log(`"${embeddingColumn}" in table "${table.name}" does not exist. Generating embeddings...`);
+                table.sdb.cacheVerbose = true;
+            }
+            options.cache
+                ? await table.cache(async () => {
+                    await table.aiEmbeddings(columnText, embeddingColumn, {
+                        createIndex: options.createIndex ?? false,
+                        cache: true,
+                        verbose: options.verbose,
+                        ollama: options.ollamaEmbeddings,
+                        model: options.embeddingsModel,
+                        contextWindow: options.embeddingsModelContextWindow,
+                        concurrent: options.embeddingsConcurrent,
+                        efConstruction: options.efConstruction,
+                        efSearch: options.efSearch,
+                        M: options.M,
+                    });
+                })
+                : await table.aiEmbeddings(columnText, embeddingColumn, {
+                    createIndex: options.createIndex ?? false,
+                    verbose: options.verbose,
+                    ollama: options.ollamaEmbeddings,
+                    model: options.embeddingsModel,
+                    contextWindow: options.embeddingsModelContextWindow,
+                    concurrent: options.embeddingsConcurrent,
+                    efConstruction: options.efConstruction,
+                    efSearch: options.efSearch,
+                    M: options.M,
+                });
+            table.sdb.cacheVerbose = previousCacheVerbose;
+        }
+        if (options.verbose) {
+            times.embeddingEnd = Date.now();
+        }
+    }
+    // Run searches based on enabled methods
+    async function vectorSearch() {
+        if (options.verbose) {
+            times.vectorSearchStart = Date.now();
+        }
+        const vectorSearchResult = await table.aiVectorSimilarity(query, embeddingColumn, nbResults, {
+            createIndex: options.createIndex ?? false,
+            cache: options.cache,
+            outputTable: `${table.name}_vector_search_results`,
+            ollama: options.ollamaEmbeddings,
+            model: options.embeddingsModel,
+            contextWindow: options.embeddingsModelContextWindow,
+            verbose: options.verbose,
+            efConstruction: options.efConstruction,
+            efSearch: options.efSearch,
+            M: options.M,
+            minSimilarity: options.vectorMinSimilarity,
+            similarityColumn: options.vectorSimilarityColumn,
+        });
+        if (options.verbose) {
+            times.vectorSearchEnd = Date.now();
+        }
+        return vectorSearchResult;
+    }
+    async function bm25Search() {
+        if (options.verbose) {
+            times.bm25Start = Date.now();
+        }
+        const bm25SearchResult = await table.bm25(query, columnId, columnText, nbResults, {
+            stemmer: options.stemmer,
+            stopwords: options.stopwords,
+            ignore: options.ignore,
+            stripAccents: options.stripAccents,
+            lower: options.lower,
+            k: options.k,
+            b: options.b,
+            conjunctive: options.conjunctive,
+            outputTable: `${table.name}_bm25_search_results`,
+            verbose: options.verbose,
+            minScore: options.bm25MinScore,
+            scoreColumn: options.bm25ScoreColumn,
+        });
+        if (options.verbose) {
+            times.bm25End = Date.now();
+        }
+        return bm25SearchResult;
+    }
+    let finalIds;
+    let vectorSearchSimilarity = [];
+    let bm25SearchScores = [];
+    if (enableVectorSearch && enableBm25) {
+        // Both methods enabled: run in parallel and fuse results
+        const [vectorSearchResult, bm25SearchResult] = await Promise.all([
+            vectorSearch(),
+            bm25Search(),
+        ]);
+        const vectorSearchResultsIds = await vectorSearchResult.getValues(columnId);
+        if (options.vectorSimilarityColumn) {
+            vectorSearchSimilarity = await vectorSearchResult.getData({
+                columns: [columnId, options.vectorSimilarityColumn],
+            });
+        }
+        const bm25SearchResultsIds = await bm25SearchResult.getValues(columnId);
+        if (options.bm25ScoreColumn) {
+            bm25SearchScores = await bm25SearchResult.getData({
+                columns: [columnId, options.bm25ScoreColumn],
+            });
+        }
+        await vectorSearchResult.removeTable();
+        await bm25SearchResult.removeTable();
+        if (options.verbose) {
+            console.log(`Vector search results IDs:`, vectorSearchResultsIds);
+            console.log(`BM25 search results IDs:`, bm25SearchResultsIds);
+        }
+        finalIds = (0, getRRFRanking_js_1.default)([vectorSearchResultsIds, bm25SearchResultsIds]);
+        if (options.verbose) {
+            console.log(`Fused results IDs:`, finalIds);
+        }
+    }
+    else if (enableVectorSearch) {
+        // Only vector search enabled
+        const vectorSearchResult = await vectorSearch();
+        finalIds = await vectorSearchResult.getValues(columnId);
+        if (options.vectorSimilarityColumn) {
+            vectorSearchSimilarity = await vectorSearchResult.getData({
+                columns: [columnId, options.vectorSimilarityColumn],
+            });
+        }
+        await vectorSearchResult.removeTable();
+        if (options.verbose) {
+            console.log(`Vector search results IDs:`, finalIds);
+        }
+    }
+    else {
+        // Only BM25 enabled
+        const bm25SearchResult = await bm25Search();
+        finalIds = await bm25SearchResult.getValues(columnId);
+        if (options.bm25ScoreColumn) {
+            bm25SearchScores = await bm25SearchResult.getData({
+                columns: [columnId, options.bm25ScoreColumn],
+            });
+        }
+        await bm25SearchResult.removeTable();
+        if (options.verbose) {
+            console.log(`BM25 search results IDs:`, finalIds);
+        }
+    }
+    const finalIdsSliced = finalIds.slice(0, nbResults);
+    if (finalIdsSliced.length === 0) {
+        // If there are no results, create an empty table with the same structure
+        await table.sdb.customQuery(`CREATE OR REPLACE TABLE "${options.outputTable ?? table.name}" AS SELECT * FROM "${table.name}" WHERE 1=0`);
+    }
+    else {
+        await table.sdb.customQuery(`CREATE OR REPLACE TABLE "${options.outputTable ?? table.name}" AS SELECT * FROM "${table.name}" WHERE "${columnId}" IN (${finalIdsSliced
+            .map((id) => (0, index_js_2.parseValue)(id))
+            .join(", ")})`);
+    }
+    let outputTableInstance;
+    if (typeof options.outputTable === "string") {
+        outputTableInstance = table.sdb.newTable(options.outputTable, structuredClone(table.projections));
+    }
+    else {
+        outputTableInstance = table;
+    }
+    if (options.vectorSimilarityColumn) {
+        if (vectorSearchSimilarity.length === 0) {
+            // If there are no similarity scores (e.g. all results were filtered out by minSimilarity), add the column with NULL values
+            await outputTableInstance.addColumn(options.vectorSimilarityColumn, "number", `NULL`);
+        }
+        else {
+            await outputTableInstance.addColumn(options.vectorSimilarityColumn, "number", `CASE ${vectorSearchSimilarity.map((d) => `WHEN "${columnId}" = ${(0, index_js_2.parseValue)(d[columnId])} THEN ${d[options.vectorSimilarityColumn]}`).join(" ")} ELSE NULL END`);
+            await outputTableInstance.round(options.vectorSimilarityColumn, {
+                decimals: 4,
+            });
+        }
+    }
+    if (options.bm25ScoreColumn) {
+        if (bm25SearchScores.length === 0) {
+            // If there are no BM25 scores (e.g. all results were filtered out by minScore), add the column with NULL values
+            await outputTableInstance.addColumn(options.bm25ScoreColumn, "number", `NULL`);
+        }
+        else {
+            await outputTableInstance.addColumn(options.bm25ScoreColumn, "number", `CASE ${bm25SearchScores.map((d) => `WHEN "${columnId}" = ${(0, index_js_2.parseValue)(d[columnId])} THEN ${d[options.bm25ScoreColumn]}`).join(" ")} ELSE NULL END`);
+            await outputTableInstance.round(options.bm25ScoreColumn, {
+                decimals: 4,
+            });
+        }
+    }
+    if (options.verbose) {
+        await outputTableInstance.logTable("all");
+        const logParts = [`\nHybrid search times:`];
+        if (enableVectorSearch) {
+            logParts.push(`- Embedding: ${(0, index_js_1.prettyDuration)(times.embeddingStart, { end: times.embeddingEnd })}`);
+        }
+        const parallelLabel = enableVectorSearch && enableBm25 ? " (parallel)" : "";
+        logParts.push(enableVectorSearch
+            ? `- Vector Search${parallelLabel}: ${(0, index_js_1.prettyDuration)(times.vectorSearchStart, {
+                end: times.vectorSearchEnd,
+            })}`
+            : `- Vector Search: disabled`);
+        logParts.push(enableBm25
+            ? `- BM25 Search${parallelLabel}: ${(0, index_js_1.prettyDuration)(times.bm25Start, { end: times.bm25End })}`
+            : `- BM25 Search: disabled`);
+        logParts.push(`- Total: ${(0, index_js_1.prettyDuration)(times.start)} (verbose adds some overhead)`);
+        console.log(logParts.join("\n") + "\n");
+    }
+    return outputTableInstance;
+}

package/script/methods/logHistogram.d.ts

@@ -0,0 +1,8 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function logHistogram(simpleTable: SimpleTable, values: string, options?: {
+    bins?: number;
+    formatLabels?: (a: number, b: number) => string;
+    compact?: boolean;
+    width?: number;
+}): Promise<void>;
+//# sourceMappingURL=logHistogram.d.ts.map

package/script/methods/logHistogram.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logHistogram.d.ts","sourceRoot":"","sources":["../../src/methods/logHistogram.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC;IAChD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,iBAmDP"}