@nshiab/simple-data-analysis 5.21.0 → 6.0.0

package/esm/methods/hybridSearch.d.ts

@@ -1,38 +1,88 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function hybridSearch(table: SimpleTable, query: string, columnId: string, columnText: string, nbResults: number, options?: {
-    cache?: boolean;
+import { type EmbeddingOptions } from "../helpers/aiOptions.js";
+/**
+ * Timing checkpoints populated by `hybridSearch` when verbose logging is enabled.
+ *
+ * @example
+ * ```ts
+ * const times: HybridSearchTimes = { start: Date.now() };
+ * ```
+ */
+export type HybridSearchTimes = {
+    /** Start of the complete hybrid-search operation. */
+    start?: number;
+    /** Start of embedding generation or cache loading. */
+    embeddingStart?: number;
+    /** End of embedding generation or cache loading. */
+    embeddingEnd?: number;
+    /** Start of vector-similarity search. */
+    vectorSearchStart?: number;
+    /** End of vector-similarity search. */
+    vectorSearchEnd?: number;
+    /** Start of BM25 text search. */
+    bm25Start?: number;
+    /** End of BM25 text search. */
+    bm25End?: number;
+};
+/**
+ * Options for combining vector similarity and BM25 text search.
+ *
+ * @example
+ * ```ts
+ * const options: HybridSearchOptions = {
+ *   embeddings: { provider: "ollama" },
+ *   vectorSearch: true,
+ *   bm25: true,
+ * };
+ * ```
+ */
+export type HybridSearchOptions = {
+    /** Options used for both stored row embeddings and the query embedding. */
+    embeddings?: EmbeddingOptions;
+    /** Logs search progress, results, and timing information when enabled. */
     verbose?: boolean;
-    embeddingsModelContextWindow?: number;
+    /** Creates an HNSW index when vector search is enabled. BM25 manages its FTS index automatically. */
     createIndex?: boolean;
-    embeddingsModel?: string;
-    ollamaEmbeddings?: boolean;
-    embeddingsConcurrent?: number;
+    /** Maximum number of stored-row embedding requests processed concurrently. */
+    embeddingsConcurrency?: number;
+    /** Language stemmer used by the BM25 full-text index. */
     stemmer?: "arabic" | "basque" | "catalan" | "danish" | "dutch" | "english" | "finnish" | "french" | "german" | "greek" | "hindi" | "hungarian" | "indonesian" | "irish" | "italian" | "lithuanian" | "nepali" | "norwegian" | "porter" | "portuguese" | "romanian" | "russian" | "serbian" | "spanish" | "swedish" | "tamil" | "turkish" | "none";
+    /** Stopword table used by the BM25 full-text index. */
     stopwords?: string;
+    /** Regular expression describing characters ignored by the BM25 index. */
     ignore?: string;
+    /** Removes accents before BM25 indexing when enabled. */
     stripAccents?: boolean;
+    /** Lowercases text before BM25 indexing when enabled. */
     lower?: boolean;
+    /** BM25 term-frequency saturation parameter. */
     k?: number;
+    /** BM25 document-length normalization parameter. */
     b?: number;
+    /** Requires every query term to match during BM25 search when enabled. */
     conjunctive?: boolean;
+    /** Enables BM25 text search. */
     bm25?: boolean;
+    /** Minimum BM25 score required for a row to be returned. */
     bm25MinScore?: number;
+    /** Adds BM25 scores under this output column name. */
     bm25ScoreColumn?: string;
+    /** Enables vector-similarity search. */
     vectorSearch?: boolean;
+    /** Minimum cosine similarity required for a vector result. */
     vectorMinSimilarity?: number;
+    /** Adds vector similarity scores under this output column name. */
     vectorSimilarityColumn?: string;
+    /** Writes results to a new table instead of replacing the current table. */
     outputTable?: string;
+    /** Candidate count used while constructing the vector index. */
     efConstruction?: number;
+    /** Candidate count used while searching the vector index. */
     efSearch?: number;
+    /** Maximum number of graph neighbors retained by the vector index. */
     M?: number;
-    times?: {
-        start?: number;
-        embeddingStart?: number;
-        embeddingEnd?: number;
-        vectorSearchStart?: number;
-        vectorSearchEnd?: number;
-        bm25Start?: number;
-        bm25End?: number;
-    };
-}): Promise<SimpleTable>;
+    /** Mutable timing checkpoints shared with `aiRAG`. */
+    times?: HybridSearchTimes;
+};
+export default function hybridSearch(table: SimpleTable, query: string, idColumn: string, textColumn: string, nbResults: number, options?: HybridSearchOptions): SimpleTable;
 //# sourceMappingURL=hybridSearch.d.ts.map

package/esm/methods/hybridSearch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybridSearch.d.ts","sourceRoot":"","sources":["../../src/methods/hybridSearch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,wBAA8B,YAAY,CACxC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE;QACN,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACE,GACL,OAAO,CAAC,WAAW,CAAC,CA6VtB"}
+{"version":3,"file":"hybridSearch.d.ts","sourceRoot":"","sources":["../../src/methods/hybridSearch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,OAAO,EACL,KAAK,gBAAgB,EAEtB,MAAM,yBAAyB,CAAC;AAKjC;;;;;;;GAOG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,yCAAyC;IACzC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uCAAuC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,0EAA0E;IAC1E,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qGAAqG;IACrG,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,8EAA8E;IAC9E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,yDAAyD;IACzD,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,yDAAyD;IACzD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,gDAAgD;IAChD,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,oDAAoD;IACpD,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,0EAA0E;IAC1E,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sDAAsD;IACtD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,8DAA8D;IAC9D,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mEAAmE;IACnE,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,sDAAsD;IACtD,KAAK,CAAC,EAAE,iBAAiB,CAAC;CAC3B,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,mBAAwB,GAChC,WAAW,CA0Bb"}

package/esm/methods/hybridSearch.js

@@ -1,7 +1,28 @@
-import { prettyDuration } from "../deps/jsr.io/@nshiab/journalism-format/1.1.9/src/index.js";
 import getRRFRanking from "../helpers/getRRFRanking.js";
-import { parseValue } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/1.0.1/src/helpers/index.js";
-export default async function hybridSearch(table, query, columnId, columnText, nbResults, options = {}) {
+import { parseValue, queueAsyncBarrier, } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js";
+import { snapshotAIOptions, } from "../helpers/aiOptions.js";
+import ensureEmbeddingColumn from "../helpers/ensureEmbeddingColumn.js";
+import { generateEmbeddingColumn } from "./aiEmbeddings.js";
+import { runAIVectorSimilarity } from "./aiVectorSimilarity.js";
+export default function hybridSearch(table, query, idColumn, textColumn, nbResults, options = {}) {
+    options = snapshotAIOptions(options);
+    const outputTable = options.outputTable === undefined
+        ? table
+        : table.sdb.newTable(options.outputTable);
+    queueAsyncBarrier(outputTable, {
+        method: "hybridSearch()",
+        parameters: {
+            query,
+            idColumn,
+            textColumn,
+            nbResults,
+            outputTable: options.outputTable,
+        },
+        execute: () => runHybridSearch(table, query, idColumn, textColumn, nbResults, options, outputTable),
+    });
+    return outputTable;
+}
+async function runHybridSearch(table, query, idColumn, textColumn, nbResults, options, outputTableInstance) {
     const enableVectorSearch = options.vectorSearch !== false;
     const enableBm25 = options.bm25 !== false;
     if (!enableVectorSearch && !enableBm25) {
@@ -16,49 +37,60 @@ export default async function hybridSearch(table, query, columnId, columnText, n
         bm25Start: 0,
         bm25End: 0,
     };
-    const embeddingColumn = `${columnText}_embeddings`;
+    const embeddingColumn = `${textColumn}_embeddings`;
     // Only generate embeddings if vector search is enabled
     if (enableVectorSearch) {
+        const { getEmbeddingIdentity } = await import("../deps/jsr.io/@nshiab/journalism-ai/2.0.6/src/index.js");
         if (options.verbose) {
             times.embeddingStart = Date.now();
         }
-        if (await table.hasColumn(embeddingColumn)) {
+        const previousCacheVerbose = table.sdb.cacheVerbose;
+        try {
             if (options.verbose) {
-                console.log(`"${embeddingColumn}" in table "${table.name}" already exist. Reusing embeddings...`);
+                table.sdb.cacheVerbose = true;
             }
-        }
-        else {
-            const previousCacheVerbose = table.sdb.cacheVerbose;
+            const embeddingOptions = {
+                verbose: options.verbose,
+                embeddings: options.embeddings,
+                concurrency: options.embeddingsConcurrency,
+            };
+            const computeEmbeddings = async () => {
+                await generateEmbeddingColumn(table, textColumn, embeddingColumn, embeddingOptions);
+            };
+            const identity = getEmbeddingIdentity(options.embeddings);
+            const embeddingStatus = await ensureEmbeddingColumn(table, textColumn, embeddingColumn, identity, async () => {
+                if (options.embeddings?.cache === false) {
+                    await computeEmbeddings();
+                    return;
+                }
+                const computeSource = Function.prototype.toString.call(computeEmbeddings);
+                const cacheProvenance = {
+                    identity,
+                    sourceColumn: textColumn,
+                    embeddingColumn,
+                };
+                Object.defineProperty(computeEmbeddings, "toString", {
+                    value: () => `${computeSource}\n${JSON.stringify(cacheProvenance)}`,
+                });
+                await table.cache(computeEmbeddings);
+            });
             if (options.verbose) {
-                console.log(`"${embeddingColumn}" in table "${table.name}" does not exist. Generating embeddings...`);
-                table.sdb.cacheVerbose = true;
+                console.log(embeddingStatus === "reused"
+                    ? `"${embeddingColumn}" in table "${table.name}" has compatible provenance. Reusing embeddings...`
+                    : `Generated compatible embeddings in "${embeddingColumn}" for table "${table.name}".`);
             }
-            options.cache
-                ? await table.cache(async () => {
-                    await table.aiEmbeddings(columnText, embeddingColumn, {
-                        createIndex: options.createIndex ?? false,
-                        cache: true,
-                        verbose: options.verbose,
-                        ollama: options.ollamaEmbeddings,
-                        model: options.embeddingsModel,
-                        contextWindow: options.embeddingsModelContextWindow,
-                        concurrent: options.embeddingsConcurrent,
-                        efConstruction: options.efConstruction,
-                        efSearch: options.efSearch,
-                        M: options.M,
-                    });
-                })
-                : await table.aiEmbeddings(columnText, embeddingColumn, {
-                    createIndex: options.createIndex ?? false,
+            if (options.createIndex) {
+                table.createVssIndex(embeddingColumn, {
+                    overwrite: false,
                     verbose: options.verbose,
-                    ollama: options.ollamaEmbeddings,
-                    model: options.embeddingsModel,
-                    contextWindow: options.embeddingsModelContextWindow,
-                    concurrent: options.embeddingsConcurrent,
                     efConstruction: options.efConstruction,
                     efSearch: options.efSearch,
                     M: options.M,
                 });
+                await table.run();
+            }
+        }
+        finally {
             table.sdb.cacheVerbose = previousCacheVerbose;
         }
         if (options.verbose) {
@@ -70,13 +102,11 @@ export default async function hybridSearch(table, query, columnId, columnText, n
         if (options.verbose) {
             times.vectorSearchStart = Date.now();
         }
-        const vectorSearchResult = await table.aiVectorSimilarity(query, embeddingColumn, nbResults, {
+        const vectorSearchResult = table.sdb.newTable(`${table.name}_vector_search_results`);
+        await runAIVectorSimilarity(table, query, embeddingColumn, nbResults, {
             createIndex: options.createIndex ?? false,
-            cache: options.cache,
             outputTable: `${table.name}_vector_search_results`,
-            ollama: options.ollamaEmbeddings,
-            model: options.embeddingsModel,
-            contextWindow: options.embeddingsModelContextWindow,
+            embeddings: options.embeddings,
             verbose: options.verbose,
             efConstruction: options.efConstruction,
             efSearch: options.efSearch,
@@ -93,7 +123,7 @@ export default async function hybridSearch(table, query, columnId, columnText, n
         if (options.verbose) {
             times.bm25Start = Date.now();
         }
-        const bm25SearchResult = await table.bm25(query, columnId, columnText, nbResults, {
+        const bm25SearchResult = table.bm25(query, idColumn, textColumn, nbResults, {
             stemmer: options.stemmer,
             stopwords: options.stopwords,
             ignore: options.ignore,
@@ -107,6 +137,7 @@ export default async function hybridSearch(table, query, columnId, columnText, n
             minScore: options.bm25MinScore,
             scoreColumn: options.bm25ScoreColumn,
         });
+        await bm25SearchResult.run();
         if (options.verbose) {
             times.bm25End = Date.now();
         }
@@ -121,16 +152,16 @@ export default async function hybridSearch(table, query, columnId, columnText, n
             vectorSearch(),
             bm25Search(),
         ]);
-        const vectorSearchResultsIds = await vectorSearchResult.getValues(columnId);
+        const vectorSearchResultsIds = await vectorSearchResult.getValues(idColumn);
         if (options.vectorSimilarityColumn) {
             vectorSearchSimilarity = await vectorSearchResult.getData({
-                columns: [columnId, options.vectorSimilarityColumn],
+                columns: [idColumn, options.vectorSimilarityColumn],
             });
         }
-        const bm25SearchResultsIds = await bm25SearchResult.getValues(columnId);
+        const bm25SearchResultsIds = await bm25SearchResult.getValues(idColumn);
         if (options.bm25ScoreColumn) {
             bm25SearchScores = await bm25SearchResult.getData({
-                columns: [columnId, options.bm25ScoreColumn],
+                columns: [idColumn, options.bm25ScoreColumn],
             });
         }
         await vectorSearchResult.removeTable();
@@ -147,10 +178,10 @@ export default async function hybridSearch(table, query, columnId, columnText, n
     else if (enableVectorSearch) {
         // Only vector search enabled
         const vectorSearchResult = await vectorSearch();
-        finalIds = await vectorSearchResult.getValues(columnId);
+        finalIds = await vectorSearchResult.getValues(idColumn);
         if (options.vectorSimilarityColumn) {
             vectorSearchSimilarity = await vectorSearchResult.getData({
-                columns: [columnId, options.vectorSimilarityColumn],
+                columns: [idColumn, options.vectorSimilarityColumn],
             });
         }
         await vectorSearchResult.removeTable();
@@ -161,10 +192,10 @@ export default async function hybridSearch(table, query, columnId, columnText, n
     else {
         // Only BM25 enabled
         const bm25SearchResult = await bm25Search();
-        finalIds = await bm25SearchResult.getValues(columnId);
+        finalIds = await bm25SearchResult.getValues(idColumn);
         if (options.bm25ScoreColumn) {
             bm25SearchScores = await bm25SearchResult.getData({
-                columns: [columnId, options.bm25ScoreColumn],
+                columns: [idColumn, options.bm25ScoreColumn],
             });
         }
         await bm25SearchResult.removeTable();
@@ -178,25 +209,18 @@ export default async function hybridSearch(table, query, columnId, columnText, n
         await table.sdb.customQuery(`CREATE OR REPLACE TABLE "${options.outputTable ?? table.name}" AS SELECT * FROM "${table.name}" WHERE 1=0`);
     }
     else {
-        await table.sdb.customQuery(`CREATE OR REPLACE TABLE "${options.outputTable ?? table.name}" AS SELECT * FROM "${table.name}" WHERE "${columnId}" IN (${finalIdsSliced
+        await table.sdb.customQuery(`CREATE OR REPLACE TABLE "${options.outputTable ?? table.name}" AS SELECT * FROM "${table.name}" WHERE "${idColumn}" IN (${finalIdsSliced
             .map((id) => parseValue(id))
             .join(", ")})`);
     }
-    let outputTableInstance;
-    if (typeof options.outputTable === "string") {
-        outputTableInstance = table.sdb.newTable(options.outputTable);
-    }
-    else {
-        outputTableInstance = table;
-    }
     if (options.vectorSimilarityColumn) {
         if (vectorSearchSimilarity.length === 0) {
             // If there are no similarity scores (e.g. all results were filtered out by minSimilarity), add the column with NULL values
-            await outputTableInstance.addColumn(options.vectorSimilarityColumn, "number", `NULL`);
+            outputTableInstance.addColumn(options.vectorSimilarityColumn, "number", `NULL`);
         }
         else {
-            await outputTableInstance.addColumn(options.vectorSimilarityColumn, "number", `CASE ${vectorSearchSimilarity.map((d) => `WHEN "${columnId}" = ${parseValue(d[columnId])} THEN ${d[options.vectorSimilarityColumn]}`).join(" ")} ELSE NULL END`);
-            await outputTableInstance.round(options.vectorSimilarityColumn, {
+            outputTableInstance.addColumn(options.vectorSimilarityColumn, "number", `CASE ${vectorSearchSimilarity.map((d) => `WHEN "${idColumn}" = ${parseValue(d[idColumn])} THEN ${d[options.vectorSimilarityColumn]}`).join(" ")} ELSE NULL END`);
+            outputTableInstance.round(options.vectorSimilarityColumn, {
                 decimals: 4,
             });
         }
@@ -204,17 +228,19 @@ export default async function hybridSearch(table, query, columnId, columnText, n
     if (options.bm25ScoreColumn) {
         if (bm25SearchScores.length === 0) {
             // If there are no BM25 scores (e.g. all results were filtered out by minScore), add the column with NULL values
-            await outputTableInstance.addColumn(options.bm25ScoreColumn, "number", `NULL`);
+            outputTableInstance.addColumn(options.bm25ScoreColumn, "number", `NULL`);
         }
         else {
-            await outputTableInstance.addColumn(options.bm25ScoreColumn, "number", `CASE ${bm25SearchScores.map((d) => `WHEN "${columnId}" = ${parseValue(d[columnId])} THEN ${d[options.bm25ScoreColumn]}`).join(" ")} ELSE NULL END`);
-            await outputTableInstance.round(options.bm25ScoreColumn, {
+            outputTableInstance.addColumn(options.bm25ScoreColumn, "number", `CASE ${bm25SearchScores.map((d) => `WHEN "${idColumn}" = ${parseValue(d[idColumn])} THEN ${d[options.bm25ScoreColumn]}`).join(" ")} ELSE NULL END`);
+            outputTableInstance.round(options.bm25ScoreColumn, {
                 decimals: 4,
             });
         }
     }
+    await outputTableInstance.run();
     if (options.verbose) {
-        await outputTableInstance.logTable("all");
+        await outputTableInstance.log("all");
+        const { prettyDuration } = await import("../deps/jsr.io/@nshiab/journalism-format/1.1.10/src/index.js");
         const logParts = [`\nHybrid search times:`];
         if (enableVectorSearch) {
             logParts.push(`- Embedding: ${prettyDuration(times.embeddingStart, { end: times.embeddingEnd })}`);
@@ -231,5 +257,4 @@ export default async function hybridSearch(table, query, columnId, columnText, n
         logParts.push(`- Total: ${prettyDuration(times.start)} (verbose adds some overhead)`);
         console.log(logParts.join("\n") + "\n");
     }
-    return outputTableInstance;
 }

package/esm/methods/loadDatawrapper.d.ts

@@ -0,0 +1,7 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type LoadDatawrapperOptions = {
+    apiKeyEnvVar?: string;
+};
+export default function loadDatawrapper(table: SimpleTable, chartId: string, options?: LoadDatawrapperOptions): SimpleTable;
+export {};
+//# sourceMappingURL=loadDatawrapper.d.ts.map

package/esm/methods/loadDatawrapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadDatawrapper.d.ts","sourceRoot":"","sources":["../../src/methods/loadDatawrapper.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,KAAK,sBAAsB,GAAG;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,eAAe,CACrC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,sBAA2B,GACnC,WAAW,CAeb"}

package/esm/methods/loadDatawrapper.js

@@ -0,0 +1,17 @@
+import { queueAsyncBarrier } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js";
+export default function loadDatawrapper(table, chartId, options = {}) {
+    options = { ...options };
+    queueAsyncBarrier(table, {
+        method: "loadDatawrapper()",
+        parameters: { chartId, options },
+        execute: async () => {
+            const { getDataDW } = await import("../deps/jsr.io/@nshiab/journalism-dataviz/1.0.21/src/index.js");
+            const data = await getDataDW(chartId, {
+                parse: true,
+                apiKey: options.apiKeyEnvVar,
+            });
+            table.loadArray(data);
+        },
+    });
+    return table;
+}

package/esm/methods/loadGeoDatawrapper.d.ts

@@ -0,0 +1,7 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type LoadGeoDatawrapperOptions = {
+    apiKeyEnvVar?: string;
+};
+export default function loadGeoDatawrapper(table: SimpleTable, chartId: string, options?: LoadGeoDatawrapperOptions): SimpleTable;
+export {};
+//# sourceMappingURL=loadGeoDatawrapper.d.ts.map

package/esm/methods/loadGeoDatawrapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadGeoDatawrapper.d.ts","sourceRoot":"","sources":["../../src/methods/loadGeoDatawrapper.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,KAAK,yBAAyB,GAAG;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,kBAAkB,CACxC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,yBAA8B,GACtC,WAAW,CAcb"}

package/esm/methods/loadGeoDatawrapper.js

@@ -0,0 +1,17 @@
+import { queueAsyncBarrier } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js";
+import loadGeoDataFromScratchFile from "../helpers/loadGeoDataFromScratchFile.js";
+export default function loadGeoDatawrapper(table, chartId, options = {}) {
+    options = { ...options };
+    queueAsyncBarrier(table, {
+        method: "loadGeoDatawrapper()",
+        parameters: { chartId, options },
+        execute: async () => {
+            const { getDataDW } = await import("../deps/jsr.io/@nshiab/journalism-dataviz/1.0.21/src/index.js");
+            const jsonString = await getDataDW(chartId, {
+                apiKey: options.apiKeyEnvVar,
+            });
+            await loadGeoDataFromScratchFile(table, jsonString);
+        },
+    });
+    return table;
+}

package/esm/methods/loadSheet.d.ts

@@ -0,0 +1,9 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type LoadSheetOptions = {
+    skip?: number;
+    apiEmailEnvVar?: string;
+    apiKeyEnvVar?: string;
+};
+export default function loadSheet(table: SimpleTable, sheetUrl: string, options?: LoadSheetOptions): SimpleTable;
+export {};
+//# sourceMappingURL=loadSheet.d.ts.map

package/esm/methods/loadSheet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadSheet.d.ts","sourceRoot":"","sources":["../../src/methods/loadSheet.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,KAAK,gBAAgB,GAAG;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,gBAAqB,GAC7B,WAAW,CAkBb"}

package/esm/methods/loadSheet.js

@@ -0,0 +1,18 @@
+import { queueAsyncBarrier } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js";
+export default function loadSheet(table, sheetUrl, options = {}) {
+    options = { ...options };
+    queueAsyncBarrier(table, {
+        method: "loadSheet()",
+        parameters: { sheetUrl, options },
+        execute: async () => {
+            const { getSheetData } = await import("../deps/jsr.io/@nshiab/journalism-google/1.1.2/src/index.js");
+            const { apiEmailEnvVar, apiKeyEnvVar, ...sheetOptions } = options;
+            table.loadArray(await getSheetData(sheetUrl, {
+                ...sheetOptions,
+                apiEmail: apiEmailEnvVar,
+                apiKey: apiKeyEnvVar,
+            }));
+        },
+    });
+    return table;
+}

package/esm/methods/logHistogram.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"logHistogram.d.ts","sourceRoot":"","sources":["../../src/methods/logHistogram.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC;IAChD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,iBAmDP"}
+{"version":3,"file":"logHistogram.d.ts","sourceRoot":"","sources":["../../src/methods/logHistogram.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC;IAChD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,iBAoDP"}

package/esm/methods/logHistogram.js

@@ -1,4 +1,3 @@
-import { logBarChart } from "../deps/jsr.io/@nshiab/journalism-dataviz/1.0.17/src/index.js";
 export default async function logHistogram(simpleTable, values, options = {}) {
     const bins = options.bins ?? 10;
     const formatLabels = options.formatLabels ?? ((a, b) => `[${a} | ${b})`);
@@ -26,7 +25,8 @@ SELECT
   bin_end,
   frequency
 FROM histogram
-ORDER BY bin_start;`, { returnDataFrom: "query" });
+ORDER BY bin_start;`, { returnData: true });
+    const { logBarChart } = await import("../deps/jsr.io/@nshiab/journalism-dataviz/1.0.21/src/index.js");
     logBarChart(data.map((d) => ({
         binRange: formatLabels(parseFloat(d.bin_start.toFixed(10)), parseFloat(d.bin_end.toFixed(10))),
         frequency: d.frequency,

package/esm/methods/toDatawrapper.d.ts

@@ -0,0 +1,9 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type ToDatawrapperOptions = {
+    apiKeyEnvVar?: string;
+    note?: string;
+    republish?: boolean;
+};
+export default function toDatawrapper(table: SimpleTable, chartId: string, options?: ToDatawrapperOptions): Promise<void>;
+export {};
+//# sourceMappingURL=toDatawrapper.d.ts.map

package/esm/methods/toDatawrapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toDatawrapper.d.ts","sourceRoot":"","sources":["../../src/methods/toDatawrapper.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,KAAK,oBAAoB,GAAG;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,wBAA8B,aAAa,CACzC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CAGf"}

package/esm/methods/toDatawrapper.js

@@ -0,0 +1,5 @@
+import publishDatawrapper from "../helpers/publishDatawrapper.js";
+export default async function toDatawrapper(table, chartId, options = {}) {
+    const data = await table.getDataAsCSV();
+    await publishDatawrapper(chartId, data, options);
+}

package/esm/methods/toGeoDatawrapper.d.ts

@@ -0,0 +1,10 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type ToGeoDatawrapperOptions = {
+    apiKeyEnvVar?: string;
+    column?: string;
+    note?: string;
+    republish?: boolean;
+};
+export default function toGeoDatawrapper(table: SimpleTable, chartId: string, options?: ToGeoDatawrapperOptions): Promise<void>;
+export {};
+//# sourceMappingURL=toGeoDatawrapper.d.ts.map

package/esm/methods/toGeoDatawrapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toGeoDatawrapper.d.ts","sourceRoot":"","sources":["../../src/methods/toGeoDatawrapper.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,KAAK,uBAAuB,GAAG;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,wBAA8B,gBAAgB,CAC5C,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,uBAA4B,GACpC,OAAO,CAAC,IAAI,CAAC,CAGf"}

package/esm/methods/toGeoDatawrapper.js

@@ -0,0 +1,5 @@
+import publishDatawrapper from "../helpers/publishDatawrapper.js";
+export default async function toGeoDatawrapper(table, chartId, options = {}) {
+    const geoData = await table.getGeoData(options.column);
+    await publishDatawrapper(chartId, JSON.stringify(geoData), options);
+}