@nshiab/simple-data-analysis 5.21.1 → 6.0.1

package/esm/methods/aiQuery.d.ts

@@ -1,21 +1,27 @@
 import type { SimpleTable } from "../index.js";
-import type { Ollama } from "ollama";
-export default function aiQuery(simpleTable: SimpleTable, prompt: string, options?: {
+import { type UnstructuredGenerationOptions } from "../helpers/aiOptions.js";
+/**
+ * Options for generating and executing a SQL query.
+ *
+ * @example
+ * ```ts
+ * const options: AIQueryOptions = {
+ *   generation: { provider: "gemini" },
+ *   outputTable: "results",
+ * };
+ * ```
+ */
+export type AIQueryOptions = {
+    /** Additional requirements appended to the SQL-generation prompt. */
     extraInstructions?: string;
-    cache?: boolean;
-    model?: string;
-    apiKey?: string;
-    vertex?: boolean;
-    project?: string;
+    /** Generation options excluding `schemaJson`, which is owned by SDA. */
+    generation?: UnstructuredGenerationOptions;
+    /** Includes model thoughts in verbose logs when the provider returns them. */
     includeThoughts?: boolean;
-    location?: string;
-    ollama?: boolean | Ollama;
-    contextWindow?: number;
-    thinkingBudget?: number;
-    thinkingLevel?: "minimal" | "low" | "medium" | "high";
-    temperature?: number;
-    safetyEnabled?: boolean;
+    /** Writes results to a new table instead of replacing the current table. */
     outputTable?: string;
+    /** Logs the request and provider response when enabled. */
     verbose?: boolean;
-}): Promise<void>;
+};
+export default function aiQuery(simpleTable: SimpleTable, prompt: string, options?: AIQueryOptions): SimpleTable;
 //# sourceMappingURL=aiQuery.d.ts.map

package/esm/methods/aiQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"aiQuery.d.ts","sourceRoot":"","sources":["../../src/methods/aiQuery.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGrC,wBAA8B,OAAO,CACnC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,iBA8CP"}
+{"version":3,"file":"aiQuery.d.ts","sourceRoot":"","sources":["../../src/methods/aiQuery.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;GAUG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,8EAA8E;IAC9E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,cAAmB,GAC3B,WAAW,CAWb"}

package/esm/methods/aiQuery.js

@@ -1,30 +1,36 @@
-import { askAI } from "../deps/jsr.io/@nshiab/journalism-ai/1.2.23/src/index.js";
-import { object, string, toJSONSchema } from "zod";
-export default async function aiQuery(simpleTable, prompt, options = {}) {
+import { snapshotAIOptions, } from "../helpers/aiOptions.js";
+import { queueAsyncBarrier } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js";
+export default function aiQuery(simpleTable, prompt, options = {}) {
+    options = snapshotAIOptions(options);
+    const outputTable = options.outputTable === undefined
+        ? simpleTable
+        : simpleTable.sdb.newTable(options.outputTable);
+    queueAsyncBarrier(outputTable, {
+        method: "aiQuery()",
+        parameters: { prompt, outputTable: options.outputTable },
+        execute: () => runAIQuery(simpleTable, prompt, options),
+    });
+    return outputTable;
+}
+async function runAIQuery(simpleTable, prompt, options) {
     const tableName = options.outputTable ?? simpleTable.name;
     const p = `I have a SQL table named "${simpleTable.name}". The data is already in it with these columns:\n\n${JSON.stringify(await simpleTable.getTypes(), null, 2)}\n\nI want you to give me a SQL query to do this:\n- ${prompt}\nThe query must ${options.outputTable
         ? `create a new table named "${tableName}" with the results`
-        : `replace the existing "${simpleTable.name}" table`}. This means the query must start with 'CREATE OR REPLACE TABLE "${tableName}"...'. Return just the query, nothing else.${options.extraInstructions ? `\n${options.extraInstructions}` : ""}`;
+        : `replace the existing "${simpleTable.name}" table`}. This means the query must start with 'CREATE OR REPLACE TABLE "${tableName}"...'. Return a JSON object matching the provided schema, with the SQL query in the "query" property. Do not include markdown or any other text.${options.extraInstructions ? `\n${options.extraInstructions}` : ""}`;
     if (options.verbose) {
         console.log("\naiQuery()");
     }
+    const [{ object, string, toJSONSchema }, { default: askAI }] = await Promise
+        .all([
+        import("zod"),
+        import("../helpers/askAI.js"),
+    ]);
     const schemaJson = toJSONSchema(object({
         query: string(),
     }));
     // Types could be improved
     const answer = await askAI(p, {
-        cache: options.cache,
-        model: options.model,
-        apiKey: options.apiKey,
-        vertex: options.vertex,
-        project: options.project,
-        location: options.location,
-        ollama: options.ollama,
-        contextWindow: options.contextWindow,
-        thinkingBudget: options.thinkingBudget,
-        thinkingLevel: options.thinkingLevel,
-        temperature: options.temperature,
-        safetyEnabled: options.safetyEnabled,
+        generation: options.generation,
         verbose: options.verbose,
         includeThoughts: options.includeThoughts,
         schemaJson,

package/esm/methods/aiRAG.d.ts

@@ -1,49 +1,24 @@
 import type SimpleTable from "../class/SimpleTable.js";
-import type { Ollama } from "ollama";
-export default function aiRAG(table: SimpleTable, query: string, columnId: string, columnText: string, nbResults: number, options?: {
-    cache?: boolean;
-    verbose?: boolean;
+import { type HybridSearchOptions } from "./hybridSearch.js";
+import type { AIRequestMetrics, UnstructuredGenerationOptions } from "../helpers/aiOptions.js";
+/**
+ * Options for retrieval and the final generation request in `aiRAG`.
+ *
+ * @example
+ * ```ts
+ * const options: AIRAGOptions = {
+ *   generation: { provider: "gemini" },
+ *   embeddings: { provider: "ollama" },
+ * };
+ * ```
+ */
+export type AIRAGOptions = Omit<HybridSearchOptions, "outputTable" | "times"> & {
+    /** Generation options excluding `schemaJson`, because `aiRAG` returns text. */
+    generation?: UnstructuredGenerationOptions;
+    /** Includes model thoughts in verbose logs when the provider returns them. */
     includeThoughts?: boolean;
-    systemPrompt?: string;
-    modelContextWindow?: number;
-    embeddingsModelContextWindow?: number;
-    createIndex?: boolean;
-    thinkingBudget?: number;
-    thinkingLevel?: "minimal" | "low" | "medium" | "high";
-    webSearch?: boolean;
-    safetyEnabled?: boolean;
-    model?: string;
-    temperature?: number;
-    apiKey?: string;
-    vertex?: boolean;
-    project?: string;
-    location?: string;
-    ollama?: boolean | Ollama;
-    metrics?: {
-        totalCost: number;
-        totalInputTokens: number;
-        totalOutputTokens: number;
-        totalRequests: number;
-    };
-    embeddingsModel?: string;
-    ollamaEmbeddings?: boolean;
-    embeddingsConcurrent?: number;
-    stemmer?: "arabic" | "basque" | "catalan" | "danish" | "dutch" | "english" | "finnish" | "french" | "german" | "greek" | "hindi" | "hungarian" | "indonesian" | "irish" | "italian" | "lithuanian" | "nepali" | "norwegian" | "porter" | "portuguese" | "romanian" | "russian" | "serbian" | "spanish" | "swedish" | "tamil" | "turkish" | "none";
-    stopwords?: string;
-    ignore?: string;
-    stripAccents?: boolean;
-    lower?: boolean;
-    k?: number;
-    b?: number;
-    conjunctive?: boolean;
-    bm25?: boolean;
-    bm25MinScore?: number;
-    bm25ScoreColumn?: string;
-    vectorSearch?: boolean;
-    vectorMinSimilarity?: number;
-    vectorSimilarityColumn?: string;
-    efConstruction?: number;
-    efSearch?: number;
-    M?: number;
-}): Promise<string>;
+    /** Mutable aggregate request metrics updated after the final provider response. */
+    metrics?: AIRequestMetrics;
+};
+export default function aiRAG(table: SimpleTable, query: string, idColumn: string, textColumn: string, nbResults: number, options?: AIRAGOptions): Promise<string>;
 //# sourceMappingURL=aiRAG.d.ts.map

package/esm/methods/aiRAG.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"aiRAG.d.ts","sourceRoot":"","sources":["../../src/methods/aiRAG.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,wBAA8B,KAAK,CACjC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,CAAC,EAAE,MAAM,CAAC;CACP,mBAuJP"}
+{"version":3,"file":"aiRAG.d.ts","sourceRoot":"","sources":["../../src/methods/aiRAG.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AACvD,OAAqB,EAAE,KAAK,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,KAAK,EACV,gBAAgB,EAChB,6BAA6B,EAC9B,MAAM,yBAAyB,CAAC;AAEjC;;;;;;;;;;GAUG;AACH,MAAM,MAAM,YAAY,GACpB,IAAI,CACJ,mBAAmB,EACnB,aAAa,GAAG,OAAO,CACxB,GACC;IACA,+EAA+E;IAC/E,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,8EAA8E;IAC9E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,mFAAmF;IACnF,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B,CAAC;AAEJ,wBAA8B,KAAK,CACjC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,YAAiB,mBAuH3B"}

package/esm/methods/aiRAG.js

@@ -1,7 +1,6 @@
-import { askAI } from "../deps/jsr.io/@nshiab/journalism-ai/1.2.23/src/index.js";
-import { prettyDuration } from "../deps/jsr.io/@nshiab/journalism-format/1.1.9/src/index.js";
 import hybridSearch from "./hybridSearch.js";
-export default async function aiRAG(table, query, columnId, columnText, nbResults, options = {}) {
+export default async function aiRAG(table, query, idColumn, textColumn, nbResults, options = {}) {
+    const { generation, includeThoughts, metrics, ...searchOptions } = options;
     const times = {
         start: Date.now(),
         embeddingStart: 0,
@@ -14,48 +13,27 @@ export default async function aiRAG(table, query, columnId, columnText, nbResult
         llmEnd: 0,
     };
     // Perform hybrid search (vector similarity + BM25 with RRF fusion)
-    const searchResultsTable = await hybridSearch(table, query, columnId, columnText, nbResults, {
-        cache: options.cache,
-        verbose: options.verbose,
-        embeddingsModelContextWindow: options.embeddingsModelContextWindow,
-        createIndex: options.createIndex,
-        embeddingsModel: options.embeddingsModel,
-        ollamaEmbeddings: options.ollamaEmbeddings,
-        embeddingsConcurrent: options.embeddingsConcurrent,
-        stemmer: options.stemmer,
-        stopwords: options.stopwords,
-        ignore: options.ignore,
-        stripAccents: options.stripAccents,
-        lower: options.lower,
-        k: options.k,
-        b: options.b,
-        conjunctive: options.conjunctive,
-        bm25: options.bm25,
-        bm25MinScore: options.bm25MinScore,
-        bm25ScoreColumn: options.bm25ScoreColumn,
-        vectorSearch: options.vectorSearch,
-        vectorMinSimilarity: options.vectorMinSimilarity,
-        vectorSimilarityColumn: options.vectorSimilarityColumn,
+    const searchResultsTable = await hybridSearch(table, query, idColumn, textColumn, nbResults, {
+        ...searchOptions,
         outputTable: `${table.name}_rag_search_results`,
-        efConstruction: options.efConstruction,
-        efSearch: options.efSearch,
-        M: options.M,
-        times: times,
+        times,
     });
-    await searchResultsTable.selectColumns([columnId, columnText]);
+    searchResultsTable.selectColumns([idColumn, textColumn]);
     // Get the retrieved data
     const retrievedData = await searchResultsTable.getData();
     // Clean up the temporary table
     await searchResultsTable.removeTable();
-    if (options.verbose) {
+    if (searchOptions.verbose) {
         times.llmStart = Date.now();
     }
+    const { default: askAI } = await import("../helpers/askAI.js");
     const response = await askAI(`Answer the following:
 - ${query}
 
 Base your answer only on the following data:\n
-${retrievedData.map((entry) => `${columnId}: ${entry[columnId]}\n\n${columnText}:\n\n${entry[columnText]}`).join("\n\n-----\n\n")}`, {
-        systemPrompt: options.systemPrompt ??
+${retrievedData.map((entry) => `${idColumn}: ${entry[idColumn]}\n\n${textColumn}:\n\n${entry[textColumn]}`).join("\n\n-----\n\n")}`, {
+        generation,
+        systemPrompt: generation?.systemPrompt ??
             `You are a focused research assistant. Your goal is to answer the user's question using ONLY the provided data.
 
 Rules of Engagement:
@@ -63,27 +41,15 @@ Rules of Engagement:
 - Strict Relevance: Only include information that directly addresses the user's query. If an entry is irrelevant, ignore it entirely.
 - Ambiguity & Contradiction: If sources conflict, present both views clearly. If the data is incomplete, state what is known and explicitly identify what is missing.
 - Groundedness: If the provided data does not contain the answer, your only response must be: "I do not have data to answer this question." Do not use outside knowledge.`,
-        cache: options.cache,
-        verbose: options.verbose,
-        includeThoughts: options.includeThoughts,
-        contextWindow: options.modelContextWindow,
-        thinkingBudget: options.thinkingBudget,
-        thinkingLevel: options.thinkingLevel,
-        webSearch: options.webSearch,
-        safetyEnabled: options.safetyEnabled,
-        model: options.model,
-        temperature: options.temperature,
-        apiKey: options.apiKey,
-        vertex: options.vertex,
-        project: options.project,
-        location: options.location,
-        ollama: options.ollama,
-        metrics: options.metrics,
+        verbose: searchOptions.verbose,
+        includeThoughts,
+        metrics,
     });
-    if (options.verbose) {
+    if (searchOptions.verbose) {
         times.llmEnd = Date.now();
-        const enableVectorSearch = options.vectorSearch !== false;
-        const enableBm25 = options.bm25 !== false;
+        const { prettyDuration } = await import("../deps/jsr.io/@nshiab/journalism-format/1.1.10/src/index.js");
+        const enableVectorSearch = searchOptions.vectorSearch !== false;
+        const enableBm25 = searchOptions.bm25 !== false;
         const parallelLabel = enableVectorSearch && enableBm25 ? " (parallel)" : "";
         const logParts = [`\nRAG process times:`];
         if (enableVectorSearch) {

package/esm/methods/aiRowByRow.d.ts

@@ -1,35 +1,47 @@
-import type { SimpleTable } from "../index.js";
-import type { Ollama } from "ollama";
-export default function aiRowByRow(simpleTable: SimpleTable, column: string, newColumn: string | string[], prompt: string, options?: {
+import type SimpleTable from "../class/SimpleTable.js";
+import type { AIRequestMetrics, GenerationOptions } from "../helpers/aiOptions.js";
+/**
+ * Options for applying a generation model to table rows.
+ *
+ * @example
+ * ```ts
+ * const options: AIRowByRowOptions = {
+ *   generation: { provider: "gemini" },
+ *   batchSize: 10,
+ *   concurrency: 5,
+ *   errorColumn: "error",
+ * };
+ * ```
+ */
+export type AIRowByRowOptions = {
+    /** Provider-specific generation options, or options for the environment-selected provider. */
+    generation?: GenerationOptions;
+    /** Number of rows sent in each request. */
     batchSize?: number;
-    concurrent?: number;
-    cache?: boolean;
+    /** Maximum number of requests processed concurrently. */
+    concurrency?: number;
+    /** Column used to store request errors instead of throwing them. */
+    errorColumn?: string;
+    /** Logs request-pool progress when enabled. */
+    logProgress?: boolean;
+    /** Validates each processed result and throws to trigger a retry. */
     test?: (result: {
         [key: string]: unknown;
     }) => void;
+    /** Number of retries after a request or validation failure. */
     retry?: number;
-    model?: string;
-    temperature?: number;
-    apiKey?: string;
-    vertex?: boolean;
-    project?: string;
-    location?: string;
-    ollama?: boolean | Ollama;
+    /** Decides whether a failed request should be retried. */
+    retryCheck?: (error: unknown) => Promise<boolean> | boolean;
+    /** Logs prompts and provider responses when enabled. */
     verbose?: boolean;
+    /** Maximum number of provider requests started per minute. Cached responses bypass this limit. */
     rateLimitPerMinute?: number;
+    /** Transforms a parsed response before validation and storage. */
     clean?: (response: unknown) => unknown;
-    contextWindow?: number;
-    thinkingBudget?: number;
-    thinkingLevel?: "minimal" | "low" | "medium" | "high";
-    safetyEnabled?: boolean;
-    webSearch?: boolean;
+    /** Additional instructions appended to the row-processing prompt. */
     extraInstructions?: string;
-    schemaJson?: unknown;
-    metrics?: {
-        totalCost: number;
-        totalInputTokens: number;
-        totalOutputTokens: number;
-        totalRequests: number;
-    };
-}): Promise<void>;
+    /** Mutable aggregate request metrics updated after each provider response. */
+    metrics?: AIRequestMetrics;
+};
+export default function aiRowByRow(table: SimpleTable, column: string, newColumn: string | string[], prompt: string, options?: AIRowByRowOptions): void;
 //# sourceMappingURL=aiRowByRow.d.ts.map

package/esm/methods/aiRowByRow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"aiRowByRow.d.ts","sourceRoot":"","sources":["../../src/methods/aiRowByRow.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGrC,wBAA8B,UAAU,CACtC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,CACN,QAAQ,EAAE,OAAO,KACd,OAAO,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACE,iBA2EP"}
+{"version":3,"file":"aiRowByRow.d.ts","sourceRoot":"","sources":["../../src/methods/aiRowByRow.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EAClB,MAAM,yBAAyB,CAAC;AAMjC;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,8FAA8F;IAC9F,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,qEAAqE;IACrE,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IACpD,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC5D,wDAAwD;IACxD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kGAAkG;IAClG,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kEAAkE;IAClE,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC;IACvC,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,8EAA8E;IAC9E,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,UAAU,CAChC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,iBAAsB,GAC9B,IAAI,CA8HN"}

package/esm/methods/aiRowByRow.js

@@ -1,42 +1,91 @@
-import { formatNumber } from "../deps/jsr.io/@nshiab/journalism-format/1.1.9/src/index.js";
-import sleep from "../helpers/sleep.js";
-import tryAI from "../helpers/tryAI.js";
-import { stringToArray } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/1.0.1/src/helpers/index.js";
-export default async function aiRowByRow(simpleTable, column, newColumn, prompt, options = {}) {
-    const newColumns = stringToArray(newColumn);
-    await simpleTable.updateWithJS(async (rows) => {
+import { snapshotAIOptions } from "../helpers/aiOptions.js";
+import processAIRowsResponse from "../helpers/processAIRowsResponse.js";
+export default function aiRowByRow(table, column, newColumn, prompt, options = {}) {
+    const newColumns = Array.isArray(newColumn) ? [...newColumn] : [newColumn];
+    options = snapshotAIOptions(options);
+    table.updateWithJS(async (rows) => {
         if (options.verbose) {
             console.log("\naiRowByRow()");
         }
         const batchSize = options.batchSize ?? 1;
-        const concurrent = options.concurrent ?? 1;
-        let requests = [];
-        let requestsNb = 1;
+        const concurrency = options.concurrency ?? 1;
+        if (!Number.isInteger(batchSize) || batchSize < 1) {
+            throw new Error("batchSize must be a positive integer.");
+        }
+        if (options.rateLimitPerMinute !== undefined &&
+            (!Number.isFinite(options.rateLimitPerMinute) ||
+                options.rateLimitPerMinute <= 0)) {
+            throw new Error("rateLimitPerMinute must be greater than 0.");
+        }
+        const [{ default: askAI }, { default: buildAIRowsRequest }, { default: runAIRequestPool },] = await Promise.all([
+            import("../helpers/askAI.js"),
+            import("../helpers/buildAIRowsRequest.js"),
+            import("../helpers/runAIRequestPool.js"),
+        ]);
+        const batches = [];
         for (let i = 0; i < rows.length; i += batchSize) {
+            batches.push(rows.slice(i, i + batchSize));
+        }
+        const requests = batches.map((batch) => {
+            const request = buildAIRowsRequest(batch, column, newColumns, prompt, options);
             if (options.verbose) {
-                console.log(`\nRequest ${requestsNb} - Processing rows ${i + 1} to ${Math.min(i + batchSize, rows.length)}... (${formatNumber((Math.min(i + batchSize, rows.length)) / rows.length * 100, {
-                    significantDigits: 3,
-                    suffix: "%",
-                })})`);
-                requestsNb++;
+                console.log(`\nPrompt:\n${request.prompt}`);
             }
-            if (requests.length < concurrent) {
-                requests.push(tryAI(i, batchSize, rows, column, newColumns, prompt, options));
+            return {
+                ...request,
+                processResponse: (response) => processAIRowsResponse(response, batch.length, newColumns, options),
+            };
+        });
+        const { results, errors } = await runAIRequestPool(requests.map((request) => (beforeRequest) => askAI(request.prompt, {
+            generation: options.generation,
+            systemPrompt: request.systemPrompt,
+            schemaJson: request.schemaJson,
+            metrics: options.metrics,
+            verbose: options.verbose,
+            processResponse: request.processResponse,
+            beforeRequest,
+        })), concurrency, {
+            retry: options.retry,
+            retryCheck: options.retryCheck,
+            minRequestIntervalMs: options.rateLimitPerMinute === undefined
+                ? undefined
+                : 60_000 / options.rateLimitPerMinute,
+            logProgress: options.logProgress,
+        });
+        if (options.errorColumn === undefined) {
+            const error = errors.find((item) => item !== undefined);
+            if (error !== undefined) {
+                throw error;
             }
-            if (requests.length === concurrent || i + batchSize >= rows.length) {
-                const start = new Date();
-                await Promise.all(requests);
-                const end = new Date();
-                const duration = end.getTime() - start.getTime();
-                // If duration is less than 10ms per request, it should means data comes from cache and we don't need to wait
-                if (typeof options.rateLimitPerMinute === "number" &&
-                    duration > 10 * requests.length && i + batchSize < rows.length) {
-                    const delay = Math.round((60 / (options.rateLimitPerMinute / concurrent)) * 1000);
-                    await sleep(delay, { start, log: options.verbose });
+        }
+        const newRows = [];
+        for (let i = 0; i < batches.length; i++) {
+            const result = results[i];
+            if (result !== undefined) {
+                for (let j = 0; j < result.length; j++) {
+                    newRows.push({
+                        ...batches[i][j],
+                        ...result[j],
+                        ...(options.errorColumn === undefined
+                            ? {}
+                            : { [options.errorColumn]: null }),
+                    });
                 }
-                requests = [];
+                continue;
+            }
+            const error = errors[i];
+            const errorMessage = error instanceof Error && error.message
+                ? error.message
+                : String(error);
+            const emptyResult = Object.fromEntries(newColumns.map((name) => [name, null]));
+            for (const row of batches[i]) {
+                newRows.push({
+                    ...row,
+                    ...emptyResult,
+                    [options.errorColumn]: errorMessage,
+                });
             }
         }
-        return rows;
+        return newRows;
     });
 }

package/esm/methods/aiVectorSimilarity.d.ts

@@ -1,14 +1,39 @@
 import type { SimpleTable } from "../index.js";
-export default function aiVectorSimilarity(simpleTable: SimpleTable, text: string, column: string, nbResults: number, options?: {
-    cache?: boolean;
+import { type EmbeddingOptions } from "../helpers/aiOptions.js";
+/**
+ * Options for embedding a query and finding similar table rows.
+ *
+ * @example
+ * ```ts
+ * const options: AIVectorSimilarityOptions = {
+ *   embeddings: { provider: "gemini" },
+ *   minSimilarity: 0.7,
+ * };
+ * ```
+ */
+export type AIVectorSimilarityOptions = {
+    /** Options used to embed the query text. */
+    embeddings?: EmbeddingOptions;
+    /** Creates a vector-similarity index on the stored embedding column. */
     createIndex?: boolean;
+    /** Replaces an existing vector-similarity index when creating one. */
     overwriteIndex?: boolean;
+    /** Writes results to a new table instead of replacing the current table. */
     outputTable?: string;
+    /** Logs embedding and index activity when enabled. */
     verbose?: boolean;
+    /** Candidate count used while constructing the vector index. */
     efConstruction?: number;
+    /** Candidate count used while searching the vector index. */
     efSearch?: number;
+    /** Maximum number of graph neighbors retained by the vector index. */
     M?: number;
+    /** Minimum cosine similarity required for a row to be returned. */
     minSimilarity?: number;
+    /** Adds cosine similarity scores under this output column name. */
     similarityColumn?: string;
-}): Promise<SimpleTable>;
+};
+export default function aiVectorSimilarity(simpleTable: SimpleTable, text: string, column: string, nbResults: number, options?: AIVectorSimilarityOptions): SimpleTable;
+/** @internal */
+export declare function runAIVectorSimilarity(simpleTable: SimpleTable, text: string, column: string, nbResults: number, options: AIVectorSimilarityOptions): Promise<void>;
 //# sourceMappingURL=aiVectorSimilarity.d.ts.map

package/esm/methods/aiVectorSimilarity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"aiVectorSimilarity.d.ts","sourceRoot":"","sources":["../../src/methods/aiVectorSimilarity.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAM/C,wBAA8B,kBAAkB,CAC9C,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACtB,wBAqEP"}
+{"version":3,"file":"aiVectorSimilarity.d.ts","sourceRoot":"","sources":["../../src/methods/aiVectorSimilarity.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAM/C,OAAO,EACL,KAAK,gBAAgB,EAEtB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;;;;;GAUG;AACH,MAAM,MAAM,yBAAyB,GAAG;IACtC,4CAA4C;IAC5C,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,wEAAwE;IACxE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,sEAAsE;IACtE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sDAAsD;IACtD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,mEAAmE;IACnE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,kBAAkB,CACxC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,yBAA8B,GACtC,WAAW,CAkBb;AAED,gBAAgB;AAChB,wBAAsB,qBAAqB,CACzC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CA+Df"}

package/esm/methods/aiVectorSimilarity.js

@@ -1,7 +1,21 @@
-import { getEmbedding } from "../deps/jsr.io/@nshiab/journalism-ai/1.2.23/src/index.js";
-import { mergeOptions, queryDB, } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/1.0.1/src/helpers/index.js";
-export default async function aiVectorSimilarity(simpleTable, text, column, nbResults, options = {}) {
-    const textEmbedding = await getEmbedding(text, options);
+import { mergeOptions, queryDB, queueAsyncBarrier, } from "../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js";
+import { snapshotAIOptions, } from "../helpers/aiOptions.js";
+export default function aiVectorSimilarity(simpleTable, text, column, nbResults, options = {}) {
+    options = snapshotAIOptions(options);
+    const outputTable = options.outputTable === undefined
+        ? simpleTable
+        : simpleTable.sdb.newTable(options.outputTable);
+    queueAsyncBarrier(outputTable, {
+        method: "aiVectorSimilarity()",
+        parameters: { text, column, nbResults, outputTable: options.outputTable },
+        execute: () => runAIVectorSimilarity(simpleTable, text, column, nbResults, options),
+    });
+    return outputTable;
+}
+/** @internal */
+export async function runAIVectorSimilarity(simpleTable, text, column, nbResults, options) {
+    const { getEmbeddingForProvider } = await import("../helpers/tryEmbedding.js");
+    const textEmbedding = await getEmbeddingForProvider(text, options.embeddings);
     const types = await simpleTable.getTypes();
     if (types[column] !== `FLOAT[${textEmbedding.length}]`) {
         await simpleTable.sdb.customQuery(`ALTER TABLE "${simpleTable.name}" ADD COLUMN "${column}_fixed_floatType" FLOAT[${textEmbedding.length}];
@@ -10,7 +24,7 @@ export default async function aiVectorSimilarity(simpleTable, text, column, nbRe
       ALTER TABLE "${simpleTable.name}" RENAME COLUMN "${column}_fixed_floatType" TO "${column}";`);
     }
     if (options.createIndex) {
-        await simpleTable.createVssIndex(column, {
+        simpleTable.createVssIndex(column, {
             overwrite: options.overwriteIndex,
             verbose: options.verbose,
             efConstruction: options.efConstruction,
@@ -45,10 +59,4 @@ export default async function aiVectorSimilarity(simpleTable, text, column, nbRe
             table: options.outputTable ?? simpleTable.name,
         },
     }));
-    if (typeof options.outputTable === "string") {
-        return simpleTable.sdb.newTable(options.outputTable);
-    }
-    else {
-        return simpleTable;
-    }
 }