@nshiab/simple-data-analysis 5.21.0 → 6.0.0

package/script/helpers/publishDatawrapper.js

@@ -0,0 +1,47 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = publishDatawrapper;
+async function publishDatawrapper(chartId, data, options = {}) {
+    const { updateDataDW, updateNotesDW, publishChartDW } = await Promise.resolve().then(() => __importStar(require("../deps/jsr.io/@nshiab/journalism-dataviz/1.0.21/src/index.js")));
+    const apiKey = { apiKey: options.apiKeyEnvVar };
+    await updateDataDW(chartId, data, apiKey);
+    if (typeof options.note === "string") {
+        await updateNotesDW(chartId, options.note, apiKey);
+    }
+    if (options.republish === true) {
+        await publishChartDW(chartId, apiKey);
+    }
+}

package/script/helpers/resolveAIProvider.d.ts

@@ -0,0 +1,7 @@
+import { type Provider } from "./resolveProvider.js";
+/** A provider capable of generating AI responses. */
+export type AIProvider = Provider;
+type AIProviderEnvironment = Record<string, string | undefined>;
+export default function resolveAIProvider(environment?: AIProviderEnvironment): AIProvider;
+export {};
+//# sourceMappingURL=resolveAIProvider.d.ts.map

package/script/helpers/resolveAIProvider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveAIProvider.d.ts","sourceRoot":"","sources":["../../src/helpers/resolveAIProvider.ts"],"names":[],"mappings":"AACA,OAAwB,EAAE,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEtE,qDAAqD;AACrD,MAAM,MAAM,UAAU,GAAG,QAAQ,CAAC;AAElC,KAAK,qBAAqB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEhE,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,WAAW,GAAE,qBAAmC,GAC/C,UAAU,CAEZ"}

package/script/helpers/resolveAIProvider.js

@@ -0,0 +1,11 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = resolveAIProvider;
+const node_process_1 = __importDefault(require("node:process"));
+const resolveProvider_js_1 = __importDefault(require("./resolveProvider.js"));
+function resolveAIProvider(environment = node_process_1.default.env) {
+    return (0, resolveProvider_js_1.default)("AI_PROVIDER", environment);
+}

package/script/helpers/resolveProvider.d.ts

@@ -0,0 +1,6 @@
+type ProviderEnvironment = Record<string, string | undefined>;
+export type Provider = "gemini" | "ollama";
+/** Resolves a provider variable with validation. */
+export default function resolveProvider(variable: "AI_PROVIDER" | "AI_EMBEDDINGS_PROVIDER", environment: ProviderEnvironment): Provider;
+export {};
+//# sourceMappingURL=resolveProvider.d.ts.map

package/script/helpers/resolveProvider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveProvider.d.ts","sourceRoot":"","sources":["../../src/helpers/resolveProvider.ts"],"names":[],"mappings":"AAAA,KAAK,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAE9D,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE3C,oDAAoD;AACpD,MAAM,CAAC,OAAO,UAAU,eAAe,CACrC,QAAQ,EAAE,aAAa,GAAG,wBAAwB,EAClD,WAAW,EAAE,mBAAmB,GAC/B,QAAQ,CAUV"}

package/script/helpers/resolveProvider.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = resolveProvider;
+/** Resolves a provider variable with validation. */
+function resolveProvider(variable, environment) {
+    const configuredProvider = environment[variable];
+    if (configuredProvider) {
+        if (configuredProvider !== "gemini" && configuredProvider !== "ollama") {
+            throw new Error(`${variable} must be either "gemini" or "ollama".`);
+        }
+        return configuredProvider;
+    }
+    return "gemini";
+}

package/script/helpers/runAIRequestPool.d.ts

@@ -0,0 +1,13 @@
+type PoolOptions = {
+    retry?: number;
+    retryCheck?: (error: unknown) => Promise<boolean> | boolean;
+    minRequestIntervalMs?: number;
+    logProgress?: boolean;
+};
+/** Runs indexed AI request tasks with bounded concurrency and retry handling. */
+export default function runAIRequestPool<T>(tasks: ((beforeRequest: () => Promise<void>) => Promise<T>)[], poolSize: number, options?: PoolOptions): Promise<{
+    results: (T | undefined)[];
+    errors: (unknown | undefined)[];
+}>;
+export {};
+//# sourceMappingURL=runAIRequestPool.d.ts.map

package/script/helpers/runAIRequestPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runAIRequestPool.d.ts","sourceRoot":"","sources":["../../src/helpers/runAIRequestPool.ts"],"names":[],"mappings":"AAEA,KAAK,WAAW,GAAG;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC5D,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,iFAAiF;AACjF,wBAA8B,gBAAgB,CAAC,CAAC,EAC9C,KAAK,EAAE,CAAC,CAAC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAC7D,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,WAAgB,GACxB,OAAO,CAAC;IACT,OAAO,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC;IAC3B,MAAM,EAAE,CAAC,OAAO,GAAG,SAAS,CAAC,EAAE,CAAC;CACjC,CAAC,CAyDD"}

package/script/helpers/runAIRequestPool.js

@@ -0,0 +1,55 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = runAIRequestPool;
+const sleep_js_1 = __importDefault(require("./sleep.js"));
+/** Runs indexed AI request tasks with bounded concurrency and retry handling. */
+async function runAIRequestPool(tasks, poolSize, options = {}) {
+    if (!Number.isInteger(poolSize) || poolSize < 1) {
+        throw new Error("poolSize must be a positive integer.");
+    }
+    const results = Array(tasks.length).fill(undefined);
+    const errors = Array(tasks.length).fill(undefined);
+    let nextIndex = 0;
+    let completed = 0;
+    let nextRequestStart = 0;
+    const beforeRequest = async () => {
+        const interval = options.minRequestIntervalMs ?? 0;
+        const now = Date.now();
+        const requestStart = Math.max(now, nextRequestStart);
+        nextRequestStart = requestStart + interval;
+        const wait = requestStart - now;
+        if (wait > 0) {
+            await (0, sleep_js_1.default)(wait);
+        }
+    };
+    const worker = async () => {
+        while (nextIndex < tasks.length) {
+            const index = nextIndex++;
+            let retries = 0;
+            while (true) {
+                try {
+                    results[index] = await tasks[index](beforeRequest);
+                    break;
+                }
+                catch (error) {
+                    const shouldRetry = retries < (options.retry ?? 0) &&
+                        (options.retryCheck ? await options.retryCheck(error) : true);
+                    if (!shouldRetry) {
+                        errors[index] = error;
+                        break;
+                    }
+                    retries++;
+                }
+            }
+            completed++;
+            if (options.logProgress) {
+                console.log(`Processed ${completed} of ${tasks.length} requests.`);
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(poolSize, tasks.length) }, () => worker()));
+    return { results, errors };
+}

package/script/helpers/sleep.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = sleep;
-const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-format/1.1.9/src/index.js");
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-format/1.1.10/src/index.js");
 function sleep(ms, options = {}) {
     return new Promise((resolve) => {
         if (options.start) {

package/script/helpers/tryEmbedding.d.ts

@@ -1,15 +1,9 @@
-import type { Ollama } from "ollama";
+import type { EmbeddingOptions } from "./aiOptions.js";
+/** Generates an embedding with an explicit provider or environment defaults. */
+export declare function getEmbeddingForProvider(text: string, embeddings?: EmbeddingOptions): Promise<number[]>;
 export default function tryEmbedding(i: number, rows: {
-    [key: string]: string | number | boolean | Date | null;
+    [key: string]: unknown;
 }[], text: string, newColumn: string, options?: {
-    cache?: boolean;
-    model?: string;
-    apiKey?: string;
-    vertex?: boolean;
-    project?: string;
-    location?: string;
-    ollama?: boolean | Ollama;
-    verbose?: boolean;
-    contextWindow?: number;
-}): Promise<number>;
+    embeddings?: EmbeddingOptions;
+}): Promise<number[]>;
 //# sourceMappingURL=tryEmbedding.d.ts.map

package/script/helpers/tryEmbedding.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tryEmbedding.d.ts","sourceRoot":"","sources":["../../src/helpers/tryEmbedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,wBAA8B,YAAY,CACxC,CAAC,EAAE,MAAM,EACT,IAAI,EAAE;IACJ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD,EAAE,EACH,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACnB,mBAOP"}
+{"version":3,"file":"tryEmbedding.d.ts","sourceRoot":"","sources":["../../src/helpers/tryEmbedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,gFAAgF;AAChF,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,gBAAgB,GAC5B,OAAO,CAAC,MAAM,EAAE,CAAC,CAEnB;AAED,wBAA8B,YAAY,CACxC,CAAC,EAAE,MAAM,EACT,IAAI,EAAE;IACJ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,EAAE,EACH,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC1B,qBAOP"}

package/script/helpers/tryEmbedding.js

@@ -1,8 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.getEmbeddingForProvider = getEmbeddingForProvider;
 exports.default = tryEmbedding;
-const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/1.2.23/src/index.js");
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/2.0.6/src/index.js");
+/** Generates an embedding with an explicit provider or environment defaults. */
+function getEmbeddingForProvider(text, embeddings) {
+    return (0, index_js_1.getEmbedding)(text, embeddings);
+}
 async function tryEmbedding(i, rows, text, newColumn, options = {}) {
     // Should be improved...
-    return rows[i][newColumn] = await (0, index_js_1.getEmbedding)(text, options);
+    return rows[i][newColumn] = await getEmbeddingForProvider(text, options.embeddings);
 }

package/script/index.d.ts

@@ -10,7 +10,7 @@
  *
  * To install the library with Node.js, use:
  * ```bash
- * npx jsr add @nshiab/simple-data-analysis
+ * npm i @nshiab/simple-data-analysis
  * ```
  *
  * To start, create a SimpleDB instance and then a SimpleTable from this instance:
@@ -18,12 +18,14 @@
  * import { SimpleDB } from "@nshiab/simple-data-analysis";
  *
  * const sdb = new SimpleDB();
- * const table = sdb.newTable("myTable"); // This returns a SimpleTable instance
- * await table.loadData("path/to/your/data.csv");
+ * const table = await sdb
+ *   .newTable("myTable")
+ *   .loadData("path/to/your/data.csv")
+ *   .log();
  *
  * // You can now perform various data analysis operations on the table.
  *
- * await sdb.done(); // Ensure to call done when you're finished.
+ * await sdb.close(); // Close the database when you're finished.
  * ```
  */
 import "./_dnt.polyfills.js";

package/script/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,OAAO,qBAAqB,CAAC;AAG7B,OAAO,EAAE,OAAO,IAAI,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,wBAAwB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,qBAAqB,CAAC;AAG7B,OAAO,EAAE,OAAO,IAAI,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,wBAAwB,CAAC"}

package/script/index.js

@@ -16,7 +16,7 @@ exports.SimpleTable = exports.SimpleDB = void 0;
  *
  * To install the library with Node.js, use:
  * ```bash
- * npx jsr add @nshiab/simple-data-analysis
+ * npm i @nshiab/simple-data-analysis
  * ```
  *
  * To start, create a SimpleDB instance and then a SimpleTable from this instance:
@@ -24,12 +24,14 @@ exports.SimpleTable = exports.SimpleDB = void 0;
  * import { SimpleDB } from "@nshiab/simple-data-analysis";
  *
  * const sdb = new SimpleDB();
- * const table = sdb.newTable("myTable"); // This returns a SimpleTable instance
- * await table.loadData("path/to/your/data.csv");
+ * const table = await sdb
+ *   .newTable("myTable")
+ *   .loadData("path/to/your/data.csv")
+ *   .log();
  *
  * // You can now perform various data analysis operations on the table.
  *
- * await sdb.done(); // Ensure to call done when you're finished.
+ * await sdb.close(); // Close the database when you're finished.
  * ```
  */
 require("./_dnt.polyfills.js");

package/script/methods/aiEmbeddings.d.ts

@@ -1,21 +1,55 @@
 import type { SimpleTable } from "../index.js";
-import type { Ollama } from "ollama";
-export default function aiEmbeddings(simpleTable: SimpleTable, column: string, newColumn: string, options?: {
+import { type EmbeddingOptions } from "../helpers/aiOptions.js";
+/**
+ * Options for generating an embedding column.
+ *
+ * @example
+ * ```ts
+ * const options: AIEmbeddingsOptions = {
+ *   embeddings: { provider: "ollama" },
+ *   concurrency: 4,
+ * };
+ * ```
+ */
+export type AIEmbeddingsOptions = {
+    /** Provider-specific embedding options, or options for the environment-selected provider. */
+    embeddings?: EmbeddingOptions;
+    /** Creates a vector-similarity index on the generated column. */
     createIndex?: boolean;
+    /** Replaces an existing vector-similarity index when creating one. */
     overwriteIndex?: boolean;
-    concurrent?: number;
-    cache?: boolean;
-    model?: string;
-    apiKey?: string;
-    vertex?: boolean;
-    project?: string;
-    location?: string;
-    ollama?: boolean | Ollama;
+    /** Maximum number of embedding requests processed concurrently. */
+    concurrency?: number;
+    /** Logs embedding progress and index creation when enabled. */
     verbose?: boolean;
+    /** Maximum request rate used to calculate delays between batches. */
     rateLimitPerMinute?: number;
-    contextWindow?: number;
+    /** Candidate count used while constructing the vector index. */
     efConstruction?: number;
+    /** Candidate count used while searching the vector index. */
     efSearch?: number;
+    /** Maximum number of graph neighbors retained by the vector index. */
     M?: number;
-}): Promise<void>;
+};
+export default function aiEmbeddings(simpleTable: SimpleTable, column: string, newColumn: string, options?: AIEmbeddingsOptions): SimpleTable;
+/**
+ * Generates every vector in an embedding column without managing provenance.
+ * Callers must wrap this operation with `ensureEmbeddingColumn`.
+ *
+ * @param simpleTable Table containing the source rows.
+ * @param column Text column to embed.
+ * @param newColumn Column that receives the generated vectors.
+ * @param options Embedding generation and concurrency options.
+ * @returns A promise that resolves after every row has been embedded.
+ *
+ * @example
+ * ```ts
+ * await generateEmbeddingColumn(table, "text", "text_embeddings", {
+ *   embeddings: { provider: "ollama", model: "nomic-embed-text" },
+ * });
+ * ```
+ *
+ * @internal
+ */
+export declare function generateEmbeddingColumn(simpleTable: SimpleTable, column: string, newColumn: string, options?: AIEmbeddingsOptions): Promise<void>;
 //# sourceMappingURL=aiEmbeddings.d.ts.map

package/script/methods/aiEmbeddings.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"aiEmbeddings.d.ts","sourceRoot":"","sources":["../../src/methods/aiEmbeddings.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,wBAA8B,YAAY,CACxC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,CAAC,EAAE,MAAM,CAAC;CACP,iBAsEP"}
+{"version":3,"file":"aiEmbeddings.d.ts","sourceRoot":"","sources":["../../src/methods/aiEmbeddings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EACL,KAAK,gBAAgB,EAEtB,MAAM,yBAAyB,CAAC;AAIjC;;;;;;;;;;GAUG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,6FAA6F;IAC7F,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,sEAAsE;IACtE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qEAAqE;IACrE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,mBAAwB,GAChC,WAAW,CAQb;AAoCD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,uBAAuB,CAC3C,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC,CAiEf"}

package/script/methods/aiEmbeddings.js

@@ -1,34 +1,119 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = aiEmbeddings;
-const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-format/1.1.9/src/index.js");
-const sleep_js_1 = __importDefault(require("../helpers/sleep.js"));
-const tryEmbedding_js_1 = __importDefault(require("../helpers/tryEmbedding.js"));
-async function aiEmbeddings(simpleTable, column, newColumn, options = {}) {
+exports.generateEmbeddingColumn = generateEmbeddingColumn;
+const aiOptions_js_1 = require("../helpers/aiOptions.js");
+const ensureEmbeddingColumn_js_1 = __importDefault(require("../helpers/ensureEmbeddingColumn.js"));
+const index_js_1 = require("../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js");
+function aiEmbeddings(simpleTable, column, newColumn, options = {}) {
+    options = (0, aiOptions_js_1.snapshotAIOptions)(options);
+    (0, index_js_1.queueAsyncBarrier)(simpleTable, {
+        method: "aiEmbeddings()",
+        parameters: { column, newColumn },
+        execute: () => runAIEmbeddings(simpleTable, column, newColumn, options),
+    });
+    return simpleTable;
+}
+async function runAIEmbeddings(simpleTable, column, newColumn, options) {
+    const { getEmbeddingIdentity } = await Promise.resolve().then(() => __importStar(require("../deps/jsr.io/@nshiab/journalism-ai/2.0.6/src/index.js")));
+    const identity = getEmbeddingIdentity(options.embeddings);
+    const embeddingStatus = await (0, ensureEmbeddingColumn_js_1.default)(simpleTable, column, newColumn, identity, () => generateEmbeddingColumn(simpleTable, column, newColumn, options));
+    if (options.verbose && embeddingStatus === "reused") {
+        console.log(`${newColumn} in table ${simpleTable.name} has compatible provenance. Reusing embeddings...`);
+    }
+    if (options.createIndex) {
+        simpleTable.createVssIndex(newColumn, {
+            overwrite: options.overwriteIndex,
+            verbose: options.verbose,
+            efConstruction: options.efConstruction,
+            efSearch: options.efSearch,
+            M: options.M,
+        });
+        await simpleTable.run();
+    }
+}
+/**
+ * Generates every vector in an embedding column without managing provenance.
+ * Callers must wrap this operation with `ensureEmbeddingColumn`.
+ *
+ * @param simpleTable Table containing the source rows.
+ * @param column Text column to embed.
+ * @param newColumn Column that receives the generated vectors.
+ * @param options Embedding generation and concurrency options.
+ * @returns A promise that resolves after every row has been embedded.
+ *
+ * @example
+ * ```ts
+ * await generateEmbeddingColumn(table, "text", "text_embeddings", {
+ *   embeddings: { provider: "ollama", model: "nomic-embed-text" },
+ * });
+ * ```
+ *
+ * @internal
+ */
+async function generateEmbeddingColumn(simpleTable, column, newColumn, options = {}) {
     await simpleTable.updateWithJS(async (rows) => {
+        const [{ formatNumber }, { default: sleep }, { default: tryEmbedding }] = await Promise.all([
+            Promise.resolve().then(() => __importStar(require("../deps/jsr.io/@nshiab/journalism-format/1.1.10/src/index.js"))),
+            Promise.resolve().then(() => __importStar(require("../helpers/sleep.js"))),
+            Promise.resolve().then(() => __importStar(require("../helpers/tryEmbedding.js"))),
+        ]);
         if (options.verbose) {
             console.log("\naiEmbeddings()");
         }
-        const concurrent = options.concurrent ?? 1;
+        const concurrency = options.concurrency ?? 1;
         let requests = [];
         for (let i = 0; i < rows.length; i++) {
             if (options.verbose) {
-                console.log(`\nProcessing row ${i + 1} of ${rows.length}... (${(0, index_js_1.formatNumber)((i + 1) / rows.length * 100, {
+                console.log(`Processing row ${i + 1} of ${rows.length}... (${formatNumber((i + 1) / rows.length * 100, {
                     significantDigits: 3,
                     suffix: "%",
                 })})`);
             }
-            if (requests.length < concurrent) {
+            if (requests.length < concurrency) {
                 const text = rows[i][column];
                 if (typeof text !== "string") {
                     throw new Error(`The column "${column}" must be a string. Found ${text} instead.`);
                 }
-                requests.push((0, tryEmbedding_js_1.default)(i, rows, text, newColumn, options));
+                requests.push(tryEmbedding(i, rows, text, newColumn, options));
             }
-            if (requests.length === concurrent || i + 1 >= rows.length) {
+            if (requests.length === concurrency || i + 1 >= rows.length) {
                 const start = new Date();
                 await Promise.all(requests);
                 const end = new Date();
@@ -36,21 +121,12 @@ async function aiEmbeddings(simpleTable, column, newColumn, options = {}) {
                 // If duration is less than 10ms per request, it should means data comes from cache and we don't need to wait
                 if (typeof options.rateLimitPerMinute === "number" &&
                     duration > 10 * requests.length && i + 1 < rows.length) {
-                    const delay = Math.round((60 / (options.rateLimitPerMinute / concurrent)) * 1000);
-                    await (0, sleep_js_1.default)(delay, { start, log: options.verbose });
+                    const delay = Math.round((60 / (options.rateLimitPerMinute / concurrency)) * 1000);
+                    await sleep(delay, { start, log: options.verbose });
                 }
                 requests = [];
             }
         }
         return rows;
     });
-    if (options.createIndex) {
-        await simpleTable.createVssIndex(newColumn, {
-            overwrite: options.overwriteIndex,
-            verbose: options.verbose,
-            efConstruction: options.efConstruction,
-            efSearch: options.efSearch,
-            M: options.M,
-        });
-    }
 }

package/script/methods/aiQuery.d.ts

@@ -1,21 +1,27 @@
 import type { SimpleTable } from "../index.js";
-import type { Ollama } from "ollama";
-export default function aiQuery(simpleTable: SimpleTable, prompt: string, options?: {
+import { type UnstructuredGenerationOptions } from "../helpers/aiOptions.js";
+/**
+ * Options for generating and executing a SQL query.
+ *
+ * @example
+ * ```ts
+ * const options: AIQueryOptions = {
+ *   generation: { provider: "gemini" },
+ *   outputTable: "results",
+ * };
+ * ```
+ */
+export type AIQueryOptions = {
+    /** Additional requirements appended to the SQL-generation prompt. */
     extraInstructions?: string;
-    cache?: boolean;
-    model?: string;
-    apiKey?: string;
-    vertex?: boolean;
-    project?: string;
+    /** Generation options excluding `schemaJson`, which is owned by SDA. */
+    generation?: UnstructuredGenerationOptions;
+    /** Includes model thoughts in verbose logs when the provider returns them. */
     includeThoughts?: boolean;
-    location?: string;
-    ollama?: boolean | Ollama;
-    contextWindow?: number;
-    thinkingBudget?: number;
-    thinkingLevel?: "minimal" | "low" | "medium" | "high";
-    temperature?: number;
-    safetyEnabled?: boolean;
+    /** Writes results to a new table instead of replacing the current table. */
     outputTable?: string;
+    /** Logs the request and provider response when enabled. */
     verbose?: boolean;
-}): Promise<void>;
+};
+export default function aiQuery(simpleTable: SimpleTable, prompt: string, options?: AIQueryOptions): SimpleTable;
 //# sourceMappingURL=aiQuery.d.ts.map

package/script/methods/aiQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"aiQuery.d.ts","sourceRoot":"","sources":["../../src/methods/aiQuery.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGrC,wBAA8B,OAAO,CACnC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,iBA8CP"}
+{"version":3,"file":"aiQuery.d.ts","sourceRoot":"","sources":["../../src/methods/aiQuery.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;GAUG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,8EAA8E;IAC9E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,cAAmB,GAC3B,WAAW,CAWb"}