@nshiab/simple-data-analysis 5.21.1 → 6.0.0

package/script/helpers/aiResponseCache.js

@@ -0,0 +1,39 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAIResponseCacheFile = getAIResponseCacheFile;
+exports.readAIResponseCache = readAIResponseCache;
+exports.removeAIResponseCache = removeAIResponseCache;
+exports.writeAIResponseCache = writeAIResponseCache;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_fs_1 = require("node:fs");
+const cacheDirectory = ".journalism-cache";
+/** Returns the cache file for an SDA-processed AI response. */
+function getAIResponseCacheFile(parameters) {
+    const hash = node_crypto_1.default
+        .createHash("sha256")
+        .update(JSON.stringify(parameters))
+        .digest("hex");
+    return `${cacheDirectory}/sda-askAI-${hash}.json`;
+}
+/** Reads an SDA-processed AI response when it exists. */
+function readAIResponseCache(cacheFile) {
+    return (0, node_fs_1.existsSync)(cacheFile)
+        ? JSON.parse((0, node_fs_1.readFileSync)(cacheFile, "utf-8"))
+        : undefined;
+}
+/** Removes an SDA-processed AI response after validation rejects it. */
+function removeAIResponseCache(cacheFile) {
+    if ((0, node_fs_1.existsSync)(cacheFile)) {
+        (0, node_fs_1.rmSync)(cacheFile);
+    }
+}
+/** Caches an AI response after SDA transformation and validation. */
+function writeAIResponseCache(cacheFile, response) {
+    if (!(0, node_fs_1.existsSync)(cacheDirectory)) {
+        (0, node_fs_1.mkdirSync)(cacheDirectory);
+    }
+    (0, node_fs_1.writeFileSync)(cacheFile, JSON.stringify(response));
+}

package/script/helpers/askAI.d.ts

@@ -0,0 +1,14 @@
+import { type AIRequestMetrics, type GenerationOptions } from "./aiOptions.js";
+export { normalizeOllamaStructuredText, parseOllamaStructuredResponse, } from "./askOllamaAdapter.js";
+type AskAIOptions<TResponse = unknown> = {
+    generation?: GenerationOptions;
+    systemPrompt?: string;
+    schemaJson?: unknown;
+    verbose?: boolean;
+    includeThoughts?: boolean;
+    metrics?: AIRequestMetrics;
+    processResponse?: (response: unknown) => TResponse | Promise<TResponse>;
+    beforeRequest?: () => Promise<void>;
+};
+export default function askAI<TResponse = unknown>(prompt: string, options?: AskAIOptions<TResponse>): Promise<TResponse>;
+//# sourceMappingURL=askAI.d.ts.map

package/script/helpers/askAI.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"askAI.d.ts","sourceRoot":"","sources":["../../src/helpers/askAI.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,gBAAgB,EAGrB,KAAK,iBAAiB,EAEvB,MAAM,gBAAgB,CAAC;AAKxB,OAAO,EACL,6BAA6B,EAC7B,6BAA6B,GAC9B,MAAM,uBAAuB,CAAC;AAE/B,KAAK,YAAY,CAAC,SAAS,GAAG,OAAO,IAAI;IACvC,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,KAAK,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACxE,aAAa,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACrC,CAAC;AAEF,wBAA8B,KAAK,CAAC,SAAS,GAAG,OAAO,EACrD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,YAAY,CAAC,SAAS,CAAM,GACpC,OAAO,CAAC,SAAS,CAAC,CAkGpB"}

package/script/helpers/askAI.js

@@ -0,0 +1,141 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseOllamaStructuredResponse = exports.normalizeOllamaStructuredText = void 0;
+exports.default = askAI;
+const node_process_1 = __importDefault(require("node:process"));
+const zod_1 = require("zod");
+const aiResponseCache_js_1 = require("./aiResponseCache.js");
+const resolveAIProvider_js_1 = __importDefault(require("./resolveAIProvider.js"));
+const aiOptions_js_1 = require("./aiOptions.js");
+const askGeminiAdapter_js_1 = __importDefault(require("./askGeminiAdapter.js"));
+const askOllamaAdapter_js_1 = __importStar(require("./askOllamaAdapter.js"));
+var askOllamaAdapter_js_2 = require("./askOllamaAdapter.js");
+Object.defineProperty(exports, "normalizeOllamaStructuredText", { enumerable: true, get: function () { return askOllamaAdapter_js_2.normalizeOllamaStructuredText; } });
+Object.defineProperty(exports, "parseOllamaStructuredResponse", { enumerable: true, get: function () { return askOllamaAdapter_js_2.parseOllamaStructuredResponse; } });
+async function askAI(prompt, options = {}) {
+    const provider = options.generation?.provider ?? (0, resolveAIProvider_js_1.default)();
+    const generationOptions = options.generation
+        ? (0, aiOptions_js_1.withoutProvider)(options.generation)
+        : {};
+    const schemaJson = options.schemaJson ?? generationOptions.schemaJson;
+    const systemPrompt = options.systemPrompt ?? generationOptions.systemPrompt;
+    const promptWithSchema = provider === "ollama" && schemaJson
+        ? `${prompt}\n\nReturn JSON matching this schema exactly:\n${JSON.stringify(schemaJson)}`
+        : prompt;
+    const processedCacheFile = generationOptions.cache !== false &&
+        options.processResponse
+        ? (0, aiResponseCache_js_1.getAIResponseCacheFile)({
+            provider,
+            prompt: promptWithSchema,
+            systemPrompt,
+            model: generationOptions.model ?? node_process_1.default.env.AI_MODEL,
+            generationOptions: {
+                ...generationOptions,
+                ollama: provider === "ollama" && "ollama" in generationOptions &&
+                    typeof generationOptions.ollama === "object"
+                    ? "custom"
+                    : undefined,
+            },
+            project: provider === "gemini" && "project" in generationOptions
+                ? generationOptions.project ?? node_process_1.default.env.AI_PROJECT
+                : undefined,
+            location: provider === "gemini" && "location" in generationOptions
+                ? generationOptions.location ?? node_process_1.default.env.AI_LOCATION
+                : undefined,
+            schemaJson,
+        })
+        : undefined;
+    if (processedCacheFile) {
+        const cachedResponse = (0, aiResponseCache_js_1.readAIResponseCache)(processedCacheFile);
+        if (cachedResponse !== undefined) {
+            try {
+                return await options.processResponse(cachedResponse);
+            }
+            catch (error) {
+                (0, aiResponseCache_js_1.removeAIResponseCache)(processedCacheFile);
+                throw error;
+            }
+        }
+    }
+    await options.beforeRequest?.();
+    const result = provider === "ollama"
+        ? await (0, askOllamaAdapter_js_1.default)(promptWithSchema, {
+            generation: generationOptions,
+            systemPrompt,
+            schemaJson,
+            disableCache: Boolean(processedCacheFile),
+        })
+        : await (0, askGeminiAdapter_js_1.default)(prompt, {
+            generation: generationOptions,
+            systemPrompt,
+            schemaJson,
+            disableCache: Boolean(processedCacheFile),
+        });
+    if (options.metrics) {
+        options.metrics.totalCost += "estimatedCost" in result
+            ? result.estimatedCost ?? 0
+            : 0;
+        options.metrics.totalInputTokens += result.promptTokenCount;
+        options.metrics.totalOutputTokens += result.outputTokenCount;
+        options.metrics.totalRequests++;
+    }
+    if (options.verbose) {
+        if (options.includeThoughts) {
+            console.log(result);
+        }
+        else {
+            const { thoughts: _thoughts, ...resultWithoutThoughts } = result;
+            console.log(resultWithoutThoughts);
+        }
+    }
+    const parsedResponse = provider === "ollama" && schemaJson
+        ? (0, askOllamaAdapter_js_1.parseOllamaStructuredResponse)(result.response)
+        : result.response;
+    const validatedResponse = schemaJson === undefined ||
+        options.processResponse
+        ? parsedResponse
+        : (0, zod_1.fromJSONSchema)(schemaJson).parse(parsedResponse);
+    const processedResponse = options.processResponse
+        ? await options.processResponse(validatedResponse)
+        : validatedResponse;
+    if (processedCacheFile) {
+        (0, aiResponseCache_js_1.writeAIResponseCache)(processedCacheFile, validatedResponse);
+    }
+    return processedResponse;
+}

package/script/helpers/askGeminiAdapter.d.ts

@@ -0,0 +1,12 @@
+import { askGemini } from "../deps/jsr.io/@nshiab/journalism-ai/2.0.6/src/index.js";
+import type { AskGeminiOptions } from "./aiOptions.js";
+type GeminiAdapterOptions = {
+    generation: AskGeminiOptions;
+    systemPrompt?: string;
+    schemaJson?: unknown;
+    disableCache?: boolean;
+};
+/** Calls `askGemini` with SDA-owned prompt and cache overrides. */
+export default function askGeminiAdapter(prompt: string, options: GeminiAdapterOptions): ReturnType<typeof askGemini>;
+export {};
+//# sourceMappingURL=askGeminiAdapter.d.ts.map

package/script/helpers/askGeminiAdapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"askGeminiAdapter.d.ts","sourceRoot":"","sources":["../../src/helpers/askGeminiAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yDAAyD,CAAC;AACpF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,KAAK,oBAAoB,GAAG;IAC1B,UAAU,EAAE,gBAAgB,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,mEAAmE;AACnE,MAAM,CAAC,OAAO,UAAU,gBAAgB,CACtC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,UAAU,CAAC,OAAO,SAAS,CAAC,CAO9B"}

package/script/helpers/askGeminiAdapter.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = askGeminiAdapter;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/2.0.6/src/index.js");
+/** Calls `askGemini` with SDA-owned prompt and cache overrides. */
+function askGeminiAdapter(prompt, options) {
+    return (0, index_js_1.askGemini)(prompt, {
+        ...options.generation,
+        systemPrompt: options.systemPrompt,
+        schemaJson: options.schemaJson,
+        cache: options.disableCache ? false : options.generation.cache,
+    });
+}

package/script/helpers/askOllamaAdapter.d.ts

@@ -0,0 +1,16 @@
+import { askOllama } from "../deps/jsr.io/@nshiab/journalism-ai/2.0.6/src/index.js";
+import type { AskOllamaOptions } from "./aiOptions.js";
+type OllamaAdapterOptions = {
+    generation: AskOllamaOptions;
+    systemPrompt?: string;
+    schemaJson?: unknown;
+    disableCache?: boolean;
+};
+/** Normalizes structured Ollama output by removing common JSON labels and fences. */
+export declare function normalizeOllamaStructuredText(response: string): string;
+/** Parses structured Ollama output while tolerating common JSON labels and fences. */
+export declare function parseOllamaStructuredResponse(response: unknown): unknown;
+/** Calls `askOllama` with SDA-owned structured-output and cache overrides. */
+export default function askOllamaAdapter(prompt: string, options: OllamaAdapterOptions): ReturnType<typeof askOllama>;
+export {};
+//# sourceMappingURL=askOllamaAdapter.d.ts.map

package/script/helpers/askOllamaAdapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"askOllamaAdapter.d.ts","sourceRoot":"","sources":["../../src/helpers/askOllamaAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yDAAyD,CAAC;AAEpF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,KAAK,oBAAoB,GAAG;IAC1B,UAAU,EAAE,gBAAgB,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,qFAAqF;AACrF,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAuBtE;AAED,sFAAsF;AACtF,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CA0BxE;AAuBD,8EAA8E;AAC9E,MAAM,CAAC,OAAO,UAAU,gBAAgB,CACtC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,UAAU,CAAC,OAAO,SAAS,CAAC,CAc9B"}

package/script/helpers/askOllamaAdapter.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeOllamaStructuredText = normalizeOllamaStructuredText;
+exports.parseOllamaStructuredResponse = parseOllamaStructuredResponse;
+exports.default = askOllamaAdapter;
+const index_js_1 = require("../deps/jsr.io/@nshiab/journalism-ai/2.0.6/src/index.js");
+const ollama_1 = require("ollama");
+/** Normalizes structured Ollama output by removing common JSON labels and fences. */
+function normalizeOllamaStructuredText(response) {
+    let json = response.trim();
+    for (let i = 0; i < 2; i++) {
+        json = json
+            .replace(/^json\s*/i, "")
+            .replace(/^[{[]?\s*```(?:json)?\s*/i, "")
+            .trim();
+    }
+    json = json.replace(/\s*```$/, "").trim();
+    const openingBraces = json.match(/{/g)?.length ?? 0;
+    const closingBraces = json.match(/}/g)?.length ?? 0;
+    if (/^\{\s*\{/.test(json) && openingBraces === closingBraces + 1) {
+        json = json.slice(1);
+    }
+    const openingBrackets = json.match(/\[/g)?.length ?? 0;
+    const closingBrackets = json.match(/]/g)?.length ?? 0;
+    if (/^\[\s*\[/.test(json) && openingBrackets === closingBrackets + 1) {
+        json = json.slice(1);
+    }
+    return json.trim();
+}
+/** Parses structured Ollama output while tolerating common JSON labels and fences. */
+function parseOllamaStructuredResponse(response) {
+    if (typeof response !== "string") {
+        return response;
+    }
+    const normalized = normalizeOllamaStructuredText(response);
+    try {
+        return JSON.parse(normalized);
+    }
+    catch (error) {
+        const candidates = [
+            [normalized.indexOf("{"), normalized.lastIndexOf("}")],
+            [normalized.indexOf("["), normalized.lastIndexOf("]")],
+        ];
+        for (const [start, end] of candidates) {
+            if (start >= 0 && end > start) {
+                try {
+                    return JSON.parse(normalized.slice(start, end + 1));
+                }
+                catch {
+                    // Try the next complete JSON container before reporting the error.
+                }
+            }
+        }
+        throw new Error(`Failed to parse Ollama response as JSON: ${error}. Response: ${response}`);
+    }
+}
+function createStructuredOllamaClient(client) {
+    return new Proxy(client, {
+        get(target, property) {
+            if (property === "chat") {
+                return async (request) => {
+                    const result = await target.chat(request);
+                    result.message.content = JSON.stringify(parseOllamaStructuredResponse(result.message.content));
+                    return result;
+                };
+            }
+            const value = Reflect.get(target, property, target);
+            return typeof value === "function" ? value.bind(target) : value;
+        },
+    });
+}
+/** Calls `askOllama` with SDA-owned structured-output and cache overrides. */
+function askOllamaAdapter(prompt, options) {
+    const customOllama = options.generation.ollama instanceof ollama_1.Ollama
+        ? options.generation.ollama
+        : undefined;
+    return (0, index_js_1.askOllama)(prompt, {
+        ...options.generation,
+        systemPrompt: options.systemPrompt,
+        ollama: options.schemaJson
+            ? createStructuredOllamaClient(customOllama ?? new ollama_1.Ollama())
+            : customOllama,
+        schemaJson: options.schemaJson,
+        cache: options.disableCache ? false : options.generation.cache,
+    });
+}

package/script/helpers/buildAIRowsRequest.d.ts

@@ -0,0 +1,16 @@
+import type { GenerationOptions } from "./aiOptions.js";
+type AIRow = {
+    [key: string]: unknown;
+};
+type AIRowsRequestOptions = {
+    generation?: GenerationOptions;
+    extraInstructions?: string;
+};
+/** Builds the shared structured request used by SDA's row-processing methods. */
+export default function buildAIRowsRequest(batch: AIRow[], column: string, newColumns: string[], prompt: string, options: AIRowsRequestOptions): {
+    prompt: string;
+    schemaJson: {};
+    systemPrompt: string;
+};
+export {};
+//# sourceMappingURL=buildAIRowsRequest.d.ts.map

package/script/helpers/buildAIRowsRequest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildAIRowsRequest.d.ts","sourceRoot":"","sources":["../../src/helpers/buildAIRowsRequest.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,KAAK,KAAK,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAExC,KAAK,oBAAoB,GAAG;IAC1B,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,iFAAiF;AACjF,MAAM,CAAC,OAAO,UAAU,kBAAkB,CACxC,KAAK,EAAE,KAAK,EAAE,EACd,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAAE,EACpB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB;;;;EAiB9B"}

package/script/helpers/buildAIRowsRequest.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildAIRowsRequest;
+const zod_1 = require("zod");
+/** Builds the shared structured request used by SDA's row-processing methods. */
+function buildAIRowsRequest(batch, column, newColumns, prompt, options) {
+    const schemaJson = options.generation?.schemaJson ??
+        (0, zod_1.toJSONSchema)((0, zod_1.array)((0, zod_1.object)(Object.fromEntries(newColumns.map((name) => [name, (0, zod_1.string)()])))));
+    const requiredSystemPrompt = `You will be provided with a JSON array of ${batch.length} string items. You must return a JSON array containing exactly ${batch.length} objects, in the same corresponding order.`;
+    const systemPrompt = options.generation?.systemPrompt
+        ? `${options.generation.systemPrompt}\n\n${requiredSystemPrompt}`
+        : requiredSystemPrompt;
+    const requestPrompt = `${prompt}\n\nHere are the ${column} values as a JSON array:\n${JSON.stringify(batch.map((row) => row[column]), null, 2)}\n\n${options.extraInstructions ? `\n${options.extraInstructions}` : ""}`;
+    return { prompt: requestPrompt, schemaJson, systemPrompt };
+}

package/script/helpers/ensureEmbeddingColumn.d.ts

@@ -0,0 +1,25 @@
+import type { EmbeddingIdentity } from "../deps/jsr.io/@nshiab/journalism-ai/2.0.6/src/index.js";
+import type SimpleTable from "../class/SimpleTable.js";
+/**
+ * Ensures that an SDA-managed embedding column exists in the requested vector
+ * space, regenerating legacy or incompatible vectors before recording their
+ * provenance.
+ *
+ * @param table Table that owns the source and embedding columns.
+ * @param sourceColumn Text column used to generate the vectors.
+ * @param embeddingColumn SDA-managed vector column.
+ * @param identity Canonical upstream embedding identity for the request.
+ * @param generate Regenerates the managed column when reuse is unsafe.
+ * @returns Whether the existing vectors were reused or regenerated.
+ *
+ * @example
+ * ```ts
+ * await ensureEmbeddingColumn(table, "text", "text_embeddings", identity, async () => {
+ *   await generateEmbeddings();
+ * });
+ * ```
+ *
+ * @internal
+ */
+export default function ensureEmbeddingColumn(table: SimpleTable, sourceColumn: string, embeddingColumn: string, identity: EmbeddingIdentity, generate: () => Promise<void>): Promise<"reused" | "generated">;
+//# sourceMappingURL=ensureEmbeddingColumn.d.ts.map

package/script/helpers/ensureEmbeddingColumn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ensureEmbeddingColumn.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureEmbeddingColumn.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yDAAyD,CAAC;AAEjG,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AA4GvD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAA8B,qBAAqB,CACjD,KAAK,EAAE,WAAW,EAClB,YAAY,EAAE,MAAM,EACpB,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,iBAAiB,EAC3B,QAAQ,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAC5B,OAAO,CAAC,QAAQ,GAAG,WAAW,CAAC,CAwCjC"}

package/script/helpers/ensureEmbeddingColumn.js

@@ -0,0 +1,111 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = ensureEmbeddingColumn;
+const index_js_1 = require("../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js");
+const METADATA_TABLE = "__sda_embedding_column_metadata";
+const METADATA_SCHEMA_VERSION = 1;
+function quoteLiteral(value) {
+    return `'${value.replaceAll("'", "''")}'`;
+}
+async function ensureMetadataTable(table) {
+    await table.sdb.customQuery(`CREATE TABLE IF NOT EXISTS ${(0, index_js_1.quoteIdentifier)(METADATA_TABLE)} (
+      table_name VARCHAR NOT NULL,
+      embedding_column VARCHAR NOT NULL,
+      source_column VARCHAR NOT NULL,
+      identity_json VARCHAR NOT NULL,
+      dimensions INTEGER NOT NULL,
+      schema_version INTEGER NOT NULL,
+      PRIMARY KEY (table_name, embedding_column)
+    );`);
+}
+async function readMetadata(table, embeddingColumn) {
+    const rows = await table.sdb.customQuery(`SELECT source_column, identity_json, dimensions, schema_version
+    FROM ${(0, index_js_1.quoteIdentifier)(METADATA_TABLE)}
+    WHERE table_name = ${quoteLiteral(table.name)}
+      AND embedding_column = ${quoteLiteral(embeddingColumn)};`, { returnData: true });
+    return rows[0];
+}
+async function deleteMetadata(table, embeddingColumn) {
+    await table.sdb.customQuery(`DELETE FROM ${(0, index_js_1.quoteIdentifier)(METADATA_TABLE)}
+    WHERE table_name = ${quoteLiteral(table.name)}
+      AND embedding_column = ${quoteLiteral(embeddingColumn)};`);
+}
+async function getDimensions(table, embeddingColumn) {
+    const type = (await table.getTypes())[embeddingColumn];
+    const fixedSize = type?.match(/\[(\d+)\]$/)?.[1];
+    if (fixedSize !== undefined) {
+        return Number(fixedSize);
+    }
+    const values = await table.getValues(embeddingColumn);
+    const vector = values.find(Array.isArray);
+    return vector?.length ?? 0;
+}
+async function dropManagedVssIndexes(table) {
+    const indexes = await table.sdb.customQuery(`SELECT index_name FROM duckdb_indexes()
+    WHERE table_name = ${quoteLiteral(table.name)};`, { returnData: true });
+    const managedIndexes = indexes.filter(({ index_name }) => index_name.startsWith("vss_cosine_index_"));
+    for (const { index_name } of managedIndexes) {
+        await table.sdb.customQuery(`DROP INDEX IF EXISTS ${(0, index_js_1.quoteIdentifier)(index_name)};`);
+    }
+    const dropped = new Set(managedIndexes.map(({ index_name }) => index_name));
+    table.indexes = table.indexes.filter((index) => !dropped.has(index.name));
+}
+async function writeMetadata(table, sourceColumn, embeddingColumn, identityJson, dimensions) {
+    await table.sdb.customQuery(`INSERT OR REPLACE INTO ${(0, index_js_1.quoteIdentifier)(METADATA_TABLE)}
+      (table_name, embedding_column, source_column, identity_json, dimensions, schema_version)
+    VALUES (
+      ${quoteLiteral(table.name)},
+      ${quoteLiteral(embeddingColumn)},
+      ${quoteLiteral(sourceColumn)},
+      ${quoteLiteral(identityJson)},
+      ${dimensions},
+      ${METADATA_SCHEMA_VERSION}
+    );`);
+}
+/**
+ * Ensures that an SDA-managed embedding column exists in the requested vector
+ * space, regenerating legacy or incompatible vectors before recording their
+ * provenance.
+ *
+ * @param table Table that owns the source and embedding columns.
+ * @param sourceColumn Text column used to generate the vectors.
+ * @param embeddingColumn SDA-managed vector column.
+ * @param identity Canonical upstream embedding identity for the request.
+ * @param generate Regenerates the managed column when reuse is unsafe.
+ * @returns Whether the existing vectors were reused or regenerated.
+ *
+ * @example
+ * ```ts
+ * await ensureEmbeddingColumn(table, "text", "text_embeddings", identity, async () => {
+ *   await generateEmbeddings();
+ * });
+ * ```
+ *
+ * @internal
+ */
+async function ensureEmbeddingColumn(table, sourceColumn, embeddingColumn, identity, generate) {
+    const identityJson = JSON.stringify(identity);
+    await ensureMetadataTable(table);
+    const columnExists = await table.hasColumn(embeddingColumn);
+    const metadata = await readMetadata(table, embeddingColumn);
+    if (columnExists && metadata !== undefined) {
+        const dimensions = await getDimensions(table, embeddingColumn);
+        if (metadata.schema_version === METADATA_SCHEMA_VERSION &&
+            metadata.source_column === sourceColumn &&
+            metadata.identity_json === identityJson &&
+            metadata.dimensions === dimensions) {
+            return "reused";
+        }
+    }
+    await deleteMetadata(table, embeddingColumn);
+    if (columnExists) {
+        await dropManagedVssIndexes(table);
+    }
+    await generate();
+    if (!(await table.hasColumn(embeddingColumn))) {
+        throw new Error(`Embedding generation did not create ${(0, index_js_1.quoteIdentifier)(embeddingColumn)} in table ${(0, index_js_1.quoteIdentifier)(table.name)}.`);
+    }
+    const dimensions = await getDimensions(table, embeddingColumn);
+    await writeMetadata(table, sourceColumn, embeddingColumn, identityJson, dimensions);
+    return "generated";
+}

package/script/helpers/loadGeoDataFromScratchFile.d.ts

@@ -0,0 +1,11 @@
+import type SimpleTable from "../class/SimpleTable.js";
+/**
+ * Loads a GeoJSON string through a temporary file for DuckDB compatibility.
+ *
+ * @param table - The table into which the GeoJSON data is loaded.
+ * @param geoData - The GeoJSON string to load.
+ * @returns A promise that resolves when the data is loaded and the temporary file is removed.
+ * @internal
+ */
+export default function loadGeoDataFromScratchFile(table: SimpleTable, geoData: string): Promise<void>;
+//# sourceMappingURL=loadGeoDataFromScratchFile.d.ts.map

package/script/helpers/loadGeoDataFromScratchFile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadGeoDataFromScratchFile.d.ts","sourceRoot":"","sources":["../../src/helpers/loadGeoDataFromScratchFile.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD;;;;;;;GAOG;AACH,wBAA8B,0BAA0B,CACtD,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAaf"}

package/script/helpers/loadGeoDataFromScratchFile.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadGeoDataFromScratchFile;
+const node_crypto_1 = require("node:crypto");
+const node_fs_1 = require("node:fs");
+const index_js_1 = require("../deps/jsr.io/@nshiab/simple-data-analysis-core/2.0.2/src/helpers/index.js");
+const temporaryDatavizDirectory = ".sda-cache/tmp/dataviz";
+/**
+ * Loads a GeoJSON string through a temporary file for DuckDB compatibility.
+ *
+ * @param table - The table into which the GeoJSON data is loaded.
+ * @param geoData - The GeoJSON string to load.
+ * @returns A promise that resolves when the data is loaded and the temporary file is removed.
+ * @internal
+ */
+async function loadGeoDataFromScratchFile(table, geoData) {
+    const path = `${temporaryDatavizDirectory}/${(0, node_crypto_1.randomUUID)()}.geojson`;
+    (0, index_js_1.createDirectory)(path);
+    try {
+        (0, node_fs_1.writeFileSync)(path, geoData);
+        table.loadGeoData(path);
+        await table.run();
+    }
+    finally {
+        if ((0, node_fs_1.existsSync)(path)) {
+            (0, node_fs_1.unlinkSync)(path);
+        }
+    }
+}

package/script/helpers/processAIRowsResponse.d.ts

@@ -0,0 +1,10 @@
+type AIRow = {
+    [key: string]: unknown;
+};
+/** Cleans and validates a batched AI response before it is cached or stored. */
+export default function processAIRowsResponse(rawResponse: unknown, batchLength: number, newColumns: string[], options: {
+    clean?: (response: unknown) => unknown;
+    test?: (result: AIRow) => void;
+}): AIRow[];
+export {};
+//# sourceMappingURL=processAIRowsResponse.d.ts.map

package/script/helpers/processAIRowsResponse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"processAIRowsResponse.d.ts","sourceRoot":"","sources":["../../src/helpers/processAIRowsResponse.ts"],"names":[],"mappings":"AAAA,KAAK,KAAK,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAExC,gFAAgF;AAChF,MAAM,CAAC,OAAO,UAAU,qBAAqB,CAC3C,WAAW,EAAE,OAAO,EACpB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC;IACvC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;CAChC,GACA,KAAK,EAAE,CA8BT"}

package/script/helpers/processAIRowsResponse.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = processAIRowsResponse;
+/** Cleans and validates a batched AI response before it is cached or stored. */
+function processAIRowsResponse(rawResponse, batchLength, newColumns, options) {
+    const response = options.clean ? options.clean(rawResponse) : rawResponse;
+    if (!Array.isArray(response)) {
+        throw new Error(`The AI returned a non-array value: ${JSON.stringify(response)}`);
+    }
+    if (response.length !== batchLength) {
+        throw new Error(`The AI returned ${response.length} values, but the batch size is ${batchLength}.`);
+    }
+    for (const item of response) {
+        if (typeof item !== "object" || item === null) {
+            throw new Error(`The AI did not return an object: ${JSON.stringify(item)}`);
+        }
+        for (const column of newColumns) {
+            if (!(column in item)) {
+                throw new Error(`The AI's response is missing the key '${column}': ${JSON.stringify(item)}`);
+            }
+        }
+        options.test?.(item);
+    }
+    return response;
+}

package/script/helpers/publishDatawrapper.d.ts

@@ -0,0 +1,8 @@
+type PublishDatawrapperOptions = {
+    apiKeyEnvVar?: string;
+    note?: string;
+    republish?: boolean;
+};
+export default function publishDatawrapper(chartId: string, data: string, options?: PublishDatawrapperOptions): Promise<void>;
+export {};
+//# sourceMappingURL=publishDatawrapper.d.ts.map

package/script/helpers/publishDatawrapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"publishDatawrapper.d.ts","sourceRoot":"","sources":["../../src/helpers/publishDatawrapper.ts"],"names":[],"mappings":"AAAA,KAAK,yBAAyB,GAAG;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,wBAA8B,kBAAkB,CAC9C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,IAAI,CAAC,CAYf"}