@nshiab/simple-data-analysis 5.21.0 → 6.0.0

package/esm/helpers/tryAI.js

@@ -1,75 +0,0 @@
-import { askAI } from "../deps/jsr.io/@nshiab/journalism-ai/1.2.23/src/index.js";
-import { array, object, string, toJSONSchema } from "zod";
-export default async function tryAI(i, batchSize, rows, column, newColumns, prompt, options = {}) {
-    const batch = rows.slice(i, i + batchSize);
-    let schemaJson;
-    if (options.schemaJson) {
-        schemaJson = options.schemaJson;
-    }
-    else {
-        const objectSchema = {};
-        for (const newColumn of newColumns) {
-            objectSchema[newColumn] = string();
-        }
-        schemaJson = toJSONSchema(array(object(objectSchema)));
-    }
-    const systemPrompt = `You will be provided with a JSON array of ${batch.length} string items. You must return a JSON array containing exactly ${batch.length} objects, in the same corresponding order.`;
-    const fullPrompt = `${prompt}\n\nHere are the ${column} values as a JSON array:\n${JSON.stringify(batch.map((d) => d[column]), null, 2)}\n\n${options.extraInstructions ? `\n${options.extraInstructions}` : ""}`;
-    const retry = options.retry ?? 1;
-    let testPassed = false;
-    let iterations = 1;
-    let newValues;
-    while (!testPassed && iterations <= retry) {
-        try {
-            newValues = await askAI(`${fullPrompt}${iterations > 1
-                ? `\nThis is your attempt #${iterations}. So get it right by following my instructions closely!`
-                : ""}`, {
-                ...options,
-                includeThoughts: options.verbose ? true : false,
-                systemPrompt: systemPrompt,
-                schemaJson,
-                test: (response) => {
-                    if (!Array.isArray(response)) {
-                        throw new Error(`The AI returned a non-array value: ${JSON.stringify(response)}`);
-                    }
-                    if (response.length !== batch.length) {
-                        throw new Error(`The AI returned ${response.length} values, but the batch size is ${batch.length}.`);
-                    }
-                    for (const item of response) {
-                        if (typeof item !== "object" || item === null) {
-                            throw new Error(`The AI did not return an object: ${JSON.stringify(item)}`);
-                        }
-                        for (const newColumn of newColumns) {
-                            if (!(newColumn in item)) {
-                                throw new Error(`The AI's response is missing the key '${newColumn}': ${JSON.stringify(item)}`);
-                            }
-                        }
-                    }
-                    if (options.test) {
-                        for (const item of response) {
-                            options.test(item);
-                        }
-                    }
-                },
-            });
-            testPassed = true;
-        }
-        catch (e) {
-            if (iterations < retry) {
-                console.log(`${console.log(e)}\nRetrying... (${iterations}/${retry})`);
-                iterations++;
-            }
-            else {
-                console.log(`${console.log(e)}\nNo more retries left. (${iterations}/${retry}).`);
-                throw e;
-            }
-        }
-    }
-    // Types could be improved
-    for (let j = 0; j <
-        newValues.length; j++) {
-        for (const newColumn of newColumns) {
-            rows[i + j][newColumn] = newValues[j][newColumn];
-        }
-    }
-}

package/esm/methods/aiRowByRowPool.d.ts

@@ -1,36 +0,0 @@
-import type SimpleTable from "../class/SimpleTable.js";
-export default function aiRowByRowPool(table: SimpleTable, column: string, newColumn: string | string[], errorColumn: string, prompt: string, poolSize: number, options: {
-    cache?: boolean;
-    batchSize?: number;
-    logProgress?: boolean;
-    verbose?: boolean;
-    includeThoughts?: boolean;
-    test?: (result: {
-        [key: string]: unknown;
-    }) => void;
-    retry?: number;
-    retryCheck?: (error: unknown) => Promise<boolean> | boolean;
-    extraInstructions?: string;
-    minRequestDurationMs?: number;
-    clean?: (response: unknown) => unknown;
-    contextWindow?: number;
-    thinkingBudget?: number;
-    thinkingLevel?: "minimal" | "low" | "medium" | "high";
-    webSearch?: boolean;
-    safetyEnabled?: boolean;
-    schemaJson?: unknown;
-    model?: string;
-    temperature?: number;
-    apiKey?: string;
-    vertex?: boolean;
-    project?: string;
-    location?: string;
-    ollama?: boolean | any;
-    metrics?: {
-        totalCost: number;
-        totalInputTokens: number;
-        totalOutputTokens: number;
-        totalRequests: number;
-    };
-}): Promise<void>;
-//# sourceMappingURL=aiRowByRowPool.d.ts.map

package/esm/methods/aiRowByRowPool.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"aiRowByRowPool.d.ts","sourceRoot":"","sources":["../../src/methods/aiRowByRowPool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,wBAA8B,cAAc,CAC1C,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC5D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,KAAK,CAAC,EAAE,CACN,QAAQ,EAAE,OAAO,KACd,OAAO,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACtD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,MAAM,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC;IACvB,OAAO,CAAC,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACH,iBA4KF"}

package/esm/methods/aiRowByRowPool.js

@@ -1,137 +0,0 @@
-import { askAIPool } from "../deps/jsr.io/@nshiab/journalism-ai/1.2.23/src/index.js";
-import { array, object, string, toJSONSchema } from "zod";
-export default async function aiRowByRowPool(table, column, newColumn, errorColumn, prompt, poolSize, options) {
-    const newColumns = Array.isArray(newColumn) ? newColumn : [newColumn];
-    await table.updateWithJS(async (rows) => {
-        if (options.verbose) {
-            console.log("\naiRowByRowPool()");
-        }
-        const batches = [];
-        for (let i = 0; i < rows.length; i += options.batchSize ?? 1) {
-            batches.push(rows.slice(i, i + (options.batchSize ?? 1)));
-        }
-        const requests = batches.map((batch) => {
-            let schemaJson;
-            if (options.schemaJson) {
-                schemaJson = options.schemaJson;
-            }
-            else {
-                const objectSchema = {};
-                for (const newColumn of newColumns) {
-                    objectSchema[newColumn] = string();
-                }
-                schemaJson = toJSONSchema(array(object(objectSchema)));
-            }
-            const systemPrompt = `You will be provided with a JSON array of ${batch.length} string items. You must return a JSON array containing exactly ${batch.length} objects, in the same corresponding order.`;
-            const batchPrompt = `${prompt}\n\nHere are the ${column} values as a JSON array:\n${JSON.stringify(batch.map((d) => d[column]), null, 2)}\n\n${options.extraInstructions ? `\n${options.extraInstructions}` : ""}`;
-            return {
-                prompt: batchPrompt,
-                options: {
-                    systemPrompt: systemPrompt,
-                    cache: options.cache,
-                    schemaJson,
-                    webSearch: options.webSearch,
-                    clean: options.clean,
-                    model: options.model,
-                    contextWindow: options.contextWindow,
-                    thinkingBudget: options.thinkingBudget,
-                    thinkingLevel: options.thinkingLevel,
-                    metrics: options.metrics,
-                    verbose: options.verbose,
-                    includeThoughts: options.includeThoughts,
-                    temperature: options.temperature,
-                    safetyEnabled: options.safetyEnabled,
-                    apiKey: options.apiKey,
-                    vertex: options.vertex,
-                    project: options.project,
-                    location: options.location,
-                    ollama: options.ollama,
-                    test: (response) => {
-                        if (!Array.isArray(response)) {
-                            throw new Error(`The AI returned a non-array value: ${JSON.stringify(response)}`);
-                        }
-                        if (response.length !== batch.length) {
-                            throw new Error(`The AI returned ${response.length} values, but the batch size is ${batch.length}.`);
-                        }
-                        for (const item of response) {
-                            if (typeof item !== "object" || item === null) {
-                                throw new Error(`The AI did not return an object: ${JSON.stringify(item)}`);
-                            }
-                            for (const newColumn of newColumns) {
-                                if (!(newColumn in item)) {
-                                    throw new Error(`The AI's response is missing the key '${newColumn}': ${JSON.stringify(item)}`);
-                                }
-                            }
-                        }
-                        if (options.test) {
-                            for (const item of response) {
-                                options.test(item);
-                            }
-                        }
-                    },
-                },
-            };
-        });
-        const { results, errors } = await askAIPool(requests, poolSize, {
-            retry: options.retry,
-            retryCheck: options.retryCheck,
-            minRequestDurationMs: options.minRequestDurationMs,
-            logProgress: options.logProgress,
-            metrics: options.metrics,
-        });
-        const resultsFlattened = results.map((d) => ({
-            index: d.index,
-            result: d.result,
-            error: null,
-        })).flat();
-        const errorsFlattened = errors.map((d) => ({
-            index: d.index,
-            result: null,
-            error: d.error instanceof Error && d.error.message
-                ? d.error.message
-                : String(d.error),
-        })).flat();
-        const resultsAndErrors = [...resultsFlattened, ...errorsFlattened].sort((a, b) => a.index - b.index);
-        const newRows = [];
-        for (let i = 0; i < batches.length; i++) {
-            const resultOrError = resultsAndErrors[i];
-            if (resultOrError.result) {
-                if (Array.isArray(resultOrError.result)) {
-                    for (let j = 0; j < resultOrError.result.length; j++) {
-                        newRows.push({
-                            ...batches[i][j],
-                            ...resultOrError.result[j],
-                            [errorColumn]: null,
-                        });
-                    }
-                }
-                else if (typeof resultOrError.result === "object" &&
-                    resultOrError.result !== null) {
-                    newRows.push({
-                        ...batches[i][0],
-                        ...resultOrError.result,
-                        [errorColumn]: null,
-                    });
-                }
-                else {
-                    throw new Error(`Unexpected result format: ${JSON.stringify(resultOrError.result)}`);
-                }
-            }
-            else if (resultOrError.error) {
-                for (const row of batches[i]) {
-                    const obj = {
-                        [errorColumn]: resultOrError.error,
-                    };
-                    for (const newColumn of newColumns) {
-                        obj[newColumn] = null;
-                    }
-                    newRows.push({
-                        ...row,
-                        ...obj,
-                    });
-                }
-            }
-        }
-        return newRows;
-    });
-}