@retab/node 0.0.48 → 0.0.52

package/dist/utils/datasets.d.ts

@@ -1,135 +0,0 @@
-import { SyncAPIResource, AsyncAPIResource } from '../resource.js';
-import { DatasetMetrics } from './display.js';
-/**
- * Advanced Dataset management utilities for ML training workflows
- * Equivalent to Python's jsonlUtils.py
- */
-export interface FinetuningJSON {
-    messages: Array<{
-        role: 'system' | 'user' | 'assistant';
-        content: string;
-    }>;
-}
-export interface DocumentAnnotationPair {
-    document: string | Buffer;
-    annotation: Record<string, any>;
-}
-export interface BatchJSONLRequest {
-    custom_id: string;
-    method: 'POST';
-    url: string;
-    body: Record<string, any>;
-}
-export interface BatchJSONLResponse {
-    id: string;
-    custom_id: string;
-    response: {
-        status_code: number;
-        request_id: string;
-        body: Record<string, any>;
-    };
-    error?: {
-        code: string;
-        message: string;
-    };
-}
-export interface AnnotationOptions {
-    model?: string;
-    temperature?: number;
-    modality?: 'native' | 'text';
-    maxConcurrency?: number;
-    reasoning_effort?: 'low' | 'medium' | 'high';
-    provider?: 'openai' | 'anthropic' | 'xai' | 'gemini';
-    idempotencyKey?: string;
-}
-export interface SaveOptions {
-    modality?: 'native' | 'text';
-    imageResolutionDpi?: number;
-    browserCanvas?: 'A3' | 'A4' | 'A5';
-}
-export declare class BaseDatasetsMixin {
-    /**
-     * Process dataset and compute comprehensive metrics
-     */
-    pprint(datasetPath: string, inputTokenPrice?: number, outputTokenPrice?: number): Promise<DatasetMetrics>;
-    /**
-     * Save document-annotation pairs as JSONL training dataset
-     */
-    save(jsonSchema: Record<string, any> | string, documentAnnotationPairsPaths: Array<{
-        document: string;
-        annotation: string;
-    }>, datasetPath: string, options?: SaveOptions): Promise<void>;
-    /**
-     * Change schema in existing dataset
-     */
-    changeSchema(inputDatasetPath: string, jsonSchema: Record<string, any> | string, outputDatasetPath?: string, inplace?: boolean): Promise<void>;
-    /**
-     * Stitch multiple documents and save as dataset
-     */
-    stitchAndSave(jsonSchema: Record<string, any> | string, pairsPaths: Array<{
-        documents: string[];
-        annotation: string;
-    }>, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
-    /**
-     * Generate annotations for documents using AI models
-     */
-    annotate(jsonSchema: Record<string, any> | string, documents: string[], datasetPath: string, options?: AnnotationOptions): Promise<void>;
-    /**
-     * Update existing annotations with new model/schema
-     */
-    updateAnnotations(jsonSchema: Record<string, any> | string, oldDatasetPath: string, newDatasetPath: string, options?: AnnotationOptions): Promise<void>;
-    /**
-     * Save batch annotation requests for OpenAI Batch API
-     */
-    saveBatchAnnotateRequests(jsonSchema: Record<string, any> | string, documents: string[], batchRequestsPath: string, options?: AnnotationOptions): Promise<void>;
-    /**
-     * Build dataset from batch API results
-     */
-    buildDatasetFromBatchResults(jsonSchema: Record<string, any> | string, batchResultsPath: string, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
-    private createSystemMessage;
-    private createUserMessage;
-    private createMultiDocumentUserMessage;
-    private generateAnnotation;
-    private generateAnnotationFromUserMessage;
-}
-export declare class Datasets extends SyncAPIResource {
-    private mixin;
-    pprint(datasetPath: string, inputTokenPrice?: number, outputTokenPrice?: number): Promise<DatasetMetrics>;
-    save(jsonSchema: Record<string, any> | string, documentAnnotationPairsPaths: Array<{
-        document: string;
-        annotation: string;
-    }>, datasetPath: string, options?: SaveOptions): Promise<void>;
-    changeSchema(inputDatasetPath: string, jsonSchema: Record<string, any> | string, outputDatasetPath?: string, inplace?: boolean): Promise<void>;
-    stitchAndSave(jsonSchema: Record<string, any> | string, pairsPaths: Array<{
-        documents: string[];
-        annotation: string;
-    }>, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
-    annotate(jsonSchema: Record<string, any> | string, documents: string[], datasetPath: string, options?: AnnotationOptions): Promise<void>;
-    updateAnnotations(jsonSchema: Record<string, any> | string, oldDatasetPath: string, newDatasetPath: string, options?: AnnotationOptions): Promise<void>;
-    saveBatchAnnotateRequests(jsonSchema: Record<string, any> | string, documents: string[], batchRequestsPath: string, options?: AnnotationOptions): Promise<void>;
-    buildDatasetFromBatchResults(jsonSchema: Record<string, any> | string, batchResultsPath: string, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
-}
-export declare class AsyncDatasets extends AsyncAPIResource {
-    private mixin;
-    pprint(datasetPath: string, inputTokenPrice?: number, outputTokenPrice?: number): Promise<DatasetMetrics>;
-    save(jsonSchema: Record<string, any> | string, documentAnnotationPairsPaths: Array<{
-        document: string;
-        annotation: string;
-    }>, datasetPath: string, options?: SaveOptions): Promise<void>;
-    changeSchema(inputDatasetPath: string, jsonSchema: Record<string, any> | string, outputDatasetPath?: string, inplace?: boolean): Promise<void>;
-    stitchAndSave(jsonSchema: Record<string, any> | string, pairsPaths: Array<{
-        documents: string[];
-        annotation: string;
-    }>, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
-    annotate(jsonSchema: Record<string, any> | string, documents: string[], datasetPath: string, options?: AnnotationOptions): Promise<void>;
-    updateAnnotations(jsonSchema: Record<string, any> | string, oldDatasetPath: string, newDatasetPath: string, options?: AnnotationOptions): Promise<void>;
-    saveBatchAnnotateRequests(jsonSchema: Record<string, any> | string, documents: string[], batchRequestsPath: string, options?: AnnotationOptions): Promise<void>;
-    buildDatasetFromBatchResults(jsonSchema: Record<string, any> | string, batchResultsPath: string, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
-}
-declare const _default: {
-    Datasets: typeof Datasets;
-    AsyncDatasets: typeof AsyncDatasets;
-    BaseDatasetsMixin: typeof BaseDatasetsMixin;
-};
-export default _default;
-//# sourceMappingURL=datasets.d.ts.map

package/dist/utils/datasets.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../../src/utils/datasets.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAmD,cAAc,EAAE,MAAM,cAAc,CAAC;AAE/F;;;GAGG;AAEH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QACtC,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAC3B,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC7C,QAAQ,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,KAAK,GAAG,QAAQ,CAAC;IACrD,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;CACpC;AAED,qBAAa,iBAAiB;IAC5B;;OAEG;IACG,MAAM,CACV,WAAW,EAAE,MAAM,EACnB,eAAe,GAAE,MAAgB,EACjC,gBAAgB,GAAE,MAAe,GAChC,OAAO,CAAC,cAAc,CAAC;IAe1B;;OAEG;IACG,IAAI,CACR,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,4BAA4B,EAAE,KAAK,CAAC;QAClC,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,EACF,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,WAAgB,GACxB,OAAO,CAAC,IAAI,CAAC;IAkChB;;OAEG;IACG,YAAY,CAChB,gBAAgB,EAAE,MAAM,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,IAAI,CAAC;IAkChB;;OAEG;IACG,aAAa,CACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,UAAU,EAAE,KAAK,CAAC;QAChB,SAAS,EAAE,MAAM,EAAE,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,EACF,WAAW,EAAE,MAAM,EACnB,QAAQ,GAAE,QAAQ,GAAG,MAAiB,GACrC,OAAO,CAAC,IAAI,CAAC;IAkChB;;OAEG;IACG,QAAQ,CACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC;IAyDhB;;OAEG;IACG,iBAAiB,CACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC;IAoDhB;;OAEG;IACG,yBAAyB,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,iBAAiB,EAAE,MAAM,EACzB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC;IA+BhB;;OAEG;IACG,4BAA4B,CAChC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,gBAAgB,EAAE,MAAM,EACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,GAAE,QAAQ,GAAG,MAAiB,GACrC,OAAO,CAAC,IAAI,CAAC;IAoDhB,OAAO,CAAC,mBAAmB;YAab,iBAAiB;YAejB,8BAA8B;YAe9B,kBAAkB;YAclB,iCAAiC;CAQhD;AAED,qBAAa,QAAS,SAAQ,eAAe;IAC3C,OAAO,CAAC,KAAK,CAA2B;IAElC,MAAM,CACV,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,MAAM,EACxB,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,cAAc,CAAC;IAIpB,IAAI,CACR,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,4BAA4B,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EAC7E,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,IAAI,CAAC;IAIV,YAAY,CAChB,gBAAgB,EAAE,MAAM,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,OAAO,CAAC,EAAE,OAAO,GAChB,OAAO,CAAC,IAAI,CAAC;IAIV,aAAa,CACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,UAAU,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EAC9D,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,GAC3B,OAAO,CAAC,IAAI,CAAC;IAIV,QAAQ,CACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,iBAAiB,CACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,yBAAyB,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,iBAAiB,EAAE,MAAM,EACzB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,4BAA4B,CAChC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,gBAAgB,EAAE,MAAM,EACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,GAC3B,OAAO,CAAC,IAAI,CAAC;CAGjB;AAED,qBAAa,aAAc,SAAQ,gBAAgB;IACjD,OAAO,CAAC,KAAK,CAA2B;IAElC,MAAM,CACV,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,MAAM,EACxB,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,cAAc,CAAC;IAIpB,IAAI,CACR,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,4BAA4B,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EAC7E,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,IAAI,CAAC;IAIV,YAAY,CAChB,gBAAgB,EAAE,MAAM,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,OAAO,CAAC,EAAE,OAAO,GAChB,OAAO,CAAC,IAAI,CAAC;IAIV,aAAa,CACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,UAAU,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EAC9D,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,GAC3B,OAAO,CAAC,IAAI,CAAC;IAIV,QAAQ,CACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,iBAAiB,CACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,yBAAyB,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,iBAAiB,EAAE,MAAM,EACzB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,4BAA4B,CAChC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,gBAAgB,EAAE,MAAM,EACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,GAC3B,OAAO,CAAC,IAAI,CAAC;CAGjB;;;;;;AAED,wBAIE"}

package/dist/utils/datasets.js

@@ -1,359 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { SyncAPIResource, AsyncAPIResource } from '../resource.js';
-import { readJSONL, writeJSONL } from './jsonl.js';
-import { displayMetrics, processDatasetAndComputeMetrics } from './display.js';
-export class BaseDatasetsMixin {
-    /**
-     * Process dataset and compute comprehensive metrics
-     */
-    async pprint(datasetPath, inputTokenPrice = 0.00015, outputTokenPrice = 0.0006) {
-        if (!fs.existsSync(datasetPath)) {
-            throw new Error(`Dataset file not found: ${datasetPath}`);
-        }
-        const metrics = await processDatasetAndComputeMetrics(datasetPath, inputTokenPrice, outputTokenPrice);
-        displayMetrics(metrics);
-        return metrics;
-    }
-    /**
-     * Save document-annotation pairs as JSONL training dataset
-     */
-    async save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options = {}) {
-        const { modality = 'native' } = options;
-        const finetuningData = [];
-        for (const { document: docPath, annotation: annPath } of documentAnnotationPairsPaths) {
-            // Read document and annotation
-            if (!fs.existsSync(docPath) || !fs.existsSync(annPath)) {
-                throw new Error(`Document or annotation file not found: ${docPath}, ${annPath}`);
-            }
-            const annotation = JSON.parse(fs.readFileSync(annPath, 'utf-8'));
-            // Create system message with schema
-            const systemMessage = this.createSystemMessage(jsonSchema, modality);
-            // Create user message with document
-            const userMessage = await this.createUserMessage(docPath, modality, options);
-            // Create assistant message with annotation
-            const assistantMessage = {
-                role: 'assistant',
-                content: JSON.stringify(annotation),
-            };
-            finetuningData.push({
-                messages: [systemMessage, userMessage, assistantMessage],
-            });
-        }
-        // Write to JSONL file
-        await writeJSONL(datasetPath, finetuningData);
-        console.log(`✅ Dataset saved to ${datasetPath} with ${finetuningData.length} examples`);
-    }
-    /**
-     * Change schema in existing dataset
-     */
-    async changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace = false) {
-        if (!fs.existsSync(inputDatasetPath)) {
-            throw new Error(`Input dataset not found: ${inputDatasetPath}`);
-        }
-        const outputPath = inplace ? inputDatasetPath : (outputDatasetPath || inputDatasetPath);
-        const tempPath = `${outputPath}.tmp`;
-        try {
-            const dataset = await readJSONL(inputDatasetPath);
-            const newSystemMessage = this.createSystemMessage(jsonSchema, 'native');
-            const updatedDataset = dataset.map((item) => ({
-                ...item,
-                messages: [
-                    newSystemMessage,
-                    ...item.messages.slice(1), // Keep user and assistant messages
-                ],
-            }));
-            await writeJSONL(tempPath, updatedDataset);
-            // Atomic move
-            fs.renameSync(tempPath, outputPath);
-            console.log(`✅ Schema updated in ${outputPath}`);
-        }
-        catch (error) {
-            // Cleanup temp file on error
-            if (fs.existsSync(tempPath)) {
-                fs.unlinkSync(tempPath);
-            }
-            throw error;
-        }
-    }
-    /**
-     * Stitch multiple documents and save as dataset
-     */
-    async stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality = 'native') {
-        const finetuningData = [];
-        for (const { documents: docPaths, annotation: annPath } of pairsPaths) {
-            if (!fs.existsSync(annPath)) {
-                throw new Error(`Annotation file not found: ${annPath}`);
-            }
-            // Verify all document files exist
-            for (const docPath of docPaths) {
-                if (!fs.existsSync(docPath)) {
-                    throw new Error(`Document file not found: ${docPath}`);
-                }
-            }
-            const annotation = JSON.parse(fs.readFileSync(annPath, 'utf-8'));
-            const systemMessage = this.createSystemMessage(jsonSchema, modality);
-            const userMessage = await this.createMultiDocumentUserMessage(docPaths, modality);
-            const assistantMessage = {
-                role: 'assistant',
-                content: JSON.stringify(annotation),
-            };
-            finetuningData.push({
-                messages: [systemMessage, userMessage, assistantMessage],
-            });
-        }
-        await writeJSONL(datasetPath, finetuningData);
-        console.log(`✅ Stitched dataset saved to ${datasetPath} with ${finetuningData.length} examples`);
-    }
-    /**
-     * Generate annotations for documents using AI models
-     */
-    async annotate(jsonSchema, documents, datasetPath, options = {}) {
-        const { model = 'gpt-4o-mini', temperature = 0.0, modality = 'native', maxConcurrency = 5, reasoning_effort = 'medium', provider = 'openai', } = options;
-        console.log(`🚀 Starting annotation of ${documents.length} documents...`);
-        const finetuningData = [];
-        const concurrencyLimit = Math.min(maxConcurrency, documents.length);
-        // Process documents in batches
-        for (let i = 0; i < documents.length; i += concurrencyLimit) {
-            const batch = documents.slice(i, i + concurrencyLimit);
-            const batchPromises = batch.map(async (docPath, index) => {
-                const globalIndex = i + index;
-                console.log(`📝 Processing document ${globalIndex + 1}/${documents.length}: ${path.basename(docPath)}`);
-                try {
-                    const annotation = await this.generateAnnotation(jsonSchema, docPath, model, temperature, modality, reasoning_effort, provider);
-                    const systemMessage = this.createSystemMessage(jsonSchema, modality);
-                    const userMessage = await this.createUserMessage(docPath, modality);
-                    const assistantMessage = {
-                        role: 'assistant',
-                        content: JSON.stringify(annotation),
-                    };
-                    return {
-                        messages: [systemMessage, userMessage, assistantMessage],
-                    };
-                }
-                catch (error) {
-                    console.error(`❌ Failed to process ${docPath}:`, error);
-                    return null;
-                }
-            });
-            const batchResults = await Promise.all(batchPromises);
-            finetuningData.push(...batchResults.filter(result => result !== null));
-        }
-        await writeJSONL(datasetPath, finetuningData);
-        console.log(`✅ Annotation complete! Generated ${finetuningData.length}/${documents.length} annotations`);
-    }
-    /**
-     * Update existing annotations with new model/schema
-     */
-    async updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options = {}) {
-        if (!fs.existsSync(oldDatasetPath)) {
-            throw new Error(`Old dataset not found: ${oldDatasetPath}`);
-        }
-        console.log(`🔄 Updating annotations from ${oldDatasetPath}...`);
-        const oldDataset = await readJSONL(oldDatasetPath);
-        const updatedDataset = [];
-        for (let i = 0; i < oldDataset.length; i++) {
-            const item = oldDataset[i];
-            console.log(`🔄 Updating annotation ${i + 1}/${oldDataset.length}`);
-            try {
-                // Extract document path from user message (this is simplified)
-                const userContent = item.messages.find(m => m.role === 'user')?.content;
-                if (!userContent) {
-                    console.warn(`⚠️  No user message found in item ${i + 1}, skipping`);
-                    continue;
-                }
-                // For this implementation, we assume the document is referenced in the user message
-                // In practice, you'd need to store document paths or reconstruct them
-                const newAnnotation = await this.generateAnnotationFromUserMessage(jsonSchema, userContent, options);
-                const systemMessage = this.createSystemMessage(jsonSchema, options.modality || 'native');
-                const assistantMessage = {
-                    role: 'assistant',
-                    content: JSON.stringify(newAnnotation),
-                };
-                updatedDataset.push({
-                    messages: [
-                        systemMessage,
-                        item.messages.find(m => m.role === 'user'),
-                        assistantMessage,
-                    ],
-                });
-            }
-            catch (error) {
-                console.error(`❌ Failed to update annotation ${i + 1}:`, error);
-            }
-        }
-        await writeJSONL(newDatasetPath, updatedDataset);
-        console.log(`✅ Updated ${updatedDataset.length}/${oldDataset.length} annotations`);
-    }
-    /**
-     * Save batch annotation requests for OpenAI Batch API
-     */
-    async saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options = {}) {
-        const { model = 'gpt-4o-mini', temperature = 0.0, modality = 'native', } = options;
-        const batchRequests = [];
-        for (let i = 0; i < documents.length; i++) {
-            const docPath = documents[i];
-            const systemMessage = this.createSystemMessage(jsonSchema, modality);
-            const userMessage = await this.createUserMessage(docPath, modality);
-            batchRequests.push({
-                custom_id: `doc_${i}_${path.basename(docPath, path.extname(docPath))}`,
-                method: 'POST',
-                url: '/v1/chat/completions',
-                body: {
-                    model,
-                    messages: [systemMessage, userMessage],
-                    temperature,
-                    response_format: { type: 'json_object' },
-                },
-            });
-        }
-        await writeJSONL(batchRequestsPath, batchRequests);
-        console.log(`✅ Saved ${batchRequests.length} batch requests to ${batchRequestsPath}`);
-    }
-    /**
-     * Build dataset from batch API results
-     */
-    async buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality = 'native') {
-        if (!fs.existsSync(batchResultsPath)) {
-            throw new Error(`Batch results file not found: ${batchResultsPath}`);
-        }
-        const batchResults = await readJSONL(batchResultsPath);
-        const finetuningData = [];
-        for (const result of batchResults) {
-            if (result.error) {
-                console.warn(`⚠️  Skipping failed request ${result.custom_id}: ${result.error.message}`);
-                continue;
-            }
-            const response = result.response.body;
-            const content = response.choices?.[0]?.message?.content;
-            if (!content) {
-                console.warn(`⚠️  No content in response for ${result.custom_id}`);
-                continue;
-            }
-            try {
-                const annotation = JSON.parse(content);
-                // Reconstruct messages (this is simplified)
-                const systemMessage = this.createSystemMessage(jsonSchema, modality);
-                // Extract user message from original request (would need to be stored)
-                const userMessage = {
-                    role: 'user',
-                    content: `Document content for ${result.custom_id}`,
-                };
-                const assistantMessage = {
-                    role: 'assistant',
-                    content: JSON.stringify(annotation),
-                };
-                finetuningData.push({
-                    messages: [systemMessage, userMessage, assistantMessage],
-                });
-            }
-            catch (error) {
-                console.warn(`⚠️  Failed to parse annotation for ${result.custom_id}:`, error);
-            }
-        }
-        await writeJSONL(datasetPath, finetuningData);
-        console.log(`✅ Built dataset with ${finetuningData.length} examples from batch results`);
-    }
-    // Helper methods
-    createSystemMessage(jsonSchema, _modality) {
-        const schemaObj = typeof jsonSchema === 'string' ? JSON.parse(jsonSchema) : jsonSchema;
-        const schemaStr = JSON.stringify(schemaObj, null, 2);
-        return {
-            role: 'system',
-            content: `You are an expert data extraction assistant. Extract information from the provided document according to the following JSON schema:\n\n${schemaStr}\n\nReturn only valid JSON that matches the schema exactly.`,
-        };
-    }
-    async createUserMessage(docPath, _modality, _options = {}) {
-        // This is a simplified implementation
-        // In practice, you'd handle different file types, base64 encoding, etc.
-        const content = fs.readFileSync(docPath, 'utf-8');
-        return {
-            role: 'user',
-            content: `Please extract data from this document:\n\n${content}`,
-        };
-    }
-    async createMultiDocumentUserMessage(docPaths, _modality) {
-        const contents = docPaths.map((docPath, index) => {
-            const content = fs.readFileSync(docPath, 'utf-8');
-            return `Document ${index + 1} (${path.basename(docPath)}):\n${content}`;
-        }).join('\n\n---\n\n');
-        return {
-            role: 'user',
-            content: `Please extract data from these documents:\n\n${contents}`,
-        };
-    }
-    async generateAnnotation(_jsonSchema, _docPath, _model, _temperature, _modality, _reasoningEffort, _provider) {
-        // This would integrate with the actual AI providers
-        // For now, return a placeholder implementation
-        throw new Error('AI provider integration not implemented in this version');
-    }
-    async generateAnnotationFromUserMessage(_jsonSchema, _userContent, _options) {
-        // This would re-generate annotation from existing user message
-        throw new Error('Annotation update not implemented in this version');
-    }
-}
-export class Datasets extends SyncAPIResource {
-    constructor() {
-        super(...arguments);
-        this.mixin = new BaseDatasetsMixin();
-    }
-    async pprint(datasetPath, inputTokenPrice, outputTokenPrice) {
-        return this.mixin.pprint(datasetPath, inputTokenPrice, outputTokenPrice);
-    }
-    async save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options) {
-        return this.mixin.save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options);
-    }
-    async changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace) {
-        return this.mixin.changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace);
-    }
-    async stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality) {
-        return this.mixin.stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality);
-    }
-    async annotate(jsonSchema, documents, datasetPath, options) {
-        return this.mixin.annotate(jsonSchema, documents, datasetPath, options);
-    }
-    async updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options) {
-        return this.mixin.updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options);
-    }
-    async saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options) {
-        return this.mixin.saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options);
-    }
-    async buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality) {
-        return this.mixin.buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality);
-    }
-}
-export class AsyncDatasets extends AsyncAPIResource {
-    constructor() {
-        super(...arguments);
-        this.mixin = new BaseDatasetsMixin();
-    }
-    async pprint(datasetPath, inputTokenPrice, outputTokenPrice) {
-        return this.mixin.pprint(datasetPath, inputTokenPrice, outputTokenPrice);
-    }
-    async save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options) {
-        return this.mixin.save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options);
-    }
-    async changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace) {
-        return this.mixin.changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace);
-    }
-    async stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality) {
-        return this.mixin.stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality);
-    }
-    async annotate(jsonSchema, documents, datasetPath, options) {
-        return this.mixin.annotate(jsonSchema, documents, datasetPath, options);
-    }
-    async updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options) {
-        return this.mixin.updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options);
-    }
-    async saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options) {
-        return this.mixin.saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options);
-    }
-    async buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality) {
-        return this.mixin.buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality);
-    }
-}
-export default {
-    Datasets,
-    AsyncDatasets,
-    BaseDatasetsMixin,
-};

package/dist/utils/display.d.ts

@@ -1,108 +0,0 @@
-/**
- * Rich display and visualization utilities for datasets and metrics
- * Equivalent to Python's display.py
- */
-export interface DatasetMetrics {
-    totalExamples: number;
-    inputTokens: {
-        total: number;
-        min: number;
-        max: number;
-        mean: number;
-        median: number;
-        p95: number;
-        p99: number;
-    };
-    outputTokens: {
-        total: number;
-        min: number;
-        max: number;
-        mean: number;
-        median: number;
-        p95: number;
-        p99: number;
-    };
-    totalTokens: {
-        total: number;
-        min: number;
-        max: number;
-        mean: number;
-        median: number;
-    };
-    estimatedCost: {
-        input: number;
-        output: number;
-        total: number;
-    };
-    messageStats: {
-        systemMessages: number;
-        userMessages: number;
-        assistantMessages: number;
-        avgMessagesPerExample: number;
-    };
-    contentAnalysis: {
-        avgSystemLength: number;
-        avgUserLength: number;
-        avgAssistantLength: number;
-        hasImages: boolean;
-        imageCount: number;
-    };
-}
-export interface TokenCountResult {
-    textTokens: number;
-    imageTokens: number;
-    totalTokens: number;
-}
-/**
- * Count tokens in text using a simple approximation
- * In production, you'd want to use tiktoken equivalent for JavaScript
- */
-export declare function countTokens(text: string, _model?: string): number;
-/**
- * Count tokens in content (text + images)
- */
-export declare function countContentTokens(content: string, _model?: string): TokenCountResult;
-/**
- * Calculate statistical metrics for an array of numbers
- */
-export declare function calculateStats(values: number[]): {
-    min: number;
-    max: number;
-    mean: number;
-    median: number;
-    p95: number;
-    p99: number;
-    total: number;
-};
-/**
- * Process dataset and compute comprehensive metrics
- */
-export declare function processDatasetAndComputeMetrics(datasetPath: string, inputTokenPrice?: number, outputTokenPrice?: number, model?: string): Promise<DatasetMetrics>;
-/**
- * Display metrics in a formatted table
- */
-export declare function displayMetrics(metrics: DatasetMetrics): void;
-/**
- * Format large numbers with appropriate units
- */
-export declare function formatNumber(num: number): string;
-/**
- * Create a simple ASCII progress bar
- */
-export declare function createProgressBar(current: number, total: number, width?: number): string;
-/**
- * Display progress with a progress bar
- */
-export declare function displayProgress(current: number, total: number, message?: string): void;
-declare const _default: {
-    processDatasetAndComputeMetrics: typeof processDatasetAndComputeMetrics;
-    displayMetrics: typeof displayMetrics;
-    countTokens: typeof countTokens;
-    countContentTokens: typeof countContentTokens;
-    calculateStats: typeof calculateStats;
-    formatNumber: typeof formatNumber;
-    createProgressBar: typeof createProgressBar;
-    displayProgress: typeof displayProgress;
-};
-export default _default;
-//# sourceMappingURL=display.d.ts.map

package/dist/utils/display.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"display.d.ts","sourceRoot":"","sources":["../../src/utils/display.ts"],"names":[],"mappings":"AAGA;;;GAGG;AAEH,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;KACb,CAAC;IACF,YAAY,EAAE;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;KACb,CAAC;IACF,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,aAAa,EAAE;QACb,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,YAAY,EAAE;QACZ,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC;IACF,eAAe,EAAE;QACf,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,SAAS,EAAE,OAAO,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,GAAE,MAAsB,GAAG,MAAM,CAKhF;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,GAAE,MAAsB,GAAG,gBAAgB,CA+BpG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG;IAChD,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf,CAuBA;AAED;;GAEG;AACH,wBAAsB,+BAA+B,CACnD,WAAW,EAAE,MAAM,EACnB,eAAe,GAAE,MAAgB,EACjC,gBAAgB,GAAE,MAAe,EACjC,KAAK,GAAE,MAAsB,GAC5B,OAAO,CAAC,cAAc,CAAC,CAiGzB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,CAoD5D;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOhD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,MAAM,CAM5F;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAUtF;;;;;;;;;;;AAED,wBASE"}