@retab/node 0.0.46 → 0.0.52

package/dist/utils/batch_processing.js

@@ -1,268 +0,0 @@
-import fs from 'fs';
-import axios from 'axios';
-import { readJSONL, writeJSONL } from './jsonl.js';
-/**
- * OpenAI Batch API client
- */
-export class OpenAIBatchProcessor {
-    constructor(options) {
-        this.apiKey = options.apiKey;
-        this.baseUrl = options.baseUrl || 'https://api.openai.com/v1';
-        this.timeout = options.timeout || 300000; // 5 minutes
-    }
-    /**
-     * Upload file for batch processing
-     */
-    async uploadFile(filePath, purpose = 'batch') {
-        if (!fs.existsSync(filePath)) {
-            throw new Error(`File not found: ${filePath}`);
-        }
-        const formData = new FormData();
-        const fileBuffer = fs.readFileSync(filePath);
-        const blob = new Blob([fileBuffer]);
-        formData.append('file', blob, filePath.split('/').pop());
-        formData.append('purpose', purpose);
-        const response = await axios.post(`${this.baseUrl}/files`, formData, {
-            headers: {
-                'Authorization': `Bearer ${this.apiKey}`,
-                'Content-Type': 'multipart/form-data',
-            },
-            timeout: this.timeout,
-        });
-        return {
-            id: response.data.id,
-            filename: response.data.filename,
-        };
-    }
-    /**
-     * Create batch job
-     */
-    async createBatch(inputFileId, endpoint, completionWindow = '24h', metadata) {
-        const response = await axios.post(`${this.baseUrl}/batches`, {
-            input_file_id: inputFileId,
-            endpoint,
-            completion_window: completionWindow,
-            metadata,
-        }, {
-            headers: {
-                'Authorization': `Bearer ${this.apiKey}`,
-                'Content-Type': 'application/json',
-            },
-            timeout: this.timeout,
-        });
-        return response.data;
-    }
-    /**
-     * Get batch job status
-     */
-    async getBatch(batchId) {
-        const response = await axios.get(`${this.baseUrl}/batches/${batchId}`, {
-            headers: {
-                'Authorization': `Bearer ${this.apiKey}`,
-            },
-            timeout: this.timeout,
-        });
-        return response.data;
-    }
-    /**
-     * Cancel batch job
-     */
-    async cancelBatch(batchId) {
-        const response = await axios.post(`${this.baseUrl}/batches/${batchId}/cancel`, {}, {
-            headers: {
-                'Authorization': `Bearer ${this.apiKey}`,
-                'Content-Type': 'application/json',
-            },
-            timeout: this.timeout,
-        });
-        return response.data;
-    }
-    /**
-     * List batch jobs
-     */
-    async listBatches(after, limit = 20) {
-        const params = new URLSearchParams();
-        if (after)
-            params.append('after', after);
-        params.append('limit', limit.toString());
-        const response = await axios.get(`${this.baseUrl}/batches?${params}`, {
-            headers: {
-                'Authorization': `Bearer ${this.apiKey}`,
-            },
-            timeout: this.timeout,
-        });
-        return response.data;
-    }
-    /**
-     * Download file content
-     */
-    async downloadFile(fileId) {
-        const response = await axios.get(`${this.baseUrl}/files/${fileId}/content`, {
-            headers: {
-                'Authorization': `Bearer ${this.apiKey}`,
-            },
-            timeout: this.timeout,
-        });
-        return response.data;
-    }
-    /**
-     * Monitor batch job progress
-     */
-    async *monitorBatch(batchId, pollInterval = 30000) {
-        const startTime = Date.now();
-        let lastProgress = 0;
-        while (true) {
-            const batch = await this.getBatch(batchId);
-            const timeElapsed = Date.now() - startTime;
-            const progress = {
-                total: batch.request_counts.total,
-                completed: batch.request_counts.completed,
-                failed: batch.request_counts.failed,
-                percentage: batch.request_counts.total > 0 ?
-                    (batch.request_counts.completed / batch.request_counts.total) * 100 : 0,
-            };
-            // Estimate time remaining
-            let estimatedTimeRemaining;
-            if (progress.completed > lastProgress && progress.completed > 0) {
-                const completionRate = progress.completed / timeElapsed;
-                const remaining = progress.total - progress.completed;
-                estimatedTimeRemaining = remaining / completionRate;
-            }
-            yield {
-                jobId: batchId,
-                status: batch.status,
-                progress,
-                timeElapsed,
-                estimatedTimeRemaining,
-            };
-            // Break if job is complete
-            if (['completed', 'failed', 'cancelled', 'expired'].includes(batch.status)) {
-                break;
-            }
-            lastProgress = progress.completed;
-            await new Promise(resolve => setTimeout(resolve, pollInterval));
-        }
-    }
-    /**
-     * Process batch end-to-end
-     */
-    async processBatch(requestsFilePath, outputFilePath, endpoint = '/v1/chat/completions', options = {}) {
-        const { completionWindow = '24h', metadata, pollInterval = 30000, showProgress = true, } = options;
-        console.log('🚀 Starting batch processing...');
-        // Upload input file
-        console.log('📤 Uploading input file...');
-        const uploadResult = await this.uploadFile(requestsFilePath);
-        console.log(`✅ File uploaded: ${uploadResult.id}`);
-        // Create batch job
-        console.log('⚙️  Creating batch job...');
-        const batch = await this.createBatch(uploadResult.id, endpoint, completionWindow, metadata);
-        console.log(`✅ Batch created: ${batch.id}`);
-        // Monitor progress
-        if (showProgress) {
-            console.log('📊 Monitoring batch progress...');
-            for await (const progress of this.monitorBatch(batch.id, pollInterval)) {
-                const { percentage, completed, total } = progress.progress;
-                const timeStr = `${Math.round(progress.timeElapsed / 1000)}s`;
-                const etaStr = progress.estimatedTimeRemaining ?
-                    ` (ETA: ${Math.round(progress.estimatedTimeRemaining / 1000)}s)` : '';
-                console.log(`   Status: ${progress.status} - ${percentage.toFixed(1)}% (${completed}/${total}) - ${timeStr}${etaStr}`);
-                if (['completed', 'failed', 'cancelled', 'expired'].includes(progress.status)) {
-                    break;
-                }
-            }
-        }
-        // Get final batch status
-        const finalBatch = await this.getBatch(batch.id);
-        if (finalBatch.status === 'completed' && finalBatch.output_file_id) {
-            console.log('📥 Downloading results...');
-            const results = await this.downloadFile(finalBatch.output_file_id);
-            fs.writeFileSync(outputFilePath, results);
-            console.log(`✅ Results saved to: ${outputFilePath}`);
-        }
-        else {
-            console.error(`❌ Batch failed with status: ${finalBatch.status}`);
-            if (finalBatch.error_file_id) {
-                const errors = await this.downloadFile(finalBatch.error_file_id);
-                console.error('Error details:', errors);
-            }
-        }
-        return finalBatch;
-    }
-}
-/**
- * Utility functions for batch processing
- */
-export const batchUtils = {
-    /**
-     * Create batch requests for chat completions
-     */
-    createChatCompletionRequests: (messages, model = 'gpt-4o-mini', options = {}) => {
-        return messages.map((msg, index) => ({
-            custom_id: msg.customId || `request_${index}`,
-            method: 'POST',
-            url: '/v1/chat/completions',
-            body: {
-                model,
-                messages: msg.messages,
-                temperature: options.temperature || 0.0,
-                max_tokens: options.maxTokens,
-                response_format: options.responseFormat,
-            },
-        }));
-    },
-    /**
-     * Create batch requests for document extraction
-     */
-    createExtractionRequests: (documents, model = 'gpt-4o-mini') => {
-        return documents.map((doc, index) => ({
-            custom_id: doc.customId || `extraction_${index}`,
-            method: 'POST',
-            url: '/v1/documents/extractions',
-            body: {
-                json_schema: doc.schema,
-                documents: [doc.document],
-                model,
-            },
-        }));
-    },
-    /**
-     * Save batch requests to JSONL file
-     */
-    saveBatchRequests: async (requests, filePath) => {
-        await writeJSONL(filePath, requests);
-        console.log(`📄 Saved ${requests.length} batch requests to ${filePath}`);
-    },
-    /**
-     * Parse batch results from JSONL file
-     */
-    parseBatchResults: async (filePath) => {
-        const results = await readJSONL(filePath);
-        return results;
-    },
-    /**
-     * Extract successful results from batch responses
-     */
-    extractSuccessfulResults: (responses) => {
-        return responses
-            .filter(response => !response.error && response.response.status_code === 200)
-            .map(response => ({
-            customId: response.custom_id,
-            result: response.response.body,
-        }));
-    },
-    /**
-     * Extract failed results from batch responses
-     */
-    extractFailedResults: (responses) => {
-        return responses
-            .filter(response => response.error || response.response.status_code !== 200)
-            .map(response => ({
-            customId: response.custom_id,
-            error: response.error || { code: 'http_error', message: `HTTP ${response.response.status_code}` },
-        }));
-    },
-};
-export default {
-    OpenAIBatchProcessor,
-    batchUtils,
-};

package/dist/utils/benchmarking.d.ts

@@ -1,115 +0,0 @@
-/**
- * Benchmarking and evaluation utilities for model comparison
- * Equivalent to Python's benchmarking.py
- */
-export interface EvaluationMetrics {
-    accuracy: number;
-    precision: number;
-    recall: number;
-    f1Score: number;
-    exactMatch: number;
-    levenshteinDistance: number;
-    jaccardSimilarity: number;
-    hammingDistance: number;
-    fieldAccuracy: Record<string, number>;
-    completeness: number;
-    errorRate: number;
-}
-export interface SingleFileEvalResult {
-    filename: string;
-    metrics: EvaluationMetrics;
-    predictions: any[];
-    groundTruths: any[];
-    differences: Record<string, any>[];
-    executionTime: number;
-}
-export interface BenchmarkResult {
-    model: string;
-    overallMetrics: EvaluationMetrics;
-    fileResults: SingleFileEvalResult[];
-    aggregateStats: {
-        meanAccuracy: number;
-        stdDevAccuracy: number;
-        meanF1: number;
-        stdDevF1: number;
-        totalFiles: number;
-        totalPredictions: number;
-    };
-    executionTime: number;
-}
-export interface DictionaryDifference {
-    field: string;
-    predicted: any;
-    groundTruth: any;
-    differenceType: 'missing' | 'extra' | 'value_mismatch' | 'type_mismatch';
-    path: string;
-}
-/**
- * Calculate Levenshtein distance between two strings
- */
-export declare function levenshteinDistance(str1: string, str2: string): number;
-/**
- * Calculate Jaccard similarity between two sets
- */
-export declare function jaccardSimilarity(set1: Set<any>, set2: Set<any>): number;
-/**
- * Calculate Hamming distance between two strings
- */
-export declare function hammingDistance(str1: string, str2: string): number;
-/**
- * Flatten nested object into dot-notation keys
- */
-export declare function flattenObject(obj: any, prefix?: string): Record<string, any>;
-/**
- * Compute detailed differences between two dictionaries
- */
-export declare function computeDictDifference(predicted: any, groundTruth: any, path?: string): DictionaryDifference[];
-/**
- * Aggregate dictionary differences across multiple examples
- */
-export declare function aggregateDictDifferences(differences: DictionaryDifference[][]): Record<string, {
-    count: number;
-    percentage: number;
-    examples: DictionaryDifference[];
-}>;
-/**
- * Calculate comprehensive evaluation metrics
- */
-export declare function calculateMetrics(predictions: any[], groundTruths: any[]): EvaluationMetrics;
-/**
- * Single file evaluation class
- */
-export declare class SingleFileEval {
-    private filename;
-    private predictions;
-    private groundTruths;
-    constructor(filename: string, predictions: any[], groundTruths: any[]);
-    evaluate(): Promise<SingleFileEvalResult>;
-}
-/**
- * Plot metrics with uncertainty (text-based visualization)
- */
-export declare function plotMetricsWithUncertainty(results: BenchmarkResult[]): void;
-/**
- * Benchmark multiple models
- */
-export declare function benchmark(models: string[], testDataPath: string, groundTruthPath: string, evaluationFunction: (model: string, testData: any[]) => Promise<any[]>): Promise<BenchmarkResult[]>;
-/**
- * Save benchmark results to file
- */
-export declare function saveBenchmarkResults(results: BenchmarkResult[], outputPath: string): Promise<void>;
-declare const _default: {
-    SingleFileEval: typeof SingleFileEval;
-    calculateMetrics: typeof calculateMetrics;
-    computeDictDifference: typeof computeDictDifference;
-    aggregateDictDifferences: typeof aggregateDictDifferences;
-    levenshteinDistance: typeof levenshteinDistance;
-    jaccardSimilarity: typeof jaccardSimilarity;
-    hammingDistance: typeof hammingDistance;
-    flattenObject: typeof flattenObject;
-    plotMetricsWithUncertainty: typeof plotMetricsWithUncertainty;
-    benchmark: typeof benchmark;
-    saveBenchmarkResults: typeof saveBenchmarkResults;
-};
-export default _default;
-//# sourceMappingURL=benchmarking.d.ts.map

package/dist/utils/benchmarking.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"benchmarking.d.ts","sourceRoot":"","sources":["../../src/utils/benchmarking.ts"],"names":[],"mappings":"AAEA;;;GAGG;AAEH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,WAAW,EAAE,GAAG,EAAE,CAAC;IACnB,YAAY,EAAE,GAAG,EAAE,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,iBAAiB,CAAC;IAClC,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,cAAc,EAAE;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,GAAG,CAAC;IACf,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,SAAS,GAAG,OAAO,GAAG,gBAAgB,GAAG,eAAe,CAAC;IACzE,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CA2BtE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAMxE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAYlE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,GAAE,MAAW,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAiBhF;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,IAAI,GAAE,MAAW,GAAG,oBAAoB,EAAE,CA6CjH;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,oBAAoB,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE;IAC9F,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,oBAAoB,EAAE,CAAC;CAClC,CAAC,CAyBD;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,GAAG,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,iBAAiB,CAmF3F;AAED;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,YAAY,CAAQ;gBAEhB,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE;IAM/D,QAAQ,IAAI,OAAO,CAAC,oBAAoB,CAAC;CA2BhD;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CA0B3E;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,YAAY,EAAE,MAAM,EACpB,eAAe,EAAE,MAAM,EACvB,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,GACrE,OAAO,CAAC,eAAe,EAAE,CAAC,CA+D5B;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,eAAe,EAAE,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;;;;;;;;;;;;;;AAED,wBAYE"}