@retab/node 0.0.0-reserved → 0.0.42

package/dist/utils/ai_models.js

@@ -0,0 +1,183 @@
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import { fileURLToPath } from 'url';
+import { ModelCardSchema } from '../types/ai_models.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const MODEL_CARDS_DIR = path.join(__dirname, '_model_cards');
+function mergeModelCards(base, override) {
+    const result = { ...base };
+    for (const [key, value] of Object.entries(override)) {
+        if (key === 'inherits') {
+            continue;
+        }
+        if (typeof value === 'object' && value !== null && key in result && typeof result[key] === 'object') {
+            result[key] = mergeModelCards(result[key], value);
+        }
+        else {
+            result[key] = value;
+        }
+    }
+    return result;
+}
+function loadModelCards(yamlFile) {
+    const yamlContent = fs.readFileSync(yamlFile, 'utf-8');
+    const rawCards = yaml.load(yamlContent);
+    const nameToCard = {};
+    // First pass: collect base cards
+    for (const card of rawCards) {
+        if (!('inherits' in card)) {
+            nameToCard[card.model] = card;
+        }
+    }
+    const finalCards = [];
+    for (const card of rawCards) {
+        if ('inherits' in card) {
+            const parent = nameToCard[card.inherits];
+            const merged = mergeModelCards(parent, card);
+            finalCards.push(ModelCardSchema.parse(merged));
+        }
+        else {
+            finalCards.push(ModelCardSchema.parse(card));
+        }
+    }
+    return finalCards;
+}
+// Create model cards directory structure if it doesn't exist
+if (!fs.existsSync(MODEL_CARDS_DIR)) {
+    fs.mkdirSync(MODEL_CARDS_DIR, { recursive: true });
+    // Create basic model card files
+    const openaiCards = [
+        {
+            model: 'gpt-4o',
+            pricing: {
+                text: { prompt: 2.5, completion: 10.0 },
+            },
+            capabilities: {
+                modalities: ['text', 'image'],
+                endpoints: ['chat_completions'],
+                features: ['streaming', 'function_calling', 'structured_outputs'],
+            },
+        },
+        {
+            model: 'gpt-4o-mini',
+            pricing: {
+                text: { prompt: 0.15, completion: 0.6 },
+            },
+            capabilities: {
+                modalities: ['text', 'image'],
+                endpoints: ['chat_completions'],
+                features: ['streaming', 'function_calling', 'structured_outputs'],
+            },
+        },
+    ];
+    fs.writeFileSync(path.join(MODEL_CARDS_DIR, 'openai.yaml'), yaml.dump(openaiCards));
+    const anthropicCards = [
+        {
+            model: 'claude-3-5-sonnet-latest',
+            pricing: {
+                text: { prompt: 3.0, completion: 15.0 },
+            },
+            capabilities: {
+                modalities: ['text', 'image'],
+                endpoints: ['chat_completions'],
+                features: ['streaming'],
+            },
+        },
+    ];
+    fs.writeFileSync(path.join(MODEL_CARDS_DIR, 'anthropic.yaml'), yaml.dump(anthropicCards));
+    // Create empty files for other providers
+    fs.writeFileSync(path.join(MODEL_CARDS_DIR, 'xai.yaml'), yaml.dump([]));
+    fs.writeFileSync(path.join(MODEL_CARDS_DIR, 'gemini.yaml'), yaml.dump([]));
+    fs.writeFileSync(path.join(MODEL_CARDS_DIR, 'auto.yaml'), yaml.dump([]));
+}
+// Load all model cards
+let modelCards = [];
+const modelCardsDict = {};
+try {
+    const cardFiles = ['openai.yaml', 'anthropic.yaml', 'xai.yaml', 'gemini.yaml', 'auto.yaml'];
+    for (const file of cardFiles) {
+        const filePath = path.join(MODEL_CARDS_DIR, file);
+        if (fs.existsSync(filePath)) {
+            modelCards = [...modelCards, ...loadModelCards(filePath)];
+        }
+    }
+    for (const card of modelCards) {
+        modelCardsDict[card.model] = card;
+    }
+}
+catch (error) {
+    console.warn('Failed to load model cards:', error);
+}
+export function getModelFromModelId(modelId) {
+    if (modelId.startsWith('ft:')) {
+        const parts = modelId.split(':');
+        return parts[1];
+    }
+    return modelId;
+}
+export function getModelCard(model) {
+    const modelName = getModelFromModelId(model);
+    if (modelName in modelCardsDict) {
+        const modelCard = ModelCardSchema.parse({ ...modelCardsDict[modelName] });
+        if (modelName !== model) {
+            // Fine-tuned model -> Change the name
+            modelCard.model = model;
+            // Remove the fine-tuning feature (if exists)
+            const features = modelCard.capabilities.features;
+            const index = features.indexOf('fine_tuning');
+            if (index > -1) {
+                features.splice(index, 1);
+            }
+        }
+        return modelCard;
+    }
+    throw new Error(`No model card found for model: ${modelName}`);
+}
+export function getProviderForModel(modelId) {
+    const modelName = getModelFromModelId(modelId);
+    // Check OpenAI models
+    if (modelName.startsWith('gpt-') || modelName.startsWith('o1') || modelName.startsWith('o3') || modelName.startsWith('o4')) {
+        return 'OpenAI';
+    }
+    // Check Anthropic models
+    if (modelName.startsWith('claude-')) {
+        return 'Anthropic';
+    }
+    // Check xAI models
+    if (modelName.startsWith('grok-')) {
+        return 'xAI';
+    }
+    // Check Gemini models
+    if (modelName.startsWith('gemini-')) {
+        return 'Gemini';
+    }
+    // Check Retab models
+    if (modelName.startsWith('auto-')) {
+        return 'Retab';
+    }
+    throw new Error(`Unknown provider for model: ${modelName}`);
+}
+export function assertValidModelExtraction(model) {
+    if (!model || typeof model !== 'string') {
+        throw new Error('Valid model must be provided for extraction');
+    }
+    // Additional validation logic can be added here
+    try {
+        getProviderForModel(model);
+    }
+    catch (error) {
+        throw new Error(`Invalid model for extraction: ${model}`);
+    }
+}
+export function assertValidModelSchemaGeneration(model) {
+    if (!model || typeof model !== 'string') {
+        throw new Error('Valid model must be provided for schema generation');
+    }
+    const validModels = ['gpt-4o-2024-11-20', 'gpt-4o-mini', 'gpt-4o'];
+    if (!validModels.includes(model)) {
+        throw new Error(`Model ${model} not valid for schema generation`);
+    }
+}
+export { modelCards, modelCardsDict };

package/dist/utils/batch_processing.d.ts

@@ -0,0 +1,227 @@
+/**
+ * Batch processing utilities for OpenAI Batch API and other providers
+ * Equivalent to Python's batch processing functionality
+ */
+export interface BatchRequest {
+    custom_id: string;
+    method: 'POST' | 'GET' | 'PUT' | 'DELETE';
+    url: string;
+    body?: Record<string, any>;
+    headers?: Record<string, string>;
+}
+export interface BatchResponse {
+    id: string;
+    custom_id: string;
+    response: {
+        status_code: number;
+        request_id: string;
+        body: Record<string, any>;
+    };
+    error?: {
+        code: string;
+        message: string;
+    };
+}
+export interface BatchJob {
+    id: string;
+    object: 'batch';
+    endpoint: string;
+    errors?: {
+        object: 'list';
+        data: Array<{
+            code: string;
+            message: string;
+            param?: string;
+            line?: number;
+        }>;
+    };
+    input_file_id: string;
+    completion_window: '24h';
+    status: 'validating' | 'failed' | 'in_progress' | 'finalizing' | 'completed' | 'expired' | 'cancelling' | 'cancelled';
+    output_file_id?: string;
+    error_file_id?: string;
+    created_at: number;
+    in_progress_at?: number;
+    expires_at?: number;
+    finalizing_at?: number;
+    completed_at?: number;
+    failed_at?: number;
+    expired_at?: number;
+    cancelling_at?: number;
+    cancelled_at?: number;
+    request_counts: {
+        total: number;
+        completed: number;
+        failed: number;
+    };
+    metadata?: Record<string, string>;
+}
+export interface BatchProcessingOptions {
+    apiKey: string;
+    baseUrl?: string;
+    timeout?: number;
+    maxRetries?: number;
+    completionWindow?: '24h';
+    metadata?: Record<string, string>;
+}
+export interface BatchProgressInfo {
+    jobId: string;
+    status: string;
+    progress: {
+        total: number;
+        completed: number;
+        failed: number;
+        percentage: number;
+    };
+    timeElapsed: number;
+    estimatedTimeRemaining?: number;
+}
+/**
+ * OpenAI Batch API client
+ */
+export declare class OpenAIBatchProcessor {
+    private apiKey;
+    private baseUrl;
+    private timeout;
+    constructor(options: BatchProcessingOptions);
+    /**
+     * Upload file for batch processing
+     */
+    uploadFile(filePath: string, purpose?: 'batch'): Promise<{
+        id: string;
+        filename: string;
+    }>;
+    /**
+     * Create batch job
+     */
+    createBatch(inputFileId: string, endpoint: string, completionWindow?: '24h', metadata?: Record<string, string>): Promise<BatchJob>;
+    /**
+     * Get batch job status
+     */
+    getBatch(batchId: string): Promise<BatchJob>;
+    /**
+     * Cancel batch job
+     */
+    cancelBatch(batchId: string): Promise<BatchJob>;
+    /**
+     * List batch jobs
+     */
+    listBatches(after?: string, limit?: number): Promise<{
+        data: BatchJob[];
+    }>;
+    /**
+     * Download file content
+     */
+    downloadFile(fileId: string): Promise<string>;
+    /**
+     * Monitor batch job progress
+     */
+    monitorBatch(batchId: string, pollInterval?: number): AsyncGenerator<BatchProgressInfo, void, unknown>;
+    /**
+     * Process batch end-to-end
+     */
+    processBatch(requestsFilePath: string, outputFilePath: string, endpoint?: string, options?: {
+        completionWindow?: '24h';
+        metadata?: Record<string, string>;
+        pollInterval?: number;
+        showProgress?: boolean;
+    }): Promise<BatchJob>;
+}
+/**
+ * Utility functions for batch processing
+ */
+export declare const batchUtils: {
+    /**
+     * Create batch requests for chat completions
+     */
+    createChatCompletionRequests: (messages: Array<{
+        messages: any[];
+        customId?: string;
+    }>, model?: string, options?: {
+        temperature?: number;
+        maxTokens?: number;
+        responseFormat?: {
+            type: string;
+        };
+    }) => BatchRequest[];
+    /**
+     * Create batch requests for document extraction
+     */
+    createExtractionRequests: (documents: Array<{
+        document: any;
+        schema: any;
+        customId?: string;
+    }>, model?: string) => BatchRequest[];
+    /**
+     * Save batch requests to JSONL file
+     */
+    saveBatchRequests: (requests: BatchRequest[], filePath: string) => Promise<void>;
+    /**
+     * Parse batch results from JSONL file
+     */
+    parseBatchResults: (filePath: string) => Promise<BatchResponse[]>;
+    /**
+     * Extract successful results from batch responses
+     */
+    extractSuccessfulResults: (responses: BatchResponse[]) => Array<{
+        customId: string;
+        result: any;
+    }>;
+    /**
+     * Extract failed results from batch responses
+     */
+    extractFailedResults: (responses: BatchResponse[]) => Array<{
+        customId: string;
+        error: any;
+    }>;
+};
+declare const _default: {
+    OpenAIBatchProcessor: typeof OpenAIBatchProcessor;
+    batchUtils: {
+        /**
+         * Create batch requests for chat completions
+         */
+        createChatCompletionRequests: (messages: Array<{
+            messages: any[];
+            customId?: string;
+        }>, model?: string, options?: {
+            temperature?: number;
+            maxTokens?: number;
+            responseFormat?: {
+                type: string;
+            };
+        }) => BatchRequest[];
+        /**
+         * Create batch requests for document extraction
+         */
+        createExtractionRequests: (documents: Array<{
+            document: any;
+            schema: any;
+            customId?: string;
+        }>, model?: string) => BatchRequest[];
+        /**
+         * Save batch requests to JSONL file
+         */
+        saveBatchRequests: (requests: BatchRequest[], filePath: string) => Promise<void>;
+        /**
+         * Parse batch results from JSONL file
+         */
+        parseBatchResults: (filePath: string) => Promise<BatchResponse[]>;
+        /**
+         * Extract successful results from batch responses
+         */
+        extractSuccessfulResults: (responses: BatchResponse[]) => Array<{
+            customId: string;
+            result: any;
+        }>;
+        /**
+         * Extract failed results from batch responses
+         */
+        extractFailedResults: (responses: BatchResponse[]) => Array<{
+            customId: string;
+            error: any;
+        }>;
+    };
+};
+export default _default;
+//# sourceMappingURL=batch_processing.d.ts.map

package/dist/utils/batch_processing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch_processing.d.ts","sourceRoot":"","sources":["../../src/utils/batch_processing.ts"],"names":[],"mappings":"AAIA;;;GAGG;AAEH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,CAAC;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAC3B,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,KAAK,CAAC;YACV,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,MAAM,CAAC;YAChB,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,IAAI,CAAC,EAAE,MAAM,CAAC;SACf,CAAC,CAAC;KACJ,CAAC;IACF,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,KAAK,CAAC;IACzB,MAAM,EAAE,YAAY,GAAG,QAAQ,GAAG,aAAa,GAAG,YAAY,GAAG,WAAW,GAAG,SAAS,GAAG,YAAY,GAAG,WAAW,CAAC;IACtH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE;QACd,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,KAAK,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE;QACR,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,WAAW,EAAE,MAAM,CAAC;IACpB,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;gBACZ,OAAO,EAAE,sBAAsB;IAM3C;;OAEG;IACG,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,OAAiB,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IA0BzG;;OAEG;IACG,WAAW,CACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,gBAAgB,GAAE,KAAa,EAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,OAAO,CAAC,QAAQ,CAAC;IAqBpB;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;IAWlD;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;IAgBrD;;OAEG;IACG,WAAW,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;KAAE,CAAC;IAepF;;OAEG;IACG,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWnD;;OAEG;IACI,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,GAAE,MAAc,GAAG,cAAc,CAAC,iBAAiB,EAAE,IAAI,EAAE,OAAO,CAAC;IA0CpH;;OAEG;IACG,YAAY,CAChB,gBAAgB,EAAE,MAAM,EACxB,cAAc,EAAE,MAAM,EACtB,QAAQ,GAAE,MAA+B,EACzC,OAAO,GAAE;QACP,gBAAgB,CAAC,EAAE,KAAK,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAClC,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,YAAY,CAAC,EAAE,OAAO,CAAC;KACnB,GACL,OAAO,CAAC,QAAQ,CAAC;CAuDrB;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;IACrB;;OAEG;6CAES,KAAK,CAAC;QAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,UAChD,MAAM,YACJ;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,cAAc,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;KACnC,KACA,YAAY,EAAE;IAejB;;OAEG;0CAEU,KAAK,CAAC;QAAE,QAAQ,EAAE,GAAG,CAAC;QAAC,MAAM,EAAE,GAAG,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,UAC5D,MAAM,KACZ,YAAY,EAAE;IAajB;;OAEG;kCACiC,YAAY,EAAE,YAAY,MAAM,KAAG,OAAO,CAAC,IAAI,CAAC;IAKpF;;OAEG;kCACiC,MAAM,KAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAKrE;;OAEG;0CACmC,aAAa,EAAE,KAAG,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,GAAG,CAAA;KAAE,CAAC;IAShG;;OAEG;sCAC+B,aAAa,EAAE,KAAG,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAA;KAAE,CAAC;CAQ5F,CAAC;;;;QApFA;;WAEG;iDAES,KAAK,CAAC;YAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,UAChD,MAAM,YACJ;YACP,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,SAAS,CAAC,EAAE,MAAM,CAAC;YACnB,cAAc,CAAC,EAAE;gBAAE,IAAI,EAAE,MAAM,CAAA;aAAE,CAAC;SACnC,KACA,YAAY,EAAE;QAejB;;WAEG;8CAEU,KAAK,CAAC;YAAE,QAAQ,EAAE,GAAG,CAAC;YAAC,MAAM,EAAE,GAAG,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,UAC5D,MAAM,KACZ,YAAY,EAAE;QAajB;;WAEG;sCACiC,YAAY,EAAE,YAAY,MAAM,KAAG,OAAO,CAAC,IAAI,CAAC;QAKpF;;WAEG;sCACiC,MAAM,KAAG,OAAO,CAAC,aAAa,EAAE,CAAC;QAKrE;;WAEG;8CACmC,aAAa,EAAE,KAAG,KAAK,CAAC;YAAE,QAAQ,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,GAAG,CAAA;SAAE,CAAC;QAShG;;WAEG;0CAC+B,aAAa,EAAE,KAAG,KAAK,CAAC;YAAE,QAAQ,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,GAAG,CAAA;SAAE,CAAC;;;AAU7F,wBAGE"}

package/dist/utils/batch_processing.js

@@ -0,0 +1,268 @@
+import fs from 'fs';
+import axios from 'axios';
+import { readJSONL, writeJSONL } from './jsonl.js';
+/**
+ * OpenAI Batch API client
+ */
+export class OpenAIBatchProcessor {
+    constructor(options) {
+        this.apiKey = options.apiKey;
+        this.baseUrl = options.baseUrl || 'https://api.openai.com/v1';
+        this.timeout = options.timeout || 300000; // 5 minutes
+    }
+    /**
+     * Upload file for batch processing
+     */
+    async uploadFile(filePath, purpose = 'batch') {
+        if (!fs.existsSync(filePath)) {
+            throw new Error(`File not found: ${filePath}`);
+        }
+        const formData = new FormData();
+        const fileBuffer = fs.readFileSync(filePath);
+        const blob = new Blob([fileBuffer]);
+        formData.append('file', blob, filePath.split('/').pop());
+        formData.append('purpose', purpose);
+        const response = await axios.post(`${this.baseUrl}/files`, formData, {
+            headers: {
+                'Authorization': `Bearer ${this.apiKey}`,
+                'Content-Type': 'multipart/form-data',
+            },
+            timeout: this.timeout,
+        });
+        return {
+            id: response.data.id,
+            filename: response.data.filename,
+        };
+    }
+    /**
+     * Create batch job
+     */
+    async createBatch(inputFileId, endpoint, completionWindow = '24h', metadata) {
+        const response = await axios.post(`${this.baseUrl}/batches`, {
+            input_file_id: inputFileId,
+            endpoint,
+            completion_window: completionWindow,
+            metadata,
+        }, {
+            headers: {
+                'Authorization': `Bearer ${this.apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            timeout: this.timeout,
+        });
+        return response.data;
+    }
+    /**
+     * Get batch job status
+     */
+    async getBatch(batchId) {
+        const response = await axios.get(`${this.baseUrl}/batches/${batchId}`, {
+            headers: {
+                'Authorization': `Bearer ${this.apiKey}`,
+            },
+            timeout: this.timeout,
+        });
+        return response.data;
+    }
+    /**
+     * Cancel batch job
+     */
+    async cancelBatch(batchId) {
+        const response = await axios.post(`${this.baseUrl}/batches/${batchId}/cancel`, {}, {
+            headers: {
+                'Authorization': `Bearer ${this.apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            timeout: this.timeout,
+        });
+        return response.data;
+    }
+    /**
+     * List batch jobs
+     */
+    async listBatches(after, limit = 20) {
+        const params = new URLSearchParams();
+        if (after)
+            params.append('after', after);
+        params.append('limit', limit.toString());
+        const response = await axios.get(`${this.baseUrl}/batches?${params}`, {
+            headers: {
+                'Authorization': `Bearer ${this.apiKey}`,
+            },
+            timeout: this.timeout,
+        });
+        return response.data;
+    }
+    /**
+     * Download file content
+     */
+    async downloadFile(fileId) {
+        const response = await axios.get(`${this.baseUrl}/files/${fileId}/content`, {
+            headers: {
+                'Authorization': `Bearer ${this.apiKey}`,
+            },
+            timeout: this.timeout,
+        });
+        return response.data;
+    }
+    /**
+     * Monitor batch job progress
+     */
+    async *monitorBatch(batchId, pollInterval = 30000) {
+        const startTime = Date.now();
+        let lastProgress = 0;
+        while (true) {
+            const batch = await this.getBatch(batchId);
+            const timeElapsed = Date.now() - startTime;
+            const progress = {
+                total: batch.request_counts.total,
+                completed: batch.request_counts.completed,
+                failed: batch.request_counts.failed,
+                percentage: batch.request_counts.total > 0 ?
+                    (batch.request_counts.completed / batch.request_counts.total) * 100 : 0,
+            };
+            // Estimate time remaining
+            let estimatedTimeRemaining;
+            if (progress.completed > lastProgress && progress.completed > 0) {
+                const completionRate = progress.completed / timeElapsed;
+                const remaining = progress.total - progress.completed;
+                estimatedTimeRemaining = remaining / completionRate;
+            }
+            yield {
+                jobId: batchId,
+                status: batch.status,
+                progress,
+                timeElapsed,
+                estimatedTimeRemaining,
+            };
+            // Break if job is complete
+            if (['completed', 'failed', 'cancelled', 'expired'].includes(batch.status)) {
+                break;
+            }
+            lastProgress = progress.completed;
+            await new Promise(resolve => setTimeout(resolve, pollInterval));
+        }
+    }
+    /**
+     * Process batch end-to-end
+     */
+    async processBatch(requestsFilePath, outputFilePath, endpoint = '/v1/chat/completions', options = {}) {
+        const { completionWindow = '24h', metadata, pollInterval = 30000, showProgress = true, } = options;
+        console.log('🚀 Starting batch processing...');
+        // Upload input file
+        console.log('📤 Uploading input file...');
+        const uploadResult = await this.uploadFile(requestsFilePath);
+        console.log(`✅ File uploaded: ${uploadResult.id}`);
+        // Create batch job
+        console.log('⚙️  Creating batch job...');
+        const batch = await this.createBatch(uploadResult.id, endpoint, completionWindow, metadata);
+        console.log(`✅ Batch created: ${batch.id}`);
+        // Monitor progress
+        if (showProgress) {
+            console.log('📊 Monitoring batch progress...');
+            for await (const progress of this.monitorBatch(batch.id, pollInterval)) {
+                const { percentage, completed, total } = progress.progress;
+                const timeStr = `${Math.round(progress.timeElapsed / 1000)}s`;
+                const etaStr = progress.estimatedTimeRemaining ?
+                    ` (ETA: ${Math.round(progress.estimatedTimeRemaining / 1000)}s)` : '';
+                console.log(`   Status: ${progress.status} - ${percentage.toFixed(1)}% (${completed}/${total}) - ${timeStr}${etaStr}`);
+                if (['completed', 'failed', 'cancelled', 'expired'].includes(progress.status)) {
+                    break;
+                }
+            }
+        }
+        // Get final batch status
+        const finalBatch = await this.getBatch(batch.id);
+        if (finalBatch.status === 'completed' && finalBatch.output_file_id) {
+            console.log('📥 Downloading results...');
+            const results = await this.downloadFile(finalBatch.output_file_id);
+            fs.writeFileSync(outputFilePath, results);
+            console.log(`✅ Results saved to: ${outputFilePath}`);
+        }
+        else {
+            console.error(`❌ Batch failed with status: ${finalBatch.status}`);
+            if (finalBatch.error_file_id) {
+                const errors = await this.downloadFile(finalBatch.error_file_id);
+                console.error('Error details:', errors);
+            }
+        }
+        return finalBatch;
+    }
+}
+/**
+ * Utility functions for batch processing
+ */
+export const batchUtils = {
+    /**
+     * Create batch requests for chat completions
+     */
+    createChatCompletionRequests: (messages, model = 'gpt-4o-mini', options = {}) => {
+        return messages.map((msg, index) => ({
+            custom_id: msg.customId || `request_${index}`,
+            method: 'POST',
+            url: '/v1/chat/completions',
+            body: {
+                model,
+                messages: msg.messages,
+                temperature: options.temperature || 0.0,
+                max_tokens: options.maxTokens,
+                response_format: options.responseFormat,
+            },
+        }));
+    },
+    /**
+     * Create batch requests for document extraction
+     */
+    createExtractionRequests: (documents, model = 'gpt-4o-mini') => {
+        return documents.map((doc, index) => ({
+            custom_id: doc.customId || `extraction_${index}`,
+            method: 'POST',
+            url: '/v1/documents/extractions',
+            body: {
+                json_schema: doc.schema,
+                documents: [doc.document],
+                model,
+            },
+        }));
+    },
+    /**
+     * Save batch requests to JSONL file
+     */
+    saveBatchRequests: async (requests, filePath) => {
+        await writeJSONL(filePath, requests);
+        console.log(`📄 Saved ${requests.length} batch requests to ${filePath}`);
+    },
+    /**
+     * Parse batch results from JSONL file
+     */
+    parseBatchResults: async (filePath) => {
+        const results = await readJSONL(filePath);
+        return results;
+    },
+    /**
+     * Extract successful results from batch responses
+     */
+    extractSuccessfulResults: (responses) => {
+        return responses
+            .filter(response => !response.error && response.response.status_code === 200)
+            .map(response => ({
+            customId: response.custom_id,
+            result: response.response.body,
+        }));
+    },
+    /**
+     * Extract failed results from batch responses
+     */
+    extractFailedResults: (responses) => {
+        return responses
+            .filter(response => response.error || response.response.status_code !== 200)
+            .map(response => ({
+            customId: response.custom_id,
+            error: response.error || { code: 'http_error', message: `HTTP ${response.response.status_code}` },
+        }));
+    },
+};
+export default {
+    OpenAIBatchProcessor,
+    batchUtils,
+};