@retab/node 0.0.0-reserved → 0.0.43

package/dist/utils/cost_calculation.js

@@ -0,0 +1,99 @@
+// Basic pricing data for common models (this would typically come from a config or API)
+const MODEL_PRICING = {
+    'gpt-4o': {
+        text: { prompt: 2.5, completion: 10.0, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+    'gpt-4o-mini': {
+        text: { prompt: 0.15, completion: 0.6, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+    'gpt-4o-2024-11-20': {
+        text: { prompt: 2.5, completion: 10.0, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+    'gpt-4o-2024-08-06': {
+        text: { prompt: 2.5, completion: 10.0, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+    'gpt-4o-mini-2024-07-18': {
+        text: { prompt: 0.15, completion: 0.6, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+    'claude-3-5-sonnet-latest': {
+        text: { prompt: 3.0, completion: 15.0, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+    'claude-3-5-sonnet-20241022': {
+        text: { prompt: 3.0, completion: 15.0, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+    'gemini-2.0-flash': {
+        text: { prompt: 0.075, completion: 0.3, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+    'gemini-2.5-pro': {
+        text: { prompt: 1.25, completion: 5.0, cached_discount: 1.0 },
+        ft_price_hike: 1.0,
+    },
+};
+/**
+ * Compute the cost of a model usage.
+ */
+export function computeCostFromModel(model, usage, currency = 'USD') {
+    const pricing = MODEL_PRICING[model];
+    if (!pricing) {
+        // Return zero cost for unknown models
+        return { value: 0, currency };
+    }
+    const promptTokens = usage.prompt_tokens || 0;
+    const completionTokens = usage.completion_tokens || 0;
+    const cachedTokens = usage.cached_tokens || 0;
+    // Calculate costs per 1M tokens
+    const promptCost = (promptTokens / 1000000) * pricing.text.prompt;
+    const completionCost = (completionTokens / 1000000) * pricing.text.completion;
+    // Apply cached discount if applicable
+    const cachedCost = (cachedTokens / 1000000) * pricing.text.prompt * pricing.text.cached_discount;
+    const totalCost = promptCost + completionCost + cachedCost;
+    return {
+        value: Math.round(totalCost * 100000) / 100000, // Round to 5 decimal places
+        currency,
+    };
+}
+export function computeCostFromModelWithBreakdown(model, usage, currency = 'USD') {
+    const pricing = MODEL_PRICING[model];
+    if (!pricing) {
+        const zeroCost = { value: 0, currency };
+        return {
+            prompt_cost: zeroCost,
+            completion_cost: zeroCost,
+            cached_cost: zeroCost,
+            total_cost: zeroCost,
+            prompt_tokens: usage.prompt_tokens || 0,
+            completion_tokens: usage.completion_tokens || 0,
+            cached_tokens: usage.cached_tokens || 0,
+        };
+    }
+    const promptTokens = usage.prompt_tokens || 0;
+    const completionTokens = usage.completion_tokens || 0;
+    const cachedTokens = usage.cached_tokens || 0;
+    const promptCostValue = (promptTokens / 1000000) * pricing.text.prompt;
+    const completionCostValue = (completionTokens / 1000000) * pricing.text.completion;
+    const cachedCostValue = (cachedTokens / 1000000) * pricing.text.prompt * pricing.text.cached_discount;
+    const promptCost = { value: Math.round(promptCostValue * 100000) / 100000, currency };
+    const completionCost = { value: Math.round(completionCostValue * 100000) / 100000, currency };
+    const cachedCost = { value: Math.round(cachedCostValue * 100000) / 100000, currency };
+    const totalCost = {
+        value: Math.round((promptCostValue + completionCostValue + cachedCostValue) * 100000) / 100000,
+        currency
+    };
+    return {
+        prompt_cost: promptCost,
+        completion_cost: completionCost,
+        cached_cost: cachedCost,
+        total_cost: totalCost,
+        prompt_tokens: promptTokens,
+        completion_tokens: completionTokens,
+        cached_tokens: cachedTokens,
+    };
+}

package/dist/utils/datasets.d.ts

@@ -0,0 +1,135 @@
+import { SyncAPIResource, AsyncAPIResource } from '../resource.js';
+import { DatasetMetrics } from './display.js';
+/**
+ * Advanced Dataset management utilities for ML training workflows
+ * Equivalent to Python's jsonlUtils.py
+ */
+export interface FinetuningJSON {
+    messages: Array<{
+        role: 'system' | 'user' | 'assistant';
+        content: string;
+    }>;
+}
+export interface DocumentAnnotationPair {
+    document: string | Buffer;
+    annotation: Record<string, any>;
+}
+export interface BatchJSONLRequest {
+    custom_id: string;
+    method: 'POST';
+    url: string;
+    body: Record<string, any>;
+}
+export interface BatchJSONLResponse {
+    id: string;
+    custom_id: string;
+    response: {
+        status_code: number;
+        request_id: string;
+        body: Record<string, any>;
+    };
+    error?: {
+        code: string;
+        message: string;
+    };
+}
+export interface AnnotationOptions {
+    model?: string;
+    temperature?: number;
+    modality?: 'native' | 'text';
+    maxConcurrency?: number;
+    reasoning_effort?: 'low' | 'medium' | 'high';
+    provider?: 'openai' | 'anthropic' | 'xai' | 'gemini';
+    idempotencyKey?: string;
+}
+export interface SaveOptions {
+    modality?: 'native' | 'text';
+    imageResolutionDpi?: number;
+    browserCanvas?: 'A3' | 'A4' | 'A5';
+}
+export declare class BaseDatasetsMixin {
+    /**
+     * Process dataset and compute comprehensive metrics
+     */
+    pprint(datasetPath: string, inputTokenPrice?: number, outputTokenPrice?: number): Promise<DatasetMetrics>;
+    /**
+     * Save document-annotation pairs as JSONL training dataset
+     */
+    save(jsonSchema: Record<string, any> | string, documentAnnotationPairsPaths: Array<{
+        document: string;
+        annotation: string;
+    }>, datasetPath: string, options?: SaveOptions): Promise<void>;
+    /**
+     * Change schema in existing dataset
+     */
+    changeSchema(inputDatasetPath: string, jsonSchema: Record<string, any> | string, outputDatasetPath?: string, inplace?: boolean): Promise<void>;
+    /**
+     * Stitch multiple documents and save as dataset
+     */
+    stitchAndSave(jsonSchema: Record<string, any> | string, pairsPaths: Array<{
+        documents: string[];
+        annotation: string;
+    }>, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
+    /**
+     * Generate annotations for documents using AI models
+     */
+    annotate(jsonSchema: Record<string, any> | string, documents: string[], datasetPath: string, options?: AnnotationOptions): Promise<void>;
+    /**
+     * Update existing annotations with new model/schema
+     */
+    updateAnnotations(jsonSchema: Record<string, any> | string, oldDatasetPath: string, newDatasetPath: string, options?: AnnotationOptions): Promise<void>;
+    /**
+     * Save batch annotation requests for OpenAI Batch API
+     */
+    saveBatchAnnotateRequests(jsonSchema: Record<string, any> | string, documents: string[], batchRequestsPath: string, options?: AnnotationOptions): Promise<void>;
+    /**
+     * Build dataset from batch API results
+     */
+    buildDatasetFromBatchResults(jsonSchema: Record<string, any> | string, batchResultsPath: string, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
+    private createSystemMessage;
+    private createUserMessage;
+    private createMultiDocumentUserMessage;
+    private generateAnnotation;
+    private generateAnnotationFromUserMessage;
+}
+export declare class Datasets extends SyncAPIResource {
+    private mixin;
+    pprint(datasetPath: string, inputTokenPrice?: number, outputTokenPrice?: number): Promise<DatasetMetrics>;
+    save(jsonSchema: Record<string, any> | string, documentAnnotationPairsPaths: Array<{
+        document: string;
+        annotation: string;
+    }>, datasetPath: string, options?: SaveOptions): Promise<void>;
+    changeSchema(inputDatasetPath: string, jsonSchema: Record<string, any> | string, outputDatasetPath?: string, inplace?: boolean): Promise<void>;
+    stitchAndSave(jsonSchema: Record<string, any> | string, pairsPaths: Array<{
+        documents: string[];
+        annotation: string;
+    }>, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
+    annotate(jsonSchema: Record<string, any> | string, documents: string[], datasetPath: string, options?: AnnotationOptions): Promise<void>;
+    updateAnnotations(jsonSchema: Record<string, any> | string, oldDatasetPath: string, newDatasetPath: string, options?: AnnotationOptions): Promise<void>;
+    saveBatchAnnotateRequests(jsonSchema: Record<string, any> | string, documents: string[], batchRequestsPath: string, options?: AnnotationOptions): Promise<void>;
+    buildDatasetFromBatchResults(jsonSchema: Record<string, any> | string, batchResultsPath: string, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
+}
+export declare class AsyncDatasets extends AsyncAPIResource {
+    private mixin;
+    pprint(datasetPath: string, inputTokenPrice?: number, outputTokenPrice?: number): Promise<DatasetMetrics>;
+    save(jsonSchema: Record<string, any> | string, documentAnnotationPairsPaths: Array<{
+        document: string;
+        annotation: string;
+    }>, datasetPath: string, options?: SaveOptions): Promise<void>;
+    changeSchema(inputDatasetPath: string, jsonSchema: Record<string, any> | string, outputDatasetPath?: string, inplace?: boolean): Promise<void>;
+    stitchAndSave(jsonSchema: Record<string, any> | string, pairsPaths: Array<{
+        documents: string[];
+        annotation: string;
+    }>, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
+    annotate(jsonSchema: Record<string, any> | string, documents: string[], datasetPath: string, options?: AnnotationOptions): Promise<void>;
+    updateAnnotations(jsonSchema: Record<string, any> | string, oldDatasetPath: string, newDatasetPath: string, options?: AnnotationOptions): Promise<void>;
+    saveBatchAnnotateRequests(jsonSchema: Record<string, any> | string, documents: string[], batchRequestsPath: string, options?: AnnotationOptions): Promise<void>;
+    buildDatasetFromBatchResults(jsonSchema: Record<string, any> | string, batchResultsPath: string, datasetPath: string, modality?: 'native' | 'text'): Promise<void>;
+}
+declare const _default: {
+    Datasets: typeof Datasets;
+    AsyncDatasets: typeof AsyncDatasets;
+    BaseDatasetsMixin: typeof BaseDatasetsMixin;
+};
+export default _default;
+//# sourceMappingURL=datasets.d.ts.map

package/dist/utils/datasets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../../src/utils/datasets.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAmD,cAAc,EAAE,MAAM,cAAc,CAAC;AAE/F;;;GAGG;AAEH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QACtC,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAC3B,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC7C,QAAQ,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,KAAK,GAAG,QAAQ,CAAC;IACrD,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;CACpC;AAED,qBAAa,iBAAiB;IAC5B;;OAEG;IACG,MAAM,CACV,WAAW,EAAE,MAAM,EACnB,eAAe,GAAE,MAAgB,EACjC,gBAAgB,GAAE,MAAe,GAChC,OAAO,CAAC,cAAc,CAAC;IAe1B;;OAEG;IACG,IAAI,CACR,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,4BAA4B,EAAE,KAAK,CAAC;QAClC,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,EACF,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,WAAgB,GACxB,OAAO,CAAC,IAAI,CAAC;IAkChB;;OAEG;IACG,YAAY,CAChB,gBAAgB,EAAE,MAAM,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,IAAI,CAAC;IAkChB;;OAEG;IACG,aAAa,CACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,UAAU,EAAE,KAAK,CAAC;QAChB,SAAS,EAAE,MAAM,EAAE,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,EACF,WAAW,EAAE,MAAM,EACnB,QAAQ,GAAE,QAAQ,GAAG,MAAiB,GACrC,OAAO,CAAC,IAAI,CAAC;IAkChB;;OAEG;IACG,QAAQ,CACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC;IAyDhB;;OAEG;IACG,iBAAiB,CACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC;IAoDhB;;OAEG;IACG,yBAAyB,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,iBAAiB,EAAE,MAAM,EACzB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC;IA+BhB;;OAEG;IACG,4BAA4B,CAChC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,gBAAgB,EAAE,MAAM,EACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,GAAE,QAAQ,GAAG,MAAiB,GACrC,OAAO,CAAC,IAAI,CAAC;IAoDhB,OAAO,CAAC,mBAAmB;YAab,iBAAiB;YAejB,8BAA8B;YAe9B,kBAAkB;YAclB,iCAAiC;CAQhD;AAED,qBAAa,QAAS,SAAQ,eAAe;IAC3C,OAAO,CAAC,KAAK,CAA2B;IAElC,MAAM,CACV,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,MAAM,EACxB,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,cAAc,CAAC;IAIpB,IAAI,CACR,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,4BAA4B,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EAC7E,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,IAAI,CAAC;IAIV,YAAY,CAChB,gBAAgB,EAAE,MAAM,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,OAAO,CAAC,EAAE,OAAO,GAChB,OAAO,CAAC,IAAI,CAAC;IAIV,aAAa,CACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,UAAU,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EAC9D,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,GAC3B,OAAO,CAAC,IAAI,CAAC;IAIV,QAAQ,CACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,iBAAiB,CACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,yBAAyB,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,iBAAiB,EAAE,MAAM,EACzB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,4BAA4B,CAChC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,gBAAgB,EAAE,MAAM,EACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,GAC3B,OAAO,CAAC,IAAI,CAAC;CAGjB;AAED,qBAAa,aAAc,SAAQ,gBAAgB;IACjD,OAAO,CAAC,KAAK,CAA2B;IAElC,MAAM,CACV,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,MAAM,EACxB,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,cAAc,CAAC;IAIpB,IAAI,CACR,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,4BAA4B,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EAC7E,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,IAAI,CAAC;IAIV,YAAY,CAChB,gBAAgB,EAAE,MAAM,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,OAAO,CAAC,EAAE,OAAO,GAChB,OAAO,CAAC,IAAI,CAAC;IAIV,aAAa,CACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,UAAU,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EAC9D,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,GAC3B,OAAO,CAAC,IAAI,CAAC;IAIV,QAAQ,CACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,iBAAiB,CACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,yBAAyB,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EAAE,EACnB,iBAAiB,EAAE,MAAM,EACzB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC;IAIV,4BAA4B,CAChC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EACxC,gBAAgB,EAAE,MAAM,EACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,GAC3B,OAAO,CAAC,IAAI,CAAC;CAGjB;;;;;;AAED,wBAIE"}

package/dist/utils/datasets.js

@@ -0,0 +1,359 @@
+import fs from 'fs';
+import path from 'path';
+import { SyncAPIResource, AsyncAPIResource } from '../resource.js';
+import { readJSONL, writeJSONL } from './jsonl.js';
+import { displayMetrics, processDatasetAndComputeMetrics } from './display.js';
+export class BaseDatasetsMixin {
+    /**
+     * Process dataset and compute comprehensive metrics
+     */
+    async pprint(datasetPath, inputTokenPrice = 0.00015, outputTokenPrice = 0.0006) {
+        if (!fs.existsSync(datasetPath)) {
+            throw new Error(`Dataset file not found: ${datasetPath}`);
+        }
+        const metrics = await processDatasetAndComputeMetrics(datasetPath, inputTokenPrice, outputTokenPrice);
+        displayMetrics(metrics);
+        return metrics;
+    }
+    /**
+     * Save document-annotation pairs as JSONL training dataset
+     */
+    async save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options = {}) {
+        const { modality = 'native' } = options;
+        const finetuningData = [];
+        for (const { document: docPath, annotation: annPath } of documentAnnotationPairsPaths) {
+            // Read document and annotation
+            if (!fs.existsSync(docPath) || !fs.existsSync(annPath)) {
+                throw new Error(`Document or annotation file not found: ${docPath}, ${annPath}`);
+            }
+            const annotation = JSON.parse(fs.readFileSync(annPath, 'utf-8'));
+            // Create system message with schema
+            const systemMessage = this.createSystemMessage(jsonSchema, modality);
+            // Create user message with document
+            const userMessage = await this.createUserMessage(docPath, modality, options);
+            // Create assistant message with annotation
+            const assistantMessage = {
+                role: 'assistant',
+                content: JSON.stringify(annotation),
+            };
+            finetuningData.push({
+                messages: [systemMessage, userMessage, assistantMessage],
+            });
+        }
+        // Write to JSONL file
+        await writeJSONL(datasetPath, finetuningData);
+        console.log(`✅ Dataset saved to ${datasetPath} with ${finetuningData.length} examples`);
+    }
+    /**
+     * Change schema in existing dataset
+     */
+    async changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace = false) {
+        if (!fs.existsSync(inputDatasetPath)) {
+            throw new Error(`Input dataset not found: ${inputDatasetPath}`);
+        }
+        const outputPath = inplace ? inputDatasetPath : (outputDatasetPath || inputDatasetPath);
+        const tempPath = `${outputPath}.tmp`;
+        try {
+            const dataset = await readJSONL(inputDatasetPath);
+            const newSystemMessage = this.createSystemMessage(jsonSchema, 'native');
+            const updatedDataset = dataset.map((item) => ({
+                ...item,
+                messages: [
+                    newSystemMessage,
+                    ...item.messages.slice(1), // Keep user and assistant messages
+                ],
+            }));
+            await writeJSONL(tempPath, updatedDataset);
+            // Atomic move
+            fs.renameSync(tempPath, outputPath);
+            console.log(`✅ Schema updated in ${outputPath}`);
+        }
+        catch (error) {
+            // Cleanup temp file on error
+            if (fs.existsSync(tempPath)) {
+                fs.unlinkSync(tempPath);
+            }
+            throw error;
+        }
+    }
+    /**
+     * Stitch multiple documents and save as dataset
+     */
+    async stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality = 'native') {
+        const finetuningData = [];
+        for (const { documents: docPaths, annotation: annPath } of pairsPaths) {
+            if (!fs.existsSync(annPath)) {
+                throw new Error(`Annotation file not found: ${annPath}`);
+            }
+            // Verify all document files exist
+            for (const docPath of docPaths) {
+                if (!fs.existsSync(docPath)) {
+                    throw new Error(`Document file not found: ${docPath}`);
+                }
+            }
+            const annotation = JSON.parse(fs.readFileSync(annPath, 'utf-8'));
+            const systemMessage = this.createSystemMessage(jsonSchema, modality);
+            const userMessage = await this.createMultiDocumentUserMessage(docPaths, modality);
+            const assistantMessage = {
+                role: 'assistant',
+                content: JSON.stringify(annotation),
+            };
+            finetuningData.push({
+                messages: [systemMessage, userMessage, assistantMessage],
+            });
+        }
+        await writeJSONL(datasetPath, finetuningData);
+        console.log(`✅ Stitched dataset saved to ${datasetPath} with ${finetuningData.length} examples`);
+    }
+    /**
+     * Generate annotations for documents using AI models
+     */
+    async annotate(jsonSchema, documents, datasetPath, options = {}) {
+        const { model = 'gpt-4o-mini', temperature = 0.0, modality = 'native', maxConcurrency = 5, reasoning_effort = 'medium', provider = 'openai', } = options;
+        console.log(`🚀 Starting annotation of ${documents.length} documents...`);
+        const finetuningData = [];
+        const concurrencyLimit = Math.min(maxConcurrency, documents.length);
+        // Process documents in batches
+        for (let i = 0; i < documents.length; i += concurrencyLimit) {
+            const batch = documents.slice(i, i + concurrencyLimit);
+            const batchPromises = batch.map(async (docPath, index) => {
+                const globalIndex = i + index;
+                console.log(`📝 Processing document ${globalIndex + 1}/${documents.length}: ${path.basename(docPath)}`);
+                try {
+                    const annotation = await this.generateAnnotation(jsonSchema, docPath, model, temperature, modality, reasoning_effort, provider);
+                    const systemMessage = this.createSystemMessage(jsonSchema, modality);
+                    const userMessage = await this.createUserMessage(docPath, modality);
+                    const assistantMessage = {
+                        role: 'assistant',
+                        content: JSON.stringify(annotation),
+                    };
+                    return {
+                        messages: [systemMessage, userMessage, assistantMessage],
+                    };
+                }
+                catch (error) {
+                    console.error(`❌ Failed to process ${docPath}:`, error);
+                    return null;
+                }
+            });
+            const batchResults = await Promise.all(batchPromises);
+            finetuningData.push(...batchResults.filter(result => result !== null));
+        }
+        await writeJSONL(datasetPath, finetuningData);
+        console.log(`✅ Annotation complete! Generated ${finetuningData.length}/${documents.length} annotations`);
+    }
+    /**
+     * Update existing annotations with new model/schema
+     */
+    async updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options = {}) {
+        if (!fs.existsSync(oldDatasetPath)) {
+            throw new Error(`Old dataset not found: ${oldDatasetPath}`);
+        }
+        console.log(`🔄 Updating annotations from ${oldDatasetPath}...`);
+        const oldDataset = await readJSONL(oldDatasetPath);
+        const updatedDataset = [];
+        for (let i = 0; i < oldDataset.length; i++) {
+            const item = oldDataset[i];
+            console.log(`🔄 Updating annotation ${i + 1}/${oldDataset.length}`);
+            try {
+                // Extract document path from user message (this is simplified)
+                const userContent = item.messages.find(m => m.role === 'user')?.content;
+                if (!userContent) {
+                    console.warn(`⚠️  No user message found in item ${i + 1}, skipping`);
+                    continue;
+                }
+                // For this implementation, we assume the document is referenced in the user message
+                // In practice, you'd need to store document paths or reconstruct them
+                const newAnnotation = await this.generateAnnotationFromUserMessage(jsonSchema, userContent, options);
+                const systemMessage = this.createSystemMessage(jsonSchema, options.modality || 'native');
+                const assistantMessage = {
+                    role: 'assistant',
+                    content: JSON.stringify(newAnnotation),
+                };
+                updatedDataset.push({
+                    messages: [
+                        systemMessage,
+                        item.messages.find(m => m.role === 'user'),
+                        assistantMessage,
+                    ],
+                });
+            }
+            catch (error) {
+                console.error(`❌ Failed to update annotation ${i + 1}:`, error);
+            }
+        }
+        await writeJSONL(newDatasetPath, updatedDataset);
+        console.log(`✅ Updated ${updatedDataset.length}/${oldDataset.length} annotations`);
+    }
+    /**
+     * Save batch annotation requests for OpenAI Batch API
+     */
+    async saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options = {}) {
+        const { model = 'gpt-4o-mini', temperature = 0.0, modality = 'native', } = options;
+        const batchRequests = [];
+        for (let i = 0; i < documents.length; i++) {
+            const docPath = documents[i];
+            const systemMessage = this.createSystemMessage(jsonSchema, modality);
+            const userMessage = await this.createUserMessage(docPath, modality);
+            batchRequests.push({
+                custom_id: `doc_${i}_${path.basename(docPath, path.extname(docPath))}`,
+                method: 'POST',
+                url: '/v1/chat/completions',
+                body: {
+                    model,
+                    messages: [systemMessage, userMessage],
+                    temperature,
+                    response_format: { type: 'json_object' },
+                },
+            });
+        }
+        await writeJSONL(batchRequestsPath, batchRequests);
+        console.log(`✅ Saved ${batchRequests.length} batch requests to ${batchRequestsPath}`);
+    }
+    /**
+     * Build dataset from batch API results
+     */
+    async buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality = 'native') {
+        if (!fs.existsSync(batchResultsPath)) {
+            throw new Error(`Batch results file not found: ${batchResultsPath}`);
+        }
+        const batchResults = await readJSONL(batchResultsPath);
+        const finetuningData = [];
+        for (const result of batchResults) {
+            if (result.error) {
+                console.warn(`⚠️  Skipping failed request ${result.custom_id}: ${result.error.message}`);
+                continue;
+            }
+            const response = result.response.body;
+            const content = response.choices?.[0]?.message?.content;
+            if (!content) {
+                console.warn(`⚠️  No content in response for ${result.custom_id}`);
+                continue;
+            }
+            try {
+                const annotation = JSON.parse(content);
+                // Reconstruct messages (this is simplified)
+                const systemMessage = this.createSystemMessage(jsonSchema, modality);
+                // Extract user message from original request (would need to be stored)
+                const userMessage = {
+                    role: 'user',
+                    content: `Document content for ${result.custom_id}`,
+                };
+                const assistantMessage = {
+                    role: 'assistant',
+                    content: JSON.stringify(annotation),
+                };
+                finetuningData.push({
+                    messages: [systemMessage, userMessage, assistantMessage],
+                });
+            }
+            catch (error) {
+                console.warn(`⚠️  Failed to parse annotation for ${result.custom_id}:`, error);
+            }
+        }
+        await writeJSONL(datasetPath, finetuningData);
+        console.log(`✅ Built dataset with ${finetuningData.length} examples from batch results`);
+    }
+    // Helper methods
+    createSystemMessage(jsonSchema, _modality) {
+        const schemaObj = typeof jsonSchema === 'string' ? JSON.parse(jsonSchema) : jsonSchema;
+        const schemaStr = JSON.stringify(schemaObj, null, 2);
+        return {
+            role: 'system',
+            content: `You are an expert data extraction assistant. Extract information from the provided document according to the following JSON schema:\n\n${schemaStr}\n\nReturn only valid JSON that matches the schema exactly.`,
+        };
+    }
+    async createUserMessage(docPath, _modality, _options = {}) {
+        // This is a simplified implementation
+        // In practice, you'd handle different file types, base64 encoding, etc.
+        const content = fs.readFileSync(docPath, 'utf-8');
+        return {
+            role: 'user',
+            content: `Please extract data from this document:\n\n${content}`,
+        };
+    }
+    async createMultiDocumentUserMessage(docPaths, _modality) {
+        const contents = docPaths.map((docPath, index) => {
+            const content = fs.readFileSync(docPath, 'utf-8');
+            return `Document ${index + 1} (${path.basename(docPath)}):\n${content}`;
+        }).join('\n\n---\n\n');
+        return {
+            role: 'user',
+            content: `Please extract data from these documents:\n\n${contents}`,
+        };
+    }
+    async generateAnnotation(_jsonSchema, _docPath, _model, _temperature, _modality, _reasoningEffort, _provider) {
+        // This would integrate with the actual AI providers
+        // For now, return a placeholder implementation
+        throw new Error('AI provider integration not implemented in this version');
+    }
+    async generateAnnotationFromUserMessage(_jsonSchema, _userContent, _options) {
+        // This would re-generate annotation from existing user message
+        throw new Error('Annotation update not implemented in this version');
+    }
+}
+export class Datasets extends SyncAPIResource {
+    constructor() {
+        super(...arguments);
+        this.mixin = new BaseDatasetsMixin();
+    }
+    async pprint(datasetPath, inputTokenPrice, outputTokenPrice) {
+        return this.mixin.pprint(datasetPath, inputTokenPrice, outputTokenPrice);
+    }
+    async save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options) {
+        return this.mixin.save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options);
+    }
+    async changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace) {
+        return this.mixin.changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace);
+    }
+    async stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality) {
+        return this.mixin.stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality);
+    }
+    async annotate(jsonSchema, documents, datasetPath, options) {
+        return this.mixin.annotate(jsonSchema, documents, datasetPath, options);
+    }
+    async updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options) {
+        return this.mixin.updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options);
+    }
+    async saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options) {
+        return this.mixin.saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options);
+    }
+    async buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality) {
+        return this.mixin.buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality);
+    }
+}
+export class AsyncDatasets extends AsyncAPIResource {
+    constructor() {
+        super(...arguments);
+        this.mixin = new BaseDatasetsMixin();
+    }
+    async pprint(datasetPath, inputTokenPrice, outputTokenPrice) {
+        return this.mixin.pprint(datasetPath, inputTokenPrice, outputTokenPrice);
+    }
+    async save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options) {
+        return this.mixin.save(jsonSchema, documentAnnotationPairsPaths, datasetPath, options);
+    }
+    async changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace) {
+        return this.mixin.changeSchema(inputDatasetPath, jsonSchema, outputDatasetPath, inplace);
+    }
+    async stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality) {
+        return this.mixin.stitchAndSave(jsonSchema, pairsPaths, datasetPath, modality);
+    }
+    async annotate(jsonSchema, documents, datasetPath, options) {
+        return this.mixin.annotate(jsonSchema, documents, datasetPath, options);
+    }
+    async updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options) {
+        return this.mixin.updateAnnotations(jsonSchema, oldDatasetPath, newDatasetPath, options);
+    }
+    async saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options) {
+        return this.mixin.saveBatchAnnotateRequests(jsonSchema, documents, batchRequestsPath, options);
+    }
+    async buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality) {
+        return this.mixin.buildDatasetFromBatchResults(jsonSchema, batchResultsPath, datasetPath, modality);
+    }
+}
+export default {
+    Datasets,
+    AsyncDatasets,
+    BaseDatasetsMixin,
+};