@retab/node 0.0.48 → 0.0.52

package/dist/utils/display.js

@@ -1,244 +0,0 @@
-import fs from 'fs';
-import { readJSONL } from './jsonl.js';
-/**
- * Count tokens in text using a simple approximation
- * In production, you'd want to use tiktoken equivalent for JavaScript
- */
-export function countTokens(text, _model = 'gpt-4o-mini') {
-    // Simple approximation: ~4 characters per token for English text
-    // This is a rough estimate; for production use tiktoken-js or similar
-    const avgCharsPerToken = 4;
-    return Math.ceil(text.length / avgCharsPerToken);
-}
-/**
- * Count tokens in content (text + images)
- */
-export function countContentTokens(content, _model = 'gpt-4o-mini') {
-    let textTokens = 0;
-    let imageTokens = 0;
-    // Check for image references (simplified detection)
-    const imagePatterns = [
-        /data:image\/[^;]+;base64,/g,
-        /!\[.*?\]\(.*?\)/g, // Markdown images
-        /<img[^>]*>/g, // HTML images
-    ];
-    let textContent = content;
-    // Count and remove image references
-    for (const pattern of imagePatterns) {
-        const matches = content.match(pattern);
-        if (matches) {
-            // OpenAI vision pricing: roughly 85 tokens per image for low detail
-            imageTokens += matches.length * 85;
-            textContent = textContent.replace(pattern, '[IMAGE]');
-        }
-    }
-    // Count text tokens
-    textTokens = countTokens(textContent, _model);
-    return {
-        textTokens,
-        imageTokens,
-        totalTokens: textTokens + imageTokens,
-    };
-}
-/**
- * Calculate statistical metrics for an array of numbers
- */
-export function calculateStats(values) {
-    if (values.length === 0) {
-        return { min: 0, max: 0, mean: 0, median: 0, p95: 0, p99: 0, total: 0 };
-    }
-    const sorted = [...values].sort((a, b) => a - b);
-    const total = values.reduce((sum, val) => sum + val, 0);
-    const mean = total / values.length;
-    const getPercentile = (p) => {
-        const index = Math.ceil((p / 100) * sorted.length) - 1;
-        return sorted[Math.max(0, index)];
-    };
-    return {
-        min: sorted[0],
-        max: sorted[sorted.length - 1],
-        mean: Math.round(mean * 100) / 100,
-        median: sorted[Math.floor(sorted.length / 2)],
-        p95: getPercentile(95),
-        p99: getPercentile(99),
-        total,
-    };
-}
-/**
- * Process dataset and compute comprehensive metrics
- */
-export async function processDatasetAndComputeMetrics(datasetPath, inputTokenPrice = 0.00015, outputTokenPrice = 0.0006, model = 'gpt-4o-mini') {
-    if (!fs.existsSync(datasetPath)) {
-        throw new Error(`Dataset file not found: ${datasetPath}`);
-    }
-    const dataset = await readJSONL(datasetPath);
-    const inputTokenCounts = [];
-    const outputTokenCounts = [];
-    const totalTokenCounts = [];
-    let systemMessages = 0;
-    let userMessages = 0;
-    let assistantMessages = 0;
-    let totalMessages = 0;
-    let systemLengths = [];
-    let userLengths = [];
-    let assistantLengths = [];
-    let imageCount = 0;
-    let hasImages = false;
-    for (const example of dataset) {
-        if (!example.messages || !Array.isArray(example.messages)) {
-            continue;
-        }
-        let exampleInputTokens = 0;
-        let exampleOutputTokens = 0;
-        for (const message of example.messages) {
-            totalMessages++;
-            const content = message.content || '';
-            const tokenCount = countContentTokens(content, model);
-            // Track content lengths
-            const contentLength = content.length;
-            switch (message.role) {
-                case 'system':
-                    systemMessages++;
-                    exampleInputTokens += tokenCount.totalTokens;
-                    systemLengths.push(contentLength);
-                    break;
-                case 'user':
-                    userMessages++;
-                    exampleInputTokens += tokenCount.totalTokens;
-                    userLengths.push(contentLength);
-                    break;
-                case 'assistant':
-                    assistantMessages++;
-                    exampleOutputTokens += tokenCount.totalTokens;
-                    assistantLengths.push(contentLength);
-                    break;
-            }
-            // Check for images
-            if (tokenCount.imageTokens > 0) {
-                hasImages = true;
-                imageCount += tokenCount.imageTokens / 85; // Rough estimate
-            }
-        }
-        inputTokenCounts.push(exampleInputTokens);
-        outputTokenCounts.push(exampleOutputTokens);
-        totalTokenCounts.push(exampleInputTokens + exampleOutputTokens);
-    }
-    const inputStats = calculateStats(inputTokenCounts);
-    const outputStats = calculateStats(outputTokenCounts);
-    const totalStats = calculateStats(totalTokenCounts);
-    return {
-        totalExamples: dataset.length,
-        inputTokens: inputStats,
-        outputTokens: outputStats,
-        totalTokens: totalStats,
-        estimatedCost: {
-            input: (inputStats.total * inputTokenPrice) / 1000,
-            output: (outputStats.total * outputTokenPrice) / 1000,
-            total: ((inputStats.total * inputTokenPrice) + (outputStats.total * outputTokenPrice)) / 1000,
-        },
-        messageStats: {
-            systemMessages,
-            userMessages,
-            assistantMessages,
-            avgMessagesPerExample: Math.round((totalMessages / dataset.length) * 100) / 100,
-        },
-        contentAnalysis: {
-            avgSystemLength: systemLengths.length > 0 ? Math.round((systemLengths.reduce((a, b) => a + b, 0) / systemLengths.length) * 100) / 100 : 0,
-            avgUserLength: userLengths.length > 0 ? Math.round((userLengths.reduce((a, b) => a + b, 0) / userLengths.length) * 100) / 100 : 0,
-            avgAssistantLength: assistantLengths.length > 0 ? Math.round((assistantLengths.reduce((a, b) => a + b, 0) / assistantLengths.length) * 100) / 100 : 0,
-            hasImages,
-            imageCount: Math.round(imageCount),
-        },
-    };
-}
-/**
- * Display metrics in a formatted table
- */
-export function displayMetrics(metrics) {
-    console.log('\n📊 Dataset Analysis Report');
-    console.log('═'.repeat(50));
-    // Basic Stats
-    console.log(`\n📈 Basic Statistics:`);
-    console.log(`   Total Examples: ${metrics.totalExamples.toLocaleString()}`);
-    console.log(`   Avg Messages/Example: ${metrics.messageStats.avgMessagesPerExample}`);
-    // Message Distribution
-    console.log(`\n💬 Message Distribution:`);
-    console.log(`   System Messages: ${metrics.messageStats.systemMessages.toLocaleString()}`);
-    console.log(`   User Messages: ${metrics.messageStats.userMessages.toLocaleString()}`);
-    console.log(`   Assistant Messages: ${metrics.messageStats.assistantMessages.toLocaleString()}`);
-    // Token Statistics
-    console.log(`\n🔢 Token Statistics:`);
-    console.log(`   Input Tokens:`);
-    console.log(`     Total: ${metrics.inputTokens.total.toLocaleString()}`);
-    console.log(`     Mean: ${metrics.inputTokens.mean.toLocaleString()}`);
-    console.log(`     Median: ${metrics.inputTokens.median.toLocaleString()}`);
-    console.log(`     Min: ${metrics.inputTokens.min.toLocaleString()}`);
-    console.log(`     Max: ${metrics.inputTokens.max.toLocaleString()}`);
-    console.log(`     95th percentile: ${metrics.inputTokens.p95.toLocaleString()}`);
-    console.log(`     99th percentile: ${metrics.inputTokens.p99.toLocaleString()}`);
-    console.log(`\n   Output Tokens:`);
-    console.log(`     Total: ${metrics.outputTokens.total.toLocaleString()}`);
-    console.log(`     Mean: ${metrics.outputTokens.mean.toLocaleString()}`);
-    console.log(`     Median: ${metrics.outputTokens.median.toLocaleString()}`);
-    console.log(`     Min: ${metrics.outputTokens.min.toLocaleString()}`);
-    console.log(`     Max: ${metrics.outputTokens.max.toLocaleString()}`);
-    console.log(`     95th percentile: ${metrics.outputTokens.p95.toLocaleString()}`);
-    console.log(`     99th percentile: ${metrics.outputTokens.p99.toLocaleString()}`);
-    // Cost Estimation
-    console.log(`\n💰 Cost Estimation:`);
-    console.log(`   Input Cost: $${metrics.estimatedCost.input.toFixed(4)}`);
-    console.log(`   Output Cost: $${metrics.estimatedCost.output.toFixed(4)}`);
-    console.log(`   Total Cost: $${metrics.estimatedCost.total.toFixed(4)}`);
-    // Content Analysis
-    console.log(`\n📝 Content Analysis:`);
-    console.log(`   Avg System Message Length: ${metrics.contentAnalysis.avgSystemLength.toLocaleString()} chars`);
-    console.log(`   Avg User Message Length: ${metrics.contentAnalysis.avgUserLength.toLocaleString()} chars`);
-    console.log(`   Avg Assistant Message Length: ${metrics.contentAnalysis.avgAssistantLength.toLocaleString()} chars`);
-    if (metrics.contentAnalysis.hasImages) {
-        console.log(`   Images Detected: ${metrics.contentAnalysis.imageCount.toLocaleString()}`);
-    }
-    console.log('\n' + '═'.repeat(50));
-}
-/**
- * Format large numbers with appropriate units
- */
-export function formatNumber(num) {
-    if (num >= 1000000) {
-        return `${(num / 1000000).toFixed(1)}M`;
-    }
-    else if (num >= 1000) {
-        return `${(num / 1000).toFixed(1)}K`;
-    }
-    return num.toLocaleString();
-}
-/**
- * Create a simple ASCII progress bar
- */
-export function createProgressBar(current, total, width = 40) {
-    const percentage = Math.min(current / total, 1);
-    const filled = Math.floor(percentage * width);
-    const empty = width - filled;
-    return `[${'█'.repeat(filled)}${' '.repeat(empty)}] ${(percentage * 100).toFixed(1)}% (${current}/${total})`;
-}
-/**
- * Display progress with a progress bar
- */
-export function displayProgress(current, total, message) {
-    const progressBar = createProgressBar(current, total);
-    const output = message ? `${message} ${progressBar}` : progressBar;
-    // Clear line and write progress (works in most terminals)
-    process.stdout.write(`\r${output}`);
-    if (current >= total) {
-        process.stdout.write('\n');
-    }
-}
-export default {
-    processDatasetAndComputeMetrics,
-    displayMetrics,
-    countTokens,
-    countContentTokens,
-    calculateStats,
-    formatNumber,
-    createProgressBar,
-    displayProgress,
-};

package/dist/utils/hash.d.ts

@@ -1,18 +0,0 @@
-/**
- * Generate a BLAKE2b hash from bytes.
- * Uses the blake2 package to match Python's hashlib.blake2b with 8-byte digest.
- */
-export declare function generateBlake2bHashFromBytes(bytes: Buffer): string;
-/**
- * Generate a BLAKE2b hash from a base64 string.
- */
-export declare function generateBlake2bHashFromBase64(base64String: string): string;
-/**
- * Generate a BLAKE2b hash from a UTF-8 string.
- */
-export declare function generateBlake2bHashFromString(inputString: string): string;
-/**
- * Generate a BLAKE2b hash from a dictionary/object.
- */
-export declare function generateBlake2bHashFromDict(inputDict: Record<string, any>): string;
-//# sourceMappingURL=hash.d.ts.map

package/dist/utils/hash.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../../src/utils/hash.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIlE;AAED;;GAEG;AACH,wBAAgB,6BAA6B,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAG1E;AAED;;GAEG;AACH,wBAAgB,6BAA6B,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAGzE;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAGlF"}

package/dist/utils/hash.js

@@ -1,31 +0,0 @@
-import * as blake2 from 'blake2';
-/**
- * Generate a BLAKE2b hash from bytes.
- * Uses the blake2 package to match Python's hashlib.blake2b with 8-byte digest.
- */
-export function generateBlake2bHashFromBytes(bytes) {
-    const hash = blake2.createHash('blake2b', { digestLength: 8 });
-    hash.update(bytes);
-    return hash.digest('hex');
-}
-/**
- * Generate a BLAKE2b hash from a base64 string.
- */
-export function generateBlake2bHashFromBase64(base64String) {
-    const bytes = Buffer.from(base64String, 'base64');
-    return generateBlake2bHashFromBytes(bytes);
-}
-/**
- * Generate a BLAKE2b hash from a UTF-8 string.
- */
-export function generateBlake2bHashFromString(inputString) {
-    const bytes = Buffer.from(inputString, 'utf-8');
-    return generateBlake2bHashFromBytes(bytes);
-}
-/**
- * Generate a BLAKE2b hash from a dictionary/object.
- */
-export function generateBlake2bHashFromDict(inputDict) {
-    const jsonString = JSON.stringify(inputDict, Object.keys(inputDict).sort());
-    return generateBlake2bHashFromString(jsonString.trim());
-}

package/dist/utils/hashing.d.ts

@@ -1,18 +0,0 @@
-/**
- * Cryptographic hashing utilities
- * Equivalent to Python's utils/hashing.py
- */
-export declare function md5(data: string | Buffer): string;
-export declare function sha256(data: string | Buffer): string;
-export declare function sha512(data: string | Buffer): string;
-export declare function hmacSha256(data: string | Buffer, secret: string): string;
-export declare function contentHash(content: any): string;
-declare const _default: {
-    md5: typeof md5;
-    sha256: typeof sha256;
-    sha512: typeof sha512;
-    hmacSha256: typeof hmacSha256;
-    contentHash: typeof contentHash;
-};
-export default _default;
-//# sourceMappingURL=hashing.d.ts.map

package/dist/utils/hashing.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"hashing.d.ts","sourceRoot":"","sources":["../../src/utils/hashing.ts"],"names":[],"mappings":"AAEA;;;GAGG;AAEH,wBAAgB,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAEjD;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAEpD;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAEpD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAExE;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,GAAG,GAAG,MAAM,CAGhD;;;;;;;;AAED,wBAME"}

package/dist/utils/hashing.js

@@ -1,28 +0,0 @@
-import { createHash, createHmac } from 'crypto';
-/**
- * Cryptographic hashing utilities
- * Equivalent to Python's utils/hashing.py
- */
-export function md5(data) {
-    return createHash('md5').update(data).digest('hex');
-}
-export function sha256(data) {
-    return createHash('sha256').update(data).digest('hex');
-}
-export function sha512(data) {
-    return createHash('sha512').update(data).digest('hex');
-}
-export function hmacSha256(data, secret) {
-    return createHmac('sha256', secret).update(data).digest('hex');
-}
-export function contentHash(content) {
-    const normalized = typeof content === 'string' ? content : JSON.stringify(content);
-    return sha256(normalized);
-}
-export default {
-    md5,
-    sha256,
-    sha512,
-    hmacSha256,
-    contentHash,
-};

package/dist/utils/index.d.ts

@@ -1,8 +0,0 @@
-export * from './stream.js';
-export * from './ai_models.js';
-export * from './json_schema_utils.js';
-export * from './jsonl.js';
-export * from './prompt_optimization.js';
-export { default as jsonlUtils } from './jsonl.js';
-export { default as promptOptimization } from './prompt_optimization.js';
-//# sourceMappingURL=index.d.ts.map

package/dist/utils/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AACA,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AAGvC,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AAGzC,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,OAAO,IAAI,kBAAkB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/utils/index.js

@@ -1,10 +0,0 @@
-// Core utilities
-export * from './stream.js';
-export * from './ai_models.js';
-export * from './json_schema_utils.js';
-// New utilities for 100% feature parity
-export * from './jsonl.js';
-export * from './prompt_optimization.js';
-// Re-export commonly used utilities
-export { default as jsonlUtils } from './jsonl.js';
-export { default as promptOptimization } from './prompt_optimization.js';

package/dist/utils/json_schema.d.ts

@@ -1,18 +0,0 @@
-export declare function generateBlake2bHashFromString(input: string): string;
-export declare function generateSchemaDataId(jsonSchema: Record<string, any>): string;
-export declare function generateSchemaId(jsonSchema: Record<string, any>): string;
-export declare function cleanSchema(schema: Record<string, any>, removeCustomFields?: boolean, fieldsToRemove?: string[]): Record<string, any>;
-export declare function loadJsonSchema(jsonSchema: Record<string, any> | string): Record<string, any>;
-export declare function expandRefs(schema: Record<string, any>): Record<string, any>;
-export declare function createReasoningSchema(schema: Record<string, any>): Record<string, any>;
-export declare function validateCurrency(currencyCode: any): string | null;
-export declare function validateCountryCode(value: any): string | null;
-export declare function validateEmailRegex(value: any): string | null;
-export declare function validatePhoneNumber(value: any): string | null;
-export declare function filterAuxiliaryFieldsJson(jsonData: any): any;
-export declare function jsonSchemaToTypescriptInterface(schema: Record<string, any>, addFieldDescription?: boolean): string;
-export declare function jsonSchemaToNlpDataStructure(schema: Record<string, any>): string;
-export declare function jsonSchemaToStrictOpenaiSchema(schema: Record<string, any>): Record<string, any>;
-export declare function unflattenDict(flatDict: Record<string, any>, separator?: string): Record<string, any>;
-export declare function schemaToTsType(schema: Record<string, any>, definitions?: Record<string, any>, visited?: Set<string>, depth?: number, maxDepth?: number, addFieldDescription?: boolean): string;
-//# sourceMappingURL=json_schema.d.ts.map

package/dist/utils/json_schema.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"json_schema.d.ts","sourceRoot":"","sources":["../../src/utils/json_schema.ts"],"names":[],"mappings":"AAIA,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGnE;AAGD,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAS5E;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAIxE;AAGD,wBAAgB,WAAW,CACzB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,kBAAkB,GAAE,OAAc,EAClC,cAAc,GAAE,MAAM,EAAO,GAC5B,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAiCrB;AAGD,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAQ5F;AAGD,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CA+B3E;AAGD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAmCtF;AAGD,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,GAAG,GAAG,MAAM,GAAG,IAAI,CAQjE;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,GAAG,GAAG,MAAM,GAAG,IAAI,CAQ7D;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,GAAG,GAAG,MAAM,GAAG,IAAI,CAO5D;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,GAAG,GAAG,MAAM,GAAG,IAAI,CAQ7D;AAGD,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,GAAG,GAAG,GAAG,CA4B5D;AAGD,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,mBAAmB,GAAE,OAAc,GAClC,MAAM,CAoCR;AAGD,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAkChF;AAGD,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAqB/F;AAGD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,SAAS,GAAE,MAAY,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAmBzG;AAGD,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,WAAW,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,EACrC,OAAO,GAAE,GAAG,CAAC,MAAM,CAAa,EAChC,KAAK,GAAE,MAAU,EACjB,QAAQ,GAAE,MAAW,EACrB,mBAAmB,GAAE,OAAc,GAClC,MAAM,CAwCR"}