convert-buddy-js 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/dist/src/{src/index.d.ts → index.d.ts} +5 -1
- package/dist/src/{src/index.js → index.js} +9 -2
- package/dist/src/index.js.map +1 -0
- package/dist/src/{src/node.d.ts → node.d.ts} +1 -1
- package/dist/src/node.js.map +1 -0
- package/dist/src/smoke-test.js.map +1 -0
- package/dist/wasm/nodejs/convert_buddy.d.ts +4 -0
- package/dist/wasm/nodejs/convert_buddy.js +12 -0
- package/dist/wasm/nodejs/convert_buddy_bg.wasm +0 -0
- package/dist/wasm/nodejs/convert_buddy_bg.wasm.d.ts +1 -0
- package/dist/wasm/web/convert_buddy.d.ts +5 -0
- package/dist/wasm/web/convert_buddy.js +12 -0
- package/dist/wasm/web/convert_buddy_bg.wasm +0 -0
- package/dist/wasm/web/convert_buddy_bg.wasm.d.ts +1 -0
- package/package.json +3 -2
- package/dist/chunk-DESHN2IK.js +0 -151
- package/dist/chunk-DESHN2IK.js.map +0 -1
- package/dist/chunk-EW7HKBIV.js +0 -193
- package/dist/chunk-EW7HKBIV.js.map +0 -1
- package/dist/chunk-KMYCJF2Y.js +0 -63
- package/dist/chunk-KMYCJF2Y.js.map +0 -1
- package/dist/src/bench/datasets.d.ts +0 -12
- package/dist/src/bench/datasets.js +0 -149
- package/dist/src/bench/datasets.js.map +0 -1
- package/dist/src/bench/runner-with-competitors.d.ts +0 -2
- package/dist/src/bench/runner-with-competitors.js +0 -330
- package/dist/src/bench/runner-with-competitors.js.map +0 -1
- package/dist/src/bench/runner.d.ts +0 -2
- package/dist/src/bench/runner.js +0 -160
- package/dist/src/bench/runner.js.map +0 -1
- package/dist/src/src/index.js.map +0 -1
- package/dist/src/src/node.js.map +0 -1
- package/dist/src/src/smoke-test.js.map +0 -1
- /package/dist/src/{src/node.js → node.js} +0 -0
- /package/dist/src/{src/smoke-test.d.ts → smoke-test.d.ts} +0 -0
- /package/dist/src/{src/smoke-test.js → smoke-test.js} +0 -0
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../bench/runner-with-competitors.ts"],"sourcesContent":["import { performance } from \"node:perf_hooks\";\r\nimport { ConvertBuddy, convert } from \"../src/index.js\";\r\nimport { createNodeTransform } from \"../src/node.js\";\r\n\r\nimport { generateCsvDataset, generateNdjsonDataset, generateXmlDataset } from \"./datasets.js\";\r\nimport * as fs from \"node:fs\";\r\nimport * as path from \"node:path\";\r\nimport { pipeline } from \"node:stream/promises\";\r\nimport { Readable, Writable } from \"node:stream\";\r\n\r\ntype BenchmarkResult = {\r\n tool: string;\r\n conversion: string;\r\n size: string;\r\n dataset: string;\r\n throughputMbps: number;\r\n latencyMs: number;\r\n memoryMb: number;\r\n recordsPerSec: number;\r\n};\r\n\r\nasync function benchmarkConversion(\r\n tool: string,\r\n conversion: string,\r\n size: string,\r\n data: Uint8Array,\r\n opts: any\r\n): Promise<BenchmarkResult> {\r\n const startMem = process.memoryUsage().heapUsed;\r\n const start = performance.now();\r\n\r\n const result = await convert(data, opts);\r\n\r\n const end = performance.now();\r\n const endMem = process.memoryUsage().heapUsed;\r\n\r\n const latencyMs = end - start;\r\n const throughputMbps = (data.length / (1024 * 1024)) / (latencyMs / 1000);\r\n const memoryMb = (endMem - startMem) / (1024 * 1024);\r\n\r\n // Estimate records (count newlines in output)\r\n const records = result.filter((b) => b === 10).length; // newline count\r\n const recordsPerSec = records / (latencyMs / 1000);\r\n\r\n return {\r\n tool,\r\n conversion,\r\n size,\r\n dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB`,\r\n throughputMbps: parseFloat(throughputMbps.toFixed(2)),\r\n latencyMs: parseFloat(latencyMs.toFixed(2)),\r\n memoryMb: parseFloat(memoryMb.toFixed(2)),\r\n recordsPerSec: parseFloat(recordsPerSec.toFixed(0)),\r\n };\r\n}\r\n\r\n// Competitor benchmark: PapaParse\r\nasync function benchmarkPapaParse(\r\n conversion: string,\r\n size: string,\r\n csvData: string\r\n): Promise<BenchmarkResult | null> {\r\n try {\r\n const papaModule = await import(\"papaparse\");\r\n const Papa = papaModule.default || papaModule;\r\n const data = new TextEncoder().encode(csvData);\r\n \r\n const startMem = process.memoryUsage().heapUsed;\r\n const start = performance.now();\r\n\r\n const result = Papa.parse(csvData, { header: true });\r\n const ndjson = result.data.map((row: any) => JSON.stringify(row)).join(\"\\n\");\r\n\r\n const end = performance.now();\r\n const endMem = process.memoryUsage().heapUsed;\r\n\r\n const latencyMs = end - start;\r\n const throughputMbps = (data.length / (1024 * 1024)) / (latencyMs / 1000);\r\n const memoryMb = (endMem - startMem) / (1024 * 1024);\r\n const recordsPerSec = result.data.length / (latencyMs / 1000);\r\n\r\n return {\r\n tool: \"PapaParse\",\r\n conversion,\r\n size,\r\n dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB`,\r\n throughputMbps: parseFloat(throughputMbps.toFixed(2)),\r\n latencyMs: parseFloat(latencyMs.toFixed(2)),\r\n memoryMb: parseFloat(memoryMb.toFixed(2)),\r\n recordsPerSec: parseFloat(recordsPerSec.toFixed(0)),\r\n };\r\n } catch (error) {\r\n console.log(`⚠️ PapaParse not available: ${error}`);\r\n return null;\r\n }\r\n}\r\n\r\n// Competitor benchmark: csv-parse\r\nasync function benchmarkCsvParse(\r\n conversion: string,\r\n size: string,\r\n csvData: string\r\n): Promise<BenchmarkResult | null> {\r\n try {\r\n const { parse } = await import(\"csv-parse/sync\");\r\n const data = new TextEncoder().encode(csvData);\r\n \r\n const startMem = process.memoryUsage().heapUsed;\r\n const start = performance.now();\r\n\r\n const records = parse(csvData, { columns: true });\r\n const ndjson = records.map((row: any) => JSON.stringify(row)).join(\"\\n\");\r\n\r\n const end = performance.now();\r\n const endMem = process.memoryUsage().heapUsed;\r\n\r\n const latencyMs = end - start;\r\n const throughputMbps = (data.length / (1024 * 1024)) / (latencyMs / 1000);\r\n const memoryMb = (endMem - startMem) / (1024 * 1024);\r\n const recordsPerSec = records.length / (latencyMs / 1000);\r\n\r\n return {\r\n tool: \"csv-parse\",\r\n conversion,\r\n size,\r\n dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB`,\r\n throughputMbps: parseFloat(throughputMbps.toFixed(2)),\r\n latencyMs: parseFloat(latencyMs.toFixed(2)),\r\n memoryMb: parseFloat(memoryMb.toFixed(2)),\r\n recordsPerSec: parseFloat(recordsPerSec.toFixed(0)),\r\n };\r\n } catch (error) {\r\n console.log(`⚠️ csv-parse not available: ${error}`);\r\n return null;\r\n }\r\n}\r\n\r\n// Competitor benchmark: fast-csv\r\nasync function benchmarkFastCsv(\r\n conversion: string,\r\n size: string,\r\n csvData: string\r\n): Promise<BenchmarkResult | null> {\r\n try {\r\n const fastCsvModule = await import(\"fast-csv\");\r\n const fastCsv = fastCsvModule.default ?? fastCsvModule;\r\n const { Readable } = await import(\"stream\");\r\n const data = new TextEncoder().encode(csvData);\r\n \r\n const startMem = process.memoryUsage().heapUsed;\r\n const start = performance.now();\r\n\r\n const rows: any[] = [];\r\n await new Promise<void>((resolve, reject) => {\r\n Readable.from([csvData])\r\n .pipe(fastCsv.parse({ headers: true }))\r\n .on(\"data\", (row) => rows.push(row))\r\n .on(\"end\", () => resolve())\r\n .on(\"error\", reject);\r\n });\r\n\r\n const ndjson = rows.map((row) => JSON.stringify(row)).join(\"\\n\");\r\n\r\n const end = performance.now();\r\n const endMem = process.memoryUsage().heapUsed;\r\n\r\n const latencyMs = end - start;\r\n const throughputMbps = (data.length / (1024 * 1024)) / (latencyMs / 1000);\r\n const memoryMb = (endMem - startMem) / (1024 * 1024);\r\n const recordsPerSec = rows.length / (latencyMs / 1000);\r\n\r\n return {\r\n tool: \"fast-csv\",\r\n conversion,\r\n size,\r\n dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB`,\r\n throughputMbps: parseFloat(throughputMbps.toFixed(2)),\r\n latencyMs: parseFloat(latencyMs.toFixed(2)),\r\n memoryMb: parseFloat(memoryMb.toFixed(2)),\r\n recordsPerSec: parseFloat(recordsPerSec.toFixed(0)),\r\n };\r\n } catch (error) {\r\n console.log(`⚠️ fast-csv not available: ${error}`);\r\n return null;\r\n }\r\n}\r\n\r\n// Competitor benchmark: fast-xml-parser\r\nasync function benchmarkFastXmlParser(\r\n conversion: string,\r\n size: string,\r\n xmlData: string\r\n): Promise<BenchmarkResult | null> {\r\n try {\r\n const fastXmlModule = await import(\"fast-xml-parser\");\r\n const XMLParser =\r\n fastXmlModule.XMLParser ?? fastXmlModule.default?.XMLParser ?? fastXmlModule.default;\r\n const data = new TextEncoder().encode(xmlData);\r\n\r\n const startMem = process.memoryUsage().heapUsed;\r\n const start = performance.now();\r\n\r\n if (typeof XMLParser !== \"function\") {\r\n throw new TypeError(\"XMLParser constructor not available\");\r\n }\r\n const parser = new XMLParser();\r\n const parsed = parser.parse(xmlData);\r\n const records = Array.isArray(parsed?.root?.record)\r\n ? parsed.root.record\r\n : parsed?.root?.record\r\n ? [parsed.root.record]\r\n : [];\r\n const ndjson = records.map((row: any) => JSON.stringify(row)).join(\"\\n\");\r\n\r\n const end = performance.now();\r\n const endMem = process.memoryUsage().heapUsed;\r\n\r\n const latencyMs = end - start;\r\n const throughputMbps = (data.length / (1024 * 1024)) / (latencyMs / 1000);\r\n const memoryMb = (endMem - startMem) / (1024 * 1024);\r\n const recordsPerSec = records.length / (latencyMs / 1000);\r\n\r\n return {\r\n tool: \"fast-xml-parser\",\r\n conversion,\r\n size,\r\n dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB`,\r\n throughputMbps: parseFloat(throughputMbps.toFixed(2)),\r\n latencyMs: parseFloat(latencyMs.toFixed(2)),\r\n memoryMb: parseFloat(memoryMb.toFixed(2)),\r\n recordsPerSec: parseFloat(recordsPerSec.toFixed(0)),\r\n };\r\n } catch (error) {\r\n console.log(`⚠️ fast-xml-parser not available: ${error}`);\r\n return null;\r\n }\r\n}\r\n\r\nasync function runBenchmarks() {\r\n console.log(\"╔════════════════════════════════════════╗\");\r\n console.log(\"║ Convert Buddy Comprehensive Benchmarks ║\");\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n\r\n const results: BenchmarkResult[] = [];\r\n\r\n // Generate datasets\r\n console.log(\"Generating datasets...\\n\");\r\n const csvSmall = generateCsvDataset(1_000, 10);\r\n const csvMedium = generateCsvDataset(10_000, 10);\r\n const csvLarge = generateCsvDataset(100_000, 10);\r\n\r\n const csvSmallStr = new TextDecoder().decode(csvSmall);\r\n const csvMediumStr = new TextDecoder().decode(csvMedium);\r\n const csvLargeStr = new TextDecoder().decode(csvLarge);\r\n\r\n const ndjsonSmall = generateNdjsonDataset(1_000, 10);\r\n const ndjsonMedium = generateNdjsonDataset(10_000, 10);\r\n const ndjsonLarge = generateNdjsonDataset(100_000, 10);\r\n\r\n const xmlSmall = generateXmlDataset(1_000, 10);\r\n const xmlMedium = generateXmlDataset(10_000, 10);\r\n const xmlLarge = generateXmlDataset(100_000, 10);\r\n const xmlSmallStr = new TextDecoder().decode(xmlSmall);\r\n const xmlMediumStr = new TextDecoder().decode(xmlMedium);\r\n const xmlLargeStr = new TextDecoder().decode(xmlLarge);\r\n\r\n // ========== CSV -> NDJSON Benchmarks ==========\r\n console.log(\"╔════════════════════════════════════════╗\");\r\n console.log(\"║ CSV → NDJSON Benchmarks ║\");\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n\r\n // Small\r\n console.log(\"Benchmarking CSV → NDJSON (small)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"CSV→NDJSON\", \"small\", csvSmall, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n })\r\n );\r\n\r\n const papaSmall = await benchmarkPapaParse(\"CSV→NDJSON\", \"small\", csvSmallStr);\r\n if (papaSmall) results.push(papaSmall);\r\n\r\n const csvParseSmall = await benchmarkCsvParse(\"CSV→NDJSON\", \"small\", csvSmallStr);\r\n if (csvParseSmall) results.push(csvParseSmall);\r\n\r\n const fastCsvSmall = await benchmarkFastCsv(\"CSV→NDJSON\", \"small\", csvSmallStr);\r\n if (fastCsvSmall) results.push(fastCsvSmall);\r\n\r\n // Medium\r\n console.log(\"Benchmarking CSV → NDJSON (medium)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"CSV→NDJSON\", \"medium\", csvMedium, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n })\r\n );\r\n\r\n const papaMedium = await benchmarkPapaParse(\"CSV→NDJSON\", \"medium\", csvMediumStr);\r\n if (papaMedium) results.push(papaMedium);\r\n\r\n const csvParseMedium = await benchmarkCsvParse(\"CSV→NDJSON\", \"medium\", csvMediumStr);\r\n if (csvParseMedium) results.push(csvParseMedium);\r\n\r\n const fastCsvMedium = await benchmarkFastCsv(\"CSV→NDJSON\", \"medium\", csvMediumStr);\r\n if (fastCsvMedium) results.push(fastCsvMedium);\r\n\r\n // Large\r\n console.log(\"Benchmarking CSV → NDJSON (large)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"CSV→NDJSON\", \"large\", csvLarge, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n })\r\n );\r\n\r\n const papaLarge = await benchmarkPapaParse(\"CSV→NDJSON\", \"large\", csvLargeStr);\r\n if (papaLarge) results.push(papaLarge);\r\n\r\n const csvParseLarge = await benchmarkCsvParse(\"CSV→NDJSON\", \"large\", csvLargeStr);\r\n if (csvParseLarge) results.push(csvParseLarge);\r\n\r\n const fastCsvLarge = await benchmarkFastCsv(\"CSV→NDJSON\", \"large\", csvLargeStr);\r\n if (fastCsvLarge) results.push(fastCsvLarge);\r\n\r\n // ========== NDJSON Benchmarks ==========\r\n console.log(\"\\n╔════════════════════════════════════════╗\");\r\n console.log(\"║ NDJSON Benchmarks ║\");\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n\r\n console.log(\"Benchmarking NDJSON → JSON (small)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"NDJSON→JSON\", \"small\", ndjsonSmall, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"json\",\r\n })\r\n );\r\n\r\n console.log(\"Benchmarking NDJSON → JSON (medium)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"NDJSON→JSON\", \"medium\", ndjsonMedium, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"json\",\r\n })\r\n );\r\n\r\n console.log(\"Benchmarking NDJSON → JSON (large)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"NDJSON→JSON\", \"large\", ndjsonLarge, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"json\",\r\n })\r\n );\r\n\r\n console.log(\"Benchmarking NDJSON passthrough (large)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"NDJSON→NDJSON\", \"large\", ndjsonLarge, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"ndjson\",\r\n })\r\n );\r\n\r\n // ========== XML Benchmarks ==========\r\n console.log(\"\\n╔════════════════════════════════════════╗\");\r\n console.log(\"║ XML → NDJSON Benchmarks ║\");\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n\r\n console.log(\"Benchmarking XML → NDJSON (small)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"XML→NDJSON\", \"small\", xmlSmall, {\r\n inputFormat: \"xml\",\r\n outputFormat: \"ndjson\",\r\n xmlConfig: { recordElement: \"record\" },\r\n })\r\n );\r\n\r\n const fastXmlSmall = await benchmarkFastXmlParser(\"XML→NDJSON\", \"small\", xmlSmallStr);\r\n if (fastXmlSmall) results.push(fastXmlSmall);\r\n\r\n console.log(\"Benchmarking XML → NDJSON (medium)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"XML→NDJSON\", \"medium\", xmlMedium, {\r\n inputFormat: \"xml\",\r\n outputFormat: \"ndjson\",\r\n xmlConfig: { recordElement: \"record\" },\r\n })\r\n );\r\n\r\n const fastXmlMedium = await benchmarkFastXmlParser(\"XML→NDJSON\", \"medium\", xmlMediumStr);\r\n if (fastXmlMedium) results.push(fastXmlMedium);\r\n\r\n console.log(\"Benchmarking XML → NDJSON (large)...\");\r\n results.push(\r\n await benchmarkConversion(\"convert-buddy\", \"XML→NDJSON\", \"large\", xmlLarge, {\r\n inputFormat: \"xml\",\r\n outputFormat: \"ndjson\",\r\n xmlConfig: { recordElement: \"record\" },\r\n })\r\n );\r\n\r\n const fastXmlLarge = await benchmarkFastXmlParser(\"XML→NDJSON\", \"large\", xmlLargeStr);\r\n if (fastXmlLarge) results.push(fastXmlLarge);\r\n\r\n // Print results\r\n console.log(\"\\n╔════════════════════════════════════════╗\");\r\n console.log(\"║ All Benchmark Results ║\");\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n console.table(results);\r\n\r\n // Save to file\r\n const outputPath = path.join(process.cwd(), \"bench-results.json\");\r\n fs.writeFileSync(outputPath, JSON.stringify(results, null, 2));\r\n console.log(`\\nResults saved to: ${outputPath}`);\r\n\r\n const printComparisonBySize = (conversion: string, title: string) => {\r\n console.log(\"\\n╔════════════════════════════════════════╗\");\r\n console.log(`║ ${title.padEnd(34)}║`);\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n\r\n for (const size of [\"small\", \"medium\", \"large\"]) {\r\n const conversionResults = results.filter(\r\n (r) => r.conversion === conversion && r.size === size\r\n );\r\n\r\n if (conversionResults.length === 0) {\r\n continue;\r\n }\r\n\r\n const sorted = conversionResults.sort((a, b) => b.throughputMbps - a.throughputMbps);\r\n console.log(`\\n${size.toUpperCase()} (${sorted[0].dataset}):`);\r\n sorted.forEach((r, i) => {\r\n const speedup =\r\n i === 0\r\n ? \"\"\r\n : ` (${(sorted[0].throughputMbps / r.throughputMbps).toFixed(2)}x slower)`;\r\n console.log(\r\n ` ${i + 1}. ${r.tool.padEnd(15)} ${r.throughputMbps.toString().padStart(8)} MB/s ${r.recordsPerSec.toLocaleString().padStart(10)} rec/s${speedup}`\r\n );\r\n });\r\n }\r\n };\r\n\r\n // Print comparison summaries by size\r\n printComparisonBySize(\"CSV→NDJSON\", \"CSV→NDJSON Comparison by Size\");\r\n printComparisonBySize(\"XML→NDJSON\", \"XML→NDJSON Comparison by Size\");\r\n\r\n // Summary statistics\r\n console.log(\"\\n╔════════════════════════════════════════╗\");\r\n console.log(\"║ Summary Statistics ║\");\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n\r\n const convertBuddyResults = results.filter((r) => r.tool === \"convert-buddy\");\r\n const avgThroughput = convertBuddyResults.reduce((sum, r) => sum + r.throughputMbps, 0) / convertBuddyResults.length;\r\n const maxThroughput = Math.max(...convertBuddyResults.map((r) => r.throughputMbps));\r\n const maxRecordsPerSec = Math.max(...convertBuddyResults.map((r) => r.recordsPerSec));\r\n\r\n console.log(`convert-buddy Performance:`);\r\n console.log(` Average Throughput: ${avgThroughput.toFixed(2)} MB/s`);\r\n console.log(` Peak Throughput: ${maxThroughput.toFixed(2)} MB/s`);\r\n console.log(` Peak Records/sec: ${maxRecordsPerSec.toLocaleString()}`);\r\n console.log(` Conversions Tested: ${new Set(convertBuddyResults.map(r => r.conversion)).size}`);\r\n console.log(` Total Benchmarks: ${convertBuddyResults.length}`);\r\n}\r\n\r\nrunBenchmarks().catch(console.error);\r\n"],"mappings":"AAAA,SAAS,mBAAmB;AAC5B,SAAuB,eAAe;AAGtC,SAAS,oBAAoB,uBAAuB,0BAA0B;AAC9E,YAAY,QAAQ;AACpB,YAAY,UAAU;AAetB,eAAe,oBACb,MACA,YACA,MACA,MACA,MAC0B;AAC1B,QAAM,WAAW,QAAQ,YAAY,EAAE;AACvC,QAAM,QAAQ,YAAY,IAAI;AAE9B,QAAM,SAAS,MAAM,QAAQ,MAAM,IAAI;AAEvC,QAAM,MAAM,YAAY,IAAI;AAC5B,QAAM,SAAS,QAAQ,YAAY,EAAE;AAErC,QAAM,YAAY,MAAM;AACxB,QAAM,iBAAkB,KAAK,UAAU,OAAO,SAAU,YAAY;AACpE,QAAM,YAAY,SAAS,aAAa,OAAO;AAG/C,QAAM,UAAU,OAAO,OAAO,CAAC,MAAM,MAAM,EAAE,EAAE;AAC/C,QAAM,gBAAgB,WAAW,YAAY;AAE7C,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,IAAI,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,IACpD,gBAAgB,WAAW,eAAe,QAAQ,CAAC,CAAC;AAAA,IACpD,WAAW,WAAW,UAAU,QAAQ,CAAC,CAAC;AAAA,IAC1C,UAAU,WAAW,SAAS,QAAQ,CAAC,CAAC;AAAA,IACxC,eAAe,WAAW,cAAc,QAAQ,CAAC,CAAC;AAAA,EACpD;AACF;AAGA,eAAe,mBACb,YACA,MACA,SACiC;AACjC,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,WAAW;AAC3C,UAAM,OAAO,WAAW,WAAW;AACnC,UAAM,OAAO,IAAI,YAAY,EAAE,OAAO,OAAO;AAE7C,UAAM,WAAW,QAAQ,YAAY,EAAE;AACvC,UAAM,QAAQ,YAAY,IAAI;AAE9B,UAAM,SAAS,KAAK,MAAM,SAAS,EAAE,QAAQ,KAAK,CAAC;AACnD,UAAM,SAAS,OAAO,KAAK,IAAI,CAAC,QAAa,KAAK,UAAU,GAAG,CAAC,EAAE,KAAK,IAAI;AAE3E,UAAM,MAAM,YAAY,IAAI;AAC5B,UAAM,SAAS,QAAQ,YAAY,EAAE;AAErC,UAAM,YAAY,MAAM;AACxB,UAAM,iBAAkB,KAAK,UAAU,OAAO,SAAU,YAAY;AACpE,UAAM,YAAY,SAAS,aAAa,OAAO;AAC/C,UAAM,gBAAgB,OAAO,KAAK,UAAU,YAAY;AAExD,WAAO;AAAA,MACL,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,SAAS,IAAI,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,MACpD,gBAAgB,WAAW,eAAe,QAAQ,CAAC,CAAC;AAAA,MACpD,WAAW,WAAW,UAAU,QAAQ,CAAC,CAAC;AAAA,MAC1C,UAAU,WAAW,SAAS,QAAQ,CAAC,CAAC;AAAA,MACxC,eAAe,WAAW,cAAc,QAAQ,CAAC,CAAC;AAAA,IACpD;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,IAAI,0CAAgC,KAAK,EAAE;AACnD,WAAO;AAAA,EACT;AACF;AAGA,eAAe,kBACb,YACA,MACA,SACiC;AACjC,MAAI;AACF,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,gBAAgB;AAC/C,UAAM,OAAO,IAAI,YAAY,EAAE,OAAO,OAAO;AAE7C,UAAM,WAAW,QAAQ,YAAY,EAAE;AACvC,UAAM,QAAQ,YAAY,IAAI;AAE9B,UAAM,UAAU,MAAM,SAAS,EAAE,SAAS,KAAK,CAAC;AAChD,UAAM,SAAS,QAAQ,IAAI,CAAC,QAAa,KAAK,UAAU,GAAG,CAAC,EAAE,KAAK,IAAI;AAEvE,UAAM,MAAM,YAAY,IAAI;AAC5B,UAAM,SAAS,QAAQ,YAAY,EAAE;AAErC,UAAM,YAAY,MAAM;AACxB,UAAM,iBAAkB,KAAK,UAAU,OAAO,SAAU,YAAY;AACpE,UAAM,YAAY,SAAS,aAAa,OAAO;AAC/C,UAAM,gBAAgB,QAAQ,UAAU,YAAY;AAEpD,WAAO;AAAA,MACL,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,SAAS,IAAI,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,MACpD,gBAAgB,WAAW,eAAe,QAAQ,CAAC,CAAC;AAAA,MACpD,WAAW,WAAW,UAAU,QAAQ,CAAC,CAAC;AAAA,MAC1C,UAAU,WAAW,SAAS,QAAQ,CAAC,CAAC;AAAA,MACxC,eAAe,WAAW,cAAc,QAAQ,CAAC,CAAC;AAAA,IACpD;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,IAAI,0CAAgC,KAAK,EAAE;AACnD,WAAO;AAAA,EACT;AACF;AAGA,eAAe,iBACb,YACA,MACA,SACiC;AACjC,MAAI;AACF,UAAM,gBAAgB,MAAM,OAAO,UAAU;AAC7C,UAAM,UAAU,cAAc,WAAW;AACzC,UAAM,EAAE,UAAAA,UAAS,IAAI,MAAM,OAAO,QAAQ;AAC1C,UAAM,OAAO,IAAI,YAAY,EAAE,OAAO,OAAO;AAE7C,UAAM,WAAW,QAAQ,YAAY,EAAE;AACvC,UAAM,QAAQ,YAAY,IAAI;AAE9B,UAAM,OAAc,CAAC;AACrB,UAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,MAAAA,UAAS,KAAK,CAAC,OAAO,CAAC,EACpB,KAAK,QAAQ,MAAM,EAAE,SAAS,KAAK,CAAC,CAAC,EACrC,GAAG,QAAQ,CAAC,QAAQ,KAAK,KAAK,GAAG,CAAC,EAClC,GAAG,OAAO,MAAM,QAAQ,CAAC,EACzB,GAAG,SAAS,MAAM;AAAA,IACvB,CAAC;AAED,UAAM,SAAS,KAAK,IAAI,CAAC,QAAQ,KAAK,UAAU,GAAG,CAAC,EAAE,KAAK,IAAI;AAE/D,UAAM,MAAM,YAAY,IAAI;AAC5B,UAAM,SAAS,QAAQ,YAAY,EAAE;AAErC,UAAM,YAAY,MAAM;AACxB,UAAM,iBAAkB,KAAK,UAAU,OAAO,SAAU,YAAY;AACpE,UAAM,YAAY,SAAS,aAAa,OAAO;AAC/C,UAAM,gBAAgB,KAAK,UAAU,YAAY;AAEjD,WAAO;AAAA,MACL,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,SAAS,IAAI,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,MACpD,gBAAgB,WAAW,eAAe,QAAQ,CAAC,CAAC;AAAA,MACpD,WAAW,WAAW,UAAU,QAAQ,CAAC,CAAC;AAAA,MAC1C,UAAU,WAAW,SAAS,QAAQ,CAAC,CAAC;AAAA,MACxC,eAAe,WAAW,cAAc,QAAQ,CAAC,CAAC;AAAA,IACpD;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,IAAI,yCAA+B,KAAK,EAAE;AAClD,WAAO;AAAA,EACT;AACF;AAGA,eAAe,uBACb,YACA,MACA,SACiC;AACjC,MAAI;AACF,UAAM,gBAAgB,MAAM,OAAO,iBAAiB;AACpD,UAAM,YACJ,cAAc,aAAa,cAAc,SAAS,aAAa,cAAc;AAC/E,UAAM,OAAO,IAAI,YAAY,EAAE,OAAO,OAAO;AAE7C,UAAM,WAAW,QAAQ,YAAY,EAAE;AACvC,UAAM,QAAQ,YAAY,IAAI;AAE9B,QAAI,OAAO,cAAc,YAAY;AACnC,YAAM,IAAI,UAAU,qCAAqC;AAAA,IAC3D;AACA,UAAM,SAAS,IAAI,UAAU;AAC7B,UAAM,SAAS,OAAO,MAAM,OAAO;AACnC,UAAM,UAAU,MAAM,QAAQ,QAAQ,MAAM,MAAM,IAC9C,OAAO,KAAK,SACZ,QAAQ,MAAM,SACZ,CAAC,OAAO,KAAK,MAAM,IACnB,CAAC;AACP,UAAM,SAAS,QAAQ,IAAI,CAAC,QAAa,KAAK,UAAU,GAAG,CAAC,EAAE,KAAK,IAAI;AAEvE,UAAM,MAAM,YAAY,IAAI;AAC5B,UAAM,SAAS,QAAQ,YAAY,EAAE;AAErC,UAAM,YAAY,MAAM;AACxB,UAAM,iBAAkB,KAAK,UAAU,OAAO,SAAU,YAAY;AACpE,UAAM,YAAY,SAAS,aAAa,OAAO;AAC/C,UAAM,gBAAgB,QAAQ,UAAU,YAAY;AAEpD,WAAO;AAAA,MACL,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,SAAS,IAAI,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,MACpD,gBAAgB,WAAW,eAAe,QAAQ,CAAC,CAAC;AAAA,MACpD,WAAW,WAAW,UAAU,QAAQ,CAAC,CAAC;AAAA,MAC1C,UAAU,WAAW,SAAS,QAAQ,CAAC,CAAC;AAAA,MACxC,eAAe,WAAW,cAAc,QAAQ,CAAC,CAAC;AAAA,IACpD;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,IAAI,gDAAsC,KAAK,EAAE;AACzD,WAAO;AAAA,EACT;AACF;AAEA,eAAe,gBAAgB;AAC7B,UAAQ,IAAI,8PAA4C;AACxD,UAAQ,IAAI,wDAA8C;AAC1D,UAAQ,IAAI,gQAA8C;AAE1D,QAAM,UAA6B,CAAC;AAGpC,UAAQ,IAAI,0BAA0B;AACtC,QAAM,WAAW,mBAAmB,KAAO,EAAE;AAC7C,QAAM,YAAY,mBAAmB,KAAQ,EAAE;AAC/C,QAAM,WAAW,mBAAmB,KAAS,EAAE;AAE/C,QAAM,cAAc,IAAI,YAAY,EAAE,OAAO,QAAQ;AACrD,QAAM,eAAe,IAAI,YAAY,EAAE,OAAO,SAAS;AACvD,QAAM,cAAc,IAAI,YAAY,EAAE,OAAO,QAAQ;AAErD,QAAM,cAAc,sBAAsB,KAAO,EAAE;AACnD,QAAM,eAAe,sBAAsB,KAAQ,EAAE;AACrD,QAAM,cAAc,sBAAsB,KAAS,EAAE;AAErD,QAAM,WAAW,mBAAmB,KAAO,EAAE;AAC7C,QAAM,YAAY,mBAAmB,KAAQ,EAAE;AAC/C,QAAM,WAAW,mBAAmB,KAAS,EAAE;AAC/C,QAAM,cAAc,IAAI,YAAY,EAAE,OAAO,QAAQ;AACrD,QAAM,eAAe,IAAI,YAAY,EAAE,OAAO,SAAS;AACvD,QAAM,cAAc,IAAI,YAAY,EAAE,OAAO,QAAQ;AAGrD,UAAQ,IAAI,8PAA4C;AACxD,UAAQ,IAAI,2DAA4C;AACxD,UAAQ,IAAI,gQAA8C;AAG1D,UAAQ,IAAI,2CAAsC;AAClD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,mBAAc,SAAS,UAAU;AAAA,MAC1E,aAAa;AAAA,MACb,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,QAAM,YAAY,MAAM,mBAAmB,mBAAc,SAAS,WAAW;AAC7E,MAAI,UAAW,SAAQ,KAAK,SAAS;AAErC,QAAM,gBAAgB,MAAM,kBAAkB,mBAAc,SAAS,WAAW;AAChF,MAAI,cAAe,SAAQ,KAAK,aAAa;AAE7C,QAAM,eAAe,MAAM,iBAAiB,mBAAc,SAAS,WAAW;AAC9E,MAAI,aAAc,SAAQ,KAAK,YAAY;AAG3C,UAAQ,IAAI,4CAAuC;AACnD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,mBAAc,UAAU,WAAW;AAAA,MAC5E,aAAa;AAAA,MACb,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,QAAM,aAAa,MAAM,mBAAmB,mBAAc,UAAU,YAAY;AAChF,MAAI,WAAY,SAAQ,KAAK,UAAU;AAEvC,QAAM,iBAAiB,MAAM,kBAAkB,mBAAc,UAAU,YAAY;AACnF,MAAI,eAAgB,SAAQ,KAAK,cAAc;AAE/C,QAAM,gBAAgB,MAAM,iBAAiB,mBAAc,UAAU,YAAY;AACjF,MAAI,cAAe,SAAQ,KAAK,aAAa;AAG7C,UAAQ,IAAI,2CAAsC;AAClD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,mBAAc,SAAS,UAAU;AAAA,MAC1E,aAAa;AAAA,MACb,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,QAAM,YAAY,MAAM,mBAAmB,mBAAc,SAAS,WAAW;AAC7E,MAAI,UAAW,SAAQ,KAAK,SAAS;AAErC,QAAM,gBAAgB,MAAM,kBAAkB,mBAAc,SAAS,WAAW;AAChF,MAAI,cAAe,SAAQ,KAAK,aAAa;AAE7C,QAAM,eAAe,MAAM,iBAAiB,mBAAc,SAAS,WAAW;AAC9E,MAAI,aAAc,SAAQ,KAAK,YAAY;AAG3C,UAAQ,IAAI,gQAA8C;AAC1D,UAAQ,IAAI,sDAA4C;AACxD,UAAQ,IAAI,gQAA8C;AAE1D,UAAQ,IAAI,4CAAuC;AACnD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,oBAAe,SAAS,aAAa;AAAA,MAC9E,aAAa;AAAA,MACb,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,UAAQ,IAAI,6CAAwC;AACpD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,oBAAe,UAAU,cAAc;AAAA,MAChF,aAAa;AAAA,MACb,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,UAAQ,IAAI,4CAAuC;AACnD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,oBAAe,SAAS,aAAa;AAAA,MAC9E,aAAa;AAAA,MACb,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,UAAQ,IAAI,4CAA4C;AACxD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,sBAAiB,SAAS,aAAa;AAAA,MAChF,aAAa;AAAA,MACb,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAGA,UAAQ,IAAI,gQAA8C;AAC1D,UAAQ,IAAI,2DAA4C;AACxD,UAAQ,IAAI,gQAA8C;AAE1D,UAAQ,IAAI,2CAAsC;AAClD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,mBAAc,SAAS,UAAU;AAAA,MAC1E,aAAa;AAAA,MACb,cAAc;AAAA,MACd,WAAW,EAAE,eAAe,SAAS;AAAA,IACvC,CAAC;AAAA,EACH;AAEA,QAAM,eAAe,MAAM,uBAAuB,mBAAc,SAAS,WAAW;AACpF,MAAI,aAAc,SAAQ,KAAK,YAAY;AAE3C,UAAQ,IAAI,4CAAuC;AACnD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,mBAAc,UAAU,WAAW;AAAA,MAC5E,aAAa;AAAA,MACb,cAAc;AAAA,MACd,WAAW,EAAE,eAAe,SAAS;AAAA,IACvC,CAAC;AAAA,EACH;AAEA,QAAM,gBAAgB,MAAM,uBAAuB,mBAAc,UAAU,YAAY;AACvF,MAAI,cAAe,SAAQ,KAAK,aAAa;AAE7C,UAAQ,IAAI,2CAAsC;AAClD,UAAQ;AAAA,IACN,MAAM,oBAAoB,iBAAiB,mBAAc,SAAS,UAAU;AAAA,MAC1E,aAAa;AAAA,MACb,cAAc;AAAA,MACd,WAAW,EAAE,eAAe,SAAS;AAAA,IACvC,CAAC;AAAA,EACH;AAEA,QAAM,eAAe,MAAM,uBAAuB,mBAAc,SAAS,WAAW;AACpF,MAAI,aAAc,SAAQ,KAAK,YAAY;AAG3C,UAAQ,IAAI,gQAA8C;AAC1D,UAAQ,IAAI,sDAA4C;AACxD,UAAQ,IAAI,gQAA8C;AAC1D,UAAQ,MAAM,OAAO;AAGrB,QAAM,aAAa,KAAK,KAAK,QAAQ,IAAI,GAAG,oBAAoB;AAChE,KAAG,cAAc,YAAY,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAC7D,UAAQ,IAAI;AAAA,oBAAuB,UAAU,EAAE;AAE/C,QAAM,wBAAwB,CAAC,YAAoB,UAAkB;AACnE,YAAQ,IAAI,gQAA8C;AAC1D,YAAQ,IAAI,WAAM,MAAM,OAAO,EAAE,CAAC,QAAG;AACrC,YAAQ,IAAI,gQAA8C;AAE1D,eAAW,QAAQ,CAAC,SAAS,UAAU,OAAO,GAAG;AAC/C,YAAM,oBAAoB,QAAQ;AAAA,QAChC,CAAC,MAAM,EAAE,eAAe,cAAc,EAAE,SAAS;AAAA,MACnD;AAEA,UAAI,kBAAkB,WAAW,GAAG;AAClC;AAAA,MACF;AAEA,YAAM,SAAS,kBAAkB,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AACnF,cAAQ,IAAI;AAAA,EAAK,KAAK,YAAY,CAAC,KAAK,OAAO,CAAC,EAAE,OAAO,IAAI;AAC7D,aAAO,QAAQ,CAAC,GAAG,MAAM;AACvB,cAAM,UACJ,MAAM,IACF,KACA,MAAM,OAAO,CAAC,EAAE,iBAAiB,EAAE,gBAAgB,QAAQ,CAAC,CAAC;AACnE,gBAAQ;AAAA,UACN,KAAK,IAAI,CAAC,KAAK,EAAE,KAAK,OAAO,EAAE,CAAC,IAAI,EAAE,eAAe,SAAS,EAAE,SAAS,CAAC,CAAC,UAAU,EAAE,cAAc,eAAe,EAAE,SAAS,EAAE,CAAC,SAAS,OAAO;AAAA,QACpJ;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA,wBAAsB,mBAAc,oCAA+B;AACnE,wBAAsB,mBAAc,oCAA+B;AAGnE,UAAQ,IAAI,gQAA8C;AAC1D,UAAQ,IAAI,sDAA4C;AACxD,UAAQ,IAAI,gQAA8C;AAE1D,QAAM,sBAAsB,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS,eAAe;AAC5E,QAAM,gBAAgB,oBAAoB,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,gBAAgB,CAAC,IAAI,oBAAoB;AAC9G,QAAM,gBAAgB,KAAK,IAAI,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC;AAClF,QAAM,mBAAmB,KAAK,IAAI,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC;AAEpF,UAAQ,IAAI,4BAA4B;AACxC,UAAQ,IAAI,yBAAyB,cAAc,QAAQ,CAAC,CAAC,OAAO;AACpE,UAAQ,IAAI,sBAAsB,cAAc,QAAQ,CAAC,CAAC,OAAO;AACjE,UAAQ,IAAI,uBAAuB,iBAAiB,eAAe,CAAC,EAAE;AACtE,UAAQ,IAAI,yBAAyB,IAAI,IAAI,oBAAoB,IAAI,OAAK,EAAE,UAAU,CAAC,EAAE,IAAI,EAAE;AAC/F,UAAQ,IAAI,uBAAuB,oBAAoB,MAAM,EAAE;AACjE;AAEA,cAAc,EAAE,MAAM,QAAQ,KAAK;","names":["Readable"]}
|
package/dist/src/bench/runner.js
DELETED
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
import { performance } from "node:perf_hooks";
|
|
2
|
-
import { convert } from "../src/index.js";
|
|
3
|
-
import { createNodeTransform } from "../src/node.js";
|
|
4
|
-
import { generateCsvDataset, generateNdjsonDataset } from "./datasets.js";
|
|
5
|
-
import * as fs from "node:fs";
|
|
6
|
-
import * as path from "node:path";
|
|
7
|
-
import { pipeline } from "node:stream/promises";
|
|
8
|
-
import { Readable, Writable } from "node:stream";
|
|
9
|
-
async function benchmarkConversion(name, data, opts) {
|
|
10
|
-
const startMem = process.memoryUsage().heapUsed;
|
|
11
|
-
const start = performance.now();
|
|
12
|
-
const result = await convert(data, opts);
|
|
13
|
-
const end = performance.now();
|
|
14
|
-
const endMem = process.memoryUsage().heapUsed;
|
|
15
|
-
const latencyMs = end - start;
|
|
16
|
-
const throughputMbps = data.length / (1024 * 1024) / (latencyMs / 1e3);
|
|
17
|
-
const memoryMb = (endMem - startMem) / (1024 * 1024);
|
|
18
|
-
const records = result.filter((b) => b === 10).length;
|
|
19
|
-
const recordsPerSec = records / (latencyMs / 1e3);
|
|
20
|
-
return {
|
|
21
|
-
name,
|
|
22
|
-
dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB`,
|
|
23
|
-
throughputMbps: parseFloat(throughputMbps.toFixed(2)),
|
|
24
|
-
latencyMs: parseFloat(latencyMs.toFixed(2)),
|
|
25
|
-
memoryMb: parseFloat(memoryMb.toFixed(2)),
|
|
26
|
-
recordsPerSec: parseFloat(recordsPerSec.toFixed(0))
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
async function benchmarkStreaming(name, data, opts, chunkSize = 64 * 1024) {
|
|
30
|
-
const startMem = process.memoryUsage().heapUsed;
|
|
31
|
-
const start = performance.now();
|
|
32
|
-
const chunks = [];
|
|
33
|
-
const transform = await createNodeTransform(opts);
|
|
34
|
-
const readable = Readable.from(
|
|
35
|
-
(async function* () {
|
|
36
|
-
for (let i = 0; i < data.length; i += chunkSize) {
|
|
37
|
-
yield Buffer.from(data.slice(i, i + chunkSize));
|
|
38
|
-
}
|
|
39
|
-
})()
|
|
40
|
-
);
|
|
41
|
-
const writable = new Writable({
|
|
42
|
-
write(chunk, encoding, callback) {
|
|
43
|
-
chunks.push(chunk);
|
|
44
|
-
callback();
|
|
45
|
-
}
|
|
46
|
-
});
|
|
47
|
-
await pipeline(readable, transform, writable);
|
|
48
|
-
const end = performance.now();
|
|
49
|
-
const endMem = process.memoryUsage().heapUsed;
|
|
50
|
-
const result = Buffer.concat(chunks);
|
|
51
|
-
const latencyMs = end - start;
|
|
52
|
-
const throughputMbps = data.length / (1024 * 1024) / (latencyMs / 1e3);
|
|
53
|
-
const memoryMb = (endMem - startMem) / (1024 * 1024);
|
|
54
|
-
const records = result.filter((b) => b === 10).length;
|
|
55
|
-
const recordsPerSec = records / (latencyMs / 1e3);
|
|
56
|
-
return {
|
|
57
|
-
name,
|
|
58
|
-
dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB (streaming)`,
|
|
59
|
-
throughputMbps: parseFloat(throughputMbps.toFixed(2)),
|
|
60
|
-
latencyMs: parseFloat(latencyMs.toFixed(2)),
|
|
61
|
-
memoryMb: parseFloat(memoryMb.toFixed(2)),
|
|
62
|
-
recordsPerSec: parseFloat(recordsPerSec.toFixed(0))
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
async function runBenchmarks() {
|
|
66
|
-
console.log("=== Convert Buddy Benchmark Suite ===\n");
|
|
67
|
-
const results = [];
|
|
68
|
-
console.log("Generating CSV datasets...");
|
|
69
|
-
const csvSmall = generateCsvDataset(1e3, 10);
|
|
70
|
-
const csvMedium = generateCsvDataset(1e5, 20);
|
|
71
|
-
const csvLarge = generateCsvDataset(1e6, 10);
|
|
72
|
-
console.log("\nBenchmarking CSV -> NDJSON (small)...");
|
|
73
|
-
results.push(
|
|
74
|
-
await benchmarkConversion("CSV->NDJSON (small)", csvSmall, {
|
|
75
|
-
inputFormat: "csv",
|
|
76
|
-
outputFormat: "ndjson",
|
|
77
|
-
profile: true
|
|
78
|
-
})
|
|
79
|
-
);
|
|
80
|
-
console.log("Benchmarking CSV -> NDJSON (medium)...");
|
|
81
|
-
results.push(
|
|
82
|
-
await benchmarkConversion("CSV->NDJSON (medium)", csvMedium, {
|
|
83
|
-
inputFormat: "csv",
|
|
84
|
-
outputFormat: "ndjson",
|
|
85
|
-
profile: true
|
|
86
|
-
})
|
|
87
|
-
);
|
|
88
|
-
console.log("Benchmarking CSV -> NDJSON (large)...");
|
|
89
|
-
results.push(
|
|
90
|
-
await benchmarkConversion("CSV->NDJSON (large)", csvLarge, {
|
|
91
|
-
inputFormat: "csv",
|
|
92
|
-
outputFormat: "ndjson",
|
|
93
|
-
profile: true
|
|
94
|
-
})
|
|
95
|
-
);
|
|
96
|
-
console.log("Benchmarking CSV -> NDJSON (streaming)...");
|
|
97
|
-
results.push(
|
|
98
|
-
await benchmarkStreaming("CSV->NDJSON (streaming)", csvMedium, {
|
|
99
|
-
inputFormat: "csv",
|
|
100
|
-
outputFormat: "ndjson",
|
|
101
|
-
profile: true
|
|
102
|
-
})
|
|
103
|
-
);
|
|
104
|
-
console.log("\nGenerating NDJSON datasets...");
|
|
105
|
-
const ndjsonSmall = generateNdjsonDataset(1e3, 10);
|
|
106
|
-
const ndjsonMedium = generateNdjsonDataset(1e5, 20);
|
|
107
|
-
const ndjsonLarge = generateNdjsonDataset(1e6, 10);
|
|
108
|
-
console.log("Benchmarking NDJSON -> JSON (small)...");
|
|
109
|
-
results.push(
|
|
110
|
-
await benchmarkConversion("NDJSON->JSON (small)", ndjsonSmall, {
|
|
111
|
-
inputFormat: "ndjson",
|
|
112
|
-
outputFormat: "json",
|
|
113
|
-
profile: true
|
|
114
|
-
})
|
|
115
|
-
);
|
|
116
|
-
console.log("Benchmarking NDJSON -> JSON (medium)...");
|
|
117
|
-
results.push(
|
|
118
|
-
await benchmarkConversion("NDJSON->JSON (medium)", ndjsonMedium, {
|
|
119
|
-
inputFormat: "ndjson",
|
|
120
|
-
outputFormat: "json",
|
|
121
|
-
profile: true
|
|
122
|
-
})
|
|
123
|
-
);
|
|
124
|
-
console.log("Benchmarking NDJSON passthrough (large)...");
|
|
125
|
-
results.push(
|
|
126
|
-
await benchmarkConversion("NDJSON passthrough (large)", ndjsonLarge, {
|
|
127
|
-
inputFormat: "ndjson",
|
|
128
|
-
outputFormat: "ndjson",
|
|
129
|
-
profile: true
|
|
130
|
-
})
|
|
131
|
-
);
|
|
132
|
-
console.log("\n=== Benchmark Results ===\n");
|
|
133
|
-
console.table(results);
|
|
134
|
-
const outputPath = path.join(process.cwd(), "bench-results.json");
|
|
135
|
-
fs.writeFileSync(
|
|
136
|
-
outputPath,
|
|
137
|
-
JSON.stringify(
|
|
138
|
-
{
|
|
139
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
140
|
-
platform: process.platform,
|
|
141
|
-
arch: process.arch,
|
|
142
|
-
nodeVersion: process.version,
|
|
143
|
-
results
|
|
144
|
-
},
|
|
145
|
-
null,
|
|
146
|
-
2
|
|
147
|
-
)
|
|
148
|
-
);
|
|
149
|
-
console.log(`
|
|
150
|
-
Results saved to: ${outputPath}`);
|
|
151
|
-
const avgThroughput = results.reduce((sum, r) => sum + r.throughputMbps, 0) / results.length;
|
|
152
|
-
const maxThroughput = Math.max(...results.map((r) => r.throughputMbps));
|
|
153
|
-
const avgMemory = results.reduce((sum, r) => sum + r.memoryMb, 0) / results.length;
|
|
154
|
-
console.log("\n=== Summary ===");
|
|
155
|
-
console.log(`Average Throughput: ${avgThroughput.toFixed(2)} MB/s`);
|
|
156
|
-
console.log(`Peak Throughput: ${maxThroughput.toFixed(2)} MB/s`);
|
|
157
|
-
console.log(`Average Memory: ${avgMemory.toFixed(2)} MB`);
|
|
158
|
-
}
|
|
159
|
-
runBenchmarks().catch(console.error);
|
|
160
|
-
//# sourceMappingURL=runner.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../bench/runner.ts"],"sourcesContent":["import { performance } from \"node:perf_hooks\";\r\nimport { ConvertBuddy, convert } from \"../src/index.js\";\r\nimport { createNodeTransform } from \"../src/node.js\";\r\nimport { generateCsvDataset, generateNdjsonDataset } from \"./datasets.js\";\r\nimport * as fs from \"node:fs\";\r\nimport * as path from \"node:path\";\r\nimport { pipeline } from \"node:stream/promises\";\r\nimport { Readable, Writable } from \"node:stream\";\r\n\r\ntype BenchmarkResult = {\r\n name: string;\r\n dataset: string;\r\n throughputMbps: number;\r\n latencyMs: number;\r\n memoryMb: number;\r\n recordsPerSec: number;\r\n};\r\n\r\nasync function benchmarkConversion(\r\n name: string,\r\n data: Uint8Array,\r\n opts: any\r\n): Promise<BenchmarkResult> {\r\n const startMem = process.memoryUsage().heapUsed;\r\n const start = performance.now();\r\n\r\n const result = await convert(data, opts);\r\n\r\n const end = performance.now();\r\n const endMem = process.memoryUsage().heapUsed;\r\n\r\n const latencyMs = end - start;\r\n const throughputMbps = (data.length / (1024 * 1024)) / (latencyMs / 1000);\r\n const memoryMb = (endMem - startMem) / (1024 * 1024);\r\n\r\n // Estimate records (count newlines in output)\r\n const records = result.filter((b) => b === 10).length; // newline count\r\n const recordsPerSec = records / (latencyMs / 1000);\r\n\r\n return {\r\n name,\r\n dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB`,\r\n throughputMbps: parseFloat(throughputMbps.toFixed(2)),\r\n latencyMs: parseFloat(latencyMs.toFixed(2)),\r\n memoryMb: parseFloat(memoryMb.toFixed(2)),\r\n recordsPerSec: parseFloat(recordsPerSec.toFixed(0)),\r\n };\r\n}\r\n\r\nasync function benchmarkStreaming(\r\n name: string,\r\n data: Uint8Array,\r\n opts: any,\r\n chunkSize: number = 64 * 1024\r\n): Promise<BenchmarkResult> {\r\n const startMem = process.memoryUsage().heapUsed;\r\n const start = performance.now();\r\n\r\n const chunks: Buffer[] = [];\r\n const transform = await createNodeTransform(opts);\r\n\r\n const readable = Readable.from(\r\n (async function* () {\r\n for (let i = 0; i < data.length; i += chunkSize) {\r\n yield Buffer.from(data.slice(i, i + chunkSize));\r\n }\r\n })()\r\n );\r\n\r\n const writable = new Writable({\r\n write(chunk, encoding, callback) {\r\n chunks.push(chunk);\r\n callback();\r\n },\r\n });\r\n\r\n await pipeline(readable, transform, writable);\r\n\r\n const end = performance.now();\r\n const endMem = process.memoryUsage().heapUsed;\r\n\r\n const result = Buffer.concat(chunks);\r\n const latencyMs = end - start;\r\n const throughputMbps = (data.length / (1024 * 1024)) / (latencyMs / 1000);\r\n const memoryMb = (endMem - startMem) / (1024 * 1024);\r\n\r\n const records = result.filter((b) => b === 10).length;\r\n const recordsPerSec = records / (latencyMs / 1000);\r\n\r\n return {\r\n name,\r\n dataset: `${(data.length / (1024 * 1024)).toFixed(2)} MB (streaming)`,\r\n throughputMbps: parseFloat(throughputMbps.toFixed(2)),\r\n latencyMs: parseFloat(latencyMs.toFixed(2)),\r\n memoryMb: parseFloat(memoryMb.toFixed(2)),\r\n recordsPerSec: parseFloat(recordsPerSec.toFixed(0)),\r\n };\r\n}\r\n\r\nasync function runBenchmarks() {\r\n console.log(\"=== Convert Buddy Benchmark Suite ===\\n\");\r\n\r\n const results: BenchmarkResult[] = [];\r\n\r\n // CSV Benchmarks\r\n console.log(\"Generating CSV datasets...\");\r\n const csvSmall = generateCsvDataset(1000, 10); // 1K rows, 10 cols\r\n const csvMedium = generateCsvDataset(100000, 20); // 100K rows, 20 cols\r\n const csvLarge = generateCsvDataset(1000000, 10); // 1M rows, 10 cols\r\n\r\n console.log(\"\\nBenchmarking CSV -> NDJSON (small)...\");\r\n results.push(\r\n await benchmarkConversion(\"CSV->NDJSON (small)\", csvSmall, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n })\r\n );\r\n\r\n console.log(\"Benchmarking CSV -> NDJSON (medium)...\");\r\n results.push(\r\n await benchmarkConversion(\"CSV->NDJSON (medium)\", csvMedium, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n })\r\n );\r\n\r\n console.log(\"Benchmarking CSV -> NDJSON (large)...\");\r\n results.push(\r\n await benchmarkConversion(\"CSV->NDJSON (large)\", csvLarge, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n })\r\n );\r\n\r\n console.log(\"Benchmarking CSV -> NDJSON (streaming)...\");\r\n results.push(\r\n await benchmarkStreaming(\"CSV->NDJSON (streaming)\", csvMedium, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n })\r\n );\r\n\r\n // NDJSON Benchmarks\r\n console.log(\"\\nGenerating NDJSON datasets...\");\r\n const ndjsonSmall = generateNdjsonDataset(1000, 10);\r\n const ndjsonMedium = generateNdjsonDataset(100000, 20);\r\n const ndjsonLarge = generateNdjsonDataset(1000000, 10);\r\n\r\n console.log(\"Benchmarking NDJSON -> JSON (small)...\");\r\n results.push(\r\n await benchmarkConversion(\"NDJSON->JSON (small)\", ndjsonSmall, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"json\",\r\n profile: true,\r\n })\r\n );\r\n\r\n console.log(\"Benchmarking NDJSON -> JSON (medium)...\");\r\n results.push(\r\n await benchmarkConversion(\"NDJSON->JSON (medium)\", ndjsonMedium, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"json\",\r\n profile: true,\r\n })\r\n );\r\n\r\n console.log(\"Benchmarking NDJSON passthrough (large)...\");\r\n results.push(\r\n await benchmarkConversion(\"NDJSON passthrough (large)\", ndjsonLarge, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n })\r\n );\r\n\r\n // Print results\r\n console.log(\"\\n=== Benchmark Results ===\\n\");\r\n console.table(results);\r\n\r\n // Save results to JSON\r\n const outputPath = path.join(process.cwd(), \"bench-results.json\");\r\n fs.writeFileSync(\r\n outputPath,\r\n JSON.stringify(\r\n {\r\n timestamp: new Date().toISOString(),\r\n platform: process.platform,\r\n arch: process.arch,\r\n nodeVersion: process.version,\r\n results,\r\n },\r\n null,\r\n 2\r\n )\r\n );\r\n\r\n console.log(`\\nResults saved to: ${outputPath}`);\r\n\r\n // Calculate summary statistics\r\n const avgThroughput =\r\n results.reduce((sum, r) => sum + r.throughputMbps, 0) / results.length;\r\n const maxThroughput = Math.max(...results.map((r) => r.throughputMbps));\r\n const avgMemory =\r\n results.reduce((sum, r) => sum + r.memoryMb, 0) / results.length;\r\n\r\n console.log(\"\\n=== Summary ===\");\r\n console.log(`Average Throughput: ${avgThroughput.toFixed(2)} MB/s`);\r\n console.log(`Peak Throughput: ${maxThroughput.toFixed(2)} MB/s`);\r\n console.log(`Average Memory: ${avgMemory.toFixed(2)} MB`);\r\n}\r\n\r\n// Run benchmarks\r\nrunBenchmarks().catch(console.error);\r\n"],"mappings":"AAAA,SAAS,mBAAmB;AAC5B,SAAuB,eAAe;AACtC,SAAS,2BAA2B;AACpC,SAAS,oBAAoB,6BAA6B;AAC1D,YAAY,QAAQ;AACpB,YAAY,UAAU;AACtB,SAAS,gBAAgB;AACzB,SAAS,UAAU,gBAAgB;AAWnC,eAAe,oBACb,MACA,MACA,MAC0B;AAC1B,QAAM,WAAW,QAAQ,YAAY,EAAE;AACvC,QAAM,QAAQ,YAAY,IAAI;AAE9B,QAAM,SAAS,MAAM,QAAQ,MAAM,IAAI;AAEvC,QAAM,MAAM,YAAY,IAAI;AAC5B,QAAM,SAAS,QAAQ,YAAY,EAAE;AAErC,QAAM,YAAY,MAAM;AACxB,QAAM,iBAAkB,KAAK,UAAU,OAAO,SAAU,YAAY;AACpE,QAAM,YAAY,SAAS,aAAa,OAAO;AAG/C,QAAM,UAAU,OAAO,OAAO,CAAC,MAAM,MAAM,EAAE,EAAE;AAC/C,QAAM,gBAAgB,WAAW,YAAY;AAE7C,SAAO;AAAA,IACL;AAAA,IACA,SAAS,IAAI,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,IACpD,gBAAgB,WAAW,eAAe,QAAQ,CAAC,CAAC;AAAA,IACpD,WAAW,WAAW,UAAU,QAAQ,CAAC,CAAC;AAAA,IAC1C,UAAU,WAAW,SAAS,QAAQ,CAAC,CAAC;AAAA,IACxC,eAAe,WAAW,cAAc,QAAQ,CAAC,CAAC;AAAA,EACpD;AACF;AAEA,eAAe,mBACb,MACA,MACA,MACA,YAAoB,KAAK,MACC;AAC1B,QAAM,WAAW,QAAQ,YAAY,EAAE;AACvC,QAAM,QAAQ,YAAY,IAAI;AAE9B,QAAM,SAAmB,CAAC;AAC1B,QAAM,YAAY,MAAM,oBAAoB,IAAI;AAEhD,QAAM,WAAW,SAAS;AAAA,KACvB,mBAAmB;AAClB,eAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,WAAW;AAC/C,cAAM,OAAO,KAAK,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC;AAAA,MAChD;AAAA,IACF,GAAG;AAAA,EACL;AAEA,QAAM,WAAW,IAAI,SAAS;AAAA,IAC5B,MAAM,OAAO,UAAU,UAAU;AAC/B,aAAO,KAAK,KAAK;AACjB,eAAS;AAAA,IACX;AAAA,EACF,CAAC;AAED,QAAM,SAAS,UAAU,WAAW,QAAQ;AAE5C,QAAM,MAAM,YAAY,IAAI;AAC5B,QAAM,SAAS,QAAQ,YAAY,EAAE;AAErC,QAAM,SAAS,OAAO,OAAO,MAAM;AACnC,QAAM,YAAY,MAAM;AACxB,QAAM,iBAAkB,KAAK,UAAU,OAAO,SAAU,YAAY;AACpE,QAAM,YAAY,SAAS,aAAa,OAAO;AAE/C,QAAM,UAAU,OAAO,OAAO,CAAC,MAAM,MAAM,EAAE,EAAE;AAC/C,QAAM,gBAAgB,WAAW,YAAY;AAE7C,SAAO;AAAA,IACL;AAAA,IACA,SAAS,IAAI,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,IACpD,gBAAgB,WAAW,eAAe,QAAQ,CAAC,CAAC;AAAA,IACpD,WAAW,WAAW,UAAU,QAAQ,CAAC,CAAC;AAAA,IAC1C,UAAU,WAAW,SAAS,QAAQ,CAAC,CAAC;AAAA,IACxC,eAAe,WAAW,cAAc,QAAQ,CAAC,CAAC;AAAA,EACpD;AACF;AAEA,eAAe,gBAAgB;AAC7B,UAAQ,IAAI,yCAAyC;AAErD,QAAM,UAA6B,CAAC;AAGpC,UAAQ,IAAI,4BAA4B;AACxC,QAAM,WAAW,mBAAmB,KAAM,EAAE;AAC5C,QAAM,YAAY,mBAAmB,KAAQ,EAAE;AAC/C,QAAM,WAAW,mBAAmB,KAAS,EAAE;AAE/C,UAAQ,IAAI,yCAAyC;AACrD,UAAQ;AAAA,IACN,MAAM,oBAAoB,uBAAuB,UAAU;AAAA,MACzD,aAAa;AAAA,MACb,cAAc;AAAA,MACd,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,UAAQ,IAAI,wCAAwC;AACpD,UAAQ;AAAA,IACN,MAAM,oBAAoB,wBAAwB,WAAW;AAAA,MAC3D,aAAa;AAAA,MACb,cAAc;AAAA,MACd,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,UAAQ,IAAI,uCAAuC;AACnD,UAAQ;AAAA,IACN,MAAM,oBAAoB,uBAAuB,UAAU;AAAA,MACzD,aAAa;AAAA,MACb,cAAc;AAAA,MACd,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,UAAQ,IAAI,2CAA2C;AACvD,UAAQ;AAAA,IACN,MAAM,mBAAmB,2BAA2B,WAAW;AAAA,MAC7D,aAAa;AAAA,MACb,cAAc;AAAA,MACd,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,UAAQ,IAAI,iCAAiC;AAC7C,QAAM,cAAc,sBAAsB,KAAM,EAAE;AAClD,QAAM,eAAe,sBAAsB,KAAQ,EAAE;AACrD,QAAM,cAAc,sBAAsB,KAAS,EAAE;AAErD,UAAQ,IAAI,wCAAwC;AACpD,UAAQ;AAAA,IACN,MAAM,oBAAoB,wBAAwB,aAAa;AAAA,MAC7D,aAAa;AAAA,MACb,cAAc;AAAA,MACd,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,UAAQ,IAAI,yCAAyC;AACrD,UAAQ;AAAA,IACN,MAAM,oBAAoB,yBAAyB,cAAc;AAAA,MAC/D,aAAa;AAAA,MACb,cAAc;AAAA,MACd,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,UAAQ,IAAI,4CAA4C;AACxD,UAAQ;AAAA,IACN,MAAM,oBAAoB,8BAA8B,aAAa;AAAA,MACnE,aAAa;AAAA,MACb,cAAc;AAAA,MACd,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,UAAQ,IAAI,+BAA+B;AAC3C,UAAQ,MAAM,OAAO;AAGrB,QAAM,aAAa,KAAK,KAAK,QAAQ,IAAI,GAAG,oBAAoB;AAChE,KAAG;AAAA,IACD;AAAA,IACA,KAAK;AAAA,MACH;AAAA,QACE,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,MAAM,QAAQ;AAAA,QACd,aAAa,QAAQ;AAAA,QACrB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,UAAQ,IAAI;AAAA,oBAAuB,UAAU,EAAE;AAG/C,QAAM,gBACJ,QAAQ,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,gBAAgB,CAAC,IAAI,QAAQ;AAClE,QAAM,gBAAgB,KAAK,IAAI,GAAG,QAAQ,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC;AACtE,QAAM,YACJ,QAAQ,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC,IAAI,QAAQ;AAE5D,UAAQ,IAAI,mBAAmB;AAC/B,UAAQ,IAAI,uBAAuB,cAAc,QAAQ,CAAC,CAAC,OAAO;AAClE,UAAQ,IAAI,oBAAoB,cAAc,QAAQ,CAAC,CAAC,OAAO;AAC/D,UAAQ,IAAI,mBAAmB,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC1D;AAGA,cAAc,EAAE,MAAM,QAAQ,KAAK;","names":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/index.ts"],"sourcesContent":["export type Format = \"csv\" | \"ndjson\" | \"json\" | \"xml\";\r\nexport type DetectInput =\r\n | Uint8Array\r\n | ArrayBuffer\r\n | string\r\n | ReadableStream<Uint8Array>\r\n | AsyncIterable<Uint8Array>;\r\n\r\nexport type CsvDetection = {\r\n delimiter: string;\r\n fields: string[];\r\n};\r\n\r\nexport type DetectOptions = {\r\n maxBytes?: number;\r\n debug?: boolean;\r\n};\r\n\r\nexport type ConvertBuddyOptions = {\r\n debug?: boolean;\r\n profile?: boolean;\r\n inputFormat?: Format;\r\n outputFormat?: Format;\r\n chunkTargetBytes?: number;\r\n parallelism?: number; // Node only - number of worker threads\r\n csvConfig?: CsvConfig;\r\n xmlConfig?: XmlConfig;\r\n};\r\n\r\nexport type CsvConfig = {\r\n delimiter?: string;\r\n quote?: string;\r\n hasHeaders?: boolean;\r\n trimWhitespace?: boolean;\r\n};\r\n\r\nexport type XmlConfig = {\r\n recordElement?: string;\r\n trimText?: boolean;\r\n includeAttributes?: boolean;\r\n expandEntities?: boolean;\r\n};\r\n\r\nexport type Stats = {\r\n bytesIn: number;\r\n bytesOut: number;\r\n chunksIn: number;\r\n recordsProcessed: number;\r\n parseTimeMs: number;\r\n transformTimeMs: number;\r\n writeTimeMs: number;\r\n maxBufferSize: number;\r\n currentPartialSize: number;\r\n throughputMbPerSec: number;\r\n};\r\n\r\ntype WasmModule = {\r\n default?: unknown;\r\n init: (debugEnabled: boolean) => void;\r\n Converter: new (debug: boolean) => {\r\n push: (chunk: Uint8Array) => Uint8Array;\r\n finish: () => Uint8Array;\r\n getStats: () => Stats;\r\n };\r\n detectFormat?: (sample: Uint8Array) => string | null | undefined;\r\n detectCsvFields?: (sample: Uint8Array) => CsvDetection | null | undefined;\r\n __wbg_set_wasm?: (wasm: unknown) => void;\r\n};\r\n\r\nasync function loadWasmModule(): Promise<WasmModule> {\r\n const isNode =\r\n typeof process !== \"undefined\" &&\r\n !!(process as any).versions?.node;\r\n\r\n if (isNode) {\r\n const { createRequire } = await import(\"node:module\");\r\n const require = createRequire(import.meta.url);\r\n const mod = require(\"../wasm-node.cjs\");\r\n return mod as WasmModule;\r\n }\r\n\r\n const wasmUrl = new URL(\"../wasm/web/convert_buddy.js\", import.meta.url);\r\n const mod = (await import(wasmUrl.href)) as unknown as WasmModule;\r\n return mod;\r\n}\r\n\r\nexport class ConvertBuddy {\r\n private converter: any;\r\n private debug: boolean;\r\n private profile: boolean;\r\n\r\n private constructor(converter: any, debug: boolean, profile: boolean) {\r\n this.converter = converter;\r\n this.debug = debug;\r\n this.profile = profile;\r\n }\r\n\r\n static async create(opts: ConvertBuddyOptions = {}): Promise<ConvertBuddy> {\r\n const debug = !!opts.debug;\r\n const profile = !!opts.profile;\r\n\r\n const wasmModule = await loadWasmModule();\r\n\r\n if (typeof wasmModule.default === \"function\") {\r\n await (wasmModule.default as () => Promise<void>)();\r\n }\r\n\r\n wasmModule.init(debug);\r\n\r\n let converter;\r\n if (opts.inputFormat && opts.outputFormat) {\r\n // Use withConfig for custom formats\r\n const Converter = (wasmModule as any).Converter;\r\n converter = Converter.withConfig(\r\n debug,\r\n opts.inputFormat,\r\n opts.outputFormat,\r\n opts.chunkTargetBytes || 1024 * 1024,\r\n profile,\r\n opts.csvConfig,\r\n opts.xmlConfig\r\n );\r\n } else {\r\n converter = new wasmModule.Converter(debug);\r\n }\r\n\r\n if (debug) console.log(\"[convert-buddy-js] initialized\", opts);\r\n return new ConvertBuddy(converter, debug, profile);\r\n }\r\n\r\n push(chunk: Uint8Array): Uint8Array {\r\n if (this.debug) console.log(\"[convert-buddy-js] push\", chunk.byteLength);\r\n return this.converter.push(chunk);\r\n }\r\n\r\n finish(): Uint8Array {\r\n if (this.debug) console.log(\"[convert-buddy-js] finish\");\r\n return this.converter.finish();\r\n }\r\n\r\n stats(): Stats {\r\n return this.converter.getStats();\r\n }\r\n}\r\n\r\nasync function readSample(\r\n input: DetectInput,\r\n maxBytes = 256 * 1024\r\n): Promise<Uint8Array> {\r\n if (typeof input === \"string\") {\r\n const encoded = new TextEncoder().encode(input);\r\n return encoded.length > maxBytes ? encoded.slice(0, maxBytes) : encoded;\r\n }\r\n\r\n if (input instanceof Uint8Array) {\r\n return input.length > maxBytes ? input.slice(0, maxBytes) : input;\r\n }\r\n\r\n if (input instanceof ArrayBuffer) {\r\n const bytes = new Uint8Array(input);\r\n return bytes.length > maxBytes ? bytes.slice(0, maxBytes) : bytes;\r\n }\r\n\r\n if (isReadableStream(input)) {\r\n const reader = input.getReader();\r\n const chunks: Uint8Array[] = [];\r\n let total = 0;\r\n\r\n while (total < maxBytes) {\r\n const { value, done } = await reader.read();\r\n if (done || !value) break;\r\n const slice = total + value.length > maxBytes\r\n ? value.slice(0, maxBytes - total)\r\n : value;\r\n chunks.push(slice);\r\n total += slice.length;\r\n }\r\n\r\n if (total >= maxBytes) {\r\n await reader.cancel();\r\n }\r\n\r\n return concatChunks(chunks, total);\r\n }\r\n\r\n const chunks: Uint8Array[] = [];\r\n let total = 0;\r\n\r\n for await (const chunk of input as AsyncIterable<Uint8Array>) {\r\n if (total >= maxBytes) {\r\n break;\r\n }\r\n const data = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);\r\n const slice = total + data.length > maxBytes\r\n ? data.slice(0, maxBytes - total)\r\n : data;\r\n chunks.push(slice);\r\n total += slice.length;\r\n }\r\n\r\n return concatChunks(chunks, total);\r\n}\r\n\r\nfunction concatChunks(chunks: Uint8Array[], total: number): Uint8Array {\r\n const result = new Uint8Array(total);\r\n let offset = 0;\r\n for (const chunk of chunks) {\r\n result.set(chunk, offset);\r\n offset += chunk.length;\r\n }\r\n return result;\r\n}\r\n\r\nfunction isReadableStream(\r\n input: DetectInput\r\n): input is ReadableStream<Uint8Array> {\r\n return typeof (input as ReadableStream<Uint8Array>)?.getReader === \"function\";\r\n}\r\n\r\nasync function loadDetectionWasm(debug: boolean): Promise<WasmModule> {\r\n const wasmModule = await loadWasmModule();\r\n if (typeof wasmModule.default === \"function\") {\r\n await (wasmModule.default as () => Promise<void>)();\r\n }\r\n wasmModule.init(debug);\r\n return wasmModule;\r\n}\r\n\r\nexport async function detectFormat(\r\n input: DetectInput,\r\n opts: DetectOptions = {}\r\n): Promise<Format | \"unknown\"> {\r\n const wasmModule = await loadDetectionWasm(!!opts.debug);\r\n const sample = await readSample(input, opts.maxBytes);\r\n const format = wasmModule.detectFormat?.(sample);\r\n return (format as Format) ?? \"unknown\";\r\n}\r\n\r\nexport async function detectCsvFieldsAndDelimiter(\r\n input: DetectInput,\r\n opts: DetectOptions = {}\r\n): Promise<CsvDetection | null> {\r\n const wasmModule = await loadDetectionWasm(!!opts.debug);\r\n const sample = await readSample(input, opts.maxBytes);\r\n const result = wasmModule.detectCsvFields?.(sample);\r\n return result ?? null;\r\n}\r\n\r\n// Web Streams TransformStream adapter\r\nexport class ConvertBuddyTransformStream extends TransformStream<Uint8Array, Uint8Array> {\r\n constructor(opts: ConvertBuddyOptions = {}) {\r\n let buddy: ConvertBuddy | null = null;\r\n\r\n super({\r\n async start(controller) {\r\n buddy = await ConvertBuddy.create(opts);\r\n },\r\n\r\n transform(chunk, controller) {\r\n if (!buddy) {\r\n throw new Error(\"ConvertBuddy not initialized\");\r\n }\r\n\r\n const output = buddy.push(chunk);\r\n if (output.length > 0) {\r\n controller.enqueue(output);\r\n }\r\n },\r\n\r\n flush(controller) {\r\n if (!buddy) {\r\n return;\r\n }\r\n\r\n const output = buddy.finish();\r\n if (output.length > 0) {\r\n controller.enqueue(output);\r\n }\r\n\r\n if (opts.profile) {\r\n const stats = buddy.stats();\r\n console.log(\"[convert-buddy] Performance Stats:\", stats);\r\n }\r\n },\r\n });\r\n }\r\n}\r\n\r\n// Utility: Convert entire buffer/string\r\nexport async function convert(\r\n input: Uint8Array | string,\r\n opts: ConvertBuddyOptions = {}\r\n): Promise<Uint8Array> {\r\n const buddy = await ConvertBuddy.create(opts);\r\n\r\n const inputBytes = typeof input === \"string\" \r\n ? new TextEncoder().encode(input)\r\n : input;\r\n\r\n const output = buddy.push(inputBytes);\r\n const final = buddy.finish();\r\n\r\n // Combine outputs\r\n const result = new Uint8Array(output.length + final.length);\r\n result.set(output, 0);\r\n result.set(final, output.length);\r\n\r\n if (opts.profile) {\r\n const stats = buddy.stats();\r\n console.log(\"[convert-buddy] Performance Stats:\", stats);\r\n }\r\n\r\n return result;\r\n}\r\n\r\n// Utility: Convert and return as string\r\nexport async function convertToString(\r\n input: Uint8Array | string,\r\n opts: ConvertBuddyOptions = {}\r\n): Promise<string> {\r\n const result = await convert(input, opts);\r\n return new TextDecoder().decode(result);\r\n}\r\n"],"mappings":"AAqEA,eAAe,iBAAsC;AACnD,QAAM,SACJ,OAAO,YAAY,eACnB,CAAC,CAAE,QAAgB,UAAU;AAE/B,MAAI,QAAQ;AACV,UAAM,EAAE,cAAc,IAAI,MAAM,OAAO,aAAa;AACpD,UAAMA,WAAU,cAAc,YAAY,GAAG;AAC7C,UAAMC,OAAMD,SAAQ,kBAAkB;AACtC,WAAOC;AAAA,EACT;AAEA,QAAM,UAAU,IAAI,IAAI,gCAAgC,YAAY,GAAG;AACvE,QAAM,MAAO,MAAM,OAAO,QAAQ;AAClC,SAAO;AACT;AAEO,MAAM,aAAa;AAAA,EAChB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,WAAgB,OAAgB,SAAkB;AACpE,SAAK,YAAY;AACjB,SAAK,QAAQ;AACb,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,aAAa,OAAO,OAA4B,CAAC,GAA0B;AACzE,UAAM,QAAQ,CAAC,CAAC,KAAK;AACrB,UAAM,UAAU,CAAC,CAAC,KAAK;AAEvB,UAAM,aAAa,MAAM,eAAe;AAExC,QAAI,OAAO,WAAW,YAAY,YAAY;AAC5C,YAAO,WAAW,QAAgC;AAAA,IACpD;AAEA,eAAW,KAAK,KAAK;AAErB,QAAI;AACJ,QAAI,KAAK,eAAe,KAAK,cAAc;AAEzC,YAAM,YAAa,WAAmB;AACtC,kBAAY,UAAU;AAAA,QACpB;AAAA,QACA,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK,oBAAoB,OAAO;AAAA,QAChC;AAAA,QACA,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AAAA,IACF,OAAO;AACL,kBAAY,IAAI,WAAW,UAAU,KAAK;AAAA,IAC5C;AAEA,QAAI,MAAO,SAAQ,IAAI,kCAAkC,IAAI;AAC7D,WAAO,IAAI,aAAa,WAAW,OAAO,OAAO;AAAA,EACnD;AAAA,EAEA,KAAK,OAA+B;AAClC,QAAI,KAAK,MAAO,SAAQ,IAAI,2BAA2B,MAAM,UAAU;AACvE,WAAO,KAAK,UAAU,KAAK,KAAK;AAAA,EAClC;AAAA,EAEA,SAAqB;AACnB,QAAI,KAAK,MAAO,SAAQ,IAAI,2BAA2B;AACvD,WAAO,KAAK,UAAU,OAAO;AAAA,EAC/B;AAAA,EAEA,QAAe;AACb,WAAO,KAAK,UAAU,SAAS;AAAA,EACjC;AACF;AAEA,eAAe,WACb,OACA,WAAW,MAAM,MACI;AACrB,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,UAAU,IAAI,YAAY,EAAE,OAAO,KAAK;AAC9C,WAAO,QAAQ,SAAS,WAAW,QAAQ,MAAM,GAAG,QAAQ,IAAI;AAAA,EAClE;AAEA,MAAI,iBAAiB,YAAY;AAC/B,WAAO,MAAM,SAAS,WAAW,MAAM,MAAM,GAAG,QAAQ,IAAI;AAAA,EAC9D;AAEA,MAAI,iBAAiB,aAAa;AAChC,UAAM,QAAQ,IAAI,WAAW,KAAK;AAClC,WAAO,MAAM,SAAS,WAAW,MAAM,MAAM,GAAG,QAAQ,IAAI;AAAA,EAC9D;AAEA,MAAI,iBAAiB,KAAK,GAAG;AAC3B,UAAM,SAAS,MAAM,UAAU;AAC/B,UAAMC,UAAuB,CAAC;AAC9B,QAAIC,SAAQ;AAEZ,WAAOA,SAAQ,UAAU;AACvB,YAAM,EAAE,OAAO,KAAK,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,CAAC,MAAO;AACpB,YAAM,QAAQA,SAAQ,MAAM,SAAS,WACjC,MAAM,MAAM,GAAG,WAAWA,MAAK,IAC/B;AACJ,MAAAD,QAAO,KAAK,KAAK;AACjB,MAAAC,UAAS,MAAM;AAAA,IACjB;AAEA,QAAIA,UAAS,UAAU;AACrB,YAAM,OAAO,OAAO;AAAA,IACtB;AAEA,WAAO,aAAaD,SAAQC,MAAK;AAAA,EACnC;AAEA,QAAM,SAAuB,CAAC;AAC9B,MAAI,QAAQ;AAEZ,mBAAiB,SAAS,OAAoC;AAC5D,QAAI,SAAS,UAAU;AACrB;AAAA,IACF;AACA,UAAM,OAAO,iBAAiB,aAAa,QAAQ,IAAI,WAAW,KAAK;AACvE,UAAM,QAAQ,QAAQ,KAAK,SAAS,WAChC,KAAK,MAAM,GAAG,WAAW,KAAK,IAC9B;AACJ,WAAO,KAAK,KAAK;AACjB,aAAS,MAAM;AAAA,EACjB;AAEA,SAAO,aAAa,QAAQ,KAAK;AACnC;AAEA,SAAS,aAAa,QAAsB,OAA2B;AACrE,QAAM,SAAS,IAAI,WAAW,KAAK;AACnC,MAAI,SAAS;AACb,aAAW,SAAS,QAAQ;AAC1B,WAAO,IAAI,OAAO,MAAM;AACxB,cAAU,MAAM;AAAA,EAClB;AACA,SAAO;AACT;AAEA,SAAS,iBACP,OACqC;AACrC,SAAO,OAAQ,OAAsC,cAAc;AACrE;AAEA,eAAe,kBAAkB,OAAqC;AACpE,QAAM,aAAa,MAAM,eAAe;AACxC,MAAI,OAAO,WAAW,YAAY,YAAY;AAC5C,UAAO,WAAW,QAAgC;AAAA,EACpD;AACA,aAAW,KAAK,KAAK;AACrB,SAAO;AACT;AAEA,eAAsB,aACpB,OACA,OAAsB,CAAC,GACM;AAC7B,QAAM,aAAa,MAAM,kBAAkB,CAAC,CAAC,KAAK,KAAK;AACvD,QAAM,SAAS,MAAM,WAAW,OAAO,KAAK,QAAQ;AACpD,QAAM,SAAS,WAAW,eAAe,MAAM;AAC/C,SAAQ,UAAqB;AAC/B;AAEA,eAAsB,4BACpB,OACA,OAAsB,CAAC,GACO;AAC9B,QAAM,aAAa,MAAM,kBAAkB,CAAC,CAAC,KAAK,KAAK;AACvD,QAAM,SAAS,MAAM,WAAW,OAAO,KAAK,QAAQ;AACpD,QAAM,SAAS,WAAW,kBAAkB,MAAM;AAClD,SAAO,UAAU;AACnB;AAGO,MAAM,oCAAoC,gBAAwC;AAAA,EACvF,YAAY,OAA4B,CAAC,GAAG;AAC1C,QAAI,QAA6B;AAEjC,UAAM;AAAA,MACJ,MAAM,MAAM,YAAY;AACtB,gBAAQ,MAAM,aAAa,OAAO,IAAI;AAAA,MACxC;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,CAAC,OAAO;AACV,gBAAM,IAAI,MAAM,8BAA8B;AAAA,QAChD;AAEA,cAAM,SAAS,MAAM,KAAK,KAAK;AAC/B,YAAI,OAAO,SAAS,GAAG;AACrB,qBAAW,QAAQ,MAAM;AAAA,QAC3B;AAAA,MACF;AAAA,MAEA,MAAM,YAAY;AAChB,YAAI,CAAC,OAAO;AACV;AAAA,QACF;AAEA,cAAM,SAAS,MAAM,OAAO;AAC5B,YAAI,OAAO,SAAS,GAAG;AACrB,qBAAW,QAAQ,MAAM;AAAA,QAC3B;AAEA,YAAI,KAAK,SAAS;AAChB,gBAAM,QAAQ,MAAM,MAAM;AAC1B,kBAAQ,IAAI,sCAAsC,KAAK;AAAA,QACzD;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAGA,eAAsB,QACpB,OACA,OAA4B,CAAC,GACR;AACrB,QAAM,QAAQ,MAAM,aAAa,OAAO,IAAI;AAE5C,QAAM,aAAa,OAAO,UAAU,WAChC,IAAI,YAAY,EAAE,OAAO,KAAK,IAC9B;AAEJ,QAAM,SAAS,MAAM,KAAK,UAAU;AACpC,QAAM,QAAQ,MAAM,OAAO;AAG3B,QAAM,SAAS,IAAI,WAAW,OAAO,SAAS,MAAM,MAAM;AAC1D,SAAO,IAAI,QAAQ,CAAC;AACpB,SAAO,IAAI,OAAO,OAAO,MAAM;AAE/B,MAAI,KAAK,SAAS;AAChB,UAAM,QAAQ,MAAM,MAAM;AAC1B,YAAQ,IAAI,sCAAsC,KAAK;AAAA,EACzD;AAEA,SAAO;AACT;AAGA,eAAsB,gBACpB,OACA,OAA4B,CAAC,GACZ;AACjB,QAAM,SAAS,MAAM,QAAQ,OAAO,IAAI;AACxC,SAAO,IAAI,YAAY,EAAE,OAAO,MAAM;AACxC;","names":["require","mod","chunks","total"]}
|
package/dist/src/src/node.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/node.ts"],"sourcesContent":["import type { Transform as NodeTransform } from \"node:stream\";\r\n\r\nimport { ConvertBuddy, type ConvertBuddyOptions } from \"./index\";\r\n\r\nexport * from \"./index\";\r\n\r\n// Node.js Transform Stream adapter\r\nasync function loadNodeTransform(): Promise<typeof import(\"node:stream\").Transform> {\r\n const isNode =\r\n typeof process !== \"undefined\" &&\r\n !!(process as any).versions?.node;\r\n\r\n if (!isNode) {\r\n throw new Error(\"createNodeTransform is only available in Node.js runtimes.\");\r\n }\r\n\r\n const streamModule = await import(\"node:stream\");\r\n return streamModule.Transform;\r\n}\r\n\r\nexport async function createNodeTransform(\r\n opts: ConvertBuddyOptions = {}\r\n): Promise<NodeTransform> {\r\n let buddy: ConvertBuddy | null = null;\r\n let initPromise: Promise<void> | null = null;\r\n\r\n const Transform = await loadNodeTransform();\r\n const transform = new Transform({\r\n async transform(chunk: Buffer, encoding: string, callback: Function) {\r\n try {\r\n if (!buddy) {\r\n if (!initPromise) {\r\n initPromise = ConvertBuddy.create(opts).then((b) => {\r\n buddy = b;\r\n });\r\n }\r\n await initPromise;\r\n }\r\n\r\n const input = new Uint8Array(chunk);\r\n const output = buddy!.push(input);\r\n\r\n if (output.length > 0) {\r\n this.push(Buffer.from(output));\r\n }\r\n\r\n callback();\r\n } catch (err) {\r\n callback(err);\r\n }\r\n },\r\n\r\n async flush(callback: Function) {\r\n try {\r\n if (buddy) {\r\n const output = buddy.finish();\r\n if (output.length > 0) {\r\n this.push(Buffer.from(output));\r\n }\r\n\r\n if (opts.profile) {\r\n const stats = buddy.stats();\r\n console.log(\"[convert-buddy] Performance Stats:\", stats);\r\n }\r\n }\r\n callback();\r\n } catch (err) {\r\n callback(err);\r\n }\r\n },\r\n });\r\n\r\n return transform;\r\n}\r\n"],"mappings":"AAEA,SAAS,oBAA8C;AAEvD,cAAc;AAGd,eAAe,oBAAqE;AAClF,QAAM,SACJ,OAAO,YAAY,eACnB,CAAC,CAAE,QAAgB,UAAU;AAE/B,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,MAAM,4DAA4D;AAAA,EAC9E;AAEA,QAAM,eAAe,MAAM,OAAO,aAAa;AAC/C,SAAO,aAAa;AACtB;AAEA,eAAsB,oBACpB,OAA4B,CAAC,GACL;AACxB,MAAI,QAA6B;AACjC,MAAI,cAAoC;AAExC,QAAM,YAAY,MAAM,kBAAkB;AAC1C,QAAM,YAAY,IAAI,UAAU;AAAA,IAC9B,MAAM,UAAU,OAAe,UAAkB,UAAoB;AACnE,UAAI;AACF,YAAI,CAAC,OAAO;AACV,cAAI,CAAC,aAAa;AAChB,0BAAc,aAAa,OAAO,IAAI,EAAE,KAAK,CAAC,MAAM;AAClD,sBAAQ;AAAA,YACV,CAAC;AAAA,UACH;AACA,gBAAM;AAAA,QACR;AAEA,cAAM,QAAQ,IAAI,WAAW,KAAK;AAClC,cAAM,SAAS,MAAO,KAAK,KAAK;AAEhC,YAAI,OAAO,SAAS,GAAG;AACrB,eAAK,KAAK,OAAO,KAAK,MAAM,CAAC;AAAA,QAC/B;AAEA,iBAAS;AAAA,MACX,SAAS,KAAK;AACZ,iBAAS,GAAG;AAAA,MACd;AAAA,IACF;AAAA,IAEA,MAAM,MAAM,UAAoB;AAC9B,UAAI;AACF,YAAI,OAAO;AACT,gBAAM,SAAS,MAAM,OAAO;AAC5B,cAAI,OAAO,SAAS,GAAG;AACrB,iBAAK,KAAK,OAAO,KAAK,MAAM,CAAC;AAAA,UAC/B;AAEA,cAAI,KAAK,SAAS;AAChB,kBAAM,QAAQ,MAAM,MAAM;AAC1B,oBAAQ,IAAI,sCAAsC,KAAK;AAAA,UACzD;AAAA,QACF;AACA,iBAAS;AAAA,MACX,SAAS,KAAK;AACZ,iBAAS,GAAG;AAAA,MACd;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO;AACT;","names":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/smoke-test.ts"],"sourcesContent":["import { ConvertBuddy, convert, convertToString } from \"./index.js\";\r\nimport { createNodeTransform } from \"../src/node.js\";\r\n\r\n\r\nasync function testBasicConversion() {\r\n console.log(\"\\n=== Test: Basic CSV to NDJSON ===\");\r\n \r\n const csvData = `name,age,city\r\nAlice,30,New York\r\nBob,25,London\r\nCharlie,35,Tokyo`;\r\n\r\n const result = await convertToString(csvData, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n profile: true,\r\n });\r\n\r\n console.log(\"Input CSV:\");\r\n console.log(csvData);\r\n console.log(\"\\nOutput NDJSON:\");\r\n console.log(result);\r\n\r\n // Validate output\r\n const lines = result.trim().split(\"\\n\");\r\n if (lines.length !== 3) {\r\n throw new Error(`Expected 3 lines, got ${lines.length}`);\r\n }\r\n\r\n const firstRecord = JSON.parse(lines[0]);\r\n if (firstRecord.name !== \"Alice\") {\r\n throw new Error(`Expected name=Alice, got ${firstRecord.name}`);\r\n }\r\n\r\n console.log(\"✓ Basic conversion test passed\");\r\n}\r\n\r\nasync function testStreamingAPI() {\r\n console.log(\"\\n=== Test: Streaming API ===\");\r\n\r\n const buddy = await ConvertBuddy.create({\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n });\r\n\r\n // Push data in chunks\r\n const header = new TextEncoder().encode(\"name,age\\n\");\r\n const row1 = new TextEncoder().encode(\"Alice,30\\n\");\r\n const row2 = new TextEncoder().encode(\"Bob,25\\n\");\r\n\r\n const out1 = buddy.push(header);\r\n const out2 = buddy.push(row1);\r\n const out3 = buddy.push(row2);\r\n const final = buddy.finish();\r\n\r\n const combined = new Uint8Array(\r\n out1.length + out2.length + out3.length + final.length\r\n );\r\n combined.set(out1, 0);\r\n combined.set(out2, out1.length);\r\n combined.set(out3, out1.length + out2.length);\r\n combined.set(final, out1.length + out2.length + out3.length);\r\n\r\n const result = new TextDecoder().decode(combined);\r\n console.log(\"Streaming result:\");\r\n console.log(result);\r\n\r\n console.log(\"✓ Streaming API test passed\");\r\n}\r\n\r\nasync function testNdjsonToJson() {\r\n console.log(\"\\n=== Test: NDJSON to JSON Array ===\");\r\n\r\n const ndjsonData = `{\"name\":\"Alice\",\"age\":30}\r\n{\"name\":\"Bob\",\"age\":25}\r\n{\"name\":\"Charlie\",\"age\":35}`;\r\n\r\n const result = await convertToString(ndjsonData, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"json\",\r\n debug: false,\r\n });\r\n\r\n console.log(\"Input NDJSON:\");\r\n console.log(ndjsonData);\r\n console.log(\"\\nOutput JSON:\");\r\n console.log(result);\r\n\r\n // Validate it's valid JSON array\r\n const parsed = JSON.parse(result);\r\n if (!Array.isArray(parsed)) {\r\n throw new Error(\"Expected JSON array\");\r\n }\r\n if (parsed.length !== 3) {\r\n throw new Error(`Expected 3 items, got ${parsed.length}`);\r\n }\r\n\r\n console.log(\"✓ NDJSON to JSON test passed\");\r\n}\r\n\r\nasync function testQuotedCsv() {\r\n console.log(\"\\n=== Test: Quoted CSV with Commas ===\");\r\n\r\n const csvData = `name,description\r\n\"Alice\",\"Hello, World\"\r\n\"Bob\",\"Test, with, commas\"\r\n\"Charlie\",\"Normal text\"`;\r\n\r\n const result = await convertToString(csvData, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n });\r\n\r\n console.log(\"Input CSV:\");\r\n console.log(csvData);\r\n console.log(\"\\nOutput NDJSON:\");\r\n console.log(result);\r\n\r\n const lines = result.trim().split(\"\\n\");\r\n const firstRecord = JSON.parse(lines[0]);\r\n \r\n if (!firstRecord.description.includes(\",\")) {\r\n throw new Error(\"Expected comma in description to be preserved\");\r\n }\r\n\r\n console.log(\"✓ Quoted CSV test passed\");\r\n}\r\n\r\nasync function testStats() {\r\n console.log(\"\\n=== Test: Performance Stats ===\");\r\n\r\n const csvData = `name,age\r\nAlice,30\r\nBob,25\r\nCharlie,35`;\r\n\r\n const result = await convert(new TextEncoder().encode(csvData), {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n });\r\n\r\n // Stats are logged automatically with profile: true\r\n console.log(\"✓ Stats test passed\");\r\n}\r\n\r\nasync function testPartialLines() {\r\n console.log(\"\\n=== Test: Partial Line Handling ===\");\r\n\r\n const buddy = await ConvertBuddy.create({\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n });\r\n\r\n // Split a line across chunks\r\n const chunk1 = new TextEncoder().encode(\"name,age\\nAli\");\r\n const chunk2 = new TextEncoder().encode(\"ce,30\\nBob,25\\n\");\r\n\r\n const out1 = buddy.push(chunk1);\r\n const out2 = buddy.push(chunk2);\r\n const final = buddy.finish();\r\n\r\n const combined = new Uint8Array(out1.length + out2.length + final.length);\r\n combined.set(out1, 0);\r\n combined.set(out2, out1.length);\r\n combined.set(final, out1.length + out2.length);\r\n\r\n const result = new TextDecoder().decode(combined);\r\n console.log(\"Result with partial lines:\");\r\n console.log(result);\r\n\r\n const lines = result.trim().split(\"\\n\");\r\n if (lines.length !== 2) {\r\n throw new Error(`Expected 2 lines, got ${lines.length}`);\r\n }\r\n\r\n const firstRecord = JSON.parse(lines[0]);\r\n if (firstRecord.name !== \"Alice\") {\r\n throw new Error(`Expected name=Alice, got ${firstRecord.name}`);\r\n }\r\n\r\n console.log(\"✓ Partial line test passed\");\r\n}\r\n\r\nasync function testLargeDataset() {\r\n console.log(\"\\n=== Test: Large Dataset Performance ===\");\r\n\r\n // Generate large CSV\r\n const rows = 10000;\r\n let csv = \"id,name,value\\n\";\r\n for (let i = 0; i < rows; i++) {\r\n csv += `${i},name_${i},${i * 100}\\n`;\r\n }\r\n\r\n const start = performance.now();\r\n const result = await convert(new TextEncoder().encode(csv), {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n });\r\n const end = performance.now();\r\n\r\n const lines = new TextDecoder().decode(result).trim().split(\"\\n\");\r\n \r\n console.log(`Processed ${rows} rows in ${(end - start).toFixed(2)}ms`);\r\n console.log(`Throughput: ${((csv.length / (1024 * 1024)) / ((end - start) / 1000)).toFixed(2)} MB/s`);\r\n console.log(`Output lines: ${lines.length}`);\r\n\r\n if (lines.length !== rows) {\r\n throw new Error(`Expected ${rows} lines, got ${lines.length}`);\r\n }\r\n\r\n console.log(\"✓ Large dataset test passed\");\r\n}\r\n\r\nasync function testEmptyAndWhitespace() {\r\n console.log(\"\\n=== Test: Empty Lines and Whitespace ===\");\r\n\r\n const csvData = `name,age\r\n\r\nAlice,30\r\n\r\nBob,25\r\n`;\r\n\r\n const result = await convertToString(csvData, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n });\r\n\r\n console.log(\"Input CSV (with empty lines):\");\r\n console.log(JSON.stringify(csvData));\r\n console.log(\"\\nOutput NDJSON:\");\r\n console.log(result);\r\n\r\n const lines = result.trim().split(\"\\n\").filter(l => l.length > 0);\r\n if (lines.length !== 2) {\r\n throw new Error(`Expected 2 non-empty lines, got ${lines.length}`);\r\n }\r\n\r\n console.log(\"✓ Empty lines test passed\");\r\n}\r\n\r\n// Run all tests\r\nasync function runAllTests() {\r\n console.log(\"╔════════════════════════════════════════╗\");\r\n console.log(\"║ Convert Buddy - Smoke Test Suite ║\");\r\n console.log(\"╚════════════════════════════════════════╝\");\r\n\r\n try {\r\n await testBasicConversion();\r\n await testStreamingAPI();\r\n await testNdjsonToJson();\r\n await testQuotedCsv();\r\n await testStats();\r\n await testPartialLines();\r\n await testLargeDataset();\r\n await testEmptyAndWhitespace();\r\n\r\n console.log(\"\\n╔════════════════════════════════════════╗\");\r\n console.log(\"║ ✓ All tests passed successfully! ║\");\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n } catch (error) {\r\n console.error(\"\\n✗ Test failed:\", error);\r\n process.exit(1);\r\n }\r\n}\r\n\r\nrunAllTests();\r\n"],"mappings":"AAAA,SAAS,cAAc,SAAS,uBAAuB;AAIvD,eAAe,sBAAsB;AACnC,UAAQ,IAAI,qCAAqC;AAEjD,QAAM,UAAU;AAAA;AAAA;AAAA;AAKhB,QAAM,SAAS,MAAM,gBAAgB,SAAS;AAAA,IAC5C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,IACP,SAAS;AAAA,EACX,CAAC;AAED,UAAQ,IAAI,YAAY;AACxB,UAAQ,IAAI,OAAO;AACnB,UAAQ,IAAI,kBAAkB;AAC9B,UAAQ,IAAI,MAAM;AAGlB,QAAM,QAAQ,OAAO,KAAK,EAAE,MAAM,IAAI;AACtC,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,IAAI,MAAM,yBAAyB,MAAM,MAAM,EAAE;AAAA,EACzD;AAEA,QAAM,cAAc,KAAK,MAAM,MAAM,CAAC,CAAC;AACvC,MAAI,YAAY,SAAS,SAAS;AAChC,UAAM,IAAI,MAAM,4BAA4B,YAAY,IAAI,EAAE;AAAA,EAChE;AAEA,UAAQ,IAAI,qCAAgC;AAC9C;AAEA,eAAe,mBAAmB;AAChC,UAAQ,IAAI,+BAA+B;AAE3C,QAAM,QAAQ,MAAM,aAAa,OAAO;AAAA,IACtC,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAGD,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,YAAY;AACpD,QAAM,OAAO,IAAI,YAAY,EAAE,OAAO,YAAY;AAClD,QAAM,OAAO,IAAI,YAAY,EAAE,OAAO,UAAU;AAEhD,QAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,QAAM,OAAO,MAAM,KAAK,IAAI;AAC5B,QAAM,OAAO,MAAM,KAAK,IAAI;AAC5B,QAAM,QAAQ,MAAM,OAAO;AAE3B,QAAM,WAAW,IAAI;AAAA,IACnB,KAAK,SAAS,KAAK,SAAS,KAAK,SAAS,MAAM;AAAA,EAClD;AACA,WAAS,IAAI,MAAM,CAAC;AACpB,WAAS,IAAI,MAAM,KAAK,MAAM;AAC9B,WAAS,IAAI,MAAM,KAAK,SAAS,KAAK,MAAM;AAC5C,WAAS,IAAI,OAAO,KAAK,SAAS,KAAK,SAAS,KAAK,MAAM;AAE3D,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,QAAQ;AAChD,UAAQ,IAAI,mBAAmB;AAC/B,UAAQ,IAAI,MAAM;AAElB,UAAQ,IAAI,kCAA6B;AAC3C;AAEA,eAAe,mBAAmB;AAChC,UAAQ,IAAI,sCAAsC;AAElD,QAAM,aAAa;AAAA;AAAA;AAInB,QAAM,SAAS,MAAM,gBAAgB,YAAY;AAAA,IAC/C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAED,UAAQ,IAAI,eAAe;AAC3B,UAAQ,IAAI,UAAU;AACtB,UAAQ,IAAI,gBAAgB;AAC5B,UAAQ,IAAI,MAAM;AAGlB,QAAM,SAAS,KAAK,MAAM,MAAM;AAChC,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,UAAM,IAAI,MAAM,qBAAqB;AAAA,EACvC;AACA,MAAI,OAAO,WAAW,GAAG;AACvB,UAAM,IAAI,MAAM,yBAAyB,OAAO,MAAM,EAAE;AAAA,EAC1D;AAEA,UAAQ,IAAI,mCAA8B;AAC5C;AAEA,eAAe,gBAAgB;AAC7B,UAAQ,IAAI,wCAAwC;AAEpD,QAAM,UAAU;AAAA;AAAA;AAAA;AAKhB,QAAM,SAAS,MAAM,gBAAgB,SAAS;AAAA,IAC5C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAED,UAAQ,IAAI,YAAY;AACxB,UAAQ,IAAI,OAAO;AACnB,UAAQ,IAAI,kBAAkB;AAC9B,UAAQ,IAAI,MAAM;AAElB,QAAM,QAAQ,OAAO,KAAK,EAAE,MAAM,IAAI;AACtC,QAAM,cAAc,KAAK,MAAM,MAAM,CAAC,CAAC;AAEvC,MAAI,CAAC,YAAY,YAAY,SAAS,GAAG,GAAG;AAC1C,UAAM,IAAI,MAAM,+CAA+C;AAAA,EACjE;AAEA,UAAQ,IAAI,+BAA0B;AACxC;AAEA,eAAe,YAAY;AACzB,UAAQ,IAAI,mCAAmC;AAE/C,QAAM,UAAU;AAAA;AAAA;AAAA;AAKhB,QAAM,SAAS,MAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,OAAO,GAAG;AAAA,IAC9D,aAAa;AAAA,IACb,cAAc;AAAA,IACd,SAAS;AAAA,EACX,CAAC;AAGD,UAAQ,IAAI,0BAAqB;AACnC;AAEA,eAAe,mBAAmB;AAChC,UAAQ,IAAI,uCAAuC;AAEnD,QAAM,QAAQ,MAAM,aAAa,OAAO;AAAA,IACtC,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAGD,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,eAAe;AACvD,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,iBAAiB;AAEzD,QAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,QAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,QAAM,QAAQ,MAAM,OAAO;AAE3B,QAAM,WAAW,IAAI,WAAW,KAAK,SAAS,KAAK,SAAS,MAAM,MAAM;AACxE,WAAS,IAAI,MAAM,CAAC;AACpB,WAAS,IAAI,MAAM,KAAK,MAAM;AAC9B,WAAS,IAAI,OAAO,KAAK,SAAS,KAAK,MAAM;AAE7C,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,QAAQ;AAChD,UAAQ,IAAI,4BAA4B;AACxC,UAAQ,IAAI,MAAM;AAElB,QAAM,QAAQ,OAAO,KAAK,EAAE,MAAM,IAAI;AACtC,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,IAAI,MAAM,yBAAyB,MAAM,MAAM,EAAE;AAAA,EACzD;AAEA,QAAM,cAAc,KAAK,MAAM,MAAM,CAAC,CAAC;AACvC,MAAI,YAAY,SAAS,SAAS;AAChC,UAAM,IAAI,MAAM,4BAA4B,YAAY,IAAI,EAAE;AAAA,EAChE;AAEA,UAAQ,IAAI,iCAA4B;AAC1C;AAEA,eAAe,mBAAmB;AAChC,UAAQ,IAAI,2CAA2C;AAGvD,QAAM,OAAO;AACb,MAAI,MAAM;AACV,WAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,WAAO,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG;AAAA;AAAA,EAClC;AAEA,QAAM,QAAQ,YAAY,IAAI;AAC9B,QAAM,SAAS,MAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,GAAG,GAAG;AAAA,IAC1D,aAAa;AAAA,IACb,cAAc;AAAA,IACd,SAAS;AAAA,EACX,CAAC;AACD,QAAM,MAAM,YAAY,IAAI;AAE5B,QAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI;AAEhE,UAAQ,IAAI,aAAa,IAAI,aAAa,MAAM,OAAO,QAAQ,CAAC,CAAC,IAAI;AACrE,UAAQ,IAAI,gBAAiB,IAAI,UAAU,OAAO,UAAW,MAAM,SAAS,MAAO,QAAQ,CAAC,CAAC,OAAO;AACpG,UAAQ,IAAI,iBAAiB,MAAM,MAAM,EAAE;AAE3C,MAAI,MAAM,WAAW,MAAM;AACzB,UAAM,IAAI,MAAM,YAAY,IAAI,eAAe,MAAM,MAAM,EAAE;AAAA,EAC/D;AAEA,UAAQ,IAAI,kCAA6B;AAC3C;AAEA,eAAe,yBAAyB;AACtC,UAAQ,IAAI,4CAA4C;AAExD,QAAM,UAAU;AAAA;AAAA;AAAA;AAAA;AAAA;AAOhB,QAAM,SAAS,MAAM,gBAAgB,SAAS;AAAA,IAC5C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAED,UAAQ,IAAI,+BAA+B;AAC3C,UAAQ,IAAI,KAAK,UAAU,OAAO,CAAC;AACnC,UAAQ,IAAI,kBAAkB;AAC9B,UAAQ,IAAI,MAAM;AAElB,QAAM,QAAQ,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,OAAO,OAAK,EAAE,SAAS,CAAC;AAChE,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,IAAI,MAAM,mCAAmC,MAAM,MAAM,EAAE;AAAA,EACnE;AAEA,UAAQ,IAAI,gCAA2B;AACzC;AAGA,eAAe,cAAc;AAC3B,UAAQ,IAAI,8PAA4C;AACxD,UAAQ,IAAI,qDAA2C;AACvD,UAAQ,IAAI,8PAA4C;AAExD,MAAI;AACF,UAAM,oBAAoB;AAC1B,UAAM,iBAAiB;AACvB,UAAM,iBAAiB;AACvB,UAAM,cAAc;AACpB,UAAM,UAAU;AAChB,UAAM,iBAAiB;AACvB,UAAM,iBAAiB;AACvB,UAAM,uBAAuB;AAE7B,YAAQ,IAAI,gQAA8C;AAC1D,YAAQ,IAAI,0DAA2C;AACvD,YAAQ,IAAI,gQAA8C;AAAA,EAC5D,SAAS,OAAO;AACd,YAAQ,MAAM,yBAAoB,KAAK;AACvC,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,YAAY;","names":[]}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|