convert-buddy-js 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +2 -3
- package/dist/browser.js +0 -2
- package/dist/browser.js.map +1 -1
- package/dist/index.d.ts +458 -1
- package/dist/index.js +36 -1
- package/dist/index.js.map +1 -1
- package/dist/node.d.ts +2 -3
- package/dist/node.js +0 -2
- package/dist/node.js.map +1 -1
- package/dist/unified-stream.js +329 -0
- package/dist/unified-stream.js.map +1 -0
- package/package.json +1 -1
- package/dist/browser-stream-controller.d.ts +0 -111
- package/dist/browser-stream-controller.js +0 -229
- package/dist/browser-stream-controller.js.map +0 -1
- package/dist/index-DiYloo2I.d.ts +0 -452
- package/dist/node-stream-controller.d.ts +0 -137
- package/dist/node-stream-controller.js +0 -277
- package/dist/node-stream-controller.js.map +0 -1
- package/dist/stream-controller.d.ts +0 -1
- package/dist/stream-controller.js +0 -277
- package/dist/stream-controller.js.map +0 -1
package/dist/browser.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export {
|
|
3
|
-
export { BrowserStreamController, BrowserStreamInput, processStream } from './browser-stream-controller.js';
|
|
1
|
+
import { ConvertOptions, Format } from './index.js';
|
|
2
|
+
export { Coerce, ConvertBuddy, ConvertBuddyOptions, ConvertBuddyTransformStream, ConvertBuddy as Converter, CsvConfig, CsvDetection, DetectInput, DetectOptions, FieldMap, JsonDetection, NdjsonDetection, ProgressCallback, Stats, StreamController, StreamInput, StreamOptions, StreamResult, StructureDetection, TransformConfig, TransformMode, XmlConfig, XmlDetection, autoDetectConfig, convertAny, convertAnyToString, createStreamController, detectCsvFieldsAndDelimiter, detectFormat, detectStructure, detectXmlElements, getExtension, getMimeType, getOptimalThreadCount, getSuggestedFilename, getThreadingInfo, isWasmThreadingSupported, processStream, streamRecords } from './index.js';
|
|
4
3
|
|
|
5
4
|
/**
|
|
6
5
|
* Browser entry point for convert-buddy-js
|
package/dist/browser.js
CHANGED
package/dist/browser.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/browser.ts"],"sourcesContent":["/**\r\n * Browser entry point for convert-buddy-js\r\n * Re-exports the main API but optimized for browser environments\r\n */\r\n\r\n// Re-export all the main functionality from index\r\nexport * from \"./index.js\";\r\n\r\n// Export browser-specific
|
|
1
|
+
{"version":3,"sources":["../src/browser.ts"],"sourcesContent":["/**\r\n * Browser entry point for convert-buddy-js\r\n * Re-exports the main API but optimized for browser environments\r\n */\r\n\r\n// Re-export all the main functionality from index\r\nexport * from \"./index.js\";\r\n\r\n// Export browser-specific utilities\r\nimport {\r\n type Format,\r\n type ConvertOptions,\r\n type ConvertBuddyOptions,\r\n convertToString as coreConvertToString,\r\n convert as coreConvert,\r\n ConvertBuddy,\r\n getMimeType,\r\n getExtension,\r\n getSuggestedFilename,\r\n} from \"./index.js\";\r\n\r\n/**\r\n * Browser-optimized convert function for strings\r\n */\r\nexport async function convertToString(\r\n input: string | Uint8Array | File | Blob | ReadableStream<Uint8Array>,\r\n opts: ConvertOptions\r\n): Promise<string> {\r\n if (typeof input === \"string\" || input instanceof Uint8Array) {\r\n return coreConvertToString(input, opts);\r\n }\r\n \r\n if (typeof File !== \"undefined\" && input instanceof File) {\r\n const arrayBuffer = await input.arrayBuffer();\r\n return coreConvertToString(new Uint8Array(arrayBuffer), opts);\r\n }\r\n \r\n if (typeof Blob !== \"undefined\" && input instanceof Blob) {\r\n const arrayBuffer = await input.arrayBuffer();\r\n return coreConvertToString(new Uint8Array(arrayBuffer), opts);\r\n }\r\n \r\n if (typeof ReadableStream !== \"undefined\" && input instanceof ReadableStream) {\r\n const chunks: Uint8Array[] = [];\r\n const reader = input.getReader();\r\n \r\n try {\r\n while (true) {\r\n const { done, value } = await reader.read();\r\n if (done) break;\r\n chunks.push(value);\r\n }\r\n } finally {\r\n reader.releaseLock();\r\n }\r\n \r\n // Combine chunks\r\n const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);\r\n const combined = new Uint8Array(totalLength);\r\n let offset = 0;\r\n for (const chunk of chunks) {\r\n combined.set(chunk, offset);\r\n offset += chunk.length;\r\n }\r\n \r\n return coreConvertToString(combined, opts);\r\n }\r\n \r\n throw new Error(\"Unsupported input type for browser environment\");\r\n}\r\n\r\n/**\r\n * Browser-optimized convert function for bytes\r\n */\r\nexport async function convert(\r\n input: string | Uint8Array | File | Blob | ReadableStream<Uint8Array>,\r\n opts: ConvertOptions\r\n): Promise<Uint8Array> {\r\n if (typeof input === \"string\" || input instanceof Uint8Array) {\r\n return coreConvert(input, opts);\r\n }\r\n \r\n if (typeof File !== \"undefined\" && input instanceof File) {\r\n const arrayBuffer = await input.arrayBuffer();\r\n return coreConvert(new Uint8Array(arrayBuffer), opts);\r\n }\r\n \r\n if (typeof Blob !== \"undefined\" && input instanceof Blob) {\r\n const arrayBuffer = await input.arrayBuffer();\r\n return coreConvert(new Uint8Array(arrayBuffer), opts);\r\n }\r\n \r\n if (typeof ReadableStream !== \"undefined\" && input instanceof ReadableStream) {\r\n const chunks: Uint8Array[] = [];\r\n const reader = input.getReader();\r\n \r\n try {\r\n while (true) {\r\n const { done, value } = await reader.read();\r\n if (done) break;\r\n chunks.push(value);\r\n }\r\n } finally {\r\n reader.releaseLock();\r\n }\r\n \r\n // Combine chunks\r\n const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);\r\n const combined = new Uint8Array(totalLength);\r\n let offset = 0;\r\n for (const chunk of chunks) {\r\n combined.set(chunk, offset);\r\n offset += chunk.length;\r\n }\r\n \r\n return coreConvert(combined, opts);\r\n }\r\n \r\n throw new Error(\"Unsupported input type for browser environment\");\r\n}\r\n\r\n/**\r\n * Convert a browser File or Blob and trigger download of the result\r\n */\r\nexport async function convertFileToFile(\r\n inputFile: File | Blob,\r\n outputFilename: string,\r\n opts: ConvertOptions\r\n): Promise<void> {\r\n const result = await convert(inputFile, opts);\r\n \r\n // Create download\r\n const mimeType = getMimeType(opts.outputFormat);\r\n const blob = new Blob([result.buffer as ArrayBuffer], { type: mimeType });\r\n const url = URL.createObjectURL(blob);\r\n \r\n const link = document.createElement(\"a\");\r\n link.href = url;\r\n link.download = outputFilename;\r\n link.style.display = \"none\";\r\n \r\n document.body.appendChild(link);\r\n link.click();\r\n \r\n // Cleanup\r\n setTimeout(() => {\r\n document.body.removeChild(link);\r\n URL.revokeObjectURL(url);\r\n }, 100);\r\n}\r\n\r\n/**\r\n * Get file type configuration for browser file pickers\r\n */\r\nexport function getFileTypeConfig(format: Format): Array<{\r\n description: string;\r\n accept: Record<string, string[]>;\r\n}> {\r\n const mimeType = getMimeType(format);\r\n const extension = `.${getExtension(format)}`;\r\n\r\n return [\r\n {\r\n description: `${format.toUpperCase()} Files`,\r\n accept: { [mimeType]: [extension] },\r\n },\r\n ];\r\n}\r\n\r\n// Legacy exports for backward compatibility\r\nexport { ConvertBuddy as Converter };"],"mappings":"AAMA,cAAc;AAGd;AAAA,EAIE,mBAAmB;AAAA,EACnB,WAAW;AAAA,EACX;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAKP,eAAsB,gBACpB,OACA,MACiB;AACjB,MAAI,OAAO,UAAU,YAAY,iBAAiB,YAAY;AAC5D,WAAO,oBAAoB,OAAO,IAAI;AAAA,EACxC;AAEA,MAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AACxD,UAAM,cAAc,MAAM,MAAM,YAAY;AAC5C,WAAO,oBAAoB,IAAI,WAAW,WAAW,GAAG,IAAI;AAAA,EAC9D;AAEA,MAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AACxD,UAAM,cAAc,MAAM,MAAM,YAAY;AAC5C,WAAO,oBAAoB,IAAI,WAAW,WAAW,GAAG,IAAI;AAAA,EAC9D;AAEA,MAAI,OAAO,mBAAmB,eAAe,iBAAiB,gBAAgB;AAC5E,UAAM,SAAuB,CAAC;AAC9B,UAAM,SAAS,MAAM,UAAU;AAE/B,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,eAAO,KAAK,KAAK;AAAA,MACnB;AAAA,IACF,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAGA,UAAM,cAAc,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,MAAM,QAAQ,CAAC;AACvE,UAAM,WAAW,IAAI,WAAW,WAAW;AAC3C,QAAI,SAAS;AACb,eAAW,SAAS,QAAQ;AAC1B,eAAS,IAAI,OAAO,MAAM;AAC1B,gBAAU,MAAM;AAAA,IAClB;AAEA,WAAO,oBAAoB,UAAU,IAAI;AAAA,EAC3C;AAEA,QAAM,IAAI,MAAM,gDAAgD;AAClE;AAKA,eAAsB,QACpB,OACA,MACqB;AACrB,MAAI,OAAO,UAAU,YAAY,iBAAiB,YAAY;AAC5D,WAAO,YAAY,OAAO,IAAI;AAAA,EAChC;AAEA,MAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AACxD,UAAM,cAAc,MAAM,MAAM,YAAY;AAC5C,WAAO,YAAY,IAAI,WAAW,WAAW,GAAG,IAAI;AAAA,EACtD;AAEA,MAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AACxD,UAAM,cAAc,MAAM,MAAM,YAAY;AAC5C,WAAO,YAAY,IAAI,WAAW,WAAW,GAAG,IAAI;AAAA,EACtD;AAEA,MAAI,OAAO,mBAAmB,eAAe,iBAAiB,gBAAgB;AAC5E,UAAM,SAAuB,CAAC;AAC9B,UAAM,SAAS,MAAM,UAAU;AAE/B,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,eAAO,KAAK,KAAK;AAAA,MACnB;AAAA,IACF,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAGA,UAAM,cAAc,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,MAAM,QAAQ,CAAC;AACvE,UAAM,WAAW,IAAI,WAAW,WAAW;AAC3C,QAAI,SAAS;AACb,eAAW,SAAS,QAAQ;AAC1B,eAAS,IAAI,OAAO,MAAM;AAC1B,gBAAU,MAAM;AAAA,IAClB;AAEA,WAAO,YAAY,UAAU,IAAI;AAAA,EACnC;AAEA,QAAM,IAAI,MAAM,gDAAgD;AAClE;AAKA,eAAsB,kBACpB,WACA,gBACA,MACe;AACf,QAAM,SAAS,MAAM,QAAQ,WAAW,IAAI;AAG5C,QAAM,WAAW,YAAY,KAAK,YAAY;AAC9C,QAAM,OAAO,IAAI,KAAK,CAAC,OAAO,MAAqB,GAAG,EAAE,MAAM,SAAS,CAAC;AACxE,QAAM,MAAM,IAAI,gBAAgB,IAAI;AAEpC,QAAM,OAAO,SAAS,cAAc,GAAG;AACvC,OAAK,OAAO;AACZ,OAAK,WAAW;AAChB,OAAK,MAAM,UAAU;AAErB,WAAS,KAAK,YAAY,IAAI;AAC9B,OAAK,MAAM;AAGX,aAAW,MAAM;AACf,aAAS,KAAK,YAAY,IAAI;AAC9B,QAAI,gBAAgB,GAAG;AAAA,EACzB,GAAG,GAAG;AACR;AAKO,SAAS,kBAAkB,QAG/B;AACD,QAAM,WAAW,YAAY,MAAM;AACnC,QAAM,YAAY,IAAI,aAAa,MAAM,CAAC;AAE1C,SAAO;AAAA,IACL;AAAA,MACE,aAAa,GAAG,OAAO,YAAY,CAAC;AAAA,MACpC,QAAQ,EAAE,CAAC,QAAQ,GAAG,CAAC,SAAS,EAAE;AAAA,IACpC;AAAA,EACF;AACF;","names":[]}
|
package/dist/index.d.ts
CHANGED
|
@@ -1 +1,458 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Unified Streaming API for ConvertBuddy
|
|
3
|
+
*
|
|
4
|
+
* This is the PRODUCTION-READY isomorphic streaming interface.
|
|
5
|
+
* Works identically in browser and Node.js environments.
|
|
6
|
+
*
|
|
7
|
+
* Design Principles:
|
|
8
|
+
* 1. Batch-based processing for high performance
|
|
9
|
+
* 2. Full backpressure control (pause/resume/abort)
|
|
10
|
+
* 3. Direct stats access through controller
|
|
11
|
+
* 4. Zero-copy when possible (WASM-parsed records preferred)
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Unified input type - works in both Node.js and Browser
|
|
16
|
+
*/
|
|
17
|
+
type StreamInput = string | Uint8Array | ArrayBuffer | ReadableStream<Uint8Array> | AsyncIterable<Uint8Array> | Blob | NodeReadableCompat;
|
|
18
|
+
/**
|
|
19
|
+
* Minimal interface for Node.js Readable stream compatibility
|
|
20
|
+
* We use duck-typing to avoid importing node:stream in browser builds
|
|
21
|
+
*/
|
|
22
|
+
interface NodeReadableCompat {
|
|
23
|
+
read?: () => unknown;
|
|
24
|
+
on?: (event: string, listener: (...args: unknown[]) => void) => unknown;
|
|
25
|
+
[Symbol.asyncIterator]?: () => AsyncIterator<unknown>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Options for stream processing
|
|
29
|
+
*/
|
|
30
|
+
interface StreamOptions extends Omit<ConvertBuddyOptions, 'onProgress'> {
|
|
31
|
+
/**
|
|
32
|
+
* Called with batches of parsed records. If async, backpressure is automatically applied.
|
|
33
|
+
* Return `false` to pause processing (call controller.resume() to continue).
|
|
34
|
+
*
|
|
35
|
+
* @param records - Array of records (batch size controlled by recordBatchSize)
|
|
36
|
+
* @param stats - Current conversion statistics
|
|
37
|
+
*/
|
|
38
|
+
onRecords?: (records: unknown[], stats: Stats) => void | boolean | Promise<void | boolean>;
|
|
39
|
+
/**
|
|
40
|
+
* Called with raw output bytes before parsing into records.
|
|
41
|
+
* Useful for writing directly to a file/stream while also processing records.
|
|
42
|
+
*/
|
|
43
|
+
onData?: (bytes: Uint8Array) => void | Promise<void>;
|
|
44
|
+
/**
|
|
45
|
+
* Called periodically with progress stats.
|
|
46
|
+
* Frequency controlled by `progressIntervalRecords`.
|
|
47
|
+
*/
|
|
48
|
+
onProgress?: (stats: Stats, recordCount: number) => void;
|
|
49
|
+
/**
|
|
50
|
+
* Trigger onProgress callback every N records. Default: 1000
|
|
51
|
+
*/
|
|
52
|
+
progressIntervalRecords?: number;
|
|
53
|
+
/**
|
|
54
|
+
* Number of records per batch when using onRecords callback or iterator.
|
|
55
|
+
* Lower = more responsive, Higher = better throughput
|
|
56
|
+
* Default: 100
|
|
57
|
+
*/
|
|
58
|
+
recordBatchSize?: number;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Stream controller returned by ConvertBuddy.stream()
|
|
62
|
+
* Provides pause/resume/abort controls and stats access
|
|
63
|
+
*/
|
|
64
|
+
interface StreamController {
|
|
65
|
+
/** Pause the stream processing */
|
|
66
|
+
pause(): void;
|
|
67
|
+
/** Resume the stream processing */
|
|
68
|
+
resume(): void;
|
|
69
|
+
/** Abort the stream processing */
|
|
70
|
+
abort(): void;
|
|
71
|
+
/** Get current statistics */
|
|
72
|
+
stats(): Stats | null;
|
|
73
|
+
/** Get current record count */
|
|
74
|
+
recordCount: number;
|
|
75
|
+
/** Check if currently paused */
|
|
76
|
+
isPaused: boolean;
|
|
77
|
+
/** Check if aborted */
|
|
78
|
+
isAborted: boolean;
|
|
79
|
+
/** Async iterator over record batches */
|
|
80
|
+
[Symbol.asyncIterator](): AsyncIterator<unknown[]>;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Result from processing a stream
|
|
84
|
+
*/
|
|
85
|
+
interface StreamResult {
|
|
86
|
+
stats: Stats;
|
|
87
|
+
recordCount: number;
|
|
88
|
+
aborted: boolean;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Process a stream with callbacks
|
|
92
|
+
* Returns controller for pause/resume/stats access
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
* ```ts
|
|
96
|
+
* const result = await processStream(file, {
|
|
97
|
+
* inputFormat: 'csv',
|
|
98
|
+
* outputFormat: 'json',
|
|
99
|
+
* onRecords: (records, stats) => {
|
|
100
|
+
* console.log(`Batch of ${records.length} records`);
|
|
101
|
+
* console.log(`Throughput: ${stats.throughputMbPerSec} MB/s`);
|
|
102
|
+
* },
|
|
103
|
+
* });
|
|
104
|
+
* console.log('Total records:', result.recordCount);
|
|
105
|
+
* ```
|
|
106
|
+
*/
|
|
107
|
+
declare function processStream(input: StreamInput, options: StreamOptions): Promise<StreamResult>;
|
|
108
|
+
/**
|
|
109
|
+
* Create a stream controller for manual control
|
|
110
|
+
*
|
|
111
|
+
* @example
|
|
112
|
+
* ```ts
|
|
113
|
+
* const controller = createStreamController(file, {
|
|
114
|
+
* inputFormat: 'csv',
|
|
115
|
+
* outputFormat: 'json',
|
|
116
|
+
* });
|
|
117
|
+
*
|
|
118
|
+
* // Process with full control
|
|
119
|
+
* for await (const batch of controller) {
|
|
120
|
+
* console.log(`Processing ${batch.length} records`);
|
|
121
|
+
* console.log('Stats:', controller.stats());
|
|
122
|
+
*
|
|
123
|
+
* if (needsSlowdown) {
|
|
124
|
+
* controller.pause();
|
|
125
|
+
* await sleep(1000);
|
|
126
|
+
* controller.resume();
|
|
127
|
+
* }
|
|
128
|
+
* }
|
|
129
|
+
* ```
|
|
130
|
+
*/
|
|
131
|
+
declare function createStreamController(input: StreamInput, options: StreamOptions): StreamController;
|
|
132
|
+
/**
|
|
133
|
+
* Helper: Create an async iterator over record batches
|
|
134
|
+
*
|
|
135
|
+
* @example
|
|
136
|
+
* ```ts
|
|
137
|
+
* for await (const batch of streamRecords(file, {
|
|
138
|
+
* inputFormat: 'csv',
|
|
139
|
+
* outputFormat: 'json',
|
|
140
|
+
* recordBatchSize: 50, // 50 records per batch
|
|
141
|
+
* })) {
|
|
142
|
+
* await Promise.all(batch.map(r => saveRecord(r)));
|
|
143
|
+
* }
|
|
144
|
+
* ```
|
|
145
|
+
*/
|
|
146
|
+
declare function streamRecords(input: StreamInput, options: StreamOptions): AsyncIterableIterator<unknown[]>;
|
|
147
|
+
|
|
148
|
+
type Format = "csv" | "ndjson" | "json" | "xml";
|
|
149
|
+
type DetectInput = Uint8Array | ArrayBuffer | string | ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
|
|
150
|
+
type CsvDetection = {
|
|
151
|
+
delimiter: string;
|
|
152
|
+
fields: string[];
|
|
153
|
+
};
|
|
154
|
+
type XmlDetection = {
|
|
155
|
+
elements: string[];
|
|
156
|
+
recordElement?: string;
|
|
157
|
+
};
|
|
158
|
+
type JsonDetection = {
|
|
159
|
+
fields: string[];
|
|
160
|
+
};
|
|
161
|
+
type NdjsonDetection = {
|
|
162
|
+
fields: string[];
|
|
163
|
+
};
|
|
164
|
+
type StructureDetection = {
|
|
165
|
+
format: Format;
|
|
166
|
+
fields: string[];
|
|
167
|
+
delimiter?: string;
|
|
168
|
+
recordElement?: string;
|
|
169
|
+
};
|
|
170
|
+
type DetectOptions = {
|
|
171
|
+
maxBytes?: number;
|
|
172
|
+
debug?: boolean;
|
|
173
|
+
};
|
|
174
|
+
type ProgressCallback = (stats: Stats) => void;
|
|
175
|
+
type ConvertBuddyOptions = {
|
|
176
|
+
debug?: boolean;
|
|
177
|
+
profile?: boolean;
|
|
178
|
+
inputFormat?: Format | "auto";
|
|
179
|
+
outputFormat?: Format;
|
|
180
|
+
chunkTargetBytes?: number;
|
|
181
|
+
parallelism?: number;
|
|
182
|
+
maxMemoryMB?: number;
|
|
183
|
+
csvConfig?: CsvConfig;
|
|
184
|
+
xmlConfig?: XmlConfig;
|
|
185
|
+
transform?: TransformConfig;
|
|
186
|
+
onProgress?: ProgressCallback;
|
|
187
|
+
progressIntervalBytes?: number;
|
|
188
|
+
};
|
|
189
|
+
type ConvertOptions = {
|
|
190
|
+
inputFormat?: Format | "auto";
|
|
191
|
+
outputFormat: Format;
|
|
192
|
+
csvConfig?: CsvConfig;
|
|
193
|
+
xmlConfig?: XmlConfig;
|
|
194
|
+
transform?: TransformConfig;
|
|
195
|
+
onProgress?: ProgressCallback;
|
|
196
|
+
};
|
|
197
|
+
type CsvConfig = {
|
|
198
|
+
delimiter?: string;
|
|
199
|
+
quote?: string;
|
|
200
|
+
hasHeaders?: boolean;
|
|
201
|
+
trimWhitespace?: boolean;
|
|
202
|
+
};
|
|
203
|
+
type XmlConfig = {
|
|
204
|
+
recordElement?: string;
|
|
205
|
+
trimText?: boolean;
|
|
206
|
+
includeAttributes?: boolean;
|
|
207
|
+
expandEntities?: boolean;
|
|
208
|
+
};
|
|
209
|
+
type TransformMode = "replace" | "augment";
|
|
210
|
+
type Coerce = {
|
|
211
|
+
type: "string";
|
|
212
|
+
} | {
|
|
213
|
+
type: "i64";
|
|
214
|
+
} | {
|
|
215
|
+
type: "f64";
|
|
216
|
+
} | {
|
|
217
|
+
type: "bool";
|
|
218
|
+
} | {
|
|
219
|
+
type: "timestamp_ms";
|
|
220
|
+
format?: "iso8601" | "unix_ms" | "unix_s";
|
|
221
|
+
};
|
|
222
|
+
type FieldMap = {
|
|
223
|
+
targetFieldName: string;
|
|
224
|
+
originFieldName?: string;
|
|
225
|
+
required?: boolean;
|
|
226
|
+
defaultValue?: string | number | boolean | null;
|
|
227
|
+
coerce?: Coerce;
|
|
228
|
+
compute?: string;
|
|
229
|
+
};
|
|
230
|
+
type TransformConfig = {
|
|
231
|
+
mode?: TransformMode;
|
|
232
|
+
fields: FieldMap[];
|
|
233
|
+
onMissingField?: "error" | "null" | "drop";
|
|
234
|
+
onMissingRequired?: "error" | "abort";
|
|
235
|
+
onCoerceError?: "error" | "null" | "dropRecord";
|
|
236
|
+
};
|
|
237
|
+
type Stats = {
|
|
238
|
+
bytesIn: number;
|
|
239
|
+
bytesOut: number;
|
|
240
|
+
chunksIn: number;
|
|
241
|
+
recordsProcessed: number;
|
|
242
|
+
parseTimeMs: number;
|
|
243
|
+
transformTimeMs: number;
|
|
244
|
+
writeTimeMs: number;
|
|
245
|
+
maxBufferSize: number;
|
|
246
|
+
currentPartialSize: number;
|
|
247
|
+
throughputMbPerSec: number;
|
|
248
|
+
};
|
|
249
|
+
declare class ConvertBuddy {
|
|
250
|
+
private converter;
|
|
251
|
+
private debug;
|
|
252
|
+
private profile;
|
|
253
|
+
private aborted;
|
|
254
|
+
private paused;
|
|
255
|
+
private onProgress?;
|
|
256
|
+
private progressIntervalBytes;
|
|
257
|
+
private lastProgressBytes;
|
|
258
|
+
private globalConfig;
|
|
259
|
+
private initialized;
|
|
260
|
+
simd: boolean;
|
|
261
|
+
/**
|
|
262
|
+
* Create a new ConvertBuddy instance with global configuration.
|
|
263
|
+
* This is useful when you want to set memory limits, debug mode, or other global settings.
|
|
264
|
+
*
|
|
265
|
+
* @example
|
|
266
|
+
* const buddy = new ConvertBuddy({ maxMemoryMB: 512, debug: true });
|
|
267
|
+
* const result = await buddy.convert(input, { outputFormat: "json" });
|
|
268
|
+
*/
|
|
269
|
+
constructor(opts?: ConvertBuddyOptions);
|
|
270
|
+
/**
|
|
271
|
+
* Convert input (string, Buffer, File, URL, etc.) to the desired output format.
|
|
272
|
+
* This is the main method for the new simplified API.
|
|
273
|
+
*
|
|
274
|
+
* @example
|
|
275
|
+
* // Auto-detect everything
|
|
276
|
+
* const buddy = new ConvertBuddy();
|
|
277
|
+
* const result = await buddy.convert("https://example.com/data.csv", { outputFormat: "json" });
|
|
278
|
+
*
|
|
279
|
+
* @example
|
|
280
|
+
* // With configuration
|
|
281
|
+
* const buddy = new ConvertBuddy({ maxMemoryMB: 512 });
|
|
282
|
+
* const result = await buddy.convert(file, { inputFormat: "csv", outputFormat: "json" });
|
|
283
|
+
*/
|
|
284
|
+
convert(input: string | Uint8Array | File | Blob | ReadableStream<Uint8Array>, opts: ConvertOptions): Promise<Uint8Array>;
|
|
285
|
+
/**
|
|
286
|
+
* Stream conversion with full control (pause/resume/abort/stats).
|
|
287
|
+
* Returns a controller that implements AsyncIterator for batch processing.
|
|
288
|
+
*
|
|
289
|
+
* @example
|
|
290
|
+
* // Process in batches
|
|
291
|
+
* const controller = cb.stream(file, { outputFormat: 'json', recordBatchSize: 100 });
|
|
292
|
+
* for await (const batch of controller) {
|
|
293
|
+
* await processBatch(batch);
|
|
294
|
+
* console.log('Stats:', controller.stats());
|
|
295
|
+
* }
|
|
296
|
+
*
|
|
297
|
+
* @example
|
|
298
|
+
* // With callbacks
|
|
299
|
+
* const controller = cb.stream(file, {
|
|
300
|
+
* outputFormat: 'json',
|
|
301
|
+
* onRecords: (records, stats) => {
|
|
302
|
+
* console.log(`Got ${records.length} records`);
|
|
303
|
+
* }
|
|
304
|
+
* });
|
|
305
|
+
*/
|
|
306
|
+
stream(input: string | Uint8Array | File | Blob | ReadableStream<Uint8Array>, opts?: ConvertOptions & {
|
|
307
|
+
onRecords?: (records: unknown[], stats: Stats) => void | boolean | Promise<void | boolean>;
|
|
308
|
+
onData?: (bytes: Uint8Array) => void | Promise<void>;
|
|
309
|
+
onProgress?: (stats: Stats, recordCount: number) => void;
|
|
310
|
+
progressIntervalRecords?: number;
|
|
311
|
+
recordBatchSize?: number;
|
|
312
|
+
}): StreamController;
|
|
313
|
+
private convertFromUrl;
|
|
314
|
+
private convertFromString;
|
|
315
|
+
private convertFromBuffer;
|
|
316
|
+
private convertFromBufferParallel;
|
|
317
|
+
private convertUsingNodejsThreadPool;
|
|
318
|
+
private convertUsingWasmThreadPool;
|
|
319
|
+
private convertUsingJsParallelism;
|
|
320
|
+
private splitIntoChunks;
|
|
321
|
+
private convertFromFile;
|
|
322
|
+
private convertFromBlob;
|
|
323
|
+
private convertFromStream;
|
|
324
|
+
/**
|
|
325
|
+
* Legacy create method for backward compatibility.
|
|
326
|
+
* Prefer using the constructor: new ConvertBuddy(opts)
|
|
327
|
+
*/
|
|
328
|
+
static create(opts?: ConvertBuddyOptions): Promise<ConvertBuddy>;
|
|
329
|
+
push(chunk: Uint8Array): Uint8Array;
|
|
330
|
+
/**
|
|
331
|
+
* Push a chunk and optionally get pre-parsed records.
|
|
332
|
+
* This is more efficient than parsing the output yourself.
|
|
333
|
+
*
|
|
334
|
+
* @param chunk - Input chunk to process
|
|
335
|
+
* @param includeRecords - If true, returns { output, records }, otherwise just output bytes
|
|
336
|
+
* @returns Either Uint8Array or { output: Uint8Array, records: any[] }
|
|
337
|
+
*/
|
|
338
|
+
pushWithRecords(chunk: Uint8Array, includeRecords?: boolean): Uint8Array | {
|
|
339
|
+
output: Uint8Array;
|
|
340
|
+
records: any[];
|
|
341
|
+
};
|
|
342
|
+
finish(): Uint8Array;
|
|
343
|
+
stats(): Stats;
|
|
344
|
+
abort(): void;
|
|
345
|
+
pause(): void;
|
|
346
|
+
resume(): void;
|
|
347
|
+
isAborted(): boolean;
|
|
348
|
+
isPaused(): boolean;
|
|
349
|
+
}
|
|
350
|
+
declare function detectFormat(input: DetectInput, opts?: DetectOptions): Promise<Format | "unknown">;
|
|
351
|
+
declare function detectStructure(input: DetectInput, formatHint?: Format, opts?: DetectOptions): Promise<StructureDetection | null>;
|
|
352
|
+
declare function detectCsvFieldsAndDelimiter(input: DetectInput, opts?: DetectOptions): Promise<CsvDetection | null>;
|
|
353
|
+
declare function detectXmlElements(input: DetectInput, opts?: DetectOptions): Promise<XmlDetection | null>;
|
|
354
|
+
declare function autoDetectConfig(input: DetectInput, opts?: DetectOptions): Promise<{
|
|
355
|
+
format: Format | "unknown";
|
|
356
|
+
csvConfig?: CsvConfig;
|
|
357
|
+
xmlConfig?: XmlConfig;
|
|
358
|
+
}>;
|
|
359
|
+
declare class ConvertBuddyTransformStream extends TransformStream<Uint8Array, Uint8Array> {
|
|
360
|
+
constructor(opts?: ConvertBuddyOptions);
|
|
361
|
+
}
|
|
362
|
+
declare function convert(input: Uint8Array | string, opts?: ConvertBuddyOptions): Promise<Uint8Array>;
|
|
363
|
+
declare function convertToString(input: Uint8Array | string, opts?: ConvertBuddyOptions): Promise<string>;
|
|
364
|
+
/**
|
|
365
|
+
* Ultra-simple standalone convert function with auto-detection.
|
|
366
|
+
* Accepts any input type (URL, File, Buffer, string, stream) and automatically detects format.
|
|
367
|
+
*
|
|
368
|
+
* @example
|
|
369
|
+
* // From URL
|
|
370
|
+
* import { convertAny } from "convert-buddy-js";
|
|
371
|
+
* const result = await convertAny("https://example.com/data.csv", { outputFormat: "json" });
|
|
372
|
+
*
|
|
373
|
+
* @example
|
|
374
|
+
* // From File (browser)
|
|
375
|
+
* const file = fileInput.files[0];
|
|
376
|
+
* const result = await convertAny(file, { outputFormat: "ndjson" });
|
|
377
|
+
*
|
|
378
|
+
* @example
|
|
379
|
+
* // From string data
|
|
380
|
+
* const result = await convertAny('{"name":"Ada"}', { outputFormat: "csv" });
|
|
381
|
+
*/
|
|
382
|
+
declare function convertAny(input: string | Uint8Array | File | Blob | ReadableStream<Uint8Array>, opts: ConvertOptions): Promise<Uint8Array>;
|
|
383
|
+
/**
|
|
384
|
+
* Ultra-simple standalone convert function that returns a string.
|
|
385
|
+
* Same as convertAny but decodes the output to a string.
|
|
386
|
+
*
|
|
387
|
+
* @example
|
|
388
|
+
* import { convertAnyToString } from "convert-buddy-js";
|
|
389
|
+
* const json = await convertAnyToString("https://example.com/data.csv", { outputFormat: "json" });
|
|
390
|
+
* console.log(JSON.parse(json));
|
|
391
|
+
*/
|
|
392
|
+
declare function convertAnyToString(input: string | Uint8Array | File | Blob | ReadableStream<Uint8Array>, opts: ConvertOptions): Promise<string>;
|
|
393
|
+
/**
|
|
394
|
+
* Get MIME type for a given format
|
|
395
|
+
*
|
|
396
|
+
* @example
|
|
397
|
+
* const mimeType = getMimeType("json"); // "application/json"
|
|
398
|
+
*/
|
|
399
|
+
declare function getMimeType(format: Format): string;
|
|
400
|
+
/**
|
|
401
|
+
* Get file extension for a given format (without the dot)
|
|
402
|
+
*
|
|
403
|
+
* @example
|
|
404
|
+
* const ext = getExtension("json"); // "json"
|
|
405
|
+
*/
|
|
406
|
+
declare function getExtension(format: Format): string;
|
|
407
|
+
/**
|
|
408
|
+
* Get suggested filename for a converted file
|
|
409
|
+
*
|
|
410
|
+
* @param originalName - Original filename
|
|
411
|
+
* @param outputFormat - Target format
|
|
412
|
+
* @param includeTimestamp - Whether to include a timestamp (default: false)
|
|
413
|
+
*
|
|
414
|
+
* @example
|
|
415
|
+
* const name = getSuggestedFilename("data.csv", "json"); // "data.json"
|
|
416
|
+
* const name = getSuggestedFilename("data.csv", "json", true); // "data_converted_1234567890.json"
|
|
417
|
+
*/
|
|
418
|
+
declare function getSuggestedFilename(originalName: string, outputFormat: Format, includeTimestamp?: boolean): string;
|
|
419
|
+
/**
|
|
420
|
+
* Get File System Access API file type configuration for showSaveFilePicker
|
|
421
|
+
*
|
|
422
|
+
* @example
|
|
423
|
+
* const types = getFileTypeConfig("json");
|
|
424
|
+
* const handle = await showSaveFilePicker({ types });
|
|
425
|
+
*/
|
|
426
|
+
declare function getFileTypeConfig(format: Format): Array<{
|
|
427
|
+
description: string;
|
|
428
|
+
accept: Record<string, string[]>;
|
|
429
|
+
}>;
|
|
430
|
+
/**
|
|
431
|
+
* Check if WASM threading is supported in the current environment
|
|
432
|
+
* @returns true if SharedArrayBuffer and Atomics are available (required for WASM threads)
|
|
433
|
+
*/
|
|
434
|
+
declare function isWasmThreadingSupported(): boolean;
|
|
435
|
+
/**
|
|
436
|
+
* Get the optimal number of worker threads based on CPU cores and WASM capabilities
|
|
437
|
+
* @returns Recommended thread count for optimal performance
|
|
438
|
+
*/
|
|
439
|
+
declare function getOptimalThreadCount(): number;
|
|
440
|
+
/**
|
|
441
|
+
* Get current threading capabilities and configuration
|
|
442
|
+
* @returns Object with threading information
|
|
443
|
+
*/
|
|
444
|
+
declare function getThreadingInfo(): {
|
|
445
|
+
wasmThreadingSupported: boolean;
|
|
446
|
+
customThreadPoolAvailable: boolean;
|
|
447
|
+
nodejsWasmThreading: boolean;
|
|
448
|
+
recommendedThreads: number;
|
|
449
|
+
currentThreads: number;
|
|
450
|
+
approach: string;
|
|
451
|
+
};
|
|
452
|
+
/**
|
|
453
|
+
* Backward compatibility alias for ConvertBuddy
|
|
454
|
+
* @deprecated Use ConvertBuddy instead
|
|
455
|
+
*/
|
|
456
|
+
declare const Converter: typeof ConvertBuddy;
|
|
457
|
+
|
|
458
|
+
export { type Coerce, ConvertBuddy, type ConvertBuddyOptions, ConvertBuddyTransformStream, type ConvertOptions, Converter, type CsvConfig, type CsvDetection, type DetectInput, type DetectOptions, type FieldMap, type Format, type JsonDetection, type NdjsonDetection, type ProgressCallback, type Stats, type StreamController, type StreamInput, type StreamOptions, type StreamResult, type StructureDetection, type TransformConfig, type TransformMode, type XmlConfig, type XmlDetection, autoDetectConfig, convert, convertAny, convertAnyToString, convertToString, createStreamController, detectCsvFieldsAndDelimiter, detectFormat, detectStructure, detectXmlElements, getExtension, getFileTypeConfig, getMimeType, getOptimalThreadCount, getSuggestedFilename, getThreadingInfo, isWasmThreadingSupported, processStream, streamRecords };
|
package/dist/index.js
CHANGED
|
@@ -157,6 +157,35 @@ class ConvertBuddy {
|
|
|
157
157
|
throw new Error("Unsupported input type");
|
|
158
158
|
}
|
|
159
159
|
}
|
|
160
|
+
/**
|
|
161
|
+
* Stream conversion with full control (pause/resume/abort/stats).
|
|
162
|
+
* Returns a controller that implements AsyncIterator for batch processing.
|
|
163
|
+
*
|
|
164
|
+
* @example
|
|
165
|
+
* // Process in batches
|
|
166
|
+
* const controller = cb.stream(file, { outputFormat: 'json', recordBatchSize: 100 });
|
|
167
|
+
* for await (const batch of controller) {
|
|
168
|
+
* await processBatch(batch);
|
|
169
|
+
* console.log('Stats:', controller.stats());
|
|
170
|
+
* }
|
|
171
|
+
*
|
|
172
|
+
* @example
|
|
173
|
+
* // With callbacks
|
|
174
|
+
* const controller = cb.stream(file, {
|
|
175
|
+
* outputFormat: 'json',
|
|
176
|
+
* onRecords: (records, stats) => {
|
|
177
|
+
* console.log(`Got ${records.length} records`);
|
|
178
|
+
* }
|
|
179
|
+
* });
|
|
180
|
+
*/
|
|
181
|
+
stream(input, opts = {}) {
|
|
182
|
+
const mergedOpts = {
|
|
183
|
+
...this.globalConfig,
|
|
184
|
+
...opts,
|
|
185
|
+
inputFormat: opts.inputFormat || this.globalConfig.inputFormat || "auto"
|
|
186
|
+
};
|
|
187
|
+
return __createStreamController(input, mergedOpts);
|
|
188
|
+
}
|
|
160
189
|
async convertFromUrl(url, opts) {
|
|
161
190
|
const response = await fetch(url);
|
|
162
191
|
if (!response.ok) {
|
|
@@ -998,6 +1027,9 @@ function getThreadingInfo() {
|
|
|
998
1027
|
};
|
|
999
1028
|
}
|
|
1000
1029
|
const Converter = ConvertBuddy;
|
|
1030
|
+
import { createStreamController as _createStreamController } from "./unified-stream.js";
|
|
1031
|
+
import { processStream, streamRecords } from "./unified-stream.js";
|
|
1032
|
+
const __createStreamController = _createStreamController;
|
|
1001
1033
|
export {
|
|
1002
1034
|
ConvertBuddy,
|
|
1003
1035
|
ConvertBuddyTransformStream,
|
|
@@ -1007,6 +1039,7 @@ export {
|
|
|
1007
1039
|
convertAny,
|
|
1008
1040
|
convertAnyToString,
|
|
1009
1041
|
convertToString,
|
|
1042
|
+
_createStreamController as createStreamController,
|
|
1010
1043
|
detectCsvFieldsAndDelimiter,
|
|
1011
1044
|
detectFormat,
|
|
1012
1045
|
detectStructure,
|
|
@@ -1017,6 +1050,8 @@ export {
|
|
|
1017
1050
|
getOptimalThreadCount,
|
|
1018
1051
|
getSuggestedFilename,
|
|
1019
1052
|
getThreadingInfo,
|
|
1020
|
-
isWasmThreadingSupported
|
|
1053
|
+
isWasmThreadingSupported,
|
|
1054
|
+
processStream,
|
|
1055
|
+
streamRecords
|
|
1021
1056
|
};
|
|
1022
1057
|
//# sourceMappingURL=index.js.map
|