convert-buddy-js 0.7.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +5 -1
- package/dist/src/index.js +190 -9
- package/dist/src/index.js.map +1 -0
- package/dist/src/node.d.ts +1 -1
- package/dist/src/node.js +56 -18
- package/dist/src/node.js.map +1 -0
- package/dist/src/smoke-test.d.ts +2 -0
- package/dist/src/smoke-test.js +210 -0
- package/dist/src/smoke-test.js.map +1 -0
- package/dist/wasm/nodejs/convert_buddy.d.ts +56 -0
- package/dist/wasm/nodejs/convert_buddy.js +649 -0
- package/dist/wasm/nodejs/convert_buddy_bg.wasm +0 -0
- package/dist/wasm/nodejs/convert_buddy_bg.wasm.d.ts +32 -0
- package/dist/wasm/nodejs/package.json +13 -0
- package/dist/wasm/web/convert_buddy.d.ts +112 -0
- package/dist/wasm/web/convert_buddy.js +707 -0
- package/dist/wasm/web/convert_buddy_bg.wasm +0 -0
- package/dist/wasm/web/convert_buddy_bg.wasm.d.ts +32 -0
- package/dist/wasm/web/package.json +17 -0
- package/package.json +5 -11
- package/wasm-node.cjs +10 -1
- package/dist/chunk-C3RSVYQF.js +0 -63
- package/dist/chunk-DESHN2IK.js +0 -151
- package/dist/chunk-WBKHAQCT.js +0 -192
package/dist/src/index.d.ts
CHANGED
|
@@ -4,6 +4,9 @@ type CsvDetection = {
|
|
|
4
4
|
delimiter: string;
|
|
5
5
|
fields: string[];
|
|
6
6
|
};
|
|
7
|
+
type XmlDetection = {
|
|
8
|
+
elements: string[];
|
|
9
|
+
};
|
|
7
10
|
type DetectOptions = {
|
|
8
11
|
maxBytes?: number;
|
|
9
12
|
debug?: boolean;
|
|
@@ -54,10 +57,11 @@ declare class ConvertBuddy {
|
|
|
54
57
|
}
|
|
55
58
|
declare function detectFormat(input: DetectInput, opts?: DetectOptions): Promise<Format | "unknown">;
|
|
56
59
|
declare function detectCsvFieldsAndDelimiter(input: DetectInput, opts?: DetectOptions): Promise<CsvDetection | null>;
|
|
60
|
+
declare function detectXmlElements(input: DetectInput, opts?: DetectOptions): Promise<XmlDetection | null>;
|
|
57
61
|
declare class ConvertBuddyTransformStream extends TransformStream<Uint8Array, Uint8Array> {
|
|
58
62
|
constructor(opts?: ConvertBuddyOptions);
|
|
59
63
|
}
|
|
60
64
|
declare function convert(input: Uint8Array | string, opts?: ConvertBuddyOptions): Promise<Uint8Array>;
|
|
61
65
|
declare function convertToString(input: Uint8Array | string, opts?: ConvertBuddyOptions): Promise<string>;
|
|
62
66
|
|
|
63
|
-
export { ConvertBuddy, type ConvertBuddyOptions, ConvertBuddyTransformStream, type CsvConfig, type CsvDetection, type DetectInput, type DetectOptions, type Format, type Stats, type XmlConfig, convert, convertToString, detectCsvFieldsAndDelimiter, detectFormat };
|
|
67
|
+
export { ConvertBuddy, type ConvertBuddyOptions, ConvertBuddyTransformStream, type CsvConfig, type CsvDetection, type DetectInput, type DetectOptions, type Format, type Stats, type XmlConfig, type XmlDetection, convert, convertToString, detectCsvFieldsAndDelimiter, detectFormat, detectXmlElements };
|
package/dist/src/index.js
CHANGED
|
@@ -1,17 +1,198 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
}
|
|
1
|
+
async function loadWasmModule() {
|
|
2
|
+
const isNode = typeof process !== "undefined" && !!process.versions?.node;
|
|
3
|
+
if (isNode) {
|
|
4
|
+
const { createRequire } = await import("node:module");
|
|
5
|
+
const require2 = createRequire(import.meta.url);
|
|
6
|
+
const mod2 = require2("../../wasm-node.cjs");
|
|
7
|
+
return mod2;
|
|
8
|
+
}
|
|
9
|
+
const wasmUrl = new URL("../wasm/web/convert_buddy.js", import.meta.url);
|
|
10
|
+
const mod = await import(wasmUrl.href);
|
|
11
|
+
return mod;
|
|
12
|
+
}
|
|
13
|
+
class ConvertBuddy {
|
|
14
|
+
converter;
|
|
15
|
+
debug;
|
|
16
|
+
profile;
|
|
17
|
+
constructor(converter, debug, profile) {
|
|
18
|
+
this.converter = converter;
|
|
19
|
+
this.debug = debug;
|
|
20
|
+
this.profile = profile;
|
|
21
|
+
}
|
|
22
|
+
static async create(opts = {}) {
|
|
23
|
+
const debug = !!opts.debug;
|
|
24
|
+
const profile = !!opts.profile;
|
|
25
|
+
const wasmModule = await loadWasmModule();
|
|
26
|
+
if (typeof wasmModule.default === "function") {
|
|
27
|
+
await wasmModule.default();
|
|
28
|
+
}
|
|
29
|
+
wasmModule.init(debug);
|
|
30
|
+
let converter;
|
|
31
|
+
if (opts.inputFormat && opts.outputFormat) {
|
|
32
|
+
const Converter = wasmModule.Converter;
|
|
33
|
+
converter = Converter.withConfig(
|
|
34
|
+
debug,
|
|
35
|
+
opts.inputFormat,
|
|
36
|
+
opts.outputFormat,
|
|
37
|
+
opts.chunkTargetBytes || 1024 * 1024,
|
|
38
|
+
profile,
|
|
39
|
+
opts.csvConfig,
|
|
40
|
+
opts.xmlConfig
|
|
41
|
+
);
|
|
42
|
+
} else {
|
|
43
|
+
converter = new wasmModule.Converter(debug);
|
|
44
|
+
}
|
|
45
|
+
if (debug) console.log("[convert-buddy-js] initialized", opts);
|
|
46
|
+
return new ConvertBuddy(converter, debug, profile);
|
|
47
|
+
}
|
|
48
|
+
push(chunk) {
|
|
49
|
+
if (this.debug) console.log("[convert-buddy-js] push", chunk.byteLength);
|
|
50
|
+
return this.converter.push(chunk);
|
|
51
|
+
}
|
|
52
|
+
finish() {
|
|
53
|
+
if (this.debug) console.log("[convert-buddy-js] finish");
|
|
54
|
+
return this.converter.finish();
|
|
55
|
+
}
|
|
56
|
+
stats() {
|
|
57
|
+
return this.converter.getStats();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async function readSample(input, maxBytes = 256 * 1024) {
|
|
61
|
+
if (typeof input === "string") {
|
|
62
|
+
const encoded = new TextEncoder().encode(input);
|
|
63
|
+
return encoded.length > maxBytes ? encoded.slice(0, maxBytes) : encoded;
|
|
64
|
+
}
|
|
65
|
+
if (input instanceof Uint8Array) {
|
|
66
|
+
return input.length > maxBytes ? input.slice(0, maxBytes) : input;
|
|
67
|
+
}
|
|
68
|
+
if (input instanceof ArrayBuffer) {
|
|
69
|
+
const bytes = new Uint8Array(input);
|
|
70
|
+
return bytes.length > maxBytes ? bytes.slice(0, maxBytes) : bytes;
|
|
71
|
+
}
|
|
72
|
+
if (isReadableStream(input)) {
|
|
73
|
+
const reader = input.getReader();
|
|
74
|
+
const chunks2 = [];
|
|
75
|
+
let total2 = 0;
|
|
76
|
+
while (total2 < maxBytes) {
|
|
77
|
+
const { value, done } = await reader.read();
|
|
78
|
+
if (done || !value) break;
|
|
79
|
+
const slice = total2 + value.length > maxBytes ? value.slice(0, maxBytes - total2) : value;
|
|
80
|
+
chunks2.push(slice);
|
|
81
|
+
total2 += slice.length;
|
|
82
|
+
}
|
|
83
|
+
if (total2 >= maxBytes) {
|
|
84
|
+
await reader.cancel();
|
|
85
|
+
}
|
|
86
|
+
return concatChunks(chunks2, total2);
|
|
87
|
+
}
|
|
88
|
+
const chunks = [];
|
|
89
|
+
let total = 0;
|
|
90
|
+
for await (const chunk of input) {
|
|
91
|
+
if (total >= maxBytes) {
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
const data = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);
|
|
95
|
+
const slice = total + data.length > maxBytes ? data.slice(0, maxBytes - total) : data;
|
|
96
|
+
chunks.push(slice);
|
|
97
|
+
total += slice.length;
|
|
98
|
+
}
|
|
99
|
+
return concatChunks(chunks, total);
|
|
100
|
+
}
|
|
101
|
+
function concatChunks(chunks, total) {
|
|
102
|
+
const result = new Uint8Array(total);
|
|
103
|
+
let offset = 0;
|
|
104
|
+
for (const chunk of chunks) {
|
|
105
|
+
result.set(chunk, offset);
|
|
106
|
+
offset += chunk.length;
|
|
107
|
+
}
|
|
108
|
+
return result;
|
|
109
|
+
}
|
|
110
|
+
function isReadableStream(input) {
|
|
111
|
+
return typeof input?.getReader === "function";
|
|
112
|
+
}
|
|
113
|
+
async function loadDetectionWasm(debug) {
|
|
114
|
+
const wasmModule = await loadWasmModule();
|
|
115
|
+
if (typeof wasmModule.default === "function") {
|
|
116
|
+
await wasmModule.default();
|
|
117
|
+
}
|
|
118
|
+
wasmModule.init(debug);
|
|
119
|
+
return wasmModule;
|
|
120
|
+
}
|
|
121
|
+
async function detectFormat(input, opts = {}) {
|
|
122
|
+
const wasmModule = await loadDetectionWasm(!!opts.debug);
|
|
123
|
+
const sample = await readSample(input, opts.maxBytes);
|
|
124
|
+
const format = wasmModule.detectFormat?.(sample);
|
|
125
|
+
return format ?? "unknown";
|
|
126
|
+
}
|
|
127
|
+
async function detectCsvFieldsAndDelimiter(input, opts = {}) {
|
|
128
|
+
const wasmModule = await loadDetectionWasm(!!opts.debug);
|
|
129
|
+
const sample = await readSample(input, opts.maxBytes);
|
|
130
|
+
const result = wasmModule.detectCsvFields?.(sample);
|
|
131
|
+
return result ?? null;
|
|
132
|
+
}
|
|
133
|
+
async function detectXmlElements(input, opts = {}) {
|
|
134
|
+
const wasmModule = await loadDetectionWasm(!!opts.debug);
|
|
135
|
+
const sample = await readSample(input, opts.maxBytes);
|
|
136
|
+
const result = wasmModule.detectXmlElements?.(sample);
|
|
137
|
+
return result ?? null;
|
|
138
|
+
}
|
|
139
|
+
class ConvertBuddyTransformStream extends TransformStream {
|
|
140
|
+
constructor(opts = {}) {
|
|
141
|
+
let buddy = null;
|
|
142
|
+
super({
|
|
143
|
+
async start(controller) {
|
|
144
|
+
buddy = await ConvertBuddy.create(opts);
|
|
145
|
+
},
|
|
146
|
+
transform(chunk, controller) {
|
|
147
|
+
if (!buddy) {
|
|
148
|
+
throw new Error("ConvertBuddy not initialized");
|
|
149
|
+
}
|
|
150
|
+
const output = buddy.push(chunk);
|
|
151
|
+
if (output.length > 0) {
|
|
152
|
+
controller.enqueue(output);
|
|
153
|
+
}
|
|
154
|
+
},
|
|
155
|
+
flush(controller) {
|
|
156
|
+
if (!buddy) {
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
const output = buddy.finish();
|
|
160
|
+
if (output.length > 0) {
|
|
161
|
+
controller.enqueue(output);
|
|
162
|
+
}
|
|
163
|
+
if (opts.profile) {
|
|
164
|
+
const stats = buddy.stats();
|
|
165
|
+
console.log("[convert-buddy] Performance Stats:", stats);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
async function convert(input, opts = {}) {
|
|
172
|
+
const buddy = await ConvertBuddy.create(opts);
|
|
173
|
+
const inputBytes = typeof input === "string" ? new TextEncoder().encode(input) : input;
|
|
174
|
+
const output = buddy.push(inputBytes);
|
|
175
|
+
const final = buddy.finish();
|
|
176
|
+
const result = new Uint8Array(output.length + final.length);
|
|
177
|
+
result.set(output, 0);
|
|
178
|
+
result.set(final, output.length);
|
|
179
|
+
if (opts.profile) {
|
|
180
|
+
const stats = buddy.stats();
|
|
181
|
+
console.log("[convert-buddy] Performance Stats:", stats);
|
|
182
|
+
}
|
|
183
|
+
return result;
|
|
184
|
+
}
|
|
185
|
+
async function convertToString(input, opts = {}) {
|
|
186
|
+
const result = await convert(input, opts);
|
|
187
|
+
return new TextDecoder().decode(result);
|
|
188
|
+
}
|
|
9
189
|
export {
|
|
10
190
|
ConvertBuddy,
|
|
11
191
|
ConvertBuddyTransformStream,
|
|
12
192
|
convert,
|
|
13
193
|
convertToString,
|
|
14
194
|
detectCsvFieldsAndDelimiter,
|
|
15
|
-
detectFormat
|
|
195
|
+
detectFormat,
|
|
196
|
+
detectXmlElements
|
|
16
197
|
};
|
|
17
198
|
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/index.ts"],"sourcesContent":["export type Format = \"csv\" | \"ndjson\" | \"json\" | \"xml\";\r\nexport type DetectInput =\r\n | Uint8Array\r\n | ArrayBuffer\r\n | string\r\n | ReadableStream<Uint8Array>\r\n | AsyncIterable<Uint8Array>;\r\n\r\nexport type CsvDetection = {\r\n delimiter: string;\r\n fields: string[];\r\n};\r\n\r\nexport type XmlDetection = {\r\n elements: string[];\r\n};\r\n\r\nexport type DetectOptions = {\r\n maxBytes?: number;\r\n debug?: boolean;\r\n};\r\n\r\nexport type ConvertBuddyOptions = {\r\n debug?: boolean;\r\n profile?: boolean;\r\n inputFormat?: Format;\r\n outputFormat?: Format;\r\n chunkTargetBytes?: number;\r\n parallelism?: number; // Node only - number of worker threads\r\n csvConfig?: CsvConfig;\r\n xmlConfig?: XmlConfig;\r\n};\r\n\r\nexport type CsvConfig = {\r\n delimiter?: string;\r\n quote?: string;\r\n hasHeaders?: boolean;\r\n trimWhitespace?: boolean;\r\n};\r\n\r\nexport type XmlConfig = {\r\n recordElement?: string;\r\n trimText?: boolean;\r\n includeAttributes?: boolean;\r\n expandEntities?: boolean;\r\n};\r\n\r\nexport type Stats = {\r\n bytesIn: number;\r\n bytesOut: number;\r\n chunksIn: number;\r\n recordsProcessed: number;\r\n parseTimeMs: number;\r\n transformTimeMs: number;\r\n writeTimeMs: number;\r\n maxBufferSize: number;\r\n currentPartialSize: number;\r\n throughputMbPerSec: number;\r\n};\r\n\r\ntype WasmModule = {\r\n default?: unknown;\r\n init: (debugEnabled: boolean) => void;\r\n Converter: new (debug: boolean) => {\r\n push: (chunk: Uint8Array) => Uint8Array;\r\n finish: () => Uint8Array;\r\n getStats: () => Stats;\r\n };\r\n detectFormat?: (sample: Uint8Array) => string | null | undefined;\r\n detectCsvFields?: (sample: Uint8Array) => CsvDetection | null | undefined;\r\n detectXmlElements?: (sample: Uint8Array) => XmlDetection | null | undefined;\r\n __wbg_set_wasm?: (wasm: unknown) => void;\r\n};\r\n\r\nasync function loadWasmModule(): Promise<WasmModule> {\r\n const isNode =\r\n typeof process !== \"undefined\" &&\r\n !!(process as any).versions?.node;\r\n\r\n if (isNode) {\r\n const { createRequire } = await import(\"node:module\");\r\n const require = createRequire(import.meta.url);\r\n const mod = require(\"../../wasm-node.cjs\");\r\n return mod as WasmModule;\r\n }\r\n\r\n const wasmUrl = new URL(\"../wasm/web/convert_buddy.js\", import.meta.url);\r\n const mod = (await import(wasmUrl.href)) as unknown as WasmModule;\r\n return mod;\r\n}\r\n\r\nexport class ConvertBuddy {\r\n private converter: any;\r\n private debug: boolean;\r\n private profile: boolean;\r\n\r\n private constructor(converter: any, debug: boolean, profile: boolean) {\r\n this.converter = converter;\r\n this.debug = debug;\r\n this.profile = profile;\r\n }\r\n\r\n static async create(opts: ConvertBuddyOptions = {}): Promise<ConvertBuddy> {\r\n const debug = !!opts.debug;\r\n const profile = !!opts.profile;\r\n\r\n const wasmModule = await loadWasmModule();\r\n\r\n if (typeof wasmModule.default === \"function\") {\r\n await (wasmModule.default as () => Promise<void>)();\r\n }\r\n\r\n wasmModule.init(debug);\r\n\r\n let converter;\r\n if (opts.inputFormat && opts.outputFormat) {\r\n // Use withConfig for custom formats\r\n const Converter = (wasmModule as any).Converter;\r\n converter = Converter.withConfig(\r\n debug,\r\n opts.inputFormat,\r\n opts.outputFormat,\r\n opts.chunkTargetBytes || 1024 * 1024,\r\n profile,\r\n opts.csvConfig,\r\n opts.xmlConfig\r\n );\r\n } else {\r\n converter = new wasmModule.Converter(debug);\r\n }\r\n\r\n if (debug) console.log(\"[convert-buddy-js] initialized\", opts);\r\n return new ConvertBuddy(converter, debug, profile);\r\n }\r\n\r\n push(chunk: Uint8Array): Uint8Array {\r\n if (this.debug) console.log(\"[convert-buddy-js] push\", chunk.byteLength);\r\n return this.converter.push(chunk);\r\n }\r\n\r\n finish(): Uint8Array {\r\n if (this.debug) console.log(\"[convert-buddy-js] finish\");\r\n return this.converter.finish();\r\n }\r\n\r\n stats(): Stats {\r\n return this.converter.getStats();\r\n }\r\n}\r\n\r\nasync function readSample(\r\n input: DetectInput,\r\n maxBytes = 256 * 1024\r\n): Promise<Uint8Array> {\r\n if (typeof input === \"string\") {\r\n const encoded = new TextEncoder().encode(input);\r\n return encoded.length > maxBytes ? encoded.slice(0, maxBytes) : encoded;\r\n }\r\n\r\n if (input instanceof Uint8Array) {\r\n return input.length > maxBytes ? input.slice(0, maxBytes) : input;\r\n }\r\n\r\n if (input instanceof ArrayBuffer) {\r\n const bytes = new Uint8Array(input);\r\n return bytes.length > maxBytes ? bytes.slice(0, maxBytes) : bytes;\r\n }\r\n\r\n if (isReadableStream(input)) {\r\n const reader = input.getReader();\r\n const chunks: Uint8Array[] = [];\r\n let total = 0;\r\n\r\n while (total < maxBytes) {\r\n const { value, done } = await reader.read();\r\n if (done || !value) break;\r\n const slice = total + value.length > maxBytes\r\n ? value.slice(0, maxBytes - total)\r\n : value;\r\n chunks.push(slice);\r\n total += slice.length;\r\n }\r\n\r\n if (total >= maxBytes) {\r\n await reader.cancel();\r\n }\r\n\r\n return concatChunks(chunks, total);\r\n }\r\n\r\n const chunks: Uint8Array[] = [];\r\n let total = 0;\r\n\r\n for await (const chunk of input as AsyncIterable<Uint8Array>) {\r\n if (total >= maxBytes) {\r\n break;\r\n }\r\n const data = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);\r\n const slice = total + data.length > maxBytes\r\n ? data.slice(0, maxBytes - total)\r\n : data;\r\n chunks.push(slice);\r\n total += slice.length;\r\n }\r\n\r\n return concatChunks(chunks, total);\r\n}\r\n\r\nfunction concatChunks(chunks: Uint8Array[], total: number): Uint8Array {\r\n const result = new Uint8Array(total);\r\n let offset = 0;\r\n for (const chunk of chunks) {\r\n result.set(chunk, offset);\r\n offset += chunk.length;\r\n }\r\n return result;\r\n}\r\n\r\nfunction isReadableStream(\r\n input: DetectInput\r\n): input is ReadableStream<Uint8Array> {\r\n return typeof (input as ReadableStream<Uint8Array>)?.getReader === \"function\";\r\n}\r\n\r\nasync function loadDetectionWasm(debug: boolean): Promise<WasmModule> {\r\n const wasmModule = await loadWasmModule();\r\n if (typeof wasmModule.default === \"function\") {\r\n await (wasmModule.default as () => Promise<void>)();\r\n }\r\n wasmModule.init(debug);\r\n return wasmModule;\r\n}\r\n\r\nexport async function detectFormat(\r\n input: DetectInput,\r\n opts: DetectOptions = {}\r\n): Promise<Format | \"unknown\"> {\r\n const wasmModule = await loadDetectionWasm(!!opts.debug);\r\n const sample = await readSample(input, opts.maxBytes);\r\n const format = wasmModule.detectFormat?.(sample);\r\n return (format as Format) ?? \"unknown\";\r\n}\r\n\r\nexport async function detectCsvFieldsAndDelimiter(\r\n input: DetectInput,\r\n opts: DetectOptions = {}\r\n): Promise<CsvDetection | null> {\r\n const wasmModule = await loadDetectionWasm(!!opts.debug);\r\n const sample = await readSample(input, opts.maxBytes);\r\n const result = wasmModule.detectCsvFields?.(sample);\r\n return result ?? null;\r\n}\r\n\r\nexport async function detectXmlElements(\r\n input: DetectInput,\r\n opts: DetectOptions = {}\r\n): Promise<XmlDetection | null> {\r\n const wasmModule = await loadDetectionWasm(!!opts.debug);\r\n const sample = await readSample(input, opts.maxBytes);\r\n const result = wasmModule.detectXmlElements?.(sample);\r\n return result ?? null;\r\n}\r\n\r\n// Web Streams TransformStream adapter\r\nexport class ConvertBuddyTransformStream extends TransformStream<Uint8Array, Uint8Array> {\r\n constructor(opts: ConvertBuddyOptions = {}) {\r\n let buddy: ConvertBuddy | null = null;\r\n\r\n super({\r\n async start(controller) {\r\n buddy = await ConvertBuddy.create(opts);\r\n },\r\n\r\n transform(chunk, controller) {\r\n if (!buddy) {\r\n throw new Error(\"ConvertBuddy not initialized\");\r\n }\r\n\r\n const output = buddy.push(chunk);\r\n if (output.length > 0) {\r\n controller.enqueue(output);\r\n }\r\n },\r\n\r\n flush(controller) {\r\n if (!buddy) {\r\n return;\r\n }\r\n\r\n const output = buddy.finish();\r\n if (output.length > 0) {\r\n controller.enqueue(output);\r\n }\r\n\r\n if (opts.profile) {\r\n const stats = buddy.stats();\r\n console.log(\"[convert-buddy] Performance Stats:\", stats);\r\n }\r\n },\r\n });\r\n }\r\n}\r\n\r\n// Utility: Convert entire buffer/string\r\nexport async function convert(\r\n input: Uint8Array | string,\r\n opts: ConvertBuddyOptions = {}\r\n): Promise<Uint8Array> {\r\n const buddy = await ConvertBuddy.create(opts);\r\n\r\n const inputBytes = typeof input === \"string\" \r\n ? new TextEncoder().encode(input)\r\n : input;\r\n\r\n const output = buddy.push(inputBytes);\r\n const final = buddy.finish();\r\n\r\n // Combine outputs\r\n const result = new Uint8Array(output.length + final.length);\r\n result.set(output, 0);\r\n result.set(final, output.length);\r\n\r\n if (opts.profile) {\r\n const stats = buddy.stats();\r\n console.log(\"[convert-buddy] Performance Stats:\", stats);\r\n }\r\n\r\n return result;\r\n}\r\n\r\n// Utility: Convert and return as string\r\nexport async function convertToString(\r\n input: Uint8Array | string,\r\n opts: ConvertBuddyOptions = {}\r\n): Promise<string> {\r\n const result = await convert(input, opts);\r\n return new TextDecoder().decode(result);\r\n}\r\n"],"mappings":"AA0EA,eAAe,iBAAsC;AACnD,QAAM,SACJ,OAAO,YAAY,eACnB,CAAC,CAAE,QAAgB,UAAU;AAE/B,MAAI,QAAQ;AACV,UAAM,EAAE,cAAc,IAAI,MAAM,OAAO,aAAa;AACpD,UAAMA,WAAU,cAAc,YAAY,GAAG;AAC7C,UAAMC,OAAMD,SAAQ,qBAAqB;AACzC,WAAOC;AAAA,EACT;AAEA,QAAM,UAAU,IAAI,IAAI,gCAAgC,YAAY,GAAG;AACvE,QAAM,MAAO,MAAM,OAAO,QAAQ;AAClC,SAAO;AACT;AAEO,MAAM,aAAa;AAAA,EAChB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,WAAgB,OAAgB,SAAkB;AACpE,SAAK,YAAY;AACjB,SAAK,QAAQ;AACb,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,aAAa,OAAO,OAA4B,CAAC,GAA0B;AACzE,UAAM,QAAQ,CAAC,CAAC,KAAK;AACrB,UAAM,UAAU,CAAC,CAAC,KAAK;AAEvB,UAAM,aAAa,MAAM,eAAe;AAExC,QAAI,OAAO,WAAW,YAAY,YAAY;AAC5C,YAAO,WAAW,QAAgC;AAAA,IACpD;AAEA,eAAW,KAAK,KAAK;AAErB,QAAI;AACJ,QAAI,KAAK,eAAe,KAAK,cAAc;AAEzC,YAAM,YAAa,WAAmB;AACtC,kBAAY,UAAU;AAAA,QACpB;AAAA,QACA,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK,oBAAoB,OAAO;AAAA,QAChC;AAAA,QACA,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AAAA,IACF,OAAO;AACL,kBAAY,IAAI,WAAW,UAAU,KAAK;AAAA,IAC5C;AAEA,QAAI,MAAO,SAAQ,IAAI,kCAAkC,IAAI;AAC7D,WAAO,IAAI,aAAa,WAAW,OAAO,OAAO;AAAA,EACnD;AAAA,EAEA,KAAK,OAA+B;AAClC,QAAI,KAAK,MAAO,SAAQ,IAAI,2BAA2B,MAAM,UAAU;AACvE,WAAO,KAAK,UAAU,KAAK,KAAK;AAAA,EAClC;AAAA,EAEA,SAAqB;AACnB,QAAI,KAAK,MAAO,SAAQ,IAAI,2BAA2B;AACvD,WAAO,KAAK,UAAU,OAAO;AAAA,EAC/B;AAAA,EAEA,QAAe;AACb,WAAO,KAAK,UAAU,SAAS;AAAA,EACjC;AACF;AAEA,eAAe,WACb,OACA,WAAW,MAAM,MACI;AACrB,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,UAAU,IAAI,YAAY,EAAE,OAAO,KAAK;AAC9C,WAAO,QAAQ,SAAS,WAAW,QAAQ,MAAM,GAAG,QAAQ,IAAI;AAAA,EAClE;AAEA,MAAI,iBAAiB,YAAY;AAC/B,WAAO,MAAM,SAAS,WAAW,MAAM,MAAM,GAAG,QAAQ,IAAI;AAAA,EAC9D;AAEA,MAAI,iBAAiB,aAAa;AAChC,UAAM,QAAQ,IAAI,WAAW,KAAK;AAClC,WAAO,MAAM,SAAS,WAAW,MAAM,MAAM,GAAG,QAAQ,IAAI;AAAA,EAC9D;AAEA,MAAI,iBAAiB,KAAK,GAAG;AAC3B,UAAM,SAAS,MAAM,UAAU;AAC/B,UAAMC,UAAuB,CAAC;AAC9B,QAAIC,SAAQ;AAEZ,WAAOA,SAAQ,UAAU;AACvB,YAAM,EAAE,OAAO,KAAK,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,CAAC,MAAO;AACpB,YAAM,QAAQA,SAAQ,MAAM,SAAS,WACjC,MAAM,MAAM,GAAG,WAAWA,MAAK,IAC/B;AACJ,MAAAD,QAAO,KAAK,KAAK;AACjB,MAAAC,UAAS,MAAM;AAAA,IACjB;AAEA,QAAIA,UAAS,UAAU;AACrB,YAAM,OAAO,OAAO;AAAA,IACtB;AAEA,WAAO,aAAaD,SAAQC,MAAK;AAAA,EACnC;AAEA,QAAM,SAAuB,CAAC;AAC9B,MAAI,QAAQ;AAEZ,mBAAiB,SAAS,OAAoC;AAC5D,QAAI,SAAS,UAAU;AACrB;AAAA,IACF;AACA,UAAM,OAAO,iBAAiB,aAAa,QAAQ,IAAI,WAAW,KAAK;AACvE,UAAM,QAAQ,QAAQ,KAAK,SAAS,WAChC,KAAK,MAAM,GAAG,WAAW,KAAK,IAC9B;AACJ,WAAO,KAAK,KAAK;AACjB,aAAS,MAAM;AAAA,EACjB;AAEA,SAAO,aAAa,QAAQ,KAAK;AACnC;AAEA,SAAS,aAAa,QAAsB,OAA2B;AACrE,QAAM,SAAS,IAAI,WAAW,KAAK;AACnC,MAAI,SAAS;AACb,aAAW,SAAS,QAAQ;AAC1B,WAAO,IAAI,OAAO,MAAM;AACxB,cAAU,MAAM;AAAA,EAClB;AACA,SAAO;AACT;AAEA,SAAS,iBACP,OACqC;AACrC,SAAO,OAAQ,OAAsC,cAAc;AACrE;AAEA,eAAe,kBAAkB,OAAqC;AACpE,QAAM,aAAa,MAAM,eAAe;AACxC,MAAI,OAAO,WAAW,YAAY,YAAY;AAC5C,UAAO,WAAW,QAAgC;AAAA,EACpD;AACA,aAAW,KAAK,KAAK;AACrB,SAAO;AACT;AAEA,eAAsB,aACpB,OACA,OAAsB,CAAC,GACM;AAC7B,QAAM,aAAa,MAAM,kBAAkB,CAAC,CAAC,KAAK,KAAK;AACvD,QAAM,SAAS,MAAM,WAAW,OAAO,KAAK,QAAQ;AACpD,QAAM,SAAS,WAAW,eAAe,MAAM;AAC/C,SAAQ,UAAqB;AAC/B;AAEA,eAAsB,4BACpB,OACA,OAAsB,CAAC,GACO;AAC9B,QAAM,aAAa,MAAM,kBAAkB,CAAC,CAAC,KAAK,KAAK;AACvD,QAAM,SAAS,MAAM,WAAW,OAAO,KAAK,QAAQ;AACpD,QAAM,SAAS,WAAW,kBAAkB,MAAM;AAClD,SAAO,UAAU;AACnB;AAEA,eAAsB,kBACpB,OACA,OAAsB,CAAC,GACO;AAC9B,QAAM,aAAa,MAAM,kBAAkB,CAAC,CAAC,KAAK,KAAK;AACvD,QAAM,SAAS,MAAM,WAAW,OAAO,KAAK,QAAQ;AACpD,QAAM,SAAS,WAAW,oBAAoB,MAAM;AACpD,SAAO,UAAU;AACnB;AAGO,MAAM,oCAAoC,gBAAwC;AAAA,EACvF,YAAY,OAA4B,CAAC,GAAG;AAC1C,QAAI,QAA6B;AAEjC,UAAM;AAAA,MACJ,MAAM,MAAM,YAAY;AACtB,gBAAQ,MAAM,aAAa,OAAO,IAAI;AAAA,MACxC;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,CAAC,OAAO;AACV,gBAAM,IAAI,MAAM,8BAA8B;AAAA,QAChD;AAEA,cAAM,SAAS,MAAM,KAAK,KAAK;AAC/B,YAAI,OAAO,SAAS,GAAG;AACrB,qBAAW,QAAQ,MAAM;AAAA,QAC3B;AAAA,MACF;AAAA,MAEA,MAAM,YAAY;AAChB,YAAI,CAAC,OAAO;AACV;AAAA,QACF;AAEA,cAAM,SAAS,MAAM,OAAO;AAC5B,YAAI,OAAO,SAAS,GAAG;AACrB,qBAAW,QAAQ,MAAM;AAAA,QAC3B;AAEA,YAAI,KAAK,SAAS;AAChB,gBAAM,QAAQ,MAAM,MAAM;AAC1B,kBAAQ,IAAI,sCAAsC,KAAK;AAAA,QACzD;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAGA,eAAsB,QACpB,OACA,OAA4B,CAAC,GACR;AACrB,QAAM,QAAQ,MAAM,aAAa,OAAO,IAAI;AAE5C,QAAM,aAAa,OAAO,UAAU,WAChC,IAAI,YAAY,EAAE,OAAO,KAAK,IAC9B;AAEJ,QAAM,SAAS,MAAM,KAAK,UAAU;AACpC,QAAM,QAAQ,MAAM,OAAO;AAG3B,QAAM,SAAS,IAAI,WAAW,OAAO,SAAS,MAAM,MAAM;AAC1D,SAAO,IAAI,QAAQ,CAAC;AACpB,SAAO,IAAI,OAAO,OAAO,MAAM;AAE/B,MAAI,KAAK,SAAS;AAChB,UAAM,QAAQ,MAAM,MAAM;AAC1B,YAAQ,IAAI,sCAAsC,KAAK;AAAA,EACzD;AAEA,SAAO;AACT;AAGA,eAAsB,gBACpB,OACA,OAA4B,CAAC,GACZ;AACjB,QAAM,SAAS,MAAM,QAAQ,OAAO,IAAI;AACxC,SAAO,IAAI,YAAY,EAAE,OAAO,MAAM;AACxC;","names":["require","mod","chunks","total"]}
|
package/dist/src/node.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Transform } from 'node:stream';
|
|
2
2
|
import { ConvertBuddyOptions } from './index.js';
|
|
3
|
-
export { ConvertBuddy, ConvertBuddyTransformStream, CsvConfig, CsvDetection, DetectInput, DetectOptions, Format, Stats, XmlConfig, convert, convertToString, detectCsvFieldsAndDelimiter, detectFormat } from './index.js';
|
|
3
|
+
export { ConvertBuddy, ConvertBuddyTransformStream, CsvConfig, CsvDetection, DetectInput, DetectOptions, Format, Stats, XmlConfig, XmlDetection, convert, convertToString, detectCsvFieldsAndDelimiter, detectFormat, detectXmlElements } from './index.js';
|
|
4
4
|
|
|
5
5
|
declare function createNodeTransform(opts?: ConvertBuddyOptions): Promise<Transform>;
|
|
6
6
|
|
package/dist/src/node.js
CHANGED
|
@@ -1,21 +1,59 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
}
|
|
1
|
+
import { ConvertBuddy } from "./index";
|
|
2
|
+
export * from "./index";
|
|
3
|
+
async function loadNodeTransform() {
|
|
4
|
+
const isNode = typeof process !== "undefined" && !!process.versions?.node;
|
|
5
|
+
if (!isNode) {
|
|
6
|
+
throw new Error("createNodeTransform is only available in Node.js runtimes.");
|
|
7
|
+
}
|
|
8
|
+
const streamModule = await import("node:stream");
|
|
9
|
+
return streamModule.Transform;
|
|
10
|
+
}
|
|
11
|
+
async function createNodeTransform(opts = {}) {
|
|
12
|
+
let buddy = null;
|
|
13
|
+
let initPromise = null;
|
|
14
|
+
const Transform = await loadNodeTransform();
|
|
15
|
+
const transform = new Transform({
|
|
16
|
+
async transform(chunk, encoding, callback) {
|
|
17
|
+
try {
|
|
18
|
+
if (!buddy) {
|
|
19
|
+
if (!initPromise) {
|
|
20
|
+
initPromise = ConvertBuddy.create(opts).then((b) => {
|
|
21
|
+
buddy = b;
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
await initPromise;
|
|
25
|
+
}
|
|
26
|
+
const input = new Uint8Array(chunk);
|
|
27
|
+
const output = buddy.push(input);
|
|
28
|
+
if (output.length > 0) {
|
|
29
|
+
this.push(Buffer.from(output));
|
|
30
|
+
}
|
|
31
|
+
callback();
|
|
32
|
+
} catch (err) {
|
|
33
|
+
callback(err);
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
async flush(callback) {
|
|
37
|
+
try {
|
|
38
|
+
if (buddy) {
|
|
39
|
+
const output = buddy.finish();
|
|
40
|
+
if (output.length > 0) {
|
|
41
|
+
this.push(Buffer.from(output));
|
|
42
|
+
}
|
|
43
|
+
if (opts.profile) {
|
|
44
|
+
const stats = buddy.stats();
|
|
45
|
+
console.log("[convert-buddy] Performance Stats:", stats);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
callback();
|
|
49
|
+
} catch (err) {
|
|
50
|
+
callback(err);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
return transform;
|
|
55
|
+
}
|
|
12
56
|
export {
|
|
13
|
-
|
|
14
|
-
ConvertBuddyTransformStream,
|
|
15
|
-
convert,
|
|
16
|
-
convertToString,
|
|
17
|
-
createNodeTransform,
|
|
18
|
-
detectCsvFieldsAndDelimiter,
|
|
19
|
-
detectFormat
|
|
57
|
+
createNodeTransform
|
|
20
58
|
};
|
|
21
59
|
//# sourceMappingURL=node.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/node.ts"],"sourcesContent":["import type { Transform as NodeTransform } from \"node:stream\";\r\n\r\nimport { ConvertBuddy, type ConvertBuddyOptions } from \"./index\";\r\n\r\nexport * from \"./index\";\r\n\r\n// Node.js Transform Stream adapter\r\nasync function loadNodeTransform(): Promise<typeof import(\"node:stream\").Transform> {\r\n const isNode =\r\n typeof process !== \"undefined\" &&\r\n !!(process as any).versions?.node;\r\n\r\n if (!isNode) {\r\n throw new Error(\"createNodeTransform is only available in Node.js runtimes.\");\r\n }\r\n\r\n const streamModule = await import(\"node:stream\");\r\n return streamModule.Transform;\r\n}\r\n\r\nexport async function createNodeTransform(\r\n opts: ConvertBuddyOptions = {}\r\n): Promise<NodeTransform> {\r\n let buddy: ConvertBuddy | null = null;\r\n let initPromise: Promise<void> | null = null;\r\n\r\n const Transform = await loadNodeTransform();\r\n const transform = new Transform({\r\n async transform(chunk: Buffer, encoding: string, callback: Function) {\r\n try {\r\n if (!buddy) {\r\n if (!initPromise) {\r\n initPromise = ConvertBuddy.create(opts).then((b) => {\r\n buddy = b;\r\n });\r\n }\r\n await initPromise;\r\n }\r\n\r\n const input = new Uint8Array(chunk);\r\n const output = buddy!.push(input);\r\n\r\n if (output.length > 0) {\r\n this.push(Buffer.from(output));\r\n }\r\n\r\n callback();\r\n } catch (err) {\r\n callback(err);\r\n }\r\n },\r\n\r\n async flush(callback: Function) {\r\n try {\r\n if (buddy) {\r\n const output = buddy.finish();\r\n if (output.length > 0) {\r\n this.push(Buffer.from(output));\r\n }\r\n\r\n if (opts.profile) {\r\n const stats = buddy.stats();\r\n console.log(\"[convert-buddy] Performance Stats:\", stats);\r\n }\r\n }\r\n callback();\r\n } catch (err) {\r\n callback(err);\r\n }\r\n },\r\n });\r\n\r\n return transform;\r\n}\r\n"],"mappings":"AAEA,SAAS,oBAA8C;AAEvD,cAAc;AAGd,eAAe,oBAAqE;AAClF,QAAM,SACJ,OAAO,YAAY,eACnB,CAAC,CAAE,QAAgB,UAAU;AAE/B,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,MAAM,4DAA4D;AAAA,EAC9E;AAEA,QAAM,eAAe,MAAM,OAAO,aAAa;AAC/C,SAAO,aAAa;AACtB;AAEA,eAAsB,oBACpB,OAA4B,CAAC,GACL;AACxB,MAAI,QAA6B;AACjC,MAAI,cAAoC;AAExC,QAAM,YAAY,MAAM,kBAAkB;AAC1C,QAAM,YAAY,IAAI,UAAU;AAAA,IAC9B,MAAM,UAAU,OAAe,UAAkB,UAAoB;AACnE,UAAI;AACF,YAAI,CAAC,OAAO;AACV,cAAI,CAAC,aAAa;AAChB,0BAAc,aAAa,OAAO,IAAI,EAAE,KAAK,CAAC,MAAM;AAClD,sBAAQ;AAAA,YACV,CAAC;AAAA,UACH;AACA,gBAAM;AAAA,QACR;AAEA,cAAM,QAAQ,IAAI,WAAW,KAAK;AAClC,cAAM,SAAS,MAAO,KAAK,KAAK;AAEhC,YAAI,OAAO,SAAS,GAAG;AACrB,eAAK,KAAK,OAAO,KAAK,MAAM,CAAC;AAAA,QAC/B;AAEA,iBAAS;AAAA,MACX,SAAS,KAAK;AACZ,iBAAS,GAAG;AAAA,MACd;AAAA,IACF;AAAA,IAEA,MAAM,MAAM,UAAoB;AAC9B,UAAI;AACF,YAAI,OAAO;AACT,gBAAM,SAAS,MAAM,OAAO;AAC5B,cAAI,OAAO,SAAS,GAAG;AACrB,iBAAK,KAAK,OAAO,KAAK,MAAM,CAAC;AAAA,UAC/B;AAEA,cAAI,KAAK,SAAS;AAChB,kBAAM,QAAQ,MAAM,MAAM;AAC1B,oBAAQ,IAAI,sCAAsC,KAAK;AAAA,UACzD;AAAA,QACF;AACA,iBAAS;AAAA,MACX,SAAS,KAAK;AACZ,iBAAS,GAAG;AAAA,MACd;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO;AACT;","names":[]}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { ConvertBuddy, convert, convertToString } from "./index.js";
|
|
2
|
+
async function testBasicConversion() {
|
|
3
|
+
console.log("\n=== Test: Basic CSV to NDJSON ===");
|
|
4
|
+
const csvData = `name,age,city
|
|
5
|
+
Alice,30,New York
|
|
6
|
+
Bob,25,London
|
|
7
|
+
Charlie,35,Tokyo`;
|
|
8
|
+
const result = await convertToString(csvData, {
|
|
9
|
+
inputFormat: "csv",
|
|
10
|
+
outputFormat: "ndjson",
|
|
11
|
+
debug: false,
|
|
12
|
+
profile: true
|
|
13
|
+
});
|
|
14
|
+
console.log("Input CSV:");
|
|
15
|
+
console.log(csvData);
|
|
16
|
+
console.log("\nOutput NDJSON:");
|
|
17
|
+
console.log(result);
|
|
18
|
+
const lines = result.trim().split("\n");
|
|
19
|
+
if (lines.length !== 3) {
|
|
20
|
+
throw new Error(`Expected 3 lines, got ${lines.length}`);
|
|
21
|
+
}
|
|
22
|
+
const firstRecord = JSON.parse(lines[0]);
|
|
23
|
+
if (firstRecord.name !== "Alice") {
|
|
24
|
+
throw new Error(`Expected name=Alice, got ${firstRecord.name}`);
|
|
25
|
+
}
|
|
26
|
+
console.log("\u2713 Basic conversion test passed");
|
|
27
|
+
}
|
|
28
|
+
async function testStreamingAPI() {
|
|
29
|
+
console.log("\n=== Test: Streaming API ===");
|
|
30
|
+
const buddy = await ConvertBuddy.create({
|
|
31
|
+
inputFormat: "csv",
|
|
32
|
+
outputFormat: "ndjson",
|
|
33
|
+
debug: false
|
|
34
|
+
});
|
|
35
|
+
const header = new TextEncoder().encode("name,age\n");
|
|
36
|
+
const row1 = new TextEncoder().encode("Alice,30\n");
|
|
37
|
+
const row2 = new TextEncoder().encode("Bob,25\n");
|
|
38
|
+
const out1 = buddy.push(header);
|
|
39
|
+
const out2 = buddy.push(row1);
|
|
40
|
+
const out3 = buddy.push(row2);
|
|
41
|
+
const final = buddy.finish();
|
|
42
|
+
const combined = new Uint8Array(
|
|
43
|
+
out1.length + out2.length + out3.length + final.length
|
|
44
|
+
);
|
|
45
|
+
combined.set(out1, 0);
|
|
46
|
+
combined.set(out2, out1.length);
|
|
47
|
+
combined.set(out3, out1.length + out2.length);
|
|
48
|
+
combined.set(final, out1.length + out2.length + out3.length);
|
|
49
|
+
const result = new TextDecoder().decode(combined);
|
|
50
|
+
console.log("Streaming result:");
|
|
51
|
+
console.log(result);
|
|
52
|
+
console.log("\u2713 Streaming API test passed");
|
|
53
|
+
}
|
|
54
|
+
async function testNdjsonToJson() {
|
|
55
|
+
console.log("\n=== Test: NDJSON to JSON Array ===");
|
|
56
|
+
const ndjsonData = `{"name":"Alice","age":30}
|
|
57
|
+
{"name":"Bob","age":25}
|
|
58
|
+
{"name":"Charlie","age":35}`;
|
|
59
|
+
const result = await convertToString(ndjsonData, {
|
|
60
|
+
inputFormat: "ndjson",
|
|
61
|
+
outputFormat: "json",
|
|
62
|
+
debug: false
|
|
63
|
+
});
|
|
64
|
+
console.log("Input NDJSON:");
|
|
65
|
+
console.log(ndjsonData);
|
|
66
|
+
console.log("\nOutput JSON:");
|
|
67
|
+
console.log(result);
|
|
68
|
+
const parsed = JSON.parse(result);
|
|
69
|
+
if (!Array.isArray(parsed)) {
|
|
70
|
+
throw new Error("Expected JSON array");
|
|
71
|
+
}
|
|
72
|
+
if (parsed.length !== 3) {
|
|
73
|
+
throw new Error(`Expected 3 items, got ${parsed.length}`);
|
|
74
|
+
}
|
|
75
|
+
console.log("\u2713 NDJSON to JSON test passed");
|
|
76
|
+
}
|
|
77
|
+
async function testQuotedCsv() {
|
|
78
|
+
console.log("\n=== Test: Quoted CSV with Commas ===");
|
|
79
|
+
const csvData = `name,description
|
|
80
|
+
"Alice","Hello, World"
|
|
81
|
+
"Bob","Test, with, commas"
|
|
82
|
+
"Charlie","Normal text"`;
|
|
83
|
+
const result = await convertToString(csvData, {
|
|
84
|
+
inputFormat: "csv",
|
|
85
|
+
outputFormat: "ndjson",
|
|
86
|
+
debug: false
|
|
87
|
+
});
|
|
88
|
+
console.log("Input CSV:");
|
|
89
|
+
console.log(csvData);
|
|
90
|
+
console.log("\nOutput NDJSON:");
|
|
91
|
+
console.log(result);
|
|
92
|
+
const lines = result.trim().split("\n");
|
|
93
|
+
const firstRecord = JSON.parse(lines[0]);
|
|
94
|
+
if (!firstRecord.description.includes(",")) {
|
|
95
|
+
throw new Error("Expected comma in description to be preserved");
|
|
96
|
+
}
|
|
97
|
+
console.log("\u2713 Quoted CSV test passed");
|
|
98
|
+
}
|
|
99
|
+
async function testStats() {
|
|
100
|
+
console.log("\n=== Test: Performance Stats ===");
|
|
101
|
+
const csvData = `name,age
|
|
102
|
+
Alice,30
|
|
103
|
+
Bob,25
|
|
104
|
+
Charlie,35`;
|
|
105
|
+
const result = await convert(new TextEncoder().encode(csvData), {
|
|
106
|
+
inputFormat: "csv",
|
|
107
|
+
outputFormat: "ndjson",
|
|
108
|
+
profile: true
|
|
109
|
+
});
|
|
110
|
+
console.log("\u2713 Stats test passed");
|
|
111
|
+
}
|
|
112
|
+
async function testPartialLines() {
|
|
113
|
+
console.log("\n=== Test: Partial Line Handling ===");
|
|
114
|
+
const buddy = await ConvertBuddy.create({
|
|
115
|
+
inputFormat: "csv",
|
|
116
|
+
outputFormat: "ndjson",
|
|
117
|
+
debug: false
|
|
118
|
+
});
|
|
119
|
+
const chunk1 = new TextEncoder().encode("name,age\nAli");
|
|
120
|
+
const chunk2 = new TextEncoder().encode("ce,30\nBob,25\n");
|
|
121
|
+
const out1 = buddy.push(chunk1);
|
|
122
|
+
const out2 = buddy.push(chunk2);
|
|
123
|
+
const final = buddy.finish();
|
|
124
|
+
const combined = new Uint8Array(out1.length + out2.length + final.length);
|
|
125
|
+
combined.set(out1, 0);
|
|
126
|
+
combined.set(out2, out1.length);
|
|
127
|
+
combined.set(final, out1.length + out2.length);
|
|
128
|
+
const result = new TextDecoder().decode(combined);
|
|
129
|
+
console.log("Result with partial lines:");
|
|
130
|
+
console.log(result);
|
|
131
|
+
const lines = result.trim().split("\n");
|
|
132
|
+
if (lines.length !== 2) {
|
|
133
|
+
throw new Error(`Expected 2 lines, got ${lines.length}`);
|
|
134
|
+
}
|
|
135
|
+
const firstRecord = JSON.parse(lines[0]);
|
|
136
|
+
if (firstRecord.name !== "Alice") {
|
|
137
|
+
throw new Error(`Expected name=Alice, got ${firstRecord.name}`);
|
|
138
|
+
}
|
|
139
|
+
console.log("\u2713 Partial line test passed");
|
|
140
|
+
}
|
|
141
|
+
async function testLargeDataset() {
|
|
142
|
+
console.log("\n=== Test: Large Dataset Performance ===");
|
|
143
|
+
const rows = 1e4;
|
|
144
|
+
let csv = "id,name,value\n";
|
|
145
|
+
for (let i = 0; i < rows; i++) {
|
|
146
|
+
csv += `${i},name_${i},${i * 100}
|
|
147
|
+
`;
|
|
148
|
+
}
|
|
149
|
+
const start = performance.now();
|
|
150
|
+
const result = await convert(new TextEncoder().encode(csv), {
|
|
151
|
+
inputFormat: "csv",
|
|
152
|
+
outputFormat: "ndjson",
|
|
153
|
+
profile: true
|
|
154
|
+
});
|
|
155
|
+
const end = performance.now();
|
|
156
|
+
const lines = new TextDecoder().decode(result).trim().split("\n");
|
|
157
|
+
console.log(`Processed ${rows} rows in ${(end - start).toFixed(2)}ms`);
|
|
158
|
+
console.log(`Throughput: ${(csv.length / (1024 * 1024) / ((end - start) / 1e3)).toFixed(2)} MB/s`);
|
|
159
|
+
console.log(`Output lines: ${lines.length}`);
|
|
160
|
+
if (lines.length !== rows) {
|
|
161
|
+
throw new Error(`Expected ${rows} lines, got ${lines.length}`);
|
|
162
|
+
}
|
|
163
|
+
console.log("\u2713 Large dataset test passed");
|
|
164
|
+
}
|
|
165
|
+
async function testEmptyAndWhitespace() {
|
|
166
|
+
console.log("\n=== Test: Empty Lines and Whitespace ===");
|
|
167
|
+
const csvData = `name,age
|
|
168
|
+
|
|
169
|
+
Alice,30
|
|
170
|
+
|
|
171
|
+
Bob,25
|
|
172
|
+
`;
|
|
173
|
+
const result = await convertToString(csvData, {
|
|
174
|
+
inputFormat: "csv",
|
|
175
|
+
outputFormat: "ndjson",
|
|
176
|
+
debug: false
|
|
177
|
+
});
|
|
178
|
+
console.log("Input CSV (with empty lines):");
|
|
179
|
+
console.log(JSON.stringify(csvData));
|
|
180
|
+
console.log("\nOutput NDJSON:");
|
|
181
|
+
console.log(result);
|
|
182
|
+
const lines = result.trim().split("\n").filter((l) => l.length > 0);
|
|
183
|
+
if (lines.length !== 2) {
|
|
184
|
+
throw new Error(`Expected 2 non-empty lines, got ${lines.length}`);
|
|
185
|
+
}
|
|
186
|
+
console.log("\u2713 Empty lines test passed");
|
|
187
|
+
}
|
|
188
|
+
async function runAllTests() {
|
|
189
|
+
console.log("\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557");
|
|
190
|
+
console.log("\u2551 Convert Buddy - Smoke Test Suite \u2551");
|
|
191
|
+
console.log("\u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D");
|
|
192
|
+
try {
|
|
193
|
+
await testBasicConversion();
|
|
194
|
+
await testStreamingAPI();
|
|
195
|
+
await testNdjsonToJson();
|
|
196
|
+
await testQuotedCsv();
|
|
197
|
+
await testStats();
|
|
198
|
+
await testPartialLines();
|
|
199
|
+
await testLargeDataset();
|
|
200
|
+
await testEmptyAndWhitespace();
|
|
201
|
+
console.log("\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557");
|
|
202
|
+
console.log("\u2551 \u2713 All tests passed successfully! \u2551");
|
|
203
|
+
console.log("\u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D\n");
|
|
204
|
+
} catch (error) {
|
|
205
|
+
console.error("\n\u2717 Test failed:", error);
|
|
206
|
+
process.exit(1);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
runAllTests();
|
|
210
|
+
//# sourceMappingURL=smoke-test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/smoke-test.ts"],"sourcesContent":["import { ConvertBuddy, convert, convertToString } from \"./index.js\";\r\nimport { createNodeTransform } from \"../src/node.js\";\r\n\r\n\r\nasync function testBasicConversion() {\r\n console.log(\"\\n=== Test: Basic CSV to NDJSON ===\");\r\n \r\n const csvData = `name,age,city\r\nAlice,30,New York\r\nBob,25,London\r\nCharlie,35,Tokyo`;\r\n\r\n const result = await convertToString(csvData, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n profile: true,\r\n });\r\n\r\n console.log(\"Input CSV:\");\r\n console.log(csvData);\r\n console.log(\"\\nOutput NDJSON:\");\r\n console.log(result);\r\n\r\n // Validate output\r\n const lines = result.trim().split(\"\\n\");\r\n if (lines.length !== 3) {\r\n throw new Error(`Expected 3 lines, got ${lines.length}`);\r\n }\r\n\r\n const firstRecord = JSON.parse(lines[0]);\r\n if (firstRecord.name !== \"Alice\") {\r\n throw new Error(`Expected name=Alice, got ${firstRecord.name}`);\r\n }\r\n\r\n console.log(\"✓ Basic conversion test passed\");\r\n}\r\n\r\nasync function testStreamingAPI() {\r\n console.log(\"\\n=== Test: Streaming API ===\");\r\n\r\n const buddy = await ConvertBuddy.create({\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n });\r\n\r\n // Push data in chunks\r\n const header = new TextEncoder().encode(\"name,age\\n\");\r\n const row1 = new TextEncoder().encode(\"Alice,30\\n\");\r\n const row2 = new TextEncoder().encode(\"Bob,25\\n\");\r\n\r\n const out1 = buddy.push(header);\r\n const out2 = buddy.push(row1);\r\n const out3 = buddy.push(row2);\r\n const final = buddy.finish();\r\n\r\n const combined = new Uint8Array(\r\n out1.length + out2.length + out3.length + final.length\r\n );\r\n combined.set(out1, 0);\r\n combined.set(out2, out1.length);\r\n combined.set(out3, out1.length + out2.length);\r\n combined.set(final, out1.length + out2.length + out3.length);\r\n\r\n const result = new TextDecoder().decode(combined);\r\n console.log(\"Streaming result:\");\r\n console.log(result);\r\n\r\n console.log(\"✓ Streaming API test passed\");\r\n}\r\n\r\nasync function testNdjsonToJson() {\r\n console.log(\"\\n=== Test: NDJSON to JSON Array ===\");\r\n\r\n const ndjsonData = `{\"name\":\"Alice\",\"age\":30}\r\n{\"name\":\"Bob\",\"age\":25}\r\n{\"name\":\"Charlie\",\"age\":35}`;\r\n\r\n const result = await convertToString(ndjsonData, {\r\n inputFormat: \"ndjson\",\r\n outputFormat: \"json\",\r\n debug: false,\r\n });\r\n\r\n console.log(\"Input NDJSON:\");\r\n console.log(ndjsonData);\r\n console.log(\"\\nOutput JSON:\");\r\n console.log(result);\r\n\r\n // Validate it's valid JSON array\r\n const parsed = JSON.parse(result);\r\n if (!Array.isArray(parsed)) {\r\n throw new Error(\"Expected JSON array\");\r\n }\r\n if (parsed.length !== 3) {\r\n throw new Error(`Expected 3 items, got ${parsed.length}`);\r\n }\r\n\r\n console.log(\"✓ NDJSON to JSON test passed\");\r\n}\r\n\r\nasync function testQuotedCsv() {\r\n console.log(\"\\n=== Test: Quoted CSV with Commas ===\");\r\n\r\n const csvData = `name,description\r\n\"Alice\",\"Hello, World\"\r\n\"Bob\",\"Test, with, commas\"\r\n\"Charlie\",\"Normal text\"`;\r\n\r\n const result = await convertToString(csvData, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n });\r\n\r\n console.log(\"Input CSV:\");\r\n console.log(csvData);\r\n console.log(\"\\nOutput NDJSON:\");\r\n console.log(result);\r\n\r\n const lines = result.trim().split(\"\\n\");\r\n const firstRecord = JSON.parse(lines[0]);\r\n \r\n if (!firstRecord.description.includes(\",\")) {\r\n throw new Error(\"Expected comma in description to be preserved\");\r\n }\r\n\r\n console.log(\"✓ Quoted CSV test passed\");\r\n}\r\n\r\nasync function testStats() {\r\n console.log(\"\\n=== Test: Performance Stats ===\");\r\n\r\n const csvData = `name,age\r\nAlice,30\r\nBob,25\r\nCharlie,35`;\r\n\r\n const result = await convert(new TextEncoder().encode(csvData), {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n });\r\n\r\n // Stats are logged automatically with profile: true\r\n console.log(\"✓ Stats test passed\");\r\n}\r\n\r\nasync function testPartialLines() {\r\n console.log(\"\\n=== Test: Partial Line Handling ===\");\r\n\r\n const buddy = await ConvertBuddy.create({\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n });\r\n\r\n // Split a line across chunks\r\n const chunk1 = new TextEncoder().encode(\"name,age\\nAli\");\r\n const chunk2 = new TextEncoder().encode(\"ce,30\\nBob,25\\n\");\r\n\r\n const out1 = buddy.push(chunk1);\r\n const out2 = buddy.push(chunk2);\r\n const final = buddy.finish();\r\n\r\n const combined = new Uint8Array(out1.length + out2.length + final.length);\r\n combined.set(out1, 0);\r\n combined.set(out2, out1.length);\r\n combined.set(final, out1.length + out2.length);\r\n\r\n const result = new TextDecoder().decode(combined);\r\n console.log(\"Result with partial lines:\");\r\n console.log(result);\r\n\r\n const lines = result.trim().split(\"\\n\");\r\n if (lines.length !== 2) {\r\n throw new Error(`Expected 2 lines, got ${lines.length}`);\r\n }\r\n\r\n const firstRecord = JSON.parse(lines[0]);\r\n if (firstRecord.name !== \"Alice\") {\r\n throw new Error(`Expected name=Alice, got ${firstRecord.name}`);\r\n }\r\n\r\n console.log(\"✓ Partial line test passed\");\r\n}\r\n\r\nasync function testLargeDataset() {\r\n console.log(\"\\n=== Test: Large Dataset Performance ===\");\r\n\r\n // Generate large CSV\r\n const rows = 10000;\r\n let csv = \"id,name,value\\n\";\r\n for (let i = 0; i < rows; i++) {\r\n csv += `${i},name_${i},${i * 100}\\n`;\r\n }\r\n\r\n const start = performance.now();\r\n const result = await convert(new TextEncoder().encode(csv), {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n profile: true,\r\n });\r\n const end = performance.now();\r\n\r\n const lines = new TextDecoder().decode(result).trim().split(\"\\n\");\r\n \r\n console.log(`Processed ${rows} rows in ${(end - start).toFixed(2)}ms`);\r\n console.log(`Throughput: ${((csv.length / (1024 * 1024)) / ((end - start) / 1000)).toFixed(2)} MB/s`);\r\n console.log(`Output lines: ${lines.length}`);\r\n\r\n if (lines.length !== rows) {\r\n throw new Error(`Expected ${rows} lines, got ${lines.length}`);\r\n }\r\n\r\n console.log(\"✓ Large dataset test passed\");\r\n}\r\n\r\nasync function testEmptyAndWhitespace() {\r\n console.log(\"\\n=== Test: Empty Lines and Whitespace ===\");\r\n\r\n const csvData = `name,age\r\n\r\nAlice,30\r\n\r\nBob,25\r\n`;\r\n\r\n const result = await convertToString(csvData, {\r\n inputFormat: \"csv\",\r\n outputFormat: \"ndjson\",\r\n debug: false,\r\n });\r\n\r\n console.log(\"Input CSV (with empty lines):\");\r\n console.log(JSON.stringify(csvData));\r\n console.log(\"\\nOutput NDJSON:\");\r\n console.log(result);\r\n\r\n const lines = result.trim().split(\"\\n\").filter(l => l.length > 0);\r\n if (lines.length !== 2) {\r\n throw new Error(`Expected 2 non-empty lines, got ${lines.length}`);\r\n }\r\n\r\n console.log(\"✓ Empty lines test passed\");\r\n}\r\n\r\n// Run all tests\r\nasync function runAllTests() {\r\n console.log(\"╔════════════════════════════════════════╗\");\r\n console.log(\"║ Convert Buddy - Smoke Test Suite ║\");\r\n console.log(\"╚════════════════════════════════════════╝\");\r\n\r\n try {\r\n await testBasicConversion();\r\n await testStreamingAPI();\r\n await testNdjsonToJson();\r\n await testQuotedCsv();\r\n await testStats();\r\n await testPartialLines();\r\n await testLargeDataset();\r\n await testEmptyAndWhitespace();\r\n\r\n console.log(\"\\n╔════════════════════════════════════════╗\");\r\n console.log(\"║ ✓ All tests passed successfully! ║\");\r\n console.log(\"╚════════════════════════════════════════╝\\n\");\r\n } catch (error) {\r\n console.error(\"\\n✗ Test failed:\", error);\r\n process.exit(1);\r\n }\r\n}\r\n\r\nrunAllTests();\r\n"],"mappings":"AAAA,SAAS,cAAc,SAAS,uBAAuB;AAIvD,eAAe,sBAAsB;AACnC,UAAQ,IAAI,qCAAqC;AAEjD,QAAM,UAAU;AAAA;AAAA;AAAA;AAKhB,QAAM,SAAS,MAAM,gBAAgB,SAAS;AAAA,IAC5C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,IACP,SAAS;AAAA,EACX,CAAC;AAED,UAAQ,IAAI,YAAY;AACxB,UAAQ,IAAI,OAAO;AACnB,UAAQ,IAAI,kBAAkB;AAC9B,UAAQ,IAAI,MAAM;AAGlB,QAAM,QAAQ,OAAO,KAAK,EAAE,MAAM,IAAI;AACtC,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,IAAI,MAAM,yBAAyB,MAAM,MAAM,EAAE;AAAA,EACzD;AAEA,QAAM,cAAc,KAAK,MAAM,MAAM,CAAC,CAAC;AACvC,MAAI,YAAY,SAAS,SAAS;AAChC,UAAM,IAAI,MAAM,4BAA4B,YAAY,IAAI,EAAE;AAAA,EAChE;AAEA,UAAQ,IAAI,qCAAgC;AAC9C;AAEA,eAAe,mBAAmB;AAChC,UAAQ,IAAI,+BAA+B;AAE3C,QAAM,QAAQ,MAAM,aAAa,OAAO;AAAA,IACtC,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAGD,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,YAAY;AACpD,QAAM,OAAO,IAAI,YAAY,EAAE,OAAO,YAAY;AAClD,QAAM,OAAO,IAAI,YAAY,EAAE,OAAO,UAAU;AAEhD,QAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,QAAM,OAAO,MAAM,KAAK,IAAI;AAC5B,QAAM,OAAO,MAAM,KAAK,IAAI;AAC5B,QAAM,QAAQ,MAAM,OAAO;AAE3B,QAAM,WAAW,IAAI;AAAA,IACnB,KAAK,SAAS,KAAK,SAAS,KAAK,SAAS,MAAM;AAAA,EAClD;AACA,WAAS,IAAI,MAAM,CAAC;AACpB,WAAS,IAAI,MAAM,KAAK,MAAM;AAC9B,WAAS,IAAI,MAAM,KAAK,SAAS,KAAK,MAAM;AAC5C,WAAS,IAAI,OAAO,KAAK,SAAS,KAAK,SAAS,KAAK,MAAM;AAE3D,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,QAAQ;AAChD,UAAQ,IAAI,mBAAmB;AAC/B,UAAQ,IAAI,MAAM;AAElB,UAAQ,IAAI,kCAA6B;AAC3C;AAEA,eAAe,mBAAmB;AAChC,UAAQ,IAAI,sCAAsC;AAElD,QAAM,aAAa;AAAA;AAAA;AAInB,QAAM,SAAS,MAAM,gBAAgB,YAAY;AAAA,IAC/C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAED,UAAQ,IAAI,eAAe;AAC3B,UAAQ,IAAI,UAAU;AACtB,UAAQ,IAAI,gBAAgB;AAC5B,UAAQ,IAAI,MAAM;AAGlB,QAAM,SAAS,KAAK,MAAM,MAAM;AAChC,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,UAAM,IAAI,MAAM,qBAAqB;AAAA,EACvC;AACA,MAAI,OAAO,WAAW,GAAG;AACvB,UAAM,IAAI,MAAM,yBAAyB,OAAO,MAAM,EAAE;AAAA,EAC1D;AAEA,UAAQ,IAAI,mCAA8B;AAC5C;AAEA,eAAe,gBAAgB;AAC7B,UAAQ,IAAI,wCAAwC;AAEpD,QAAM,UAAU;AAAA;AAAA;AAAA;AAKhB,QAAM,SAAS,MAAM,gBAAgB,SAAS;AAAA,IAC5C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAED,UAAQ,IAAI,YAAY;AACxB,UAAQ,IAAI,OAAO;AACnB,UAAQ,IAAI,kBAAkB;AAC9B,UAAQ,IAAI,MAAM;AAElB,QAAM,QAAQ,OAAO,KAAK,EAAE,MAAM,IAAI;AACtC,QAAM,cAAc,KAAK,MAAM,MAAM,CAAC,CAAC;AAEvC,MAAI,CAAC,YAAY,YAAY,SAAS,GAAG,GAAG;AAC1C,UAAM,IAAI,MAAM,+CAA+C;AAAA,EACjE;AAEA,UAAQ,IAAI,+BAA0B;AACxC;AAEA,eAAe,YAAY;AACzB,UAAQ,IAAI,mCAAmC;AAE/C,QAAM,UAAU;AAAA;AAAA;AAAA;AAKhB,QAAM,SAAS,MAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,OAAO,GAAG;AAAA,IAC9D,aAAa;AAAA,IACb,cAAc;AAAA,IACd,SAAS;AAAA,EACX,CAAC;AAGD,UAAQ,IAAI,0BAAqB;AACnC;AAEA,eAAe,mBAAmB;AAChC,UAAQ,IAAI,uCAAuC;AAEnD,QAAM,QAAQ,MAAM,aAAa,OAAO;AAAA,IACtC,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAGD,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,eAAe;AACvD,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,iBAAiB;AAEzD,QAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,QAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,QAAM,QAAQ,MAAM,OAAO;AAE3B,QAAM,WAAW,IAAI,WAAW,KAAK,SAAS,KAAK,SAAS,MAAM,MAAM;AACxE,WAAS,IAAI,MAAM,CAAC;AACpB,WAAS,IAAI,MAAM,KAAK,MAAM;AAC9B,WAAS,IAAI,OAAO,KAAK,SAAS,KAAK,MAAM;AAE7C,QAAM,SAAS,IAAI,YAAY,EAAE,OAAO,QAAQ;AAChD,UAAQ,IAAI,4BAA4B;AACxC,UAAQ,IAAI,MAAM;AAElB,QAAM,QAAQ,OAAO,KAAK,EAAE,MAAM,IAAI;AACtC,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,IAAI,MAAM,yBAAyB,MAAM,MAAM,EAAE;AAAA,EACzD;AAEA,QAAM,cAAc,KAAK,MAAM,MAAM,CAAC,CAAC;AACvC,MAAI,YAAY,SAAS,SAAS;AAChC,UAAM,IAAI,MAAM,4BAA4B,YAAY,IAAI,EAAE;AAAA,EAChE;AAEA,UAAQ,IAAI,iCAA4B;AAC1C;AAEA,eAAe,mBAAmB;AAChC,UAAQ,IAAI,2CAA2C;AAGvD,QAAM,OAAO;AACb,MAAI,MAAM;AACV,WAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,WAAO,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG;AAAA;AAAA,EAClC;AAEA,QAAM,QAAQ,YAAY,IAAI;AAC9B,QAAM,SAAS,MAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,GAAG,GAAG;AAAA,IAC1D,aAAa;AAAA,IACb,cAAc;AAAA,IACd,SAAS;AAAA,EACX,CAAC;AACD,QAAM,MAAM,YAAY,IAAI;AAE5B,QAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI;AAEhE,UAAQ,IAAI,aAAa,IAAI,aAAa,MAAM,OAAO,QAAQ,CAAC,CAAC,IAAI;AACrE,UAAQ,IAAI,gBAAiB,IAAI,UAAU,OAAO,UAAW,MAAM,SAAS,MAAO,QAAQ,CAAC,CAAC,OAAO;AACpG,UAAQ,IAAI,iBAAiB,MAAM,MAAM,EAAE;AAE3C,MAAI,MAAM,WAAW,MAAM;AACzB,UAAM,IAAI,MAAM,YAAY,IAAI,eAAe,MAAM,MAAM,EAAE;AAAA,EAC/D;AAEA,UAAQ,IAAI,kCAA6B;AAC3C;AAEA,eAAe,yBAAyB;AACtC,UAAQ,IAAI,4CAA4C;AAExD,QAAM,UAAU;AAAA;AAAA;AAAA;AAAA;AAAA;AAOhB,QAAM,SAAS,MAAM,gBAAgB,SAAS;AAAA,IAC5C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,OAAO;AAAA,EACT,CAAC;AAED,UAAQ,IAAI,+BAA+B;AAC3C,UAAQ,IAAI,KAAK,UAAU,OAAO,CAAC;AACnC,UAAQ,IAAI,kBAAkB;AAC9B,UAAQ,IAAI,MAAM;AAElB,QAAM,QAAQ,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,OAAO,OAAK,EAAE,SAAS,CAAC;AAChE,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,IAAI,MAAM,mCAAmC,MAAM,MAAM,EAAE;AAAA,EACnE;AAEA,UAAQ,IAAI,gCAA2B;AACzC;AAGA,eAAe,cAAc;AAC3B,UAAQ,IAAI,8PAA4C;AACxD,UAAQ,IAAI,qDAA2C;AACvD,UAAQ,IAAI,8PAA4C;AAExD,MAAI;AACF,UAAM,oBAAoB;AAC1B,UAAM,iBAAiB;AACvB,UAAM,iBAAiB;AACvB,UAAM,cAAc;AACpB,UAAM,UAAU;AAChB,UAAM,iBAAiB;AACvB,UAAM,iBAAiB;AACvB,UAAM,uBAAuB;AAE7B,YAAQ,IAAI,gQAA8C;AAC1D,YAAQ,IAAI,0DAA2C;AACvD,YAAQ,IAAI,gQAA8C;AAAA,EAC5D,SAAS,OAAO;AACd,YAAQ,MAAM,yBAAoB,KAAK;AACvC,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,YAAY;","names":[]}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/* tslint:disable */
|
|
2
|
+
/* eslint-disable */
|
|
3
|
+
/**
|
|
4
|
+
* Detect CSV fields and delimiter from a sample of bytes.
|
|
5
|
+
*/
|
|
6
|
+
export function detectCsvFields(sample: Uint8Array): any;
|
|
7
|
+
export function init(debug_enabled: boolean): void;
|
|
8
|
+
/**
|
|
9
|
+
* Detect XML elements from a sample of bytes.
|
|
10
|
+
*/
|
|
11
|
+
export function detectXmlElements(sample: Uint8Array): any;
|
|
12
|
+
/**
|
|
13
|
+
* Detect the input format from a sample of bytes.
|
|
14
|
+
*/
|
|
15
|
+
export function detectFormat(sample: Uint8Array): string | undefined;
|
|
16
|
+
/**
|
|
17
|
+
* A streaming converter state machine.
|
|
18
|
+
* Converts between CSV, NDJSON, JSON, and XML formats with high performance.
|
|
19
|
+
*/
|
|
20
|
+
export class Converter {
|
|
21
|
+
free(): void;
|
|
22
|
+
/**
|
|
23
|
+
* Create a new converter with specific configuration
|
|
24
|
+
*/
|
|
25
|
+
static withConfig(debug: boolean, input_format: string, output_format: string, chunk_target_bytes: number, enable_stats: boolean, csv_config: any, xml_config: any): Converter;
|
|
26
|
+
constructor(debug: boolean);
|
|
27
|
+
/**
|
|
28
|
+
* Push a chunk of bytes. Returns converted output bytes for that chunk.
|
|
29
|
+
*/
|
|
30
|
+
push(chunk: Uint8Array): Uint8Array;
|
|
31
|
+
/**
|
|
32
|
+
* Finish the stream and return any remaining buffered output.
|
|
33
|
+
*/
|
|
34
|
+
finish(): Uint8Array;
|
|
35
|
+
/**
|
|
36
|
+
* Get performance statistics
|
|
37
|
+
*/
|
|
38
|
+
getStats(): Stats;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Performance statistics for the converter
|
|
42
|
+
*/
|
|
43
|
+
export class Stats {
|
|
44
|
+
private constructor();
|
|
45
|
+
free(): void;
|
|
46
|
+
readonly parse_time_ms: number;
|
|
47
|
+
readonly write_time_ms: number;
|
|
48
|
+
readonly max_buffer_size: number;
|
|
49
|
+
readonly records_processed: number;
|
|
50
|
+
readonly transform_time_ms: number;
|
|
51
|
+
readonly current_partial_size: number;
|
|
52
|
+
readonly throughput_mb_per_sec: number;
|
|
53
|
+
readonly bytes_in: number;
|
|
54
|
+
readonly bytes_out: number;
|
|
55
|
+
readonly chunks_in: number;
|
|
56
|
+
}
|