convert-buddy-js 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -0
- package/dist/{chunk-B44HYXEP.js → chunk-27H3T556.js} +11 -3
- package/dist/src/index.d.ts +54 -0
- package/dist/src/index.js +15 -0
- package/package.json +10 -7
- package/dist/chunk-HFHFJO2R.js +0 -44
- package/dist/chunk-VUNV25KB.js +0 -16
- package/dist/index.d.ts +0 -13
- package/dist/index.js +0 -7
package/README.md
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# convert-buddy-js
|
|
2
|
+
|
|
3
|
+
A high-performance, streaming-first parser and converter for CSV, XML, NDJSON, and JSON. `convert-buddy-js` is a TypeScript wrapper around a Rust/WASM core, offering fast parsing and multiple usage styles for Node.js and browsers.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install convert-buddy-js
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Usage
|
|
12
|
+
|
|
13
|
+
### Convert a full string or buffer
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { convertToString } from "convert-buddy-js";
|
|
17
|
+
|
|
18
|
+
const csv = `name,age\nAda,36\nLinus,54`;
|
|
19
|
+
|
|
20
|
+
const output = await convertToString(csv, {
|
|
21
|
+
inputFormat: "csv",
|
|
22
|
+
outputFormat: "ndjson",
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
console.log(output);
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### Manual streaming (chunked)
|
|
29
|
+
|
|
30
|
+
```ts
|
|
31
|
+
import { ConvertBuddy } from "convert-buddy-js";
|
|
32
|
+
|
|
33
|
+
const buddy = await ConvertBuddy.create({
|
|
34
|
+
inputFormat: "xml",
|
|
35
|
+
outputFormat: "ndjson",
|
|
36
|
+
xmlConfig: { recordElement: "row", includeAttributes: true },
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const chunkOutput = buddy.push(new Uint8Array([/* bytes */]));
|
|
40
|
+
const finalOutput = buddy.finish();
|
|
41
|
+
|
|
42
|
+
console.log(buddy.stats());
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### Node.js Transform stream
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
import { createNodeTransform } from "convert-buddy-js";
|
|
49
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
50
|
+
|
|
51
|
+
const transform = await createNodeTransform({
|
|
52
|
+
inputFormat: "csv",
|
|
53
|
+
outputFormat: "ndjson",
|
|
54
|
+
csvConfig: { hasHeaders: true },
|
|
55
|
+
profile: true,
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
createReadStream("input.csv")
|
|
59
|
+
.pipe(transform)
|
|
60
|
+
.pipe(createWriteStream("output.ndjson"));
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Web Streams
|
|
64
|
+
|
|
65
|
+
```ts
|
|
66
|
+
import { ConvertBuddyTransformStream } from "convert-buddy-js";
|
|
67
|
+
|
|
68
|
+
const transform = new ConvertBuddyTransformStream({
|
|
69
|
+
inputFormat: "csv",
|
|
70
|
+
outputFormat: "ndjson",
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
const response = await fetch("/data.csv");
|
|
74
|
+
const outputStream = response.body?.pipeThrough(transform);
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Configuration
|
|
78
|
+
|
|
79
|
+
### Formats
|
|
80
|
+
|
|
81
|
+
- `csv`
|
|
82
|
+
- `xml`
|
|
83
|
+
- `ndjson`
|
|
84
|
+
- `json`
|
|
85
|
+
|
|
86
|
+
### CSV options
|
|
87
|
+
|
|
88
|
+
```ts
|
|
89
|
+
{
|
|
90
|
+
csvConfig: {
|
|
91
|
+
delimiter: ",",
|
|
92
|
+
quote: '"',
|
|
93
|
+
hasHeaders: true,
|
|
94
|
+
trimWhitespace: false,
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
### XML options
|
|
100
|
+
|
|
101
|
+
```ts
|
|
102
|
+
{
|
|
103
|
+
xmlConfig: {
|
|
104
|
+
recordElement: "row",
|
|
105
|
+
trimText: true,
|
|
106
|
+
includeAttributes: true,
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Performance options
|
|
112
|
+
|
|
113
|
+
```ts
|
|
114
|
+
{
|
|
115
|
+
chunkTargetBytes: 1024 * 1024,
|
|
116
|
+
parallelism: 4,
|
|
117
|
+
profile: true,
|
|
118
|
+
debug: false,
|
|
119
|
+
}
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## How it works
|
|
123
|
+
|
|
124
|
+
- **Rust core** (`crates/convert-buddy`) implements streaming parsers and stats tracking.
|
|
125
|
+
- **WASM bindings** are generated via `wasm-bindgen` and bundled into this package.
|
|
126
|
+
- **TypeScript wrapper** (`src/index.ts`) exposes the `ConvertBuddy` class and stream adapters.
|
|
127
|
+
|
|
128
|
+
### Build (repository)
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
npm install
|
|
132
|
+
npm -w convert-buddy-js run build
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
### Benchmarks (repository)
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
npm -w convert-buddy-js run bench:competitors
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Comparison to similar tools
|
|
142
|
+
|
|
143
|
+
Convert Buddy targets multi-format conversion with a unified, streaming API. Most existing libraries specialize:
|
|
144
|
+
|
|
145
|
+
- CSV-only parsers (e.g., PapaParse, `csv-parse`, `fast-csv`)
|
|
146
|
+
- XML-only parsers
|
|
147
|
+
- JSON/NDJSON-only utilities
|
|
148
|
+
|
|
149
|
+
**Where Convert Buddy shines**
|
|
150
|
+
- Large dataset throughput (WASM + fast-path parsing)
|
|
151
|
+
- Streaming conversions without loading full files into memory
|
|
152
|
+
- Unified API across CSV, XML, NDJSON, JSON
|
|
153
|
+
|
|
154
|
+
**Where others may be better**
|
|
155
|
+
- Tiny inputs (WASM setup overhead can dominate)
|
|
156
|
+
- Advanced format-specific features
|
|
157
|
+
- Long-tail ecosystem plugins
|
|
158
|
+
|
|
159
|
+
Benchmarks live in `packages/convert-buddy-js/bench/` and include honest cases where Convert Buddy is slower to help users choose the right tool.
|
|
160
|
+
|
|
161
|
+
## License
|
|
162
|
+
|
|
163
|
+
MIT
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
// src/index.ts
|
|
2
|
-
import { Transform } from "stream";
|
|
3
2
|
async function loadWasmModule() {
|
|
4
3
|
const isNode = typeof process !== "undefined" && !!process.versions?.node;
|
|
5
4
|
if (isNode) {
|
|
@@ -56,9 +55,18 @@ var ConvertBuddy = class _ConvertBuddy {
|
|
|
56
55
|
return this.converter.getStats();
|
|
57
56
|
}
|
|
58
57
|
};
|
|
59
|
-
function
|
|
58
|
+
async function loadNodeTransform() {
|
|
59
|
+
const isNode = typeof process !== "undefined" && !!process.versions?.node;
|
|
60
|
+
if (!isNode) {
|
|
61
|
+
throw new Error("createNodeTransform is only available in Node.js runtimes.");
|
|
62
|
+
}
|
|
63
|
+
const streamModule = await import("stream");
|
|
64
|
+
return streamModule.Transform;
|
|
65
|
+
}
|
|
66
|
+
async function createNodeTransform(opts = {}) {
|
|
60
67
|
let buddy = null;
|
|
61
68
|
let initPromise = null;
|
|
69
|
+
const Transform = await loadNodeTransform();
|
|
62
70
|
const transform = new Transform({
|
|
63
71
|
async transform(chunk, encoding, callback) {
|
|
64
72
|
try {
|
|
@@ -158,4 +166,4 @@ export {
|
|
|
158
166
|
convert,
|
|
159
167
|
convertToString
|
|
160
168
|
};
|
|
161
|
-
//# sourceMappingURL=chunk-
|
|
169
|
+
//# sourceMappingURL=chunk-27H3T556.js.map
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
|
+
|
|
3
|
+
type Format = "csv" | "ndjson" | "json" | "xml";
|
|
4
|
+
type ConvertBuddyOptions = {
|
|
5
|
+
debug?: boolean;
|
|
6
|
+
profile?: boolean;
|
|
7
|
+
inputFormat?: Format;
|
|
8
|
+
outputFormat?: Format;
|
|
9
|
+
chunkTargetBytes?: number;
|
|
10
|
+
parallelism?: number;
|
|
11
|
+
csvConfig?: CsvConfig;
|
|
12
|
+
xmlConfig?: XmlConfig;
|
|
13
|
+
};
|
|
14
|
+
type CsvConfig = {
|
|
15
|
+
delimiter?: string;
|
|
16
|
+
quote?: string;
|
|
17
|
+
hasHeaders?: boolean;
|
|
18
|
+
trimWhitespace?: boolean;
|
|
19
|
+
};
|
|
20
|
+
type XmlConfig = {
|
|
21
|
+
recordElement?: string;
|
|
22
|
+
trimText?: boolean;
|
|
23
|
+
includeAttributes?: boolean;
|
|
24
|
+
};
|
|
25
|
+
type Stats = {
|
|
26
|
+
bytesIn: number;
|
|
27
|
+
bytesOut: number;
|
|
28
|
+
chunksIn: number;
|
|
29
|
+
recordsProcessed: number;
|
|
30
|
+
parseTimeMs: number;
|
|
31
|
+
transformTimeMs: number;
|
|
32
|
+
writeTimeMs: number;
|
|
33
|
+
maxBufferSize: number;
|
|
34
|
+
currentPartialSize: number;
|
|
35
|
+
throughputMbPerSec: number;
|
|
36
|
+
};
|
|
37
|
+
declare class ConvertBuddy {
|
|
38
|
+
private converter;
|
|
39
|
+
private debug;
|
|
40
|
+
private profile;
|
|
41
|
+
private constructor();
|
|
42
|
+
static create(opts?: ConvertBuddyOptions): Promise<ConvertBuddy>;
|
|
43
|
+
push(chunk: Uint8Array): Uint8Array;
|
|
44
|
+
finish(): Uint8Array;
|
|
45
|
+
stats(): Stats;
|
|
46
|
+
}
|
|
47
|
+
declare function createNodeTransform(opts?: ConvertBuddyOptions): Promise<Transform>;
|
|
48
|
+
declare class ConvertBuddyTransformStream extends TransformStream<Uint8Array, Uint8Array> {
|
|
49
|
+
constructor(opts?: ConvertBuddyOptions);
|
|
50
|
+
}
|
|
51
|
+
declare function convert(input: Uint8Array | string, opts?: ConvertBuddyOptions): Promise<Uint8Array>;
|
|
52
|
+
declare function convertToString(input: Uint8Array | string, opts?: ConvertBuddyOptions): Promise<string>;
|
|
53
|
+
|
|
54
|
+
export { ConvertBuddy, type ConvertBuddyOptions, ConvertBuddyTransformStream, type CsvConfig, type Format, type Stats, type XmlConfig, convert, convertToString, createNodeTransform };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ConvertBuddy,
|
|
3
|
+
ConvertBuddyTransformStream,
|
|
4
|
+
convert,
|
|
5
|
+
convertToString,
|
|
6
|
+
createNodeTransform
|
|
7
|
+
} from "../chunk-27H3T556.js";
|
|
8
|
+
export {
|
|
9
|
+
ConvertBuddy,
|
|
10
|
+
ConvertBuddyTransformStream,
|
|
11
|
+
convert,
|
|
12
|
+
convertToString,
|
|
13
|
+
createNodeTransform
|
|
14
|
+
};
|
|
15
|
+
//# sourceMappingURL=index.js.map
|
package/package.json
CHANGED
|
@@ -1,22 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "convert-buddy-js",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "TypeScript wrapper for convert-buddy (Rust/WASM core)",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
|
-
"main": "./dist/index.js",
|
|
8
|
-
"types": "./dist/index.d.ts",
|
|
7
|
+
"main": "./dist/src/index.js",
|
|
8
|
+
"types": "./dist/src/index.d.ts",
|
|
9
9
|
"exports": {
|
|
10
10
|
".": {
|
|
11
|
-
"types": "./dist/index.d.ts",
|
|
12
|
-
"default": "./dist/index.js"
|
|
11
|
+
"types": "./dist/src/index.d.ts",
|
|
12
|
+
"default": "./dist/src/index.js"
|
|
13
13
|
}
|
|
14
14
|
},
|
|
15
15
|
"files": [
|
|
16
|
-
"dist/index.js",
|
|
17
|
-
"dist/index.d.ts",
|
|
16
|
+
"dist/src/index.js",
|
|
17
|
+
"dist/src/index.d.ts",
|
|
18
18
|
"dist/chunk-*.js",
|
|
19
19
|
"dist/chunk-*.d.ts",
|
|
20
|
+
"dist/*.js",
|
|
21
|
+
"dist/*.d.ts",
|
|
20
22
|
"wasm",
|
|
21
23
|
"wasm-node.cjs",
|
|
22
24
|
"!dist/smoke-test.*",
|
|
@@ -45,6 +47,7 @@
|
|
|
45
47
|
"@types/papaparse": "^5.3.15",
|
|
46
48
|
"csv-parse": "^5.6.0",
|
|
47
49
|
"fast-csv": "^5.0.1",
|
|
50
|
+
"fast-xml-parser": "^4.5.1",
|
|
48
51
|
"papaparse": "^5.4.1",
|
|
49
52
|
"tsup": "^8.0.0",
|
|
50
53
|
"typescript": "^5.5.0"
|
package/dist/chunk-HFHFJO2R.js
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
// src/index.ts
|
|
2
|
-
async function loadWasmModule() {
|
|
3
|
-
const isNode = typeof process !== "undefined" && !!process.versions?.node;
|
|
4
|
-
if (isNode) {
|
|
5
|
-
const { createRequire } = await import("module");
|
|
6
|
-
const require2 = createRequire(import.meta.url);
|
|
7
|
-
const mod2 = require2("../wasm-node.cjs");
|
|
8
|
-
return mod2;
|
|
9
|
-
}
|
|
10
|
-
const mod = await import("../wasm/web/convert_buddy.js");
|
|
11
|
-
return mod;
|
|
12
|
-
}
|
|
13
|
-
var ConvertBuddy = class _ConvertBuddy {
|
|
14
|
-
converter;
|
|
15
|
-
debug;
|
|
16
|
-
constructor(converter, debug) {
|
|
17
|
-
this.converter = converter;
|
|
18
|
-
this.debug = debug;
|
|
19
|
-
}
|
|
20
|
-
static async create(opts = {}) {
|
|
21
|
-
const debug = !!opts.debug;
|
|
22
|
-
const wasmModule = await loadWasmModule();
|
|
23
|
-
if (typeof wasmModule.default === "function") {
|
|
24
|
-
await wasmModule.default();
|
|
25
|
-
}
|
|
26
|
-
wasmModule.init(debug);
|
|
27
|
-
const converter = new wasmModule.Converter(debug);
|
|
28
|
-
if (debug) console.log("[convert-buddy-js] initialized with debug logging");
|
|
29
|
-
return new _ConvertBuddy(converter, debug);
|
|
30
|
-
}
|
|
31
|
-
push(chunk) {
|
|
32
|
-
if (this.debug) console.log("[convert-buddy-js] push", chunk.byteLength);
|
|
33
|
-
return this.converter.push(chunk);
|
|
34
|
-
}
|
|
35
|
-
finish() {
|
|
36
|
-
if (this.debug) console.log("[convert-buddy-js] finish");
|
|
37
|
-
return this.converter.finish();
|
|
38
|
-
}
|
|
39
|
-
};
|
|
40
|
-
|
|
41
|
-
export {
|
|
42
|
-
ConvertBuddy
|
|
43
|
-
};
|
|
44
|
-
//# sourceMappingURL=chunk-HFHFJO2R.js.map
|
package/dist/chunk-VUNV25KB.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
2
|
-
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
3
|
-
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
4
|
-
}) : x)(function(x) {
|
|
5
|
-
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
6
|
-
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
7
|
-
});
|
|
8
|
-
var __commonJS = (cb, mod) => function __require2() {
|
|
9
|
-
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
export {
|
|
13
|
-
__require,
|
|
14
|
-
__commonJS
|
|
15
|
-
};
|
|
16
|
-
//# sourceMappingURL=chunk-VUNV25KB.js.map
|
package/dist/index.d.ts
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
type ConvertBuddyOptions = {
|
|
2
|
-
debug?: boolean;
|
|
3
|
-
};
|
|
4
|
-
declare class ConvertBuddy {
|
|
5
|
-
private converter;
|
|
6
|
-
private debug;
|
|
7
|
-
private constructor();
|
|
8
|
-
static create(opts?: ConvertBuddyOptions): Promise<ConvertBuddy>;
|
|
9
|
-
push(chunk: Uint8Array): Uint8Array;
|
|
10
|
-
finish(): Uint8Array;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export { ConvertBuddy, type ConvertBuddyOptions };
|