jtcsv 2.2.8 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -1
- package/bin/jtcsv.js +891 -821
- package/bin/jtcsv.ts +2534 -0
- package/csv-to-json.js +168 -145
- package/dist/jtcsv-core.cjs.js +1407 -0
- package/dist/jtcsv-core.cjs.js.map +1 -0
- package/dist/jtcsv-core.esm.js +1379 -0
- package/dist/jtcsv-core.esm.js.map +1 -0
- package/dist/jtcsv-core.umd.js +1413 -0
- package/dist/jtcsv-core.umd.js.map +1 -0
- package/dist/jtcsv-full.cjs.js +1912 -0
- package/dist/jtcsv-full.cjs.js.map +1 -0
- package/dist/jtcsv-full.esm.js +1880 -0
- package/dist/jtcsv-full.esm.js.map +1 -0
- package/dist/jtcsv-full.umd.js +1918 -0
- package/dist/jtcsv-full.umd.js.map +1 -0
- package/dist/jtcsv-workers.esm.js +759 -0
- package/dist/jtcsv-workers.esm.js.map +1 -0
- package/dist/jtcsv-workers.umd.js +773 -0
- package/dist/jtcsv-workers.umd.js.map +1 -0
- package/dist/jtcsv.cjs.js +61 -19
- package/dist/jtcsv.cjs.js.map +1 -1
- package/dist/jtcsv.esm.js +61 -19
- package/dist/jtcsv.esm.js.map +1 -1
- package/dist/jtcsv.umd.js +61 -19
- package/dist/jtcsv.umd.js.map +1 -1
- package/errors.js +188 -2
- package/examples/advanced/conditional-transformations.js +446 -0
- package/examples/advanced/conditional-transformations.ts +446 -0
- package/examples/advanced/csv-parser.worker.js +89 -0
- package/examples/advanced/csv-parser.worker.ts +89 -0
- package/examples/advanced/nested-objects-example.js +306 -0
- package/examples/advanced/nested-objects-example.ts +306 -0
- package/examples/advanced/performance-optimization.js +504 -0
- package/examples/advanced/performance-optimization.ts +504 -0
- package/examples/advanced/run-demo-server.js +116 -0
- package/examples/advanced/run-demo-server.ts +116 -0
- package/examples/advanced/web-worker-usage.html +874 -0
- package/examples/async-multithreaded-example.ts +335 -0
- package/examples/cli-advanced-usage.md +288 -0
- package/examples/cli-batch-processing.ts +38 -0
- package/examples/cli-tool.js +0 -3
- package/examples/cli-tool.ts +183 -0
- package/examples/error-handling.js +21 -7
- package/examples/error-handling.ts +356 -0
- package/examples/express-api.js +0 -3
- package/examples/express-api.ts +164 -0
- package/examples/large-dataset-example.js +0 -3
- package/examples/large-dataset-example.ts +204 -0
- package/examples/ndjson-processing.js +1 -1
- package/examples/ndjson-processing.ts +456 -0
- package/examples/plugin-excel-exporter.js +3 -4
- package/examples/plugin-excel-exporter.ts +406 -0
- package/examples/react-integration.tsx +637 -0
- package/examples/schema-validation.ts +640 -0
- package/examples/simple-usage.js +254 -254
- package/examples/simple-usage.ts +194 -0
- package/examples/streaming-example.js +4 -5
- package/examples/streaming-example.ts +419 -0
- package/examples/web-workers-advanced.ts +28 -0
- package/index.d.ts +1 -3
- package/index.js +15 -1
- package/json-save.js +1 -0
- package/json-to-csv.js +160 -18
- package/package.json +69 -10
- package/plugins/express-middleware/README.md +21 -2
- package/plugins/express-middleware/example.js +3 -4
- package/plugins/express-middleware/example.ts +135 -0
- package/plugins/express-middleware/index.d.ts +1 -1
- package/plugins/express-middleware/index.js +270 -118
- package/plugins/express-middleware/index.ts +557 -0
- package/plugins/fastify-plugin/index.js +2 -4
- package/plugins/fastify-plugin/index.ts +443 -0
- package/plugins/hono/index.ts +226 -0
- package/plugins/nestjs/index.ts +201 -0
- package/plugins/nextjs-api/examples/ConverterComponent.tsx +386 -0
- package/plugins/nextjs-api/examples/api-convert.js +0 -2
- package/plugins/nextjs-api/examples/api-convert.ts +67 -0
- package/plugins/nextjs-api/index.tsx +339 -0
- package/plugins/nextjs-api/route.js +2 -3
- package/plugins/nextjs-api/route.ts +370 -0
- package/plugins/nuxt/index.ts +94 -0
- package/plugins/nuxt/runtime/composables/useJtcsv.ts +100 -0
- package/plugins/nuxt/runtime/plugin.ts +71 -0
- package/plugins/remix/index.js +1 -1
- package/plugins/remix/index.ts +260 -0
- package/plugins/sveltekit/index.js +1 -1
- package/plugins/sveltekit/index.ts +301 -0
- package/plugins/trpc/index.ts +267 -0
- package/src/browser/browser-functions.ts +402 -0
- package/src/browser/core.js +92 -0
- package/src/browser/core.ts +152 -0
- package/src/browser/csv-to-json-browser.d.ts +3 -0
- package/src/browser/csv-to-json-browser.js +36 -14
- package/src/browser/csv-to-json-browser.ts +264 -0
- package/src/browser/errors-browser.ts +303 -0
- package/src/browser/extensions/plugins.js +92 -0
- package/src/browser/extensions/plugins.ts +93 -0
- package/src/browser/extensions/workers.js +39 -0
- package/src/browser/extensions/workers.ts +39 -0
- package/src/browser/globals.d.ts +5 -0
- package/src/browser/index.ts +192 -0
- package/src/browser/json-to-csv-browser.d.ts +3 -0
- package/src/browser/json-to-csv-browser.js +13 -3
- package/src/browser/json-to-csv-browser.ts +262 -0
- package/src/browser/streams.js +12 -2
- package/src/browser/streams.ts +336 -0
- package/src/browser/workers/csv-parser.worker.ts +377 -0
- package/src/browser/workers/worker-pool.ts +548 -0
- package/src/core/delimiter-cache.js +22 -8
- package/src/core/delimiter-cache.ts +310 -0
- package/src/core/node-optimizations.ts +449 -0
- package/src/core/plugin-system.js +29 -11
- package/src/core/plugin-system.ts +400 -0
- package/src/core/transform-hooks.ts +558 -0
- package/src/engines/fast-path-engine-new.ts +347 -0
- package/src/engines/fast-path-engine.ts +854 -0
- package/src/errors.ts +72 -0
- package/src/formats/ndjson-parser.ts +469 -0
- package/src/formats/tsv-parser.ts +334 -0
- package/src/index-with-plugins.js +16 -9
- package/src/index-with-plugins.ts +395 -0
- package/src/types/index.ts +255 -0
- package/src/utils/bom-utils.js +259 -0
- package/src/utils/bom-utils.ts +373 -0
- package/src/utils/encoding-support.js +124 -0
- package/src/utils/encoding-support.ts +155 -0
- package/src/utils/schema-validator.js +19 -19
- package/src/utils/schema-validator.ts +819 -0
- package/src/utils/transform-loader.js +1 -1
- package/src/utils/transform-loader.ts +389 -0
- package/src/utils/zod-adapter.js +170 -0
- package/src/utils/zod-adapter.ts +280 -0
- package/src/web-server/index.js +10 -10
- package/src/web-server/index.ts +683 -0
- package/src/workers/csv-multithreaded.ts +310 -0
- package/src/workers/csv-parser.worker.ts +227 -0
- package/src/workers/worker-pool.ts +409 -0
- package/stream-csv-to-json.js +26 -8
- package/stream-json-to-csv.js +1 -0
package/bin/jtcsv.js
CHANGED
|
@@ -7,14 +7,27 @@
|
|
|
7
7
|
* with streaming, batch processing, and all security features.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
const fs = require(
|
|
11
|
-
const path = require(
|
|
12
|
-
const { pipeline } = require(
|
|
13
|
-
const jtcsv = require(
|
|
14
|
-
const transformLoader = require(
|
|
15
|
-
const schemaValidator = require(
|
|
16
|
-
|
|
17
|
-
const VERSION = require(
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { pipeline } = require('stream/promises');
|
|
13
|
+
const jtcsv = require('../index.js');
|
|
14
|
+
const transformLoader = require('../src/utils/transform-loader');
|
|
15
|
+
const schemaValidator = require('../src/utils/schema-validator');
|
|
16
|
+
|
|
17
|
+
const VERSION = require('../package.json').version;
|
|
18
|
+
|
|
19
|
+
// Read all stdin as UTF-8
|
|
20
|
+
function readStdin() {
|
|
21
|
+
return new Promise((resolve, reject) => {
|
|
22
|
+
let data = '';
|
|
23
|
+
process.stdin.setEncoding('utf8');
|
|
24
|
+
process.stdin.on('data', (chunk) => {
|
|
25
|
+
data += chunk;
|
|
26
|
+
});
|
|
27
|
+
process.stdin.on('end', () => resolve(data));
|
|
28
|
+
process.stdin.on('error', reject);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
18
31
|
|
|
19
32
|
// Function to apply transform from JavaScript file
|
|
20
33
|
async function applyTransform(data, transformFile) {
|
|
@@ -23,36 +36,36 @@ async function applyTransform(data, transformFile) {
|
|
|
23
36
|
const transformModule = require(transformPath);
|
|
24
37
|
|
|
25
38
|
// Check if module exports a function
|
|
26
|
-
if (typeof transformModule ===
|
|
39
|
+
if (typeof transformModule === 'function') {
|
|
27
40
|
return transformModule(data);
|
|
28
|
-
} else if (typeof transformModule.default ===
|
|
41
|
+
} else if (typeof transformModule.default === 'function') {
|
|
29
42
|
return transformModule.default(data);
|
|
30
|
-
} else if (typeof transformModule.transform ===
|
|
43
|
+
} else if (typeof transformModule.transform === 'function') {
|
|
31
44
|
return transformModule.transform(data);
|
|
32
45
|
} else {
|
|
33
46
|
throw new Error(
|
|
34
|
-
`Transform file must export a function. Found: ${typeof transformModule}
|
|
47
|
+
`Transform file must export a function. Found: ${typeof transformModule}`
|
|
35
48
|
);
|
|
36
49
|
}
|
|
37
50
|
} catch (error) {
|
|
38
51
|
throw new Error(
|
|
39
|
-
`Failed to apply transform from ${transformFile}: ${error.message}
|
|
52
|
+
`Failed to apply transform from ${transformFile}: ${error.message}`
|
|
40
53
|
);
|
|
41
54
|
}
|
|
42
55
|
}
|
|
43
56
|
|
|
44
57
|
// ANSI colors for terminal output
|
|
45
58
|
const colors = {
|
|
46
|
-
reset:
|
|
47
|
-
bright:
|
|
48
|
-
dim:
|
|
49
|
-
red:
|
|
50
|
-
green:
|
|
51
|
-
yellow:
|
|
52
|
-
blue:
|
|
53
|
-
magenta:
|
|
54
|
-
cyan:
|
|
55
|
-
white:
|
|
59
|
+
reset: '\x1b[0m',
|
|
60
|
+
bright: '\x1b[1m',
|
|
61
|
+
dim: '\x1b[2m',
|
|
62
|
+
red: '\x1b[31m',
|
|
63
|
+
green: '\x1b[32m',
|
|
64
|
+
yellow: '\x1b[33m',
|
|
65
|
+
blue: '\x1b[34m',
|
|
66
|
+
magenta: '\x1b[35m',
|
|
67
|
+
cyan: '\x1b[36m',
|
|
68
|
+
white: '\x1b[37m'
|
|
56
69
|
};
|
|
57
70
|
|
|
58
71
|
function color(text, colorName) {
|
|
@@ -61,124 +74,128 @@ function color(text, colorName) {
|
|
|
61
74
|
|
|
62
75
|
function showHelp() {
|
|
63
76
|
console.log(`
|
|
64
|
-
${color(
|
|
65
|
-
${color(
|
|
77
|
+
${color('jtcsv CLI v' + VERSION, 'cyan')}
|
|
78
|
+
${color('The Complete JSON↔CSV Converter for Node.js', 'dim')}
|
|
66
79
|
|
|
67
|
-
${color(
|
|
80
|
+
${color('USAGE:', 'bright')}
|
|
68
81
|
jtcsv [command] [options] [file...]
|
|
69
82
|
|
|
70
|
-
${color(
|
|
71
|
-
${color(
|
|
72
|
-
${color(
|
|
73
|
-
${color(
|
|
74
|
-
${color(
|
|
75
|
-
${color(
|
|
76
|
-
${color(
|
|
77
|
-
${color(
|
|
78
|
-
${color(
|
|
79
|
-
${color(
|
|
80
|
-
${color(
|
|
81
|
-
${color(
|
|
82
|
-
${color(
|
|
83
|
-
${color(
|
|
84
|
-
${color(
|
|
85
|
-
${color(
|
|
86
|
-
${color(
|
|
87
|
-
|
|
88
|
-
${color(
|
|
89
|
-
${color(
|
|
90
|
-
${color(
|
|
91
|
-
${color(
|
|
92
|
-
${color(
|
|
93
|
-
|
|
94
|
-
${color(
|
|
95
|
-
${color(
|
|
96
|
-
${color(
|
|
97
|
-
${color(
|
|
98
|
-
|
|
99
|
-
${color(
|
|
100
|
-
${color(
|
|
83
|
+
${color('MAIN COMMANDS:', 'bright')}
|
|
84
|
+
${color('json-to-csv', 'green')} Convert JSON to CSV (alias: json2csv)
|
|
85
|
+
${color('csv-to-json', 'green')} Convert CSV to JSON (alias: csv2json)
|
|
86
|
+
${color('ndjson-to-csv', 'green')} Convert NDJSON to CSV
|
|
87
|
+
${color('csv-to-ndjson', 'green')} Convert CSV to NDJSON
|
|
88
|
+
${color('ndjson-to-json', 'green')} Convert NDJSON to JSON array
|
|
89
|
+
${color('json-to-ndjson', 'green')} Convert JSON array to NDJSON
|
|
90
|
+
${color('save-json', 'yellow')} Save data as JSON file
|
|
91
|
+
${color('save-csv', 'yellow')} Save data as CSV file
|
|
92
|
+
${color('stream', 'yellow')} Streaming conversion for large files
|
|
93
|
+
${color('batch', 'yellow')} Batch process multiple files
|
|
94
|
+
${color('preprocess', 'magenta')} Preprocess JSON with deep unwrapping
|
|
95
|
+
${color('unwrap', 'magenta')} Flatten nested JSON structures (alias: flatten)
|
|
96
|
+
${color('tui', 'magenta')} Launch Terminal User Interface (@jtcsv/tui)
|
|
97
|
+
${color('web', 'magenta')} Launch Web Interface (http://localhost:3000)
|
|
98
|
+
${color('help', 'blue')} Show this help message
|
|
99
|
+
${color('version', 'blue')} Show version information
|
|
100
|
+
|
|
101
|
+
${color('STREAMING SUBCOMMANDS:', 'bright')}
|
|
102
|
+
${color('stream json-to-csv', 'dim')} Stream JSON to CSV
|
|
103
|
+
${color('stream csv-to-json', 'dim')} Stream CSV to JSON
|
|
104
|
+
${color('stream file-to-csv', 'dim')} Stream file to CSV
|
|
105
|
+
${color('stream file-to-json', 'dim')} Stream file to JSON
|
|
106
|
+
|
|
107
|
+
${color('BATCH SUBCOMMANDS:', 'bright')}
|
|
108
|
+
${color('batch json-to-csv', 'dim')} Batch convert JSON files to CSV
|
|
109
|
+
${color('batch csv-to-json', 'dim')} Batch convert CSV files to JSON
|
|
110
|
+
${color('batch process', 'dim')} Process mixed file types
|
|
111
|
+
|
|
112
|
+
${color('EXAMPLES:', 'bright')}
|
|
113
|
+
${color('Convert JSON file to CSV:', 'dim')}
|
|
101
114
|
jtcsv json-to-csv input.json output.csv --delimiter=,
|
|
102
115
|
|
|
103
|
-
${color(
|
|
116
|
+
${color('Convert CSV file to JSON:', 'dim')}
|
|
104
117
|
jtcsv csv-to-json input.csv output.json --parse-numbers --auto-detect
|
|
105
118
|
|
|
106
|
-
${color(
|
|
119
|
+
${color('Save data as JSON file:', 'dim')}
|
|
107
120
|
jtcsv save-json data.json output.json --pretty
|
|
108
121
|
|
|
109
|
-
${color(
|
|
122
|
+
${color('Save data as CSV file:', 'dim')}
|
|
110
123
|
jtcsv save-csv data.csv output.csv --delimiter=, --transform=transform.js
|
|
111
124
|
|
|
112
|
-
${color(
|
|
125
|
+
${color('Stream large JSON file to CSV:', 'dim')}
|
|
113
126
|
jtcsv stream json-to-csv large.json output.csv --max-records=1000000
|
|
114
127
|
|
|
115
|
-
${color(
|
|
128
|
+
${color('Stream CSV file to JSON:', 'dim')}
|
|
116
129
|
jtcsv stream csv-to-json large.csv output.json --max-rows=500000
|
|
117
130
|
|
|
118
|
-
${color(
|
|
131
|
+
${color('Preprocess complex JSON:', 'dim')}
|
|
119
132
|
jtcsv preprocess complex.json simplified.json --max-depth=3
|
|
120
133
|
|
|
121
|
-
${color(
|
|
134
|
+
${color('Batch convert JSON files:', 'dim')}
|
|
122
135
|
jtcsv batch json-to-csv "data/*.json" "output/" --delimiter=;
|
|
123
136
|
|
|
124
|
-
${color(
|
|
137
|
+
${color('Launch TUI interface:', 'dim')}
|
|
125
138
|
jtcsv tui
|
|
126
139
|
|
|
127
|
-
${color(
|
|
140
|
+
${color('Launch Web interface:', 'dim')}
|
|
128
141
|
jtcsv web --port=3000
|
|
129
142
|
|
|
130
|
-
${color(
|
|
131
|
-
${color(
|
|
132
|
-
${color(
|
|
133
|
-
${color(
|
|
134
|
-
${color(
|
|
135
|
-
${color(
|
|
136
|
-
${color(
|
|
137
|
-
${color(
|
|
138
|
-
${color(
|
|
139
|
-
${color(
|
|
140
|
-
${color(
|
|
141
|
-
${color(
|
|
142
|
-
${color(
|
|
143
|
-
${color(
|
|
144
|
-
${color(
|
|
145
|
-
${color(
|
|
146
|
-
${color(
|
|
147
|
-
${color(
|
|
148
|
-
${color(
|
|
149
|
-
${color(
|
|
150
|
-
${color(
|
|
151
|
-
${color(
|
|
152
|
-
${color(
|
|
153
|
-
${color(
|
|
154
|
-
${color(
|
|
155
|
-
${color(
|
|
156
|
-
${color(
|
|
157
|
-
${color(
|
|
158
|
-
${color(
|
|
159
|
-
${color(
|
|
160
|
-
${color(
|
|
161
|
-
${color(
|
|
162
|
-
${color(
|
|
163
|
-
${color(
|
|
164
|
-
${color(
|
|
165
|
-
${color(
|
|
166
|
-
${color(
|
|
167
|
-
${color(
|
|
168
|
-
${color(
|
|
143
|
+
${color('CONVERSION OPTIONS:', 'bright')}
|
|
144
|
+
${color('--delimiter=', 'cyan')}CHAR CSV delimiter (default: ;)
|
|
145
|
+
${color('--auto-detect', 'cyan')} Auto-detect delimiter (default: true)
|
|
146
|
+
${color('--candidates=', 'cyan')}LIST Delimiter candidates (default: ;,\t|)
|
|
147
|
+
${color('--no-headers', 'cyan')} Exclude headers from CSV output
|
|
148
|
+
${color('--parse-numbers', 'cyan')} Parse numeric values in CSV
|
|
149
|
+
${color('--parse-booleans', 'cyan')} Parse boolean values in CSV
|
|
150
|
+
${color('--no-trim', 'cyan')} Don't trim whitespace from CSV values
|
|
151
|
+
${color('--no-fast-path', 'cyan')} Disable fast-path parser (force quote-aware)
|
|
152
|
+
${color('--fast-path-mode=', 'cyan')}MODE Fast path output mode (objects|compact)
|
|
153
|
+
${color('--rename=', 'cyan')}JSON Rename columns (JSON map)
|
|
154
|
+
${color('--template=', 'cyan')}JSON Column order template (JSON object)
|
|
155
|
+
${color('--no-injection-protection', 'cyan')} Disable CSV injection protection
|
|
156
|
+
${color('--no-rfc4180', 'cyan')} Disable RFC 4180 compliance
|
|
157
|
+
${color('--max-records=', 'cyan')}N Maximum records to process
|
|
158
|
+
${color('--max-rows=', 'cyan')}N Maximum rows to process
|
|
159
|
+
${color('--pretty', 'cyan')} Pretty print JSON output
|
|
160
|
+
${color('--schema=', 'cyan')}JSON JSON schema for validation and formatting
|
|
161
|
+
${color('--transform=', 'cyan')}JS Custom transform function (JavaScript file)
|
|
162
|
+
${color('PREPROCESS OPTIONS:', 'bright')}
|
|
163
|
+
${color('--max-depth=', 'cyan')}N Maximum recursion depth (default: 5)
|
|
164
|
+
${color('--flatten', 'cyan')} Flatten nested objects into dot notation
|
|
165
|
+
${color('--flatten-separator=', 'cyan')}CHAR Separator for flattened keys (default: .)
|
|
166
|
+
${color('--flatten-max-depth=', 'cyan')}N Maximum flattening depth (default: 3)
|
|
167
|
+
${color('--array-handling=', 'cyan')}MODE Array handling: stringify|join|expand (default: stringify)
|
|
168
|
+
${color('--unwrap-arrays', 'cyan')} Unwrap arrays to strings
|
|
169
|
+
${color('--stringify-objects', 'cyan')} Stringify complex objects
|
|
170
|
+
${color('STREAMING OPTIONS:', 'bright')}
|
|
171
|
+
${color('--chunk-size=', 'cyan')}N Chunk size in bytes (default: 65536)
|
|
172
|
+
${color('--buffer-size=', 'cyan')}N Buffer size in records (default: 1000)
|
|
173
|
+
${color('--add-bom', 'cyan')} Add UTF-8 BOM for Excel compatibility
|
|
174
|
+
${color('BATCH OPTIONS:', 'bright')}
|
|
175
|
+
${color('--recursive', 'cyan')} Process directories recursively
|
|
176
|
+
${color('--pattern=', 'cyan')}GLOB File pattern to match
|
|
177
|
+
${color('--output-dir=', 'cyan')}DIR Output directory for batch processing
|
|
178
|
+
${color('--overwrite', 'cyan')} Overwrite existing files
|
|
179
|
+
${color('--parallel=', 'cyan')}N Parallel processing limit (default: 4)
|
|
180
|
+
${color('GENERAL OPTIONS:', 'bright')}
|
|
181
|
+
${color('--silent', 'cyan')} Suppress all output except errors
|
|
182
|
+
${color('--verbose', 'cyan')} Show detailed progress information
|
|
183
|
+
${color('--debug', 'cyan')} Show debug information
|
|
184
|
+
${color('--dry-run', 'cyan')} Show what would be done without actually doing it
|
|
185
|
+
${color('SECURITY FEATURES:', 'bright')}
|
|
169
186
|
• CSV injection protection (enabled by default)
|
|
170
187
|
• Path traversal protection
|
|
171
188
|
• Input validation and sanitization
|
|
172
189
|
• Size limits to prevent DoS attacks
|
|
173
190
|
• Schema validation support
|
|
174
191
|
|
|
175
|
-
${color(
|
|
192
|
+
${color('PERFORMANCE FEATURES:', 'bright')}
|
|
176
193
|
• Streaming for files >100MB
|
|
177
194
|
• Batch processing with parallel execution
|
|
178
195
|
• Memory-efficient preprocessing
|
|
179
196
|
• Configurable buffer sizes
|
|
180
197
|
|
|
181
|
-
${color(
|
|
198
|
+
${color('LEARN MORE:', 'dim')}
|
|
182
199
|
GitHub: https://github.com/Linol-Hamelton/jtcsv
|
|
183
200
|
Issues: https://github.com/Linol-Hamelton/jtcsv/issues
|
|
184
201
|
Documentation: https://github.com/Linol-Hamelton/jtcsv#readme
|
|
@@ -197,47 +214,49 @@ function showVersion() {
|
|
|
197
214
|
|
|
198
215
|
async function convertJsonToCsv(inputFile, outputFile, options) {
|
|
199
216
|
const startTime = Date.now();
|
|
217
|
+
const writingToStdout = outputFile === "-";
|
|
218
|
+
const silent = options.silent || writingToStdout;
|
|
200
219
|
|
|
201
220
|
try {
|
|
202
221
|
// Read input file
|
|
203
|
-
const inputData = await fs.promises.readFile(inputFile,
|
|
222
|
+
const inputData = inputFile === '-' ? await readStdin() : await fs.promises.readFile(inputFile, 'utf8');
|
|
204
223
|
const jsonData = JSON.parse(inputData);
|
|
205
224
|
|
|
206
225
|
if (!Array.isArray(jsonData)) {
|
|
207
|
-
throw new Error(
|
|
226
|
+
throw new Error('JSON data must be an array of objects');
|
|
208
227
|
}
|
|
209
228
|
|
|
210
|
-
if (!
|
|
229
|
+
if (!silent) {
|
|
211
230
|
console.log(
|
|
212
231
|
color(
|
|
213
232
|
`Converting ${jsonData.length.toLocaleString()} records...`,
|
|
214
|
-
|
|
215
|
-
)
|
|
233
|
+
'dim'
|
|
234
|
+
)
|
|
216
235
|
);
|
|
217
236
|
}
|
|
218
237
|
|
|
219
238
|
if (options.transform) {
|
|
220
|
-
if (!
|
|
239
|
+
if (!silent) {
|
|
221
240
|
console.log(
|
|
222
|
-
color(`Applying transform from: ${options.transform}`,
|
|
241
|
+
color(`Applying transform from: ${options.transform}`, 'dim')
|
|
223
242
|
);
|
|
224
243
|
}
|
|
225
244
|
try {
|
|
226
245
|
transformedData = transformLoader.applyTransform(
|
|
227
246
|
jsonData,
|
|
228
|
-
options.transform
|
|
247
|
+
options.transform
|
|
229
248
|
);
|
|
230
|
-
if (!
|
|
249
|
+
if (!silent) {
|
|
231
250
|
console.log(
|
|
232
251
|
color(
|
|
233
252
|
`✓ Transform applied to ${transformedData.length} records`,
|
|
234
|
-
|
|
235
|
-
)
|
|
253
|
+
'green'
|
|
254
|
+
)
|
|
236
255
|
);
|
|
237
256
|
}
|
|
238
257
|
} catch (transformError) {
|
|
239
258
|
console.error(
|
|
240
|
-
color(`✗ Transform error: ${transformError.message}`,
|
|
259
|
+
color(`✗ Transform error: ${transformError.message}`, 'red')
|
|
241
260
|
);
|
|
242
261
|
if (options.debug) {
|
|
243
262
|
console.error(transformError.stack);
|
|
@@ -256,6 +275,10 @@ async function convertJsonToCsv(inputFile, outputFile, options) {
|
|
|
256
275
|
preventCsvInjection: options.preventCsvInjection,
|
|
257
276
|
rfc4180Compliant: options.rfc4180Compliant,
|
|
258
277
|
schema: options.schema, // Add schema option
|
|
278
|
+
flatten: options.flatten,
|
|
279
|
+
flattenSeparator: options.flattenSeparator,
|
|
280
|
+
flattenMaxDepth: options.flattenMaxDepth,
|
|
281
|
+
arrayHandling: options.arrayHandling
|
|
259
282
|
};
|
|
260
283
|
|
|
261
284
|
// Apply transform function if provided
|
|
@@ -268,31 +291,35 @@ async function convertJsonToCsv(inputFile, outputFile, options) {
|
|
|
268
291
|
const csvData = jtcsv.jsonToCsv(transformedData, jtcsvOptions);
|
|
269
292
|
|
|
270
293
|
// Write output file
|
|
271
|
-
|
|
294
|
+
if (writingToStdout) {
|
|
295
|
+
process.stdout.write(csvData);
|
|
296
|
+
} else {
|
|
297
|
+
await fs.promises.writeFile(outputFile, csvData, 'utf8');
|
|
298
|
+
}
|
|
272
299
|
|
|
273
300
|
const elapsed = Date.now() - startTime;
|
|
274
|
-
if (!
|
|
301
|
+
if (!silent) {
|
|
275
302
|
console.log(
|
|
276
303
|
color(
|
|
277
304
|
`✓ Converted ${transformedData.length.toLocaleString()} records in ${elapsed}ms`,
|
|
278
|
-
|
|
279
|
-
)
|
|
305
|
+
'green'
|
|
306
|
+
)
|
|
280
307
|
);
|
|
281
308
|
console.log(
|
|
282
309
|
color(
|
|
283
310
|
` Output: ${outputFile} (${csvData.length.toLocaleString()} bytes)`,
|
|
284
|
-
|
|
285
|
-
)
|
|
311
|
+
'dim'
|
|
312
|
+
)
|
|
286
313
|
);
|
|
287
314
|
}
|
|
288
315
|
|
|
289
316
|
return {
|
|
290
317
|
records: transformedData.length,
|
|
291
318
|
bytes: csvData.length,
|
|
292
|
-
time: elapsed
|
|
319
|
+
time: elapsed
|
|
293
320
|
};
|
|
294
321
|
} catch (error) {
|
|
295
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
322
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
296
323
|
if (options.debug) {
|
|
297
324
|
console.error(error.stack);
|
|
298
325
|
}
|
|
@@ -302,10 +329,12 @@ async function convertJsonToCsv(inputFile, outputFile, options) {
|
|
|
302
329
|
|
|
303
330
|
async function convertCsvToJson(inputFile, outputFile, options) {
|
|
304
331
|
const startTime = Date.now();
|
|
332
|
+
const writingToStdout = outputFile === "-";
|
|
333
|
+
const silent = options.silent || writingToStdout;
|
|
305
334
|
|
|
306
335
|
try {
|
|
307
|
-
if (!
|
|
308
|
-
console.log(color(
|
|
336
|
+
if (!silent) {
|
|
337
|
+
console.log(color('Reading CSV file...', 'dim'));
|
|
309
338
|
}
|
|
310
339
|
|
|
311
340
|
// Prepare options for jtcsv
|
|
@@ -321,36 +350,36 @@ async function convertCsvToJson(inputFile, outputFile, options) {
|
|
|
321
350
|
maxRows: options.maxRows,
|
|
322
351
|
useFastPath: options.useFastPath,
|
|
323
352
|
fastPathMode: options.fastPathMode,
|
|
324
|
-
schema: options.schema
|
|
353
|
+
schema: options.schema // Add schema option if supported
|
|
325
354
|
};
|
|
326
355
|
|
|
327
356
|
// Read and convert CSV
|
|
328
|
-
const jsonData = await jtcsv.readCsvAsJson(inputFile, jtcsvOptions);
|
|
357
|
+
const jsonData = inputFile === '-' ? jtcsv.csvToJson(await readStdin(), jtcsvOptions) : await jtcsv.readCsvAsJson(inputFile, jtcsvOptions);
|
|
329
358
|
|
|
330
359
|
// Apply transform if specified
|
|
331
360
|
let transformedData = jsonData;
|
|
332
361
|
if (options.transform) {
|
|
333
|
-
if (!
|
|
362
|
+
if (!silent) {
|
|
334
363
|
console.log(
|
|
335
|
-
color(`Applying transform from: ${options.transform}`,
|
|
364
|
+
color(`Applying transform from: ${options.transform}`, 'dim')
|
|
336
365
|
);
|
|
337
366
|
}
|
|
338
367
|
try {
|
|
339
368
|
transformedData = transformLoader.applyTransform(
|
|
340
369
|
jsonData,
|
|
341
|
-
options.transform
|
|
370
|
+
options.transform
|
|
342
371
|
);
|
|
343
|
-
if (!
|
|
372
|
+
if (!silent) {
|
|
344
373
|
console.log(
|
|
345
374
|
color(
|
|
346
375
|
`✓ Transform applied to ${transformedData.length} rows`,
|
|
347
|
-
|
|
348
|
-
)
|
|
376
|
+
'green'
|
|
377
|
+
)
|
|
349
378
|
);
|
|
350
379
|
}
|
|
351
380
|
} catch (transformError) {
|
|
352
381
|
console.error(
|
|
353
|
-
color(`✗ Transform error: ${transformError.message}`,
|
|
382
|
+
color(`✗ Transform error: ${transformError.message}`, 'red')
|
|
354
383
|
);
|
|
355
384
|
if (options.debug) {
|
|
356
385
|
console.error(transformError.stack);
|
|
@@ -365,31 +394,35 @@ async function convertCsvToJson(inputFile, outputFile, options) {
|
|
|
365
394
|
: JSON.stringify(transformedData);
|
|
366
395
|
|
|
367
396
|
// Write output file
|
|
368
|
-
|
|
397
|
+
if (writingToStdout) {
|
|
398
|
+
process.stdout.write(jsonOutput);
|
|
399
|
+
} else {
|
|
400
|
+
await fs.promises.writeFile(outputFile, jsonOutput, 'utf8');
|
|
401
|
+
}
|
|
369
402
|
|
|
370
403
|
const elapsed = Date.now() - startTime;
|
|
371
|
-
if (!
|
|
404
|
+
if (!silent) {
|
|
372
405
|
console.log(
|
|
373
406
|
color(
|
|
374
407
|
`✓ Converted ${transformedData.length.toLocaleString()} rows in ${elapsed}ms`,
|
|
375
|
-
|
|
376
|
-
)
|
|
408
|
+
'green'
|
|
409
|
+
)
|
|
377
410
|
);
|
|
378
411
|
console.log(
|
|
379
412
|
color(
|
|
380
413
|
` Output: ${outputFile} (${jsonOutput.length.toLocaleString()} bytes)`,
|
|
381
|
-
|
|
382
|
-
)
|
|
414
|
+
'dim'
|
|
415
|
+
)
|
|
383
416
|
);
|
|
384
417
|
}
|
|
385
418
|
|
|
386
419
|
return {
|
|
387
420
|
rows: transformedData.length,
|
|
388
421
|
bytes: jsonOutput.length,
|
|
389
|
-
time: elapsed
|
|
422
|
+
time: elapsed
|
|
390
423
|
};
|
|
391
424
|
} catch (error) {
|
|
392
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
425
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
393
426
|
if (options.debug) {
|
|
394
427
|
console.error(error.stack);
|
|
395
428
|
}
|
|
@@ -402,10 +435,10 @@ async function saveAsCsv(inputFile, outputFile, options) {
|
|
|
402
435
|
|
|
403
436
|
try {
|
|
404
437
|
// Read input file
|
|
405
|
-
const inputData = await fs.promises.readFile(inputFile,
|
|
438
|
+
const inputData = await fs.promises.readFile(inputFile, 'utf8');
|
|
406
439
|
|
|
407
440
|
if (!options.silent) {
|
|
408
|
-
console.log(color(
|
|
441
|
+
console.log(color('Saving CSV file...', 'dim'));
|
|
409
442
|
}
|
|
410
443
|
|
|
411
444
|
// Apply transform if specified
|
|
@@ -413,7 +446,7 @@ async function saveAsCsv(inputFile, outputFile, options) {
|
|
|
413
446
|
if (options.transform) {
|
|
414
447
|
if (!options.silent) {
|
|
415
448
|
console.log(
|
|
416
|
-
color(`Applying transform from: ${options.transform}`,
|
|
449
|
+
color(`Applying transform from: ${options.transform}`, 'dim')
|
|
417
450
|
);
|
|
418
451
|
}
|
|
419
452
|
try {
|
|
@@ -424,26 +457,26 @@ async function saveAsCsv(inputFile, outputFile, options) {
|
|
|
424
457
|
hasHeaders: options.hasHeaders,
|
|
425
458
|
trim: options.trim,
|
|
426
459
|
parseNumbers: options.parseNumbers,
|
|
427
|
-
parseBooleans: options.parseBooleans
|
|
460
|
+
parseBooleans: options.parseBooleans
|
|
428
461
|
});
|
|
429
462
|
|
|
430
463
|
const transformedJson = transformLoader.applyTransform(
|
|
431
464
|
parsedData,
|
|
432
|
-
options.transform
|
|
465
|
+
options.transform
|
|
433
466
|
);
|
|
434
467
|
|
|
435
468
|
// Конвертировать обратно в CSV
|
|
436
469
|
transformedData = jtcsv.jsonToCsv(transformedJson, {
|
|
437
470
|
delimiter: options.delimiter,
|
|
438
|
-
includeHeaders: options.includeHeaders
|
|
471
|
+
includeHeaders: options.includeHeaders
|
|
439
472
|
});
|
|
440
473
|
|
|
441
474
|
if (!options.silent) {
|
|
442
|
-
console.log(color(
|
|
475
|
+
console.log(color('✓ Transform applied', 'green'));
|
|
443
476
|
}
|
|
444
477
|
} catch (transformError) {
|
|
445
478
|
console.error(
|
|
446
|
-
color(`✗ Transform error: ${transformError.message}`,
|
|
479
|
+
color(`✗ Transform error: ${transformError.message}`, 'red')
|
|
447
480
|
);
|
|
448
481
|
if (options.debug) {
|
|
449
482
|
console.error(transformError.stack);
|
|
@@ -453,17 +486,17 @@ async function saveAsCsv(inputFile, outputFile, options) {
|
|
|
453
486
|
}
|
|
454
487
|
|
|
455
488
|
// Write output file
|
|
456
|
-
await fs.promises.writeFile(outputFile, transformedData,
|
|
489
|
+
await fs.promises.writeFile(outputFile, transformedData, 'utf8');
|
|
457
490
|
|
|
458
491
|
const elapsed = Date.now() - startTime;
|
|
459
492
|
if (!options.silent) {
|
|
460
|
-
console.log(color(`✓ Saved CSV in ${elapsed}ms`,
|
|
461
|
-
console.log(color(` Output: ${outputFile}`,
|
|
493
|
+
console.log(color(`✓ Saved CSV in ${elapsed}ms`, 'green'));
|
|
494
|
+
console.log(color(` Output: ${outputFile}`, 'dim'));
|
|
462
495
|
}
|
|
463
496
|
|
|
464
497
|
return { time: elapsed };
|
|
465
498
|
} catch (error) {
|
|
466
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
499
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
467
500
|
if (options.debug) {
|
|
468
501
|
console.error(error.stack);
|
|
469
502
|
}
|
|
@@ -476,15 +509,15 @@ async function saveAsJson(inputFile, outputFile, options) {
|
|
|
476
509
|
|
|
477
510
|
try {
|
|
478
511
|
// Read input file
|
|
479
|
-
const inputData = await fs.promises.readFile(inputFile,
|
|
512
|
+
const inputData = await fs.promises.readFile(inputFile, 'utf8');
|
|
480
513
|
const jsonData = JSON.parse(inputData);
|
|
481
514
|
|
|
482
515
|
if (!options.silent) {
|
|
483
516
|
console.log(
|
|
484
517
|
color(
|
|
485
|
-
`Saving ${Array.isArray(jsonData) ? jsonData.length.toLocaleString() +
|
|
486
|
-
|
|
487
|
-
)
|
|
518
|
+
`Saving ${Array.isArray(jsonData) ? jsonData.length.toLocaleString() + ' records' : 'object'}...`,
|
|
519
|
+
'dim'
|
|
520
|
+
)
|
|
488
521
|
);
|
|
489
522
|
}
|
|
490
523
|
|
|
@@ -493,20 +526,20 @@ async function saveAsJson(inputFile, outputFile, options) {
|
|
|
493
526
|
if (options.transform) {
|
|
494
527
|
if (!options.silent) {
|
|
495
528
|
console.log(
|
|
496
|
-
color(`Applying transform from: ${options.transform}`,
|
|
529
|
+
color(`Applying transform from: ${options.transform}`, 'dim')
|
|
497
530
|
);
|
|
498
531
|
}
|
|
499
532
|
try {
|
|
500
533
|
transformedData = transformLoader.applyTransform(
|
|
501
534
|
jsonData,
|
|
502
|
-
options.transform
|
|
535
|
+
options.transform
|
|
503
536
|
);
|
|
504
537
|
if (!options.silent) {
|
|
505
|
-
console.log(color(
|
|
538
|
+
console.log(color('✓ Transform applied', 'green'));
|
|
506
539
|
}
|
|
507
540
|
} catch (transformError) {
|
|
508
541
|
console.error(
|
|
509
|
-
color(`✗ Transform error: ${transformError.message}`,
|
|
542
|
+
color(`✗ Transform error: ${transformError.message}`, 'red')
|
|
510
543
|
);
|
|
511
544
|
if (options.debug) {
|
|
512
545
|
console.error(transformError.stack);
|
|
@@ -517,7 +550,7 @@ async function saveAsJson(inputFile, outputFile, options) {
|
|
|
517
550
|
|
|
518
551
|
// Prepare options for jtcsv
|
|
519
552
|
const jtcsvOptions = {
|
|
520
|
-
prettyPrint: options.pretty
|
|
553
|
+
prettyPrint: options.pretty
|
|
521
554
|
};
|
|
522
555
|
|
|
523
556
|
// Save as JSON
|
|
@@ -525,13 +558,13 @@ async function saveAsJson(inputFile, outputFile, options) {
|
|
|
525
558
|
|
|
526
559
|
const elapsed = Date.now() - startTime;
|
|
527
560
|
if (!options.silent) {
|
|
528
|
-
console.log(color(`✓ Saved JSON in ${elapsed}ms`,
|
|
529
|
-
console.log(color(` Output: ${outputFile}`,
|
|
561
|
+
console.log(color(`✓ Saved JSON in ${elapsed}ms`, 'green'));
|
|
562
|
+
console.log(color(` Output: ${outputFile}`, 'dim'));
|
|
530
563
|
}
|
|
531
564
|
|
|
532
565
|
return { time: elapsed };
|
|
533
566
|
} catch (error) {
|
|
534
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
567
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
535
568
|
if (options.debug) {
|
|
536
569
|
console.error(error.stack);
|
|
537
570
|
}
|
|
@@ -548,16 +581,16 @@ async function convertNdjsonToCsv(inputFile, outputFile, options) {
|
|
|
548
581
|
|
|
549
582
|
try {
|
|
550
583
|
if (!options.silent) {
|
|
551
|
-
console.log(color(
|
|
584
|
+
console.log(color('Converting NDJSON to CSV...', 'dim'));
|
|
552
585
|
}
|
|
553
586
|
|
|
554
587
|
// Read NDJSON file
|
|
555
|
-
const inputData = await fs.promises.readFile(inputFile,
|
|
588
|
+
const inputData = await fs.promises.readFile(inputFile, 'utf8');
|
|
556
589
|
const jsonData = jtcsv.ndjsonToJson(inputData);
|
|
557
590
|
|
|
558
591
|
if (!options.silent) {
|
|
559
592
|
console.log(
|
|
560
|
-
color(`Parsed ${jsonData.length.toLocaleString()} records from NDJSON`,
|
|
593
|
+
color(`Parsed ${jsonData.length.toLocaleString()} records from NDJSON`, 'dim')
|
|
561
594
|
);
|
|
562
595
|
}
|
|
563
596
|
|
|
@@ -568,38 +601,38 @@ async function convertNdjsonToCsv(inputFile, outputFile, options) {
|
|
|
568
601
|
renameMap: options.renameMap,
|
|
569
602
|
template: options.template,
|
|
570
603
|
preventCsvInjection: options.preventCsvInjection,
|
|
571
|
-
rfc4180Compliant: options.rfc4180Compliant
|
|
604
|
+
rfc4180Compliant: options.rfc4180Compliant
|
|
572
605
|
};
|
|
573
606
|
|
|
574
607
|
// Convert to CSV
|
|
575
608
|
const csvData = jtcsv.jsonToCsv(jsonData, jtcsvOptions);
|
|
576
609
|
|
|
577
610
|
// Write output file
|
|
578
|
-
await fs.promises.writeFile(outputFile, csvData,
|
|
611
|
+
await fs.promises.writeFile(outputFile, csvData, 'utf8');
|
|
579
612
|
|
|
580
613
|
const elapsed = Date.now() - startTime;
|
|
581
614
|
if (!options.silent) {
|
|
582
615
|
console.log(
|
|
583
616
|
color(
|
|
584
617
|
`✓ Converted ${jsonData.length.toLocaleString()} records in ${elapsed}ms`,
|
|
585
|
-
|
|
586
|
-
)
|
|
618
|
+
'green'
|
|
619
|
+
)
|
|
587
620
|
);
|
|
588
621
|
console.log(
|
|
589
622
|
color(
|
|
590
623
|
` Output: ${outputFile} (${csvData.length.toLocaleString()} bytes)`,
|
|
591
|
-
|
|
592
|
-
)
|
|
624
|
+
'dim'
|
|
625
|
+
)
|
|
593
626
|
);
|
|
594
627
|
}
|
|
595
628
|
|
|
596
629
|
return {
|
|
597
630
|
records: jsonData.length,
|
|
598
631
|
bytes: csvData.length,
|
|
599
|
-
time: elapsed
|
|
632
|
+
time: elapsed
|
|
600
633
|
};
|
|
601
634
|
} catch (error) {
|
|
602
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
635
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
603
636
|
if (options.debug) {
|
|
604
637
|
console.error(error.stack);
|
|
605
638
|
}
|
|
@@ -612,7 +645,7 @@ async function convertCsvToNdjson(inputFile, outputFile, options) {
|
|
|
612
645
|
|
|
613
646
|
try {
|
|
614
647
|
if (!options.silent) {
|
|
615
|
-
console.log(color(
|
|
648
|
+
console.log(color('Converting CSV to NDJSON...', 'dim'));
|
|
616
649
|
}
|
|
617
650
|
|
|
618
651
|
// Prepare options for jtcsv
|
|
@@ -624,7 +657,7 @@ async function convertCsvToNdjson(inputFile, outputFile, options) {
|
|
|
624
657
|
renameMap: options.renameMap,
|
|
625
658
|
trim: options.trim,
|
|
626
659
|
parseNumbers: options.parseNumbers,
|
|
627
|
-
parseBooleans: options.parseBooleans
|
|
660
|
+
parseBooleans: options.parseBooleans
|
|
628
661
|
};
|
|
629
662
|
|
|
630
663
|
// Read and convert CSV
|
|
@@ -634,31 +667,31 @@ async function convertCsvToNdjson(inputFile, outputFile, options) {
|
|
|
634
667
|
const ndjsonData = jtcsv.jsonToNdjson(jsonData);
|
|
635
668
|
|
|
636
669
|
// Write output file
|
|
637
|
-
await fs.promises.writeFile(outputFile, ndjsonData,
|
|
670
|
+
await fs.promises.writeFile(outputFile, ndjsonData, 'utf8');
|
|
638
671
|
|
|
639
672
|
const elapsed = Date.now() - startTime;
|
|
640
673
|
if (!options.silent) {
|
|
641
674
|
console.log(
|
|
642
675
|
color(
|
|
643
676
|
`✓ Converted ${jsonData.length.toLocaleString()} rows in ${elapsed}ms`,
|
|
644
|
-
|
|
645
|
-
)
|
|
677
|
+
'green'
|
|
678
|
+
)
|
|
646
679
|
);
|
|
647
680
|
console.log(
|
|
648
681
|
color(
|
|
649
682
|
` Output: ${outputFile} (${ndjsonData.length.toLocaleString()} bytes)`,
|
|
650
|
-
|
|
651
|
-
)
|
|
683
|
+
'dim'
|
|
684
|
+
)
|
|
652
685
|
);
|
|
653
686
|
}
|
|
654
687
|
|
|
655
688
|
return {
|
|
656
689
|
rows: jsonData.length,
|
|
657
690
|
bytes: ndjsonData.length,
|
|
658
|
-
time: elapsed
|
|
691
|
+
time: elapsed
|
|
659
692
|
};
|
|
660
693
|
} catch (error) {
|
|
661
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
694
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
662
695
|
if (options.debug) {
|
|
663
696
|
console.error(error.stack);
|
|
664
697
|
}
|
|
@@ -671,11 +704,11 @@ async function convertNdjsonToJson(inputFile, outputFile, options) {
|
|
|
671
704
|
|
|
672
705
|
try {
|
|
673
706
|
if (!options.silent) {
|
|
674
|
-
console.log(color(
|
|
707
|
+
console.log(color('Converting NDJSON to JSON array...', 'dim'));
|
|
675
708
|
}
|
|
676
709
|
|
|
677
710
|
// Read NDJSON file
|
|
678
|
-
const inputData = await fs.promises.readFile(inputFile,
|
|
711
|
+
const inputData = await fs.promises.readFile(inputFile, 'utf8');
|
|
679
712
|
const jsonData = jtcsv.ndjsonToJson(inputData);
|
|
680
713
|
|
|
681
714
|
// Format JSON
|
|
@@ -684,31 +717,31 @@ async function convertNdjsonToJson(inputFile, outputFile, options) {
|
|
|
684
717
|
: JSON.stringify(jsonData);
|
|
685
718
|
|
|
686
719
|
// Write output file
|
|
687
|
-
await fs.promises.writeFile(outputFile, jsonOutput,
|
|
720
|
+
await fs.promises.writeFile(outputFile, jsonOutput, 'utf8');
|
|
688
721
|
|
|
689
722
|
const elapsed = Date.now() - startTime;
|
|
690
723
|
if (!options.silent) {
|
|
691
724
|
console.log(
|
|
692
725
|
color(
|
|
693
726
|
`✓ Converted ${jsonData.length.toLocaleString()} records in ${elapsed}ms`,
|
|
694
|
-
|
|
695
|
-
)
|
|
727
|
+
'green'
|
|
728
|
+
)
|
|
696
729
|
);
|
|
697
730
|
console.log(
|
|
698
731
|
color(
|
|
699
732
|
` Output: ${outputFile} (${jsonOutput.length.toLocaleString()} bytes)`,
|
|
700
|
-
|
|
701
|
-
)
|
|
733
|
+
'dim'
|
|
734
|
+
)
|
|
702
735
|
);
|
|
703
736
|
}
|
|
704
737
|
|
|
705
738
|
return {
|
|
706
739
|
records: jsonData.length,
|
|
707
740
|
bytes: jsonOutput.length,
|
|
708
|
-
time: elapsed
|
|
741
|
+
time: elapsed
|
|
709
742
|
};
|
|
710
743
|
} catch (error) {
|
|
711
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
744
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
712
745
|
if (options.debug) {
|
|
713
746
|
console.error(error.stack);
|
|
714
747
|
}
|
|
@@ -721,46 +754,46 @@ async function convertJsonToNdjson(inputFile, outputFile, options) {
|
|
|
721
754
|
|
|
722
755
|
try {
|
|
723
756
|
if (!options.silent) {
|
|
724
|
-
console.log(color(
|
|
757
|
+
console.log(color('Converting JSON array to NDJSON...', 'dim'));
|
|
725
758
|
}
|
|
726
759
|
|
|
727
760
|
// Read JSON file
|
|
728
|
-
const inputData = await fs.promises.readFile(inputFile,
|
|
761
|
+
const inputData = await fs.promises.readFile(inputFile, 'utf8');
|
|
729
762
|
const jsonData = JSON.parse(inputData);
|
|
730
763
|
|
|
731
764
|
if (!Array.isArray(jsonData)) {
|
|
732
|
-
throw new Error(
|
|
765
|
+
throw new Error('JSON data must be an array of objects');
|
|
733
766
|
}
|
|
734
767
|
|
|
735
768
|
// Convert to NDJSON
|
|
736
769
|
const ndjsonData = jtcsv.jsonToNdjson(jsonData);
|
|
737
770
|
|
|
738
771
|
// Write output file
|
|
739
|
-
await fs.promises.writeFile(outputFile, ndjsonData,
|
|
772
|
+
await fs.promises.writeFile(outputFile, ndjsonData, 'utf8');
|
|
740
773
|
|
|
741
774
|
const elapsed = Date.now() - startTime;
|
|
742
775
|
if (!options.silent) {
|
|
743
776
|
console.log(
|
|
744
777
|
color(
|
|
745
778
|
`✓ Converted ${jsonData.length.toLocaleString()} records in ${elapsed}ms`,
|
|
746
|
-
|
|
747
|
-
)
|
|
779
|
+
'green'
|
|
780
|
+
)
|
|
748
781
|
);
|
|
749
782
|
console.log(
|
|
750
783
|
color(
|
|
751
784
|
` Output: ${outputFile} (${ndjsonData.length.toLocaleString()} bytes)`,
|
|
752
|
-
|
|
753
|
-
)
|
|
785
|
+
'dim'
|
|
786
|
+
)
|
|
754
787
|
);
|
|
755
788
|
}
|
|
756
789
|
|
|
757
790
|
return {
|
|
758
791
|
records: jsonData.length,
|
|
759
792
|
bytes: ndjsonData.length,
|
|
760
|
-
time: elapsed
|
|
793
|
+
time: elapsed
|
|
761
794
|
};
|
|
762
795
|
} catch (error) {
|
|
763
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
796
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
764
797
|
if (options.debug) {
|
|
765
798
|
console.error(error.stack);
|
|
766
799
|
}
|
|
@@ -777,18 +810,18 @@ async function unwrapJson(inputFile, outputFile, options) {
|
|
|
777
810
|
|
|
778
811
|
try {
|
|
779
812
|
if (!options.silent) {
|
|
780
|
-
console.log(color(
|
|
813
|
+
console.log(color('Unwrapping/flattening nested JSON...', 'dim'));
|
|
781
814
|
}
|
|
782
815
|
|
|
783
816
|
// Read JSON file
|
|
784
|
-
const inputData = await fs.promises.readFile(inputFile,
|
|
817
|
+
const inputData = await fs.promises.readFile(inputFile, 'utf8');
|
|
785
818
|
const jsonData = JSON.parse(inputData);
|
|
786
819
|
|
|
787
820
|
const maxDepth = options.maxDepth || 10;
|
|
788
|
-
const separator = options.flattenPrefix ||
|
|
821
|
+
const separator = options.flattenPrefix || '_';
|
|
789
822
|
|
|
790
823
|
// Flatten function
|
|
791
|
-
function flattenObject(obj, prefix =
|
|
824
|
+
function flattenObject(obj, prefix = '', depth = 0) {
|
|
792
825
|
if (depth >= maxDepth) {
|
|
793
826
|
return { [prefix.slice(0, -1)]: JSON.stringify(obj) };
|
|
794
827
|
}
|
|
@@ -798,12 +831,12 @@ async function unwrapJson(inputFile, outputFile, options) {
|
|
|
798
831
|
for (const [key, value] of Object.entries(obj)) {
|
|
799
832
|
const newKey = prefix + key;
|
|
800
833
|
|
|
801
|
-
if (value && typeof value ===
|
|
834
|
+
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
802
835
|
Object.assign(result, flattenObject(value, newKey + separator, depth + 1));
|
|
803
836
|
} else if (Array.isArray(value)) {
|
|
804
837
|
// Flatten arrays
|
|
805
838
|
if (options.unwrapArrays) {
|
|
806
|
-
result[newKey] = value.join(
|
|
839
|
+
result[newKey] = value.join(', ');
|
|
807
840
|
} else {
|
|
808
841
|
result[newKey] = JSON.stringify(value);
|
|
809
842
|
}
|
|
@@ -820,7 +853,7 @@ async function unwrapJson(inputFile, outputFile, options) {
|
|
|
820
853
|
unwrappedData = jsonData.map(item => flattenObject(item));
|
|
821
854
|
if (!options.silent) {
|
|
822
855
|
console.log(
|
|
823
|
-
color(`Processing ${jsonData.length.toLocaleString()} records...`,
|
|
856
|
+
color(`Processing ${jsonData.length.toLocaleString()} records...`, 'dim')
|
|
824
857
|
);
|
|
825
858
|
}
|
|
826
859
|
} else {
|
|
@@ -833,7 +866,7 @@ async function unwrapJson(inputFile, outputFile, options) {
|
|
|
833
866
|
: JSON.stringify(unwrappedData);
|
|
834
867
|
|
|
835
868
|
// Write output file
|
|
836
|
-
await fs.promises.writeFile(outputFile, jsonOutput,
|
|
869
|
+
await fs.promises.writeFile(outputFile, jsonOutput, 'utf8');
|
|
837
870
|
|
|
838
871
|
const elapsed = Date.now() - startTime;
|
|
839
872
|
const recordCount = Array.isArray(unwrappedData) ? unwrappedData.length : 1;
|
|
@@ -842,24 +875,24 @@ async function unwrapJson(inputFile, outputFile, options) {
|
|
|
842
875
|
console.log(
|
|
843
876
|
color(
|
|
844
877
|
`✓ Unwrapped ${recordCount.toLocaleString()} record(s) in ${elapsed}ms`,
|
|
845
|
-
|
|
846
|
-
)
|
|
878
|
+
'green'
|
|
879
|
+
)
|
|
847
880
|
);
|
|
848
881
|
console.log(
|
|
849
882
|
color(
|
|
850
883
|
` Output: ${outputFile} (${jsonOutput.length.toLocaleString()} bytes)`,
|
|
851
|
-
|
|
852
|
-
)
|
|
884
|
+
'dim'
|
|
885
|
+
)
|
|
853
886
|
);
|
|
854
887
|
}
|
|
855
888
|
|
|
856
889
|
return {
|
|
857
890
|
records: recordCount,
|
|
858
891
|
bytes: jsonOutput.length,
|
|
859
|
-
time: elapsed
|
|
892
|
+
time: elapsed
|
|
860
893
|
};
|
|
861
894
|
} catch (error) {
|
|
862
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
895
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
863
896
|
if (options.debug) {
|
|
864
897
|
console.error(error.stack);
|
|
865
898
|
}
|
|
@@ -872,12 +905,12 @@ async function preprocessJson(inputFile, outputFile, options) {
|
|
|
872
905
|
|
|
873
906
|
try {
|
|
874
907
|
// Read input file
|
|
875
|
-
const inputData = await fs.promises.readFile(inputFile,
|
|
908
|
+
const inputData = await fs.promises.readFile(inputFile, 'utf8');
|
|
876
909
|
const jsonData = JSON.parse(inputData);
|
|
877
910
|
|
|
878
911
|
if (!Array.isArray(jsonData)) {
|
|
879
912
|
throw new Error(
|
|
880
|
-
|
|
913
|
+
'JSON data must be an array of objects for preprocessing'
|
|
881
914
|
);
|
|
882
915
|
}
|
|
883
916
|
|
|
@@ -885,8 +918,8 @@ async function preprocessJson(inputFile, outputFile, options) {
|
|
|
885
918
|
console.log(
|
|
886
919
|
color(
|
|
887
920
|
`Preprocessing ${jsonData.length.toLocaleString()} records...`,
|
|
888
|
-
|
|
889
|
-
)
|
|
921
|
+
'dim'
|
|
922
|
+
)
|
|
890
923
|
);
|
|
891
924
|
}
|
|
892
925
|
|
|
@@ -898,25 +931,25 @@ async function preprocessJson(inputFile, outputFile, options) {
|
|
|
898
931
|
if (options.transform) {
|
|
899
932
|
if (!options.silent) {
|
|
900
933
|
console.log(
|
|
901
|
-
color(`Applying transform from: ${options.transform}`,
|
|
934
|
+
color(`Applying transform from: ${options.transform}`, 'dim')
|
|
902
935
|
);
|
|
903
936
|
}
|
|
904
937
|
try {
|
|
905
938
|
transformedData = transformLoader.applyTransform(
|
|
906
939
|
processedData,
|
|
907
|
-
options.transform
|
|
940
|
+
options.transform
|
|
908
941
|
);
|
|
909
942
|
if (!options.silent) {
|
|
910
943
|
console.log(
|
|
911
944
|
color(
|
|
912
945
|
`✓ Transform applied to ${transformedData.length} records`,
|
|
913
|
-
|
|
914
|
-
)
|
|
946
|
+
'green'
|
|
947
|
+
)
|
|
915
948
|
);
|
|
916
949
|
}
|
|
917
950
|
} catch (transformError) {
|
|
918
951
|
console.error(
|
|
919
|
-
color(`✗ Transform error: ${transformError.message}`,
|
|
952
|
+
color(`✗ Transform error: ${transformError.message}`, 'red')
|
|
920
953
|
);
|
|
921
954
|
if (options.debug) {
|
|
922
955
|
console.error(transformError.stack);
|
|
@@ -930,7 +963,7 @@ async function preprocessJson(inputFile, outputFile, options) {
|
|
|
930
963
|
const maxDepth = options.maxDepth || 5;
|
|
931
964
|
transformedData.forEach((item) => {
|
|
932
965
|
for (const key in item) {
|
|
933
|
-
if (item[key] && typeof item[key] ===
|
|
966
|
+
if (item[key] && typeof item[key] === 'object') {
|
|
934
967
|
item[key] = jtcsv.deepUnwrap(item[key], 0, maxDepth);
|
|
935
968
|
}
|
|
936
969
|
}
|
|
@@ -943,31 +976,31 @@ async function preprocessJson(inputFile, outputFile, options) {
|
|
|
943
976
|
: JSON.stringify(transformedData);
|
|
944
977
|
|
|
945
978
|
// Write output file
|
|
946
|
-
await fs.promises.writeFile(outputFile, jsonOutput,
|
|
979
|
+
await fs.promises.writeFile(outputFile, jsonOutput, 'utf8');
|
|
947
980
|
|
|
948
981
|
const elapsed = Date.now() - startTime;
|
|
949
982
|
if (!options.silent) {
|
|
950
983
|
console.log(
|
|
951
984
|
color(
|
|
952
985
|
`✓ Preprocessed ${transformedData.length.toLocaleString()} records in ${elapsed}ms`,
|
|
953
|
-
|
|
954
|
-
)
|
|
986
|
+
'green'
|
|
987
|
+
)
|
|
955
988
|
);
|
|
956
989
|
console.log(
|
|
957
990
|
color(
|
|
958
991
|
` Output: ${outputFile} (${jsonOutput.length.toLocaleString()} bytes)`,
|
|
959
|
-
|
|
960
|
-
)
|
|
992
|
+
'dim'
|
|
993
|
+
)
|
|
961
994
|
);
|
|
962
995
|
}
|
|
963
996
|
|
|
964
997
|
return {
|
|
965
998
|
records: transformedData.length,
|
|
966
999
|
bytes: jsonOutput.length,
|
|
967
|
-
time: elapsed
|
|
1000
|
+
time: elapsed
|
|
968
1001
|
};
|
|
969
1002
|
} catch (error) {
|
|
970
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
1003
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
971
1004
|
if (options.debug) {
|
|
972
1005
|
console.error(error.stack);
|
|
973
1006
|
}
|
|
@@ -985,29 +1018,29 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
985
1018
|
|
|
986
1019
|
try {
|
|
987
1020
|
if (!options.silent) {
|
|
988
|
-
console.log(color(
|
|
1021
|
+
console.log(color('Streaming JSON to CSV...', 'dim'));
|
|
989
1022
|
}
|
|
990
1023
|
|
|
991
1024
|
// Create streams
|
|
992
|
-
const readStream = fs.createReadStream(inputFile,
|
|
993
|
-
const writeStream = fs.createWriteStream(outputFile,
|
|
1025
|
+
const readStream = fs.createReadStream(inputFile, 'utf8');
|
|
1026
|
+
const writeStream = fs.createWriteStream(outputFile, 'utf8');
|
|
994
1027
|
|
|
995
1028
|
// Add UTF-8 BOM if requested
|
|
996
1029
|
if (options.addBOM) {
|
|
997
|
-
writeStream.write(
|
|
1030
|
+
writeStream.write('\uFEFF');
|
|
998
1031
|
}
|
|
999
1032
|
|
|
1000
1033
|
// Parse JSON stream
|
|
1001
|
-
let buffer =
|
|
1002
|
-
|
|
1034
|
+
let buffer = '';
|
|
1035
|
+
const isFirstChunk = true;
|
|
1003
1036
|
let headersWritten = false;
|
|
1004
1037
|
|
|
1005
|
-
readStream.on(
|
|
1038
|
+
readStream.on('data', (chunk) => {
|
|
1006
1039
|
buffer += chunk;
|
|
1007
1040
|
|
|
1008
1041
|
// Try to parse complete JSON objects
|
|
1009
|
-
const lines = buffer.split(
|
|
1010
|
-
buffer = lines.pop() ||
|
|
1042
|
+
const lines = buffer.split('\n');
|
|
1043
|
+
buffer = lines.pop() || '';
|
|
1011
1044
|
|
|
1012
1045
|
for (const line of lines) {
|
|
1013
1046
|
if (line.trim()) {
|
|
@@ -1018,7 +1051,7 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1018
1051
|
if (options.renameMap) {
|
|
1019
1052
|
finalObj = {};
|
|
1020
1053
|
for (const [oldKey, newKey] of Object.entries(
|
|
1021
|
-
options.renameMap
|
|
1054
|
+
options.renameMap
|
|
1022
1055
|
)) {
|
|
1023
1056
|
if (oldKey in obj) {
|
|
1024
1057
|
finalObj[newKey] = obj[oldKey];
|
|
@@ -1036,7 +1069,7 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1036
1069
|
// Write headers on first object
|
|
1037
1070
|
if (!headersWritten && options.includeHeaders !== false) {
|
|
1038
1071
|
const headers = Object.keys(finalObj);
|
|
1039
|
-
writeStream.write(headers.join(options.delimiter ||
|
|
1072
|
+
writeStream.write(headers.join(options.delimiter || ';') + '\n');
|
|
1040
1073
|
headersWritten = true;
|
|
1041
1074
|
}
|
|
1042
1075
|
|
|
@@ -1046,15 +1079,15 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1046
1079
|
.map((value) => {
|
|
1047
1080
|
const str = String(value);
|
|
1048
1081
|
if (
|
|
1049
|
-
str.includes(options.delimiter ||
|
|
1082
|
+
str.includes(options.delimiter || ';') ||
|
|
1050
1083
|
str.includes('"') ||
|
|
1051
|
-
str.includes(
|
|
1084
|
+
str.includes('\n')
|
|
1052
1085
|
) {
|
|
1053
1086
|
return `"${str.replace(/"/g, '""')}"`;
|
|
1054
1087
|
}
|
|
1055
1088
|
return str;
|
|
1056
1089
|
})
|
|
1057
|
-
.join(options.delimiter ||
|
|
1090
|
+
.join(options.delimiter || ';') + '\n';
|
|
1058
1091
|
|
|
1059
1092
|
writeStream.write(row);
|
|
1060
1093
|
|
|
@@ -1063,8 +1096,8 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1063
1096
|
process.stdout.write(
|
|
1064
1097
|
color(
|
|
1065
1098
|
` Processed ${recordCount.toLocaleString()} records\r`,
|
|
1066
|
-
|
|
1067
|
-
)
|
|
1099
|
+
'dim'
|
|
1100
|
+
)
|
|
1068
1101
|
);
|
|
1069
1102
|
}
|
|
1070
1103
|
} catch (error) {
|
|
@@ -1073,8 +1106,8 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1073
1106
|
console.warn(
|
|
1074
1107
|
color(
|
|
1075
1108
|
` Warning: Skipping invalid JSON line: ${error.message}`,
|
|
1076
|
-
|
|
1077
|
-
)
|
|
1109
|
+
'yellow'
|
|
1110
|
+
)
|
|
1078
1111
|
);
|
|
1079
1112
|
}
|
|
1080
1113
|
}
|
|
@@ -1082,7 +1115,7 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1082
1115
|
}
|
|
1083
1116
|
});
|
|
1084
1117
|
|
|
1085
|
-
readStream.on(
|
|
1118
|
+
readStream.on('end', async () => {
|
|
1086
1119
|
// Process remaining buffer
|
|
1087
1120
|
if (buffer.trim()) {
|
|
1088
1121
|
try {
|
|
@@ -1109,7 +1142,7 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1109
1142
|
|
|
1110
1143
|
if (!headersWritten && options.includeHeaders !== false) {
|
|
1111
1144
|
const headers = Object.keys(finalObj);
|
|
1112
|
-
writeStream.write(headers.join(options.delimiter ||
|
|
1145
|
+
writeStream.write(headers.join(options.delimiter || ';') + '\n');
|
|
1113
1146
|
}
|
|
1114
1147
|
|
|
1115
1148
|
const row =
|
|
@@ -1117,15 +1150,15 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1117
1150
|
.map((value) => {
|
|
1118
1151
|
const str = String(value);
|
|
1119
1152
|
if (
|
|
1120
|
-
str.includes(options.delimiter ||
|
|
1153
|
+
str.includes(options.delimiter || ';') ||
|
|
1121
1154
|
str.includes('"') ||
|
|
1122
|
-
str.includes(
|
|
1155
|
+
str.includes('\n')
|
|
1123
1156
|
) {
|
|
1124
1157
|
return `"${str.replace(/"/g, '""')}"`;
|
|
1125
1158
|
}
|
|
1126
1159
|
return str;
|
|
1127
1160
|
})
|
|
1128
|
-
.join(options.delimiter ||
|
|
1161
|
+
.join(options.delimiter || ';') + '\n';
|
|
1129
1162
|
|
|
1130
1163
|
writeStream.write(row);
|
|
1131
1164
|
} catch (error) {
|
|
@@ -1136,31 +1169,31 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
|
|
|
1136
1169
|
writeStream.end();
|
|
1137
1170
|
|
|
1138
1171
|
// Wait for write stream to finish
|
|
1139
|
-
await new Promise((resolve) => writeStream.on(
|
|
1172
|
+
await new Promise((resolve) => writeStream.on('finish', resolve));
|
|
1140
1173
|
|
|
1141
1174
|
const elapsed = Date.now() - startTime;
|
|
1142
1175
|
if (!options.silent) {
|
|
1143
1176
|
console.log(
|
|
1144
1177
|
color(
|
|
1145
1178
|
`\n✓ Streamed ${recordCount.toLocaleString()} records in ${elapsed}ms`,
|
|
1146
|
-
|
|
1147
|
-
)
|
|
1179
|
+
'green'
|
|
1180
|
+
)
|
|
1148
1181
|
);
|
|
1149
|
-
console.log(color(` Output: ${outputFile}`,
|
|
1182
|
+
console.log(color(` Output: ${outputFile}`, 'dim'));
|
|
1150
1183
|
}
|
|
1151
1184
|
});
|
|
1152
1185
|
|
|
1153
|
-
readStream.on(
|
|
1154
|
-
console.error(color(`✗ Stream error: ${error.message}`,
|
|
1186
|
+
readStream.on('error', (error) => {
|
|
1187
|
+
console.error(color(`✗ Stream error: ${error.message}`, 'red'));
|
|
1155
1188
|
process.exit(1);
|
|
1156
1189
|
});
|
|
1157
1190
|
|
|
1158
|
-
writeStream.on(
|
|
1159
|
-
console.error(color(`✗ Write error: ${error.message}`,
|
|
1191
|
+
writeStream.on('error', (error) => {
|
|
1192
|
+
console.error(color(`✗ Write error: ${error.message}`, 'red'));
|
|
1160
1193
|
process.exit(1);
|
|
1161
1194
|
});
|
|
1162
1195
|
} catch (error) {
|
|
1163
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
1196
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
1164
1197
|
if (options.debug) {
|
|
1165
1198
|
console.error(error.stack);
|
|
1166
1199
|
}
|
|
@@ -1174,35 +1207,37 @@ async function streamCsvToJson(inputFile, outputFile, options) {
|
|
|
1174
1207
|
|
|
1175
1208
|
try {
|
|
1176
1209
|
if (!options.silent) {
|
|
1177
|
-
console.log(color(
|
|
1210
|
+
console.log(color('Streaming CSV to JSON...', 'dim'));
|
|
1178
1211
|
}
|
|
1179
1212
|
|
|
1180
1213
|
// Create streams
|
|
1181
|
-
const readStream = fs.createReadStream(inputFile,
|
|
1182
|
-
const writeStream = fs.createWriteStream(outputFile,
|
|
1214
|
+
const readStream = fs.createReadStream(inputFile, 'utf8');
|
|
1215
|
+
const writeStream = fs.createWriteStream(outputFile, 'utf8');
|
|
1183
1216
|
|
|
1184
1217
|
// Write JSON array opening bracket
|
|
1185
|
-
writeStream.write(
|
|
1218
|
+
writeStream.write('[\n');
|
|
1186
1219
|
|
|
1187
|
-
let buffer =
|
|
1220
|
+
let buffer = '';
|
|
1188
1221
|
let isFirstRow = true;
|
|
1189
1222
|
let headers = [];
|
|
1190
1223
|
|
|
1191
|
-
readStream.on(
|
|
1224
|
+
readStream.on('data', (chunk) => {
|
|
1192
1225
|
buffer += chunk;
|
|
1193
1226
|
|
|
1194
1227
|
// Process complete lines
|
|
1195
|
-
const lines = buffer.split(
|
|
1196
|
-
buffer = lines.pop() ||
|
|
1228
|
+
const lines = buffer.split('\n');
|
|
1229
|
+
buffer = lines.pop() || '';
|
|
1197
1230
|
|
|
1198
1231
|
for (let i = 0; i < lines.length; i++) {
|
|
1199
1232
|
const line = lines[i].trim();
|
|
1200
|
-
if (!line)
|
|
1233
|
+
if (!line) {
|
|
1234
|
+
continue;
|
|
1235
|
+
}
|
|
1201
1236
|
|
|
1202
1237
|
rowCount++;
|
|
1203
1238
|
|
|
1204
1239
|
// Parse CSV line
|
|
1205
|
-
const fields = parseCsvLineSimple(line, options.delimiter ||
|
|
1240
|
+
const fields = parseCsvLineSimple(line, options.delimiter || ';');
|
|
1206
1241
|
|
|
1207
1242
|
// First row might be headers
|
|
1208
1243
|
if (rowCount === 1 && options.hasHeaders !== false) {
|
|
@@ -1225,18 +1260,29 @@ async function streamCsvToJson(inputFile, outputFile, options) {
|
|
|
1225
1260
|
let value = fields[j];
|
|
1226
1261
|
|
|
1227
1262
|
// Parse numbers if enabled
|
|
1228
|
-
if (options.parseNumbers
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1263
|
+
if (options.parseNumbers) {
|
|
1264
|
+
// Fast numeric detection
|
|
1265
|
+
const trimmed = value.trim();
|
|
1266
|
+
const firstChar = trimmed.charAt(0);
|
|
1267
|
+
if ((firstChar >= '0' && firstChar <= '9') || firstChar === '-' || firstChar === '.') {
|
|
1268
|
+
const num = parseFloat(trimmed);
|
|
1269
|
+
if (!isNaN(num) && isFinite(num)) {
|
|
1270
|
+
if (String(num) === trimmed || (trimmed.includes('.') && !isNaN(Number(trimmed)))) {
|
|
1271
|
+
value = num;
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1232
1274
|
}
|
|
1233
1275
|
}
|
|
1234
1276
|
|
|
1235
1277
|
// Parse booleans if enabled
|
|
1236
1278
|
if (options.parseBooleans) {
|
|
1237
1279
|
const lowerValue = value.toLowerCase();
|
|
1238
|
-
if (lowerValue ===
|
|
1239
|
-
|
|
1280
|
+
if (lowerValue === 'true') {
|
|
1281
|
+
value = true;
|
|
1282
|
+
}
|
|
1283
|
+
if (lowerValue === 'false') {
|
|
1284
|
+
value = false;
|
|
1285
|
+
}
|
|
1240
1286
|
}
|
|
1241
1287
|
|
|
1242
1288
|
obj[finalHeader] = value;
|
|
@@ -1245,26 +1291,26 @@ async function streamCsvToJson(inputFile, outputFile, options) {
|
|
|
1245
1291
|
// Write JSON object
|
|
1246
1292
|
const jsonStr = JSON.stringify(obj);
|
|
1247
1293
|
if (!isFirstRow) {
|
|
1248
|
-
writeStream.write(
|
|
1294
|
+
writeStream.write(',\n');
|
|
1249
1295
|
}
|
|
1250
|
-
writeStream.write(
|
|
1296
|
+
writeStream.write(' ' + jsonStr);
|
|
1251
1297
|
isFirstRow = false;
|
|
1252
1298
|
|
|
1253
1299
|
// Show progress
|
|
1254
1300
|
if (options.verbose && rowCount % 10000 === 0) {
|
|
1255
1301
|
process.stdout.write(
|
|
1256
|
-
color(` Processed ${rowCount.toLocaleString()} rows\r`,
|
|
1302
|
+
color(` Processed ${rowCount.toLocaleString()} rows\r`, 'dim')
|
|
1257
1303
|
);
|
|
1258
1304
|
}
|
|
1259
1305
|
}
|
|
1260
1306
|
});
|
|
1261
1307
|
|
|
1262
|
-
readStream.on(
|
|
1308
|
+
readStream.on('end', async () => {
|
|
1263
1309
|
// Process remaining buffer
|
|
1264
1310
|
if (buffer.trim()) {
|
|
1265
1311
|
const fields = parseCsvLineSimple(
|
|
1266
1312
|
buffer.trim(),
|
|
1267
|
-
options.delimiter ||
|
|
1313
|
+
options.delimiter || ';'
|
|
1268
1314
|
);
|
|
1269
1315
|
|
|
1270
1316
|
if (fields.length > 0) {
|
|
@@ -1287,43 +1333,43 @@ async function streamCsvToJson(inputFile, outputFile, options) {
|
|
|
1287
1333
|
|
|
1288
1334
|
const jsonStr = JSON.stringify(obj);
|
|
1289
1335
|
if (!isFirstRow) {
|
|
1290
|
-
writeStream.write(
|
|
1336
|
+
writeStream.write(',\n');
|
|
1291
1337
|
}
|
|
1292
|
-
writeStream.write(
|
|
1338
|
+
writeStream.write(' ' + jsonStr);
|
|
1293
1339
|
}
|
|
1294
1340
|
}
|
|
1295
1341
|
}
|
|
1296
1342
|
|
|
1297
1343
|
// Write JSON array closing bracket
|
|
1298
|
-
writeStream.write(
|
|
1344
|
+
writeStream.write('\n]');
|
|
1299
1345
|
writeStream.end();
|
|
1300
1346
|
|
|
1301
1347
|
// Wait for write stream to finish
|
|
1302
|
-
await new Promise((resolve) => writeStream.on(
|
|
1348
|
+
await new Promise((resolve) => writeStream.on('finish', resolve));
|
|
1303
1349
|
|
|
1304
1350
|
const elapsed = Date.now() - startTime;
|
|
1305
1351
|
if (!options.silent) {
|
|
1306
1352
|
console.log(
|
|
1307
1353
|
color(
|
|
1308
1354
|
`\n✓ Streamed ${(rowCount - (options.hasHeaders !== false ? 1 : 0)).toLocaleString()} rows in ${elapsed}ms`,
|
|
1309
|
-
|
|
1310
|
-
)
|
|
1355
|
+
'green'
|
|
1356
|
+
)
|
|
1311
1357
|
);
|
|
1312
|
-
console.log(color(` Output: ${outputFile}`,
|
|
1358
|
+
console.log(color(` Output: ${outputFile}`, 'dim'));
|
|
1313
1359
|
}
|
|
1314
1360
|
});
|
|
1315
1361
|
|
|
1316
|
-
readStream.on(
|
|
1317
|
-
console.error(color(`✗ Stream error: ${error.message}`,
|
|
1362
|
+
readStream.on('error', (error) => {
|
|
1363
|
+
console.error(color(`✗ Stream error: ${error.message}`, 'red'));
|
|
1318
1364
|
process.exit(1);
|
|
1319
1365
|
});
|
|
1320
1366
|
|
|
1321
|
-
writeStream.on(
|
|
1322
|
-
console.error(color(`✗ Write error: ${error.message}`,
|
|
1367
|
+
writeStream.on('error', (error) => {
|
|
1368
|
+
console.error(color(`✗ Write error: ${error.message}`, 'red'));
|
|
1323
1369
|
process.exit(1);
|
|
1324
1370
|
});
|
|
1325
1371
|
} catch (error) {
|
|
1326
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
1372
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
1327
1373
|
if (options.debug) {
|
|
1328
1374
|
console.error(error.stack);
|
|
1329
1375
|
}
|
|
@@ -1334,7 +1380,7 @@ async function streamCsvToJson(inputFile, outputFile, options) {
|
|
|
1334
1380
|
// Simple CSV line parser for streaming
|
|
1335
1381
|
function parseCsvLineSimple(line, delimiter) {
|
|
1336
1382
|
const fields = [];
|
|
1337
|
-
let currentField =
|
|
1383
|
+
let currentField = '';
|
|
1338
1384
|
let inQuotes = false;
|
|
1339
1385
|
|
|
1340
1386
|
for (let i = 0; i < line.length; i++) {
|
|
@@ -1351,7 +1397,7 @@ function parseCsvLineSimple(line, delimiter) {
|
|
|
1351
1397
|
}
|
|
1352
1398
|
} else if (char === delimiter && !inQuotes) {
|
|
1353
1399
|
fields.push(currentField);
|
|
1354
|
-
currentField =
|
|
1400
|
+
currentField = '';
|
|
1355
1401
|
} else {
|
|
1356
1402
|
currentField += char;
|
|
1357
1403
|
}
|
|
@@ -1371,32 +1417,32 @@ async function batchJsonToCsv(inputPattern, outputDir, options) {
|
|
|
1371
1417
|
try {
|
|
1372
1418
|
let glob;
|
|
1373
1419
|
try {
|
|
1374
|
-
glob = require(
|
|
1420
|
+
glob = require('glob');
|
|
1375
1421
|
} catch (error) {
|
|
1376
1422
|
console.error(
|
|
1377
1423
|
color(
|
|
1378
1424
|
'✗ Error: The "glob" module is required for batch processing',
|
|
1379
|
-
|
|
1380
|
-
)
|
|
1425
|
+
'red'
|
|
1426
|
+
)
|
|
1381
1427
|
);
|
|
1382
|
-
console.error(color(
|
|
1383
|
-
console.error(color(
|
|
1428
|
+
console.error(color(' Install it with: npm install glob', 'cyan'));
|
|
1429
|
+
console.error(color(' Or update jtcsv: npm update jtcsv', 'cyan'));
|
|
1384
1430
|
process.exit(1);
|
|
1385
1431
|
}
|
|
1386
1432
|
const files = glob.sync(inputPattern, {
|
|
1387
1433
|
absolute: true,
|
|
1388
|
-
nodir: true
|
|
1434
|
+
nodir: true
|
|
1389
1435
|
});
|
|
1390
1436
|
|
|
1391
1437
|
if (files.length === 0) {
|
|
1392
1438
|
console.error(
|
|
1393
|
-
color(`✗ No files found matching pattern: ${inputPattern}`,
|
|
1439
|
+
color(`✗ No files found matching pattern: ${inputPattern}`, 'red')
|
|
1394
1440
|
);
|
|
1395
1441
|
process.exit(1);
|
|
1396
1442
|
}
|
|
1397
1443
|
|
|
1398
1444
|
if (!options.silent) {
|
|
1399
|
-
console.log(color(`Found ${files.length} files to process...`,
|
|
1445
|
+
console.log(color(`Found ${files.length} files to process...`, 'dim'));
|
|
1400
1446
|
}
|
|
1401
1447
|
|
|
1402
1448
|
// Create output directory if it doesn't exist
|
|
@@ -1409,32 +1455,32 @@ async function batchJsonToCsv(inputPattern, outputDir, options) {
|
|
|
1409
1455
|
for (let i = 0; i < files.length; i += parallelLimit) {
|
|
1410
1456
|
const batch = files.slice(i, i + parallelLimit);
|
|
1411
1457
|
const promises = batch.map(async (file) => {
|
|
1412
|
-
const fileName = path.basename(file,
|
|
1458
|
+
const fileName = path.basename(file, '.json');
|
|
1413
1459
|
const outputFile = path.join(outputDir, `${fileName}.csv`);
|
|
1414
1460
|
|
|
1415
1461
|
if (!options.silent && options.verbose) {
|
|
1416
|
-
console.log(color(` Processing: ${file}`,
|
|
1462
|
+
console.log(color(` Processing: ${file}`, 'dim'));
|
|
1417
1463
|
}
|
|
1418
1464
|
|
|
1419
1465
|
try {
|
|
1420
1466
|
const result = await convertJsonToCsv(file, outputFile, {
|
|
1421
1467
|
...options,
|
|
1422
|
-
silent: true
|
|
1468
|
+
silent: true // Suppress individual file output
|
|
1423
1469
|
});
|
|
1424
1470
|
|
|
1425
1471
|
if (!options.silent) {
|
|
1426
1472
|
console.log(
|
|
1427
1473
|
color(
|
|
1428
1474
|
` ✓ ${fileName}.json → ${fileName}.csv (${result.records} records)`,
|
|
1429
|
-
|
|
1430
|
-
)
|
|
1475
|
+
'green'
|
|
1476
|
+
)
|
|
1431
1477
|
);
|
|
1432
1478
|
}
|
|
1433
1479
|
|
|
1434
1480
|
return { file, success: true, ...result };
|
|
1435
1481
|
} catch (error) {
|
|
1436
1482
|
if (!options.silent) {
|
|
1437
|
-
console.log(color(` ✗ ${fileName}.json: ${error.message}`,
|
|
1483
|
+
console.log(color(` ✗ ${fileName}.json: ${error.message}`, 'red'));
|
|
1438
1484
|
}
|
|
1439
1485
|
return { file, success: false, error: error.message };
|
|
1440
1486
|
}
|
|
@@ -1449,8 +1495,8 @@ async function batchJsonToCsv(inputPattern, outputDir, options) {
|
|
|
1449
1495
|
console.log(
|
|
1450
1496
|
color(
|
|
1451
1497
|
` Progress: ${processed}/${files.length} (${percent}%)`,
|
|
1452
|
-
|
|
1453
|
-
)
|
|
1498
|
+
'dim'
|
|
1499
|
+
)
|
|
1454
1500
|
);
|
|
1455
1501
|
}
|
|
1456
1502
|
}
|
|
@@ -1463,15 +1509,15 @@ async function batchJsonToCsv(inputPattern, outputDir, options) {
|
|
|
1463
1509
|
|
|
1464
1510
|
if (!options.silent) {
|
|
1465
1511
|
console.log(
|
|
1466
|
-
color(`\n✓ Batch processing completed in ${elapsed}ms`,
|
|
1512
|
+
color(`\n✓ Batch processing completed in ${elapsed}ms`, 'green')
|
|
1467
1513
|
);
|
|
1468
1514
|
console.log(
|
|
1469
|
-
color(` Successful: ${successful}/${files.length} files`,
|
|
1515
|
+
color(` Successful: ${successful}/${files.length} files`, 'dim')
|
|
1470
1516
|
);
|
|
1471
1517
|
console.log(
|
|
1472
|
-
color(` Total records: ${totalRecords.toLocaleString()}`,
|
|
1518
|
+
color(` Total records: ${totalRecords.toLocaleString()}`, 'dim')
|
|
1473
1519
|
);
|
|
1474
|
-
console.log(color(` Output directory: ${outputDir}`,
|
|
1520
|
+
console.log(color(` Output directory: ${outputDir}`, 'dim'));
|
|
1475
1521
|
}
|
|
1476
1522
|
|
|
1477
1523
|
return {
|
|
@@ -1479,10 +1525,10 @@ async function batchJsonToCsv(inputPattern, outputDir, options) {
|
|
|
1479
1525
|
successful,
|
|
1480
1526
|
totalRecords,
|
|
1481
1527
|
time: elapsed,
|
|
1482
|
-
results
|
|
1528
|
+
results
|
|
1483
1529
|
};
|
|
1484
1530
|
} catch (error) {
|
|
1485
|
-
console.error(color(`✗ Batch processing error: ${error.message}`,
|
|
1531
|
+
console.error(color(`✗ Batch processing error: ${error.message}`, 'red'));
|
|
1486
1532
|
if (options.debug) {
|
|
1487
1533
|
console.error(error.stack);
|
|
1488
1534
|
}
|
|
@@ -1496,32 +1542,32 @@ async function batchCsvToJson(inputPattern, outputDir, options) {
|
|
|
1496
1542
|
try {
|
|
1497
1543
|
let glob;
|
|
1498
1544
|
try {
|
|
1499
|
-
glob = require(
|
|
1545
|
+
glob = require('glob');
|
|
1500
1546
|
} catch (error) {
|
|
1501
1547
|
console.error(
|
|
1502
1548
|
color(
|
|
1503
1549
|
'✗ Error: The "glob" module is required for batch processing',
|
|
1504
|
-
|
|
1505
|
-
)
|
|
1550
|
+
'red'
|
|
1551
|
+
)
|
|
1506
1552
|
);
|
|
1507
|
-
console.error(color(
|
|
1508
|
-
console.error(color(
|
|
1553
|
+
console.error(color(' Install it with: npm install glob', 'cyan'));
|
|
1554
|
+
console.error(color(' Or update jtcsv: npm update jtcsv', 'cyan'));
|
|
1509
1555
|
process.exit(1);
|
|
1510
1556
|
}
|
|
1511
1557
|
const files = glob.sync(inputPattern, {
|
|
1512
1558
|
absolute: true,
|
|
1513
|
-
nodir: true
|
|
1559
|
+
nodir: true
|
|
1514
1560
|
});
|
|
1515
1561
|
|
|
1516
1562
|
if (files.length === 0) {
|
|
1517
1563
|
console.error(
|
|
1518
|
-
color(`✗ No files found matching pattern: ${inputPattern}`,
|
|
1564
|
+
color(`✗ No files found matching pattern: ${inputPattern}`, 'red')
|
|
1519
1565
|
);
|
|
1520
1566
|
process.exit(1);
|
|
1521
1567
|
}
|
|
1522
1568
|
|
|
1523
1569
|
if (!options.silent) {
|
|
1524
|
-
console.log(color(`Found ${files.length} files to process...`,
|
|
1570
|
+
console.log(color(`Found ${files.length} files to process...`, 'dim'));
|
|
1525
1571
|
}
|
|
1526
1572
|
|
|
1527
1573
|
// Create output directory if it doesn't exist
|
|
@@ -1534,32 +1580,32 @@ async function batchCsvToJson(inputPattern, outputDir, options) {
|
|
|
1534
1580
|
for (let i = 0; i < files.length; i += parallelLimit) {
|
|
1535
1581
|
const batch = files.slice(i, i + parallelLimit);
|
|
1536
1582
|
const promises = batch.map(async (file) => {
|
|
1537
|
-
const fileName = path.basename(file,
|
|
1583
|
+
const fileName = path.basename(file, '.csv');
|
|
1538
1584
|
const outputFile = path.join(outputDir, `${fileName}.json`);
|
|
1539
1585
|
|
|
1540
1586
|
if (!options.silent && options.verbose) {
|
|
1541
|
-
console.log(color(` Processing: ${file}`,
|
|
1587
|
+
console.log(color(` Processing: ${file}`, 'dim'));
|
|
1542
1588
|
}
|
|
1543
1589
|
|
|
1544
1590
|
try {
|
|
1545
1591
|
const result = await convertCsvToJson(file, outputFile, {
|
|
1546
1592
|
...options,
|
|
1547
|
-
silent: true
|
|
1593
|
+
silent: true // Suppress individual file output
|
|
1548
1594
|
});
|
|
1549
1595
|
|
|
1550
1596
|
if (!options.silent) {
|
|
1551
1597
|
console.log(
|
|
1552
1598
|
color(
|
|
1553
1599
|
` ✓ ${fileName}.csv → ${fileName}.json (${result.rows} rows)`,
|
|
1554
|
-
|
|
1555
|
-
)
|
|
1600
|
+
'green'
|
|
1601
|
+
)
|
|
1556
1602
|
);
|
|
1557
1603
|
}
|
|
1558
1604
|
|
|
1559
1605
|
return { file, success: true, ...result };
|
|
1560
1606
|
} catch (error) {
|
|
1561
1607
|
if (!options.silent) {
|
|
1562
|
-
console.log(color(` ✗ ${fileName}.csv: ${error.message}`,
|
|
1608
|
+
console.log(color(` ✗ ${fileName}.csv: ${error.message}`, 'red'));
|
|
1563
1609
|
}
|
|
1564
1610
|
return { file, success: false, error: error.message };
|
|
1565
1611
|
}
|
|
@@ -1574,8 +1620,8 @@ async function batchCsvToJson(inputPattern, outputDir, options) {
|
|
|
1574
1620
|
console.log(
|
|
1575
1621
|
color(
|
|
1576
1622
|
` Progress: ${processed}/${files.length} (${percent}%)`,
|
|
1577
|
-
|
|
1578
|
-
)
|
|
1623
|
+
'dim'
|
|
1624
|
+
)
|
|
1579
1625
|
);
|
|
1580
1626
|
}
|
|
1581
1627
|
}
|
|
@@ -1588,13 +1634,13 @@ async function batchCsvToJson(inputPattern, outputDir, options) {
|
|
|
1588
1634
|
|
|
1589
1635
|
if (!options.silent) {
|
|
1590
1636
|
console.log(
|
|
1591
|
-
color(`\n✓ Batch processing completed in ${elapsed}ms`,
|
|
1637
|
+
color(`\n✓ Batch processing completed in ${elapsed}ms`, 'green')
|
|
1592
1638
|
);
|
|
1593
1639
|
console.log(
|
|
1594
|
-
color(` Successful: ${successful}/${files.length} files`,
|
|
1640
|
+
color(` Successful: ${successful}/${files.length} files`, 'dim')
|
|
1595
1641
|
);
|
|
1596
|
-
console.log(color(` Total rows: ${totalRows.toLocaleString()}`,
|
|
1597
|
-
console.log(color(` Output directory: ${outputDir}`,
|
|
1642
|
+
console.log(color(` Total rows: ${totalRows.toLocaleString()}`, 'dim'));
|
|
1643
|
+
console.log(color(` Output directory: ${outputDir}`, 'dim'));
|
|
1598
1644
|
}
|
|
1599
1645
|
|
|
1600
1646
|
return {
|
|
@@ -1602,10 +1648,10 @@ async function batchCsvToJson(inputPattern, outputDir, options) {
|
|
|
1602
1648
|
successful,
|
|
1603
1649
|
totalRows,
|
|
1604
1650
|
time: elapsed,
|
|
1605
|
-
results
|
|
1651
|
+
results
|
|
1606
1652
|
};
|
|
1607
1653
|
} catch (error) {
|
|
1608
|
-
console.error(color(`✗ Batch processing error: ${error.message}`,
|
|
1654
|
+
console.error(color(`✗ Batch processing error: ${error.message}`, 'red'));
|
|
1609
1655
|
if (options.debug) {
|
|
1610
1656
|
console.error(error.stack);
|
|
1611
1657
|
}
|
|
@@ -1619,34 +1665,34 @@ async function batchProcessMixed(inputPattern, outputDir, options) {
|
|
|
1619
1665
|
try {
|
|
1620
1666
|
let glob;
|
|
1621
1667
|
try {
|
|
1622
|
-
glob = require(
|
|
1668
|
+
glob = require('glob');
|
|
1623
1669
|
} catch (error) {
|
|
1624
1670
|
console.error(
|
|
1625
1671
|
color(
|
|
1626
1672
|
'✗ Error: The "glob" module is required for batch processing',
|
|
1627
|
-
|
|
1628
|
-
)
|
|
1673
|
+
'red'
|
|
1674
|
+
)
|
|
1629
1675
|
);
|
|
1630
|
-
console.error(color(
|
|
1631
|
-
console.error(color(
|
|
1676
|
+
console.error(color(' Install it with: npm install glob', 'cyan'));
|
|
1677
|
+
console.error(color(' Or update jtcsv: npm update jtcsv', 'cyan'));
|
|
1632
1678
|
process.exit(1);
|
|
1633
1679
|
}
|
|
1634
1680
|
|
|
1635
1681
|
// Находим все файлы
|
|
1636
1682
|
const files = glob.sync(inputPattern, {
|
|
1637
1683
|
absolute: true,
|
|
1638
|
-
nodir: true
|
|
1684
|
+
nodir: true
|
|
1639
1685
|
});
|
|
1640
1686
|
|
|
1641
1687
|
if (files.length === 0) {
|
|
1642
1688
|
console.error(
|
|
1643
|
-
color(`✗ No files found matching pattern: ${inputPattern}`,
|
|
1689
|
+
color(`✗ No files found matching pattern: ${inputPattern}`, 'red')
|
|
1644
1690
|
);
|
|
1645
1691
|
process.exit(1);
|
|
1646
1692
|
}
|
|
1647
1693
|
|
|
1648
1694
|
if (!options.silent) {
|
|
1649
|
-
console.log(color(`Found ${files.length} files to process...`,
|
|
1695
|
+
console.log(color(`Found ${files.length} files to process...`, 'dim'));
|
|
1650
1696
|
}
|
|
1651
1697
|
|
|
1652
1698
|
// Создаем выходную директорию
|
|
@@ -1657,23 +1703,23 @@ async function batchProcessMixed(inputPattern, outputDir, options) {
|
|
|
1657
1703
|
|
|
1658
1704
|
// Группируем файлы по типу
|
|
1659
1705
|
const jsonFiles = files.filter((file) =>
|
|
1660
|
-
file.toLowerCase().endsWith(
|
|
1706
|
+
file.toLowerCase().endsWith('.json')
|
|
1661
1707
|
);
|
|
1662
1708
|
const csvFiles = files.filter((file) =>
|
|
1663
|
-
file.toLowerCase().endsWith(
|
|
1709
|
+
file.toLowerCase().endsWith('.csv')
|
|
1664
1710
|
);
|
|
1665
1711
|
const otherFiles = files.filter(
|
|
1666
1712
|
(file) =>
|
|
1667
|
-
!file.toLowerCase().endsWith(
|
|
1668
|
-
!file.toLowerCase().endsWith(
|
|
1713
|
+
!file.toLowerCase().endsWith('.json') &&
|
|
1714
|
+
!file.toLowerCase().endsWith('.csv')
|
|
1669
1715
|
);
|
|
1670
1716
|
|
|
1671
1717
|
if (otherFiles.length > 0 && !options.silent) {
|
|
1672
1718
|
console.log(
|
|
1673
1719
|
color(
|
|
1674
1720
|
` Warning: Skipping ${otherFiles.length} non-JSON/CSV files`,
|
|
1675
|
-
|
|
1676
|
-
)
|
|
1721
|
+
'yellow'
|
|
1722
|
+
)
|
|
1677
1723
|
);
|
|
1678
1724
|
}
|
|
1679
1725
|
|
|
@@ -1681,34 +1727,34 @@ async function batchProcessMixed(inputPattern, outputDir, options) {
|
|
|
1681
1727
|
for (let i = 0; i < jsonFiles.length; i += parallelLimit) {
|
|
1682
1728
|
const batch = jsonFiles.slice(i, i + parallelLimit);
|
|
1683
1729
|
const promises = batch.map(async (file) => {
|
|
1684
|
-
const fileName = path.basename(file,
|
|
1730
|
+
const fileName = path.basename(file, '.json');
|
|
1685
1731
|
const outputFile = path.join(outputDir, `${fileName}.csv`);
|
|
1686
1732
|
|
|
1687
1733
|
if (!options.silent && options.verbose) {
|
|
1688
|
-
console.log(color(` Processing JSON: ${file}`,
|
|
1734
|
+
console.log(color(` Processing JSON: ${file}`, 'dim'));
|
|
1689
1735
|
}
|
|
1690
1736
|
|
|
1691
1737
|
try {
|
|
1692
1738
|
const result = await convertJsonToCsv(file, outputFile, {
|
|
1693
1739
|
...options,
|
|
1694
|
-
silent: true
|
|
1740
|
+
silent: true
|
|
1695
1741
|
});
|
|
1696
1742
|
|
|
1697
1743
|
if (!options.silent) {
|
|
1698
1744
|
console.log(
|
|
1699
1745
|
color(
|
|
1700
1746
|
` ✓ ${fileName}.json → ${fileName}.csv (${result.records} records)`,
|
|
1701
|
-
|
|
1702
|
-
)
|
|
1747
|
+
'green'
|
|
1748
|
+
)
|
|
1703
1749
|
);
|
|
1704
1750
|
}
|
|
1705
1751
|
|
|
1706
|
-
return { file, type:
|
|
1752
|
+
return { file, type: 'json', success: true, ...result };
|
|
1707
1753
|
} catch (error) {
|
|
1708
1754
|
if (!options.silent) {
|
|
1709
|
-
console.log(color(` ✗ ${fileName}.json: ${error.message}`,
|
|
1755
|
+
console.log(color(` ✗ ${fileName}.json: ${error.message}`, 'red'));
|
|
1710
1756
|
}
|
|
1711
|
-
return { file, type:
|
|
1757
|
+
return { file, type: 'json', success: false, error: error.message };
|
|
1712
1758
|
}
|
|
1713
1759
|
});
|
|
1714
1760
|
|
|
@@ -1720,34 +1766,34 @@ async function batchProcessMixed(inputPattern, outputDir, options) {
|
|
|
1720
1766
|
for (let i = 0; i < csvFiles.length; i += parallelLimit) {
|
|
1721
1767
|
const batch = csvFiles.slice(i, i + parallelLimit);
|
|
1722
1768
|
const promises = batch.map(async (file) => {
|
|
1723
|
-
const fileName = path.basename(file,
|
|
1769
|
+
const fileName = path.basename(file, '.csv');
|
|
1724
1770
|
const outputFile = path.join(outputDir, `${fileName}.json`);
|
|
1725
1771
|
|
|
1726
1772
|
if (!options.silent && options.verbose) {
|
|
1727
|
-
console.log(color(` Processing CSV: ${file}`,
|
|
1773
|
+
console.log(color(` Processing CSV: ${file}`, 'dim'));
|
|
1728
1774
|
}
|
|
1729
1775
|
|
|
1730
1776
|
try {
|
|
1731
1777
|
const result = await convertCsvToJson(file, outputFile, {
|
|
1732
1778
|
...options,
|
|
1733
|
-
silent: true
|
|
1779
|
+
silent: true
|
|
1734
1780
|
});
|
|
1735
1781
|
|
|
1736
1782
|
if (!options.silent) {
|
|
1737
1783
|
console.log(
|
|
1738
1784
|
color(
|
|
1739
1785
|
` ✓ ${fileName}.csv → ${fileName}.json (${result.rows} rows)`,
|
|
1740
|
-
|
|
1741
|
-
)
|
|
1786
|
+
'green'
|
|
1787
|
+
)
|
|
1742
1788
|
);
|
|
1743
1789
|
}
|
|
1744
1790
|
|
|
1745
|
-
return { file, type:
|
|
1791
|
+
return { file, type: 'csv', success: true, ...result };
|
|
1746
1792
|
} catch (error) {
|
|
1747
1793
|
if (!options.silent) {
|
|
1748
|
-
console.log(color(` ✗ ${fileName}.csv: ${error.message}`,
|
|
1794
|
+
console.log(color(` ✗ ${fileName}.csv: ${error.message}`, 'red'));
|
|
1749
1795
|
}
|
|
1750
|
-
return { file, type:
|
|
1796
|
+
return { file, type: 'csv', success: false, error: error.message };
|
|
1751
1797
|
}
|
|
1752
1798
|
});
|
|
1753
1799
|
|
|
@@ -1763,21 +1809,21 @@ async function batchProcessMixed(inputPattern, outputDir, options) {
|
|
|
1763
1809
|
|
|
1764
1810
|
if (!options.silent) {
|
|
1765
1811
|
console.log(
|
|
1766
|
-
color(`\n✓ Mixed batch processing completed in ${elapsed}ms`,
|
|
1812
|
+
color(`\n✓ Mixed batch processing completed in ${elapsed}ms`, 'green')
|
|
1767
1813
|
);
|
|
1768
1814
|
console.log(
|
|
1769
|
-
color(` Successful: ${successful}/${files.length} files`,
|
|
1815
|
+
color(` Successful: ${successful}/${files.length} files`, 'dim')
|
|
1770
1816
|
);
|
|
1771
1817
|
console.log(
|
|
1772
1818
|
color(
|
|
1773
1819
|
` JSON files: ${jsonFiles.length}, CSV files: ${csvFiles.length}`,
|
|
1774
|
-
|
|
1775
|
-
)
|
|
1820
|
+
'dim'
|
|
1821
|
+
)
|
|
1776
1822
|
);
|
|
1777
1823
|
console.log(
|
|
1778
|
-
color(` Total records: ${totalRecords.toLocaleString()}`,
|
|
1824
|
+
color(` Total records: ${totalRecords.toLocaleString()}`, 'dim')
|
|
1779
1825
|
);
|
|
1780
|
-
console.log(color(` Output directory: ${outputDir}`,
|
|
1826
|
+
console.log(color(` Output directory: ${outputDir}`, 'dim'));
|
|
1781
1827
|
}
|
|
1782
1828
|
|
|
1783
1829
|
return {
|
|
@@ -1788,10 +1834,10 @@ async function batchProcessMixed(inputPattern, outputDir, options) {
|
|
|
1788
1834
|
successful,
|
|
1789
1835
|
totalRecords,
|
|
1790
1836
|
time: elapsed,
|
|
1791
|
-
results
|
|
1837
|
+
results
|
|
1792
1838
|
};
|
|
1793
1839
|
} catch (error) {
|
|
1794
|
-
console.error(color(`✗ Batch processing error: ${error.message}`,
|
|
1840
|
+
console.error(color(`✗ Batch processing error: ${error.message}`, 'red'));
|
|
1795
1841
|
if (options.debug) {
|
|
1796
1842
|
console.error(error.stack);
|
|
1797
1843
|
}
|
|
@@ -1805,9 +1851,9 @@ async function batchProcessMixed(inputPattern, outputDir, options) {
|
|
|
1805
1851
|
|
|
1806
1852
|
function parseOptions(args) {
|
|
1807
1853
|
const options = {
|
|
1808
|
-
delimiter:
|
|
1854
|
+
delimiter: ';',
|
|
1809
1855
|
autoDetect: true,
|
|
1810
|
-
candidates: [
|
|
1856
|
+
candidates: [';', ',', '\t', '|'],
|
|
1811
1857
|
hasHeaders: true,
|
|
1812
1858
|
includeHeaders: true,
|
|
1813
1859
|
renameMap: undefined,
|
|
@@ -1816,7 +1862,7 @@ function parseOptions(args) {
|
|
|
1816
1862
|
parseNumbers: false,
|
|
1817
1863
|
parseBooleans: false,
|
|
1818
1864
|
useFastPath: true,
|
|
1819
|
-
fastPathMode:
|
|
1865
|
+
fastPathMode: 'objects',
|
|
1820
1866
|
preventCsvInjection: true,
|
|
1821
1867
|
rfc4180Compliant: true,
|
|
1822
1868
|
maxRecords: undefined,
|
|
@@ -1831,168 +1877,192 @@ function parseOptions(args) {
|
|
|
1831
1877
|
unwrapArrays: false,
|
|
1832
1878
|
stringifyObjects: false,
|
|
1833
1879
|
recursive: false,
|
|
1834
|
-
pattern:
|
|
1835
|
-
outputDir:
|
|
1880
|
+
pattern: '**/*',
|
|
1881
|
+
outputDir: './output',
|
|
1836
1882
|
overwrite: false,
|
|
1837
1883
|
parallel: 4,
|
|
1838
1884
|
chunkSize: 65536,
|
|
1839
1885
|
bufferSize: 1000,
|
|
1840
1886
|
schema: undefined,
|
|
1841
1887
|
transform: undefined,
|
|
1842
|
-
flattenPrefix:
|
|
1888
|
+
flattenPrefix: '_',
|
|
1889
|
+
flatten: false,
|
|
1890
|
+
flattenSeparator: '.',
|
|
1891
|
+
flattenMaxDepth: 3,
|
|
1892
|
+
arrayHandling: 'stringify'
|
|
1843
1893
|
};
|
|
1844
1894
|
|
|
1845
1895
|
const files = [];
|
|
1846
1896
|
|
|
1847
|
-
for (let i = 0; i < args.length; i++) {
|
|
1848
|
-
const arg = args[i];
|
|
1849
|
-
|
|
1850
|
-
if (arg
|
|
1851
|
-
|
|
1897
|
+
for (let i = 0; i < args.length; i++) {
|
|
1898
|
+
const arg = args[i];
|
|
1899
|
+
|
|
1900
|
+
if (arg === '-') {
|
|
1901
|
+
files.push(arg);
|
|
1902
|
+
continue;
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
if (arg.startsWith('--')) {
|
|
1906
|
+
const [key, value] = arg.slice(2).split('=');
|
|
1852
1907
|
|
|
1853
1908
|
switch (key) {
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
options.fastPathMode !==
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1909
|
+
case 'delimiter':
|
|
1910
|
+
options.delimiter = value || ',';
|
|
1911
|
+
options.autoDetect = false;
|
|
1912
|
+
break;
|
|
1913
|
+
case 'auto-detect':
|
|
1914
|
+
options.autoDetect = value !== 'false';
|
|
1915
|
+
break;
|
|
1916
|
+
case 'candidates':
|
|
1917
|
+
options.candidates = value ? value.split(',') : [';', ',', '\t', '|'];
|
|
1918
|
+
break;
|
|
1919
|
+
case 'no-headers':
|
|
1920
|
+
options.includeHeaders = false;
|
|
1921
|
+
options.hasHeaders = false;
|
|
1922
|
+
break;
|
|
1923
|
+
case 'parse-numbers':
|
|
1924
|
+
options.parseNumbers = true;
|
|
1925
|
+
break;
|
|
1926
|
+
case 'parse-booleans':
|
|
1927
|
+
options.parseBooleans = true;
|
|
1928
|
+
break;
|
|
1929
|
+
case 'no-trim':
|
|
1930
|
+
options.trim = false;
|
|
1931
|
+
break;
|
|
1932
|
+
case 'no-fast-path':
|
|
1933
|
+
options.useFastPath = false;
|
|
1934
|
+
break;
|
|
1935
|
+
case 'fast-path':
|
|
1936
|
+
options.useFastPath = value !== 'false';
|
|
1937
|
+
break;
|
|
1938
|
+
case 'fast-path-mode':
|
|
1939
|
+
options.fastPathMode = value || 'objects';
|
|
1940
|
+
if (
|
|
1941
|
+
options.fastPathMode !== 'objects' &&
|
|
1942
|
+
options.fastPathMode !== 'compact'
|
|
1943
|
+
) {
|
|
1944
|
+
throw new Error('Invalid --fast-path-mode value (objects|compact)');
|
|
1945
|
+
}
|
|
1946
|
+
break;
|
|
1947
|
+
case 'rename':
|
|
1948
|
+
try {
|
|
1949
|
+
const jsonStr = value || '{}';
|
|
1950
|
+
const cleanStr = jsonStr
|
|
1951
|
+
.replace(/^'|'$/g, '')
|
|
1952
|
+
.replace(/^"|"$/g, '');
|
|
1953
|
+
options.renameMap = JSON.parse(cleanStr);
|
|
1954
|
+
} catch (e) {
|
|
1955
|
+
throw new Error(`Invalid JSON in --rename option: ${e.message}`);
|
|
1956
|
+
}
|
|
1957
|
+
break;
|
|
1958
|
+
case 'template':
|
|
1959
|
+
try {
|
|
1960
|
+
const jsonStr = value || '{}';
|
|
1961
|
+
const cleanStr = jsonStr
|
|
1962
|
+
.replace(/^'|'$/g, '')
|
|
1963
|
+
.replace(/^"|"$/g, '');
|
|
1964
|
+
options.template = JSON.parse(cleanStr);
|
|
1965
|
+
} catch (e) {
|
|
1966
|
+
throw new Error(`Invalid JSON in --template option: ${e.message}`);
|
|
1967
|
+
}
|
|
1968
|
+
break;
|
|
1969
|
+
case 'no-injection-protection':
|
|
1970
|
+
options.preventCsvInjection = false;
|
|
1971
|
+
break;
|
|
1972
|
+
case 'no-rfc4180':
|
|
1973
|
+
options.rfc4180Compliant = false;
|
|
1974
|
+
break;
|
|
1975
|
+
case 'max-records':
|
|
1976
|
+
options.maxRecords = parseInt(value, 10);
|
|
1977
|
+
break;
|
|
1978
|
+
case 'max-rows':
|
|
1979
|
+
options.maxRows = parseInt(value, 10);
|
|
1980
|
+
break;
|
|
1981
|
+
case 'max-depth':
|
|
1982
|
+
options.maxDepth = parseInt(value, 10) || 5;
|
|
1983
|
+
break;
|
|
1984
|
+
case 'pretty':
|
|
1985
|
+
options.pretty = true;
|
|
1986
|
+
break;
|
|
1987
|
+
case 'flatten':
|
|
1988
|
+
options.flatten = true;
|
|
1989
|
+
break;
|
|
1990
|
+
case 'flatten-separator':
|
|
1991
|
+
options.flattenSeparator = value || '.';
|
|
1992
|
+
break;
|
|
1993
|
+
case 'flatten-max-depth':
|
|
1994
|
+
options.flattenMaxDepth = parseInt(value, 10) || 3;
|
|
1995
|
+
break;
|
|
1996
|
+
case 'array-handling':
|
|
1997
|
+
options.arrayHandling = value || 'stringify';
|
|
1998
|
+
if (!['stringify', 'join', 'expand'].includes(options.arrayHandling)) {
|
|
1999
|
+
throw new Error('Invalid --array-handling value (stringify|join|expand)');
|
|
2000
|
+
}
|
|
2001
|
+
break;
|
|
2002
|
+
case 'silent':
|
|
2003
|
+
options.silent = true;
|
|
2004
|
+
break;
|
|
2005
|
+
case 'verbose':
|
|
2006
|
+
options.verbose = true;
|
|
2007
|
+
break;
|
|
2008
|
+
case 'debug':
|
|
2009
|
+
options.debug = true;
|
|
2010
|
+
break;
|
|
2011
|
+
case 'dry-run':
|
|
2012
|
+
options.dryRun = true;
|
|
2013
|
+
break;
|
|
2014
|
+
case 'add-bom':
|
|
2015
|
+
options.addBOM = true;
|
|
2016
|
+
break;
|
|
2017
|
+
case 'unwrap-arrays':
|
|
2018
|
+
options.unwrapArrays = true;
|
|
2019
|
+
break;
|
|
2020
|
+
case 'stringify-objects':
|
|
2021
|
+
options.stringifyObjects = true;
|
|
2022
|
+
break;
|
|
2023
|
+
case 'recursive':
|
|
2024
|
+
options.recursive = true;
|
|
2025
|
+
break;
|
|
2026
|
+
case 'pattern':
|
|
2027
|
+
options.pattern = value || '**/*';
|
|
2028
|
+
break;
|
|
2029
|
+
case 'output-dir':
|
|
2030
|
+
options.outputDir = value || './output';
|
|
2031
|
+
break;
|
|
2032
|
+
case 'overwrite':
|
|
2033
|
+
options.overwrite = true;
|
|
2034
|
+
break;
|
|
2035
|
+
case 'parallel':
|
|
2036
|
+
options.parallel = parseInt(value, 10) || 4;
|
|
2037
|
+
break;
|
|
2038
|
+
case 'chunk-size':
|
|
2039
|
+
options.chunkSize = parseInt(value, 10) || 65536;
|
|
2040
|
+
break;
|
|
2041
|
+
case 'buffer-size':
|
|
2042
|
+
options.bufferSize = parseInt(value, 10) || 1000;
|
|
2043
|
+
break;
|
|
2044
|
+
case 'schema':
|
|
2045
|
+
try {
|
|
2046
|
+
const jsonStr = value || '{}';
|
|
2047
|
+
const cleanStr = jsonStr
|
|
2048
|
+
.replace(/^'|'$/g, '')
|
|
2049
|
+
.replace(/^"|"$/g, '');
|
|
2050
|
+
options.schema = JSON.parse(cleanStr);
|
|
2051
|
+
} catch (e) {
|
|
2052
|
+
throw new Error(`Invalid JSON in --schema option: ${e.message}`);
|
|
2053
|
+
}
|
|
2054
|
+
break;
|
|
2055
|
+
case 'transform':
|
|
2056
|
+
options.transform = value;
|
|
2057
|
+
break;
|
|
2058
|
+
case 'port':
|
|
2059
|
+
options.port = parseInt(value, 10) || 3000;
|
|
2060
|
+
break;
|
|
2061
|
+
case 'host':
|
|
2062
|
+
options.host = value || 'localhost';
|
|
2063
|
+
break;
|
|
1994
2064
|
}
|
|
1995
|
-
} else if (!arg.startsWith(
|
|
2065
|
+
} else if (!arg.startsWith('-')) {
|
|
1996
2066
|
files.push(arg);
|
|
1997
2067
|
}
|
|
1998
2068
|
}
|
|
@@ -2006,21 +2076,21 @@ function parseOptions(args) {
|
|
|
2006
2076
|
|
|
2007
2077
|
async function launchTUI() {
|
|
2008
2078
|
try {
|
|
2009
|
-
console.log(color(
|
|
2010
|
-
console.log(color(
|
|
2079
|
+
console.log(color('Launching Terminal User Interface...', 'cyan'));
|
|
2080
|
+
console.log(color('Press Ctrl+Q to exit', 'dim'));
|
|
2011
2081
|
|
|
2012
|
-
const JtcsvTUI = require(
|
|
2082
|
+
const JtcsvTUI = require('@jtcsv/tui');
|
|
2013
2083
|
const tui = new JtcsvTUI();
|
|
2014
2084
|
tui.start();
|
|
2015
2085
|
} catch (error) {
|
|
2016
|
-
if (error.code ===
|
|
2017
|
-
console.error(color(
|
|
2018
|
-
console.log(color(
|
|
2019
|
-
console.log(color(
|
|
2020
|
-
console.log(color(
|
|
2021
|
-
console.log(color(
|
|
2086
|
+
if (error.code === 'MODULE_NOT_FOUND') {
|
|
2087
|
+
console.error(color('Error: @jtcsv/tui is not installed', 'red'));
|
|
2088
|
+
console.log(color('Install it with:', 'dim'));
|
|
2089
|
+
console.log(color(' npm install @jtcsv/tui', 'cyan'));
|
|
2090
|
+
console.log(color('\nOr use the CLI interface instead:', 'dim'));
|
|
2091
|
+
console.log(color(' jtcsv help', 'cyan'));
|
|
2022
2092
|
} else {
|
|
2023
|
-
console.error(color(`Error: ${error.message}`,
|
|
2093
|
+
console.error(color(`Error: ${error.message}`, 'red'));
|
|
2024
2094
|
}
|
|
2025
2095
|
process.exit(1);
|
|
2026
2096
|
}
|
|
@@ -2028,13 +2098,13 @@ async function launchTUI() {
|
|
|
2028
2098
|
|
|
2029
2099
|
async function launchWebUI(options = {}) {
|
|
2030
2100
|
try {
|
|
2031
|
-
const webServer = require(
|
|
2101
|
+
const webServer = require('../src/web-server');
|
|
2032
2102
|
webServer.startServer({
|
|
2033
2103
|
port: options.port || 3000,
|
|
2034
2104
|
host: options.host || 'localhost'
|
|
2035
2105
|
});
|
|
2036
2106
|
} catch (error) {
|
|
2037
|
-
console.error(color(`Error: ${error.message}`,
|
|
2107
|
+
console.error(color(`Error: ${error.message}`, 'red'));
|
|
2038
2108
|
if (options.debug) {
|
|
2039
2109
|
console.error(error.stack);
|
|
2040
2110
|
}
|
|
@@ -2043,78 +2113,78 @@ async function launchWebUI(options = {}) {
|
|
|
2043
2113
|
}
|
|
2044
2114
|
|
|
2045
2115
|
async function startBasicTUI() {
|
|
2046
|
-
const readline = require(
|
|
2116
|
+
const readline = require('readline');
|
|
2047
2117
|
|
|
2048
2118
|
const rl = readline.createInterface({
|
|
2049
2119
|
input: process.stdin,
|
|
2050
|
-
output: process.stdout
|
|
2120
|
+
output: process.stdout
|
|
2051
2121
|
});
|
|
2052
2122
|
|
|
2053
2123
|
console.clear();
|
|
2054
|
-
console.log(color(
|
|
2055
|
-
console.log(color(
|
|
2056
|
-
console.log(color(
|
|
2124
|
+
console.log(color('╔══════════════════════════════════════╗', 'cyan'));
|
|
2125
|
+
console.log(color('║ JTCSV Terminal Interface ║', 'cyan'));
|
|
2126
|
+
console.log(color('╚══════════════════════════════════════╝', 'cyan'));
|
|
2057
2127
|
console.log();
|
|
2058
|
-
console.log(color(
|
|
2059
|
-
console.log(
|
|
2060
|
-
console.log(
|
|
2061
|
-
console.log(
|
|
2062
|
-
console.log(
|
|
2063
|
-
console.log(
|
|
2128
|
+
console.log(color('Select operation:', 'bright'));
|
|
2129
|
+
console.log(' 1. JSON → CSV');
|
|
2130
|
+
console.log(' 2. CSV → JSON');
|
|
2131
|
+
console.log(' 3. Preprocess JSON');
|
|
2132
|
+
console.log(' 4. Batch Processing');
|
|
2133
|
+
console.log(' 5. Exit');
|
|
2064
2134
|
console.log();
|
|
2065
2135
|
|
|
2066
|
-
rl.question(color(
|
|
2136
|
+
rl.question(color('Enter choice (1-5): ', 'cyan'), async (choice) => {
|
|
2067
2137
|
switch (choice) {
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2138
|
+
case '1':
|
|
2139
|
+
await runJsonToCsvTUI(rl);
|
|
2140
|
+
break;
|
|
2141
|
+
case '2':
|
|
2142
|
+
await runCsvToJsonTUI(rl);
|
|
2143
|
+
break;
|
|
2144
|
+
case '3':
|
|
2145
|
+
console.log(color('Preprocess feature coming soon...', 'yellow'));
|
|
2146
|
+
rl.close();
|
|
2147
|
+
break;
|
|
2148
|
+
case '4':
|
|
2149
|
+
console.log(color('Batch processing coming soon...', 'yellow'));
|
|
2150
|
+
rl.close();
|
|
2151
|
+
break;
|
|
2152
|
+
case '5':
|
|
2153
|
+
console.log(color('Goodbye!', 'green'));
|
|
2154
|
+
rl.close();
|
|
2155
|
+
process.exit(0);
|
|
2156
|
+
break;
|
|
2157
|
+
default:
|
|
2158
|
+
console.log(color('Invalid choice', 'red'));
|
|
2159
|
+
rl.close();
|
|
2160
|
+
process.exit(1);
|
|
2091
2161
|
}
|
|
2092
2162
|
});
|
|
2093
2163
|
}
|
|
2094
2164
|
|
|
2095
2165
|
async function runJsonToCsvTUI(rl) {
|
|
2096
2166
|
console.clear();
|
|
2097
|
-
console.log(color(
|
|
2167
|
+
console.log(color('JSON → CSV Conversion', 'cyan'));
|
|
2098
2168
|
console.log();
|
|
2099
2169
|
|
|
2100
|
-
rl.question(
|
|
2101
|
-
rl.question(
|
|
2102
|
-
rl.question(
|
|
2170
|
+
rl.question('Input JSON file: ', (inputFile) => {
|
|
2171
|
+
rl.question('Output CSV file: ', async (outputFile) => {
|
|
2172
|
+
rl.question('Delimiter (default: ;): ', async (delimiter) => {
|
|
2103
2173
|
try {
|
|
2104
|
-
console.log(color(
|
|
2174
|
+
console.log(color('\nConverting...', 'dim'));
|
|
2105
2175
|
|
|
2106
2176
|
const result = await convertJsonToCsv(inputFile, outputFile, {
|
|
2107
|
-
delimiter: delimiter ||
|
|
2108
|
-
silent: false
|
|
2177
|
+
delimiter: delimiter || ';',
|
|
2178
|
+
silent: false
|
|
2109
2179
|
});
|
|
2110
2180
|
|
|
2111
|
-
console.log(color(
|
|
2112
|
-
rl.question(
|
|
2181
|
+
console.log(color('\n✓ Conversion complete!', 'green'));
|
|
2182
|
+
rl.question('\nPress Enter to continue...', () => {
|
|
2113
2183
|
rl.close();
|
|
2114
2184
|
startBasicTUI();
|
|
2115
2185
|
});
|
|
2116
2186
|
} catch (error) {
|
|
2117
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
2187
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
2118
2188
|
rl.close();
|
|
2119
2189
|
process.exit(1);
|
|
2120
2190
|
}
|
|
@@ -2125,29 +2195,29 @@ async function runJsonToCsvTUI(rl) {
|
|
|
2125
2195
|
|
|
2126
2196
|
async function runCsvToJsonTUI(rl) {
|
|
2127
2197
|
console.clear();
|
|
2128
|
-
console.log(color(
|
|
2198
|
+
console.log(color('CSV → JSON Conversion', 'cyan'));
|
|
2129
2199
|
console.log();
|
|
2130
2200
|
|
|
2131
|
-
rl.question(
|
|
2132
|
-
rl.question(
|
|
2133
|
-
rl.question(
|
|
2134
|
-
rl.question(
|
|
2201
|
+
rl.question('Input CSV file: ', (inputFile) => {
|
|
2202
|
+
rl.question('Output JSON file: ', async (outputFile) => {
|
|
2203
|
+
rl.question('Delimiter (default: ;): ', async (delimiter) => {
|
|
2204
|
+
rl.question('Pretty print? (y/n): ', async (pretty) => {
|
|
2135
2205
|
try {
|
|
2136
|
-
console.log(color(
|
|
2206
|
+
console.log(color('\nConverting...', 'dim'));
|
|
2137
2207
|
|
|
2138
2208
|
const result = await convertCsvToJson(inputFile, outputFile, {
|
|
2139
|
-
delimiter: delimiter ||
|
|
2140
|
-
pretty: pretty.toLowerCase() ===
|
|
2141
|
-
silent: false
|
|
2209
|
+
delimiter: delimiter || ';',
|
|
2210
|
+
pretty: pretty.toLowerCase() === 'y',
|
|
2211
|
+
silent: false
|
|
2142
2212
|
});
|
|
2143
2213
|
|
|
2144
|
-
console.log(color(
|
|
2145
|
-
rl.question(
|
|
2214
|
+
console.log(color('\n✓ Conversion complete!', 'green'));
|
|
2215
|
+
rl.question('\nPress Enter to continue...', () => {
|
|
2146
2216
|
rl.close();
|
|
2147
2217
|
startBasicTUI();
|
|
2148
2218
|
});
|
|
2149
2219
|
} catch (error) {
|
|
2150
|
-
console.error(color(`✗ Error: ${error.message}`,
|
|
2220
|
+
console.error(color(`✗ Error: ${error.message}`, 'red'));
|
|
2151
2221
|
rl.close();
|
|
2152
2222
|
process.exit(1);
|
|
2153
2223
|
}
|
|
@@ -2174,10 +2244,10 @@ async function main() {
|
|
|
2174
2244
|
|
|
2175
2245
|
// Handle dry run
|
|
2176
2246
|
if (options.dryRun) {
|
|
2177
|
-
console.log(color(
|
|
2247
|
+
console.log(color('DRY RUN - No files will be modified', 'yellow'));
|
|
2178
2248
|
console.log(`Command: ${command}`);
|
|
2179
|
-
console.log(`Files: ${files.join(
|
|
2180
|
-
console.log(
|
|
2249
|
+
console.log(`Files: ${files.join(', ')}`);
|
|
2250
|
+
console.log('Options:', options);
|
|
2181
2251
|
return;
|
|
2182
2252
|
}
|
|
2183
2253
|
|
|
@@ -2188,244 +2258,244 @@ async function main() {
|
|
|
2188
2258
|
}
|
|
2189
2259
|
|
|
2190
2260
|
switch (command) {
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2261
|
+
// Main conversion commands
|
|
2262
|
+
case 'json-to-csv':
|
|
2263
|
+
case 'json2csv':
|
|
2264
|
+
if (files.length < 2) {
|
|
2265
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2266
|
+
console.log(
|
|
2267
|
+
color('Usage: jtcsv json-to-csv input.json output.csv', 'cyan')
|
|
2268
|
+
);
|
|
2269
|
+
process.exit(1);
|
|
2270
|
+
}
|
|
2271
|
+
await convertJsonToCsv(files[0], files[1], options);
|
|
2272
|
+
break;
|
|
2203
2273
|
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2274
|
+
case 'csv-to-json':
|
|
2275
|
+
case 'csv2json':
|
|
2276
|
+
if (files.length < 2) {
|
|
2277
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2278
|
+
console.log(
|
|
2279
|
+
color('Usage: jtcsv csv-to-json input.csv output.json', 'cyan')
|
|
2280
|
+
);
|
|
2281
|
+
process.exit(1);
|
|
2282
|
+
}
|
|
2283
|
+
await convertCsvToJson(files[0], files[1], options);
|
|
2284
|
+
break;
|
|
2215
2285
|
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2286
|
+
case 'save-json':
|
|
2287
|
+
if (files.length < 2) {
|
|
2288
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2289
|
+
console.log(
|
|
2290
|
+
color('Usage: jtcsv save-json input.json output.json', 'cyan')
|
|
2291
|
+
);
|
|
2292
|
+
process.exit(1);
|
|
2293
|
+
}
|
|
2294
|
+
await saveAsJson(files[0], files[1], options);
|
|
2295
|
+
break;
|
|
2226
2296
|
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2297
|
+
case 'save-csv':
|
|
2298
|
+
if (files.length < 2) {
|
|
2299
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2300
|
+
console.log(
|
|
2301
|
+
color('Usage: jtcsv save-csv input.csv output.csv', 'cyan')
|
|
2302
|
+
);
|
|
2303
|
+
process.exit(1);
|
|
2304
|
+
}
|
|
2305
|
+
await saveAsCsv(files[0], files[1], options);
|
|
2306
|
+
break;
|
|
2237
2307
|
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2308
|
+
case 'preprocess':
|
|
2309
|
+
if (files.length < 2) {
|
|
2310
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2311
|
+
console.log(
|
|
2312
|
+
color('Usage: jtcsv preprocess input.json output.json', 'cyan')
|
|
2313
|
+
);
|
|
2314
|
+
process.exit(1);
|
|
2315
|
+
}
|
|
2316
|
+
await preprocessJson(files[0], files[1], options);
|
|
2317
|
+
break;
|
|
2248
2318
|
|
|
2249
2319
|
// NDJSON commands
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2320
|
+
case 'ndjson-to-csv':
|
|
2321
|
+
if (files.length < 2) {
|
|
2322
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2323
|
+
console.log(
|
|
2324
|
+
color('Usage: jtcsv ndjson-to-csv input.ndjson output.csv', 'cyan')
|
|
2325
|
+
);
|
|
2326
|
+
process.exit(1);
|
|
2327
|
+
}
|
|
2328
|
+
await convertNdjsonToCsv(files[0], files[1], options);
|
|
2329
|
+
break;
|
|
2260
2330
|
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2331
|
+
case 'csv-to-ndjson':
|
|
2332
|
+
if (files.length < 2) {
|
|
2333
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2334
|
+
console.log(
|
|
2335
|
+
color('Usage: jtcsv csv-to-ndjson input.csv output.ndjson', 'cyan')
|
|
2336
|
+
);
|
|
2337
|
+
process.exit(1);
|
|
2338
|
+
}
|
|
2339
|
+
await convertCsvToNdjson(files[0], files[1], options);
|
|
2340
|
+
break;
|
|
2271
2341
|
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2342
|
+
case 'ndjson-to-json':
|
|
2343
|
+
if (files.length < 2) {
|
|
2344
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2345
|
+
console.log(
|
|
2346
|
+
color('Usage: jtcsv ndjson-to-json input.ndjson output.json', 'cyan')
|
|
2347
|
+
);
|
|
2348
|
+
process.exit(1);
|
|
2349
|
+
}
|
|
2350
|
+
await convertNdjsonToJson(files[0], files[1], options);
|
|
2351
|
+
break;
|
|
2282
2352
|
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2353
|
+
case 'json-to-ndjson':
|
|
2354
|
+
if (files.length < 2) {
|
|
2355
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2356
|
+
console.log(
|
|
2357
|
+
color('Usage: jtcsv json-to-ndjson input.json output.ndjson', 'cyan')
|
|
2358
|
+
);
|
|
2359
|
+
process.exit(1);
|
|
2360
|
+
}
|
|
2361
|
+
await convertJsonToNdjson(files[0], files[1], options);
|
|
2362
|
+
break;
|
|
2293
2363
|
|
|
2294
2364
|
// Unwrap/Flatten command
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2365
|
+
case 'unwrap':
|
|
2366
|
+
case 'flatten':
|
|
2367
|
+
if (files.length < 2) {
|
|
2368
|
+
console.error(color('Error: Input and output files required', 'red'));
|
|
2369
|
+
console.log(
|
|
2370
|
+
color('Usage: jtcsv unwrap input.json output.json', 'cyan')
|
|
2371
|
+
);
|
|
2372
|
+
process.exit(1);
|
|
2373
|
+
}
|
|
2374
|
+
await unwrapJson(files[0], files[1], options);
|
|
2375
|
+
break;
|
|
2306
2376
|
|
|
2307
2377
|
// Streaming commands
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2378
|
+
case 'stream':
|
|
2379
|
+
if (args.length < 2) {
|
|
2380
|
+
console.error(
|
|
2381
|
+
color('Error: Streaming mode requires subcommand', 'red')
|
|
2382
|
+
);
|
|
2383
|
+
console.log(
|
|
2384
|
+
color(
|
|
2385
|
+
'Usage: jtcsv stream [json-to-csv|csv-to-json|file-to-csv|file-to-json]',
|
|
2386
|
+
'cyan'
|
|
2387
|
+
)
|
|
2388
|
+
);
|
|
2389
|
+
process.exit(1);
|
|
2390
|
+
}
|
|
2321
2391
|
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2392
|
+
const streamCommand = args[1].toLowerCase();
|
|
2393
|
+
// Для stream команд нужно парсить опции начиная с 3-го аргумента (после stream и подкоманды)
|
|
2394
|
+
const streamArgs = args.slice(2);
|
|
2395
|
+
const { options: streamOptions, files: streamFiles } =
|
|
2326
2396
|
parseOptions(streamArgs);
|
|
2327
2397
|
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
if (streamOptions.addBOM) {
|
|
2339
|
-
writeStream.write("\uFEFF");
|
|
2340
|
-
}
|
|
2341
|
-
|
|
2342
|
-
const transformStream = jtcsv.createJsonToCsvStream(streamOptions);
|
|
2343
|
-
await pipeline(readStream, transformStream, writeStream);
|
|
2398
|
+
if (streamCommand === 'json-to-csv' && streamFiles.length >= 2) {
|
|
2399
|
+
await streamJsonToCsv(streamFiles[0], streamFiles[1], streamOptions);
|
|
2400
|
+
} else if (streamCommand === 'csv-to-json' && streamFiles.length >= 2) {
|
|
2401
|
+
await streamCsvToJson(streamFiles[0], streamFiles[1], streamOptions);
|
|
2402
|
+
} else if (streamCommand === 'file-to-csv' && streamFiles.length >= 2) {
|
|
2403
|
+
// Use jtcsv streaming API if available
|
|
2404
|
+
try {
|
|
2405
|
+
const readStream = fs.createReadStream(streamFiles[0], 'utf8');
|
|
2406
|
+
const writeStream = fs.createWriteStream(streamFiles[1], 'utf8');
|
|
2344
2407
|
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
console.error(color(`✗ Streaming error: ${error.message}`, "red"));
|
|
2348
|
-
process.exit(1);
|
|
2408
|
+
if (streamOptions.addBOM) {
|
|
2409
|
+
writeStream.write('\uFEFF');
|
|
2349
2410
|
}
|
|
2350
|
-
} else if (streamCommand === "file-to-json" && streamFiles.length >= 2) {
|
|
2351
|
-
// Use jtcsv streaming API if available
|
|
2352
|
-
try {
|
|
2353
|
-
const readStream = fs.createReadStream(streamFiles[0], "utf8");
|
|
2354
|
-
const writeStream = fs.createWriteStream(streamFiles[1], "utf8");
|
|
2355
2411
|
|
|
2356
|
-
|
|
2357
|
-
|
|
2412
|
+
const transformStream = jtcsv.createJsonToCsvStream(streamOptions);
|
|
2413
|
+
await pipeline(readStream, transformStream, writeStream);
|
|
2358
2414
|
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
process.exit(1);
|
|
2363
|
-
}
|
|
2364
|
-
} else {
|
|
2365
|
-
console.error(
|
|
2366
|
-
color("Error: Invalid streaming command or missing files", "red"),
|
|
2367
|
-
);
|
|
2415
|
+
console.log(color('✓ File streamed successfully', 'green'));
|
|
2416
|
+
} catch (error) {
|
|
2417
|
+
console.error(color(`✗ Streaming error: ${error.message}`, 'red'));
|
|
2368
2418
|
process.exit(1);
|
|
2369
2419
|
}
|
|
2370
|
-
|
|
2420
|
+
} else if (streamCommand === 'file-to-json' && streamFiles.length >= 2) {
|
|
2421
|
+
// Use jtcsv streaming API if available
|
|
2422
|
+
try {
|
|
2423
|
+
const readStream = fs.createReadStream(streamFiles[0], 'utf8');
|
|
2424
|
+
const writeStream = fs.createWriteStream(streamFiles[1], 'utf8');
|
|
2371
2425
|
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
console.
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
);
|
|
2426
|
+
const transformStream = jtcsv.createCsvToJsonStream(streamOptions);
|
|
2427
|
+
await pipeline(readStream, transformStream, writeStream);
|
|
2428
|
+
|
|
2429
|
+
console.log(color('✓ File streamed successfully', 'green'));
|
|
2430
|
+
} catch (error) {
|
|
2431
|
+
console.error(color(`✗ Streaming error: ${error.message}`, 'red'));
|
|
2379
2432
|
process.exit(1);
|
|
2380
2433
|
}
|
|
2434
|
+
} else {
|
|
2435
|
+
console.error(
|
|
2436
|
+
color('Error: Invalid streaming command or missing files', 'red')
|
|
2437
|
+
);
|
|
2438
|
+
process.exit(1);
|
|
2439
|
+
}
|
|
2440
|
+
break;
|
|
2441
|
+
|
|
2442
|
+
// Batch processing commands
|
|
2443
|
+
case 'batch':
|
|
2444
|
+
if (args.length < 2) {
|
|
2445
|
+
console.error(color('Error: Batch mode requires subcommand', 'red'));
|
|
2446
|
+
console.log(
|
|
2447
|
+
color('Usage: jtcsv batch [json-to-csv|csv-to-json|process]', 'cyan')
|
|
2448
|
+
);
|
|
2449
|
+
process.exit(1);
|
|
2450
|
+
}
|
|
2381
2451
|
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2452
|
+
const batchCommand = args[1].toLowerCase();
|
|
2453
|
+
// Для batch команд нужно парсить опции начиная с 3-го аргумента (после batch и подкоманды)
|
|
2454
|
+
const batchArgs = args.slice(2);
|
|
2455
|
+
const { options: batchOptions, files: batchFiles } =
|
|
2386
2456
|
parseOptions(batchArgs);
|
|
2387
2457
|
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2458
|
+
if (batchCommand === 'json-to-csv' && batchFiles.length >= 2) {
|
|
2459
|
+
await batchJsonToCsv(batchFiles[0], batchFiles[1], batchOptions);
|
|
2460
|
+
} else if (batchCommand === 'csv-to-json' && batchFiles.length >= 2) {
|
|
2461
|
+
await batchCsvToJson(batchFiles[0], batchFiles[1], batchOptions);
|
|
2462
|
+
} else if (batchCommand === 'process' && files.length >= 2) {
|
|
2463
|
+
await batchProcessMixed(files[0], files[1], options);
|
|
2464
|
+
} else {
|
|
2465
|
+
console.error(
|
|
2466
|
+
color('Error: Invalid batch command or missing files', 'red')
|
|
2467
|
+
);
|
|
2468
|
+
process.exit(1);
|
|
2469
|
+
}
|
|
2470
|
+
break;
|
|
2401
2471
|
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2472
|
+
// TUI command
|
|
2473
|
+
case 'tui':
|
|
2474
|
+
await launchTUI();
|
|
2475
|
+
break;
|
|
2406
2476
|
|
|
2407
2477
|
// Web UI command
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2478
|
+
case 'web':
|
|
2479
|
+
await launchWebUI(options);
|
|
2480
|
+
break;
|
|
2411
2481
|
|
|
2412
2482
|
// Help and version
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2483
|
+
case 'help':
|
|
2484
|
+
case '--help':
|
|
2485
|
+
case '-h':
|
|
2486
|
+
showHelp();
|
|
2487
|
+
break;
|
|
2418
2488
|
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2489
|
+
case 'version':
|
|
2490
|
+
case '-v':
|
|
2491
|
+
case '--version':
|
|
2492
|
+
showVersion();
|
|
2493
|
+
break;
|
|
2424
2494
|
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2495
|
+
default:
|
|
2496
|
+
console.error(color(`Error: Unknown command '${command}'`, 'red'));
|
|
2497
|
+
console.log(color('Use jtcsv help for available commands', 'cyan'));
|
|
2498
|
+
process.exit(1);
|
|
2429
2499
|
}
|
|
2430
2500
|
}
|
|
2431
2501
|
|
|
@@ -2433,17 +2503,17 @@ async function main() {
|
|
|
2433
2503
|
// ERROR HANDLING
|
|
2434
2504
|
// ============================================================================
|
|
2435
2505
|
|
|
2436
|
-
process.on(
|
|
2437
|
-
console.error(color(`\n✗ Uncaught error: ${error.message}`,
|
|
2506
|
+
process.on('uncaughtException', (error) => {
|
|
2507
|
+
console.error(color(`\n✗ Uncaught error: ${error.message}`, 'red'));
|
|
2438
2508
|
if (process.env.DEBUG) {
|
|
2439
2509
|
console.error(error.stack);
|
|
2440
2510
|
}
|
|
2441
2511
|
process.exit(1);
|
|
2442
2512
|
});
|
|
2443
2513
|
|
|
2444
|
-
process.on(
|
|
2514
|
+
process.on('unhandledRejection', (error) => {
|
|
2445
2515
|
console.error(
|
|
2446
|
-
color(`\n✗ Unhandled promise rejection: ${error.message}`,
|
|
2516
|
+
color(`\n✗ Unhandled promise rejection: ${error.message}`, 'red')
|
|
2447
2517
|
);
|
|
2448
2518
|
if (process.env.DEBUG) {
|
|
2449
2519
|
console.error(error.stack);
|
|
@@ -2454,7 +2524,7 @@ process.on("unhandledRejection", (error) => {
|
|
|
2454
2524
|
// Run main function
|
|
2455
2525
|
if (require.main === module) {
|
|
2456
2526
|
main().catch((error) => {
|
|
2457
|
-
console.error(color(`\n✗ Fatal error: ${error.message}`,
|
|
2527
|
+
console.error(color(`\n✗ Fatal error: ${error.message}`, 'red'));
|
|
2458
2528
|
process.exit(1);
|
|
2459
2529
|
});
|
|
2460
2530
|
}
|