@winkjs/composer 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/emitter.js +44 -14
- package/src/core/emitter-manager/mqtt/index.js +2 -1
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +4 -4
- package/src/core/env-vars.js +44 -3
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +95 -36
- package/src/core/source-manager/mqtt/dedup.js +3 -3
- package/src/core/source-manager/mqtt/index.js +1 -1
- package/src/core/source-manager/mqtt/status.js +67 -14
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/index.js +55 -17
- package/src/core/storage-manager/questdb/persist-plan.js +32 -8
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/callback-guard/index.js +155 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +4 -3
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +4 -3
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +41 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +176 -38
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/emitter-manager/mqtt/mqtt-store.js +0 -538
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
|
@@ -1,207 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Console Formatter for WinkComposer
|
|
3
|
-
*
|
|
4
|
-
* Provides rich, colored console output with structured formatting
|
|
5
|
-
* for development, debugging, and edge device monitoring.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
// ANSI color codes for terminal output
|
|
9
|
-
const colors = {
|
|
10
|
-
reset: '\x1b[0m',
|
|
11
|
-
bright: '\x1b[1m',
|
|
12
|
-
dim: '\x1b[2m',
|
|
13
|
-
|
|
14
|
-
// Foreground colors
|
|
15
|
-
black: '\x1b[30m',
|
|
16
|
-
red: '\x1b[31m',
|
|
17
|
-
green: '\x1b[32m',
|
|
18
|
-
yellow: '\x1b[33m',
|
|
19
|
-
blue: '\x1b[34m',
|
|
20
|
-
magenta: '\x1b[35m',
|
|
21
|
-
cyan: '\x1b[36m',
|
|
22
|
-
white: '\x1b[37m',
|
|
23
|
-
gray: '\x1b[90m',
|
|
24
|
-
|
|
25
|
-
// Background colors
|
|
26
|
-
bgRed: '\x1b[41m',
|
|
27
|
-
bgYellow: '\x1b[43m',
|
|
28
|
-
bgGreen: '\x1b[42m'
|
|
29
|
-
};
|
|
30
|
-
|
|
31
|
-
// Method-specific configurations
|
|
32
|
-
const methodConfig = {
|
|
33
|
-
log: {
|
|
34
|
-
color: colors.reset,
|
|
35
|
-
symbol: ' ',
|
|
36
|
-
label: 'INFO'
|
|
37
|
-
},
|
|
38
|
-
warn: {
|
|
39
|
-
color: colors.yellow,
|
|
40
|
-
symbol: '⚠️ ',
|
|
41
|
-
label: 'WARN'
|
|
42
|
-
},
|
|
43
|
-
error: {
|
|
44
|
-
color: colors.red,
|
|
45
|
-
symbol: '✗ ',
|
|
46
|
-
label: 'ERROR'
|
|
47
|
-
},
|
|
48
|
-
table: {
|
|
49
|
-
color: colors.cyan,
|
|
50
|
-
symbol: '📊',
|
|
51
|
-
label: 'DATA'
|
|
52
|
-
},
|
|
53
|
-
debug: {
|
|
54
|
-
color: colors.gray,
|
|
55
|
-
symbol: '🔍',
|
|
56
|
-
label: 'DEBUG'
|
|
57
|
-
},
|
|
58
|
-
success: {
|
|
59
|
-
color: colors.green,
|
|
60
|
-
symbol: '✓ ',
|
|
61
|
-
label: 'OK'
|
|
62
|
-
},
|
|
63
|
-
critical: {
|
|
64
|
-
color: `${colors.bright}${colors.bgRed}${colors.white}`,
|
|
65
|
-
symbol: '🔴',
|
|
66
|
-
label: 'CRITICAL'
|
|
67
|
-
}
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* Format timestamp for console output
|
|
72
|
-
*/
|
|
73
|
-
const formatTimestamp = function () {
|
|
74
|
-
const now = new Date();
|
|
75
|
-
const time = now.toTimeString().split( ' ' )[ 0 ]; // HH:MM:SS
|
|
76
|
-
const ms = now.getMilliseconds().toString().padStart( 3, '0' );
|
|
77
|
-
return `${colors.gray}[${time}.${ms}]${colors.reset}`;
|
|
78
|
-
}; // formatTimestamp()
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Format node identifier
|
|
82
|
-
*/
|
|
83
|
-
const formatNodeId = function ( partitionKey ) {
|
|
84
|
-
const partition = partitionKey ? `/${partitionKey}` : '';
|
|
85
|
-
return `${colors.cyan}[${partition}]${colors.reset}`;
|
|
86
|
-
}; // formatNodeId()
|
|
87
|
-
|
|
88
|
-
/**
|
|
89
|
-
* Format structured data for console
|
|
90
|
-
*/
|
|
91
|
-
const formatData = function ( data, indent = 2 ) {
|
|
92
|
-
if ( data === null || data === undefined ) {
|
|
93
|
-
return `${colors.gray}(empty)${colors.reset}`;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
if ( typeof data === 'string' || typeof data === 'number' || typeof data === 'boolean' ) {
|
|
97
|
-
return String( data );
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
// For objects, create indented key-value display
|
|
101
|
-
const spaces = ' '.repeat( indent );
|
|
102
|
-
const lines = [];
|
|
103
|
-
|
|
104
|
-
for ( const [ key, value ] of Object.entries( data ) ) {
|
|
105
|
-
const formattedKey = `${colors.bright}${key}:${colors.reset}`;
|
|
106
|
-
|
|
107
|
-
// Handle different value types
|
|
108
|
-
let formattedValue;
|
|
109
|
-
if ( typeof value === 'object' && value !== null ) {
|
|
110
|
-
// Nested object - use JSON stringify with indent
|
|
111
|
-
formattedValue = JSON.stringify( value, null, 2 )
|
|
112
|
-
.split( '\n' )
|
|
113
|
-
.map( ( line, i ) => ( i === 0 ? line : spaces + line ) )
|
|
114
|
-
.join( '\n' );
|
|
115
|
-
} else if ( typeof value === 'string' ) {
|
|
116
|
-
formattedValue = `"${value}"`;
|
|
117
|
-
} else {
|
|
118
|
-
formattedValue = String( value );
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
lines.push( `${spaces}${formattedKey} ${formattedValue}` );
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
return lines.join( '\n' );
|
|
125
|
-
}; // formatData()
|
|
126
|
-
|
|
127
|
-
/**
|
|
128
|
-
* Create formatted console output
|
|
129
|
-
*/
|
|
130
|
-
const createConsoleFormatter = function () {
|
|
131
|
-
|
|
132
|
-
/**
|
|
133
|
-
* Format and emit to console
|
|
134
|
-
*
|
|
135
|
-
* @param {string} method - Console method (log, warn, error, table)
|
|
136
|
-
* @param {Object} data - Data to output
|
|
137
|
-
* @param {string} [partitionKey] - Optional partition identifier
|
|
138
|
-
* @param {Object} [metadata] - Optional metadata (UNS, etc.)
|
|
139
|
-
*/
|
|
140
|
-
const format = function ( method, data, partitionKey, metadata ) {
|
|
141
|
-
const config = methodConfig[ method ] || methodConfig.log;
|
|
142
|
-
|
|
143
|
-
// Build header line
|
|
144
|
-
const timestamp = formatTimestamp();
|
|
145
|
-
const nodeId = formatNodeId( partitionKey );
|
|
146
|
-
const header = `${config.symbol} ${timestamp} ${nodeId}`;
|
|
147
|
-
|
|
148
|
-
// Handle table method specially
|
|
149
|
-
if ( method === 'table' && ( Array.isArray( data ) || typeof data === 'object' ) ) {
|
|
150
|
-
console.log( `${header} ${config.color}[TABLE]${colors.reset}` );
|
|
151
|
-
console.table( data );
|
|
152
|
-
return;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// Handle critical alerts with emphasis
|
|
156
|
-
if ( method === 'critical' ) {
|
|
157
|
-
console.log( `\n${config.color} ${config.label} ${colors.reset}` );
|
|
158
|
-
console.log( header );
|
|
159
|
-
console.log( formatData( data, 4 ) );
|
|
160
|
-
console.log( `${config.color} ${'═'.repeat( 40 )} ${colors.reset}\n` );
|
|
161
|
-
return;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
// Standard formatted output
|
|
165
|
-
const output = [];
|
|
166
|
-
output.push( header );
|
|
167
|
-
|
|
168
|
-
// Add metadata if present (UNS path, etc.)
|
|
169
|
-
if ( metadata?.uns ) {
|
|
170
|
-
output.push( ` ${colors.gray}UNS: ${metadata.uns}${colors.reset}` );
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
// Format main data
|
|
174
|
-
if ( typeof data === 'string' ) {
|
|
175
|
-
output.push( ` ${config.color}${data}${colors.reset}` );
|
|
176
|
-
} else {
|
|
177
|
-
output.push( formatData( data, 2 ) );
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
// Output using appropriate console method
|
|
181
|
-
const consoleMethod = [ 'log', 'warn', 'error', 'table' ].includes( method ) ? method : 'log';
|
|
182
|
-
|
|
183
|
-
console[ consoleMethod ]( output.join( '\n' ) );
|
|
184
|
-
}; // format()
|
|
185
|
-
|
|
186
|
-
/**
|
|
187
|
-
* Create separator line for visual grouping
|
|
188
|
-
*/
|
|
189
|
-
const separator = function ( label, char = '═' ) {
|
|
190
|
-
const line = char.repeat( 40 );
|
|
191
|
-
if ( label ) {
|
|
192
|
-
console.log( `${colors.cyan}${line}${colors.reset}` );
|
|
193
|
-
console.log( `${colors.bright}${colors.cyan} ${label}${colors.reset}` );
|
|
194
|
-
console.log( `${colors.cyan}${line}${colors.reset}` );
|
|
195
|
-
} else {
|
|
196
|
-
console.log( `${colors.gray}${line}${colors.reset}` );
|
|
197
|
-
}
|
|
198
|
-
}; // separator()
|
|
199
|
-
|
|
200
|
-
return {
|
|
201
|
-
format: format,
|
|
202
|
-
separator: separator,
|
|
203
|
-
colors: colors // Export for custom formatting
|
|
204
|
-
};
|
|
205
|
-
}; // createConsoleFormatter()
|
|
206
|
-
|
|
207
|
-
export default createConsoleFormatter;
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
export { default as init } from './init.js';
|
|
2
|
-
export { default as update } from './update.js';
|
|
3
|
-
export { default as reset } from './reset.js';
|
|
4
|
-
// export { default as recompute } from './recompute.js';
|
|
5
|
-
// export { default as publishTo } from './publish-to.js';
|
|
6
|
-
// export * from './introspect.js';
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
// import { getNodeType } from './introspect.js';
|
|
2
|
-
|
|
3
|
-
const init = function ( spec ) {
|
|
4
|
-
const state = Object.create( null );
|
|
5
|
-
|
|
6
|
-
// CSV source configuration
|
|
7
|
-
state.source = spec.source; // URL or file path
|
|
8
|
-
state.mode = spec.mode || 'stream'; // 'stream' or 'batch'
|
|
9
|
-
state.rate = spec.rate || 100; // Messages per second (for stream mode)
|
|
10
|
-
state.loop = spec.loop !== false; // Loop when reaching end
|
|
11
|
-
state.columns = spec.columns || 'auto'; // Column mapping
|
|
12
|
-
|
|
13
|
-
// Parsing options for Papaparse
|
|
14
|
-
state.parseOptions = {
|
|
15
|
-
header: true,
|
|
16
|
-
dynamicTyping: true,
|
|
17
|
-
skipEmptyLines: true,
|
|
18
|
-
...( spec.parseOptions || {} )
|
|
19
|
-
};
|
|
20
|
-
|
|
21
|
-
// Runtime state
|
|
22
|
-
state.data = null; // Parsed CSV data
|
|
23
|
-
state.currentIndex = 0;
|
|
24
|
-
state.lastEmitTime = 0;
|
|
25
|
-
state.isLoaded = false;
|
|
26
|
-
state.isPaused = false;
|
|
27
|
-
|
|
28
|
-
// For browser environment - store fetch promise
|
|
29
|
-
state.loadPromise = null;
|
|
30
|
-
|
|
31
|
-
state.nodeType = 'CSV'; // getNodeType();
|
|
32
|
-
|
|
33
|
-
return state;
|
|
34
|
-
}; // init()
|
|
35
|
-
|
|
36
|
-
export default init;
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
// nodes/csv-source/load.js
|
|
2
|
-
import SimpleCSV from './simple-parser.js';
|
|
3
|
-
|
|
4
|
-
const loadCSV = async function ( state ) {
|
|
5
|
-
try {
|
|
6
|
-
let csvText;
|
|
7
|
-
|
|
8
|
-
// Check if source looks like CSV data (has newlines or commas)
|
|
9
|
-
const isEmbeddedData = state.source.includes('\n') ||
|
|
10
|
-
(state.source.includes(',') && !state.source.endsWith('.csv'));
|
|
11
|
-
|
|
12
|
-
// Determine environment and load CSV text
|
|
13
|
-
if ( typeof window === 'object' ) {
|
|
14
|
-
// Browser environment
|
|
15
|
-
if ( isEmbeddedData ) {
|
|
16
|
-
// Embedded data
|
|
17
|
-
csvText = state.source;
|
|
18
|
-
} else if ( state.source.startsWith( 'http' ) || state.source.startsWith( '/' ) ) {
|
|
19
|
-
// Fetch from URL
|
|
20
|
-
const response = await fetch( state.source );
|
|
21
|
-
csvText = await response.text();
|
|
22
|
-
} else {
|
|
23
|
-
// Assume it's embedded data if not a URL
|
|
24
|
-
csvText = state.source;
|
|
25
|
-
}
|
|
26
|
-
} else if ( typeof process === 'object' && process.versions && process.versions.node ) {
|
|
27
|
-
// Node.js environment
|
|
28
|
-
if ( isEmbeddedData ) {
|
|
29
|
-
// Embedded data
|
|
30
|
-
csvText = state.source;
|
|
31
|
-
} else if ( state.source.startsWith( 'http' ) ) {
|
|
32
|
-
// Fetch from URL
|
|
33
|
-
const response = await fetch( state.source );
|
|
34
|
-
csvText = await response.text();
|
|
35
|
-
} else {
|
|
36
|
-
// Read from file
|
|
37
|
-
const fs = await import( 'fs/promises' );
|
|
38
|
-
csvText = await fs.readFile( state.source, 'utf8' );
|
|
39
|
-
}
|
|
40
|
-
} else {
|
|
41
|
-
throw new Error( 'Unable to determine environment for CSV loading' );
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// Parse CSV using SimpleCSV
|
|
45
|
-
const parsed = SimpleCSV.parse( csvText, state.parseOptions );
|
|
46
|
-
|
|
47
|
-
// Handle any parsing errors
|
|
48
|
-
if ( parsed.errors && parsed.errors.length > 0 ) {
|
|
49
|
-
console.warn( 'CSV parsing warnings:', parsed.errors );
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
// Store parsed data
|
|
53
|
-
state.data = parsed.data;
|
|
54
|
-
state.isLoaded = true;
|
|
55
|
-
state.currentIndex = 0;
|
|
56
|
-
|
|
57
|
-
// Log summary
|
|
58
|
-
console.log( `Loaded ${state.data.length} rows from CSV` );
|
|
59
|
-
if ( parsed.meta && parsed.meta.fields ) {
|
|
60
|
-
console.log( `Columns: ${parsed.meta.fields.join( ', ' )}` );
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
} catch ( error ) {
|
|
64
|
-
console.error( 'Failed to load CSV:', error );
|
|
65
|
-
state.data = [];
|
|
66
|
-
state.isLoaded = false;
|
|
67
|
-
}
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
export default loadCSV;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
// nodes/csv-source/reset.js
|
|
2
|
-
const reset = function ( state ) {
|
|
3
|
-
// Reset to initial state
|
|
4
|
-
state.currentIndex = 0;
|
|
5
|
-
state.lastEmitTime = 0;
|
|
6
|
-
state.isPaused = false;
|
|
7
|
-
|
|
8
|
-
// Don't clear the data - just reset the reading position
|
|
9
|
-
// This allows resuming from the beginning without reloading
|
|
10
|
-
|
|
11
|
-
return true;
|
|
12
|
-
}; // reset()
|
|
13
|
-
|
|
14
|
-
export default reset;
|
|
@@ -1,216 +0,0 @@
|
|
|
1
|
-
// nodes/csv-source/simple-csv-parser.js
|
|
2
|
-
|
|
3
|
-
const SimpleCSV = {
|
|
4
|
-
parse: function ( csvText, config = {} ) {
|
|
5
|
-
// Default configuration
|
|
6
|
-
const options = {
|
|
7
|
-
delimiter: config.delimiter || ',',
|
|
8
|
-
header: config.header !== false,
|
|
9
|
-
dynamicTyping: config.dynamicTyping !== false,
|
|
10
|
-
skipEmptyLines: config.skipEmptyLines !== false,
|
|
11
|
-
transformHeader: config.transformHeader || null,
|
|
12
|
-
transform: config.transform || null,
|
|
13
|
-
comments: config.comments || false,
|
|
14
|
-
quoteChar: config.quoteChar || '"',
|
|
15
|
-
parseDates: config.parseDates || false
|
|
16
|
-
};
|
|
17
|
-
|
|
18
|
-
const result = {
|
|
19
|
-
data: [],
|
|
20
|
-
errors: [],
|
|
21
|
-
meta: {
|
|
22
|
-
delimiter: options.delimiter,
|
|
23
|
-
linebreak: '\n',
|
|
24
|
-
aborted: false,
|
|
25
|
-
fields: []
|
|
26
|
-
}
|
|
27
|
-
};
|
|
28
|
-
|
|
29
|
-
try {
|
|
30
|
-
// Normalize line breaks
|
|
31
|
-
csvText = csvText.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
32
|
-
|
|
33
|
-
// Split into lines (handling quoted newlines)
|
|
34
|
-
const lines = this.splitLines( csvText, options.quoteChar );
|
|
35
|
-
|
|
36
|
-
// Filter empty lines and comments
|
|
37
|
-
const processedLines = lines.filter( (line, index) => {
|
|
38
|
-
if ( options.skipEmptyLines && line.trim() === '' ) return false;
|
|
39
|
-
if ( options.comments && line.trim().startsWith( options.comments ) ) return false;
|
|
40
|
-
return true;
|
|
41
|
-
});
|
|
42
|
-
|
|
43
|
-
if ( processedLines.length === 0 ) {
|
|
44
|
-
return result;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
// Parse header if present
|
|
48
|
-
let headers = [];
|
|
49
|
-
let dataStartIndex = 0;
|
|
50
|
-
|
|
51
|
-
if ( options.header ) {
|
|
52
|
-
headers = this.parseLine( processedLines[0], options );
|
|
53
|
-
|
|
54
|
-
// Transform headers if function provided
|
|
55
|
-
if ( options.transformHeader ) {
|
|
56
|
-
headers = headers.map( (h) => options.transformHeader( h ) );
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
result.meta.fields = headers;
|
|
60
|
-
dataStartIndex = 1;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
// Parse data rows
|
|
64
|
-
for ( let i = dataStartIndex; i < processedLines.length; i += 1 ) {
|
|
65
|
-
const values = this.parseLine( processedLines[i], options );
|
|
66
|
-
|
|
67
|
-
if ( values.length === 0 ) continue;
|
|
68
|
-
|
|
69
|
-
// Create row object or array
|
|
70
|
-
let row;
|
|
71
|
-
if ( options.header && headers.length > 0 ) {
|
|
72
|
-
row = {};
|
|
73
|
-
headers.forEach( (header, index) => {
|
|
74
|
-
let value = values[index];
|
|
75
|
-
|
|
76
|
-
// Apply dynamic typing
|
|
77
|
-
if ( options.dynamicTyping ) {
|
|
78
|
-
value = this.castValue( value, options );
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
// Apply transform if provided
|
|
82
|
-
if ( options.transform ) {
|
|
83
|
-
value = options.transform( value, header );
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
row[header] = value;
|
|
87
|
-
});
|
|
88
|
-
} else {
|
|
89
|
-
row = values.map( (value) => {
|
|
90
|
-
if ( options.dynamicTyping ) {
|
|
91
|
-
return this.castValue( value );
|
|
92
|
-
}
|
|
93
|
-
return value;
|
|
94
|
-
});
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
result.data.push( row );
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
} catch ( error ) {
|
|
101
|
-
result.errors.push({
|
|
102
|
-
type: 'ParseError',
|
|
103
|
-
code: 'ParseError',
|
|
104
|
-
message: error.message,
|
|
105
|
-
row: result.data.length
|
|
106
|
-
});
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
return result;
|
|
110
|
-
},
|
|
111
|
-
|
|
112
|
-
// Split CSV into lines, respecting quoted newlines
|
|
113
|
-
splitLines: function ( text, quoteChar ) {
|
|
114
|
-
const lines = [];
|
|
115
|
-
let currentLine = '';
|
|
116
|
-
let inQuotes = false;
|
|
117
|
-
|
|
118
|
-
for ( let i = 0; i < text.length; i += 1 ) {
|
|
119
|
-
const char = text[i];
|
|
120
|
-
const nextChar = text[i + 1];
|
|
121
|
-
|
|
122
|
-
if ( char === quoteChar ) {
|
|
123
|
-
// Check if it's an escaped quote
|
|
124
|
-
if ( nextChar === quoteChar && inQuotes ) {
|
|
125
|
-
currentLine += quoteChar;
|
|
126
|
-
i += 1; // Skip next quote
|
|
127
|
-
} else {
|
|
128
|
-
inQuotes = !inQuotes;
|
|
129
|
-
currentLine += char;
|
|
130
|
-
}
|
|
131
|
-
} else if ( char === '\n' && !inQuotes ) {
|
|
132
|
-
lines.push( currentLine );
|
|
133
|
-
currentLine = '';
|
|
134
|
-
} else {
|
|
135
|
-
currentLine += char;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
// Don't forget the last line
|
|
140
|
-
if ( currentLine !== '' ) {
|
|
141
|
-
lines.push( currentLine );
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
return lines;
|
|
145
|
-
},
|
|
146
|
-
|
|
147
|
-
// Parse a single line into values
|
|
148
|
-
parseLine: function ( line, options ) {
|
|
149
|
-
const values = [];
|
|
150
|
-
let currentValue = '';
|
|
151
|
-
let inQuotes = false;
|
|
152
|
-
const delimiter = options.delimiter;
|
|
153
|
-
const quoteChar = options.quoteChar;
|
|
154
|
-
|
|
155
|
-
for ( let i = 0; i < line.length; i += 1 ) {
|
|
156
|
-
const char = line[i];
|
|
157
|
-
const nextChar = line[i + 1];
|
|
158
|
-
|
|
159
|
-
if ( char === quoteChar ) {
|
|
160
|
-
if ( inQuotes && nextChar === quoteChar ) {
|
|
161
|
-
// Escaped quote
|
|
162
|
-
currentValue += quoteChar;
|
|
163
|
-
i += 1; // Skip next quote
|
|
164
|
-
} else if ( inQuotes && ( nextChar === delimiter || nextChar === undefined || i === line.length - 1 ) ) {
|
|
165
|
-
// End of quoted value
|
|
166
|
-
inQuotes = false;
|
|
167
|
-
} else if ( !inQuotes && ( currentValue === '' || currentValue === ' '.repeat( currentValue.length ) ) ) {
|
|
168
|
-
// Start of quoted value
|
|
169
|
-
inQuotes = true;
|
|
170
|
-
} else {
|
|
171
|
-
currentValue += char;
|
|
172
|
-
}
|
|
173
|
-
} else if ( char === delimiter && !inQuotes ) {
|
|
174
|
-
// End of value
|
|
175
|
-
values.push( currentValue.trim() );
|
|
176
|
-
currentValue = '';
|
|
177
|
-
} else {
|
|
178
|
-
currentValue += char;
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
// Don't forget the last value
|
|
183
|
-
values.push( currentValue.trim() );
|
|
184
|
-
|
|
185
|
-
return values;
|
|
186
|
-
},
|
|
187
|
-
|
|
188
|
-
// Convert string values to appropriate types
|
|
189
|
-
castValue: function ( value, options ) {
|
|
190
|
-
if ( value === '' || value === null || value === undefined ) {
|
|
191
|
-
return value;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
// Boolean
|
|
195
|
-
if ( value.toLowerCase() === 'true' ) return true;
|
|
196
|
-
if ( value.toLowerCase() === 'false' ) return false;
|
|
197
|
-
|
|
198
|
-
// Number
|
|
199
|
-
if ( !isNaN( value ) && !isNaN( parseFloat( value ) ) ) {
|
|
200
|
-
return value.indexOf( '.' ) > -1 ? parseFloat( value ) : parseInt( value, 10 );
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
// Date (only if parseDates is true)
|
|
204
|
-
if ( options.parseDates && (/^\d{4}-\d{2}-\d{2}/).test( value ) ) {
|
|
205
|
-
const date = new Date( value );
|
|
206
|
-
if ( !isNaN( date.getTime() ) ) {
|
|
207
|
-
return date;
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
// String (default)
|
|
212
|
-
return value;
|
|
213
|
-
}
|
|
214
|
-
};
|
|
215
|
-
|
|
216
|
-
export default SimpleCSV;
|