meadow-integration 1.0.19 → 1.0.21

package/source/services/parser/Service-FileParser-CSV.js

@@ -0,0 +1,263 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libFS = require('fs');
+const libReadline = require('readline');
+
+const defaultCSVParserOptions = (
+	{
+		delimiter: ',',
+		quoteChar: '"',
+		hasHeaders: true,
+		skipRows: 0,
+		commentPrefix: '',
+		trim: true,
+		chunkSize: 100
+	});
+
+class MeadowIntegrationFileParserCSV extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultCSVParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParserCSV';
+
+		this._headers = null;
+	}
+
+	/**
+	 * Parse a single CSV line into an array of values.
+	 * Handles quoted fields (including embedded commas and escaped quotes).
+	 *
+	 * @param {string} pLine - Raw CSV line
+	 * @param {string} pDelimiter - Field delimiter character
+	 * @param {string} pQuoteChar - Quote character
+	 * @param {boolean} pTrim - Whether to trim field values
+	 * @returns {Array<string>} Parsed field values
+	 */
+	_parseCSVLine(pLine, pDelimiter, pQuoteChar, pTrim)
+	{
+		let tmpDelimiter = pDelimiter || ',';
+		let tmpQuoteChar = pQuoteChar || '"';
+		let tmpValues = [];
+		let tmpCurrent = '';
+		let tmpInQuotes = false;
+
+		for (let i = 0; i < pLine.length; i++)
+		{
+			let tmpChar = pLine[i];
+
+			if (tmpChar === tmpQuoteChar)
+			{
+				if (tmpInQuotes && pLine[i + 1] === tmpQuoteChar)
+				{
+					// Escaped quote (doubled)
+					tmpCurrent += tmpQuoteChar;
+					i++;
+				}
+				else
+				{
+					tmpInQuotes = !tmpInQuotes;
+				}
+			}
+			else if (tmpChar === tmpDelimiter && !tmpInQuotes)
+			{
+				tmpValues.push(pTrim ? tmpCurrent.trim() : tmpCurrent);
+				tmpCurrent = '';
+			}
+			else
+			{
+				tmpCurrent += tmpChar;
+			}
+		}
+
+		tmpValues.push(pTrim ? tmpCurrent.trim() : tmpCurrent);
+		return tmpValues;
+	}
+
+	/**
+	 * Parse a CSV file using streaming readline.
+	 * Fires chunkCallback with arrays of records as they accumulate.
+	 * Fires completionCallback when the file is fully consumed.
+	 *
+	 * @param {string} pFilePath - Absolute path to the CSV file
+	 * @param {object} pOptions - Parser options (overrides instance options)
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) per chunk
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpChunkSize = tmpOptions.chunkSize || 100;
+		let tmpHasHeaders = tmpOptions.hasHeaders !== false;
+		let tmpSkipRows = parseInt(tmpOptions.skipRows, 10) || 0;
+		let tmpCommentPrefix = tmpOptions.commentPrefix || '';
+		let tmpTrim = tmpOptions.trim !== false;
+		let tmpDelimiter = tmpOptions.delimiter || ',';
+		let tmpQuoteChar = tmpOptions.quoteChar || '"';
+
+		this._headers = null;
+
+		let tmpLineIndex = 0;
+		let tmpRecordCount = 0;
+		let tmpChunkBuffer = [];
+
+		const tmpReadline = libReadline.createInterface(
+			{
+				input: libFS.createReadStream(pFilePath),
+				crlfDelay: Infinity
+			});
+
+		tmpReadline.on('line',
+			(pLine) =>
+			{
+				// Skip comment lines
+				if (tmpCommentPrefix && pLine.startsWith(tmpCommentPrefix))
+				{
+					return;
+				}
+
+				// Skip header/preamble rows
+				if (tmpLineIndex < tmpSkipRows)
+				{
+					tmpLineIndex++;
+					return;
+				}
+
+				let tmpValues = this._parseCSVLine(pLine, tmpDelimiter, tmpQuoteChar, tmpTrim);
+
+				// First non-skipped, non-comment line becomes headers
+				if (tmpHasHeaders && !this._headers)
+				{
+					this._headers = tmpValues;
+					tmpLineIndex++;
+					return;
+				}
+
+				let tmpRecord;
+				if (this._headers)
+				{
+					tmpRecord = {};
+					for (let i = 0; i < this._headers.length; i++)
+					{
+						tmpRecord[this._headers[i]] = (tmpValues && tmpValues[i] !== undefined) ? tmpValues[i] : '';
+					}
+				}
+				else
+				{
+					tmpRecord = tmpValues || [];
+				}
+
+				tmpChunkBuffer.push(tmpRecord);
+				tmpRecordCount++;
+				tmpLineIndex++;
+
+				if (tmpChunkBuffer.length >= tmpChunkSize)
+				{
+					pChunkCallback(null, tmpChunkBuffer.splice(0, tmpChunkBuffer.length));
+				}
+			});
+
+		tmpReadline.on('close',
+			() =>
+			{
+				if (tmpChunkBuffer.length > 0)
+				{
+					pChunkCallback(null, tmpChunkBuffer.splice(0, tmpChunkBuffer.length));
+				}
+				return pCompletionCallback(null, tmpRecordCount);
+			});
+
+		tmpReadline.on('error',
+			(pError) =>
+			{
+				return pCompletionCallback(pError);
+			});
+	}
+
+	/**
+	 * Parse CSV content string into a full array of records.
+	 *
+	 * @param {string} pContent - Raw CSV text
+	 * @param {object} pOptions - Parser options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpHasHeaders = tmpOptions.hasHeaders !== false;
+		let tmpSkipRows = parseInt(tmpOptions.skipRows, 10) || 0;
+		let tmpCommentPrefix = tmpOptions.commentPrefix || '';
+		let tmpTrim = tmpOptions.trim !== false;
+		let tmpDelimiter = tmpOptions.delimiter || ',';
+		let tmpQuoteChar = tmpOptions.quoteChar || '"';
+
+		let tmpLines = pContent.split('\n');
+		let tmpHeaders = null;
+		let tmpRecords = [];
+		let tmpLineIndex = 0;
+
+		for (let i = 0; i < tmpLines.length; i++)
+		{
+			let tmpLine = tmpLines[i];
+
+			// Strip trailing \r for Windows line endings
+			if (tmpLine.length > 0 && tmpLine[tmpLine.length - 1] === '\r')
+			{
+				tmpLine = tmpLine.slice(0, -1);
+			}
+
+			// Skip comment lines
+			if (tmpCommentPrefix && tmpLine.startsWith(tmpCommentPrefix))
+			{
+				continue;
+			}
+
+			// Skip preamble rows
+			if (tmpLineIndex < tmpSkipRows)
+			{
+				tmpLineIndex++;
+				continue;
+			}
+
+			// Skip blank lines
+			if (!tmpLine || tmpLine.trim().length === 0)
+			{
+				tmpLineIndex++;
+				continue;
+			}
+
+			let tmpValues = this._parseCSVLine(tmpLine, tmpDelimiter, tmpQuoteChar, tmpTrim);
+
+			if (tmpHasHeaders && !tmpHeaders)
+			{
+				tmpHeaders = tmpValues;
+				tmpLineIndex++;
+				continue;
+			}
+
+			let tmpRecord;
+			if (tmpHeaders)
+			{
+				tmpRecord = {};
+				for (let j = 0; j < tmpHeaders.length; j++)
+				{
+					tmpRecord[tmpHeaders[j]] = (tmpValues && tmpValues[j] !== undefined) ? tmpValues[j] : '';
+				}
+			}
+			else
+			{
+				tmpRecord = tmpValues || [];
+			}
+
+			tmpRecords.push(tmpRecord);
+			tmpLineIndex++;
+		}
+
+		return fCallback(null, tmpRecords);
+	}
+}
+
+module.exports = MeadowIntegrationFileParserCSV;

package/source/services/parser/Service-FileParser-FixedWidth.js

@@ -0,0 +1,158 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libFS = require('fs');
+const libReadline = require('readline');
+
+const defaultFixedWidthParserOptions = (
+	{
+		skipLines: 0,
+		columns: []
+	});
+
+class MeadowIntegrationFileParserFixedWidth extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultFixedWidthParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParserFixedWidth';
+	}
+
+	/**
+	 * Extract fields from a fixed-width line using a columns definition.
+	 * Column start positions are 1-based.
+	 *
+	 * @param {string} pLine - Raw text line
+	 * @param {Array} pColumns - Array of {name, start, width}
+	 * @returns {object} Extracted record
+	 */
+	_parseLine(pLine, pColumns)
+	{
+		let tmpRecord = {};
+		for (let i = 0; i < pColumns.length; i++)
+		{
+			let tmpCol = pColumns[i];
+			// start is 1-based
+			let tmpStartIdx = (parseInt(tmpCol.start, 10) || 1) - 1;
+			let tmpWidth = parseInt(tmpCol.width, 10) || 0;
+			let tmpValue = pLine.substring(tmpStartIdx, tmpStartIdx + tmpWidth).trim();
+			tmpRecord[tmpCol.name] = tmpValue;
+		}
+		return tmpRecord;
+	}
+
+	/**
+	 * Parse a fixed-width file using streaming readline.
+	 *
+	 * @param {string} pFilePath - Absolute path to the fixed-width file
+	 * @param {object} pOptions - Parser options: skipLines, columns, chunkSize
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) per chunk
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpColumns = tmpOptions.columns || [];
+		let tmpSkipLines = parseInt(tmpOptions.skipLines, 10) || 0;
+		let tmpChunkSize = parseInt(tmpOptions.chunkSize, 10) || 100;
+
+		if (!tmpColumns || tmpColumns.length === 0)
+		{
+			return pCompletionCallback(new Error('FixedWidth parser requires options.columns array of {name, start, width}'));
+		}
+
+		let tmpLineIndex = 0;
+		let tmpRecordCount = 0;
+		let tmpChunkBuffer = [];
+
+		const tmpReadline = libReadline.createInterface(
+			{
+				input: libFS.createReadStream(pFilePath),
+				crlfDelay: Infinity
+			});
+
+		tmpReadline.on('line',
+			(pLine) =>
+			{
+				if (tmpLineIndex < tmpSkipLines)
+				{
+					tmpLineIndex++;
+					return;
+				}
+
+				// Skip blank lines
+				if (!pLine || pLine.trim().length === 0)
+				{
+					tmpLineIndex++;
+					return;
+				}
+
+				let tmpRecord = this._parseLine(pLine, tmpColumns);
+				tmpChunkBuffer.push(tmpRecord);
+				tmpRecordCount++;
+				tmpLineIndex++;
+
+				if (tmpChunkBuffer.length >= tmpChunkSize)
+				{
+					pChunkCallback(null, tmpChunkBuffer.splice(0, tmpChunkBuffer.length));
+				}
+			});
+
+		tmpReadline.on('close',
+			() =>
+			{
+				if (tmpChunkBuffer.length > 0)
+				{
+					pChunkCallback(null, tmpChunkBuffer.splice(0, tmpChunkBuffer.length));
+				}
+				return pCompletionCallback(null, tmpRecordCount);
+			});
+
+		tmpReadline.on('error',
+			(pError) =>
+			{
+				return pCompletionCallback(pError);
+			});
+	}
+
+	/**
+	 * Parse fixed-width content string into a full array of records.
+	 *
+	 * @param {string} pContent - Raw fixed-width text
+	 * @param {object} pOptions - Parser options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpColumns = tmpOptions.columns || [];
+		let tmpSkipLines = parseInt(tmpOptions.skipLines, 10) || 0;
+
+		if (!tmpColumns || tmpColumns.length === 0)
+		{
+			return fCallback(new Error('FixedWidth parser requires options.columns array of {name, start, width}'));
+		}
+
+		let tmpLines = pContent.split('\n');
+		let tmpRecords = [];
+
+		for (let i = tmpSkipLines; i < tmpLines.length; i++)
+		{
+			let tmpLine = tmpLines[i];
+
+			// Skip blank lines
+			if (!tmpLine || tmpLine.trim().length === 0)
+			{
+				continue;
+			}
+
+			tmpRecords.push(this._parseLine(tmpLine, tmpColumns));
+		}
+
+		return fCallback(null, tmpRecords);
+	}
+}
+
+module.exports = MeadowIntegrationFileParserFixedWidth;

package/source/services/parser/Service-FileParser-JSON.js

@@ -0,0 +1,255 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libFS = require('fs');
+
+const defaultJSONParserOptions = (
+	{
+		rootPath: '',
+		flattenNested: false,
+		flattenDelimiter: '_'
+	});
+
+class MeadowIntegrationFileParserJSON extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultJSONParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParserJSON';
+	}
+
+	/**
+	 * Navigate a nested object using a dot-separated path with optional
+	 * array index notation (e.g. "Results.series[0].data").
+	 *
+	 * @param {object} pObject - The object to navigate
+	 * @param {string} pPath - Dot-separated path, segments may include [n]
+	 * @returns {*} The resolved value, or null if the path is invalid
+	 */
+	_resolveDataPath(pObject, pPath)
+	{
+		if (!pPath || typeof pPath !== 'string')
+		{
+			return pObject;
+		}
+
+		let tmpSegments = pPath.split('.');
+		let tmpCurrent = pObject;
+
+		for (let i = 0; i < tmpSegments.length; i++)
+		{
+			if (tmpCurrent === null || tmpCurrent === undefined || typeof tmpCurrent !== 'object')
+			{
+				return null;
+			}
+
+			let tmpSegment = tmpSegments[i];
+			// Check for array index notation: name[index]
+			let tmpMatch = tmpSegment.match(/^([^\[]+)\[(\d+)\]$/);
+			if (tmpMatch)
+			{
+				let tmpKey = tmpMatch[1];
+				let tmpIndex = parseInt(tmpMatch[2], 10);
+				if (!(tmpKey in tmpCurrent) || !Array.isArray(tmpCurrent[tmpKey]))
+				{
+					return null;
+				}
+				tmpCurrent = tmpCurrent[tmpKey][tmpIndex];
+			}
+			else
+			{
+				if (!(tmpSegment in tmpCurrent))
+				{
+					return null;
+				}
+				tmpCurrent = tmpCurrent[tmpSegment];
+			}
+		}
+
+		return tmpCurrent;
+	}
+
+	/**
+	 * Flatten a nested object into a single-level object using a delimiter.
+	 *
+	 * @param {object} pObject - Nested object
+	 * @param {string} pDelimiter - Key delimiter (default '_')
+	 * @param {string} pPrefix - Key prefix for recursion
+	 * @returns {object} Flat object
+	 */
+	_flattenObject(pObject, pDelimiter, pPrefix)
+	{
+		let tmpDelimiter = pDelimiter || '_';
+		let tmpPrefix = pPrefix || '';
+		let tmpResult = {};
+
+		let tmpKeys = Object.keys(pObject);
+		for (let i = 0; i < tmpKeys.length; i++)
+		{
+			let tmpKey = tmpKeys[i];
+			let tmpFullKey = tmpPrefix ? `${tmpPrefix}${tmpDelimiter}${tmpKey}` : tmpKey;
+			let tmpValue = pObject[tmpKey];
+
+			if (tmpValue !== null && typeof tmpValue === 'object' && !Array.isArray(tmpValue))
+			{
+				let tmpNested = this._flattenObject(tmpValue, tmpDelimiter, tmpFullKey);
+				let tmpNestedKeys = Object.keys(tmpNested);
+				for (let j = 0; j < tmpNestedKeys.length; j++)
+				{
+					tmpResult[tmpNestedKeys[j]] = tmpNested[tmpNestedKeys[j]];
+				}
+			}
+			else
+			{
+				tmpResult[tmpFullKey] = tmpValue;
+			}
+		}
+
+		return tmpResult;
+	}
+
+	/**
+	 * Resolve parsed JSON to a records array, applying rootPath navigation.
+	 *
+	 * @param {*} pParsed - Parsed JSON value
+	 * @param {object} pOptions - Parser options
+	 * @returns {Array|null} Array of records or null on failure
+	 */
+	_resolveRecords(pParsed, pOptions)
+	{
+		let tmpData = pParsed;
+
+		if (pOptions && pOptions.rootPath)
+		{
+			tmpData = this._resolveDataPath(pParsed, pOptions.rootPath);
+			if (tmpData === null || tmpData === undefined)
+			{
+				return null;
+			}
+		}
+
+		let tmpRecords;
+		if (Array.isArray(tmpData))
+		{
+			tmpRecords = tmpData;
+		}
+		else if (typeof tmpData === 'object' && tmpData !== null)
+		{
+			// Common envelope keys
+			if (Array.isArray(tmpData.data))
+			{
+				tmpRecords = tmpData.data;
+			}
+			else if (Array.isArray(tmpData.records))
+			{
+				tmpRecords = tmpData.records;
+			}
+			else if (Array.isArray(tmpData.rows))
+			{
+				tmpRecords = tmpData.rows;
+			}
+			else
+			{
+				tmpRecords = [tmpData];
+			}
+		}
+		else
+		{
+			return null;
+		}
+
+		return tmpRecords;
+	}
+
+	/**
+	 * Parse a JSON file into an array of records.
+	 * Reads the entire file into memory.
+	 *
+	 * @param {string} pFilePath - Absolute path to the JSON file
+	 * @param {object} pOptions - Parser options
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) once with all records
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+
+		let tmpContent;
+		try
+		{
+			tmpContent = libFS.readFileSync(pFilePath, 'utf8');
+		}
+		catch (pError)
+		{
+			return pCompletionCallback(new Error(`JSON file read error: ${pError.message}`));
+		}
+
+		this.parseContent(tmpContent, tmpOptions,
+			(pError, pRecords) =>
+			{
+				if (pError)
+				{
+					return pCompletionCallback(pError);
+				}
+				pChunkCallback(null, pRecords);
+				return pCompletionCallback(null, pRecords.length);
+			});
+	}
+
+	/**
+	 * Parse JSON content string into a full array of records.
+	 *
+	 * @param {string} pContent - Raw JSON text
+	 * @param {object} pOptions - Parser options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpFlattenNested = tmpOptions.flattenNested || false;
+		let tmpFlattenDelimiter = tmpOptions.flattenDelimiter || '_';
+
+		let tmpParsed;
+		try
+		{
+			tmpParsed = JSON.parse(pContent);
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`JSON parse error: ${pError.message}`));
+		}
+
+		let tmpRecords = this._resolveRecords(tmpParsed, tmpOptions);
+		if (tmpRecords === null)
+		{
+			if (tmpOptions.rootPath)
+			{
+				return fCallback(new Error(`rootPath '${tmpOptions.rootPath}' not found in JSON content`));
+			}
+			return fCallback(new Error(`Could not resolve records from JSON content`));
+		}
+
+		if (tmpFlattenNested)
+		{
+			let tmpFlattened = [];
+			for (let i = 0; i < tmpRecords.length; i++)
+			{
+				if (tmpRecords[i] !== null && typeof tmpRecords[i] === 'object')
+				{
+					tmpFlattened.push(this._flattenObject(tmpRecords[i], tmpFlattenDelimiter));
+				}
+				else
+				{
+					tmpFlattened.push(tmpRecords[i]);
+				}
+			}
+			return fCallback(null, tmpFlattened);
+		}
+
+		return fCallback(null, tmpRecords);
+	}
+}
+
+module.exports = MeadowIntegrationFileParserJSON;