meadow-integration 1.0.18 → 1.0.20

package/source/services/parser/Service-FileParser-FixedWidth.js

@@ -0,0 +1,158 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libFS = require('fs');
+const libReadline = require('readline');
+
+const defaultFixedWidthParserOptions = (
+	{
+		skipLines: 0,
+		columns: []
+	});
+
+class MeadowIntegrationFileParserFixedWidth extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultFixedWidthParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParserFixedWidth';
+	}
+
+	/**
+	 * Extract fields from a fixed-width line using a columns definition.
+	 * Column start positions are 1-based.
+	 *
+	 * @param {string} pLine - Raw text line
+	 * @param {Array} pColumns - Array of {name, start, width}
+	 * @returns {object} Extracted record
+	 */
+	_parseLine(pLine, pColumns)
+	{
+		let tmpRecord = {};
+		for (let i = 0; i < pColumns.length; i++)
+		{
+			let tmpCol = pColumns[i];
+			// start is 1-based
+			let tmpStartIdx = (parseInt(tmpCol.start, 10) || 1) - 1;
+			let tmpWidth = parseInt(tmpCol.width, 10) || 0;
+			let tmpValue = pLine.substring(tmpStartIdx, tmpStartIdx + tmpWidth).trim();
+			tmpRecord[tmpCol.name] = tmpValue;
+		}
+		return tmpRecord;
+	}
+
+	/**
+	 * Parse a fixed-width file using streaming readline.
+	 *
+	 * @param {string} pFilePath - Absolute path to the fixed-width file
+	 * @param {object} pOptions - Parser options: skipLines, columns, chunkSize
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) per chunk
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpColumns = tmpOptions.columns || [];
+		let tmpSkipLines = parseInt(tmpOptions.skipLines, 10) || 0;
+		let tmpChunkSize = parseInt(tmpOptions.chunkSize, 10) || 100;
+
+		if (!tmpColumns || tmpColumns.length === 0)
+		{
+			return pCompletionCallback(new Error('FixedWidth parser requires options.columns array of {name, start, width}'));
+		}
+
+		let tmpLineIndex = 0;
+		let tmpRecordCount = 0;
+		let tmpChunkBuffer = [];
+
+		const tmpReadline = libReadline.createInterface(
+			{
+				input: libFS.createReadStream(pFilePath),
+				crlfDelay: Infinity
+			});
+
+		tmpReadline.on('line',
+			(pLine) =>
+			{
+				if (tmpLineIndex < tmpSkipLines)
+				{
+					tmpLineIndex++;
+					return;
+				}
+
+				// Skip blank lines
+				if (!pLine || pLine.trim().length === 0)
+				{
+					tmpLineIndex++;
+					return;
+				}
+
+				let tmpRecord = this._parseLine(pLine, tmpColumns);
+				tmpChunkBuffer.push(tmpRecord);
+				tmpRecordCount++;
+				tmpLineIndex++;
+
+				if (tmpChunkBuffer.length >= tmpChunkSize)
+				{
+					pChunkCallback(null, tmpChunkBuffer.splice(0, tmpChunkBuffer.length));
+				}
+			});
+
+		tmpReadline.on('close',
+			() =>
+			{
+				if (tmpChunkBuffer.length > 0)
+				{
+					pChunkCallback(null, tmpChunkBuffer.splice(0, tmpChunkBuffer.length));
+				}
+				return pCompletionCallback(null, tmpRecordCount);
+			});
+
+		tmpReadline.on('error',
+			(pError) =>
+			{
+				return pCompletionCallback(pError);
+			});
+	}
+
+	/**
+	 * Parse fixed-width content string into a full array of records.
+	 *
+	 * @param {string} pContent - Raw fixed-width text
+	 * @param {object} pOptions - Parser options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpColumns = tmpOptions.columns || [];
+		let tmpSkipLines = parseInt(tmpOptions.skipLines, 10) || 0;
+
+		if (!tmpColumns || tmpColumns.length === 0)
+		{
+			return fCallback(new Error('FixedWidth parser requires options.columns array of {name, start, width}'));
+		}
+
+		let tmpLines = pContent.split('\n');
+		let tmpRecords = [];
+
+		for (let i = tmpSkipLines; i < tmpLines.length; i++)
+		{
+			let tmpLine = tmpLines[i];
+
+			// Skip blank lines
+			if (!tmpLine || tmpLine.trim().length === 0)
+			{
+				continue;
+			}
+
+			tmpRecords.push(this._parseLine(tmpLine, tmpColumns));
+		}
+
+		return fCallback(null, tmpRecords);
+	}
+}
+
+module.exports = MeadowIntegrationFileParserFixedWidth;

package/source/services/parser/Service-FileParser-JSON.js

@@ -0,0 +1,255 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libFS = require('fs');
+
+const defaultJSONParserOptions = (
+	{
+		rootPath: '',
+		flattenNested: false,
+		flattenDelimiter: '_'
+	});
+
+class MeadowIntegrationFileParserJSON extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultJSONParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParserJSON';
+	}
+
+	/**
+	 * Navigate a nested object using a dot-separated path with optional
+	 * array index notation (e.g. "Results.series[0].data").
+	 *
+	 * @param {object} pObject - The object to navigate
+	 * @param {string} pPath - Dot-separated path, segments may include [n]
+	 * @returns {*} The resolved value, or null if the path is invalid
+	 */
+	_resolveDataPath(pObject, pPath)
+	{
+		if (!pPath || typeof pPath !== 'string')
+		{
+			return pObject;
+		}
+
+		let tmpSegments = pPath.split('.');
+		let tmpCurrent = pObject;
+
+		for (let i = 0; i < tmpSegments.length; i++)
+		{
+			if (tmpCurrent === null || tmpCurrent === undefined || typeof tmpCurrent !== 'object')
+			{
+				return null;
+			}
+
+			let tmpSegment = tmpSegments[i];
+			// Check for array index notation: name[index]
+			let tmpMatch = tmpSegment.match(/^([^\[]+)\[(\d+)\]$/);
+			if (tmpMatch)
+			{
+				let tmpKey = tmpMatch[1];
+				let tmpIndex = parseInt(tmpMatch[2], 10);
+				if (!(tmpKey in tmpCurrent) || !Array.isArray(tmpCurrent[tmpKey]))
+				{
+					return null;
+				}
+				tmpCurrent = tmpCurrent[tmpKey][tmpIndex];
+			}
+			else
+			{
+				if (!(tmpSegment in tmpCurrent))
+				{
+					return null;
+				}
+				tmpCurrent = tmpCurrent[tmpSegment];
+			}
+		}
+
+		return tmpCurrent;
+	}
+
+	/**
+	 * Flatten a nested object into a single-level object using a delimiter.
+	 *
+	 * @param {object} pObject - Nested object
+	 * @param {string} pDelimiter - Key delimiter (default '_')
+	 * @param {string} pPrefix - Key prefix for recursion
+	 * @returns {object} Flat object
+	 */
+	_flattenObject(pObject, pDelimiter, pPrefix)
+	{
+		let tmpDelimiter = pDelimiter || '_';
+		let tmpPrefix = pPrefix || '';
+		let tmpResult = {};
+
+		let tmpKeys = Object.keys(pObject);
+		for (let i = 0; i < tmpKeys.length; i++)
+		{
+			let tmpKey = tmpKeys[i];
+			let tmpFullKey = tmpPrefix ? `${tmpPrefix}${tmpDelimiter}${tmpKey}` : tmpKey;
+			let tmpValue = pObject[tmpKey];
+
+			if (tmpValue !== null && typeof tmpValue === 'object' && !Array.isArray(tmpValue))
+			{
+				let tmpNested = this._flattenObject(tmpValue, tmpDelimiter, tmpFullKey);
+				let tmpNestedKeys = Object.keys(tmpNested);
+				for (let j = 0; j < tmpNestedKeys.length; j++)
+				{
+					tmpResult[tmpNestedKeys[j]] = tmpNested[tmpNestedKeys[j]];
+				}
+			}
+			else
+			{
+				tmpResult[tmpFullKey] = tmpValue;
+			}
+		}
+
+		return tmpResult;
+	}
+
+	/**
+	 * Resolve parsed JSON to a records array, applying rootPath navigation.
+	 *
+	 * @param {*} pParsed - Parsed JSON value
+	 * @param {object} pOptions - Parser options
+	 * @returns {Array|null} Array of records or null on failure
+	 */
+	_resolveRecords(pParsed, pOptions)
+	{
+		let tmpData = pParsed;
+
+		if (pOptions && pOptions.rootPath)
+		{
+			tmpData = this._resolveDataPath(pParsed, pOptions.rootPath);
+			if (tmpData === null || tmpData === undefined)
+			{
+				return null;
+			}
+		}
+
+		let tmpRecords;
+		if (Array.isArray(tmpData))
+		{
+			tmpRecords = tmpData;
+		}
+		else if (typeof tmpData === 'object' && tmpData !== null)
+		{
+			// Common envelope keys
+			if (Array.isArray(tmpData.data))
+			{
+				tmpRecords = tmpData.data;
+			}
+			else if (Array.isArray(tmpData.records))
+			{
+				tmpRecords = tmpData.records;
+			}
+			else if (Array.isArray(tmpData.rows))
+			{
+				tmpRecords = tmpData.rows;
+			}
+			else
+			{
+				tmpRecords = [tmpData];
+			}
+		}
+		else
+		{
+			return null;
+		}
+
+		return tmpRecords;
+	}
+
+	/**
+	 * Parse a JSON file into an array of records.
+	 * Reads the entire file into memory.
+	 *
+	 * @param {string} pFilePath - Absolute path to the JSON file
+	 * @param {object} pOptions - Parser options
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) once with all records
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+
+		let tmpContent;
+		try
+		{
+			tmpContent = libFS.readFileSync(pFilePath, 'utf8');
+		}
+		catch (pError)
+		{
+			return pCompletionCallback(new Error(`JSON file read error: ${pError.message}`));
+		}
+
+		this.parseContent(tmpContent, tmpOptions,
+			(pError, pRecords) =>
+			{
+				if (pError)
+				{
+					return pCompletionCallback(pError);
+				}
+				pChunkCallback(null, pRecords);
+				return pCompletionCallback(null, pRecords.length);
+			});
+	}
+
+	/**
+	 * Parse JSON content string into a full array of records.
+	 *
+	 * @param {string} pContent - Raw JSON text
+	 * @param {object} pOptions - Parser options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpFlattenNested = tmpOptions.flattenNested || false;
+		let tmpFlattenDelimiter = tmpOptions.flattenDelimiter || '_';
+
+		let tmpParsed;
+		try
+		{
+			tmpParsed = JSON.parse(pContent);
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`JSON parse error: ${pError.message}`));
+		}
+
+		let tmpRecords = this._resolveRecords(tmpParsed, tmpOptions);
+		if (tmpRecords === null)
+		{
+			if (tmpOptions.rootPath)
+			{
+				return fCallback(new Error(`rootPath '${tmpOptions.rootPath}' not found in JSON content`));
+			}
+			return fCallback(new Error(`Could not resolve records from JSON content`));
+		}
+
+		if (tmpFlattenNested)
+		{
+			let tmpFlattened = [];
+			for (let i = 0; i < tmpRecords.length; i++)
+			{
+				if (tmpRecords[i] !== null && typeof tmpRecords[i] === 'object')
+				{
+					tmpFlattened.push(this._flattenObject(tmpRecords[i], tmpFlattenDelimiter));
+				}
+				else
+				{
+					tmpFlattened.push(tmpRecords[i]);
+				}
+			}
+			return fCallback(null, tmpFlattened);
+		}
+
+		return fCallback(null, tmpRecords);
+	}
+}
+
+module.exports = MeadowIntegrationFileParserJSON;

package/source/services/parser/Service-FileParser-XLSX.js

@@ -0,0 +1,194 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libFS = require('fs');
+
+const defaultXLSXParserOptions = (
+	{
+		sheetName: '',
+		sheetIndex: 0,
+		headerRow: 1,
+		dataStartRow: 2,
+		maxFileSizeMB: 50
+	});
+
+class MeadowIntegrationFileParserXLSX extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultXLSXParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParserXLSX';
+	}
+
+	/**
+	 * Parse an XLSX file into an array of records.
+	 * Entire file is read into memory. Enforces maxFileSizeMB guard.
+	 *
+	 * @param {string} pFilePath - Absolute path to the XLSX file
+	 * @param {object} pOptions - Parser options
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) once with all records
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpMaxFileSizeMB = parseFloat(tmpOptions.maxFileSizeMB) || 50;
+		let tmpMaxBytes = tmpMaxFileSizeMB * 1024 * 1024;
+
+		let tmpStat;
+		try
+		{
+			tmpStat = libFS.statSync(pFilePath);
+		}
+		catch (pError)
+		{
+			return pCompletionCallback(new Error(`XLSX file stat error: ${pError.message}`));
+		}
+
+		if (tmpStat.size > tmpMaxBytes)
+		{
+			return pCompletionCallback(new Error(`XLSX file size ${(tmpStat.size / 1024 / 1024).toFixed(1)}MB exceeds maxFileSizeMB limit of ${tmpMaxFileSizeMB}MB`));
+		}
+
+		let tmpBuffer;
+		try
+		{
+			tmpBuffer = libFS.readFileSync(pFilePath);
+		}
+		catch (pError)
+		{
+			return pCompletionCallback(new Error(`XLSX file read error: ${pError.message}`));
+		}
+
+		this._parseBuffer(tmpBuffer, tmpOptions,
+			(pError, pRecords) =>
+			{
+				if (pError)
+				{
+					return pCompletionCallback(pError);
+				}
+				pChunkCallback(null, pRecords);
+				return pCompletionCallback(null, pRecords.length);
+			});
+	}
+
+	/**
+	 * Parse XLSX content (Buffer) into a full array of records.
+	 * Content must be a Buffer containing xlsx file bytes.
+	 *
+	 * @param {Buffer|string} pContent - XLSX file as Buffer (or base64 string)
+	 * @param {object} pOptions - Parser options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpBuffer = Buffer.isBuffer(pContent) ? pContent : Buffer.from(pContent, 'base64');
+		return this._parseBuffer(tmpBuffer, tmpOptions, fCallback);
+	}
+
+	/**
+	 * Internal: parse an xlsx Buffer into records using the xlsx library.
+	 *
+	 * @param {Buffer} pBuffer - XLSX bytes
+	 * @param {object} pOptions - Merged options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	_parseBuffer(pBuffer, pOptions, fCallback)
+	{
+		let tmpXLSX;
+		try
+		{
+			tmpXLSX = require('xlsx');
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`xlsx library not available: ${pError.message}`));
+		}
+
+		let tmpWorkbook;
+		try
+		{
+			tmpWorkbook = tmpXLSX.read(pBuffer, { type: 'buffer' });
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`XLSX parse error: ${pError.message}`));
+		}
+
+		// Determine sheet to use
+		let tmpSheetName;
+		if (pOptions.sheetName && typeof pOptions.sheetName === 'string' && pOptions.sheetName.length > 0)
+		{
+			tmpSheetName = pOptions.sheetName;
+		}
+		else
+		{
+			let tmpSheetIndex = parseInt(pOptions.sheetIndex, 10) || 0;
+			tmpSheetName = tmpWorkbook.SheetNames[tmpSheetIndex];
+		}
+
+		if (!tmpSheetName || !tmpWorkbook.Sheets[tmpSheetName])
+		{
+			return fCallback(new Error(`XLSX sheet '${tmpSheetName}' not found in workbook`));
+		}
+
+		let tmpSheet = tmpWorkbook.Sheets[tmpSheetName];
+		let tmpHeaderRow = parseInt(pOptions.headerRow, 10) || 1;
+		let tmpDataStartRow = parseInt(pOptions.dataStartRow, 10) || 2;
+
+		// When headerRow and dataStartRow are at their defaults (1 and 2),
+		// use xlsx's built-in sheet_to_json which handles this automatically
+		if (tmpHeaderRow === 1 && tmpDataStartRow === 2)
+		{
+			try
+			{
+				let tmpRecords = tmpXLSX.utils.sheet_to_json(tmpSheet);
+				return fCallback(null, tmpRecords);
+			}
+			catch (pError)
+			{
+				return fCallback(new Error(`XLSX sheet_to_json error: ${pError.message}`));
+			}
+		}
+
+		// Custom header/data row offsets: read as raw array first
+		try
+		{
+			let tmpRawRows = tmpXLSX.utils.sheet_to_json(tmpSheet, { header: 1 });
+
+			// Header row is 1-based; convert to 0-based index
+			let tmpHeaderIdx = tmpHeaderRow - 1;
+			let tmpDataIdx = tmpDataStartRow - 1;
+
+			if (tmpHeaderIdx >= tmpRawRows.length)
+			{
+				return fCallback(new Error(`XLSX headerRow ${tmpHeaderRow} is beyond sheet row count`));
+			}
+
+			let tmpHeaders = tmpRawRows[tmpHeaderIdx];
+			let tmpRecords = [];
+
+			for (let i = tmpDataIdx; i < tmpRawRows.length; i++)
+			{
+				let tmpRow = tmpRawRows[i];
+				let tmpRecord = {};
+				for (let j = 0; j < tmpHeaders.length; j++)
+				{
+					tmpRecord[tmpHeaders[j]] = (tmpRow && tmpRow[j] !== undefined) ? tmpRow[j] : '';
+				}
+				tmpRecords.push(tmpRecord);
+			}
+
+			return fCallback(null, tmpRecords);
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`XLSX custom row parse error: ${pError.message}`));
+		}
+	}
+}
+
+module.exports = MeadowIntegrationFileParserXLSX;