meadow-integration 1.0.19 → 1.0.21

package/source/services/parser/Service-FileParser-XLSX.js

@@ -0,0 +1,194 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libFS = require('fs');
+
+const defaultXLSXParserOptions = (
+	{
+		sheetName: '',
+		sheetIndex: 0,
+		headerRow: 1,
+		dataStartRow: 2,
+		maxFileSizeMB: 50
+	});
+
+class MeadowIntegrationFileParserXLSX extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultXLSXParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParserXLSX';
+	}
+
+	/**
+	 * Parse an XLSX file into an array of records.
+	 * Entire file is read into memory. Enforces maxFileSizeMB guard.
+	 *
+	 * @param {string} pFilePath - Absolute path to the XLSX file
+	 * @param {object} pOptions - Parser options
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) once with all records
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpMaxFileSizeMB = parseFloat(tmpOptions.maxFileSizeMB) || 50;
+		let tmpMaxBytes = tmpMaxFileSizeMB * 1024 * 1024;
+
+		let tmpStat;
+		try
+		{
+			tmpStat = libFS.statSync(pFilePath);
+		}
+		catch (pError)
+		{
+			return pCompletionCallback(new Error(`XLSX file stat error: ${pError.message}`));
+		}
+
+		if (tmpStat.size > tmpMaxBytes)
+		{
+			return pCompletionCallback(new Error(`XLSX file size ${(tmpStat.size / 1024 / 1024).toFixed(1)}MB exceeds maxFileSizeMB limit of ${tmpMaxFileSizeMB}MB`));
+		}
+
+		let tmpBuffer;
+		try
+		{
+			tmpBuffer = libFS.readFileSync(pFilePath);
+		}
+		catch (pError)
+		{
+			return pCompletionCallback(new Error(`XLSX file read error: ${pError.message}`));
+		}
+
+		this._parseBuffer(tmpBuffer, tmpOptions,
+			(pError, pRecords) =>
+			{
+				if (pError)
+				{
+					return pCompletionCallback(pError);
+				}
+				pChunkCallback(null, pRecords);
+				return pCompletionCallback(null, pRecords.length);
+			});
+	}
+
+	/**
+	 * Parse XLSX content (Buffer) into a full array of records.
+	 * Content must be a Buffer containing xlsx file bytes.
+	 *
+	 * @param {Buffer|string} pContent - XLSX file as Buffer (or base64 string)
+	 * @param {object} pOptions - Parser options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpBuffer = Buffer.isBuffer(pContent) ? pContent : Buffer.from(pContent, 'base64');
+		return this._parseBuffer(tmpBuffer, tmpOptions, fCallback);
+	}
+
+	/**
+	 * Internal: parse an xlsx Buffer into records using the xlsx library.
+	 *
+	 * @param {Buffer} pBuffer - XLSX bytes
+	 * @param {object} pOptions - Merged options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	_parseBuffer(pBuffer, pOptions, fCallback)
+	{
+		let tmpXLSX;
+		try
+		{
+			tmpXLSX = require('xlsx');
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`xlsx library not available: ${pError.message}`));
+		}
+
+		let tmpWorkbook;
+		try
+		{
+			tmpWorkbook = tmpXLSX.read(pBuffer, { type: 'buffer' });
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`XLSX parse error: ${pError.message}`));
+		}
+
+		// Determine sheet to use
+		let tmpSheetName;
+		if (pOptions.sheetName && typeof pOptions.sheetName === 'string' && pOptions.sheetName.length > 0)
+		{
+			tmpSheetName = pOptions.sheetName;
+		}
+		else
+		{
+			let tmpSheetIndex = parseInt(pOptions.sheetIndex, 10) || 0;
+			tmpSheetName = tmpWorkbook.SheetNames[tmpSheetIndex];
+		}
+
+		if (!tmpSheetName || !tmpWorkbook.Sheets[tmpSheetName])
+		{
+			return fCallback(new Error(`XLSX sheet '${tmpSheetName}' not found in workbook`));
+		}
+
+		let tmpSheet = tmpWorkbook.Sheets[tmpSheetName];
+		let tmpHeaderRow = parseInt(pOptions.headerRow, 10) || 1;
+		let tmpDataStartRow = parseInt(pOptions.dataStartRow, 10) || 2;
+
+		// When headerRow and dataStartRow are at their defaults (1 and 2),
+		// use xlsx's built-in sheet_to_json which handles this automatically
+		if (tmpHeaderRow === 1 && tmpDataStartRow === 2)
+		{
+			try
+			{
+				let tmpRecords = tmpXLSX.utils.sheet_to_json(tmpSheet);
+				return fCallback(null, tmpRecords);
+			}
+			catch (pError)
+			{
+				return fCallback(new Error(`XLSX sheet_to_json error: ${pError.message}`));
+			}
+		}
+
+		// Custom header/data row offsets: read as raw array first
+		try
+		{
+			let tmpRawRows = tmpXLSX.utils.sheet_to_json(tmpSheet, { header: 1 });
+
+			// Header row is 1-based; convert to 0-based index
+			let tmpHeaderIdx = tmpHeaderRow - 1;
+			let tmpDataIdx = tmpDataStartRow - 1;
+
+			if (tmpHeaderIdx >= tmpRawRows.length)
+			{
+				return fCallback(new Error(`XLSX headerRow ${tmpHeaderRow} is beyond sheet row count`));
+			}
+
+			let tmpHeaders = tmpRawRows[tmpHeaderIdx];
+			let tmpRecords = [];
+
+			for (let i = tmpDataIdx; i < tmpRawRows.length; i++)
+			{
+				let tmpRow = tmpRawRows[i];
+				let tmpRecord = {};
+				for (let j = 0; j < tmpHeaders.length; j++)
+				{
+					tmpRecord[tmpHeaders[j]] = (tmpRow && tmpRow[j] !== undefined) ? tmpRow[j] : '';
+				}
+				tmpRecords.push(tmpRecord);
+			}
+
+			return fCallback(null, tmpRecords);
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`XLSX custom row parse error: ${pError.message}`));
+		}
+	}
+}
+
+module.exports = MeadowIntegrationFileParserXLSX;

package/source/services/parser/Service-FileParser-XML.js

@@ -0,0 +1,190 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libFS = require('fs');
+
+const defaultXMLParserOptions = (
+	{
+		recordPath: '',
+		attributePrefix: '@_',
+		ignoreAttributes: false
+	});
+
+class MeadowIntegrationFileParserXML extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultXMLParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParserXML';
+	}
+
+	/**
+	 * Walk an XML-parsed object looking for the first array of object records.
+	 * Recurses one level at a time: checks direct children first, then recurses.
+	 *
+	 * @param {object} pObject - Parsed XML object node
+	 * @returns {Array|null} First array of objects found, or null
+	 */
+	_extractXMLRecords(pObject)
+	{
+		if (!pObject || typeof pObject !== 'object')
+		{
+			return null;
+		}
+
+		let tmpKeys = Object.keys(pObject);
+
+		// First pass: look for array-valued keys whose elements are objects
+		for (let i = 0; i < tmpKeys.length; i++)
+		{
+			let tmpValue = pObject[tmpKeys[i]];
+			if (Array.isArray(tmpValue) && tmpValue.length > 0 && typeof tmpValue[0] === 'object')
+			{
+				return tmpValue;
+			}
+		}
+
+		// Second pass: recurse into object-valued keys
+		for (let i = 0; i < tmpKeys.length; i++)
+		{
+			let tmpValue = pObject[tmpKeys[i]];
+			if (typeof tmpValue === 'object' && !Array.isArray(tmpValue))
+			{
+				let tmpResult = this._extractXMLRecords(tmpValue);
+				if (tmpResult)
+				{
+					return tmpResult;
+				}
+			}
+		}
+
+		return null;
+	}
+
+	/**
+	 * Navigate a parsed XML object using a dot-separated recordPath.
+	 *
+	 * @param {object} pParsed - Parsed XML object
+	 * @param {string} pRecordPath - Dot-separated path to the records array
+	 * @returns {Array|null} Records array or null
+	 */
+	_resolveRecordPath(pParsed, pRecordPath)
+	{
+		let tmpParts = pRecordPath.split('.');
+		let tmpCurrent = pParsed;
+
+		for (let i = 0; i < tmpParts.length; i++)
+		{
+			if (!tmpCurrent || typeof tmpCurrent !== 'object' || !(tmpParts[i] in tmpCurrent))
+			{
+				return null;
+			}
+			tmpCurrent = tmpCurrent[tmpParts[i]];
+		}
+
+		return Array.isArray(tmpCurrent) ? tmpCurrent : [tmpCurrent];
+	}
+
+	/**
+	 * Parse an XML file into an array of records.
+	 * Reads the entire file into memory.
+	 *
+	 * @param {string} pFilePath - Absolute path to the XML file
+	 * @param {object} pOptions - Parser options
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) once with all records
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+
+		let tmpContent;
+		try
+		{
+			tmpContent = libFS.readFileSync(pFilePath, 'utf8');
+		}
+		catch (pError)
+		{
+			return pCompletionCallback(new Error(`XML file read error: ${pError.message}`));
+		}
+
+		this.parseContent(tmpContent, tmpOptions,
+			(pError, pRecords) =>
+			{
+				if (pError)
+				{
+					return pCompletionCallback(pError);
+				}
+				pChunkCallback(null, pRecords);
+				return pCompletionCallback(null, pRecords.length);
+			});
+	}
+
+	/**
+	 * Parse XML content string into a full array of records.
+	 *
+	 * @param {string} pContent - Raw XML text
+	 * @param {object} pOptions - Parser options
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpRecordPath = tmpOptions.recordPath || '';
+		let tmpAttributePrefix = tmpOptions.attributePrefix || '@_';
+		let tmpIgnoreAttributes = tmpOptions.ignoreAttributes === true;
+
+		let tmpXMLParser;
+		try
+		{
+			let libFastXMLParser = require('fast-xml-parser');
+			tmpXMLParser = new libFastXMLParser.XMLParser(
+				{
+					ignoreAttributes: tmpIgnoreAttributes,
+					attributeNamePrefix: tmpAttributePrefix
+				});
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`fast-xml-parser library not available: ${pError.message}`));
+		}
+
+		let tmpParsed;
+		try
+		{
+			tmpParsed = tmpXMLParser.parse(pContent);
+		}
+		catch (pError)
+		{
+			return fCallback(new Error(`XML parse error: ${pError.message}`));
+		}
+
+		let tmpRecords;
+
+		if (tmpRecordPath)
+		{
+			tmpRecords = this._resolveRecordPath(tmpParsed, tmpRecordPath);
+			if (!tmpRecords)
+			{
+				return fCallback(new Error(`recordPath '${tmpRecordPath}' not found in XML`));
+			}
+		}
+		else
+		{
+			// Smart extraction: walk tree looking for first array of objects
+			tmpRecords = this._extractXMLRecords(tmpParsed);
+		}
+
+		if (!tmpRecords)
+		{
+			// Wrap the entire parsed result as a single record
+			tmpRecords = [tmpParsed];
+		}
+
+		return fCallback(null, tmpRecords);
+	}
+}
+
+module.exports = MeadowIntegrationFileParserXML;

package/source/services/parser/Service-FileParser.js

@@ -0,0 +1,142 @@
+'use strict';
+
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libPath = require('path');
+
+const defaultFileParserOptions = (
+	{
+		format: ''
+	});
+
+// Extension to format mapping
+const EXTENSION_FORMAT_MAP = (
+	{
+		'.csv': 'csv',
+		'.tsv': 'csv',
+		'.txt': 'csv',
+		'.json': 'json',
+		'.jsonl': 'json',
+		'.xlsx': 'xlsx',
+		'.xlsm': 'xlsx',
+		'.xls': 'xlsx',
+		'.xml': 'xml',
+		'.fw': 'fixedwidth',
+		'.dat': 'fixedwidth'
+	});
+
+class MeadowIntegrationFileParser extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		let tmpOptions = Object.assign({}, defaultFileParserOptions, pOptions);
+		super(pFable, tmpOptions, pServiceHash);
+
+		this.serviceType = 'MeadowIntegrationFileParser';
+
+		// Register sub-parser service types
+		this.fable.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationFileParserCSV', require('./Service-FileParser-CSV.js'));
+		this.fable.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationFileParserJSON', require('./Service-FileParser-JSON.js'));
+		this.fable.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationFileParserXLSX', require('./Service-FileParser-XLSX.js'));
+		this.fable.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationFileParserXML', require('./Service-FileParser-XML.js'));
+		this.fable.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationFileParserFixedWidth', require('./Service-FileParser-FixedWidth.js'));
+	}
+
+	/**
+	 * Detect the format of a file from its extension, then from a content prefix.
+	 *
+	 * @param {string} pFilePath - File path (used for extension detection)
+	 * @param {string} [pContentPrefix] - First bytes of content for content-based detection
+	 * @returns {string} Format string: 'csv', 'json', 'xlsx', 'xml', 'fixedwidth'
+	 */
+	detectFormat(pFilePath, pContentPrefix)
+	{
+		// Extension-based detection
+		if (pFilePath && typeof pFilePath === 'string')
+		{
+			let tmpExt = libPath.extname(pFilePath).toLowerCase();
+			if (tmpExt && EXTENSION_FORMAT_MAP[tmpExt])
+			{
+				return EXTENSION_FORMAT_MAP[tmpExt];
+			}
+		}
+
+		// Content-based detection
+		if (pContentPrefix && typeof pContentPrefix === 'string')
+		{
+			let tmpTrimmed = pContentPrefix.trim();
+			if (tmpTrimmed.startsWith('[') || tmpTrimmed.startsWith('{'))
+			{
+				return 'json';
+			}
+			if (tmpTrimmed.startsWith('<?xml') || tmpTrimmed.startsWith('<'))
+			{
+				return 'xml';
+			}
+		}
+
+		return 'csv';
+	}
+
+	/**
+	 * Get the appropriate sub-parser service for a given format.
+	 *
+	 * @param {string} pFormat - Format string
+	 * @returns {object} Sub-parser service instance
+	 */
+	_getParser(pFormat)
+	{
+		switch (pFormat)
+		{
+			case 'json':
+				return this.fable.MeadowIntegrationFileParserJSON;
+			case 'xlsx':
+				return this.fable.MeadowIntegrationFileParserXLSX;
+			case 'xml':
+				return this.fable.MeadowIntegrationFileParserXML;
+			case 'fixedwidth':
+				return this.fable.MeadowIntegrationFileParserFixedWidth;
+			case 'csv':
+			default:
+				return this.fable.MeadowIntegrationFileParserCSV;
+		}
+	}
+
+	/**
+	 * Parse a file using streaming, dispatching to the appropriate sub-parser.
+	 * Format is determined from options.format, then from file extension, then content.
+	 *
+	 * @param {string} pFilePath - Absolute path to the file
+	 * @param {object} pOptions - Parser options; pOptions.format overrides detection
+	 * @param {function} pChunkCallback - Called with (pError, pRecords) as records are ready
+	 * @param {function} pCompletionCallback - Called with (pError, pTotalCount) when done
+	 */
+	parseFile(pFilePath, pOptions, pChunkCallback, pCompletionCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpFormat = tmpOptions.format ? tmpOptions.format.toLowerCase() : this.detectFormat(pFilePath);
+		let tmpParser = this._getParser(tmpFormat);
+
+		this.fable.log.info(`FileParser: parsing [${pFilePath}] as format [${tmpFormat}]`);
+		return tmpParser.parseFile(pFilePath, tmpOptions, pChunkCallback, pCompletionCallback);
+	}
+
+	/**
+	 * Parse content using a full-array (non-streaming) interface.
+	 * Format is determined from options.format, then from content prefix detection.
+	 *
+	 * @param {string|Buffer} pContent - Raw file content
+	 * @param {object} pOptions - Parser options; pOptions.format overrides detection
+	 * @param {function} fCallback - Called with (pError, pRecords)
+	 */
+	parseContent(pContent, pOptions, fCallback)
+	{
+		let tmpOptions = Object.assign({}, this.options, pOptions);
+		let tmpContentPrefix = Buffer.isBuffer(pContent) ? '' : (pContent || '').substring(0, 100);
+		let tmpFormat = tmpOptions.format ? tmpOptions.format.toLowerCase() : this.detectFormat('', tmpContentPrefix);
+		let tmpParser = this._getParser(tmpFormat);
+
+		return tmpParser.parseContent(pContent, tmpOptions, fCallback);
+	}
+}
+
+module.exports = MeadowIntegrationFileParser;

package/source/views/MappingEditor-SchemaUtils.js

@@ -0,0 +1,71 @@
+const MICRODDL_TYPE_MAP =
+{
+	'@': { DataType: 'AutoIdentity', Label: 'Auto ID', HasSize: false },
+	'%': { DataType: 'GUID', Label: 'GUID', HasSize: false },
+	'$': { DataType: 'String', Label: 'String', HasSize: true },
+	'*': { DataType: 'Text', Label: 'Text', HasSize: false },
+	'#': { DataType: 'Numeric', Label: 'Numeric', HasSize: false },
+	'.': { DataType: 'Decimal', Label: 'Decimal', HasSize: true },
+	'&': { DataType: 'DateTime', Label: 'Date/Time', HasSize: false },
+	'^': { DataType: 'Boolean', Label: 'Boolean', HasSize: false }
+};
+
+const DATATYPE_TO_SYMBOL = {};
+for (let tmpSymbol in MICRODDL_TYPE_MAP)
+{
+	DATATYPE_TO_SYMBOL[MICRODDL_TYPE_MAP[tmpSymbol].DataType] = tmpSymbol;
+}
+
+function columnsToMicroDDL(pColumns, pTableName)
+{
+	let tmpTableName = (pTableName || 'Untitled').replace(/[^a-zA-Z0-9_]/g, '');
+	let tmpLines = ['!' + tmpTableName];
+
+	for (let i = 0; i < pColumns.length; i++)
+	{
+		let tmpCol = pColumns[i];
+		let tmpSymbol = DATATYPE_TO_SYMBOL[tmpCol.DataType] || '$';
+		let tmpLine = tmpSymbol + (tmpCol.Name || 'Column' + i);
+
+		if (MICRODDL_TYPE_MAP[tmpSymbol].HasSize && tmpCol.Size)
+		{
+			tmpLine += ' ' + tmpCol.Size;
+		}
+
+		tmpLines.push(tmpLine);
+	}
+
+	return tmpLines.join('\n');
+}
+
+function microDDLToColumns(pDDL)
+{
+	let tmpLines = pDDL.split('\n');
+	let tmpColumns = [];
+
+	for (let i = 0; i < tmpLines.length; i++)
+	{
+		let tmpLine = tmpLines[i].trim();
+		if (!tmpLine || tmpLine.startsWith('!') || tmpLine.startsWith('//') || tmpLine.startsWith('--') || tmpLine.startsWith('->'))
+		{
+			continue;
+		}
+
+		let tmpSymbol = tmpLine.charAt(0);
+		if (MICRODDL_TYPE_MAP.hasOwnProperty(tmpSymbol))
+		{
+			let tmpRest = tmpLine.substring(1).trim();
+			let tmpParts = tmpRest.split(/\s+/);
+			tmpColumns.push(
+			{
+				Name: tmpParts[0] || '',
+				DataType: MICRODDL_TYPE_MAP[tmpSymbol].DataType,
+				Size: tmpParts[1] || ''
+			});
+		}
+	}
+
+	return tmpColumns;
+}
+
+module.exports = { MICRODDL_TYPE_MAP, DATATYPE_TO_SYMBOL, columnsToMicroDDL, microDDLToColumns };