meadow-integration 1.0.1 → 1.0.4

package/source/restserver/endpoints/Endpoint-ObjectArrayToCSV.js

@@ -0,0 +1,231 @@
+/**
+ * POST /1.0/Comprehension/ToCSV
+ *
+ * Convert a comprehension or array of objects to CSV format.
+ *
+ * Request body (JSON):
+ * {
+ *   "Records": [ {...}, {...}, ... ],          // Array of record objects to convert
+ *   "Entity": "MyEntity"                      // (optional) Entity name if passing a comprehension
+ * }
+ *
+ * OR comprehension-based:
+ * {
+ *   "Comprehension": { ... },                 // Comprehension object
+ *   "Entity": "MyEntity"                      // (optional) Entity; auto-inferred if single
+ * }
+ *
+ * OR file-based:
+ *
+ * POST /1.0/Comprehension/ToCSVFromFile
+ * {
+ *   "File": "/path/to/records.json",          // JSON file (array or comprehension)
+ *   "Entity": "MyEntity"                      // (optional) Entity name
+ * }
+ *
+ * Response: CSV text (Content-Type: text/csv)
+ */
+module.exports = function(pFable, pOrator)
+{
+	// In-memory conversion
+	pOrator.serviceServer.postWithBodyParser('/1.0/Comprehension/ToCSV',
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+
+			let tmpRecordArray = extractRecordArray(pFable, tmpBody);
+			if (tmpRecordArray.Error)
+			{
+				pResponse.send(400, tmpRecordArray);
+				return fNext();
+			}
+
+			let tmpCSV = generateCSV(tmpRecordArray.Records);
+			pResponse.setHeader('Content-Type', 'text/csv');
+			pResponse.setHeader('Content-Disposition', 'attachment; filename="export.csv"');
+			pResponse.sendRaw(200, tmpCSV);
+			return fNext();
+		});
+
+	// File-based conversion
+	pOrator.serviceServer.postWithBodyParser('/1.0/Comprehension/ToCSVFromFile',
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+
+			if (!tmpBody.File || (typeof(tmpBody.File) !== 'string'))
+			{
+				pResponse.send(400, { Error: 'No valid File path provided in request body.' });
+				return fNext();
+			}
+
+			pFable.instantiateServiceProvider('FilePersistence');
+
+			let tmpFilePath = pFable.FilePersistence.resolvePath(tmpBody.File);
+
+			if (!pFable.FilePersistence.existsSync(tmpFilePath))
+			{
+				pResponse.send(404, { Error: `File [${tmpFilePath}] does not exist.` });
+				return fNext();
+			}
+
+			let tmpRawRecordSet;
+			try
+			{
+				tmpRawRecordSet = JSON.parse(pFable.FilePersistence.readFileSync(tmpFilePath));
+			}
+			catch (pError)
+			{
+				pResponse.send(400, { Error: `Error parsing JSON file: ${pError.message}` });
+				return fNext();
+			}
+
+			// Build a body-like object for extraction
+			let tmpExtractBody = { Entity: tmpBody.Entity };
+			if (Array.isArray(tmpRawRecordSet))
+			{
+				tmpExtractBody.Records = tmpRawRecordSet;
+			}
+			else if (typeof(tmpRawRecordSet) === 'object')
+			{
+				tmpExtractBody.Comprehension = tmpRawRecordSet;
+			}
+
+			let tmpRecordArray = extractRecordArray(pFable, tmpExtractBody);
+			if (tmpRecordArray.Error)
+			{
+				pResponse.send(400, tmpRecordArray);
+				return fNext();
+			}
+
+			let tmpCSV = generateCSV(tmpRecordArray.Records);
+			pResponse.setHeader('Content-Type', 'text/csv');
+			pResponse.setHeader('Content-Disposition', 'attachment; filename="export.csv"');
+			pResponse.sendRaw(200, tmpCSV);
+			return fNext();
+		});
+};
+
+function extractRecordArray(pFable, pBody)
+{
+	// Direct array
+	if (pBody.Records && Array.isArray(pBody.Records))
+	{
+		if (pBody.Records.length < 1)
+		{
+			return { Error: 'Records array is empty.' };
+		}
+		return { Records: pBody.Records };
+	}
+
+	// Comprehension
+	if (pBody.Comprehension && (typeof(pBody.Comprehension) === 'object'))
+	{
+		let tmpEntity = pBody.Entity;
+
+		if (tmpEntity)
+		{
+			if (pBody.Comprehension[tmpEntity])
+			{
+				let tmpRecords;
+				if (Array.isArray(pBody.Comprehension[tmpEntity]))
+				{
+					tmpRecords = pBody.Comprehension[tmpEntity];
+				}
+				else
+				{
+					tmpRecords = Object.values(pBody.Comprehension[tmpEntity]);
+				}
+				if (tmpRecords.length < 1)
+				{
+					return { Error: 'No records found for the specified entity.' };
+				}
+				return { Records: tmpRecords };
+			}
+			else
+			{
+				return { Error: `Entity [${tmpEntity}] not found in comprehension.` };
+			}
+		}
+		else
+		{
+			// Auto-detect single entity
+			let tmpKeys = Object.keys(pBody.Comprehension);
+			if (tmpKeys.length === 1 && typeof(pBody.Comprehension[tmpKeys[0]]) === 'object' && !Array.isArray(pBody.Comprehension[tmpKeys[0]]))
+			{
+				pFable.log.info(`Auto-detected entity [${tmpKeys[0]}] from comprehension.`);
+				let tmpRecords = Object.values(pBody.Comprehension[tmpKeys[0]]);
+				if (tmpRecords.length < 1)
+				{
+					return { Error: 'No records found in the auto-detected entity.' };
+				}
+				return { Records: tmpRecords };
+			}
+			else if (tmpKeys.length > 1)
+			{
+				return { Error: `Multiple entities found [${tmpKeys.join(', ')}]. Please specify an Entity.` };
+			}
+			else
+			{
+				return { Error: 'No entities found in the comprehension.' };
+			}
+		}
+	}
+
+	return { Error: 'No valid Records array or Comprehension object provided.' };
+}
+
+function flattenObject(pObject, pAddressPrefix)
+{
+	let tmpPrefix = pAddressPrefix || '';
+	let tmpFlattenedObject = {};
+	for (const [pKey, pValue] of Object.entries(pObject))
+	{
+		const pPropertyPath = tmpPrefix ? `${tmpPrefix}.${pKey}` : pKey;
+		if (pValue && typeof pValue === 'object' && !Array.isArray(pValue))
+		{
+			Object.assign(tmpFlattenedObject, flattenObject(pValue, pPropertyPath));
+		}
+		else
+		{
+			tmpFlattenedObject[pPropertyPath] = pValue;
+		}
+	}
+	return tmpFlattenedObject;
+}
+
+function escapeCSVValue(pValue)
+{
+	if (pValue === null || pValue === undefined) return '';
+	const str = String(pValue);
+	return /[",\n]/.test(str) ? `"${str.replace(/"/g, '""')}"` : str;
+}
+
+function generateCSV(pRecords)
+{
+	const tmpAllKeysSet = new Set();
+	const tmpFlattenedRecords = [];
+
+	// First pass: flatten and collect all keys
+	for (const tmpRecord of pRecords)
+	{
+		const tmpFlattenedObject = flattenObject(tmpRecord);
+		tmpFlattenedRecords.push(tmpFlattenedObject);
+		for (const tmpKey of Object.keys(tmpFlattenedObject))
+		{
+			tmpAllKeysSet.add(tmpKey);
+		}
+	}
+
+	const tmpAllObjectKeys = Array.from(tmpAllKeysSet).sort();
+
+	// Build CSV string
+	let tmpCSV = tmpAllObjectKeys.join(',') + '\n';
+	for (const tmpFlatRecord of tmpFlattenedRecords)
+	{
+		const tmpRow = tmpAllObjectKeys.map((pKey) => escapeCSVValue(tmpFlatRecord[pKey])).join(',');
+		tmpCSV += tmpRow + '\n';
+	}
+
+	return tmpCSV;
+}

package/source/restserver/endpoints/Endpoint-TSVCheck.js

@@ -0,0 +1,93 @@
+const libFS = require('fs');
+const libPath = require('path');
+const libReadline = require('readline');
+
+/**
+ * POST /1.0/TSV/Check
+ *
+ * Analyze a TSV file for statistics.
+ *
+ * Request body (JSON):
+ * {
+ *   "File": "/absolute/path/to/file.tsv",     // The TSV file to analyze
+ *   "Records": false,                          // (optional) Include full record dump
+ *   "QuoteDelimiter": "\""                     // (optional) Quote delimiter character
+ * }
+ *
+ * Response: JSON statistics object
+ */
+module.exports = function(pFable, pOrator)
+{
+	pOrator.serviceServer.postWithBodyParser('/1.0/TSV/Check',
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+
+			if (!tmpBody.File || (typeof(tmpBody.File) !== 'string'))
+			{
+				pResponse.send(400, { Error: 'No valid File path provided in request body.' });
+				return fNext();
+			}
+
+			pFable.instantiateServiceProvider('FilePersistence');
+			pFable.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationTabularCheck', require('../../services/tabular/Service-TabularCheck.js'));
+
+			// Create a fresh CSVParser for each request to reset header state
+			let tmpCSVParser = pFable.instantiateServiceProviderWithoutRegistration('CSVParser');
+
+			// Configure for TSV
+			tmpCSVParser.Delimiter = '\t';
+			if (tmpBody.QuoteDelimiter)
+			{
+				tmpCSVParser.QuoteCharacter = tmpBody.QuoteDelimiter;
+			}
+
+			let tmpInputFilePath = pFable.FilePersistence.resolvePath(tmpBody.File);
+
+			if (!pFable.FilePersistence.existsSync(tmpInputFilePath))
+			{
+				pResponse.send(404, { Error: `File [${tmpInputFilePath}] does not exist.` });
+				return fNext();
+			}
+
+			let tmpStatistics = pFable.MeadowIntegrationTabularCheck.newStatisticsObject(tmpInputFilePath);
+			let tmpStoreFullRecord = (tmpBody.Records === true);
+
+			if (tmpStoreFullRecord)
+			{
+				tmpStatistics.Records = [];
+			}
+
+			const tmpReadline = libReadline.createInterface(
+				{
+					input: libFS.createReadStream(tmpInputFilePath),
+					crlfDelay: Infinity,
+				});
+
+			tmpReadline.on('line',
+				(pLine) =>
+				{
+					const tmpRecord = tmpCSVParser.parseCSVLine(pLine);
+					if (tmpRecord)
+					{
+						pFable.MeadowIntegrationTabularCheck.collectStatistics(tmpRecord, tmpStatistics, tmpStoreFullRecord);
+					}
+				});
+
+			tmpReadline.on('close',
+				() =>
+				{
+					pFable.log.info(`TSV Check: ${tmpStatistics.RowCount} rows, ${tmpStatistics.ColumnCount} columns in [${tmpInputFilePath}].`);
+					pResponse.send(200, tmpStatistics);
+					return fNext();
+				});
+
+			tmpReadline.on('error',
+				(pError) =>
+				{
+					pFable.log.error(`TSV Check error reading file [${tmpInputFilePath}]: ${pError}`, pError);
+					pResponse.send(500, { Error: `Error reading TSV file: ${pError.message}` });
+					return fNext();
+				});
+		});
+};

package/source/restserver/endpoints/Endpoint-TSVTransform.js

@@ -0,0 +1,191 @@
+const libFS = require('fs');
+const libPath = require('path');
+const libReadline = require('readline');
+
+/**
+ * POST /1.0/TSV/Transform
+ *
+ * Transform a TSV file into a comprehension.
+ *
+ * Request body (JSON):
+ * {
+ *   "File": "/absolute/path/to/file.tsv",            // The TSV file to transform
+ *   "Entity": "MyEntity",                             // (optional) Entity name
+ *   "GUIDName": "GUIDMyEntity",                       // (optional) GUID column name
+ *   "GUIDTemplate": "{~D:Record.id~}",                // (optional) Pict template for GUID
+ *   "Mappings": { "Col1": "{~D:Record.col1~}" },      // (optional) Column mappings object
+ *   "MappingConfiguration": { ... },                  // (optional) Full explicit mapping config
+ *   "IncomingComprehension": { ... },                 // (optional) Existing comprehension to merge into
+ *   "Extended": false,                                // (optional) Return full operation state
+ *   "QuoteDelimiter": "\""                            // (optional) Quote delimiter character
+ * }
+ *
+ * Response: Comprehension JSON (or extended state if Extended=true)
+ */
+module.exports = function(pFable, pOrator)
+{
+	pOrator.serviceServer.postWithBodyParser('/1.0/TSV/Transform',
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+
+			if (!tmpBody.File || (typeof(tmpBody.File) !== 'string'))
+			{
+				pResponse.send(400, { Error: 'No valid File path provided in request body.' });
+				return fNext();
+			}
+
+			pFable.instantiateServiceProvider('FilePersistence');
+			pFable.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationTabularTransform', require('../../services/tabular/Service-TabularTransform.js'));
+
+			// Create a fresh CSVParser for each request to reset header state
+			let tmpCSVParser = pFable.instantiateServiceProviderWithoutRegistration('CSVParser');
+
+			// Configure for TSV
+			tmpCSVParser.Delimiter = '\t';
+			if (tmpBody.QuoteDelimiter)
+			{
+				tmpCSVParser.QuoteCharacter = tmpBody.QuoteDelimiter;
+			}
+
+			let tmpInputFilePath = pFable.FilePersistence.resolvePath(tmpBody.File);
+
+			if (!pFable.FilePersistence.existsSync(tmpInputFilePath))
+			{
+				pResponse.send(404, { Error: `File [${tmpInputFilePath}] does not exist.` });
+				return fNext();
+			}
+
+			let tmpMappingOutcome = pFable.MeadowIntegrationTabularTransform.newMappingOutcomeObject();
+
+			// Apply user configuration from request body
+			if (tmpBody.Entity)
+			{
+				tmpMappingOutcome.UserConfiguration.Entity = tmpBody.Entity;
+			}
+			if (tmpBody.GUIDName)
+			{
+				tmpMappingOutcome.UserConfiguration.GUIDName = tmpBody.GUIDName;
+			}
+			if (tmpBody.GUIDTemplate)
+			{
+				tmpMappingOutcome.UserConfiguration.GUIDTemplate = tmpBody.GUIDTemplate;
+			}
+			if (tmpBody.Mappings && (typeof(tmpBody.Mappings) === 'object'))
+			{
+				tmpMappingOutcome.UserConfiguration.Mappings = tmpBody.Mappings;
+			}
+
+			// Apply explicit mapping configuration
+			if (tmpBody.MappingConfiguration && (typeof(tmpBody.MappingConfiguration) === 'object'))
+			{
+				tmpMappingOutcome.ExplicitConfiguration = tmpBody.MappingConfiguration;
+			}
+
+			// Apply incoming comprehension
+			if (tmpBody.IncomingComprehension && (typeof(tmpBody.IncomingComprehension) === 'object'))
+			{
+				tmpMappingOutcome.ExistingComprehension = tmpBody.IncomingComprehension;
+				tmpMappingOutcome.Comprehension = JSON.parse(JSON.stringify(tmpBody.IncomingComprehension));
+			}
+
+			const tmpReadline = libReadline.createInterface(
+				{
+					input: libFS.createReadStream(tmpInputFilePath),
+					crlfDelay: Infinity,
+				});
+
+			tmpReadline.on('line',
+				(pLine) =>
+				{
+					const tmpIncomingRecord = tmpCSVParser.parseCSVLine(pLine);
+					tmpMappingOutcome.ParsedRowCount++;
+
+					if (tmpIncomingRecord)
+					{
+						if (!tmpMappingOutcome.ImplicitConfiguration)
+						{
+							tmpMappingOutcome.ImplicitConfiguration = pFable.MeadowIntegrationTabularTransform.generateMappingConfigurationPrototype(libPath.basename(tmpInputFilePath), tmpIncomingRecord);
+
+							if ((!tmpMappingOutcome.ExplicitConfiguration) || (typeof(tmpMappingOutcome.ExplicitConfiguration) != 'object'))
+							{
+								tmpMappingOutcome.Configuration = Object.assign({}, tmpMappingOutcome.ImplicitConfiguration, tmpMappingOutcome.UserConfiguration);
+							}
+							else
+							{
+								tmpMappingOutcome.Configuration = Object.assign({}, tmpMappingOutcome.ImplicitConfiguration, tmpMappingOutcome.ExplicitConfiguration, tmpMappingOutcome.UserConfiguration);
+							}
+
+							if (!('GUIDName' in tmpMappingOutcome.Configuration))
+							{
+								tmpMappingOutcome.Configuration.GUIDName = `GUID${tmpMappingOutcome.Configuration.Entity}`;
+							}
+
+							if (!(tmpMappingOutcome.Configuration.Entity in tmpMappingOutcome.Comprehension))
+							{
+								tmpMappingOutcome.Comprehension[tmpMappingOutcome.Configuration.Entity] = {};
+							}
+						}
+
+						let tmpMappingRecordSolution = (
+							{
+								IncomingRecord: tmpIncomingRecord,
+								MappingConfiguration: tmpMappingOutcome.Configuration,
+								MappingOutcome: tmpMappingOutcome,
+								RowIndex: tmpMappingOutcome.ParsedRowCount,
+								NewRecordsGUIDUniqueness: [],
+								NewRecordPrototype: {},
+								Fable: pFable,
+								Pict: pFable,
+								AppData: pFable.AppData
+							});
+
+						// Run the solvers for this record
+						let tmpSolverResultsObject = {};
+						if (tmpMappingOutcome.Configuration.Solvers && Array.isArray(tmpMappingOutcome.Configuration.Solvers))
+						{
+							for (let i = 0; i < tmpMappingOutcome.Configuration.Solvers.length; i++)
+							{
+								let tmpSolver = tmpMappingOutcome.Configuration.Solvers[i];
+								pFable.ExpressionParser.solve(tmpSolver, tmpMappingRecordSolution, tmpSolverResultsObject, pFable.manifest, tmpMappingRecordSolution);
+							}
+						}
+
+						if (tmpMappingOutcome.Configuration.MultipleGUIDUniqueness && tmpMappingRecordSolution.NewRecordsGUIDUniqueness.length > 0)
+						{
+							for (let i = 0; i < tmpMappingRecordSolution.NewRecordsGUIDUniqueness.length; i++)
+							{
+								pFable.MeadowIntegrationTabularTransform.addRecordToComprehension(tmpIncomingRecord, tmpMappingOutcome, tmpMappingRecordSolution.NewRecordPrototype, tmpMappingRecordSolution.NewRecordsGUIDUniqueness[i]);
+							}
+						}
+						else if (!tmpMappingOutcome.Configuration.MultipleGUIDUniqueness)
+						{
+							pFable.MeadowIntegrationTabularTransform.addRecordToComprehension(tmpIncomingRecord, tmpMappingOutcome, tmpMappingRecordSolution.NewRecordPrototype);
+						}
+					}
+				});
+
+			tmpReadline.on('close',
+				() =>
+				{
+					pFable.log.info(`TSV Transform: Parsed ${tmpMappingOutcome.ParsedRowCount} rows from [${tmpInputFilePath}].`);
+					if (tmpBody.Extended)
+					{
+						pResponse.send(200, tmpMappingOutcome);
+					}
+					else
+					{
+						pResponse.send(200, tmpMappingOutcome.Comprehension);
+					}
+					return fNext();
+				});
+
+			tmpReadline.on('error',
+				(pError) =>
+				{
+					pFable.log.error(`TSV Transform error reading file [${tmpInputFilePath}]: ${pError}`, pError);
+					pResponse.send(500, { Error: `Error reading TSV file: ${pError.message}` });
+					return fNext();
+				});
+		});
+};