meadow-integration 1.0.35 → 1.0.37

package/package.json

@@ -1,7 +1,38 @@
 {
     "name": "meadow-integration",
-    "version": "1.0.35",
+    "version": "1.0.37",
     "description": "Meadow Data Integration",
+    "retoldBeacon": {
+        "displayName": "Meadow Integration",
+        "description": "Parse / transform / LabWriter capabilities that seed operations dispatch to via the Ultravisor.",
+        "category": "integration",
+        "mode": "capability-provider",
+        "providerPath": "./source/Meadow-Integration-BeaconProvider.js",
+        "capability": "MeadowIntegration",
+        "healthCheck": {
+            "path": "/"
+        },
+        "defaultPort": 54400,
+        "requiresUltravisor": true,
+        "configForm": {
+            "Fields": []
+        },
+        "docker": {
+            "image": "retold-beacon-host-meadow-integration",
+            "dockerfile": "retold-beacon-host-meadow-integration.Dockerfile",
+            "exposedPort": 54400,
+            "hostPackage": "retold-beacon-host",
+            "dataMountPath": "/app/data",
+            "configMountPath": "/app/data/config.json",
+            "extraMounts": [
+                {
+                    "Source": "seed_datasets",
+                    "Target": "/app/seed_datasets",
+                    "ReadOnly": true
+                }
+            ]
+        }
+    },
     "bin": {
         "mdwint": "source/cli/Meadow-Integration-CLI-Run.js"
     },
@@ -19,7 +50,7 @@
     "devDependencies": {
         "meadow-connection-sqlite": "^1.0.18",
         "pict-docuserve": "^0.1.5",
-        "quackage": "^1.1.1"
+        "quackage": "^1.1.2"
     },
     "mocha": {
         "diff": true,
@@ -40,13 +71,13 @@
         ]
     },
     "dependencies": {
-        "fable": "^3.1.70",
+        "fable": "^3.1.71",
         "fable-serviceproviderbase": "^3.0.19",
         "fast-xml-parser": "^4.4.1",
-        "meadow": "^2.0.35",
+        "meadow": "^2.0.37",
         "meadow-connection-mssql": "^1.0.21",
         "meadow-connection-mysql": "^1.0.17",
-        "orator": "^6.0.4",
+        "orator": "^6.1.0",
         "orator-serviceserver-restify": "^2.0.10",
         "pict-section-flow": "^0.0.17",
         "pict-service-commandlineutility": "^1.0.19",

package/source/Meadow-Integration-BeaconProvider.js

@@ -0,0 +1,293 @@
+/**
+ * Meadow Integration — Ultravisor Beacon Capability Provider
+ *
+ * Wraps meadow-integration's FileParser + TabularTransform services, plus a
+ * small LabWriter action that posts records to a databeacon's dynamic
+ * /1.0/<Entity> endpoint, as an ultravisor-beacon CapabilityProvider.
+ *
+ * Loaded by retold-beacon-host via `--provider meadow-integration/source/Meadow-Integration-BeaconProvider.js`
+ * at container runtime.  Discovery + Dockerfile selection is driven by the
+ * `retoldBeacon` stanza in this module's package.json; ultravisor-lab
+ * scans that stanza the same way it does for every other published beacon.
+ *
+ * Capabilities (two are registered so a single host process can serve both):
+ *   MeadowIntegration.ParseContent     raw content (CSV/JSON/XML) → records
+ *   MeadowIntegration.ParseFile        JSON file on disk → records
+ *   MeadowIntegration.TransformRecords apply a MappingConfiguration
+ *   LabWriter.BulkInsertViaBeacon      POST records to a beacon endpoint
+ *
+ * Constructor signature: `(pProviderConfig, pPict)`.
+ *   pProviderConfig -- user-saved config blob from the lab's form ({} is fine).
+ *   pPict           -- the host's pict instance; meadow-integration's
+ *                      services register through its serviceManager and we
+ *                      grab the instances off the pict afterward.
+ */
+'use strict';
+
+const libFS = require('fs');
+
+const libMeadowIntegrationFileParser = require('./services/parser/Service-FileParser.js');
+const libMeadowIntegrationTabularTransform = require('./services/tabular/Service-TabularTransform.js');
+
+class MeadowIntegrationBeaconProvider
+{
+	constructor(pProviderConfig, pPict)
+	{
+		this._Config = pProviderConfig || {};
+
+		if (!pPict)
+		{
+			throw new Error('meadow-integration beacon provider requires the host pict instance.');
+		}
+
+		pPict.serviceManager.addServiceType('MeadowIntegrationFileParser', libMeadowIntegrationFileParser);
+		pPict.serviceManager.instantiateServiceProvider('MeadowIntegrationFileParser');
+		pPict.serviceManager.addServiceType('TabularTransform', libMeadowIntegrationTabularTransform);
+		pPict.serviceManager.instantiateServiceProvider('TabularTransform');
+
+		this._parser    = pPict.MeadowIntegrationFileParser;
+		this._transform = pPict.TabularTransform;
+	}
+
+	/**
+	 * Called by retold-beacon-host.  Registers both capabilities on the
+	 * given ultravisor-beacon.  Using the explicit register() hook (rather
+	 * than the `Capability` + `actions` pair the base class convention
+	 * expects) so a single provider module can contribute more than one
+	 * capability.
+	 */
+	register(pBeacon)
+	{
+		pBeacon.registerCapability(
+			{
+				Capability: 'LabWriter',
+				Name:       'LabWriterProvider',
+				actions:
+				{
+					BulkInsertViaBeacon: this._buildBulkInsertAction()
+				}
+			});
+
+		pBeacon.registerCapability(
+			{
+				Capability: 'MeadowIntegration',
+				Name:       'MeadowIntegrationProvider',
+				actions:
+				{
+					ParseContent:     this._buildParseContentAction(),
+					ParseFile:        this._buildParseFileAction(),
+					TransformRecords: this._buildTransformRecordsAction()
+				}
+			});
+	}
+
+	// ── LabWriter actions ──────────────────────────────────────────────────
+
+	_buildBulkInsertAction()
+	{
+		return {
+			Description: 'POSTs records to a target beacon\'s /1.0/<Entity> endpoint',
+			SettingsSchema:
+			[
+				{ Name: 'BeaconURL',  DataType: 'String', Required: true },
+				{ Name: 'EntityName', DataType: 'String', Required: true },
+				{ Name: 'Records',    DataType: 'Array',  Required: true }
+			],
+			Handler: (pWorkItem, pContext, fCallback) =>
+			{
+				try
+				{
+					let tmpSettings  = pWorkItem.Settings || {};
+					let tmpBeaconURL = tmpSettings.BeaconURL;
+					let tmpEntity    = tmpSettings.EntityName;
+					let tmpRecords   = tmpSettings.Records;
+					if (!tmpBeaconURL) { return fCallback(new Error('BeaconURL is required.')); }
+					if (!tmpEntity)    { return fCallback(new Error('EntityName is required.')); }
+					// Defensive: catch the "engine left an unresolved {~D:~} string
+					// in Records" failure mode loudly instead of iterating over
+					// the string's characters and POSTing garbage. The execution
+					// engine's template resolver SHOULD substitute Array-typed
+					// settings whose value is a single whole-string reference,
+					// but if it doesn't (older engine, mis-wired graph) this
+					// catches it at the boundary with a useful message.
+					if (tmpRecords === undefined || tmpRecords === null) { tmpRecords = []; }
+					if (!Array.isArray(tmpRecords))
+					{
+						return fCallback(new Error(
+							`BulkInsertViaBeacon: Records must be an array, got ${typeof(tmpRecords)}` +
+							(typeof(tmpRecords) === 'string'
+								? ` (looks like an unresolved template: "${tmpRecords.slice(0, 80)}")`
+								: '')));
+					}
+
+					let tmpURL    = new URL(tmpBeaconURL);
+					let tmpHttp   = tmpURL.protocol === 'https:' ? require('https') : require('http');
+					let tmpBase   = (tmpURL.pathname || '').replace(/\/$/, '');
+
+					let tmpInserted = 0;
+					let tmpFailed   = 0;
+					let tmpIdx      = 0;
+					let tmpLastErr  = '';
+
+					let tmpNext = () =>
+					{
+						if (tmpIdx >= tmpRecords.length)
+						{
+							return fCallback(null,
+								{
+									Outputs:
+									{
+										InsertedCount: tmpInserted,
+										FailedCount:   tmpFailed,
+										TotalCount:    tmpRecords.length,
+										LastError:     tmpLastErr
+									},
+									Log: []
+								});
+						}
+						let tmpRec  = tmpRecords[tmpIdx++];
+						let tmpBody = JSON.stringify(tmpRec);
+						let tmpReq  = tmpHttp.request(
+							{
+								host:    tmpURL.hostname,
+								port:    tmpURL.port,
+								path:    `${tmpBase}/${tmpEntity}`,
+								method:  'POST',
+								headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(tmpBody) },
+								timeout: 15000
+							},
+							(pRes) =>
+							{
+								let tmpChunks = [];
+								pRes.on('data', (c) => tmpChunks.push(c));
+								pRes.on('end', () =>
+									{
+										if (pRes.statusCode >= 400)
+										{
+											tmpFailed++;
+											tmpLastErr = `HTTP ${pRes.statusCode}: ` + Buffer.concat(tmpChunks).toString('utf8').slice(0, 200);
+										}
+										else { tmpInserted++; }
+										setImmediate(tmpNext);
+									});
+							});
+						tmpReq.on('error',   (pErr) => { tmpFailed++; tmpLastErr = pErr.message; setImmediate(tmpNext); });
+						tmpReq.on('timeout', () => { tmpReq.destroy(); tmpFailed++; tmpLastErr = 'timeout'; setImmediate(tmpNext); });
+						tmpReq.write(tmpBody);
+						tmpReq.end();
+					};
+					tmpNext();
+				}
+				catch (pEx) { return fCallback(pEx); }
+			}
+		};
+	}
+
+	// ── MeadowIntegration actions ──────────────────────────────────────────
+
+	_buildParseContentAction()
+	{
+		return {
+			Description: 'Parse raw content (CSV/JSON/XML) into records',
+			SettingsSchema:
+			[
+				{ Name: 'Content', DataType: 'String', Required: true },
+				{ Name: 'Format',  DataType: 'String', Required: false }
+			],
+			Handler: (pWorkItem, pContext, fCallback) =>
+			{
+				try
+				{
+					let tmpSettings = pWorkItem.Settings || {};
+					this._parser.parseContent(tmpSettings.Content || '',
+						{ format: tmpSettings.Format || 'auto' },
+						(pErr, pResult) =>
+						{
+							if (pErr) { return fCallback(pErr); }
+							let tmpRecords = Array.isArray(pResult) ? pResult : (pResult && pResult.Records) || [];
+							return fCallback(null, { Outputs: { Records: tmpRecords, Count: tmpRecords.length }, Log: [] });
+						});
+				}
+				catch (pEx) { return fCallback(pEx); }
+			}
+		};
+	}
+
+	_buildParseFileAction()
+	{
+		return {
+			Description: 'Parse a file from disk into records',
+			SettingsSchema:
+			[
+				{ Name: 'FilePath', DataType: 'String', Required: true },
+				{ Name: 'Format',   DataType: 'String', Required: false }
+			],
+			Handler: (pWorkItem, pContext, fCallback) =>
+			{
+				try
+				{
+					let tmpSettings = pWorkItem.Settings || {};
+					if (!tmpSettings.FilePath) { return fCallback(new Error('FilePath is required.')); }
+
+					// Seed fixtures are small JSON files; simplest reliable
+					// path is to load + parse directly rather than stream.
+					let tmpContent;
+					try { tmpContent = libFS.readFileSync(tmpSettings.FilePath, 'utf8'); }
+					catch (pReadErr) { return fCallback(pReadErr); }
+
+					let tmpRecords;
+					try { tmpRecords = JSON.parse(tmpContent); }
+					catch (pParseErr) { return fCallback(pParseErr); }
+					if (!Array.isArray(tmpRecords)) { tmpRecords = [tmpRecords]; }
+
+					return fCallback(null, { Outputs: { Records: tmpRecords, Count: tmpRecords.length }, Log: [] });
+				}
+				catch (pEx) { return fCallback(pEx); }
+			}
+		};
+	}
+
+	_buildTransformRecordsAction()
+	{
+		return {
+			Description: 'Apply a MappingConfiguration to records via TabularTransform',
+			SettingsSchema:
+			[
+				{ Name: 'Records',              DataType: 'Array',  Required: true },
+				{ Name: 'MappingConfiguration', DataType: 'Object', Required: true },
+				{ Name: 'EntityName',           DataType: 'String', Required: false }
+			],
+			Handler: (pWorkItem, pContext, fCallback) =>
+			{
+				try
+				{
+					let tmpSettings = pWorkItem.Settings || {};
+					let tmpRecords  = tmpSettings.Records || [];
+					let tmpMapping  = tmpSettings.MappingConfiguration || {};
+					let tmpEntity   = tmpSettings.EntityName || (tmpMapping && tmpMapping.Entity) || 'Record';
+					this._transform.transform(tmpRecords, tmpMapping,
+						(pErr, pResult) =>
+						{
+							if (pErr) { return fCallback(pErr); }
+							let tmpOut = (pResult && pResult.Records) ? pResult : { Records: pResult || [] };
+							return fCallback(null,
+								{
+									Outputs:
+									{
+										Records:     tmpOut.Records || [],
+										Count:       (tmpOut.Records || []).length,
+										ParsedCount: tmpRecords.length,
+										BadRecords:  tmpOut.BadRecords || [],
+										Entity:      tmpEntity
+									},
+									Log: []
+								});
+						});
+				}
+				catch (pEx) { return fCallback(pEx); }
+			}
+		};
+	}
+}
+
+module.exports = MeadowIntegrationBeaconProvider;