@twin.org/engine-types 0.0.1-next.37 → 0.0.1-next.39

package/dist/cjs/index.cjs

@@ -11,7 +11,9 @@ var entityStorageConnectorDynamodb = require('@twin.org/entity-storage-connector
 var entityStorageConnectorFile = require('@twin.org/entity-storage-connector-file');
 var entityStorageConnectorGcpFirestore = require('@twin.org/entity-storage-connector-gcp-firestore');
 var entityStorageConnectorMemory = require('@twin.org/entity-storage-connector-memory');
+var entityStorageConnectorMongodb = require('@twin.org/entity-storage-connector-mongodb');
 var entityStorageConnectorMysql = require('@twin.org/entity-storage-connector-mysql');
+var entityStorageConnectorPostgresql = require('@twin.org/entity-storage-connector-postgresql');
 var entityStorageConnectorScylladb = require('@twin.org/entity-storage-connector-scylladb');
 var entityStorageModels = require('@twin.org/entity-storage-models');
 var entityStorageService = require('@twin.org/entity-storage-service');
@@ -26,6 +28,10 @@ var blobStorageConnectorIpfs = require('@twin.org/blob-storage-connector-ipfs');
 var blobStorageConnectorMemory = require('@twin.org/blob-storage-connector-memory');
 var blobStorageModels = require('@twin.org/blob-storage-models');
 var blobStorageService = require('@twin.org/blob-storage-service');
+var dataProcessingConverters = require('@twin.org/data-processing-converters');
+var dataProcessingExtractors = require('@twin.org/data-processing-extractors');
+var dataProcessingModels = require('@twin.org/data-processing-models');
+var dataProcessingService = require('@twin.org/data-processing-service');
 var eventBusConnectorLocal = require('@twin.org/event-bus-connector-local');
 var eventBusModels = require('@twin.org/event-bus-models');
 var eventBusService = require('@twin.org/event-bus-service');
@@ -214,7 +220,15 @@ const EntityStorageConnectorType = {
     /**
      * MySqlDb.
      */
-    MySqlDb: "mysql"
+    MySqlDb: "mysql",
+    /**
+     * MongoDb.
+     */
+    MongoDb: "mongodb",
+    /**
+     * Postgre SQL.
+     */
+    PostgreSql: "postgresql"
 };
 
 // Copyright 2024 IOTA Stiftung.
@@ -325,6 +339,26 @@ function initialiseEntityStorageConnector(engineCore, context, typeCustom, schem
                 }
             });
         }
+        else if (type === EntityStorageConnectorType.MongoDb) {
+            entityStorageConnector = new entityStorageConnectorMongodb.MongoDbEntityStorageConnector({
+                entitySchema: schema,
+                ...entityStorageConfig.options,
+                config: {
+                    ...entityStorageConfig.options.config,
+                    collection: `${entityStorageConfig.options.tablePrefix ?? ""}${instanceName}`
+                }
+            });
+        }
+        else if (type === EntityStorageConnectorType.PostgreSql) {
+            entityStorageConnector = new entityStorageConnectorPostgresql.PostgreSqlEntityStorageConnector({
+                entitySchema: schema,
+                ...entityStorageConfig.options,
+                config: {
+                    ...entityStorageConfig.options.config,
+                    tableName: `${entityStorageConfig.options.tablePrefix ?? ""}${instanceName}`
+                }
+            });
+        }
         else {
             throw new core.GeneralError("engineCore", "connectorUnknownType", {
                 type,
@@ -712,6 +746,162 @@ function initialiseBlobStorageComponent(engineCore, context, instanceConfig, ove
     return finalInstanceType;
 }
 
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Data converter connector types.
+ */
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const DataConverterConnectorType = {
+    /**
+     * JSON.
+     */
+    Json: "json",
+    /**
+     * Xml.
+     */
+    Xml: "xml"
+};
+
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Data extractor connector types.
+ */
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const DataExtractorConnectorType = {
+    /**
+     * JSON Path.
+     */
+    JsonPath: "json-path"
+};
+
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Data processing component types.
+ */
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const DataProcessingComponentType = {
+    /**
+     * Service.
+     */
+    Service: "service"
+};
+
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Initialise the data converter connector.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the connector type is unknown.
+ */
+function initialiseDataConverterConnector(engineCore, context, instanceConfig, overrideInstanceType) {
+    engineCore.logInfo(core.I18n.formatMessage("engineCore.configuring", {
+        element: `Data Converter Connector: ${instanceConfig.type}`
+    }));
+    const type = instanceConfig.type;
+    let connector;
+    let instanceType;
+    if (type === DataConverterConnectorType.Json) {
+        connector = new dataProcessingConverters.JsonConverterConnector();
+        instanceType = dataProcessingConverters.JsonConverterConnector.NAMESPACE;
+    }
+    else if (type === DataConverterConnectorType.Xml) {
+        connector = new dataProcessingConverters.XmlConverterConnector();
+        instanceType = dataProcessingConverters.XmlConverterConnector.NAMESPACE;
+    }
+    else {
+        throw new core.GeneralError("engineCore", "connectorUnknownType", {
+            type,
+            connectorType: "dataConverterConnector"
+        });
+    }
+    const finalInstanceType = overrideInstanceType ?? instanceType;
+    context.componentInstances.push({
+        instanceType: finalInstanceType,
+        component: connector
+    });
+    dataProcessingModels.DataConverterConnectorFactory.register(finalInstanceType, () => connector);
+    return finalInstanceType;
+}
+/**
+ * Initialise the data extractor connector.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the connector type is unknown.
+ */
+function initialiseDataExtractorConnector(engineCore, context, instanceConfig, overrideInstanceType) {
+    engineCore.logInfo(core.I18n.formatMessage("engineCore.configuring", {
+        element: `Data Extractor Connector: ${instanceConfig.type}`
+    }));
+    const type = instanceConfig.type;
+    let connector;
+    let instanceType;
+    if (type === DataExtractorConnectorType.JsonPath) {
+        connector = new dataProcessingExtractors.JsonPathExtractorConnector();
+        instanceType = dataProcessingExtractors.JsonPathExtractorConnector.NAMESPACE;
+    }
+    else {
+        throw new core.GeneralError("engineCore", "connectorUnknownType", {
+            type,
+            connectorType: "dataExtractorConnector"
+        });
+    }
+    const finalInstanceType = overrideInstanceType ?? instanceType;
+    context.componentInstances.push({
+        instanceType: finalInstanceType,
+        component: connector
+    });
+    dataProcessingModels.DataExtractorConnectorFactory.register(finalInstanceType, () => connector);
+    return finalInstanceType;
+}
+/**
+ * Initialise the data processing component.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the component type is unknown.
+ */
+function initialiseDataProcessingComponent(engineCore, context, instanceConfig, overrideInstanceType) {
+    engineCore.logInfo(core.I18n.formatMessage("engineCore.configuring", {
+        element: `Data Processing Component: ${instanceConfig.type}`
+    }));
+    const type = instanceConfig.type;
+    let component;
+    let instanceType;
+    if (type === DataProcessingComponentType.Service) {
+        dataProcessingService.initSchema();
+        initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.extractionRuleGroupStorageConnectorType, "ExtractionRuleGroup");
+        component = new dataProcessingService.DataProcessingService({
+            ...instanceConfig.options
+        });
+        instanceType = dataProcessingService.DataProcessingService.NAMESPACE;
+    }
+    else {
+        throw new core.GeneralError("engineCore", "componentUnknownType", {
+            type,
+            componentType: "dataProcessingComponent"
+        });
+    }
+    const finalInstanceType = overrideInstanceType ?? instanceType;
+    context.componentInstances.push({
+        instanceType: finalInstanceType,
+        component
+    });
+    core.ComponentFactory.register(finalInstanceType, () => component);
+    return finalInstanceType;
+}
+
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 /**
@@ -2150,6 +2340,9 @@ exports.AuditableItemStreamComponentType = AuditableItemStreamComponentType;
 exports.BackgroundTaskConnectorType = BackgroundTaskConnectorType;
 exports.BlobStorageComponentType = BlobStorageComponentType;
 exports.BlobStorageConnectorType = BlobStorageConnectorType;
+exports.DataConverterConnectorType = DataConverterConnectorType;
+exports.DataExtractorConnectorType = DataExtractorConnectorType;
+exports.DataProcessingComponentType = DataProcessingComponentType;
 exports.DltConfigType = DltConfigType;
 exports.EntityStorageComponentType = EntityStorageComponentType;
 exports.EntityStorageConnectorType = EntityStorageConnectorType;
@@ -2183,6 +2376,9 @@ exports.initialiseAuditableItemStreamComponent = initialiseAuditableItemStreamCo
 exports.initialiseBackgroundTaskConnector = initialiseBackgroundTaskConnector;
 exports.initialiseBlobStorageComponent = initialiseBlobStorageComponent;
 exports.initialiseBlobStorageConnector = initialiseBlobStorageConnector;
+exports.initialiseDataConverterConnector = initialiseDataConverterConnector;
+exports.initialiseDataExtractorConnector = initialiseDataExtractorConnector;
+exports.initialiseDataProcessingComponent = initialiseDataProcessingComponent;
 exports.initialiseEntityStorageComponent = initialiseEntityStorageComponent;
 exports.initialiseEntityStorageConnector = initialiseEntityStorageConnector;
 exports.initialiseEventBusComponent = initialiseEventBusComponent;

package/dist/esm/index.mjs

@@ -9,7 +9,9 @@ import { DynamoDbEntityStorageConnector } from '@twin.org/entity-storage-connect
 import { FileEntityStorageConnector } from '@twin.org/entity-storage-connector-file';
 import { FirestoreEntityStorageConnector } from '@twin.org/entity-storage-connector-gcp-firestore';
 import { MemoryEntityStorageConnector } from '@twin.org/entity-storage-connector-memory';
+import { MongoDbEntityStorageConnector } from '@twin.org/entity-storage-connector-mongodb';
 import { MySqlEntityStorageConnector } from '@twin.org/entity-storage-connector-mysql';
+import { PostgreSqlEntityStorageConnector } from '@twin.org/entity-storage-connector-postgresql';
 import { ScyllaDBTableConnector } from '@twin.org/entity-storage-connector-scylladb';
 import { EntityStorageConnectorFactory } from '@twin.org/entity-storage-models';
 import { EntityStorageService } from '@twin.org/entity-storage-service';
@@ -24,40 +26,44 @@ import { IpfsBlobStorageConnector } from '@twin.org/blob-storage-connector-ipfs'
 import { MemoryBlobStorageConnector } from '@twin.org/blob-storage-connector-memory';
 import { BlobStorageConnectorFactory } from '@twin.org/blob-storage-models';
 import { initSchema as initSchema$3, BlobStorageService } from '@twin.org/blob-storage-service';
+import { JsonConverterConnector, XmlConverterConnector } from '@twin.org/data-processing-converters';
+import { JsonPathExtractorConnector } from '@twin.org/data-processing-extractors';
+import { DataConverterConnectorFactory, DataExtractorConnectorFactory } from '@twin.org/data-processing-models';
+import { initSchema as initSchema$4, DataProcessingService } from '@twin.org/data-processing-service';
 import { LocalEventBusConnector } from '@twin.org/event-bus-connector-local';
 import { EventBusConnectorFactory } from '@twin.org/event-bus-models';
 import { EventBusService } from '@twin.org/event-bus-service';
-import { EntityStorageFaucetConnector, EntityStorageWalletConnector, initSchema as initSchema$d } from '@twin.org/wallet-connector-entity-storage';
+import { EntityStorageFaucetConnector, EntityStorageWalletConnector, initSchema as initSchema$e } from '@twin.org/wallet-connector-entity-storage';
 import { IotaFaucetConnector, IotaWalletConnector } from '@twin.org/wallet-connector-iota';
 import { IotaRebasedFaucetConnector, IotaRebasedWalletConnector } from '@twin.org/wallet-connector-iota-rebased';
 import { FaucetConnectorFactory, WalletConnectorFactory } from '@twin.org/wallet-models';
-import { initSchema as initSchema$4, EntityStorageIdentityConnector, EntityStorageIdentityProfileConnector, EntityStorageIdentityResolverConnector } from '@twin.org/identity-connector-entity-storage';
+import { initSchema as initSchema$5, EntityStorageIdentityConnector, EntityStorageIdentityProfileConnector, EntityStorageIdentityResolverConnector } from '@twin.org/identity-connector-entity-storage';
 import { IotaIdentityConnector, IotaIdentityResolverConnector } from '@twin.org/identity-connector-iota';
 import { IotaRebasedIdentityConnector, IotaRebasedIdentityResolverConnector } from '@twin.org/identity-connector-iota-rebased';
 import { IdentityConnectorFactory, IdentityProfileConnectorFactory, IdentityResolverConnectorFactory } from '@twin.org/identity-models';
 import { IdentityService, IdentityProfileService, IdentityResolverService } from '@twin.org/identity-service';
-import { initSchema as initSchema$5, ImmutableProofService } from '@twin.org/immutable-proof-service';
-import { initSchema as initSchema$6, EntityStorageImmutableStorageConnector } from '@twin.org/immutable-storage-connector-entity-storage';
+import { initSchema as initSchema$6, ImmutableProofService } from '@twin.org/immutable-proof-service';
+import { initSchema as initSchema$7, EntityStorageImmutableStorageConnector } from '@twin.org/immutable-storage-connector-entity-storage';
 import { IotaImmutableStorageConnector } from '@twin.org/immutable-storage-connector-iota';
 import { IotaRebasedImmutableStorageConnector } from '@twin.org/immutable-storage-connector-iota-rebased';
 import { ImmutableStorageConnectorFactory } from '@twin.org/immutable-storage-models';
 import { ConsoleLoggingConnector } from '@twin.org/logging-connector-console';
-import { initSchema as initSchema$7, EntityStorageLoggingConnector } from '@twin.org/logging-connector-entity-storage';
+import { initSchema as initSchema$8, EntityStorageLoggingConnector } from '@twin.org/logging-connector-entity-storage';
 import { MultiLoggingConnector, LoggingConnectorFactory } from '@twin.org/logging-models';
 import { LoggingService } from '@twin.org/logging-service';
 import { AwsMessagingEmailConnector, AwsMessagingSmsConnector, AwsMessagingPushNotificationConnector } from '@twin.org/messaging-connector-aws';
-import { initSchema as initSchema$8, EntityStorageMessagingEmailConnector, EntityStorageMessagingSmsConnector, EntityStorageMessagingPushNotificationConnector } from '@twin.org/messaging-connector-entity-storage';
+import { initSchema as initSchema$9, EntityStorageMessagingEmailConnector, EntityStorageMessagingSmsConnector, EntityStorageMessagingPushNotificationConnector } from '@twin.org/messaging-connector-entity-storage';
 import { MessagingEmailConnectorFactory, MessagingSmsConnectorFactory, MessagingPushNotificationsConnectorFactory } from '@twin.org/messaging-models';
-import { initSchema as initSchema$9, MessagingService } from '@twin.org/messaging-service';
-import { initSchema as initSchema$a, EntityStorageNftConnector } from '@twin.org/nft-connector-entity-storage';
+import { initSchema as initSchema$a, MessagingService } from '@twin.org/messaging-service';
+import { initSchema as initSchema$b, EntityStorageNftConnector } from '@twin.org/nft-connector-entity-storage';
 import { IotaNftConnector } from '@twin.org/nft-connector-iota';
 import { IotaRebasedNftConnector } from '@twin.org/nft-connector-iota-rebased';
 import { NftConnectorFactory } from '@twin.org/nft-models';
 import { NftService } from '@twin.org/nft-service';
-import { initSchema as initSchema$b, EntityStorageTelemetryConnector } from '@twin.org/telemetry-connector-entity-storage';
+import { initSchema as initSchema$c, EntityStorageTelemetryConnector } from '@twin.org/telemetry-connector-entity-storage';
 import { TelemetryConnectorFactory } from '@twin.org/telemetry-models';
 import { TelemetryService } from '@twin.org/telemetry-service';
-import { initSchema as initSchema$c, EntityStorageVaultConnector } from '@twin.org/vault-connector-entity-storage';
+import { initSchema as initSchema$d, EntityStorageVaultConnector } from '@twin.org/vault-connector-entity-storage';
 import { HashicorpVaultConnector } from '@twin.org/vault-connector-hashicorp';
 import { VaultConnectorFactory } from '@twin.org/vault-models';
 
@@ -212,7 +218,15 @@ const EntityStorageConnectorType = {
     /**
      * MySqlDb.
      */
-    MySqlDb: "mysql"
+    MySqlDb: "mysql",
+    /**
+     * MongoDb.
+     */
+    MongoDb: "mongodb",
+    /**
+     * Postgre SQL.
+     */
+    PostgreSql: "postgresql"
 };
 
 // Copyright 2024 IOTA Stiftung.
@@ -323,6 +337,26 @@ function initialiseEntityStorageConnector(engineCore, context, typeCustom, schem
                 }
             });
         }
+        else if (type === EntityStorageConnectorType.MongoDb) {
+            entityStorageConnector = new MongoDbEntityStorageConnector({
+                entitySchema: schema,
+                ...entityStorageConfig.options,
+                config: {
+                    ...entityStorageConfig.options.config,
+                    collection: `${entityStorageConfig.options.tablePrefix ?? ""}${instanceName}`
+                }
+            });
+        }
+        else if (type === EntityStorageConnectorType.PostgreSql) {
+            entityStorageConnector = new PostgreSqlEntityStorageConnector({
+                entitySchema: schema,
+                ...entityStorageConfig.options,
+                config: {
+                    ...entityStorageConfig.options.config,
+                    tableName: `${entityStorageConfig.options.tablePrefix ?? ""}${instanceName}`
+                }
+            });
+        }
         else {
             throw new GeneralError("engineCore", "connectorUnknownType", {
                 type,
@@ -710,6 +744,162 @@ function initialiseBlobStorageComponent(engineCore, context, instanceConfig, ove
     return finalInstanceType;
 }
 
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Data converter connector types.
+ */
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const DataConverterConnectorType = {
+    /**
+     * JSON.
+     */
+    Json: "json",
+    /**
+     * Xml.
+     */
+    Xml: "xml"
+};
+
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Data extractor connector types.
+ */
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const DataExtractorConnectorType = {
+    /**
+     * JSON Path.
+     */
+    JsonPath: "json-path"
+};
+
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Data processing component types.
+ */
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const DataProcessingComponentType = {
+    /**
+     * Service.
+     */
+    Service: "service"
+};
+
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Initialise the data converter connector.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the connector type is unknown.
+ */
+function initialiseDataConverterConnector(engineCore, context, instanceConfig, overrideInstanceType) {
+    engineCore.logInfo(I18n.formatMessage("engineCore.configuring", {
+        element: `Data Converter Connector: ${instanceConfig.type}`
+    }));
+    const type = instanceConfig.type;
+    let connector;
+    let instanceType;
+    if (type === DataConverterConnectorType.Json) {
+        connector = new JsonConverterConnector();
+        instanceType = JsonConverterConnector.NAMESPACE;
+    }
+    else if (type === DataConverterConnectorType.Xml) {
+        connector = new XmlConverterConnector();
+        instanceType = XmlConverterConnector.NAMESPACE;
+    }
+    else {
+        throw new GeneralError("engineCore", "connectorUnknownType", {
+            type,
+            connectorType: "dataConverterConnector"
+        });
+    }
+    const finalInstanceType = overrideInstanceType ?? instanceType;
+    context.componentInstances.push({
+        instanceType: finalInstanceType,
+        component: connector
+    });
+    DataConverterConnectorFactory.register(finalInstanceType, () => connector);
+    return finalInstanceType;
+}
+/**
+ * Initialise the data extractor connector.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the connector type is unknown.
+ */
+function initialiseDataExtractorConnector(engineCore, context, instanceConfig, overrideInstanceType) {
+    engineCore.logInfo(I18n.formatMessage("engineCore.configuring", {
+        element: `Data Extractor Connector: ${instanceConfig.type}`
+    }));
+    const type = instanceConfig.type;
+    let connector;
+    let instanceType;
+    if (type === DataExtractorConnectorType.JsonPath) {
+        connector = new JsonPathExtractorConnector();
+        instanceType = JsonPathExtractorConnector.NAMESPACE;
+    }
+    else {
+        throw new GeneralError("engineCore", "connectorUnknownType", {
+            type,
+            connectorType: "dataExtractorConnector"
+        });
+    }
+    const finalInstanceType = overrideInstanceType ?? instanceType;
+    context.componentInstances.push({
+        instanceType: finalInstanceType,
+        component: connector
+    });
+    DataExtractorConnectorFactory.register(finalInstanceType, () => connector);
+    return finalInstanceType;
+}
+/**
+ * Initialise the data processing component.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the component type is unknown.
+ */
+function initialiseDataProcessingComponent(engineCore, context, instanceConfig, overrideInstanceType) {
+    engineCore.logInfo(I18n.formatMessage("engineCore.configuring", {
+        element: `Data Processing Component: ${instanceConfig.type}`
+    }));
+    const type = instanceConfig.type;
+    let component;
+    let instanceType;
+    if (type === DataProcessingComponentType.Service) {
+        initSchema$4();
+        initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.extractionRuleGroupStorageConnectorType, "ExtractionRuleGroup");
+        component = new DataProcessingService({
+            ...instanceConfig.options
+        });
+        instanceType = DataProcessingService.NAMESPACE;
+    }
+    else {
+        throw new GeneralError("engineCore", "componentUnknownType", {
+            type,
+            componentType: "dataProcessingComponent"
+        });
+    }
+    const finalInstanceType = overrideInstanceType ?? instanceType;
+    context.componentInstances.push({
+        instanceType: finalInstanceType,
+        component
+    });
+    ComponentFactory.register(finalInstanceType, () => component);
+    return finalInstanceType;
+}
+
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 /**
@@ -964,7 +1154,7 @@ function initialiseIdentityConnector(engineCore, context, instanceConfig, overri
         instanceType = IotaRebasedIdentityConnector.NAMESPACE;
     }
     else if (type === IdentityConnectorType.EntityStorage) {
-        initSchema$4({ includeProfile: false });
+        initSchema$5({ includeProfile: false });
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.didDocumentEntityStorageType, "IdentityDocument");
         connector = new EntityStorageIdentityConnector({
             vaultConnectorType: context.defaultTypes.vaultConnector,
@@ -1060,7 +1250,7 @@ function initialiseIdentityProfileConnector(engineCore, context, instanceConfig,
     let connector;
     let instanceType;
     if (type === IdentityProfileConnectorType.EntityStorage) {
-        initSchema$4({ includeDocument: false });
+        initSchema$5({ includeDocument: false });
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.profileEntityStorageType, "IdentityProfile");
         connector = new EntityStorageIdentityProfileConnector(instanceConfig.options);
         instanceType = EntityStorageIdentityProfileConnector.NAMESPACE;
@@ -1192,7 +1382,7 @@ function initialiseIdentityResolverConnector(engineCore, context, instanceConfig
         instanceType = IotaRebasedIdentityResolverConnector.NAMESPACE;
     }
     else if (type === IdentityResolverConnectorType.EntityStorage) {
-        initSchema$4({ includeProfile: false });
+        initSchema$5({ includeProfile: false });
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.didDocumentEntityStorageType, "IdentityDocument");
         connector = new EntityStorageIdentityResolverConnector({
             vaultConnectorType: context.defaultTypes.vaultConnector,
@@ -1275,7 +1465,7 @@ function initialiseImmutableProofComponent(engineCore, context, instanceConfig,
     let component;
     let instanceType;
     if (type === ImmutableProofComponentType.Service) {
-        initSchema$5();
+        initSchema$6();
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.immutableProofEntityStorageType, "ImmutableProof");
         component = new ImmutableProofService({
             vaultConnectorType: context.defaultTypes.vaultConnector,
@@ -1366,7 +1556,7 @@ function initialiseImmutableStorageConnector(engineCore, context, instanceConfig
         instanceType = IotaRebasedImmutableStorageConnector.NAMESPACE;
     }
     else if (type === ImmutableStorageConnectorType.EntityStorage) {
-        initSchema$6();
+        initSchema$7();
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.immutableStorageEntityStorageType, "ImmutableItem");
         connector = new EntityStorageImmutableStorageConnector(instanceConfig.options);
         instanceType = EntityStorageImmutableStorageConnector.NAMESPACE;
@@ -1443,7 +1633,7 @@ function initialiseLoggingConnector(engineCore, context, instanceConfig, overrid
         instanceType = ConsoleLoggingConnector.NAMESPACE;
     }
     else if (type === LoggingConnectorType.EntityStorage) {
-        initSchema$7();
+        initSchema$8();
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.logEntryStorageConnectorType, "LogEntry");
         connector = new EntityStorageLoggingConnector(instanceConfig.options);
         instanceType = EntityStorageLoggingConnector.NAMESPACE;
@@ -1581,7 +1771,7 @@ function initialiseMessagingEmailConnector(engineCore, context, instanceConfig,
     let connector;
     let instanceType;
     if (type === MessagingEmailConnectorType.EntityStorage) {
-        initSchema$8({ email: true, sms: false, pushNotification: false });
+        initSchema$9({ email: true, sms: false, pushNotification: false });
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.messagingEmailEntryStorageConnectorType, "EmailEntry");
         connector = new EntityStorageMessagingEmailConnector({
             loggingConnectorType: context.defaultTypes.loggingConnector,
@@ -1624,7 +1814,7 @@ function initialiseMessagingSmsConnector(engineCore, context, instanceConfig, ov
     let connector;
     let instanceType;
     if (type === MessagingSmsConnectorType.EntityStorage) {
-        initSchema$8({ email: false, sms: true, pushNotification: false });
+        initSchema$9({ email: false, sms: true, pushNotification: false });
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.messagingSmsEntryStorageConnectorType, "SmsEntry");
         connector = new EntityStorageMessagingSmsConnector({
             loggingConnectorType: context.defaultTypes.loggingConnector,
@@ -1667,7 +1857,7 @@ function initialiseMessagingPushNotificationConnector(engineCore, context, insta
     let connector;
     let instanceType;
     if (type === MessagingPushNotificationConnectorType.EntityStorage) {
-        initSchema$8({ email: false, sms: false, pushNotification: true });
+        initSchema$9({ email: false, sms: false, pushNotification: true });
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.messagingDeviceEntryStorageConnectorType, "PushNotificationDeviceEntry");
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.messagingMessageEntryStorageConnectorType, "PushNotificationMessageEntry");
         connector = new EntityStorageMessagingPushNotificationConnector({
@@ -1711,7 +1901,7 @@ function initialiseMessagingComponent(engineCore, context, instanceConfig, overr
     let component;
     let instanceType;
     if (type === MessagingComponentType.Service) {
-        initSchema$9();
+        initSchema$a();
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.templateEntryStorageConnectorType, "TemplateEntry");
         component = new MessagingService({
             messagingEmailConnectorType: context.defaultTypes.messagingEmailConnector,
@@ -1786,7 +1976,7 @@ function initialiseNftConnector(engineCore, context, instanceConfig, overrideIns
     let connector;
     let instanceType;
     if (type === NftConnectorType.EntityStorage) {
-        initSchema$a();
+        initSchema$b();
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.nftEntityStorageType, "Nft");
         connector = new EntityStorageNftConnector(instanceConfig.options);
         instanceType = EntityStorageNftConnector.NAMESPACE;
@@ -1899,7 +2089,7 @@ function initialiseTelemetryConnector(engineCore, context, instanceConfig, overr
     let connector;
     let instanceType;
     if (type === TelemetryConnectorType.EntityStorage) {
-        initSchema$b();
+        initSchema$c();
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.telemetryMetricStorageConnectorType, "TelemetryMetric");
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.telemetryMetricValueStorageConnectorType, "TelemetryMetricValue");
         connector = new EntityStorageTelemetryConnector({
@@ -1990,7 +2180,7 @@ function initialiseVaultConnector(engineCore, context, instanceConfig, overrideI
     let connector;
     let instanceType;
     if (type === VaultConnectorType.EntityStorage) {
-        initSchema$c();
+        initSchema$d();
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.vaultKeyEntityStorageType, "VaultKey");
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.vaultSecretEntityStorageType, "VaultSecret");
         connector = new EntityStorageVaultConnector(instanceConfig.options);
@@ -2112,7 +2302,7 @@ function initialiseWalletStorage(engineCore, context, instanceConfig, overrideIn
     const type = instanceConfig.type;
     if (type === WalletConnectorType.Iota) ;
     else if (type === WalletConnectorType.EntityStorage) {
-        initSchema$d();
+        initSchema$e();
         initialiseEntityStorageConnector(engineCore, context, instanceConfig.options?.walletAddressEntityStorageType, "WalletAddress");
     }
     else {
@@ -2141,4 +2331,4 @@ const DltConfigType = {
     IotaRebased: "iota-rebased"
 };
 
-export { AttestationComponentType, AttestationConnectorType, AuditableItemGraphComponentType, AuditableItemStreamComponentType, BackgroundTaskConnectorType, BlobStorageComponentType, BlobStorageConnectorType, DltConfigType, EntityStorageComponentType, EntityStorageConnectorType, EventBusComponentType, EventBusConnectorType, FaucetConnectorType, IdentityComponentType, IdentityConnectorType, IdentityProfileComponentType, IdentityProfileConnectorType, IdentityResolverComponentType, IdentityResolverConnectorType, ImmutableProofComponentType, ImmutableStorageConnectorType, LoggingComponentType, LoggingConnectorType, MessagingComponentType, MessagingEmailConnectorType, MessagingPushNotificationConnectorType, MessagingSmsConnectorType, NftComponentType, NftConnectorType, TelemetryComponentType, TelemetryConnectorType, VaultConnectorType, WalletConnectorType, initialiseAttestationComponent, initialiseAttestationConnector, initialiseAuditableItemGraphComponent, initialiseAuditableItemStreamComponent, initialiseBackgroundTaskConnector, initialiseBlobStorageComponent, initialiseBlobStorageConnector, initialiseEntityStorageComponent, initialiseEntityStorageConnector, initialiseEventBusComponent, initialiseEventBusConnector, initialiseFaucetConnector, initialiseIdentityComponent, initialiseIdentityConnector, initialiseIdentityProfileComponent, initialiseIdentityProfileConnector, initialiseIdentityResolverComponent, initialiseIdentityResolverConnector, initialiseImmutableProofComponent, initialiseImmutableStorageConnector, initialiseLoggingComponent, initialiseLoggingConnector, initialiseMessagingComponent, initialiseMessagingEmailConnector, initialiseMessagingPushNotificationConnector, initialiseMessagingSmsConnector, initialiseNftComponent, initialiseNftConnector, initialiseTelemetryComponent, initialiseTelemetryConnector, initialiseVaultConnector, initialiseWalletConnector, initialiseWalletStorage };
+export { AttestationComponentType, AttestationConnectorType, AuditableItemGraphComponentType, AuditableItemStreamComponentType, BackgroundTaskConnectorType, BlobStorageComponentType, BlobStorageConnectorType, DataConverterConnectorType, DataExtractorConnectorType, DataProcessingComponentType, DltConfigType, EntityStorageComponentType, EntityStorageConnectorType, EventBusComponentType, EventBusConnectorType, FaucetConnectorType, IdentityComponentType, IdentityConnectorType, IdentityProfileComponentType, IdentityProfileConnectorType, IdentityResolverComponentType, IdentityResolverConnectorType, ImmutableProofComponentType, ImmutableStorageConnectorType, LoggingComponentType, LoggingConnectorType, MessagingComponentType, MessagingEmailConnectorType, MessagingPushNotificationConnectorType, MessagingSmsConnectorType, NftComponentType, NftConnectorType, TelemetryComponentType, TelemetryConnectorType, VaultConnectorType, WalletConnectorType, initialiseAttestationComponent, initialiseAttestationConnector, initialiseAuditableItemGraphComponent, initialiseAuditableItemStreamComponent, initialiseBackgroundTaskConnector, initialiseBlobStorageComponent, initialiseBlobStorageConnector, initialiseDataConverterConnector, initialiseDataExtractorConnector, initialiseDataProcessingComponent, initialiseEntityStorageComponent, initialiseEntityStorageConnector, initialiseEventBusComponent, initialiseEventBusConnector, initialiseFaucetConnector, initialiseIdentityComponent, initialiseIdentityConnector, initialiseIdentityProfileComponent, initialiseIdentityProfileConnector, initialiseIdentityResolverComponent, initialiseIdentityResolverConnector, initialiseImmutableProofComponent, initialiseImmutableStorageConnector, initialiseLoggingComponent, initialiseLoggingConnector, initialiseMessagingComponent, initialiseMessagingEmailConnector, initialiseMessagingPushNotificationConnector, initialiseMessagingSmsConnector, initialiseNftComponent, initialiseNftConnector, initialiseTelemetryComponent, initialiseTelemetryConnector, initialiseVaultConnector, initialiseWalletConnector, initialiseWalletStorage };

package/dist/types/components/dataProcessing.d.ts

@@ -0,0 +1,35 @@
+import type { IEngineCore, IEngineCoreContext } from "@twin.org/engine-models";
+import type { DataConverterConnectorConfig } from "../models/config/dataConverterConnectorConfig";
+import type { DataExtractorConnectorConfig } from "../models/config/dataExtractorConnectorConfig";
+import type { DataProcessingComponentConfig } from "../models/config/dataProcessingComponentConfig";
+import type { IEngineConfig } from "../models/IEngineConfig";
+/**
+ * Initialise the data converter connector.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the connector type is unknown.
+ */
+export declare function initialiseDataConverterConnector(engineCore: IEngineCore<IEngineConfig>, context: IEngineCoreContext<IEngineConfig>, instanceConfig: DataConverterConnectorConfig, overrideInstanceType?: string): string | undefined;
+/**
+ * Initialise the data extractor connector.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the connector type is unknown.
+ */
+export declare function initialiseDataExtractorConnector(engineCore: IEngineCore<IEngineConfig>, context: IEngineCoreContext<IEngineConfig>, instanceConfig: DataExtractorConnectorConfig, overrideInstanceType?: string): string | undefined;
+/**
+ * Initialise the data processing component.
+ * @param engineCore The engine core.
+ * @param context The context for the engine.
+ * @param instanceConfig The instance config.
+ * @param overrideInstanceType The instance type to override the default.
+ * @returns The name of the instance created.
+ * @throws GeneralError if the component type is unknown.
+ */
+export declare function initialiseDataProcessingComponent(engineCore: IEngineCore<IEngineConfig>, context: IEngineCoreContext<IEngineConfig>, instanceConfig: DataProcessingComponentConfig, overrideInstanceType?: string): string | undefined;

package/dist/types/index.d.ts

@@ -3,6 +3,7 @@ export * from "./components/auditableItemGraph";
 export * from "./components/auditableItemStream";
 export * from "./components/backgroundTask";
 export * from "./components/blobStorage";
+export * from "./components/dataProcessing";
 export * from "./components/entityStorage";
 export * from "./components/eventBus";
 export * from "./components/faucet";
@@ -24,6 +25,9 @@ export * from "./models/config/auditableItemStreamComponentConfig";
 export * from "./models/config/backgroundTaskConnectorConfig";
 export * from "./models/config/blobStorageComponentConfig";
 export * from "./models/config/blobStorageConnectorConfig";
+export * from "./models/config/dataConverterConnectorConfig";
+export * from "./models/config/dataExtractorConnectorConfig";
+export * from "./models/config/dataProcessingComponentConfig";
 export * from "./models/config/dltConfig";
 export * from "./models/config/entityStorageComponentConfig";
 export * from "./models/config/entityStorageConnectorConfig";
@@ -58,6 +62,9 @@ export * from "./models/types/auditableItemStreamComponentType";
 export * from "./models/types/backgroundTaskConnectorType";
 export * from "./models/types/blobStorageComponentType";
 export * from "./models/types/blobStorageConnectorType";
+export * from "./models/types/dataConverterConnectorType";
+export * from "./models/types/dataExtractorConnectorType";
+export * from "./models/types/dataProcessingComponentType";
 export * from "./models/types/dltConfigType";
 export * from "./models/types/entityStorageComponentType";
 export * from "./models/types/entityStorageConnectorType";

package/dist/types/models/IEngineConfig.d.ts

@@ -6,6 +6,9 @@ import type { AuditableItemStreamComponentConfig } from "./config/auditableItemS
 import type { BackgroundTaskConnectorConfig } from "./config/backgroundTaskConnectorConfig";
 import type { BlobStorageComponentConfig } from "./config/blobStorageComponentConfig";
 import type { BlobStorageConnectorConfig } from "./config/blobStorageConnectorConfig";
+import type { DataConverterConnectorConfig } from "./config/dataConverterConnectorConfig";
+import type { DataExtractorConnectorConfig } from "./config/dataExtractorConnectorConfig";
+import type { DataProcessingComponentConfig } from "./config/dataProcessingComponentConfig";
 import type { DltConfig } from "./config/dltConfig";
 import type { EntityStorageComponentConfig } from "./config/entityStorageComponentConfig";
 import type { EntityStorageConnectorConfig } from "./config/entityStorageConnectorConfig";
@@ -162,7 +165,7 @@ export interface IEngineConfig extends IEngineCoreConfig {
          */
         attestationConnector?: IEngineCoreTypeConfig<AttestationConnectorConfig>[];
         /**
-         * Attestation component profile options which can be overridden by individual components by specifying types other than default.
+         * Attestation component options which can be overridden by individual components by specifying types other than default.
          */
         attestationComponent?: IEngineCoreTypeConfig<AttestationComponentConfig>[];
         /**
@@ -170,8 +173,20 @@ export interface IEngineConfig extends IEngineCoreConfig {
          */
         auditableItemGraphComponent?: IEngineCoreTypeConfig<AuditableItemGraphComponentConfig>[];
         /**
-         * Auditable item stream component profile options which can be overridden by individual components by specifying types other than default.
+         * Auditable item stream component  options which can be overridden by individual components by specifying types other than default.
          */
         auditableItemStreamComponent?: IEngineCoreTypeConfig<AuditableItemStreamComponentConfig>[];
+        /**
+         * Data converter connector options which can be overridden by individual components by specifying types other than default.
+         */
+        dataConverterConnector?: IEngineCoreTypeConfig<DataConverterConnectorConfig>[];
+        /**
+         * Data extractor connector options which can be overridden by individual components by specifying types other than default.
+         */
+        dataExtractorConnector?: IEngineCoreTypeConfig<DataExtractorConnectorConfig>[];
+        /**
+         * Date processing options which can be overridden by individual components by specifying types other than default.
+         */
+        dataProcessingComponent?: IEngineCoreTypeConfig<DataProcessingComponentConfig>[];
     };
 }

package/dist/types/models/config/dataConverterConnectorConfig.d.ts

@@ -0,0 +1,11 @@
+import type { DataConverterConnectorType } from "../types/dataConverterConnectorType";
+/**
+ * Data converter connector config types.
+ */
+export type DataConverterConnectorConfig = {
+    type: typeof DataConverterConnectorType.Json;
+    options?: never;
+} | {
+    type: typeof DataConverterConnectorType.Xml;
+    options?: never;
+};

package/dist/types/models/config/dataExtractorConnectorConfig.d.ts

@@ -0,0 +1,8 @@
+import type { DataExtractorConnectorType } from "../types/dataExtractorConnectorType";
+/**
+ * Data extractor connector config types.
+ */
+export type DataExtractorConnectorConfig = {
+    type: typeof DataExtractorConnectorType.JsonPath;
+    options?: never;
+};

package/dist/types/models/config/dataProcessingComponentConfig.d.ts

@@ -0,0 +1,9 @@
+import type { IDataProcessingServiceConstructorOptions } from "@twin.org/data-processing-service";
+import type { DataProcessingComponentType } from "../types/dataProcessingComponentType";
+/**
+ * Data processing component config types.
+ */
+export type DataProcessingComponentConfig = {
+    type: typeof DataProcessingComponentType.Service;
+    options?: IDataProcessingServiceConstructorOptions;
+};

package/dist/types/models/config/entityStorageConnectorConfig.d.ts

@@ -2,7 +2,9 @@ import type { ICosmosDbEntityStorageConnectorConstructorOptions } from "@twin.or
 import type { IDynamoDbEntityStorageConnectorConstructorOptions } from "@twin.org/entity-storage-connector-dynamodb";
 import type { IFileEntityStorageConnectorConstructorOptions } from "@twin.org/entity-storage-connector-file";
 import type { IFirestoreEntityStorageConnectorConstructorOptions } from "@twin.org/entity-storage-connector-gcp-firestore";
+import type { IMongoDbEntityStorageConnectorConstructorOptions } from "@twin.org/entity-storage-connector-mongodb";
 import type { IMySqlEntityStorageConnectorConstructorOptions } from "@twin.org/entity-storage-connector-mysql";
+import type { IPostgreSqlEntityStorageConnectorConstructorOptions } from "@twin.org/entity-storage-connector-postgresql";
 import type { IScyllaDBTableConnectorConstructorOptions } from "@twin.org/entity-storage-connector-scylladb";
 import type { EntityStorageConnectorType } from "../types/entityStorageConnectorType";
 /**
@@ -46,4 +48,16 @@ export type EntityStorageConnectorConfig = {
         config: Omit<IMySqlEntityStorageConnectorConstructorOptions["config"], "tableName">;
         tablePrefix?: string;
     };
+} | {
+    type: typeof EntityStorageConnectorType.MongoDb;
+    options: Omit<IMongoDbEntityStorageConnectorConstructorOptions, "entitySchema" | "config"> & {
+        config: Omit<IMongoDbEntityStorageConnectorConstructorOptions["config"], "collection">;
+        tablePrefix?: string;
+    };
+} | {
+    type: typeof EntityStorageConnectorType.PostgreSql;
+    options: Omit<IPostgreSqlEntityStorageConnectorConstructorOptions, "entitySchema" | "config"> & {
+        config: Omit<IPostgreSqlEntityStorageConnectorConstructorOptions["config"], "tableName">;
+        tablePrefix?: string;
+    };
 };

package/dist/types/models/types/dataConverterConnectorType.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Data converter connector types.
+ */
+export declare const DataConverterConnectorType: {
+    /**
+     * JSON.
+     */
+    readonly Json: "json";
+    /**
+     * Xml.
+     */
+    readonly Xml: "xml";
+};
+/**
+ * Data converter connector types.
+ */
+export type DataConverterConnectorType = (typeof DataConverterConnectorType)[keyof typeof DataConverterConnectorType];

package/dist/types/models/types/dataExtractorConnectorType.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Data extractor connector types.
+ */
+export declare const DataExtractorConnectorType: {
+    /**
+     * JSON Path.
+     */
+    readonly JsonPath: "json-path";
+};
+/**
+ * Data extractor connector types.
+ */
+export type DataExtractorConnectorType = (typeof DataExtractorConnectorType)[keyof typeof DataExtractorConnectorType];

package/dist/types/models/types/dataProcessingComponentType.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Data processing component types.
+ */
+export declare const DataProcessingComponentType: {
+    /**
+     * Service.
+     */
+    readonly Service: "service";
+};
+/**
+ * Data processing component types.
+ */
+export type DataProcessingComponentType = (typeof DataProcessingComponentType)[keyof typeof DataProcessingComponentType];

package/dist/types/models/types/entityStorageConnectorType.d.ts

@@ -30,6 +30,14 @@ export declare const EntityStorageConnectorType: {
      * MySqlDb.
      */
     readonly MySqlDb: "mysql";
+    /**
+     * MongoDb.
+     */
+    readonly MongoDb: "mongodb";
+    /**
+     * Postgre SQL.
+     */
+    readonly PostgreSql: "postgresql";
 };
 /**
  * Entity storage connector types.

package/docs/reference/functions/initialiseDataConverterConnector.md

@@ -0,0 +1,41 @@
+# Function: initialiseDataConverterConnector()
+
+> **initialiseDataConverterConnector**(`engineCore`, `context`, `instanceConfig`, `overrideInstanceType`?): `string` \| `undefined`
+
+Initialise the data converter connector.
+
+## Parameters
+
+### engineCore
+
+`IEngineCore`\<[`IEngineConfig`](../interfaces/IEngineConfig.md)\>
+
+The engine core.
+
+### context
+
+`IEngineCoreContext`\<[`IEngineConfig`](../interfaces/IEngineConfig.md)\>
+
+The context for the engine.
+
+### instanceConfig
+
+[`DataConverterConnectorConfig`](../type-aliases/DataConverterConnectorConfig.md)
+
+The instance config.
+
+### overrideInstanceType?
+
+`string`
+
+The instance type to override the default.
+
+## Returns
+
+`string` \| `undefined`
+
+The name of the instance created.
+
+## Throws
+
+GeneralError if the connector type is unknown.

package/docs/reference/functions/initialiseDataExtractorConnector.md

@@ -0,0 +1,41 @@
+# Function: initialiseDataExtractorConnector()
+
+> **initialiseDataExtractorConnector**(`engineCore`, `context`, `instanceConfig`, `overrideInstanceType`?): `string` \| `undefined`
+
+Initialise the data extractor connector.
+
+## Parameters
+
+### engineCore
+
+`IEngineCore`\<[`IEngineConfig`](../interfaces/IEngineConfig.md)\>
+
+The engine core.
+
+### context
+
+`IEngineCoreContext`\<[`IEngineConfig`](../interfaces/IEngineConfig.md)\>
+
+The context for the engine.
+
+### instanceConfig
+
+[`DataExtractorConnectorConfig`](../type-aliases/DataExtractorConnectorConfig.md)
+
+The instance config.
+
+### overrideInstanceType?
+
+`string`
+
+The instance type to override the default.
+
+## Returns
+
+`string` \| `undefined`
+
+The name of the instance created.
+
+## Throws
+
+GeneralError if the connector type is unknown.

package/docs/reference/functions/initialiseDataProcessingComponent.md

@@ -0,0 +1,41 @@
+# Function: initialiseDataProcessingComponent()
+
+> **initialiseDataProcessingComponent**(`engineCore`, `context`, `instanceConfig`, `overrideInstanceType`?): `string` \| `undefined`
+
+Initialise the data processing component.
+
+## Parameters
+
+### engineCore
+
+`IEngineCore`\<[`IEngineConfig`](../interfaces/IEngineConfig.md)\>
+
+The engine core.
+
+### context
+
+`IEngineCoreContext`\<[`IEngineConfig`](../interfaces/IEngineConfig.md)\>
+
+The context for the engine.
+
+### instanceConfig
+
+[`DataProcessingComponentConfig`](../type-aliases/DataProcessingComponentConfig.md)
+
+The instance config.
+
+### overrideInstanceType?
+
+`string`
+
+The instance type to override the default.
+
+## Returns
+
+`string` \| `undefined`
+
+The name of the instance created.
+
+## Throws
+
+GeneralError if the component type is unknown.

package/docs/reference/index.md

@@ -13,6 +13,9 @@
 - [BackgroundTaskConnectorConfig](type-aliases/BackgroundTaskConnectorConfig.md)
 - [BlobStorageComponentConfig](type-aliases/BlobStorageComponentConfig.md)
 - [BlobStorageConnectorConfig](type-aliases/BlobStorageConnectorConfig.md)
+- [DataConverterConnectorConfig](type-aliases/DataConverterConnectorConfig.md)
+- [DataExtractorConnectorConfig](type-aliases/DataExtractorConnectorConfig.md)
+- [DataProcessingComponentConfig](type-aliases/DataProcessingComponentConfig.md)
 - [DltConfig](type-aliases/DltConfig.md)
 - [EntityStorageComponentConfig](type-aliases/EntityStorageComponentConfig.md)
 - [EntityStorageConnectorConfig](type-aliases/EntityStorageConnectorConfig.md)
@@ -46,6 +49,9 @@
 - [BackgroundTaskConnectorType](type-aliases/BackgroundTaskConnectorType.md)
 - [BlobStorageComponentType](type-aliases/BlobStorageComponentType.md)
 - [BlobStorageConnectorType](type-aliases/BlobStorageConnectorType.md)
+- [DataConverterConnectorType](type-aliases/DataConverterConnectorType.md)
+- [DataExtractorConnectorType](type-aliases/DataExtractorConnectorType.md)
+- [DataProcessingComponentType](type-aliases/DataProcessingComponentType.md)
 - [DltConfigType](type-aliases/DltConfigType.md)
 - [EntityStorageComponentType](type-aliases/EntityStorageComponentType.md)
 - [EntityStorageConnectorType](type-aliases/EntityStorageConnectorType.md)
@@ -82,6 +88,9 @@
 - [BackgroundTaskConnectorType](variables/BackgroundTaskConnectorType.md)
 - [BlobStorageComponentType](variables/BlobStorageComponentType.md)
 - [BlobStorageConnectorType](variables/BlobStorageConnectorType.md)
+- [DataConverterConnectorType](variables/DataConverterConnectorType.md)
+- [DataExtractorConnectorType](variables/DataExtractorConnectorType.md)
+- [DataProcessingComponentType](variables/DataProcessingComponentType.md)
 - [DltConfigType](variables/DltConfigType.md)
 - [EntityStorageComponentType](variables/EntityStorageComponentType.md)
 - [EntityStorageConnectorType](variables/EntityStorageConnectorType.md)
@@ -118,6 +127,9 @@
 - [initialiseBackgroundTaskConnector](functions/initialiseBackgroundTaskConnector.md)
 - [initialiseBlobStorageConnector](functions/initialiseBlobStorageConnector.md)
 - [initialiseBlobStorageComponent](functions/initialiseBlobStorageComponent.md)
+- [initialiseDataConverterConnector](functions/initialiseDataConverterConnector.md)
+- [initialiseDataExtractorConnector](functions/initialiseDataExtractorConnector.md)
+- [initialiseDataProcessingComponent](functions/initialiseDataProcessingComponent.md)
 - [initialiseEntityStorageConnector](functions/initialiseEntityStorageConnector.md)
 - [initialiseEntityStorageComponent](functions/initialiseEntityStorageComponent.md)
 - [initialiseEventBusConnector](functions/initialiseEventBusConnector.md)

package/docs/reference/interfaces/IEngineConfig.md

@@ -202,7 +202,7 @@ Attestation connector options which can be overridden by individual components b
 
 > `optional` **attestationComponent**: `IEngineCoreTypeConfig`\<[`AttestationComponentConfig`](../type-aliases/AttestationComponentConfig.md)\>[]
 
-Attestation component profile options which can be overridden by individual components by specifying types other than default.
+Attestation component options which can be overridden by individual components by specifying types other than default.
 
 #### auditableItemGraphComponent?
 
@@ -214,7 +214,25 @@ Auditable item graph component options which can be overridden by individual com
 
 > `optional` **auditableItemStreamComponent**: `IEngineCoreTypeConfig`\<[`AuditableItemStreamComponentConfig`](../type-aliases/AuditableItemStreamComponentConfig.md)\>[]
 
-Auditable item stream component profile options which can be overridden by individual components by specifying types other than default.
+Auditable item stream component  options which can be overridden by individual components by specifying types other than default.
+
+#### dataConverterConnector?
+
+> `optional` **dataConverterConnector**: `IEngineCoreTypeConfig`\<[`DataConverterConnectorConfig`](../type-aliases/DataConverterConnectorConfig.md)\>[]
+
+Data converter connector options which can be overridden by individual components by specifying types other than default.
+
+#### dataExtractorConnector?
+
+> `optional` **dataExtractorConnector**: `IEngineCoreTypeConfig`\<[`DataExtractorConnectorConfig`](../type-aliases/DataExtractorConnectorConfig.md)\>[]
+
+Data extractor connector options which can be overridden by individual components by specifying types other than default.
+
+#### dataProcessingComponent?
+
+> `optional` **dataProcessingComponent**: `IEngineCoreTypeConfig`\<[`DataProcessingComponentConfig`](../type-aliases/DataProcessingComponentConfig.md)\>[]
+
+Date processing options which can be overridden by individual components by specifying types other than default.
 
 #### Overrides
 

package/docs/reference/type-aliases/DataConverterConnectorConfig.md

@@ -0,0 +1,5 @@
+# Type Alias: DataConverterConnectorConfig
+
+> **DataConverterConnectorConfig**: \{ `type`: *typeof* [`Json`](../variables/DataConverterConnectorType.md#json); `options`: `never`; \} \| \{ `type`: *typeof* [`Xml`](../variables/DataConverterConnectorType.md#xml); `options`: `never`; \}
+
+Data converter connector config types.

package/docs/reference/type-aliases/DataConverterConnectorType.md

@@ -0,0 +1,5 @@
+# Type Alias: DataConverterConnectorType
+
+> **DataConverterConnectorType**: *typeof* [`DataConverterConnectorType`](../variables/DataConverterConnectorType.md)\[keyof *typeof* [`DataConverterConnectorType`](../variables/DataConverterConnectorType.md)\]
+
+Data converter connector types.

package/docs/reference/type-aliases/DataExtractorConnectorConfig.md

@@ -0,0 +1,15 @@
+# Type Alias: DataExtractorConnectorConfig
+
+> **DataExtractorConnectorConfig**: `object`
+
+Data extractor connector config types.
+
+## Type declaration
+
+### type
+
+> **type**: *typeof* [`JsonPath`](../variables/DataExtractorConnectorType.md#jsonpath)
+
+### options?
+
+> `optional` **options**: `never`

package/docs/reference/type-aliases/DataExtractorConnectorType.md

@@ -0,0 +1,5 @@
+# Type Alias: DataExtractorConnectorType
+
+> **DataExtractorConnectorType**: *typeof* [`DataExtractorConnectorType`](../variables/DataExtractorConnectorType.md)\[keyof *typeof* [`DataExtractorConnectorType`](../variables/DataExtractorConnectorType.md)\]
+
+Data extractor connector types.

package/docs/reference/type-aliases/DataProcessingComponentConfig.md

@@ -0,0 +1,15 @@
+# Type Alias: DataProcessingComponentConfig
+
+> **DataProcessingComponentConfig**: `object`
+
+Data processing component config types.
+
+## Type declaration
+
+### type
+
+> **type**: *typeof* [`Service`](../variables/DataProcessingComponentType.md#service)
+
+### options?
+
+> `optional` **options**: `IDataProcessingServiceConstructorOptions`

package/docs/reference/type-aliases/DataProcessingComponentType.md

@@ -0,0 +1,5 @@
+# Type Alias: DataProcessingComponentType
+
+> **DataProcessingComponentType**: *typeof* [`DataProcessingComponentType`](../variables/DataProcessingComponentType.md)\[keyof *typeof* [`DataProcessingComponentType`](../variables/DataProcessingComponentType.md)\]
+
+Data processing component types.

package/docs/reference/type-aliases/EntityStorageConnectorConfig.md

@@ -1,5 +1,5 @@
 # Type Alias: EntityStorageConnectorConfig
 
-> **EntityStorageConnectorConfig**: \{ `type`: *typeof* [`File`](../variables/EntityStorageConnectorType.md#file); `options`: `Omit`\<`IFileEntityStorageConnectorConstructorOptions`, `"entitySchema"`\> & `object`; \} \| \{ `type`: *typeof* [`Memory`](../variables/EntityStorageConnectorType.md#memory); `options`: `never`; \} \| \{ `type`: *typeof* [`AwsDynamoDb`](../variables/EntityStorageConnectorType.md#awsdynamodb); `options`: `Omit`\<`IDynamoDbEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`AzureCosmosDb`](../variables/EntityStorageConnectorType.md#azurecosmosdb); `options`: `Omit`\<`ICosmosDbEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`GcpFirestoreDb`](../variables/EntityStorageConnectorType.md#gcpfirestoredb); `options`: `Omit`\<`IFirestoreEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`ScyllaDb`](../variables/EntityStorageConnectorType.md#scylladb); `options`: `Omit`\<`IScyllaDBTableConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`MySqlDb`](../variables/EntityStorageConnectorType.md#mysqldb); `options`: `Omit`\<`IMySqlEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \}
+> **EntityStorageConnectorConfig**: \{ `type`: *typeof* [`File`](../variables/EntityStorageConnectorType.md#file); `options`: `Omit`\<`IFileEntityStorageConnectorConstructorOptions`, `"entitySchema"`\> & `object`; \} \| \{ `type`: *typeof* [`Memory`](../variables/EntityStorageConnectorType.md#memory); `options`: `never`; \} \| \{ `type`: *typeof* [`AwsDynamoDb`](../variables/EntityStorageConnectorType.md#awsdynamodb); `options`: `Omit`\<`IDynamoDbEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`AzureCosmosDb`](../variables/EntityStorageConnectorType.md#azurecosmosdb); `options`: `Omit`\<`ICosmosDbEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`GcpFirestoreDb`](../variables/EntityStorageConnectorType.md#gcpfirestoredb); `options`: `Omit`\<`IFirestoreEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`ScyllaDb`](../variables/EntityStorageConnectorType.md#scylladb); `options`: `Omit`\<`IScyllaDBTableConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`MySqlDb`](../variables/EntityStorageConnectorType.md#mysqldb); `options`: `Omit`\<`IMySqlEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`MongoDb`](../variables/EntityStorageConnectorType.md#mongodb); `options`: `Omit`\<`IMongoDbEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \} \| \{ `type`: *typeof* [`PostgreSql`](../variables/EntityStorageConnectorType.md#postgresql); `options`: `Omit`\<`IPostgreSqlEntityStorageConnectorConstructorOptions`, `"entitySchema"` \| `"config"`\> & `object`; \}
 
 Entity storage connector config types.

package/docs/reference/variables/DataConverterConnectorType.md

@@ -0,0 +1,19 @@
+# Variable: DataConverterConnectorType
+
+> `const` **DataConverterConnectorType**: `object`
+
+Data converter connector types.
+
+## Type declaration
+
+### Json
+
+> `readonly` **Json**: `"json"` = `"json"`
+
+JSON.
+
+### Xml
+
+> `readonly` **Xml**: `"xml"` = `"xml"`
+
+Xml.

package/docs/reference/variables/DataExtractorConnectorType.md

@@ -0,0 +1,13 @@
+# Variable: DataExtractorConnectorType
+
+> `const` **DataExtractorConnectorType**: `object`
+
+Data extractor connector types.
+
+## Type declaration
+
+### JsonPath
+
+> `readonly` **JsonPath**: `"json-path"` = `"json-path"`
+
+JSON Path.

package/docs/reference/variables/DataProcessingComponentType.md

@@ -0,0 +1,13 @@
+# Variable: DataProcessingComponentType
+
+> `const` **DataProcessingComponentType**: `object`
+
+Data processing component types.
+
+## Type declaration
+
+### Service
+
+> `readonly` **Service**: `"service"` = `"service"`
+
+Service.

package/docs/reference/variables/EntityStorageConnectorType.md

@@ -47,3 +47,15 @@ GCP Firestore.
 > `readonly` **MySqlDb**: `"mysql"` = `"mysql"`
 
 MySqlDb.
+
+### MongoDb
+
+> `readonly` **MongoDb**: `"mongodb"` = `"mongodb"`
+
+MongoDb.
+
+### PostgreSql
+
+> `readonly` **PostgreSql**: `"postgresql"` = `"postgresql"`
+
+Postgre SQL.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@twin.org/engine-types",
-	"version": "0.0.1-next.37",
+	"version": "0.0.1-next.39",
 	"description": "Types to use in an engine.",
 	"repository": {
 		"type": "git",
@@ -18,7 +18,7 @@
 		"@twin.org/crypto": "next",
 		"@twin.org/data-core": "next",
 		"@twin.org/data-schema-org": "next",
-		"@twin.org/engine-models": "0.0.1-next.37",
+		"@twin.org/engine-models": "0.0.1-next.39",
 		"@twin.org/entity": "next",
 		"@twin.org/modules": "next",
 		"@twin.org/nameof": "next"
@@ -41,12 +41,18 @@
 		"@twin.org/blob-storage-connector-memory": "next",
 		"@twin.org/blob-storage-models": "next",
 		"@twin.org/blob-storage-service": "next",
+		"@twin.org/data-processing-converters": "next",
+		"@twin.org/data-processing-extractors": "next",
+		"@twin.org/data-processing-models": "next",
+		"@twin.org/data-processing-service": "next",
 		"@twin.org/entity-storage-connector-cosmosdb": "next",
 		"@twin.org/entity-storage-connector-dynamodb": "next",
 		"@twin.org/entity-storage-connector-file": "next",
 		"@twin.org/entity-storage-connector-gcp-firestore": "next",
 		"@twin.org/entity-storage-connector-memory": "next",
+		"@twin.org/entity-storage-connector-mongodb": "next",
 		"@twin.org/entity-storage-connector-mysql": "next",
+		"@twin.org/entity-storage-connector-postgresql": "next",
 		"@twin.org/entity-storage-connector-scylladb": "next",
 		"@twin.org/entity-storage-models": "next",
 		"@twin.org/entity-storage-service": "next",