@twin.org/node-core 0.9.2-next.8 → 0.9.2

package/dist/es/builders/engineEnvBuilder.js

@@ -2,13 +2,14 @@
 // SPDX-License-Identifier: Apache-2.0.
 import path from "node:path";
 import { Is, Mutex } from "@twin.org/core";
+import { EngineCloneMode } from "@twin.org/engine-models";
 import { AttestationComponentType, AttestationConnectorType, AuditableItemGraphComponentType, AuditableItemStreamComponentType, AutomationComponentType, BackgroundTaskComponentType, BlobStorageComponentType, BlobStorageConnectorType, ContextIdHandlerComponentType, DataProcessingComponentType, DataspaceControlPlaneComponentType, DataspaceDataPlaneComponentType, DltConfigType, DocumentManagementComponentType, EngineTypeHelper, EntityStorageConnectorType, EventBusComponentType, EventBusConnectorType, FaucetConnectorType, FederatedCatalogueComponentType, HealthComponentType, IdentityComponentType, IdentityConnectorType, IdentityProfileComponentType, IdentityProfileConnectorType, IdentityResolverComponentType, IdentityResolverConnectorType, ImmutableProofComponentType, LoggingComponentType, LoggingConnectorType, MessagingAdminComponentType, MessagingComponentType, MessagingEmailConnectorType, MessagingPushNotificationConnectorType, MessagingSmsConnectorType, MetricsCollectorComponentType, MetricsProducerComponentType, NftComponentType, NftConnectorType, NotarizationComponentType, NotarizationConnectorType, PlatformComponentType, RightsManagementPapComponentType, RightsManagementPdpComponentType, RightsManagementPepComponentType, RightsManagementPipComponentType, RightsManagementPmpComponentType, RightsManagementPnapComponentType, RightsManagementPnpComponentType, RightsManagementPxpComponentType, SchemaVersionMigrationComponentType, TaskSchedulerComponentType, TelemetryComponentType, TelemetryConnectorType, TenantAdminComponentType, TracingComponentType, TracingConnectorType, TrustComponentType, TrustVerifierComponentType, VaultConnectorType, WalletConnectorType } from "@twin.org/engine-types";
 import { OpenTelemetryExporterTypes } from "@twin.org/logging-connector-opentelemetry";
 import { OpenTelemetryReaderTypes } from "@twin.org/telemetry-connector-opentelemetry";
 import { OpenTelemetryProcessorTypes } from "@twin.org/tracing-connector-opentelemetry";
 import { CONTEXT_ID_HANDLER_FEATURE_DID, CONTEXT_ID_HANDLER_FEATURE_TENANT } from "../defaults.js";
 import { isAuthEntityStorageRequired } from "./engineServerEnvBuilder.js";
-import { commaSeparatedListToArray, envBoolean, envCount, envDateTime, envInteger, envMinToMs, envMinutes, envMs, envSecToMs, envSeconds } from "./helper/envHelpers.js";
+import { commaSeparatedListToArray, envArray, envBoolean, envCount, envDateTime, envInteger, envMinToMs, envMinutes, envMs, envSecToMs, envSeconds } from "./helper/envHelpers.js";
 /**
  * Build the engine core configuration from environment variables.
  * @param envVars The environment variables.
@@ -23,7 +24,11 @@ export async function buildEngineConfiguration(envVars) {
     const coreConfig = {
         debug: envBoolean(envVars, "debug", false),
         silent: envBoolean(envVars, "silent", false),
-        silentLoggers: commaSeparatedListToArray(envVars.loggingSilentComponents),
+        silentComponents: {
+            logging: commaSeparatedListToArray(envVars.loggingSilentComponents),
+            telemetry: commaSeparatedListToArray(envVars.telemetrySilentComponents),
+            tracing: commaSeparatedListToArray(envVars.tracingSilentComponents)
+        },
         types: {}
     };
     const mutexTimeoutMs = envMs(envVars, "mutexTimeoutDefault");
@@ -156,7 +161,11 @@ async function configureEntityStorage(coreConfig, envVars) {
                     localDataCenter: envVars.scylladbLocalDataCenter ?? "",
                     keyspace: envVars.scylladbKeyspace ?? "",
                     port: envInteger(envVars, "scylladbPort"),
-                    mutexTimeoutMs: envMs(envVars, "entityStorageMutexTimeout")
+                    mutexTimeoutMs: envMs(envVars, "entityStorageMutexTimeout"),
+                    pool: {
+                        coreConnectionsPerHost: envCount(envVars, "scylladbPoolCoreConnectionsPerHost"),
+                        maxRequestsPerConnection: envCount(envVars, "scylladbPoolMaxRequestsPerConnection")
+                    }
                 },
                 tablePrefix: envVars.entityStorageTablePrefix
             }
@@ -172,7 +181,15 @@ async function configureEntityStorage(coreConfig, envVars) {
                     user: envVars.mySqlUser ?? "",
                     password: envVars.mySqlPassword ?? "",
                     database: envVars.mySqlDatabase ?? "",
-                    mutexTimeoutMs: envMs(envVars, "entityStorageMutexTimeout")
+                    mutexTimeoutMs: envMs(envVars, "entityStorageMutexTimeout"),
+                    pool: {
+                        connectionLimit: envCount(envVars, "mySqlPoolConnectionLimit"),
+                        maxIdle: envCount(envVars, "mySqlPoolMaxIdle"),
+                        idleTimeout: envMs(envVars, "mySqlPoolIdleTimeout"),
+                        enableKeepAlive: envBoolean(envVars, "mySqlPoolEnableKeepAlive"),
+                        waitForConnections: envBoolean(envVars, "mySqlPoolWaitForConnections"),
+                        queueLimit: envCount(envVars, "mySqlPoolQueueLimit")
+                    }
                 },
                 tablePrefix: envVars.entityStorageTablePrefix
             }
@@ -188,7 +205,13 @@ async function configureEntityStorage(coreConfig, envVars) {
                     user: envVars.mongoDbUser ?? "",
                     password: envVars.mongoDbPassword ?? "",
                     database: envVars.mongoDbDatabase ?? "",
-                    mutexTimeoutMs: envMs(envVars, "entityStorageMutexTimeout")
+                    mutexTimeoutMs: envMs(envVars, "entityStorageMutexTimeout"),
+                    pool: {
+                        maxPoolSize: envCount(envVars, "mongoDbPoolMaxPoolSize"),
+                        minPoolSize: envCount(envVars, "mongoDbPoolMinPoolSize"),
+                        maxIdleTimeMs: envMs(envVars, "mongoDbPoolMaxIdleTime"),
+                        waitQueueTimeoutMs: envMs(envVars, "mongoDbPoolWaitQueueTimeout")
+                    }
                 },
                 tablePrefix: envVars.entityStorageTablePrefix
             }
@@ -204,7 +227,13 @@ async function configureEntityStorage(coreConfig, envVars) {
                     user: envVars.postgreSqlUser ?? "",
                     password: envVars.postgreSqlPassword ?? "",
                     database: envVars.postgreSqlDatabase ?? "",
-                    mutexTimeoutMs: envMs(envVars, "entityStorageMutexTimeout")
+                    mutexTimeoutMs: envMs(envVars, "entityStorageMutexTimeout"),
+                    pool: {
+                        max: envCount(envVars, "postgreSqlPoolMax"),
+                        idleTimeout: envSeconds(envVars, "postgreSqlPoolIdleTimeout"),
+                        connectTimeout: envSeconds(envVars, "postgreSqlPoolConnectTimeout"),
+                        maxLifetime: envSeconds(envVars, "postgreSqlPoolMaxLifetime")
+                    }
                 },
                 tablePrefix: envVars.entityStorageTablePrefix
             }
@@ -497,39 +526,71 @@ async function configureEventBus(coreConfig, envVars) {
  */
 async function configureTelemetry(coreConfig, envVars) {
     coreConfig.types.telemetryConnector ??= [];
-    if (envVars.telemetryConnector === TelemetryConnectorType.EntityStorage) {
-        coreConfig.types.telemetryConnector.push({
-            type: TelemetryConnectorType.EntityStorage,
-            options: {
-                config: {
-                    mutexTimeoutMs: envMs(envVars, "telemetryMutexTimeout")
+    const telemetryConnectorTypes = commaSeparatedListToArray(envVars.telemetryConnector);
+    let additionalConnectorCount = 0;
+    for (const telemetryConnector of telemetryConnectorTypes) {
+        if (telemetryConnector === TelemetryConnectorType.Silent) {
+            coreConfig.types.telemetryConnector.push({
+                type: TelemetryConnectorType.Silent
+            });
+            additionalConnectorCount++;
+        }
+        else if (telemetryConnector === TelemetryConnectorType.EntityStorage) {
+            coreConfig.types.telemetryConnector.push({
+                type: TelemetryConnectorType.EntityStorage,
+                options: {
+                    config: {
+                        batchSize: envCount(envVars, "telemetryBatchSize"),
+                        batchIntervalMs: envSecToMs(envVars, "telemetryBatchFlushInterval"),
+                        maxCacheSize: envCount(envVars, "telemetryMaxCacheSize"),
+                        mutexTimeoutMs: envMs(envVars, "telemetryMutexTimeout"),
+                        metricDefinitionCacheCapacity: envCount(envVars, "telemetryMetricDefinitionCacheCapacity"),
+                        metricDefinitionCacheTtiMs: envMs(envVars, "telemetryMetricDefinitionCacheTtiMs")
+                    }
                 }
+            });
+            additionalConnectorCount++;
+        }
+        else if (telemetryConnector === TelemetryConnectorType.OpenTelemetry) {
+            let readers;
+            if (envVars.openTelemetryReader === OpenTelemetryReaderTypes.Prometheus) {
+                readers = {
+                    prometheus: {
+                        type: OpenTelemetryReaderTypes.Prometheus,
+                        port: envCount(envVars, "openTelemetryPrometheusPort")
+                    }
+                };
             }
-        });
-    }
-    else if (envVars.telemetryConnector === TelemetryConnectorType.OpenTelemetry) {
-        let readers;
-        if (envVars.openTelemetryReader === OpenTelemetryReaderTypes.Prometheus) {
-            readers = {
-                prometheus: {
-                    type: OpenTelemetryReaderTypes.Prometheus,
-                    port: envCount(envVars, "openTelemetryPrometheusPort")
+            coreConfig.types.telemetryConnector.push({
+                type: TelemetryConnectorType.OpenTelemetry,
+                options: {
+                    config: {
+                        meterName: envVars.openTelemetryMeterName,
+                        meterVersion: envVars.openTelemetryMeterVersion,
+                        readers
+                    }
                 }
-            };
+            });
+            additionalConnectorCount++;
         }
+    }
+    // If more than one telemetry connector, then we need to add a multi connector
+    // and set it as the default one
+    if (additionalConnectorCount > 1) {
         coreConfig.types.telemetryConnector.push({
-            type: TelemetryConnectorType.OpenTelemetry,
+            type: TelemetryConnectorType.Multi,
             options: {
-                config: {
-                    mutexTimeoutMs: envMs(envVars, "telemetryMutexTimeout"),
-                    meterName: envVars.openTelemetryMeterName,
-                    meterVersion: envVars.openTelemetryMeterVersion,
-                    readers
-                }
-            }
+                telemetryConnectorTypes
+            },
+            isDefault: true
         });
     }
-    if (coreConfig.types.telemetryConnector.length > 0) {
+    else if (additionalConnectorCount > 0) {
+        // If only one connector, then we set it as the default one
+        coreConfig.types.telemetryConnector[coreConfig.types.telemetryConnector.length - 1].isDefault =
+            true;
+    }
+    if (additionalConnectorCount > 0) {
         coreConfig.types.telemetryComponent ??= [];
         coreConfig.types.telemetryComponent.push({ type: TelemetryComponentType.Service });
     }
@@ -541,7 +602,7 @@ async function configureTelemetry(coreConfig, envVars) {
  * @returns A promise that resolves when the metrics collector configuration has been applied.
  */
 async function configureMetricsCollector(coreConfig, envVars) {
-    if (isTelemetryRequired(envVars)) {
+    if (isTelemetryEnabled(envVars)) {
         coreConfig.types.metricsCollectorComponent ??= [];
         coreConfig.types.metricsCollectorComponent.push({
             type: MetricsCollectorComponentType.Service,
@@ -550,7 +611,7 @@ async function configureMetricsCollector(coreConfig, envVars) {
                     intervalMs: envSecToMs(envVars, "telemetryMetricsCollectorInterval")
                 }
             },
-            isCloneable: false
+            cloneMode: EngineCloneMode.Never
         });
         coreConfig.types.metricsProducerComponent ??= [];
         const metricsProducers = commaSeparatedListToArray(envVars.telemetryMetricsProducers ??
@@ -559,7 +620,7 @@ async function configureMetricsCollector(coreConfig, envVars) {
             coreConfig.types.metricsProducerComponent.push({
                 type: producerType,
                 options: { maxHistory: envCount(envVars, "telemetryMetricsProducerMaxHistory") },
-                isCloneable: false
+                cloneMode: EngineCloneMode.Never
             });
         }
     }
@@ -572,38 +633,66 @@ async function configureMetricsCollector(coreConfig, envVars) {
  */
 async function configureTracing(coreConfig, envVars) {
     coreConfig.types.tracingConnector ??= [];
-    if (envVars.tracingConnector === TracingConnectorType.EntityStorage) {
-        coreConfig.types.tracingConnector.push({
-            type: TracingConnectorType.EntityStorage,
-            options: {
-                config: {
-                    mutexTimeoutMs: envMs(envVars, "tracingMutexTimeout")
+    const tracingConnectorTypes = commaSeparatedListToArray(envVars.tracingConnector);
+    let additionalConnectorCount = 0;
+    for (const tracingConnector of tracingConnectorTypes) {
+        if (tracingConnector === TracingConnectorType.Silent) {
+            coreConfig.types.tracingConnector.push({
+                type: TracingConnectorType.Silent
+            });
+            additionalConnectorCount++;
+        }
+        else if (tracingConnector === TracingConnectorType.EntityStorage) {
+            coreConfig.types.tracingConnector.push({
+                type: TracingConnectorType.EntityStorage,
+                options: {
+                    config: {
+                        mutexTimeoutMs: envMs(envVars, "tracingMutexTimeout")
+                    }
                 }
+            });
+            additionalConnectorCount++;
+        }
+        else if (tracingConnector === TracingConnectorType.OpenTelemetry) {
+            const otelTracingConfig = {
+                tracerName: envVars.openTelemetryTracingTracerName,
+                tracerVersion: envVars.openTelemetryTracingTracerVersion
+            };
+            if (Is.stringValue(envVars.openTelemetryTracingEndpoint)) {
+                otelTracingConfig.exporters = {
+                    otlp: {
+                        endpoint: envVars.openTelemetryTracingEndpoint,
+                        processor: envVars.openTelemetryTracingProcessor ??
+                            OpenTelemetryProcessorTypes.Batch
+                    }
+                };
             }
-        });
-    }
-    else if (envVars.tracingConnector === TracingConnectorType.OpenTelemetry) {
-        const otelTracingConfig = {
-            tracerName: envVars.openTelemetryTracingTracerName,
-            tracerVersion: envVars.openTelemetryTracingTracerVersion
-        };
-        if (Is.stringValue(envVars.openTelemetryTracingEndpoint)) {
-            otelTracingConfig.exporters = {
-                otlp: {
-                    endpoint: envVars.openTelemetryTracingEndpoint,
-                    processor: envVars.openTelemetryTracingProcessor ??
-                        OpenTelemetryProcessorTypes.Batch
+            coreConfig.types.tracingConnector.push({
+                type: TracingConnectorType.OpenTelemetry,
+                options: {
+                    config: otelTracingConfig
                 }
-            };
+            });
+            additionalConnectorCount++;
         }
+    }
+    // If more than one tracing connector, then we need to add a multi connector
+    // and set it as the default one
+    if (additionalConnectorCount > 1) {
         coreConfig.types.tracingConnector.push({
-            type: TracingConnectorType.OpenTelemetry,
+            type: TracingConnectorType.Multi,
             options: {
-                config: otelTracingConfig
-            }
+                tracingConnectorTypes
+            },
+            isDefault: true
         });
     }
-    if (coreConfig.types.tracingConnector.length > 0) {
+    else if (additionalConnectorCount > 0) {
+        // If only one connector, then we set it as the default one
+        coreConfig.types.tracingConnector[coreConfig.types.tracingConnector.length - 1].isDefault =
+            true;
+    }
+    if (additionalConnectorCount > 0) {
         coreConfig.types.tracingComponent ??= [];
         coreConfig.types.tracingComponent.push({ type: TracingComponentType.Service });
     }
@@ -650,7 +739,7 @@ async function configureHealth(coreConfig, envVars) {
                     initialInterval: envSecToMs(envVars, "healthStartupInterval")
                 }
             },
-            isCloneable: false
+            cloneMode: EngineCloneMode.Never
         });
     }
 }
@@ -662,13 +751,12 @@ async function configureHealth(coreConfig, envVars) {
  */
 async function configureSchemaMigration(coreConfig, envVars) {
     const isSchemaMigrationEnabled = envBoolean(envVars, "schemaMigrationEnabled", true);
-    if (isSchemaMigrationEnabled) {
-        coreConfig.types.schemaVersionMigrationComponent ??= [];
-        coreConfig.types.schemaVersionMigrationComponent.push({
-            type: SchemaVersionMigrationComponentType.Service,
-            isCloneable: false
-        });
-    }
+    coreConfig.types.schemaVersionMigrationComponent ??= [];
+    coreConfig.types.schemaVersionMigrationComponent.push({
+        type: SchemaVersionMigrationComponentType.Service,
+        options: { config: { enabled: isSchemaMigrationEnabled } },
+        cloneMode: EngineCloneMode.Never
+    });
 }
 /**
  * Configures the platform.
@@ -685,7 +773,8 @@ async function configurePlatform(coreConfig, envVars) {
             config: {
                 isMultiTenant: isTenantEnabled
             }
-        }
+        },
+        cloneMode: EngineCloneMode.Always
     });
 }
 /**
@@ -713,12 +802,14 @@ async function configureContextIdHandlers(coreConfig, envVars) {
     coreConfig.types.contextIdHandlerComponent ??= [];
     coreConfig.types.contextIdHandlerComponent.push({
         type: ContextIdHandlerComponentType.Did,
-        features: [CONTEXT_ID_HANDLER_FEATURE_DID]
+        features: [CONTEXT_ID_HANDLER_FEATURE_DID],
+        cloneMode: EngineCloneMode.Always
     });
     if (envBoolean(envVars, "tenantEnabled", false)) {
         coreConfig.types.contextIdHandlerComponent.push({
             type: ContextIdHandlerComponentType.Tenant,
-            features: [CONTEXT_ID_HANDLER_FEATURE_TENANT]
+            features: [CONTEXT_ID_HANDLER_FEATURE_TENANT],
+            cloneMode: EngineCloneMode.Always
         });
     }
 }
@@ -778,16 +869,6 @@ async function configureMessaging(coreConfig, envVars) {
             });
         }
         else if (envVars.messagingPushNotificationConnector === MessagingPushNotificationConnectorType.Aws) {
-            let messagingApps;
-            if (Is.stringValue(envVars.awsMessagingPushNotificationApplications)) {
-                try {
-                    messagingApps = JSON.parse(envVars.awsMessagingPushNotificationApplications);
-                }
-                catch { }
-            }
-            else if (Is.array(envVars.awsMessagingPushNotificationApplications)) {
-                messagingApps = envVars.awsMessagingPushNotificationApplications;
-            }
             coreConfig.types.messagingPushNotificationConnector.push({
                 type: MessagingPushNotificationConnectorType.Aws,
                 options: {
@@ -797,7 +878,7 @@ async function configureMessaging(coreConfig, envVars) {
                         accessKeyId: envVars.awsSesAccessKeyId,
                         secretAccessKey: envVars.awsSesSecretAccessKey,
                         endpoint: envVars.awsSesEndpoint,
-                        applicationsSettings: messagingApps ?? []
+                        applicationsSettings: envArray(envVars, "awsMessagingPushNotificationApplications", [])
                     }
                 }
             });
@@ -973,7 +1054,9 @@ async function configureIdentity(coreConfig, envVars) {
                     walletAddressIndex: envCount(envVars, "identityWalletAddressIndex") ?? 0,
                     didResolutionCacheTtlMs: envMs(envVars, "identityDidResolutionCacheTtl"),
                     didResolutionCacheCapacity: envCount(envVars, "identityDidResolutionCacheCapacity"),
-                    didResolutionCacheMutexTimeoutMs: envMs(envVars, "identityDidResolutionCacheMutexTimeout")
+                    didResolutionCacheMutexTimeoutMs: envMs(envVars, "identityDidResolutionCacheMutexTimeout"),
+                    didResolutionRetries: envCount(envVars, "identityDidResolutionRetries"),
+                    didResolutionRetryDelayMs: envMs(envVars, "identityDidResolutionRetryDelay")
                 }
             }
         });
@@ -1407,7 +1490,9 @@ async function configureDataspace(coreConfig, envVars) {
                     stalledNegotiationTimeoutMs: envSecToMs(envVars, "dataspaceStalledNegotiationTimeout"),
                     stalledTransferTimeoutMs: envSecToMs(envVars, "dataspaceStalledTransferTimeout"),
                     providerTransferIdleTimeoutMs: envSecToMs(envVars, "dataspaceProviderTransferIdleTimeout"),
-                    providerTransferPolicySweepIntervalMs: envSecToMs(envVars, "dataspaceProviderTransferPolicySweepInterval")
+                    providerTransferPolicySweepIntervalMs: envSecToMs(envVars, "dataspaceProviderTransferPolicySweepInterval"),
+                    agreementUnusedThresholdMs: envMs(envVars, "dataspaceAgreementUnusedThreshold"),
+                    agreementSweepIntervalMs: envMs(envVars, "dataspaceAgreementSweepInterval")
                 }
             },
             isDefault: true
@@ -1484,7 +1569,9 @@ export function isTrustRequired(envVars) {
  * @returns True if dataspace or verifiable storage is enabled.
  */
 export function isBackgroundTasksRequired(envVars) {
-    return envBoolean(envVars, "dataspaceEnabled", false) || isImmutableProofRequired(envVars);
+    return (envBoolean(envVars, "dataspaceEnabled", false) ||
+        isImmutableProofRequired(envVars) ||
+        envBoolean(envVars, "healthEnabled", false));
 }
 /**
  * Checks if the immutable proof subsystem is required.
@@ -1544,13 +1631,21 @@ export function isAutomationRequired(envVars) {
     return isRightsManagementRequired(envVars);
 }
 /**
- * Checks if the telemetry subsystem is required.
+ * Checks if the telemetry subsystem is enabled.
  * Returns true when any component that depends on the telemetry subsystem is enabled.
  * @param envVars The environment variables.
  * @returns True if telemetry is enabled.
  */
-export function isTelemetryRequired(envVars) {
-    return (envVars.telemetryConnector === TelemetryConnectorType.EntityStorage ||
-        envVars.telemetryConnector === TelemetryConnectorType.OpenTelemetry);
+export function isTelemetryEnabled(envVars) {
+    return commaSeparatedListToArray(envVars.telemetryConnector).some(t => t === TelemetryConnectorType.EntityStorage || t === TelemetryConnectorType.OpenTelemetry);
+}
+/**
+ * Checks if the tracing subsystem is enabled.
+ * Returns true when any component that depends on the tracing subsystem is enabled.
+ * @param envVars The environment variables.
+ * @returns True if tracing is enabled.
+ */
+export function isTracingEnabled(envVars) {
+    return commaSeparatedListToArray(envVars.tracingConnector).some(t => t === TracingConnectorType.EntityStorage || t === TracingConnectorType.OpenTelemetry);
 }
 //# sourceMappingURL=engineEnvBuilder.js.map