@conduitplatform/module-tools 0.17.0-alpha.6 → 0.17.0-alpha.8

package/dist/index.d.cts

@@ -163,6 +163,7 @@ declare abstract class ManagedModule<T> extends ConduitServiceModule {
     private _pendingConfigUpdates;
     private _processingConfig;
     protected constructor(moduleName: string, peerManifestRoot?: string);
+    protected get configInitialized(): boolean;
     get name(): string;
     get address(): string;
     start(): Promise<void>;
@@ -484,9 +485,26 @@ interface ImportModelAdapter {
  */
 declare function importModel(adapter: ImportModelAdapter, records: Record<string, unknown>[], options?: SanitizeForExportOptions): Promise<ImportResultEntry>;
 
+declare const convictConfigParser: (config: Indexable) => Indexable;
+
+declare function containsRedactedMarker(value: unknown): boolean;
+declare function redactSensitiveConfig<T>(config: T, schema?: unknown): T;
+declare function restoreRedactedSecrets<T>(incoming: T, current: T, schema?: unknown): T;
+
+declare function stableConfigJson(value: unknown): string;
+declare function storedConfigsEquivalent(left: unknown, right: unknown): boolean;
+declare function reconcileStoredModuleConfig<T>(args: {
+    stored: T;
+    migrated: T;
+    configureOverride: (config: T) => Promise<T>;
+}): Promise<{
+    config: T;
+    persisted: boolean;
+}>;
+
 declare class ModuleError extends GrpcError {
     debugLogInfo?: string;
     constructor(errorDefinition: ModuleErrorDefinition, debugLogInfo?: string);
 }
 
-export { ConduitActiveSchema, ConduitBoolean, ConduitDate, ConduitJson, ConduitNumber, ConduitObjectId, type ConduitPackageJson, type ConduitPeerWatchEntry, type ConduitPeersManifest, ConduitRelation, type ConduitService, ConduitServiceModule, ConduitString, ConfigController, DEFAULT_STRIP_FIELDS, type ExportModelAdapter, type ExportResult, type ExportableResource, GrpcServer, type ImportModelAdapter, type ImportResult, type ImportResultEntry, ManagedModule, ModuleError, ModuleLifecycleStage, type RequestHandlers, RouteBuilder, type RouterRequestHandler, type RouterRequestHandlerCallback, RoutingManager, type SanitizeForExportOptions, type SocketEventHandler, type SocketRequestHandler, type SocketRequestHandlerCallback, type SocketRequestHandlers, addServiceToServer, createParsedRouterRequest, createServer, exportModel, importModel, initializeSdk, linearBackoffTimeout, linearBackoffTimeoutAsync, merge, readConduitPeersManifest, sanitizeDocumentsForExport, stripInternalFields, wrapFunctionsAsync, wrapGrpcFunctions, wrapRouterGrpcFunction };
+export { ConduitActiveSchema, ConduitBoolean, ConduitDate, ConduitJson, ConduitNumber, ConduitObjectId, type ConduitPackageJson, type ConduitPeerWatchEntry, type ConduitPeersManifest, ConduitRelation, type ConduitService, ConduitServiceModule, ConduitString, ConfigController, DEFAULT_STRIP_FIELDS, type ExportModelAdapter, type ExportResult, type ExportableResource, GrpcServer, type ImportModelAdapter, type ImportResult, type ImportResultEntry, ManagedModule, ModuleError, ModuleLifecycleStage, type RequestHandlers, RouteBuilder, type RouterRequestHandler, type RouterRequestHandlerCallback, RoutingManager, type SanitizeForExportOptions, type SocketEventHandler, type SocketRequestHandler, type SocketRequestHandlerCallback, type SocketRequestHandlers, addServiceToServer, containsRedactedMarker, convictConfigParser, createParsedRouterRequest, createServer, exportModel, importModel, initializeSdk, linearBackoffTimeout, linearBackoffTimeoutAsync, merge, readConduitPeersManifest, reconcileStoredModuleConfig, redactSensitiveConfig, restoreRedactedSecrets, sanitizeDocumentsForExport, stableConfigJson, storedConfigsEquivalent, stripInternalFields, wrapFunctionsAsync, wrapGrpcFunctions, wrapRouterGrpcFunction };

package/dist/index.d.ts

@@ -163,6 +163,7 @@ declare abstract class ManagedModule<T> extends ConduitServiceModule {
     private _pendingConfigUpdates;
     private _processingConfig;
     protected constructor(moduleName: string, peerManifestRoot?: string);
+    protected get configInitialized(): boolean;
     get name(): string;
     get address(): string;
     start(): Promise<void>;
@@ -484,9 +485,26 @@ interface ImportModelAdapter {
  */
 declare function importModel(adapter: ImportModelAdapter, records: Record<string, unknown>[], options?: SanitizeForExportOptions): Promise<ImportResultEntry>;
 
+declare const convictConfigParser: (config: Indexable) => Indexable;
+
+declare function containsRedactedMarker(value: unknown): boolean;
+declare function redactSensitiveConfig<T>(config: T, schema?: unknown): T;
+declare function restoreRedactedSecrets<T>(incoming: T, current: T, schema?: unknown): T;
+
+declare function stableConfigJson(value: unknown): string;
+declare function storedConfigsEquivalent(left: unknown, right: unknown): boolean;
+declare function reconcileStoredModuleConfig<T>(args: {
+    stored: T;
+    migrated: T;
+    configureOverride: (config: T) => Promise<T>;
+}): Promise<{
+    config: T;
+    persisted: boolean;
+}>;
+
 declare class ModuleError extends GrpcError {
     debugLogInfo?: string;
     constructor(errorDefinition: ModuleErrorDefinition, debugLogInfo?: string);
 }
 
-export { ConduitActiveSchema, ConduitBoolean, ConduitDate, ConduitJson, ConduitNumber, ConduitObjectId, type ConduitPackageJson, type ConduitPeerWatchEntry, type ConduitPeersManifest, ConduitRelation, type ConduitService, ConduitServiceModule, ConduitString, ConfigController, DEFAULT_STRIP_FIELDS, type ExportModelAdapter, type ExportResult, type ExportableResource, GrpcServer, type ImportModelAdapter, type ImportResult, type ImportResultEntry, ManagedModule, ModuleError, ModuleLifecycleStage, type RequestHandlers, RouteBuilder, type RouterRequestHandler, type RouterRequestHandlerCallback, RoutingManager, type SanitizeForExportOptions, type SocketEventHandler, type SocketRequestHandler, type SocketRequestHandlerCallback, type SocketRequestHandlers, addServiceToServer, createParsedRouterRequest, createServer, exportModel, importModel, initializeSdk, linearBackoffTimeout, linearBackoffTimeoutAsync, merge, readConduitPeersManifest, sanitizeDocumentsForExport, stripInternalFields, wrapFunctionsAsync, wrapGrpcFunctions, wrapRouterGrpcFunction };
+export { ConduitActiveSchema, ConduitBoolean, ConduitDate, ConduitJson, ConduitNumber, ConduitObjectId, type ConduitPackageJson, type ConduitPeerWatchEntry, type ConduitPeersManifest, ConduitRelation, type ConduitService, ConduitServiceModule, ConduitString, ConfigController, DEFAULT_STRIP_FIELDS, type ExportModelAdapter, type ExportResult, type ExportableResource, GrpcServer, type ImportModelAdapter, type ImportResult, type ImportResultEntry, ManagedModule, ModuleError, ModuleLifecycleStage, type RequestHandlers, RouteBuilder, type RouterRequestHandler, type RouterRequestHandlerCallback, RoutingManager, type SanitizeForExportOptions, type SocketEventHandler, type SocketRequestHandler, type SocketRequestHandlerCallback, type SocketRequestHandlers, addServiceToServer, containsRedactedMarker, convictConfigParser, createParsedRouterRequest, createServer, exportModel, importModel, initializeSdk, linearBackoffTimeout, linearBackoffTimeoutAsync, merge, readConduitPeersManifest, reconcileStoredModuleConfig, redactSensitiveConfig, restoreRedactedSecrets, sanitizeDocumentsForExport, stableConfigJson, storedConfigsEquivalent, stripInternalFields, wrapFunctionsAsync, wrapGrpcFunctions, wrapRouterGrpcFunction };

package/dist/index.esm.js

@@ -199,7 +199,7 @@ function addServiceToServer(server, protoPath, descriptorObject, functions) {
 // src/helpers/wrapGrpcFunctions.ts
 import { createVerifier } from "fast-jwt";
 import { status } from "@grpc/grpc-js";
-import { ConduitGrpcSdk } from "@conduitplatform/grpc-sdk";
+import { ConduitGrpcSdk, GrpcError } from "@conduitplatform/grpc-sdk";
 function wrapGrpcFunctions(functions, postponeRestart, noAuth = false) {
   const grpcKey = process.env.GRPC_KEY;
   const wrappedFunctions = {};
@@ -232,11 +232,19 @@ function wrapGrpcFunctions(functions, postponeRestart, noAuth = false) {
       try {
         invoked = functions[name](call, callback);
       } catch (error) {
-        return throwError(callback, error.message);
+        return throwError(
+          callback,
+          error.message,
+          error instanceof GrpcError ? error.code : status.INTERNAL
+        );
       }
       if (typeof invoked?.then === "function") {
         invoked.then().catch((error) => {
-          return throwError(callback, error.message);
+          return throwError(
+            callback,
+            error.message,
+            error instanceof GrpcError ? error.code : status.INTERNAL
+          );
         });
       }
     };
@@ -1234,9 +1242,84 @@ async function setupLoki(module, instance) {
   }
 }
 
-// src/utilities/initializeSdk.ts
-import winston2 from "winston";
+// src/logging/fileTransports.ts
+import fs from "fs";
+import os from "os";
 import path3 from "path";
+import winston2 from "winston";
+import DailyRotateFile from "winston-daily-rotate-file";
+var DEFAULT_RETENTION_DAYS = 3;
+var DEFAULT_MAX_SIZE = "20m";
+var SIZE_STRING = /^(?:0\.)?\d+[kmg]$/i;
+function positiveIntFromEnv(raw, fallback) {
+  if (raw === void 0) return fallback;
+  const trimmed = raw.trim();
+  if (!/^[0-9]+$/.test(trimmed)) return fallback;
+  const value = Number(trimmed);
+  if (!Number.isSafeInteger(value) || value <= 0) return fallback;
+  return value;
+}
+function logFileMaxSize() {
+  const raw = process.env.LOG_FILE_MAX_SIZE?.trim();
+  if (!raw) return DEFAULT_MAX_SIZE;
+  if (SIZE_STRING.test(raw)) return raw.toLowerCase();
+  if (/^[0-9]+$/.test(raw)) {
+    const bytes = Number(raw);
+    if (Number.isSafeInteger(bytes) && bytes > 0) return bytes;
+  }
+  return DEFAULT_MAX_SIZE;
+}
+function formatMeta(meta) {
+  if (Array.isArray(meta)) {
+    if (meta.length === 0) return "";
+    return meta.map((item) => JSON.stringify(item)).join(" ");
+  }
+  if (meta !== null && typeof meta === "object") {
+    if (Object.keys(meta).length === 0) return "";
+    return JSON.stringify(meta);
+  }
+  return "";
+}
+function createFileFormat() {
+  return winston2.format.combine(
+    winston2.format.errors({ stack: true }),
+    winston2.format.timestamp({
+      format: "YYYY-MM-DD HH:mm:ss"
+    }),
+    winston2.format.printf((info) => {
+      const metaText = formatMeta(info.meta);
+      const line = `[${info.timestamp}] [${info.level.toUpperCase()}]: ${info.message}`;
+      if (info.stack) {
+        return `${line} ${info.stack}`;
+      }
+      return metaText ? `${line} ${metaText}` : line;
+    })
+  );
+}
+function rotateTransport(logDir, name, level) {
+  return new DailyRotateFile({
+    dirname: logDir,
+    filename: `${name}-%DATE%.log`,
+    datePattern: "YYYY-MM-DD",
+    level,
+    maxFiles: `${positiveIntFromEnv(process.env.LOG_RETENTION_DAYS, DEFAULT_RETENTION_DAYS)}d`,
+    maxSize: logFileMaxSize(),
+    zippedArchive: false,
+    auditFile: path3.join(logDir, `.${name}-audit.json`),
+    format: createFileFormat()
+  });
+}
+function createFileLogTransports() {
+  if (process.env.LOG_TO_FILE?.trim() === "false") {
+    return [];
+  }
+  const logDir = process.env.LOG_FILE_DIR?.trim() || path3.join(os.tmpdir(), "conduit-logs");
+  fs.mkdirSync(logDir, { recursive: true });
+  return [
+    rotateTransport(logDir, "combined", "info"),
+    rotateTransport(logDir, "errors", "error")
+  ];
+}
 
 // src/metrics/index.ts
 import {
@@ -1635,17 +1718,9 @@ function clientMiddleware() {
 var initializeSdk = (coreUrl, moduleName, watchModules = true, serviceHealthStatusGetter = () => HealthCheckStatus2.SERVING) => {
   let logger;
   try {
-    logger = new ConduitLogger([
-      new winston2.transports.File({
-        filename: path3.join(__dirname, ".logs/combined.log"),
-        level: "info"
-      }),
-      new winston2.transports.File({
-        filename: path3.join(__dirname, ".logs/errors.log"),
-        level: "error"
-      })
-    ]);
-  } catch (e) {
+    const fileTransports = createFileLogTransports();
+    logger = fileTransports.length > 0 ? new ConduitLogger(fileTransports) : new ConduitLogger();
+  } catch {
     logger = new ConduitLogger();
   }
   const grpcSdk = new ConduitGrpcSdk7(
@@ -1751,6 +1826,110 @@ var convictConfigParser = (config) => {
   return config;
 };
 
+// src/utilities/redactSensitiveConfig.ts
+var WELL_KNOWN_SECRET_KEYS = /^(apiKey|api_key|password|secret|privateKey|private_key)$/i;
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isSchemaLeaf(value) {
+  return isRecord(value) && ("default" in value || "format" in value || "type" in value) && !isRecord(value._cvtProperties);
+}
+function isSensitiveLeaf(value) {
+  return isSchemaLeaf(value) && value.sensitive === true;
+}
+function unwrapSchema(schema) {
+  if (isRecord(schema) && isRecord(schema._cvtProperties)) {
+    return schema._cvtProperties;
+  }
+  return schema;
+}
+var REDACTED_MARKER = "[REDACTED]";
+function containsRedactedMarker(value) {
+  if (value === REDACTED_MARKER) return true;
+  if (Array.isArray(value)) {
+    return value.some(containsRedactedMarker);
+  }
+  if (!isRecord(value)) return false;
+  return Object.values(value).some(containsRedactedMarker);
+}
+function redactSensitiveConfig(config, schema) {
+  if (!isRecord(config)) return config;
+  const redacted = Array.isArray(config) ? [...config] : { ...config };
+  const node = unwrapSchema(schema);
+  for (const [key, value] of Object.entries(redacted)) {
+    const childSchema = isRecord(node) ? node[key] : void 0;
+    if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) {
+      if (typeof value === "string" && value.length > 0) {
+        redacted[key] = REDACTED_MARKER;
+      }
+      continue;
+    }
+    if (isRecord(value) || Array.isArray(value)) {
+      redacted[key] = redactSensitiveConfig(
+        value,
+        childSchema
+      );
+    }
+  }
+  return redacted;
+}
+function restoreRedactedSecrets(incoming, current, schema) {
+  if (Array.isArray(incoming) && Array.isArray(current)) {
+    return incoming.map(
+      (item, index) => restoreRedactedSecrets(item, current[index], schema)
+    );
+  }
+  if (!isRecord(incoming) || !isRecord(current)) return incoming;
+  const restored = { ...incoming };
+  const node = unwrapSchema(schema);
+  for (const [key, value] of Object.entries(restored)) {
+    const childSchema = isRecord(node) ? node[key] : void 0;
+    const currentValue = current[key];
+    if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) {
+      if (value === REDACTED_MARKER && typeof currentValue === "string" && currentValue.length > 0 && currentValue !== REDACTED_MARKER) {
+        restored[key] = currentValue;
+      }
+      continue;
+    }
+    if ((isRecord(value) || Array.isArray(value)) && (isRecord(currentValue) || Array.isArray(currentValue))) {
+      restored[key] = restoreRedactedSecrets(value, currentValue, childSchema);
+    }
+  }
+  return restored;
+}
+
+// src/utilities/reconcileModuleConfig.ts
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function sortJson(value) {
+  if (Array.isArray(value)) {
+    return value.map(sortJson);
+  }
+  if (!isRecord2(value)) {
+    return value;
+  }
+  return Object.fromEntries(
+    Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, nested]) => [key, sortJson(nested)])
+  );
+}
+function stableConfigJson(value) {
+  return JSON.stringify(sortJson(value));
+}
+function storedConfigsEquivalent(left, right) {
+  return stableConfigJson(left) === stableConfigJson(right);
+}
+async function reconcileStoredModuleConfig(args) {
+  if (storedConfigsEquivalent(args.stored, args.migrated)) {
+    return { config: args.migrated, persisted: false };
+  }
+  if (containsRedactedMarker(args.migrated)) {
+    return { config: args.migrated, persisted: false };
+  }
+  const persisted = await args.configureOverride(args.migrated);
+  return { config: persisted, persisted: true };
+}
+
 // src/routing/RoutingController.ts
 import { status as status4 } from "@grpc/grpc-js";
 var RoutingController = class {
@@ -1803,6 +1982,9 @@ var ManagedModule = class extends ConduitServiceModule {
       throw new Error("Failed to initialize grpcSdk");
     }
   }
+  get configInitialized() {
+    return this._configInitialized;
+  }
   get name() {
     return this._moduleName;
   }
@@ -1994,9 +2176,10 @@ var ManagedModule = class extends ConduitServiceModule {
         });
       }
       let config = JSON.parse(call.request.newConfig);
-      config = merge(this.config.getProperties(), config);
-      config = await this.preConfig(config);
       const previousConfig = this.config.getProperties();
+      config = merge(previousConfig, config);
+      config = restoreRedactedSecrets(config, previousConfig, this.configSchema);
+      config = await this.preConfig(config);
       try {
         this.config.load(config).validate({
           allowed: "warn"
@@ -2072,7 +2255,21 @@ var ManagedModule = class extends ConduitServiceModule {
         this.configOverride
       );
       ConfigController.getInstance();
-      if (config) ConfigController.getInstance().config = config;
+      if (config) {
+        const migrated = await this.preConfig(config);
+        this.config.load(migrated).validate({
+          allowed: "warn"
+        });
+        const persistable = this.config.getProperties();
+        const reconciled = await reconcileStoredModuleConfig({
+          stored: config,
+          migrated: persistable,
+          configureOverride: (next) => this.grpcSdk.config.configure(next, convictConfigParser(configSchema), true)
+        });
+        config = reconciled.config;
+        this.config.load(config);
+        ConfigController.getInstance().config = this.config.getProperties();
+      }
       if (!config || config.active || !config.hasOwnProperty("active"))
         await this.onConfig();
     }
@@ -2082,8 +2279,8 @@ var ManagedModule = class extends ConduitServiceModule {
 };
 
 // src/classes/ModuleError.ts
-import { GrpcError } from "@conduitplatform/grpc-sdk";
-var ModuleError = class extends GrpcError {
+import { GrpcError as GrpcError2 } from "@conduitplatform/grpc-sdk";
+var ModuleError = class extends GrpcError2 {
   debugLogInfo;
   constructor(errorDefinition, debugLogInfo) {
     const { grpcCode, conduitCode, message } = errorDefinition;
@@ -2116,6 +2313,8 @@ export {
   RouteBuilder,
   RoutingManager,
   addServiceToServer,
+  containsRedactedMarker,
+  convictConfigParser,
   createParsedRouterRequest,
   createServer,
   exportModel,
@@ -2125,7 +2324,12 @@ export {
   linearBackoffTimeoutAsync,
   merge,
   readConduitPeersManifest,
+  reconcileStoredModuleConfig,
+  redactSensitiveConfig,
+  restoreRedactedSecrets,
   sanitizeDocumentsForExport,
+  stableConfigJson,
+  storedConfigsEquivalent,
   stripInternalFields,
   wrapFunctionsAsync,
   wrapGrpcFunctions,