@conduitplatform/module-tools 0.17.0-alpha.6 → 0.17.0-alpha.8

package/README.mdx

@@ -6,6 +6,19 @@ This library provides a framework to build micro-services in NodeJS that can be
 
 - run npm build
 
+## Logging
+
+Modules log to stdout. When `LOKI_URL` is set, the same logs are also shipped to Loki. File logging is on unless `LOG_TO_FILE=false`.
+
+| Variable             | Default                 | Description                                                                                                                                     |
+| -------------------- | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| `LOG_TO_FILE`        | unset (on)              | Set to `false` to skip file logs. Any other value, or leaving it unset, writes them.                                                            |
+| `LOG_FILE_DIR`       | `{tmpdir}/conduit-logs` | Directory for `combined-YYYY-MM-DD.log` and `errors-YYYY-MM-DD.log`. It is created when missing.                                                |
+| `LOG_RETENTION_DAYS` | `3`                     | Days of log files to keep. Older files are deleted. Non-positive or non-integer values use the default.                                         |
+| `LOG_FILE_MAX_SIZE`  | `20m`                   | Max size of each file before a same-day size roll. Accepts `k`, `m`, or `g` suffixes, or a positive byte count. Invalid values use the default. |
+
+Files roll once per day and are not gzipped. Metadata in file lines is single-line JSON.
+
 ## Features ✔️
 
 - Service registration & discovery

package/dist/index.cjs

@@ -48,6 +48,8 @@ __export(index_exports, {
   RouteBuilder: () => RouteBuilder,
   RoutingManager: () => RoutingManager,
   addServiceToServer: () => addServiceToServer,
+  containsRedactedMarker: () => containsRedactedMarker,
+  convictConfigParser: () => convictConfigParser,
   createParsedRouterRequest: () => createParsedRouterRequest,
   createServer: () => createServer,
   exportModel: () => exportModel,
@@ -57,7 +59,12 @@ __export(index_exports, {
   linearBackoffTimeoutAsync: () => linearBackoffTimeoutAsync,
   merge: () => merge,
   readConduitPeersManifest: () => readConduitPeersManifest,
+  reconcileStoredModuleConfig: () => reconcileStoredModuleConfig,
+  redactSensitiveConfig: () => redactSensitiveConfig,
+  restoreRedactedSecrets: () => restoreRedactedSecrets,
   sanitizeDocumentsForExport: () => sanitizeDocumentsForExport,
+  stableConfigJson: () => stableConfigJson,
+  storedConfigsEquivalent: () => storedConfigsEquivalent,
   stripInternalFields: () => stripInternalFields,
   wrapFunctionsAsync: () => wrapFunctionsAsync,
   wrapGrpcFunctions: () => wrapGrpcFunctions,
@@ -284,11 +291,19 @@ function wrapGrpcFunctions(functions, postponeRestart, noAuth = false) {
       try {
         invoked = functions[name](call, callback);
       } catch (error) {
-        return throwError(callback, error.message);
+        return throwError(
+          callback,
+          error.message,
+          error instanceof import_grpc_sdk3.GrpcError ? error.code : import_grpc_js2.status.INTERNAL
+        );
       }
       if (typeof invoked?.then === "function") {
         invoked.then().catch((error) => {
-          return throwError(callback, error.message);
+          return throwError(
+            callback,
+            error.message,
+            error instanceof import_grpc_sdk3.GrpcError ? error.code : import_grpc_js2.status.INTERNAL
+          );
         });
       }
     };
@@ -1268,9 +1283,84 @@ async function setupLoki(module2, instance) {
   }
 }
 
-// src/utilities/initializeSdk.ts
-var import_winston2 = __toESM(require("winston"), 1);
+// src/logging/fileTransports.ts
+var import_fs = __toESM(require("fs"), 1);
+var import_os = __toESM(require("os"), 1);
 var import_path2 = __toESM(require("path"), 1);
+var import_winston2 = __toESM(require("winston"), 1);
+var import_winston_daily_rotate_file = __toESM(require("winston-daily-rotate-file"), 1);
+var DEFAULT_RETENTION_DAYS = 3;
+var DEFAULT_MAX_SIZE = "20m";
+var SIZE_STRING = /^(?:0\.)?\d+[kmg]$/i;
+function positiveIntFromEnv(raw, fallback) {
+  if (raw === void 0) return fallback;
+  const trimmed = raw.trim();
+  if (!/^[0-9]+$/.test(trimmed)) return fallback;
+  const value = Number(trimmed);
+  if (!Number.isSafeInteger(value) || value <= 0) return fallback;
+  return value;
+}
+function logFileMaxSize() {
+  const raw = process.env.LOG_FILE_MAX_SIZE?.trim();
+  if (!raw) return DEFAULT_MAX_SIZE;
+  if (SIZE_STRING.test(raw)) return raw.toLowerCase();
+  if (/^[0-9]+$/.test(raw)) {
+    const bytes = Number(raw);
+    if (Number.isSafeInteger(bytes) && bytes > 0) return bytes;
+  }
+  return DEFAULT_MAX_SIZE;
+}
+function formatMeta(meta) {
+  if (Array.isArray(meta)) {
+    if (meta.length === 0) return "";
+    return meta.map((item) => JSON.stringify(item)).join(" ");
+  }
+  if (meta !== null && typeof meta === "object") {
+    if (Object.keys(meta).length === 0) return "";
+    return JSON.stringify(meta);
+  }
+  return "";
+}
+function createFileFormat() {
+  return import_winston2.default.format.combine(
+    import_winston2.default.format.errors({ stack: true }),
+    import_winston2.default.format.timestamp({
+      format: "YYYY-MM-DD HH:mm:ss"
+    }),
+    import_winston2.default.format.printf((info) => {
+      const metaText = formatMeta(info.meta);
+      const line = `[${info.timestamp}] [${info.level.toUpperCase()}]: ${info.message}`;
+      if (info.stack) {
+        return `${line} ${info.stack}`;
+      }
+      return metaText ? `${line} ${metaText}` : line;
+    })
+  );
+}
+function rotateTransport(logDir, name, level) {
+  return new import_winston_daily_rotate_file.default({
+    dirname: logDir,
+    filename: `${name}-%DATE%.log`,
+    datePattern: "YYYY-MM-DD",
+    level,
+    maxFiles: `${positiveIntFromEnv(process.env.LOG_RETENTION_DAYS, DEFAULT_RETENTION_DAYS)}d`,
+    maxSize: logFileMaxSize(),
+    zippedArchive: false,
+    auditFile: import_path2.default.join(logDir, `.${name}-audit.json`),
+    format: createFileFormat()
+  });
+}
+function createFileLogTransports() {
+  if (process.env.LOG_TO_FILE?.trim() === "false") {
+    return [];
+  }
+  const logDir = process.env.LOG_FILE_DIR?.trim() || import_path2.default.join(import_os.default.tmpdir(), "conduit-logs");
+  import_fs.default.mkdirSync(logDir, { recursive: true });
+  return [
+    rotateTransport(logDir, "combined", "info"),
+    rotateTransport(logDir, "errors", "error")
+  ];
+}
 
 // src/metrics/index.ts
 var import_prom_client = require("prom-client");
@@ -1659,17 +1749,9 @@ function clientMiddleware() {
 var initializeSdk = (coreUrl, moduleName, watchModules = true, serviceHealthStatusGetter = () => import_grpc_sdk13.HealthCheckStatus.SERVING) => {
   let logger;
   try {
-    logger = new ConduitLogger([
-      new import_winston2.default.transports.File({
-        filename: import_path2.default.join(__dirname, ".logs/combined.log"),
-        level: "info"
-      }),
-      new import_winston2.default.transports.File({
-        filename: import_path2.default.join(__dirname, ".logs/errors.log"),
-        level: "error"
-      })
-    ]);
-  } catch (e) {
+    const fileTransports = createFileLogTransports();
+    logger = fileTransports.length > 0 ? new ConduitLogger(fileTransports) : new ConduitLogger();
+  } catch {
     logger = new ConduitLogger();
   }
   const grpcSdk = new import_grpc_sdk13.ConduitGrpcSdk(
@@ -1775,6 +1857,110 @@ var convictConfigParser = (config) => {
   return config;
 };
 
+// src/utilities/redactSensitiveConfig.ts
+var WELL_KNOWN_SECRET_KEYS = /^(apiKey|api_key|password|secret|privateKey|private_key)$/i;
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isSchemaLeaf(value) {
+  return isRecord(value) && ("default" in value || "format" in value || "type" in value) && !isRecord(value._cvtProperties);
+}
+function isSensitiveLeaf(value) {
+  return isSchemaLeaf(value) && value.sensitive === true;
+}
+function unwrapSchema(schema) {
+  if (isRecord(schema) && isRecord(schema._cvtProperties)) {
+    return schema._cvtProperties;
+  }
+  return schema;
+}
+var REDACTED_MARKER = "[REDACTED]";
+function containsRedactedMarker(value) {
+  if (value === REDACTED_MARKER) return true;
+  if (Array.isArray(value)) {
+    return value.some(containsRedactedMarker);
+  }
+  if (!isRecord(value)) return false;
+  return Object.values(value).some(containsRedactedMarker);
+}
+function redactSensitiveConfig(config, schema) {
+  if (!isRecord(config)) return config;
+  const redacted = Array.isArray(config) ? [...config] : { ...config };
+  const node = unwrapSchema(schema);
+  for (const [key, value] of Object.entries(redacted)) {
+    const childSchema = isRecord(node) ? node[key] : void 0;
+    if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) {
+      if (typeof value === "string" && value.length > 0) {
+        redacted[key] = REDACTED_MARKER;
+      }
+      continue;
+    }
+    if (isRecord(value) || Array.isArray(value)) {
+      redacted[key] = redactSensitiveConfig(
+        value,
+        childSchema
+      );
+    }
+  }
+  return redacted;
+}
+function restoreRedactedSecrets(incoming, current, schema) {
+  if (Array.isArray(incoming) && Array.isArray(current)) {
+    return incoming.map(
+      (item, index) => restoreRedactedSecrets(item, current[index], schema)
+    );
+  }
+  if (!isRecord(incoming) || !isRecord(current)) return incoming;
+  const restored = { ...incoming };
+  const node = unwrapSchema(schema);
+  for (const [key, value] of Object.entries(restored)) {
+    const childSchema = isRecord(node) ? node[key] : void 0;
+    const currentValue = current[key];
+    if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) {
+      if (value === REDACTED_MARKER && typeof currentValue === "string" && currentValue.length > 0 && currentValue !== REDACTED_MARKER) {
+        restored[key] = currentValue;
+      }
+      continue;
+    }
+    if ((isRecord(value) || Array.isArray(value)) && (isRecord(currentValue) || Array.isArray(currentValue))) {
+      restored[key] = restoreRedactedSecrets(value, currentValue, childSchema);
+    }
+  }
+  return restored;
+}
+
+// src/utilities/reconcileModuleConfig.ts
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function sortJson(value) {
+  if (Array.isArray(value)) {
+    return value.map(sortJson);
+  }
+  if (!isRecord2(value)) {
+    return value;
+  }
+  return Object.fromEntries(
+    Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, nested]) => [key, sortJson(nested)])
+  );
+}
+function stableConfigJson(value) {
+  return JSON.stringify(sortJson(value));
+}
+function storedConfigsEquivalent(left, right) {
+  return stableConfigJson(left) === stableConfigJson(right);
+}
+async function reconcileStoredModuleConfig(args) {
+  if (storedConfigsEquivalent(args.stored, args.migrated)) {
+    return { config: args.migrated, persisted: false };
+  }
+  if (containsRedactedMarker(args.migrated)) {
+    return { config: args.migrated, persisted: false };
+  }
+  const persisted = await args.configureOverride(args.migrated);
+  return { config: persisted, persisted: true };
+}
+
 // src/routing/RoutingController.ts
 var import_grpc_js5 = require("@grpc/grpc-js");
 var RoutingController = class {
@@ -1827,6 +2013,9 @@ var ManagedModule = class extends ConduitServiceModule {
       throw new Error("Failed to initialize grpcSdk");
     }
   }
+  get configInitialized() {
+    return this._configInitialized;
+  }
   get name() {
     return this._moduleName;
   }
@@ -2018,9 +2207,10 @@ var ManagedModule = class extends ConduitServiceModule {
         });
       }
       let config = JSON.parse(call.request.newConfig);
-      config = merge(this.config.getProperties(), config);
-      config = await this.preConfig(config);
       const previousConfig = this.config.getProperties();
+      config = merge(previousConfig, config);
+      config = restoreRedactedSecrets(config, previousConfig, this.configSchema);
+      config = await this.preConfig(config);
       try {
         this.config.load(config).validate({
           allowed: "warn"
@@ -2096,7 +2286,21 @@ var ManagedModule = class extends ConduitServiceModule {
         this.configOverride
       );
       ConfigController.getInstance();
-      if (config) ConfigController.getInstance().config = config;
+      if (config) {
+        const migrated = await this.preConfig(config);
+        this.config.load(migrated).validate({
+          allowed: "warn"
+        });
+        const persistable = this.config.getProperties();
+        const reconciled = await reconcileStoredModuleConfig({
+          stored: config,
+          migrated: persistable,
+          configureOverride: (next) => this.grpcSdk.config.configure(next, convictConfigParser(configSchema), true)
+        });
+        config = reconciled.config;
+        this.config.load(config);
+        ConfigController.getInstance().config = this.config.getProperties();
+      }
       if (!config || config.active || !config.hasOwnProperty("active"))
         await this.onConfig();
     }
@@ -2141,6 +2345,8 @@ var ModuleError = class extends import_grpc_sdk15.GrpcError {
   RouteBuilder,
   RoutingManager,
   addServiceToServer,
+  containsRedactedMarker,
+  convictConfigParser,
   createParsedRouterRequest,
   createServer,
   exportModel,
@@ -2150,7 +2356,12 @@ var ModuleError = class extends import_grpc_sdk15.GrpcError {
   linearBackoffTimeoutAsync,
   merge,
   readConduitPeersManifest,
+  reconcileStoredModuleConfig,
+  redactSensitiveConfig,
+  restoreRedactedSecrets,
   sanitizeDocumentsForExport,
+  stableConfigJson,
+  storedConfigsEquivalent,
   stripInternalFields,
   wrapFunctionsAsync,
   wrapGrpcFunctions,