@conduitplatform/module-tools 0.17.0-alpha.3 → 0.17.0-alpha.7

package/dist/index.d.cts

@@ -163,6 +163,7 @@ declare abstract class ManagedModule<T> extends ConduitServiceModule {
     private _pendingConfigUpdates;
     private _processingConfig;
     protected constructor(moduleName: string, peerManifestRoot?: string);
+    protected get configInitialized(): boolean;
     get name(): string;
     get address(): string;
     start(): Promise<void>;
@@ -342,7 +343,7 @@ declare function wrapGrpcFunctions(functions: {
     [name: string]: Function;
 };
 
-declare function createParsedRouterRequest(params: Params, urlParams?: Params, queryParams?: Params, bodyParams?: Params, path?: string, headers?: Headers, context?: Context, cookies?: Cookies): ParsedRouterRequest;
+declare function createParsedRouterRequest(params: Params, urlParams?: Params, queryParams?: Params, bodyParams?: Params, path?: string, headers?: Headers, context?: Context, cookies?: Cookies, rawHeaders?: string[], rawBody?: Buffer): ParsedRouterRequest;
 
 type RouterRequestHandler = (call: ParsedRouterRequest) => Promise<UnparsedRouterResponse>;
 type RouterRequestHandlerCallback = (call: ParsedRouterRequest, callback: (response: UnparsedRouterResponse) => void) => Promise<UnparsedRouterResponse | undefined | void>;
@@ -406,6 +407,8 @@ declare class RouteBuilder {
     bodyParams(params: ConduitModel): RouteBuilder;
     strictParams(strict: boolean): RouteBuilder;
     rateLimit(maxRequests: number, resetInterval: number): RouteBuilder;
+    captureRawBody(capture: boolean): RouteBuilder;
+    captureRawHeaders(capture: boolean): RouteBuilder;
     return(name: string, fields: ConduitReturn): RouteBuilder;
     handler(fn: RequestHandlers): RouteBuilder;
     add(): void;
@@ -482,9 +485,26 @@ interface ImportModelAdapter {
  */
 declare function importModel(adapter: ImportModelAdapter, records: Record<string, unknown>[], options?: SanitizeForExportOptions): Promise<ImportResultEntry>;
 
+declare const convictConfigParser: (config: Indexable) => Indexable;
+
+declare function containsRedactedMarker(value: unknown): boolean;
+declare function redactSensitiveConfig<T>(config: T, schema?: unknown): T;
+declare function restoreRedactedSecrets<T>(incoming: T, current: T, schema?: unknown): T;
+
+declare function stableConfigJson(value: unknown): string;
+declare function storedConfigsEquivalent(left: unknown, right: unknown): boolean;
+declare function reconcileStoredModuleConfig<T>(args: {
+    stored: T;
+    migrated: T;
+    configureOverride: (config: T) => Promise<T>;
+}): Promise<{
+    config: T;
+    persisted: boolean;
+}>;
+
 declare class ModuleError extends GrpcError {
     debugLogInfo?: string;
     constructor(errorDefinition: ModuleErrorDefinition, debugLogInfo?: string);
 }
 
-export { ConduitActiveSchema, ConduitBoolean, ConduitDate, ConduitJson, ConduitNumber, ConduitObjectId, type ConduitPackageJson, type ConduitPeerWatchEntry, type ConduitPeersManifest, ConduitRelation, type ConduitService, ConduitServiceModule, ConduitString, ConfigController, DEFAULT_STRIP_FIELDS, type ExportModelAdapter, type ExportResult, type ExportableResource, GrpcServer, type ImportModelAdapter, type ImportResult, type ImportResultEntry, ManagedModule, ModuleError, ModuleLifecycleStage, type RequestHandlers, RouteBuilder, type RouterRequestHandler, type RouterRequestHandlerCallback, RoutingManager, type SanitizeForExportOptions, type SocketEventHandler, type SocketRequestHandler, type SocketRequestHandlerCallback, type SocketRequestHandlers, addServiceToServer, createParsedRouterRequest, createServer, exportModel, importModel, initializeSdk, linearBackoffTimeout, linearBackoffTimeoutAsync, merge, readConduitPeersManifest, sanitizeDocumentsForExport, stripInternalFields, wrapFunctionsAsync, wrapGrpcFunctions, wrapRouterGrpcFunction };
+export { ConduitActiveSchema, ConduitBoolean, ConduitDate, ConduitJson, ConduitNumber, ConduitObjectId, type ConduitPackageJson, type ConduitPeerWatchEntry, type ConduitPeersManifest, ConduitRelation, type ConduitService, ConduitServiceModule, ConduitString, ConfigController, DEFAULT_STRIP_FIELDS, type ExportModelAdapter, type ExportResult, type ExportableResource, GrpcServer, type ImportModelAdapter, type ImportResult, type ImportResultEntry, ManagedModule, ModuleError, ModuleLifecycleStage, type RequestHandlers, RouteBuilder, type RouterRequestHandler, type RouterRequestHandlerCallback, RoutingManager, type SanitizeForExportOptions, type SocketEventHandler, type SocketRequestHandler, type SocketRequestHandlerCallback, type SocketRequestHandlers, addServiceToServer, containsRedactedMarker, convictConfigParser, createParsedRouterRequest, createServer, exportModel, importModel, initializeSdk, linearBackoffTimeout, linearBackoffTimeoutAsync, merge, readConduitPeersManifest, reconcileStoredModuleConfig, redactSensitiveConfig, restoreRedactedSecrets, sanitizeDocumentsForExport, stableConfigJson, storedConfigsEquivalent, stripInternalFields, wrapFunctionsAsync, wrapGrpcFunctions, wrapRouterGrpcFunction };

package/dist/index.d.ts

@@ -163,6 +163,7 @@ declare abstract class ManagedModule<T> extends ConduitServiceModule {
     private _pendingConfigUpdates;
     private _processingConfig;
     protected constructor(moduleName: string, peerManifestRoot?: string);
+    protected get configInitialized(): boolean;
     get name(): string;
     get address(): string;
     start(): Promise<void>;
@@ -342,7 +343,7 @@ declare function wrapGrpcFunctions(functions: {
     [name: string]: Function;
 };
 
-declare function createParsedRouterRequest(params: Params, urlParams?: Params, queryParams?: Params, bodyParams?: Params, path?: string, headers?: Headers, context?: Context, cookies?: Cookies): ParsedRouterRequest;
+declare function createParsedRouterRequest(params: Params, urlParams?: Params, queryParams?: Params, bodyParams?: Params, path?: string, headers?: Headers, context?: Context, cookies?: Cookies, rawHeaders?: string[], rawBody?: Buffer): ParsedRouterRequest;
 
 type RouterRequestHandler = (call: ParsedRouterRequest) => Promise<UnparsedRouterResponse>;
 type RouterRequestHandlerCallback = (call: ParsedRouterRequest, callback: (response: UnparsedRouterResponse) => void) => Promise<UnparsedRouterResponse | undefined | void>;
@@ -406,6 +407,8 @@ declare class RouteBuilder {
     bodyParams(params: ConduitModel): RouteBuilder;
     strictParams(strict: boolean): RouteBuilder;
     rateLimit(maxRequests: number, resetInterval: number): RouteBuilder;
+    captureRawBody(capture: boolean): RouteBuilder;
+    captureRawHeaders(capture: boolean): RouteBuilder;
     return(name: string, fields: ConduitReturn): RouteBuilder;
     handler(fn: RequestHandlers): RouteBuilder;
     add(): void;
@@ -482,9 +485,26 @@ interface ImportModelAdapter {
  */
 declare function importModel(adapter: ImportModelAdapter, records: Record<string, unknown>[], options?: SanitizeForExportOptions): Promise<ImportResultEntry>;
 
+declare const convictConfigParser: (config: Indexable) => Indexable;
+
+declare function containsRedactedMarker(value: unknown): boolean;
+declare function redactSensitiveConfig<T>(config: T, schema?: unknown): T;
+declare function restoreRedactedSecrets<T>(incoming: T, current: T, schema?: unknown): T;
+
+declare function stableConfigJson(value: unknown): string;
+declare function storedConfigsEquivalent(left: unknown, right: unknown): boolean;
+declare function reconcileStoredModuleConfig<T>(args: {
+    stored: T;
+    migrated: T;
+    configureOverride: (config: T) => Promise<T>;
+}): Promise<{
+    config: T;
+    persisted: boolean;
+}>;
+
 declare class ModuleError extends GrpcError {
     debugLogInfo?: string;
     constructor(errorDefinition: ModuleErrorDefinition, debugLogInfo?: string);
 }
 
-export { ConduitActiveSchema, ConduitBoolean, ConduitDate, ConduitJson, ConduitNumber, ConduitObjectId, type ConduitPackageJson, type ConduitPeerWatchEntry, type ConduitPeersManifest, ConduitRelation, type ConduitService, ConduitServiceModule, ConduitString, ConfigController, DEFAULT_STRIP_FIELDS, type ExportModelAdapter, type ExportResult, type ExportableResource, GrpcServer, type ImportModelAdapter, type ImportResult, type ImportResultEntry, ManagedModule, ModuleError, ModuleLifecycleStage, type RequestHandlers, RouteBuilder, type RouterRequestHandler, type RouterRequestHandlerCallback, RoutingManager, type SanitizeForExportOptions, type SocketEventHandler, type SocketRequestHandler, type SocketRequestHandlerCallback, type SocketRequestHandlers, addServiceToServer, createParsedRouterRequest, createServer, exportModel, importModel, initializeSdk, linearBackoffTimeout, linearBackoffTimeoutAsync, merge, readConduitPeersManifest, sanitizeDocumentsForExport, stripInternalFields, wrapFunctionsAsync, wrapGrpcFunctions, wrapRouterGrpcFunction };
+export { ConduitActiveSchema, ConduitBoolean, ConduitDate, ConduitJson, ConduitNumber, ConduitObjectId, type ConduitPackageJson, type ConduitPeerWatchEntry, type ConduitPeersManifest, ConduitRelation, type ConduitService, ConduitServiceModule, ConduitString, ConfigController, DEFAULT_STRIP_FIELDS, type ExportModelAdapter, type ExportResult, type ExportableResource, GrpcServer, type ImportModelAdapter, type ImportResult, type ImportResultEntry, ManagedModule, ModuleError, ModuleLifecycleStage, type RequestHandlers, RouteBuilder, type RouterRequestHandler, type RouterRequestHandlerCallback, RoutingManager, type SanitizeForExportOptions, type SocketEventHandler, type SocketRequestHandler, type SocketRequestHandlerCallback, type SocketRequestHandlers, addServiceToServer, containsRedactedMarker, convictConfigParser, createParsedRouterRequest, createServer, exportModel, importModel, initializeSdk, linearBackoffTimeout, linearBackoffTimeoutAsync, merge, readConduitPeersManifest, reconcileStoredModuleConfig, redactSensitiveConfig, restoreRedactedSecrets, sanitizeDocumentsForExport, stableConfigJson, storedConfigsEquivalent, stripInternalFields, wrapFunctionsAsync, wrapGrpcFunctions, wrapRouterGrpcFunction };

package/dist/index.esm.js

@@ -199,7 +199,7 @@ function addServiceToServer(server, protoPath, descriptorObject, functions) {
 // src/helpers/wrapGrpcFunctions.ts
 import { createVerifier } from "fast-jwt";
 import { status } from "@grpc/grpc-js";
-import { ConduitGrpcSdk } from "@conduitplatform/grpc-sdk";
+import { ConduitGrpcSdk, GrpcError } from "@conduitplatform/grpc-sdk";
 function wrapGrpcFunctions(functions, postponeRestart, noAuth = false) {
   const grpcKey = process.env.GRPC_KEY;
   const wrappedFunctions = {};
@@ -232,11 +232,19 @@ function wrapGrpcFunctions(functions, postponeRestart, noAuth = false) {
       try {
         invoked = functions[name](call, callback);
       } catch (error) {
-        return throwError(callback, error.message);
+        return throwError(
+          callback,
+          error.message,
+          error instanceof GrpcError ? error.code : status.INTERNAL
+        );
       }
       if (typeof invoked?.then === "function") {
         invoked.then().catch((error) => {
-          return throwError(callback, error.message);
+          return throwError(
+            callback,
+            error.message,
+            error instanceof GrpcError ? error.code : status.INTERNAL
+          );
         });
       }
     };
@@ -250,7 +258,7 @@ function throwError(callback, message, code = status.INTERNAL) {
 }
 
 // src/helpers/GrpcToParsedRouterRequest.ts
-function createParsedRouterRequest(params, urlParams, queryParams, bodyParams, path5, headers, context, cookies) {
+function createParsedRouterRequest(params, urlParams, queryParams, bodyParams, path5, headers, context, cookies, rawHeaders, rawBody) {
   return {
     request: {
       params,
@@ -259,6 +267,8 @@ function createParsedRouterRequest(params, urlParams, queryParams, bodyParams, p
       bodyParams: bodyParams ?? {},
       path: path5 ?? "",
       headers,
+      rawHeaders: rawHeaders ?? [],
+      rawBody: rawBody ?? Buffer.alloc(0),
       context: context ?? {},
       cookies: cookies ?? {}
     }
@@ -469,6 +479,14 @@ var RouteBuilder = class {
     this._options.rateLimit = { maxRequests, resetInterval };
     return this;
   }
+  captureRawBody(capture) {
+    this._options.captureRawBody = capture;
+    return this;
+  }
+  captureRawHeaders(capture) {
+    this._options.captureRawHeaders = capture;
+    return this;
+  }
   return(name, fields) {
     this._returns = new ConduitRouteReturnDefinition(name, fields);
     return this;
@@ -620,6 +638,12 @@ function wrapRouterGrpcFunction(fun, routerType) {
       routerRequest = !call.request.hasOwnProperty("event");
       if (routerRequest) {
         call.request.headers = parseRequestData(call.request.headers);
+        call.request.rawHeaders = call.request.rawHeaders ? JSON.parse(call.request.rawHeaders) : [];
+        if (call.request.rawBody && !Buffer.isBuffer(call.request.rawBody)) {
+          call.request.rawBody = Buffer.from(call.request.rawBody);
+        } else if (!call.request.rawBody) {
+          call.request.rawBody = Buffer.alloc(0);
+        }
       }
     } catch (e) {
       ConduitGrpcSdk3.Metrics?.increment(`${routerType}_grpc_errors_total`);
@@ -1735,6 +1759,110 @@ var convictConfigParser = (config) => {
   return config;
 };
 
+// src/utilities/redactSensitiveConfig.ts
+var WELL_KNOWN_SECRET_KEYS = /^(apiKey|api_key|password|secret|privateKey|private_key)$/i;
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isSchemaLeaf(value) {
+  return isRecord(value) && ("default" in value || "format" in value || "type" in value) && !isRecord(value._cvtProperties);
+}
+function isSensitiveLeaf(value) {
+  return isSchemaLeaf(value) && value.sensitive === true;
+}
+function unwrapSchema(schema) {
+  if (isRecord(schema) && isRecord(schema._cvtProperties)) {
+    return schema._cvtProperties;
+  }
+  return schema;
+}
+var REDACTED_MARKER = "[REDACTED]";
+function containsRedactedMarker(value) {
+  if (value === REDACTED_MARKER) return true;
+  if (Array.isArray(value)) {
+    return value.some(containsRedactedMarker);
+  }
+  if (!isRecord(value)) return false;
+  return Object.values(value).some(containsRedactedMarker);
+}
+function redactSensitiveConfig(config, schema) {
+  if (!isRecord(config)) return config;
+  const redacted = Array.isArray(config) ? [...config] : { ...config };
+  const node = unwrapSchema(schema);
+  for (const [key, value] of Object.entries(redacted)) {
+    const childSchema = isRecord(node) ? node[key] : void 0;
+    if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) {
+      if (typeof value === "string" && value.length > 0) {
+        redacted[key] = REDACTED_MARKER;
+      }
+      continue;
+    }
+    if (isRecord(value) || Array.isArray(value)) {
+      redacted[key] = redactSensitiveConfig(
+        value,
+        childSchema
+      );
+    }
+  }
+  return redacted;
+}
+function restoreRedactedSecrets(incoming, current, schema) {
+  if (Array.isArray(incoming) && Array.isArray(current)) {
+    return incoming.map(
+      (item, index) => restoreRedactedSecrets(item, current[index], schema)
+    );
+  }
+  if (!isRecord(incoming) || !isRecord(current)) return incoming;
+  const restored = { ...incoming };
+  const node = unwrapSchema(schema);
+  for (const [key, value] of Object.entries(restored)) {
+    const childSchema = isRecord(node) ? node[key] : void 0;
+    const currentValue = current[key];
+    if (isSensitiveLeaf(childSchema) || WELL_KNOWN_SECRET_KEYS.test(key)) {
+      if (value === REDACTED_MARKER && typeof currentValue === "string" && currentValue.length > 0 && currentValue !== REDACTED_MARKER) {
+        restored[key] = currentValue;
+      }
+      continue;
+    }
+    if ((isRecord(value) || Array.isArray(value)) && (isRecord(currentValue) || Array.isArray(currentValue))) {
+      restored[key] = restoreRedactedSecrets(value, currentValue, childSchema);
+    }
+  }
+  return restored;
+}
+
+// src/utilities/reconcileModuleConfig.ts
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function sortJson(value) {
+  if (Array.isArray(value)) {
+    return value.map(sortJson);
+  }
+  if (!isRecord2(value)) {
+    return value;
+  }
+  return Object.fromEntries(
+    Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, nested]) => [key, sortJson(nested)])
+  );
+}
+function stableConfigJson(value) {
+  return JSON.stringify(sortJson(value));
+}
+function storedConfigsEquivalent(left, right) {
+  return stableConfigJson(left) === stableConfigJson(right);
+}
+async function reconcileStoredModuleConfig(args) {
+  if (storedConfigsEquivalent(args.stored, args.migrated)) {
+    return { config: args.migrated, persisted: false };
+  }
+  if (containsRedactedMarker(args.migrated)) {
+    return { config: args.migrated, persisted: false };
+  }
+  const persisted = await args.configureOverride(args.migrated);
+  return { config: persisted, persisted: true };
+}
+
 // src/routing/RoutingController.ts
 import { status as status4 } from "@grpc/grpc-js";
 var RoutingController = class {
@@ -1787,6 +1915,9 @@ var ManagedModule = class extends ConduitServiceModule {
       throw new Error("Failed to initialize grpcSdk");
     }
   }
+  get configInitialized() {
+    return this._configInitialized;
+  }
   get name() {
     return this._moduleName;
   }
@@ -1978,9 +2109,10 @@ var ManagedModule = class extends ConduitServiceModule {
         });
       }
       let config = JSON.parse(call.request.newConfig);
-      config = merge(this.config.getProperties(), config);
-      config = await this.preConfig(config);
       const previousConfig = this.config.getProperties();
+      config = merge(previousConfig, config);
+      config = restoreRedactedSecrets(config, previousConfig, this.configSchema);
+      config = await this.preConfig(config);
       try {
         this.config.load(config).validate({
           allowed: "warn"
@@ -2056,7 +2188,21 @@ var ManagedModule = class extends ConduitServiceModule {
         this.configOverride
       );
       ConfigController.getInstance();
-      if (config) ConfigController.getInstance().config = config;
+      if (config) {
+        const migrated = await this.preConfig(config);
+        this.config.load(migrated).validate({
+          allowed: "warn"
+        });
+        const persistable = this.config.getProperties();
+        const reconciled = await reconcileStoredModuleConfig({
+          stored: config,
+          migrated: persistable,
+          configureOverride: (next) => this.grpcSdk.config.configure(next, convictConfigParser(configSchema), true)
+        });
+        config = reconciled.config;
+        this.config.load(config);
+        ConfigController.getInstance().config = this.config.getProperties();
+      }
       if (!config || config.active || !config.hasOwnProperty("active"))
         await this.onConfig();
     }
@@ -2066,8 +2212,8 @@ var ManagedModule = class extends ConduitServiceModule {
 };
 
 // src/classes/ModuleError.ts
-import { GrpcError } from "@conduitplatform/grpc-sdk";
-var ModuleError = class extends GrpcError {
+import { GrpcError as GrpcError2 } from "@conduitplatform/grpc-sdk";
+var ModuleError = class extends GrpcError2 {
   debugLogInfo;
   constructor(errorDefinition, debugLogInfo) {
     const { grpcCode, conduitCode, message } = errorDefinition;
@@ -2100,6 +2246,8 @@ export {
   RouteBuilder,
   RoutingManager,
   addServiceToServer,
+  containsRedactedMarker,
+  convictConfigParser,
   createParsedRouterRequest,
   createServer,
   exportModel,
@@ -2109,7 +2257,12 @@ export {
   linearBackoffTimeoutAsync,
   merge,
   readConduitPeersManifest,
+  reconcileStoredModuleConfig,
+  redactSensitiveConfig,
+  restoreRedactedSecrets,
   sanitizeDocumentsForExport,
+  stableConfigJson,
+  storedConfigsEquivalent,
   stripInternalFields,
   wrapFunctionsAsync,
   wrapGrpcFunctions,