@voxgig/sdkgen 4.32.1 → 4.33.0

package/project/.sdk/tm/java/utility/MakeOptions.java

@@ -1,6 +1,7 @@
 package JAVAPACKAGE.utility;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -60,6 +61,38 @@ final class MakeOptions {
       opts.remove("auth");
     }
 
+    Map<String, Object> config = ctx.config;
+    if (config == null) {
+      config = new LinkedHashMap<>();
+    }
+    Map<String, Object> cfgopts = Helpers.toMapAny(config.get("options"));
+    if (cfgopts == null) {
+      cfgopts = new LinkedHashMap<>();
+    }
+
+    // The secret registry exists BEFORE validation, fed from the raw input, so
+    // the constructor's own rejection of a mistyped credential is clean too.
+    // Both clean blocks are cloned for the reason the options merge below is,
+    // and an absent one is left out: merge lets a null replace everything.
+    List<Map<String, Object>> cleanblocks = Arrays.asList(
+        Helpers.toMapAny(cfgopts.get("clean")), Helpers.toMapAny(options.get("clean")));
+    List<Object> cleanmerge = new ArrayList<>();
+    cleanmerge.add(new LinkedHashMap<String, Object>());
+    cleanmerge.add(Struct.clone(Schema.optspec().get("clean")));
+    for (Map<String, Object> block : cleanblocks) {
+      if (block != null) {
+        cleanmerge.add(Struct.clone(block));
+      }
+    }
+    final Map<String, Object> cleancfg =
+        Clean.makeCleanConfig(Helpers.toMapAny(Struct.merge(cleanmerge)));
+    Clean.addSensitiveOptions(cleancfg, without(options, "clean"));
+    for (Map<String, Object> block : cleanblocks) {
+      for (String raw : Clean.splitvalues(block == null ? null : block.get("values"))) {
+        Clean.add(cleancfg, raw);
+      }
+    }
+
     // Feature add-order. options.feature may be given as an ordered LIST of
     // { name, active, ...opts } entries (the list position IS the order in
     // which features are added), or as a { name: {opts} } map. Normalize a
@@ -85,15 +118,6 @@ final class MakeOptions {
       opts.put("feature", fmap);
     }
 
-    Map<String, Object> config = ctx.config;
-    if (config == null) {
-      config = new LinkedHashMap<>();
-    }
-    Map<String, Object> cfgopts = Helpers.toMapAny(config.get("options"));
-    if (cfgopts == null) {
-      cfgopts = new LinkedHashMap<>();
-    }
-
     // THE OPTION SPEC IS GENERATED, NOT WRITTEN HERE.
     //
     // Built from the model: `main.kit.optspec` for the standard options,
@@ -125,7 +149,16 @@ final class MakeOptions {
 
     Map<String, Object> vopts = new LinkedHashMap<>();
     vopts.put("errs", new ArrayList<>());
-    Object validated = Struct.validate(merged, optspec, vopts);
+    Object validated;
+    try {
+      validated = Struct.validate(merged, optspec, vopts);
+    }
+    catch (IllegalArgumentException err) {
+      // The message quotes the offending value; a rewrite is the only way to
+      // clean a JDK exception, so the type is kept and the text replaced.
+      throw new IllegalArgumentException(
+          (String) Clean.cleanWith(cleancfg, String.valueOf(err.getMessage())), err.getCause());
+    }
     opts = (Map<String, Object>) validated;
 
     // Restore the suppression the optspec default would otherwise erase.
@@ -198,22 +231,6 @@ final class MakeOptions {
       opts.put("base", resolved.toString());
     }
 
-    // Derived clean config.
-    String cleanKeys = "key,token,id";
-    Object ck = Struct.getpath(opts, List.of("clean", "keys"));
-    if (ck instanceof String) {
-      cleanKeys = (String) ck;
-    }
-
-    List<String> filtered = new ArrayList<>();
-    for (String p : cleanKeys.split(",")) {
-      p = p.trim();
-      if (!"".equals(p)) {
-        filtered.add(Struct.escre(p));
-      }
-    }
-    String keyre = String.join("|", filtered);
-
     // Resolve the feature add-order: an explicit list order (above) wins;
     // otherwise order the map test-first, then the remaining names sorted, so
     // the outcome is deterministic and `test` is always the base transport.
@@ -249,14 +266,21 @@ final class MakeOptions {
     }
 
     Map<String, Object> derived = new LinkedHashMap<>();
-    Map<String, Object> derivedClean = new LinkedHashMap<>();
-    if (!"".equals(keyre)) {
-      derivedClean.put("keyre", keyre);
-    }
-    derived.put("clean", derivedClean);
+    derived.put("clean", cleancfg);
     derived.put("featureorder", featureorder);
     opts.put("__derived__", derived);
 
+    // Again over the merged result: the config's own defaults can carry one.
+    Clean.addSensitiveOptions(cleancfg, without(opts, "clean", "__derived__"));
+
     return opts;
   }
+
+  private static Map<String, Object> without(Map<String, Object> map, String... keys) {
+    Map<String, Object> out = new LinkedHashMap<>(map);
+    for (String key : keys) {
+      out.remove(key);
+    }
+    return out;
+  }
 }

package/project/.sdk/tm/java/utility/Register.java

@@ -9,6 +9,7 @@ public final class Register {
 
   public static void registerAll(Utility u) {
     u.clean = Clean::clean;
+    u.cleanAdd = Clean::cleanAdd;
     u.done = Done::done;
     u.makeError = MakeError::makeError;
     u.featureAdd = FeatureAdd::featureAdd;
@@ -74,6 +75,9 @@ public final class Register {
       case "clean":
         if (val instanceof Utility.CleanFn) { u.clean = (Utility.CleanFn) val; return true; }
         return false;
+      case "cleanAdd":
+        if (val instanceof Utility.CleanAddFn) { u.cleanAdd = (Utility.CleanAddFn) val; return true; }
+        return false;
       case "makeError":
         if (val instanceof Utility.MakeErrorFn) { u.makeError = (Utility.MakeErrorFn) val; return true; }
         return false;

package/project/.sdk/tm/js/src/Context.js

@@ -119,8 +119,10 @@ class Context {
   }
 
 
+  // The serialised context leaves the pipeline (a logger, an error dump), so
+  // it is cleaned; the live fields stay raw for the pipeline's own use.
   toJSON() {
-    return {
+    const record = {
       id: this.id,
       op: this.op,
       spec: this.spec,
@@ -129,6 +131,8 @@ class Context {
       response: this.response,
       meta: this.meta,
     }
+    const clean = this.utility?.clean
+    return 'function' === typeof clean ? clean(this, record) : record
   }
 
   toString() {

package/project/.sdk/tm/js/src/feature/audit/AuditFeature.js

@@ -55,7 +55,7 @@ class AuditFeature extends BaseFeature {
     }
     this._seen.add(ctx)
     this._seq++
-    const record = {
+    const record = ctx.utility.clean(ctx, {
       seq: this._seq,
       ts: this._now(),
       actor: (ctx.ctrl && ctx.ctrl.actor) || this._options.actor || 'anonymous',
@@ -64,7 +64,7 @@ class AuditFeature extends BaseFeature {
       outcome,
       status: ctx.result && ctx.result.status,
       correlationId: ctx.id,
-    }
+    })
 
     const client = this._client
     const recs = client._audit.records

package/project/.sdk/tm/js/src/feature/cost/CostFeature.js

@@ -258,7 +258,7 @@ class CostFeature extends BaseFeature {
     this._bump(cost.actors, actor, amount)
 
     this._seq++
-    const record = {
+    const record = ctx.utility.clean(ctx, {
       seq: this._seq,
       entity,
       op: opname,
@@ -267,7 +267,7 @@ class CostFeature extends BaseFeature {
       currency: cost.currency,
       source,
       attempts: pending.attempts,
-    }
+    })
     cost.last = record
 
     const sink = this._options.sink

package/project/.sdk/tm/js/src/feature/debug/DebugFeature.js

@@ -38,7 +38,7 @@ class DebugFeature extends BaseFeature {
       op: ((ctx.op && ctx.op.entity) || '_') + '.' + ((ctx.op && ctx.op.name) || '_'),
       method: spec.method,
       url: spec.url || spec.path,
-      headers: this._redact(spec.headers),
+      headers: this._redact(ctx, spec.headers),
       start: this._now(),
       status: undefined,
       ok: undefined,
@@ -77,7 +77,7 @@ class DebugFeature extends BaseFeature {
 
 
   _finish(ctx, ok) {
-    const entry = this._entries.get(ctx)
+    let entry = this._entries.get(ctx)
     if (null == entry) {
       return
     }
@@ -88,6 +88,11 @@ class DebugFeature extends BaseFeature {
       entry.status = ctx.result.status
     }
 
+    // The whole entry leaves through the buffer and the callback: the url
+    // and the error message can carry a query credential the header mask
+    // above never saw.
+    entry = ctx.utility.clean(ctx, entry)
+
     const client = this._client
     const buf = client._debug.entries
     buf.push(entry)
@@ -103,17 +108,18 @@ class DebugFeature extends BaseFeature {
   }
 
 
-  _redact(headers) {
+  // The core clean rules apply (clean.keys, every registered value); the
+  // feature's own `redact` list ADDS header names on top of them.
+  _redact(ctx, headers) {
     if (null == headers) {
       return {}
     }
-    const patterns = this._options.redact ||
-      ['authorization', 'cookie', 'set-cookie', 'api-key', 'apikey', 'x-api-key', 'idempotency-key']
+    const patterns = (this._options.redact || []).map((n) => String(n).toLowerCase())
     const out = {}
     for (const k of Object.keys(headers)) {
-      out[k] = patterns.indexOf(k.toLowerCase()) >= 0 ? '<redacted>' : headers[k]
+      out[k] = patterns.indexOf(k.toLowerCase()) >= 0 ? '[redacted]' : headers[k]
     }
-    return out
+    return ctx.utility.clean(ctx, out)
   }
 
 

package/project/.sdk/tm/js/src/feature/log/LogFeature.js

@@ -35,9 +35,9 @@ class LogFeature extends BaseFeature {
         let level = this._options.level || 'info'
 
         logger = Pino({ name: 'log', level }, pretty)
-
-        this._logger = logger
       }
+
+      this._logger = logger
     }
   }
 
@@ -83,15 +83,19 @@ class LogFeature extends BaseFeature {
     this._loghook('PreResult', ctx)
   }
 
+  // A log line leaves the pipeline, so it carries the cleaned record: the
+  // spec after auth holds the credential, and a logger serialises whatever
+  // it is handed.
   _loghook(hook, ctx, level) {
     level = level || 'info'
     if (this._logger) {
-      this._logger[level]({
+      const clean = ctx.utility.clean
+      this._logger[level](clean(ctx, {
         hook,
         op: ctx.op,
         spec: ctx.spec,
-        ctx
-      })
+        ctx: 'function' === typeof ctx.toJSON ? ctx.toJSON() : ctx,
+      }))
     }
   }
 }

package/project/.sdk/tm/js/src/feature/proxy/ProxyFeature.js

@@ -20,6 +20,7 @@ class ProxyFeature extends BaseFeature {
   _options = {}
   _url
   _noProxy = []
+  _ctx
 
 
   init(ctx, options) {
@@ -43,6 +44,22 @@ class ProxyFeature extends BaseFeature {
     this._noProxy = ('string' === typeof noProxy ? noProxy.split(/\s*,\s*/) : (noProxy || []))
       .filter((s) => null != s && '' !== s)
 
+    // A proxy URL may carry credentials as userinfo, from the option or the
+    // environment, and neither is under a sensitive key name.
+    this._ctx = ctx
+    if ('string' === typeof this._url) {
+      try {
+        const parsed = new URL(this._url)
+        for (const part of [parsed.username, parsed.password]) {
+          if ('' !== part) {
+            ctx.utility.cleanAdd(ctx, part)
+            try { ctx.utility.cleanAdd(ctx, decodeURIComponent(part)) } catch (_e) { }
+          }
+        }
+      }
+      catch (_e) { }
+    }
+
     const self = this
     const utility = ctx.utility
     const inner = utility.fetcher
@@ -98,7 +115,8 @@ class ProxyFeature extends BaseFeature {
   _track(url) {
     const client = this._client
     if (null == client._proxy) {
-      client._proxy = { routed: 0, url: this._url }
+      const ctx = this._ctx
+      client._proxy = { routed: 0, url: null == ctx ? this._url : ctx.utility.clean(ctx, this._url) }
     }
     client._proxy.routed++
   }

package/project/.sdk/tm/js/src/feature/secrets/SecretsFeature.js

@@ -111,6 +111,7 @@ class SecretsFeature extends BaseFeature {
       (null == xopts ? undefined : xopts.refresh)
 
     if ('string' === typeof explicit && '' !== explicit) {
+      this._register(explicit)
       providers.push({
         kind: 'memory',
         name: 'options',
@@ -163,6 +164,16 @@ class SecretsFeature extends BaseFeature {
   }
 
 
+  // Every value this feature resolves or buys is a secret the SDK handles,
+  // and none arrives under an option key the intake registration saw.
+  _register(value) {
+    const ctx = this._client._rootctx
+    if (null != ctx && null != ctx.utility && 'function' === typeof ctx.utility.cleanAdd) {
+      ctx.utility.cleanAdd(ctx, value)
+    }
+  }
+
+
   PreSpec(_ctx) {
     return this.resolve()
   }
@@ -217,6 +228,7 @@ class SecretsFeature extends BaseFeature {
     }
 
     const found = await this._sekreto.try(this._secretname)
+    this._register(found)
 
     if (null == this._exchange) {
       if (undefined !== found) {
@@ -346,6 +358,8 @@ class SecretsFeature extends BaseFeature {
         "secrets: token exchange returned no '" + x.response + "' field from " + url)
     }
 
+    this._register(token)
+
     return token
   }
 

package/project/.sdk/tm/js/src/feature/telemetry/TelemetryFeature.js

@@ -90,13 +90,15 @@ class TelemetryFeature extends BaseFeature {
     span.durationMs = Math.max(0, span.end - span.start)
     span.ok = ok
 
+    const out = ctx.utility.clean(ctx, span)
+
     const client = this._client
     client._telemetry.active--
-    client._telemetry.spans.push(span)
+    client._telemetry.spans.push(out)
 
     const exporter = this._options.exporter
     if ('function' === typeof exporter) {
-      try { exporter(span) } catch (_e) { }
+      try { exporter(out) } catch (_e) { }
     }
   }