@voxgig/sdkgen 4.32.1 → 4.33.0

package/project/.sdk/tm/js/src/utility/CleanUtility.js

@@ -1,31 +1,304 @@
 
-const {
-  walk, size, pad, slice, clone
-} = require('./StructUtility')
+const { OPTSPEC } = require('../Schema')
 
-// Clean request data by partially hiding sensitive values.
+
+// Everything that leaves the pipeline passes through clean: the error, the
+// explain record, the serialised context, and whatever a feature emits.
+// Two layers: every registered secret VALUE (and its encoded forms) is
+// replaced wherever it appears in a string, and every value under a
+// sensitive KEY name is masked whatever it holds. Inside the pipeline data
+// stays raw, so a hook can still read the header it must add to.
+
+const MAXDEPTH = 32
+const CIRCULAR = '[circular]'
+
+
+function normkey(key) {
+  return String(key).toLowerCase().replace(/[-_]/g, '')
+}
+
+
+function splitkeys(keys) {
+  return String(null == keys ? '' : keys)
+    .split(/\s*,\s*/)
+    .map(normkey)
+    .filter((k) => '' !== k)
+}
+
+
+// The comma-separated literal values a caller registers; a list is taken
+// as-is for a caller that has one.
+function splitvalues(values) {
+  if (Array.isArray(values)) {
+    return values.filter((v) => 'string' === typeof v)
+  }
+  return String(null == values ? '' : values)
+    .split(/\s*,\s*/)
+    .filter((v) => '' !== v)
+}
+
+
+// The spec carries numbers as strings, so every target reads it alike.
+function count(val, dflt) {
+  const n = Math.floor(Number(val))
+  return Number.isFinite(n) && 0 <= n ? n : dflt
+}
+
+
+// The derived block makeOptions builds; a context without options (makeError
+// is reached with a bare one) falls back to the schema defaults, so nothing
+// leaves raw for want of a constructor.
+function cleanConfig(ctx) {
+  const derived = ctx?.options?.__derived__?.clean
+  if (null != derived) {
+    return derived
+  }
+  return makeCleanConfig(OPTSPEC?.clean)
+}
+
+
+function makeCleanConfig(cleanopts) {
+  const opts = cleanopts || {}
+  return {
+    active: false !== opts.active,
+    keys: splitkeys(opts.keys),
+    values: [],
+    mask: 'string' === typeof opts.mask ? opts.mask : '[redacted]',
+    hint: count(opts.hint, 0),
+    min: Math.max(1, count(opts.min, 4)),
+  }
+}
+
+
+// The encoded forms a value travels in: Basic and Bearer both carry base64,
+// a query credential is percent-encoded, and a JSON dump escapes it.
+function forms(value) {
+  const out = [value]
+  const add = (s) => { if ('' !== s && !out.includes(s)) out.push(s) }
+  try { add(Buffer.from(value, 'utf8').toString('base64')) } catch (_e) { }
+  try { add(encodeURIComponent(value)) } catch (_e) { }
+  try { add(JSON.stringify(value).slice(1, -1)) } catch (_e) { }
+  return out
+}
+
+
+// Register a secret value. Idempotent; shorter than `min` is not a secret
+// the SDK can mask without blanking ordinary text.
+function cleanAdd(ctx, value) {
+  const cfg = cleanConfig(ctx)
+  if ('string' !== typeof value || value.length < cfg.min) {
+    return
+  }
+  let changed = false
+  for (const form of forms(value)) {
+    if (form.length >= cfg.min && !cfg.values.includes(form)) {
+      cfg.values.push(form)
+      changed = true
+    }
+  }
+  if (changed) {
+    cfg.values.sort((a, b) => b.length - a.length)
+  }
+}
+
+
+function maskValue(cfg, value) {
+  if (0 < cfg.hint && value.length > 2 * cfg.hint) {
+    return cfg.mask + value.slice(-cfg.hint)
+  }
+  return cfg.mask
+}
+
+
+function cleanString(cfg, text) {
+  let out = text
+  for (const value of cfg.values) {
+    if (out.includes(value)) {
+      out = out.split(value).join(maskValue(cfg, value))
+    }
+  }
+  return out
+}
+
+
+function sensitiveKey(cfg, key) {
+  if (null == key || 'number' === typeof key) {
+    return false
+  }
+  const nk = normkey(key)
+  for (const k of cfg.keys) {
+    if (nk.includes(k)) {
+      return true
+    }
+  }
+  return false
+}
+
+
+// A plain-data copy of what is about to leave: toJSON is honoured (an entity
+// serialises as its data, a context as its record), functions are dropped,
+// cycles are cut, and no live object is shared with the copy - masking the
+// copy must never mask the pipeline's own spec.
+function snapshot(cfg, val, key, depth, seen) {
+  if (null == val) {
+    return val
+  }
+
+  const t = typeof val
+
+  if ('string' === t) {
+    return sensitiveKey(cfg, key) ? maskValue(cfg, val) : cleanString(cfg, val)
+  }
+
+  if ('function' === t) {
+    return undefined
+  }
+
+  if ('object' !== t) {
+    return sensitiveKey(cfg, key) ? cfg.mask : val
+  }
+
+  if (MAXDEPTH <= depth || seen.includes(val)) {
+    return CIRCULAR
+  }
+
+  if (sensitiveKey(cfg, key)) {
+    return cfg.mask
+  }
+
+  seen.push(val)
+  try {
+    if (Array.isArray(val)) {
+      const out = []
+      for (let i = 0; i < val.length; i++) {
+        out.push(snapshot(cfg, val[i], i, depth + 1, seen))
+      }
+      return out
+    }
+
+    if (val instanceof Error) {
+      const out = {
+        message: cleanString(cfg, String(val.message)),
+        stack: cleanString(cfg, String(val.stack || '')),
+      }
+      for (const k of Object.keys(val)) {
+        const v = snapshot(cfg, val[k], k, depth + 1, seen)
+        if (undefined !== v) out[k] = v
+      }
+      return out
+    }
+
+    if ('function' === typeof val.toJSON) {
+      const json = val.toJSON()
+      return json === val ? plain(cfg, val, depth, seen) : snapshot(cfg, json, key, depth + 1, seen)
+    }
+
+    return plain(cfg, val, depth, seen)
+  }
+  finally {
+    seen.pop()
+  }
+}
+
+
+function plain(cfg, val, depth, seen) {
+  const out = {}
+  for (const k of Object.keys(val)) {
+    const v = snapshot(cfg, val[k], k, depth + 1, seen)
+    if (undefined !== v) out[cleanName(cfg, out, k)] = v
+  }
+  return out
+}
+
+
+// A registered value used as a property name is masked like any other
+// string; names that mask alike take a counter, so none is lost.
+function cleanName(cfg, out, key) {
+  const name = cleanString(cfg, key)
+  if (name === key || !Object.prototype.hasOwnProperty.call(out, name)) {
+    return name
+  }
+  let i = 1
+  while (Object.prototype.hasOwnProperty.call(out, name + '#' + i)) i++
+  return name + '#' + i
+}
+
+
+// Clean a value on its way out. A string is redacted; an Error is redacted
+// IN PLACE (it is about to be thrown, and its identity matters to the
+// caller); anything else comes back as a masked plain-data copy.
 function clean(ctx, val) {
-  const options = ctx.options
-
-  const cleankeyre = options?.__derived__?.clean?.keyre
-  const hintsize = 4
-
-  /*
-  if (null != cleankeyre) {
-    val = walk(clone(val), (key, subval) => {
-      if (cleankeyre.exec(key) && 'string' === typeof subval) {
-        const len = size(subval)
-        const hint = (hintsize * 4) < len ? slice(subval, 0, hintsize) : ''
-        subval = pad(hint, len, '*')
+  const cfg = cleanConfig(ctx)
+
+  if (false === cfg.active) {
+    return val
+  }
+
+  if ('string' === typeof val) {
+    return cleanString(cfg, val)
+  }
+
+  if (val instanceof Error) {
+    val.message = cleanString(cfg, String(val.message))
+    if ('string' === typeof val.stack) {
+      val.stack = cleanString(cfg, val.stack)
+    }
+    for (const k of Object.keys(val)) {
+      let v = val[k]
+      if ('string' === typeof v) {
+        v = sensitiveKey(cfg, k) ? maskValue(cfg, v) : cleanString(cfg, v)
+      }
+      else if (null != v && 'object' === typeof v) {
+        v = snapshot(cfg, v, k, 1, [])
       }
-      return subval
-    })
+      const name = cleanString(cfg, k)
+      if (name !== k) {
+        delete val[k]
+      }
+      val[name === k ? k : cleanName(cfg, val, k)] = v
+    }
+    return val
   }
-  */
 
-  return val
+  return snapshot(cfg, val, undefined, 0, [])
+}
+
+
+// Is this key name sensitive under the context's clean configuration?
+function cleanKey(ctx, key) {
+  return sensitiveKey(cleanConfig(ctx), key)
+}
+
+
+// Every scalar under a sensitive name, at any depth and of any shape: a
+// credential mistyped as an object or a number is still a credential, and
+// the validation error that rejects it quotes it.
+function cleanAddSensitive(ctx, val, under = false, depth = 0, seen = [], parent) {
+  if (null == val || MAXDEPTH <= depth) {
+    return
+  }
+  const t = typeof val
+  if ('string' === t || 'number' === t || 'bigint' === t) {
+    if (under) cleanAdd(ctx, String(val))
+    return
+  }
+  if ('object' !== t || seen.includes(val)) {
+    return
+  }
+  seen.push(val)
+  // The keys under `feature` name features, not fields: `secrets` is no secret.
+  const named = !(1 === depth && 'feature' === parent)
+  for (const k of Object.keys(val)) {
+    cleanAddSensitive(ctx, val[k], under || (named && cleanKey(ctx, k)), depth + 1, seen, k)
+  }
 }
 
+
 module.exports = {
-  clean
+  clean,
+  cleanAdd,
+  cleanAddSensitive,
+  cleanKey,
+  makeCleanConfig,
+  splitvalues,
 }

package/project/.sdk/tm/js/src/utility/DoneUtility.js

@@ -3,12 +3,8 @@ const { clean } = require('./CleanUtility')
 
 function done(ctx) {
   const error = ctx.utility.makeError
-  const delprop = ctx.utility.struct.delprop
 
-  if (ctx.ctrl.explain) {
-    ctx.ctrl.explain = clean(ctx, ctx.ctrl.explain)
-    delprop(ctx.ctrl.explain.result, 'err')
-  }
+  cleanExplain(ctx)
 
   if (ctx.result && ctx.result.ok) {
     return ctx.result.resdata
@@ -17,6 +13,22 @@ function done(ctx) {
   return error(ctx)
 }
 
+// In place: the caller may hold the record, and a stream copies only its ctrl.
+function cleanExplain(ctx) {
+  const explain = ctx.ctrl.explain
+  if (null == explain || 'object' !== typeof explain) {
+    return
+  }
+  const cleaned = clean(ctx, explain)
+  if (cleaned !== explain) {
+    for (const k of Object.keys(explain)) delete explain[k]
+    Object.assign(explain, cleaned)
+  }
+  // With clean off, explain.result is the live result makeError reads.
+  if (null != explain.result) explain.result = ctx.utility.struct.delprop({ ...explain.result }, 'err')
+}
+
 module.exports = {
+  cleanExplain,
   done
 }

package/project/.sdk/tm/js/src/utility/MakeErrorUtility.js

@@ -18,9 +18,25 @@ function makeError(ctx, err) {
   err = undefined === err ? reserr : err
   err = err || ctx.error('unknown', 'unknown error')
 
+  // A hook or fetcher may reject with a plain value; clean returns a masked
+  // copy of that rather than changing it, so the copy is what leaves.
+  if (!(err instanceof Error)) {
+    const copy = clean(ctx, err)
+    const text = 'string' === typeof copy ? copy : String(copy?.message ?? 'unknown error')
+    err = Object.assign(new Error(text), 'object' === typeof copy ? copy : {})
+  }
+
   const errmsg = err.message || 'unknown error'
-  const msg = 'ProjectNameSDK: ' + op.name + ': ' + errmsg
-  err.message = clean(ctx, msg)
+  err.message = 'ProjectNameSDK: ' + op.name + ': ' + errmsg
+
+  // The context stays reachable for a debugger, but not for a serialiser:
+  // a transport error carries none, and a pipeline error's is enumerable
+  // only until here.
+  if (null != err.ctx) {
+    Object.defineProperty(err, 'ctx', { value: err.ctx, enumerable: false, writable: true })
+  }
+
+  clean(ctx, err)
 
   if (result.err) {
     delprop(result, 'err')

package/project/.sdk/tm/js/src/utility/MakeOptionsUtility.js

@@ -1,6 +1,8 @@
 
 const { OPTSPEC } = require('../Schema')
 
+const { clean, cleanAdd, cleanAddSensitive, makeCleanConfig, splitvalues } = require('./CleanUtility')
+
 
 function makeOptions(ctx) {
   const utility = ctx.utility
@@ -10,7 +12,6 @@ function makeOptions(ctx) {
   const setprop = struct.setprop
   const merge = struct.merge
   const validate = struct.validate
-  const escre = struct.escre
 
   // `auth: null` is the documented way to disable auth outright, and
   // prepareAuth honours it before it ever reads the apikey. But it cannot
@@ -33,6 +34,21 @@ function makeOptions(ctx) {
     delete opts.auth
   }
 
+  // The secret registry exists BEFORE validation, fed from the raw input, so
+  // the constructor's own rejection of a mistyped credential is clean too.
+  let config = ctx.config || {}
+  let cfgopts = config.options || {}
+
+  // An absent block is left out, or merge would erase the defaults.
+  const layer = (b) => null != b && 'object' === typeof b && !Array.isArray(b) ? b : {}
+  const cleancfg = makeCleanConfig(merge([{}, OPTSPEC.clean,
+    struct.clone(layer(cfgopts.clean)), layer(opts.clean)]))
+  const cleanctx = { options: { __derived__: { clean: cleancfg } } }
+  cleanAddSensitive(cleanctx, settings(opts))
+  for (const raw of [...splitvalues(cfgopts.clean?.values), ...splitvalues(opts.clean?.values)]) {
+    cleanAdd(cleanctx, raw)
+  }
+
   // Feature add-order. `options.feature` may be given as an ordered ARRAY of
   // { name, active, ...opts } entries (the array position IS the order in
   // which features are added), or as a { name: {opts} } map. Normalize an
@@ -57,8 +73,6 @@ function makeOptions(ctx) {
     setprop(utility, key, val)
   }
 
-  let config = ctx.config || {}
-  let cfgopts = config.options || {}
 
   // THE OPTION SPEC IS GENERATED, NOT WRITTEN HERE.
   //
@@ -80,7 +94,12 @@ function makeOptions(ctx) {
   // contaminate every instance constructed after it.
   opts = merge([{}, struct.clone(cfgopts), opts])
 
-  opts = validate(opts, optspec)
+  try {
+    opts = validate(opts, optspec)
+  }
+  catch (err) {
+    throw clean(cleanctx, err)
+  }
 
   // The platform fetch, supplied AFTER validate rather than as a spec
   // default. `system.fetch` is declared `$ANY`, which passes a caller's own
@@ -147,24 +166,29 @@ function makeOptions(ctx) {
   }
 
   opts.__derived__ = {
-    clean: {
-      keyre: undefined
-    },
+    clean: cleancfg,
     featureorder,
   }
 
-  const keyre = opts.clean.keys
-    .split(/\s*,\s*/)
-    .filter((s) => null != s && '' !== s)
-    .map((key) => escre(key)).join('|')
-
-  if ('' != keyre) {
-    opts.__derived__.clean.keyre = keyre
-  }
+  // Again over the merged result: the config's own defaults can carry one.
+  cleanAddSensitive({ options: opts }, settings(opts))
 
   return opts
 }
 
+// Registration skips entity blocks: entity settings and seed records hold no credential.
+function settings(opts) {
+  const noent = (b) => null != b && 'object' === typeof b && !Array.isArray(b)
+    ? { ...b, entity: undefined } : b
+  const feature = Array.isArray(opts.feature) ? opts.feature.map(noent)
+    : null != opts.feature && 'object' === typeof opts.feature
+      ? Object.fromEntries(Object.entries(opts.feature).map(([k, v]) => [k, noent(v)]))
+      : opts.feature
+  return { ...opts, clean: undefined, __derived__: undefined, entity: undefined,
+    test: noent(opts.test), feature }
+}
+
+
 module.exports = {
   makeOptions
 }

package/project/.sdk/tm/js/src/utility/Utility.js

@@ -1,7 +1,7 @@
 
 
-const { clean } = require('./CleanUtility')
-const { done } = require('./DoneUtility')
+const { clean, cleanAdd } = require('./CleanUtility')
+const { cleanExplain, done } = require('./DoneUtility')
 const { makeError } = require('./MakeErrorUtility')
 const { featureAdd } = require('./FeatureAddUtility')
 const { featureHook } = require('./FeatureHookUtility')
@@ -37,6 +37,8 @@ const { StructUtility } = require('./StructUtility')
 class Utility {
 
   clean = clean
+  cleanAdd = cleanAdd
+  cleanExplain = cleanExplain
   done = done
   makeError = makeError
   featureAdd = featureAdd

package/project/.sdk/tm/js/test/feature/harness.js

@@ -80,6 +80,9 @@ function makeClient(spec) {
 
   const utility = {
     struct,
+    // Features pass every record they emit through the SDK's own clean.
+    clean: stdutil.clean,
+    cleanAdd: stdutil.cleanAdd,
     fetcher: server,
     param: (ctx, name) => {
       const p = (ctx.spec && ctx.spec.params) || {}
@@ -88,10 +91,16 @@ function makeClient(spec) {
     },
   }
 
+  // One clean registry per client, as makeOptions gives a real one, so a value
+  // a feature registers with cleanAdd is masked by every clean after it.
+  const derived = stdutil.makeOptions({
+    utility, options: { feature: { test: { active: true } } }, config: {},
+  }).__derived__
+
   const client = {
     _mode: spec.mode || 'test',
     _features: [],
-    _options: { base, headers: spec.headers || {}, feature: {} },
+    _options: { base, headers: spec.headers || {}, feature: {}, __derived__: derived },
     options() { return this._options },
     utility() { return utility },
   }
@@ -111,6 +120,7 @@ function makeClient(spec) {
       id: 'C' + idseq,
       client,
       utility,
+      options: client._options,
       out: {},
       ctrl: over.ctrl || {},
       meta: {},

package/project/.sdk/tm/js/test/feature.test.js

@@ -384,7 +384,7 @@ describe('feature', () => {
       const entries = h.client._debug.entries
       strictEqual(entries.length, 1) // ring buffer capped at max
       strictEqual(seen.length, 2)
-      strictEqual(seen[0].headers.authorization, '<redacted>')
+      strictEqual(seen[0].headers.authorization, '[redacted]')
     })
 
     test('captures failures', { skip: skipWithout('netsim') }, async () => {
@@ -518,6 +518,13 @@ describe('feature', () => {
       await h.op({ op: 'load' })
       strictEqual(rec.calls[0].fetchdef.proxy, undefined)
     })
+
+    test('masks the credentials in the URL it reports', async () => {
+      const h = makeClient({ features: [{ name: 'proxy', options: { url: 'http://pxuser:pxs3cret@proxy:8080' } }] })
+      await h.op({ op: 'load' })
+      strictEqual(h.client._proxy.url.includes('pxs3cret'), false)
+      strictEqual(h.client._proxy.url.includes('pxuser'), false)
+    })
   })
 
 
@@ -763,7 +770,7 @@ describe('feature', () => {
     const h = makeClient({ features: [{ name: 'debug', options: { now: () => 7, redact: ['x-secret'] } }] })
     await h.op({ op: 'load', headers: { 'x-secret': 'hide', 'x-ok': 'show' } })
     const e = h.client._debug.entries[0]
-    strictEqual(e.headers['x-secret'], '<redacted>')
+    strictEqual(e.headers['x-secret'], '[redacted]')
     strictEqual(e.headers['x-ok'], 'show')
   })
 

package/project/.sdk/tm/kotlin/core/Context.kt

@@ -244,4 +244,23 @@ class Context(ctxmap: MutableMap<String, Any?>?, basectx: Context?) {
   fun makeError(code: String, msg: String): SdkError {
     return SdkError(code, msg, this)
   }
+
+  // The serialised context leaves the pipeline (a logger, an error dump), so
+  // it is cleaned; the live fields stay raw for the pipeline's own use.
+  fun toMap(): Any? {
+    val record = linkedMapOf<String, Any?>()
+    record["id"] = this.id
+    record["op"] = this.op
+    record["spec"] = this.spec
+    record["entity"] = this.entity
+    record["result"] = this.result
+    record["response"] = this.response
+    record["meta"] = this.meta
+    val u = this.utility ?: return record
+    return u.clean(this, record)
+  }
+
+  override fun toString(): String {
+    return "Context " + Struct.jsonify(this.toMap())
+  }
 }

package/project/.sdk/tm/kotlin/core/SdkClient.kt

@@ -81,6 +81,11 @@ abstract class SdkClient(sdkopts: MutableMap<String, Any?>?) {
     this.util.featureHook(this.rootctx, "PostConstruct")
   }
 
+  // The options hold the credential; a printed client shows only its name.
+  override fun toString(): String {
+    return this.javaClass.simpleName + "(mode=" + this.mode + ")"
+  }
+
   fun optionsMap(): MutableMap<String, Any?> {
     val out = Struct.clone(this.options)
     if (out is MutableMap<*, *>) {
@@ -262,12 +267,14 @@ abstract class SdkClient(sdkopts: MutableMap<String, Any?>?) {
 
     val out = linkedMapOf<String, Any?>()
 
+    // A raw request returns its failure rather than passing it through
+    // makeError, so it is cleaned here.
     val fetchdef: MutableMap<String, Any?>
     try {
       fetchdef = this.prepare(fetchargs)
     } catch (err: RuntimeException) {
       out["ok"] = false
-      out["err"] = err
+      out["err"] = utility.clean(ctx, err)
       return out
     }
 
@@ -277,7 +284,7 @@ abstract class SdkClient(sdkopts: MutableMap<String, Any?>?) {
       fetched = utility.fetcher(ctx, if (url is String) url else "", fetchdef)
     } catch (err: RuntimeException) {
       out["ok"] = false
-      out["err"] = err
+      out["err"] = utility.clean(ctx, err)
       return out
     }
 

package/project/.sdk/tm/kotlin/core/Utility.kt

@@ -14,6 +14,7 @@ typealias FetcherFn = (Context, String, MutableMap<String, Any?>) -> Any?
 class Utility private constructor(register: Boolean) {
 
   lateinit var clean: (Context, Any?) -> Any?
+  lateinit var cleanAdd: (Context, Any?) -> Unit
   lateinit var done: (Context) -> Any?
   lateinit var makeError: (Context, RuntimeException?) -> Any?
   lateinit var featureAdd: (Context, Feature) -> Unit
@@ -59,6 +60,7 @@ class Utility private constructor(register: Boolean) {
     val u = Utility(false)
 
     u.clean = this.clean
+    u.cleanAdd = this.cleanAdd
     u.done = this.done
     u.makeError = this.makeError
     u.featureAdd = this.featureAdd

package/project/.sdk/tm/kotlin/feature/AuditFeature.kt

@@ -3,6 +3,7 @@ package KOTLINPACKAGE.feature
 import java.util.function.Consumer
 
 import KOTLINPACKAGE.core.Context
+import KOTLINPACKAGE.core.Helpers
 import KOTLINPACKAGE.core.SdkClient
 
 // Audit trail. Emits a structured record for every operation — who (actor),
@@ -65,18 +66,19 @@ class AuditFeature : BaseFeature("audit", "0.0.1", true) {
     val entity = ctx.op.entity
     val opname = ctx.op.name
 
-    val record = linkedMapOf<String, Any?>()
-    record["seq"] = this.seq
-    record["ts"] = FeatureOptions.foptNow(this.options).getAsLong()
-    record["actor"] = actor
-    record["entity"] = entity
-    record["op"] = opname
-    record["outcome"] = outcome
-    record["correlationId"] = ctx.id
+    val raw = linkedMapOf<String, Any?>()
+    raw["seq"] = this.seq
+    raw["ts"] = FeatureOptions.foptNow(this.options).getAsLong()
+    raw["actor"] = actor
+    raw["entity"] = entity
+    raw["op"] = opname
+    raw["outcome"] = outcome
+    raw["correlationId"] = ctx.id
     val result = ctx.result
     if (result != null) {
-      record["status"] = result.status
+      raw["status"] = result.status
     }
+    val record = Helpers.toMapAny(ctx.utility?.clean?.invoke(ctx, raw)) ?: raw
 
     this.records.add(record)
     val max = FeatureOptions.foptInt(this.options, "max", 1000)