@voxgig/sdkgen 4.32.1 → 4.34.0

package/project/.sdk/tm/kotlin/utility/CoreUtil.kt

@@ -1,26 +1,461 @@
 package KOTLINPACKAGE.utility
 
+import java.io.PrintWriter
+import java.io.StringWriter
+import java.nio.charset.StandardCharsets
+import java.util.Base64
+
 import KOTLINPACKAGE.core.Context
+import KOTLINPACKAGE.core.Entity
 import KOTLINPACKAGE.core.Helpers
+import KOTLINPACKAGE.core.Operation
+import KOTLINPACKAGE.core.Response
 import KOTLINPACKAGE.core.Result
+import KOTLINPACKAGE.core.Schema
 import KOTLINPACKAGE.core.SdkError
 import KOTLINPACKAGE.core.Spec
+import KOTLINPACKAGE.utility.struct.Struct
 
 fun makeContext(ctxmap: MutableMap<String, Any?>?, basectx: Context?): Context {
   return Context(ctxmap, basectx)
 }
 
-fun clean(ctx: Context, value: Any?): Any? {
-  return value
+// Everything that leaves the pipeline passes through clean; inside it data
+// stays raw, so a hook can still read the header it must add to. See
+// docs/explanation/secret-redaction.md.
+
+private const val MAXDEPTH = 32
+private const val CIRCULAR = "[circular]"
+
+// The derived clean block: built by makeOptions from the raw input, grown by
+// cleanAdd for as long as the client lives (features register later).
+class CleanConfig(
+  val active: Boolean,
+  val keys: List<String>,
+  val values: MutableList<String>,
+  val mask: String,
+  val hint: Int,
+  val min: Int,
+) {
+  // The options map this sits in can itself be printed; the registry never is.
+  override fun toString(): String = "CleanConfig(active=$active)"
+}
+
+private fun normkey(key: Any?): String {
+  return key.toString().lowercase().replace("-", "").replace("_", "")
+}
+
+private val SPLIT_RE = Regex("\\s*,\\s*")
+
+private fun splitkeys(keys: Any?): List<String> {
+  return (keys?.toString() ?: "").split(SPLIT_RE).map { normkey(it) }.filter { "" != it }
 }
 
+fun splitvalues(values: Any?): List<String> {
+  if (values is List<*>) {
+    return values.filterIsInstance<String>()
+  }
+  return (values?.toString() ?: "").split(SPLIT_RE).filter { "" != it }
+}
+
+private fun count(v: Any?, dflt: Int): Int {
+  val n: Double = when (v) {
+    is Number -> v.toDouble()
+    is String -> v.trim().toDoubleOrNull() ?: return dflt
+    else -> return dflt
+  }
+  if (n.isNaN() || n.isInfinite() || n < 0) {
+    return dflt
+  }
+  return Math.floor(n).toInt()
+}
+
+fun makeCleanConfig(cleanopts: Any?): CleanConfig {
+  val opts = Helpers.toMapAny(cleanopts) ?: linkedMapOf()
+  return CleanConfig(
+    active = false != opts["active"],
+    keys = splitkeys(opts["keys"]),
+    values = mutableListOf(),
+    mask = (opts["mask"] as? String) ?: "[redacted]",
+    hint = count(opts["hint"], 0),
+    min = Math.max(1, count(opts["min"], 4)),
+  )
+}
+
+// A context without options (makeError accepts a bare one) still masks by
+// the schema defaults.
+private fun cleanConfig(ctx: Context?): CleanConfig {
+  val derived = Helpers.toMapAny(ctx?.options?.get("__derived__"))?.get("clean")
+  if (derived is CleanConfig) {
+    return derived
+  }
+  return makeCleanConfig(Schema.optspec["clean"])
+}
+
+// encodeURIComponent, byte for byte: URLEncoder differs on space and `~!'()*`.
+private fun percentEncode(value: String): String {
+  val out = StringBuilder()
+  for (b in value.toByteArray(StandardCharsets.UTF_8)) {
+    val c = b.toInt() and 0xff
+    val ch = c.toChar()
+    if ((ch in 'A'..'Z') || (ch in 'a'..'z') || (ch in '0'..'9') || "-_.!~*'()".indexOf(ch) >= 0) {
+      out.append(ch)
+    } else {
+      out.append('%').append(String.format("%02X", c))
+    }
+  }
+  return out.toString()
+}
+
+// The encoded forms a value travels in.
+private fun forms(value: String): List<String> {
+  val out = mutableListOf(value)
+  fun add(s: String) {
+    if ("" != s && !out.contains(s)) {
+      out.add(s)
+    }
+  }
+  try {
+    add(Base64.getEncoder().encodeToString(value.toByteArray(StandardCharsets.UTF_8)))
+  } catch (e: RuntimeException) {
+  }
+  try {
+    add(percentEncode(value))
+  } catch (e: RuntimeException) {
+  }
+  try {
+    val json = Struct.jsonify(value)
+    if (json.length >= 2) {
+      add(json.substring(1, json.length - 1))
+    }
+  } catch (e: RuntimeException) {
+  }
+  return out
+}
+
+internal fun registerValue(cfg: CleanConfig, value: Any?) {
+  if (value !is String || value.length < cfg.min) {
+    return
+  }
+  synchronized(cfg) {
+    var changed = false
+    for (form in forms(value)) {
+      if (form.length >= cfg.min && !cfg.values.contains(form)) {
+        cfg.values.add(form)
+        changed = true
+      }
+    }
+    if (changed) {
+      cfg.values.sortByDescending { it.length }
+    }
+  }
+}
+
+fun cleanAdd(ctx: Context, value: Any?) {
+  registerValue(cleanConfig(ctx), value)
+}
+
+// Every scalar under a sensitive name in the options, at any depth and of any
+// shape: a credential mistyped as a map or a number is still one, and a
+// message can quote it. A key under `feature` names a feature, not a field,
+// so a feature called secrets does not make its settings secret.
+// Entity blocks (entity settings, seeded records) hold no credential.
+internal fun registerSensitive(cfg: CleanConfig, opts: Map<String, Any?>) {
+  val seen = mutableListOf<Any>()
+  for ((k, v) in opts) {
+    if ("entity" == k) {
+      continue
+    }
+    val under = sensitiveKey(cfg, k)
+    when {
+      "feature" == k && v is Map<*, *> -> for (fopts in v.values) {
+        addSensitive(cfg, noEntity(fopts), under, 2, seen)
+      }
+      "feature" == k && v is List<*> -> for (fopts in v) {
+        addSensitive(cfg, noEntity(fopts), under, 2, seen)
+      }
+      else -> addSensitive(cfg, if ("test" == k) noEntity(v) else v, under, 1, seen)
+    }
+  }
+}
+
+private fun noEntity(block: Any?): Any? =
+  if (block is Map<*, *>) block.filterKeys { it != "entity" } else block
+
+private fun addSensitive(cfg: CleanConfig, v: Any?, under: Boolean, depth: Int, seen: MutableList<Any>) {
+  if (v == null || MAXDEPTH <= depth) {
+    return
+  }
+  if (v is String || v is Number) {
+    if (under) {
+      registerValue(cfg, v.toString())
+      registerValue(cfg, Struct.stringify(v))
+    }
+    return
+  }
+  if (seen.any { it === v }) {
+    return
+  }
+  seen.add(v)
+  when (v) {
+    is Map<*, *> -> for ((k, item) in v) {
+      addSensitive(cfg, item, under || sensitiveKey(cfg, k), depth + 1, seen)
+    }
+    is Collection<*> -> for (item in v) {
+      addSensitive(cfg, item, under, depth + 1, seen)
+    }
+    is Array<*> -> for (item in v) {
+      addSensitive(cfg, item, under, depth + 1, seen)
+    }
+  }
+}
+
+private fun maskValue(cfg: CleanConfig, value: String): String {
+  if (0 < cfg.hint && value.length > 2 * cfg.hint) {
+    return cfg.mask + value.substring(value.length - cfg.hint)
+  }
+  return cfg.mask
+}
+
+private fun cleanString(cfg: CleanConfig, text: String): String {
+  val values = synchronized(cfg) { cfg.values.toList() }
+  var out = text
+  for (value in values) {
+    if (out.contains(value)) {
+      out = out.replace(value, maskValue(cfg, value))
+    }
+  }
+  return out
+}
+
+private fun sensitiveKey(cfg: CleanConfig, key: Any?): Boolean {
+  if (key == null || key is Number) {
+    return false
+  }
+  val nk = normkey(key)
+  for (k in cfg.keys) {
+    if (nk.contains(k)) {
+      return true
+    }
+  }
+  return false
+}
+
+private val DROP = Any()
+
+private fun isfunc(v: Any): Boolean {
+  if (v is Function<*>) {
+    return true
+  }
+  val cls = v.javaClass
+  return cls.isSynthetic || cls.interfaces.any { it.name.startsWith("java.util.function.") }
+}
+
+private fun stackOf(err: Throwable): String {
+  val sw = StringWriter()
+  err.printStackTrace(PrintWriter(sw))
+  return sw.toString()
+}
+
+// The SDK's own objects have no serialisation hook, so this is the record
+// each one leaves as. The context stays raw on the error and is not here.
+private fun record(v: Any): MutableMap<String, Any?> {
+  val out = linkedMapOf<String, Any?>()
+  when (v) {
+    is Throwable -> {
+      out["message"] = v.message ?: ""
+      out["stack"] = stackOf(v)
+      if (v is SdkError) {
+        out["sdk"] = v.sdk
+        out["code"] = v.code
+        out["status"] = v.status
+        out["result"] = v.result
+        out["spec"] = v.spec
+      }
+    }
+    is Spec -> {
+      out["method"] = v.method
+      out["base"] = v.base
+      out["prefix"] = v.prefix
+      out["suffix"] = v.suffix
+      out["path"] = v.path
+      out["url"] = v.url
+      out["params"] = v.params
+      out["query"] = v.query
+      out["headers"] = v.headers
+      out["body"] = v.body
+      out["step"] = v.step
+    }
+    is Result -> {
+      out["ok"] = v.ok
+      out["status"] = v.status
+      out["statusText"] = v.statusText
+      out["headers"] = v.headers
+      out["body"] = v.body
+      out["resdata"] = v.resdata
+      out["resmatch"] = v.resmatch
+      out["err"] = v.err
+      out["paging"] = v.paging
+    }
+    is Response -> {
+      out["status"] = v.status
+      out["statusText"] = v.statusText
+      out["headers"] = v.headers
+      out["body"] = v.body
+      out["err"] = v.err
+    }
+    is Operation -> {
+      out["entity"] = v.entity
+      out["name"] = v.name
+      out["input"] = v.input
+    }
+    is Context -> {
+      out["id"] = v.id
+      out["op"] = v.op
+      out["spec"] = v.spec
+      out["entity"] = v.entity
+      out["result"] = v.result
+      out["response"] = v.response
+      out["meta"] = v.meta
+    }
+    is Entity -> {
+      out["name"] = v.name
+      out["data"] = try { v.data() } catch (e: RuntimeException) { null }
+    }
+  }
+  return out
+}
+
+// A masked plain-data copy: functions dropped, cycles cut, and nothing
+// shared with the live value, whose spec must stay raw.
 @Suppress("UNCHECKED_CAST")
+private fun snapshot(cfg: CleanConfig, v: Any?, key: Any?, depth: Int, seen: MutableList<Any>): Any? {
+  if (v == null || v === Struct.UNDEF) {
+    return v
+  }
+
+  if (v is String) {
+    return if (sensitiveKey(cfg, key)) maskValue(cfg, v) else cleanString(cfg, v)
+  }
+
+  if (isfunc(v)) {
+    return DROP
+  }
+
+  if (v is Number || v is Boolean || v is Char) {
+    return if (sensitiveKey(cfg, key)) cfg.mask else v
+  }
+
+  if (MAXDEPTH <= depth || seen.any { it === v }) {
+    return CIRCULAR
+  }
+
+  if (sensitiveKey(cfg, key)) {
+    return cfg.mask
+  }
+
+  seen.add(v)
+  try {
+    return when (v) {
+      is List<*> -> {
+        val out = mutableListOf<Any?>()
+        v.forEachIndexed { i, item ->
+          val s = snapshot(cfg, item, i, depth + 1, seen)
+          if (s !== DROP) {
+            out.add(s)
+          }
+        }
+        out
+      }
+      is Map<*, *> -> plain(cfg, v, depth, seen)
+      is Throwable, is Spec, is Result, is Response, is Operation, is Context, is Entity ->
+        plain(cfg, record(v), depth, seen)
+      else -> cleanString(cfg, v.toString())
+    }
+  } finally {
+    seen.removeAt(seen.size - 1)
+  }
+}
+
+private fun plain(cfg: CleanConfig, v: Map<*, *>, depth: Int, seen: MutableList<Any>): MutableMap<String, Any?> {
+  val out = linkedMapOf<String, Any?>()
+  for ((k, item) in v) {
+    val s = snapshot(cfg, item, k, depth + 1, seen)
+    if (s !== DROP) {
+      out[cleanName(cfg, out, k.toString())] = s
+    }
+  }
+  return out
+}
+
+// A registered value used as a property name is masked like any other
+// string; names that mask alike take a counter, so none is lost.
+private fun cleanName(cfg: CleanConfig, out: Map<String, Any?>, key: String): String {
+  val name = cleanString(cfg, key)
+  if (name == key || !out.containsKey(name)) {
+    return name
+  }
+  var i = 1
+  while (out.containsKey(name + "#" + i)) {
+    i++
+  }
+  return name + "#" + i
+}
+
+internal fun cleanWith(cfg: CleanConfig, value: Any?): Any? {
+  if (!cfg.active) {
+    return value
+  }
+
+  if (value is String) {
+    return cleanString(cfg, value)
+  }
+
+  // An SdkError is cleaned in place, since it is about to be thrown.
+  if (value is SdkError) {
+    value.msg = cleanString(cfg, value.msg)
+    value.code = cleanString(cfg, value.code)
+    value.result = snapshot(cfg, value.result, "result", 1, mutableListOf())
+    value.spec = snapshot(cfg, value.spec, "spec", 1, mutableListOf())
+    return value
+  }
+
+  // A foreign exception's message cannot be rewritten, so it is replaced by
+  // an SdkError with the cleaned message and the original frames; the
+  // original is not attached, as a printed stack trace would show it.
+  if (value is Throwable) {
+    val out = SdkError("", cleanString(cfg, value.message ?: value.toString()), null)
+    out.stackTrace = value.stackTrace
+    return out
+  }
+
+  val out = snapshot(cfg, value, null, 0, mutableListOf())
+  return if (out === DROP) null else out
+}
+
+fun clean(ctx: Context, value: Any?): Any? {
+  return cleanWith(cleanConfig(ctx), value)
+}
+
+fun cleanKey(ctx: Context, key: Any?): Boolean {
+  return sensitiveKey(cleanConfig(ctx), key)
+}
+
+// The caller holds the explain map, so the cleaned copy is written back into
+// it rather than swapped for it.
+private fun cleanExplain(ctx: Context) {
+  val explain = ctx.ctrl.explain ?: return
+  val cleaned = Helpers.toMapAny(clean(ctx, explain))
+  if (cleaned != null && cleaned !== explain) {
+    explain.clear()
+    explain.putAll(cleaned)
+  }
+}
+
 fun done(ctx: Context): Any? {
-  val explain = ctx.ctrl.explain
-  if (explain != null) {
-    val cleaned = clean(ctx, explain) as MutableMap<String, Any?>
-    ctx.ctrl.explain = cleaned
-    val rm = Helpers.toMapAny(cleaned["result"])
+  if (ctx.ctrl.explain != null) {
+    cleanExplain(ctx)
+    val rm = Helpers.toMapAny(ctx.ctrl.explain!!["result"])
     rm?.remove("err")
   }
 
@@ -64,21 +499,22 @@ fun makeError(ctx: Context, errIn: RuntimeException?): Any? {
 
   val spec = ctx.spec
 
-  val explain = ctx.ctrl.explain
-  if (explain != null) {
+  if (ctx.ctrl.explain != null) {
+    cleanExplain(ctx)
     val errRecord = linkedMapOf<String, Any?>()
     errRecord["message"] = msg
-    explain["err"] = errRecord
+    ctx.ctrl.explain!!["err"] = errRecord
   }
 
   var code = ""
   if (err is SdkError) {
-    code = err.code
+    code = clean(ctx, err.code) as String
   }
 
   val sdkErr = SdkError(code, msg, ctx)
   sdkErr.result = clean(ctx, result)
   sdkErr.spec = clean(ctx, spec)
+  sdkErr.status = result.status
 
   ctx.ctrl.err = sdkErr
 

package/project/.sdk/tm/kotlin/utility/MakeOptions.kt

@@ -3,6 +3,7 @@ package KOTLINPACKAGE.utility
 import KOTLINPACKAGE.core.Context
 import KOTLINPACKAGE.core.Helpers
 import KOTLINPACKAGE.core.Schema
+import KOTLINPACKAGE.core.SdkError
 import KOTLINPACKAGE.core.Utility
 import KOTLINPACKAGE.utility.struct.Struct
 
@@ -41,6 +42,32 @@ fun makeOptions(ctx: Context): MutableMap<String, Any?> {
   // check on the value, which cannot distinguish them.
   val authSuppressed = options.containsKey("auth") && null == options["auth"]
 
+  var config = ctx.config
+  if (config == null) {
+    config = linkedMapOf()
+  }
+  var cfgopts = Helpers.toMapAny(config["options"])
+  if (cfgopts == null) {
+    cfgopts = linkedMapOf()
+  }
+
+  // The secret registry exists BEFORE validation, fed from the raw input, so
+  // the constructor's own rejection of a mistyped credential is clean too.
+  // Both clean blocks are cloned for the reason the options merge below is,
+  // and an absent one is left out: merge lets a null replace everything.
+  val cleanblocks = listOf(Helpers.toMapAny(cfgopts["clean"]), Helpers.toMapAny(options["clean"]))
+  val cleanmerge = mutableListOf<Any?>(linkedMapOf<String, Any?>(), Struct.clone(Schema.optspec["clean"]))
+  for (block in cleanblocks.filterNotNull()) {
+    cleanmerge.add(Struct.clone(block))
+  }
+  val cleancfg = makeCleanConfig(Struct.merge(cleanmerge))
+  registerSensitive(cleancfg, options - "clean")
+  for (block in cleanblocks) {
+    for (raw in splitvalues(block?.get("values"))) {
+      registerValue(cleancfg, raw)
+    }
+  }
+
   var opts = Struct.clone(options) as MutableMap<String, Any?>
 
   if (authSuppressed) {
@@ -71,15 +98,6 @@ fun makeOptions(ctx: Context): MutableMap<String, Any?> {
     opts["feature"] = fmap
   }
 
-  var config = ctx.config
-  if (config == null) {
-    config = linkedMapOf()
-  }
-  var cfgopts = Helpers.toMapAny(config["options"])
-  if (cfgopts == null) {
-    cfgopts = linkedMapOf()
-  }
-
   // THE OPTION SPEC IS GENERATED, NOT WRITTEN HERE.
   //
   // Built from the model: `main.kit.optspec` for the standard options, plus
@@ -111,7 +129,13 @@ fun makeOptions(ctx: Context): MutableMap<String, Any?> {
 
   val vopts = linkedMapOf<String, Any?>()
   vopts["errs"] = mutableListOf<Any?>()
-  val validated = Struct.validate(merged, optspec, vopts)
+  val validated = try {
+    Struct.validate(merged, optspec, vopts)
+  } catch (err: RuntimeException) {
+    // A rejection quotes the value it rejected.
+    throw SdkError("options_invalid",
+      cleanWith(cleancfg, err.message ?: err.toString()) as String, ctx)
+  }
   opts = validated as MutableMap<String, Any?>
 
   // Restore the suppression the optspec default would otherwise erase.
@@ -131,22 +155,6 @@ fun makeOptions(ctx: Context): MutableMap<String, Any?> {
     }
   }
 
-  // Derived clean config.
-  var cleanKeys = "key,token,id"
-  val ck = Struct.getpath(opts, listOf("clean", "keys"))
-  if (ck is String) {
-    cleanKeys = ck
-  }
-
-  val filtered = mutableListOf<String>()
-  for (pRaw in cleanKeys.split(",")) {
-    val p = pRaw.trim()
-    if ("" != p) {
-      filtered.add(Struct.escre(p))
-    }
-  }
-  val keyre = filtered.joinToString("|")
-
   // Resolve the feature add-order: an explicit list order (above) wins;
   // otherwise order the map test-first, then the remaining names sorted, so
   // the outcome is deterministic and `test` is always the base transport.
@@ -177,14 +185,13 @@ fun makeOptions(ctx: Context): MutableMap<String, Any?> {
   }
 
   val derived = linkedMapOf<String, Any?>()
-  val derivedClean = linkedMapOf<String, Any?>()
-  if ("" != keyre) {
-    derivedClean["keyre"] = keyre
-  }
-  derived["clean"] = derivedClean
+  derived["clean"] = cleancfg
   derived["featureorder"] = featureorder
   opts["__derived__"] = derived
 
+  // Again over the merged result: the config's own defaults can carry one.
+  registerSensitive(cleancfg, opts - "clean" - "__derived__")
+
   return opts
 }
 

package/project/.sdk/tm/kotlin/utility/Register.kt

@@ -7,6 +7,7 @@ object Register {
 
   fun registerAll(u: Utility) {
     u.clean = ::clean
+    u.cleanAdd = ::cleanAdd
     u.done = ::done
     u.makeError = ::makeError
     u.featureAdd = ::featureAdd

package/project/.sdk/tm/lua/core/context.lua

@@ -1,5 +1,6 @@
 -- ProjectName SDK context
 
+local json = require("dkjson")
 local vs = require("utility.struct.struct")
 local Control = require("core.control")
 local Operation = require("core.operation")
@@ -217,4 +218,46 @@ function Context:make_error(code, msg)
 end
 
 
+-- The record a logger or an error dump sees; the live fields stay raw for
+-- the pipeline's own use. Raw here, so clean's snapshot can take it.
+function Context:to_record()
+  return {
+    id = self.id,
+    op = self.op,
+    spec = self.spec,
+    entity = self.entity,
+    result = self.result,
+    response = self.response,
+    meta = self.meta,
+  }
+end
+
+
+-- The serialised context leaves the pipeline, so it is cleaned.
+function Context:to_json()
+  local record = self:to_record()
+  local clean = self.utility ~= nil and self.utility.clean or nil
+  if type(clean) == "function" then
+    record = clean(self, record)
+  end
+  local ok, text = pcall(json.encode, record, {
+    exception = function() return "null" end,
+  })
+  if ok and type(text) == "string" then
+    return text
+  end
+  return json.encode({ id = self.id })
+end
+
+
+function Context:__tostring()
+  return "Context " .. self:to_json()
+end
+
+
+function Context.__tojson(self)
+  return self:to_json()
+end
+
+
 return Context