@voxgig/sdkgen 4.32.1 → 4.34.0

package/project/.sdk/tm/lua/utility/clean.lua

@@ -1,7 +1,431 @@
 -- ProjectName SDK utility: clean
+--
+-- Everything that leaves the pipeline passes through clean; inside it data
+-- stays raw, so a hook can still read the header it must add to. The lua
+-- port of tm/ts/src/utility/CleanUtility.ts.
 
+local json = require("dkjson")
+
+local MAXDEPTH = 32
+local CIRCULAR = "[circular]"
+
+-- The schema's own default, for a context that reaches clean before any
+-- options exist (the raw template tree has no generated schema to read).
+local DEFAULT_KEYS = "key,secret,token,password,passwd,authorization,cookie,credential,signature"
+
+local B64_ALPHABET =
+  "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+
+
+local function base64(text)
+  local out = {}
+  local size = #text
+  local index = 1
+
+  while index <= size do
+    local rest = size - index + 1
+    local a = string.byte(text, index)
+    local b = string.byte(text, index + 1) or 0
+    local c = string.byte(text, index + 2) or 0
+    local word = (a << 16) | (b << 8) | c
+
+    local chunk = {}
+    for shift = 18, 0, -6 do
+      local at = (word >> shift) & 0x3f
+      chunk[#chunk + 1] = string.sub(B64_ALPHABET, at + 1, at + 1)
+    end
+
+    if rest == 1 then
+      chunk[3] = "="
+      chunk[4] = "="
+    elseif rest == 2 then
+      chunk[4] = "="
+    end
+
+    out[#out + 1] = table.concat(chunk)
+    index = index + 3
+  end
+
+  return table.concat(out)
+end
+
+
+-- encodeURIComponent: everything but the RFC 3986 unreserved set and !*'().
+local function percent_encode(text)
+  return (string.gsub(text, "[^%w%-_%.~!%*'%(%)]", function(c)
+    return string.format("%%%02X", string.byte(c))
+  end))
+end
+
+
+local function json_escape(text)
+  local ok, encoded = pcall(json.encode, text)
+  if ok and type(encoded) == "string" and #encoded >= 2 then
+    return string.sub(encoded, 2, -2)
+  end
+  return text
+end
+
+
+local function trim(s)
+  return (string.match(s, "^%s*(.-)%s*$"))
+end
+
+
+local function normkey(key)
+  return (string.gsub(string.lower(tostring(key)), "[-_]", ""))
+end
+
+
+local function splitkeys(keys)
+  local out = {}
+  if keys == nil then
+    return out
+  end
+  for part in string.gmatch(tostring(keys), "[^,]+") do
+    local k = normkey(trim(part))
+    if k ~= "" then
+      out[#out + 1] = k
+    end
+  end
+  return out
+end
+
+
+local function splitvalues(values)
+  local out = {}
+  if type(values) == "table" then
+    for _, v in ipairs(values) do
+      if type(v) == "string" then
+        out[#out + 1] = v
+      end
+    end
+    return out
+  end
+  if values == nil then
+    return out
+  end
+  for part in string.gmatch(tostring(values), "[^,]+") do
+    local v = trim(part)
+    if v ~= "" then
+      out[#out + 1] = v
+    end
+  end
+  return out
+end
+
+
+local function count(val, dflt)
+  local n = tonumber(val)
+  if n == nil or n < 0 then
+    return dflt
+  end
+  return math.floor(n)
+end
+
+
+local function make_clean_config(cleanopts)
+  local opts = type(cleanopts) == "table" and cleanopts or {}
+  return {
+    active = opts.active ~= false,
+    keys = splitkeys(opts.keys == nil and DEFAULT_KEYS or opts.keys),
+    values = {},
+    mask = type(opts.mask) == "string" and opts.mask or "[redacted]",
+    hint = count(opts.hint, 0),
+    min = math.max(1, count(opts.min, 4)),
+  }
+end
+
+
+-- A context without options (make_error accepts a bare one) still masks by
+-- the schema defaults.
+local function clean_config(ctx)
+  if type(ctx) == "table" and type(ctx.options) == "table" then
+    local derived = ctx.options["__derived__"]
+    if type(derived) == "table" and type(derived.clean) == "table"
+      and type(derived.clean.values) == "table"
+    then
+      return derived.clean
+    end
+  end
+
+  local ok, schema = pcall(require, "schema")
+  local spec = nil
+  if ok and type(schema) == "table" and type(schema.OPTSPEC) == "table" then
+    spec = schema.OPTSPEC.clean
+  end
+  return make_clean_config(spec)
+end
+
+
+local function contains(list, value)
+  for _, v in ipairs(list) do
+    if v == value then
+      return true
+    end
+  end
+  return false
+end
+
+
+-- The encoded forms a value travels in.
+local function forms(value)
+  local out = { value }
+  for _, form in ipairs({ base64(value), percent_encode(value), json_escape(value) }) do
+    if form ~= "" and not contains(out, form) then
+      out[#out + 1] = form
+    end
+  end
+  return out
+end
+
+
+local function clean_add_util(ctx, value)
+  local cfg = clean_config(ctx)
+  if type(value) ~= "string" or #value < cfg.min then
+    return
+  end
+  local changed = false
+  for _, form in ipairs(forms(value)) do
+    if #form >= cfg.min and not contains(cfg.values, form) then
+      cfg.values[#cfg.values + 1] = form
+      changed = true
+    end
+  end
+  if changed then
+    table.sort(cfg.values, function(a, b) return #a > #b end)
+  end
+end
+
+
+local function mask_value(cfg, value)
+  if 0 < cfg.hint and #value > 2 * cfg.hint then
+    return cfg.mask .. string.sub(value, -cfg.hint)
+  end
+  return cfg.mask
+end
+
+
+local function replace_all(text, needle, repl)
+  local out = {}
+  local start = 1
+  while true do
+    local i, j = string.find(text, needle, start, true)
+    if i == nil then
+      break
+    end
+    out[#out + 1] = string.sub(text, start, i - 1)
+    out[#out + 1] = repl
+    start = j + 1
+  end
+  out[#out + 1] = string.sub(text, start)
+  return table.concat(out)
+end
+
+
+local function clean_string(cfg, text)
+  local out = text
+  for _, value in ipairs(cfg.values) do
+    if string.find(out, value, 1, true) ~= nil then
+      out = replace_all(out, value, mask_value(cfg, value))
+    end
+  end
+  return out
+end
+
+
+local function sensitive_key(cfg, key)
+  if key == nil or type(key) == "number" then
+    return false
+  end
+  local nk = normkey(key)
+  for _, k in ipairs(cfg.keys) do
+    if string.find(nk, k, 1, true) ~= nil then
+      return true
+    end
+  end
+  return false
+end
+
+
+-- Table keys in a stable order: pairs has none, and the collision counter
+-- below must not depend on it.
+local function sorted_keys(tbl)
+  local keys = {}
+  for k in pairs(tbl) do
+    keys[#keys + 1] = k
+  end
+  table.sort(keys, function(a, b)
+    local ta, tb = type(a), type(b)
+    if ta ~= tb then
+      return ta < tb
+    end
+    if ta == "number" or ta == "string" then
+      return a < b
+    end
+    return tostring(a) < tostring(b)
+  end)
+  return keys
+end
+
+
+-- A registered value used as a property name is masked like any other
+-- string; names that mask alike take a counter, so none is lost.
+local function clean_name(cfg, out, key)
+  if type(key) ~= "string" then
+    return key
+  end
+  local name = clean_string(cfg, key)
+  if name == key or out[name] == nil then
+    return name
+  end
+  local i = 1
+  while out[name .. "#" .. i] ~= nil do
+    i = i + 1
+  end
+  return name .. "#" .. i
+end
+
+
+-- A masked plain-data copy: an object's own record (to_record) honoured,
+-- functions dropped, cycles cut, and nothing shared with the live value,
+-- whose spec must stay raw.
+local function snapshot(cfg, val, key, depth, seen)
+  local t = type(val)
+
+  if val == nil then
+    return nil
+  end
+
+  if t == "string" then
+    if sensitive_key(cfg, key) then
+      return mask_value(cfg, val)
+    end
+    return clean_string(cfg, val)
+  end
+
+  if t == "function" or t == "thread" or t == "userdata" then
+    return nil
+  end
+
+  if t ~= "table" then
+    if sensitive_key(cfg, key) then
+      return cfg.mask
+    end
+    return val
+  end
+
+  if MAXDEPTH <= depth or seen[val] then
+    return CIRCULAR
+  end
+
+  if sensitive_key(cfg, key) then
+    return cfg.mask
+  end
+
+  seen[val] = true
+
+  local mt = getmetatable(val)
+  if mt ~= nil and type(mt.to_record) == "function" then
+    local record = mt.to_record(val)
+    if type(record) == "table" and record ~= val then
+      local out = snapshot(cfg, record, key, depth + 1, seen)
+      seen[val] = nil
+      return out
+    end
+  end
+
+  local out = {}
+  for _, k in ipairs(sorted_keys(val)) do
+    local cv = snapshot(cfg, val[k], k, depth + 1, seen)
+    if cv ~= nil then
+      out[clean_name(cfg, out, k)] = cv
+    end
+  end
+
+  seen[val] = nil
+  return out
+end
+
+
+-- An SDK error is cleaned in place, since it is about to be returned.
 local function clean_util(ctx, val)
-  return val
+  local cfg = clean_config(ctx)
+
+  if cfg.active == false then
+    return val
+  end
+
+  if type(val) == "string" then
+    return clean_string(cfg, val)
+  end
+
+  if type(val) == "table" and val.is_sdk_error == true then
+    for _, k in ipairs(sorted_keys(val)) do
+      local v = val[k]
+      if type(v) == "string" then
+        if k ~= "msg" and sensitive_key(cfg, k) then
+          v = mask_value(cfg, v)
+        else
+          v = clean_string(cfg, v)
+        end
+      elseif type(v) == "table" then
+        v = snapshot(cfg, v, k, 1, {})
+      end
+      local name = clean_name(cfg, {}, k)
+      if name ~= k then
+        val[k] = nil
+        name = clean_name(cfg, val, k)
+      end
+      val[name] = v
+    end
+    return val
+  end
+
+  return snapshot(cfg, val, nil, 0, {})
 end
 
-return clean_util
+
+local function clean_key_util(ctx, key)
+  return sensitive_key(clean_config(ctx), key)
+end
+
+
+-- Every scalar under a sensitive name, at any depth and of any shape: a
+-- credential mistyped as a table or a number is still a credential, and
+-- the validation error that rejects it quotes it.
+local function clean_add_sensitive_util(ctx, val, under, depth, seen)
+  depth = depth or 0
+  seen = seen or {}
+  local t = type(val)
+  if val == nil or MAXDEPTH <= depth then
+    return
+  end
+  if t == "string" or t == "number" then
+    if under then
+      local text = tostring(val)
+      if math.type(val) == "float" and val == math.floor(val) then
+        text = string.format("%d", val)
+      end
+      clean_add_util(ctx, text)
+    end
+    return
+  end
+  if t ~= "table" or seen[val] then
+    return
+  end
+  seen[val] = true
+  for k, v in pairs(val) do
+    clean_add_sensitive_util(ctx, v, under or clean_key_util(ctx, k), depth + 1, seen)
+  end
+end
+
+
+return {
+  clean = clean_util,
+  clean_add = clean_add_util,
+  clean_add_sensitive = clean_add_sensitive_util,
+  clean_key = clean_key_util,
+  make_clean_config = make_clean_config,
+  splitvalues = splitvalues,
+}

package/project/.sdk/tm/lua/utility/clean_explain.lua

@@ -0,0 +1,34 @@
+-- ProjectName SDK utility: clean_explain
+
+-- In place: the caller may hold the record, and a stream copies only its
+-- ctrl. With clean off explain.result is the live result make_error reads,
+-- so err is pruned from a copy of it.
+local function clean_explain_util(ctx)
+  local explain = ctx.ctrl.explain
+  if type(explain) ~= "table" then
+    return
+  end
+
+  local cleaned = ctx.utility.clean(ctx, explain)
+  if type(cleaned) == "table" and cleaned ~= explain then
+    for k in pairs(explain) do
+      explain[k] = nil
+    end
+    for k, v in pairs(cleaned) do
+      explain[k] = v
+    end
+  end
+
+  local result = explain["result"]
+  if type(result) == "table" then
+    local pruned = {}
+    for k, v in pairs(result) do
+      if k ~= "err" then
+        pruned[k] = v
+      end
+    end
+    explain["result"] = pruned
+  end
+end
+
+return clean_explain_util

package/project/.sdk/tm/lua/utility/done.lua

@@ -1,13 +1,7 @@
 -- ProjectName SDK utility: done
 
 local function done_util(ctx)
-  if ctx.ctrl.explain ~= nil then
-    ctx.ctrl.explain = ctx.utility.clean(ctx, ctx.ctrl.explain)
-    local explain_result = ctx.ctrl.explain["result"]
-    if type(explain_result) == "table" then
-      explain_result["err"] = nil
-    end
-  end
+  ctx.utility.clean_explain(ctx)
 
   if ctx.result ~= nil and ctx.result.ok then
     return ctx.result.resdata, nil

package/project/.sdk/tm/lua/utility/make_error.lua

@@ -49,6 +49,7 @@ local function make_error_util(ctx, err)
 
   local spec = ctx.spec
 
+  ctx.utility.clean_explain(ctx)
   if ctx.ctrl.explain ~= nil then
     ctx.ctrl.explain["err"] = { message = msg }
   end
@@ -61,8 +62,9 @@ local function make_error_util(ctx, err)
   -- `err.status` instead of reaching into `err.result`.
   sdk_err.status = result.status or -1
 
+  -- A hook's own error supplies the code as well as the message.
   if type(err) == "table" and getmetatable(err) == ProjectNameError then
-    sdk_err.code = err.code
+    sdk_err.code = ctx.utility.clean(ctx, err.code)
   end
 
   ctx.ctrl.err = sdk_err

package/project/.sdk/tm/lua/utility/make_options.lua

@@ -2,6 +2,7 @@
 
 local vs = require("utility.struct.struct")
 local schema = require("schema")
+local cleanmod = require("utility.clean")
 
 -- See the call site in make_options for why this exists.
 local function densify_lists(v, seen)
@@ -48,6 +49,43 @@ local function densify_lists(v, seen)
   return v
 end
 
+local function noentity(settings)
+  if type(settings) ~= "table" then
+    return settings
+  end
+  local out = {}
+  for k, v in pairs(settings) do
+    if k ~= "entity" then
+      out[k] = v
+    end
+  end
+  return out
+end
+
+-- The options to scan for secrets. The feature map is keyed by feature
+-- names, not field names, so it is scanned as a list: `secrets` must not
+-- make every setting of that feature a secret. Entity blocks hold entity
+-- settings and seeded records, never a credential, so none is scanned. The
+-- raw scan still sees the feature list form, whose entries each carry `name`.
+local function secret_scan(opts, omit)
+  local out = {}
+  for k, v in pairs(opts) do
+    if not omit[k] and k ~= "entity" then
+      if k == "feature" and type(v) == "table" then
+        local list = {}
+        for _, fopts in pairs(v) do
+          list[#list + 1] = noentity(fopts)
+        end
+        v = list
+      elseif k == "test" then
+        v = noentity(v)
+      end
+      out[k] = v
+    end
+  end
+  return out
+end
+
 local function copy_data(v)
   if type(v) ~= 'table' then
     return v
@@ -121,6 +159,32 @@ local function make_options_util(ctx)
     opts = {}
   end
 
+  local config = ctx.config or {}
+  local cfgopts = {}
+  local co = config["options"]
+  if type(co) == "table" then
+    cfgopts = co
+  end
+
+  -- The secret registry exists BEFORE validation, fed from the raw input, so
+  -- the constructor's own rejection of a mistyped credential is clean too.
+  local specclean = type(schema.OPTSPEC) == "table" and schema.OPTSPEC.clean or nil
+  local cfgclean = type(cfgopts.clean) == "table" and cfgopts.clean or {}
+  local rawclean = type(opts.clean) == "table" and opts.clean or {}
+  local cleancfg = cleanmod.make_clean_config(vs.merge({
+    {},
+    type(specclean) == "table" and vs.clone(specclean) or {},
+    vs.clone(cfgclean),
+    vs.clone(rawclean),
+  }))
+  local cleanctx = { options = { __derived__ = { clean = cleancfg } } }
+  cleanmod.clean_add_sensitive(cleanctx, secret_scan(opts, { clean = true, extend = true }))
+  for _, block in ipairs({ cfgclean, rawclean }) do
+    for _, raw in ipairs(cleanmod.splitvalues(block.values)) do
+      cleanmod.clean_add(cleanctx, raw)
+    end
+  end
+
   -- Feature add-order. options.feature may be given as an ordered LIST of
   -- { name = ..., active = ..., ... } entries (the list position IS the order
   -- in which features are added), or as a { name = {opts} } map. Normalize a
@@ -146,13 +210,6 @@ local function make_options_util(ctx)
     opts.feature = fmap
   end
 
-  local config = ctx.config or {}
-  local cfgopts = {}
-  local co = config["options"]
-  if type(co) == "table" then
-    cfgopts = co
-  end
-
   -- THE OPTION SPEC IS GENERATED, NOT WRITTEN HERE.
   --
   -- `schema.OPTSPEC` is built from the model: `main.kit.optspec` for
@@ -215,7 +272,10 @@ local function make_options_util(ctx)
   -- feature reading it refuses it by its own rule, with its own message.
   merged = densify_lists(merged)
 
-  local validated = vs.validate(merged, optspec)
+  local vok, validated = pcall(vs.validate, merged, optspec)
+  if not vok then
+    error(cleanmod.clean(cleanctx, validated), 0)
+  end
   if type(validated) ~= "table" then
     validated = {}
   end
@@ -282,22 +342,6 @@ local function make_options_util(ctx)
     opts["extend"] = extend
   end
 
-  -- Derived clean config.
-  local clean_keys = "key,token,id"
-  local ck = vs.getpath(opts, "clean.keys")
-  if type(ck) == "string" then
-    clean_keys = ck
-  end
-
-  local parts = {}
-  for part in string.gmatch(clean_keys, "[^,]+") do
-    local trimmed = part:match("^%s*(.-)%s*$")
-    if trimmed ~= "" then
-      table.insert(parts, vs.escre(trimmed))
-    end
-  end
-  local keyre = table.concat(parts, "|")
-
   -- Resolve the feature add-order: an explicit list order (above) wins;
   -- otherwise order the map test-first, then the remaining names sorted, so
   -- the outcome is deterministic and `test` is always the base transport.
@@ -354,12 +398,14 @@ local function make_options_util(ctx)
     end
   end
 
-  local derived = { clean = {} }
-  if keyre ~= "" then
-    derived.clean = { keyre = keyre }
-  end
-  derived.featureorder = featureorder
-  opts["__derived__"] = derived
+  -- Again over the merged result: the config's own defaults can carry one.
+  cleanmod.clean_add_sensitive(cleanctx,
+    secret_scan(opts, { clean = true, extend = true, __derived__ = true }))
+
+  opts["__derived__"] = {
+    clean = cleancfg,
+    featureorder = featureorder,
+  }
 
   return opts
 end

package/project/.sdk/tm/lua/utility/register.lua

@@ -3,6 +3,7 @@
 local Utility = require("core.utility_type")
 
 local clean = require("utility.clean")
+local clean_explain = require("utility.clean_explain")
 local done = require("utility.done")
 local make_error = require("utility.make_error")
 local feature_add = require("utility.feature_add")
@@ -35,7 +36,9 @@ local transform_response = require("utility.transform_response")
 
 
 local function register_all(u)
-  u.clean = clean
+  u.clean = clean.clean
+  u.clean_add = clean.clean_add
+  u.clean_explain = clean_explain
   u.done = done
   u.make_error = make_error
   u.feature_add = feature_add

package/project/.sdk/tm/ocaml/Makefile

@@ -87,6 +87,10 @@ SDK = $(RUNTIME) sdk_config.ml $(ENTITIES) sdk_client.ml sdk_error.ml
 
 ENTITY_TESTS = $(sort $(wildcard test/*_entity_test.ml) $(wildcard test/*_direct_test.ml))
 
+# The canary sweep (generated: test/clean_test.ml), which prints the
+# `clean: swept N surface(s), M leak(s)` line sdkgen's lane reads.
+CLEAN_TESTS = $(sort $(wildcard test/clean_test.ml))
+
 # Doc-example structural gate (present only when the readme phase generated it).
 # Wildcard so an absent file is simply skipped, never a missing-target error.
 DOC_TESTS = $(sort $(wildcard test/readme_examples_test.ml))
@@ -96,7 +100,7 @@ DOC_TESTS = $(sort $(wildcard test/readme_examples_test.ml))
 SDK_TEST = $(RUNTIME) sdk_config.ml $(ENTITIES) sdk_client.ml sdk_error.ml \
   test/testutil.ml test/harness.ml \
   test/t_pipeline.ml test/t_feature.ml test/custom_utility.ml \
-  $(ENTITY_TESTS) $(DOC_TESTS) $(FEATURE_TESTS) test/t_main.ml
+  $(ENTITY_TESTS) $(CLEAN_TESTS) $(DOC_TESTS) $(FEATURE_TESTS) test/t_main.ml
 
 # The struct corpus needs only the struct library and the corpus engine —
 # omni carries its own JSON reader and regex engine, so nothing of the SDK