@voxgig/sdkgen 4.32.1 → 4.34.0

package/project/.sdk/tm/ts/src/utility/CleanUtility.ts

@@ -1,24 +1,302 @@
 
 import { Context } from '../types'
+import { OPTSPEC } from '../Schema'
 
 
-import {
-  walk, size, pad, slice, clone
-} from './StructUtility'
+// Everything that leaves the pipeline passes through clean; inside it data
+// stays raw, so a hook can still read the header it must add to. See
+// docs/explanation/secret-redaction.md.
 
+const MAXDEPTH = 32
+const CIRCULAR = '[circular]'
 
-// Clean request data by partially hiding sensitive values.
+
+type CleanConfig = {
+  active: boolean
+  keys: string[]
+  values: string[]
+  mask: string
+  hint: number
+  min: number
+}
+
+
+function normkey(key: any): string {
+  return String(key).toLowerCase().replace(/[-_]/g, '')
+}
+
+
+function splitkeys(keys: any): string[] {
+  return String(null == keys ? '' : keys)
+    .split(/\s*,\s*/)
+    .map(normkey)
+    .filter((k: string) => '' !== k)
+}
+
+
+function splitvalues(values: any): string[] {
+  if (Array.isArray(values)) {
+    return values.filter((v: any) => 'string' === typeof v)
+  }
+  return String(null == values ? '' : values)
+    .split(/\s*,\s*/)
+    .filter((v: string) => '' !== v)
+}
+
+
+function count(val: any, dflt: number): number {
+  const n = Math.floor(Number(val))
+  return Number.isFinite(n) && 0 <= n ? n : dflt
+}
+
+
+// A context without options (makeError accepts a bare one) still masks by
+// the schema defaults.
+function cleanConfig(ctx: any): CleanConfig {
+  const derived = ctx?.options?.__derived__?.clean
+  if (null != derived) {
+    return derived
+  }
+  return makeCleanConfig((OPTSPEC as any)?.clean)
+}
+
+
+function makeCleanConfig(cleanopts: any): CleanConfig {
+  const opts = cleanopts || {}
+  return {
+    active: false !== opts.active,
+    keys: splitkeys(opts.keys),
+    values: [],
+    mask: 'string' === typeof opts.mask ? opts.mask : '[redacted]',
+    hint: count(opts.hint, 0),
+    min: Math.max(1, count(opts.min, 4)),
+  }
+}
+
+
+// The encoded forms a value travels in.
+function forms(value: string): string[] {
+  const out = [value]
+  const add = (s: string) => { if ('' !== s && !out.includes(s)) out.push(s) }
+  try { add(Buffer.from(value, 'utf8').toString('base64')) } catch (_e) { }
+  try { add(encodeURIComponent(value)) } catch (_e) { }
+  try { add(JSON.stringify(value).slice(1, -1)) } catch (_e) { }
+  return out
+}
+
+
+function cleanAdd(ctx: Context, value: any): void {
+  const cfg = cleanConfig(ctx)
+  if ('string' !== typeof value || value.length < cfg.min) {
+    return
+  }
+  let changed = false
+  for (const form of forms(value)) {
+    if (form.length >= cfg.min && !cfg.values.includes(form)) {
+      cfg.values.push(form)
+      changed = true
+    }
+  }
+  if (changed) {
+    cfg.values.sort((a, b) => b.length - a.length)
+  }
+}
+
+
+function maskValue(cfg: CleanConfig, value: string): string {
+  if (0 < cfg.hint && value.length > 2 * cfg.hint) {
+    return cfg.mask + value.slice(-cfg.hint)
+  }
+  return cfg.mask
+}
+
+
+function cleanString(cfg: CleanConfig, text: string): string {
+  let out = text
+  for (const value of cfg.values) {
+    if (out.includes(value)) {
+      out = out.split(value).join(maskValue(cfg, value))
+    }
+  }
+  return out
+}
+
+
+function sensitiveKey(cfg: CleanConfig, key: any): boolean {
+  if (null == key || 'number' === typeof key) {
+    return false
+  }
+  const nk = normkey(key)
+  for (const k of cfg.keys) {
+    if (nk.includes(k)) {
+      return true
+    }
+  }
+  return false
+}
+
+
+// A masked plain-data copy: toJSON honoured, functions dropped, cycles cut,
+// and nothing shared with the live value, whose spec must stay raw.
+function snapshot(cfg: CleanConfig, val: any, key: any, depth: number, seen: any[]): any {
+  if (null == val) {
+    return val
+  }
+
+  const t = typeof val
+
+  if ('string' === t) {
+    return sensitiveKey(cfg, key) ? maskValue(cfg, val) : cleanString(cfg, val)
+  }
+
+  if ('function' === t) {
+    return undefined
+  }
+
+  if ('object' !== t) {
+    return sensitiveKey(cfg, key) ? cfg.mask : val
+  }
+
+  if (MAXDEPTH <= depth || seen.includes(val)) {
+    return CIRCULAR
+  }
+
+  if (sensitiveKey(cfg, key)) {
+    return cfg.mask
+  }
+
+  seen.push(val)
+  try {
+    if (Array.isArray(val)) {
+      const out: any[] = []
+      for (let i = 0; i < val.length; i++) {
+        out.push(snapshot(cfg, val[i], i, depth + 1, seen))
+      }
+      return out
+    }
+
+    if (val instanceof Error) {
+      const out: any = {
+        message: cleanString(cfg, String(val.message)),
+        stack: cleanString(cfg, String(val.stack || '')),
+      }
+      for (const k of Object.keys(val)) {
+        const v = snapshot(cfg, (val as any)[k], k, depth + 1, seen)
+        if (undefined !== v) out[k] = v
+      }
+      return out
+    }
+
+    if ('function' === typeof (val as any).toJSON) {
+      const json = (val as any).toJSON()
+      return json === val ? plain(cfg, val, depth, seen) : snapshot(cfg, json, key, depth + 1, seen)
+    }
+
+    return plain(cfg, val, depth, seen)
+  }
+  finally {
+    seen.pop()
+  }
+}
+
+
+function plain(cfg: CleanConfig, val: any, depth: number, seen: any[]): any {
+  const out: any = {}
+  for (const k of Object.keys(val)) {
+    const v = snapshot(cfg, val[k], k, depth + 1, seen)
+    if (undefined !== v) out[cleanName(cfg, out, k)] = v
+  }
+  return out
+}
+
+
+// A registered value used as a property name is masked like any other
+// string; names that mask alike take a counter, so none is lost.
+function cleanName(cfg: CleanConfig, out: any, key: string): string {
+  const name = cleanString(cfg, key)
+  if (name === key || !Object.prototype.hasOwnProperty.call(out, name)) {
+    return name
+  }
+  let i = 1
+  while (Object.prototype.hasOwnProperty.call(out, name + '#' + i)) i++
+  return name + '#' + i
+}
+
+
+// An Error is cleaned in place, since it is about to be thrown.
 function clean(ctx: Context, val: any) {
-  const options = ctx.options
+  const cfg = cleanConfig(ctx)
 
-  const cleankeyre = options?.__derived__?.clean?.keyre
-  const hintsize = 4
+  if (false === cfg.active) {
+    return val
+  }
+
+  if ('string' === typeof val) {
+    return cleanString(cfg, val)
+  }
+
+  if (val instanceof Error) {
+    val.message = cleanString(cfg, String(val.message))
+    if ('string' === typeof val.stack) {
+      val.stack = cleanString(cfg, val.stack)
+    }
+    for (const k of Object.keys(val)) {
+      let v = (val as any)[k]
+      if ('string' === typeof v) {
+        v = sensitiveKey(cfg, k) ? maskValue(cfg, v) : cleanString(cfg, v)
+      }
+      else if (null != v && 'object' === typeof v) {
+        v = snapshot(cfg, v, k, 1, [])
+      }
+      const name = cleanString(cfg, k)
+      if (name !== k) {
+        delete (val as any)[k]
+      }
+      (val as any)[name === k ? k : cleanName(cfg, val, k)] = v
+    }
+    return val
+  }
+
+  return snapshot(cfg, val, undefined, 0, [])
+}
+
+
+function cleanKey(ctx: Context, key: any): boolean {
+  return sensitiveKey(cleanConfig(ctx), key)
+}
 
 
-  return val
+// Every scalar under a sensitive name, at any depth and of any shape: a
+// credential mistyped as an object or a number is still a credential, and
+// the validation error that rejects it quotes it.
+function cleanAddSensitive(
+  ctx: Context, val: any, under = false, depth = 0, seen: any[] = [], parent?: string
+): void {
+  if (null == val || MAXDEPTH <= depth) {
+    return
+  }
+  const t = typeof val
+  if ('string' === t || 'number' === t || 'bigint' === t) {
+    if (under) cleanAdd(ctx, String(val))
+    return
+  }
+  if ('object' !== t || seen.includes(val)) {
+    return
+  }
+  seen.push(val)
+  // The keys under `feature` name features, not fields: `secrets` is no secret.
+  const named = !(1 === depth && 'feature' === parent)
+  for (const k of Object.keys(val)) {
+    cleanAddSensitive(ctx, val[k], under || (named && cleanKey(ctx, k)), depth + 1, seen, k)
+  }
 }
 
 
 export {
-  clean
+  clean,
+  cleanAdd,
+  cleanAddSensitive,
+  cleanKey,
+  makeCleanConfig,
+  splitvalues,
 }

package/project/.sdk/tm/ts/src/utility/DoneUtility.ts

@@ -7,12 +7,8 @@ import { clean } from './CleanUtility'
 
 function done(ctx: Context) {
   const error = ctx.utility.makeError
-  const delprop = ctx.utility.struct.delprop
 
-  if (ctx.ctrl.explain) {
-    ctx.ctrl.explain = clean(ctx, ctx.ctrl.explain)
-    delprop(ctx.ctrl.explain.result, 'err')
-  }
+  cleanExplain(ctx)
 
   if (ctx.result && ctx.result.ok) {
     return ctx.result.resdata
@@ -22,6 +18,23 @@ function done(ctx: Context) {
 }
 
 
+// In place: the caller may hold the record, and a stream copies only its ctrl.
+function cleanExplain(ctx: Context) {
+  const explain = ctx.ctrl.explain
+  if (null == explain || 'object' !== typeof explain) {
+    return
+  }
+  const cleaned = clean(ctx, explain)
+  if (cleaned !== explain) {
+    for (const k of Object.keys(explain)) delete explain[k]
+    Object.assign(explain, cleaned)
+  }
+  // With clean off, explain.result is the live result makeError reads.
+  if (null != explain.result) explain.result = ctx.utility.struct.delprop({ ...explain.result }, 'err')
+}
+
+
 export {
+  cleanExplain,
   done
 }

package/project/.sdk/tm/ts/src/utility/MakeErrorUtility.ts

@@ -22,9 +22,23 @@ function makeError(ctx: Context, err?: any) {
   err = undefined === err ? reserr : err
   err = err || ctx.error('unknown', 'unknown error')
 
+  // A hook or fetcher may reject with a plain value; clean returns a masked
+  // copy of that rather than changing it, so the copy is what leaves.
+  if (!(err instanceof Error)) {
+    const copy = clean(ctx, err)
+    const text = 'string' === typeof copy ? copy : String(copy?.message ?? 'unknown error')
+    err = Object.assign(new Error(text), 'object' === typeof copy ? copy : {})
+  }
+
   const errmsg = err.message || 'unknown error'
-  const msg = 'ProjectNameSDK: ' + op.name + ': ' + errmsg
-  err.message = clean(ctx, msg)
+  err.message = 'ProjectNameSDK: ' + op.name + ': ' + errmsg
+
+  // Reachable for a debugger, invisible to a serialiser.
+  if (null != err.ctx) {
+    Object.defineProperty(err, 'ctx', { value: err.ctx, enumerable: false, writable: true })
+  }
+
+  clean(ctx, err)
 
   if (result.err) {
     delprop(result, 'err')
@@ -43,16 +57,12 @@ function makeError(ctx: Context, err?: any) {
   err.result = clean(ctx, result)
   err.spec = clean(ctx, spec)
 
-  // Promote the HTTP status to the top level, so a consumer can branch on
-  // `err.status` / `err.notFound` instead of reaching into `err.result`.
+  // So a consumer branches on `err.status`, not on the shape of `err.result`.
   err.status = null == result.status ? -1 : result.status
 
   ctx.ctrl.err = err
 
-  // Fire PreUnexpected so observability features (metrics, telemetry, audit,
-  // debug) close/record error paths that never reach PreDone (e.g. a PrePoint
-  // rbac short-circuit). Fires after ctx.ctrl.err is set so hooks can read the
-  // error; features guard against double-recording when PreDone already fired.
+  // Closes error paths that never reach PreDone (e.g. an rbac short-circuit).
   if (null != ctx.client && null != ctx.utility &&
     'function' === typeof ctx.utility.featureHook) {
     ctx.utility.featureHook(ctx, 'PreUnexpected')

package/project/.sdk/tm/ts/src/utility/MakeOptionsUtility.ts

@@ -2,6 +2,8 @@
 import { Context } from '../types'
 import { OPTSPEC } from '../Schema'
 
+import { clean, cleanAdd, cleanAddSensitive, makeCleanConfig, splitvalues } from './CleanUtility'
+
 
 function makeOptions(ctx: Context) {
   const utility = ctx.utility
@@ -11,12 +13,25 @@ function makeOptions(ctx: Context) {
   const setprop = struct.setprop
   const merge = struct.merge
   const validate = struct.validate
-  const escre = struct.escre
 
   let opts = { ...(options || {}) }
 
   const authSuppressed = null === (options || {}).auth
 
+  let config = ctx.config || {}
+  let cfgopts = config.options || {}
+
+  // The registry exists BEFORE validation, so rejecting a mistyped credential
+  // is clean too. An absent block is left out, or merge would erase the defaults.
+  const layer = (b: any) => null != b && 'object' === typeof b && !Array.isArray(b) ? b : {}
+  const cleancfg = makeCleanConfig(merge([{}, (OPTSPEC as any).clean,
+    struct.clone(layer(cfgopts.clean)), layer(opts.clean)]))
+  const cleanctx: any = { options: { __derived__: { clean: cleancfg } } }
+  cleanAddSensitive(cleanctx, settings(opts))
+  for (const raw of [...splitvalues(cfgopts.clean?.values), ...splitvalues(opts.clean?.values)]) {
+    cleanAdd(cleanctx, raw)
+  }
+
   let featureorder: string[] = []
   if (Array.isArray(opts.feature)) {
     const fmap: any = {}
@@ -35,9 +50,6 @@ function makeOptions(ctx: Context) {
     setprop(utility, key, val)
   }
 
-  let config = ctx.config || {}
-  let cfgopts = config.options || {}
-
   const optspec = OPTSPEC
 
   // Clone the config side before merging: `config` is a module-level
@@ -46,7 +58,12 @@ function makeOptions(ctx: Context) {
   // contaminate every instance constructed after it.
   opts = merge([{}, struct.clone(cfgopts), opts])
 
-  opts = validate(opts, optspec)
+  try {
+    opts = validate(opts, optspec)
+  }
+  catch (err: any) {
+    throw clean(cleanctx, err)
+  }
 
   opts.system = opts.system || {}
   if (null == opts.system.fetch) {
@@ -96,25 +113,30 @@ function makeOptions(ctx: Context) {
   }
 
   opts.__derived__ = {
-    clean: {
-      keyre: undefined
-    },
+    clean: cleancfg,
     featureorder,
   }
 
-  const keyre = opts.clean.keys
-    .split(/\s*,\s*/)
-    .filter((s: string) => null != s && '' !== s)
-    .map((key: string) => escre(key)).join('|')
-
-  if ('' != keyre) {
-    opts.__derived__.clean.keyre = keyre
-  }
+  // Again over the merged result: the config's own defaults can carry one.
+  cleanAddSensitive({ options: opts } as any, settings(opts))
 
   return opts
 }
 
 
+// Registration skips entity blocks: entity settings and seed records hold no credential.
+function settings(opts: any): any {
+  const noent = (b: any) => null != b && 'object' === typeof b && !Array.isArray(b)
+    ? { ...b, entity: undefined } : b
+  const feature = Array.isArray(opts.feature) ? opts.feature.map(noent)
+    : null != opts.feature && 'object' === typeof opts.feature
+      ? Object.fromEntries(Object.entries(opts.feature).map(([k, v]) => [k, noent(v)]))
+      : opts.feature
+  return { ...opts, clean: undefined, __derived__: undefined, entity: undefined,
+    test: noent(opts.test), feature }
+}
+
+
 export {
   makeOptions
 }

package/project/.sdk/tm/ts/src/utility/Utility.ts

@@ -1,7 +1,7 @@
 
 
-import { clean } from './CleanUtility'
-import { done } from './DoneUtility'
+import { clean, cleanAdd } from './CleanUtility'
+import { cleanExplain, done } from './DoneUtility'
 import { makeError } from './MakeErrorUtility'
 import { featureAdd } from './FeatureAddUtility'
 import { featureHook } from './FeatureHookUtility'
@@ -37,6 +37,8 @@ import { StructUtility } from './StructUtility'
 class Utility {
 
   clean = clean
+  cleanAdd = cleanAdd
+  cleanExplain = cleanExplain
   done = done
   makeError = makeError
   featureAdd = featureAdd

package/project/.sdk/tm/ts/test/feature/harness.ts

@@ -80,6 +80,9 @@ function makeClient(spec: {
 
   const utility: any = {
     struct,
+    // Features pass every record they emit through the SDK's own clean.
+    clean: (stdutil as any).clean,
+    cleanAdd: (stdutil as any).cleanAdd,
     fetcher: server,
     param: (ctx: any, name: string) => {
       const p = (ctx.spec && ctx.spec.params) || {}
@@ -88,10 +91,16 @@ function makeClient(spec: {
     },
   }
 
+  // One clean registry per client, as makeOptions gives a real one, so a value
+  // a feature registers with cleanAdd is masked by every clean after it.
+  const derived = (stdutil as any).makeOptions({
+    utility, options: { feature: { test: { active: true } } }, config: {},
+  }).__derived__
+
   const client: any = {
     _mode: spec.mode || 'test',
     _features: [],
-    _options: { base, headers: spec.headers || {}, feature: {} },
+    _options: { base, headers: spec.headers || {}, feature: {}, __derived__: derived },
     options() { return this._options },
     utility() { return utility },
   }
@@ -111,6 +120,7 @@ function makeClient(spec: {
       id: 'C' + idseq,
       client,
       utility,
+      options: client._options,
       out: {},
       ctrl: over.ctrl || {},
       meta: {},

package/project/.sdk/tm/ts/test/feature.test.ts

@@ -384,7 +384,7 @@ describe('feature', () => {
       const entries = h.client._debug.entries
       strictEqual(entries.length, 1) // ring buffer capped at max
       strictEqual(seen.length, 2)
-      strictEqual(seen[0].headers.authorization, '<redacted>')
+      strictEqual(seen[0].headers.authorization, '[redacted]')
     })
 
     test('captures failures', { skip: skipWithout('netsim') }, async () => {
@@ -518,6 +518,13 @@ describe('feature', () => {
       await h.op({ op: 'load' })
       strictEqual(rec.calls[0].fetchdef.proxy, undefined)
     })
+
+    test('masks the credentials in the URL it reports', async () => {
+      const h = makeClient({ features: [{ name: 'proxy', options: { url: 'http://pxuser:pxs3cret@proxy:8080' } }] })
+      await h.op({ op: 'load' })
+      strictEqual(h.client._proxy.url.includes('pxs3cret'), false)
+      strictEqual(h.client._proxy.url.includes('pxuser'), false)
+    })
   })
 
 
@@ -763,7 +770,7 @@ describe('feature', () => {
     const h = makeClient({ features: [{ name: 'debug', options: { now: () => 7, redact: ['x-secret'] } }] })
     await h.op({ op: 'load', headers: { 'x-secret': 'hide', 'x-ok': 'show' } })
     const e = h.client._debug.entries[0]
-    strictEqual(e.headers['x-secret'], '<redacted>')
+    strictEqual(e.headers['x-secret'], '[redacted]')
     strictEqual(e.headers['x-ok'], 'show')
   })
 

package/project/.sdk/tm/zig/core/context.zig

@@ -238,6 +238,40 @@ pub const Context = struct {
         return self.utility orelse unreachable;
     }
 
+    // The record a logger or an error dump sees, cleaned; the live fields
+    // stay raw for the pipeline's own use.
+    pub fn to_value(self: *Context) Value {
+        const record = h.omap();
+        h.setp(record, "id", h.vstr(self.id));
+        h.setp(record, "op", h.jo(&.{
+            .{ "entity", h.vstr(self.op.entity) },
+            .{ "name", h.vstr(self.op.name) },
+            .{ "input", h.vstr(self.op.input) },
+        }));
+        if (self.spec) |sp| h.setp(record, "spec", sp.to_value());
+        if (self.entity) |e| h.setp(record, "entity", h.vstr(e.get_name()));
+        if (self.result) |r| h.setp(record, "result", r.to_value());
+        if (self.response) |rp| {
+            h.setp(record, "response", h.jo(&.{
+                .{ "status", h.vnum(rp.status) },
+                .{ "statusText", h.vstr(rp.status_text) },
+                .{ "headers", rp.headers },
+                .{ "body", rp.body },
+            }));
+        }
+        h.setp(record, "meta", self.meta);
+        return utility_mod.clean_util(self, record);
+    }
+
+    pub fn to_json(self: *Context) []const u8 {
+        return h.jsonify_compact(self.to_value());
+    }
+
+    pub fn format(self: *const Context, writer: *std.Io.Writer) std.Io.Writer.Error!void {
+        try writer.writeAll("Context ");
+        try writer.writeAll(@constCast(self).to_json());
+    }
+
     // --- ctx.out staging helpers ---
 
     pub fn out_get(self: *Context, key: []const u8) ?OutVal {

package/project/.sdk/tm/zig/core/error.zig

@@ -3,6 +3,7 @@
 // with the rich error object stashed on the context's pending_err, then read
 // back by makeError. This templated file only needs the project name.
 
+const std = @import("std");
 const vs = @import("voxgig-struct");
 const mem = @import("mem.zig");
 const Value = vs.JsonValue;
@@ -11,7 +12,8 @@ pub const ProjectNameError = struct {
     sdk: []const u8 = "ProjectName",
     code: []const u8,
     msg: []const u8,
-    // Cleaned snapshots attached by makeError (null until then).
+    // Cleaned snapshots attached by makeError (null until then). The context
+    // is not on the error at all, so what prints is what makeError cleaned.
     result: Value = .{ .null = {} },
     spec: Value = .{ .null = {} },
 
@@ -22,6 +24,25 @@ pub const ProjectNameError = struct {
         e.* = .{ .sdk = "ProjectName", .code = code, .msg = msg };
         return e;
     }
+
+    pub fn to_json(self: *const ProjectNameError) []const u8 {
+        const out = Value.makeMap(mem.a()) catch return "{}";
+        out.object.put("sdk", .{ .string = self.sdk }) catch {};
+        out.object.put("code", .{ .string = self.code }) catch {};
+        out.object.put("msg", .{ .string = self.msg }) catch {};
+        out.object.put("result", self.result) catch {};
+        out.object.put("spec", self.spec) catch {};
+        return vs.jsonifyCompact(mem.a(), out) catch "{}";
+    }
+
+    pub fn to_string(self: *const ProjectNameError) []const u8 {
+        const spec = vs.jsonifyCompact(mem.a(), self.spec) catch "null";
+        return std.fmt.allocPrint(mem.a(), "{s} [{s}] spec={s}", .{ self.msg, self.code, spec }) catch self.msg;
+    }
+
+    pub fn format(self: *const ProjectNameError, writer: *std.Io.Writer) std.Io.Writer.Error!void {
+        try writer.writeAll(self.to_string());
+    }
 };
 
 // The pipeline error set. The payload travels via ctx.pending_err.