@voxgig/sdkgen 4.32.0 → 4.33.0

package/project/.sdk/tm/csharp/utility/MakeOptions.cs

@@ -1,6 +1,7 @@
 // ProjectName SDK utility: makeOptions - merge, validate and derive the
 // client options.
 
+using System.Collections;
 using System.Text.RegularExpressions;
 
 using Voxgig.Struct;
@@ -134,6 +135,40 @@ public static partial class SdkUtility
             opts.Remove("auth");
         }
 
+        var config = ctx.Config ?? new Dictionary<string, object?>();
+        var cfgopts = config.TryGetValue("options", out var co) &&
+            co is Dictionary<string, object?> cm
+            ? cm : new Dictionary<string, object?>();
+
+        // The secret registry exists BEFORE validation, fed from the raw
+        // input, so the constructor's own rejection of a mistyped credential
+        // is clean too.
+        var cleanlayers = new List<object?>
+        {
+            new Dictionary<string, object?>(),
+            StructUtils.Clone(StructUtils.GetProp(SdkSchema.Optspec, "clean")),
+            StructUtils.Clone(StructUtils.GetProp(cfgopts, "clean")) ?? new Dictionary<string, object?>(),
+        };
+        if (opts.TryGetValue("clean", out var cleanraw) && cleanraw != null)
+        {
+            cleanlayers.Add(cleanraw);
+        }
+        var cleancfg = MakeCleanConfig(StructUtils.Merge(cleanlayers));
+        var cleanctx = new Context(new Dictionary<string, object?>
+        {
+            ["options"] = new Dictionary<string, object?>
+            {
+                ["__derived__"] = new Dictionary<string, object?> { ["clean"] = cleancfg },
+            },
+        }, null);
+        CleanAddOptions(cleanctx, opts, "clean");
+        foreach (var raw in CleanSplitValues(
+                StructUtils.GetPath(cfgopts, StructUtils.Jt("clean", "values")))
+            .Concat(CleanSplitValues(StructUtils.GetPath(opts, StructUtils.Jt("clean", "values")))))
+        {
+            CleanAddUtil(cleanctx, raw);
+        }
+
         // Feature add-order. options.feature may be given as an ordered LIST of
         // { name, active, ...opts } entries (the list position IS the order in
         // which features are added), or as a { name: {opts} } map. Normalize a
@@ -159,11 +194,6 @@ public static partial class SdkUtility
             opts["feature"] = fmap;
         }
 
-        var config = ctx.Config ?? new Dictionary<string, object?>();
-        var cfgopts = config.TryGetValue("options", out var co) &&
-            co is Dictionary<string, object?> cm
-            ? cm : new Dictionary<string, object?>();
-
         // THE OPTION SPEC IS GENERATED, NOT WRITTEN HERE.
         //
         // Built from the model: `main.kit.optspec` for the standard options,
@@ -192,7 +222,24 @@ public static partial class SdkUtility
             StructUtils.Clone(cfgopts),
             opts,
         });
-        var validated = StructUtils.Validate(merged, optspec);
+        object? validated;
+        try
+        {
+            validated = StructUtils.Validate(merged, optspec);
+        }
+        catch (Exception verr)
+        {
+            // The struct port's message quotes the offending value, which is
+            // the credential when that is what was mistyped. Exception's
+            // message is read-only, so a changed message travels as the SDK's
+            // own error; an unchanged one is the original, rethrown.
+            var cleaned = CleanUtil(cleanctx, verr.Message) as string ?? verr.Message;
+            if (cleaned == verr.Message)
+            {
+                throw;
+            }
+            throw new ProjectNameError("options_invalid", cleaned, null);
+        }
         opts = validated as Dictionary<string, object?> ?? new Dictionary<string, object?>();
 
         // Restore the suppression the optspec default would otherwise erase.
@@ -263,20 +310,6 @@ public static partial class SdkUtility
             }
         }
 
-        // Derived clean config.
-        var cleanKeys = "key,token,id";
-        if (StructUtils.GetPath(opts, StructUtils.Jt("clean", "keys")) is string cks)
-        {
-            cleanKeys = cks;
-        }
-
-        var filtered = cleanKeys.Split(',')
-            .Select(p => p.Trim())
-            .Where(p => p != "")
-            .Select(StructUtils.EscRe)
-            .ToList();
-        var keyre = string.Join("|", filtered);
-
         // Resolve the feature add-order: an explicit list order (above) wins;
         // otherwise order the map test-first, then the remaining names sorted,
         // so the outcome is deterministic and `test` is always the base
@@ -319,20 +352,77 @@ public static partial class SdkUtility
             }
         }
 
-        var derived = new Dictionary<string, object?>
+        opts["__derived__"] = new Dictionary<string, object?>
         {
-            ["clean"] = new Dictionary<string, object?>(),
+            ["clean"] = cleancfg,
+            ["featureorder"] = featureorder,
         };
-        if (keyre != "")
+
+        // Again over the merged result: the config's own defaults can carry one.
+        CleanAddOptions(cleanctx, opts, "clean", "__derived__");
+
+        return opts;
+    }
+
+    // A feature's name is not a field name: a feature called `secrets` does
+    // not make every one of its options a secret. Entity blocks (entity
+    // settings, seeded records) hold no credential.
+    private static void CleanAddOptions(Context cleanctx, Dictionary<string, object?> opts,
+        params string[] omit)
+    {
+        var top = CleanOmit(opts, omit.Append("feature").Append("entity").ToArray());
+        if (top.ContainsKey("test"))
+        {
+            top["test"] = CleanNoEntity(top["test"]);
+        }
+        CleanAddSensitive(cleanctx, top);
+        var feature = opts.GetValueOrDefault("feature");
+        if (feature is IDictionary fmap)
+        {
+            foreach (DictionaryEntry kv in fmap)
+            {
+                CleanAddSensitive(cleanctx, CleanNoEntity(kv.Value));
+            }
+        }
+        else if (feature is IList flist)
+        {
+            foreach (var entry in flist)
+            {
+                CleanAddSensitive(cleanctx, CleanNoEntity(entry));
+            }
+        }
+        else
+        {
+            CleanAddSensitive(cleanctx, feature);
+        }
+    }
+
+    private static object? CleanNoEntity(object? block)
+    {
+        if (block is not IDictionary dict)
         {
-            derived["clean"] = new Dictionary<string, object?>
+            return block;
+        }
+        var out_ = new Dictionary<string, object?>();
+        foreach (DictionaryEntry kv in dict)
+        {
+            var key = Convert.ToString(kv.Key) ?? "";
+            if (key != "entity")
             {
-                ["keyre"] = keyre,
-            };
+                out_[key] = kv.Value;
+            }
         }
-        derived["featureorder"] = featureorder;
-        opts["__derived__"] = derived;
+        return out_;
+    }
 
-        return opts;
+    private static Dictionary<string, object?> CleanOmit(Dictionary<string, object?> opts,
+        params string[] keys)
+    {
+        var out_ = new Dictionary<string, object?>(opts);
+        foreach (var key in keys)
+        {
+            out_.Remove(key);
+        }
+        return out_;
     }
 }

package/project/.sdk/tm/csharp/utility/MakeResponse.cs

@@ -35,21 +35,31 @@ public static partial class SdkUtility
 
         spec.Step = "response";
 
-        utility.ResultBasic(ctx);
-        utility.ResultHeaders(ctx);
-        utility.ResultBody(ctx);
+        // A shaping failure (a body that is not JSON) is the operation's
+        // result, as in ts: carried on result.Err so the pipeline still
+        // reaches Done, which is what cleans the explain record.
+        try
+        {
+            utility.ResultBasic(ctx);
+            utility.ResultHeaders(ctx);
+            utility.ResultBody(ctx);
 
-        // GraphQL reports failures as a top-level `errors` array under HTTP
-        // 200, so ResultBasic's status check never sees them. Lift them
-        // here, before the response transform tries to unwrap data that is
-        // not there.
-        utility.GraphqlErrors(ctx);
+            // GraphQL reports failures as a top-level `errors` array under HTTP
+            // 200, so ResultBasic's status check never sees them. Lift them
+            // here, before the response transform tries to unwrap data that is
+            // not there.
+            utility.GraphqlErrors(ctx);
 
-        utility.TransformResponse(ctx);
+            utility.TransformResponse(ctx);
 
-        if (result.Err == null)
+            if (result.Err == null)
+            {
+                result.Ok = true;
+            }
+        }
+        catch (Exception err)
         {
-            result.Ok = true;
+            result.Err = err;
         }
 
         if (ctx.Ctrl.Explain != null)

package/project/.sdk/tm/csharp/utility/Register.cs

@@ -9,6 +9,7 @@ public static partial class SdkUtility
     public static void RegisterAll(Utility u)
     {
         u.Clean = CleanUtil;
+        u.CleanAdd = CleanAddUtil;
         u.Done = DoneUtil;
         u.MakeError = MakeErrorUtil;
         u.FeatureAdd = FeatureAddUtil;

package/project/.sdk/tm/elixir/lib/projectname/context.ex

@@ -21,6 +21,25 @@ defmodule ProjectName.Context do
 
   def make_error(ctx, code, msg), do: Error.new(code, msg, ctx)
 
+  # The serialised context leaves the pipeline (a logger, an error dump), so
+  # it is the cleaned record; the live fields stay raw for the pipeline's use.
+  # A context node itself inspects as a handle, so this is the SDK's own
+  # serialisation of one.
+  def to_data(ctx) do
+    record =
+      S.jm([
+        "id", S.getprop(ctx, "id"),
+        "op", S.getprop(ctx, "op"),
+        "spec", S.getprop(ctx, "spec"),
+        "entity", entity_name(S.getprop(ctx, "entity")),
+        "result", S.getprop(ctx, "result"),
+        "response", S.getprop(ctx, "response"),
+        "meta", S.getprop(ctx, "meta")
+      ])
+
+    ProjectName.Feature.clean(ctx, record)
+  end
+
   def new(ctxmap \\ nil, basectx \\ nil) do
     cm = if S.ismap(ctxmap), do: ctxmap, else: nil
     gp = fn k -> H.get_ctx_prop(cm, k) end

package/project/.sdk/tm/elixir/lib/projectname/entity_base.ex

@@ -153,7 +153,7 @@ defmodule ProjectName.EntityBase do
     # materialised items so stream always yields.
     items =
       if S.isfunc(stream_fn) do
-        stream_fn.()
+        cleaned(ctx, stream_fn)
       else
         rd = if result != nil, do: S.getprop(result, "resdata"), else: nil
 
@@ -167,6 +167,27 @@ defmodule ProjectName.EntityBase do
     items
     |> Stream.take_while(fn _ -> not (S.isfunc(signal) and signal.() == true) end)
     |> Stream.map(&stream_unwrap/1)
+    |> guarded(ctx)
+  end
+
+  # The caller iterates after run_op has returned, so what the stream raises
+  # is cleaned here, as run_op's rescue would clean it. The caller's own
+  # reducer runs inside the reduce, so what it raises is cleaned too.
+  defp cleaned(ctx, step) do
+    step.()
+  rescue
+    e -> reraise(Utility.clean_exception(ctx, e), __STACKTRACE__)
+  end
+
+  defp guarded(enum, ctx) do
+    fn acc, fun -> reduce_step(ctx, fn -> Enumerable.reduce(enum, acc, fun) end) end
+  end
+
+  defp reduce_step(ctx, step) do
+    case cleaned(ctx, step) do
+      {:suspended, acc, cont} -> {:suspended, acc, fn a -> reduce_step(ctx, fn -> cont.(a) end) end}
+      done -> done
+    end
   end
 
   # Unwrap an entity node to its bare record; recurse into chunk lists.

package/project/.sdk/tm/elixir/lib/projectname/error.ex

@@ -13,4 +13,28 @@ defmodule ProjectName.Error do
   def new(code \\ "", msg \\ "", ctx \\ nil) do
     %__MODULE__{code: code, msg: msg, sdk: "ProjectName", ctx: ctx}
   end
+
+  # The HTTP status, promoted so a caller can branch without reaching into
+  # the result's shape; -1 when there was no response.
+  def status(%__MODULE__{result: result}) do
+    s = if Voxgig.Struct.ismap(result), do: Voxgig.Struct.getprop(result, "status"), else: nil
+    if is_number(s), do: trunc(s), else: -1
+  end
+
+  def not_found?(%__MODULE__{} = e), do: 404 == status(e)
+end
+
+# The context holds the live spec and options, and an error is what gets
+# logged: the printed form carries what make_error attached, which is already
+# cleaned, and never the context. A struct node prints as its JSON, since a
+# bare handle says nothing.
+defimpl Inspect, for: ProjectName.Error do
+  import Inspect.Algebra
+  alias Voxgig.Struct, as: S
+
+  def inspect(e, opts) do
+    show = fn v -> if S.isnode(v), do: S.jsonify(v, S.jm(["indent", 0])), else: v end
+    fields = [code: e.code, msg: e.msg, sdk: e.sdk, result: show.(e.result), spec: show.(e.spec)]
+    concat(["#ProjectName.Error<", to_doc(fields, opts), ">"])
+  end
 end

package/project/.sdk/tm/elixir/lib/projectname/feature/audit.ex

@@ -76,7 +76,7 @@ defmodule ProjectName.Feature.Audit do
       status = if result != nil, do: S.getprop(result, "status"), else: nil
 
       record =
-        S.jm([
+        Feature.clean(ctx, S.jm([
           "seq", seq,
           "ts", Feature.now(f),
           "actor", actor,
@@ -85,7 +85,7 @@ defmodule ProjectName.Feature.Audit do
           "outcome", outcome,
           "status", status,
           "correlationId", S.getprop(ctx, "id")
-        ])
+        ]))
 
       client = S.getprop(f, "client")
       records = S.getprop(S.getprop(client, "_audit"), "records")

package/project/.sdk/tm/elixir/lib/projectname/feature/cost.ex

@@ -261,7 +261,7 @@ defmodule ProjectName.Feature.Cost do
     S.setprop(f, "seq", seq)
 
     record =
-      S.jm([
+      Feature.clean(ctx, S.jm([
         "seq", seq,
         "entity", entity,
         "op", opname,
@@ -270,7 +270,7 @@ defmodule ProjectName.Feature.Cost do
         "currency", S.getprop(cost, "currency"),
         "source", source,
         "attempts", S.getprop(pending, "attempts")
-      ])
+      ]))
 
     S.setprop(cost, "last", record)
 

package/project/.sdk/tm/elixir/lib/projectname/feature/debug.ex

@@ -54,7 +54,7 @@ defmodule ProjectName.Feature.Debug do
           "method", if(spec != nil, do: S.getprop(spec, "method"), else: nil),
           "url",
           if(spec != nil, do: H.or_(S.getprop(spec, "url"), S.getprop(spec, "path")), else: nil),
-          "headers", redact(f, if(spec != nil, do: S.getprop(spec, "headers"), else: nil)),
+          "headers", redact(f, ctx, if(spec != nil, do: S.getprop(spec, "headers"), else: nil)),
           "start", F.now(f),
           "status", nil,
           "ok", nil,
@@ -137,6 +137,11 @@ defmodule ProjectName.Feature.Debug do
         S.setprop(entry, "status", S.getprop(result, "status"))
       end
 
+      # The whole entry leaves through the buffer and the callback: the url
+      # and the error message can carry a query credential the header mask
+      # never saw.
+      entry = F.clean(ctx, entry)
+
       client = S.getprop(f, "client")
       buf = S.getprop(S.getprop(client, "_debug"), "entries")
       F.list_push(buf, entry)
@@ -158,32 +163,27 @@ defmodule ProjectName.Feature.Debug do
     end
   end
 
-  defp redact(f, headers) do
+  # The core clean rules apply (clean.keys, every registered value); the
+  # feature's own `redact` list ADDS header names on top of them.
+  defp redact(f, ctx, headers) do
     if headers == nil do
       S.jm([])
     else
       patterns =
-        opt_list(S.getprop(F.opts(f), "redact"), [
-          "authorization",
-          "cookie",
-          "set-cookie",
-          "api-key",
-          "apikey",
-          "x-api-key",
-          "idempotency-key"
-        ])
+        opt_list(S.getprop(F.opts(f), "redact"), [])
+        |> Enum.map(fn p -> String.downcase(to_string(p)) end)
 
       out = S.jm([])
 
       Enum.each(H.entries(headers), fn {k, v} ->
         if String.downcase(to_string(k)) in patterns do
-          S.setprop(out, k, "<redacted>")
+          S.setprop(out, k, "[redacted]")
         else
           S.setprop(out, k, v)
         end
       end)
 
-      out
+      F.clean(ctx, out)
     end
   end
 

package/project/.sdk/tm/elixir/lib/projectname/feature/log.ex

@@ -28,11 +28,13 @@ defmodule ProjectName.Feature.Log do
       logger = S.getprop(F.opts(f), "logger")
 
       logger =
-        if S.ismap(logger) do
+        if S.ismap(logger) or S.isfunc(logger) do
           logger
         else
           mk = fn level ->
-            fn msg -> IO.write(:stderr, "[" <> level <> "] " <> msg <> "\n") end
+            fn record ->
+              IO.write(:stderr, "[" <> level <> "] " <> S.jsonify(record, S.jm(["indent", 0])) <> "\n")
+            end
           end
 
           S.jm(["info", mk.("INFO"), "debug", mk.("DEBUG"), "warn", mk.("WARN"), "error", mk.("ERROR")])
@@ -44,6 +46,9 @@ defmodule ProjectName.Feature.Log do
     nil
   end
 
+  # A log line leaves the pipeline, so it carries the cleaned record: the
+  # spec after auth holds the credential, and a logger serialises whatever it
+  # is handed. `logger` is a fn of the record, or a map with an "info" fn.
   defp loghook(f, hook, ctx) do
     if F.active?(f) do
       logger = S.getprop(f, "logger")
@@ -51,9 +56,17 @@ defmodule ProjectName.Feature.Log do
       if logger != nil do
         op = S.getprop(ctx, "op")
         opname = if op != nil, do: S.getprop(op, "name"), else: ""
-        msg = "hook=" <> hook <> " op=" <> to_string(opname)
-        log_fn = S.getprop(logger, "info")
-        if S.isfunc(log_fn), do: log_fn.(msg)
+
+        record =
+          F.clean(ctx, S.jm([
+            "hook", hook,
+            "op", opname,
+            "spec", S.getprop(ctx, "spec"),
+            "ctx", ProjectName.Context.to_data(ctx)
+          ]))
+
+        log_fn = if S.isfunc(logger), do: logger, else: S.getprop(logger, "info")
+        if S.isfunc(log_fn), do: log_fn.(record)
       end
     end
 

package/project/.sdk/tm/elixir/lib/projectname/feature/proxy.ex

@@ -52,7 +52,25 @@ defmodule ProjectName.Feature.Proxy do
 
       np_list = Enum.filter(np_list, fn s -> s != nil and s != "" end)
 
+      # A proxy URL may carry credentials as userinfo, from the option or
+      # the environment, and neither is under a sensitive key name.
+      if is_binary(url) do
+        case URI.parse(url).userinfo do
+          nil ->
+            nil
+
+          ui ->
+            Enum.each(String.split(ui, ":", parts: 2), fn part ->
+              if part != "" do
+                F.clean_add(ctx, part)
+                F.clean_add(ctx, URI.decode(part))
+              end
+            end)
+        end
+      end
+
       S.setprop(f, "url", url)
+      S.setprop(f, "clean_url", if(is_binary(url), do: F.clean(ctx, url), else: url))
       S.setprop(f, "no_proxy", S.jt(np_list))
 
       utility = S.getprop(ctx, "utility")
@@ -112,7 +130,7 @@ defmodule ProjectName.Feature.Proxy do
 
   defp track(f) do
     client = S.getprop(f, "client")
-    track = F.track_node(client, "_proxy", S.jm(["routed", 0, "url", S.getprop(f, "url")]))
+    track = F.track_node(client, "_proxy", S.jm(["routed", 0, "url", S.getprop(f, "clean_url")]))
     S.setprop(track, "routed", S.getprop(track, "routed") + 1)
   end
 

package/project/.sdk/tm/elixir/lib/projectname/feature/secrets.ex

@@ -229,6 +229,8 @@ defmodule ProjectName.Feature.Secrets do
         if xopts == nil, do: nil, else: S.getprop(xopts, "refresh")
       end
 
+    register(f, explicit)
+
     first =
       if is_binary(explicit) and explicit != "" do
         [%Sekreto.ProviderSpec{
@@ -431,7 +433,18 @@ defmodule ProjectName.Feature.Secrets do
     end
   end
 
+  # Every value the chain resolves and every token bought can leave through
+  # the SDK's diagnostics, so each is registered with the clean utility the
+  # moment this feature sees it.
+  defp register(f, v) do
+    client = S.getprop(f, "client")
+    ctx = if client != nil, do: S.getprop(client, "_rootctx"), else: nil
+    if is_binary(v) and ctx != nil, do: F.clean_add(ctx, v)
+    nil
+  end
+
   defp settle(f, found) do
+    register(f, found)
     x = S.getprop(f, "_exchange")
 
     if x == nil do
@@ -637,11 +650,13 @@ defmodule ProjectName.Feature.Secrets do
     if S.getprop(client, "mode") != "live" do
       token = "test-" <> x.response
       S.setprop(f, "_cred", token)
+      register(f, token)
       {:ok, token}
     else
       case buy_once(f, x) do
         {:ok, token} ->
           S.setprop(f, "_cred", token)
+          register(f, token)
           {:ok, token}
 
         other ->

package/project/.sdk/tm/elixir/lib/projectname/feature/telemetry.ex

@@ -132,15 +132,17 @@ defmodule ProjectName.Feature.Telemetry do
       S.setprop(span, "durationMs", max(0, fin - S.getprop(span, "start")))
       S.setprop(span, "ok", ok)
 
+      out = F.clean(ctx, span)
+
       telemetry = S.getprop(S.getprop(f, "client"), "_telemetry")
       S.setprop(telemetry, "active", S.getprop(telemetry, "active") - 1)
-      F.list_push(S.getprop(telemetry, "spans"), span)
+      F.list_push(S.getprop(telemetry, "spans"), out)
 
       exporter = S.getprop(F.opts(f), "exporter")
 
       if S.isfunc(exporter) do
         try do
-          exporter.(span)
+          exporter.(out)
         rescue
           _ -> nil
         end

package/project/.sdk/tm/elixir/lib/projectname/feature.ex

@@ -42,6 +42,28 @@ defmodule ProjectName.Feature do
 
   def active?(f), do: S.getprop(f, "active") == true
 
+  # The utility's clean, or the shipped one when a caller-built utility (the
+  # test harness) carries no such member. What a feature emits, it cleans.
+  def clean(ctx, v) do
+    case util_member(ctx, "clean") do
+      nil -> ProjectName.Utility.clean_impl(ctx, v)
+      f -> f.(ctx, v)
+    end
+  end
+
+  def clean_add(ctx, v) do
+    case util_member(ctx, "clean_add") do
+      nil -> ProjectName.Utility.clean_add_impl(ctx, v)
+      f -> f.(ctx, v)
+    end
+  end
+
+  defp util_member(ctx, name) do
+    u = if S.ismap(ctx), do: S.getprop(ctx, "utility"), else: nil
+    f = if S.ismap(u), do: S.getprop(u, name), else: nil
+    if S.isfunc(f), do: f, else: nil
+  end
+
   # Injectable clock (ms) — options.now() when a function, else wall clock.
   def now(f) do
     n = S.getprop(opts(f), "now")