@voxgig/sdkgen 4.32.1 → 4.34.0

package/project/.sdk/tm/py/pkg/core/context.py

@@ -207,3 +207,20 @@ class ProjectNameContext:
 
     def make_error(self, code, msg):
         return ProjectNameError(code, msg, self)
+
+    # The serialised context leaves the pipeline (a logger, an error dump),
+    # so it is cleaned; the live fields stay raw for the pipeline's own use.
+    def to_json(self):
+        record = {
+            "id": self.id,
+            "op": self.op,
+            "spec": self.spec,
+            "result": self.result,
+            "response": self.response,
+            "meta": self.meta,
+        }
+        clean = getattr(self.utility, "clean", None) if self.utility is not None else None
+        return clean(self, record) if callable(clean) else record
+
+    def __repr__(self):
+        return self.__class__.__name__ + "(" + repr(self.to_json()) + ")"

package/project/.sdk/tm/py/pkg/core/error.py

@@ -4,15 +4,56 @@ from __future__ import annotations
 
 
 class ProjectNameError(Exception):
+    # The context holds the live spec and options, and an error is what gets
+    # logged: a slot keeps it reachable for a debugger and out of `vars()`,
+    # pickling and every attribute dump.
+    __slots__ = ("_ctx",)
+
     def __init__(self, code="", msg="", ctx=None):
         super().__init__(msg)
         self.is_sdk_error = True
         self.sdk = "ProjectName"
         self.code = code
         self.msg = msg
-        self.ctx = ctx
+        self._ctx = ctx
+        self.status = -1
         self.result = None
         self.spec = None
 
+    @property
+    def ctx(self):
+        return getattr(self, "_ctx", None)
+
+    @ctx.setter
+    def ctx(self, value):
+        self._ctx = value
+
+    # `err.not_found` rather than a magic number at every call site.
+    @property
+    def not_found(self):
+        return 404 == self.status
+
+    # What make_error attached is already cleaned; the context is not part
+    # of the record.
+    def to_json(self):
+        return {
+            "sdk": self.sdk,
+            "code": self.code,
+            "message": self.msg,
+            "status": self.status,
+            "result": self.result,
+            "spec": self.spec,
+        }
+
     def __str__(self):
         return self.msg
+
+    def __repr__(self):
+        return (self.__class__.__name__ + "(code=" + repr(self.code) +
+                ", msg=" + repr(self.msg) + ", status=" + repr(self.status) + ")")
+
+    def __getstate__(self):
+        return dict(self.__dict__)
+
+    def __reduce__(self):
+        return (self.__class__, (self.code, self.msg), self.__getstate__())

package/project/.sdk/tm/py/pkg/core/utility_type.py

@@ -11,6 +11,8 @@ class ProjectNameUtility:
 
         # All utility functions are set by the register module
         self.clean = None
+        self.clean_add = None
+        self.clean_explain = None
         self.done = None
         self.make_error = None
         self.feature_add = None
@@ -50,6 +52,8 @@ class ProjectNameUtility:
     def copy(cls, src):
         u = cls()
         u.clean = src.clean
+        u.clean_add = src.clean_add
+        u.clean_explain = src.clean_explain
         u.done = src.done
         u.make_error = src.make_error
         u.feature_add = src.feature_add

package/project/.sdk/tm/py/pkg/feature/audit_feature.py

@@ -67,7 +67,7 @@ class ProjectNameAuditFeature(ProjectNameBaseFeature):
         if actor is None:
             actor = "anonymous"
 
-        record = {
+        record = ctx.utility.clean(ctx, {
             "seq": self.seq,
             "ts": self._now(),
             "actor": actor,
@@ -76,7 +76,7 @@ class ProjectNameAuditFeature(ProjectNameBaseFeature):
             "outcome": outcome,
             "status": ctx.result.status if ctx.result is not None else None,
             "correlationId": ctx.id,
-        }
+        })
 
         records = self.client._audit["records"]
         records.append(record)

package/project/.sdk/tm/py/pkg/feature/cost_feature.py

@@ -249,7 +249,7 @@ class ProjectNameCostFeature(ProjectNameBaseFeature):
         self._bump(cost["actors"], actor, amount)
 
         self.seq += 1
-        record = {
+        record = ctx.utility.clean(ctx, {
             "seq": self.seq,
             "entity": entity,
             "op": opname,
@@ -258,7 +258,7 @@ class ProjectNameCostFeature(ProjectNameBaseFeature):
             "currency": cost["currency"],
             "source": source,
             "attempts": pending["attempts"],
-        }
+        })
         cost["last"] = record
 
         sink = self.options.get("sink")

package/project/.sdk/tm/py/pkg/feature/debug_feature.py

@@ -8,10 +8,11 @@ from projectname_sdk.feature.base_feature import ProjectNameBaseFeature
 
 # Request/response capture for debugging. Records a bounded ring buffer of
 # per-operation traces — method, URL, redacted headers, response status and
-# timing — on `client._debug["entries"]`. Sensitive header values (matching
-# `redact`, default authorization/cookie/api-key style names) are masked.
-# An optional `onEntry` callback receives each finished entry (e.g. to
-# stream to a console). `max` caps the buffer (default 100).
+# timing — on `client._debug["entries"]`. Every entry passes through the
+# SDK's clean rules (clean.keys and every registered secret value); the
+# `redact` option ADDS header names on top of them. An optional `onEntry`
+# callback receives each finished entry (e.g. to stream to a console).
+# `max` caps the buffer (default 100).
 class ProjectNameDebugFeature(ProjectNameBaseFeature):
     def __init__(self):
         super().__init__()
@@ -47,7 +48,7 @@ class ProjectNameDebugFeature(ProjectNameBaseFeature):
             "op": opname,
             "method": spec.method if spec is not None else None,
             "url": (spec.url or spec.path) if spec is not None else None,
-            "headers": self._redact(spec.headers if spec is not None else None),
+            "headers": self._redact(ctx, spec.headers if spec is not None else None),
             "start": self._now(),
             "status": None,
             "ok": None,
@@ -99,6 +100,11 @@ class ProjectNameDebugFeature(ProjectNameBaseFeature):
         if entry["status"] is None and result is not None:
             entry["status"] = result.status
 
+        # The whole entry leaves through the buffer and the callback: the url
+        # and the error message can carry a query credential the header mask
+        # above never saw.
+        entry = ctx.utility.clean(ctx, entry)
+
         buf = self.client._debug["entries"]
         buf.append(entry)
         mx = 100 if self.options.get("max") is None else self.options.get("max")
@@ -112,20 +118,19 @@ class ProjectNameDebugFeature(ProjectNameBaseFeature):
             except Exception:
                 pass
 
-    def _redact(self, headers):
+    # The core clean rules apply (clean.keys, every registered value); the
+    # feature's own `redact` list ADDS header names on top of them.
+    def _redact(self, ctx, headers):
         if headers is None:
             return {}
-        patterns = self.options.get("redact") or [
-            "authorization", "cookie", "set-cookie", "api-key",
-            "apikey", "x-api-key", "idempotency-key",
-        ]
+        patterns = [str(n).lower() for n in (self.options.get("redact") or [])]
         out = {}
         for key in headers:
             if str(key).lower() in patterns:
-                out[key] = "<redacted>"
+                out[key] = "[redacted]"
             else:
                 out[key] = headers[key]
-        return out
+        return ctx.utility.clean(ctx, out)
 
     def _now(self):
         now = self.options.get("now")

package/project/.sdk/tm/py/pkg/feature/log_feature.py

@@ -44,7 +44,8 @@ class ProjectNameLogFeature(ProjectNameBaseFeature):
         if ctx.op is not None:
             opname = ctx.op.name
 
-        msg = "hook=" + hook + " op=" + opname
+        # A log line leaves the pipeline, so it is cleaned like any record.
+        msg = ctx.utility.clean(ctx, "hook=" + hook + " op=" + opname)
 
         log_fn = self.logger.get(level)
         if callable(log_fn):

package/project/.sdk/tm/py/pkg/feature/proxy_feature.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 import os
 import re
+from urllib.parse import unquote, urlsplit
 
 from projectname_sdk.feature.base_feature import ProjectNameBaseFeature
 
@@ -27,6 +28,7 @@ class ProjectNameProxyFeature(ProjectNameBaseFeature):
         self.options = {}
         self.url = None
         self.no_proxy = []
+        self._ctx = None
 
     def init(self, ctx, options):
         self.client = ctx.client
@@ -54,6 +56,19 @@ class ProjectNameProxyFeature(ProjectNameBaseFeature):
             no_proxy = re.split(r"\s*,\s*", no_proxy)
         self.no_proxy = [s for s in (no_proxy or []) if s is not None and s != ""]
 
+        # A proxy URL may carry credentials as userinfo, from the option or
+        # the environment, and neither is under a sensitive key name.
+        self._ctx = ctx
+        if isinstance(self.url, str):
+            try:
+                parsed = urlsplit(self.url)
+                for part in (parsed.username, parsed.password):
+                    if part:
+                        ctx.utility.clean_add(ctx, part)
+                        ctx.utility.clean_add(ctx, unquote(part))
+            except Exception:
+                pass
+
         utility = ctx.utility
         inner = utility.fetcher
 
@@ -99,6 +114,8 @@ class ProjectNameProxyFeature(ProjectNameBaseFeature):
         client = self.client
         track = getattr(client, "_proxy", None)
         if track is None:
-            track = {"routed": 0, "url": self.url}
+            ctx = self._ctx
+            url = self.url if ctx is None else ctx.utility.clean(ctx, self.url)
+            track = {"routed": 0, "url": url}
             client._proxy = track
         track["routed"] += 1

package/project/.sdk/tm/py/pkg/feature/secrets_feature.py

@@ -153,6 +153,7 @@ class ProjectNameSecretsFeature(ProjectNameBaseFeature):
             explicit = xopts.get("refresh") if isinstance(xopts, dict) else None
 
         if isinstance(explicit, str) and "" != explicit:
+            self._register(explicit)
             providers.append({
                 "kind": "memory",
                 "name": "options",
@@ -198,6 +199,14 @@ class ProjectNameSecretsFeature(ProjectNameBaseFeature):
     def sekreto(self):
         return self._sekreto
 
+    # Every value this feature resolves or buys is a secret the SDK handles,
+    # and none arrives under an option key the intake registration saw.
+    def _register(self, value):
+        ctx = getattr(self._client, "_rootctx", None)
+        utility = getattr(ctx, "utility", None)
+        if callable(getattr(utility, "clean_add", None)):
+            utility.clean_add(ctx, value)
+
     def PreSpec(self, _ctx):
         return self.resolve()
 
@@ -231,6 +240,7 @@ class ProjectNameSecretsFeature(ProjectNameBaseFeature):
         # RAISES for "a store could not answer" - only the miss falls
         # through.
         found = self._sekreto.try_(self._secretname)
+        self._register(found)
 
         if self._exchange is None:
             if found is not None:
@@ -367,6 +377,8 @@ class ProjectNameSecretsFeature(ProjectNameBaseFeature):
                 "secrets: token exchange returned no '" + x["response"]
                 + "' field from " + url)
 
+        self._register(token)
+
         return token
 
     # Buy a token and try the request again when the API says the current

package/project/.sdk/tm/py/pkg/feature/telemetry_feature.py

@@ -101,14 +101,16 @@ class ProjectNameTelemetryFeature(ProjectNameBaseFeature):
         span["durationMs"] = max(0, span["end"] - span["start"])
         span["ok"] = ok
 
+        out = ctx.utility.clean(ctx, span)
+
         telemetry = self.client._telemetry
         telemetry["active"] -= 1
-        telemetry["spans"].append(span)
+        telemetry["spans"].append(out)
 
         exporter = self.options.get("exporter")
         if callable(exporter):
             try:
-                exporter(span)
+                exporter(out)
             except Exception:
                 pass
 

package/project/.sdk/tm/py/pkg/utility/clean.py

@@ -1,5 +1,315 @@
 # ProjectName SDK utility: clean
 
+from __future__ import annotations
+import base64
+import json
+from urllib.parse import quote
 
+from projectname_sdk.schema import OPTSPEC
+
+
+# Everything that leaves the pipeline passes through clean: the error, the
+# explain record, the serialised context, and whatever a feature emits.
+# Two layers: every registered secret VALUE (and its encoded forms) is
+# replaced wherever it appears in a string, and every value under a
+# sensitive KEY name is masked whatever it holds. Inside the pipeline data
+# stays raw, so a hook can still read the header it must add to.
+
+_MAXDEPTH = 32
+_CIRCULAR = "[circular]"
+
+# A dropped value (a function) - distinct from None, which is data.
+_DROP = object()
+
+
+def _normkey(key):
+    return str(key).lower().replace("-", "").replace("_", "")
+
+
+def _splitkeys(keys):
+    out = []
+    for part in str("" if keys is None else keys).split(","):
+        k = _normkey(part.strip())
+        if k != "":
+            out.append(k)
+    return out
+
+
+# The comma-separated literal values a caller registers; a list is taken
+# as-is for a caller that has one.
+def split_values(values):
+    if isinstance(values, list):
+        return [v for v in values if isinstance(v, str)]
+    out = []
+    for part in str("" if values is None else values).split(","):
+        v = part.strip()
+        if v != "":
+            out.append(v)
+    return out
+
+
+# The spec carries numbers as strings, so every target reads it alike.
+def _count(val, dflt):
+    try:
+        n = int(float(val))
+    except (TypeError, ValueError):
+        return dflt
+    return n if n >= 0 else dflt
+
+
+def _options_of(ctx):
+    if isinstance(ctx, dict):
+        return ctx.get("options")
+    return getattr(ctx, "options", None)
+
+
+# The derived block make_options builds; a context without options
+# (make_error is reached with a bare one) falls back to the schema defaults,
+# so nothing leaves raw for want of a constructor.
+def _clean_config(ctx):
+    options = _options_of(ctx)
+    derived = options.get("__derived__") if isinstance(options, dict) else None
+    cfg = derived.get("clean") if isinstance(derived, dict) else None
+    if isinstance(cfg, dict) and isinstance(cfg.get("values"), list):
+        return cfg
+    return make_clean_config(OPTSPEC.get("clean"))
+
+
+def make_clean_config(cleanopts):
+    opts = cleanopts if isinstance(cleanopts, dict) else {}
+    mask = opts.get("mask")
+    return {
+        "active": opts.get("active") is not False,
+        "keys": _splitkeys(opts.get("keys")),
+        "values": [],
+        "mask": mask if isinstance(mask, str) else "[redacted]",
+        "hint": _count(opts.get("hint"), 0),
+        "min": max(1, _count(opts.get("min"), 4)),
+    }
+
+
+# The encoded forms a value travels in: Basic and Bearer both carry base64,
+# a query credential is percent-encoded, and a JSON dump escapes it.
+def _forms(value):
+    out = [value]
+
+    def add(s):
+        if s != "" and s not in out:
+            out.append(s)
+
+    try:
+        add(base64.b64encode(value.encode("utf-8")).decode("ascii"))
+    except Exception:
+        pass
+    try:
+        add(quote(value, safe="!~*'()"))
+    except Exception:
+        pass
+    try:
+        add(json.dumps(value, ensure_ascii=False)[1:-1])
+        add(json.dumps(value)[1:-1])
+    except Exception:
+        pass
+    return out
+
+
+# Register a secret value. Idempotent; shorter than `min` is not a secret
+# the SDK can mask without blanking ordinary text.
+def clean_add_util(ctx, value):
+    cfg = _clean_config(ctx)
+    if not isinstance(value, str) or len(value) < cfg["min"]:
+        return
+    values = cfg["values"]
+    changed = False
+    for form in _forms(value):
+        if len(form) >= cfg["min"] and form not in values:
+            values.append(form)
+            changed = True
+    if changed:
+        values.sort(key=len, reverse=True)
+
+
+# Every scalar under a sensitive name, at any depth and of any shape: a
+# credential mistyped as a map or a number is still a credential, and the
+# validation error that rejects it quotes it.
+def clean_add_sensitive(ctx, val, under=False, depth=0, seen=None):
+    if val is None or _MAXDEPTH <= depth or isinstance(val, bool):
+        return
+    if isinstance(val, (str, int, float)):
+        if under:
+            text = str(int(val)) if isinstance(val, float) and val.is_integer() else str(val)
+            clean_add_util(ctx, text)
+        return
+    if not isinstance(val, (dict, list, tuple)):
+        return
+    seen = [] if seen is None else seen
+    if id(val) in seen:
+        return
+    seen.append(id(val))
+    if isinstance(val, dict):
+        for k, v in val.items():
+            clean_add_sensitive(ctx, v, under or clean_key(ctx, k), depth + 1, seen)
+    else:
+        for v in val:
+            clean_add_sensitive(ctx, v, under, depth + 1, seen)
+
+
+def _mask_value(cfg, value):
+    hint = cfg["hint"]
+    if 0 < hint and len(value) > 2 * hint:
+        return cfg["mask"] + value[-hint:]
+    return cfg["mask"]
+
+
+def _clean_string(cfg, text):
+    out = text
+    for value in cfg["values"]:
+        if value in out:
+            out = out.replace(value, _mask_value(cfg, value))
+    return out
+
+
+def _sensitive_key(cfg, key):
+    if key is None or isinstance(key, bool) or isinstance(key, (int, float)):
+        return False
+    nk = _normkey(key)
+    for k in cfg["keys"]:
+        if k in nk:
+            return True
+    return False
+
+
+def _is_function(val):
+    return callable(val) and not isinstance(val, type)
+
+
+# A plain-data copy of what is about to leave: a `to_json` hook is honoured
+# (a context serialises as its record), functions are dropped, cycles are
+# cut, and no live object is shared with the copy - masking the copy must
+# never mask the pipeline's own spec.
+def _snapshot(cfg, val, key, depth, seen):
+    if val is None:
+        return None
+
+    if isinstance(val, str):
+        return _mask_value(cfg, val) if _sensitive_key(cfg, key) else _clean_string(cfg, val)
+
+    if isinstance(val, (bool, int, float)):
+        return cfg["mask"] if _sensitive_key(cfg, key) else val
+
+    if _is_function(val):
+        return _DROP
+
+    if _MAXDEPTH <= depth or id(val) in seen:
+        return _CIRCULAR
+
+    if _sensitive_key(cfg, key):
+        return cfg["mask"]
+
+    seen.append(id(val))
+    try:
+        if isinstance(val, dict):
+            return _plain(cfg, val.items(), depth, seen)
+
+        if isinstance(val, (list, tuple, set, frozenset)):
+            out = []
+            for i, item in enumerate(val):
+                c = _snapshot(cfg, item, i, depth + 1, seen)
+                out.append(None if c is _DROP else c)
+            return out
+
+        if isinstance(val, BaseException):
+            out = {"message": _clean_string(cfg, str(val))}
+            for k, v in _public(val):
+                c = _snapshot(cfg, v, k, depth + 1, seen)
+                if c is not _DROP:
+                    out[k] = c
+            return out
+
+        to_json = getattr(val, "to_json", None)
+        if callable(to_json):
+            rec = to_json()
+            if rec is not val:
+                return _snapshot(cfg, rec, key, depth + 1, seen)
+
+        if hasattr(val, "__dict__"):
+            return _plain(cfg, _public(val), depth, seen)
+
+        return _clean_string(cfg, str(val))
+    finally:
+        seen.pop()
+
+
+# An object's own public attributes: the private ones hold the pipeline's
+# live references (an entity's client, an error's context).
+def _public(obj):
+    return [(k, v) for k, v in vars(obj).items()
+            if not str(k).startswith("_")]
+
+
+def _plain(cfg, items, depth, seen):
+    out = {}
+    for k, v in items:
+        c = _snapshot(cfg, v, k, depth + 1, seen)
+        if c is not _DROP:
+            out[_clean_name(cfg, out, str(k))] = c
+    return out
+
+
+# A registered value used as a property name is masked like any other
+# string; names that mask alike take a counter, so none is lost.
+def _clean_name(cfg, out, key):
+    name = _clean_string(cfg, key)
+    if name == key or name not in out:
+        return name
+    i = 1
+    while name + "#" + str(i) in out:
+        i += 1
+    return name + "#" + str(i)
+
+
+# An exception is cleaned IN PLACE: it is about to be raised, and its
+# identity matters to the caller. The message lives in `args`.
+def _clean_exception(cfg, err):
+    args = tuple(_clean_string(cfg, a) if isinstance(a, str) else a for a in err.args)
+    if args != err.args:
+        err.args = args
+
+    for k, v in _public(err):
+        if isinstance(v, str):
+            v = _mask_value(cfg, v) if _sensitive_key(cfg, k) else _clean_string(cfg, v)
+        elif v is not None and not isinstance(v, (bool, int, float)) and not _is_function(v):
+            v = _snapshot(cfg, v, k, 1, [])
+        name = _clean_string(cfg, k)
+        if name != k:
+            delattr(err, k)
+            name = _clean_name(cfg, vars(err), k)
+        setattr(err, name, v)
+
+    notes = getattr(err, "__notes__", None)
+    if isinstance(notes, list):
+        err.__notes__ = [_clean_string(cfg, n) if isinstance(n, str) else n for n in notes]
+
+
+# Clean a value on its way out. A string is redacted; an exception is
+# redacted in place; anything else comes back as a masked plain-data copy.
 def clean_util(ctx, val):
-    return val
+    cfg = _clean_config(ctx)
+
+    if cfg["active"] is False:
+        return val
+
+    if isinstance(val, str):
+        return _clean_string(cfg, val)
+
+    if isinstance(val, BaseException):
+        _clean_exception(cfg, val)
+        return val
+
+    out = _snapshot(cfg, val, None, 0, [])
+    return None if out is _DROP else out
+
+
+# Is this key name sensitive under the context's clean configuration?
+def clean_key(ctx, key):
+    return _sensitive_key(_clean_config(ctx), key)

package/project/.sdk/tm/py/pkg/utility/done.py

@@ -2,11 +2,7 @@
 
 
 def done_util(ctx):
-    if ctx.ctrl.explain is not None:
-        ctx.ctrl.explain = ctx.utility.clean(ctx, ctx.ctrl.explain)
-        explain_result = ctx.ctrl.explain.get("result") if isinstance(ctx.ctrl.explain, dict) else None
-        if isinstance(explain_result, dict):
-            explain_result.pop("err", None)
+    clean_explain_util(ctx)
 
     if ctx.result is not None and ctx.result.ok:
         return ctx.result.resdata
@@ -14,3 +10,20 @@ def done_util(ctx):
     # make_error raises on the default (throw) path; only returns bare
     # resdata when throw_err is explicitly disabled.
     return ctx.utility.make_error(ctx, None)
+
+
+# In place: the caller holds this very dict (ctrl is built from it), and a
+# stream copies only its ctrl, so a reassignment would leave the raw record
+# in their hands.
+def clean_explain_util(ctx):
+    explain = ctx.ctrl.explain
+    if not isinstance(explain, dict):
+        return
+    cleaned = ctx.utility.clean(ctx, explain)
+    if isinstance(cleaned, dict) and cleaned is not explain:
+        explain.clear()
+        explain.update(cleaned)
+    # With clean off, explain.result is the live result object, not a dict.
+    result = explain.get("result")
+    if isinstance(result, dict):
+        explain["result"] = {k: v for k, v in result.items() if k != "err"}