@voxgig/sdkgen 4.34.0 → 4.34.2

package/project/.sdk/tm/py/pkg/utility/prepare_headers.py

@@ -2,6 +2,22 @@
 
 from __future__ import annotations
 from projectname_sdk.utility.voxgig_struct import voxgig_struct as vs
+from projectname_sdk.utility.param import call_args
+from projectname_sdk.utility.media import media_headers
+
+
+# The form style of a cookie parameter: a list repeats the name, a map sends
+# its own keys, and every value is percent-encoded.
+def _cookie_pair(wire, val):
+    def esc(v):
+        return vs.escurl(vs.stringify(v))
+    if vs.islist(val):
+        pairs = [wire + "=" + esc(item) for item in val]
+    elif vs.ismap(val):
+        pairs = [vs.escurl(key) + "=" + esc(val[key]) for key in vs.keysof(val)]
+    else:
+        pairs = [wire + "=" + esc(val)]
+    return "; ".join(pairs)
 
 
 def prepare_headers_util(ctx):
@@ -13,26 +29,46 @@ def prepare_headers_util(ctx):
         cloned = vs.clone(headers)
         if isinstance(cloned, dict):
             out = cloned
+    out = media_headers(ctx.point, out)
+
+    # A header argument replaces a default of the same name, whatever its case.
+    for _name, orig, val in call_args(ctx, "header"):
+        if val is not None:
+            wire = orig.lower()
+            for key in [k for k in out if isinstance(k, str) and k.lower() == wire]:
+                del out[key]
+            out[wire] = vs.stringify(val)
 
-    # A header parameter travels as a header, under the name the definition
-    # gives it, and only from this call's own arguments. It replaces a default
-    # of the same name, whatever its case.
-    hl = vs.getpath(ctx.point, "args.header") if ctx.point is not None else None
-    if isinstance(hl, list):
-        for hd in hl:
-            name = vs.getprop(hd, "name")
-            if not isinstance(name, str) or name == "":
-                continue
-            orig = vs.getprop(hd, "orig")
-            if not isinstance(orig, str) or orig == "":
-                orig = name
-            val = vs.getprop(ctx.reqmatch or {}, name)
-            if val is None:
-                val = vs.getprop(ctx.reqdata or {}, name)
-            if val is not None:
-                wire = orig.lower()
-                for key in [k for k in out if isinstance(k, str) and k.lower() == wire]:
-                    del out[key]
-                out[wire] = vs.stringify(val)
+    # A cookie argument travels in the cookie header, form serialized and
+    # percent-encoded, replacing a cookie of the same name among those the
+    # caller's headers already send.
+    sent = [(orig, val) for _name, orig, val in call_args(ctx, "cookie") if val is not None]
+    if sent:
+        names = [n for orig, val in sent
+                 for n in ([vs.escurl(k) for k in vs.keysof(val)] if vs.ismap(val) else [orig])]
+        kept = []
+        for key in [k for k in out if isinstance(k, str) and k.lower() == "cookie"]:
+            given = out.pop(key)
+            if isinstance(given, str):
+                kept.extend(cookie_keep(given, names))
+        for orig, val in sent:
+            pair = _cookie_pair(orig, val)
+            if pair != "":
+                kept.append(pair)
+        if kept:
+            out["cookie"] = "; ".join(kept)
 
     return out
+
+
+def cookie_keep(header, names):
+    """The caller's cookie pieces with the named cookies removed.
+
+    A cookie is one ;-delimited piece, whatever its value holds.
+    """
+    kept = []
+    for piece in header.split(";"):
+        cookie = piece.strip()
+        if cookie != "" and cookie.split("=", 1)[0].strip() not in names:
+            kept.append(cookie)
+    return kept

package/project/.sdk/tm/py/pkg/utility/prepare_method.py

@@ -27,3 +27,10 @@ def prepare_method_util(ctx):
     # The silent-pass inline runner hid a stray "GET" fallback here; the
     # shared corpus (prepareMethod, opname "bad" -> null) pins it now.
     return METHOD_MAP.get(opname)
+
+
+# Whether a comma-separated allow option names the item: whole names, any case.
+def allowed(names, item):
+    want = item.upper() if isinstance(item, str) else ""
+    return "" != want and isinstance(names, str) and any(
+        name.strip().upper() == want for name in names.split(","))

package/project/.sdk/tm/py/pkg/utility/prepare_query.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 from projectname_sdk.utility.voxgig_struct import voxgig_struct as vs
+from projectname_sdk.utility.param import call_args
 
 
 def _contains_param(params, s):
@@ -29,14 +30,17 @@ def prepare_query_util(ctx):
                 name = vs.getprop(pd, "name")
                 if isinstance(name, str):
                     params.append(name)
-        # A header parameter travels in the headers, which prepare_headers
-        # fills.
-        hl = vs.getpath(point, "args.header")
-        if isinstance(hl, list):
-            for hd in hl:
-                name = vs.getprop(hd, "name")
-                if isinstance(name, str):
-                    params.append(name)
+        # A header or cookie parameter travels in the headers, which
+        # prepare_headers fills, unless a query parameter shares its name:
+        # then both are sent.
+        ql = vs.getpath(point, "args.query")
+        declared = [vs.getprop(qd, "name") for qd in ql] if isinstance(ql, list) else []
+        for located in (vs.getpath(point, "args.header"), vs.getpath(point, "args.cookie")):
+            if isinstance(located, list):
+                for hd in located:
+                    name = vs.getprop(hd, "name")
+                    if isinstance(name, str) and name not in declared:
+                        params.append(name)
 
     # A query parameter travels under the name the definition gives it, its
     # orig, which the model may have renamed for the caller.
@@ -60,4 +64,9 @@ def prepare_query_util(ctx):
                     and not _contains_param(params, key):
                 out[wire.get(key, key)] = val
 
+    # A create or update passes its query arguments in its data.
+    for name, orig, val in call_args(ctx, "query"):
+        if val is not None and not _contains_param(params, name):
+            out[orig] = val
+
     return out

package/project/.sdk/tm/py/pkg/utility/result_body.py

@@ -1,5 +1,10 @@
 # ProjectName SDK utility: result_body
 
+import re
+
+
+PREVIEW_LENGTH = 160
+
 
 def result_body_util(ctx):
     response = ctx.response
@@ -9,5 +14,51 @@ def result_body_util(ctx):
         if response is not None and response.json_func is not None and response.body is not None:
             json_data = response.json_func()
             result.body = json_data
+        if response is not None and getattr(response, "unreadable", False):
+            sent = ctx.spec.headers if ctx.spec is not None else None
+            result.err = unreadable_body(ctx, result.status, result.headers, response.body,
+                                         sent, result.err)
 
     return result
+
+
+# A body that is not JSON. An HTTP failure keeps its own error, with the
+# response described; otherwise the code tells a wrong content type from
+# malformed JSON.
+def unreadable_body(ctx, status, headers, text, sent, failed):
+    ctype = _header_value(headers, "content-type")
+    agent = ctx.utility.clean(ctx, _header_value(sent, "user-agent")) or "transport default"
+    detail = ("HTTP " + str(status) + ", content-type " + (ctype or "none") +
+              ", user-agent " + str(agent) +
+              ("" if text is None else ", body: " + _preview(ctx, text)))
+
+    if failed is not None:
+        if isinstance(failed, str):
+            return failed + " (" + detail + ")"
+        message = str(failed) + " (" + detail + ")"
+        if hasattr(failed, "msg"):
+            failed.msg = message
+        failed.args = (message,)
+        return failed
+
+    if "" == ctype or "json" in ctype.lower():
+        return ctx.make_error("response_json_invalid",
+                              "response: body is not valid JSON (" + detail + ")")
+    return ctx.make_error("response_content_type",
+                          "response: expected JSON, got " + ctype + " (" + detail + ")")
+
+
+def _header_value(headers, name):
+    if not isinstance(headers, dict):
+        return ""
+    for key, val in headers.items():
+        if name == str(key).lower():
+            return str(val)
+    return ""
+
+
+# Cleaned whole: a secret the bound would split could leave its prefix.
+def _preview(ctx, text):
+    flat = ctx.utility.clean(ctx, re.sub(r"\s+", " ", str(text)).strip())
+    flat = str(flat)
+    return flat[:PREVIEW_LENGTH] + "..." if len(flat) > PREVIEW_LENGTH else flat

package/project/.sdk/tm/py/pkg/utility/transform_request.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 from projectname_sdk.utility.voxgig_struct import voxgig_struct as vs
 from projectname_sdk.core.helpers import to_map
+from projectname_sdk.utility.param import call_args
 
 
 # `$action` selects the point (see make_point_util); it is never an API field,
@@ -11,14 +12,22 @@ def _strip_action(reqdata):
     return _omit(reqdata, ["$action"])
 
 
-# A header argument travels as a header, which prepare_headers_util sends, so
-# the body is built from the request data without it.
-def _header_arg_names(point):
-    hl = vs.getpath(point, "args.header") if point is not None else None
-    if not isinstance(hl, list):
-        return []
-    names = [vs.getprop(hd, "name") for hd in hl]
-    return [n for n in names if isinstance(n, str) and n != ""]
+# A header, cookie or query argument travels where prepare_headers_util or
+# prepare_query_util sends it, so the body is built from the request data
+# without it, unless the point marks it as a field the body keeps.
+def _routed_arg_names(ctx):
+    return [name for name, _orig, _val in
+            call_args(ctx, "header") + call_args(ctx, "cookie") + call_args(ctx, "query")
+            if not _field_arg(ctx, name)]
+
+
+def _field_arg(ctx, name):
+    for kind in ("header", "cookie", "query"):
+        defs = vs.getpath(ctx.point, "args." + kind) if ctx.point is not None else None
+        for arg in defs if isinstance(defs, list) else []:
+            if vs.getprop(arg, "name") == name and vs.getprop(arg, "field") is True:
+                return True
+    return False
 
 
 def _omit(reqdata, names):
@@ -34,7 +43,7 @@ def transform_request_util(ctx):
     if spec is not None:
         spec.step = "reqform"
 
-    data = _omit(ctx.reqdata, _header_arg_names(point))
+    data = _omit(ctx.reqdata, _routed_arg_names(ctx))
 
     transform = to_map(vs.getprop(point, "transform"))
     if transform is None:

package/project/.sdk/tm/py/test/feature_harness.py

@@ -97,7 +97,7 @@ def recording_server(reply=None):
 def default_method(op):
     if op == "create":
         return "POST"
-    if op == "update":
+    if op == "update" or op == "patch":
         return "PATCH"
     if op == "remove":
         return "DELETE"

package/project/.sdk/tm/py/test/runner.py

@@ -13,7 +13,7 @@ class ProjectNameTestRunner:
     @staticmethod
     def load_env_local():
         try:
-            with open("../../.env.local", "r") as f:
+            with open("../../.env.local", "r", encoding="utf-8") as f:
                 content = f.read()
         except (FileNotFoundError, IOError):
             return
@@ -74,7 +74,7 @@ class ProjectNameTestRunner:
             return ProjectNameTestRunner._test_control
         ctrl_path = os.path.join(os.path.dirname(__file__), "sdk-test-control.json")
         try:
-            with open(ctrl_path, "r") as f:
+            with open(ctrl_path, "r", encoding="utf-8") as f:
                 ProjectNameTestRunner._test_control = json.load(f)
         except (FileNotFoundError, IOError, ValueError):
             ProjectNameTestRunner._test_control = {
@@ -195,3 +195,78 @@ def live_client_options():
 
 def live_delay_ms():
     return ProjectNameTestRunner.live_delay_ms()
+
+
+def live_miss(strict, reason):
+    """A live check that did not pass, as main.kit.test.live.strict decides:
+    strict fails the test, lenient skips it with the same reason."""
+    import pytest
+    if strict:
+        pytest.fail(reason, pytrace=False)
+    pytest.skip(reason)
+
+
+def live_empty(reason):
+    """An account holding no record for the test to read skips either way."""
+    import pytest
+    pytest.skip(reason)
+
+
+def live_list(data):
+    """A live list response's records: the body, or the first list an
+    envelope object holds."""
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        for value in data.values():
+            if isinstance(value, list):
+                return value
+    return None
+
+
+def live_describe(result):
+    """A live response for a message: the SDK's error, or else its status
+    and content type, never its body."""
+    err = result.get("err") if isinstance(result, dict) else None
+    if err is not None:
+        return str(err)
+    headers = result.get("headers") if isinstance(result, dict) else None
+    ctype = None
+    if isinstance(headers, dict):
+        ctype = next((v for k, v in headers.items() if str(k).lower() == "content-type"), None)
+    return "HTTP " + str(result.get("status") if isinstance(result, dict) else None) + \
+        (" " + str(ctype).split(";")[0].strip() if ctype else "")
+
+
+def live_existing(setup, strict, name, list_call):
+    """Finds the record a create-less flow reads live: the first its list
+    returns, put where the flow reads the fixture's existing records."""
+    try:
+        found = list_call()
+    except Exception as err:
+        live_miss(strict, f"Live list discovery failed: {err}")
+    if not isinstance(found, list):
+        live_miss(strict, "Live list discovery returned no list")
+    if 0 == len(found):
+        live_empty(f"The account has no {name} record to load")
+    existing = setup["data"].setdefault("existing", {})
+    existing[name] = {"live01": entity_data(found[0])}
+
+
+def live_observe(live_env, strict):
+    """A flow test decorator: in a lenient live run a failing check skips
+    the test, observing the live API rather than failing on it."""
+    import functools
+
+    def wrap(fn):
+        @functools.wraps(fn)
+        def run(*args, **kwargs):
+            try:
+                return fn(*args, **kwargs)
+            except Exception as err:
+                if strict or "TRUE" != os.environ.get(live_env):
+                    raise
+                import pytest
+                pytest.skip(f"live run, main.kit.test.live.strict is false: {err!r}")
+        return run
+    return wrap

package/project/.sdk/tm/py/test/test_concurrency.py

@@ -0,0 +1,177 @@
+# ProjectName SDK concurrency test
+#
+# Requests in flight at once on one client. Each resolves its operation
+# through the cache the client's root context shares with every request, and
+# registers and cleans secrets through the one registry the client holds.
+
+import sys
+import threading
+
+from projectname_sdk import ProjectNameSDK
+
+
+ROUNDS = 50
+WIDTH = 8
+OPS = 32
+MASKED = "a [redacted] b [redacted] c"
+
+
+# Runs body on WIDTH threads released together, and returns what they raised.
+def _at_once(body):
+    start = threading.Barrier(WIDTH)
+    raised = []
+
+    def run(n):
+        try:
+            start.wait()
+            body(n)
+        except BaseException as err:
+            raised.append(err)
+
+    threads = [threading.Thread(target=run, args=(n,)) for n in range(WIDTH)]
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join()
+    return raised
+
+
+# A live client whose transport answers at once.
+def _live_client():
+    def fetch(url, fetchdef):
+        return {
+            "status": 200,
+            "statusText": "OK",
+            "headers": {},
+            "json": lambda: {"ok": True},
+        }, None
+
+    return ProjectNameSDK({
+        "base": "http://concurrency.test/api",
+        "allow": {"op": "direct"},
+        "system": {"fetch": fetch},
+    })
+
+
+def _added_secret(rnd, n, k):
+    return "ADDED-SECRET-%d-%d-%d" % (rnd, n, k)
+
+
+# Half the threads register their secrets; the rest run body(n) until those
+# are done or body returns False. Returns what the threads raised.
+def _while_registering(utility, root, rnd, body):
+    lock = threading.Lock()
+    registering = [WIDTH // 2]
+    registered = threading.Event()
+
+    def work(n):
+        if n < WIDTH // 2:
+            try:
+                for k in range(OPS):
+                    utility.clean_add(root, _added_secret(rnd, n, k))
+            finally:
+                with lock:
+                    registering[0] -= 1
+                    if 0 == registering[0]:
+                        registered.set()
+            return
+        while not registered.is_set():
+            if not body(n):
+                return
+
+    return _at_once(work)
+
+
+# The first secret registered in the round that a clean leaves raw.
+def _unmasked_secret(utility, root, rnd):
+    for n in range(WIDTH // 2):
+        for k in range(OPS):
+            added = _added_secret(rnd, n, k)
+            if "[redacted]" != utility.clean(root, added):
+                return added
+    return None
+
+
+class TestConcurrency:
+
+    # Threads switch as often as the interpreter allows, so a race a few
+    # bytecodes wide has room to show.
+    def setup_method(self):
+        self._interval = sys.getswitchinterval()
+        sys.setswitchinterval(1e-6)
+
+    def teardown_method(self):
+        sys.setswitchinterval(self._interval)
+
+    def test_concurrent_resolutions_share_one_cached_operation(self):
+        for rnd in range(ROUNDS):
+            client = ProjectNameSDK.test(None, None)
+            utility = client.get_utility()
+            root = client.get_root_ctx()
+            ops = [[None] * OPS for _ in range(WIDTH)]
+
+            def resolve(n):
+                for k in range(OPS):
+                    ops[n][k] = utility.make_context({"opname": "op%d" % k}, root).op
+
+            raised = _at_once(resolve)
+
+            assert [] == raised, "round %d raised: %r" % (rnd, raised)
+            for k in range(OPS):
+                cached = utility.make_context({"opname": "op%d" % k}, root).op
+                for n in range(WIDTH):
+                    assert ops[n][k] is cached, \
+                        "round %d: op%d resolved to more than one Operation" % (rnd, k)
+
+    # Secrets registered on some threads while others clean: every clean
+    # masks what was registered before it, the longer secret whole, and no
+    # registration is lost.
+    def test_concurrent_registration_keeps_every_secret_masked(self):
+        for rnd in range(ROUNDS // 4):
+            client = ProjectNameSDK.test(None, None)
+            utility = client.get_utility()
+            root = client.get_root_ctx()
+            inner = "INNER-SECRET-%d" % rnd
+            utility.clean_add(root, inner)
+            utility.clean_add(root, "OUTER-" + inner + "-TAIL")
+            text = "a " + inner + " b OUTER-" + inner + "-TAIL c"
+            assert MASKED == utility.clean(root, text)
+
+            wrong = []
+
+            def clean(n):
+                got = utility.clean(root, text)
+                if MASKED != got:
+                    wrong.append(got)
+                    return False
+                return True
+
+            raised = _while_registering(utility, root, rnd, clean)
+
+            assert [] == raised, "round %d raised: %r" % (rnd, raised)
+            assert [] == wrong, "round %d cleaned to: %r" % (rnd, wrong)
+            added = _unmasked_secret(utility, root, rnd)
+            assert added is None, "round %d: %s was registered but not masked" % (rnd, added)
+
+    # Requests on one client while secrets register on it: each request
+    # copies the client's options, the registry among them.
+    def test_concurrent_requests_survive_registration(self):
+        for rnd in range(ROUNDS // 4):
+            client = _live_client()
+            utility = client.get_utility()
+            root = client.get_root_ctx()
+            failed = []
+
+            def request(n):
+                res = client.direct({"path": "p%d" % n})
+                if True is not res.get("ok"):
+                    failed.append(res)
+                    return False
+                return True
+
+            raised = _while_registering(utility, root, rnd, request)
+
+            assert [] == raised, "round %d raised: %r" % (rnd, raised)
+            assert [] == failed, "round %d, a request failed: %r" % (rnd, failed)
+            added = _unmasked_secret(utility, root, rnd)
+            assert added is None, "round %d: %s was registered but not masked" % (rnd, added)