@voxgig/sdkgen 4.34.1 → 4.34.2

package/project/.sdk/tm/go/test/concurrency_test.go

@@ -0,0 +1,214 @@
+package sdktest
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	sdk "GOMODULE"
+)
+
+// Requests in flight at once on one client. Each resolves its operation
+// through the cache the client's root context shares with every request, and
+// registers and cleans secrets through the one registry the client holds.
+
+const concurrencyRounds = 200
+const concurrencyWidth = 8
+const concurrencyOps = 32
+
+// concurrencyClient is a live client whose transport answers at once.
+func concurrencyClient() *sdk.ProjectNameSDK {
+	return sdk.NewProjectNameSDK(map[string]any{
+		"base":  "http://concurrency.test/api",
+		"allow": map[string]any{"op": "direct"},
+		"system": map[string]any{
+			"fetch": func(url string, fetchdef map[string]any) (map[string]any, error) {
+				return map[string]any{
+					"status":     200,
+					"statusText": "OK",
+					"headers":    map[string]any{},
+					"json":       func() any { return map[string]any{"ok": true} },
+				}, nil
+			},
+		},
+	})
+}
+
+// atOnce runs body on concurrencyWidth goroutines released together.
+func atOnce(body func(n int)) {
+	var ready, done sync.WaitGroup
+	start := make(chan struct{})
+	ready.Add(concurrencyWidth)
+	done.Add(concurrencyWidth)
+	for i := 0; i < concurrencyWidth; i++ {
+		go func(n int) {
+			defer done.Done()
+			ready.Done()
+			<-start
+			body(n)
+		}(i)
+	}
+	ready.Wait()
+	close(start)
+	done.Wait()
+}
+
+func TestConcurrentFirstRequests(t *testing.T) {
+	for round := 0; round < concurrencyRounds; round++ {
+		// A fresh client each round, so every request in it is a first request.
+		client := concurrencyClient()
+		results := make([]map[string]any, concurrencyWidth)
+		errs := make([]error, concurrencyWidth)
+		atOnce(func(n int) {
+			results[n], errs[n] = client.Direct(map[string]any{"path": fmt.Sprintf("p%d", n)})
+		})
+		for n := range results {
+			if errs[n] != nil || results[n]["ok"] != true {
+				t.Fatalf("round %d, request %d failed: %v %v", round, n, errs[n], results[n])
+			}
+		}
+	}
+}
+
+func TestConcurrentOperationResolution(t *testing.T) {
+	for round := 0; round < concurrencyRounds; round++ {
+		client := concurrencyClient()
+		utility := client.GetUtility()
+		root := client.GetRootCtx()
+		ops := make([][]*sdk.Operation, concurrencyWidth)
+		atOnce(func(n int) {
+			ops[n] = make([]*sdk.Operation, concurrencyOps)
+			for k := range ops[n] {
+				ops[n][k] = utility.MakeContext(map[string]any{"opname": fmt.Sprintf("op%d", k)}, root).Op
+			}
+		})
+		for k := 0; k < concurrencyOps; k++ {
+			cached := utility.MakeContext(map[string]any{"opname": fmt.Sprintf("op%d", k)}, root).Op
+			for n := range ops {
+				if ops[n][k] != cached {
+					t.Fatalf("round %d: op%d resolved to more than one Operation", round, k)
+				}
+			}
+		}
+	}
+}
+
+func addedSecret(round, n, k int) string {
+	return fmt.Sprintf("ADDED-SECRET-%d-%d-%d", round, n, k)
+}
+
+// registerSecrets registers goroutine n's secrets.
+func registerSecrets(utility *sdk.Utility, root *sdk.Context, round, n int) {
+	for k := 0; k < concurrencyOps; k++ {
+		utility.CleanAdd(root, addedSecret(round, n, k))
+	}
+}
+
+// unmaskedSecret is the first secret registered in the round that a clean
+// leaves raw, or "".
+func unmaskedSecret(utility *sdk.Utility, root *sdk.Context, round int) string {
+	for n := 0; n < concurrencyWidth/2; n++ {
+		for k := 0; k < concurrencyOps; k++ {
+			added := addedSecret(round, n, k)
+			if got := utility.Clean(root, added); got != "[redacted]" {
+				return added
+			}
+		}
+	}
+	return ""
+}
+
+// whileRegistering has half the goroutines register their secrets, and the
+// rest run body until those are done or body returns false.
+func whileRegistering(utility *sdk.Utility, root *sdk.Context, round int, body func(n int) bool) {
+	var registering sync.WaitGroup
+	registering.Add(concurrencyWidth / 2)
+	registered := make(chan struct{})
+	go func() {
+		registering.Wait()
+		close(registered)
+	}()
+	atOnce(func(n int) {
+		if n < concurrencyWidth/2 {
+			defer registering.Done()
+			registerSecrets(utility, root, round, n)
+			return
+		}
+		for {
+			select {
+			case <-registered:
+				return
+			default:
+			}
+			if !body(n) {
+				return
+			}
+		}
+	})
+}
+
+// Secrets registered on some goroutines while others clean: every clean masks
+// what was registered before it, the longer secret whole, and no registration
+// is lost.
+func TestConcurrentCleanRegistry(t *testing.T) {
+	const masked = "a [redacted] b [redacted] c"
+	for round := 0; round < concurrencyRounds/4; round++ {
+		client := concurrencyClient()
+		utility := client.GetUtility()
+		root := client.GetRootCtx()
+		inner := fmt.Sprintf("INNER-SECRET-%d", round)
+		utility.CleanAdd(root, inner)
+		utility.CleanAdd(root, "OUTER-"+inner+"-TAIL")
+		text := "a " + inner + " b OUTER-" + inner + "-TAIL c"
+		if got := utility.Clean(root, text); got != masked {
+			t.Fatalf("round %d cleaned to: %v", round, got)
+		}
+
+		wrong := make([]any, concurrencyWidth)
+		whileRegistering(utility, root, round, func(n int) bool {
+			if got := utility.Clean(root, text); got != masked {
+				wrong[n] = got
+				return false
+			}
+			return true
+		})
+
+		for n, got := range wrong {
+			if got != nil {
+				t.Fatalf("round %d, cleaner %d cleaned to: %v", round, n, got)
+			}
+		}
+		if raw := unmaskedSecret(utility, root, round); raw != "" {
+			t.Fatalf("round %d: %s was registered but not masked", round, raw)
+		}
+	}
+}
+
+// Requests on one client while secrets register on it: each request copies
+// the client's options, the registry among them.
+func TestConcurrentRequestsWhileRegistering(t *testing.T) {
+	for round := 0; round < concurrencyRounds/4; round++ {
+		client := concurrencyClient()
+		utility := client.GetUtility()
+		root := client.GetRootCtx()
+
+		failed := make([]any, concurrencyWidth)
+		whileRegistering(utility, root, round, func(n int) bool {
+			res, err := client.Direct(map[string]any{"path": fmt.Sprintf("p%d", n)})
+			if err != nil || res["ok"] != true {
+				failed[n] = fmt.Sprint(err, res)
+				return false
+			}
+			return true
+		})
+
+		for n, got := range failed {
+			if got != nil {
+				t.Fatalf("round %d, request %d failed: %v", round, n, got)
+			}
+		}
+		if raw := unmaskedSecret(utility, root, round); raw != "" {
+			t.Fatalf("round %d: %s was registered but not masked", round, raw)
+		}
+	}
+}

package/project/.sdk/tm/go/utility/transform_request.go

@@ -49,7 +49,7 @@ func stripAction(reqdata any) any {
 
 // A header, cookie or query argument travels where prepareHeadersUtil or
 // prepareQueryUtil sends it, so the body is built from the request data
-// without it, unless the entity declares it as a field too.
+// without it, unless the point marks it as a field the body keeps.
 func routedArgNames(ctx *core.Context) []string {
 	names := []string{}
 	for _, arg := range append(append(callArgs(ctx, "header"), callArgs(ctx, "cookie")...), callArgs(ctx, "query")...) {

package/project/.sdk/tm/java/core/Context.java

@@ -4,6 +4,7 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadLocalRandom;
 
 import JAVAPACKAGE.utility.struct.Struct;
@@ -151,7 +152,7 @@ public class Context {
       this.opmap = basectx.opmap;
     }
     if (this.opmap == null) {
-      this.opmap = new LinkedHashMap<>();
+      this.opmap = new ConcurrentHashMap<>();
     }
 
     // Data
@@ -260,8 +261,10 @@ public class Context {
 
     Operation op = new Operation(opdef);
 
-    this.opmap.put(cacheKey, op);
-    return op;
+    // Requests on other threads share this cache; every one of them gets
+    // the Operation stored first.
+    Operation stored = this.opmap.putIfAbsent(cacheKey, op);
+    return null == stored ? op : stored;
   }
 
   public SdkError makeError(String code, String msg) {

package/project/.sdk/tm/java/test/ConcurrencyTest.java

@@ -0,0 +1,221 @@
+package JAVAPACKAGE.sdktest;
+
+// Requests in flight at once on one client. Each resolves its operation
+// through the cache the client's root context shares with every request, and
+// registers and cleans secrets through the one registry the client holds.
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.function.BiFunction;
+import java.util.function.IntConsumer;
+import java.util.function.Supplier;
+
+import org.junit.jupiter.api.Test;
+
+import JAVAPACKAGE.core.Context;
+import JAVAPACKAGE.core.Operation;
+import JAVAPACKAGE.core.ProjectNameSDK;
+import JAVAPACKAGE.core.Utility;
+
+public class ConcurrencyTest {
+
+  static final int ROUNDS = 200;
+  static final int WIDTH = 8;
+  static final int OPS = 32;
+
+  static Map<String, Object> map(Object... kv) {
+    Map<String, Object> out = new LinkedHashMap<>();
+    for (int i = 0; i + 1 < kv.length; i += 2) {
+      out.put((String) kv[i], kv[i + 1]);
+    }
+    return out;
+  }
+
+  // A live client whose transport answers at once.
+  static ProjectNameSDK liveClient() {
+    BiFunction<String, Map<String, Object>, Map<String, Object>> fetch =
+        (url, fetchdef) -> map(
+            "status", 200,
+            "statusText", "OK",
+            "headers", new LinkedHashMap<String, Object>(),
+            "json", (Supplier<Object>) () -> map("ok", true));
+    return new ProjectNameSDK(map(
+        "base", "http://concurrency.test/api",
+        "allow", map("op", "direct"),
+        "system", map("fetch", fetch)));
+  }
+
+  // Runs body on WIDTH threads released together, and returns what they threw.
+  static List<Throwable> atOnce(IntConsumer body) throws InterruptedException {
+    CyclicBarrier start = new CyclicBarrier(WIDTH);
+    ConcurrentLinkedQueue<Throwable> thrown = new ConcurrentLinkedQueue<>();
+    List<Thread> threads = new ArrayList<>();
+    for (int i = 0; i < WIDTH; i++) {
+      final int n = i;
+      Thread thread = new Thread(() -> {
+        try {
+          start.await();
+          body.accept(n);
+        }
+        catch (Throwable t) {
+          thrown.add(t);
+        }
+      });
+      thread.start();
+      threads.add(thread);
+    }
+    for (Thread thread : threads) {
+      thread.join();
+    }
+    return new ArrayList<>(thrown);
+  }
+
+  @Test
+  public void concurrentFirstRequestsSucceed() throws InterruptedException {
+    for (int round = 0; round < ROUNDS; round++) {
+      // A fresh client each round, so every request in it is a first request.
+      ProjectNameSDK client = liveClient();
+      List<Map<String, Object>> results = new ArrayList<>();
+      for (int n = 0; n < WIDTH; n++) {
+        results.add(null);
+      }
+      List<Throwable> thrown = atOnce(n -> results.set(n, client.direct(map("path", "p" + n))));
+
+      assertTrue(thrown.isEmpty(), "round " + round + " threw: " + thrown);
+      for (int n = 0; n < WIDTH; n++) {
+        Map<String, Object> res = results.get(n);
+        assertEquals(true, null == res ? null : res.get("ok"),
+            "round " + round + ", request " + n + " failed: " + res);
+      }
+    }
+  }
+
+  @Test
+  public void concurrentResolutionsShareOneCachedOperation() throws InterruptedException {
+    for (int round = 0; round < ROUNDS; round++) {
+      ProjectNameSDK client = liveClient();
+      Utility utility = client.getUtility();
+      Context root = client.getRootCtx();
+      Operation[][] ops = new Operation[WIDTH][OPS];
+      List<Throwable> thrown = atOnce(n -> {
+        for (int k = 0; k < OPS; k++) {
+          ops[n][k] = utility.makeContext.apply(map("opname", "op" + k), root).op;
+        }
+      });
+
+      assertTrue(thrown.isEmpty(), "round " + round + " threw: " + thrown);
+      for (int k = 0; k < OPS; k++) {
+        Operation cached = utility.makeContext.apply(map("opname", "op" + k), root).op;
+        for (int n = 0; n < WIDTH; n++) {
+          assertSame(cached, ops[n][k],
+              "round " + round + ": op" + k + " resolved to more than one Operation");
+        }
+      }
+    }
+  }
+
+  static String addedSecret(int round, int n, int k) {
+    return "ADDED-SECRET-" + round + "-" + n + "-" + k;
+  }
+
+  // Registers thread n's secrets, then counts the thread out of registering.
+  static void registerSecrets(Utility utility, Context root, int round, int n,
+      CountDownLatch registering) {
+    try {
+      for (int k = 0; k < OPS; k++) {
+        utility.cleanAdd.apply(root, addedSecret(round, n, k));
+      }
+    }
+    finally {
+      registering.countDown();
+    }
+  }
+
+  static void assertEverySecretMasked(Utility utility, Context root, int round) {
+    for (int n = 0; n < WIDTH / 2; n++) {
+      for (int k = 0; k < OPS; k++) {
+        String added = addedSecret(round, n, k);
+        assertEquals("[redacted]", utility.clean.apply(root, added),
+            "round " + round + ": " + added + " was registered but not masked");
+      }
+    }
+  }
+
+  // Secrets registered on some threads while others clean: every clean masks
+  // what was registered before it, the longer secret whole, and no
+  // registration is lost.
+  @Test
+  public void concurrentRegistrationKeepsEverySecretMasked() throws InterruptedException {
+    for (int round = 0; round < ROUNDS / 4; round++) {
+      final int r = round;
+      ProjectNameSDK client = liveClient();
+      Utility utility = client.getUtility();
+      Context root = client.getRootCtx();
+      String inner = "INNER-SECRET-" + round;
+      utility.cleanAdd.apply(root, inner);
+      utility.cleanAdd.apply(root, "OUTER-" + inner + "-TAIL");
+      String text = "a " + inner + " b OUTER-" + inner + "-TAIL c";
+      String masked = "a [redacted] b [redacted] c";
+      assertEquals(masked, utility.clean.apply(root, text));
+
+      CountDownLatch registering = new CountDownLatch(WIDTH / 2);
+      ConcurrentLinkedQueue<Object> wrong = new ConcurrentLinkedQueue<>();
+      List<Throwable> thrown = atOnce(n -> {
+        if (n < WIDTH / 2) {
+          registerSecrets(utility, root, r, n, registering);
+          return;
+        }
+        while (0 < registering.getCount()) {
+          Object got = utility.clean.apply(root, text);
+          if (!masked.equals(got)) {
+            wrong.add(String.valueOf(got));
+          }
+        }
+      });
+
+      assertTrue(thrown.isEmpty(), "round " + round + " threw: " + thrown);
+      assertTrue(wrong.isEmpty(), "round " + round + " cleaned to: " + wrong);
+      assertEverySecretMasked(utility, root, round);
+    }
+  }
+
+  // Requests on one client while secrets register on it: each request copies
+  // the client's options, the registry among them.
+  @Test
+  public void concurrentRequestsSurviveRegistration() throws InterruptedException {
+    for (int round = 0; round < ROUNDS / 4; round++) {
+      final int r = round;
+      ProjectNameSDK client = liveClient();
+      Utility utility = client.getUtility();
+      Context root = client.getRootCtx();
+      CountDownLatch registering = new CountDownLatch(WIDTH / 2);
+      ConcurrentLinkedQueue<Object> failed = new ConcurrentLinkedQueue<>();
+      List<Throwable> thrown = atOnce(n -> {
+        if (n < WIDTH / 2) {
+          registerSecrets(utility, root, r, n, registering);
+          return;
+        }
+        while (0 < registering.getCount()) {
+          Map<String, Object> res = client.direct(map("path", "p" + n));
+          if (!Boolean.TRUE.equals(res.get("ok"))) {
+            failed.add(String.valueOf(res));
+            return;
+          }
+        }
+      });
+
+      assertTrue(thrown.isEmpty(), "round " + round + " threw: " + thrown);
+      assertTrue(failed.isEmpty(), "round " + round + ", a request failed: " + failed);
+      assertEverySecretMasked(utility, root, round);
+    }
+  }
+}

package/project/.sdk/tm/java/utility/Clean.java

@@ -14,6 +14,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -37,8 +38,8 @@ import JAVAPACKAGE.utility.struct.Struct;
 //
 // The configuration is the derived block MakeOptions builds
 // (`options.__derived__.clean`): active, keys, values, mask, hint, min. It
-// is a plain map so the registry stays MUTABLE after construction - the
-// auth step and the features register what they handle later.
+// is a map so the registry can grow after construction - the auth step and
+// the features register what they handle later.
 @SuppressWarnings({"unchecked"})
 final class Clean {
 
@@ -67,7 +68,9 @@ final class Clean {
 
   static Map<String, Object> makeCleanConfig(Map<String, Object> cleanopts) {
     Map<String, Object> opts = cleanopts == null ? new LinkedHashMap<>() : cleanopts;
-    Map<String, Object> cfg = new LinkedHashMap<>();
+    // Read without the monitor add() holds, by a clean or a request copying
+    // the options: each read sees the list the last registration put.
+    Map<String, Object> cfg = new ConcurrentHashMap<>();
     cfg.put("active", !Boolean.FALSE.equals(opts.get("active")));
     cfg.put("keys", splitkeys(opts.get("keys")));
     cfg.put("values", new ArrayList<String>());
@@ -148,10 +151,8 @@ final class Clean {
   }
 
   private static List<String> values(Map<String, Object> cfg) {
-    if (!(cfg.get("values") instanceof List)) {
-      cfg.put("values", new ArrayList<String>());
-    }
-    return (List<String>) cfg.get("values");
+    Object values = cfg.get("values");
+    return values instanceof List ? (List<String>) values : List.of();
   }
 
   private static String mask(Map<String, Object> cfg) {
@@ -193,21 +194,25 @@ final class Clean {
   }
 
   // Register a secret value. Idempotent; shorter than `min` is not a secret
-  // the SDK can mask without blanking ordinary text.
+  // the SDK can mask without blanking ordinary text. Requests on other
+  // threads clean while one registers, so a registration publishes a new
+  // list and never changes a published one.
   static void add(Map<String, Object> cfg, Object value) {
     if (!(value instanceof String) || ((String) value).length() < min(cfg)) {
       return;
     }
-    List<String> values = values(cfg);
-    boolean changed = false;
-    for (String form : forms((String) value)) {
-      if (form.length() >= min(cfg) && !values.contains(form)) {
-        values.add(form);
-        changed = true;
+    synchronized (cfg) {
+      List<String> values = new ArrayList<>(values(cfg));
+      int had = values.size();
+      for (String form : forms((String) value)) {
+        if (form.length() >= min(cfg) && !values.contains(form)) {
+          values.add(form);
+        }
+      }
+      if (had < values.size()) {
+        values.sort((a, b) -> b.length() - a.length());
+        cfg.put("values", List.copyOf(values));
       }
-    }
-    if (changed) {
-      values.sort((a, b) -> b.length() - a.length());
     }
   }
 

package/project/.sdk/tm/java/utility/TransformRequest.java

@@ -45,7 +45,7 @@ final class TransformRequest {
 
   // A header, cookie or query argument travels where PrepareHeaders or
   // PrepareQuery sends it, so the body is built from the request data without
-  // it, unless the entity declares it as a field too.
+  // it, unless the point marks it as a field the body keeps.
   private static List<String> routedArgNames(Context ctx) {
     List<String> names = new ArrayList<>();
     for (String kind : List.of("header", "cookie", "query")) {

package/project/.sdk/tm/js/src/utility/TransformRequestUtility.js

@@ -50,7 +50,7 @@ function stripAction(reqdata) {
 
 // A header, cookie or query argument travels where prepareHeaders or
 // prepareQuery sends it, so the body is built from the request data without
-// it, unless the entity declares it as a field too.
+// it, unless the point marks it as a field the body keeps.
 function routedArgNames(ctx) {
   return callArgs(ctx, 'header').concat(callArgs(ctx, 'cookie'), callArgs(ctx, 'query'))
     .map((arg) => arg.name)

package/project/.sdk/tm/kotlin/core/Context.kt

@@ -1,5 +1,6 @@
 package KOTLINPACKAGE.core
 
+import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.ThreadLocalRandom
 
 import KOTLINPACKAGE.utility.struct.Struct
@@ -143,7 +144,7 @@ class Context(ctxmap: MutableMap<String, Any?>?, basectx: Context?) {
       this.opmap = basectx.opmap
     }
     if (this.opmap == null) {
-      this.opmap = linkedMapOf()
+      this.opmap = ConcurrentHashMap()
     }
 
     // Data
@@ -237,8 +238,9 @@ class Context(ctxmap: MutableMap<String, Any?>?, basectx: Context?) {
 
     val op = Operation(opdef)
 
-    this.opmap!![cacheKey] = op
-    return op
+    // Requests on other threads share this cache; every one of them gets
+    // the Operation stored first.
+    return this.opmap!!.putIfAbsent(cacheKey, op) ?: op
   }
 
   fun makeError(code: String, msg: String): SdkError {