@voxgig/sdkgen 4.34.1 → 4.34.2

package/project/.sdk/tm/csharp/test/ConcurrencyTest.cs

@@ -0,0 +1,238 @@
+// Requests in flight at once on one client. Each resolves its operation
+// through the cache the client's root context shares with every request, and
+// registers and cleans secrets through the one registry the client holds.
+
+using System.Collections.Concurrent;
+
+using Xunit;
+
+using ProjectNameSdk;
+
+namespace ProjectNameSdk.Test;
+
+public class ConcurrencyTest
+{
+    private const int Rounds = 200;
+    private const int Width = 8;
+    private const int Ops = 32;
+
+    private static Dictionary<string, object?> Answer(string url, Dictionary<string, object?> fetchdef)
+    {
+        return new Dictionary<string, object?>
+        {
+            ["status"] = 200,
+            ["statusText"] = "OK",
+            ["headers"] = new Dictionary<string, object?>(),
+            ["json"] = (Func<object?>)(() => new Dictionary<string, object?> { ["ok"] = true }),
+        };
+    }
+
+    // A live client whose transport answers at once.
+    private static ProjectNameSDK LiveClient()
+    {
+        return new ProjectNameSDK(new Dictionary<string, object?>
+        {
+            ["base"] = "http://concurrency.test/api",
+            ["allow"] = new Dictionary<string, object?> { ["op"] = "direct" },
+            ["system"] = new Dictionary<string, object?>
+            {
+                ["fetch"] = (Func<string, Dictionary<string, object?>,
+                    Dictionary<string, object?>>)Answer,
+            },
+        });
+    }
+
+    // Runs body on Width threads released together, and returns what they threw.
+    private static List<Exception> AtOnce(Action<int> body)
+    {
+        var start = new Barrier(Width);
+        var thrown = new ConcurrentQueue<Exception>();
+        var threads = new List<Thread>();
+        for (var i = 0; i < Width; i++)
+        {
+            var n = i;
+            var thread = new Thread(() =>
+            {
+                try
+                {
+                    start.SignalAndWait();
+                    body(n);
+                }
+                catch (Exception e)
+                {
+                    thrown.Enqueue(e);
+                }
+            });
+            thread.Start();
+            threads.Add(thread);
+        }
+        threads.ForEach(t => t.Join());
+        return thrown.ToList();
+    }
+
+    [Fact]
+    public void ConcurrentFirstRequestsSucceed()
+    {
+        for (var round = 0; round < Rounds; round++)
+        {
+            // A fresh client each round, so every request in it is a first request.
+            var client = LiveClient();
+            var results = new Dictionary<string, object?>?[Width];
+            var thrown = AtOnce(n =>
+                results[n] = client.Direct(new Dictionary<string, object?> { ["path"] = "p" + n }));
+
+            Assert.True(0 == thrown.Count, "round " + round + " threw:\n" + string.Join("\n", thrown));
+            for (var n = 0; n < Width; n++)
+            {
+                var res = results[n];
+                Assert.True(Equals(true, res?["ok"]), "round " + round + ", request " + n +
+                    " failed: " + (res?.GetValueOrDefault("err") ?? "no result"));
+            }
+        }
+    }
+
+    [Fact]
+    public void ConcurrentResolutionsShareOneCachedOperation()
+    {
+        for (var round = 0; round < Rounds; round++)
+        {
+            var client = LiveClient();
+            var utility = client.GetUtility();
+            var root = client.GetRootCtx();
+            var ops = new Operation?[Width, Ops];
+            var thrown = AtOnce(n =>
+            {
+                for (var k = 0; k < Ops; k++)
+                {
+                    ops[n, k] = utility.MakeContext(
+                        new Dictionary<string, object?> { ["opname"] = "op" + k }, root).Op;
+                }
+            });
+
+            Assert.True(0 == thrown.Count, "round " + round + " threw:\n" + string.Join("\n", thrown));
+            for (var k = 0; k < Ops; k++)
+            {
+                var cached = utility.MakeContext(
+                    new Dictionary<string, object?> { ["opname"] = "op" + k }, root).Op;
+                for (var n = 0; n < Width; n++)
+                {
+                    Assert.True(ReferenceEquals(cached, ops[n, k]),
+                        "round " + round + ": op" + k + " resolved to more than one Operation");
+                }
+            }
+        }
+    }
+
+    private static string AddedSecret(int round, int n, int k)
+    {
+        return "ADDED-SECRET-" + round + "-" + n + "-" + k;
+    }
+
+    // Registers thread n's secrets, then counts the thread out of registering.
+    private static void RegisterSecrets(Utility utility, Context root, int round, int n,
+        CountdownEvent registering)
+    {
+        try
+        {
+            for (var k = 0; k < Ops; k++)
+            {
+                utility.CleanAdd(root, AddedSecret(round, n, k));
+            }
+        }
+        finally
+        {
+            registering.Signal();
+        }
+    }
+
+    private static void AssertEverySecretMasked(Utility utility, Context root, int round)
+    {
+        for (var n = 0; n < Width / 2; n++)
+        {
+            for (var k = 0; k < Ops; k++)
+            {
+                var added = AddedSecret(round, n, k);
+                Assert.True("[redacted]" == utility.Clean(root, added) as string,
+                    "round " + round + ": " + added + " was registered but not masked");
+            }
+        }
+    }
+
+    // Secrets registered on some threads while others clean: every clean
+    // masks what was registered before it, the longer secret whole, and no
+    // registration is lost.
+    [Fact]
+    public void ConcurrentRegistrationKeepsEverySecretMasked()
+    {
+        for (var round = 0; round < Rounds / 4; round++)
+        {
+            var client = LiveClient();
+            var utility = client.GetUtility();
+            var root = client.GetRootCtx();
+            var inner = "INNER-SECRET-" + round;
+            utility.CleanAdd(root, inner);
+            utility.CleanAdd(root, "OUTER-" + inner + "-TAIL");
+            var text = "a " + inner + " b OUTER-" + inner + "-TAIL c";
+            Assert.Equal("a [redacted] b [redacted] c", utility.Clean(root, text));
+
+            var registering = new CountdownEvent(Width / 2);
+            var wrong = new ConcurrentQueue<string>();
+            var thrown = AtOnce(n =>
+            {
+                if (n < Width / 2)
+                {
+                    RegisterSecrets(utility, root, round, n, registering);
+                    return;
+                }
+                while (!registering.IsSet)
+                {
+                    var got = utility.Clean(root, text) as string;
+                    if ("a [redacted] b [redacted] c" != got)
+                    {
+                        wrong.Enqueue(got ?? "null");
+                    }
+                }
+            });
+
+            Assert.True(0 == thrown.Count, "round " + round + " threw:\n" + string.Join("\n", thrown));
+            Assert.True(wrong.IsEmpty, "round " + round + " cleaned to: " + string.Join(" | ", wrong));
+            AssertEverySecretMasked(utility, root, round);
+        }
+    }
+
+    // Requests on one client while secrets register on it: each request
+    // copies the client's options, the registry among them.
+    [Fact]
+    public void ConcurrentRequestsSurviveRegistration()
+    {
+        for (var round = 0; round < Rounds / 4; round++)
+        {
+            var client = LiveClient();
+            var utility = client.GetUtility();
+            var root = client.GetRootCtx();
+            var registering = new CountdownEvent(Width / 2);
+            var failed = new ConcurrentQueue<string>();
+            var thrown = AtOnce(n =>
+            {
+                if (n < Width / 2)
+                {
+                    RegisterSecrets(utility, root, round, n, registering);
+                    return;
+                }
+                while (!registering.IsSet)
+                {
+                    var res = client.Direct(new Dictionary<string, object?> { ["path"] = "p" + n });
+                    if (!Equals(true, res.GetValueOrDefault("ok")))
+                    {
+                        failed.Enqueue(Convert.ToString(res.GetValueOrDefault("err")) ?? "no error");
+                        return;
+                    }
+                }
+            });
+
+            Assert.True(0 == thrown.Count, "round " + round + " threw:\n" + string.Join("\n", thrown));
+            Assert.True(failed.IsEmpty, "round " + round + ", a request failed: " + string.Join(" | ", failed));
+            AssertEverySecretMasked(utility, root, round);
+        }
+    }
+}

package/project/.sdk/tm/csharp/utility/Clean.cs

@@ -21,10 +21,13 @@ public sealed class CleanConfig
 {
     public bool Active = true;
     public List<string> Keys = new();
+    // Requests on other threads clean while one registers, so a registration
+    // publishes a new list and never changes a published one.
     public List<string> Values = new();
     public string Mask = "[redacted]";
     public int Hint;
     public int Min = 4;
+    internal readonly object Registering = new();
 }
 
 public static partial class SdkUtility
@@ -152,18 +155,17 @@ public static partial class SdkUtility
         {
             return;
         }
-        var changed = false;
-        foreach (var form in CleanForms(s))
+        lock (cfg.Registering)
         {
-            if (form.Length >= cfg.Min && !cfg.Values.Contains(form))
+            var added = CleanForms(s).Where(f => f.Length >= cfg.Min && !cfg.Values.Contains(f)).ToList();
+            if (0 == added.Count)
             {
-                cfg.Values.Add(form);
-                changed = true;
+                return;
             }
-        }
-        if (changed)
-        {
-            cfg.Values.Sort((a, b) => b.Length.CompareTo(a.Length));
+            var values = new List<string>(cfg.Values);
+            values.AddRange(added);
+            values.Sort((a, b) => b.Length.CompareTo(a.Length));
+            Volatile.Write(ref cfg.Values, values);
         }
     }
 
@@ -179,7 +181,7 @@ public static partial class SdkUtility
     private static string CleanString(CleanConfig cfg, string text)
     {
         var out_ = text;
-        foreach (var value in cfg.Values)
+        foreach (var value in Volatile.Read(ref cfg.Values))
         {
             if (out_.Contains(value))
             {

package/project/.sdk/tm/csharp/utility/TransformRequest.cs

@@ -49,7 +49,7 @@ public static partial class SdkUtility
 
     // A header, cookie or query argument travels where PrepareHeadersUtil or
     // PrepareQueryUtil sends it, so the body is built from the request data
-    // without it, unless the entity declares it as a field too.
+    // without it, unless the point marks it as a field the body keeps.
     private static List<string> RoutedArgNames(Context ctx)
     {
         return CallArgs(ctx, "header").Concat(CallArgs(ctx, "cookie")).Concat(CallArgs(ctx, "query"))

package/project/.sdk/tm/elixir/lib/projectname/context.ex

@@ -150,6 +150,9 @@ defmodule ProjectName.Context do
             S.jm(["entity", entname, "name", opname, "input", inpt, "points", points])
           )
 
+        # Racing processes may each build and store an Operation, and an entry
+        # lost to a concurrent write is rebuilt; an Operation is a read-only
+        # descriptor, so a duplicate is harmless.
         S.setprop(opmap, cache_key, op)
         op
     end

package/project/.sdk/tm/elixir/lib/projectname/entity_base.ex

@@ -108,7 +108,8 @@ defmodule ProjectName.EntityBase do
   # materialises the whole result). When the streaming feature is active the
   # result carries a `stream` closure and this yields from it (honouring
   # chunkSize); otherwise it falls back to the materialised items, so stream
-  # always yields. Records are unwrapped to bare struct maps (matching list).
+  # always yields. An entity is yielded as its record, the struct map its
+  # data_get/1 returns.
   # `callopts` parameterises the call:
   #   - ctrl:   per-call pipeline control (threaded onto the op ctx);
   #   - body:   an enumerable/list payload for outbound (upload) streaming,

package/project/.sdk/tm/elixir/lib/projectname/utility.ex

@@ -204,24 +204,45 @@ defmodule ProjectName.Utility do
   def clean_add_impl(ctx, value) do
     cfg = clean_config(ctx)
     minlen = numof(S.getprop(cfg, "min"), 4)
-    values = S.getprop(cfg, "values")
 
-    if is_binary(value) and String.length(value) >= minlen and S.islist(values) do
-      have = list_values(values)
+    if is_binary(value) and String.length(value) >= minlen and S.islist(S.getprop(cfg, "values")) do
+      forms = Enum.filter(clean_forms(value), &(String.length(&1) >= minlen))
 
-      add =
-        clean_forms(value)
-        |> Enum.filter(fn f -> String.length(f) >= minlen and f not in have end)
+      # A registration reads the registry and writes a new one, so a request
+      # in another process registering at once would drop it; they take turns.
+      have = list_values(S.getprop(cfg, "values"))
 
-      # Longest first, so a value is never masked by a substring of itself.
-      if add != [] do
-        S.setprop(cfg, "values", S.jt(Enum.sort_by(have ++ add, &(-String.length(&1)))))
+      if Enum.any?(forms, &(&1 not in have)) do
+        registering(fn -> publish(cfg, forms) end)
       end
     end
 
     nil
   end
 
+  # Retried after a millisecond rather than after global's back-off sleep,
+  # which grows to seconds; the lock is released if its holder dies.
+  defp registering(fun) do
+    case :global.trans({{__MODULE__, :clean_registry}, self()}, fun, [node()], 0) do
+      :aborted ->
+        Process.sleep(1)
+        registering(fun)
+
+      done ->
+        done
+    end
+  end
+
+  defp publish(cfg, forms) do
+    have = list_values(S.getprop(cfg, "values"))
+    add = Enum.reject(forms, &(&1 in have))
+
+    # Longest first, so a value is never masked by a substring of itself.
+    if add != [] do
+      S.setprop(cfg, "values", S.jt(Enum.sort_by(have ++ add, &(-String.length(&1)))))
+    end
+  end
+
   defp mask_value(cfg, value) do
     hint = numof(S.getprop(cfg, "hint"), 0)
     mask = strof(S.getprop(cfg, "mask"))
@@ -2280,7 +2301,7 @@ defmodule ProjectName.Utility do
 
   # A header, cookie or query argument travels where prepare_headers_impl or
   # prepare_query_impl sends it, so the body is built from the request data
-  # without it, unless the entity declares it as a field too.
+  # without it, unless the point marks it as a field the body keeps.
   defp routed_arg_names(ctx) do
     (call_args(ctx, "header") ++ call_args(ctx, "cookie") ++ call_args(ctx, "query"))
     |> Enum.map(&elem(&1, 0))

package/project/.sdk/tm/elixir/test/concurrency_test.exs

@@ -0,0 +1,176 @@
+# ProjectName SDK concurrency test
+#
+# Requests in flight at once on one client, each in its own process, register
+# and clean secrets through the one registry the client holds.
+
+defmodule ProjectName.ConcurrencyTest do
+  use ExUnit.Case
+
+  alias ProjectName.Utility
+  alias Voxgig.Struct, as: S
+
+  @rounds 3
+  @width 8
+  @added 16
+  @masked "a [redacted] b [redacted] c"
+
+  # Runs body in @width processes, released once every one is waiting, and
+  # returns what they raised.
+  defp at_once(body) do
+    parent = self()
+    waiting = :counters.new(1, [:atomics])
+
+    tasks =
+      for n <- 0..(@width - 1) do
+        Task.async(fn ->
+          :counters.add(waiting, 1, 1)
+          send(parent, {:ready, self()})
+
+          receive do
+            :go -> :ok
+          end
+
+          seen = :counters.get(waiting, 1)
+
+          try do
+            body.(n)
+            {seen, nil}
+          rescue
+            err -> {seen, err}
+          end
+        end)
+      end
+
+    ready =
+      for _ <- tasks do
+        receive do
+          {:ready, pid} -> pid
+        end
+      end
+
+    Enum.each(ready, &send(&1, :go))
+    outcomes = Task.await_many(tasks, :infinity)
+
+    for {seen, _} <- outcomes do
+      assert @width == seen, "a process was released with #{seen} of #{@width} waiting"
+    end
+
+    for {_, err} <- outcomes, nil != err, do: err
+  end
+
+  # A live client whose transport answers at once.
+  defp live_client do
+    fetch = fn _url, _fetchdef ->
+      {S.jm([
+         "status", 200,
+         "statusText", "OK",
+         "headers", S.jm([]),
+         "json", fn -> S.jm(["ok", true]) end
+       ]), nil}
+    end
+
+    ProjectName.new(
+      S.jm([
+        "base", "http://concurrency.test/api",
+        "allow", S.jm(["op", "direct"]),
+        "system", S.jm(["fetch", fetch])
+      ])
+    )
+  end
+
+  defp register_secrets(root, round, n, registering) do
+    try do
+      for k <- 1..@added, do: Utility.clean_add(root, "ADDED-SECRET-#{round}-#{n}-#{k}")
+    after
+      :counters.sub(registering, 1, 1)
+    end
+  end
+
+  defp assert_every_secret_masked(root, round) do
+    for n <- 0..(div(@width, 2) - 1), k <- 1..@added do
+      added = "ADDED-SECRET-#{round}-#{n}-#{k}"
+
+      assert "[redacted]" == Utility.clean(root, added),
+             "round #{round}: #{added} was registered but not masked"
+    end
+  end
+
+  # A cleaner pauses between passes, so the registrars contend with each other.
+  defp clean_while(root, text, registering) do
+    if 0 < :counters.get(registering, 1) do
+      got = Utility.clean(root, text)
+
+      if @masked != got do
+        raise "cleaned to: " <> inspect(got)
+      end
+
+      Process.sleep(1)
+      clean_while(root, text, registering)
+    end
+  end
+
+  # Secrets registered in some processes while others clean: every clean masks
+  # what was registered before it, the longer secret whole, and no
+  # registration is lost.
+  test "concurrent registration keeps every secret masked" do
+    for round <- 1..@rounds do
+      client = ProjectName.test()
+      root = ProjectName.get_root_ctx(client)
+      inner = "INNER-SECRET-#{round}"
+      Utility.clean_add(root, inner)
+      Utility.clean_add(root, "OUTER-#{inner}-TAIL")
+      text = "a #{inner} b OUTER-#{inner}-TAIL c"
+      assert @masked == Utility.clean(root, text)
+
+      registering = :counters.new(1, [:atomics])
+      :counters.put(registering, 1, div(@width, 2))
+
+      raised =
+        at_once(fn n ->
+          if n < div(@width, 2) do
+            register_secrets(root, round, n, registering)
+          else
+            clean_while(root, text, registering)
+          end
+        end)
+
+      assert [] == raised, "round #{round} raised: #{inspect(raised)}"
+      assert_every_secret_masked(root, round)
+    end
+  end
+
+  defp request_while(client, n, registering) do
+    if 0 < :counters.get(registering, 1) do
+      res = ProjectName.direct(client, S.jm(["path", "p#{n}"]))
+
+      if true != S.getprop(res, "ok") do
+        raise "a request failed: " <> inspect(S.getprop(res, "err"))
+      end
+
+      request_while(client, n, registering)
+    end
+  end
+
+  # Requests on one client while secrets register on it: each request copies
+  # the client's options, the registry among them.
+  test "concurrent requests survive registration" do
+    for round <- 1..@rounds do
+      client = live_client()
+      root = ProjectName.get_root_ctx(client)
+      registering = :counters.new(1, [:atomics])
+      :counters.put(registering, 1, div(@width, 2))
+
+      raised =
+        at_once(fn n ->
+          if n < div(@width, 2) do
+            register_secrets(root, round, n, registering)
+          else
+            request_while(client, n, registering)
+          end
+        end)
+
+      assert [] == raised, "round #{round} raised: #{inspect(raised)}"
+      assert_every_secret_masked(root, round)
+    end
+  end
+end

package/project/.sdk/tm/go/core/context.go

@@ -280,9 +280,13 @@ func (ctx *Context) resolveOp(opname string) *Operation {
 		"points": targets,
 	})
 
+	// Every request racing to build this Operation gets the one stored first.
 	ctx.Opmu.Lock()
+	defer ctx.Opmu.Unlock()
+	if stored, ok := ctx.Opmap[cacheKey]; ok && stored != nil {
+		return stored
+	}
 	ctx.Opmap[cacheKey] = op
-	ctx.Opmu.Unlock()
 	return op
 }