@voxgig/sdkgen 4.34.0 → 4.34.2

package/project/.sdk/tm/rust/utility/prepare_query.rs

@@ -2,6 +2,7 @@ use std::rc::Rc;
 
 use crate::core::context::Context;
 use crate::core::helpers::{getp, setp};
+use crate::utility::param::call_args;
 use crate::utility::voxgigstruct::Value;
 
 pub fn prepare_query_util(ctx: &Rc<Context>) -> Value {
@@ -45,14 +46,28 @@ pub fn prepare_query_util(ctx: &Rc<Context>) -> Value {
         false
     };
 
-    // A header parameter travels in the headers, which prepare_headers fills.
+    // A header or cookie parameter travels in the headers, which prepare_headers
+    // fills, unless a query parameter shares its name: then both are sent.
     let aheader = getp(&getp(&point, "args"), "header");
+    let acookie = getp(&getp(&point, "args"), "cookie");
+    let declared = getp(&getp(&point, "args"), "query");
     let in_header = |key: &str| -> bool {
-        if let Value::List(hl) = &aheader {
-            for hd in hl.borrow().iter() {
-                if let Value::Str(s) = getp(hd, "name") {
+        if let Value::List(ql) = &declared {
+            for qd in ql.borrow().iter() {
+                if let Value::Str(s) = getp(qd, "name") {
                     if s == key {
-                        return true;
+                        return false;
+                    }
+                }
+            }
+        }
+        for located in [&aheader, &acookie] {
+            if let Value::List(hl) = located {
+                for hd in hl.borrow().iter() {
+                    if let Value::Str(s) = getp(hd, "name") {
+                        if s == key {
+                            return true;
+                        }
                     }
                 }
             }
@@ -92,5 +107,12 @@ pub fn prepare_query_util(ctx: &Rc<Context>) -> Value {
         }
     }
 
+    // A create or update passes its query arguments in its data.
+    for (name, wire, val) in call_args(ctx, "query") {
+        if !val.is_noval() && !val.is_null() && !contains(&name) && !in_args(&name) && !in_header(&name) {
+            setp(&out, &wire, val);
+        }
+    }
+
     out
 }

package/project/.sdk/tm/rust/utility/result_body.rs

@@ -3,6 +3,7 @@ use std::rc::Rc;
 
 use crate::core::context::Context;
 use crate::core::helpers::call_json;
+use crate::core::response::unreadable_body;
 use crate::core::result::SdkResult;
 use crate::utility::voxgigstruct::Value;
 
@@ -12,13 +13,25 @@ pub fn result_body_util(ctx: &Rc<Context>) -> Option<Rc<RefCell<SdkResult>>> {
 
     if let Some(result) = &result {
         if let Some(response) = &response {
-            let (json, body) = {
+            let (json, body, unreadable) = {
                 let r = response.borrow();
-                (r.json.clone(), r.body.clone())
+                (r.json.clone(), r.body.clone(), r.unreadable)
             };
             if matches!(json, Value::Func(_)) && !body.is_noval() && !body.is_null() {
                 result.borrow_mut().body = call_json(&json);
             }
+            if unreadable {
+                let sent = match ctx.spec.borrow().clone() {
+                    Some(spec) => spec.borrow().headers.clone(),
+                    None => Value::Noval,
+                };
+                let (status, headers, failed) = {
+                    let r = result.borrow();
+                    (r.status, r.headers.clone(), r.err.clone())
+                };
+                result.borrow_mut().err =
+                    Some(unreadable_body(ctx, status, &headers, &body, &sent, failed));
+            }
         }
     }
 

package/project/.sdk/tm/rust/utility/transform_request.rs

@@ -2,6 +2,7 @@ use std::rc::Rc;
 
 use crate::core::context::Context;
 use crate::core::helpers::{getp, jo, setp, to_map};
+use crate::utility::param::call_args;
 use crate::utility::voxgigstruct as vs;
 use crate::utility::voxgigstruct::Value;
 
@@ -11,20 +12,32 @@ fn strip_action(reqdata: Value) -> Value {
     omit_keys(reqdata, &["$action".to_string()])
 }
 
-// A header argument travels as a header, which prepare_headers_util sends, so
-// the body is built from the request data without it.
-fn header_arg_names(point: &Value) -> Vec<String> {
-    match getp(&getp(point, "args"), "header") {
-        Value::List(hl) => hl
-            .borrow()
-            .iter()
-            .filter_map(|hd| match getp(hd, "name") {
-                Value::Str(n) if !n.is_empty() => Some(n),
-                _ => None,
-            })
-            .collect(),
-        _ => Vec::new(),
+// A header, cookie or query argument travels where prepare_headers_util or
+// prepare_query_util sends it, so the body is built from the request data
+// without it, unless the point marks it as a field the body keeps.
+fn routed_arg_names(ctx: &Rc<Context>) -> Vec<String> {
+    let mut names = Vec::new();
+    for kind in ["header", "cookie", "query"] {
+        for (name, _, _) in call_args(ctx, kind) {
+            if !field_arg(ctx, &name) {
+                names.push(name);
+            }
+        }
     }
+    names
+}
+
+fn field_arg(ctx: &Rc<Context>, name: &str) -> bool {
+    let point = ctx.point.borrow().clone();
+    ["header", "cookie", "query"].iter().any(|kind| {
+        match getp(&getp(&point, "args"), kind) {
+            Value::List(al) => al.borrow().iter().any(|ad| {
+                matches!(getp(ad, "name"), Value::Str(ref n) if n == name)
+                    && matches!(getp(ad, "field"), Value::Bool(true))
+            }),
+            _ => false,
+        }
+    })
 }
 
 fn omit_keys(reqdata: Value, names: &[String]) -> Value {
@@ -59,7 +72,7 @@ pub fn transform_request_util(ctx: &Rc<Context>) -> Value {
         sp.borrow_mut().step = "reqform".to_string();
     }
 
-    let reqdata = omit_keys(ctx.reqdata.borrow().clone(), &header_arg_names(&point));
+    let reqdata = omit_keys(ctx.reqdata.borrow().clone(), &routed_arg_names(ctx));
 
     let transform = to_map(&getp(&point, "transform"));
     if transform.is_noval() {

package/project/.sdk/tm/scala/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) $$const.year$$ Voxgig
+Copyright (c) $$const.year$$ PROJECTPUBLISHER
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/project/.sdk/tm/scala/Makefile

@@ -16,12 +16,16 @@ build:
 # Main classes are NAMED here rather than globbed, so a per-feature suite has
 # to be listed. SecretsTestMain drives the feature through the `extend` seam
 # and so runs in every scala SDK, model-activated or not. SdkCleanTestMain is
-# the canary sweep: no credential leaves the SDK in any form.
+# the canary sweep: no credential leaves the SDK in any form. SdkServerTestMain
+# checks how a templated base URL takes its server variables.
+# SdkConcurrencyTestMain races first requests on one client.
 test:
 	scala-cli run . --main-class OmniSmoke
 	scala-cli run . --main-class SdkTestMain
 	scala-cli run . --main-class SdkEntityTestMain
 	scala-cli run . --main-class SdkCleanTestMain
+	scala-cli run . --main-class SdkServerTestMain
+	scala-cli run . --main-class SdkConcurrencyTestMain
 	scala-cli run . --main-class SecretsTestMain
 	scala-cli run . --main-class Runner -- ../.sdk/test/test.json
 	scala-cli run . --main-class PrimaryCorpusMain -- ../.sdk/test/test.json

package/project/.sdk/tm/scala/core/Context.scala

@@ -1,7 +1,7 @@
 package SCALAPACKAGE.core
 
 import java.util.{ArrayList, LinkedHashMap, List => JList, Map => JMap}
-import java.util.concurrent.ThreadLocalRandom
+import java.util.concurrent.{ConcurrentHashMap, ThreadLocalRandom}
 import SCALAPACKAGE.utility.struct.Struct
 
 // Per-operation context threaded through the pipeline and feature hooks.
@@ -85,7 +85,7 @@ class Context(ctxmap: JMap[String, Object], basectx: Context) {
     // Opmap
     Helpers.getCtxProp(ctxmap, "opmap") match { case om: JMap[_, _] => opmap = om.asInstanceOf[JMap[String, Operation]]; case _ => }
     if (opmap == null && basectx != null) opmap = basectx.opmap
-    if (opmap == null) opmap = new LinkedHashMap[String, Operation]()
+    if (opmap == null) opmap = new ConcurrentHashMap[String, Operation]()
 
     // Data
     data = Helpers.toMapAny(Helpers.getCtxProp(ctxmap, "data"))
@@ -133,7 +133,7 @@ class Context(ctxmap: JMap[String, Object], basectx: Context) {
     val opcfg = Struct.getpath(config, java.util.List.of("entity", entname, "op", opname))
 
     var input = "match"
-    if ("update" == opname || "create" == opname) input = "data"
+    if ("update" == opname || "create" == opname || "patch" == opname) input = "data"
 
     var points: JList[Object] = null
     opcfg match {
@@ -153,8 +153,10 @@ class Context(ctxmap: JMap[String, Object], basectx: Context) {
     opdef.put("points", points)
 
     val newop = new Operation(opdef)
-    opmap.put(cacheKey, newop)
-    newop
+    // Requests on other threads share this cache; every one of them gets
+    // the Operation stored first.
+    val stored = opmap.putIfAbsent(cacheKey, newop)
+    if (stored != null) stored else newop
   }
 
   def makeError(code: String, msg: String): SdkError = new SdkError(code, msg, this)

package/project/.sdk/tm/scala/core/Contracts.scala

@@ -18,6 +18,7 @@ trait SdkEntity extends Entity {
   def list(reqmatch: JMap[String, Object], ctrl: JMap[String, Object]): Object
   def create(reqdata: JMap[String, Object], ctrl: JMap[String, Object]): Object
   def update(reqdata: JMap[String, Object], ctrl: JMap[String, Object]): Object
+  def patch(reqdata: JMap[String, Object], ctrl: JMap[String, Object]): Object
   def remove(reqmatch: JMap[String, Object], ctrl: JMap[String, Object]): Object
   // Runs `action` through the pipeline and returns an Iterator over result
   // items (the streaming feature's incremental output when active, else the

package/project/.sdk/tm/scala/core/Helpers.scala

@@ -10,6 +10,13 @@ object Helpers {
   // underlying API spec doesn't define. The static SdkEntity contract
   // requires every CRUD method on every entity, so absent ops must still be
   // callable — they error at runtime instead of failing to compile.
+  // Whether a comma-separated allow option names the item: whole names, any case.
+  def allowed(names: Object, item: String): Boolean = names match {
+    case list: String if null != item && item.nonEmpty =>
+      list.split(",", -1).exists(_.trim.equalsIgnoreCase(item))
+    case _ => false
+  }
+
   def unsupportedOp(opname: String, entityname: String): SdkError =
     new SdkError("op_unsupported",
       "operation '" + opname + "' not supported by entity '" + entityname + "'", null)

package/project/.sdk/tm/scala/core/Response.scala

@@ -13,6 +13,8 @@ class Response(resmap: JMap[String, Object]) {
   var jsonFunc: Supplier[Object] = null
   var body: Object = null
   var err: RuntimeException = null
+  // Set by a transport that could not read a non-blank body as JSON.
+  var unreadable: Boolean = false
 
   locally {
     val s = Struct.getprop(resmap, "status")
@@ -33,5 +35,58 @@ class Response(resmap: JMap[String, Object]) {
       case e: RuntimeException => err = e
       case _ =>
     }
+
+    unreadable = java.lang.Boolean.TRUE == Struct.getprop(resmap, "unreadable")
+  }
+}
+
+object Response {
+  private val PreviewLength = 160
+
+  // A body that is not JSON. An HTTP failure keeps its own error, with the
+  // response described; otherwise the code tells a wrong content type from
+  // malformed JSON.
+  def unreadableBody(ctx: Context, status: Int, headers: Object, text: Object, sent: Object,
+      failed: RuntimeException): RuntimeException = {
+    val ctype = headerValue(headers, "content-type")
+    val agent = clean(ctx, headerValue(sent, "user-agent"))
+    val detail = "HTTP " + status + ", content-type " + (if (ctype.isEmpty) "none" else ctype) +
+      ", user-agent " + (if (agent.isEmpty) "transport default" else agent) +
+      (if (text == null) "" else ", body: " + preview(ctx, text))
+
+    failed match {
+      case null if ctype.isEmpty || ctype.toLowerCase.contains("json") =>
+        ctx.makeError("response_json_invalid", "response: body is not valid JSON (" + detail + ")")
+      case null =>
+        ctx.makeError("response_content_type",
+          "response: expected JSON, got " + ctype + " (" + detail + ")")
+      case e: SdkError =>
+        e.msg += " (" + detail + ")"
+        e
+      case e =>
+        new RuntimeException(e.getMessage + " (" + detail + ")", e)
+    }
+  }
+
+  private def headerValue(headers: Object, name: String): String = headers match {
+    case hm: JMap[_, _] =>
+      val it = hm.entrySet().iterator()
+      while (it.hasNext) {
+        val e = it.next()
+        if (name.equalsIgnoreCase(String.valueOf(e.getKey))) return String.valueOf(e.getValue)
+      }
+      ""
+    case _ => ""
+  }
+
+  private def clean(ctx: Context, s: String): String =
+    if (ctx.utility == null || ctx.utility.clean == null) s
+    else String.valueOf(ctx.utility.clean(ctx, s))
+
+  // Cleaned whole: a secret the bound would split could leave its prefix.
+  private def preview(ctx: Context, text: Object): String = {
+    val flat = clean(ctx, String.valueOf(text).replaceAll("\\s+", " ").trim)
+    if (flat.codePointCount(0, flat.length) <= PreviewLength) flat
+    else flat.substring(0, flat.offsetByCodePoints(0, PreviewLength)) + "..."
   }
 }

package/project/.sdk/tm/scala/core/SdkClient.scala

@@ -106,6 +106,14 @@ abstract class SdkClient(options0: JMap[String, Object]) {
     val path = Struct.getprop(fetchargs, "path") match { case s: String => s; case _ => "" }
     var method = Struct.getprop(fetchargs, "method") match { case s: String => s; case _ => "" }
     if ("" == method) method = "GET"
+    method = method.toUpperCase(java.util.Locale.ROOT)
+
+    val allowMethod = Struct.getpath(opts, java.util.List.of("allow", "method"))
+    if (!Helpers.allowed(allowMethod, method)) {
+      throw ctx.makeError("spec_method_allow",
+        "Method \"" + method + "\" not allowed by SDK option allow.method value: \"" +
+          (allowMethod match { case s: String => s; case _ => "" }) + "\"")
+    }
 
     var params = Helpers.toMapAny(Struct.getprop(fetchargs, "params"))
     if (params == null) params = new LinkedHashMap[String, Object]()
@@ -149,10 +157,7 @@ abstract class SdkClient(options0: JMap[String, Object]) {
 
   // Is this raw-access op permitted by the SDK's allow.op option?
   private def opAllowed(op: String): Boolean =
-    Struct.getpath(this.options, java.util.List.of("allow", "op")) match {
-      case s: String => s.contains(op)
-      case _ => false
-    }
+    Helpers.allowed(Struct.getpath(this.options, java.util.List.of("allow", "op")), op)
 
   private def opDenied(op: String): JMap[String, Object] = {
     val allow = Struct.getpath(this.options, java.util.List.of("allow", "op")) match {
@@ -278,17 +283,25 @@ abstract class SdkClient(options0: JMap[String, Object]) {
         val noBody = status == 204 || status == 304 || "0" == contentLength
 
         var jsonData: Object = null
+        var bodyErr: RuntimeException = null
         if (!noBody) {
           Struct.getprop(fm, "json") match {
             case jf: Supplier[_] => jsonData = jf.asInstanceOf[Supplier[Object]].get()
             case _ =>
           }
+          if (java.lang.Boolean.TRUE == Struct.getprop(fm, "unreadable")) {
+            val failed = if (status >= 200 && status < 300) null else ctx.makeError(
+              "request_status", "request: " + status + ": " + Struct.getprop(fm, "statusText"))
+            bodyErr = Response.unreadableBody(ctx, status, headers, Struct.getprop(fm, "body"),
+              fetchdef.get("headers"), failed)
+          }
         }
 
-        out.put("ok", java.lang.Boolean.valueOf(status >= 200 && status < 300))
+        out.put("ok", java.lang.Boolean.valueOf(bodyErr == null && status >= 200 && status < 300))
         out.put("status", java.lang.Integer.valueOf(status))
         out.put("headers", headers)
         out.put("data", jsonData)
+        if (bodyErr != null) out.put("err", cleanErr(ctx, bodyErr))
         out
       case _ =>
         out.put("ok", java.lang.Boolean.FALSE)

package/project/.sdk/tm/scala/core/Spec.scala

@@ -18,6 +18,9 @@ class Spec(specmap: JMap[String, Object]) {
   var body: Object = null
   var url: String = ""
   var path: String = ""
+  // The query parameters prepareAuth placed: the credential, which the
+  // request sends and the entity's match leaves out.
+  var authquery: JList[String] = new java.util.ArrayList[String]()
 
   def this() = this(null)
 

package/project/.sdk/tm/scala/feature/TestFeature.scala

@@ -45,9 +45,42 @@ class TestFeature extends BaseFeature("test", "0.0.1", true) {
   // and the response transform unwraps that key on the way back. Returning the
   // bare payload means the transform unwraps a property that is not there and
   // the caller gets nothing. Mirrors the go/ts/lua/php mocks.
+  // The record the mock keeps: the request data without `$body`, which only the
+  // wire carries.
+  private def record(reqdata: JMap[String, Object]): JMap[String, Object] = {
+    if (reqdata == null) return null
+    val out = new LinkedHashMap[String, Object](reqdata)
+    out.remove("$body")
+    out
+  }
+
+  // The key a list's response transform
+  // ["`$EACH`", "body", {"`$MERGE`": "`.<key>`"}] reads each item's record under.
+  private def itemEnvelopeKey(restf: Object): String = restf match {
+    case spec: JList[_] if 3 == spec.size && "`$EACH`" == spec.get(0) && "body" == spec.get(1) =>
+      spec.get(2) match {
+        case m: JMap[_, _] => m.get("`$MERGE`") match {
+          case merge: String => "^`\\.([^.`$]+)`$".r.findFirstMatchIn(merge).map(_.group(1)).orNull
+          case _ => null
+        }
+        case _ => null
+      }
+    case _ => null
+  }
+
   private def envelope(ctx: Context, data: Object): Object = {
     if (data == null || ctx == null || ctx.point == null) return data
     val tm = Struct.getprop(ctx.point, "transform")
+    val key = itemEnvelopeKey(Struct.getprop(tm, "res"))
+    if (null != key && data.isInstanceOf[JList[_]]) {
+      val items = new java.util.ArrayList[Object]()
+      data.asInstanceOf[JList[Object]].forEach { item =>
+        val wrapped = new LinkedHashMap[String, Object]()
+        wrapped.put(key, item)
+        items.add(wrapped)
+      }
+      return items
+    }
     Struct.getprop(tm, "res") match {
       case spec: String =>
         // Rebuild whatever nesting the transform unwraps. Multi-segment on purpose:
@@ -124,7 +157,7 @@ class TestFeature extends BaseFeature("test", "0.0.1", true) {
       while (it.hasNext) Struct.delprop(it.next(), "$KEY")
       val out = Struct.clone(found)
       respond(ctx, 200, out, null)
-    } else if ("update" == op.name) {
+    } else if ("update" == op.name || "patch" == op.name) {
       var updateMatch = new LinkedHashMap[String, Object]()
       if (ctx.reqdata != null) {
         if (ctx.reqdata.containsKey("id")) updateMatch.put("id", ctx.reqdata.get("id"))
@@ -141,7 +174,7 @@ class TestFeature extends BaseFeature("test", "0.0.1", true) {
       val ent = Struct.getelem(found, java.lang.Integer.valueOf(0))
       // update miss: 404, never another record
       if (ent == null) return respond(ctx, 404, null, extra("statusText", "Not found"))
-      ent match { case _: JMap[_, _] if ctx.reqdata != null => Struct.merge(Struct.jt(ent, ctx.reqdata)); case _ => }
+      ent match { case _: JMap[_, _] if ctx.reqdata != null => Struct.merge(Struct.jt(ent, record(ctx.reqdata))); case _ => }
       Struct.delprop(ent, "$KEY")
       val out = Struct.clone(ent)
       respond(ctx, 200, out, null)
@@ -166,7 +199,7 @@ class TestFeature extends BaseFeature("test", "0.0.1", true) {
           java.lang.Integer.valueOf(r.nextInt(0x10000)), java.lang.Integer.valueOf(r.nextInt(0x10000)))
       }
 
-      val ent = Struct.clone(ctx.reqdata)
+      val ent = Struct.clone(record(ctx.reqdata))
       ent match {
         case entm0: JMap[_, _] =>
           val entm = entm0.asInstanceOf[JMap[String, Object]]

package/project/.sdk/tm/scala/feature/TimeoutFeature.scala

@@ -2,6 +2,7 @@ package SCALAPACKAGE.feature
 
 import java.util.{Map => JMap}
 import java.util.concurrent.{CompletableFuture, CompletionException, TimeUnit, TimeoutException}
+import java.util.concurrent.atomic.AtomicLong
 import java.util.function.Supplier
 import SCALAPACKAGE.core.{Context, FetcherFn, SdkClient}
 
@@ -28,28 +29,53 @@ class TimeoutFeature extends BaseFeature("timeout", "0.0.1", true) {
     val deadline = FeatureOptions.foptInt(this.options, "ms", 30000)
     if (deadline <= 0) return inner(ctx, url, fetchdef)
 
-    val sup: Supplier[Object] = () => inner(ctx, url, fetchdef)
+    // The deadline runs from here, not from the wait below: a caller paused
+    // between the two would otherwise find a late response complete and take
+    // it. The worker notes when it finished, so a response or a failure after
+    // the deadline is a timeout however late the caller looks.
+    val now = FeatureOptions.foptNow(this.options)
+    val start = now.getAsLong
+    val arrived = new AtomicLong(Long.MaxValue)
+    val sup: Supplier[Object] = () => {
+      try inner(ctx, url, fetchdef)
+      finally arrived.set(now.getAsLong)
+    }
     val fut = CompletableFuture.supplyAsync(sup)
 
-    try fut.get(deadline.toLong, TimeUnit.MILLISECONDS)
+    try {
+      val remaining = Math.max(0L, deadline - (now.getAsLong - start))
+      val out = try fut.get(remaining, TimeUnit.MILLISECONDS)
+      catch {
+        case e: java.util.concurrent.ExecutionException =>
+          if (deadline < arrived.get - start) throw timeout(ctx, deadline)
+          throw unwrap(e)
+      }
+      if (deadline < arrived.get - start) throw timeout(ctx, deadline)
+      out
+    }
     catch {
-      case _: TimeoutException =>
-        track(deadline)
-        throw ctx.makeError("timeout", "Request exceeded timeout of " + deadline + "ms")
-      case e: java.util.concurrent.ExecutionException =>
-        var cause = e.getCause
-        cause match { case ce: CompletionException if ce.getCause != null => cause = ce.getCause; case _ => }
-        cause match {
-          case re: RuntimeException => throw re
-          case err: Error => throw err
-          case _ => throw new RuntimeException(cause)
-        }
+      case _: TimeoutException => throw timeout(ctx, deadline)
       case _: InterruptedException =>
         Thread.currentThread().interrupt()
         throw new RuntimeException("interrupted")
     }
   }
 
+  private def unwrap(e: java.util.concurrent.ExecutionException): Throwable = {
+    var cause = e.getCause
+    cause match { case ce: CompletionException if ce.getCause != null => cause = ce.getCause; case _ => }
+    cause match {
+      case re: RuntimeException => re
+      case err: Error => err
+      case _ => new RuntimeException(cause)
+    }
+  }
+
+  private def timeout(ctx: Context, deadline: Int): Throwable = {
+    track(deadline)
+    ctx.makeError("timeout", "Request exceeded timeout of " + deadline + "ms")
+  }
+
   private def track(deadline: Int): Unit = {
     this.count += 1
     this.ms = deadline

package/project/.sdk/tm/scala/feature/ValidateFeature.scala

@@ -171,6 +171,8 @@ class ValidateFeature extends BaseFeature("validate", "0.0.1", true) {
     // fields will never name it - and under `strict` every custom-action call
     // would be rejected for the one key that made it reachable.
     out.remove("$action")
+    // Nor is `$body`, the raw request body.
+    out.remove("$body")
 
     out
   }