@voxgig/sdkgen 4.2.5 → 4.2.6

package/bin/voxgig-sdkgen

@@ -8,7 +8,7 @@ const { Shape, One, Skip } = require('shape')
 
 const { SdkGen } = require('../dist/sdkgen.js')
 
-const VERSION = '4.2.5'
+const VERSION = '4.2.6'
 const KONSOLE = console
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voxgig/sdkgen",
-  "version": "4.2.5",
+  "version": "4.2.6",
   "main": "dist/sdkgen.js",
   "type": "commonjs",
   "engines": {

package/project/.sdk/src/cmp/go/utility_go.ts

@@ -7,6 +7,7 @@ import {
   canonKey,
   canonScalarKey,
   each,
+  opParams,
   exampleVarName,
   names,
 } from '@voxgig/sdkgen'
@@ -36,13 +37,25 @@ function projectPath(suffix?: string): string {
 // has no params and the request shape mirrors the entity fields). Returns
 // undefined when neither is present.
 function paramCanonType(entity: any, op: any, paramName: string): unknown {
-  const points = op && op.points ? each(op.points) : []
-  for (const pt of points as any[]) {
-    const params = pt && pt.args && pt.args.params ? each(pt.args.params) : []
-    const found = (params as any[]).find((p: any) => p && p.name === paramName)
-    if (found) {
-      return found.type
-    }
+  // opParams, NOT a raw walk of op.points.
+  //
+  // opParams drops points flagged with select['$action'] and merges what is
+  // left; the typed-model generator reaches the op's params through it, so
+  // anything else is a DIFFERENT set of params wearing the same name.
+  //
+  // A raw walk returned the first match on ANY point, action points included.
+  // github's `action` entity has a field `owner` (`$OBJECT`, "A GitHub user")
+  // and seven create points whose path carries `{owner}` (a string) — so the
+  // type said Record<string, any> and the doc example said 'example_owner',
+  // and ts/README.md stopped compiling. Same for workflow_id: `number` in the
+  // type, quoted string in the example.
+  //
+  // Deriving from opParams is what actually makes good on the promise below —
+  // that the docs and the generated types cannot disagree.
+  const params = op ? each(opParams(op)) : []
+  const found = (params as any[]).find((p: any) => p && p.name === paramName)
+  if (found) {
+    return found.type
   }
   const field = (entity && entity.fields ? each(entity.fields) : [])
     .find((f: any) => f && f.name === paramName) as any

package/project/.sdk/src/cmp/js/utility_js.ts

@@ -6,6 +6,7 @@ import {
   canonKey,
   canonScalarKey,
   each,
+  opParams,
 } from '@voxgig/sdkgen'
 
 import {
@@ -28,13 +29,25 @@ import {
 // does. Falls back to the entity field of the same name (used when the op
 // has no params). Returns undefined when neither is present.
 function paramCanonType(entity: any, op: any, paramName: string): unknown {
-  const points = op && op.points ? each(op.points) : []
-  for (const pt of points as any[]) {
-    const params = pt && pt.args && pt.args.params ? each(pt.args.params) : []
-    const found = (params as any[]).find((p: any) => p && p.name === paramName)
-    if (found) {
-      return found.type
-    }
+  // opParams, NOT a raw walk of op.points.
+  //
+  // opParams drops points flagged with select['$action'] and merges what is
+  // left; the typed-model generator reaches the op's params through it, so
+  // anything else is a DIFFERENT set of params wearing the same name.
+  //
+  // A raw walk returned the first match on ANY point, action points included.
+  // github's `action` entity has a field `owner` (`$OBJECT`, "A GitHub user")
+  // and seven create points whose path carries `{owner}` (a string) — so the
+  // type said Record<string, any> and the doc example said 'example_owner',
+  // and ts/README.md stopped compiling. Same for workflow_id: `number` in the
+  // type, quoted string in the example.
+  //
+  // Deriving from opParams is what actually makes good on the promise below —
+  // that the docs and the generated types cannot disagree.
+  const params = op ? each(opParams(op)) : []
+  const found = (params as any[]).find((p: any) => p && p.name === paramName)
+  if (found) {
+    return found.type
   }
   const field = (entity && entity.fields ? each(entity.fields) : [])
     .find((f: any) => f && f.name === paramName) as any

package/project/.sdk/src/cmp/php/ReadmeExamplesTest_php.ts

@@ -110,7 +110,16 @@ ${entityLines}
 
     // PHP-level errors that indicate a real bug in a documented example (as
     // opposed to an expected not-found / domain error, which is tolerated).
-    private const FATAL = '/(Call to undefined method|Call to undefined function|Call to a member function|ArgumentCountError|Too few arguments|Undefined constant|Uncaught TypeError)/';
+    // A whitelist, so an expected domain error ("404: Not found") that an
+    // example deliberately catches stays tolerated.
+    //
+    // ParseError and "Parse error" are in it because runnable blocks are no
+    // longer linted: test_php_snippets_have_valid_syntax now skips anything the
+    // run pass executes, on the grounds that running a block proves it parses.
+    // That only holds if the run pass can SEE a parse failure — and without
+    // these two patterns it could not, so a syntax error in a runnable example
+    // would have passed both gates.
+    private const FATAL = '/(Call to undefined method|Call to undefined function|Call to a member function|ArgumentCountError|Too few arguments|Undefined constant|Uncaught TypeError|ParseError|Parse error)/';
 
     // The three documentation sources this gate covers.
     private function docs(): array
@@ -228,11 +237,25 @@ ${entityLines}
         $this->assertNotEmpty($this->phpBlocks(), 'docs should contain php examples');
     }
 
-    /** Every php block across all three docs must parse (php -l). */
+    /**
+     * Every php block that is NOT executed must parse (php -l).
+     *
+     * Runnable blocks are deliberately excluded: test_php_examples_run_offline
+     * executes them, and a syntax error there fails that gate with a better
+     * message than a lint pass gives. Linting them as well spawned a second php
+     * process per runnable block for a result the run already establishes.
+     *
+     * The non-runnable blocks — illustrations and signature snippets — are
+     * never executed, so this is the ONLY syntax check they get, and dropping
+     * it entirely would lose that coverage.
+     */
     public function test_php_snippets_have_valid_syntax(): void
     {
         $failures = [];
         foreach ($this->phpBlocks() as $blk) {
+            if ($this->isRunnable($blk['code'])) {
+                continue;
+            }
             $block = $blk['code'];
             $code = preg_match('/^\\s*<\\?php/', $block) ? $block : "<?php\\n" . $block;
             $tmp = tempnam(sys_get_temp_dir(), 'readme_php_') . '.php';
@@ -312,27 +335,136 @@ ${entityLines}
         $ran = 0;
         $failures = [];
         $sdk = __DIR__ . '/../${sdkfile}';
+
+        // BATCHED. One php process for every runnable snippet, not one each.
+        // A repo with 277 entities documents hundreds of examples, and a
+        // process spawn per snippet dominated the php suite's wall clock.
+        //
+        // Each snippet is included inside a closure, so it gets its own
+        // variable scope, and is bracketed by markers so output is attributed
+        // back to the snippet that produced it. The interpreter is SHARED,
+        // which is the cost of batching: a snippet that mutates process-wide
+        // state (the SDK config singleton) or calls exit() affects what
+        // follows it.
+        //
+        // That is why the batch is not trusted blindly. A snippet whose END
+        // marker never arrives — killed by a fatal, or by an exit() that took
+        // the rest of the batch with it — is RE-RUN on its own, in a fresh
+        // process, and that isolated result is the one that counts. So the
+        // fast path is a single spawn, and the moment isolation actually
+        // matters the harness falls back to the old one-process-per-snippet
+        // behaviour for the affected snippets only.
+        $runnable = [];
         foreach ($this->phpBlocks() as $blk) {
-            if (!$this->isRunnable($blk['code'])) {
-                continue;
+            if ($this->isRunnable($blk['code'])) {
+                $runnable[] = $blk;
+            }
+        }
+        $ran = count($runnable);
+
+        if ($ran > 0) {
+            $dir = sys_get_temp_dir() . '/readme_batch_' . getmypid() . '_' . bin2hex(random_bytes(4));
+            @mkdir($dir, 0700, true);
+            $paths = [];
+            foreach ($runnable as $i => $blk) {
+                $f = $dir . '/snip_' . $i . '.php';
+                file_put_contents($f, $this->toRunner($blk['code'], $sdk));
+                $paths[$i] = $f;
             }
-            $ran++;
-            $runner = $this->toRunner($blk['code'], $sdk);
-            $tmp = tempnam(sys_get_temp_dir(), 'readme_run_') . '.php';
-            file_put_contents($tmp, $runner);
+
+            $driver = $dir . '/_driver.php';
+            file_put_contents($driver, $this->batchDriver($paths));
             $out = [];
             $rc = 0;
-            exec('php ' . escapeshellarg($tmp) . ' 2>&1', $out, $rc);
-            @unlink($tmp);
+            exec('php ' . escapeshellarg($driver) . ' 2>&1', $out, $rc);
             $text = implode("\\n", $out);
-            if (preg_match(self::FATAL, $text) === 1) {
-                $failures[] = $blk['doc'] . ' #' . $blk['n'] . ' (exit ' . $rc . "):\\n" . $text . "\\n" . $blk['code'];
+
+            foreach ($runnable as $i => $blk) {
+                $seg = $this->batchSegment($text, $i);
+                if ($seg === null) {
+                    // No END marker: the batch died inside or before this
+                    // snippet. Re-run it alone so the verdict is isolated.
+                    $solo = [];
+                    $src = 0;
+                    exec('php ' . escapeshellarg($paths[$i]) . ' 2>&1', $solo, $src);
+                    $seg = implode("\\n", $solo);
+                    $rc = $src;
+                }
+                if (preg_match(self::FATAL, $seg) === 1) {
+                    $failures[] = $blk['doc'] . ' #' . $blk['n'] . ' (exit ' . $rc . "):\\n" . $seg . "\\n" . $blk['code'];
+                }
             }
+
+            foreach ($paths as $f) { @unlink($f); }
+            @unlink($driver);
+            @rmdir($dir);
         }
         $this->assertGreaterThan(0, $ran, 'expected at least one runnable example to execute');
         $this->assertSame([], $failures, "docs php examples raised a real error when run offline:\\n" . implode("\\n\\n", $failures));
     }
 
+    /**
+     * The batch driver: include each snippet in its own closure scope, marked
+     * so output can be attributed back. The shutdown handler is what makes a
+     * fatal recoverable — it prints the END marker for whichever snippet was
+     * in flight, so the harness can tell "this one died" from "the batch
+     * stopped before reaching it", and re-run only what it must.
+     */
+    private function batchDriver(array $paths): string
+    {
+        // NOWDOC, not heredoc. The driver is PHP source that must survive
+        // being embedded verbatim: a heredoc interpolates it, and $e['message']
+        // with a quoted key is a parse error under interpolation. A nowdoc
+        // substitutes nothing, so the only thing injected is the file list.
+        $tpl = <<<'DRIVER'
+<?php
+$__files = __VOX_FILES__;
+$__current = null;
+register_shutdown_function(function () use (&$__current) {
+    if ($__current !== null) {
+        $e = error_get_last();
+        if ($e !== null) {
+            echo "\\nFATAL: " . $e['message'] . "\\n";
+        }
+        echo "\\n@@VOXEND " . $__current . "\\n";
+    }
+});
+foreach ($__files as $__i => $__f) {
+    $__current = $__i;
+    echo "\\n@@VOXBEGIN " . $__i . "\\n";
+    try {
+        (function ($__path) { include $__path; })($__f);
+    } catch (\\Throwable $__t) {
+        echo "\\nFATAL: " . get_class($__t) . ": " . $__t->getMessage() . "\\n";
+    }
+    echo "\\n@@VOXEND " . $__i . "\\n";
+    $__current = null;
+}
+DRIVER;
+        return str_replace('__VOX_FILES__', var_export($paths, true), $tpl);
+    }
+
+    /**
+     * The output between this snippet's BEGIN and END markers, or null when the
+     * END marker is absent — meaning the batch never got past it, and the
+     * caller must re-run it in isolation rather than guess.
+     */
+    private function batchSegment(string $text, int $i): ?string
+    {
+        $begin = '@@VOXBEGIN ' . $i;
+        $end = '@@VOXEND ' . $i;
+        $b = strpos($text, $begin);
+        if ($b === false) {
+            return null;
+        }
+        $b += strlen($begin);
+        $e = strpos($text, $end, $b);
+        if ($e === false) {
+            return null;
+        }
+        return trim(substr($text, $b, $e - $b));
+    }
+
     /**
      * COMPLETENESS GATE: every fenced php block is partitioned into exactly one
      * of {executed, syntaxchecked-nonrunnable, illustration}. The three counts

package/project/.sdk/src/cmp/py/ReadmeExamplesTest_py.ts

@@ -484,6 +484,80 @@ def _is_not_found_error(last):
     return ("404" in last) or ("not found" in low) or ("notfound" in low)
 
 
+def _run_batch(items, label):
+    # BATCHED. One interpreter for every runnable block in this document,
+    # instead of one process each. A 269-entity SDK documents hundreds of
+    # examples and the spawn cost dominated the suite.
+    #
+    # Each block is exec'd in a FRESH globals dict, so names do not leak
+    # between snippets, and is bracketed by markers so output is attributed
+    # back. The interpreter is shared, which is the cost: a snippet that
+    # mutates process-wide state or dies hard affects what follows.
+    #
+    # So the batch is not trusted blindly. Any snippet whose END marker is
+    # missing is re-run on its own by the caller, and that isolated verdict is
+    # the one that counts.
+    driver = (
+        "import sys, traceback" + _NL
+        + "_S = __VOX_SOURCES__" + _NL
+        + "for _i, _src in _S:" + _NL
+        + "    print('@@VOXBEGIN ' + str(_i), flush=True)" + _NL
+        + "    try:" + _NL
+        + "        _g = {'__name__': '__main__'}" + _NL
+        + "        exec(compile(_src, '<snippet>', 'exec'), _g)" + _NL
+        + "    except BaseException:" + _NL
+        + "        traceback.print_exc(file=sys.stdout)" + _NL
+        + "    print('@@VOXEND ' + str(_i), flush=True)" + _NL
+    )
+    driver = driver.replace("__VOX_SOURCES__", repr(items))
+
+    env = dict(os.environ)
+    env["PYTHONDONTWRITEBYTECODE"] = "1"
+    env["PYTHONPATH"] = _PY_ROOT + os.pathsep + env.get("PYTHONPATH", "")
+    proc = subprocess.run(
+        [sys.executable, "-c", driver],
+        cwd=_PY_ROOT, env=env, capture_output=True, text=True,
+    )
+    return (proc.stdout or "") + (proc.stderr or "")
+
+
+def _batch_segment(text, index):
+    # The output between this snippet's markers, or None when the END marker
+    # never arrived — the batch died in or before it, so only an isolated
+    # re-run can say what happened.
+    begin = "@@VOXBEGIN " + str(index)
+    end = "@@VOXEND " + str(index)
+    b = text.find(begin)
+    if b < 0:
+        return None
+    b += len(begin)
+    e = text.find(end, b)
+    if e < 0:
+        return None
+    return text[b:e].strip()
+
+
+def _classify_batch_error(seg, source, label, index):
+    # Same verdict as _run_source, read off a traceback in the batch output.
+    if not seg:
+        return None
+    errlines = [ln for ln in seg.split(_NL) if ln.strip()]
+    if not errlines or "Traceback (most recent call last)" not in seg:
+        return None
+    last = errlines[-1]
+    exc_type = last.split(":", 1)[0].strip().split(".")[-1]
+    detail = (
+        label + " python block #" + str(index) + ": " + last + _NL + _NL
+        + "--- executed source ---" + _NL + source
+        + _NL + _NL + "--- batch output ---" + _NL + seg
+    )
+    if exc_type in _PROGRAMMING_ERROR_NAMES:
+        return "PROGRAMMING ERROR in " + detail
+    if _is_not_found_error(last):
+        return None
+    return "RUNTIME ERROR in " + detail
+
+
 def _run_source(source, label, index):
     # Run one rewritten block in a subprocess. Returns None when it exits 0 or
     # fails with the single tolerated not-found/404 domain error; otherwise
@@ -536,6 +610,7 @@ def _completeness_gate(label, blocks):
     illustration = 0
     compiled = 0
     failures = []
+    _pending = []
 
     for i, block in enumerate(blocks):
         kind = _classify(block)
@@ -553,14 +628,24 @@ def _completeness_gate(label, blocks):
                 )
                 continue
             executed += 1
-            msg = _run_source(source, label, i)
-            if msg is not None:
-                failures.append(msg)
+            _pending.append((i, source))
         elif kind == "illustration":
             illustration += 1
         else:
             compiled += 1
 
+    if _pending:
+        _text = _run_batch(_pending, label)
+        for _i, _src in _pending:
+            _seg = _batch_segment(_text, _i)
+            if _seg is None:
+                # Batch never finished this one — isolate it.
+                _msg = _run_source(_src, label, _i)
+            else:
+                _msg = _classify_batch_error(_seg, _src, label, _i)
+            if _msg is not None:
+                failures.append(_msg)
+
     print(
         _NL + "[readme-examples] " + label + " python blocks: total="
         + str(total) + " executed=" + str(executed) + " compiled="

package/project/.sdk/src/cmp/rb/ReadmeExamplesTest_rb.ts

@@ -82,6 +82,7 @@ const ReadmeExamplesTest = cmp(function ReadmeExamplesTest(props: any) {
 require "minitest/autorun"
 require "tempfile"
 require "open3"
+require "tmpdir"
 
 class ReadmeExamplesTest < Minitest::Test
   # The three documentation sources this gate covers.
@@ -227,17 +228,46 @@ ${entityLines}
   # domain errors (e.g. "404: Not found") never match FATAL, so caught not-found
   # cases stay tolerated.
   def test_ruby_examples_run_offline
-    ran = 0
     failures = []
-    ruby_blocks.each do |blk|
-      next unless runnable?(blk[:code])
-      ran += 1
-      Tempfile.create(["readme_run_", ".rb"]) do |f|
-        f.write(to_runner(blk[:code]))
-        f.flush
-        out, status = Open3.capture2e("ruby", f.path)
-        if out =~ FATAL
-          failures << "#{blk[:doc]} ##{blk[:n]} (exit #{status.exitstatus}):\\n#{out}\\n#{blk[:code]}"
+    runnable = ruby_blocks.select { |blk| runnable?(blk[:code]) }
+    ran = runnable.length
+
+    # BATCHED. One ruby process for every runnable block, not one each. A
+    # spawn per snippet dominated this suite on the large SDKs.
+    #
+    # Each snippet is loaded inside its own anonymous Module, so constants and
+    # methods do not collide, and is bracketed by markers so output is
+    # attributed back. The interpreter is SHARED, which is the cost of
+    # batching: a snippet that mutates global state or calls exit! affects
+    # what follows it.
+    #
+    # So the batch is not trusted blindly. Any snippet whose END marker never
+    # arrives is re-run on its own, in a fresh process, and that isolated
+    # result is the one that counts.
+    if ran > 0
+      Dir.mktmpdir("readme_batch_") do |dir|
+        paths = {}
+        runnable.each_with_index do |blk, i|
+          path = File.join(dir, "snip_#{i}.rb")
+          File.write(path, to_runner(blk[:code]))
+          paths[i] = path
+        end
+
+        driver = File.join(dir, "_driver.rb")
+        File.write(driver, batch_driver(paths))
+        out, status = Open3.capture2e("ruby", driver)
+
+        runnable.each_with_index do |blk, i|
+          seg = batch_segment(out, i)
+          code = status.exitstatus
+          if seg.nil?
+            solo, sstatus = Open3.capture2e("ruby", paths[i])
+            seg = solo
+            code = sstatus.exitstatus
+          end
+          if seg =~ FATAL
+            failures << "#{blk[:doc]} ##{blk[:n]} (exit #{code}):\\n#{seg}\\n#{blk[:code]}"
+          end
         end
       end
     end
@@ -245,6 +275,49 @@ ${entityLines}
     assert_equal [], failures, "docs ruby examples raised a real error when run offline:\\n#{failures.join("\\n\\n")}"
   end
 
+  # The batch driver: load each snippet into its own anonymous Module so its
+  # constants and methods cannot collide with another's. The at_exit hook is
+  # what makes a hard death recoverable — it emits the END marker for whichever
+  # snippet was in flight, so the harness can tell "this one died" from "the
+  # batch never reached it", and re-run only what it must.
+  def batch_driver(paths)
+    list = paths.keys.sort.map { |i| [i, paths[i]] }
+    <<~DRIVER
+      files = #{list.inspect}
+      current = nil
+      at_exit do
+        if current
+          puts
+          puts "@@VOXEND \\#{current}"
+        end
+      end
+      files.each do |(i, path)|
+        current = i
+        puts
+        puts "@@VOXBEGIN \\#{i}"
+        begin
+          Module.new.module_eval(File.read(path), path)
+        rescue Exception => e
+          puts "FATAL: \\#{e.class}: \\#{e.message}"
+        end
+        puts
+        puts "@@VOXEND \\#{i}"
+        current = nil
+      end
+    DRIVER
+  end
+
+  # Output between this snippet's markers, or nil when the END marker is
+  # absent — the batch never got past it, so only an isolated re-run can say.
+  def batch_segment(text, i)
+    b = text.index("@@VOXBEGIN #{i}")
+    return nil if b.nil?
+    b += "@@VOXBEGIN #{i}".length
+    e = text.index("@@VOXEND #{i}", b)
+    return nil if e.nil?
+    text[b...e].strip
+  end
+
   # COMPLETENESS GATE: every fenced ruby block is partitioned into exactly one of
   # {executed, syntaxchecked-nonrunnable, illustration}. The three counts must
   # sum to the total. A block that references the SDK but is neither runnable (so

package/project/.sdk/src/cmp/ts/utility_ts.ts

@@ -6,6 +6,7 @@ import {
   canonKey,
   canonScalarKey,
   each,
+  opParams,
 } from '@voxgig/sdkgen'
 
 import {
@@ -29,13 +30,25 @@ import {
 // has no params and the generated match type is `Partial<Entity>`). Returns
 // undefined when neither is present.
 function paramCanonType(entity: any, op: any, paramName: string): unknown {
-  const points = op && op.points ? each(op.points) : []
-  for (const pt of points as any[]) {
-    const params = pt && pt.args && pt.args.params ? each(pt.args.params) : []
-    const found = (params as any[]).find((p: any) => p && p.name === paramName)
-    if (found) {
-      return found.type
-    }
+  // opParams, NOT a raw walk of op.points.
+  //
+  // opParams drops points flagged with select['$action'] and merges what is
+  // left; the typed-model generator reaches the op's params through it, so
+  // anything else is a DIFFERENT set of params wearing the same name.
+  //
+  // A raw walk returned the first match on ANY point, action points included.
+  // github's `action` entity has a field `owner` (`$OBJECT`, "A GitHub user")
+  // and seven create points whose path carries `{owner}` (a string) — so the
+  // type said Record<string, any> and the doc example said 'example_owner',
+  // and ts/README.md stopped compiling. Same for workflow_id: `number` in the
+  // type, quoted string in the example.
+  //
+  // Deriving from opParams is what actually makes good on the promise below —
+  // that the docs and the generated types cannot disagree.
+  const params = op ? each(opParams(op)) : []
+  const found = (params as any[]).find((p: any) => p && p.name === paramName)
+  if (found) {
+    return found.type
   }
   const field = (entity && entity.fields ? each(entity.fields) : [])
     .find((f: any) => f && f.name === paramName) as any

package/project/sdkgen-package.json

@@ -3,7 +3,7 @@
     "package": 1
   },
   "name": "@voxgig/sdkgen",
-  "version": "4.2.5",
+  "version": "4.2.6",
   "provides": {
     "target": [
       "c",