haveapi 0.30.2 → 0.31.0

data/benchmarks/performance_compare.rb

@@ -8,6 +8,8 @@ module HaveAPI
 end
 require 'digest'
 require 'json'
+require 'open3'
+require 'rbconfig'
 
 module HaveAPI
   module Performance
@@ -18,6 +20,7 @@ module HaveAPI
     MAX_ITERATIONS = 100_000
     REQUEST_LIMIT_SECONDS = 30.0
     SETUP_LIMIT_SECONDS = 120.0
+    BENCHMARK_INPUTS = %w[servers/ruby/benchmarks/Gemfile servers/ruby/benchmarks/Gemfile.lock servers/ruby/benchmarks/shell.nix].freeze
 
     def self.complex_cases(profile)
       sizes = profile == 'weekly' ? [32] : [16]
@@ -30,6 +33,34 @@ module HaveAPI
     RUNTIME_POLICY = { 'rack_env' => 'test', 'locale' => 'C.UTF-8', 'timezone' => 'UTC', 'yjit' => false, 'rjit' => false }.freeze
     class ContractError < StandardError; end
 
+    def self.gemspec(root, path)
+      script = <<~RUBY
+        require 'json'
+        spec = Gem::Specification.load(File.basename(ARGV.fetch(0)))
+        abort 'Cannot load gemspec' unless spec
+        puts JSON.generate(ruby: spec.required_ruby_version.to_s,
+                           dependencies: spec.dependencies.map { |d| [d.name, d.type.to_s, d.requirement.to_s] }.sort)
+      RUBY
+      environment = (ENV.keys.grep(/\ABUNDLE/) + %w[RUBYOPT RUBYLIB RUBYGEMS_GEMDEPS]).to_h { |key| [key, nil] }
+      output, errors, status = Open3.capture3(environment, RbConfig.ruby, '-e', script, path, chdir: File.dirname(File.join(root, path)))
+      raise ContractError, "Cannot load source gemspec #{path}: #{errors.strip}" unless status.success?
+
+      specification = JSON.parse(output)
+      unless Gem::Requirement.new(specification.fetch('ruby')).satisfied_by?(Gem::Version.new(RUBY_VERSION))
+        raise ContractError, "Benchmark Ruby does not satisfy #{path}"
+      end
+
+      specification.fetch('dependencies').each do |name, type, requirement|
+        next unless type == 'runtime'
+        next if %w[haveapi haveapi-client haveapi-rust-client].include?(name)
+
+        unless Gem::Requirement.new(requirement.split(',').map(&:strip)).satisfied_by?(Gem::Specification.find_by_name(name).version)
+          raise ContractError, "Benchmark dependencies do not satisfy #{name} #{requirement}"
+        end
+      end
+      specification
+    end
+
     # One reviewed maintenance mapping, not a general cherry-pick certificate.
     # Both live entry points verify Git; replay verifies the retained raw delta.
     module Release
@@ -166,7 +197,10 @@ module HaveAPI
         receipt = inputs.fetch('release_versions')
         raise ContractError, 'Incomplete release input graph' unless receipt.fetch('contents').keys.sort == paths(profile).sort
         raise ContractError, 'Release input normalization mismatch' unless receipt == capture(receipt.fetch('contents'))
-        raise ContractError, 'Release normalized graph mismatch' unless inputs.fetch('manifests') == receipt.fetch('normalized_sha256')
+
+        expected = receipt.fetch('normalized_sha256')
+        expected = expected.except('flake.nix', 'flake.lock') if inputs.has_key?('benchmark')
+        raise ContractError, 'Release normalized graph mismatch' unless inputs.fetch('manifests') == expected
 
         receipt.fetch('raw_sha256').each do |path, digest|
           raise ContractError, 'Release input escaped verified source' unless source.fetch('files_sha256').fetch(path) == digest
@@ -175,7 +209,7 @@ module HaveAPI
       end
 
       def self.graph(inputs)
-        inputs.except('release_versions', 'json_preparation')
+        inputs.except('release_versions', 'json_preparation', 'contributor_flakes')
       end
 
       def self.html(body, own)
@@ -211,6 +245,19 @@ module HaveAPI
       raise ContractError, 'Weekly baseline must be the fixed reference'
     end
 
+    def self.push_pair!(selection, baseline:, candidate:)
+      return false unless selection['push_pair']
+
+      expected = { 'event_name' => 'push', 'before' => baseline, 'after' => candidate }
+      unless selection['event_name'] == 'push' && selection['profile'] == 'quick' && selection['push_pair'] == expected &&
+             selection['baseline_sha'] == baseline && selection['candidate_sha'] == candidate && baseline != candidate &&
+             [baseline, candidate].all? { |sha| sha.is_a?(String) && sha.match?(/\A[0-9a-f]{40}\z/) && sha != '0' * 40 }
+        raise ContractError, 'Invalid quick push event pair'
+      end
+
+      true
+    end
+
     # Canonicalization ignores map order only. Arrays and scalar types retain
     # their meaning; duplicate JSON or symbol/string-normalized keys fail closed.
     module Identity
@@ -270,6 +317,9 @@ module HaveAPI
     end
 
     class Comparator
+      POLICY = 'per-revision-response-stability-v1'.freeze
+      SOURCE_SHA256 = Digest::SHA256.file(__FILE__).hexdigest.freeze
+
       def initialize(samples, budgets, budget_sha256:)
         @samples = samples
         @budgets = budgets
@@ -357,6 +407,37 @@ module HaveAPI
             raise ContractError, 'Loaded runtime not in verified source tree' unless selected.fetch('files_sha256').fetch(path) == sha256
           end
           inputs = observation.fetch('inputs')
+          if inputs.has_key?('benchmark')
+            tool = inputs.fetch('benchmark')
+            raise ContractError, 'Incomplete benchmark inputs' unless tool.keys.sort == BENCHMARK_INPUTS.sort
+
+            tool.each do |path, input|
+              unless Digest::SHA256.hexdigest(input.fetch('content')) == input.fetch('sha256') &&
+                     @samples.fetch('inputs').fetch('benchmark').fetch(path) == input
+                raise ContractError, 'Benchmark input digest mismatch'
+              end
+            end
+            unless inputs.fetch('locks') == { 'servers/ruby/benchmarks/Gemfile.lock' => tool.fetch('servers/ruby/benchmarks/Gemfile.lock').fetch('sha256') }
+              raise ContractError, 'Benchmark lock identity mismatch'
+            end
+
+            inputs.fetch('contributor_flakes').each do |path, input|
+              unless %w[flake.nix flake.lock].include?(path) && Digest::SHA256.hexdigest(input.fetch('content')) == selected.fetch('files_sha256').fetch(path)
+                raise ContractError, 'Contributor flake escaped source'
+              end
+            end
+            unless inputs.fetch('contributor_flakes').keys.sort == %w[flake.lock flake.nix]
+              raise ContractError, 'Missing contributor flake provenance'
+            end
+
+            inputs.fetch('manifests').each do |path, digest|
+              next if Release::JSON_GEMFILES.include?(path) || lineage
+
+              raise ContractError, 'Dependency manifest escaped source' unless selected.fetch('files_sha256').fetch(path) == digest
+            end
+          elsif inputs.has_key?('contributor_flakes')
+            raise ContractError, 'Contributor flake separation without benchmark inputs'
+          end
           if lineage
             Release.validate_inputs(inputs, selected, @samples.fetch('profile'))
           elsif inputs.has_key?('release_versions')
@@ -474,14 +555,28 @@ module HaveAPI
         end
       end
 
+      def output_counts(outputs, digest = 'canonical_sha256')
+        outputs.each_with_object(Hash.new(0)) do |output, counts|
+          identity = JSON.generate(output.values_at('http_status', 'envelope_status', digest))
+          counts[identity] += output.fetch('count')
+        end
+      end
+
       def same_outputs(left, right)
-        summaries = [left, right].map do |outputs|
-          outputs.each_with_object(Hash.new(0)) do |output, counts|
-            identity = JSON.generate(output.values_at('http_status', 'envelope_status', 'canonical_sha256'))
-            counts[identity] += output.fetch('count')
-          end
+        output_counts(left) == output_counts(right)
+      end
+
+      def response_changes(pairs)
+        outputs = %w[baseline candidate].to_h do |role|
+          [role, pairs.flat_map { |pair| pair.fetch(role).fetch('outputs') }]
         end
-        summaries.first == summaries.last
+        bytes = outputs.transform_values do |examples|
+          examples.sum { |output| output.fetch('body_bytes') * output.fetch('count') }.to_f / examples.sum { |output| output.fetch('count') }
+        end
+        delta = bytes.fetch('candidate') - bytes.fetch('baseline')
+        { 'raw_changed' => output_counts(outputs.fetch('baseline'), 'body_sha256') != output_counts(outputs.fetch('candidate'), 'body_sha256'),
+          'canonical_changed' => !same_outputs(outputs.fetch('baseline'), outputs.fetch('candidate')),
+          'bytes_per_request' => bytes.merge('delta' => delta, 'delta_percent' => 100.0 * delta / bytes.fetch('baseline')) }
       end
 
       def compare_case(item, budget)
@@ -499,7 +594,8 @@ module HaveAPI
           raise ContractError, 'Uncalibrated allocation ceiling' unless reference > 0 && ceiling == (reference * 1.25).ceil + 32
         end
 
-        canonical_outputs = canonical_counters = nil
+        outputs_by_source = {}
+        canonical_counters = nil
         (self_pairs + pairs).each_with_index do |pair, index|
           expected_order = if index < self_pairs.length
                              %w[baseline baseline]
@@ -530,8 +626,10 @@ module HaveAPI
               raise ContractError, 'Worker workload order mismatch'
             end
 
-            canonical_outputs ||= sample.fetch('outputs')
-            raise ContractError, 'Response identity differs between operations/revisions' unless same_outputs(canonical_outputs, sample['outputs'])
+            outputs_by_source[actual_sha] ||= sample.fetch('outputs')
+            unless same_outputs(outputs_by_source.fetch(actual_sha), sample.fetch('outputs'))
+              raise ContractError, "Response identity differs within revision #{actual_sha}"
+            end
           end
         end
         classifications = %w[wall_seconds cpu_seconds].to_h do |metric|
@@ -566,9 +664,10 @@ module HaveAPI
                            'pass'
                          end
         item.slice('id', 'dimensions', 'iterations')
-            .merge('classification' => classification, 'timing' => classifications,
+            .merge('classification' => classification, 'timing' => classifications, 'responses' => response_changes(pairs),
                    'correctness' => { 'status' => 'pass', 'validated_samples' => 2 * (self_pairs.length + pairs.length),
-                                      'operations_per_sample' => iterations, 'distinct_responses' => canonical_outputs.length },
+                                      'operations_per_sample' => iterations, 'distinct_responses' => outputs_by_source.fetch(@samples.fetch('baseline_sha')).length,
+                                      'distinct_responses_by_source' => outputs_by_source.transform_values(&:length) },
                    'pilot' => item.fetch('pilot', {}).slice('selected_iterations', 'warmup_iterations', 'target_seconds',
                                                             'baseline_batch_seconds', 'adequate_signal', 'capped', 'calibration_adjusted'),
                    'allocations' => { 'baseline_median' => base_allocations, 'candidate_median' => candidate_allocations,

data/benchmarks/shell.nix

@@ -0,0 +1,21 @@
+let
+  pkgs = import (builtins.fetchTree {
+    type = "github";
+    owner = "NixOS";
+    repo = "nixpkgs";
+    rev = "02c087f3fe673c9e317f206a627cb6b910336dbe";
+    narHash = "sha256-IoNisKcGbUjdd45qCqywGMCYmZ5EnkxezvLII0FWwNA=";
+  }) { };
+in
+pkgs.mkShell {
+  name = "haveapi-benchmarks";
+  packages = with pkgs; [ ruby_3_4 git gnumake pkg-config openssl libffi ];
+  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib ];
+  shellHook = ''
+    for name in ''${!BUNDLE@}; do unset "$name"; done
+    unset RUBYOPT RUBYLIB RUBYGEMS_GEMDEPS
+    export GEM_HOME="''${HAVEAPI_BENCHMARK_GEM_HOME:-${toString ./.}/.gems}"
+    export GEM_PATH="$GEM_HOME:${pkgs.ruby_3_4}/lib/ruby/gems/3.4.0"
+    export PATH="$GEM_HOME/bin:$PATH"
+  '';
+}

data/doc/client-examples.md

@@ -2,20 +2,23 @@
 
 The Ruby server renders examples from the current user's API description.
 JavaScript and TypeScript examples cover callback and async clients; Python
-examples cover sync and async clients. These clients need no server configuration.
+examples cover sync and async clients. These clients need no server
+configuration.
 Ruby, PHP, CLI, curl and HTTP examples are also available. A sample
 that a client cannot represent shows a limitation instead of dropping input or
 inventing a successful result.
 
-Go and Rust tabs require a profile for a generated SDK. Their package versions
-are independent of HaveAPI and the API version. Without a profile, the server
+Go, Rust and generated TypeScript tabs require a profile for a generated SDK.
+Their package versions are independent of HaveAPI and the API version. Without a
+profile, the server
 hides those clients from the overview and all initialization, authentication and
 action panels. Install the matching generator gem only when configuring its
-profile; ordinary servers do not depend on either generator.
+profile; ordinary servers do not depend on these generators.
 
 ## Configure a generated SDK
 
-Register profiles before mounting the server. For an SDK that matches the current
+Register profiles before mounting the server. For an SDK that matches the
+current
 API, configure its package identity:
 
 ```ruby
@@ -25,25 +28,34 @@ api.documentation_clients('1.0',
   },
   rust: {
     crate_name: 'widget-client'
+  },
+  typescript: {
+    package_name: '@example/widget-client'
   }
 )
 api.mount('/api/')
 ```
 
 `generator_version` defaults to the server's `HaveAPI::VERSION`. Install the
-optional `haveapi-go-client` or `haveapi-rust-client` generator gem at that
-version. An explicit `generator_version` requires the installed generator to
-report that version; it never downloads one. It is independent of the SDK and
-API versions.
-
-At the end of mount, the server builds one private structural model per configured
-API version. The model includes authentication resources, routes and changes from
+matching generator in the server's Ruby environment: `haveapi-go-client`,
+`haveapi-rust-client`, or the source-distributed `haveapi-typescript-client`.
+See the [TypeScript generator guide](../clients/typescript/README.md) for its
+setup. An explicit `generator_version` requires the installed generator to
+report
+that version; the server never downloads one. This version is independent of the
+SDK and API versions.
+
+At the end of mount, the server builds one private structural model per
+configured
+API version. The model includes authentication resources, routes and changes
+from
 post-mount hooks, so canonical names and parent arguments use the complete API
 declaration. The private copy excludes examples, prose and parameter defaults.
 HTML requests use the current user's authorized description, fields and samples;
 they never expose the private model or its initialization diagnostics. Without
-a configured profile, the server loads no optional generator and shows no Go or
-Rust panels.
+a configured profile, the server loads no optional generator and hides the Go,
+Rust and generated TypeScript panels. The callback and async TypeScript
+examples remain separate and need no profile.
 
 Automatic profiles assume that the installed SDK matches the current server's
 generation contract. A package name or version does not prove its contents.
@@ -53,8 +65,9 @@ schema override below when the SDK represents an older or restricted API.
 
 ### Optional SDK versions and sources
 
-Set `module_version: 'v1.2.3'` for Go or `crate_version: '1.2.3'` for Rust when a
-specific package version is needed. Omit the field or use nil to let the user's
+Set `module_version: 'v1.2.3'` for Go, `crate_version: '1.2.3'` for Rust or
+`package_version: '1.2.3'` for TypeScript when a specific package version is
+needed. Omit the field or use nil to let the user's
 package manager select a release. The server does not query a registry or claim
 to have resolved a version. Omitted versions produce these installation hints:
 
@@ -63,15 +76,25 @@ go get example.org/widget/client@latest
 cargo add widget-client
 # Blocking Rust variant:
 cargo add widget-client --features blocking
+npm install @example/widget-client haveapi-client@0.30.2
 ```
 
 Go resolves `@latest` by its normal rules. Cargo selects a compatible release
 according to its resolver, manifest and toolchain; existing lockfiles still
 apply. Explicit Cargo versions remain version requirements, without an added
-exact `=` pin. Latest selection can change over time; supply a version to control
+exact `=` pin. Latest selection can change over time; supply a version to
+control
 the selection. Cargo never receives a literal `version = "latest"`. Rust
 examples also show their Tokio and serde_json requirements.
 
+TypeScript accepts ordinary lowercase npm package names, including scoped names,
+and optional concrete SemVer versions. Tags, version ranges, Git URLs, local
+paths
+and custom factories are not profile options. Configure a package that users can
+install; the server does not check its availability or publish it. Generated
+snippets use the package's public `createClient` export and the published
+`haveapi-client` 0.30.2 runtime.
+
 For an unpublished Rust SDK, supply an HTTPS Git URL and a full commit SHA:
 
 ```ruby
@@ -118,13 +141,20 @@ api.documentation_clients('1.0',
     snapshot_path: '/srv/widget-sdk/api-1.0.snapshot.json',
     generator_config_path: '/srv/widget-sdk/generator.toml',
     generator_version: '0.30.0'
+  },
+  typescript: {
+    package_name: '@example/widget-client',
+    package_version: '1.2.3',
+    snapshot_path: '/srv/widget-sdk/api-1.0.snapshot.json',
+    generator_version: '0.30.0'
   }
 )
 ```
 
 Paths name trusted absolute local files. `description_path` is Go's saved
-machine-readable description, not HTML. `snapshot_path` is Rust's saved schema
-and capture metadata. They select an exact SDK shape in place of the current
+machine-readable description, not HTML. For Rust and TypeScript, `snapshot_path`
+selects the saved schema and capture metadata. These files select an exact SDK
+shape in place of the current
 server declaration. Keep the files private and retain them with the SDK release.
 
 Rust's legacy `generator_config_path` supplies the crate name/version and full
@@ -138,7 +168,8 @@ Saved files are read and validated during registration. Automatic models are
 initialized at the end of mount. Missing files or generators, actual generator
 version mismatches, unknown fields, conflicting identities and invalid schemas
 fail startup. Mounting also rejects unavailable API versions and mismatched Rust
-snapshot versions. Failed mounting publishes no profiles and leaves no usable
+and TypeScript snapshot versions. Failed mounting publishes no profiles and
+leaves no usable
 server application. An older explicitly pinned generator can serve a suitable
 legacy file profile, but automatic/inline Rust mode requires the new factory
 interface and never guesses a fallback.
@@ -165,7 +196,8 @@ Construction copies and freezes Hash, Array and String data without freezing
 the caller's containers. Each request owns its description snapshot.
 `context.with(...)` creates a distinct frozen context, sharing unchanged fields
 by identity and normalizing only explicitly supplied fields. A shallow-frozen
-input container still needs copying; explicitly supplying an existing owned Hash,
+input container still needs copying; explicitly supplying an existing owned
+Hash,
 Array or String also copies it. Renderers must treat context-owned data as
 read-only.
 
@@ -185,16 +217,21 @@ overview and every panel before rendering.
 
 Generated-client adapters belong to their generator gems and consume context
 readers without depending on the server gem. They resolve names using the
-complete private generator model and never run a compiler, formatter or generator
-in an HTML request. Keep adapter rendering read-only: configured adapters are frozen
-and shared, while authorized description snapshots belong to individual requests.
+complete private generator model and never run a compiler, formatter or
+generator
+in an HTML request. Keep adapter rendering read-only: configured adapters are
+frozen
+and shared, while authorized description snapshots belong to individual
+requests.
 Derived contexts within one request share unchanged owned data. Supplying a
-resource, action or sample still copies that supplied subtree, so a resource with
+resource, action or sample still copies that supplied subtree, so a resource
+with
 many actions can remain costly to render.
 
 ## Verification
 
-`make test-documentation` checks renderers, contexts, profiles, rendered HTML and
+`make test-documentation` checks renderers, contexts, profiles, rendered HTML
+and
 local assets. Prepare the component bundles and JavaScript development
 dependencies with `make release-deps` first. The target builds the JavaScript
 package before compiling TypeScript examples against its public declarations;
@@ -220,8 +257,10 @@ nix develop .#server-ruby --command bash -c \
   'make --no-print-directory benchmark-documentation > /tmp/documentation.json'
 ```
 
-The target synchronizes shared documentation before running the harness. Standard
-output is JSON; shell/dependency preparation happens before the redirected command.
+The target synchronizes shared documentation before running the harness.
+Standard
+output is JSON; shell/dependency preparation happens before the redirected
+command.
 No network server, database, generated SDK compilation or formatter is involved.
 The synthetic fixture uses Basic users `admin` and `member`, both with the local
 test password `fixture-only`.
@@ -229,8 +268,10 @@ test password `fixture-only`.
 Defaults measure two shapes with eight ordinary actions each: eight resources
 with one action, and one resource with eight actions. Each ordinary action also
 has a privileged sample; the first resource has an additional admin-only action.
-Context lanes derive a prepared parent with a changed profile or a constant small
-sample. The HTML lane performs one real Rack request per trial, with all available
+Context lanes derive a prepared parent with a changed profile or a constant
+small
+sample. The HTML lane performs one real Rack request per trial, with all
+available
 dynamic renderer panels. Use a minimal smoke run or a larger repeated campaign:
 
 ```sh
@@ -242,8 +283,10 @@ make benchmark-documentation \
 ```
 
 Run these `make` commands inside the prepared shell. `--shape custom --resources
-N --actions N` controls both dimensions independently; `--lanes context` isolates
-derivation and `--lanes html` isolates requests. `--user member` selects restricted
+N --actions N` controls both dimensions independently; `--lanes context`
+isolates
+derivation and `--lanes html` isolates requests. `--user member` selects
+restricted
 help. `--help` lists all options. `--rust` enables the fixture's matching Rust
 profile, includes both Rust variants in HTML, and adds separate async/blocking
 action and authentication adapter lanes. It requires the optional generator's
@@ -269,13 +312,16 @@ Rust lanes are never silently omitted. See the
 
 Every lane reports each trial and the median wall seconds, process CPU seconds
 and allocated objects, with the operation count per trial. Wall time uses a
-monotonic clock. A full GC runs before each trial, outside its timer; GC triggered
+monotonic clock. A full GC runs before each trial, outside its timer; GC
+triggered
 by the measured work remains included. Mount/profile construction has separate
 setup metrics. Ownership construction, description selection, warmups, output
 validation, hashing and JSON serialization are outside lane measurements.
 Reports include fixture dimensions, renderer IDs, profile mode, Ruby runtime,
-host, Git HEAD/status/diff digest, loaded source digests, dependency versions and
-available lockfile/harness/fixture digests. HTML and Rust outputs include status, bytes and
+host, Git HEAD/status/diff digest, loaded source digests, dependency versions
+and
+available lockfile/harness/fixture digests. HTML and Rust outputs include
+status, bytes and
 SHA-256; failed or missing output causes a nonzero exit.
 
 For a before/after comparison, retain one exact harness and fixture and point