duel 0.1.0

data/sig/duel/testing.rbs

@@ -0,0 +1,91 @@
+module Duel
+  # What a plugin's tests need: its prepare blocks run as a worker's would
+  # be, its metrics measuring a block, and whole comparisons of two small
+  # projects.
+  module Testing
+    extend Testing
+
+    # What Testing.compare found.
+    class Comparison < Data
+      # A Result for each benchmark.
+      attr_reader results: Array[Result]
+
+      # What the run printed.
+      attr_reader output: String
+
+      # As `duel` would have it: no benchmark worse, nothing across the board
+      # (see #passed?).
+      # @api private
+      attr_reader passed: bool
+
+      # @api private
+      def self.new: (Array[Result] results, String output, bool passed) -> Comparison
+
+      # Whether it passed, as `duel` would have it.
+      def passed?: () -> bool
+
+      # The benchmark's Result.
+      def []: (String name) -> Result?
+    end
+
+    # What #measure measured, by name, and what metrics said of it.
+    class Measurement < Hash[String, Metric::value]
+      # What metrics said of it, by measure.
+      attr_reader details: Hash[String, String]
+
+      # @api private
+      attr_writer details: Hash[String, String]
+    end
+
+    # Where prepare blocks' builds are kept, for every test of this process.
+    # @api private
+    CACHE: String
+
+    # Each config's metrics.
+    # @api private
+    MADE: Hash[Config, [Hash[String, Hash[Symbol, untyped]], Array[Metric]]]
+
+    # Runs +config+'s prepare blocks as they'd prepare a worker, sets the
+    # environment they give while the block runs, yields it, and stops
+    # what they started after.
+    def prepare: [T] (Config config, ?name: String, ?root: String, ?cache: String) { (Hash[String, String?] env) -> T } -> T
+
+    # What +config+'s metrics measure of the block, as one counted sample
+    # of +n+ iterations of a benchmark would, with its hooks (each
+    # iteration inside around_iteration hooks, if it has them). The
+    # block gets the benchmark's Context.
+    def measure: (Config config, ?n: Integer, ?only: (:count | :timing)?, **untyped hooks) { (Context context) -> untyped } -> Measurement
+
+    # Runs the block with +env+ set, and the environment as it was after.
+    # @api private
+    def with_env: [T] (Hash[String, String?] env) { () -> T } -> T
+
+    # Each config's worker's Stops.
+    # @api private
+    STOPS: Hash[Config, Stops]
+
+    # Runs the block as a worker of +config+ would, then ends it.
+    def in_worker: [T] (Config config) { () -> T } -> T
+
+    # +config+'s metrics, made once (its in_worker blocks run first).
+    def metrics: (Config config) -> Array[Metric]
+
+    # Compares the project at +base+ with the one at +candidate+, in
+    # worker processes, as `duel` does, with +settings+ (as `duel`'s
+    # options name them: processes:, timeout:, filter:, limit:...).
+    def compare: (String base, String candidate, **untyped settings) -> Comparison
+
+    # A stand-in Sample of +n+ iterations of a benchmark, measuring
+    # nothing.
+    def sample: (?n: Integer, ?name: String, ?only: only?, ?peak_memory: bool) -> Sample
+
+    # A stand-in reading of running totals, and what was read with them.
+    def reading: [T, C] (?Snapshot::totals[T] totals, ?C? context) -> Snapshot::Reading[T, C?]
+
+    # What running totals' details block says of +stretches+ of +sample+.
+    def details: (singleton(Snapshot) metric, Snapshot::stretches[Metric::value, untyped] stretches, ?Sample sample) -> Snapshot::explained
+
+    # Writes a project into +dir+: +files+, by path, with their contents.
+    def project: (String dir, Hash[String, String] files) -> String
+  end
+end

data/sig/duel/version.rbs

@@ -0,0 +1,3 @@
+module Duel
+  VERSION: String
+end

data/sig/duel/worker.rbs

@@ -0,0 +1,74 @@
+module Duel
+  # A worker process: loads one version of the code and the candidate's
+  # benchmarks, and answers the runner's requests, one JSON object a line.
+  # @api private
+  class Worker
+    type hello = { "benchmarks" => Array[Definition::summary], "metrics" => Array[Metric::spec], "ruby" => String,
+                   "memory" => bool, "profiler" => bool, "yjit" => bool, "extensions" => Hash[String, String],
+                   "gem_dirs" => Array[String] }
+
+    type ok = { "ok" => true }
+
+    type sampled = { "timed" => Hash[String, Metric::value], "counted" => Hash[String, Metric::value], "heap" => Integer,
+                     ?"details" => Hash[String, String] }
+
+    type memory = { "peak" => Integer }
+
+    type error = { "error" => String, "backtrace" => Array[String] }
+
+    type reply = ok | sampled | memory | Profile::profile | error
+
+    BUILT_IN: Array[singleton(Metric)]
+
+    @metrics: Array[Metric]
+    @specs: Array[Metric::spec]
+    @benchmarks: Hash[String, Definition]
+    @hooks: Hash[Symbol, Array[hook]]
+    @harnesses: Hash[String, Harness]
+
+    def self.main: (String version_root, String candidate_root, String control, ?String? config_path) -> void
+
+    # The native extensions loaded, by feature, at their real paths.
+    def self.extensions: (?Array[String] features, ?Array[String] load_path) -> Hash[String, String]
+
+    # The directories gems are installed in, as real paths ending in "/".
+    def self.gem_dirs: (?Array[String] dirs) -> Array[String]
+
+    # Whether StackProf is installed, for profiles: on CRuby only.
+    def self.profiler?: (?String engine) -> bool
+
+    # Whether YJIT is on, by Ruby.
+    YJIT: ^() -> bool
+
+    # Whether YJIT is on, for a longer warm-up.
+    def self.yjit?: () -> bool
+
+    def initialize: (?Array[singleton(Metric)] plugin_metrics, ?Hash[String, Hash[Symbol, untyped]] measures,
+                     ?hooks: Hash[Symbol, Array[hook]]) -> void
+
+    # Closes its metrics, last first, as it ends.
+    def close: () -> void
+
+    # +metrics+ made, and their measures' specs, with +measures+' changes.
+    def self.made: (Array[singleton(Metric)] metrics, Hash[String, Hash[Symbol, untyped]] measures) -> [Array[Metric], Array[Metric::spec]]
+
+    # What the runner learns first.
+    def hello: () -> hello
+
+    # Answers a request (a parsed JSON object). Errors are answered, not
+    # raised.
+    def handle: (Hash[String, untyped] request) -> reply
+
+    private
+
+    def setup: (String name, bool trial) -> ok
+
+    def sample: (String name, Integer n, bool count, Integer runs) -> sampled
+
+    def memory: (String name, Integer n) -> memory
+
+    def profile: (String name, Integer n, Integer runs) -> Profile::profile
+
+    def teardown: (String name) -> ok
+  end
+end

data/sig/duel.rbs

@@ -0,0 +1,276 @@
+# Performance regression guards: every benchmark runs on two versions of
+# a codebase side by side, the baseline and the candidate, and the
+# candidate fails if it's slower at any of them.
+module Duel
+  # What Duel raises when something is used wrong, or fails (see Failed).
+  class Error < StandardError
+  end
+
+  # What a benchmark is judged on, if not both its timings and its
+  # counts (see DSL#duel).
+  type only = :count | :timing
+
+  # A benchmark's body, run in its Context: given the iterations to do
+  # when it takes a parameter, else run once an iteration.
+  # @api private
+  type body = ^() [self: Context] -> void | ^(Integer n) [self: Context] -> void
+
+  # A hook (see HOOKS): a block, or a lambda, given the benchmark (setup,
+  # teardown and before_sample, if it takes it), or the Iteration to run
+  # (around_iteration), or nothing (after_iterations). A file's, or a
+  # benchmark's own, runs in each benchmark's Context.
+  type hook = ^(?untyped) [self: Context] -> untyped
+
+  # A benchmark's own hooks, by name (see DSL#duel).
+  # @api private
+  type hooks = Hash[Symbol, hook]
+
+  # A benchmark (see DSL#duel), defined in +file+, whose hooks (its
+  # file's, and its own) it gets. Or one of a whole process (see
+  # DSL#duel_process). What a metric sees of it (Sample#benchmark): its
+  # name, only and peak_memory.
+  class Definition < Struct[String | only | bool | body | hooks | Definition::process | nil]
+    # A process benchmark's process: its command, or Ruby code.
+    # @api private
+    type process = { "command" => Array[String]?, "ruby" => String?, "runs" => Integer? }
+
+    # What the runner learns of it (see #to_h).
+    # @api private
+    type summary = { "name" => String, "only" => String?, "peak_memory" => bool, "process" => process? }
+
+    # Its name, as DSL#duel was given it.
+    attr_reader name: String
+
+    # @api private
+    attr_writer name: String
+
+    # What it's judged on (DSL#duel's only:): nil for both its timings
+    # and its counts.
+    attr_reader only: only?
+
+    # @api private
+    attr_writer only: only?
+
+    # Whether its peak memory is judged too.
+    attr_reader peak_memory: bool
+
+    # @api private
+    attr_writer peak_memory: bool
+
+    # @api private
+    attr_accessor file: String
+
+    # (None, for a process benchmark, which is never run in a worker.)
+    # @api private
+    attr_accessor body: body
+
+    # Its own hooks, by name (see DSL#duel).
+    # @api private
+    attr_accessor hooks: hooks
+
+    # Its process, for a process benchmark.
+    # @api private
+    attr_accessor process: process?
+
+    # @api private
+    def self.new: (String name, only? only, bool peak_memory, String file, body? body, ?hooks? hooks, ?process? process) -> instance
+
+    # Whose CPU time it takes: the benchmark thread's, or the process's.
+    # @api private
+    def cpu: () -> (:thread | :process)
+
+    # What the runner learns of it.
+    # @api private
+    def to_h: () -> summary
+  end
+
+  # The iterations of a benchmark judged on its counts alone.
+  # @api private
+  COUNTS_ONLY: Integer
+
+  # What a benchmark may be judged on: nil for both.
+  # @api private
+  ONLY: Array[only?]
+
+  # The hooks, by name, in the order a benchmark's life runs them.
+  HOOKS: Array[Symbol]
+
+  # No hooks, by name.
+  # @api private
+  NO_HOOKS: Hash[Symbol, Array[hook]]
+
+  # What a worker's blocks (Config#in_worker) are given: blocks to run
+  # when the worker ends.
+  class Stops
+    @blocks: Array[^() -> void]
+
+    # @api private
+    def initialize: () -> void
+
+    # Runs the block when the worker ends.
+    def at_stop: () { () -> void } -> void
+
+    # Runs them, last first.
+    # @api private
+    def run: () -> void
+  end
+
+  # What Config#in_worker returns: what its block returned, in the
+  # worker.
+  class InWorker[T]
+    # The value, boxed (so a nil or false one is made too), or nil outside
+    # a worker.
+    @made: [T]?
+
+    # The worker's. Outside a worker, it fails.
+    def value: () -> T
+
+    # @api private
+    def made: (T value) -> void
+
+    # @api private
+    def forget: () -> void
+  end
+
+  self.@config: Config?
+  self.@benchmarks: Hash[String, Definition]?
+  self.@file_hooks: Hash[String, Hash[Symbol, Array[hook]]]?
+  self.@plugins: Hash[Symbol, Module & _Plugin]?
+
+  # A plugin (see Duel.plugin): what configures Duel for it, given the
+  # configuration and the options it's used with.
+  interface _Plugin
+    def configure: (Config config, **untyped options) -> untyped
+  end
+
+  # The configuration: the project's .duel.rb, or the conventions.
+  def self.config: () -> Config
+
+  # @api private
+  def self.config=: (Config config) -> Config
+
+  # Yields the configuration, as .duel.rb does.
+  def self.configure: [T] () { (Config config) -> T } -> T
+
+  self.@measuring: bool?
+
+  # Whether a benchmark is being measured right now: in a worker, inside a
+  # sample, but not in its untimed blocks (or in setup).
+  def self.measuring?: () -> bool
+
+  # Fails while a benchmark is being measured: for a helper, +name+d,
+  # meant for setting up.
+  def self.unmeasured!: (String name) -> void
+
+  # Runs the block with measuring? as +measuring+.
+  # @api private
+  def self.measured: [T] (bool measuring) { () -> T } -> T
+
+  # Loads a config file (.duel.rb): a fresh configuration, rooted where
+  # the file is.
+  # @api private
+  def self.load_config: (String path) -> Config
+
+  # Where +method+ is defined ("file:line"), or whose it is.
+  # @api private
+  def self.where: (Method | UnboundMethod method) -> String
+
+  # Every benchmark defined, by name.
+  # @api private
+  def self.benchmarks: () -> Hash[String, Definition]
+
+  # Each file's hooks, by its path, then by name.
+  # @api private
+  def self.file_hooks: () -> Hash[String, Hash[Symbol, Array[hook]]]
+
+  # Adds +block+ as a hook of the file of benchmarks running, or of
+  # +file+, the one that gave it.
+  # @api private
+  def self.file_hook: (Symbol name, Proc? block, String file) -> void
+
+  # +block+, a hook +name+d, or a failure if it isn't one.
+  # @api private
+  def self.hook: (Symbol name, untyped block) -> untyped
+
+  # Registers a plugin, +name+d as its file is required (see
+  # Config#use): a module whose configure Config#use calls, and whose
+  # DSL module, if it has one, has its DSL methods. Returns its name.
+  def self.plugin: (Symbol name, Module & _Plugin plugin) -> Symbol
+
+  # Every plugin registered, by name.
+  # @api private
+  def self.plugins: () -> Hash[Symbol, Module & _Plugin]
+
+  self.@defining_file: String?
+
+  # The file of benchmarks that called the DSL method running, if one is.
+  # @api private
+  def self.defining_file: () -> String?
+
+  # Runs the block with +file+ as the defining file, unless one is already.
+  # @api private
+  def self.defining: [T] (String? file) { () -> T } -> T
+
+  # What files of benchmarks call (the worker gives it to its main
+  # object).
+  module DSL
+    # Adds the public methods of a plugin's DSL module to the DSL (a name
+    # it has, or every object has, fails), as .duel.rb uses the plugin.
+    # @api private
+    def self.add: (Module methods) -> Array[Symbol]
+
+    # Adds one of a plugin's DSL module's methods to the DSL, unless it's
+    # there already.
+    # @api private
+    def self.define: (Symbol name, Module methods) -> void
+
+    # The plugin's DSL module each method DSL.add added is from.
+    # @api private
+    DEFINED: Hash[Symbol, Module]
+
+    # Defines a benchmark: its block is timed, run as many times as a
+    # sample needs or, with a parameter, once a sample, given how many
+    # iterations to do. Its own hooks (see HOOKS) run after its file's:
+    # an around_iteration one runs around each iteration, given it to
+    # run, and only the iteration is measured.
+    def duel: (String name, ?only: only?, ?peak_memory: bool, **hook hooks) { (?Integer n) [self: Context] -> void } -> void
+
+    # A benchmark's own hooks, checked.
+    # @api private
+    def self.hooks: (String name, Hash[Symbol, untyped] hooks, ^(?) [self: Context] -> untyped body) -> hooks
+
+    # Defines a benchmark of a whole process: a command (as Process.spawn
+    # takes it), or Ruby code, run fresh for each version in every sample
+    # (the fastest of +runs+).
+    def duel_process: (String name, *String command, ?ruby: String?, ?runs: Integer?, **untyped options) -> void
+
+    # En garde.
+    # @api private
+    alias `⚔️` duel
+
+    # @api private
+    alias ⚔ duel
+
+    # The file's hooks (see HOOKS), for every benchmark in it, run in its
+    # context: setup once in each worker before a benchmark's samples,
+    # teardown after them, and before_sample before every sample, not
+    # measured, each given the benchmark if it takes it.
+    def setup: () { (?Definition benchmark) [self: Context] -> void } -> void
+
+    def teardown: () { (?Definition benchmark) [self: Context] -> void } -> void
+
+    def before_sample: () { (?Definition benchmark) [self: Context] -> void } -> void
+
+    # Runs the block around each iteration of every benchmark in the
+    # file, given the iteration to run: only the iteration is measured.
+    def around_iteration: () { (Iteration iteration) [self: Context] -> void } -> void
+
+    # Runs the block after each sample's iterations of every benchmark in
+    # the file, measured with them.
+    def after_iterations: () { () [self: Context] -> void } -> void
+
+    # The file that called the DSL method calling this.
+    # @api private
+    def self.caller_file: () -> String
+  end
+end

data/sig/manifest.yaml

@@ -0,0 +1,4 @@
+# The standard libraries Duel's signatures use, for rbs collection.
+dependencies:
+  - name: optparse
+  - name: socket

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: duel
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Erik Berlin
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+description: Duel runs your benchmarks on two versions of your code side by side,
+  the baseline (where your branch left main, say) and the candidate, and fails if
+  the candidate is slower at any of them. Timings are judged by a distribution-free
+  confidence interval for paired CPU-time ratios and confirmed in fresh processes,
+  so it fails on real regressions, not noise.
+email:
+- sferik@gmail.com
+executables:
+- duel
+extensions: []
+extra_rdoc_files: []
+files:
+- AGENT_PROMPT.md
+- CHANGELOG.md
+- LICENSE
+- README.md
+- docs/README.md
+- docs/benchmarks.md
+- docs/ci.md
+- docs/configuration.md
+- docs/how-it-works.md
+- docs/json.md
+- docs/plugins.md
+- docs/troubleshooting.md
+- exe/duel
+- lib/duel.rb
+- lib/duel/across.rb
+- lib/duel/baseline.rb
+- lib/duel/cli.rb
+- lib/duel/config.rb
+- lib/duel/context.rb
+- lib/duel/init.rb
+- lib/duel/json_report.rb
+- lib/duel/lock.rb
+- lib/duel/metric.rb
+- lib/duel/native.rb
+- lib/duel/process_sample.rb
+- lib/duel/profile.rb
+- lib/duel/rake_task.rb
+- lib/duel/resource.rb
+- lib/duel/rubies.rb
+- lib/duel/runner.rb
+- lib/duel/stats.rb
+- lib/duel/summary.rb
+- lib/duel/templates/benchmark_duel.rb.erb
+- lib/duel/templates/duel.rb.erb
+- lib/duel/templates/perf.yml.erb
+- lib/duel/testing.rb
+- lib/duel/version.rb
+- lib/duel/worker.rb
+- sig/duel.rbs
+- sig/duel/across.rbs
+- sig/duel/baseline.rbs
+- sig/duel/cli.rbs
+- sig/duel/config.rbs
+- sig/duel/context.rbs
+- sig/duel/init.rbs
+- sig/duel/json_report.rbs
+- sig/duel/lock.rbs
+- sig/duel/metric.rbs
+- sig/duel/native.rbs
+- sig/duel/process_sample.rbs
+- sig/duel/profile.rbs
+- sig/duel/rake_task.rbs
+- sig/duel/resource.rbs
+- sig/duel/rubies.rbs
+- sig/duel/runner.rbs
+- sig/duel/stats.rbs
+- sig/duel/summary.rbs
+- sig/duel/testing.rbs
+- sig/duel/version.rbs
+- sig/duel/worker.rbs
+- sig/manifest.yaml
+homepage: https://github.com/sferik/duel-ruby
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/sferik/duel-ruby
+  changelog_uri: https://github.com/sferik/duel-ruby/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/sferik/duel-ruby/issues
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.23
+specification_version: 4
+summary: 'Performance regression guards: fail when a change makes your code slower'
+test_files: []