ibex 0.1.0 → 0.3.0

data/lib/ibex/cli/fuzz.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+require "shellwords"
+require_relative "../bounded_subprocess"
+require_relative "../fuzz"
+require_relative "fuzz_regressions"
+
+module Ibex
+  # CLI entry point for bounded grammar-derived differential fuzzing.
+  module CLIFuzz
+    include CLIFuzzRegressions
+
+    # @rbs!
+    #   private def print_help: (OptionParser) -> Integer
+    #   private def input_path: (Array[String]) -> String
+    #   private def normalize_grammar_path: (String) -> IR::Grammar
+    #   private def handle_grammar_warnings: (IR::Grammar, String) -> void
+    #   private def warning_categories: (String) -> Array[Symbol]
+    #   private def select_configuration_mode: (String) -> void
+
+    private
+
+    # @rbs (Array[String] arguments) -> Integer
+    def run_fuzz_command(arguments)
+      parser = fuzz_option_parser
+      remaining = parser.parse(arguments)
+      return print_help(parser) if @options[:help]
+
+      path = input_path(remaining)
+      grammar = normalize_grammar_path(path)
+      handle_grammar_warnings(grammar, path)
+      external = external_fuzz_target
+      fuzzer = Fuzz.new(
+        grammar,
+        seed: @options.fetch(:fuzz_seed, 0), count: @options.fetch(:fuzz_count, 100),
+        max_tokens: @options.fetch(:fuzz_max_tokens, 32), max_depth: @options.fetch(:fuzz_max_depth, 16),
+        max_expansions: @options.fetch(:fuzz_max_expansions, Samples::DEFAULT_MAX_EXPANSIONS),
+        max_actions: @options.fetch(:fuzz_max_actions, Fuzz::DEFAULT_MAX_ACTIONS),
+        max_stack: @options.fetch(:fuzz_max_stack, Fuzz::DEFAULT_MAX_STACK),
+        coverage_guided: @options.fetch(:fuzz_coverage_guided, false),
+        path_length: @options.fetch(:fuzz_path_length, 2),
+        against: external&.[](0), against_description: external&.[](1)
+      )
+      write_fuzz_report(fuzzer.run)
+      0
+    rescue Fuzz::Mismatch => e
+      raise unless fuzzer && path
+
+      begin
+        write_minimized_fuzz_mismatch(fuzzer, e, path, external)
+      rescue Fuzz::BudgetExceeded => budget
+        write_fuzz_budget_report(budget, external, phase: "reduction")
+      end
+    rescue Fuzz::BudgetExceeded => e
+      write_fuzz_budget_report(e, external, phase: "comparison")
+    end
+
+    # @rbs () -> OptionParser
+    def fuzz_option_parser
+      OptionParser.new do |options|
+        options.banner = "Usage: ibex fuzz [options] grammarfile"
+        add_fuzz_generation_options(options)
+        add_fuzz_execution_options(options)
+        options.on("--mode=MODE", %w[default extended], "grammar mode") { |value| select_configuration_mode(value) }
+        options.on("--warnings=CATEGORIES", "all, error, all,error, or none") do |value|
+          @options[:warnings] = warning_categories(value)
+        end
+        options.on("--help", "show help") { @options[:help] = true }
+      end
+    end
+
+    # @rbs (OptionParser options) -> void
+    def add_fuzz_generation_options(options)
+      options.on("--count=N", Integer, "generated sentences (default 100)") do |value|
+        @options[:fuzz_count] = positive_fuzz_option!("count", value)
+      end
+      options.on("--seed=N", Integer, "deterministic seed (default 0)") { |value| @options[:fuzz_seed] = value }
+      options.on("--max-tokens=N", Integer, "maximum generated tokens") do |value|
+        @options[:fuzz_max_tokens] = positive_fuzz_option!("max-tokens", value)
+      end
+      options.on("--max-depth=N", Integer, "maximum random expansion depth") do |value|
+        @options[:fuzz_max_depth] = positive_fuzz_option!("max-depth", value)
+      end
+      options.on("--max-expansions=N", Integer, "total derivation work budget") do |value|
+        @options[:fuzz_max_expansions] = positive_fuzz_option!("max-expansions", value)
+      end
+    end
+
+    # @rbs (OptionParser options) -> void
+    def add_fuzz_execution_options(options)
+      options.on("--max-actions=N", Integer, "actions per simulated sentence") do |value|
+        @options[:fuzz_max_actions] = positive_fuzz_option!("max-actions", value)
+      end
+      options.on("--max-stack=N", Integer, "simulated parser stack depth") do |value|
+        @options[:fuzz_max_stack] = positive_fuzz_option!("max-stack", value)
+      end
+      options.on("--coverage-guided", "prefer uncovered production paths") do
+        @options[:fuzz_coverage_guided] = true
+      end
+      options.on("--path-length=N", Integer, "coverage path length: 1 or 2") do |value|
+        raise OptionParser::InvalidArgument, "--path-length must be 1 or 2" unless [1, 2].include?(value)
+
+        @options[:fuzz_path_length] = value
+      end
+      add_fuzz_external_options(options)
+    end
+
+    # @rbs (OptionParser options) -> void
+    def add_fuzz_external_options(options)
+      options.on("--against=COMMAND", "compare with an explicit JSON-token subprocess") do |value|
+        @options[:fuzz_against] = value
+      end
+      options.on("--against-runtime=DESCRIPTION", "required target runtime configuration for --against") do |value|
+        raise OptionParser::InvalidArgument, "--against-runtime must not be empty" if value.empty?
+
+        @options[:fuzz_against_runtime] = value
+      end
+      options.on("--against-timeout=SECONDS", Integer, "target timeout per sentence (default 10)") do |value|
+        @options[:fuzz_against_timeout] = positive_fuzz_option!("against-timeout", value)
+      end
+      options.on("--against-max-output=N", Integer, "target stdout/stderr byte budget") do |value|
+        @options[:fuzz_against_max_output] = positive_fuzz_option!("against-max-output", value)
+      end
+      options.on("--max-reduction-trials=N", Integer, "automatic mismatch-reduction trial budget") do |value|
+        @options[:fuzz_max_reduction_trials] = positive_fuzz_option!("max-reduction-trials", value)
+      end
+      options.on("--regression-dir=DIR", "saved minimized regressions (default test/fuzz/regressions)") do |value|
+        raise OptionParser::InvalidArgument, "--regression-dir must not be empty" if value.empty?
+
+        @options[:fuzz_regression_dir] = value
+      end
+      options.on("--[no-]save-regression", "persist minimized mismatches (default enabled)") do |value|
+        @options[:fuzz_save_regression] = value
+      end
+      options.on("--format=FORMAT", %w[json text], "json or text (default json)") do |value|
+        @options[:fuzz_format] = value
+      end
+    end
+
+    # @rbs () -> fuzz_external_target?
+    def external_fuzz_target
+      command = @options[:fuzz_against]
+      runtime = @options[:fuzz_against_runtime]
+      if command.nil?
+        raise OptionParser::InvalidArgument, "--against-runtime requires --against" if runtime
+
+        return
+      end
+      unless runtime
+        raise OptionParser::InvalidArgument,
+              "--against requires --against-runtime so the comparison target is reproducible"
+      end
+
+      arguments = Shellwords.split(command)
+      raise OptionParser::InvalidArgument, "--against command must not be empty" if arguments.empty?
+
+      timeout = @options.fetch(:fuzz_against_timeout, BoundedSubprocess::DEFAULT_TIMEOUT_SECONDS)
+      max_output = @options.fetch(:fuzz_against_max_output, BoundedSubprocess::DEFAULT_MAX_OUTPUT_BYTES)
+      subprocess = BoundedSubprocess.new(timeout_seconds: timeout, max_output_bytes: max_output)
+      runner = lambda do |tokens|
+        external_fuzz_outcome(subprocess, arguments, tokens, timeout, max_output)
+      end #: ^(Array[String]) -> Symbol
+      description = {
+        command: command, target_runtime: runtime,
+        timeout_seconds: timeout, max_output_bytes: max_output,
+        host_ruby_engine: RUBY_ENGINE, host_ruby_version: RUBY_VERSION,
+        host_platform: RUBY_PLATFORM
+      } #: fuzz_external_description
+      [runner, description]
+    end
+
+    # @rbs (BoundedSubprocess subprocess, Array[String] arguments, Array[String] tokens,
+    #   Integer timeout, Integer max_output) -> Symbol
+    def external_fuzz_outcome(subprocess, arguments, tokens, timeout, max_output)
+      result = subprocess.run(arguments, input: JSON.generate(tokens))
+      if result.timed_out || result.output_limited
+        raise Fuzz::BudgetExceeded.new(
+          message: "external target exceeded its #{result.timed_out ? 'timeout' : 'output limit'}",
+          bounds: { timeout_seconds: timeout, max_output_bytes: max_output }
+        )
+      end
+      unless result.status.exited? && [0, 1].include?(result.status.exitstatus)
+        detail = result.stderr.empty? ? result.stdout : result.stderr
+        raise Ibex::Error,
+              "(fuzz):1:1: external target did not exit with 0 or 1: #{detail}"
+      end
+
+      result.status.success? ? :accepted : :error
+    end
+
+    # @rbs (String name, Integer value) -> Integer
+    def positive_fuzz_option!(name, value)
+      return value if value.positive?
+
+      raise OptionParser::InvalidArgument, "--#{name} must be positive"
+    end
+  end
+end

data/lib/ibex/cli/fuzz_regressions.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "digest"
+require "fileutils"
+require "json"
+
+module Ibex
+  # @rbs!
+  #   type fuzz_external_description = {
+  #     command: String,
+  #     target_runtime: String,
+  #     timeout_seconds: Integer,
+  #     max_output_bytes: Integer,
+  #     host_ruby_engine: String,
+  #     host_ruby_version: String,
+  #     host_platform: String
+  #   }
+  #   type fuzz_external_target = [
+  #     ^(Array[String]) -> Symbol,
+  #     fuzz_external_description
+  #   ]
+
+  # Automatic bounded minimization and persistence for fuzz differences.
+  module CLIFuzzRegressions
+    # @rbs!
+    #   private def atomic_write_ir: (String path, String source) -> void
+
+    private
+
+    # @rbs (Fuzz fuzzer, Fuzz::Mismatch mismatch, String grammar_path,
+    #   fuzz_external_target? external) -> Integer
+    def write_minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
+      details = minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
+      report = {
+        ibex_report: "fuzz", schema_version: 1, result: "difference", mismatch: details
+      } #: Hash[Symbol, untyped]
+      report[:external] = external[1] if external
+      write_fuzz_report(report)
+      1
+    end
+
+    # @rbs (Hash[Symbol, untyped] report) -> void
+    def write_fuzz_report(report)
+      if @options.fetch(:fuzz_format, "json") == "json"
+        @stdout.puts JSON.pretty_generate(report)
+        return
+      end
+
+      @stdout.puts("result=#{report.fetch(:result)}")
+      case report.fetch(:result)
+      when "no_difference_within_bounds"
+        @stdout.puts("seed=#{report.fetch(:seed)} bounds=#{report.fetch(:bounds).inspect}")
+        @stdout.puts("no difference found within the declared bounds; this is not a proof of equivalence")
+      when "difference"
+        mismatch = report.fetch(:mismatch) #: Hash[Symbol, untyped]
+        @stdout.puts("kind=#{mismatch.fetch(:kind)} tokens=#{mismatch.fetch(:tokens).inspect}")
+        @stdout.puts("minimized=#{mismatch.fetch(:minimized_tokens).inspect} " \
+                     "reduction_complete=#{mismatch.dig(:reduction, :complete)}")
+        @stdout.puts("regression=#{mismatch.dig(:regression, :path)}") if mismatch[:regression]
+      when "budget_exhausted"
+        @stdout.puts("search incomplete: #{report.dig(:budget, :message)}")
+      end
+      external = report[:external] #: fuzz_external_description?
+      return unless external
+
+      @stdout.puts("target_runtime=#{external.fetch(:target_runtime)}")
+      @stdout.puts("target_command=#{external.fetch(:command)}")
+    end
+
+    # @rbs (Fuzz::BudgetExceeded error,
+    #   fuzz_external_target? external, phase: String) -> Integer
+    def write_fuzz_budget_report(error, external, phase:)
+      report = {
+        ibex_report: "fuzz", schema_version: 1, result: "budget_exhausted",
+        budget: error.details.merge(phase: phase)
+      } #: Hash[Symbol, untyped]
+      report[:external] = external[1] if external
+      write_fuzz_report(report)
+      2
+    end
+
+    # @rbs (Fuzz fuzzer, Fuzz::Mismatch mismatch, String grammar_path,
+    #   fuzz_external_target? external) -> Hash[Symbol, untyped]
+    def minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
+      result = fuzzer.minimize(
+        mismatch, max_trials: @options.fetch(:fuzz_max_reduction_trials, 1_000)
+      )
+      details = mismatch.details.dup #: Hash[Symbol, untyped]
+      details[:minimized_tokens] = result.items
+      details[:reduction] = {
+        original_size: result.original_size, minimized_size: result.items.length,
+        trials: result.trials, complete: result.complete
+      }
+      regression = persist_fuzz_regression(
+        mismatch.details, result, grammar_path, external&.[](1)
+      )
+      details[:regression] = regression if regression
+      details
+    end
+
+    # @rbs (fuzz_mismatch mismatch, DeltaReducer::Result result, String grammar_path,
+    #   fuzz_external_description? external) -> Hash[Symbol, String]?
+    def persist_fuzz_regression(mismatch, result, grammar_path, external)
+      return if @options.fetch(:fuzz_save_regression, true) == false
+
+      directory = @options.fetch(:fuzz_regression_dir, "test/fuzz/regressions")
+      FileUtils.mkdir_p(directory)
+      document = fuzz_regression_document(mismatch, result, grammar_path, external)
+      source = "#{JSON.pretty_generate(document)}\n"
+      digest = Digest::SHA256.hexdigest(source)
+      seed = @options.fetch(:fuzz_seed, 0)
+      path = File.join(directory, "fuzz-seed-#{seed}-#{digest.slice(0, 16)}.json")
+      validate_fuzz_regression_target!(path)
+      atomic_write_ir(path, source)
+      { path: path, sha256: digest }
+    end
+
+    # @rbs (String path) -> void
+    def validate_fuzz_regression_target!(path)
+      return unless File.symlink?(path) || (File.exist?(path) && File.stat(path).nlink > 1)
+
+      raise Ibex::Error, "(fuzz):1:1: regression output refuses symlinks and files with multiple hard links"
+    end
+
+    # @rbs (fuzz_mismatch mismatch, DeltaReducer::Result result, String grammar_path,
+    #   fuzz_external_description? external) -> Hash[Symbol, Object?]
+    def fuzz_regression_document(mismatch, result, grammar_path, external)
+      document = {
+        ibex_report: "fuzz-regression", schema_version: 1, grammar: grammar_path,
+        seed: @options.fetch(:fuzz_seed, 0), kind: mismatch[:kind],
+        sentence: mismatch[:sentence],
+        algorithms: mismatch[:outcomes].keys.reject { |name| name == :external }.map(&:to_s),
+        original_tokens: mismatch[:tokens], minimized_tokens: result.items,
+        bounds: mismatch[:bounds],
+        reduction: {
+          original_size: result.original_size, minimized_size: result.items.length,
+          trials: result.trials, complete: result.complete
+        }
+      } #: Hash[Symbol, Object?]
+      document[:external] = external if external
+      document
+    end
+  end
+end

data/lib/ibex/cli/generation_artifacts.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "../artifact_set"
+require_relative "../generation_input"
+require_relative "../generation_transaction"
+
+module Ibex
+  # Collects, verifies, and transactionally publishes CLI generation outputs.
+  module CLIGenerationArtifacts
+    # @rbs!
+    #   private def report_status: (String) -> void
+    #   private def verify_file: (String, String, String) -> void
+    #   private def default_output_path: (String, String) -> String
+    #   private def configuration_value: (String) -> Object?
+    #   private def effective_configuration: () -> Configuration::Resolver
+
+    private
+
+    # @rbs () -> void
+    def begin_artifact_generation
+      @generation_artifacts = ArtifactSet.new
+      @generation_statuses = [] #: Array[String]
+      @generation_sources = [] #: Array[String]
+      @generation_inputs = [] #: Array[GenerationInput]
+      @generation_automaton = nil
+    end
+
+    # @rbs (Symbol kind, String path, String content, ?mode: Integer?, ?status: bool) -> Artifact
+    def register_artifact(kind, path, content, mode: nil, status: false)
+      artifact = generation_artifacts.add(kind: kind, path: path, content: content, mode: mode)
+      @generation_statuses << "wrote #{path}" if status
+      artifact
+    end
+
+    # @rbs (Array[String] source_paths) -> Integer
+    def finish_artifact_generation(_source_paths)
+      add_generation_manifest if @options[:manifest]
+      return 0 if @defer_generation_publication
+
+      ensure_generation_inputs_stable!
+      if @options[:verify_output]
+        verify_artifact_generation
+        ensure_generation_inputs_stable!
+      elsif !generation_artifacts.empty?
+        GenerationTransaction.new(
+          generation_artifacts,
+          warning: ->(message) { @stderr.puts("ibex: warning: #{message}") },
+          stability_check: -> { generation_inputs_stable? },
+          source_records: @generation_inputs
+        ).commit
+        @generation_statuses.each { |message| report_status(message) }
+      end
+      0
+    end
+
+    # @rbs () -> void
+    def add_generation_manifest
+      require_relative "../generation_manifest"
+
+      path = manifest_output_path
+      source = GenerationManifest.render(
+        generation_artifacts,
+        source_records: @generation_inputs,
+        options: generation_manifest_options
+      )
+      register_artifact(:manifest, path, source, status: true)
+    end
+
+    # @rbs () -> void
+    def verify_artifact_generation
+      generation_artifacts.each do |artifact|
+        verify_file(artifact.path, artifact.content, artifact_label(artifact.kind))
+      end
+      parser = generation_artifacts.find { |artifact| artifact.kind == :parser }
+      report_status("verified #{parser.path}") if parser
+    end
+
+    # @rbs (Symbol kind) -> String
+    def artifact_label(kind)
+      {
+        parser: "parser",
+        rbs: "RBS signature",
+        action_source: "action source",
+        manifest: "generation manifest"
+      }.fetch(kind, kind.to_s.tr("_", " "))
+    end
+
+    # @rbs () -> ArtifactSet
+    def generation_artifacts
+      @generation_artifacts || raise(Ibex::Error, "(generation):1:1: artifact collection is not active")
+    end
+
+    # @rbs () -> bool
+    def generation_inputs_stable?
+      @generation_inputs.all?(&:current?)
+    end
+
+    # @rbs () -> void
+    def ensure_generation_inputs_stable!
+      return if generation_inputs_stable?
+
+      raise GenerationTransaction::SourceChanged, "(generation):1:1: source changed while rendering outputs"
+    end
+
+    # @rbs (String path, String source) -> GenerationInput
+    def record_generation_input(path, source)
+      input = GenerationInput.new(path, source)
+      @generation_inputs.reject! { |entry| entry.path == input.path }
+      @generation_inputs << input
+      @generation_sources = @generation_inputs.map(&:path)
+      input
+    end
+
+    # @rbs () -> String
+    def manifest_output_path
+      configured = @options[:manifest]
+      return configured if configured.is_a?(String) && !configured.empty?
+
+      parser = generation_artifacts.find { |artifact| artifact.kind == :parser }
+      raise Ibex::Error, "(generation):1:1: generation manifest requires a parser artifact" unless parser
+
+      default_output_path(parser.path, ".ibex.json")
+    end
+
+    # @rbs () -> Hash[String, Object?]
+    def generation_manifest_options
+      keys = %i[
+        action_source algorithm counterexample_max_configurations counterexample_max_tokens debug
+        embedded emit entry_isolation executable frozen line_convert line_convert_all messages mode omit_actions rbs
+        superclass table
+      ]
+      options = {} #: Hash[String, Object?]
+      keys.each do |key|
+        value = @options[key]
+        options[key.to_s] = value unless value.nil?
+      end
+      options["cst_trivia"] = effective_cst_trivia.to_s
+      append_ir_manifest_options(options)
+      options
+    end
+
+    # @rbs (Hash[String, Object?] options) -> void
+    def append_ir_manifest_options(options)
+      automaton = @generation_automaton
+      return unless automaton
+
+      grammar = automaton.grammar
+      contract = grammar.parser_contract
+      options["grammar_ir"] = {
+        "schema_version" => grammar.schema_version,
+        "digest" => automaton.grammar_digest,
+        "parser_contract" => stringify_contract(contract)
+      }
+      options["automaton_ir"] = {
+        "schema_version" => automaton.schema_version,
+        "algorithm" => automaton.algorithm,
+        "entry_construction" => automaton.entry_construction,
+        "construction_authority" => construction_authority
+      }
+      options["effective_configuration"] = manifest_effective_configuration
+    end
+
+    # @rbs () -> String
+    def construction_authority
+      @options[:from] == "automaton-ir" ? "embedded_automaton" : "grammar_contract"
+    end
+
+    # @rbs () -> Array[Hash[String, Object?]]
+    def manifest_effective_configuration
+      values = effective_configuration.to_h.fetch("configuration")
+      return values unless @options[:from] == "automaton-ir"
+
+      values.reject { |entry| %w[parser.algorithm parser.entries].include?(entry.fetch("key")) }
+    end
+
+    # @rbs (IR::ParserContract contract) -> Hash[String, Object?]
+    def stringify_contract(contract)
+      contract.to_h.to_h do |name, entry|
+        [name.to_s, entry.to_h { |key, value| [key.to_s, value] }]
+      end
+    end
+
+    # @rbs () -> Symbol
+    def effective_cst_trivia
+      configuration_value("cst.trivia") #: Symbol
+    end
+  end
+end

data/lib/ibex/cli/generation_error_messages.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  # Optional example-keyed error-message support for parser generation.
+  module CLIGenerationErrorMessages
+    # @rbs!
+    #   private def record_generation_input: (String, String) -> GenerationInput
+
+    private
+
+    # @rbs (OptionParser options) -> void
+    def add_error_messages_generation_option(options)
+      options.on("--messages=FILE", "embed example-keyed syntax error messages") do |value|
+        @options[:messages] = value
+      end
+    end
+
+    # @rbs (IR::Automaton automaton) -> Hash[Integer, String | { id: String, message: String }]
+    def configured_error_messages(automaton)
+      path = @options[:messages]
+      return {} unless path
+
+      require_relative "../error_messages"
+      source = File.binread(path)
+      record_generation_input(path, source)
+      document = ErrorMessages.parse(source, file: path)
+      ErrorMessages.records_for(document, automaton, file: path)
+    end
+
+    # @rbs () -> void
+    def validate_messages_options
+      return unless @options[:messages]
+      raise Ibex::Error, "(cli):1:1: --messages is available only with --emit=ruby" unless @options[:emit] == "ruby"
+
+      raise Ibex::Error, "(cli):1:1: --messages cannot be combined with --check-only" if @options[:check_only]
+    end
+  end
+end