ibex 0.1.0 → 0.2.0

data/lib/ibex/generation_transaction_validation.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  # Canonical target, source, symlink, and lock identity checks for generation.
+  module GenerationTransactionValidation
+    # @rbs!
+    #   private def lock_path: (String) -> String
+
+    private
+
+    # Follow a final symlink while preserving the symlink path itself during publication.
+    # @rbs (String path, ?Hash[String, bool] seen, ?validate_links: bool) -> String
+    def resolved_target(path, seen = {}, validate_links: true)
+      expanded = File.expand_path(path)
+      if File.symlink?(expanded)
+        raise Errno::ELOOP, expanded if seen[expanded]
+
+        seen[expanded] = true
+        linked = File.expand_path(File.readlink(expanded), File.dirname(expanded))
+        return resolved_target(linked, seen, validate_links: validate_links)
+      end
+      return existing_target(expanded, validate_links) if File.exist?(expanded)
+
+      parent = File.realpath(File.dirname(expanded))
+      raise Errno::ENOTDIR, parent unless File.directory?(parent)
+
+      File.join(parent, File.basename(expanded))
+    end
+
+    # @rbs (String expanded, bool validate_links) -> String
+    def existing_target(expanded, validate_links)
+      target = File.realpath(expanded)
+      validate_existing_target!(target) if validate_links
+      target
+    end
+
+    # @rbs (String target) -> void
+    def validate_existing_target!(target)
+      stat = File.stat(target)
+      unless stat.file?
+        raise GenerationTransaction::Error,
+              "(generation):1:1: output target is not a regular file: #{target}"
+      end
+      return unless stat.nlink > 1
+
+      raise GenerationTransaction::Error,
+            "(generation):1:1: output target has multiple hard links: #{target}"
+    end
+
+    # @rbs () -> void
+    def validate_source_boundaries!
+      source_paths = @source_records.map(&:path)
+      @records.each do |record|
+        target = record.fetch(:target)
+        collision = source_paths.find { |source| paths_alias?(target, source) }
+        next unless collision
+
+        raise GenerationTransaction::Error,
+              "(generation):1:1: artifact target aliases generation input: #{target} (#{collision})"
+      end
+    end
+
+    # @rbs (Array[String] lock_paths) -> void
+    def validate_lock_boundaries!(lock_paths)
+      protected_paths = protected_generation_paths
+      lock_paths.each do |path|
+        if File.symlink?(path)
+          raise GenerationTransaction::Error,
+                "(generation):1:1: generation lock must not be a symlink: #{path}"
+        end
+
+        collision = protected_paths.find { |protected| paths_alias?(path, protected) }
+        next unless collision
+
+        raise GenerationTransaction::Error,
+              "(generation):1:1: generation lock aliases a protected path: #{path} (#{collision})"
+      end
+    end
+
+    # @rbs (String path, File lock) -> void
+    def validate_lock_identity!(path, lock)
+      path_stat = File.lstat(path)
+      lock_stat = lock.stat
+      unless stable_lock_identity?(path_stat, lock_stat)
+        raise GenerationTransaction::Error,
+              "(generation):1:1: generation lock identity changed while opening: #{path}"
+      end
+      return unless protected_generation_stats.any? { |stat| same_inode?(stat, lock_stat) }
+
+      raise GenerationTransaction::Error,
+            "(generation):1:1: generation lock aliases a protected inode: #{path}"
+    end
+
+    # @rbs () -> Array[String]
+    def protected_generation_paths
+      @source_records.map(&:path) + @records.map { |record| record.fetch(:target) }
+    end
+
+    # @rbs () -> Array[File::Stat]
+    def protected_generation_stats
+      protected_generation_paths.filter_map { |path| File.stat(path) if File.exist?(path) }
+    end
+
+    # @rbs (File::Stat path_stat, File::Stat lock_stat) -> bool
+    def stable_lock_identity?(path_stat, lock_stat)
+      path_stat.file? && path_stat.nlink == 1 && same_inode?(path_stat, lock_stat)
+    end
+
+    # @rbs (File::Stat left, File::Stat right) -> bool
+    def same_inode?(left, right)
+      left.dev == right.dev && left.ino == right.ino
+    end
+
+    # @rbs (String left, String right) -> bool
+    def paths_alias?(left, right)
+      return true if File.expand_path(left) == File.expand_path(right)
+      return true if portable_target_key(left) == portable_target_key(right)
+      return false unless File.exist?(left) && File.exist?(right)
+
+      File.identical?(left, right)
+    rescue SystemCallError
+      true
+    end
+
+    # Reject target names that collide on common case-insensitive or
+    # normalization-insensitive filesystems, even when the current filesystem
+    # would permit both names.
+    # @rbs (String path) -> [String, String]
+    def portable_target_key(path)
+      basename = File.basename(path)
+      folded = if basename.encoding == Encoding::UTF_8 && basename.valid_encoding?
+                 basename.unicode_normalize(:nfc).downcase(:fold).unicode_normalize(:nfc)
+               else
+                 basename.downcase
+               end
+      [File.dirname(path), folded]
+    end
+
+    # @rbs (String phase) -> void
+    def ensure_sources_stable!(phase)
+      validate_active_locks!(phase)
+      validate_current_targets!(phase)
+      return if @stability_check.call
+
+      raise GenerationTransaction::SourceChanged, "(generation):1:1: source changed #{phase}"
+    end
+
+    # @rbs (String phase) -> void
+    def validate_active_locks!(phase)
+      @locked_paths.zip(@locks).each do |path, lock|
+        next if stable_lock_identity?(File.lstat(path), lock.stat)
+
+        raise GenerationTransaction::SourceChanged,
+              "(generation):1:1: generation lock changed #{phase}: #{path}"
+      end
+    rescue SystemCallError => e
+      raise GenerationTransaction::SourceChanged,
+            "(generation):1:1: cannot revalidate generation lock #{phase}: #{e.message}"
+    end
+
+    # @rbs (String phase) -> void
+    def validate_current_targets!(phase)
+      sources = @source_records.map(&:path)
+      @records.each do |record|
+        validate_current_target!(record, sources, phase)
+      end
+    rescue SystemCallError => e
+      raise GenerationTransaction::SourceChanged,
+            "(generation):1:1: cannot revalidate artifact target #{phase}: #{e.message}"
+    end
+
+    # @rbs (Hash[Symbol, untyped] record, Array[String] sources, String phase) -> void
+    def validate_current_target!(record, sources, phase)
+      artifact = record.fetch(:artifact)
+      current = resolved_target(artifact.path, validate_links: false)
+      unless current == record.fetch(:target)
+        raise GenerationTransaction::SourceChanged,
+              "(generation):1:1: artifact target changed #{phase}: #{artifact.path}"
+      end
+      return unless sources.any? { |source| paths_alias?(current, source) }
+
+      raise GenerationTransaction::SourceChanged,
+            "(generation):1:1: artifact target aliases an input #{phase}: #{artifact.path}"
+    end
+
+    # @rbs () -> void
+    def validate_locked_targets!
+      current = @records.map { |record| lock_path(record.fetch(:target)) }.uniq.sort
+      return if current == @locked_paths
+
+      raise GenerationTransaction::Error,
+            "(generation):1:1: output target changed while acquiring generation locks"
+    end
+  end
+end

data/lib/ibex/grammar_tests.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "json"
+require "rbconfig"
+require "tempfile"
+require "timeout"
+require "tmpdir"
+
+module Ibex
+  # Executes grammar-declared source examples against one generated parser.
+  module GrammarTests
+    DEFAULT_TIMEOUT = 10
+    RESULT_MARKER = "IBEX_GRAMMAR_TEST_RESULT="
+
+    Result = Struct.new(
+      :expectation, #: Symbol
+      :actual, #: Symbol
+      :error_class, #: String?
+      :error_message, #: String?
+      :location, #: IR::location
+      :production_ids, #: Array[Integer]
+      keyword_init: true
+    )
+
+    class Result
+      # @rbs () -> bool
+      def passed? = expectation == actual
+    end
+
+    ProductionCoverage = Struct.new(
+      :covered_ids, #: Array[Integer]
+      :missing_ids, #: Array[Integer]
+      :production_count, #: Integer
+      keyword_init: true
+    )
+
+    class ProductionCoverage
+      # @rbs () -> Float
+      def percentage
+        return 100.0 if production_count.zero?
+
+        covered_ids.length.fdiv(production_count) * 100
+      end
+
+      # @rbs (Integer minimum) -> bool
+      def meets?(minimum)
+        covered_ids.length * 100 >= production_count * minimum
+      end
+
+      # @rbs () -> bool
+      def complete? = missing_ids.empty?
+    end
+
+    # Runs all examples in one isolated process while creating a fresh parser
+    # instance for every case.
+    class Runner
+      CHILD_RUNNER = <<~'RUBY'
+        require "json"
+
+        load ARGV.fetch(0)
+        parser_class = ARGV.fetch(1).split("::").reject(&:empty?).reduce(Object) do |scope, name|
+          scope.const_get(name, false)
+        end
+        tests = JSON.parse(ARGV.fetch(2))
+        results = tests.map do |test|
+          production_ids = []
+          begin
+            parser = parser_class.new
+            raise NoMethodError, "#{parser_class} must define parse(source)" unless parser.respond_to?(:parse)
+
+            parser.observe do |event|
+              production_ids << event.data.fetch("production_id") if event.type == :reduce
+            end
+            parser.parse(test.fetch("source"))
+            result = { "actual" => "accept", "error_class" => nil, "error_message" => nil }
+          rescue Ibex::Runtime::ParseError => error
+            result = { "actual" => "reject", "error_class" => error.class.name, "error_message" => error.message }
+          rescue SystemExit, SignalException, StandardError => error
+            result = { "actual" => "error", "error_class" => error.class.name, "error_message" => error.message }
+          end
+          result.merge("production_ids" => production_ids)
+        end
+        puts "IBEX_GRAMMAR_TEST_RESULT=#{JSON.generate(results)}"
+      RUBY
+
+      # @rbs @automaton: IR::Automaton
+      # @rbs @timeout: Integer
+
+      # @rbs (IR::Automaton automaton, ?timeout: Integer) -> void
+      def initialize(automaton, timeout: DEFAULT_TIMEOUT)
+        raise ArgumentError, "timeout must be a positive Integer" unless timeout.is_a?(Integer) && timeout.positive?
+
+        @automaton = automaton
+        @timeout = timeout
+      end
+
+      # @rbs () -> Array[Result]
+      def run
+        grammar = @automaton.grammar
+        raise Ibex::Error, "(test):1:1: grammar declares no %test cases" if grammar.grammar_tests.empty?
+        unless grammar.parser_parameters.empty?
+          raise Ibex::Error, "(test):1:1: %test cannot instantiate a parser with required %param declarations"
+        end
+
+        Dir.mktmpdir("ibex-grammar-tests") do |directory|
+          parser_path = File.join(directory, "parser.rb")
+          runner_path = File.join(directory, "runner.rb")
+          File.binwrite(parser_path, generated_parser)
+          File.binwrite(runner_path, CHILD_RUNNER)
+          execute_child(parser_path, runner_path)
+        end
+      end
+
+      # @rbs (Array[Result] results) -> ProductionCoverage
+      def production_coverage(results)
+        production_count = @automaton.grammar.productions.length
+        covered_ids = results.flat_map(&:production_ids).uniq.sort.freeze
+        missing_ids = ((0...production_count).to_a - covered_ids).freeze
+        ProductionCoverage.new(
+          covered_ids: covered_ids, missing_ids: missing_ids, production_count: production_count
+        ).freeze
+      end
+
+      private
+
+      # @rbs () -> String
+      def generated_parser
+        Codegen::Ruby.new(@automaton, table: :compact, embedded: true, line_convert: true).generate
+      end
+
+      # @rbs (String parser_path, String runner_path) -> Array[Result]
+      def execute_child(parser_path, runner_path)
+        payload = JSON.generate(@automaton.grammar.grammar_tests)
+        stdout = Tempfile.new("ibex-grammar-test-stdout")
+        stderr = Tempfile.new("ibex-grammar-test-stderr")
+        stdout_path = stdout.path || raise(Ibex::Error, "(test):1:1: missing temporary stdout path")
+        stderr_path = stderr.path || raise(Ibex::Error, "(test):1:1: missing temporary stderr path")
+        pid = spawn(
+          RbConfig.ruby, runner_path, parser_path, @automaton.grammar.class_name, payload,
+          out: stdout_path, err: stderr_path
+        )
+        status = wait_for_child(pid)
+        output = File.binread(stdout_path)
+        errors = File.binread(stderr_path)
+        raise_child_failure(status, errors) unless status.success?
+
+        decode_results(output)
+      ensure
+        stdout&.close!
+        stderr&.close!
+      end
+
+      # @rbs (Integer pid) -> Process::Status
+      def wait_for_child(pid)
+        Timeout.timeout(@timeout) { Process.wait2(pid).fetch(1) }
+      rescue Timeout::Error
+        Process.kill("KILL", pid)
+        Process.wait(pid)
+        raise Ibex::Error, "(test):1:1: grammar tests exceeded #{@timeout} seconds"
+      end
+
+      # @rbs (Process::Status status, String errors) -> bot
+      def raise_child_failure(status, errors)
+        detail = errors.lines.last&.strip
+        suffix = detail && !detail.empty? ? ": #{detail}" : ""
+        raise Ibex::Error, "(test):1:1: grammar test process exited with status #{status.exitstatus}#{suffix}"
+      end
+
+      # @rbs (String output) -> Array[Result]
+      def decode_results(output)
+        line = output.lines.reverse.find { |candidate| candidate.start_with?(RESULT_MARKER) }
+        raise Ibex::Error, "(test):1:1: grammar test process produced no result" unless line
+
+        documents = JSON.parse(line.delete_prefix(RESULT_MARKER).strip)
+        tests = @automaton.grammar.grammar_tests
+        unless documents.is_a?(Array) && documents.length == tests.length
+          raise Ibex::Error, "(test):1:1: grammar test process returned an invalid result count"
+        end
+
+        tests.zip(documents).map { |test, document| build_result(test, document) }
+      rescue JSON::ParserError, ArgumentError => e
+        raise Ibex::Error, "(test):1:1: invalid grammar test process result: #{e.message}"
+      end
+
+      # @rbs (IR::grammar_test test, untyped document) -> Result
+      def build_result(test, document)
+        unless document.is_a?(Hash) && %w[accept reject error].include?(document["actual"])
+          raise Ibex::Error, "(test):1:1: grammar test process returned an invalid case result"
+        end
+
+        production_ids = document["production_ids"]
+        production_count = @automaton.grammar.productions.length
+        unless production_ids.is_a?(Array) &&
+               production_ids.all? { |id| id.is_a?(Integer) && id.between?(0, production_count - 1) }
+          raise Ibex::Error, "(test):1:1: grammar test process returned invalid production coverage"
+        end
+
+        Result.new(
+          expectation: test[:expectation], actual: document.fetch("actual").to_sym,
+          error_class: document["error_class"], error_message: document["error_message"],
+          location: test[:loc], production_ids: production_ids.freeze
+        ).freeze
+      end
+    end
+  end
+end

data/lib/ibex/ir/automaton_ir.rb

@@ -74,17 +74,33 @@ module Ibex
       attr_reader :grammar_digest #: String
       attr_reader :grammar #: Grammar
       attr_reader :states #: Array[AutomatonState]
+      attr_reader :entry_states #: Hash[String, Integer]
       attr_reader :conflict_summary #: conflict_summary
       attr_reader :schema_version #: Integer
 
       # @rbs (grammar: Grammar, states: Array[AutomatonState], conflict_summary: conflict_summary,
-      #   ?algorithm: String, ?grammar_digest: String?, ?schema_version: Integer) -> void
+      #   ?algorithm: String, ?grammar_digest: String?, ?schema_version: Integer,
+      #   ?entry_states: Hash[String, Integer]?) -> void
       def initialize(grammar:, states:, conflict_summary:, algorithm: "lalr1", grammar_digest: nil,
-                     schema_version: SCHEMA_VERSION)
+                     schema_version: SCHEMA_VERSION, entry_states: nil)
+        unless SUPPORTED_SCHEMA_VERSIONS.include?(schema_version)
+          raise Ibex::Error, "unsupported automaton schema_version #{schema_version.inspect}"
+        end
+
+        grammar = Migration.to_v2(grammar) if schema_version >= 2 && grammar.schema_version != schema_version
+        raise Ibex::Error, "automaton migration did not produce a grammar" unless grammar.is_a?(Grammar)
+        unless grammar.schema_version == schema_version
+          raise Ibex::Error,
+                "automaton schema_version #{schema_version} requires Grammar IR v#{schema_version}, " \
+                "got v#{grammar.schema_version}"
+        end
+
         @algorithm = algorithm.freeze
         @grammar = grammar
         @grammar_digest = (grammar_digest || digest_for(grammar)).freeze
         @states = states.freeze
+        @entry_states = IR.deep_freeze(entry_states || { grammar.start => 0 })
+        validate_entry_states
         @conflict_summary = IR.deep_freeze(conflict_summary)
         @schema_version = schema_version
         freeze
@@ -92,13 +108,30 @@ module Ibex
 
       # @rbs () -> Hash[Symbol, untyped]
       def to_h
-        { ibex_ir: "automaton", schema_version: @schema_version, algorithm: @algorithm,
-          grammar_digest: @grammar_digest, grammar: @grammar.to_h,
-          states: @states.map { |state| state.to_h(@grammar) }, conflict_summary: @conflict_summary }
+        value = { ibex_ir: "automaton", schema_version: @schema_version, algorithm: @algorithm,
+                  grammar_digest: @grammar_digest, grammar: @grammar.to_h,
+                  states: @states.map { |state| state.to_h(@grammar) },
+                  conflict_summary: @conflict_summary } #: Hash[Symbol, untyped]
+        value[:entry_states] = @entry_states unless @entry_states == { @grammar.start => 0 }
+        value
       end
 
       private
 
+      # @rbs () -> void
+      def validate_entry_states
+        ordered = @grammar.starts.select { |name| @entry_states.key?(name) }
+        unless @entry_states.any? && @entry_states.keys == ordered
+          raise Ibex::Error, "(ir):1:1: automaton entry states must be an ordered subset of grammar starts"
+        end
+
+        @entry_states.each do |name, state|
+          next if state.between?(0, @states.length - 1)
+
+          raise Ibex::Error, "(ir):1:1: entry #{name} references missing state #{state}"
+        end
+      end
+
       # @rbs (Grammar grammar) -> String
       def digest_for(grammar)
         require "digest"