ibex 0.2.0 → 0.4.0

data/lib/ibex/bison_import/tokenizer.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module BisonImport
+    # Iterative scanner for the punctuation needed to recover Bison rules.
+    class Tokenizer
+      # One structural token with a byte-oriented source position.
+      class Token
+        attr_reader :type #: Symbol
+        attr_reader :value #: String
+        attr_reader :line #: Integer
+        attr_reader :column #: Integer
+
+        # @rbs (type: Symbol, value: String, line: Integer, column: Integer) -> void
+        def initialize(type:, value:, line:, column:)
+          @type = type
+          @value = value.freeze
+          @line = line
+          @column = column
+          freeze
+        end
+      end
+
+      IDENTIFIER_START = /[A-Za-z_$.]/ #: Regexp
+      IDENTIFIER_CONTINUE = /[A-Za-z0-9_$.-]/ #: Regexp
+
+      # @rbs (String source, start_line: Integer, max_tokens: Integer) -> void
+      def initialize(source, start_line:, max_tokens:)
+        @source = source
+        @index = 0
+        @line = start_line
+        @column = 1
+        @max_tokens = max_tokens
+      end
+
+      # @rbs () -> Array[Token]
+      def tokenize
+        tokens = [] #: Array[Token]
+        until eof?
+          skip_ignored
+          break if eof?
+
+          tokens << scan_token
+          check_token_budget(tokens.length)
+        end
+        tokens.freeze
+      end
+
+      private
+
+      # @rbs () -> Token
+      def scan_token
+        line = @line
+        column = @column
+        byte = current_byte || raise(Ibex::Error, "(bison-import):#{line}:#{column}: unexpected end of input")
+        if identifier_start?(byte)
+          Token.new(type: :symbol, value: scan_identifier, line: line, column: column)
+        elsif [34, 39].include?(byte)
+          Token.new(type: :literal, value: scan_quoted(byte), line: line, column: column)
+        elsif byte == 123
+          Token.new(type: :action, value: scan_action, line: line, column: column)
+        elsif byte == 37
+          Token.new(type: :directive, value: scan_directive, line: line, column: column)
+        elsif byte == 60
+          Token.new(type: :tag, value: scan_delimited(60, 62), line: line, column: column)
+        elsif byte == 91
+          Token.new(type: :tag, value: scan_delimited(91, 93), line: line, column: column)
+        else
+          value = byte ? byte.chr : ""
+          advance
+          Token.new(type: punctuation_type(value), value: value, line: line, column: column)
+        end
+      end
+
+      # @rbs () -> void
+      def skip_ignored
+        loop do
+          advance while whitespace?(current_byte)
+          if current_byte == 47 && peek_byte == 42
+            skip_block_comment
+          elsif (current_byte == 47 && peek_byte == 47) || current_byte == 35
+            skip_line
+          else
+            break
+          end
+        end
+      end
+
+      # @rbs () -> String
+      def scan_identifier
+        start = @index
+        advance while identifier_continue?(current_byte)
+        @source.byteslice(start, @index - start) || ""
+      end
+
+      # @rbs () -> String
+      def scan_directive
+        start = @index
+        advance
+        advance if current_byte == 37
+        advance while identifier_continue?(current_byte)
+        @source.byteslice(start, @index - start) || ""
+      end
+
+      # @rbs (Integer quote) -> String
+      def scan_quoted(quote)
+        start = @index
+        advance
+        until eof?
+          byte = current_byte
+          advance
+          if byte == 92
+            advance unless eof?
+          elsif byte == quote
+            break
+          end
+        end
+        @source.byteslice(start, @index - start) || ""
+      end
+
+      # @rbs () -> String
+      def scan_action
+        advance
+        start = @index
+        depth = 1
+        until eof?
+          byte = current_byte
+          if skip_action_opaque?(byte)
+            next
+          elsif byte == 123
+            depth += 1
+            advance
+          elsif byte == 125
+            depth -= 1
+            if depth.zero?
+              value = @source.byteslice(start, @index - start) || ""
+              advance
+              return value
+            end
+            advance
+          else
+            advance
+          end
+        end
+        raise Ibex::Error, "(bison-import):#{@line}:#{@column}: unterminated C action"
+      end
+
+      # @rbs (Integer? byte) -> bool
+      def skip_action_opaque?(byte)
+        if byte && [34, 39].include?(byte)
+          scan_quoted(byte)
+        elsif byte == 47 && peek_byte == 42
+          skip_block_comment
+        elsif byte == 47 && peek_byte == 47
+          skip_line
+        else
+          return false
+        end
+        true
+      end
+
+      # @rbs (Integer opener, Integer closer) -> String
+      def scan_delimited(opener, closer)
+        start = @index
+        advance
+        until eof? || current_byte == closer
+          current_byte == 92 ? 2.times { advance unless eof? } : advance
+        end
+        advance if current_byte == closer
+        value = @source.byteslice(start, @index - start) || ""
+        return value if value.getbyte(0) == opener
+
+        ""
+      end
+
+      # @rbs () -> void
+      def skip_block_comment
+        advance
+        advance
+        until eof?
+          if current_byte == 42 && peek_byte == 47
+            advance
+            advance
+            return
+          end
+          advance
+        end
+      end
+
+      # @rbs () -> void
+      def skip_line
+        advance until eof? || current_byte == 10
+        advance if current_byte == 10
+      end
+
+      # @rbs (String value) -> Symbol
+      def punctuation_type(value)
+        case value
+        when ":" then :colon
+        when "|" then :pipe
+        when ";" then :semicolon
+        else :other
+        end
+      end
+
+      # @rbs (Integer? byte) -> bool
+      def identifier_start?(byte)
+        byte ? IDENTIFIER_START.match?(byte.chr) : false
+      end
+
+      # @rbs (Integer? byte) -> bool
+      def identifier_continue?(byte)
+        byte ? IDENTIFIER_CONTINUE.match?(byte.chr) : false
+      end
+
+      # @rbs (Integer? byte) -> bool
+      def whitespace?(byte)
+        !byte.nil? && [9, 10, 11, 12, 13, 32].include?(byte)
+      end
+
+      # @rbs (Integer count) -> void
+      def check_token_budget(count)
+        return if count <= @max_tokens
+
+        raise BudgetExceeded.new(
+          result: "budget_exhausted", phase: "tokenization", max_tokens: @max_tokens
+        )
+      end
+
+      # @rbs () -> Integer?
+      def current_byte
+        @source.getbyte(@index)
+      end
+
+      # @rbs () -> Integer?
+      def peek_byte
+        @source.getbyte(@index + 1)
+      end
+
+      # @rbs () -> bool
+      def eof?
+        @index >= @source.bytesize
+      end
+
+      # @rbs () -> void
+      def advance
+        return if eof?
+
+        if current_byte == 10
+          @line += 1
+          @column = 1
+        else
+          @column += 1
+        end
+        @index += 1
+      end
+    end
+  end
+end

data/lib/ibex/bison_import.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "error"
+require_relative "ir"
+
+module Ibex
+  # Clean-room, nonexecuting import of Bison grammar structure.
+  module BisonImport
+    DIRECTIVES = {
+      "token" => :converted,
+      "left" => :converted,
+      "right" => :converted,
+      "nonassoc" => :converted,
+      "precedence" => :converted,
+      "start" => :converted,
+      "expect" => :converted,
+      "expect-rr" => :converted,
+      "empty" => :converted,
+      "prec" => :converted,
+      "nterm" => :recognized_metadata,
+      "type" => :recognized_metadata,
+      "union" => :recognized_metadata,
+      "destructor" => :recognized_metadata,
+      "printer" => :recognized_metadata,
+      "param" => :recognized_metadata,
+      "parse-param" => :recognized_metadata,
+      "lex-param" => :recognized_metadata,
+      "define" => :recognized_metadata,
+      "code" => :recognized_metadata,
+      "dprec" => :unsupported,
+      "merge" => :unsupported,
+      "glr-parser" => :unsupported,
+      "nondeterministic-parser" => :unsupported,
+      "locations" => :unsupported,
+      "initial-action" => :unsupported,
+      "language" => :unsupported,
+      "skeleton" => :unsupported,
+      "require" => :unsupported,
+      "output" => :unsupported,
+      "file-prefix" => :unsupported,
+      "name-prefix" => :unsupported,
+      "verbose" => :unsupported,
+      "yacc" => :unsupported,
+      "debug" => :unsupported,
+      "token-table" => :unsupported,
+      "no-lines" => :unsupported,
+      "header" => :unsupported,
+      "defines" => :unsupported,
+      "error-verbose" => :unsupported,
+      "pure-parser" => :unsupported
+    }.freeze #: Hash[String, Symbol]
+
+    FOREIGN_ACTION_SENTINEL = "__ibex_foreign_action_c__" #: String
+    STRUCTURAL_STATUS_MARKER = "ibex-bison-structural-status" #: String
+    STRUCTURE_NEUTRAL_UNSUPPORTED = %w[
+      initial-action locations language skeleton require output file-prefix name-prefix
+      verbose yacc debug token-table no-lines header defines error-verbose pure-parser
+    ].freeze #: Array[String]
+
+    class BudgetExceeded < Ibex::Error
+      attr_reader :details #: Hash[Symbol, Object?]
+
+      # @rbs (Hash[Symbol, Object?] details) -> void
+      def initialize(details)
+        @details = IR.deep_freeze(details)
+        super("(bison-import):1:1: configured import budget was exhausted")
+      end
+    end
+
+    # One directive occurrence and its original source position.
+    class Directive
+      attr_reader :name #: String
+      attr_reader :status #: Symbol
+      attr_reader :line #: Integer
+      attr_reader :column #: Integer
+      attr_reader :detail #: String
+
+      # @rbs (name: String, status: Symbol, line: Integer, column: Integer, detail: String) -> void
+      def initialize(name:, status:, line:, column:, detail:)
+        @name = name.freeze
+        @status = status
+        @line = line
+        @column = column
+        @detail = detail.freeze
+        freeze
+      end
+
+      # @rbs () -> Hash[Symbol, Object?]
+      def to_h
+        { name: @name, status: @status, line: @line, column: @column, detail: @detail }
+      end
+    end
+
+    # One opaque C action and its mechanical reference rewrite.
+    class Action
+      attr_reader :id #: Integer
+      attr_reader :line #: Integer
+      attr_reader :column #: Integer
+      attr_reader :original #: String
+      attr_reader :transformed #: String
+      attr_reader :encoded #: String
+
+      # @rbs (id: Integer, line: Integer, column: Integer, original: String, transformed: String,
+      #   encoded: String) -> void
+      def initialize(id:, line:, column:, original:, transformed:, encoded:)
+        @id = id
+        @line = line
+        @column = column
+        @original = original.freeze
+        @transformed = transformed.freeze
+        @encoded = encoded.freeze
+        freeze
+      end
+
+      # @rbs () -> Hash[Symbol, Object?]
+      def to_h
+        {
+          id: @id, line: @line, column: @column,
+          original: @original, transformed: @transformed,
+          source_encoding: "hex_sentinel"
+        }
+      end
+    end
+
+    # Immutable source artifact and versioned analysis report.
+    class Result
+      attr_reader :source #: String
+      attr_reader :file #: String
+      attr_reader :class_name #: String
+      attr_reader :directives #: Array[Directive]
+      attr_reader :actions #: Array[Action]
+      attr_reader :rule_count #: Integer
+      attr_reader :bounds #: Hash[Symbol, Integer]
+
+      # @rbs (source: String, file: String, class_name: String, directives: Array[Directive],
+      #   actions: Array[Action], rule_count: Integer, bounds: Hash[Symbol, Integer]) -> void
+      def initialize(source:, file:, class_name:, directives:, actions:, rule_count:, bounds:)
+        @source = source.freeze
+        @file = file.freeze
+        @class_name = class_name.freeze
+        @directives = directives.freeze
+        @actions = actions.freeze
+        @rule_count = rule_count
+        @bounds = IR.deep_freeze(bounds)
+        freeze
+      end
+
+      # @rbs () -> Hash[Symbol, Object?]
+      def to_h
+        unsupported = @directives.select { |directive| directive.status == :unsupported }
+        structural = structural_unsupported
+        {
+          ibex_report: "bison_import",
+          schema_version: 1,
+          result: unsupported.empty? ? "imported" : "imported_with_unsupported",
+          file: @file,
+          class_name: @class_name,
+          generated_source: @source,
+          directive_contract: DIRECTIVES.sort.to_h,
+          directives: @directives.map(&:to_h),
+          unsupported: unsupported.map(&:to_h),
+          structurally_complete: structural.empty?,
+          structural_unsupported: structural.map(&:to_h),
+          actions: @actions.map(&:to_h),
+          counts: {
+            rules: @rule_count,
+            actions: @actions.length,
+            unsupported_directives: unsupported.length,
+            structural_unsupported_directives: structural.length
+          },
+          bounds: @bounds,
+          statement: "C semantic actions are opaque analysis data; Ruby generation is refused."
+        }
+      end
+
+      # @rbs () -> Array[Directive]
+      def structural_unsupported
+        @directives.select do |directive|
+          directive.status == :unsupported &&
+            !STRUCTURE_NEUTRAL_UNSUPPORTED.include?(directive.name)
+        end
+      end
+
+      # @rbs () -> bool
+      def structurally_complete?
+        structural_unsupported.empty?
+      end
+    end
+
+    # @rbs (String source) -> bool
+    def bison_source?(source)
+      source.lines.count { |line| line.match?(%r{^\s*%%(?:\s|/|$)}) } >= 2
+    end
+    module_function :bison_source?
+  end
+end
+
+require_relative "bison_import/tokenizer"
+require_relative "bison_import/importer"

data/lib/ibex/bounded_subprocess.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "tempfile"
+
+module Ibex
+  # Captures one shell-free child process under wall-clock and output budgets.
+  class BoundedSubprocess
+    DEFAULT_TIMEOUT_SECONDS = 10 #: Integer
+    DEFAULT_MAX_OUTPUT_BYTES = 1_048_576 #: Integer
+    TERMINATION_GRACE_SECONDS = 0.25 #: Float
+    POLL_SECONDS = 0.01 #: Float
+
+    # Immutable observation of one bounded child-process execution.
+    class Result
+      attr_reader :stdout #: String
+      attr_reader :stderr #: String
+      attr_reader :status #: Process::Status
+      attr_reader :timed_out #: bool
+      attr_reader :output_limited #: bool
+
+      # @rbs (stdout: String, stderr: String, status: Process::Status,
+      #   timed_out: bool, output_limited: bool) -> void
+      def initialize(stdout:, stderr:, status:, timed_out:, output_limited:)
+        @stdout = stdout.freeze
+        @stderr = stderr.freeze
+        @status = status
+        @timed_out = timed_out
+        @output_limited = output_limited
+        freeze
+      end
+    end
+
+    # @rbs (timeout_seconds: Integer, max_output_bytes: Integer) -> void
+    def initialize(timeout_seconds: DEFAULT_TIMEOUT_SECONDS,
+                   max_output_bytes: DEFAULT_MAX_OUTPUT_BYTES)
+      raise ArgumentError, "timeout_seconds must be positive" unless timeout_seconds.positive?
+      raise ArgumentError, "max_output_bytes must be positive" unless max_output_bytes.positive?
+
+      @timeout_seconds = timeout_seconds
+      @max_output_bytes = max_output_bytes
+    end
+
+    # @rbs (Array[String] command, input: String) -> Result
+    def run(command, input:)
+      raise ArgumentError, "command must not be empty" if command.empty?
+
+      Tempfile.create("ibex-subprocess-input") do |stdin|
+        Tempfile.create("ibex-subprocess-stdout") do |stdout|
+          Tempfile.create("ibex-subprocess-stderr") do |stderr|
+            run_with_files(command, input, stdin, stdout, stderr)
+          end
+        end
+      end
+    end
+
+    private
+
+    # @rbs (Array[String] command, String input, File stdin, File stdout, File stderr) -> Result
+    def run_with_files(command, input, stdin, stdout, stderr)
+      stdin.binmode
+      stdin.write(input)
+      stdin.flush
+      stdin.rewind
+      pid = spawn_child(command, stdin, stdout, stderr)
+      status, timed_out, output_limited = wait(pid, stdout.path, stderr.path)
+      terminate_descendants(pid)
+      Result.new(
+        stdout: read_bounded(stdout.path), stderr: read_bounded(stderr.path),
+        status: status, timed_out: timed_out, output_limited: output_limited
+      ).freeze
+    ensure
+      terminate(pid) if pid && !status
+    end
+
+    # @rbs (Array[String] command, File stdin, File stdout, File stderr) -> Integer
+    def spawn_child(command, stdin, stdout, stderr)
+      options = { in: stdin, out: stdout, err: stderr } #: Hash[Symbol, Object?]
+      options[:pgroup] = true if process_groups?
+      spawn(*command, **options)
+    end
+
+    # @rbs (Integer pid, String stdout_path, String stderr_path) ->
+    #   [Process::Status, bool, bool]
+    def wait(pid, stdout_path, stderr_path)
+      deadline = monotonic_time + @timeout_seconds
+      loop do
+        waited = Process.waitpid2(pid, Process::WNOHANG)
+        return [waited.last, false, false] if waited
+
+        return [terminate(pid), false, true] if output_exceeded?(stdout_path, stderr_path)
+        return [terminate(pid), true, false] if monotonic_time >= deadline
+
+        sleep(POLL_SECONDS)
+      end
+    end
+
+    # @rbs (Integer pid) -> Process::Status
+    def terminate(pid)
+      signal_process_tree("TERM", pid)
+      deadline = monotonic_time + TERMINATION_GRACE_SECONDS
+      loop do
+        waited = Process.waitpid2(pid, Process::WNOHANG)
+        return waited.last if waited
+        break if monotonic_time >= deadline
+
+        sleep(POLL_SECONDS)
+      end
+      signal_process_tree("KILL", pid)
+      Process.waitpid2(pid).last
+    rescue Errno::ESRCH, Errno::ECHILD
+      process_status(pid)
+    end
+
+    # @rbs (Integer pid) -> void
+    def terminate_descendants(pid)
+      return unless process_groups?
+      return unless process_group_alive?(pid)
+
+      deadline = monotonic_time + TERMINATION_GRACE_SECONDS
+      Process.kill("TERM", -pid)
+      sleep(POLL_SECONDS) while process_group_alive?(pid) && monotonic_time < deadline
+      Process.kill("KILL", -pid) if process_group_alive?(pid)
+    rescue Errno::ESRCH
+      nil
+    end
+
+    # @rbs (String signal, Integer pid) -> void
+    def signal_process_tree(signal, pid)
+      Process.kill(signal, process_groups? ? -pid : pid)
+    end
+
+    # @rbs (Integer pid) -> bool
+    def process_group_alive?(pid)
+      Process.kill(0, -pid)
+      true
+    rescue Errno::ESRCH
+      false
+    rescue Errno::EPERM
+      true
+    end
+
+    # @rbs () -> bool
+    def process_groups?
+      !RUBY_PLATFORM.match?(/mswin|mingw|cygwin/)
+    end
+
+    # @rbs (Integer pid) -> Process::Status
+    def process_status(pid)
+      waited = Process.waitpid2(pid, Process::WNOHANG)
+      return waited.last if waited
+
+      raise Ibex::Error, "(subprocess):1:1: child #{pid} exited without a wait status"
+    end
+
+    # @rbs (String stdout_path, String stderr_path) -> bool
+    def output_exceeded?(stdout_path, stderr_path)
+      File.size(stdout_path) > @max_output_bytes || File.size(stderr_path) > @max_output_bytes
+    end
+
+    # @rbs (String path) -> String
+    def read_bounded(path)
+      File.binread(path, @max_output_bytes)
+    end
+
+    # @rbs () -> Float
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/ibex/cli/ambiguity.rb

@@ -12,6 +12,8 @@ module Ibex
     #   private def input_path: (Array[String]) -> String
     #   private def positive_counterexample_limit: (Integer, String) -> Integer
     #   private def normalize_grammar_path: (String) -> IR::Grammar
+    #   private def configuration_value: (String) -> untyped
+    #   private def set_configuration_option: (Symbol, Object?) -> void
 
     private
 
@@ -22,9 +24,10 @@ module Ibex
       return print_help(parser) if @options[:help]
       raise Ibex::Error, "(cli):1:1: check command requires --ambiguity" unless @options[:check_ambiguity]
 
-      grammar = normalize_grammar_path(input_path(remaining))
+      grammar = activate_analysis_grammar(normalize_grammar_path(input_path(remaining)))
       automaton = LALR::Builder.new(
-        grammar, algorithm: @options[:algorithm] || :lalr, entry_isolation: @options[:entry_isolation] == true
+        grammar, algorithm: configuration_value("parser.algorithm"),
+                 entry_isolation: configuration_value("parser.entries") == :isolated
       ).build
       check = Codegen::Ambiguity.new(
         automaton,
@@ -53,14 +56,20 @@ module Ibex
           @options[:check_format] = value
         end
         options.on("--algorithm=NAME", %w[slr lalr ielr lr1], "parser construction algorithm") do |value|
-          @options[:algorithm] = value.to_sym
+          set_configuration_option(:algorithm, value.to_sym)
         end
         options.on("--entry-isolation", "build independent state sets for each start symbol") do
-          @options[:entry_isolation] = true
+          set_configuration_option(:entry_isolation, true)
+        end
+        options.on("--mode=MODE", %w[default extended], "grammar mode") do |value|
+          set_configuration_option(:mode, value.to_sym)
         end
-        options.on("--mode=MODE", %w[default extended], "grammar mode") { |value| @options[:mode] = value.to_sym }
         options.on("--help", "show help") { @options[:help] = true }
       end
     end
+
+    # @rbs!
+    #   private def activate_analysis_grammar: (IR::Grammar, ?options: Hash[Symbol, Object?],
+    #     ?explicit_keys: Array[Symbol]) -> IR::Grammar
   end
 end