@ai-ecoverse/wasix-ruby 3.4.11-1

package/lib/ruby/3.4.11/syntax_suggest/cli.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "optparse"
+
+module SyntaxSuggest
+  # All the logic of the exe/syntax_suggest CLI in one handy spot
+  #
+  #   Cli.new(argv: ["--help"]).call
+  #   Cli.new(argv: ["<path/to/file>.rb"]).call
+  #   Cli.new(argv: ["<path/to/file>.rb", "--record=tmp"]).call
+  #   Cli.new(argv: ["<path/to/file>.rb", "--terminal"]).call
+  #
+  class Cli
+    attr_accessor :options
+
+    # ARGV is Everything passed to the executable, does not include executable name
+    #
+    # All other intputs are dependency injection for testing
+    def initialize(argv:, exit_obj: Kernel, io: $stdout, env: ENV)
+      @options = {}
+      @parser = nil
+      options[:record_dir] = env["SYNTAX_SUGGEST_RECORD_DIR"]
+      options[:record_dir] = "tmp" if env["DEBUG"]
+      options[:terminal] = SyntaxSuggest::DEFAULT_VALUE
+
+      @io = io
+      @argv = argv
+      @exit_obj = exit_obj
+    end
+
+    def call
+      if @argv.empty?
+        # Display help if raw command
+        parser.parse! %w[--help]
+        return
+      else
+        # Mutates @argv
+        parse
+        return if options[:exit]
+      end
+
+      file_name = @argv.first
+      if file_name.nil?
+        @io.puts "No file given"
+        @exit_obj.exit(1)
+        return
+      end
+
+      file = Pathname(file_name)
+      if !file.exist?
+        @io.puts "file not found: #{file.expand_path} "
+        @exit_obj.exit(1)
+        return
+      end
+
+      @io.puts "Record dir: #{options[:record_dir]}" if options[:record_dir]
+
+      display = SyntaxSuggest.call(
+        io: @io,
+        source: file.read,
+        filename: file.expand_path,
+        terminal: options.fetch(:terminal, SyntaxSuggest::DEFAULT_VALUE),
+        record_dir: options[:record_dir]
+      )
+
+      if display.document_ok?
+        @io.puts "Syntax OK"
+        @exit_obj.exit(0)
+      else
+        @exit_obj.exit(1)
+      end
+    end
+
+    def parse
+      parser.parse!(@argv)
+
+      self
+    end
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = <<~EOM
+          Usage: syntax_suggest <file> [options]
+
+          Parses a ruby source file and searches for syntax error(s) such as
+          unexpected `end', expecting end-of-input.
+
+          Example:
+
+            $ syntax_suggest dog.rb
+
+            # ...
+
+              > 10  defdog
+              > 15  end
+
+          ENV options:
+
+            SYNTAX_SUGGEST_RECORD_DIR=<dir>
+
+            Records the steps used to search for a syntax error
+            to the given directory
+
+          Options:
+        EOM
+
+        opts.version = SyntaxSuggest::VERSION
+
+        opts.on("--help", "Help - displays this message") do |v|
+          @io.puts opts
+          options[:exit] = true
+          @exit_obj.exit
+        end
+
+        opts.on("--record <dir>", "Records the steps used to search for a syntax error to the given directory") do |v|
+          options[:record_dir] = v
+        end
+
+        opts.on("--terminal", "Enable terminal highlighting") do |v|
+          options[:terminal] = true
+        end
+
+        opts.on("--no-terminal", "Disable terminal highlighting") do |v|
+          options[:terminal] = false
+        end
+      end
+    end
+  end
+end

package/lib/ruby/3.4.11/syntax_suggest/code_block.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Multiple lines form a singular CodeBlock
+  #
+  # Source code is made of multiple CodeBlocks.
+  #
+  # Example:
+  #
+  #   code_block.to_s # =>
+  #     #   def foo
+  #     #     puts "foo"
+  #     #   end
+  #
+  #   code_block.valid? # => true
+  #   code_block.in_valid? # => false
+  #
+  #
+  class CodeBlock
+    UNSET = Object.new.freeze
+    attr_reader :lines, :starts_at, :ends_at
+
+    def initialize(lines: [])
+      @lines = Array(lines)
+      @valid = UNSET
+      @deleted = false
+      @starts_at = @lines.first.number
+      @ends_at = @lines.last.number
+    end
+
+    def delete
+      @deleted = true
+    end
+
+    def deleted?
+      @deleted
+    end
+
+    def visible_lines
+      @lines.select(&:visible?).select(&:not_empty?)
+    end
+
+    def mark_invisible
+      @lines.map(&:mark_invisible)
+    end
+
+    def is_end?
+      to_s.strip == "end"
+    end
+
+    def hidden?
+      @lines.all?(&:hidden?)
+    end
+
+    # This is used for frontier ordering, we are searching from
+    # the largest indentation to the smallest. This allows us to
+    # populate an array with multiple code blocks then call `sort!`
+    # on it without having to specify the sorting criteria
+    def <=>(other)
+      out = current_indent <=> other.current_indent
+      return out if out != 0
+
+      # Stable sort
+      starts_at <=> other.starts_at
+    end
+
+    def current_indent
+      @current_indent ||= lines.select(&:not_empty?).map(&:indent).min || 0
+    end
+
+    def invalid?
+      !valid?
+    end
+
+    def valid?
+      if @valid == UNSET
+        # Performance optimization
+        #
+        # If all the lines were previously hidden
+        # and we expand to capture additional empty
+        # lines then the result cannot be invalid
+        #
+        # That means there's no reason to re-check all
+        # lines with the parser (which is expensive).
+        # Benchmark in commit message
+        @valid = if lines.all? { |l| l.hidden? || l.empty? }
+          true
+        else
+          SyntaxSuggest.valid?(lines.map(&:original).join)
+        end
+      else
+        @valid
+      end
+    end
+
+    def to_s
+      @lines.join
+    end
+  end
+end

package/lib/ruby/3.4.11/syntax_suggest/code_frontier.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # The main function of the frontier is to hold the edges of our search and to
+  # evaluate when we can stop searching.
+
+  # There are three main phases in the algorithm:
+  #
+  # 1. Sanitize/format input source
+  # 2. Search for invalid blocks
+  # 3. Format invalid blocks into something meaningful
+  #
+  # The Code frontier is a critical part of the second step
+  #
+  # ## Knowing where we've been
+  #
+  # Once a code block is generated it is added onto the frontier. Then it will be
+  # sorted by indentation and frontier can be filtered. Large blocks that fully enclose a
+  # smaller block will cause the smaller block to be evicted.
+  #
+  #   CodeFrontier#<<(block) # Adds block to frontier
+  #   CodeFrontier#pop # Removes block from frontier
+  #
+  # ## Knowing where we can go
+  #
+  # Internally the frontier keeps track of "unvisited" lines which are exposed via `next_indent_line`
+  # when called, this method returns, a line of code with the highest indentation.
+  #
+  # The returned line of code can be used to build a CodeBlock and then that code block
+  # is added back to the frontier. Then, the lines are removed from the
+  # "unvisited" so we don't double-create the same block.
+  #
+  #   CodeFrontier#next_indent_line # Shows next line
+  #   CodeFrontier#register_indent_block(block) # Removes lines from unvisited
+  #
+  # ## Knowing when to stop
+  #
+  # The frontier knows how to check the entire document for a syntax error. When blocks
+  # are added onto the frontier, they're removed from the document. When all code containing
+  # syntax errors has been added to the frontier, the document will be parsable without a
+  # syntax error and the search can stop.
+  #
+  #   CodeFrontier#holds_all_syntax_errors? # Returns true when frontier holds all syntax errors
+  #
+  # ## Filtering false positives
+  #
+  # Once the search is completed, the frontier may have multiple blocks that do not contain
+  # the syntax error. To limit the result to the smallest subset of "invalid blocks" call:
+  #
+  #   CodeFrontier#detect_invalid_blocks
+  #
+  class CodeFrontier
+    def initialize(code_lines:, unvisited: UnvisitedLines.new(code_lines: code_lines))
+      @code_lines = code_lines
+      @unvisited = unvisited
+      @queue = PriorityEngulfQueue.new
+
+      @check_next = true
+    end
+
+    def count
+      @queue.length
+    end
+
+    # Performance optimization
+    #
+    # Parsing with ripper is expensive
+    # If we know we don't have any blocks with invalid
+    # syntax, then we know we cannot have found
+    # the incorrect syntax yet.
+    #
+    # When an invalid block is added onto the frontier
+    # check document state
+    private def can_skip_check?
+      check_next = @check_next
+      @check_next = false
+
+      if check_next
+        false
+      else
+        true
+      end
+    end
+
+    # Returns true if the document is valid with all lines
+    # removed. By default it checks all blocks in present in
+    # the frontier array, but can be used for arbitrary arrays
+    # of codeblocks as well
+    def holds_all_syntax_errors?(block_array = @queue, can_cache: true)
+      return false if can_cache && can_skip_check?
+
+      without_lines = block_array.to_a.flat_map do |block|
+        block.lines
+      end
+
+      SyntaxSuggest.valid_without?(
+        without_lines: without_lines,
+        code_lines: @code_lines
+      )
+    end
+
+    # Returns a code block with the largest indentation possible
+    def pop
+      @queue.pop
+    end
+
+    def next_indent_line
+      @unvisited.peek
+    end
+
+    def expand?
+      return false if @queue.empty?
+      return true if @unvisited.empty?
+
+      frontier_indent = @queue.peek.current_indent
+      unvisited_indent = next_indent_line.indent
+
+      if ENV["SYNTAX_SUGGEST_DEBUG"]
+        puts "```"
+        puts @queue.peek
+        puts "```"
+        puts "  @frontier indent:  #{frontier_indent}"
+        puts "  @unvisited indent: #{unvisited_indent}"
+      end
+
+      # Expand all blocks before moving to unvisited lines
+      frontier_indent >= unvisited_indent
+    end
+
+    # Keeps track of what lines have been added to blocks and which are not yet
+    # visited.
+    def register_indent_block(block)
+      @unvisited.visit_block(block)
+      self
+    end
+
+    # When one element fully encapsulates another we remove the smaller
+    # block from the frontier. This prevents double expansions and all-around
+    # weird behavior. However this guarantee is quite expensive to maintain
+    def register_engulf_block(block)
+    end
+
+    # Add a block to the frontier
+    #
+    # This method ensures the frontier always remains sorted (in indentation order)
+    # and that each code block's lines are removed from the indentation hash so we
+    # don't re-evaluate the same line multiple times.
+    def <<(block)
+      @unvisited.visit_block(block)
+
+      @queue.push(block)
+
+      @check_next = true if block.invalid?
+
+      self
+    end
+
+    # Example:
+    #
+    #   combination([:a, :b, :c, :d])
+    #   # => [[:a], [:b], [:c], [:d], [:a, :b], [:a, :c], [:a, :d], [:b, :c], [:b, :d], [:c, :d], [:a, :b, :c], [:a, :b, :d], [:a, :c, :d], [:b, :c, :d], [:a, :b, :c, :d]]
+    def self.combination(array)
+      guesses = []
+      1.upto(array.length).each do |size|
+        guesses.concat(array.combination(size).to_a)
+      end
+      guesses
+    end
+
+    # Given that we know our syntax error exists somewhere in our frontier, we want to find
+    # the smallest possible set of blocks that contain all the syntax errors
+    def detect_invalid_blocks
+      self.class.combination(@queue.to_a.select(&:invalid?)).detect do |block_array|
+        holds_all_syntax_errors?(block_array, can_cache: false)
+      end || []
+    end
+  end
+end

package/lib/ruby/3.4.11/syntax_suggest/code_line.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Represents a single line of code of a given source file
+  #
+  # This object contains metadata about the line such as
+  # amount of indentation, if it is empty or not, and
+  # lexical data, such as if it has an `end` or a keyword
+  # in it.
+  #
+  # Visibility of lines can be toggled off. Marking a line as invisible
+  # indicates that it should not be used for syntax checks.
+  # It's functionally the same as commenting it out.
+  #
+  # Example:
+  #
+  #   line = CodeLine.from_source("def foo\n").first
+  #   line.number => 1
+  #   line.empty? # => false
+  #   line.visible? # => true
+  #   line.mark_invisible
+  #   line.visible? # => false
+  #
+  class CodeLine
+    TRAILING_SLASH = ("\\" + $/).freeze
+
+    # Returns an array of CodeLine objects
+    # from the source string
+    def self.from_source(source, lines: nil)
+      lines ||= source.lines
+      lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
+      lines.map.with_index do |line, index|
+        CodeLine.new(
+          line: line,
+          index: index,
+          lex: lex_array_for_line[index + 1]
+        )
+      end
+    end
+
+    attr_reader :line, :index, :lex, :line_number, :indent
+    def initialize(line:, index:, lex:)
+      @lex = lex
+      @line = line
+      @index = index
+      @original = line
+      @line_number = @index + 1
+      strip_line = line.dup
+      strip_line.lstrip!
+
+      @indent = if (@empty = strip_line.empty?)
+        line.length - 1 # Newline removed from strip_line is not "whitespace"
+      else
+        line.length - strip_line.length
+      end
+
+      set_kw_end
+    end
+
+    # Used for stable sort via indentation level
+    #
+    # Ruby's sort is not "stable" meaning that when
+    # multiple elements have the same value, they are
+    # not guaranteed to return in the same order they
+    # were put in.
+    #
+    # So when multiple code lines have the same indentation
+    # level, they're sorted by their index value which is unique
+    # and consistent.
+    #
+    # This is mostly needed for consistency of the test suite
+    def indent_index
+      @indent_index ||= [indent, index]
+    end
+    alias_method :number, :line_number
+
+    # Returns true if the code line is determined
+    # to contain a keyword that matches with an `end`
+    #
+    # For example: `def`, `do`, `begin`, `ensure`, etc.
+    def is_kw?
+      @is_kw
+    end
+
+    # Returns true if the code line is determined
+    # to contain an `end` keyword
+    def is_end?
+      @is_end
+    end
+
+    # Used to hide lines
+    #
+    # The search alorithm will group lines into blocks
+    # then if those blocks are determined to represent
+    # valid code they will be hidden
+    def mark_invisible
+      @line = ""
+    end
+
+    # Means the line was marked as "invisible"
+    # Confusingly, "empty" lines are visible...they
+    # just don't contain any source code other than a newline ("\n").
+    def visible?
+      !line.empty?
+    end
+
+    # Opposite or `visible?` (note: different than `empty?`)
+    def hidden?
+      !visible?
+    end
+
+    # An `empty?` line is one that was originally left
+    # empty in the source code, while a "hidden" line
+    # is one that we've since marked as "invisible"
+    def empty?
+      @empty
+    end
+
+    # Opposite of `empty?` (note: different than `visible?`)
+    def not_empty?
+      !empty?
+    end
+
+    # Renders the given line
+    #
+    # Also allows us to represent source code as
+    # an array of code lines.
+    #
+    # When we have an array of code line elements
+    # calling `join` on the array will call `to_s`
+    # on each element, which essentially converts
+    # it back into it's original source string.
+    def to_s
+      line
+    end
+
+    # When the code line is marked invisible
+    # we retain the original value of it's line
+    # this is useful for debugging and for
+    # showing extra context
+    #
+    # DisplayCodeWithLineNumbers will render
+    # all lines given to it, not just visible
+    # lines, it uses the original method to
+    # obtain them.
+    attr_reader :original
+
+    # Comparison operator, needed for equality
+    # and sorting
+    def <=>(other)
+      index <=> other.index
+    end
+
+    # [Not stable API]
+    #
+    # Lines that have a `on_ignored_nl` type token and NOT
+    # a `BEG` type seem to be a good proxy for the ability
+    # to join multiple lines into one.
+    #
+    # This predicate method is used to determine when those
+    # two criteria have been met.
+    #
+    # The one known case this doesn't handle is:
+    #
+    #     Ripper.lex <<~EOM
+    #       a &&
+    #        b ||
+    #        c
+    #     EOM
+    #
+    # For some reason this introduces `on_ignore_newline` but with BEG type
+    def ignore_newline_not_beg?
+      @ignore_newline_not_beg
+    end
+
+    # Determines if the given line has a trailing slash
+    #
+    #     lines = CodeLine.from_source(<<~EOM)
+    #       it "foo" \
+    #     EOM
+    #     expect(lines.first.trailing_slash?).to eq(true)
+    #
+    def trailing_slash?
+      last = @lex.last
+
+      # Older versions of prism diverged slightly from Ripper in compatibility mode
+      case last&.type
+      when :on_sp
+        last.token == TRAILING_SLASH
+      when :on_tstring_end
+        true
+      else
+        false
+      end
+    end
+
+    # Endless method detection
+    #
+    # From https://github.com/ruby/irb/commit/826ae909c9c93a2ddca6f9cfcd9c94dbf53d44ab
+    # Detecting a "oneliner" seems to need a state machine.
+    # This can be done by looking mostly at the "state" (last value):
+    #
+    #   ENDFN -> BEG (token = '=' ) -> END
+    #
+    private def set_kw_end
+      oneliner_count = 0
+      in_oneliner_def = nil
+
+      kw_count = 0
+      end_count = 0
+
+      @ignore_newline_not_beg = false
+      @lex.each do |lex|
+        kw_count += 1 if lex.is_kw?
+        end_count += 1 if lex.is_end?
+
+        if lex.type == :on_ignored_nl
+          @ignore_newline_not_beg = !lex.expr_beg?
+        end
+
+        if in_oneliner_def.nil?
+          in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
+        elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
+          # Continue
+        elsif lex.state.allbits?(Ripper::EXPR_BEG)
+          in_oneliner_def = :BODY if lex.token == "="
+        elsif lex.state.allbits?(Ripper::EXPR_END)
+          # We found an endless method, count it
+          oneliner_count += 1 if in_oneliner_def == :BODY
+
+          in_oneliner_def = nil
+        else
+          in_oneliner_def = nil
+        end
+      end
+
+      kw_count -= oneliner_count
+
+      @is_kw = (kw_count - end_count) > 0
+      @is_end = (end_count - kw_count) > 0
+    end
+  end
+end