syntax_suggest 0.0.1

data/lib/syntax_suggest/code_line.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Represents a single line of code of a given source file
+  #
+  # This object contains metadata about the line such as
+  # amount of indentation, if it is empty or not, and
+  # lexical data, such as if it has an `end` or a keyword
+  # in it.
+  #
+  # Visibility of lines can be toggled off. Marking a line as invisible
+  # indicates that it should not be used for syntax checks.
+  # It's functionally the same as commenting it out.
+  #
+  # Example:
+  #
+  #   line = CodeLine.from_source("def foo\n").first
+  #   line.number => 1
+  #   line.empty? # => false
+  #   line.visible? # => true
+  #   line.mark_invisible
+  #   line.visible? # => false
+  #
+  class CodeLine
+    TRAILING_SLASH = ("\\" + $/).freeze
+
+    # Returns an array of CodeLine objects
+    # from the source string
+    def self.from_source(source, lines: nil)
+      lines ||= source.lines
+      lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
+      lines.map.with_index do |line, index|
+        CodeLine.new(
+          line: line,
+          index: index,
+          lex: lex_array_for_line[index + 1]
+        )
+      end
+    end
+
+    attr_reader :line, :index, :lex, :line_number, :indent
+    def initialize(line:, index:, lex:)
+      @lex = lex
+      @line = line
+      @index = index
+      @original = line
+      @line_number = @index + 1
+      strip_line = line.dup
+      strip_line.lstrip!
+
+      if strip_line.empty?
+        @empty = true
+        @indent = 0
+      else
+        @empty = false
+        @indent = line.length - strip_line.length
+      end
+
+      set_kw_end
+    end
+
+    # Used for stable sort via indentation level
+    #
+    # Ruby's sort is not "stable" meaning that when
+    # multiple elements have the same value, they are
+    # not guaranteed to return in the same order they
+    # were put in.
+    #
+    # So when multiple code lines have the same indentation
+    # level, they're sorted by their index value which is unique
+    # and consistent.
+    #
+    # This is mostly needed for consistency of the test suite
+    def indent_index
+      @indent_index ||= [indent, index]
+    end
+    alias_method :number, :line_number
+
+    # Returns true if the code line is determined
+    # to contain a keyword that matches with an `end`
+    #
+    # For example: `def`, `do`, `begin`, `ensure`, etc.
+    def is_kw?
+      @is_kw
+    end
+
+    # Returns true if the code line is determined
+    # to contain an `end` keyword
+    def is_end?
+      @is_end
+    end
+
+    # Used to hide lines
+    #
+    # The search alorithm will group lines into blocks
+    # then if those blocks are determined to represent
+    # valid code they will be hidden
+    def mark_invisible
+      @line = ""
+    end
+
+    # Means the line was marked as "invisible"
+    # Confusingly, "empty" lines are visible...they
+    # just don't contain any source code other than a newline ("\n").
+    def visible?
+      !line.empty?
+    end
+
+    # Opposite or `visible?` (note: different than `empty?`)
+    def hidden?
+      !visible?
+    end
+
+    # An `empty?` line is one that was originally left
+    # empty in the source code, while a "hidden" line
+    # is one that we've since marked as "invisible"
+    def empty?
+      @empty
+    end
+
+    # Opposite of `empty?` (note: different than `visible?`)
+    def not_empty?
+      !empty?
+    end
+
+    # Renders the given line
+    #
+    # Also allows us to represent source code as
+    # an array of code lines.
+    #
+    # When we have an array of code line elements
+    # calling `join` on the array will call `to_s`
+    # on each element, which essentially converts
+    # it back into it's original source string.
+    def to_s
+      line
+    end
+
+    # When the code line is marked invisible
+    # we retain the original value of it's line
+    # this is useful for debugging and for
+    # showing extra context
+    #
+    # DisplayCodeWithLineNumbers will render
+    # all lines given to it, not just visible
+    # lines, it uses the original method to
+    # obtain them.
+    attr_reader :original
+
+    # Comparison operator, needed for equality
+    # and sorting
+    def <=>(other)
+      index <=> other.index
+    end
+
+    # [Not stable API]
+    #
+    # Lines that have a `on_ignored_nl` type token and NOT
+    # a `BEG` type seem to be a good proxy for the ability
+    # to join multiple lines into one.
+    #
+    # This predicate method is used to determine when those
+    # two criteria have been met.
+    #
+    # The one known case this doesn't handle is:
+    #
+    #     Ripper.lex <<~EOM
+    #       a &&
+    #        b ||
+    #        c
+    #     EOM
+    #
+    # For some reason this introduces `on_ignore_newline` but with BEG type
+    def ignore_newline_not_beg?
+      @ignore_newline_not_beg
+    end
+
+    # Determines if the given line has a trailing slash
+    #
+    #     lines = CodeLine.from_source(<<~EOM)
+    #       it "foo" \
+    #     EOM
+    #     expect(lines.first.trailing_slash?).to eq(true)
+    #
+    def trailing_slash?
+      last = @lex.last
+      return false unless last
+      return false unless last.type == :on_sp
+
+      last.token == TRAILING_SLASH
+    end
+
+    # Endless method detection
+    #
+    # From https://github.com/ruby/irb/commit/826ae909c9c93a2ddca6f9cfcd9c94dbf53d44ab
+    # Detecting a "oneliner" seems to need a state machine.
+    # This can be done by looking mostly at the "state" (last value):
+    #
+    #   ENDFN -> BEG (token = '=' ) -> END
+    #
+    private def set_kw_end
+      oneliner_count = 0
+      in_oneliner_def = nil
+
+      kw_count = 0
+      end_count = 0
+
+      @ignore_newline_not_beg = false
+      @lex.each do |lex|
+        kw_count += 1 if lex.is_kw?
+        end_count += 1 if lex.is_end?
+
+        if lex.type == :on_ignored_nl
+          @ignore_newline_not_beg = !lex.expr_beg?
+        end
+
+        if in_oneliner_def.nil?
+          in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
+        elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
+          # Continue
+        elsif lex.state.allbits?(Ripper::EXPR_BEG)
+          in_oneliner_def = :BODY if lex.token == "="
+        elsif lex.state.allbits?(Ripper::EXPR_END)
+          # We found an endless method, count it
+          oneliner_count += 1 if in_oneliner_def == :BODY
+
+          in_oneliner_def = nil
+        else
+          in_oneliner_def = nil
+        end
+      end
+
+      kw_count -= oneliner_count
+
+      @is_kw = (kw_count - end_count) > 0
+      @is_end = (end_count - kw_count) > 0
+    end
+  end
+end

data/lib/syntax_suggest/code_search.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Searches code for a syntax error
+  #
+  # There are three main phases in the algorithm:
+  #
+  # 1. Sanitize/format input source
+  # 2. Search for invalid blocks
+  # 3. Format invalid blocks into something meaninful
+  #
+  # This class handles the part.
+  #
+  # The bulk of the heavy lifting is done in:
+  #
+  #  - CodeFrontier (Holds information for generating blocks and determining if we can stop searching)
+  #  - ParseBlocksFromLine (Creates blocks into the frontier)
+  #  - BlockExpand (Expands existing blocks to search more code)
+  #
+  # ## Syntax error detection
+  #
+  # When the frontier holds the syntax error, we can stop searching
+  #
+  #   search = CodeSearch.new(<<~EOM)
+  #     def dog
+  #       def lol
+  #     end
+  #   EOM
+  #
+  #   search.call
+  #
+  #   search.invalid_blocks.map(&:to_s) # =>
+  #   # => ["def lol\n"]
+  #
+  class CodeSearch
+    private
+
+    attr_reader :frontier
+
+    public
+
+    attr_reader :invalid_blocks, :record_dir, :code_lines
+
+    def initialize(source, record_dir: DEFAULT_VALUE)
+      record_dir = if record_dir == DEFAULT_VALUE
+        ENV["SYNTAX_SUGGEST_RECORD_DIR"] || ENV["SYNTAX_SUGGEST_DEBUG"] ? "tmp" : nil
+      else
+        record_dir
+      end
+
+      if record_dir
+        @record_dir = SyntaxSuggest.record_dir(record_dir)
+        @write_count = 0
+      end
+
+      @tick = 0
+      @source = source
+      @name_tick = Hash.new { |hash, k| hash[k] = 0 }
+      @invalid_blocks = []
+
+      @code_lines = CleanDocument.new(source: source).call.lines
+
+      @frontier = CodeFrontier.new(code_lines: @code_lines)
+      @block_expand = BlockExpand.new(code_lines: @code_lines)
+      @parse_blocks_from_indent_line = ParseBlocksFromIndentLine.new(code_lines: @code_lines)
+    end
+
+    # Used for debugging
+    def record(block:, name: "record")
+      return unless @record_dir
+      @name_tick[name] += 1
+      filename = "#{@write_count += 1}-#{name}-#{@name_tick[name]}-(#{block.starts_at}__#{block.ends_at}).txt"
+      if ENV["SYNTAX_SUGGEST_DEBUG"]
+        puts "\n\n==== #{filename} ===="
+        puts "\n```#{block.starts_at}..#{block.ends_at}"
+        puts block.to_s
+        puts "```"
+        puts "  block indent:      #{block.current_indent}"
+      end
+      @record_dir.join(filename).open(mode: "a") do |f|
+        document = DisplayCodeWithLineNumbers.new(
+          lines: @code_lines.select(&:visible?),
+          terminal: false,
+          highlight_lines: block.lines
+        ).call
+
+        f.write("    Block lines: #{block.starts_at..block.ends_at} (#{name}) \n\n#{document}")
+      end
+    end
+
+    def push(block, name:)
+      record(block: block, name: name)
+
+      block.mark_invisible if block.valid?
+      frontier << block
+    end
+
+    # Parses the most indented lines into blocks that are marked
+    # and added to the frontier
+    def create_blocks_from_untracked_lines
+      max_indent = frontier.next_indent_line&.indent
+
+      while (line = frontier.next_indent_line) && (line.indent == max_indent)
+        @parse_blocks_from_indent_line.each_neighbor_block(frontier.next_indent_line) do |block|
+          push(block, name: "add")
+        end
+      end
+    end
+
+    # Given an already existing block in the frontier, expand it to see
+    # if it contains our invalid syntax
+    def expand_existing
+      block = frontier.pop
+      return unless block
+
+      record(block: block, name: "before-expand")
+
+      block = @block_expand.call(block)
+      push(block, name: "expand")
+    end
+
+    # Main search loop
+    def call
+      until frontier.holds_all_syntax_errors?
+        @tick += 1
+
+        if frontier.expand?
+          expand_existing
+        else
+          create_blocks_from_untracked_lines
+        end
+      end
+
+      @invalid_blocks.concat(frontier.detect_invalid_blocks)
+      @invalid_blocks.sort_by! { |block| block.starts_at }
+      self
+    end
+  end
+end

data/lib/syntax_suggest/core_ext.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# Ruby 3.2+ has a cleaner way to hook into Ruby that doesn't use `require`
+if SyntaxError.method_defined?(:detailed_message)
+  module SyntaxSuggest
+    class MiniStringIO
+      def initialize(isatty: $stderr.isatty)
+        @string = +""
+        @isatty = isatty
+      end
+
+      attr_reader :isatty
+      def puts(value = $/, **)
+        @string << value
+      end
+
+      attr_reader :string
+    end
+  end
+
+  SyntaxError.prepend Module.new {
+    def detailed_message(highlight: true, syntax_suggest: true, **kwargs)
+      return super unless syntax_suggest
+
+      require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
+
+      message = super
+      file = if highlight
+        SyntaxSuggest::PathnameFromMessage.new(super(highlight: false, **kwargs)).call.name
+      else
+        SyntaxSuggest::PathnameFromMessage.new(message).call.name
+      end
+
+      io = SyntaxSuggest::MiniStringIO.new
+
+      if file
+        SyntaxSuggest.call(
+          io: io,
+          source: file.read,
+          filename: file,
+          terminal: highlight
+        )
+        annotation = io.string
+
+        annotation + message
+      else
+        message
+      end
+    rescue => e
+      if ENV["SYNTAX_SUGGEST_DEBUG"]
+        $stderr.warn(e.message)
+        $stderr.warn(e.backtrace)
+      end
+
+      # Ignore internal errors
+      message
+    end
+  }
+else
+  autoload :Pathname, "pathname"
+
+  # Monkey patch kernel to ensure that all `require` calls call the same
+  # method
+  module Kernel
+    module_function
+
+    alias_method :syntax_suggest_original_require, :require
+    alias_method :syntax_suggest_original_require_relative, :require_relative
+    alias_method :syntax_suggest_original_load, :load
+
+    def load(file, wrap = false)
+      syntax_suggest_original_load(file)
+    rescue SyntaxError => e
+      require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
+
+      SyntaxSuggest.handle_error(e)
+    end
+
+    def require(file)
+      syntax_suggest_original_require(file)
+    rescue SyntaxError => e
+      require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
+
+      SyntaxSuggest.handle_error(e)
+    end
+
+    def require_relative(file)
+      if Pathname.new(file).absolute?
+        syntax_suggest_original_require file
+      else
+        relative_from = caller_locations(1..1).first
+        relative_from_path = relative_from.absolute_path || relative_from.path
+        syntax_suggest_original_require File.expand_path("../#{file}", relative_from_path)
+      end
+    rescue SyntaxError => e
+      require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
+
+      SyntaxSuggest.handle_error(e)
+    end
+  end
+end

data/lib/syntax_suggest/display_code_with_line_numbers.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Outputs code with highlighted lines
+  #
+  # Whatever is passed to this class will be rendered
+  # even if it is "marked invisible" any filtering of
+  # output should be done before calling this class.
+  #
+  #   DisplayCodeWithLineNumbers.new(
+  #     lines: lines,
+  #     highlight_lines: [lines[2], lines[3]]
+  #   ).call
+  #   # =>
+  #       1
+  #       2  def cat
+  #     ❯ 3    Dir.chdir
+  #     ❯ 4    end
+  #       5  end
+  #       6
+  class DisplayCodeWithLineNumbers
+    TERMINAL_HIGHLIGHT = "\e[1;3m" # Bold, italics
+    TERMINAL_END = "\e[0m"
+
+    def initialize(lines:, highlight_lines: [], terminal: false)
+      @lines = Array(lines).sort
+      @terminal = terminal
+      @highlight_line_hash = Array(highlight_lines).each_with_object({}) { |line, h| h[line] = true }
+      @digit_count = @lines.last&.line_number.to_s.length
+    end
+
+    def call
+      @lines.map do |line|
+        format_line(line)
+      end.join
+    end
+
+    private def format_line(code_line)
+      # Handle trailing slash lines
+      code_line.original.lines.map.with_index do |contents, i|
+        format(
+          empty: code_line.empty?,
+          number: (code_line.number + i).to_s,
+          contents: contents,
+          highlight: @highlight_line_hash[code_line]
+        )
+      end.join
+    end
+
+    private def format(contents:, number:, empty:, highlight: false)
+      string = +""
+      string << if highlight
+        "❯ "
+      else
+        "  "
+      end
+
+      string << number.rjust(@digit_count).to_s
+      if empty
+        string << contents
+      else
+        string << "  "
+        string << TERMINAL_HIGHLIGHT if @terminal && highlight
+        string << contents
+        string << TERMINAL_END if @terminal
+      end
+      string
+    end
+  end
+end

data/lib/syntax_suggest/display_invalid_blocks.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative "capture_code_context"
+require_relative "display_code_with_line_numbers"
+
+module SyntaxSuggest
+  # Used for formatting invalid blocks
+  class DisplayInvalidBlocks
+    attr_reader :filename
+
+    def initialize(code_lines:, blocks:, io: $stderr, filename: nil, terminal: DEFAULT_VALUE)
+      @io = io
+      @blocks = Array(blocks)
+      @filename = filename
+      @code_lines = code_lines
+
+      @terminal = terminal == DEFAULT_VALUE ? io.isatty : terminal
+    end
+
+    def document_ok?
+      @blocks.none? { |b| !b.hidden? }
+    end
+
+    def call
+      if document_ok?
+        @io.puts "Syntax OK"
+        return self
+      end
+
+      if filename
+        @io.puts("--> #{filename}")
+        @io.puts
+      end
+      @blocks.each do |block|
+        display_block(block)
+      end
+
+      self
+    end
+
+    private def display_block(block)
+      # Build explanation
+      explain = ExplainSyntax.new(
+        code_lines: block.lines
+      ).call
+
+      # Enhance code output
+      # Also handles several ambiguious cases
+      lines = CaptureCodeContext.new(
+        blocks: block,
+        code_lines: @code_lines
+      ).call
+
+      # Build code output
+      document = DisplayCodeWithLineNumbers.new(
+        lines: lines,
+        terminal: @terminal,
+        highlight_lines: block.lines
+      ).call
+
+      # Output syntax error explanation
+      explain.errors.each do |e|
+        @io.puts e
+      end
+      @io.puts
+
+      # Output code
+      @io.puts(document)
+    end
+
+    private def code_with_context
+      lines = CaptureCodeContext.new(
+        blocks: @blocks,
+        code_lines: @code_lines
+      ).call
+
+      DisplayCodeWithLineNumbers.new(
+        lines: lines,
+        terminal: @terminal,
+        highlight_lines: @invalid_lines
+      ).call
+    end
+  end
+end