dead_end 1.2.0 → 2.0.0

data/lib/dead_end/code_line.rb

@@ -4,44 +4,47 @@ module DeadEnd
   # Represents a single line of code of a given source file
   #
   # This object contains metadata about the line such as
-  # amount of indentation. An if it is empty or not.
+  # amount of indentation, if it is empty or not, and
+  # lexical data, such as if it has an `end` or a keyword
+  # in it.
   #
-  # While a given search for syntax errors is being performed
-  # state about the search can be stored in individual lines such
-  # as :valid or :invalid.
-  #
-  # Visibility of lines can be toggled on and off.
+  # Visibility of lines can be toggled off. Marking a line as invisible
+  # indicates that it should not be used for syntax checks.
+  # It's functionally the same as commenting it out.
   #
   # Example:
   #
-  #   line = CodeLine.new(line: "def foo\n", index: 0)
-  #   line.line_number => 1
+  #   line = CodeLine.from_source("def foo\n").first
+  #   line.number => 1
   #   line.empty? # => false
   #   line.visible? # => true
   #   line.mark_invisible
   #   line.visible? # => false
   #
-  # A CodeBlock is made of multiple CodeLines
-  #
-  # Marking a line as invisible indicates that it should not be used
-  # for syntax checks. It's essentially the same as commenting it out
-  #
-  # Marking a line as invisible also lets the overall program know
-  # that it should not check that area for syntax errors.
   class CodeLine
     TRAILING_SLASH = ("\\" + $/).freeze
 
-    def self.parse(source)
+    # Returns an array of CodeLine objects
+    # from the source string
+    def self.from_source(source)
+      lex_array_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
       source.lines.map.with_index do |line, index|
-        CodeLine.new(line: line, index: index)
+        CodeLine.new(
+          line: line,
+          index: index,
+          lex: lex_array_for_line[index + 1]
+        )
       end
     end
 
-    attr_reader :line, :index, :indent, :original_line
+    attr_reader :line, :index, :lex, :line_number, :indent
+    def initialize(line:, index:, lex:)
+      @lex = lex
+      @line = line
+      @index = index
+      @original = line.freeze
+      @line_number = @index + 1
 
-    def initialize(line:, index:)
-      @original_line = line.freeze
-      @line = @original_line
       if line.strip.empty?
         @empty = true
         @indent = 0
@@ -49,102 +52,182 @@ module DeadEnd
         @empty = false
         @indent = SpaceCount.indent(line)
       end
-      @index = index
-      @status = nil # valid, invalid, unknown
-      @invalid = false
 
-      lex_detect!
-    end
-
-    private def lex_detect!
-      lex_array = LexAll.new(source: line)
       kw_count = 0
       end_count = 0
-      lex_array.each_with_index do |lex, index|
-        next unless lex.type == :on_kw
-
-        case lex.token
-        when "if", "unless", "while", "until"
-          # Only count if/unless when it's not a "trailing" if/unless
-          # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375
-          kw_count += 1 unless lex.expr_label?
-        when "def", "case", "for", "begin", "class", "module", "do"
-          kw_count += 1
-        when "end"
-          end_count += 1
-        end
+      @lex.each do |lex|
+        kw_count += 1 if lex.is_kw?
+        end_count += 1 if lex.is_end?
       end
 
-      @is_comment = lex_array.detect { |lex| lex.type != :on_sp }&.type == :on_comment
-      return if @is_comment
+      kw_count -= oneliner_method_count
+
       @is_kw = (kw_count - end_count) > 0
       @is_end = (end_count - kw_count) > 0
-      @is_trailing_slash = lex_array.last.token == TRAILING_SLASH
-    end
-
-    alias_method :original, :original_line
-
-    def trailing_slash?
-      @is_trailing_slash
     end
 
+    # Used for stable sort via indentation level
+    #
+    # Ruby's sort is not "stable" meaning that when
+    # multiple elements have the same value, they are
+    # not guaranteed to return in the same order they
+    # were put in.
+    #
+    # So when multiple code lines have the same indentation
+    # level, they're sorted by their index value which is unique
+    # and consistent.
+    #
+    # This is mostly needed for consistency of the test suite
     def indent_index
       @indent_index ||= [indent, index]
     end
+    alias_method :number, :line_number
 
-    def <=>(other)
-      index <=> other.index
-    end
-
-    def is_comment?
-      @is_comment
-    end
-
-    def not_comment?
-      !is_comment?
-    end
-
+    # Returns true if the code line is determined
+    # to contain a keyword that matches with an `end`
+    #
+    # For example: `def`, `do`, `begin`, `ensure`, etc.
     def is_kw?
       @is_kw
     end
 
+    # Returns true if the code line is determined
+    # to contain an `end` keyword
     def is_end?
       @is_end
     end
 
+    # Used to hide lines
+    #
+    # The search alorithm will group lines into blocks
+    # then if those blocks are determined to represent
+    # valid code they will be hidden
     def mark_invisible
       @line = ""
-      self
-    end
-
-    def mark_visible
-      @line = @original_line
-      self
     end
 
+    # Means the line was marked as "invisible"
+    # Confusingly, "empty" lines are visible...they
+    # just don't contain any source code other than a newline ("\n").
     def visible?
       !line.empty?
     end
 
+    # Opposite or `visible?` (note: different than `empty?`)
     def hidden?
       !visible?
     end
 
-    def line_number
-      index + 1
+    # An `empty?` line is one that was originally left
+    # empty in the source code, while a "hidden" line
+    # is one that we've since marked as "invisible"
+    def empty?
+      @empty
     end
-    alias_method :number, :line_number
 
+    # Opposite of `empty?` (note: different than `visible?`)
     def not_empty?
       !empty?
     end
 
-    def empty?
-      @empty
-    end
-
+    # Renders the given line
+    #
+    # Also allows us to represent source code as
+    # an array of code lines.
+    #
+    # When we have an array of code line elements
+    # calling `join` on the array will call `to_s`
+    # on each element, which essentially converts
+    # it back into it's original source string.
     def to_s
       line
     end
+
+    # When the code line is marked invisible
+    # we retain the original value of it's line
+    # this is useful for debugging and for
+    # showing extra context
+    #
+    # DisplayCodeWithLineNumbers will render
+    # all lines given to it, not just visible
+    # lines, it uses the original method to
+    # obtain them.
+    attr_reader :original
+
+    # Comparison operator, needed for equality
+    # and sorting
+    def <=>(other)
+      index <=> other.index
+    end
+
+    # [Not stable API]
+    #
+    # Lines that have a `on_ignored_nl` type token and NOT
+    # a `BEG` type seem to be a good proxy for the ability
+    # to join multiple lines into one.
+    #
+    # This predicate method is used to determine when those
+    # two criteria have been met.
+    #
+    # The one known case this doesn't handle is:
+    #
+    #     Ripper.lex <<~EOM
+    #       a &&
+    #        b ||
+    #        c
+    #     EOM
+    #
+    # For some reason this introduces `on_ignore_newline` but with BEG type
+    def ignore_newline_not_beg?
+      lex_value = lex.detect { |l| l.type == :on_ignored_nl }
+      !!(lex_value && !lex_value.expr_beg?)
+    end
+
+    # Determines if the given line has a trailing slash
+    #
+    #     lines = CodeLine.from_source(<<~EOM)
+    #       it "foo" \
+    #     EOM
+    #     expect(lines.first.trailing_slash?).to eq(true)
+    #
+    def trailing_slash?
+      last = @lex.last
+      return false unless last
+      return false unless last.type == :on_sp
+
+      last.token == TRAILING_SLASH
+    end
+
+    # Endless method detection
+    #
+    # From https://github.com/ruby/irb/commit/826ae909c9c93a2ddca6f9cfcd9c94dbf53d44ab
+    # Detecting a "oneliner" seems to need a state machine.
+    # This can be done by looking mostly at the "state" (last value):
+    #
+    #   ENDFN -> BEG (token = '=' ) -> END
+    #
+    private def oneliner_method_count
+      oneliner_count = 0
+      in_oneliner_def = nil
+
+      @lex.each do |lex|
+        if in_oneliner_def.nil?
+          in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
+        elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
+          # Continue
+        elsif lex.state.allbits?(Ripper::EXPR_BEG)
+          in_oneliner_def = :BODY if lex.token == "="
+        elsif lex.state.allbits?(Ripper::EXPR_END)
+          # We found an endless method, count it
+          oneliner_count += 1 if in_oneliner_def == :BODY
+
+          in_oneliner_def = nil
+        else
+          in_oneliner_def = nil
+        end
+      end
+
+      oneliner_count
+    end
   end
 end

data/lib/dead_end/code_search.rb

@@ -3,11 +3,19 @@
 module DeadEnd
   # Searches code for a syntax error
   #
+  # There are three main phases in the algorithm:
+  #
+  # 1. Sanitize/format input source
+  # 2. Search for invalid blocks
+  # 3. Format invalid blocks into something meaninful
+  #
+  # This class handles the part.
+  #
   # The bulk of the heavy lifting is done in:
   #
   #  - CodeFrontier (Holds information for generating blocks and determining if we can stop searching)
   #  - ParseBlocksFromLine (Creates blocks into the frontier)
-  #  - BlockExpand (Expands existing blocks to search more code
+  #  - BlockExpand (Expands existing blocks to search more code)
   #
   # ## Syntax error detection
   #
@@ -31,28 +39,24 @@ module DeadEnd
 
     public
 
-    public
-
     attr_reader :invalid_blocks, :record_dir, :code_lines
 
     def initialize(source, record_dir: ENV["DEAD_END_RECORD_DIR"] || ENV["DEBUG"] ? "tmp" : nil)
-      @source = source
       if record_dir
         @time = Time.now.strftime("%Y-%m-%d-%H-%M-%s-%N")
         @record_dir = Pathname(record_dir).join(@time).tap { |p| p.mkpath }
         @write_count = 0
       end
-      code_lines = source.lines.map.with_index do |line, i|
-        CodeLine.new(line: line, index: i)
-      end
 
-      @code_lines = TrailingSlashJoin.new(code_lines: code_lines).call
+      @tick = 0
+      @source = source
+      @name_tick = Hash.new { |hash, k| hash[k] = 0 }
+      @invalid_blocks = []
+
+      @code_lines = CleanDocument.new(source: source).call.lines
 
       @frontier = CodeFrontier.new(code_lines: @code_lines)
-      @invalid_blocks = []
-      @name_tick = Hash.new { |hash, k| hash[k] = 0 }
-      @tick = 0
-      @block_expand = BlockExpand.new(code_lines: code_lines)
+      @block_expand = BlockExpand.new(code_lines: @code_lines)
       @parse_blocks_from_indent_line = ParseBlocksFromIndentLine.new(code_lines: @code_lines)
     end
 
@@ -63,10 +67,10 @@ module DeadEnd
       filename = "#{@write_count += 1}-#{name}-#{@name_tick[name]}.txt"
       if ENV["DEBUG"]
         puts "\n\n==== #{filename} ===="
-        puts "\n```#{block.starts_at}:#{block.ends_at}"
+        puts "\n```#{block.starts_at}..#{block.ends_at}"
         puts block.to_s
         puts "```"
-        puts "  block indent:     #{block.current_indent}"
+        puts "  block indent:      #{block.current_indent}"
       end
       @record_dir.join(filename).open(mode: "a") do |f|
         display = DisplayInvalidBlocks.new(
@@ -122,26 +126,8 @@ module DeadEnd
       push(block, name: "expand")
     end
 
-    def sweep_heredocs
-      HeredocBlockParse.new(
-        source: @source,
-        code_lines: @code_lines
-      ).call.each do |block|
-        push(block, name: "heredoc")
-      end
-    end
-
-    def sweep_comments
-      lines = @code_lines.select(&:is_comment?)
-      return if lines.empty?
-      block = CodeBlock.new(lines: lines)
-      sweep(block: block, name: "comments")
-    end
-
     # Main search loop
     def call
-      sweep_heredocs
-      sweep_comments
       until frontier.holds_all_syntax_errors?
         @tick += 1
 

data/lib/dead_end/display_code_with_line_numbers.rb

@@ -7,7 +7,6 @@ module DeadEnd
   # even if it is "marked invisible" any filtering of
   # output should be done before calling this class.
   #
-  #
   #   DisplayCodeWithLineNumbers.new(
   #     lines: lines,
   #     highlight_lines: [lines[2], lines[3]]

data/lib/dead_end/display_invalid_blocks.rb

@@ -37,8 +37,10 @@ module DeadEnd
 
     private def found_invalid_blocks
       @io.puts
-      @io.puts banner
-      @io.puts
+      if banner
+        @io.puts banner
+        @io.puts
+      end
       @io.puts("file: #{filename}") if filename
       @io.puts <<~EOM
         simplified:

data/lib/dead_end/fyi.rb

@@ -4,3 +4,5 @@ require_relative "auto"
 
 DeadEnd.send(:remove_const, :SEARCH_SOURCE_ON_ERROR_DEFAULT)
 DeadEnd::SEARCH_SOURCE_ON_ERROR_DEFAULT = false
+
+warn "DEPRECATED: calling `require 'dead_end/fyi'` is deprecated, `require 'dead_end'` instead"

data/lib/dead_end/internals.rb

@@ -1,8 +1,7 @@
 # frozen_string_literal: true
 
-#
 # This is the top level file, but is moved to `internals`
-# so the top level file can instead enable the "automatic" behavior
+# so the top level require can instead enable the "automatic" behavior
 
 require_relative "version"
 
@@ -15,7 +14,7 @@ require "timeout"
 module DeadEnd
   class Error < StandardError; end
   SEARCH_SOURCE_ON_ERROR_DEFAULT = true
-  TIMEOUT_DEFAULT = ENV.fetch("DEAD_END_TIMEOUT", 5).to_i
+  TIMEOUT_DEFAULT = ENV.fetch("DEAD_END_TIMEOUT", 1).to_i
 
   def self.handle_error(e, search_source_on_error: SEARCH_SOURCE_ON_ERROR_DEFAULT)
     raise e unless e.message.include?("end-of-input")
@@ -33,8 +32,6 @@ module DeadEnd
       )
     end
 
-    warn ""
-    warn ""
     raise e
   end
 
@@ -145,14 +142,13 @@ end
 
 require_relative "code_line"
 require_relative "code_block"
+require_relative "code_search"
 require_relative "code_frontier"
-require_relative "display_invalid_blocks"
-require_relative "around_block_scan"
-require_relative "block_expand"
-require_relative "parse_blocks_from_indent_line"
+require_relative "clean_document"
 
-require_relative "code_search"
-require_relative "who_dis_syntax_error"
-require_relative "heredoc_block_parse"
 require_relative "lex_all"
-require_relative "trailing_slash_join"
+require_relative "block_expand"
+require_relative "around_block_scan"
+require_relative "who_dis_syntax_error"
+require_relative "display_invalid_blocks"
+require_relative "parse_blocks_from_indent_line"

data/lib/dead_end/lex_all.rb

@@ -24,6 +24,10 @@ module DeadEnd
       @lex.map! { |(line, _), type, token, state| LexValue.new(line, type, token, state) }
     end
 
+    def to_a
+      @lex
+    end
+
     def each
       return @lex.each unless block_given?
       @lex.each do |x|
@@ -31,34 +35,14 @@ module DeadEnd
       end
     end
 
-    def last
-      @lex.last
+    def [](index)
+      @lex[index]
     end
 
-    # Value object for accessing lex values
-    #
-    # This lex:
-    #
-    #   [1, 0], :on_ident, "describe", CMDARG
-    #
-    # Would translate into:
-    #
-    #  lex.line # => 1
-    #  lex.type # => :on_indent
-    #  lex.token # => "describe"
-    class LexValue
-      attr_reader :line, :type, :token, :state
-
-      def initialize(line, type, token, state)
-        @line = line
-        @type = type
-        @token = token
-        @state = state
-      end
-
-      def expr_label?
-        state.allbits?(Ripper::EXPR_LABEL)
-      end
+    def last
+      @lex.last
     end
   end
 end
+
+require_relative "lex_value"

data/lib/dead_end/lex_value.rb

@@ -0,0 +1,62 @@
+module DeadEnd
+  # Value object for accessing lex values
+  #
+  # This lex:
+  #
+  #   [1, 0], :on_ident, "describe", CMDARG
+  #
+  # Would translate into:
+  #
+  #  lex.line # => 1
+  #  lex.type # => :on_indent
+  #  lex.token # => "describe"
+  class LexValue
+    attr_reader :line, :type, :token, :state
+
+    def initialize(line, type, token, state)
+      @line = line
+      @type = type
+      @token = token
+      @state = state
+
+      set_kw_end
+    end
+
+    private def set_kw_end
+      @is_end = false
+      @is_kw = false
+      return if type != :on_kw
+
+      case token
+      when "if", "unless", "while", "until"
+        # Only count if/unless when it's not a "trailing" if/unless
+        # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375
+        @is_kw = true unless expr_label?
+      when "def", "case", "for", "begin", "class", "module", "do"
+        @is_kw = true
+      when "end"
+        @is_end = true
+      end
+    end
+
+    def ignore_newline?
+      type == :on_ignored_nl
+    end
+
+    def is_end?
+      @is_end
+    end
+
+    def is_kw?
+      @is_kw
+    end
+
+    def expr_beg?
+      state.anybits?(Ripper::EXPR_BEG)
+    end
+
+    def expr_label?
+      state.allbits?(Ripper::EXPR_LABEL)
+    end
+  end
+end

data/lib/dead_end/parse_blocks_from_indent_line.rb

@@ -4,7 +4,7 @@ module DeadEnd
   # This class is responsible for generating initial code blocks
   # that will then later be expanded.
   #
-  # The biggest concern when guessing about code blocks, is accidentally
+  # The biggest concern when guessing code blocks, is accidentally
   # grabbing one that contains only an "end". In this example:
   #
   #   def dog

data/lib/dead_end/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeadEnd
-  VERSION = "1.2.0"
+  VERSION = "2.0.0"
 end

data/lib/dead_end/who_dis_syntax_error.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DeadEnd
-  # Determines what type of syntax error is in the source
+  # Determines what type of syntax error that is in the source
   #
   # Example:
   #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dead_end
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 2.0.0
 platform: ruby
 authors:
 - schneems
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-08 00:00:00.000000000 Z
+date: 2021-10-11 00:00:00.000000000 Z
 dependencies: []
 description: When you get an "unexpected end" in your syntax this gem helps you find
   it
@@ -40,6 +40,7 @@ files:
 - lib/dead_end/auto.rb
 - lib/dead_end/block_expand.rb
 - lib/dead_end/capture_code_context.rb
+- lib/dead_end/clean_document.rb
 - lib/dead_end/code_block.rb
 - lib/dead_end/code_frontier.rb
 - lib/dead_end/code_line.rb
@@ -47,11 +48,10 @@ files:
 - lib/dead_end/display_code_with_line_numbers.rb
 - lib/dead_end/display_invalid_blocks.rb
 - lib/dead_end/fyi.rb
-- lib/dead_end/heredoc_block_parse.rb
 - lib/dead_end/internals.rb
 - lib/dead_end/lex_all.rb
+- lib/dead_end/lex_value.rb
 - lib/dead_end/parse_blocks_from_indent_line.rb
-- lib/dead_end/trailing_slash_join.rb
 - lib/dead_end/version.rb
 - lib/dead_end/who_dis_syntax_error.rb
 homepage: https://github.com/zombocom/dead_end.git

data/lib/dead_end/heredoc_block_parse.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module DeadEnd
-  # Takes in a source, and returns blocks containing each heredoc
-  class HeredocBlockParse
-    private
-
-    attr_reader :code_lines, :lex
-
-    public
-
-    def initialize(source:, code_lines:)
-      @code_lines = code_lines
-      @lex = LexAll.new(source: source)
-    end
-
-    def call
-      blocks = []
-      beginning = []
-      @lex.each do |lex|
-        case lex.type
-        when :on_heredoc_beg
-          beginning << lex.line
-        when :on_heredoc_end
-          start_index = beginning.pop - 1
-          end_index = lex.line - 1
-          blocks << CodeBlock.new(lines: code_lines[start_index..end_index])
-        end
-      end
-
-      blocks
-    end
-  end
-end