syntax_search 0.1.1 → 0.2.0

data/lib/syntax_search/code_line.rb

@@ -29,7 +29,7 @@ module SyntaxErrorSearch
   # Marking a line as invisible also lets the overall program know
   # that it should not check that area for syntax errors.
   class CodeLine
-    attr_reader :line, :index, :indent
+    attr_reader :line, :index, :indent, :original_line
 
     def initialize(line: , index:)
       @original_line = line.freeze
@@ -39,6 +39,45 @@ module SyntaxErrorSearch
       @indent = SpaceCount.indent(line)
       @status = nil # valid, invalid, unknown
       @invalid = false
+
+      @kw_count = 0
+      @end_count = 0
+      @lex = LexAll.new(source: line)
+      @lex.each do |lex|
+        next unless lex.type == :on_kw
+
+        case lex.token
+        when 'def', 'case', 'for', 'begin', 'class', 'module', 'if', 'unless', 'while', 'until' , 'do'
+          @kw_count += 1
+        when 'end'
+          @end_count += 1
+        end
+      end
+
+      @is_comment = true if @lex.detect {|lex| lex.type != :on_sp}&.type == :on_comment
+
+      @is_kw = (@kw_count - @end_count) > 0
+      @is_end = (@end_count - @kw_count) > 0
+    end
+
+    def <=>(b)
+      self.index <=> b.index
+    end
+
+    def is_comment?
+      @is_comment
+    end
+
+    def not_comment?
+      !is_comment?
+    end
+
+    def is_kw?
+      @is_kw
+    end
+
+    def is_end?
+      @is_end
     end
 
     def mark_invalid
@@ -72,6 +111,8 @@ module SyntaxErrorSearch
       index + 1
     end
 
+    alias :number :line_number
+
     def not_empty?
       !empty?
     end

data/lib/syntax_search/code_search.rb

@@ -3,15 +3,16 @@
 module SyntaxErrorSearch
   # Searches code for a syntax error
   #
-  # The bulk of the heavy lifting is done by the CodeFrontier
+  # The bulk of the heavy lifting is done in:
   #
-  # The flow looks like this:
+  #  - CodeFrontier (Holds information for generating blocks and determining if we can stop searching)
+  #  - ParseBlocksFromLine (Creates blocks into the frontier)
+  #  - BlockExpand (Expands existing blocks to search more code
   #
   # ## Syntax error detection
   #
   # When the frontier holds the syntax error, we can stop searching
   #
-  #
   #   search = CodeSearch.new(<<~EOM)
   #     def dog
   #       def lol
@@ -23,42 +24,51 @@ module SyntaxErrorSearch
   #   search.invalid_blocks.map(&:to_s) # =>
   #   # => ["def lol\n"]
   #
-  #
   class CodeSearch
     private; attr_reader :frontier; public
     public; attr_reader :invalid_blocks, :record_dir, :code_lines
 
-    def initialize(string, record_dir: ENV["SYNTAX_SEARCH_RECORD_DIR"])
+    def initialize(source, record_dir: ENV["SYNTAX_SEARCH_RECORD_DIR"] || ENV["DEBUG"] ? "tmp" : nil)
+      @source = source
       if record_dir
         @time = Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')
         @record_dir = Pathname(record_dir).join(@time).tap {|p| p.mkpath }
         @write_count = 0
       end
-      @code_lines = string.lines.map.with_index do |line, i|
+      @code_lines = source.lines.map.with_index do |line, i|
         CodeLine.new(line: line, index: i)
       end
       @frontier = CodeFrontier.new(code_lines: @code_lines)
       @invalid_blocks = []
       @name_tick = Hash.new {|hash, k| hash[k] = 0 }
       @tick = 0
-      @scan = IndentScan.new(code_lines: @code_lines)
+      @block_expand = BlockExpand.new(code_lines: code_lines)
+      @parse_blocks_from_indent_line = ParseBlocksFromIndentLine.new(code_lines: @code_lines)
     end
 
+    # Used for debugging
     def record(block:, name: "record")
       return if !@record_dir
       @name_tick[name] += 1
       filename = "#{@write_count += 1}-#{name}-#{@name_tick[name]}.txt"
+      if ENV["DEBUG"]
+        puts "\n\n==== #{filename} ===="
+        puts "\n```#{block.starts_at}:#{block.ends_at}"
+        puts "#{block.to_s}"
+        puts "```"
+        puts "  block indent:     #{block.current_indent}"
+      end
       @record_dir.join(filename).open(mode: "a") do |f|
         display = DisplayInvalidBlocks.new(
           blocks: block,
-          terminal: false
+          terminal: false,
+          code_lines: @code_lines,
         )
         f.write(display.indent display.code_with_lines)
       end
     end
 
-    def push_if_invalid(block, name: )
-      frontier.register(block)
+    def push(block, name: )
       record(block: block, name: name)
 
       if block.valid?
@@ -69,33 +79,63 @@ module SyntaxErrorSearch
       end
     end
 
+    # Removes the block without putting it back in the frontier
+    def sweep(block:, name: )
+      record(block: block, name: name)
+
+      block.lines.each(&:mark_invisible)
+      frontier.register_indent_block(block)
+    end
+
+    # Parses the most indented lines into blocks that are marked
+    # and added to the frontier
     def add_invalid_blocks
       max_indent = frontier.next_indent_line&.indent
 
       while (line = frontier.next_indent_line) && (line.indent == max_indent)
-        neighbors = @scan.neighbors_from_top(frontier.next_indent_line)
 
-        @scan.each_neighbor_block(frontier.next_indent_line) do |block|
+        @parse_blocks_from_indent_line.each_neighbor_block(frontier.next_indent_line) do |block|
           record(block: block, name: "add")
-          if block.valid?
-            block.lines.each(&:mark_invisible)
-          end
-        end
 
-        block = CodeBlock.new(lines: neighbors, code_lines: @code_lines)
-        push_if_invalid(block, name: "add")
+          block.mark_invisible if block.valid?
+          push(block, name: "add")
+        end
       end
     end
 
+    # Given an already existing block in the frontier, expand it to see
+    # if it contains our invalid syntax
     def expand_invalid_block
       block = frontier.pop
       return unless block
 
-      block.expand_until_next_boundry
-      push_if_invalid(block, name: "expand")
+      record(block: block, name: "pop")
+
+      # block = block.expand_until_next_boundry
+      block = @block_expand.call(block)
+      push(block, name: "expand")
+    end
+
+    def sweep_heredocs
+      HeredocBlockParse.new(
+        source: @source,
+        code_lines: @code_lines
+      ).call.each do |block|
+        push(block, name: "heredoc")
+      end
+    end
+
+    def sweep_comments
+      lines = @code_lines.select(&:is_comment?)
+      return if lines.empty?
+      block = CodeBlock.new(lines: lines)
+      sweep(block: block, name: "comments")
     end
 
+    # Main search loop
     def call
+      sweep_heredocs
+      sweep_comments
       until frontier.holds_all_syntax_errors?
         @tick += 1
 

data/lib/syntax_search/display_code_with_line_numbers.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SyntaxErrorSearch
+  # Outputs code with highlighted lines
+  #
+  # Whatever is passed to this class will be rendered
+  # even if it is "marked invisible" any filtering of
+  # output should be done before calling this class.
+  #
+  #
+  #   DisplayCodeWithLineNumbers.new(
+  #     lines: lines,
+  #     highlight_lines: [lines[2], lines[3]]
+  #   ).call
+  #   # =>
+  #       1
+  #       2  def cat
+  #     ❯ 3    Dir.chdir
+  #     ❯ 4    end
+  #       5  end
+  #       6
+  class DisplayCodeWithLineNumbers
+    TERMINAL_HIGHLIGHT = "\e[1;3m" # Bold, italics
+    TERMINAL_END = "\e[0m"
+
+    def initialize(lines: , highlight_lines: [], terminal: false)
+      @lines = lines.sort
+      @terminal = terminal
+      @highlight_line_hash = highlight_lines.each_with_object({}) {|line, h| h[line] = true  }
+      @digit_count = @lines.last&.line_number.to_s.length
+    end
+
+    def call
+      @lines.map do |line|
+        string = String.new("")
+        if @highlight_line_hash[line]
+          string << "❯ "
+        else
+          string << "  "
+        end
+
+        number = line.line_number.to_s.rjust(@digit_count)
+        string << number.to_s
+        if line.empty?
+          string << line.original_line
+        else
+          string << "  "
+          string << TERMINAL_HIGHLIGHT if @terminal && @highlight_line_hash[line] # Bold, italics
+          string << line.original_line
+          string << TERMINAL_END if @terminal
+        end
+        string
+      end.join
+    end
+  end
+end

data/lib/syntax_search/display_invalid_blocks.rb

@@ -1,25 +1,28 @@
 # frozen_string_literal: true
 
+require_relative "capture_code_context"
+require_relative "display_code_with_line_numbers"
+
 module SyntaxErrorSearch
   # Used for formatting invalid blocks
   class DisplayInvalidBlocks
     attr_reader :filename
 
-    def initialize(blocks:, io: $stderr, filename: nil, terminal: false)
+    def initialize(code_lines: ,blocks:, io: $stderr, filename: nil, terminal: false, invalid_type: :unmatched_end)
       @terminal = terminal
       @filename = filename
       @io = io
 
       @blocks = Array(blocks)
-      @lines = @blocks.map(&:lines).flatten
-      @code_lines = @blocks.first&.code_lines || []
-      @digit_count = @code_lines.last&.line_number.to_s.length
 
-      @invalid_line_hash = @lines.each_with_object({}) {|line, h| h[line] = true  }
+      @invalid_lines = @blocks.map(&:lines).flatten
+      @code_lines = code_lines
+
+      @invalid_type = invalid_type
     end
 
     def call
-      if @blocks.any?
+      if @blocks.any? { |b| !b.hidden? }
         found_invalid_blocks
       else
         @io.puts "Syntax OK"
@@ -33,15 +36,28 @@ module SyntaxErrorSearch
     end
 
     private def found_invalid_blocks
-      @io.puts <<~EOM
+      case @invalid_type
+      when :missing_end
+        @io.puts <<~EOM
 
-        SyntaxErrorSearch: A syntax error was detected
+          SyntaxSearch: Missing `end` detected
 
-        This code has an unmatched `end` this is caused by either
-        missing a syntax keyword (`def`,  `do`, etc.) or inclusion
-        of an extra `end` line
+          This code has a missing `end`. Ensure that all
+          syntax keywords (`def`, `do`, etc.) have a matching `end`.
+
+        EOM
+      when :unmatched_end
+        @io.puts <<~EOM
+
+          SyntaxSearch: Unmatched `end` detected
+
+          This code has an unmatched `end`. Ensure that all `end` lines
+          in your code have a matching syntax keyword  (`def`,  `do`, etc.)
+          and that you don't have any extra `end` lines.
+
+        EOM
+      end
 
-      EOM
       @io.puts("file: #{filename}") if filename
       @io.puts <<~EOM
         simplified:
@@ -50,50 +66,35 @@ module SyntaxErrorSearch
       EOM
     end
 
-    def indent(string, with: "  ")
+    def indent(string, with: "    ")
       string.each_line.map {|l| with  + l }.join
     end
 
     def code_block
       string = String.new("")
-      string << "```\n"
-      # string << "#".rjust(@digit_count) + " filename: #{filename}\n\n" if filename
-      string << code_with_lines
-      string << "```\n"
+      string << code_with_context
       string
     end
 
-    def terminal_end
-      "\e[0m"
-    end
-
-    def terminal_highlight
-      "\e[1;3m" # Bold, italics
+    def code_with_context
+      lines = CaptureCodeContext.new(
+        blocks: @blocks,
+        code_lines: @code_lines
+      ).call
+
+      DisplayCodeWithLineNumbers.new(
+        lines: lines,
+        terminal: @terminal,
+        highlight_lines: @invalid_lines,
+      ).call
     end
 
     def code_with_lines
-      @code_lines.map do |line|
-        next if line.hidden?
-
-        string = String.new("")
-        if @invalid_line_hash[line]
-          string << "❯ "
-        else
-          string << "  "
-        end
-
-        number = line.line_number.to_s.rjust(@digit_count)
-        string << number.to_s
-        if line.empty?
-          string << line.to_s
-        else
-          string << "  "
-          string << terminal_highlight if @terminal && @invalid_line_hash[line] # Bold, italics
-          string << line.to_s
-          string << terminal_end if @terminal
-        end
-        string
-      end.join
+      DisplayCodeWithLineNumbers.new(
+        lines: @code_lines.select(&:visible?),
+        terminal: @terminal,
+        highlight_lines: @invalid_lines,
+      ).call
     end
   end
 end

data/lib/syntax_search/heredoc_block_parse.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SyntaxErrorSearch
+  # Takes in a source, and returns blocks containing each heredoc
+  class HeredocBlockParse
+    private; attr_reader :code_lines, :lex; public
+
+    def initialize(source:, code_lines: )
+      @code_lines = code_lines
+      @lex = LexAll.new(source: source)
+    end
+
+    def call
+      blocks = []
+      beginning = []
+      @lex.each do |lex|
+        case lex.type
+        when :on_heredoc_beg
+          beginning << lex.line
+        when :on_heredoc_end
+          start_index = beginning.pop - 1
+          end_index = lex.line - 1
+          blocks << CodeBlock.new(lines: code_lines[start_index..end_index])
+        end
+      end
+
+      blocks
+    end
+  end
+end

data/lib/syntax_search/lex_all.rb

@@ -0,0 +1,58 @@
+module SyntaxErrorSearch
+  # Ripper.lex is not guaranteed to lex the entire source document
+  #
+  # lex = LexAll.new(source: source)
+  # lex.each do |value|
+  #   puts value.line
+  # end
+  class LexAll
+    include Enumerable
+
+    def initialize(source: )
+      @lex = Ripper.lex(source)
+      lineno = @lex.last&.first&.first + 1
+      source_lines = source.lines
+      last_lineno = source_lines.count
+
+      until lineno >= last_lineno
+        lines = source_lines[lineno..-1]
+
+        @lex.concat(Ripper.lex(lines.join, '-', lineno + 1))
+        lineno = @lex.last&.first&.first + 1
+      end
+
+      @lex.map! {|(line, _), type, token| LexValue.new(line, _, type, token) }
+    end
+
+    def each
+      return @lex.each unless block_given?
+      @lex.each do |x|
+        yield x
+      end
+    end
+
+    def last
+      @lex.last
+    end
+
+    # Value object for accessing lex values
+    #
+    # This lex:
+    #
+    #   [1, 0], :on_ident, "describe", CMDARG
+    #
+    # Would translate into:
+    #
+    #  lex.line # => 1
+    #  lex.type # => :on_indent
+    #  lex.token # => "describe"
+    class LexValue
+      attr_reader :line, :type, :token
+      def initialize(line, _, type, token)
+        @line = line
+        @type = type
+        @token = token
+      end
+    end
+  end
+end