syntax_search 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77f054eda8b4e25dcaf0306818ae17bb1ff3da9c52f2d606141ff44cd343c51d
-  data.tar.gz: 44f4347d5078d0250286e78a08ff730b02094aeb05897c9088963381a95e9071
+  metadata.gz: 66acadf6481512affd39e4a1fb691ed41a6be537a8a63801f71316863d30494d
+  data.tar.gz: 1bcb85706516cbe5c3f53a1ed48bf5215479672952fc62d2e3221c583e468b22
 SHA512:
-  metadata.gz: 6ccc7e2ed620689614bdfae8943b8c15b81b4567d3b42e2c734ac7974699ed231272ce52929eff9d49eb25edd843ae3f7fa2398d0552b77665be779a4412461f
-  data.tar.gz: 1e866b4aaafb534f946da5e00efb50536a4d20bf0e1527f8641fb298cca5cc6b7d0e1f744f63da443b9920500d7d4ddf0113ffa8bde2b4689cd6793538a12b6e
+  metadata.gz: 4e60777bd284d78436db8608f34d11b4b5aa8456bfe0c9507ddf3a0e3b97b902a2d00d1537433ed914c637a0729ccb253fc9047fb6db9e3b18a8cc8d0294ae06
+  data.tar.gz: 2fb1b35e70507f4b358be4b767cbf266c2d47c55efba1b77d7ca5f260cac4e77d519fc7439f55c78c5294c0c87240d15b4cab3721c8cfd580b518f5051b7cd0c

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## HEAD (unreleased)
 
+## 0.2.0
+
+- Simplify large file output so minimal context around the invalid section is shown (https://github.com/zombocom/syntax_search/pull/26)
+- Block expansion is now lexically aware of keywords (def/do/end etc.) (https://github.com/zombocom/syntax_search/pull/24)
+- Fix bug where not all of a source is lexed which is used in heredoc detection/removal (https://github.com/zombocom/syntax_search/pull/23)
+
 ## 0.1.5
 
 - Strip out heredocs in documents first (https://github.com/zombocom/syntax_search/pull/19)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    syntax_search (0.1.5)
+    syntax_search (0.2.0)
 
 GEM
   remote: https://rubygems.org/

data/exe/syntax_search

@@ -62,6 +62,7 @@ if file.nil? || file.empty?
 end
 
 file = Pathname(file)
+options[:record_dir] = "tmp" if ENV["DEBUG"]
 
 $stderr.puts "Record dir: #{options[:record_dir]}"  if options[:record_dir]
 

data/lib/syntax_search.rb

@@ -142,3 +142,4 @@ require_relative "syntax_search/parse_blocks_from_indent_line"
 require_relative "syntax_search/code_search"
 require_relative "syntax_search/who_dis_syntax_error"
 require_relative "syntax_search/heredoc_block_parse"
+require_relative "syntax_search/lex_all"

data/lib/syntax_search/around_block_scan.rb

@@ -32,9 +32,11 @@ module SyntaxErrorSearch
       @code_lines = code_lines
       @orig_before_index = block.lines.first.index
       @orig_after_index = block.lines.last.index
+      @orig_indent = block.current_indent
       @skip_array = []
       @after_array = []
       @before_array = []
+      @stop_after_kw = false
     end
 
     def skip(name)
@@ -42,29 +44,129 @@ module SyntaxErrorSearch
       self
     end
 
+    def stop_after_kw
+      @stop_after_kw = true
+      self
+    end
+
     def scan_while(&block)
+      stop_next = false
+
+      kw_count = 0
+      end_count = 0
       @before_index = before_lines.reverse_each.take_while do |line|
+        next false if stop_next
         next true if @skip_array.detect {|meth| line.send(meth) }
 
+        kw_count += 1 if line.is_kw?
+        end_count += 1 if line.is_end?
+        if @stop_after_kw && kw_count > end_count
+          stop_next = true
+        end
+
         block.call(line)
       end.reverse.first&.index
 
+      stop_next = false
+      kw_count = 0
+      end_count = 0
       @after_index = after_lines.take_while do |line|
+        next false if stop_next
         next true if @skip_array.detect {|meth| line.send(meth) }
 
+        kw_count += 1 if line.is_kw?
+        end_count += 1 if line.is_end?
+        if @stop_after_kw && end_count > kw_count
+          stop_next = true
+        end
+
         block.call(line)
       end.last&.index
       self
     end
 
+    def capture_neighbor_context
+      lines = []
+      kw_count = 0
+      end_count = 0
+      before_lines.reverse.each do |line|
+        next if line.empty?
+        break if line.indent < @orig_indent
+        next if line.indent != @orig_indent
+
+        kw_count += 1 if line.is_kw?
+        end_count += 1 if line.is_end?
+        if kw_count != 0 && kw_count == end_count
+          lines << line
+          break
+        end
+
+        lines << line
+      end
+
+      lines.reverse!
+
+      kw_count = 0
+      end_count = 0
+      after_lines.each do |line|
+        # puts "line: #{line.number} #{line.original_line}, indent: #{line.indent}, #{line.empty?} #{line.indent == @orig_indent}"
+
+        next if line.empty?
+        break if line.indent < @orig_indent
+        next if line.indent != @orig_indent
+
+        kw_count += 1 if line.is_kw?
+        end_count += 1 if line.is_end?
+        if kw_count != 0 && kw_count == end_count
+          lines << line
+          break
+        end
+
+        lines << line
+      end
+      lines.select! {|line| !line.is_comment? }
+
+      lines
+    end
+
+    def on_falling_indent
+      last_indent = @orig_indent
+      before_lines.reverse.each do |line|
+        next if line.empty?
+        if line.indent < last_indent
+          yield line
+          last_indent = line.indent
+        end
+      end
+
+      last_indent = @orig_indent
+      after_lines.each do |line|
+        next if line.empty?
+        if line.indent < last_indent
+          yield line
+          last_indent = line.indent
+        end
+      end
+    end
+
+    def scan_neighbors
+      self.scan_while {|line| line.not_empty? && line.indent >= @orig_indent }
+    end
+
     def scan_adjacent_indent
       before_indent = @code_lines[@orig_before_index.pred]&.indent || 0
       after_indent = @code_lines[@orig_after_index.next]&.indent || 0
 
       indent = [before_indent, after_indent].min
-      @before_index = before_index.pred if before_indent >= indent
-      @after_index = after_index.next if after_indent >= indent
+      self.scan_while {|line| line.not_empty? && line.indent >= indent }
+
+      self
+    end
 
+    def start_at_next_line
+      before_index; after_index
+      @before_index -= 1
+      @after_index += 1
       self
     end
 
@@ -73,11 +175,11 @@ module SyntaxErrorSearch
     end
 
     def before_index
-      @before_index || @orig_before_index
+      @before_index ||= @orig_before_index
     end
 
     def after_index
-      @after_index || @orig_after_index
+      @after_index ||= @orig_after_index
     end
 
     private def before_lines

data/lib/syntax_search/block_expand.rb

@@ -44,21 +44,17 @@ module SyntaxErrorSearch
 
     def expand_indent(block)
       block = AroundBlockScan.new(code_lines: @code_lines, block: block)
+        .skip(:hidden?)
+        .stop_after_kw
         .scan_adjacent_indent
         .code_block
-
-      # Handle if/else/end case
-      if (next_block = expand_neighbors(block, grab_empty: false))
-        return next_block
-      else
-        return block
-      end
     end
 
     def expand_neighbors(block, grab_empty: true)
       scan = AroundBlockScan.new(code_lines: @code_lines, block: block)
         .skip(:hidden?)
-        .scan_while {|line| line.not_empty? && line.indent >= block.current_indent }
+        .stop_after_kw
+        .scan_neighbors
 
       # Slurp up empties
       if grab_empty

data/lib/syntax_search/capture_code_context.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module SyntaxErrorSearch
+
+  # Given a block, this method will capture surrounding
+  # code to give the user more context for the location of
+  # the problem.
+  #
+  # Return is an array of CodeLines to be rendered.
+  #
+  # Surrounding code is captured regardless of visible state
+  #
+  #   puts block.to_s # => "def bark"
+  #
+  #   context = CaptureCodeContext.new(
+  #     blocks: block,
+  #     code_lines: code_lines
+  #   )
+  #
+  #   puts context.call.join
+  #   # =>
+  #     class Dog
+  #       def bark
+  #     end
+  #
+  class CaptureCodeContext
+    attr_reader :code_lines
+
+    def initialize(blocks: , code_lines:)
+      @blocks = Array(blocks)
+      @code_lines = code_lines
+      @visible_lines = @blocks.map(&:visible_lines).flatten
+      @lines_to_output = @visible_lines.dup
+    end
+
+    def call
+      @blocks.each do |block|
+        around_lines = AroundBlockScan.new(code_lines: @code_lines, block: block)
+          .start_at_next_line
+          .capture_neighbor_context
+
+        around_lines -= block.lines
+
+        @lines_to_output.concat(around_lines)
+
+        AroundBlockScan.new(
+          block: block,
+          code_lines: @code_lines,
+        ).on_falling_indent do |line|
+          @lines_to_output << line
+        end
+      end
+
+      @lines_to_output.select!(&:not_empty?)
+      @lines_to_output.select!(&:not_comment?)
+      @lines_to_output.uniq!
+      @lines_to_output.sort!
+
+      return @lines_to_output
+    end
+  end
+end

data/lib/syntax_search/code_block.rb

@@ -23,6 +23,10 @@ module SyntaxErrorSearch
       @lines = Array(lines)
     end
 
+    def visible_lines
+      @lines.select(&:visible?).select(&:not_empty?)
+    end
+
     def mark_invisible
       @lines.map(&:mark_invisible)
     end
@@ -48,7 +52,11 @@ module SyntaxErrorSearch
     # populate an array with multiple code blocks then call `sort!`
     # on it without having to specify the sorting criteria
     def <=>(other)
-      self.current_indent <=> other.current_indent
+      out = self.current_indent <=> other.current_indent
+      return out if out != 0
+
+      # Stable sort
+      self.starts_at <=> other.starts_at
     end
 
     def current_indent

data/lib/syntax_search/code_line.rb

@@ -29,7 +29,7 @@ module SyntaxErrorSearch
   # Marking a line as invisible also lets the overall program know
   # that it should not check that area for syntax errors.
   class CodeLine
-    attr_reader :line, :index, :indent
+    attr_reader :line, :index, :indent, :original_line
 
     def initialize(line: , index:)
       @original_line = line.freeze
@@ -39,6 +39,45 @@ module SyntaxErrorSearch
       @indent = SpaceCount.indent(line)
       @status = nil # valid, invalid, unknown
       @invalid = false
+
+      @kw_count = 0
+      @end_count = 0
+      @lex = LexAll.new(source: line)
+      @lex.each do |lex|
+        next unless lex.type == :on_kw
+
+        case lex.token
+        when 'def', 'case', 'for', 'begin', 'class', 'module', 'if', 'unless', 'while', 'until' , 'do'
+          @kw_count += 1
+        when 'end'
+          @end_count += 1
+        end
+      end
+
+      @is_comment = true if @lex.detect {|lex| lex.type != :on_sp}&.type == :on_comment
+
+      @is_kw = (@kw_count - @end_count) > 0
+      @is_end = (@end_count - @kw_count) > 0
+    end
+
+    def <=>(b)
+      self.index <=> b.index
+    end
+
+    def is_comment?
+      @is_comment
+    end
+
+    def not_comment?
+      !is_comment?
+    end
+
+    def is_kw?
+      @is_kw
+    end
+
+    def is_end?
+      @is_end
     end
 
     def mark_invalid
@@ -72,6 +111,8 @@ module SyntaxErrorSearch
       index + 1
     end
 
+    alias :number :line_number
+
     def not_empty?
       !empty?
     end

data/lib/syntax_search/code_search.rb

@@ -28,7 +28,7 @@ module SyntaxErrorSearch
     private; attr_reader :frontier; public
     public; attr_reader :invalid_blocks, :record_dir, :code_lines
 
-    def initialize(source, record_dir: ENV["SYNTAX_SEARCH_RECORD_DIR"])
+    def initialize(source, record_dir: ENV["SYNTAX_SEARCH_RECORD_DIR"] || ENV["DEBUG"] ? "tmp" : nil)
       @source = source
       if record_dir
         @time = Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')
@@ -79,6 +79,14 @@ module SyntaxErrorSearch
       end
     end
 
+    # Removes the block without putting it back in the frontier
+    def sweep(block:, name: )
+      record(block: block, name: name)
+
+      block.lines.each(&:mark_invisible)
+      frontier.register_indent_block(block)
+    end
+
     # Parses the most indented lines into blocks that are marked
     # and added to the frontier
     def add_invalid_blocks
@@ -108,7 +116,6 @@ module SyntaxErrorSearch
       push(block, name: "expand")
     end
 
-
     def sweep_heredocs
       HeredocBlockParse.new(
         source: @source,
@@ -118,9 +125,17 @@ module SyntaxErrorSearch
       end
     end
 
+    def sweep_comments
+      lines = @code_lines.select(&:is_comment?)
+      return if lines.empty?
+      block = CodeBlock.new(lines: lines)
+      sweep(block: block, name: "comments")
+    end
+
     # Main search loop
     def call
       sweep_heredocs
+      sweep_comments
       until frontier.holds_all_syntax_errors?
         @tick += 1
 

data/lib/syntax_search/display_code_with_line_numbers.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SyntaxErrorSearch
+  # Outputs code with highlighted lines
+  #
+  # Whatever is passed to this class will be rendered
+  # even if it is "marked invisible" any filtering of
+  # output should be done before calling this class.
+  #
+  #
+  #   DisplayCodeWithLineNumbers.new(
+  #     lines: lines,
+  #     highlight_lines: [lines[2], lines[3]]
+  #   ).call
+  #   # =>
+  #       1
+  #       2  def cat
+  #     ❯ 3    Dir.chdir
+  #     ❯ 4    end
+  #       5  end
+  #       6
+  class DisplayCodeWithLineNumbers
+    TERMINAL_HIGHLIGHT = "\e[1;3m" # Bold, italics
+    TERMINAL_END = "\e[0m"
+
+    def initialize(lines: , highlight_lines: [], terminal: false)
+      @lines = lines.sort
+      @terminal = terminal
+      @highlight_line_hash = highlight_lines.each_with_object({}) {|line, h| h[line] = true  }
+      @digit_count = @lines.last&.line_number.to_s.length
+    end
+
+    def call
+      @lines.map do |line|
+        string = String.new("")
+        if @highlight_line_hash[line]
+          string << "❯ "
+        else
+          string << "  "
+        end
+
+        number = line.line_number.to_s.rjust(@digit_count)
+        string << number.to_s
+        if line.empty?
+          string << line.original_line
+        else
+          string << "  "
+          string << TERMINAL_HIGHLIGHT if @terminal && @highlight_line_hash[line] # Bold, italics
+          string << line.original_line
+          string << TERMINAL_END if @terminal
+        end
+        string
+      end.join
+    end
+  end
+end

data/lib/syntax_search/display_invalid_blocks.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "capture_code_context"
+require_relative "display_code_with_line_numbers"
+
 module SyntaxErrorSearch
   # Used for formatting invalid blocks
   class DisplayInvalidBlocks
@@ -11,11 +14,10 @@ module SyntaxErrorSearch
       @io = io
 
       @blocks = Array(blocks)
-      @lines = @blocks.map(&:lines).flatten
+
+      @invalid_lines = @blocks.map(&:lines).flatten
       @code_lines = code_lines
-      @digit_count = @code_lines.last&.line_number.to_s.length
 
-      @invalid_line_hash = @lines.each_with_object({}) {|line, h| h[line] = true  }
       @invalid_type = invalid_type
     end
 
@@ -70,41 +72,29 @@ module SyntaxErrorSearch
 
     def code_block
       string = String.new("")
-      string << code_with_lines
+      string << code_with_context
       string
     end
 
-    def terminal_end
-      "\e[0m"
-    end
-
-    def terminal_highlight
-      "\e[1;3m" # Bold, italics
+    def code_with_context
+      lines = CaptureCodeContext.new(
+        blocks: @blocks,
+        code_lines: @code_lines
+      ).call
+
+      DisplayCodeWithLineNumbers.new(
+        lines: lines,
+        terminal: @terminal,
+        highlight_lines: @invalid_lines,
+      ).call
     end
 
     def code_with_lines
-      @code_lines.map do |line|
-        next if line.hidden?
-
-        string = String.new("")
-        if @invalid_line_hash[line]
-          string << "❯ "
-        else
-          string << "  "
-        end
-
-        number = line.line_number.to_s.rjust(@digit_count)
-        string << number.to_s
-        if line.empty?
-          string << line.to_s
-        else
-          string << "  "
-          string << terminal_highlight if @terminal && @invalid_line_hash[line] # Bold, italics
-          string << line.to_s
-          string << terminal_end if @terminal
-        end
-        string
-      end.join
+      DisplayCodeWithLineNumbers.new(
+        lines: @code_lines.select(&:visible?),
+        terminal: @terminal,
+        highlight_lines: @invalid_lines,
+      ).call
     end
   end
 end

data/lib/syntax_search/heredoc_block_parse.rb

@@ -7,19 +7,19 @@ module SyntaxErrorSearch
 
     def initialize(source:, code_lines: )
       @code_lines = code_lines
-      @lex = Ripper.lex(source)
+      @lex = LexAll.new(source: source)
     end
 
     def call
       blocks = []
       beginning = []
-      @lex.each do |(line, col), event, *_|
-        case event
+      @lex.each do |lex|
+        case lex.type
         when :on_heredoc_beg
-          beginning << line
+          beginning << lex.line
         when :on_heredoc_end
           start_index = beginning.pop - 1
-          end_index = line - 1
+          end_index = lex.line - 1
           blocks << CodeBlock.new(lines: code_lines[start_index..end_index])
         end
       end

data/lib/syntax_search/lex_all.rb

@@ -0,0 +1,58 @@
+module SyntaxErrorSearch
+  # Ripper.lex is not guaranteed to lex the entire source document
+  #
+  # lex = LexAll.new(source: source)
+  # lex.each do |value|
+  #   puts value.line
+  # end
+  class LexAll
+    include Enumerable
+
+    def initialize(source: )
+      @lex = Ripper.lex(source)
+      lineno = @lex.last&.first&.first + 1
+      source_lines = source.lines
+      last_lineno = source_lines.count
+
+      until lineno >= last_lineno
+        lines = source_lines[lineno..-1]
+
+        @lex.concat(Ripper.lex(lines.join, '-', lineno + 1))
+        lineno = @lex.last&.first&.first + 1
+      end
+
+      @lex.map! {|(line, _), type, token| LexValue.new(line, _, type, token) }
+    end
+
+    def each
+      return @lex.each unless block_given?
+      @lex.each do |x|
+        yield x
+      end
+    end
+
+    def last
+      @lex.last
+    end
+
+    # Value object for accessing lex values
+    #
+    # This lex:
+    #
+    #   [1, 0], :on_ident, "describe", CMDARG
+    #
+    # Would translate into:
+    #
+    #  lex.line # => 1
+    #  lex.type # => :on_indent
+    #  lex.token # => "describe"
+    class LexValue
+      attr_reader :line, :type, :token
+      def initialize(line, _, type, token)
+        @line = line
+        @type = type
+        @token = token
+      end
+    end
+  end
+end

data/lib/syntax_search/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SyntaxErrorSearch
-  VERSION = "0.1.5"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: syntax_search
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - schneems
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-12-02 00:00:00.000000000 Z
+date: 2020-12-07 00:00:00.000000000 Z
 dependencies: []
 description: When you get an "unexpected end" in your syntax this gem helps you find
   it
@@ -39,13 +39,16 @@ files:
 - lib/syntax_search/around_block_scan.rb
 - lib/syntax_search/auto.rb
 - lib/syntax_search/block_expand.rb
+- lib/syntax_search/capture_code_context.rb
 - lib/syntax_search/code_block.rb
 - lib/syntax_search/code_frontier.rb
 - lib/syntax_search/code_line.rb
 - lib/syntax_search/code_search.rb
+- lib/syntax_search/display_code_with_line_numbers.rb
 - lib/syntax_search/display_invalid_blocks.rb
 - lib/syntax_search/fyi.rb
 - lib/syntax_search/heredoc_block_parse.rb
+- lib/syntax_search/lex_all.rb
 - lib/syntax_search/parse_blocks_from_indent_line.rb
 - lib/syntax_search/version.rb
 - lib/syntax_search/who_dis_syntax_error.rb