openc3 5.0.6

data/lib/openc3/utilities/ruby_lex_utils.rb

@@ -0,0 +1,568 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'irb/ruby-lex'
+require 'stringio'
+
+if RUBY_VERSION >= "2.7"
+
+  # Clear the $VERBOSE global since we're overriding methods
+  old_verbose = $VERBOSE; $VERBOSE = nil
+  class RubyLex
+    attr_accessor :indent
+    attr_accessor :line_no
+    attr_accessor :exp_line_no
+    attr_accessor :tokens
+    attr_accessor :code_block_open
+    attr_accessor :ltype
+    attr_accessor :line
+    attr_accessor :continue
+
+    def reinitialize
+      @line_no = 1
+      @prompt = nil
+      initialize_input()
+    end
+  end
+  $VERBOSE = old_verbose
+
+  class RubyLexUtils
+    # Regular expression to detect blank lines
+    BLANK_LINE_REGEX  = /^\s*$/
+    # Regular expression to detect lines containing only 'else'
+    LONELY_ELSE_REGEX = /^\s*else\s*$/
+
+    KEY_KEYWORDS = [
+      'class'.freeze,
+      'module'.freeze,
+      'def'.freeze,
+      'undef'.freeze,
+      'begin'.freeze,
+      'rescue'.freeze,
+      'ensure'.freeze,
+      'end'.freeze,
+      'if'.freeze,
+      'unless'.freeze,
+      'then'.freeze,
+      'elsif'.freeze,
+      'else'.freeze,
+      'case'.freeze,
+      'when'.freeze,
+      'while'.freeze,
+      'until'.freeze,
+      'for'.freeze,
+      'break'.freeze,
+      'next'.freeze,
+      'redo'.freeze,
+      'retry'.freeze,
+      'in'.freeze,
+      'do'.freeze,
+      'return'.freeze,
+      'alias'.freeze
+    ]
+
+    # Create a new RubyLex and StringIO to hold the text to operate on
+    def initialize
+      @lex    = RubyLex.new
+      @lex_io = StringIO.new('')
+    end
+
+    if RUBY_VERSION >= "3.0"
+      def ripper_lex_without_warning(code)
+        RubyLex.ripper_lex_without_warning(code)
+      end
+    else
+      def ripper_lex_without_warning(code)
+        @lex.ripper_lex_without_warning(code)
+      end
+    end
+
+    # @param text [String]
+    # @return [Boolean] Whether the text contains the 'begin' keyword
+    def contains_begin?(text)
+      @lex.reinitialize
+      @lex_io.string = text
+      @lex.set_input(@lex_io)
+      tokens = ripper_lex_without_warning(text)
+      tokens.each do |token|
+        if token[1] == :on_kw and token[2] == 'begin'
+          return true
+        end
+      end
+      return false
+    end
+
+    # @param text [String]
+    # @return [Boolean] Whether the text contains a Ruby keyword
+    def contains_keyword?(text)
+      @lex.reinitialize
+      @lex_io.string = text
+      @lex.set_input(@lex_io)
+      tokens = ripper_lex_without_warning(text)
+      tokens.each do |token|
+        if token[1] == :on_kw
+          if KEY_KEYWORDS.include?(token[2])
+            return true
+          end
+        elsif token[1] == :on_lbrace and !token[3].allbits?(Ripper::EXPR_BEG | Ripper::EXPR_LABEL)
+          return true
+        end
+      end
+      return false
+    end
+
+    # @param text [String]
+    # @return [Boolean] Whether the text contains a keyword which starts a block.
+    #   i.e. 'do', '{', or 'begin'
+    def contains_block_beginning?(text)
+      @lex.reinitialize
+      @lex_io.string = text
+      @lex.set_input(@lex_io)
+      tokens = ripper_lex_without_warning(text)
+      tokens.each do |token|
+        if token[1] == :on_kw
+          if token[2] == 'begin' || token[2] == 'do'
+            return true
+          end
+        elsif token[1] == :on_lbrace
+          return true
+        end
+      end
+      return false
+    end
+
+    # @param text [String]
+    # @param progress_dialog [OpenC3::ProgressDialog] If this is set, the overall
+    #   progress will be set as the processing progresses
+    # @return [String] The text with all comments removed
+    def remove_comments(text, progress_dialog = nil)
+      @lex.reinitialize
+      @lex_io.string = text
+      @lex.set_input(@lex_io)
+      comments_removed = ""
+      token_count = 0
+      progress = 0.0
+      tokens = ripper_lex_without_warning(text)
+      tokens.each do |token|
+        token_count += 1
+        if token[1] != :on_comment
+          comments_removed << token[2]
+        else
+          newline_count = token[2].count("\n")
+          comments_removed << ("\n" * newline_count)
+        end
+        if progress_dialog and token_count % 10000 == 0
+          progress += 0.01
+          progress = 0.0 if progress >= 0.99
+          progress_dialog.set_overall_progress(progress)
+        end
+      end
+
+      return comments_removed
+    end
+
+    # Yields each lexed segment and if the segment is instrumentable
+    #
+    # @param text [String]
+    # @yieldparam line [String] The entire line
+    # @yieldparam instrumentable [Boolean] Whether the line is instrumentable
+    # @yieldparam inside_begin [Integer] The level of indentation
+    # @yieldparam line_no [Integer] The current line number
+    def each_lexed_segment(text)
+      inside_begin = false
+      indent = 0
+      lex = RubyLex.new
+      lex_io = StringIO.new(text)
+      lex.set_input(lex_io)
+      lex.line = ''
+      while lexed = lex.lex
+        lex.line_no += lexed.count("\n")
+        lex.line.concat lexed
+        next if lex.ltype or lex.continue
+
+        # Detect the beginning and end of begin blocks so we can not catch exceptions there
+        if indent == 0 and contains_begin?(lex.line)
+          inside_begin = true
+          indent = lex.indent
+        else
+          indent += lex.indent if indent > 0
+        end
+
+        if inside_begin and indent <= 0
+          indent = 0
+          inside_begin = false
+        end
+
+        loop do # loop to allow restarting for nested conditions
+          # Yield blank lines and lonely else lines before the actual line
+          while (index = lex.line.index("\n"))
+            line = lex.line[0..index]
+            if BLANK_LINE_REGEX.match?(line)
+              yield line, true, inside_begin, lex.exp_line_no
+              lex.exp_line_no += 1
+              lex.line = lex.line[(index + 1)..-1]
+            elsif LONELY_ELSE_REGEX.match?(line)
+              yield line, false, inside_begin, lex.exp_line_no
+              lex.exp_line_no += 1
+              lex.line = lex.line[(index + 1)..-1]
+            else
+              break
+            end
+          end
+
+          if contains_keyword?(lex.line)
+            if contains_block_beginning?(lex.line)
+              section = ''
+              lex.line.each_line do |lexed_part|
+                section << lexed_part
+                if contains_block_beginning?(section)
+                  yield section, false, inside_begin, lex.exp_line_no
+                  break
+                end
+                lex.exp_line_no += 1
+              end
+              lex.exp_line_no += 1
+              remainder = lex.line[(section.length)..-1]
+              lex.line = remainder
+              next unless remainder.empty?
+            else
+              yield lex.line, false, inside_begin, lex.exp_line_no
+            end
+          elsif !lex.line.empty?
+            num_left_brackets  = lex.line.count('{')
+            num_right_brackets = lex.line.count('}')
+            if num_left_brackets != num_right_brackets
+              # Don't instrument lines with unequal numbers of { and } brackets
+              yield lex.line, false, inside_begin, lex.exp_line_no
+            else
+              yield lex.line, true, inside_begin, lex.exp_line_no
+            end
+          end
+          lex.line = ''
+          lex.exp_line_no = lex.line_no
+          lex.indent = 0
+          break
+        end # loop do
+      end # while lexed
+    end # def each_lexed_segment
+  end
+
+else
+
+  # Clear the $VERBOSE global since we're overriding methods
+  old_verbose = $VERBOSE; $VERBOSE = nil
+  class RubyLex
+    if self.method_defined?(:indent)
+      attr_writer :indent
+    else
+      attr_accessor :indent
+    end
+    # @return [Integer] The expression line number. This can differ from the
+    #   actual line number due to white space and Ruby control keywords.
+    attr_accessor :exp_line_no
+
+    # Resets the RubyLex in preparation of parsing a line
+    def reinitialize
+      @seek                      = 0
+      @exp_line_no               = 1
+      @line_no                   = 1
+      @base_char_no              = 0
+      @char_no                   = 0
+      @rests.clear
+      @readed.clear
+      @here_readed.clear
+      @indent = 0
+      @indent_stack.clear
+      @lex_state                 = EXPR_BEG
+      @space_seen                = false
+      @here_header               = false
+      @continue                  = false
+      @line                      = ''
+      @skip_space                = false
+      @readed_auto_clean_up      = false
+      @exception_on_syntax_error = true
+      @prompt                    = nil
+    end
+
+    # Monkey patch to keep this from looping forever if the string never is closed with a right brace
+    def identify_string_dvar
+      getc
+
+      reserve_continue = @continue
+      reserve_ltype = @ltype
+      reserve_indent = @indent
+      reserve_indent_stack = @indent_stack
+      reserve_state = @lex_state
+      reserve_quoted = @quoted
+
+      @ltype = nil
+      @quoted = nil
+      @indent = 0
+      @indent_stack = []
+      @lex_state = EXPR_BEG
+
+      loop do
+        @continue = false
+        prompt
+        tk = token
+        break if tk.nil? # This is the patch
+        if @ltype or @continue or @indent >= 0
+          next
+        end
+        break if tk.kind_of?(TkRBRACE)
+      end
+    ensure
+      @continue = reserve_continue
+      @ltype = reserve_ltype
+      @indent = reserve_indent
+      @indent_stack = reserve_indent_stack
+      @lex_state = reserve_state
+      @quoted = reserve_quoted
+    end
+  end
+  $VERBOSE = old_verbose
+
+  class RubyLexUtils
+    # Regular expression to detect blank lines
+    BLANK_LINE_REGEX  = /^\s*$/
+    # Regular expression to detect lines containing only 'else'
+    LONELY_ELSE_REGEX = /^\s*else\s*$/
+
+    # Ruby keywords
+    KEYWORD_TOKENS = [RubyToken::TkCLASS,
+                      RubyToken::TkMODULE,
+                      RubyToken::TkDEF,
+                      RubyToken::TkUNDEF,
+                      RubyToken::TkBEGIN,
+                      RubyToken::TkRESCUE,
+                      RubyToken::TkENSURE,
+                      RubyToken::TkEND,
+                      RubyToken::TkIF,
+                      RubyToken::TkUNLESS,
+                      RubyToken::TkTHEN,
+                      RubyToken::TkELSIF,
+                      RubyToken::TkELSE,
+                      RubyToken::TkCASE,
+                      RubyToken::TkWHEN,
+                      RubyToken::TkWHILE,
+                      RubyToken::TkUNTIL,
+                      RubyToken::TkFOR,
+                      RubyToken::TkBREAK,
+                      RubyToken::TkNEXT,
+                      RubyToken::TkREDO,
+                      RubyToken::TkRETRY,
+                      RubyToken::TkIN,
+                      RubyToken::TkDO,
+                      RubyToken::TkRETURN,
+                      RubyToken::TkIF_MOD,
+                      RubyToken::TkUNLESS_MOD,
+                      RubyToken::TkWHILE_MOD,
+                      RubyToken::TkUNTIL_MOD,
+                      RubyToken::TkALIAS,
+                      RubyToken::TklBEGIN,
+                      RubyToken::TklEND,
+                      RubyToken::TkfLBRACE]
+
+    # Ruby keywords which define the beginning of a block: do, {, begin
+    BLOCK_BEGINNING_TOKENS = [RubyToken::TkDO,
+                              RubyToken::TkfLBRACE,
+                              RubyToken::TkBEGIN]
+
+    # Create a new RubyLex and StringIO to hold the text to operate on
+    def initialize
+      @lex    = RubyLex.new
+      @lex_io = StringIO.new('')
+    end
+
+    # @param text [String]
+    # @return [Boolean] Whether the text contains the 'begin' keyword
+    def contains_begin?(text)
+      @lex.reinitialize
+      @lex.exception_on_syntax_error = false
+      @lex_io.string = text
+      @lex.set_input(@lex_io)
+      while token = @lex.token
+        if token.class == RubyToken::TkBEGIN
+          return true
+        end
+      end
+      return false
+    end
+
+    # @param text [String]
+    # @return [Boolean] Whether the text contains a Ruby keyword
+    def contains_keyword?(text)
+      @lex.reinitialize
+      @lex.exception_on_syntax_error = false
+      @lex_io.string = text
+      @lex.set_input(@lex_io)
+      while token = @lex.token
+        if KEYWORD_TOKENS.include?(token.class)
+          return true
+        end
+      end
+      return false
+    end
+
+    # @param text [String]
+    # @return [Boolean] Whether the text contains a keyword which starts a block.
+    #   i.e. 'do', '{', or 'begin'
+    def contains_block_beginning?(text)
+      @lex.reinitialize
+      @lex.exception_on_syntax_error = false
+      @lex_io.string = text
+      @lex.set_input(@lex_io)
+      while token = @lex.token
+        if BLOCK_BEGINNING_TOKENS.include?(token.class)
+          return true
+        end
+      end
+      return false
+    end
+
+    # @param text [String]
+    # @param progress_dialog [OpenC3::ProgressDialog] If this is set, the overall
+    #   progress will be set as the processing progresses
+    # @return [String] The text with all comments removed
+    def remove_comments(text, progress_dialog = nil)
+      comments_removed = text.clone
+      @lex.reinitialize
+      @lex.exception_on_syntax_error = false
+      @lex_io.string = text
+      @lex.set_input(@lex_io)
+      need_remove = nil
+      delete_ranges = []
+      token_count = 0
+      progress = 0.0
+      while token = @lex.token
+        token_count += 1
+        if need_remove
+          delete_ranges << (need_remove..(token.seek - 1))
+          need_remove = nil
+        end
+        if token.class == RubyToken::TkCOMMENT
+          need_remove = token.seek
+        end
+        if progress_dialog and token_count % 10000 == 0
+          progress += 0.01
+          progress = 0.0 if progress >= 0.99
+          progress_dialog.set_overall_progress(progress)
+        end
+      end
+
+      if need_remove
+        delete_ranges << (need_remove..(text.length - 1))
+        need_remove = nil
+      end
+
+      delete_count = 0
+      delete_ranges.reverse_each do |range|
+        delete_count += 1
+        comments_removed[range] = ''
+        if progress_dialog and delete_count % 10000 == 0
+          progress += 0.01
+          progress = 0.0 if progress >= 0.99
+          progress_dialog.set_overall_progress(progress)
+        end
+      end
+
+      return comments_removed
+    end
+
+    # Yields each lexed segment and if the segment is instrumentable
+    #
+    # @param text [String]
+    # @yieldparam line [String] The entire line
+    # @yieldparam instrumentable [Boolean] Whether the line is instrumentable
+    # @yieldparam inside_begin [Integer] The level of indentation
+    # @yieldparam line_no [Integer] The current line number
+    def each_lexed_segment(text)
+      inside_begin = false
+      inside_indent = nil
+      lex = RubyLex.new
+      lex.exception_on_syntax_error = false
+      lex_io = StringIO.new(text)
+      lex.set_input(lex_io)
+
+      while lexed = lex.lex
+        line_no = lex.exp_line_no
+
+        if inside_indent.nil? and contains_begin?(lexed)
+          inside_indent = lex.indent - 1
+          inside_begin = true
+        end
+
+        if lex.indent == inside_indent
+          inside_indent = nil
+          inside_begin = false
+        end
+
+        loop do # loop to allow restarting for nested conditions
+          # Yield blank lines and lonely else lines before the actual line
+          while (index = lexed.index("\n"))
+            line = lexed[0..index]
+            if BLANK_LINE_REGEX.match?(line)
+              yield line, true, inside_begin, line_no
+              line_no += 1
+              lexed = lexed[(index + 1)..-1]
+            elsif LONELY_ELSE_REGEX.match?(line)
+              yield line, false, inside_begin, line_no
+              line_no += 1
+              lexed = lexed[(index + 1)..-1]
+            else
+              break
+            end
+          end
+
+          if contains_keyword?(lexed)
+            if contains_block_beginning?(lexed)
+              section = ''
+              lexed.each_line do |lexed_part|
+                section << lexed_part
+                if contains_block_beginning?(section)
+                  yield section, false, inside_begin, line_no
+                  break
+                end
+                line_no += 1
+              end
+              line_no += 1
+              remainder = lexed[(section.length)..-1]
+              lexed = remainder
+              next unless remainder.empty?
+            else
+              yield lexed, false, inside_begin, line_no
+            end
+          elsif !lexed.empty?
+            num_left_brackets  = lexed.count('{')
+            num_right_brackets = lexed.count('}')
+            if num_left_brackets != num_right_brackets
+              # Don't instrument lines with unequal numbers of { and } brackets
+              yield lexed, false, inside_begin, line_no
+            else
+              yield lexed, true, inside_begin, line_no
+            end
+          end
+          lex.exp_line_no = lex.line_no
+          break
+        end # loop do
+      end # while lexed
+    end # def each_lexed_segment
+  end
+
+end  # RUBY_VERSION < "2.7"