loxxy 0.4.07 → 0.4.08

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7cd3e438270c62a4863cddfd1b737182993ea205d84985e8862a25c74bb2e8c4
-  data.tar.gz: f6bd428ebc505e238392cbb04e5fb1207eb34752f974ae8f81a23798e4ef51b5
+  metadata.gz: e7e447625d999f4b95129d7feff1586c6759cb929a8faaa82e3ecc94105d0268
+  data.tar.gz: 00a4801454ffd09b2d4e810fd15d0f9b40fea13f46e6d4eed7eba4cab8d35ebe
 SHA512:
-  metadata.gz: 77ac1d09cb77b8afab98c21bb2f8544fd2961473efd1acaae3134a2aba0ed7138270aa86f384fe2ce1f76d865c8cd049dc3b8379b954c70992cc53b068244593
-  data.tar.gz: 9fa0e0c3c3d96600bc4e8cd0447383a287be6ff483fb2a843056c87175239bc5fd319e0f11cfe8ab496652a773c91d91f8c05176275b07873f08ffad0de7a1f4
+  metadata.gz: 4a59f746cb7b8086ff3e1252c3fc0846229cb6927dcc4d62ad261c5e0c2fec4b1a75176cab6788cedb5c57a2fe9e7bec94d92d7dc63a2c455609a18761f78043
+  data.tar.gz: 66f72710a3f422e4431c88c44b56419f727711c1fbf339491f1fcb2f8f18edb0f4a21ce49a66ee4325ccf3b2576b0b94490d74f96c922a9d340d476e43660881

data/CHANGELOG.md

@@ -1,4 +1,10 @@
-## [0.4.07] - 2022-11-21
+## [0.4.08] - 2022-04-09
+- Refactoring of the `Scanner` class.
+
+### Changed
+- `Frontend::Scanner` class: major code refactoring. 
+
+## [0.4.07] - 2021-11-21
 - Minor fixes; dependency towards Rubies 3+ allowed...
 
 ### Fixed

data/lib/loxxy/front_end/scanner.rb

@@ -20,6 +20,15 @@ module Loxxy
     # Delimiters: e.g. parentheses '(',  ')'
     # Separators: e.g. comma
     class Scanner
+      PATT_BLOCK_COMMENT_BEGIN = /\/\*/.freeze
+      PATT_BLOCK_COMMENT_END = /\*\//.freeze
+      PATT_COMPARISON = /[!=><]=?/.freeze
+      PATT_IDENTIFIER = /[a-zA-Z_][a-zA-Z_0-9]*/.freeze
+      PATT_LINE_COMMENT = /\/\/[^\r\n]*/.freeze
+      PATT_NEWLINE = /(?:\r\n)|\r|\n/.freeze
+      PATT_NUMBER = /\d+(?:\.\d+)?/.freeze
+      PATT_WHITESPACE = /[ \t\f]+/.freeze
+
       # @return [StringScanner] Low-level input scanner
       attr_reader(:scanner)
 
@@ -31,7 +40,7 @@ module Loxxy
 
       # One or two special character tokens.
       # These are enumerated in section 4.2.1 Token type
-      @@lexeme2name = {
+      Lexeme2name = {
         '(' => 'LEFT_PAREN',
         ')' => 'RIGHT_PAREN',
         '{' => 'LEFT_BRACE',
@@ -77,16 +86,16 @@ module Loxxy
       # Constructor. Initialize a tokenizer for Lox input.
       # @param source [String] Lox text to tokenize.
       def initialize(source = nil)
-        @scanner = StringScanner.new('')
-        start_with(source) if source
+        reset
+        input = source || ''
+        @scanner = StringScanner.new(input)
       end
 
       # Reset the tokenizer and make the given text, the current input.
       # @param source [String] Lox text to tokenize.
       def start_with(source)
+        reset
         @scanner.string = source
-        @lineno = 1
-        @line_start = 0
       end
 
       # Scan the source and return an array of tokens.
@@ -99,39 +108,79 @@ module Loxxy
         end
         tok_sequence << build_token('EOF', nil)
 
-        return tok_sequence
+        tok_sequence
       end
 
       private
 
-      def _next_token
-        skip_intertoken_spaces
-        curr_ch = scanner.peek(1)
-        return nil if curr_ch.nil? || curr_ch.empty?
+      def reset
+        @state = :default
+        @lineno = 1
+        @line_start = 0
+      end
 
+      def _next_token
+        nesting_level = 0
         token = nil
 
-        if '(){},.;+-/*'.include? curr_ch
-          # Single delimiter or separator character
-          token = build_token(@@lexeme2name[curr_ch], scanner.getch)
-        elsif (lexeme = scanner.scan(/[!=><]=?/))
-          # One or two special character tokens
-          token = build_token(@@lexeme2name[lexeme], lexeme)
-        elsif scanner.scan(/"/) # Start of string detected...
-          token = build_string_token
-        elsif (lexeme = scanner.scan(/\d+(?:\.\d+)?/))
-          token = build_token('NUMBER', lexeme)
-        elsif (lexeme = scanner.scan(/[a-zA-Z_][a-zA-Z_0-9]*/))
-          keyw = @@keywords[lexeme]
-          tok_type = keyw ? keyw.upcase : 'IDENTIFIER'
-          token = build_token(tok_type, lexeme)
-        else # Unknown token
-          col = scanner.pos - @line_start + 1
-          _erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
-          raise ScanError, "Error: [line #{lineno}:#{col}]: Unexpected character."
-        end
+        # Loop until end of input reached or token found
+        until token || scanner.eos?
+          if scanner.skip(PATT_NEWLINE)
+            next_line_scanned
+            next
+          end
 
-        return token
+          case @state
+            when :default
+              next if scanner.skip(PATT_WHITESPACE) # Skip whitespaces
+
+              curr_ch = scanner.peek(1)
+
+              token = if scanner.skip(PATT_LINE_COMMENT)
+                next
+              elsif scanner.skip(PATT_BLOCK_COMMENT_BEGIN)
+                @state = :in_block_comment
+                nesting_level = 1
+                next
+              elsif '(){},.;+-/*'.include? curr_ch
+                # Single delimiter or separator character
+                build_token(Lexeme2name[curr_ch], scanner.getch)
+              elsif (lexeme = scanner.scan(PATT_COMPARISON))
+                # One or two special character tokens
+                build_token(Lexeme2name[lexeme], lexeme)
+              elsif scanner.scan(/"/) # Start of string detected...
+                build_string_token
+              elsif (lexeme = scanner.scan(PATT_NUMBER))
+                build_token('NUMBER', lexeme)
+              elsif (lexeme = scanner.scan(PATT_IDENTIFIER))
+                keyw = @@keywords[lexeme]
+                tok_type = keyw ? keyw.upcase : 'IDENTIFIER'
+                build_token(tok_type, lexeme)
+              else # Unknown token
+                col = scanner.pos - @line_start + 1
+                _erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+                raise ScanError, "Error: [line #{lineno}:#{col}]: Unexpected character."
+              end
+
+            when :in_block_comment
+              comment_part = scanner.scan_until(/(?:\/\*)|(?:\*\/)|(?:(?:\r\n)|\r|\n)/)
+              unterminated_comment unless comment_part
+
+              case scanner.matched
+                when PATT_NEWLINE
+                  next_line_scanned
+                when PATT_BLOCK_COMMENT_END
+                  nesting_level -= 1
+                  @state = :default if nesting_level.zero?
+                when PATT_BLOCK_COMMENT_BEGIN
+                  nesting_level += 1
+              end
+              next
+          end # case
+        end # until
+
+        unterminated_comment unless nesting_level.zero?
+        token
       end
 
       def build_token(aSymbolName, aLexeme)
@@ -214,56 +263,12 @@ module Loxxy
         Rley::Lexical::Literal.new(lox_string, lexeme, 'STRING', pos)
       end
 
-      # Skip non-significant whitespaces and comments.
-      # Advance the scanner until something significant is found.
-      def skip_intertoken_spaces
-        loop do
-          ws_found = scanner.skip(/[ \t\f]+/) ? true : false
-          nl_found = scanner.skip(/(?:\r\n)|\r|\n/)
-          if nl_found
-            ws_found = true
-            next_line
-          end
-          cmt_found = false
-          if scanner.scan(/\/(\/|\*)/)
-            cmt_found = true
-            case scanner.matched
-              when '//'
-                scanner.skip(/[^\r\n]*(?:(?:\r\n)|\r|\n)?/)
-                next_line
-              when '/*'
-                skip_block_comment
-                next
-            end
-          end
-          break unless ws_found || cmt_found
-        end
-
-        scanner.pos
-      end
-
-      def skip_block_comment
-        nesting_level = 1
-        loop do
-          comment_part = scanner.scan_until(/(?:\/\*)|(?:\*\/)|(?:(?:\r\n)|\r|\n)/)
-          unless comment_part
-            msg = "Unterminated '/* ... */' block comment on line #{lineno}"
-            raise ScanError, msg
-          end
-
-          case scanner.matched
-            when /(?:(?:\r\n)|\r|\n)/
-              next_line
-            when '*/'
-              nesting_level -= 1
-              break if nesting_level.zero?
-            when '/*'
-              nesting_level += 1
-          end
-        end
+      def unterminated_comment
+        msg = "Unterminated '/* ... */' block comment on line #{lineno}"
+        raise ScanError, msg
       end
 
-      def next_line
+      def next_line_scanned
         @lineno += 1
         @line_start = scanner.pos
       end

data/lib/loxxy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Loxxy
-  VERSION = '0.4.07'
+  VERSION = '0.4.08'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: loxxy
 version: !ruby/object:Gem::Version
-  version: 0.4.07
+  version: 0.4.08
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-21 00:00:00.000000000 Z
+date: 2022-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rley