rley 0.8.08 → 0.8.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55b425fac78001960b95935a8dc8de88d1c0826bea5fba5fae72455f53cb084a
-  data.tar.gz: fdf0f9ed39985923134cd3ecc4bdd9325c9aab783437ab1d61a07e333ad3f2eb
+  metadata.gz: f7ad228834b0f81ead8bc7ba0276c46b15d1d26e6c041bca00737bc5e224642b
+  data.tar.gz: 8877d597137a6aa3548abe6bdeeb5c51d9e94479094c78db377680657131a7d2
 SHA512:
-  metadata.gz: e63b7a5df081fa326b48f7f2e2cedbaf877ed377927e408278e3c8078a30fdcc69714ff234ba98dccc1eb3b05eaea53ebb39846a43e2cb794b883d91ce1c6953
-  data.tar.gz: e60bc7b045fe8d7203ddf26ecd603646137d43cbbbe79b533b2088c0f7d15bb46bc1dd58ae4d811ba8d78327fdcb0045cfddec4d708a76968d33c1602bf71cf4
+  metadata.gz: b7f4da7fafc790cd5c737310b811b4b5e51d5f37c37373ce04089c73e8ade7c318d3638ee1b345e165b7cbd59cc7636038ad866f19352f8208d8de759c47006d
+  data.tar.gz: 611344a1d84c111e3280cac654082670eecc27f4da781e9f32e5bfdc21f52a1ff3c17cb812de2b346cfbf29247c5976321368cebcf55e0463860f5f7281ab61a

data/.rubocop.yml

@@ -5,6 +5,9 @@ AllCops:
 Gemspec/DateAssignment:
   Enabled: true
 
+Gemspec/RequireMFA: # new in 1.23
+  Enabled: false
+
 Layout/ArgumentAlignment:
   Enabled: false
 
@@ -130,6 +133,9 @@ Lint/RaiseException:
 
 Lint/RedundantDirGlobSort:
   Enabled: true
+  
+Lint/RefinementImportMethods: # new in 1.27
+  Enabled: true  
 
 Lint/RequireRelativeSelfPath:
   Enabled: true
@@ -161,6 +167,9 @@ Lint/UnusedMethodArgument:
 Lint/UselessAccessModifier:
   Enabled: true
 
+Lint/UselessRuby2Keywords: # new in 1.23
+  Enabled: true
+
 Lint/Void:
   Enabled: false
 
@@ -200,6 +209,9 @@ Metrics/ModuleLength:
 Metrics/PerceivedComplexity:
   Enabled: false
 
+Naming/BlockForwarding: # new in 1.24
+  Enabled: true
+
 Naming/ConstantName:
   Enabled: false
 
@@ -278,6 +290,12 @@ Style/ExpandPathArguments:
 Style/ExponentialNotation:
   Enabled: true
 
+Style/FileRead: # new in 1.24
+  Enabled: true
+
+Style/FileWrite: # new in 1.24
+  Enabled: true
+
 Style/GuardClause:
   Enabled: false
 
@@ -302,6 +320,9 @@ Style/InPatternThen:
 Style/InverseMethods:
   Enabled: false
 
+Style/MapToHash: # new in 1.24
+  Enabled: true
+
 Style/MissingRespondToMissing:
   Enabled: false
 
@@ -310,6 +331,9 @@ Style/MultilineInPatternThen:
 
 Style/NegatedIfElseCondition:
   Enabled: true
+  
+Style/NestedFileDirname: # new in 1.26
+  Enabled: true
 
 Style/Next:
   Enabled: false
@@ -326,6 +350,9 @@ Style/NumberedParametersLimit:
 Style/NumericLiterals:
   Enabled: false
 
+Style/OpenStructUse: # new in 1.23
+  Enabled: true
+
 Style/QuotedSymbols:
   Enabled: true
 
@@ -334,6 +361,9 @@ Style/RaiseArgs:
 
 Style/RedundantArgument:
   Enabled: true
+  
+Style/RedundantInitialize: # new in 1.27
+  Enabled: true  
 
 Style/RedundantReturn:
   Enabled: false
@@ -349,9 +379,9 @@ Style/RegexpLiteral:
 
 Style/PercentLiteralDelimiters:
   Enabled: false
-  
+
 Style/SelectByRegexp:
-  Enabled: true  
+  Enabled: true
 
 Style/StderrPuts:
   Enabled: false

data/CHANGELOG.md

@@ -1,7 +1,18 @@
+### 0.8.10 / 2022-04-08
+- Refactoring of `RGN::Tokenizer` class.
+* [CHANGE] Class `RGN::Tokenizer` changes.
+
+### 0.8.09 / 2022-01-28
+- Added code for coming tutorial
+
+* [NEW] Folder `tutorial` contains code used in tutorial (in wiki)
+* [NEW] Folders `TOML\iter_1` .. `TOML\iter_2` contains code for a TOML parser
+* [CHANGE] File `.rubocop.yml` updated to take into account new 1.25 cops.
+* [CHANGE] File `.rubocop.yml` updated to take into account new 1.25 cops.
+
 ### 0.8.08 / 2021-10-31
 - Fixes in example files, Refactored module `Notation` renamed `to `RGN´
 
-
 * [CHANGE] Module `Notation` changed to `RGN`.
 
 * [FIX] `examples/general/calc_iter1/CalcLexer#initialize` now strips whitespaces at end of expression to parse.
@@ -21,7 +32,7 @@
 
 * [NEW] Folder `example\tokenizer` contains an example with a scanner that was generated.
 * [NEW] Class `Rley::Lexical::Literal` a specialization of Token class for literal tokens.
-* [CHANGE] File `.rubocp.yml` updated to take into account new 1.21 and 1.22 cops.
+* [CHANGE] File `.rubocop.yml` updated to take into account new 1.21 and 1.22 cops.
 * [DELETED] Class `Rley::Parser::ParserTracer` this class was no more in use.
 * [DELETED] Class `Rley::Syntax::Literal` this class was unused.
 * [DELETED] Class `Rley::Syntax::VerbatimSymbol` this class was unused.

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2021 Dimitri Geshef
+Copyright (c) 2014-2022 Dimitri Geshef
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -458,5 +458,5 @@ standard parser textbooks. Here are a few references (and links) of papers on GF
 
 Copyright
 ---------
-Copyright (c) 2014-2020, Dimitri Geshef.  
+Copyright (c) 2014-2022, Dimitri Geshef.  
 __Rley__ is released under the MIT License see [LICENSE.txt](https://github.com/famished-tiger/Rley/blob/master/LICENSE.txt) for details.

data/lib/rley/constants.rb

@@ -5,7 +5,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.8.08'
+  Version = '0.8.11'
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm"

data/lib/rley/rgn/ast_node.rb

@@ -36,7 +36,7 @@ module Rley
 
       # Abstract method (must be overriden in subclasses).
       # Part of the 'visitee' role in Visitor design pattern.
-      # @param _visitor [LoxxyTreeVisitor] the visitor
+      # @param _visitor [ASTVisitor] the visitor
       def accept(_visitor)
         raise NotImplementedError
       end

data/lib/rley/rgn/parser.rb

@@ -35,9 +35,9 @@ module Rley
         @engine.use_grammar(Rley::RGN::RGNGrammar)
       end
 
-      # Parse the given Lox program into a parse tree.
-      # @param source [String] Lox program to parse
-      # @return [Rley::ParseTree] A parse tree equivalent to the Lox input.
+      # Parse the given RGN snippet into a parse tree.
+      # @param source [String] Snippet to parse
+      # @return [Rley::ParseTree] A parse tree equivalent to the RGN input.
       def parse(source)
         lexer = Tokenizer.new(source)
         result = engine.parse(lexer.tokens)

data/lib/rley/rgn/symbol_node.rb

@@ -28,7 +28,7 @@ module Rley
 
       # Abstract method (must be overriden in subclasses).
       # Part of the 'visitee' role in Visitor design pattern.
-      # @param visitor [LoxxyTreeVisitor] the visitor
+      # @param visitor [RGN::ASTVisitor] the visitor
       def accept(visitor)
         visitor.visit_symbol_node(self)
       end

data/lib/rley/rgn/tokenizer.rb

@@ -14,6 +14,13 @@ module Rley
     # Delimiters: e.g. parentheses '(',  ')'
     # Separators: e.g. comma
     class Tokenizer
+      PATT_KEY = /[a-zA-Z_][a-zA-Z_0-9]*:/.freeze
+      PATT_INTEGER = /\d+/.freeze
+      PATT_NEWLINE = /(?:\r\n)|\r|\n/.freeze
+      PATT_STRING_START = /"|'/.freeze
+      PATT_SYMBOL = /[^?*+,:(){}\s]+/.freeze
+      PATT_WHITESPACE = /[ \t\f]+/.freeze
+
       # @return [StringScanner] Low-level input scanner
       attr_reader(:scanner)
 
@@ -24,7 +31,7 @@ module Rley
       attr_reader(:line_start)
 
       # One or two special character tokens.
-      @@lexeme2name = {
+      Lexeme2name = {
         '(' => 'LEFT_PAREN',
         ')' => 'RIGHT_PAREN',
         '{' => 'LEFT_BRACE',
@@ -39,21 +46,21 @@ module Rley
       # Here are all the implemented Rley notation keywords
       @@keywords = %w[
         match_closest repeat
-      ].map { |x| [x, x] }.to_h
+      ].to_h { |x| [x, x] }
 
-      # Constructor. Initialize a tokenizer for Lox input.
-      # @param source [String] Lox text to tokenize.
+      # Constructor. Initialize a tokenizer for RGN input.
+      # @param source [String] RGN text to tokenize.
       def initialize(source = nil)
-        @scanner = StringScanner.new('')
-        start_with(source) if source
+        reset
+        input = source || ''
+        @scanner = StringScanner.new(input)
       end
 
       # Reset the tokenizer and make the given text, the current input.
-      # @param source [String] Lox text to tokenize.
+      # @param source [String] RGN text to tokenize.
       def start_with(source)
+        reset
         @scanner.string = source
-        @lineno = 1
-        @line_start = 0
       end
 
       # Scan the source and return an array of tokens.
@@ -65,47 +72,64 @@ module Rley
           tok_sequence << token unless token.nil?
         end
 
-        return tok_sequence
+        tok_sequence
       end
 
       private
 
-      def _next_token
-        pos_before = scanner.pos
-        skip_intertoken_spaces
-        ws_found = true if scanner.pos > pos_before
-        curr_ch = scanner.peek(1)
-        return nil if curr_ch.nil? || curr_ch.empty?
+      def reset
+        @lineno = 1
+        @line_start = 0
+      end
 
+      def _next_token
         token = nil
+        ws_found = false
 
-        if '(){},'.include? curr_ch
-          # Single delimiter, separator or character
-          token = build_token(@@lexeme2name[curr_ch], scanner.getch)
-        elsif '?*+,'.include? curr_ch # modifier character
-          # modifiers without prefix text are symbols
-          symb = ws_found ? 'SYMBOL' : @@lexeme2name[curr_ch]
-          token = build_token(symb, scanner.getch)
-        elsif (lexeme = scanner.scan(/\.\./))
-          # One or two special character tokens
-          token = build_token(@@lexeme2name[lexeme], lexeme)
-        elsif scanner.check(/"|'/) # Start of string detected...
-          token = build_string_token
-        elsif (lexeme = scanner.scan(/\d+/))
-          token = build_token('INT_LIT', lexeme)
-        elsif (lexeme = scanner.scan(/[a-zA-Z_][a-zA-Z_0-9]*:/))
-          keyw = @@keywords[lexeme.chop!]
-          token = build_token('KEY', lexeme) if keyw
-          # ... error case
-        elsif (lexeme = scanner.scan(/[^?*+,:(){}\s]+/))
-           token = build_token('SYMBOL', lexeme)
-        else # Unknown token
-          col = scanner.pos - @line_start + 1
-          _erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
-          raise ScanError, "Error: [line #{lineno}:#{col}]: Unexpected character."
-        end
+        # Loop until end of input reached or token found
+        until token || scanner.eos?
 
-        return token
+          nl_found = scanner.skip(PATT_NEWLINE)
+          if nl_found
+            next_line_scanned
+            next
+          end
+          if scanner.skip(PATT_WHITESPACE) # Skip whitespaces
+            ws_found = true
+            next
+          end
+
+          curr_ch = scanner.peek(1)
+
+          if '(){},'.include? curr_ch
+            # Single delimiter, separator or character
+            token = build_token(Lexeme2name[curr_ch], scanner.getch)
+          elsif '?*+,'.include? curr_ch # modifier character
+            # modifiers without prefix text are symbols
+            symb = (ws_found || nl_found) ? 'SYMBOL' : Lexeme2name[curr_ch]
+            token = build_token(symb, scanner.getch)
+          elsif (lexeme = scanner.scan(/\.\./))
+            # One or two special character tokens
+            token = build_token(Lexeme2name[lexeme], lexeme)
+          elsif scanner.check(PATT_STRING_START) # Start of string detected...
+            token = build_string_token
+          elsif (lexeme = scanner.scan(PATT_INTEGER))
+            token = build_token('INT_LIT', lexeme)
+          elsif (lexeme = scanner.scan(PATT_KEY))
+            keyw = @@keywords[lexeme.chop!]
+            token = build_token('KEY', lexeme) if keyw
+            # ... error case
+          elsif (lexeme = scanner.scan(PATT_SYMBOL))
+             token = build_token('SYMBOL', lexeme)
+          else # Unknown token
+            col = scanner.pos - @line_start + 1
+            _erroneous = curr_ch.nil? ? '' : scanner.scan(/./)
+            raise ScanError, "Error: [line #{lineno}:#{col}]: Unexpected character."
+          end
+          ws_found = false
+        end # until
+
+        token
       end
 
       def build_token(aSymbolName, aLexeme)
@@ -154,24 +178,8 @@ module Rley
         Rley::Lexical::Token.new(literal, 'STR_LIT', pos)
       end
 
-      # Skip non-significant whitespaces and comments.
-      # Advance the scanner until something significant is found.
-      def skip_intertoken_spaces
-        loop do
-          ws_found = scanner.skip(/[ \t\f]+/) ? true : false
-          nl_found = scanner.skip(/(?:\r\n)|\r|\n/)
-          if nl_found
-            ws_found = true
-            next_line
-          end
-
-          break unless ws_found
-        end
-
-        scanner.pos
-      end
-
-      def next_line
+      # Event: next line detected.
+      def next_line_scanned
         @lineno += 1
         @line_start = scanner.pos
       end

data/lib/rley/syntax/grm_symbol.rb

@@ -8,10 +8,6 @@ module Rley # This module is used as a namespace
       # @return [String] The name of the grammar symbol
       attr_reader(:name)
 
-      # An indicator that tells whether the grammar symbol can generate a
-      # non-empty string of terminals.
-      attr_writer(:generative)
-
       # Constructor.
       # aName [String] The name of the grammar symbol.
       def initialize(aName)

data/lib/rley/syntax/non_terminal.rb

@@ -7,6 +7,10 @@ module Rley # This module is used as a namespace
     # A non-terminal symbol (sometimes called a syntactic variable) represents
     # a composition of terminal or non-terminal symbols
     class NonTerminal < GrmSymbol
+      # An indicator that tells whether the grammar symbol can generate a
+      # non-empty string of terminals.
+      attr_writer(:generative)
+
       # A non-terminal symbol is nullable if it can match an empty string.
       attr_writer(:nullable)
 

data/lib/rley/syntax/terminal.rb

@@ -7,25 +7,29 @@ module Rley # This module is used as a namespace
     # A terminal symbol represents a class of words in the language
     # defined the grammar.
     class Terminal < GrmSymbol
-      # Constructor.
-      # @param aName [String] The name of the grammar symbol.
-      def initialize(aName)
-        super(aName)
-        self.generative = true
+      # An indicator that tells whether the grammar symbol can generate a
+      # non-empty string of terminals.
+      # @return [TrueClass]
+      def generative?
+        true
       end
 
       # Return true iff the symbol is a terminal
+      # @return [TrueClass]
       def terminal?
-        return true
+        true
       end
 
       # @return [false] Return true if the symbol derives
       # the empty string. As terminal symbol corresponds to a input token
       # it is by definition non-nullable.
+      # @return [FalseClass]
       def nullable?
         false
       end
 
+      # Return a readable text representation of the instance
+      # @return [String] The symbol name
       def to_s
         name
       end

data/spec/rley/parser/gfg_earley_parser_spec.rb

@@ -22,7 +22,6 @@ require_relative '../support/expectation_helper'
 require_relative '../../../lib/rley/parser/gfg_earley_parser'
 
 module Rley # Open this namespace to avoid module qualifier prefixes
-  # rubocop: disable Metrics/BlockLength
   module Parser # Open this namespace to avoid module qualifier prefixes
     describe GFGEarleyParser do
       include GrammarABCHelper # Mix-in module with builder for grammar abc
@@ -1038,6 +1037,5 @@ MSG
       end # context
     end # describe
   end # module
-  # rubocop: enable Metrics/BlockLength
 end # module
 # End of file

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: rley
 version: !ruby/object:Gem::Version
-  version: 0.8.08
+  version: 0.8.11
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-31 00:00:00.000000000 Z
+date: 2022-04-17 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: prime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -296,14 +310,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Ruby implementation of the Earley's parsing algorithm
@@ -330,14 +344,6 @@ test_files:
 - spec/rley/lexical/literal_spec.rb
 - spec/rley/lexical/token_range_spec.rb
 - spec/rley/lexical/token_spec.rb
-- spec/rley/parser/dangling_else_spec.rb
-- spec/rley/parser/error_reason_spec.rb
-- spec/rley/parser/gfg_chart_spec.rb
-- spec/rley/parser/gfg_earley_parser_spec.rb
-- spec/rley/parser/gfg_parsing_spec.rb
-- spec/rley/parser/parse_entry_set_spec.rb
-- spec/rley/parser/parse_entry_spec.rb
-- spec/rley/parser/parse_walker_factory_spec.rb
 - spec/rley/parse_forest_visitor_spec.rb
 - spec/rley/parse_rep/ambiguous_parse_spec.rb
 - spec/rley/parse_rep/ast_builder_spec.rb
@@ -347,6 +353,14 @@ test_files:
 - spec/rley/parse_rep/parse_forest_factory_spec.rb
 - spec/rley/parse_rep/parse_tree_factory_spec.rb
 - spec/rley/parse_tree_visitor_spec.rb
+- spec/rley/parser/dangling_else_spec.rb
+- spec/rley/parser/error_reason_spec.rb
+- spec/rley/parser/gfg_chart_spec.rb
+- spec/rley/parser/gfg_earley_parser_spec.rb
+- spec/rley/parser/gfg_parsing_spec.rb
+- spec/rley/parser/parse_entry_set_spec.rb
+- spec/rley/parser/parse_entry_spec.rb
+- spec/rley/parser/parse_walker_factory_spec.rb
 - spec/rley/ptree/non_terminal_node_spec.rb
 - spec/rley/ptree/parse_tree_node_spec.rb
 - spec/rley/ptree/parse_tree_spec.rb