srl_ruby 0.0.1

data/lib/regex/wildcard.rb

@@ -0,0 +1,23 @@
+# File: wildcard.rb
+
+require_relative 'atomic_expression' # Access the superclass
+
+module Regex # This module is used as a namespace
+  # A wildcard matches any character (except for the newline).
+  class Wildcard < AtomicExpression
+    # Constructor
+    def initialize()
+      super
+    end
+
+    protected
+
+    # Conversion method re-definition.
+    # Purpose: Return the String representation of the expression.
+    def text_repr()
+      return '.'
+    end
+  end # class
+end # module
+
+# End of file

data/lib/srl_ruby/ast_builder.rb

@@ -0,0 +1,384 @@
+require 'stringio'
+require_relative 'regex_repr'
+
+module SrlRuby
+  # The purpose of a ASTBuilder is to build piece by piece an AST
+  # (Abstract Syntax Tree) from a sequence of input tokens and
+  # visit events produced by walking over a GFGParsing object.
+  # Uses the Builder GoF pattern.
+  # The Builder pattern creates a complex object
+  # (say, a parse tree) from simpler objects (terminal and non-terminal
+  # nodes) and using a step by step approach.
+  class ASTBuilder < Rley::ParseRep::ASTBaseBuilder
+    Terminal2NodeClass = {}.freeze
+
+    attr_reader :options
+
+    protected
+    
+    def terminal2node()
+      Terminal2NodeClass
+    end
+
+    # Overriding method.
+    # Factory method for creating a node object for the given
+    # input token.
+    # @param aTerminal [Terminal] Terminal symbol associated with the token
+    # @param aTokenPosition [Integer] Position of token in the input stream
+    # @param aToken [Token] The input token
+    def new_leaf_node(_production, _terminal, aTokenPosition, aToken)
+      node = Rley::PTree::TerminalNode.new(aToken, aTokenPosition)
+
+      return node
+    end
+
+    def multiplicity(lowerBound, upperBound)
+      return SRL::Regex::Multiplicity.new(lowerBound, upperBound, :greedy)
+    end
+
+    def string_literal(aString, to_escape = true)
+      if aString.size > 1
+        chars = []
+        aString.each_char do |ch|
+          if to_escape && Regex::Character::MetaChars.include?(ch)
+            chars << Regex::Character.new("\\")
+          end
+          chars << Regex::Character.new(ch)
+        end
+        result = Regex::Concatenation.new(*chars)
+      elsif to_escape && Regex::Character::MetaChars.include?(aString)
+        backslash = Regex::Character.new("\\")
+        a_string = Regex::Character.new(aString)
+        result = Regex::Concatenation.new(backslash, a_string)
+      else
+        result = Regex::Character.new(aString)
+      end
+
+      return result
+    end
+
+    def char_range(lowerBound, upperBound)
+      # TODO fix module nesting
+      lower = Regex::Character.new(lowerBound)
+      upper = Regex::Character.new(upperBound)
+      return Regex::CharRange.new(lower, upper)
+    end
+
+    def char_class(toNegate, *theChildren)
+      Regex::CharClass.new(toNegate, *theChildren)
+    end
+
+    def char_shorthand(shortName)
+      Regex::CharShorthand.new(shortName)
+    end
+
+    def wildcard()
+      Regex::Wildcard.new
+    end
+
+    def repetition(expressionToRepeat, aMultiplicity)
+      return Regex::Repetition.new(expressionToRepeat, aMultiplicity)
+    end
+    
+    def begin_anchor
+      return Regex::Anchor.new('^')
+    end
+    
+    # rule('expression' => %w[pattern separator flags]).as 'flagged_expr'
+    def reduce_flagged_expr(_production, aRange, theTokens, theChildren)
+      @options = theChildren[2] if theChildren[2]
+      return_first_child(aRange, theTokens, theChildren)
+    end
+
+    # rule('pattern' => %w[pattern separator quantifiable]).as 'pattern_sequence'
+    def reduce_pattern_sequence(_production, _range, _tokens, theChildren)
+      return Regex::Concatenation.new(theChildren[0], theChildren[2])
+    end
+
+    # rule('flags' => %[flags separator single_flag]).as 'flag_sequence'
+    def reduce_flag_sequence(_production, _range, _tokens, theChildren)
+      theChildren[0] << theChildren[2]
+    end
+
+    # rule('single_flag' => %w[CASE INSENSITIVE]).as 'case_insensitive'
+    def reduce_case_insensitive(_production, _range, _tokens, _children)
+      return [Regex::MatchOption.new(:IGNORECASE, true)]
+    end
+
+    # rule('single_flag' => %w[MULTI LINE]).as 'multi_line'
+    def reduce_multi_line(_production, _range, _tokens, _children)
+      return [Regex::MatchOption.new(:MULTILINE, true)]
+    end
+
+    # rule('single_flag' => %w[ALL LAZY]).as 'all_lazy'
+    def reduce_all_lazy(_production, _range, _tokens, _children)
+      return [Regex::MatchOption.new(:ALL_LAZY, true)]
+    end
+
+    # rule 'quantifiable' => %w[begin_anchor anchorable end_anchor]
+    def reduce_pinned_quantifiable(_production, _range, _tokens, theChildren)
+      theChildren[1].begin_anchor = theChildren[0]
+      theChildren[1].end_anchor = theChildren[2]
+      return theChildren[1]
+    end
+
+    # rule 'quantifiable' => %w[begin_anchor anchorable]
+    def reduce_begin_anchor_quantifiable(_production, _range, _tokens, theChildren)
+      theChildren[1].begin_anchor = theChildren[0]
+      return theChildren[1]
+    end
+
+    # rule 'quantifiable' => %w[anchorable end_anchor]
+    def reduce_end_anchor_quantifiable(_production, _range, _tokens, theChildren)
+      theChildren[0].end_anchor = theChildren[1]
+      return theChildren[0]
+    end
+
+    # rule 'begin_anchor' => %w[STARTS WITH]
+    def reduce_starts_with(_production, _range, _tokens, _children)
+      begin_anchor
+    end
+
+    # rule 'begin_anchor' => %w[BEGIN WITH]  
+    def reduce_begin_with(_production, _range, _tokens, _children)
+      begin_anchor
+    end  
+
+    # rule 'end_anchor' => %w[MUST END].as 'end_anchor'
+    def reduce_end_anchor(_production, _range, _tokens, _children)
+      return Regex::Anchor.new('$')
+    end
+
+    # rule('anchorable' => %w[assertable assertion]).as 'asserted_anchorable'
+    def reduce_asserted_anchorable(_production, _range, _tokens, theChildren)
+      assertion = theChildren.last
+      assertion.children.unshift(theChildren[0])
+      return assertion
+    end
+
+    # rule('assertion' => %w[IF FOLLOWED BY assertable]).as 'if_followed'
+    def reduce_if_followed(_production, _range, _tokens, theChildren)
+      return Regex::Lookaround.new(theChildren.last, :ahead, :positive)
+    end
+
+    # rule('assertion' => %w[IF NOT FOLLOWED BY assertable]).as 'if_not_followed'
+    def reduce_if_not_followed(_production, _range, _tokens, theChildren)
+      return Regex::Lookaround.new(theChildren.last, :ahead, :negative)
+    end
+
+    # rule('assertion' => %w[IF ALREADY HAD assertable]).as 'if_had'
+    def reduce_if_had(_production, _range, _tokens, theChildren)
+      return Regex::Lookaround.new(theChildren.last, :behind, :positive)
+    end
+
+    # rule('assertion' => %w[IF NOT ALREADY HAD assertable]).as 'if_not_had'
+    def reduce_if_not_had(_production, _range, _tokens, theChildren)
+      return Regex::Lookaround.new(theChildren.last, :behind, :negative)
+    end
+
+    # rule('assertable' => %w[term quantifier]).as 'quantified_assertable'
+    def reduce_quantified_assertable(_production, _range, _tokens, theChildren)
+      quantifier = theChildren[1]
+      term = theChildren[0]
+      repetition(term, quantifier)
+    end
+
+    # rule('letter_range' => %w[LETTER FROM LETTER_LIT TO LETTER_LIT]).as 'lowercase_from_to'
+    def reduce_lowercase_from_to(_production, _range, _tokens, theChildren)
+      lower = theChildren[2].token.lexeme
+      upper =  theChildren[4].token.lexeme
+      ch_range = char_range(lower, upper)
+      char_class(false, ch_range)
+    end
+
+    # rule('letter_range' => %w[UPPERCASE LETTER FROM LETTER_LIT TO LETTER_LIT]).as 'uppercase_from_to'
+    def reduce_uppercase_from_to(_production, _range, _tokens, theChildren)
+      lower = theChildren[3].token.lexeme
+      upper =  theChildren[5].token.lexeme
+      ch_range = char_range(lower.upcase, upper.upcase)
+      char_class(false, ch_range)
+    end
+
+    # rule('letter_range' => 'LETTER').as 'any_lowercase'
+    def reduce_any_lowercase(_production, _range, _tokens, _children)
+      ch_range = char_range('a', 'z')
+      char_class(false, ch_range)
+    end
+
+    # rule('letter_range' => %w[UPPERCASE LETTER]).as 'any_uppercase'
+    def reduce_any_uppercase(_production, _range, _tokens, _children)
+      ch_range = char_range('A', 'Z')
+      char_class(false, ch_range)
+    end
+
+    # rule('digit_range' => %w[digit_or_number FROM DIGIT_LIT TO DIGIT_LIT]).as 'digits_from_to'
+    def reduce_digits_from_to(aProduction, aRange, theTokens, theChildren)
+      reduce_lowercase_from_to(aProduction, aRange, theTokens, theChildren)
+    end
+
+    # rule('digit_range' => 'digit_or_number').as 'simple_digit_range'
+    def reduce_simple_digit_range(_production, _range, _tokens, _children)
+      char_shorthand('d')
+    end
+
+    # rule('character_class' => %w[ANY CHARACTER]).as 'any_character'
+    def reduce_any_character(_production, _range, _tokens, _children)
+      char_shorthand('w')
+    end
+
+    # rule('character_class' => %w[NO CHARACTER]).as 'no_character'
+    def reduce_no_character(_production, _range, _tokens, _children)
+      char_shorthand('W')
+    end
+
+    # rule('character_class' => 'WHITESPACE').as 'whitespace'
+    def reduce_whitespace(_production, _range, _tokens, _children)
+      char_shorthand('s')
+    end
+
+    # rule('character_class' => %w[NO WHITESPACE]).as 'no_whitespace'
+    def reduce_no_whitespace(_production, _range, _tokens, _children)
+      char_shorthand('S')
+    end
+
+    # rule('character_class' => 'ANYTHING').as 'anything'
+    def reduce_anything(_production, _range, _tokens, _children)
+      wildcard
+    end
+
+    # rule('alternation' => %w[ANY OF LPAREN alternatives RPAREN]).as 'any_of'
+    def reduce_one_of(_production, _range, _tokens, theChildren)
+      raw_literal = theChildren[-1].token.lexeme.dup
+      alternatives = raw_literal.chars.map { |ch| Regex::Character.new(ch) }
+      # TODO check other implementations
+      return Regex::CharClass.new(false, *alternatives)
+    end
+
+    # rule('special_char' => 'TAB').as 'tab'
+    def reduce_tab(_production, _range, _tokens, _children)
+      Regex::Character.new('\t')
+    end
+
+    # rule('special_char' => 'BACKSLASH').as 'backslash'
+    def reduce_backslash(_production, _range, _tokens, _children)
+      Regex::Character.new('\\')
+    end
+
+    # rule('special_char' => %w[NEW LINE]).as 'new_line'
+    def reduce_new_line(_production, _range, _tokens, _children)
+      # TODO: control portability
+      Regex::Character.new('\n')
+    end
+
+    # rule('literal' => %w[LITERALLY STRING_LIT]).as 'literally'
+    def reduce_literally(_production, _range, _tokens, theChildren)
+      # What if literal is empty?...
+
+      raw_literal = theChildren[-1].token.lexeme.dup
+      return string_literal(raw_literal)
+    end
+
+    # rule('alternation' => %w[ANY OF LPAREN alternatives RPAREN]).as 'any_of'
+    def reduce_any_of(_production, _range, _tokens, theChildren)
+      return Regex::Alternation.new(*theChildren[3])
+    end
+
+    # rule('alternatives' => %w[alternatives separator quantifiable]).as 'alternative_list'
+    def reduce_alternative_list(_production, _range, _tokens, theChildren)
+      return theChildren[0] << theChildren[-1]
+    end
+
+    # rule('alternatives' => 'quantifiable').as 'simple_alternative'
+    def reduce_simple_alternative(_production, _range, _tokens, theChildren)
+      return [theChildren.last]
+    end
+
+    # rule('grouping' => %w[LPAREN pattern RPAREN]).as 'grouping_parenthenses'
+    def reduce_grouping_parenthenses(_production, _range, _tokens, theChildren)
+      return Regex::NonCapturingGroup.new(theChildren[1])
+    end
+
+    # rule('capturing_group' => %w[CAPTURE assertable]).as 'capture'
+    def reduce_capture(_production, _range, _tokens, theChildren)
+      return Regex::CapturingGroup.new(theChildren[1])
+    end
+
+    # rule('capturing_group' => %w[CAPTURE assertable UNTIL assertable]).as
+    #   'capture_until'
+    def reduce_capture_until(_production, _range, _tokens, theChildren)
+      group = Regex::CapturingGroup.new(theChildren[1])
+      return Regex::Concatenation.new(group, theChildren[3])
+    end
+
+    # rule('capturing_group' => %w[CAPTURE assertable AS var_name]).as 
+    #   'named_capture'
+    def reduce_named_capture(_production, _range, _tokens, theChildren)
+      name = theChildren[3].token.lexeme.dup
+      return Regex::CapturingGroup.new(theChildren[1], name)
+    end
+
+    # rule('capturing_group' => %w[CAPTURE assertable AS var_name 
+    #   UNTIL assertable]).as 'named_capture_until'
+    def reduce_named_capture_until(_production, _range, _tokens, theChildren)
+      name = theChildren[3].token.lexeme.dup
+      group = Regex::CapturingGroup.new(theChildren[1], name)
+      return Regex::Concatenation.new(group, theChildren[5])
+    end
+    
+    # rule('quantifier' => 'ONCE').as 'once'
+    def reduce_once(_production, _range, _tokens, _children)
+      multiplicity(1, 1)
+    end
+    
+    # rule('quantifier' => 'TWICE').as 'twice'
+    def reduce_twice(_production, _range, _tokens, _children)
+      multiplicity(2, 2)
+    end  
+
+    # rule('quantifier' => %w[EXACTLY count TIMES]).as 'exactly'
+    def reduce_exactly(_production, _range, _tokens, theChildren)
+      count = theChildren[1].token.lexeme.to_i
+      multiplicity(count, count)
+    end
+
+    # rule('quantifier' => %w[BETWEEN count AND count times_suffix]).as
+    #   'between_and'
+    def reduce_between_and(_production, _range, _tokens, theChildren)
+      lower = theChildren[1].token.lexeme.to_i
+      upper = theChildren[3].token.lexeme.to_i
+      multiplicity(lower, upper)
+    end
+    
+    # rule('quantifier' => 'OPTIONAL').as 'optional'
+    def reduce_optional(_production, _range, _tokens, _children)
+      multiplicity(0, 1)
+    end
+
+    # rule('quantifier' => %w[ONCE OR MORE]).as 'once_or_more'
+    def reduce_once_or_more(_production, _range, _tokens, _children)
+      multiplicity(1, :more)
+    end
+    
+    # rule('quantifier' => %w[NEVER OR MORE]).as 'never_or_more'
+    def reduce_never_or_more(_production, _range, _tokens, _children)
+      multiplicity(0, :more)
+    end
+    
+    # rule('quantifier' => %w[AT LEAST count TIMES]).as 'at_least' 
+    def reduce_at_least(_production, _range, _tokens, theChildren)
+      count = theChildren[2].token.lexeme.to_i
+      multiplicity(count, :more)
+    end 
+    
+    # rule('times_suffix' => 'TIMES').as 'times_keyword'
+    def reduce_times_keyword(_production, _range, _tokens, _children)
+      return nil
+    end
+    
+    # rule('times_suffix' => []).as 'times_dropped'
+    def reduce_times_dropped(_production, _range, _tokens, _children)
+      return nil
+    end  
+  end # class
+end # module
+# End of file

data/lib/srl_ruby/grammar.rb

@@ -0,0 +1,106 @@
+# Grammar for SRL (Simple Regex Language)
+require 'rley' # Load the gem
+module SrlRuby
+  ########################################
+  # Work in progress.
+  # This is a very partial grammar of SRL.
+  # It will be expanded with the coming versions of Rley
+  builder = Rley::Syntax::GrammarBuilder.new do
+    add_terminals('LPAREN', 'RPAREN', 'COMMA')
+    add_terminals('DIGIT_LIT', 'INTEGER', 'LETTER_LIT')
+    add_terminals('LITERALLY', 'STRING_LIT')
+    add_terminals('BEGIN', 'STARTS', 'WITH')
+    add_terminals('MUST', 'END')
+    add_terminals('UPPERCASE', 'LETTER', 'FROM', 'TO')
+    add_terminals('DIGIT', 'NUMBER', 'ANY', 'NO')
+    add_terminals('CHARACTER', 'WHITESPACE', 'ANYTHING')
+    add_terminals('TAB', 'BACKSLASH', 'NEW', 'LINE')
+    add_terminals('OF', 'ONE')
+    add_terminals('EXACTLY', 'TIMES', 'ONCE', 'TWICE')
+    add_terminals('BETWEEN', 'AND', 'OPTIONAL', 'OR')
+    add_terminals('MORE', 'NEVER', 'AT', 'LEAST')
+    add_terminals('IF', 'FOLLOWED', 'BY', 'NOT')
+    add_terminals('ALREADY', 'HAD')
+    add_terminals('CAPTURE', 'AS', 'UNTIL')
+    add_terminals('CASE', 'INSENSITIVE', 'MULTI', 'ALL')
+    add_terminals('LAZY')
+
+    rule('srl' => 'expression').as 'start_rule'
+    rule('expression' => %w[pattern separator flags]).as 'flagged_expr'
+    rule('expression' => 'pattern').as 'simple_expr'
+    rule('pattern' => %w[pattern separator quantifiable]).as 'pattern_sequence'
+    rule('pattern' => 'quantifiable').as 'basic_pattern'
+    rule('separator' => 'COMMA').as 'comma_separator'
+    rule('separator' => []).as 'void_separator'
+    rule('flags' => %w[flags separator single_flag]).as 'flag_sequence'
+    rule('single_flag' => %w[CASE INSENSITIVE]).as 'case_insensitive'
+    rule('single_flag' => %w[MULTI LINE]).as 'multi_line'
+    rule('single_flag' => %w[ALL LAZY]).as 'all_lazy'
+    rule('quantifiable' => %w[begin_anchor anchorable end_anchor]).as 'pinned_quantifiable'
+    rule('quantifiable' => %w[begin_anchor anchorable]).as 'begin_anchor_quantifiable'
+    rule('quantifiable' => %w[anchorable end_anchor]).as 'end_anchor_quantifiable'
+    rule('quantifiable' => 'anchorable').as 'simple_quantifiable'
+    rule('begin_anchor' => %w[STARTS WITH]).as 'starts_with'
+    rule('begin_anchor' => %w[BEGIN WITH]).as 'begin_with'
+    rule('end_anchor' => %w[MUST END]).as 'end_anchor'
+    rule('anchorable' => 'assertable').as 'simple_anchorable'
+    rule('anchorable' => %w[assertable assertion]).as 'asserted_anchorable'
+    rule('assertion' => %w[IF FOLLOWED BY assertable]).as 'if_followed'
+    rule('assertion' => %w[IF NOT FOLLOWED BY assertable]).as 'if_not_followed'
+    rule('assertion' => %w[IF ALREADY HAD assertable]).as 'if_had'
+    rule('assertion' => %w[IF NOT ALREADY HAD assertable]).as 'if_not_had'
+    rule('assertable' => 'term').as 'simple_assertable'
+    rule('assertable' => %w[term quantifier]).as 'quantified_assertable'
+    rule('term' => 'atom').as 'atom_term'
+    rule('term' => 'alternation').as 'alternation_term'
+    rule('term' => 'grouping').as 'grouping_term'
+    rule('term' => 'capturing_group').as 'capturing_group_atom'
+    rule('atom' => 'letter_range').as 'letter_range_atom'
+    rule('atom' => 'digit_range').as 'digit_range_atom'
+    rule('atom' => 'character_class').as 'character_class_atom'
+    rule('atom' => 'special_char').as 'special_char_atom'
+    rule('atom' => 'literal').as 'literal_atom'
+    rule('letter_range' => %w[LETTER FROM LETTER_LIT TO LETTER_LIT]).as 'lowercase_from_to'
+    rule('letter_range' => %w[UPPERCASE LETTER FROM LETTER_LIT TO LETTER_LIT]).as 'uppercase_from_to'
+    rule('letter_range' => 'LETTER').as 'any_lowercase'
+    rule('letter_range' => %w[UPPERCASE LETTER]).as 'any_uppercase'
+    rule('digit_range' => %w[digit_or_number FROM DIGIT_LIT TO DIGIT_LIT]).as 'digits_from_to'
+    rule('digit_range' => 'digit_or_number').as 'simple_digit_range'
+    rule('character_class' => %w[ANY CHARACTER]).as 'any_character'
+    rule('character_class' => %w[NO CHARACTER]).as 'no_character'
+    rule('character_class' => 'WHITESPACE').as 'whitespace'
+    rule('character_class' => %w[NO WHITESPACE]).as 'no_whitespace'
+    rule('character_class' => 'ANYTHING').as 'anything'
+    rule('character_class' => %w[ONE OF STRING_LIT]).as 'one_of'
+    rule('special_char' => 'TAB').as 'tab'
+    rule('special_char' => 'BACKSLASH').as 'backslash'
+    rule('special_char' => %w[NEW LINE]).as 'new_line'
+    rule('literal' => %w[LITERALLY STRING_LIT]).as 'literally'
+    rule('alternation' => %w[ANY OF LPAREN alternatives RPAREN]).as 'any_of'
+    rule('alternatives' => %w[alternatives separator quantifiable]).as 'alternative_list'
+    rule('alternatives' => 'quantifiable').as 'simple_alternative'
+    rule('grouping' => %w[LPAREN pattern RPAREN]).as 'grouping_parenthenses'
+    rule('capturing_group' => %w[CAPTURE assertable]).as 'capture'
+    rule('capturing_group' => %w[CAPTURE assertable UNTIL assertable]).as 'capture_until'
+    rule('capturing_group' => %w[CAPTURE assertable AS var_name]).as 'named_capture'
+    rule('capturing_group' => %w[CAPTURE assertable AS var_name UNTIL assertable]).as 'named_capture_until'
+    rule('var_name' => 'STRING_LIT').as 'var_name'
+    rule('quantifier' => 'ONCE').as 'once'
+    rule('quantifier' => 'TWICE').as 'twice'
+    rule('quantifier' => %w[EXACTLY count TIMES]).as 'exactly'
+    rule('quantifier' => %w[BETWEEN count AND count times_suffix]).as 'between_and'
+    rule('quantifier' => 'OPTIONAL').as 'optional'
+    rule('quantifier' => %w[ONCE OR MORE]).as 'once_or_more'
+    rule('quantifier' => %w[NEVER OR MORE]).as 'never_or_more'
+    rule('quantifier' => %w[AT LEAST count TIMES]).as 'at_least'
+    rule('digit_or_number' => 'DIGIT').as 'digit_keyword'
+    rule('digit_or_number' => 'NUMBER').as 'number_keyword'
+    rule('count' => 'DIGIT_LIT').as 'single_digit'
+    rule('count' => 'INTEGER').as 'integer_count'
+    rule('times_suffix' => 'TIMES').as 'times_keyword'
+    rule('times_suffix' => []).as 'times_dropped'
+  end
+
+  # And now build the grammar and make it accessible via a global constant
+  Grammar = builder.grammar
+end # module

data/lib/srl_ruby/regex_repr.rb

@@ -0,0 +1,13 @@
+require_relative '../regex/character'
+require_relative '../regex/char_range'
+require_relative '../regex/concatenation'
+require_relative '../regex/multiplicity'
+require_relative '../regex/repetition'
+require_relative '../regex/char_class'
+require_relative '../regex/char_shorthand'
+require_relative '../regex/wildcard'
+require_relative '../regex/alternation'
+require_relative '../regex/non_capturing_group'
+require_relative '../regex/anchor'
+require_relative '../regex/lookaround'
+require_relative '../regex/capturing_group'

data/lib/srl_ruby/tokenizer.rb

@@ -0,0 +1,147 @@
+# File: srl_tokenizer.rb
+# Tokenizer for SRL (Simple Regex Language)
+require 'strscan'
+require 'rley' # Load the Rley gem
+
+module SrlRuby
+  # The tokenizer should recognize:
+  # Keywords: as, capture, letter
+  # Integer literals including single digit
+  # String literals (quote delimited)
+  # Single character literal
+  # Delimiters: parentheses '(' and ')'
+  # Separators: comma (optional)
+  class Tokenizer
+    attr_reader(:scanner)
+    attr_reader(:lineno)
+    attr_reader(:line_start)
+
+    @@lexeme2name = {
+      '(' => 'LPAREN',
+      ')' => 'RPAREN',
+      ',' => 'COMMA'
+    }.freeze
+
+    # Here are all the SRL keywords (in uppercase)
+    @@keywords = %w[
+      ALL
+      ALREADY
+      AND
+      ANY
+      ANYTHING
+      AS
+      AT
+      BACKSLASH
+      BEGIN
+      BETWEEN
+      BY
+      CAPTURE
+      CASE
+      CHARACTER
+      DIGIT
+      END
+      EXACTLY
+      FOLLOWED
+      FROM
+      HAD
+      IF
+      INSENSITIVE
+      LAZY
+      LEAST
+      LETTER
+      LINE
+      LITERALLY
+      MORE
+      MULTI
+      MUST
+      NEVER
+      NEW
+      NO
+      NOT
+      NUMBER
+      OF
+      ONCE
+      ONE
+      OPTIONAL
+      OR
+      STARTS
+      TAB
+      TIMES
+      TO
+      TWICE
+      UNTIL
+      UPPERCASE
+      WHITESPACE
+      WITH
+    ].map { |x| [x, x] } .to_h
+
+    class ScanError < StandardError; end
+
+    def initialize(source)
+      @scanner = StringScanner.new(source)
+      @lineno = 1
+    end
+
+    def tokens()
+      tok_sequence = []
+      until @scanner.eos?
+        token = _next_token
+        tok_sequence << token unless token.nil?
+      end
+
+      return tok_sequence
+    end
+
+    private
+
+    def _next_token()
+      skip_whitespaces
+      curr_ch = scanner.peek(1)
+      return nil if curr_ch.nil? || curr_ch.empty?
+
+      token = nil
+
+      if '(),'.include? curr_ch
+        # Delimiters, separators => single character token
+        token = build_token(@@lexeme2name[curr_ch], scanner.getch)
+      elsif (lexeme = scanner.scan(/[0-9]{2,}/))
+        token = build_token('INTEGER', lexeme) # An integer has 2..* digits
+      elsif (lexeme = scanner.scan(/[0-9]/))
+        token = build_token('DIGIT_LIT', lexeme)
+      elsif (lexeme = scanner.scan(/[a-zA-Z]{2,}/))
+        token = build_token(@@keywords[lexeme.upcase], lexeme)
+        # TODO: handle case unknown identifier
+      elsif (lexeme = scanner.scan(/[a-zA-Z]((?=\s)|$)/))
+        token = build_token('LETTER_LIT', lexeme)
+      elsif (lexeme = scanner.scan(/"([^"]|\\")*"/)) # Double quotes literal?
+        unquoted = lexeme.gsub(/(^")|("$)/, '')
+        token = build_token('STRING_LIT', unquoted)
+      elsif (lexeme = scanner.scan(/'([^']|\\')*'/)) # Single quotes literal?
+        unquoted = lexeme.gsub(/(^')|('$)/, '')
+        token = build_token('STRING_LIT', unquoted)
+      else # Unknown token
+        erroneous = curr_ch.nil? ? '' : curr_ch
+        sequel = scanner.scan(/.{1,20}/)
+        erroneous += sequel unless sequel.nil?
+        raise ScanError.new("Unknown token #{erroneous}")
+      end
+
+      return token
+    end
+
+    def build_token(aSymbolName, aLexeme)
+      begin
+        token = Rley::Lexical::Token.new(aLexeme, aSymbolName)
+      rescue StandardError
+        puts "Failing with '#{aSymbolName}' and '#{aLexeme}'"
+        raise ex
+      end
+
+      return token
+    end
+
+    def skip_whitespaces()
+      scanner.scan(/[ \t\f\n\r]+/)
+    end
+  end # class
+end # module

data/lib/srl_ruby/version.rb

@@ -0,0 +1,3 @@
+module SrlRuby
+  VERSION = '0.0.1'.freeze
+end

data/lib/srl_ruby.rb

@@ -0,0 +1,4 @@
+require_relative './srl_ruby/version'
+
+module SrlRuby # This module is used as a namespace
+end