twowaysql 0.2.0

data/lib/twowaysql/parser.rb

@@ -0,0 +1,489 @@
+#
+# DO NOT MODIFY!!!!
+# This file is automatically generated by racc 1.4.5
+# from racc grammer file "lib/twowaysql/parser.y".
+#
+
+require 'racc/parser'
+
+
+module TwoWaySQL
+
+  class Parser < Racc::Parser
+
+module_eval <<'..end lib/twowaysql/parser.y modeval..id42b80b1728', 'lib/twowaysql/parser.y', 128
+
+require 'strscan'
+
+def initialize(opts={})
+  opts = {
+    :debug => true,
+    :preserve_space => true,
+    :preserve_comment => true,
+    :preserve_eol => true
+  }.merge(opts)
+  @yydebug = opts[:debug]
+  @preserve_space = opts[:preserve_space]
+  @preserve_comment = opts[:preserve_comment]
+  @preserve_eol = opts[:preserve_eol]
+  @num_questions = 0
+end
+
+BEGIN_SUBSTITUTION          = '(\/|\#)\*([^\*]+)\*\1'
+PAREN_EXAMPLE               = '\([^\)]+\)'
+SUBSTITUTION_PATTERN        = /\A#{BEGIN_SUBSTITUTION}\s*/
+PAREN_SUBSTITUTION_PATTERN  = /\A#{BEGIN_SUBSTITUTION}\s*#{PAREN_EXAMPLE}/
+
+CONDITIONAL_PATTERN   = /\A(\/|\#)\*(IF)\s+([^\*]+)\s*\*\1/
+BEGIN_END_PATTERN     = /\A(\/|\#)\*(BEGIN|END)\s*\*\1/
+QUOTED_STRING_PATTERN = /\A(\'(?:[^\']+|\'\')*\')/   ## quoted string
+SPLIT_TOKEN_PATTERN   = /\A(\S+?)(?=\s*(?:(?:\/|\#)\*|-{2,}|\(|\)|\,))/  ## stop on delimiters --,/*,#*,',',(,)
+ELSE_PATTERN          = /\A\-{2,}\s*ELSE\s*/
+AND_PATTERN           = /\A(\s*AND\s+)/
+OR_PATTERN            = /\A(\s*OR\s+)/
+LITERAL_PATTERN       = /\A([^;\s]+)/
+SPACES_PATTERN        = /\A(\s+)/
+QUESTION_PATTERN      = /\A\?/
+COMMA_PATTERN         = /\A\,/
+LPAREN_PATTERN        = /\A\(/
+RPAREN_PATTERN        = /\A\)/
+ACTUAL_COMMENT_PATTERN          = /\A(\/|\#)\*\s+(.+)\s*\*\1/  ## start with spaces
+SEMICOLON_AT_INPUT_END_PATTERN  = /\A\;\s*\Z/
+UNMATCHED_COMMENT_START_PATTERN = /\A(?:(?:\/|\#)\*)/
+
+
+def parse( io )
+  @q = []
+  io.each do |line|
+    s = StringScanner.new(line.rstrip)
+    until s.eos? do
+      case
+      when s.scan(AND_PATTERN)
+        @q.push [ :AND, s[1] ]
+      when s.scan(OR_PATTERN)
+        @q.push [ :OR, s[1] ]
+      when s.scan(SPACES_PATTERN)
+        @q.push [ :SPACES, s[1] ] if @preserve_space
+      when s.scan(QUESTION_PATTERN)
+        @q.push [ :QUESTION, nil ]
+      when s.scan(COMMA_PATTERN)
+        @q.push [ :COMMA, ',' ]
+      when s.scan(LPAREN_PATTERN)
+        @q.push [ :LPAREN, '(' ]
+      when s.scan(RPAREN_PATTERN)
+        @q.push [ :RPAREN, ')' ]
+      when s.scan(ELSE_PATTERN)
+        @q.push [ :ELSE, nil ]
+      when s.scan(ACTUAL_COMMENT_PATTERN)
+        @q.push [ :ACTUAL_COMMENT, s[2] ] if @preserve_comment
+      when s.scan(BEGIN_END_PATTERN)
+        @q.push [ s[2].intern, nil ]
+      when s.scan(CONDITIONAL_PATTERN)
+        @q.push [ s[2].intern, s[3] ]
+      when s.scan(PAREN_SUBSTITUTION_PATTERN)
+        @q.push [ :PAREN_SUBSTITUTION, s[2] ]
+      when s.scan(SUBSTITUTION_PATTERN)
+        @q.push [ :SUBSTITUTION, s[2] ]
+      when s.scan(QUOTED_STRING_PATTERN)
+        @q.push [ :QUOTED, s[1] ]
+      when s.scan(SPLIT_TOKEN_PATTERN)
+        @q.push [ :CHARS, s[1] ]
+      when s.scan(UNMATCHED_COMMENT_START_PATTERN)   ## unmatched comment start, '/*','#*'
+        raise Racc::ParseError, "## unmatched comment. cannot parse [#{s.rest}]"
+      when s.scan(LITERAL_PATTERN)   ## other string token
+        @q.push [ :CHARS, s[1] ]
+      when s.scan(SEMICOLON_AT_INPUT_END_PATTERN)
+        #drop semicolon at input end
+      else
+        raise Racc::ParseError, "## cannot parse [#{s.rest}]"
+      end
+    end
+      
+    @q.push [ :EOL, nil ] if @preserve_eol
+  end
+    
+  @q.push [ false, nil ]
+    
+  ## cal racc's private parse method
+  do_parse
+end
+
+def next_token
+  @q.shift
+end
+..end lib/twowaysql/parser.y modeval..id42b80b1728
+
+##### racc 1.4.5 generates ###
+
+racc_reduce_table = [
+ 0, 0, :racc_error,
+ 1, 20, :_reduce_1,
+ 0, 21, :_reduce_2,
+ 2, 21, :_reduce_3,
+ 1, 22, :_reduce_none,
+ 1, 22, :_reduce_none,
+ 1, 22, :_reduce_none,
+ 3, 25, :_reduce_7,
+ 4, 24, :_reduce_8,
+ 2, 27, :_reduce_9,
+ 0, 27, :_reduce_10,
+ 1, 26, :_reduce_none,
+ 1, 26, :_reduce_none,
+ 1, 26, :_reduce_none,
+ 2, 28, :_reduce_14,
+ 2, 29, :_reduce_15,
+ 1, 23, :_reduce_16,
+ 1, 23, :_reduce_17,
+ 1, 23, :_reduce_18,
+ 1, 23, :_reduce_19,
+ 1, 23, :_reduce_20,
+ 1, 23, :_reduce_21,
+ 1, 23, :_reduce_22,
+ 1, 23, :_reduce_23,
+ 1, 23, :_reduce_24,
+ 1, 23, :_reduce_25,
+ 1, 23, :_reduce_26,
+ 1, 23, :_reduce_none,
+ 2, 30, :_reduce_28,
+ 3, 30, :_reduce_29,
+ 2, 30, :_reduce_30,
+ 3, 30, :_reduce_31,
+ 1, 30, :_reduce_32 ]
+
+racc_reduce_n = 33
+
+racc_shift_n = 44
+
+racc_action_table = [
+     9,    36,    14,    43,    17,    19,    21,    22,    23,     4,
+     6,     8,    11,    13,    15,    16,    18,     9,    24,    14,
+     3,    17,    19,    21,    22,    23,     4,     6,     8,    11,
+    13,    15,    16,    18,     9,   nil,    14,   nil,    17,    19,
+    21,    22,    23,     4,     6,     8,    11,    13,    15,    16,
+    18,     9,   nil,    14,   nil,    17,    19,    21,    22,    23,
+     4,     6,     8,    11,    13,    15,    16,    18,     9,    35,
+    14,   nil,    17,    19,    21,    22,    23,     4,     6,     8,
+    11,    13,    15,    16,    18,    32,    33,    34,    27,    29,
+    27,    29,    40,    41 ]
+
+racc_action_check = [
+    39,    26,    39,    37,    39,    39,    39,    39,    39,    39,
+    39,    39,    39,    39,    39,    39,    39,     2,     3,     2,
+     1,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,    38,   nil,    38,   nil,    38,    38,
+    38,    38,    38,    38,    38,    38,    38,    38,    38,    38,
+    38,    31,   nil,    31,   nil,    31,    31,    31,    31,    31,
+    31,    31,    31,    31,    31,    31,    31,    31,    25,    25,
+    25,   nil,    25,    25,    25,    25,    25,    25,    25,    25,
+    25,    25,    25,    25,    25,    16,    16,    16,    36,    36,
+    14,    14,    34,    34 ]
+
+racc_action_pointer = [
+   nil,    20,    15,    18,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,    84,   nil,    77,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,    66,    -4,   nil,   nil,   nil,
+   nil,    49,   nil,   nil,    84,   nil,    82,     0,    32,    -2,
+   nil,   nil,   nil,   nil ]
+
+racc_action_default = [
+    -2,   -33,    -1,   -33,   -21,    -3,   -22,    -4,   -23,    -2,
+    -5,   -24,    -6,   -25,    -2,   -26,   -33,   -18,   -32,   -19,
+   -27,   -16,   -17,   -20,    44,   -33,   -10,    -2,   -11,    -2,
+   -12,   -13,   -30,   -28,   -33,    -7,    -2,   -33,   -14,   -15,
+   -31,   -29,    -9,    -8 ]
+
+racc_goto_table = [
+     2,    26,     1,    37,   nil,   nil,   nil,   nil,   nil,    25,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,    42,   nil,   nil,   nil,    38,   nil,    39 ]
+
+racc_goto_check = [
+     2,     7,     1,     8,   nil,   nil,   nil,   nil,   nil,     2,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,     7,   nil,   nil,   nil,     2,   nil,     2 ]
+
+racc_goto_pointer = [
+   nil,     2,     0,   nil,   nil,   nil,   nil,   -13,   -23,   nil,
+   nil,   nil ]
+
+racc_goto_default = [
+   nil,   nil,    31,     5,     7,    10,    12,   nil,   nil,    28,
+    30,    20 ]
+
+racc_token_table = {
+ false => 0,
+ Object.new => 1,
+ :BEGIN => 2,
+ :END => 3,
+ :IF => 4,
+ :ELSE => 5,
+ :AND => 6,
+ :OR => 7,
+ :CHARS => 8,
+ :QUOTED => 9,
+ :SPACES => 10,
+ :COMMA => 11,
+ :LPAREN => 12,
+ :RPAREN => 13,
+ :QUESTION => 14,
+ :ACTUAL_COMMENT => 15,
+ :EOL => 16,
+ :SUBSTITUTION => 17,
+ :PAREN_SUBSTITUTION => 18 }
+
+racc_use_result_var = true
+
+racc_nt_base = 19
+
+Racc_arg = [
+ racc_action_table,
+ racc_action_check,
+ racc_action_default,
+ racc_action_pointer,
+ racc_goto_table,
+ racc_goto_check,
+ racc_goto_default,
+ racc_goto_pointer,
+ racc_nt_base,
+ racc_reduce_table,
+ racc_token_table,
+ racc_shift_n,
+ racc_reduce_n,
+ racc_use_result_var ]
+
+Racc_token_to_s_table = [
+'$end',
+'error',
+'BEGIN',
+'END',
+'IF',
+'ELSE',
+'AND',
+'OR',
+'CHARS',
+'QUOTED',
+'SPACES',
+'COMMA',
+'LPAREN',
+'RPAREN',
+'QUESTION',
+'ACTUAL_COMMENT',
+'EOL',
+'SUBSTITUTION',
+'PAREN_SUBSTITUTION',
+'$start',
+'sql',
+'stmt_list',
+'stmt',
+'primary',
+'if_stmt',
+'begin_stmt',
+'sub_stmt',
+'else_stmt',
+'and_stmt',
+'or_stmt',
+'substitution']
+
+Racc_debug_parser = false
+
+##### racc system variables end #####
+
+ # reduce 0 omitted
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 8
+  def _reduce_1( val, _values, result )
+                  result = RootNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 13
+  def _reduce_2( val, _values, result )
+                  result = []
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 17
+  def _reduce_3( val, _values, result )
+                  result.push val[1]
+   result
+  end
+.,.,
+
+ # reduce 4 omitted
+
+ # reduce 5 omitted
+
+ # reduce 6 omitted
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 26
+  def _reduce_7( val, _values, result )
+                  result = BeginNode.new( val[1] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 31
+  def _reduce_8( val, _values, result )
+                  result = IfNode.new( val[0], val[1], val[2] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 36
+  def _reduce_9( val, _values, result )
+                  result = val[1]
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 40
+  def _reduce_10( val, _values, result )
+                  result = nil
+   result
+  end
+.,.,
+
+ # reduce 11 omitted
+
+ # reduce 12 omitted
+
+ # reduce 13 omitted
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 49
+  def _reduce_14( val, _values, result )
+                  result = SubStatementNode.new( val[0], val[1] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 54
+  def _reduce_15( val, _values, result )
+                  result = SubStatementNode.new( val[0], val[1] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 59
+  def _reduce_16( val, _values, result )
+                  result = LiteralNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 63
+  def _reduce_17( val, _values, result )
+                  result = LiteralNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 67
+  def _reduce_18( val, _values, result )
+                  result = LiteralNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 71
+  def _reduce_19( val, _values, result )
+                  result = LiteralNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 75
+  def _reduce_20( val, _values, result )
+                  result = LiteralNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 79
+  def _reduce_21( val, _values, result )
+                  result = LiteralNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 83
+  def _reduce_22( val, _values, result )
+                  result = LiteralNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 87
+  def _reduce_23( val, _values, result )
+                  result = LiteralNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 92
+  def _reduce_24( val, _values, result )
+                  @num_questions += 1
+                  result = QuestionNode.new( @num_questions )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 96
+  def _reduce_25( val, _values, result )
+                  result = CommentNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 100
+  def _reduce_26( val, _values, result )
+                  result = EolNode.new
+   result
+  end
+.,.,
+
+ # reduce 27 omitted
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 106
+  def _reduce_28( val, _values, result )
+                  result = SubstitutionNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 110
+  def _reduce_29( val, _values, result )
+                  result = SubstitutionNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 114
+  def _reduce_30( val, _values, result )
+                  result = SubstitutionNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 118
+  def _reduce_31( val, _values, result )
+                  result = SubstitutionNode.new( val[0] )
+   result
+  end
+.,.,
+
+module_eval <<'.,.,', 'lib/twowaysql/parser.y', 122
+  def _reduce_32( val, _values, result )
+                  result = ParenSubstitutionNode.new( val[0] )
+   result
+  end
+.,.,
+
+ def _reduce_none( val, _values, result )
+  result
+ end
+
+  end   # class Parser
+
+end   # module TwoWaySQL

data/lib/twowaysql/parser.y

@@ -0,0 +1,226 @@
+class TwoWaySQL::Parser
+
+rule
+
+sql            : stmt_list
+                {
+                  result = RootNode.new( val[0] )
+                }
+
+stmt_list      :
+                {
+                  result = []
+                }
+               | stmt_list stmt
+                {
+                  result.push val[1]
+                }
+
+stmt           : primary
+               | if_stmt
+               | begin_stmt
+
+begin_stmt     : BEGIN stmt_list END
+                {
+                  result = BeginNode.new( val[1] )
+                }
+
+if_stmt        : IF sub_stmt else_stmt END
+                {
+                  result = IfNode.new( val[0], val[1], val[2] )
+                }
+
+else_stmt      : ELSE sub_stmt
+                {
+                  result = val[1]
+                }
+               |
+                {
+                  result = nil
+                }
+
+sub_stmt       : and_stmt
+               | or_stmt
+               | stmt_list
+
+and_stmt       : AND stmt_list
+                {
+                  result = SubStatementNode.new( val[0], val[1] )
+                }
+
+or_stmt        : OR stmt_list
+                {
+                  result = SubStatementNode.new( val[0], val[1] )
+                }
+
+primary        : CHARS
+                {
+                  result = LiteralNode.new( val[0] )
+                }
+               | QUOTED
+                {
+                  result = LiteralNode.new( val[0] )
+                }
+               | AND
+                {
+                  result = LiteralNode.new( val[0] )
+                }
+               | OR
+                {
+                  result = LiteralNode.new( val[0] )
+                }
+               | SPACES
+                {
+                  result = LiteralNode.new( val[0] )
+                }
+               | COMMA
+                {
+                  result = LiteralNode.new( val[0] )
+                }
+               | LPAREN
+                {
+                  result = LiteralNode.new( val[0] )
+                }
+               | RPAREN
+                {
+                  result = LiteralNode.new( val[0] )
+                }
+               | QUESTION
+                {
+                  @num_questions += 1
+                  result = QuestionNode.new( @num_questions )
+                }
+               | ACTUAL_COMMENT
+                {
+                  result = CommentNode.new( val[0] )
+                }
+               | EOL
+                {
+                  result = EolNode.new
+                }
+               | substitution
+
+substitution   : SUBSTITUTION QUOTED
+                {
+                  result = SubstitutionNode.new( val[0] )
+                }
+               | SUBSTITUTION SPACES QUOTED
+                {
+                  result = SubstitutionNode.new( val[0] )
+                }
+               | SUBSTITUTION CHARS
+                {
+                  result = SubstitutionNode.new( val[0] )
+                }
+               | SUBSTITUTION SPACES CHARS
+                {
+                  result = SubstitutionNode.new( val[0] )
+                }
+               | PAREN_SUBSTITUTION
+                {
+                  result = ParenSubstitutionNode.new( val[0] )
+                }
+
+end
+
+
+---- inner
+
+require 'strscan'
+
+def initialize(opts={})
+  opts = {
+    :debug => true,
+    :preserve_space => true,
+    :preserve_comment => true,
+    :preserve_eol => true
+  }.merge(opts)
+  @yydebug = opts[:debug]
+  @preserve_space = opts[:preserve_space]
+  @preserve_comment = opts[:preserve_comment]
+  @preserve_eol = opts[:preserve_eol]
+  @num_questions = 0
+end
+
+BEGIN_SUBSTITUTION          = '(\/|\#)\*([^\*]+)\*\1'
+PAREN_EXAMPLE               = '\([^\)]+\)'
+SUBSTITUTION_PATTERN        = /\A#{BEGIN_SUBSTITUTION}\s*/
+PAREN_SUBSTITUTION_PATTERN  = /\A#{BEGIN_SUBSTITUTION}\s*#{PAREN_EXAMPLE}/
+
+CONDITIONAL_PATTERN   = /\A(\/|\#)\*(IF)\s+([^\*]+)\s*\*\1/
+BEGIN_END_PATTERN     = /\A(\/|\#)\*(BEGIN|END)\s*\*\1/
+QUOTED_STRING_PATTERN = /\A(\'(?:[^\']+|\'\')*\')/   ## quoted string
+SPLIT_TOKEN_PATTERN   = /\A(\S+?)(?=\s*(?:(?:\/|\#)\*|-{2,}|\(|\)|\,))/  ## stop on delimiters --,/*,#*,',',(,)
+ELSE_PATTERN          = /\A\-{2,}\s*ELSE\s*/
+AND_PATTERN           = /\A(\s*AND\s+)/
+OR_PATTERN            = /\A(\s*OR\s+)/
+LITERAL_PATTERN       = /\A([^;\s]+)/
+SPACES_PATTERN        = /\A(\s+)/
+QUESTION_PATTERN      = /\A\?/
+COMMA_PATTERN         = /\A\,/
+LPAREN_PATTERN        = /\A\(/
+RPAREN_PATTERN        = /\A\)/
+ACTUAL_COMMENT_PATTERN          = /\A(\/|\#)\*\s+(.+)\s*\*\1/  ## start with spaces
+SEMICOLON_AT_INPUT_END_PATTERN  = /\A\;\s*\Z/
+UNMATCHED_COMMENT_START_PATTERN = /\A(?:(?:\/|\#)\*)/
+
+
+def parse( io )
+  @q = []
+  io.each do |line|
+    s = StringScanner.new(line.rstrip)
+    until s.eos? do
+      case
+      when s.scan(AND_PATTERN)
+        @q.push [ :AND, s[1] ]
+      when s.scan(OR_PATTERN)
+        @q.push [ :OR, s[1] ]
+      when s.scan(SPACES_PATTERN)
+        @q.push [ :SPACES, s[1] ] if @preserve_space
+      when s.scan(QUESTION_PATTERN)
+        @q.push [ :QUESTION, nil ]
+      when s.scan(COMMA_PATTERN)
+        @q.push [ :COMMA, ',' ]
+      when s.scan(LPAREN_PATTERN)
+        @q.push [ :LPAREN, '(' ]
+      when s.scan(RPAREN_PATTERN)
+        @q.push [ :RPAREN, ')' ]
+      when s.scan(ELSE_PATTERN)
+        @q.push [ :ELSE, nil ]
+      when s.scan(ACTUAL_COMMENT_PATTERN)
+        @q.push [ :ACTUAL_COMMENT, s[2] ] if @preserve_comment
+      when s.scan(BEGIN_END_PATTERN)
+        @q.push [ s[2].intern, nil ]
+      when s.scan(CONDITIONAL_PATTERN)
+        @q.push [ s[2].intern, s[3] ]
+      when s.scan(PAREN_SUBSTITUTION_PATTERN)
+        @q.push [ :PAREN_SUBSTITUTION, s[2] ]
+      when s.scan(SUBSTITUTION_PATTERN)
+        @q.push [ :SUBSTITUTION, s[2] ]
+      when s.scan(QUOTED_STRING_PATTERN)
+        @q.push [ :QUOTED, s[1] ]
+      when s.scan(SPLIT_TOKEN_PATTERN)
+        @q.push [ :CHARS, s[1] ]
+      when s.scan(UNMATCHED_COMMENT_START_PATTERN)   ## unmatched comment start, '/*','#*'
+        raise Racc::ParseError, "## unmatched comment. cannot parse [#{s.rest}]"
+      when s.scan(LITERAL_PATTERN)   ## other string token
+        @q.push [ :CHARS, s[1] ]
+      when s.scan(SEMICOLON_AT_INPUT_END_PATTERN)
+        #drop semicolon at input end
+      else
+        raise Racc::ParseError, "## cannot parse [#{s.rest}]"
+      end
+    end
+      
+    @q.push [ :EOL, nil ] if @preserve_eol
+  end
+    
+  @q.push [ false, nil ]
+    
+  ## cal racc's private parse method
+  do_parse
+end
+
+def next_token
+  @q.shift
+end