racc 1.4.14 → 1.4.15

data/test/assets/huia.y

@@ -0,0 +1,318 @@
+# Copyright (c) 2014 James Harton
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+class Huia::Parser
+
+  token
+    IDENTIFIER EQUAL PLUS MINUS ASTERISK FWD_SLASH COLON FLOAT INTEGER STRING
+    EXPO INDENT OUTDENT OPAREN CPAREN DOT SIGNATURE NL EOF PIPE COMMA NIL TRUE
+    FALSE EQUALITY CALL SELF CONSTANT CHAR DOUBLE_TICK_STRING
+    DOUBLE_TICK_STRING_END INTERPOLATE_START INTERPOLATE_END BOX LSQUARE
+    RSQUARE FACES LFACE RFACE BANG TILDE RETURN NOT_EQUALITY OR AND GT LT
+    GTE LTE AT
+
+  prechigh
+    left EXPO
+    left BANG TILDE
+    left ASTERISK FWD_SLASH PERCENT
+    left PLUS MINUS
+
+    right EQUAL
+  preclow
+
+  rule
+    statements:   statement
+                | statements statement             { return scope }
+
+    statement:  expr eol                           { return scope.append val[0] }
+              | expr                               { return scope.append val[0] }
+              | eol                                { return scope }
+
+    eol:        NL | EOF
+    nlq:        NL |
+
+    expr:        literal
+               | grouped_expr
+               | binary_op
+               | unary_op
+               | method_call
+               | constant
+               | variable
+               | array
+               | hash
+               | return
+
+    return:            return_expr
+                     | return_nil
+    return_expr:       RETURN expr                      { return n(:Return, val[1]) }
+    return_nil:        RETURN                           { return n(:Return, n(:Nil)) }
+
+    array:             empty_array
+                     | array_list
+
+    empty_array:       BOX                              { return n :Array }
+
+    array_list:        LSQUARE array_items RSQUARE      { return val[1] }
+    array_items:       expr                             { return n :Array, [val[0]] }
+                     | array_items COMMA expr           { val[0].append(val[2]); return val[0] }
+
+    hash:              empty_hash
+                     | hash_list
+    empty_hash:        FACES                            { return n :Hash }
+    hash_list:         LFACE hash_items RFACE           { return val[1] }
+    hash_items:        hash_item                        { return n :Hash, val[0] }
+                     | hash_items COMMA hash_item       { val[0].append(val[2]); return val[0] }
+    hash_item:         expr COLON expr                  { return n :HashItem, val[0], val[2] }
+
+    constant:          CONSTANT                         { return constant val[0] }
+
+    indented:          indented_w_stmts
+                     | indented_w_expr
+                     | indented_wo_stmts
+    indented_w_stmts:  indent statements outdent        { return val[0] }
+    indented_w_expr:   indent expr outdent              { return val[0].append(val[1]) }
+    indented_wo_stmts: indent outdent                   { return val[0] }
+    outdent:           OUTDENT { return pop_scope }
+
+
+    indent_w_args:     indent_pipe indent_args PIPE nlq INDENT { return val[0] }
+    indent_pipe:       PIPE   { return push_scope }
+    indent_wo_args:    INDENT { return push_scope }
+    indent:            indent_w_args
+                     | indent_wo_args
+
+    indent_args:       indent_arg
+                     | indent_args COMMA indent_arg
+    indent_arg:        arg_var                             { return scope.add_argument val[0] }
+                     | arg_var EQUAL expr                  { return n :Assignment, val[0], val[2] }
+    arg_var:           IDENTIFIER                          { return n :Variable, val[0] }
+
+    method_call:            method_call_on_object
+                          | method_call_on_self
+                          | method_call_on_closure
+    method_call_on_object:  expr DOT call_signature        { return n :MethodCall, val[0], val[2] }
+                          | expr DOT IDENTIFIER            { return n :MethodCall, val[0], n(:CallSignature, val[2]) }
+    method_call_on_self:    call_signature                 { return n :MethodCall, scope_instance, val[0] }
+
+    method_call_on_closure: AT call_signature              { return n :MethodCall, this_closure, val[1] }
+                          | AT IDENTIFIER                  { return n :MethodCall, this_closure, n(:CallSignature, val[1]) }
+
+    call_signature:         call_arguments
+                          | call_simple_name
+    call_simple_name:       CALL                           { return n :CallSignature, val[0] }
+    call_argument:          SIGNATURE call_passed_arg      { return n :CallSignature, val[0], [val[1]] }
+    call_passed_arg:        call_passed_simple
+                          | call_passed_indented
+    call_passed_simple:     expr
+                          | expr NL
+    call_passed_indented:   indented
+                          | indented NL
+    call_arguments:         call_argument                  { return val[0] }
+                          | call_arguments call_argument   { return val[0].concat_signature val[1] }
+
+    grouped_expr: OPAREN expr CPAREN            { return n :Expression, val[1] }
+
+    variable:  IDENTIFIER                       { return allocate_local val[0] }
+
+    binary_op: assignment
+             | addition
+             | subtraction
+             | multiplication
+             | division
+             | exponentiation
+             | modulo
+             | equality
+             | not_equality
+             | logical_or
+             | logical_and
+             | greater_than
+             | less_than
+             | greater_or_eq
+             | less_or_eq
+
+    assignment:     IDENTIFIER EQUAL expr  { return allocate_local_assignment val[0], val[2] }
+    addition:       expr PLUS expr         { return binary val[0], val[2], 'plus:' }
+    subtraction:    expr MINUS expr        { return binary val[0], val[2], 'minus:' }
+    multiplication: expr ASTERISK expr     { return binary val[0], val[2], 'multiplyBy:' }
+    division:       expr FWD_SLASH expr    { return binary val[0], val[2], 'divideBy:' }
+    exponentiation: expr EXPO expr         { return binary val[0], val[2], 'toThePowerOf:' }
+    modulo:         expr PERCENT expr      { return binary val[0], val[2], 'moduloOf:' }
+    equality:       expr EQUALITY expr     { return binary val[0], val[2], 'isEqualTo:' }
+    not_equality:   expr NOT_EQUALITY expr { return binary val[0], val[2], 'isNotEqualTo:' }
+    logical_or:     expr OR expr           { return binary val[0], val[2], 'logicalOr:' }
+    logical_and:    expr AND expr          { return binary val[0], val[2], 'logicalAnd:' }
+    greater_than:   expr GT expr           { return binary val[0], val[2], 'isGreaterThan:' }
+    less_than:      expr LT expr           { return binary val[0], val[2], 'isLessThan:' }
+    greater_or_eq:  expr GTE expr          { return binary val[0], val[2], 'isGreaterOrEqualTo:' }
+    less_or_eq:     expr LTE expr          { return binary val[0], val[2], 'isLessOrEqualTo:' }
+
+    unary_op:  unary_not
+             | unary_plus
+             | unary_minus
+             | unary_complement
+
+    unary_not:        BANG  expr { return unary val[1], 'unaryNot' }
+    unary_plus:       PLUS  expr { return unary val[1], 'unaryPlus' }
+    unary_minus:      MINUS expr { return unary val[1], 'unaryMinus' }
+    unary_complement: TILDE expr { return unary val[1], 'unaryComplement' }
+
+    literal:   integer
+             | float
+             | string
+             | nil
+             | true
+             | false
+             | self
+
+    float:          FLOAT                       { return n :Float,   val[0] }
+    integer:        INTEGER                     { return n :Integer, val[0] }
+    nil:            NIL                         { return n :Nil }
+    true:           TRUE                        { return n :True }
+    false:          FALSE                       { return n :False }
+    self:           SELF                        { return n :Self }
+
+    string:         STRING                      { return n :String,  val[0] }
+                  | interpolated_string
+                  | empty_string
+
+    interpolated_string: DOUBLE_TICK_STRING interpolated_string_contents DOUBLE_TICK_STRING_END { return val[1] }
+    interpolation:       INTERPOLATE_START expr INTERPOLATE_END { return val[1] }
+    interpolated_string_contents:   interpolated_string_chunk   { return n :InterpolatedString, val[0] }
+                                  | interpolated_string_contents interpolated_string_chunk { val[0].append(val[1]); return val[0] }
+    interpolated_string_chunk:   chars         { return val[0] }
+                               | interpolation { return to_string(val[0]) }
+    empty_string: DOUBLE_TICK_STRING DOUBLE_TICK_STRING_END { return n :String, '' }
+
+    chars:   CHAR       { return n :String, val[0] }
+           | chars CHAR { val[0].append(val[1]); return val[0] }
+end
+
+---- inner
+
+attr_accessor :lexer, :scopes, :state
+
+def initialize lexer
+  @lexer  = lexer
+  @state  = []
+  @scopes = []
+  push_scope
+end
+
+def ast
+  @ast ||= do_parse
+  @scopes.first
+end
+
+def on_error t, val, vstack
+  line = lexer.line
+  col  = lexer.column
+  message = "Unexpected #{token_to_str t} at #{lexer.filename} line #{line}:#{col}:\n\n"
+
+  start = line - 5 > 0 ? line - 5 : 0
+  i_size = line.to_s.size
+  (start..(start + 5)).each do |i|
+    message << sprintf("\t%#{i_size}d: %s\n", i, lexer.get_line(i))
+    message << "\t#{' ' * i_size}  #{'-' * (col - 1)}^\n" if i == line
+  end
+
+  raise SyntaxError, message
+end
+
+def next_token
+  nt = lexer.next_computed_token
+  # just use a state stack for now, we'll have to do something
+  # more sophisticated soon.
+  if nt && nt.first == :state
+    if nt.last
+      state.push << nt.last
+    else
+      state.pop
+    end
+    next_token
+  else
+    nt
+  end
+end
+
+def push_scope
+  new_scope = Huia::AST::Scope.new scope
+  new_scope.file   = lexer.filename
+  new_scope.line   = lexer.line
+  new_scope.column = lexer.column
+  scopes.push new_scope
+  new_scope
+end
+
+def pop_scope
+  scopes.pop
+end
+
+def scope
+  scopes.last
+end
+
+def binary left, right, method
+  node(:MethodCall, left, node(:CallSignature, method, [right]))
+end
+
+def unary left, method
+  node(:MethodCall, left, node(:CallSignature, method))
+end
+
+def node type, *args
+  Huia::AST.const_get(type).new(*args).tap do |n|
+    n.file   = lexer.filename
+    n.line   = lexer.line
+    n.column = lexer.column
+  end
+end
+alias n node
+
+def allocate_local name
+  node(:Variable, name).tap do |n|
+    scope.allocate_local n
+  end
+end
+
+def allocate_local_assignment name, value
+  node(:Assignment, name, value).tap do |n|
+    scope.allocate_local n
+  end
+end
+
+def this_closure
+  allocate_local('@')
+end
+
+def scope_instance
+  node(:ScopeInstance, scope)
+end
+
+def constant name
+  return scope_instance if name == 'self'
+  node(:Constant, name)
+end
+
+def to_string expr
+  node(:MethodCall, expr, node(:CallSignature, 'toString'))
+end

data/test/assets/journey.y

@@ -0,0 +1,47 @@
+class Journey::Parser
+
+token SLASH LITERAL SYMBOL LPAREN RPAREN DOT STAR OR
+
+rule
+  expressions
+    : expressions expression  { result = Cat.new(val.first, val.last) }
+    | expression              { result = val.first }
+    | or
+    ;
+  expression
+    : terminal
+    | group
+    | star
+    ;
+  group
+    : LPAREN expressions RPAREN { result = Group.new(val[1]) }
+    ;
+  or
+    : expressions OR expression { result = Or.new([val.first, val.last]) }
+    ;
+  star
+    : STAR       { result = Star.new(Symbol.new(val.last)) }
+    ;
+  terminal
+    : symbol
+    | literal
+    | slash
+    | dot
+    ;
+  slash
+    : SLASH              { result = Slash.new('/') }
+    ;
+  symbol
+    : SYMBOL             { result = Symbol.new(val.first) }
+    ;
+  literal
+    : LITERAL            { result = Literal.new(val.first) }
+  dot
+    : DOT                { result = Dot.new(val.first) }
+    ;
+
+end
+
+---- header
+
+require 'journey/parser_extras'

data/test/assets/liquor.y

@@ -0,0 +1,313 @@
+# Copyright (c) 2012-2013 Peter Zotov  <whitequark@whitequark.org>
+#              2012 Yaroslav Markin  <yaroslav@markin.net>
+#              2012 Nate Gadgibalaev  <nat@xnsv.ru>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+class Liquor::Parser
+  token comma dot endtag ident integer keyword lblock lblock2 lbracket
+        linterp lparen op_div op_eq op_gt op_geq op_lt op_leq op_minus
+        op_mod op_mul op_neq op_not op_plus pipe plaintext rblock
+        rbracket rinterp rparen string tag_ident
+
+  prechigh
+    left dot
+    nonassoc op_uminus op_not
+    left op_mul op_div op_mod
+    left op_plus op_minus
+    left op_eq op_neq op_lt op_leq op_gt op_geq
+    left op_and
+    left op_or
+  preclow
+
+  expect 15
+
+  start block
+
+rule
+  block: /* empty */
+      { result = [] }
+    | plaintext block
+      { result = [ val[0], *val[1] ] }
+    | interp block
+      { result = [ val[0], *val[1] ] }
+    | tag block
+      { result = [ val[0], *val[1] ] }
+
+  interp:
+      linterp expr rinterp
+      { result = [ :interp, retag(val), val[1] ] }
+    | linterp filter_chain rinterp
+      { result = [ :interp, retag(val), val[1] ] }
+
+  primary_expr:
+      ident
+    | lparen expr rparen
+      { result = [ val[1][0], retag(val), *val[1][2..-1] ] }
+
+  expr:
+      integer
+    | string
+    | tuple
+    | ident function_args
+      { result = [ :call,   retag(val), val[0], val[1] ] }
+    | expr lbracket expr rbracket
+      { result = [ :index,  retag(val), val[0], val[2] ] }
+    | expr dot ident function_args
+      { result = [ :external, retag(val), val[0], val[2], val[3] ] }
+    | expr dot ident
+      { result = [ :external, retag(val), val[0], val[2], nil ] }
+    | op_minus expr =op_uminus
+      { result = [ :uminus, retag(val), val[1] ] }
+    | op_not expr
+      { result = [ :not, retag(val), val[1] ] }
+    | expr op_mul expr
+      { result = [ :mul, retag(val), val[0], val[2] ] }
+    | expr op_div expr
+      { result = [ :div, retag(val), val[0], val[2] ] }
+    | expr op_mod expr
+      { result = [ :mod, retag(val), val[0], val[2] ] }
+    | expr op_plus expr
+      { result = [ :plus, retag(val), val[0], val[2] ] }
+    | expr op_minus expr
+      { result = [ :minus, retag(val), val[0], val[2] ] }
+    | expr op_eq expr
+      { result = [ :eq, retag(val), val[0], val[2] ] }
+    | expr op_neq expr
+      { result = [ :neq, retag(val), val[0], val[2] ] }
+    | expr op_lt expr
+      { result = [ :lt, retag(val), val[0], val[2] ] }
+    | expr op_leq expr
+      { result = [ :leq, retag(val), val[0], val[2] ] }
+    | expr op_gt expr
+      { result = [ :gt, retag(val), val[0], val[2] ] }
+    | expr op_geq expr
+      { result = [ :geq, retag(val), val[0], val[2] ] }
+    | expr op_and expr
+      { result = [ :and, retag(val), val[0], val[2] ] }
+    | expr op_or expr
+      { result = [ :or, retag(val), val[0], val[2] ] }
+    | primary_expr
+
+  tuple:
+      lbracket tuple_content rbracket
+      { result = [ :tuple, retag(val), val[1].compact ] }
+
+  tuple_content:
+      expr comma tuple_content
+      { result = [ val[0], *val[2] ] }
+    | expr
+      { result = [ val[0] ] }
+    | /* empty */
+      { result = [ ] }
+
+  function_args:
+      lparen function_args_inside rparen
+      { result = [ :args, retag(val), *val[1] ] }
+
+  function_args_inside:
+      expr function_keywords
+      { result = [ val[0], val[1][2] ] }
+    | function_keywords
+      { result = [ nil,    val[0][2] ] }
+
+  function_keywords:
+      keyword expr function_keywords
+      { name = val[0][2].to_sym
+        tail = val[2][2]
+        loc  = retag([ val[0], val[1] ])
+
+        if tail.include? name
+          @errors << SyntaxError.new("duplicate keyword argument `#{val[0][2]}'",
+              tail[name][1])
+        end
+
+        hash = {
+          name => [ val[1][0], loc, *val[1][2..-1] ]
+        }.merge(tail)
+
+        result = [ :keywords, retag([ loc, val[2] ]), hash ]
+      }
+    | /* empty */
+      { result = [ :keywords, nil, {} ] }
+
+  filter_chain:
+      expr pipe filter_chain_cont
+      { result = [ val[0], *val[2] ].
+            reduce { |tree, node| node[3][2] = tree; node }
+      }
+
+  filter_chain_cont:
+      filter_call pipe filter_chain_cont
+      { result = [ val[0], *val[2] ] }
+    | filter_call
+      { result = [ val[0] ] }
+
+  filter_call:
+      ident function_keywords
+      { ident_loc = val[0][1]
+        empty_args_loc = { line:  ident_loc[:line],
+                           start: ident_loc[:end] + 1,
+                           end:   ident_loc[:end] + 1, }
+        result = [ :call, val[0][1], val[0],
+                   [ :args, val[1][1] || empty_args_loc, nil, val[1][2] ] ]
+      }
+
+  tag:
+      lblock ident expr tag_first_cont
+      { result = [ :tag, retag(val), val[1], val[2], *reduce_tag_args(val[3][2]) ] }
+    | lblock ident tag_first_cont
+      { result = [ :tag, retag(val), val[1], nil,    *reduce_tag_args(val[2][2]) ] }
+
+  # Racc cannot do lookahead across rules. I had to add states
+  # explicitly to avoid S/R conflicts. You are not expected to
+  # understand this.
+
+  tag_first_cont:
+      rblock
+      { result = [ :cont,  retag(val), [] ] }
+    | keyword tag_first_cont2
+      { result = [ :cont,  retag(val), [ val[0], *val[1][2] ] ] }
+
+  tag_first_cont2:
+      rblock block lblock2 tag_next_cont
+      { result = [ :cont2, val[0][1],  [ [:block, val[0][1], val[1] ], *val[3] ] ] }
+    | expr tag_first_cont
+      { result = [ :cont2, retag(val), [ val[0], *val[1][2] ] ] }
+
+  tag_next_cont:
+      endtag rblock
+      { result = [] }
+    | keyword tag_next_cont2
+      { result = [ val[0], *val[1] ] }
+
+  tag_next_cont2:
+      rblock block lblock2 tag_next_cont
+      { result = [ [:block, val[0][1], val[1] ], *val[3] ] }
+    | expr keyword tag_next_cont3
+      { result = [ val[0], val[1], *val[2] ] }
+
+  tag_next_cont3:
+      rblock block lblock2 tag_next_cont
+      { result = [ [:block, val[0][1], val[1] ], *val[3] ] }
+    | expr tag_next_cont
+      { result = [ val[0], *val[1] ] }
+
+---- inner
+  attr_reader :errors, :ast
+
+  def initialize(tags={})
+    super()
+
+    @errors = []
+    @ast    = nil
+    @tags   = tags
+  end
+
+  def success?
+    @errors.empty?
+  end
+
+  def parse(string, name='(code)')
+    @errors.clear
+    @name = name
+    @ast  = nil
+
+    begin
+      @stream = Lexer.lex(string, @name, @tags)
+      @ast = do_parse
+    rescue Liquor::SyntaxError => e
+      @errors << e
+    end
+
+    success?
+  end
+
+  def next_token
+    tok = @stream.shift
+    [ tok[0], tok ] if tok
+  end
+
+  TOKEN_NAME_MAP = {
+    :comma    => ',',
+    :dot      => '.',
+    :lblock   => '{%',
+    :rblock   => '%}',
+    :linterp  => '{{',
+    :rinterp  => '}}',
+    :lbracket => '[',
+    :rbracket => ']',
+    :lparen   => '(',
+    :rparen   => ')',
+    :pipe     => '|',
+    :op_not   => '!',
+    :op_mul   => '*',
+    :op_div   => '/',
+    :op_mod   => '%',
+    :op_plus  => '+',
+    :op_minus => '-',
+    :op_eq    => '==',
+    :op_neq   => '!=',
+    :op_lt    => '<',
+    :op_leq   => '<=',
+    :op_gt    => '>',
+    :op_geq   => '>=',
+    :keyword  => 'keyword argument name',
+    :kwarg    => 'keyword argument',
+    :ident    => 'identifier',
+  }
+
+  def on_error(error_token_id, error_token, value_stack)
+    if token_to_str(error_token_id) == "$end"
+      raise Liquor::SyntaxError.new("unexpected end of program", {
+        file: @name
+      })
+    else
+      type, (loc, value) = error_token
+      type = TOKEN_NAME_MAP[type] || type
+
+      raise Liquor::SyntaxError.new("unexpected token `#{type}'", loc)
+    end
+  end
+
+  def retag(nodes)
+    loc = nodes.map { |node| node[1] }.compact
+    first, *, last = loc
+    return first if last.nil?
+
+    {
+      file:  first[:file],
+      line:  first[:line],
+      start: first[:start],
+      end:    last[:end],
+    }
+  end
+
+  def reduce_tag_args(list)
+    list.each_slice(2).reduce([]) { |args, (k, v)|
+      if v[0] == :block
+        args << [ :blockarg, retag([ k, v ]), k, v[2] || [] ]
+      else
+        args << [ :kwarg,    retag([ k, v ]), k, v          ]
+      end
+    }
+  end