rubex 0.0.1 → 0.1

data/lib/rubex/parser.racc

@@ -1,53 +1,724 @@
 class Rubex::Parser
-token 
-  kDEF kEND kRETURN kDTYPE_I32 tLPAREN tRPAREN tIDENTIFIER tPLUS tNL tCOMMA
+token
+  kDEF kEND kRETURN kPRINT  kIF    kELSIF  kELSE   kTHEN  kSTATIC_ARRAY kFOR 
+  kDOT_EACH kDO     kTRUE   kFALSE kNIL    kSTRUCT kUNION kALIAS        kLIB 
+  kCFUNC    kSELF   kNULL   kFWD   kATTACH kRAISE  kBREAK kBLOCK_GIVEN  kYIELD
+  kBEGIN    kRESCUE kENSURE kWHILE  kCLASS kDATA_VAR
+
+  kDTYPE_UINT   kDTYPE_LINT kDTYPE_LLINT  kDTYPE_CHAR   kDTYPE_I8   kDTYPE_I16
+  kDTYPE_I32    kDTYPE_I64  kDTYPE_UI8    kDTYPE_UI16   kDTYPE_UI32 kDTYPE_UI64
+  kDTYPE_INT    kDTYPE_F32  kDTYPE_F64    kDTYPE_LF64   kDTYPE_ROBJ kDTYPE_ULINT
+  kDTYPE_ULLINT kDTYPE_VOID kDTYPE_SIZE_T kDTYPE_RB_STR kDTYPE_UCHAR kDTYPE_BOOL
+  tCUSTOM_DTYPE kDTYPE_RB_ARR kDTYPE_RB_HSH
+
+  tLPAREN tRPAREN tLSQUARE tRSQUARE tIDENTIFIER tNL     tCOMMA tSQUOTE tSCOLON
+  EOF     tDOT    tQMARK   tSYMBOL  tLBRACE     tRBRACE tCOLON
+
+  tPLUS tMINUS tSTAR tDIVIDE tEXPO      tMODULUS tASSIGN tEQ tLT tLTEQ tGT
+  tGTEQ tANDOP tOROP tBANG   tOP_ASSIGN tSTRING  kSIZEOF
+  
+  tBIT_AND tBIT_OR tBIT_XOR tBIT_LSHIFT tBIT_RSHIFT tBIT_NOT
+
+  tINTEGER tFLOAT tSINGLE_CHAR
 
 prechigh
-  left tEXPO
-  left tMULTIPLY tDIVIDE tMODULUS
+  right tBANG tBIT_NOT
+  right ADDRESS_OF
+  right tEXPO
+  right TYPECAST
+  right kSIZEOF
+
+  right UMINUS
+  left tSTAR tDIVIDE tMODULUS
   left tPLUS tMINUS
+  left tBIT_OR tBIT_XOR
+  left tBIT_AND
+  left tBIT_LSHIFT tBIT_RSHIFT
+  left tLT tLTEQ tGT tGTEQ
+  nonassoc tEQ tNEQ
+  left tANDOP
+  left tOROP
 
-  right tEQUAL
+  right tASSIGN tOP_ASSIGN
 preclow
 
 rule
 
   program:
-    stmts { result = Node.new(val[0]) }
+    opt_terms top_compstmts opt_terms {result = Node.new(val[1])}
 
-  stmts:
-    stmts stmt tNL { result = [val[0], val[1]] }
-  | stmt tNL { result = [val[0]] }
+  top_compstmts:
+    top_compstmt { result = [val[0]] }
+  | top_compstmts terms top_compstmt { result = [*val[0], val[2]] }
+
+  top_compstmt:
+    c_bindings {}
+  | top_stmts terms { result = val[0] }
+
+  top_stmts:
+    { result = [] }
+  | top_stmt { result = [val[0]] }
+  | top_stmts terms top_stmt { result = [*val[0], val[2]] }
+
+  top_stmt:
+    methods {}
+  | klass {}
+  | attached_klass {}
+  | struct_or_union_def {}
+
+  klass:
+    kCLASS tIDENTIFIER opt_inheritance eol top_stmts opt_terms kEND
+      { 
+        raise Rubex::SyntaxError unless val[1][0].match(/[A-Z]/)
+        result = TopStatement::Klass.new val[1], val[2], val[4]
+      }
+
+  attached_klass:
+    kCLASS tIDENTIFIER kATTACH dtype opt_inheritance eol top_stmts terms kEND
+      {
+        raise Rubex::SyntaxError unless val[1][0].match(/[A-Z]/)
+        result = TopStatement::AttachedKlass.new(val[1], val[3], val[4], val[6], 
+          location)
+      }
+
+  opt_inheritance:
+    { result = nil }
+  | tLT tIDENTIFIER { result = val[1] }
+
+  c_bindings:
+    kLIB tSTRING opt_compilation_cmds c_declarations terms kEND
+      {  
+        result = TopStatement::CBindings.new val[1], val[2], val[3], location
+      }
+
+  opt_compilation_cmds:
+    { result = [{}]  }
+  | compilation_cmd { result = [val[0]] }
+  | opt_compilation_cmds tCOMMA compilation_cmd { result = [*val[0], val[2]] }
+
+  compilation_cmd:
+    tIDENTIFIER tCOLON tSTRING 
+      { 
+        if val[0] == 'link'
+          result = { link: val[2] }
+        end
+      }
+
+  c_declarations:
+    {}
+  | c_declaration { result = [val[0]] }
+  | c_declarations terms c_declaration { result = [*val[0], val[2]] }
+
+  c_declaration:
+    c_function_declaration_or_var_decl
+  | struct_or_union_def
+  | alias_stmt
+
+  c_function_declaration_or_var_decl:
+    dtype opt_star tIDENTIFIER opt_c_func_arg_list
+      { 
+        if !val[3].empty? # function decl
+          val[3].pop # HACK: because declared external C functions don't have implicit self argument.
+          result = Statement::CFunctionDecl.new val[0], val[1], val[2], val[3]
+        else # var/macro decl
+          result = variable_decl_nodes([{
+            dtype: val[0], 
+            variables: [{
+              ptr_level: val[1],
+              ident: val[2]
+            }]
+          }])[0]
+        end
+      }
+
+  c_func_args:
+    { result = [] }
+  | c_func_normal_arg { result = [Expression::ArgDeclaration.new(val[0])] }
+  | c_func_args tCOMMA c_func_normal_arg
+      { 
+        result = [*val[0], Expression::ArgDeclaration.new(val[2])] 
+      }
+
+  opt_c_func_arg_list:
+      { result = Statement::ArgumentList.new([]) }
+  | tLPAREN c_func_args tRPAREN
+      { 
+        # self is a compulsory implicit argument for C methods.
+        val[1] << Expression::ArgDeclaration.new(
+            { dtype: 'object', variables: [ {ident: 'self' }] })
+        result = Statement::ArgumentList.new(val[1])
+      }
+
+  struct_or_union_def:
+    kSTRUCT tIDENTIFIER eol var_decl_stmts opt_terms kEND 
+      { 
+        add_dtype_to_lexer val[1]
+        result = Statement::CStructOrUnionDef.new(val[0], val[1], val[3].flatten,
+          location)
+      }
+
+  var_decl_stmts:
+    { result = [] }
+  | var_decl_stmt { result = [*val[0]] } 
+  | var_decl_stmts terms var_decl_stmt { result = [*val[0], val[2]] }
+
+  var_decl_stmt:
+    dtype var_decls
+      {
+        result = variable_decl_nodes([{ dtype: val[0], variables: val[1] }])
+      }
+
+  var_decls:
+    var_decls tCOMMA normal_decl { result = [*val[0], val[2]] }
+  | normal_decl { result = [val[0]] }
   
-  stmt: 
-    kDEF fname f_arglist stmts kEND 
+  alias_stmt:
+    kALIAS ident_or_custom_dtype tASSIGN aliased_type
+      { # TODO: take hash from aliased_type and turn into nodes.
+        add_dtype_to_lexer val[1]
+        result = Statement::Alias.new val[1], val[3], location
+      }
+
+  ident_or_custom_dtype:
+    tIDENTIFIER | tCUSTOM_DTYPE {}
+
+  aliased_type:
+    dtype opt_star
       { 
-        result = RubyMethodDef.new(val[1], val[2])
-        result.add_statements val[3]
+        result =  {
+          dtype: val[0],
+          variables: [
+            {
+              ptr_level: val[1],
+              ident: ""
+            }
+          ]
+        }
+      }
+  | kSTRUCT tIDENTIFIER opt_star
+      { 
+        result = {
+          dtype: "#{val[0]}#{val[1]}",
+          variables: [
+            {
+              ptr_level: val[2],
+              ident: ""
+            }
+          ]
+        }
+      }
+  | dtype opt_star tLPAREN opt_star opt_identifier tRPAREN opt_c_func_arg_list
+      {
+        result = {
+          dtype: val[0],
+          variables: [
+            {
+              ptr_level: val[3],
+              ident:{
+                return_ptr_level: val[1],
+                arg_list: val[6],
+                name: val[4]
+              }              
+            }
+          ]
+        }
       }
-  | kRETURN expr { result = Statement::Return.new val[1] }
-  | {}
 
-  fname:
-    tIDENTIFIER {}
+  opt_identifier:
+    { result = nil }
+  | tIDENTIFIER {}
+
+  methods:
+    ruby_method { result = val[0] }
+  | c_function { result = val[0] }
+
+  ruby_method:
+    kDEF opt_singleton f_name f_opt_arglist compstmt kEND
+      {
+        result = TopStatement::RubyMethodDef.new(
+          val[2], val[3], val[4], singleton: val[1])
+      }
+
+  c_function:
+    kCFUNC dtype opt_star tIDENTIFIER opt_c_func_arg_list eol compstmt kEND
+      { 
+        if val[4].empty? # since last arg of cfunc must be self.
+          val[4] = Statement::ArgumentList.new(
+            [
+              Expression::ArgDeclaration.new(
+                { dtype: 'object', variables: [ { ident: 'self' }] }
+              )
+            ]
+          ) 
+        end
+        result = TopStatement::CFunctionDef.new(val[1], val[2], val[3], 
+          val[4], val[6])
+      }
+
+  c_func_normal_arg:
+    dtype opt_star
+      {
+        result = {
+          dtype: val[0],
+          variables: [
+            {
+              ptr_level: val[1],
+              ident: ""
+            }
+          ]
+        } 
+      }
+  | dtype normal_decl { result = { dtype: val[0]  , variables: [val[1]] } }
+  | normal_decl       { result = { dtype: 'object', variables: [val[0]] } }
+
+  opt_singleton:
+    { result = false }
+  | kSELF tDOT { result = true }
+
+  compstmt:
+    stmts opt_terms { result = val[0] }
+
+  stmts:
+    { result = [] }
+  | stmt { result = [*val[0]] }
+  | stmts terms stmt { result = [*val[0], *val[2]] }
+
+
+  stmt:
+    var_decl_or_init { result = variable_decl_nodes val }
+  | kRETURN opt_expr { result = Statement::Return.new val[1], location }
+  | kPRINT opt_lparen command_arg_list opt_rparen 
+      { result = Statement::Print.new val[2], location }
+  | normal_init
+      {
+        match = val[0]
+        result = Statement::Assign.new match[:name], match[:value], location
+      }
+  | kIF expr then compstmt if_tail kEND
+      {
+        result = Statement::IfBlock.new val[1], [*val[3]], val[4], location
+      }
+  | stmt kIF expr
+      {
+        set_location
+        result = Statement::IfBlock.new val[2], [*val[0]], nil, location
+      }
+  | kFOR for_expr kDO compstmt kEND
+      {
+        result = Statement::For.new *val[1], val[3], location
+      }
+  | kWHILE expr kDO compstmt kEND
+      {
+        result = Statement::While.new val[1], val[3], location
+      }
+  | op_assign {}
+  | struct_or_union_def
+  | forward_declaration
+  | alias_stmt
+  | expr { result = Statement::Expression.new(val[0], location) }
+  | kRAISE opt_lparen command_arg_list opt_rparen
+      { 
+        result = Statement::Raise.new(Statement::ActualArgList.new(val[2]))
+      }
+  | kBREAK { result = Statement::Break.new(location) }
+  | kYIELD opt_lparen command_arg_list opt_rparen
+      {
+        result = Statement::Yield.new(Statement::ActualArgList.new(val[2]))
+      }
+  | begin_block
+
+  begin_block:
+    kBEGIN terms compstmt opt_begin_block_tails kEND
+      { 
+        result = Statement::BeginBlock::Begin.new val[2], val[3], location
+      }
+
+  opt_begin_block_tails:
+    opt_rescue_blocks opt_else_block opt_ensure_block
+      { 
+        tails = []
+        tails.concat(val[0]) if val[0]
+        tails << val[1]      if val[1]
+        tails << val[2]      if val[2]
+
+        result = tails
+      }
+
+  opt_rescue_blocks:
+      { result = nil }
+  | rescue_block { result = [val[0]] }
+  | opt_rescue_blocks rescue_block { result = [*val[0], val[1]]  }
+
+  rescue_block:
+    kRESCUE tIDENTIFIER terms compstmt
+      { 
+        err = Expression::Name.new(val[1]) 
+        result = Statement::BeginBlock::Rescue.new err, nil, val[3] || [], location 
+      }
+
+  opt_else_block:
+      { result = nil }
+  | kELSE terms compstmt
+      { result = Statement::BeginBlock::Else.new val[2] || [], location }
+
+  opt_ensure_block:
+      { result = nil }
+  | kENSURE terms compstmt
+      { result = Statement::BeginBlock::Ensure.new val[2] || [], location }
+
+  forward_declaration:
+    kFWD kSTRUCT tIDENTIFIER
+    {
+      val.flatten!
+      add_dtype_to_lexer val[2]
+      result = Statement::ForwardDecl.new val[1], val[2], location
+    }
+
+  op_assign:
+    atomic_value tOP_ASSIGN expr
+      {
+        #  TODO: error if lvalue is a literal
+        result = op_assign val
+      }
+
+  for_expr:
+    tLPAREN actual_for_expr tRPAREN { result = val[1] }
+  | actual_for_expr
+
+  actual_for_expr:
+    expr_value lt_or_lteq tIDENTIFIER lt_or_lteq expr_value { result = val }
+  | expr_value gt_or_gteq tIDENTIFIER gt_or_gteq expr_value { result = val }
+
+  lt_or_lteq:
+    tLT | tLTEQ
+
+  gt_or_gteq:
+    tGT | tGTEQ
+
+  then:
+    kTHEN | eol
+
+  if_tail:
+    {}
+  | opt_else {}
+  | kELSIF expr then compstmt if_tail
+      {
+        result = Statement::IfBlock::Elsif.new val[1], [*val[3]], val[4], location
+      }
 
-  f_arglist:
-    tLPAREN f_args tRPAREN tNL { result = val[1] }
+  opt_else:
+    kELSE compstmt { result = Statement::IfBlock::Else.new val[1], location }
+
+  var_decl_or_init:
+    dtype decls_or_inits
+      {
+        result = {
+          dtype: val[0],
+          variables: val[1]
+        }
+      }
+
+  decls_or_inits:
+    decls_or_inits tCOMMA var_init { result = [*val[0], val[2]] }
+  | decls_or_inits tCOMMA normal_decl { result = [*val[0], val[2]] }
+  | normal_decl { result = [val[0]] }
+  | var_init { result = [val[0]] }
+
+  normal_decl:
+    # possible function pointer declaration
+    opt_star tLPAREN opt_star actual_normal_decl tRPAREN opt_c_func_arg_list 
+      {
+        result = {}
+
+        if val[5] # function (pointer) decl
+          result[:ptr_level] = val[2]
+          result[:ident] = {
+            :return_ptr_level => val[0],
+            :name             => val[3],
+            :arg_list         => val[5]
+          }
+        else
+          result[:ptr_level] = val[0].join val[2]
+          result[:ident]    = val[3]
+        end
+      }
+  | opt_star actual_normal_decl opt_c_func_arg_list
+      {
+        result = {}
+
+        if !val[2].empty?
+          result[:ptr_level] = val[0]
+          result[:ident] = {
+            :name => val[1],
+            :arg_list => val[2]
+          }
+        else
+          result[:ptr_level] = val[0]
+          result[:ident] = val[1]
+        end
+      }
+
+  actual_normal_decl: tIDENTIFIER | array_ref
+
+  var_init:
+    normal_decl tASSIGN expr
+      {
+        result = val[0]
+        value = val[2]
+
+        value.c_array = true if value.is_a?(Expression::Literal::ArrayLit)
+        result[:value] = value
+      }
+
+  opt_star:
+    { result = nil }
+  | opt_star tSTAR { result = val.join }
+
+  array_list:
+    { result = [] }
+  | expr { result = [*val[0]] }
+  | array_list tCOMMA expr { result = [*val[0], val[2]] }
+
+  normal_init:
+    tIDENTIFIER tASSIGN expr 
+      { 
+        result = { 
+          name: Expression::Name.new(val[0]),
+          value: val[2]
+        } 
+      }
+  | array_ref tASSIGN expr   { result = { name: val[0], value: val[2] } }
+  | method_or_attr tASSIGN expr
+    {
+      result = {
+        name: Expression::CommandCall.new(val[0][0], val[0][1], []),
+        value: val[2]
+      }
+    }
+
+  literal:
+    tINTEGER     { result = Expression::Literal::Int.new val[0]    }
+  | tFLOAT       { result = Expression::Literal::Double.new val[0] }
+  | tSINGLE_CHAR { result = Expression::Literal::Char.new val[0]   }
+  | tSTRING      { result = Expression::Literal::StringLit.new val[0] }
+  | kTRUE        { result = Expression::Literal::True.new 'Qtrue' }
+  | kFALSE       { result = Expression::Literal::False.new 'Qfalse' }
+  | kNIL         { result = Expression::Literal::Nil.new 'Qnil' }
+  | tSYMBOL      { result = Expression::Literal::RubySymbol.new val[0] }
+  | kNULL        { result = Expression::Literal::CNull.new val[0] }
+  | tLSQUARE opt_eols array_list tRSQUARE 
+      { result = Expression::Literal::ArrayLit.new val[2] }
+  | tLBRACE opt_eols hash_data opt_eols tRBRACE
+      { result = Expression::Literal::HashLit.new val[2] }
+
+  hash_data:
+    { result = [] }
+  | hash_data_k_v { result = [val[0]] }
+  | hash_data tCOMMA opt_eols hash_data_k_v { result = [*val[0], val[3]] }
+
+  hash_data_k_v:
+    atomic_value rocket atomic_value { result = [val[0], val[2]] }
+
+  rocket:
+    tASSIGN tGT
+
+  f_name:
+    tIDENTIFIER tQMARK { result = "#{val[0]}#{val[1]}" }
+  | tIDENTIFIER tBANG  { result = "#{val[0]}#{val[1]}" }
+  | tIDENTIFIER        { result = val[0] }
+  | kNIL tQMARK        { result = "#{val[0]}#{val[1]}" }
+
+  f_opt_arglist:
+    { result = Statement::ArgumentList.new([]) }
+  | tLPAREN f_args tRPAREN eol { result = Statement::ArgumentList.new val[1] }
 
   f_args:
-    f_args tCOMMA f_normal_arg 
+    f_normal_arg 
       { 
-        result = ArgumentList.new
-        result.push val[0]
-        result.push val[2]
-      } 
-  | f_normal_arg { }
+        result = [Expression::ArgDeclaration.new(val[0])] 
+      }
+  | f_args tCOMMA f_normal_arg
+      {
+        result = [*val[0], Expression::ArgDeclaration.new(val[2])]
+      }
 
   f_normal_arg:
-    kDTYPE_I32 tIDENTIFIER { result = CBaseType.new val[0], val[1] }
+    dtype tIDENTIFIER opt_default_value
+      { 
+        result = { 
+          dtype: val[0],
+          variables: [
+            {
+              ident: val[1],
+              value: val[2]
+            }
+          ]
+        }
+      }
+  | tIDENTIFIER opt_default_value
+      {
+        result = { 
+          dtype: 'object',
+          variables: [
+            {
+              ident: val[0],
+              value: val[1]
+            }
+          ]
+        }
+      }
+
+  opt_default_value:
+    { result = nil }
+  |  tASSIGN atomic_value { result = val[1] }
 
   expr:
-    tIDENTIFIER tPLUS tIDENTIFIER { result = Expression::Addition.new val[0], val[2] }
+    actual_expr  {}
+  | typecast actual_expr   =TYPECAST 
+      {
+        val[1].typecast = val[0]
+        result = val[1]
+      }
+  | expr    tANDOP    expr  { result = binary_op val }
+  | expr    tOROP     expr  { result = binary_op val }
+
+  typecast:
+    tLT dtype opt_star tGT { result = Expression::Typecast.new(val[1], val[2]) }
+
+  actual_expr:
+    expr_value                   {}
+  | expr_cmp                     {}  
+  | tBANG   expr                 { result = unary_op val }
+  | tLPAREN expr      tRPAREN    { result = val[1] }
+
+  expr_value:
+    expr_value tPLUS       expr_value { result = binary_op val }
+  | expr_value tMINUS      expr_value { result = binary_op val }
+  | expr_value tSTAR       expr_value { result = binary_op val }
+  | expr_value tDIVIDE     expr_value { result = binary_op val }
+  | expr_value tEXPO       expr_value { result = binary_op val }
+  | expr_value tMODULUS    expr_value { result = binary_op val }
+  | expr_value tBIT_AND    expr_value { result = binary_op val }
+  | expr_value tBIT_OR     expr_value { result = binary_op val }
+  | expr_value tBIT_XOR    expr_value { result = binary_op val }
+  | expr_value tBIT_LSHIFT expr_value { result = binary_op val }
+  | expr_value tBIT_RSHIFT expr_value { result = binary_op val }
+  | tMINUS     expr_value  =UMINUS    { result = unary_op val  }
+  | tBIT_AND   expr_value  =ADDRESS_OF{ result = unary_op val  }
+  | tBIT_NOT   expr_value             { result = unary_op val  } 
+  | tLPAREN    expr_value  tRPAREN    { result = val[1] }
+  | atomic_value {}
+
+  expr_cmp:
+    expr_value tEQ       expr_value { result = binary_op val }
+  | expr_value tNEQ      expr_value { result = binary_op val }
+  | expr_value tLT       expr_value { result = binary_op val }
+  | expr_value tLTEQ     expr_value { result = binary_op val }
+  | expr_value tGT       expr_value { result = binary_op val }
+  | expr_value tGTEQ     expr_value { result = binary_op val }
+  | tLPAREN    expr_cmp  tRPAREN    { result = val[1] }
+
+  atomic_value:
+    array_ref {}
+  | literal {}
+  | tIDENTIFIER command_opt_args
+      {
+        args = val[1]
+        result =
+        if args.empty?
+          Expression::Name.new val[0]
+        else
+          Expression::CommandCall.new nil, val[0], args
+        end
+      }
+  | command_call
+  | kSIZEOF tLPAREN dtype opt_star tRPAREN 
+      { 
+        result = Expression::SizeOf.new(val[2], val[3])
+      }
+  | kBLOCK_GIVEN { result = Expression::BlockGiven.new }
+  | kDATA_VAR 
+      { 
+        result = Expression::ElementRef.new('data',
+          Expression::Literal::Int.new('0'))
+      }
+
+  command_call:
+    method_or_attr command_opt_args
+      {
+        result = Expression::CommandCall.new val[0][0], val[0][1], val[1]
+      }
+
+  method_or_attr:
+    atomic_value tDOT f_name
+      {
+        result = [val[0], val[2]]
+      }
+  | atomic_value tDOT array_ref
+    {
+      result = [val[0], val[2]]
+    }
+
+  command_opt_args:
+    { result = [] }
+  | tLPAREN command_arg_list tRPAREN { result = val[1] }
+
+  command_arg_list:
+    { result = [] }
+  | expr { result = val }
+  | command_arg_list tCOMMA expr { result = [*val[0], val[2]] }
+
+  array_ref:
+    tIDENTIFIER tLSQUARE expr tRSQUARE
+      {
+        result = Expression::ElementRef.new val[0], val[2]
+      }
+
+  dtype:
+    kDTYPE_UINT   | kDTYPE_LINT   | kDTYPE_LLINT | kDTYPE_CHAR   | kDTYPE_I8   |
+    kDTYPE_I16    | kDTYPE_I32    | kDTYPE_I64   | kDTYPE_UI8    | kDTYPE_UI16 |
+    kDTYPE_UI32   | kDTYPE_UI64   | kDTYPE_INT   | kDTYPE_F32    | kDTYPE_F64  |
+    kDTYPE_LF64   | kDTYPE_ROBJ   | kDTYPE_ULINT | kDTYPE_ULLINT | kDTYPE_VOID |
+    kDTYPE_SIZE_T | kDTYPE_RB_STR | kDTYPE_UCHAR | kDTYPE_BOOL   | tCUSTOM_DTYPE |
+    kDTYPE_RB_ARR | kDTYPE_RB_HSH
+      {
+        result = val[0]
+      }
+
+  opt_expr:
+    { result = nil }
+  | expr
+
+  opt_lparen:
+    {}
+  | tLPAREN
+
+  opt_rparen:
+    {}
+  | tRPAREN
+
+  opt_terms:
+    {}
+  | terms {}
+
+  terms:
+    term {}
+  | terms term {}
+
+  term:
+    tNL {}
+  | tSCOLON
+
+  opt_eols:
+    {}
+  | eols
+
+  eols:
+    eol
+  | eols eol
+
+  eol:
+    tNL {result = val[0]}
 end
 
 ---- header
@@ -57,11 +728,188 @@ require_relative 'ast.rb'
 include Rubex::AST
 
 ---- inner
+
+attr_reader :lineno, :location, :string
+
 def parse file_name
   @lexer = Rubex::Lexer.new
+  @yydebug = true
+  @custom_dtypes = {}
+  @prev_token = nil
   @lexer.parse_file file_name
 end
 
+def set_location
+  @location = @lexer.location
+end
+
+def set_string
+  @string = @lexer.ss.string
+end
+
+def set_lineno
+  @lineno = @lexer.lineno
+end
+
 def next_token
-  @lexer.next_token
+  set_location
+  set_string
+  set_lineno
+
+  t = @lexer.next_token
+  if !t.nil?
+    if t[0] == :tIDENTIFIER
+      if @custom_dtypes.has_key?(t[1]) &&
+        !(@prev_token[0] == :kSTRUCT || @prev_token[0] == :tDOT)
+        t = [:tCUSTOM_DTYPE, t[1]]
+      else
+        t = check_for_primitive_dtype(t)
+        t = check_for_keyword(t)
+      end
+    end
+  end
+
+  @prev_token = t
+  t
+end
+
+def check_for_keyword token
+  match = token[1]
+  if match == "def"
+    return [:kDEF, match]
+  elsif match == "cfunc"
+    return [:kCFUNC, match]
+  elsif match == "self"
+    return [:kSELF, match]
+  elsif match == "do"
+    return [:kDO, match]
+  elsif match == "end"
+    return [:kEND, match]
+  elsif match == "return"
+    return [:kRETURN, match]
+  elsif match == "print"
+    return [:kPRINT , match]
+  elsif match == "if"
+    return [:kIF    , match]
+  elsif match == "elsif"
+    return [:kELSIF , match]
+  elsif match == "else"
+    return [:kELSE  , match]
+  elsif match == "then"
+    return [:kTHEN  , match]
+  elsif match == "sizeof"
+    return [:kSIZEOF, match]
+  elsif match == "raise"
+    return [:kRAISE, match]
+  elsif match == "break"
+    return [:kBREAK, match]
+  elsif match == "begin"
+    return [:kBEGIN, match]
+  elsif match == "ensure"
+    return [:kENSURE, match]
+  elsif match == "rescue"
+    return [:kRESCUE, match]
+  end
+
+  return token
+end
+
+def check_for_primitive_dtype token
+  match = token[1]
+  if match == "char"
+    return [:kDTYPE_CHAR, match]
+  elsif match == "i8"
+    return [:kDTYPE_I8, match]
+  elsif match == "i16"
+    return [:kDTYPE_I16, match]
+  elsif match == "i32"
+    return [:kDTYPE_I32, match]
+  elsif match == "i64"
+    return [:kDTYPE_I64, match]
+  elsif match == "u8"
+    return [:kDTYPE_UI8, match]
+  elsif match == "u16"
+    return [:kDTYPE_UI16, match]
+  elsif match == "u32"
+    return [:kDTYPE_UI32, match]
+  elsif match == "u64"
+    return [:kDTYPE_UI64, match]
+  elsif match == "int"
+    return [:kDTYPE_INT, match]
+  elsif match == "long"
+    return [:kDTYPE_LINT, match]
+  elsif match == "f32"
+    return [:kDTYPE_F32, match]
+  elsif match == "float"
+    return [:kDTYPE_F32, match]
+  elsif match == "f64"
+    return [:kDTYPE_F64, match]
+  elsif match == "double"
+    return [:kDTYPE_F64, match]
+  elsif match == "object"
+    return[:kDTYPE_ROBJ, match]
+  elsif match == "void"
+    return[:kDTYPE_VOID, match]
+  elsif match == "size_t"
+    return [:kDTYPE_SIZE_T, match]
+  elsif match == "str"
+    return [:kDTYPE_RB_STR, match]
+  elsif match == "arr"
+    return [:kDTYPE_RB_ARR, match]
+  elsif match == "hsh"
+    return [:kDTYPE_RB_HSH, match]
+  elsif match == "yield"
+    return [:kYIELD, match]
+  elsif match == "bool"
+    return [:kDTYPE_BOOL, match]
+  end
+
+  token
+end
+
+def binary_op val
+  Expression::Binary.new val[0], val[1], val[2]
+end
+
+def unary_op val
+  Expression::Unary.new val[0], val[1]
+end
+
+# expr, op_assign, expr => expr = expr op expr
+def op_assign val
+  Statement::Assign.new(val[0], binary_op([val[0], val[1][0], val[2]]), location)
+end
+
+def variable_decl_nodes val
+  variables = val[0]
+  type = variables[:dtype]
+
+  result = variables[:variables].map do |var|
+    ident = var[:ident]
+    ptr_level = var[:ptr_level]
+
+    statement =
+    if ident.is_a?(Hash) # only if function pointer
+      dtype = { dtype: type, ident: ident }
+      Statement::CPtrDecl.new(dtype, ident[:name], var[:value], ptr_level,
+        location)
+    else
+      if ptr_level
+        Statement::CPtrDecl.new(type, var[:ident], var[:value], ptr_level,
+          location)
+      elsif ident.is_a?(Expression::ElementRef)
+        Statement::CArrayDecl.new(type, var[:ident], var[:value], location)
+      else
+        Statement::VarDecl.new(type, var[:ident], var[:value], location)
+      end
+    end
+
+    statement
+  end
+
+  result
+end
+
+def add_dtype_to_lexer dtype
+  @custom_dtypes[dtype] = true
 end