koi-reference-parser 0.0.2 → 0.0.3

data/README.rdoc

@@ -21,6 +21,7 @@ And turns it into an {Abstract Syntax Tree (AST)}[http://en.wikipedia.org/wiki/A
             <AdditionOperator "+">
             <IntegerLiteral "2">
             
+The AST is also available in a far more portable format based on nested hashes by calling the to_hash method on the root AST node.
 
 === Installation
 
@@ -36,6 +37,9 @@ This parser is normally installed as part of Koi's default toolchain. However if
   include KoiReferenceParser
   
   ast = Parser.parse( program_text )
+  
+  # Get the AST as a portable nested hash
+  ast_hash = ast.to_hash
 
 === Author & Credits
 

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.3

data/lib/parser/koi-reference-parser.treetop

@@ -1,5 +1,14 @@
+# This file is a Parsing Expression Grammar (PEG) for the Koi
+# programming language. It is interpreted by the PEG engine Treetop.
+
+# More info on PEGs: http://en.wikipedia.org/wiki/Parsing_expression_grammar
+# More info on Treetop: http://treetop.rubyforge.org/
+
 grammar KoiReferenceParser
 
+  ##################
+  # Program structure
+
   rule block
     statement+ <Block>
   end
@@ -8,6 +17,9 @@ grammar KoiReferenceParser
     space? ( hash_assignment / assignment / if / unless / function_call ) ';'? space? <Statement>
   end
   
+  ##################
+  # Literals
+  
   rule nil
     "nil" <NilLiteral>
   end
@@ -36,6 +48,24 @@ grammar KoiReferenceParser
     '$'? [a-zA-Z] [a-zA-Z0-9_]* <Identifier>
   end
   
+  ##################
+  # Hash literals
+  
+  rule hash
+    '{' key_value_list? '}' <HashLiteral>
+  end
+  
+  rule key_value_list
+    key_value+ <KeyValueList>
+  end
+  
+  rule key_value
+    space? expression space? '=>' space? expression space? ','? <KeyValue>
+  end
+  
+  ##################
+  # Assignment
+  
   rule hash_assignment
     identifier hash_accessor_list space? assignment_operator space? ( hash / function_definition / expression ) <HashAssignment>
   end
@@ -70,18 +100,15 @@ grammar KoiReferenceParser
     'function(' space? identifier space? ')' 
       block
     space? 'end' <FunctionDefinition>
-  end
-  
+  end  
+
   ##################
-  # Whitespace
+  # Hash component access
   
-  rule space
-    [\s]+ <Whitespace>
+  rule hash_access
+    identifier hash_accessor_list <HashAccess>
   end
   
-  ##################
-  # Hashes
-  
   rule hash_accessor_list
     hash_accessor+ <HashAccessorList>
   end
@@ -89,26 +116,6 @@ grammar KoiReferenceParser
   rule hash_accessor
     '[' expression ']' <HashAccessor>
   end
-  
-  rule hash
-    '{' key_value_list? '}' <HashLiteral>
-  end
-  
-  rule key_value_list
-    key_value+ <KeyValueList>
-  end
-  
-  rule key_value
-    space? expression space? '=>' space? expression space? ','? <KeyValue>
-  end
-  
-  rule key
-    space? expression space? <Key>
-  end
-  
-  rule value
-    space? expression space? <Value>
-  end
 
   ##################
   # Expressions
@@ -117,12 +124,8 @@ grammar KoiReferenceParser
     space? (hash_access / comparative / additive) <Expression>
   end
   
-  rule hash_access
-    identifier hash_accessor_list <HashAccess>
-  end
-  
   rule comparative
-    additive space? (equality_operator / inequality_operator / greater_than_operator / less_than_operator) space? additive <ComparativeExpression>
+    additive space? comparative_operator space? additive <ComparativeExpression>
   end
   
   rule additive
@@ -130,12 +133,6 @@ grammar KoiReferenceParser
     /
     multitive 
   end
-  
-  rule additive_operator
-    addition_operator 
-    /
-    subtraction_operator 
-  end
 
   rule multitive
     primary space? multitive_operator space? multitive <MultitiveExpression>
@@ -143,17 +140,26 @@ grammar KoiReferenceParser
     primary 
   end
   
-  rule multitive_operator
-    multiplication_operator 
-    /
-    division_operator  
-  end  
-  
   rule primary
     function_call / nil / false / true / identifier / float / integer / string 
     /
     '(' space? expression space? ')' <Expression>
   end
+  
+  ##################
+  # Operator sets
+  
+  rule comparative_operator
+    equality_operator / inequality_operator / greater_than_operator / less_than_operator
+  end
+  
+  rule additive_operator
+    addition_operator / subtraction_operator 
+  end
+  
+  rule multitive_operator
+    multiplication_operator / division_operator  
+  end
 
   ##################
   # Operators
@@ -193,5 +199,12 @@ grammar KoiReferenceParser
   rule less_than_operator
     '<' <LessThanOperator>
   end
+  
+  ##################
+  # Whitespace
+  
+  rule space
+    [\s]+
+  end
 
 end

data/lib/parser/parser.rb

@@ -8,21 +8,12 @@ class Parser
     tree = @@parser.parse(data)
     
     if(tree.nil?)
-      if( data.length > 80 )
-        error_detail = "\n\n" + data.slice(@@parser.index - 120, 80).gsub("\n", ";")
-        error_detail += "\n" + data.slice(@@parser.index - 40, 80).gsub("\n", ";")
-        error_detail += "\n" + " "*40 + "^\n"
-      else
-        error_detail = "\n\n" + data.gsub("\n", ";")
-        error_detail += "\n" + " " * @@parser.index + "^\n"
-      end
-      raise ParseError, "Parse error at index: #{@@parser.index}#{error_detail}"
+      raise ParseError, "Parse error at offset: #{@@parser.index}"
     end
     
     # clean up the tree by removing all nodes of default type 'SyntaxNode'
     self.clean_tree(tree)
-    # clean up the tree further by removing all whitespace nodes
-    self.clean_whitespace(tree)
+    
     return tree
   end
   
@@ -33,11 +24,5 @@ class Parser
       root_node.elements.delete_if{|node| node.class.name == "Treetop::Runtime::SyntaxNode" }
       root_node.elements.each {|node| self.clean_tree(node) }
     end
-    
-    def self.clean_whitespace(root_node)
-      return if(root_node.elements.nil?)
-      root_node.elements.delete_if{|node| node.class.name == "KoiReferenceParser::Whitespace" }
-      root_node.elements.each {|node| self.clean_whitespace(node) }
-    end
 
 end

data/lib/parser/syntax_nodes.rb

@@ -57,18 +57,9 @@ module KoiReferenceParser
     class KeyValue < Treetop::Runtime::SyntaxNode
     end
     
-    class Key < Treetop::Runtime::SyntaxNode
-    end
-    
-    class Value < Treetop::Runtime::SyntaxNode
-    end
-    
     class Identifier < Treetop::Runtime::SyntaxNode
     end
     
-    class Whitespace < Treetop::Runtime::SyntaxNode
-    end
-    
     class AssignmentOperator < Treetop::Runtime::SyntaxNode
     end
     

data/test/{parser/functional → functional}/function_call_as_argument.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_helper.rb'))
 
 class FunctionCallAsArgumentTest < Test::Unit::TestCase
 

data/test/{parser/functional → functional}/simple_program_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_helper.rb'))
 
 class SimpleProgramTest < Test::Unit::TestCase
 

data/test/helpers/assert_assigns.rb

@@ -0,0 +1,9 @@
+def assert_assigns(node_type, value, tree)
+  assert_kind_of Block, tree
+  assert_kind_of Statement, tree.elements.last
+  assert_kind_of Assignment, tree.elements.last.elements.last
+  assert_kind_of Identifier, tree.elements.last.elements.last.elements[0]
+  assert_kind_of AssignmentOperator, tree.elements.last.elements.last.elements[1]
+  assert_kind_of node_type, tree.elements.last.elements.last.elements[2]
+  assert_equal value, tree.elements.last.elements.last.elements[2].text_value unless(value.nil?)
+end

data/test/helpers/assert_assigns_expression.rb

@@ -0,0 +1,10 @@
+def assert_assigns_expression(node_type, value, tree)
+  assert_kind_of Block, tree
+  assert_kind_of Statement, tree.elements.last
+  assert_kind_of Assignment, tree.elements.last.elements.last
+  assert_kind_of Identifier, tree.elements.last.elements.last.elements[0]
+  assert_kind_of AssignmentOperator, tree.elements.last.elements.last.elements[1]
+  assert_kind_of Expression, tree.elements.last.elements.last.elements[2]
+  assert_kind_of node_type, tree.elements.last.elements.last.elements[2].elements.first
+  assert_equal value, tree.elements.last.elements.last.elements[2].elements.first.text_value unless(value.nil?)
+end

data/test/helpers/assert_expression.rb

@@ -0,0 +1,11 @@
+def assert_expression(tree, expression_type, operator)
+  assert_kind_of Block, tree
+  assert_kind_of Statement, tree.elements.first
+  assert_kind_of Assignment, tree.elements.first.elements.last
+  assert_kind_of Identifier, tree.elements.first.elements.last.elements[0]
+  assert_kind_of AssignmentOperator, tree.elements.first.elements.last.elements[1]
+  assert_kind_of Expression, tree.elements.first.elements.last.elements[2]
+  assert_kind_of expression_type, tree.elements.first.elements.last.elements[2].elements.first
+  assert_equal 3, tree.elements.first.elements.last.elements[2].elements.first.elements.size
+  assert_kind_of operator, tree.elements.first.elements.last.elements[2].elements.first.elements[1] unless(operator.nil?)
+end

data/test/helpers/assert_identifier.rb

@@ -0,0 +1,6 @@
+def assert_identifier(tree)
+  assert_kind_of Block, tree
+  assert_kind_of Statement, tree.elements.first
+  assert_kind_of Assignment, tree.elements.first.elements.last
+  assert_kind_of Identifier, tree.elements.first.elements.last.elements.first
+end

data/test/test_helper.rb

@@ -4,6 +4,7 @@ require 'test/unit'
 require_files = []
 require_files << File.join(File.dirname(__FILE__), '..', 'lib', 'koi-reference-parser.rb')
 require_files.concat Dir[File.join(File.dirname(__FILE__), 'setup', '*.rb')]
+require_files.concat Dir[File.join(File.dirname(__FILE__), 'helpers', '*.rb')]
 
 require_files.each do |file|
   require File.expand_path(file)

data/test/unit/assignment/assignment_of_hash_value_test.rb

@@ -0,0 +1,19 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
+
+class AssignmentOfHashValueTest < Test::Unit::TestCase
+
+  include KoiReferenceParser
+  
+  test "assignment of hash key" do
+    tree = Parser.parse('test = test[1]')
+    assert_kind_of Block, tree
+    assert_kind_of Statement, tree.elements.first
+    assert_kind_of Assignment, tree.elements.first.elements.first
+    assert_kind_of Identifier, tree.elements.first.elements.first.elements[0]
+    assert_kind_of Expression, tree.elements.first.elements.first.elements[2]
+    assert_kind_of HashAccess, tree.elements.first.elements.first.elements[2].elements.first
+    assert_kind_of Identifier, tree.elements.first.elements.first.elements[2].elements.first.elements[0]
+    assert_kind_of HashAccessorList, tree.elements.first.elements.first.elements[2].elements.first.elements[1]
+  end
+  
+end

data/test/{parser/unit/hash_accessor_test.rb → unit/assignment/assignment_to_hash_test.rb}

@@ -1,9 +1,9 @@
 require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
 
-class HashAccessorTest < Test::Unit::TestCase
+class AssignmentToHashTest < Test::Unit::TestCase
 
   include KoiReferenceParser
-
+  
   test "assignment to hash key" do
     tree = Parser.parse('test[1] = 1')
     assert_kind_of Block, tree
@@ -25,17 +25,5 @@ class HashAccessorTest < Test::Unit::TestCase
     assert_kind_of HashAccessor, tree.elements.first.elements.first.elements[1].elements.first
     assert_equal 3, tree.elements.first.elements.first.elements[1].elements.length
   end
-  
-  test "assignment of hash key" do
-    tree = Parser.parse('test = test[1]')
-    assert_kind_of Block, tree
-    assert_kind_of Statement, tree.elements.first
-    assert_kind_of Assignment, tree.elements.first.elements.first
-    assert_kind_of Identifier, tree.elements.first.elements.first.elements[0]
-    assert_kind_of Expression, tree.elements.first.elements.first.elements[2]
-    assert_kind_of HashAccess, tree.elements.first.elements.first.elements[2].elements.first
-    assert_kind_of Identifier, tree.elements.first.elements.first.elements[2].elements.first.elements[0]
-    assert_kind_of HashAccessorList, tree.elements.first.elements.first.elements[2].elements.first.elements[1]
-  end
 
 end

data/test/unit/assignment/boolean_assignment_test.rb

@@ -0,0 +1,17 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
+
+class BooleanAssignmentTest < Test::Unit::TestCase
+
+  include KoiReferenceParser
+  
+  test "assignment of boolean true" do
+    tree = Parser.parse('test = true')
+    assert_assigns_expression(TrueLiteral, nil, tree)
+  end
+  
+  test "assignment of boolean false" do
+    tree = Parser.parse('test = false')
+    assert_assigns_expression(FalseLiteral, nil, tree)
+  end
+  
+end

data/test/unit/assignment/float_assignment_test.rb

@@ -0,0 +1,22 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
+
+class FloatAssignmentTest < Test::Unit::TestCase
+
+  include KoiReferenceParser
+  
+  test "assignment of float" do
+    tree = Parser.parse('test = 99.0009')
+    assert_assigns_expression(FloatLiteral, '99.0009', tree)
+  end
+  
+  test "assignment of explicitly positive float" do
+    tree = Parser.parse('test = +99.0009')
+    assert_assigns_expression(FloatLiteral, '+99.0009', tree)
+  end
+  
+  test "assignment of explicitly negative float" do
+    tree = Parser.parse('test = -99.0009')
+    assert_assigns_expression(FloatLiteral, '-99.0009', tree)
+  end
+  
+end

data/test/unit/assignment/function_call_assignment_test.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
+
+class FunctionCallAssignmentTest < Test::Unit::TestCase
+
+  include KoiReferenceParser
+  
+  test "assignment of result of function call" do
+    tree = Parser.parse('test = function(args)')
+    assert_assigns_expression(FunctionCall, nil, tree)
+  end
+  
+end

data/test/unit/assignment/function_definition_test.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
+
+class FunctionDefinitionAssignmentTest < Test::Unit::TestCase
+
+  include KoiReferenceParser
+  
+  test "assignment of function definition" do
+    tree = Parser.parse('test = function(args) blah = 1 end')
+    assert_assigns(FunctionDefinition, nil, tree)
+  end
+  
+end

data/test/unit/assignment/hash_literal_assignment_test.rb

@@ -0,0 +1,27 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'test_helper.rb'))
+
+class HashLiteralAssignmentTest < Test::Unit::TestCase
+
+  include KoiReferenceParser
+  
+  test "assignment of empty hash literal" do
+    tree = Parser.parse('test = {}')
+    assert_assigns(HashLiteral, nil, tree)
+  end
+  
+  test "assignment of hash literal with single key-value pair" do
+    tree = Parser.parse('test = { 1 => 2 }')
+    assert_assigns(HashLiteral, nil, tree)
+  end
+  
+  test "assignment of hash literal with 2 key-value pairs" do
+    tree = Parser.parse('test = { 1 => 2, 2 => 2 }')
+    assert_assigns(HashLiteral, nil, tree)
+  end
+  
+  test "assignment of hash literal with 3 key-value pairs" do
+    tree = Parser.parse('test = { 1 => 2, 2 => 2, 3 => 2 }')
+    assert_assigns(HashLiteral, nil, tree)
+  end 
+  
+end