rockit 0.7.1

data/rakefile

@@ -0,0 +1,200 @@
+require 'rake'
+require 'rake/clean'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+
+#require 'rubygems'
+
+#require 'rbconfig'
+#require 'find'
+#require 'ftools'
+#require 'fileutils'
+#include Config
+
+PROJECT = "rockit"
+
+VERSION_FILE = "VERSION"
+MANIFEST_FILE = "Manifest"
+README_FILE = "README"
+TODO_FILE = "TODO"
+RAKE_FILE = "rakefile"
+CHANGELOG = "Changelog"
+
+
+task :default => :ptest
+
+
+#############################################################################
+# Version related
+#############################################################################
+
+# Read the version from the version file
+def Version()
+  File.open(VERSION_FILE) {|fh| fh.read.strip}
+end
+
+# Update the version num on disc
+def update_version
+  v = Version().split(".").map {|vp| vp.to_i}
+  new_v = yield(v)
+  File.open(VERSION_FILE, "w") {|fh| fh.write(new_v)}
+  puts Version()
+end
+
+def inc_tiny; update_version {|v| "#{v[0]}.#{v[1]}.#{v[2]+1}\n"}; end
+def inc_minor; update_version {|v| "#{v[0]}.#{v[1]+1}.0\n"}; end
+def inc_major; update_version {|v| "#{v[0]+1}.0.0\n"}; end
+
+task :version do
+  puts Version()
+end
+task :inc_tiny  do
+  inc_tiny()
+end
+task :inc_minor do 
+  inc_minor()
+end
+task :inc_major do
+  inc_major()
+end
+
+#############################################################################
+# Test related
+#############################################################################
+
+PackratTestFiles = FileList["tests/unit/packrat/test*.rb"]
+PackratAcceptanceTestFiles = FileList["tests/acceptance/packrat/**/atest*.rb"]
+
+task :setup_for_test do
+  $: << "lib"
+end
+
+def load_files(list)
+  list.each {|f| puts "Loading #{f}"; require f}
+end
+
+task :ptest => [:setup_for_test] do
+  load_files PackratTestFiles
+end
+
+task :patest => [:setup_for_test] do
+  load_files PackratAcceptanceTestFiles
+end
+
+task :unit_tests => [:setup_for_test] do
+  load_files FileList["tests/unit/**/test*.rb"]
+end
+
+task :acceptance_tests do
+  load_files FileList["tests/acceptance/**/test*.rb"]
+end
+
+task :utest => :unit_tests
+task :ut => :unit_tests
+task :atest => :acceptance_tests
+task :at => :acceptance_tests
+
+#############################################################################
+# Gem and package related
+#############################################################################
+
+BaseFileToIncludeGlobs = [
+    "README", "TODO", "BUGS", "LICENSE", "VERSION",
+    "rakefile", "Manifest",
+    "lib/packrat/**/*",
+    "lib/rockit/prettyprint/**/*",
+    "lib/rockit/tree/**/*",
+    "lib/util/**/*",
+]
+
+TestFilesToIncludeGlobs = [
+    "tests/acceptance/**/*", 
+    "tests/unit/**/*",
+]
+
+PkgFileGlobs = BaseFileToIncludeGlobs + TestFilesToIncludeGlobs
+
+GemSpec = Gem::Specification.new do |s|
+  s.name = PROJECT
+  s.version = Version()
+  s.platform = Gem::Platform::RUBY
+  s.summary = "Rockit is the Ruby Object-oriented Compiler construction toolKIT"
+  s.author = "Robert Feldt"
+  s.email = "Robert.Feldt@gmail.com"
+  s.homepage = "http://www.pronovomundo.com/projects/ruby/rockit"
+
+  s.description = <<EOS
+Rockit is a potent parser generator and gives you AST's (Abstract Syntax Tree's) which you can pattern match and pretty-print. Rockit does not distinghuish between lexing and parsing so the generated parsers are scanner-/lexer-less. The vision is to extend Rockit with more advanced compiler-related abilities including back-ends and code generation. However, currently the focus is on parsing and AST-related tasks such as transformation.
+EOS
+
+  s.require_path = "lib"
+  s.requirements << "none"
+
+  s.has_rdoc = false
+  # Dependency on rake?
+
+  s.files = []
+  PkgFileGlobs.each do |globpat|
+    fs = Dir.glob(globpat).delete_if {|i| i.include?(".svn")}
+    s.files += fs
+  end
+
+  s.extensions = []
+end
+
+Rake::GemPackageTask.new(GemSpec) do |pkg|
+  pkg.need_zip = true
+  pkg.need_tar = true
+end
+
+# Reformat the 'svn log' output to become the Changelog file. We assume
+# log entries are single lines with each log event starting with an asterisk
+# "*" and use those as split markers. We also insert newlines before the
+# svn log event separators "-----------------------...---".
+def reformat_svn_log_to_changelog(svnLogAsString)
+  s = svnLogAsString.split(/, \*/).join(",\n\n* ")
+  s.split(/^------/).join("\n------")
+end
+
+task :changelog do
+  # We must run 'svn up' first to make sure the log is recent
+  system "svn up"
+  File.open("Changelog", "w") do |fh|
+    fh.write reformat_svn_log_to_changelog(`svn log`)
+  end
+end
+
+#############################################################################
+# Rubyforge related
+#############################################################################
+
+# Create a package (a name for a top-level "item" which can then have multiple
+# releases where each release can have multiple files)
+def create_package(packageName = "rockit")
+  system "rubyforge create_package #{PROJECT} #{packageName}"
+end
+
+# Release a <file> with a given <release_name> in the given <package>.
+def release_file(file, release_name, package = PROJECT)
+  group_id = PROJECT
+  system "rubyforge add_release #{group_id} #{package} #{release_name} #{file}"
+end
+
+# Delete a package (and all its files)
+def delete_package(packageName)
+  system "rubyforge create_package rockit #{ARGV.last}"
+end
+
+# We use a temporary package name for now, should be PROJECT later...
+PACKAGE_NAME = "1799"  # if of the test package in rockit group
+
+desc "Create package and release to rubyforge"
+task :rubyforge_release => [:package] do
+  system "rubyforge login"
+  release_name = "#{PROJECT}-#{Version()}"
+  files_to_release = FileList["pkg/#{release_name}.*"]
+  files_to_release.each do |file|
+    puts "Releasing #{file} to Rubyforge"
+    release_file(file, release_name, PACKAGE_NAME)
+  end
+end

data/tests/acceptance/packrat/minibasic/atest_minibasic.rb

@@ -0,0 +1,45 @@
+require 'test/unit'
+
+require File.join(File.dirname(__FILE__), "minibasic")
+
+class ATestMiniBasic < Test::Unit::TestCase
+  def assert_parse(exp, str)
+    res = MiniBasic::Parser.parse_string(str)
+    assert_equal(exp, res)
+  end
+
+  def assert_stmt(expStmt, str)
+    assert_parse([expStmt], str)
+  end
+
+  include MiniBasic::Grammar::ASTs
+
+  def test_01
+    assert_stmt(PrintLn["PRINTLN"], "PRINTLN")
+  end
+
+  def test_02
+    assert_stmt(Read["READ", :A], "READ A")
+  end
+
+  def test_03
+    assert_stmt(Print["PRINT", :BC], "PRINT  BC")
+  end
+
+  def test_04
+    assert_stmt(Assign[:D, ":=", 1], "D := 1")
+    assert_stmt(Assign[:D, ":=", 1], "D :=1")
+    assert_stmt(Assign[:D, ":=", 1], "D:=1")
+  end
+
+  def test_05
+    assert_stmt(For["FOR", :A, ":=", 1, "TO", 3, 
+                 [Assign[:S, ":=", OpExpr[:S, "+", :A]],
+                  Print["PRINT", :S],
+                 ],
+                 "NEXT"], "FOR A := 1 TO 3 S:= S + A PRINT S NEXT")
+  end
+
+  def atest_06    
+  end
+end

data/tests/acceptance/packrat/minibasic/minibasic.rb

@@ -0,0 +1,137 @@
+#############################################################################
+# Grammar and interpreter for a mini version of Basic. 
+# Author: Robert Feldt
+#
+# The grammar is based on the minibasic example in SableCC.
+#
+# Here's the copyright notice from the original file SableCC file:
+#
+# Copyright (C) 1997, 1998, 1999 J-Meg inc.  All rights reserved.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation; either version 2.1 of the License, or
+# (at your option) any later version.
+#
+# This file is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+# License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this file (in the file "COPYING-LESSER"); if not,
+# write to the Free Software Foundation, Inc., 59 Temple Place,
+# Suite 330, Boston, MA  02111-1307  USA
+#
+# If you have any question, send an electronic message to
+# Etienne M. Gagnon, M.Sc. (egagnon@j-meg.com), or write to:
+#
+# J-Meg inc.
+# 11348 Brunet
+# Montreal-Nord (Quebec)
+# H1G 5G1  Canada
+#############################################################################
+require 'packrat/grammar'
+
+module MiniBasic
+  class Interpreter
+    def initialize
+      # For variables and their values. Default value is 0.
+      @vars = Hash.new(0)
+    end
+
+    def eval(sexpr)
+      case sexpr.first
+      when :Statements
+        sexpr.statements.each {|stmt| mb_eval(stmt)}
+      when "If"
+        if mb_eval(sexpr.condition) # What is true and false in basic?
+          mb_eval(sexpr.statements)
+        elsif sexpr.optelse
+          mb_eval(sexpr.optelse[2])
+        end
+      when "For"
+        for i in (mb_eval(sexpr.from)..mb_eval(sexpr.to))
+          $vars[sexpr.ident.id] = i
+          mb_eval(sexpr.statements)
+        end
+      when "Read"
+        print "? "; STDOUT.flush
+        $vars[sexpr.ident.id] = STDIN.gets.to_i   # Error catching?!
+      when "Print"
+        print mb_eval(sexpr.message); STDOUT.flush
+      when "PrintLn"
+        print "\n"; STDOUT.flush
+      when "Assignment"
+        $vars[sexpr.ident.id] = mb_eval(sexpr.expression)
+      when "Condition"
+        map = {">" => :>, "<" => :<, "=" => :==}
+        mb_eval(sexpr.left).send(map[sexpr.op], mb_eval(sexpr.right))
+      when "BinExpr"
+        map = {"+"=>:+, "-"=>:-, "*"=>:*, "/"=>"/".intern, "MOD"=>"%".intern }
+        mb_eval(sexpr.left).send(map[sexpr.op], mb_eval(sexpr.right))
+      when "String"
+        sexpr.value[1..-2] # Skip leading and trailing double quotes
+      when "Identifier"
+        $vars[sexpr.id]
+      when "Number"
+        sexpr.lexeme.to_i
+      end
+    end
+  end
+
+  Grammar = Packrat::Grammar.new do
+    start_symbol :Program
+
+    # Spacing (S) and Forced Spacing (FS) 
+    S  = hidden(/\s*/)
+    FS = hidden(/\s\s*/)
+
+    prod :Program, [S, :Statements, eos(), lift(0)]
+
+    prod :Statements, [plus(:Statement), lift(0)]
+
+    rule :Statement, [
+      ['IF', FS, :Condition, FS, 'THEN', FS,
+	:Statements, FS,
+	maybe(:OptElse), S,
+	'ENDIF', S, ast(:If)],
+
+      ['FOR', FS, :Identifier, S, ':=', S, :Expr, FS, 'TO', FS, :Expr, S,
+	:Statements, S,
+	'NEXT', S, ast(:For)],
+
+      ['READ', FS, :Identifier, S, ast(:Read)],
+
+      ['PRINTLN', S, ast(:PrintLn)],
+
+      ['PRINT', FS, any(:Expr, :String), S, ast(:Print)],
+
+      [:Identifier, S, ':=', S, :Expr, S, ast(:Assign)],
+    ]
+
+    prod :OptElse, ['ELSE', FS, :Statements, lift(1)]
+
+    prod :Condition, [:Expr, S, any('<', '>', '='), S, :Expr, ast(:Cond)]
+
+    # This is crude! No precedence levels or handling of associativity.
+    rule :Expr, [
+      [:BaseExpr, S, any('+', '-', '*', '/', 'MOD'), S, :BaseExpr,
+       ast(:OpExpr)
+      ],
+      [:BaseExpr, lift(0)],
+    ]
+
+    rule :BaseExpr, [
+      [:Number, lift(0)],
+      [:Identifier, lift(0)],
+      ['(', S, :Expr, S, ')', lift(2)],
+    ]
+
+    prod :String, [/"[^"]*"/, lift(0)]
+    prod :Identifier, [/[A-Z]([A-Z0-9])*/, lift(0) {|r| r.intern}]
+    prod :Number, [/[0-9]+/, lift(0) {|r| r.to_i}]
+  end
+
+  Parser = Grammar.interpreting_parser
+end

data/tests/acceptance/rockit/dparser/atest_any_operator.rb

@@ -0,0 +1,33 @@
+require 'test/unit'
+require 'rockit/dparser'
+
+class ATestAnyOperator < Test::Unit::TestCase
+  AnyTestGrammar = Rockit::DParser::Grammar.new do
+    start :S, [["a", any("b", "c"), any(:D, :E), ast(:S, 0, 1, 2)]]
+    ("d".."e").map {|s| term(s.upcase.intern, s)}
+  end
+
+  Parser = AnyTestGrammar.new_parser
+
+  def test_01
+    ast = Parser.parse "abd"
+    assert_equal("a", ast[0])
+    assert_equal("b", ast[1])
+    assert_equal("d", ast[2])
+
+    ast = Parser.parse "abe"
+    assert_equal("a", ast[0])
+    assert_equal("b", ast[1])
+    assert_equal("e", ast[2])
+
+    ast = Parser.parse "acd"
+    assert_equal("a", ast[0])
+    assert_equal("c", ast[1])
+    assert_equal("d", ast[2])
+
+    ast = Parser.parse "ace"
+    assert_equal("a", ast[0])
+    assert_equal("c", ast[1])
+    assert_equal("e", ast[2])
+  end
+end

data/tests/acceptance/rockit/dparser/atest_arithmetic_grammar.rb

@@ -0,0 +1,30 @@
+require 'test/unit'
+require 'rockit/dparser'
+
+class ATestArithmeticGrammar < Test::Unit::TestCase
+  ArithmeticTestGrammar = Rockit::DParser::Grammar.new do
+    start :Expr, [
+      [:Digit, "+", :Digit, ast(:Add, 0, 2)],
+      [:Digit, "*", :Digit, ast(:Mult, 0, 2)],
+    ]
+    term :Digit, /[0-9]+/
+  end
+
+  def test_03_use_parser
+    parser = ArithmeticTestGrammar.new_parser
+
+    ast = parser.parse "12+34"
+    assert_equal("Add", ast.name)
+    assert_kind_of(parser.astclass_of_name(:Add), ast)
+    assert_equal(2, ast.num_children)
+    assert_equal("12", ast[0])
+    assert_equal("34", ast[1])
+
+    ast = parser.parse "567*980"
+    assert_equal("Mult", ast.name)
+    assert_kind_of(parser.astclass_of_name(:Mult), ast)
+    assert_equal(2, ast.num_children)
+    assert_equal("567", ast[0])
+    assert_equal("980", ast[1])
+  end
+end

data/tests/acceptance/rockit/dparser/atest_list_operator.rb

@@ -0,0 +1,57 @@
+require 'test/unit'
+require 'rockit/dparser'
+
+class ATestListOperator < Test::Unit::TestCase
+  ListTestGrammar = Rockit::DParser::Grammar.new do
+    start :S, [list(:Number, ",")], value(0)
+    term :Number, [/[0-9]+/]
+  end
+
+  Parser = ListTestGrammar.new_parser
+  
+  def test_01_two_value_list
+    ary = Parser.parse("1, 2")
+    assert_equal(["1", "2"], ary)
+  end
+
+  def test_02_multi_value_list
+    ary = Parser.parse("1, 2, 34, 567,890")
+    assert_equal(["1", "2", "34", "567", "890"], ary)
+  end
+
+  def test_03_single_value_list
+    ary = Parser.parse("9731")
+    assert_equal(["9731"], ary)
+  end
+
+  ListAndAstTestGrammar = Rockit::DParser::Grammar.new do
+    start :S, [list(:Digit)], ast(:S, 0)
+    term :Digit, [/[0-9]+/]
+  end
+
+  AstParser = ListAndAstTestGrammar.new_parser
+
+  def test_04_multiple_occurences_and_ast
+    ast = AstParser.parse("7,5,6,9")
+    assert_equal("S", ast.name)
+    assert_kind_of(AstParser.astclass_of_name(:S), ast)
+    assert_equal(1, ast.num_children)
+    assert_equal(["7", "5", "6", "9"], ast[0])
+  end
+
+  def test_05_single_occurence_and_ast
+    ast = AstParser.parse("745")
+    assert_equal("S", ast.name)
+    assert_kind_of(AstParser.astclass_of_name(:S), ast)
+    assert_equal(1, ast.num_children)
+    assert_equal(["745"], ast[0])
+  end
+
+  def atest_06_no_occurence_and_ast
+    ast = AstParser.parse("")
+    assert_equal("S", ast.name)
+    assert_kind_of(AstParser.astclass_of_name(:S), ast)
+    assert_equal(1, ast.num_children)
+    assert_equal([], ast[0])
+  end
+end

data/tests/acceptance/rockit/dparser/atest_mult_operator.rb

@@ -0,0 +1,60 @@
+require 'test/unit'
+require 'rockit/dparser'
+
+class ATestMultOperator < Test::Unit::TestCase
+  MultTestGrammar = Rockit::DParser::Grammar.new do
+    start :S, [mult(:Digit)], value(0)
+    term :Digit, [/[0-9]/]
+  end
+
+  Parser = MultTestGrammar.new_parser
+
+  def test_01_multiple_occurences
+    (2..20).each do |i|
+      digit = rand(10).to_s
+      ary = Parser.parse(digit * i)
+      assert_equal((1..i).map {digit}, ary)
+    end
+  end
+
+  def test_02_single_occurence
+    ary = Parser.parse("7")
+    assert_equal(["7"], ary)
+  end
+
+  def test_03_no_occurence
+    ary = Parser.parse("")
+    assert_equal([], ary)
+  end
+
+  MultAndAstTestGrammar = Rockit::DParser::Grammar.new do
+    start :S, [mult(:Digit)], ast(:S, 0)
+    term :Digit, [/[0-9]/]
+  end
+
+  AstParser = MultAndAstTestGrammar.new_parser
+
+  def test_04_multiple_occurences_and_ast
+    ast = AstParser.parse("7 5 6 9")
+    assert_equal("S", ast.name)
+    assert_kind_of(AstParser.astclass_of_name(:S), ast)
+    assert_equal(1, ast.num_children)
+    assert_equal(["7", "5", "6", "9"], ast[0])
+  end
+
+  def test_05_single_occurence_and_ast
+    ast = AstParser.parse("7")
+    assert_equal("S", ast.name)
+    assert_kind_of(AstParser.astclass_of_name(:S), ast)
+    assert_equal(1, ast.num_children)
+    assert_equal(["7"], ast[0])
+  end
+
+  def test_06_no_occurence_and_ast
+    ast = AstParser.parse("")
+    assert_equal("S", ast.name)
+    assert_kind_of(AstParser.astclass_of_name(:S), ast)
+    assert_equal(1, ast.num_children)
+    assert_equal([], ast[0])
+  end
+end

data/tests/acceptance/rockit/dparser/atest_operator_grammar.rb

@@ -0,0 +1,61 @@
+require 'test/unit'
+require 'rockit/dparser'
+
+class ATestOperatorGrammar < Test::Unit::TestCase
+  MaybeTestGrammar = Rockit::DParser::Grammar.new do
+    start :M, [[:A, maybe(:Digit)]], ast(:M, 0, 1)
+    term :A, ["a"]
+    term :Digit, [/[0-9]+/]
+  end
+
+  def test_01_use_maybe_parser
+    parser = MaybeTestGrammar.new_parser
+
+    ast = parser.parse "a1"
+    assert_equal("M", ast.name)
+    assert_kind_of(parser.astclass_of_name(:M), ast)
+    assert_equal(2, ast.num_children)
+    assert_equal("a", ast[0])
+    assert_equal("1", ast[1])    
+
+    ast = parser.parse "a"
+    assert_equal("M", ast.name)
+    assert_kind_of(parser.astclass_of_name(:M), ast)
+    assert_equal(2, ast.num_children)
+    assert_equal("a", ast[0])
+    assert_equal(nil, ast[1])    
+  end
+
+  OperatorTestGrammar = Rockit::DParser::Grammar.new do
+    start :Str, [[:A, plus(:B), plus(:A), mult(:C), maybe(:D)]], 
+      ast(:Str, 0, 1, 3, 4)
+    term :A, ["a"]
+    term :B, ["b"]
+    term :C, ["c"]
+    term :D, ["d"]
+  end
+
+  def test_02_use_parser
+    parser = OperatorTestGrammar.new_parser
+    ast = parser.parse "abbbac"
+    assert_equal("Str", ast.name)
+    assert_kind_of(parser.astclass_of_name(:Str), ast)
+    assert_equal(4, ast.num_children)
+    assert_equal("a", ast[0])
+    assert_equal(["b", "b", "b"], ast[1])
+    assert_equal(["c"], ast[2])
+    assert_equal(nil, ast[3])
+  end
+
+  def test_03_use_parser
+    parser = OperatorTestGrammar.new_parser
+    ast = parser.parse "a b a c  c d"
+    assert_equal("Str", ast.name)
+    assert_kind_of(parser.astclass_of_name(:Str), ast)
+    assert_equal(4, ast.num_children)
+    assert_equal("a", ast[0])
+    assert_equal(["b"], ast[1])
+    assert_equal(["c", "c"], ast[2])
+    assert_equal("d", ast[3])
+  end
+end

data/tests/acceptance/rockit/dparser/atest_plus_operator.rb

@@ -0,0 +1,55 @@
+require 'test/unit'
+require 'rockit/dparser'
+
+class ATestPlusOperator < Test::Unit::TestCase
+  PlusTestGrammar = Rockit::DParser::Grammar.new do
+    start :S, [plus(:A)], value(0)
+    term :A, ["a"]
+  end
+
+  Parser = PlusTestGrammar.new_parser
+
+
+  def test_01_multiple_occurences
+    (2..20).each do |i|
+      ary = Parser.parse("a" * i)
+      assert_equal((1..i).map {"a"}, ary)
+    end
+  end
+
+  def test_02_single_occurence
+    ary = Parser.parse("a")
+    assert_equal(["a"], ary)
+  end
+
+  def test_03_no_occurence
+    assert_raises(RuntimeError) {Parser.parse("b")}
+  end
+
+  PlusAndAstTestGrammar = Rockit::DParser::Grammar.new do
+    start :S, [plus(:Digit)], ast(:S, 0)
+    term :Digit, [/[0-9]/]
+  end
+
+  AstParser = PlusAndAstTestGrammar.new_parser
+
+  def test_04_multiple_occurences_and_ast
+    ast = AstParser.parse("7 5 6 9")
+    assert_equal("S", ast.name)
+    assert_kind_of(AstParser.astclass_of_name(:S), ast)
+    assert_equal(1, ast.num_children)
+    assert_equal(["7", "5", "6", "9"], ast[0])
+  end
+
+  def test_05_single_occurence_and_ast
+    ast = AstParser.parse("7")
+    assert_equal("S", ast.name)
+    assert_kind_of(AstParser.astclass_of_name(:S), ast)
+    assert_equal(1, ast.num_children)
+    assert_equal(["7"], ast[0])
+  end
+
+  def test_06_no_occurence_and_ast
+    assert_raises(RuntimeError) {AstParser.parse("b")}
+  end
+end

data/tests/acceptance/rockit/dparser/atest_samples_calculator.rb

@@ -0,0 +1,14 @@
+require 'test/unit'
+require 'rockit/dparser'
+require 'calculator/calculator'
+
+require File.join(File.dirname(__FILE__), "calc_tests_common")
+
+class ATestSamplesCalculator < Test::Unit::TestCase
+  include CalcTestsCommon
+
+  def setup
+    @grammar = Samples::CalculatorGrammar
+    @calculator = Samples::Calculator.new
+  end
+end