citrus 1.1.0 → 1.2.0

data/benchmark/seqpar.citrus

@@ -0,0 +1,14 @@
+grammar SeqPar
+  rule statement
+      'par ' (statement ' ')+ 'end'
+    | 'sequence' ' ' (statement ' ')+ 'end'
+    | 'seq' ' ' (statement ' ')+ 'end'
+    | ('fit' [\s] (statement ' ')+ 'end') {
+        def foo
+          "foo"
+        end
+      }
+    | 'art'+ [ ] (statement ' ')+ 'end'
+    | [A-Z] [a-zA-z0-9]*
+  end
+end

data/benchmark/seqpar.gnuplot

@@ -0,0 +1,15 @@
+f1(x) = a*x
+a = 0.5
+fit f1(x) 'before.dat' using 1:2 via a
+
+f2(x) = b*x
+b = 0.5
+fit f2(x) 'after.dat' using 1:2 via b
+
+set xlabel "Length of input"
+set ylabel "CPU time to parse"
+
+plot a*x        title 'a*x (Before)',\
+     b*x        title 'b*x (After)',\
+     "before.dat" using 1:2 title 'Before', \
+     "after.dat" using 1:2 title 'After'

data/benchmark/seqpar.rb

@@ -0,0 +1,110 @@
+# Benchmarking written by Bernard Lambeau and Jason Garber of the Treetop
+# project.
+#
+# To test your optimizations:
+#   1. Run ruby seqpar.rb
+#   2. cp after.dat before.dat
+#   3. Make your modifications to the Citrus code
+#   4. Run ruby seqpar.rb
+#   5. Run gnuplot seqpar.gnuplot
+#
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'citrus'
+require 'benchmark'
+
+srand(47562) # So it runs the same each time
+
+class Array
+  def sum
+    inject(0) {|m, x| m + x }
+  end
+
+  def mean
+    sum / size
+  end
+end
+
+class SeqParBenchmark
+  OPERATORS = ["seq", "fit", "art" * 5, "par", "sequence"]
+
+  def initialize
+    @where = File.expand_path('..', __FILE__)
+    Citrus.load(File.join(@where, 'seqpar'))
+    @grammar = SeqPar
+  end
+
+  # Checks the grammar
+  def check
+    [ "Task",
+      "seq Task end",
+      "par Task end",
+      "seq Task Task end",
+      "par Task Task end",
+      "par seq Task end Task end",
+      "par seq seq Task end end Task end",
+      "seq Task par seq Task end Task end Task end"
+    ].each do |input|
+      @grammar.parse(input)
+    end
+  end
+
+  # Generates an input text
+  def generate(depth=0)
+    return "Task" if depth > 7
+    return "seq #{generate(depth + 1)} end" if depth == 0
+
+    which = rand(OPERATORS.length)
+
+    case which
+    when 0
+      "Task"
+    else
+      raise unless OPERATORS[which]
+      buffer = "#{OPERATORS[which]} "
+      0.upto(rand(4) + 1) do
+        buffer << generate(depth + 1) << " "
+      end
+      buffer << "end"
+      buffer
+    end
+  end
+
+  # Launches benchmarking
+  def benchmark
+    number_by_size = Hash.new {|h,k| h[k] = 0}
+    time_by_size = Hash.new {|h,k| h[k] = 0}
+    0.upto(250) do |i|
+      input = generate
+      length = input.length
+      puts "Launching #{i}: #{input.length}"
+      # puts input
+      tms = Benchmark.measure { @grammar.parse(input) }
+      number_by_size[length] += 1
+      time_by_size[length] += tms.total * 1000
+    end
+    # puts number_by_size.inspect
+    # puts time_by_size.inspect
+
+    File.open(File.join(@where, 'after.dat'), 'w') do |dat|
+      number_by_size.keys.sort.each do |size|
+        dat << "#{size} #{(time_by_size[size]/number_by_size[size]).truncate}\n"
+      end
+    end
+
+    if File.exists?(File.join(@where, 'before.dat'))
+      before = {}
+      performance_increases = []
+      File.foreach(File.join(@where, 'before.dat')) do |line|
+        size, time = line.split(' ')
+        before[size] = time
+      end
+      File.foreach(File.join(@where, 'after.dat')) do |line|
+        size, time = line.split(' ')
+        performance_increases << (before[size].to_f - time.to_f) / before[size].to_f unless time == "0" || before[size] == "0"
+      end
+      puts "Average performance increase: #{(performance_increases.mean * 100 * 10).round / 10.0}%"
+    end
+  end
+end
+
+SeqParBenchmark.new.benchmark

data/citrus.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name = 'citrus'
-  s.version = '1.1.0'
-  s.date = '2010-05-18'
+  s.version = '1.2.0'
+  s.date = '2010-06-02'
 
   s.summary = 'Parsing Expressions for Ruby'
   s.description = 'Parsing Expressions for Ruby'
@@ -11,9 +11,12 @@ Gem::Specification.new do |s|
 
   s.require_paths = %w< lib >
 
-  s.files = Dir['lib/**/*.rb'] +
+  s.files = Dir['benchmark/*.rb'] +
+    Dir['benchmark/*.citrus'] +
+    Dir['benchmark/*.gnuplot'] +
     Dir['examples/**/*'] +
     Dir['extras/**/*'] +
+    Dir['lib/**/*.rb'] +
     Dir['test/*.rb'] +
     %w< citrus.gemspec Rakefile README >
 

data/extras/citrus.vim

@@ -15,8 +15,8 @@ syn case match
 
 syn match ctDoubleColon "::" contained
 syn match ctConstant "\u\w*" contained
-syn match ctVariable "\l\w*" contained
 syn match ctModule "\(\(::\)\?\u\w*\)\+" contains=ctDoubleColon,ctConstant contained
+syn match ctVariable "\a[a-zA-Z_-]*" contained
 
 " Comments
 syn match ctComment "#.*" contains=@Spell
@@ -56,7 +56,7 @@ syn match ctRule    "\<rule\>"    nextgroup=ctVariable skipwhite skipnl containe
 
 " Blocks
 syn region ctGrammarBlock start="\<grammar\>" matchgroup=ctGrammar end="\<end\>" contains=ctComment,ctGrammar,ctInclude,ctRoot,ctRuleBlock fold
-syn region ctRuleBlock start="\<rule\>" matchgroup=ctRule end="\<end\>" contains=ALLBUT,ctRequire,ctGrammar,ctInclude,ctRoot,ctConstant,ctVariable fold
+syn region ctRuleBlock start="\<rule\>" matchgroup=ctRule end="\<end\>" contains=ALLBUT,ctRequire,ctGrammar,ctInclude,ctRoot,ctConstant fold
 
 " Groups
 hi def link ctComment       Comment
@@ -84,7 +84,7 @@ hi def link ctStringDelimiter Delimiter
 hi def link ctRegexpSpecial  ctStringSpecial
 hi def link ctStringSpecial  Special
 
-hi def link ctQuantifier    ctOperator
+hi def link ctQuantifier    Number
 hi def link ctOperator      Operator
 
 let b:current_syntax = "citrus"

data/lib/citrus.rb

@@ -4,11 +4,11 @@
 #
 # http://github.com/mjijackson/citrus
 module Citrus
-  VERSION = [1, 1, 0]
+  VERSION = [1, 2, 0]
 
   Infinity = 1.0 / 0
 
-  autoload 'File', 'citrus/file'
+  autoload :File, 'citrus/file'
 
   # Returns the current version of Citrus as a string.
   def self.version
@@ -28,8 +28,7 @@ module Citrus
   # Evaluates the given Citrus parsing expression grammar +code+ in the global
   # scope. Returns an array of any grammar modules that were created.
   def self.eval(code)
-    file = File.parse(code)
-    file.value
+    File.parse(code).value
   end
 
   # This error is raised whenever a parse fails.
@@ -147,7 +146,6 @@ module Citrus
     # grammar.
     def rule(name, obj=nil)
       sym = name.to_sym
-
       obj = Proc.new.call if block_given?
 
       if obj
@@ -166,7 +164,8 @@ module Citrus
       raise "Cannot create rule \"#{name}\": " + e.message
     end
 
-    # Gets/sets the +name+ of the root rule of this grammar.
+    # Gets/sets the +name+ of the root rule of this grammar. If no root rule is
+    # explicitly specified, the name of this grammar's first rule is returned.
     def root(name=nil)
       @root = name.to_sym if name
       # The first rule in a grammar is the default root.
@@ -241,24 +240,47 @@ module Citrus
       rule
     end
 
-    # Parses the given +string+ from the given +offset+ using the rules in this
-    # grammar. A ParseError is raised if there is no match made or if
-    # +consume_all+ is +true+ and the entire input string cannot be consumed.
-    def parse(string, offset=0, enable_memo=false, consume_all=true)
-      raise "No root rule specified" unless root
+    # Parses the given input +string+ using the given +options+. If no match can
+    # be made, a ParseError is raised. See #default_parse_options for a detailed
+    # description of available parse options.
+    def parse(string, options={})
+      opts = default_parse_options.merge(options)
+
+      raise "No root rule specified" unless opts[:root]
 
-      root_rule = rule(root)
+      root_rule = rule(opts[:root])
       raise "No rule named \"#{root}\"" unless root_rule
 
-      input = Input.new(string, enable_memo)
-      match = input.match(root_rule, offset)
+      input = Input.new(string, opts[:enable_memo])
+      match = input.match(root_rule, opts[:offset])
 
-      if !match || (consume_all && match.length != string.length)
+      if !match || (opts[:consume_all] && match.length != string.length)
         raise ParseError.new(input)
       end
 
       match
     end
+
+    # The default set of options that is used in #parse. The options hash may
+    # have any of the following keys:
+    #
+    # offset::        The offset at which the parse should start. Defaults to 0.
+    # root::          The name of the root rule to use for the parse. Defaults
+    #                 to the name supplied by calling #root.
+    # consume_all::   If this is +true+ and the entire input string cannot be
+    #                 consumed, a ParseError will be raised. Defaults to +true+.
+    # enable_memo::   If this is +true+ the matches generated during a parse are
+    #                 memoized. This technique (also known as Packrat parsing)
+    #                 guarantees parsers will operate in linear time but costs
+    #                 significantly more in terms of time and memory required.
+    #                 Defaults to +false+.
+    def default_parse_options
+      { :offset => 0,
+        :root => root,
+        :consume_all => true,
+        :enable_memo => false
+      }
+    end
   end
 
   # This class represents the core of the parsing algorithm. It wraps the input
@@ -339,11 +361,11 @@ module Citrus
       end
     end
 
-    @uniq_id = 0
+    @unique_id = 0
 
     # Generates a new rule id.
     def self.new_id
-      @uniq_id += 1
+      @unique_id += 1
     end
 
     # The grammar this rule belongs to.
@@ -397,7 +419,7 @@ module Citrus
   private
 
     def extend_match(match)
-      match.extend(ext) if ext
+      match.ext = ext if ext
     end
 
     def create_match(data, offset)
@@ -446,15 +468,15 @@ module Citrus
   end
 
   # An Alias is a Proxy for a rule in the same grammar. It is used in rule
-  # definitions when a rule calls some other rule by name. The PEG notation is
-  # simply the name of another rule without any other punctuation, e.g.:
+  # definitions when a rule calls some other rule by name. The Citrus notation
+  # is simply the name of another rule without any other punctuation, e.g.:
   #
   #     name
   #
   class Alias
     include Proxy
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       rule_name.to_s
     end
@@ -473,15 +495,15 @@ module Citrus
 
   # A Super is a Proxy for a rule of the same name that was defined previously
   # in the grammar's inheritance chain. Thus, Super's work like Ruby's +super+,
-  # only for rules in a grammar instead of methods in a module. The PEG notation
-  # is the word +super+ without any other punctuation, e.g.:
+  # only for rules in a grammar instead of methods in a module. The Citrus
+  # notation is the word +super+ without any other punctuation, e.g.:
   #
   #     super
   #
   class Super
     include Proxy
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       'super'
     end
@@ -510,13 +532,13 @@ module Citrus
     # The actual String or Regexp object this rule uses to match.
     attr_reader :rule
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       rule.inspect
     end
   end
 
-  # A FixedWidth is a Terminal that matches based on its length. The PEG
+  # A FixedWidth is a Terminal that matches based on its length. The Citrus
   # notation is any sequence of characters enclosed in either single or double
   # quotes, e.g.:
   #
@@ -540,13 +562,13 @@ module Citrus
 
   # An Expression is a Terminal that has the same semantics as a regular
   # expression in Ruby. The expression must match at the beginning of the input
-  # (index 0). The PEG notation is identical to Ruby's regular expression
+  # (index 0). The Citrus notation is identical to Ruby's regular expression
   # notation, e.g.:
   #
   #     /expr/
   #
-  # Character classes and the dot symbol may also be used in PEG notation for
-  # compatibility with other PEG implementations, e.g.:
+  # Character classes and the dot symbol may also be used in Citrus notation for
+  # compatibility with other parsing expression implementations, e.g.:
   #
   #     [a-zA-Z]
   #     .
@@ -602,7 +624,7 @@ module Citrus
   end
 
   # An AndPredicate is a Predicate that contains a rule that must match. Upon
-  # success an empty match is returned and no input is consumed. The PEG
+  # success an empty match is returned and no input is consumed. The Citrus
   # notation is any expression preceeded by an ampersand, e.g.:
   #
   #     &expr
@@ -616,14 +638,14 @@ module Citrus
       create_match('', offset) if input.match(rule, offset)
     end
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       '&' + rule.embed
     end
   end
 
   # A NotPredicate is a Predicate that contains a rule that must not match. Upon
-  # success an empty match is returned and no input is consumed. The PEG
+  # success an empty match is returned and no input is consumed. The Citrus
   # notation is any expression preceeded by an exclamation mark, e.g.:
   #
   #     !expr
@@ -637,14 +659,14 @@ module Citrus
       create_match('', offset) unless input.match(rule, offset)
     end
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       '!' + rule.embed
     end
   end
 
   # A Label is a Predicate that applies a new name to any matches made by its
-  # rule. The PEG notation is any sequence of word characters (i.e.
+  # rule. The Citrus notation is any sequence of word characters (i.e.
   # <tt>[a-zA-Z0-9_]</tt>) followed by a colon, followed by any other
   # expression, e.g.:
   #
@@ -673,14 +695,14 @@ module Citrus
       end
     end
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       label.to_s + ':' + rule.embed
     end
   end
 
   # A Repeat is a Predicate that specifies a minimum and maximum number of times
-  # its rule must match. The PEG notation is an integer, +N+, followed by an
+  # its rule must match. The Citrus notation is an integer, +N+, followed by an
   # asterisk, followed by another integer, +M+, all of which follow any other
   # expression, e.g.:
   #
@@ -721,23 +743,29 @@ module Citrus
       create_match(matches, offset) if @range.include?(matches.length)
     end
 
+    # The minimum number of times this rule must match.
+    def min
+      @range.begin
+    end
+
+    # The maximum number of times this rule may match.
+    def max
+      @range.end
+    end
+
     # Returns the operator this rule uses as a string. Will be one of
     # <tt>+</tt>, <tt>?</tt>, or <tt>N*M</tt>.
     def operator
-      unless @operator
-        m = [@range.begin, @range.end].map do |n|
-          n == 0 || n == Infinity ? '' : n.to_s
+      @operator ||= case [min, max]
+        when [0, 0] then ''
+        when [0, 1] then '?'
+        when [1, Infinity] then '+'
+        else
+          [min, max].map {|n| n == 0 || n == Infinity ? '' : n.to_s }.join('*')
         end
-        @operator = case m
-          when ['', '1'] then '?'
-          when ['1', ''] then '+'
-          else m.join('*')
-          end
-      end
-      @operator
     end
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       rule.embed + operator
     end
@@ -753,8 +781,8 @@ module Citrus
     end
   end
 
-  # A Choice is a List where only one rule must match. The PEG notation is two
-  # or more expressions separated by a vertical bar, e.g.:
+  # A Choice is a List where only one rule must match. The Citrus notation is
+  # two or more expressions separated by a vertical bar, e.g.:
   #
   #     expr | expr
   #
@@ -771,14 +799,14 @@ module Citrus
       nil
     end
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       rules.map {|r| r.embed }.join(' | ')
     end
   end
 
-  # A Sequence is a List where all rules must match. The PEG notation is two or
-  # more expressions separated by a space, e.g.:
+  # A Sequence is a List where all rules must match. The Citrus notation is two
+  # or more expressions separated by a space, e.g.:
   #
   #     expr expr
   #
@@ -799,7 +827,7 @@ module Citrus
       create_match(matches, offset) if matches.length == rules.length
     end
 
-    # Returns the PEG notation of this rule as a string.
+    # Returns the Citrus notation of this rule as a string.
     def to_s
       rules.map {|r| r.embed }.join(' ')
     end
@@ -829,6 +857,9 @@ module Citrus
     # the label.
     attr_accessor :name
 
+    # A module that will be used to extend this match.
+    attr_accessor :ext
+
     # The offset in the input at which this match occurred.
     attr_reader :offset
 
@@ -894,9 +925,23 @@ module Citrus
     # Uses #match to allow sub-matches of this match to be called by name as
     # instance methods.
     def method_missing(sym, *args)
-      m = first(sym)
-      return m if m
-      raise 'No match named "%s" in %s (%s)' % [sym, self, name]
+      # Extend this object only when needed and immediately redefine
+      # #method_missing so that the new version is used on all future calls.
+      extend(ext) if ext
+      redefine_method_missing!
+      __send__(sym, *args)
+    end
+
+  private
+
+    def redefine_method_missing! # :nodoc:
+      instance_eval(<<-RUBY, __FILE__, __LINE__ + 1)
+        def method_missing(sym, *args)
+          m = first(sym)
+          return m if m
+          raise 'No match named "%s" in %s (%s)' % [sym, self, name]
+        end
+      RUBY
     end
   end
 end

data/lib/citrus/file.rb

@@ -179,7 +179,7 @@ module Citrus
     end
 
     rule :rule_name do
-      all(/[a-z][a-zA-Z0-9_]*/, :space) {
+      all(/[a-z][a-zA-Z_-]*/, :space) {
         def value
           first.text
         end

data/test/{calc_peg_test.rb → calc_file_test.rb}

@@ -6,6 +6,6 @@ end
 
 Citrus.load(File.dirname(__FILE__) + '/../examples/calc')
 
-class CalcPEGTest < Test::Unit::TestCase
+class CalcFileTest < Test::Unit::TestCase
   include CalcTests
 end

data/test/file_test.rb

@@ -333,6 +333,10 @@ class CitrusFileTest < Test::Unit::TestCase
     match = grammar.parse('some_rule ')
     assert(match)
     assert('some_rule', match.value)
+
+    assert_raise ParseError do
+      match = grammar.parse('some_rule1')
+    end
   end
 
   def test_terminal

data/test/repeat_test.rb

@@ -51,22 +51,27 @@ class RepeatTest < Test::Unit::TestCase
   end
 
   def test_operator
-    rule = Repeat.new(1, 2, '')
+    rule = Repeat.new(1, 2)
     assert_equal('1*2', rule.operator)
   end
 
+  def test_operator_empty
+    rule = Repeat.new(0, 0)
+    assert_equal('', rule.operator)
+  end
+
   def test_operator_asterisk
-    rule = Repeat.new(0, Infinity, '')
+    rule = Repeat.new(0, Infinity)
     assert_equal('*', rule.operator)
   end
 
   def test_operator_question_mark
-    rule = Repeat.new(0, 1, '')
+    rule = Repeat.new(0, 1)
     assert_equal('?', rule.operator)
   end
 
   def test_operator_plus
-    rule = Repeat.new(1, Infinity, '')
+    rule = Repeat.new(1, Infinity)
     assert_equal('+', rule.operator)
   end
 

data/test/rule_test.rb

@@ -3,7 +3,9 @@ require File.dirname(__FILE__) + '/helper'
 class RuleTest < Test::Unit::TestCase
 
   module MatchModule
-    def a_test; end
+    def a_test
+      :test
+    end
   end
 
   NumericProc = Proc.new {
@@ -23,8 +25,7 @@ class RuleTest < Test::Unit::TestCase
     rule.ext = MatchModule
     match = rule.match(input('a'))
     assert(match)
-    assert_kind_of(MatchModule, match)
-    assert_respond_to(match, :a_test)
+    assert_equal(:test, match.a_test)
   end
 
   def test_numeric_proc
@@ -41,7 +42,6 @@ class RuleTest < Test::Unit::TestCase
     rule.ext = NumericModule
     match = rule.match(input('1'))
     assert(match)
-    assert_kind_of(NumericModule, match)
     assert_equal(1, match.to_i)
     assert_instance_of(Float, match.to_f)
   end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
-  - 1
+  - 2
   - 0
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors: 
 - Michael Jackson
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-18 00:00:00 -06:00
+date: 2010-06-02 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -50,17 +50,20 @@ extensions: []
 extra_rdoc_files: 
 - README
 files: 
-- lib/citrus/debug.rb
-- lib/citrus/file.rb
-- lib/citrus/sugar.rb
-- lib/citrus.rb
+- benchmark/seqpar.rb
+- benchmark/seqpar.citrus
+- benchmark/seqpar.gnuplot
 - examples/calc.citrus
 - examples/calc.rb
 - examples/calc_sugar.rb
 - extras/citrus.vim
+- lib/citrus/debug.rb
+- lib/citrus/file.rb
+- lib/citrus/sugar.rb
+- lib/citrus.rb
 - test/alias_test.rb
 - test/and_predicate_test.rb
-- test/calc_peg_test.rb
+- test/calc_file_test.rb
 - test/calc_sugar_test.rb
 - test/calc_test.rb
 - test/choice_test.rb
@@ -117,7 +120,7 @@ summary: Parsing Expressions for Ruby
 test_files: 
 - test/alias_test.rb
 - test/and_predicate_test.rb
-- test/calc_peg_test.rb
+- test/calc_file_test.rb
 - test/calc_sugar_test.rb
 - test/calc_test.rb
 - test/choice_test.rb