gherkin 0.0.3

data/spec/gherkin/shared/py_string_spec.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+module Gherkin
+  module Lexer
+    shared_examples_for "a Gherkin lexer lexing py_strings" do
+            
+      def ps(content)
+        '"""%s"""' % ("\n" + content + "\n")
+      end
+      
+      it "should provide the amount of indentation of the triple quotes to the listener" do
+str = <<EOS
+Feature: some feature
+  Scenario: some scenario
+    Given foo
+    """
+      Hello
+    Goodbye
+    """
+    Then bar
+EOS
+        @listener.should_receive(:py_string).with("  Hello\nGoodbye", 4)
+        @lexer.scan(str)
+      end
+
+      it "should parse a simple py_string" do
+        @listener.should_receive(:py_string).with("I am a py_string", 1)
+        @lexer.scan ps("I am a py_string")
+      end
+
+      it "should parse an empty py_string" do
+        @listener.should_receive(:py_string).with("", 4)
+        @lexer.scan("Feature: Hi\nScenario: Hi\nGiven a step\n\"\"\"\n\"\"\"")
+      end
+
+      it "should treat a string containing only newlines as only newlines" do
+py_string = <<EOS
+"""
+
+
+
+"""
+EOS
+        @listener.should_receive(:py_string).with("\n\n", 1)
+        @lexer.scan(py_string)
+      end
+      
+      it "should parse content separated by two newlines" do
+        @lexer.scan ps("A\n\nB")
+        @listener.to_sexp.should == [
+          [:py_string, "A\n\nB", 1],
+        ]
+      end
+      
+      it "should parse a multiline string" do
+        @listener.should_receive(:py_string).with("A\nB\nC\nD", 1)
+        @lexer.scan ps("A\nB\nC\nD")
+      end
+            
+      it "should ignore unescaped quotes inside the string delimeters" do
+        @listener.should_receive(:py_string).with("What does \"this\" mean?", 1)
+        @lexer.scan ps('What does "this" mean?')
+      end
+      
+      it "should preserve whitespace within the triple quotes" do
+str = <<EOS
+    """
+      Line one
+ Line two
+    """
+EOS
+        @listener.should_receive(:py_string).with("  Line one\nLine two", 1)
+        @lexer.scan(str)
+      end
+            
+      it "should preserve tabs within the content" do
+        @listener.should_receive(:py_string).with("I have\tsome tabs\nInside\t\tthe content", 1)
+        @lexer.scan ps("I have\tsome tabs\nInside\t\tthe content")
+      end
+  
+      it "should handle complex py_strings" do
+py_string = <<EOS
+# Feature comment
+@one
+Feature: Sample
+
+  @two @three
+  Scenario: Missing
+    Given missing
+
+1 scenario (1 passed)
+1 step (1 passed)
+
+EOS
+        
+        @listener.should_receive(:py_string).with(py_string, 1)
+        @lexer.scan ps(py_string)
+      end
+ 
+      it "should allow whitespace after the closing py_string delimiter" do
+str = <<EOS
+    """
+      Line one
+    """           
+EOS
+        @listener.should_receive(:py_string).with("  Line one", 1)
+        @lexer.scan(str)
+      end
+    end
+  end
+end

data/spec/gherkin/shared/table_spec.rb

@@ -0,0 +1,97 @@
+#encoding: utf-8
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+module Gherkin
+  module Lexer
+    shared_examples_for "a Gherkin lexer lexing tables" do
+      tables = {
+        "|a|b|\n"        => [%w{a b}],
+        "|a|b|c|\n"      => [%w{a b c}],
+        "|c|d|\n|e|f|\n" => [%w{c d}, %w{e f}]
+      }
+    
+      tables.each do |text, expected|
+        it "should parse #{text}" do
+          @listener.should_receive(:table).with(t(expected), 1)
+          @lexer.scan(text.dup)
+        end
+      end
+
+      it "should parse a table with many columns" do
+        @listener.should_receive(:table).with(t([%w{a b c d e f g h i j k l m n o p}]), 1)
+        @lexer.scan("|a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|\n")
+      end
+    
+      it "should parse a multicharacter cell content" do
+        @listener.should_receive(:table).with(t([%w{foo bar}]), 1)
+        @lexer.scan("| foo | bar |\n")
+      end
+    
+      it "should parse cells with spaces within the content" do
+        @listener.should_receive(:table).with(t([["Dill pickle", "Valencia orange"], ["Ruby red grapefruit", "Tire iron"]]), 1)
+        @lexer.scan("| Dill pickle | Valencia orange |\n| Ruby red grapefruit | Tire iron |\n")
+      end
+      
+      it "should allow utf-8" do
+        #  Fails in 1.9.1! 
+        #  'Gherkin::Lexer::Table should allow utf-8 with using == to evaluate' FAILED 
+        #    expected: [[:table, [["ůﻚ", "2"]], 1]],
+        #         got: [[:table, [["\xC5\xAF\xEF\xBB\x9A", "2"]], 1]] (using ==)
+        #  BUT, simply running:
+        #     [[:table, [["ůﻚ", "2"]], 1]].should == [[:table, [["\xC5\xAF\xEF\xBB\x9A", "2"]], 1]] 
+        #  passes
+        #
+        @lexer.scan(" | ůﻚ | 2 | \n")
+        @listener.to_sexp.should == [
+          [:table, [["ůﻚ", "2"]], 1]
+        ]
+      end 
+
+      it "should allow utf-8 using should_receive" do
+        @listener.should_receive(:table).with(t([['繁體中文  而且','並且','繁體中文  而且','並且']]), 1)
+        @lexer.scan("| 繁體中文  而且|並且| 繁體中文  而且|並且|\n")
+      end
+
+      it "should parse a 2x2 table" do
+        @listener.should_receive(:table).with(t([%w{1 2}, %w{3 4}]), 1)
+        @lexer.scan("| 1 | 2 |\n| 3 | 4 |\n")
+      end
+
+      it "should parse a 2x2 table with several newlines" do
+        @listener.should_receive(:table).with(t([%w{1 2}, %w{3 4}]), 1)
+        @lexer.scan("| 1 | 2 |\n| 3 | 4 |\n\n\n")
+      end
+
+      it "should parse a 2x2 table with empty cells" do
+        @listener.should_receive(:table).with(t([['1', ''], ['', '4']]), 1)
+        @lexer.scan("| 1 |  |\n|| 4 |\n")
+      end
+    
+      it "should parse a 1x2 table that does not end in a newline" do
+        @listener.should_receive(:table).with(t([%w{1 2}]), 1)
+        @lexer.scan("| 1 | 2 |")
+      end
+
+      it "should parse a 1x2 table without spaces and newline" do
+        @listener.should_receive(:table).with(t([%w{1 2}]), 1)
+        @lexer.scan("|1|2|\n")
+      end
+      
+      it "should parse a row with whitespace after the rows" do
+        @listener.should_receive(:table).with(t([%w{1 2}, %w{a b}]), 1)
+        @lexer.scan("| 1 | 2 | \n | a | b | \n")
+      end
+      
+      it "should parse a table with lots of whitespace" do
+        @listener.should_receive(:table).with(t([["abc", "123"]]), 1)
+        @lexer.scan("  \t| \t   abc\t| \t123\t \t\t| \t\t   \t \t\n  ")
+      end
+      
+      it "should raise LexingError for rows that aren't closed" do
+        lambda { 
+          @lexer.scan("|| oh hello \n") 
+        }.should raise_error(/Parsing error on line 1: '|| oh hello/)
+      end
+    end
+  end
+end

data/spec/gherkin/shared/tags_spec.rb

@@ -0,0 +1,50 @@
+#encoding: utf-8
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+module Gherkin
+  module Lexer
+    shared_examples_for "a Gherkin lexer lexing tags" do
+      it "should lex a single tag" do
+        @listener.should_receive(:tag).with("dog", 1)
+        @lexer.scan("@dog\n")
+      end
+  
+      it "should lex multiple tags" do
+        @listener.should_receive(:tag).twice
+        @lexer.scan("@dog @cat\n")
+      end
+  
+      it "should lex UTF-8 tags" do
+        @listener.should_receive(:tag).with("シナリオテンプレート", 1)
+        @lexer.scan("@シナリオテンプレート\n")
+      end
+        
+      it "should lex mixed tags" do
+        @listener.should_receive(:tag).with("wip", 1).ordered
+        @listener.should_receive(:tag).with("Значения", 1).ordered
+        @lexer.scan("@wip @Значения\n")
+      end
+  
+      it "should lex wacky identifiers" do
+        @listener.should_receive(:tag).exactly(4).times
+        @lexer.scan("@BJ-x98.77 @BJ-z12.33 @O_o" "@#not_a_comment\n")
+      end
+
+      # TODO: Ask on ML for opinions about this one
+      it "should lex tags without spaces between them?" do
+        @listener.should_receive(:tag).twice
+        @lexer.scan("@one@two\n")
+      end
+    
+      it "should not lex tags beginning with two @@ signs" do
+        @listener.should_not_receive(:tag)
+        lambda { @lexer.scan("@@test\n") }.should raise_error(/Lexing error on line 1/)
+      end
+    
+      it "should not lex a lone @ sign" do
+        @listener.should_not_receive(:tag)
+        lambda { @lexer.scan("@\n") }.should raise_error(/Lexing error on line 1/)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,53 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'spec/gherkin'))
+require 'gherkin'
+require 'gherkin/sexp_recorder'
+require 'rubygems'
+require 'spec'
+require 'spec/autorun'
+require 'shared/lexer_spec'
+require 'shared/tags_spec'
+require 'shared/py_string_spec'
+require 'shared/table_spec'
+
+module GherkinSpecHelper
+  def scan_file(file)
+    @lexer.scan(File.new(File.dirname(__FILE__) + "/gherkin/fixtures/" + file).read)
+  end
+end
+
+Spec::Runner.configure do |c|
+  c.include(GherkinSpecHelper)
+end
+
+# Allows comparison of Java List with Ruby Array (tables)
+Spec::Matchers.define :t do |expected|
+  match do |table|
+    def table.inspect
+      "t " + self.map{|row| row.map{|cell| cell}}.inspect
+    end
+    table.map{|row| row.map{|cell| cell}}.should == expected
+  end
+end
+
+Spec::Matchers.define :a do |expected|
+  match do |array|
+    def array.inspect
+      "a " + self.map{|e| e.to_sym}.inspect
+    end
+    array.map{|e| e.to_sym}.should == expected
+  end
+end
+
+Spec::Matchers.define :sym do |expected|
+  match do |actual|
+    expected.to_s == actual.to_s
+  end
+end
+
+Spec::Matchers.define :allow do |event|
+  match do |parser|
+    parser.expected.index(event)
+  end  
+end

data/tasks/bench.rake

@@ -0,0 +1,176 @@
+%w{/../lib /bench}.each do |l| 
+  $LOAD_PATH << File.expand_path(File.dirname(__FILE__) + l)
+end
+
+require 'benchmark'
+
+GENERATED_FEATURES = File.expand_path(File.dirname(__FILE__) + "/bench/generated")
+
+class RandomFeatureGenerator
+  def initialize(number)
+    require 'faker'
+    require 'feature_builder'
+    
+    @number = number
+  end
+  
+  def generate
+    @number.times do
+      name = catch_phrase
+      feature = FeatureBuilder.new(name) do |f|
+        num_scenarios = rand_in(1..10)
+        num_scenarios.times do
+          f.scenario(bs) do |steps|
+            num_steps = rand_in(3..10)
+            num_steps.times do
+              steps.step(sentence, self)
+            end
+          end
+        end
+      end      
+      write feature.to_s, name
+    end    
+  end
+  
+  def write(content, name)
+    File.open(GENERATED_FEATURES + "/#{name.downcase.gsub(/[\s\-\/]/, '_')}.feature", "w+") do |file|
+      file << content
+    end
+  end
+  
+  def rand_in(range)
+    ary = range.to_a
+    ary[rand(ary.length - 1)]
+  end
+  
+  def catch_phrase
+    Faker::Company.catch_phrase
+  end
+  
+  def bs
+    Faker::Company.bs.capitalize
+  end
+  
+  def sentence
+    Faker::Lorem.sentence
+  end
+
+  def table_cell
+    Faker::Lorem.words(rand(2)+1).join(" ")
+  end
+end
+
+class Benchmarker
+  def initialize
+    @features = Dir[GENERATED_FEATURES + "/**/*feature"]
+  end
+  
+  def report(lexer)
+    Benchmark.bm do |x|
+      x.report("#{lexer}:") { send :"run_#{lexer}" }
+    end
+  end
+  
+  def report_all 
+    Benchmark.bmbm do |x|
+      x.report("c_gherkin:") { run_c_gherkin }
+      x.report("rb_gherkin:") { run_rb_gherkin }
+      x.report("cucumber:") { run_cucumber }
+      x.report("tt:") { run_tt }
+    end
+  end
+  
+  def run_cucumber
+    require 'cucumber'
+    require 'logger'
+    step_mother = Cucumber::StepMother.new
+    logger = Logger.new(STDOUT)
+    logger.level = Logger::INFO
+    step_mother.log = logger
+    step_mother.load_plain_text_features(@features)
+  end
+  
+  def run_tt
+    require 'cucumber'
+    # Using Cucumber's Treetop lexer, but never calling #build to build the AST
+    lexer = Cucumber::Parser::NaturalLanguage.new(nil, 'en').lexer
+    @features.each do |file|
+      source = IO.read(file)
+      parse_tree = lexer.parse(source)
+      if parse_tree.nil?
+        raise Cucumber::Lexer::SyntaxError.new(lexer, file, 0)
+      end
+    end
+  end
+
+  def run_rb_gherkin    
+    require 'gherkin'
+    require 'null_listener'
+    listener = NullListener.new
+    @features.each do |feature|
+      lexer = Gherkin::Feature.new('en', listener)
+      lexer.scan(File.read(feature))
+    end
+  end
+
+  def run_c_gherkin    
+    require 'gherkin'
+    require 'null_listener'
+    listener = NullListener.new
+    @features.each_with_index do |feature, idx|
+      lexer = Gherkin::Feature.new('Native', listener)
+      lexer.scan(File.read(feature))
+    end
+  end
+end
+
+desc "Generate 500 random features and benchmark Cucumber, Treetop and Gherkin with them"
+task :bench => ["bench:clean", "bench:gen"] do
+  benchmarker = Benchmarker.new
+  benchmarker.report_all
+end
+
+namespace :bench do
+  desc "Generate [number] features with random content, or 500 features if number is not provided"
+  task :gen, :number do |t, args|
+    args.with_defaults(:number => 500)
+    generator = RandomFeatureGenerator.new(args.number.to_i)
+    generator.generate    
+  end
+
+  desc "Benchmark Cucumber AST building from the features in tasks/bench/generated"
+  task :cucumber do
+    benchmarker = Benchmarker.new
+    benchmarker.report("cucumber")
+  end
+  
+  desc "Benchmark the Treetop lexer with the features in tasks/bench/generated"
+  task :tt do
+    benchmarker = Benchmarker.new
+    benchmarker.report("tt")
+  end
+
+  desc "Benchmark the Ruby Gherkin lexer with the features in tasks/bench/generated"
+  task :rb_gherkin do
+    benchmarker = Benchmarker.new
+    benchmarker.report("rb_gherkin")
+  end
+
+  desc "Benchmark the C Gherkin lexer with the features in tasks/bench/generated"
+  task :c_gherkin do
+    benchmarker = Benchmarker.new
+    benchmarker.report("c_gherkin")
+  end
+
+  desc "Show basic statistics about the features in tasks/bench/generated"
+  task :stats do
+    ["Feature", "Scenario", "Given"].each do |kw|
+      sh "grep #{kw} #{GENERATED_FEATURES}/* | wc -l"
+    end
+  end
+
+  desc "Remove all generated features in tasks/bench/generated"
+  task :clean do
+    rm_f FileList[GENERATED_FEATURES + "/**/*feature"]
+  end
+end