gherkin 0.0.3-universal-java-1.5 → 0.0.4-universal-java-1.5

data/ragel/lexer.java.rl.erb

@@ -7,7 +7,7 @@ import gherkin.Lexer;
 import gherkin.Listener;
 import gherkin.LexingError;
 
-public class <%= i18n_lexer_class_name %> implements Lexer {
+public class <%= @i18n.capitalize %> implements Lexer {
   %%{
     machine lexer;
     alphtype byte;
@@ -72,10 +72,12 @@ public class <%= i18n_lexer_class_name %> implements Lexer {
 
     action store_comment_content {
       listener.comment(substring(data, contentStart, p).trim(), lineNumber);
+      keywordStart = -1;
     }
 
     action store_tag_content {
       listener.tag(substring(data, contentStart, p).trim(), currentLine);
+      keywordStart = -1;
     }
 
     action inc_line_number {
@@ -134,12 +136,12 @@ public class <%= i18n_lexer_class_name %> implements Lexer {
       }
     }
 
-    include lexer_common "lexer_common.<%= i18n_language %>.rl"; 
+    include lexer_common "lexer_common.<%= @i18n %>.rl"; 
   }%%
 
     private final Listener listener;
 
-    public <%= i18n_lexer_class_name %>(Listener listener) {
+    public <%= @i18n.capitalize %>(Listener listener) {
         this.listener = listener;
     }
 

data/ragel/lexer.rb.rl.erb

@@ -1,12 +1,9 @@
+require 'gherkin/core_ext/array'
+
 module Gherkin
   module RbLexer
-    class <%= i18n_lexer_class_name %> #:nodoc:
+    class <%= @i18n.capitalize %> #:nodoc:
       %%{
-        # patterns:
-        # * data[start...end].pack("c*").strip_of_some_sort
-        # * changing the end point of the range according to next_keyword_start
-        # * methods taking the machine state because Ragel doesn't seem to know about ivars
- 
         machine lexer;
  
         action begin_content {
@@ -24,7 +21,7 @@ module Gherkin
         }
 
         action store_pystring_content {
-          con = unindent(@start_col, data[@content_start...@next_keyword_start-1].pack("c*").sub(/(\r?\n)?( )*\Z/, ''))
+          con = unindent(@start_col, data[@content_start...@next_keyword_start-1].utf8_pack("c*").sub(/(\r?\n)?( )*\Z/, ''))
           @listener.py_string(con, @current_line) 
         }
  
@@ -59,18 +56,20 @@ module Gherkin
         }
 
         action store_step_content {
-          con = data[@content_start...p].pack("c*").strip
+          con = data[@content_start...p].utf8_pack("c*").strip
           @listener.step(@keyword, con, @current_line)
         }
  
         action store_comment_content {
-          con = data[@content_start...p].pack("c*").strip
+          con = data[@content_start...p].utf8_pack("c*").strip
           @listener.comment(con, @line_number)
+          @keyword_start = nil
         }
  
         action store_tag_content {
-          con = data[@content_start...p].pack("c*").strip
+          con = data[@content_start...p].utf8_pack("c*").strip
           @listener.tag(con, @current_line)
+          @keyword_start = nil
         }
  
         action inc_line_number {
@@ -86,7 +85,7 @@ module Gherkin
         }
  
         action end_keyword {
-          @keyword = data[@keyword_start...p].pack("c*").sub(/:$/,'').strip
+          @keyword = data[@keyword_start...p].utf8_pack("c*").sub(/:$/,'').strip
           @keyword_start = nil
         }
  
@@ -109,7 +108,7 @@ module Gherkin
         }
 
         action store_cell_content {
-          con = data[@content_start...p].pack("c*").strip
+          con = data[@content_start...p].utf8_pack("c*").strip
           current_row << con
         }
 
@@ -130,7 +129,7 @@ module Gherkin
           end
         }
 
-        include lexer_common "lexer_common.<%= i18n_language %>.rl"; 
+        include lexer_common "lexer_common.<%= @i18n %>.rl"; 
       }%%
  
       def initialize(listener)
@@ -159,13 +158,13 @@ module Gherkin
  
       def store_keyword_content(event, data, p, eof)
         end_point = (!@next_keyword_start or (p == eof)) ? p : @next_keyword_start
-        con = yield data[@content_start...end_point].pack("c*")
+        con = yield data[@content_start...end_point].utf8_pack("c*")
         @listener.send(event, @keyword, con, @current_line)
       end
       
       def current_line_content(data, p)
         rest = data[@last_newline..-1]
-        rest[0..rest.index(10)||-1].pack("c*").strip
+        rest[0..rest.index(10)||-1].utf8_pack("c*").strip
       end
     end
   end

data/ragel/lexer_common.rl.erb

@@ -2,12 +2,12 @@
   machine lexer_common;
 
   # Language specific
-  I18N_Feature = <%= i18n['feature'] %> >start_keyword %end_keyword; 
-  I18N_Background = <%= i18n['background'] %> >start_keyword %end_keyword;
-  I18N_ScenarioOutline = <%= i18n['scenario_outline'] %> >start_keyword %end_keyword;
-  I18N_Scenario = <%= i18n['scenario'] %> >start_keyword %end_keyword;
-  I18N_Step = (<%= i18n['given'] %> | <%= i18n['when'] %> | <%= i18n['and'] %> | <%= i18n['then'] %> | <%= i18n['but'] %>) >start_keyword %end_keyword;
-  I18N_Examples = <%= i18n['examples'] %> >start_keyword %end_keyword;
+  I18N_Feature = <%= keywords['feature'] %> >start_keyword %end_keyword; 
+  I18N_Background = <%= keywords['background'] %> >start_keyword %end_keyword;
+  I18N_ScenarioOutline = <%= keywords['scenario_outline'] %> >start_keyword %end_keyword;
+  I18N_Scenario = <%= keywords['scenario'] %> >start_keyword %end_keyword;
+  I18N_Step = (<%= keywords['given'] %> | <%= keywords['when'] %> | <%= keywords['and'] %> | <%= keywords['then'] %> | <%= keywords['but'] %>) >start_keyword %end_keyword;
+  I18N_Examples = <%= keywords['examples'] %> >start_keyword %end_keyword;
 
   EOF = '%_FEATURE_END_%'; # Explicit EOF added before scanning begins 
   EOL = ('\r'? '\n') @inc_line_number @last_newline;

data/spec/gherkin/fixtures/1.feature

@@ -0,0 +1,8 @@
+Feature: Logging in
+  So that I can be myself
+  # Comment
+  Scenario: Anonymous user can get a login form.
+    Scenery here
+
+  @tag
+  Scenario: Another one

data/spec/gherkin/fixtures/complex.feature

@@ -1,6 +1,6 @@
 #Comment on line 1
+#Comment on line 2
 @tag1 @tag2
-#Comment on line 3
 Feature: Feature Text
   In order to test multiline forms
   As a ragel writer
@@ -40,4 +40,4 @@ Feature: Feature Text
       """
       Makes Homer something something
       """
-    Then crazy
+    Then crazy

data/spec/gherkin/shared/lexer_spec.rb

@@ -362,15 +362,28 @@ Given I am a step
           ]
         end        
       end
-   
+
+      describe "Comment or tag between Feature elements where previous narrative starts with same letter as a keyword" do
+        it "should lex this feature properly" do
+          scan_file("1.feature")
+          @listener.to_sexp.should == [
+            [:feature, "Feature", "Logging in\nSo that I can be myself", 1],
+            [:comment, "# Comment", 3],
+            [:scenario, "Scenario", "Anonymous user can get a login form.\nScenery here", 4],
+            [:tag, "tag", 7],
+            [:scenario, "Scenario", "Another one", 8]
+          ]
+        end
+      end   
+
       describe "A complex feature with tags, comments, multiple scenarios, and multiple steps and tables" do
         it "should find things in the right order" do
           scan_file("complex.feature")
           @listener.to_sexp.should == [
             [:comment, "#Comment on line 1", 1],
-            [:tag, "tag1", 2],
-            [:tag, "tag2", 2],
-            [:comment, "#Comment on line 3", 3],
+            [:comment, "#Comment on line 2", 2],
+            [:tag, "tag1", 3],
+            [:tag, "tag2", 3],
             [:feature, "Feature", "Feature Text\nIn order to test multiline forms\nAs a ragel writer\nI need to check for complex combinations", 4],
             [:comment, "#Comment on line 9", 9],
             [:comment, "#Comment on line 11", 11],

data/spec/gherkin/shared/py_string_spec.rb

@@ -107,6 +107,18 @@ EOS
         @listener.should_receive(:py_string).with("  Line one", 1)
         @lexer.scan(str)
       end
+
+      it "should preserve the last newline(s) at the end of a py_string" do
+str = <<EOS
+     """
+     PyString text
+
+
+     """
+EOS
+        @listener.should_receive(:py_string).with("PyString text\n\n",1)
+        @lexer.scan(str)
+      end        
     end
   end
 end

data/tasks/bench.rake

@@ -74,6 +74,7 @@ class Benchmarker
   def report_all 
     Benchmark.bmbm do |x|
       x.report("c_gherkin:") { run_c_gherkin }
+      x.report("c_gherkin_no_parser:") { run_c_gherkin_no_parser }
       x.report("rb_gherkin:") { run_rb_gherkin }
       x.report("cucumber:") { run_cucumber }
       x.report("tt:") { run_tt }
@@ -93,32 +94,42 @@ class Benchmarker
   def run_tt
     require 'cucumber'
     # Using Cucumber's Treetop lexer, but never calling #build to build the AST
-    lexer = Cucumber::Parser::NaturalLanguage.new(nil, 'en').lexer
+    lexer = Cucumber::Parser::NaturalLanguage.new(nil, 'en').parser
     @features.each do |file|
       source = IO.read(file)
       parse_tree = lexer.parse(source)
       if parse_tree.nil?
-        raise Cucumber::Lexer::SyntaxError.new(lexer, file, 0)
+        raise Cucumber::Parser::SyntaxError.new(lexer, file, 0)
       end
     end
   end
 
   def run_rb_gherkin    
     require 'gherkin'
+    require 'gherkin/rb_lexer'
     require 'null_listener'
-    listener = NullListener.new
     @features.each do |feature|
-      lexer = Gherkin::Feature.new('en', listener)
+      parser = Gherkin::Parser.new(NullListener.new, true, "root")
+      lexer = Gherkin::RbLexer['en'].new(parser)
       lexer.scan(File.read(feature))
     end
   end
 
-  def run_c_gherkin    
+  def run_c_gherkin
     require 'gherkin'
     require 'null_listener'
-    listener = NullListener.new
-    @features.each_with_index do |feature, idx|
-      lexer = Gherkin::Feature.new('Native', listener)
+    @features.each do |feature|
+      parser = Gherkin::Parser.new(NullListener.new, true, "root")
+      lexer = Gherkin::CLexer['en'].new(parser)
+      lexer.scan(File.read(feature))
+    end
+  end
+
+  def run_c_gherkin_no_parser
+    require 'gherkin'
+    require 'null_listener'
+    @features.each do |feature|
+      lexer = Gherkin::CLexer['en'].new(NullListener.new)
       lexer.scan(File.read(feature))
     end
   end
@@ -144,24 +155,30 @@ namespace :bench do
     benchmarker.report("cucumber")
   end
   
-  desc "Benchmark the Treetop lexer with the features in tasks/bench/generated"
+  desc "Benchmark the Treetop parser with the features in tasks/bench/generated"
   task :tt do
     benchmarker = Benchmarker.new
     benchmarker.report("tt")
   end
 
-  desc "Benchmark the Ruby Gherkin lexer with the features in tasks/bench/generated"
+  desc "Benchmark the Ruby Gherkin lexer+parser with the features in tasks/bench/generated"
   task :rb_gherkin do
     benchmarker = Benchmarker.new
     benchmarker.report("rb_gherkin")
   end
 
-  desc "Benchmark the C Gherkin lexer with the features in tasks/bench/generated"
+  desc "Benchmark the C Gherkin lexer+parser with the features in tasks/bench/generated"
   task :c_gherkin do
     benchmarker = Benchmarker.new
     benchmarker.report("c_gherkin")
   end
 
+  desc "Benchmark the C Gherkin lexer (no parser) with the features in tasks/bench/generated"
+  task :c_gherkin_no_parser do
+    benchmarker = Benchmarker.new
+    benchmarker.report("c_gherkin_no_parser")
+  end
+
   desc "Show basic statistics about the features in tasks/bench/generated"
   task :stats do
     ["Feature", "Scenario", "Given"].each do |kw|

data/tasks/compile.rake

@@ -0,0 +1,70 @@
+require File.dirname(__FILE__) + '/ragel_task'
+
+CLEAN.include [
+  '**/*.{o,bundle,jar,so,obj,pdb,lib,def,exp,log}', 'ext',
+  'java/target',
+  'ragel/i18n/*.rl',
+  'lib/gherkin/rb_lexer/*.rb',
+  'ext/**/*.c',
+  'java/src/gherkin/lexer/*.java'
+]
+
+desc "Compile the Java extensions"
+task :jar do
+  sh("ant -f java/build.xml")
+end
+
+YAML.load_file(File.dirname(__FILE__) + '/../lib/gherkin/i18n.yml').each do |i18n, keywords|
+  i18n = i18n.gsub(/[\s-]/, '')
+
+  java = RagelTask.new('java', i18n, keywords)
+  rb   = RagelTask.new('rb', i18n, keywords)
+
+  task :jar     => java.target
+  task :jar     => rb.target
+
+  begin
+    require 'rake/extensiontask'
+    c = RagelTask.new('c', i18n, keywords)
+
+    extconf = "ext/gherkin_lexer_#{i18n}/extconf.rb"
+
+    file extconf do
+      FileUtils.mkdir(File.dirname(extconf)) unless File.directory?(File.dirname(extconf))
+      File.open(extconf, "w") do |io|
+        io.write(<<-EOF)
+require 'mkmf'
+dir_config("gherkin_lexer_#{i18n}")
+have_library("c", "main")
+create_makefile("gherkin_lexer_#{i18n}")
+EOF
+      end
+    end
+
+    Rake::ExtensionTask.new("gherkin_lexer_#{i18n}") do |ext|
+      if ENV['RUBY_CC_VERSION']
+        ext.cross_compile = true
+        ext.cross_platform = 'i386-mingw32'
+      end
+    end
+
+    task :compile => c.target
+    task :compile => rb.target
+
+    # The way tasks are defined with compile:xxx (but without namespace) in rake-compiler forces us
+    # to use these hacks for setting up dependencies. Ugly!
+    Rake::Task["compile:gherkin_lexer_#{i18n}"].prerequisites.unshift(extconf)
+    Rake::Task["compile:gherkin_lexer_#{i18n}"].prerequisites.unshift(c.target)
+    Rake::Task["compile:gherkin_lexer_#{i18n}"].prerequisites.unshift(rb.target)
+
+    Rake::Task["compile"].prerequisites.unshift(extconf)
+    Rake::Task["compile"].prerequisites.unshift(c.target)
+    Rake::Task["compile"].prerequisites.unshift(rb.target)
+  rescue LoadError
+    unless defined?($c_warned)
+      warn "WARNING: Rake::ExtensionTask not installed. Skipping C compilation." 
+      $c_warned = true
+      task :compile # no-op
+    end
+  end
+end

data/tasks/ragel_task.rb

@@ -0,0 +1,83 @@
+require 'yaml'
+require 'erb'
+
+class RagelTask
+  RL_OUTPUT_DIR = File.dirname(__FILE__) + "/../ragel/i18n"
+
+  def initialize(lang, i18n, keywords)
+    @lang     = lang
+    @i18n     = i18n
+    @keywords = keywords
+    define_tasks
+  end
+
+  def define_tasks
+    file target => [lang_ragel, common_ragel] do
+      mkdir_p(File.dirname(target)) unless File.directory?(File.dirname(target))
+      sh "ragel #{flags} #{lang_ragel} -o #{target}"
+    end
+
+    file lang_ragel => lang_erb do
+      impl = ERB.new(IO.read(lang_erb)).result(binding)
+      write(impl, lang_ragel)
+    end
+
+    file common_ragel => common_erb  do
+      keywords = prep_keywords
+      common = ERB.new(IO.read(common_erb)).result(binding)
+      write(common, common_ragel)
+    end
+  end
+
+  def target
+    {
+      'c'    => "ext/gherkin_lexer_#{@i18n}/gherkin_lexer_#{@i18n}.c",
+      'java' => "java/src/gherkin/lexer/#{@i18n.capitalize}.java",
+      'rb'   => "lib/gherkin/rb_lexer/#{@i18n}.rb"
+    }[@lang]
+  end
+
+  def common_ragel
+    RL_OUTPUT_DIR + "/lexer_common.#{@i18n}.rl"
+  end
+
+  def common_erb
+    File.dirname(__FILE__) + '/../ragel/lexer_common.rl.erb'
+  end
+
+  def lang_ragel
+    RL_OUTPUT_DIR + "/#{@i18n}.#{@lang}.rl"
+  end
+
+  def lang_erb
+    File.dirname(__FILE__) + "/../ragel/lexer.#{@lang}.rl.erb"
+  end
+
+  def flags
+    {
+      'c'    => '-C',
+      'java' => '-J',
+      'rb'   => '-R'
+    }[@lang]
+  end
+
+  def prep_keywords
+    keywords = @keywords.dup
+    delimited_keywords = %w{feature background scenario scenario_outline examples}
+    bare_keywords = %w{given when then and but}
+    all_keywords = delimited_keywords + bare_keywords
+    
+    all_keywords.each       { |k| keywords[k] = keywords[k].split("|") }
+    delimited_keywords.each { |k| keywords[k].map! { |v| v += ':'} }
+    bare_keywords.each      { |k| keywords[k].map! { |v| (v + ' ').sub(/< $/, '')} }
+    all_keywords.each       { |k| keywords[k] = '("' + keywords[k].join('" | "') + '")' }    
+    keywords
+  end
+
+  def write(content, filename)
+    mkdir_p(File.dirname(filename)) unless File.directory?(File.dirname(filename))
+    File.open(filename, "wb") do |file|
+      file.write(content)
+    end
+  end  
+end