gherkin 0.0.4-i386-mswin32 → 1.0.0-i386-mswin32

data/README.rdoc

@@ -19,9 +19,11 @@ rake clobber
 
 == Release process
 
-Run just "rake" for each platform (1.8.6, 1.8.7, 1.9, jruby) to make sure all is green.
+Run just "rake clean spec cucumber" for each platform (1.8.6, 1.8.7, 1.9, jruby) to make sure all is green.
 
-1) Bump version in the VERSION file
+1) rvm 1.8.7
+2) Bump version in the VERSION file
+3) rake clean jar compile
 2) rake gemspec
 3) Commit everything
 4) rake release

data/Rakefile

@@ -46,4 +46,6 @@ end
 
 Dir['tasks/**/*.rake'].each { |rake| load rake }
 
-task :default => [:jar, :compile, :spec, :cucumber]
+task :default  => [:spec, :cucumber]
+task :spec     => defined?(JRUBY_VERSION) ? :jar : :compile
+task :cucumber => defined?(JRUBY_VERSION) ? :jar : :compile

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :minor: 0
-:patch: 4
-:major: 0
+:patch: 0
+:major: 1

data/gherkin.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gherkin}
-  s.version = "0.0.4"
+  s.version = "1.0.0"
   s.platform = %q{i386-mswin32}
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Mike Sassak", "Gregory Hnatiuk", "Aslak Helles\303\270y"]
-  s.date = %q{2009-11-28}
+  s.date = %q{2009-12-01}
   s.default_executable = %q{gherkin}
   s.description = %q{A fast Gherkin lexer in Ragel}
   s.email = %q{cukes@googlegroups.com}
@@ -51,6 +51,7 @@ Gem::Specification.new do |s|
      "lib/gherkin.rb",
      "lib/gherkin/c_lexer.rb",
      "lib/gherkin/core_ext/array.rb",
+     "lib/gherkin/i18n.rb",
      "lib/gherkin/i18n.yml",
      "lib/gherkin/i18n_lexer.rb",
      "lib/gherkin/java_lexer.rb",
@@ -64,46 +65,6 @@ Gem::Specification.new do |s|
      "lib/gherkin/rb_lexer/README.rdoc",
      "lib/gherkin/rb_parser.rb",
      "lib/gherkin/tools/pretty_printer.rb",
-     "lib/gherkin_lexer_ar.so",
-     "lib/gherkin_lexer_bg.so",
-     "lib/gherkin_lexer_cat.so",
-     "lib/gherkin_lexer_cs.so",
-     "lib/gherkin_lexer_cy.so",
-     "lib/gherkin_lexer_da.so",
-     "lib/gherkin_lexer_de.so",
-     "lib/gherkin_lexer_en.so",
-     "lib/gherkin_lexer_enau.so",
-     "lib/gherkin_lexer_enlol.so",
-     "lib/gherkin_lexer_entx.so",
-     "lib/gherkin_lexer_es.so",
-     "lib/gherkin_lexer_et.so",
-     "lib/gherkin_lexer_fi.so",
-     "lib/gherkin_lexer_fr.so",
-     "lib/gherkin_lexer_he.so",
-     "lib/gherkin_lexer_hr.so",
-     "lib/gherkin_lexer_hu.so",
-     "lib/gherkin_lexer_id.so",
-     "lib/gherkin_lexer_it.so",
-     "lib/gherkin_lexer_ja.so",
-     "lib/gherkin_lexer_ko.so",
-     "lib/gherkin_lexer_lt.so",
-     "lib/gherkin_lexer_lv.so",
-     "lib/gherkin_lexer_nl.so",
-     "lib/gherkin_lexer_no.so",
-     "lib/gherkin_lexer_pl.so",
-     "lib/gherkin_lexer_pt.so",
-     "lib/gherkin_lexer_ro.so",
-     "lib/gherkin_lexer_ro2.so",
-     "lib/gherkin_lexer_ru.so",
-     "lib/gherkin_lexer_se.so",
-     "lib/gherkin_lexer_sk.so",
-     "lib/gherkin_lexer_sr.so",
-     "lib/gherkin_lexer_srLatn.so",
-     "lib/gherkin_lexer_tr.so",
-     "lib/gherkin_lexer_uz.so",
-     "lib/gherkin_lexer_vi.so",
-     "lib/gherkin_lexer_zhCN.so",
-     "lib/gherkin_lexer_zhTW.so",
      "nativegems.sh",
      "ragel/i18n/.gitignore",
      "ragel/lexer.c.rl.erb",

data/lib/gherkin/i18n.rb

@@ -0,0 +1,80 @@
+module Gherkin
+  class I18n
+    KEYWORD_KEYS = %w{name native feature background scenario scenario_outline examples given when then and but}
+    LANGUAGES    = YAML.load_file(File.dirname(__FILE__) + '/i18n.yml')
+
+    class << self
+      # Used by code generators for other lexer tools like pygments lexer and textmate bundle
+      def all
+        LANGUAGES.keys.sort.map{|key| get(key)}
+      end
+
+      def get(key)
+        languages[key] ||= new(key)
+      end
+
+      def languages
+        @languages ||= {}
+      end
+    end
+
+    def initialize(key)
+      @key = key
+      @keywords = LANGUAGES[key]
+      raise "Language not supported: #{key.inspect}" if @key.nil?
+      @keywords['grammar_name'] = @keywords['name'].gsub(/\s/, '')
+      @parser = nil
+    end
+
+    def sanitized_key
+      @key.gsub(/[\s-]/, '')
+    end
+
+    def incomplete?
+      KEYWORD_KEYS.detect{|key| @keywords[key].nil?}
+    end
+
+    def feature_keywords
+      keywords('feature')
+    end
+
+    def scenario_keywords
+      keywords('scenario')
+    end
+
+    def scenario_outline_keywords
+      keywords('scenario_outline')
+    end
+
+    def background_keywords
+      keywords('background')
+    end
+
+    def examples_keywords
+      keywords('examples')
+    end
+
+    def but_keywords(space=true)
+      keywords('but', space)
+    end
+
+    def and_keywords(space=true)
+      keywords('and', space)
+    end
+
+    def step_keywords
+      %w{given when then and but}.map{|key| keywords(key, true)}.flatten.uniq
+    end
+
+    def keywords(key, space=false)
+      raise "No #{key} in #{@keywords.inspect}" if @keywords[key].nil?
+      @keywords[key].split('|').map{|kw| space ? keyword_space(kw) : kw}
+    end
+
+    private
+
+    def keyword_space(val)
+      (val + ' ').sub(/< $/,'')
+    end
+  end
+end

data/ragel/lexer.c.rl.erb

@@ -7,6 +7,8 @@
 
 #ifdef HAVE_RUBY_RE_H
 #include <ruby/re.h>
+#else
+#include <re.h>
 #endif
 
 #ifdef HAVE_RUBY_ENCODING_H
@@ -232,7 +234,7 @@ static VALUE rb_eGherkinLexerError;
     }
   }
 
-  include lexer_common "lexer_common.<%= @i18n %>.rl";   
+  include lexer_common "lexer_common.<%= @i18n.sanitized_key %>.rl";   
 
 }%%
 
@@ -350,7 +352,7 @@ static VALUE CLexer_scan(VALUE self, VALUE input)
   rb_str_append(input_copy, rb_str_new2("\n%_FEATURE_END_%"));
   char *data = RSTRING_PTR(input_copy);
   long len = RSTRING_LEN(input_copy);
- 
+  
   if (len == 0) { 
     rb_raise(rb_eGherkinLexerError, "No content to lex.");
   } else {
@@ -387,14 +389,14 @@ static VALUE CLexer_scan(VALUE self, VALUE input)
   }
 }
 
-void Init_gherkin_lexer_<%= @i18n %>()
+void Init_gherkin_lexer_<%= @i18n.sanitized_key %>()
 {
   mGherkin = rb_define_module("Gherkin");
   mLexer = rb_const_get(mGherkin, rb_intern("Lexer"));
   rb_eGherkinLexerError = rb_const_get(mLexer, rb_intern("LexingError"));
 
   mCLexer = rb_define_module_under(mGherkin, "CLexer");
-  cI18nLexer = rb_define_class_under(mCLexer, "<%= @i18n.capitalize %>", rb_cObject);
+  cI18nLexer = rb_define_class_under(mCLexer, "<%= @i18n.sanitized_key.capitalize %>", rb_cObject);
   rb_define_alloc_func(cI18nLexer, CLexer_alloc);
   rb_define_method(cI18nLexer, "initialize", CLexer_init, 1);
   rb_define_method(cI18nLexer, "scan", CLexer_scan, 1);

data/ragel/lexer.java.rl.erb

@@ -7,7 +7,7 @@ import gherkin.Lexer;
 import gherkin.Listener;
 import gherkin.LexingError;
 
-public class <%= @i18n.capitalize %> implements Lexer {
+public class <%= @i18n.sanitized_key.capitalize %> implements Lexer {
   %%{
     machine lexer;
     alphtype byte;
@@ -136,12 +136,12 @@ public class <%= @i18n.capitalize %> implements Lexer {
       }
     }
 
-    include lexer_common "lexer_common.<%= @i18n %>.rl"; 
+    include lexer_common "lexer_common.<%= @i18n.sanitized_key %>.rl"; 
   }%%
 
     private final Listener listener;
 
-    public <%= @i18n.capitalize %>(Listener listener) {
+    public <%= @i18n.sanitized_key.capitalize %>(Listener listener) {
         this.listener = listener;
     }
 

data/ragel/lexer.rb.rl.erb

@@ -2,7 +2,7 @@ require 'gherkin/core_ext/array'
 
 module Gherkin
   module RbLexer
-    class <%= @i18n.capitalize %> #:nodoc:
+    class <%= @i18n.sanitized_key.capitalize %> #:nodoc:
       %%{
         machine lexer;
  
@@ -129,7 +129,7 @@ module Gherkin
           end
         }
 
-        include lexer_common "lexer_common.<%= @i18n %>.rl"; 
+        include lexer_common "lexer_common.<%= @i18n.sanitized_key %>.rl"; 
       }%%
  
       def initialize(listener)

data/ragel/lexer_common.rl.erb

@@ -2,12 +2,12 @@
   machine lexer_common;
 
   # Language specific
-  I18N_Feature = <%= keywords['feature'] %> >start_keyword %end_keyword; 
-  I18N_Background = <%= keywords['background'] %> >start_keyword %end_keyword;
-  I18N_ScenarioOutline = <%= keywords['scenario_outline'] %> >start_keyword %end_keyword;
-  I18N_Scenario = <%= keywords['scenario'] %> >start_keyword %end_keyword;
-  I18N_Step = (<%= keywords['given'] %> | <%= keywords['when'] %> | <%= keywords['and'] %> | <%= keywords['then'] %> | <%= keywords['but'] %>) >start_keyword %end_keyword;
-  I18N_Examples = <%= keywords['examples'] %> >start_keyword %end_keyword;
+  I18N_Feature = (<%= ragel_list(@i18n.feature_keywords) %> ':') >start_keyword %end_keyword; 
+  I18N_Background = (<%= ragel_list(@i18n.background_keywords) %> ':') >start_keyword %end_keyword;
+  I18N_ScenarioOutline = (<%= ragel_list(@i18n.scenario_outline_keywords) %> ':') >start_keyword %end_keyword;
+  I18N_Scenario = (<%= ragel_list(@i18n.scenario_keywords) %> ':') >start_keyword %end_keyword;
+  I18N_Step = <%= ragel_list(@i18n.step_keywords) %> >start_keyword %end_keyword;
+  I18N_Examples = (<%= ragel_list(@i18n.examples_keywords) %> ':') >start_keyword %end_keyword;
 
   EOF = '%_FEATURE_END_%'; # Explicit EOF added before scanning begins 
   EOL = ('\r'? '\n') @inc_line_number @last_newline;

data/tasks/bench.rake

@@ -73,8 +73,8 @@ class Benchmarker
   
   def report_all 
     Benchmark.bmbm do |x|
-      x.report("c_gherkin:") { run_c_gherkin }
-      x.report("c_gherkin_no_parser:") { run_c_gherkin_no_parser }
+      x.report("native_gherkin:") { run_native_gherkin }
+      x.report("native_gherkin_no_parser:") { run_native_gherkin_no_parser }
       x.report("rb_gherkin:") { run_rb_gherkin }
       x.report("cucumber:") { run_cucumber }
       x.report("tt:") { run_tt }
@@ -115,21 +115,21 @@ class Benchmarker
     end
   end
 
-  def run_c_gherkin
+  def run_native_gherkin
     require 'gherkin'
     require 'null_listener'
     @features.each do |feature|
       parser = Gherkin::Parser.new(NullListener.new, true, "root")
-      lexer = Gherkin::CLexer['en'].new(parser)
+      lexer = Gherkin::Lexer['en'].new(parser)
       lexer.scan(File.read(feature))
     end
   end
 
-  def run_c_gherkin_no_parser
+  def run_native_gherkin_no_parser
     require 'gherkin'
     require 'null_listener'
     @features.each do |feature|
-      lexer = Gherkin::CLexer['en'].new(NullListener.new)
+      lexer = Gherkin::Lexer['en'].new(NullListener.new)
       lexer.scan(File.read(feature))
     end
   end
@@ -167,16 +167,16 @@ namespace :bench do
     benchmarker.report("rb_gherkin")
   end
 
-  desc "Benchmark the C Gherkin lexer+parser with the features in tasks/bench/generated"
-  task :c_gherkin do
+  desc "Benchmark the ntive Gherkin lexer+parser with the features in tasks/bench/generated"
+  task :native_gherkin do
     benchmarker = Benchmarker.new
-    benchmarker.report("c_gherkin")
+    benchmarker.report("native_gherkin")
   end
 
-  desc "Benchmark the C Gherkin lexer (no parser) with the features in tasks/bench/generated"
-  task :c_gherkin_no_parser do
+  desc "Benchmark the native Gherkin lexer (no parser) with the features in tasks/bench/generated"
+  task :native_gherkin_no_parser do
     benchmarker = Benchmarker.new
-    benchmarker.report("c_gherkin_no_parser")
+    benchmarker.report("native_gherkin_no_parser")
   end
 
   desc "Show basic statistics about the features in tasks/bench/generated"

data/tasks/compile.rake

@@ -1,4 +1,5 @@
 require File.dirname(__FILE__) + '/ragel_task'
+require 'gherkin/i18n'
 
 CLEAN.include [
   '**/*.{o,bundle,jar,so,obj,pdb,lib,def,exp,log}', 'ext',
@@ -14,34 +15,33 @@ task :jar do
   sh("ant -f java/build.xml")
 end
 
-YAML.load_file(File.dirname(__FILE__) + '/../lib/gherkin/i18n.yml').each do |i18n, keywords|
-  i18n = i18n.gsub(/[\s-]/, '')
-
-  java = RagelTask.new('java', i18n, keywords)
-  rb   = RagelTask.new('rb', i18n, keywords)
+Gherkin::I18n.all.each do |i18n|
+  java = RagelTask.new('java', i18n)
+  rb   = RagelTask.new('rb', i18n)
 
   task :jar     => java.target
   task :jar     => rb.target
 
   begin
     require 'rake/extensiontask'
-    c = RagelTask.new('c', i18n, keywords)
+    c = RagelTask.new('c', i18n)
 
-    extconf = "ext/gherkin_lexer_#{i18n}/extconf.rb"
+    extconf = "ext/gherkin_lexer_#{i18n.sanitized_key}/extconf.rb"
 
     file extconf do
       FileUtils.mkdir(File.dirname(extconf)) unless File.directory?(File.dirname(extconf))
       File.open(extconf, "w") do |io|
         io.write(<<-EOF)
 require 'mkmf'
-dir_config("gherkin_lexer_#{i18n}")
+$CFLAGS << ' -O0 -Wall -Werror'
+dir_config("gherkin_lexer_#{i18n.sanitized_key}")
 have_library("c", "main")
-create_makefile("gherkin_lexer_#{i18n}")
+create_makefile("gherkin_lexer_#{i18n.sanitized_key}")
 EOF
       end
     end
 
-    Rake::ExtensionTask.new("gherkin_lexer_#{i18n}") do |ext|
+    Rake::ExtensionTask.new("gherkin_lexer_#{i18n.sanitized_key}") do |ext|
       if ENV['RUBY_CC_VERSION']
         ext.cross_compile = true
         ext.cross_platform = 'i386-mingw32'
@@ -53,9 +53,9 @@ EOF
 
     # The way tasks are defined with compile:xxx (but without namespace) in rake-compiler forces us
     # to use these hacks for setting up dependencies. Ugly!
-    Rake::Task["compile:gherkin_lexer_#{i18n}"].prerequisites.unshift(extconf)
-    Rake::Task["compile:gherkin_lexer_#{i18n}"].prerequisites.unshift(c.target)
-    Rake::Task["compile:gherkin_lexer_#{i18n}"].prerequisites.unshift(rb.target)
+    Rake::Task["compile:gherkin_lexer_#{i18n.sanitized_key}"].prerequisites.unshift(extconf)
+    Rake::Task["compile:gherkin_lexer_#{i18n.sanitized_key}"].prerequisites.unshift(c.target)
+    Rake::Task["compile:gherkin_lexer_#{i18n.sanitized_key}"].prerequisites.unshift(rb.target)
 
     Rake::Task["compile"].prerequisites.unshift(extconf)
     Rake::Task["compile"].prerequisites.unshift(c.target)

data/tasks/ragel_task.rb

@@ -4,10 +4,9 @@ require 'erb'
 class RagelTask
   RL_OUTPUT_DIR = File.dirname(__FILE__) + "/../ragel/i18n"
 
-  def initialize(lang, i18n, keywords)
+  def initialize(lang, i18n)
     @lang     = lang
     @i18n     = i18n
-    @keywords = keywords
     define_tasks
   end
 
@@ -18,27 +17,24 @@ class RagelTask
     end
 
     file lang_ragel => lang_erb do
-      impl = ERB.new(IO.read(lang_erb)).result(binding)
-      write(impl, lang_ragel)
+      write(ERB.new(IO.read(lang_erb)).result(binding), lang_ragel)
     end
 
     file common_ragel => common_erb  do
-      keywords = prep_keywords
-      common = ERB.new(IO.read(common_erb)).result(binding)
-      write(common, common_ragel)
+      write(ERB.new(IO.read(common_erb)).result(binding), common_ragel)
     end
   end
 
   def target
     {
-      'c'    => "ext/gherkin_lexer_#{@i18n}/gherkin_lexer_#{@i18n}.c",
-      'java' => "java/src/gherkin/lexer/#{@i18n.capitalize}.java",
-      'rb'   => "lib/gherkin/rb_lexer/#{@i18n}.rb"
+      'c'    => "ext/gherkin_lexer_#{@i18n.sanitized_key}/gherkin_lexer_#{@i18n.sanitized_key}.c",
+      'java' => "java/src/gherkin/lexer/#{@i18n.sanitized_key.capitalize}.java",
+      'rb'   => "lib/gherkin/rb_lexer/#{@i18n.sanitized_key}.rb"
     }[@lang]
   end
 
   def common_ragel
-    RL_OUTPUT_DIR + "/lexer_common.#{@i18n}.rl"
+    RL_OUTPUT_DIR + "/lexer_common.#{@i18n.sanitized_key}.rl"
   end
 
   def common_erb
@@ -46,7 +42,7 @@ class RagelTask
   end
 
   def lang_ragel
-    RL_OUTPUT_DIR + "/#{@i18n}.#{@lang}.rl"
+    RL_OUTPUT_DIR + "/#{@i18n.sanitized_key}.#{@lang}.rl"
   end
 
   def lang_erb
@@ -61,23 +57,14 @@ class RagelTask
     }[@lang]
   end
 
-  def prep_keywords
-    keywords = @keywords.dup
-    delimited_keywords = %w{feature background scenario scenario_outline examples}
-    bare_keywords = %w{given when then and but}
-    all_keywords = delimited_keywords + bare_keywords
-    
-    all_keywords.each       { |k| keywords[k] = keywords[k].split("|") }
-    delimited_keywords.each { |k| keywords[k].map! { |v| v += ':'} }
-    bare_keywords.each      { |k| keywords[k].map! { |v| (v + ' ').sub(/< $/, '')} }
-    all_keywords.each       { |k| keywords[k] = '("' + keywords[k].join('" | "') + '")' }    
-    keywords
-  end
-
   def write(content, filename)
     mkdir_p(File.dirname(filename)) unless File.directory?(File.dirname(filename))
     File.open(filename, "wb") do |file|
       file.write(content)
     end
-  end  
+  end
+
+  def ragel_list(keywords)
+    "(#{keywords.map{|keyword| %{"#{keyword}"}}.join(' | ')})"
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: gherkin
 version: !ruby/object:Gem::Version 
-  version: 0.0.4
+  version: 1.0.0
 platform: i386-mswin32
 authors: 
 - Mike Sassak
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-28 00:00:00 +01:00
+date: 2009-12-01 00:00:00 +01:00
 default_executable: gherkin
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -85,6 +85,7 @@ files:
 - lib/gherkin.rb
 - lib/gherkin/c_lexer.rb
 - lib/gherkin/core_ext/array.rb
+- lib/gherkin/i18n.rb
 - lib/gherkin/i18n.yml
 - lib/gherkin/i18n_lexer.rb
 - lib/gherkin/java_lexer.rb
@@ -98,46 +99,6 @@ files:
 - lib/gherkin/rb_lexer/README.rdoc
 - lib/gherkin/rb_parser.rb
 - lib/gherkin/tools/pretty_printer.rb
-- lib/gherkin_lexer_ar.so
-- lib/gherkin_lexer_bg.so
-- lib/gherkin_lexer_cat.so
-- lib/gherkin_lexer_cs.so
-- lib/gherkin_lexer_cy.so
-- lib/gherkin_lexer_da.so
-- lib/gherkin_lexer_de.so
-- lib/gherkin_lexer_en.so
-- lib/gherkin_lexer_enau.so
-- lib/gherkin_lexer_enlol.so
-- lib/gherkin_lexer_entx.so
-- lib/gherkin_lexer_es.so
-- lib/gherkin_lexer_et.so
-- lib/gherkin_lexer_fi.so
-- lib/gherkin_lexer_fr.so
-- lib/gherkin_lexer_he.so
-- lib/gherkin_lexer_hr.so
-- lib/gherkin_lexer_hu.so
-- lib/gherkin_lexer_id.so
-- lib/gherkin_lexer_it.so
-- lib/gherkin_lexer_ja.so
-- lib/gherkin_lexer_ko.so
-- lib/gherkin_lexer_lt.so
-- lib/gherkin_lexer_lv.so
-- lib/gherkin_lexer_nl.so
-- lib/gherkin_lexer_no.so
-- lib/gherkin_lexer_pl.so
-- lib/gherkin_lexer_pt.so
-- lib/gherkin_lexer_ro.so
-- lib/gherkin_lexer_ro2.so
-- lib/gherkin_lexer_ru.so
-- lib/gherkin_lexer_se.so
-- lib/gherkin_lexer_sk.so
-- lib/gherkin_lexer_sr.so
-- lib/gherkin_lexer_srLatn.so
-- lib/gherkin_lexer_tr.so
-- lib/gherkin_lexer_uz.so
-- lib/gherkin_lexer_vi.so
-- lib/gherkin_lexer_zhCN.so
-- lib/gherkin_lexer_zhTW.so
 - nativegems.sh
 - ragel/i18n/.gitignore
 - ragel/lexer.c.rl.erb