gherkin 0.0.3-universal-java-1.5 → 0.0.4-universal-java-1.5

data/.gitignore

@@ -3,6 +3,5 @@
 coverage
 rdoc
 pkg
-*.dot
-*.png
-*.so
+ext
+tmp

data/README.rdoc

@@ -17,6 +17,25 @@ rake spec cucumber
 
 rake clobber
 
+== Release process
+
+Run just "rake" for each platform (1.8.6, 1.8.7, 1.9, jruby) to make sure all is green.
+
+1) Bump version in the VERSION file
+2) rake gemspec
+3) Commit everything
+4) rake release
+5) ./nativegems.sh
+6) gem push pkg/... (for each native gem)
+
+TODO: Also build windows gem with dll using rake-compiler. MinGW gem can be done on OS X/Linux, but
+the one for the old Ruby one-click installers must be built with Visual Studio/nmake.
+
+== Build windows gems on OS X
+
+* http://www.copiousfreetime.org/articles/2008/10/12/building-gems-for-windows.html
+* rake-compiler docs
+
 == Notes
 
 Ragel supports Ruby, but it's much slower than C. The ruby target will be used for development.

data/Rakefile

@@ -4,9 +4,6 @@ require 'rubygems'
 require 'rake'
 require 'rake/clean'
 
-JRUBY   = defined?(JRUBY_VERSION)
-WINDOWS = Config::CONFIG['host_os'] =~ /mswin|mingw/
-
 begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
@@ -18,22 +15,26 @@ begin
     gem.authors = ["Mike Sassak", "Gregory Hnatiuk", "Aslak Hellesøy"]
     gem.executables = ["gherkin"]
     gem.add_development_dependency "rspec", "1.2.9"
-    gem.add_development_dependency "cucumber"
+    gem.add_development_dependency "cucumber", "0.4.4"
+    gem.add_development_dependency "rake-compiler", "0.6.0" unless defined?(JRUBY_VERSION)
     
-    # Jeweler only includes files in git by default. Add the generated ones.
-    gem.files += FileList['lib/gherkin/rb_lexer/*.rb']
-
-    if(JRUBY)
-      gem.platform = Gem::Platform::CURRENT
+    case ENV['PLATFORM']
+    when 'universal-java-1.5'
+      gem.platform = 'universal-java-1.5'
       gem.files += FileList['lib/gherkin.jar']
       gem.extensions = []
-    elsif(WINDOWS)
-      gem.platform = Gem::Platform::CURRENT
-      gem.files += FileList['lib/gherkin_lexer.dll']
+    when 'i386-mswin32'
+      gem.platform = 'i386-mswin32'
+      gem.files += FileList['lib/*.so']
+      gem.extensions = []
+    when 'i386-mingw32'
+      gem.platform = 'i386-mingw32'
+      gem.files += FileList['lib/*.so']
       gem.extensions = []
     else
-      gem.files += FileList['ext/gherkin_lexer/*.{c,h}']
-      gem.extensions = 'ext/gherkin_lexer/extconf.rb'
+      gem.files += FileList['lib/gherkin/rb_lexer/*.rb']
+      gem.files += FileList['ext/**/*.c']
+      gem.extensions = FileList['ext/**/extconf.rb']
     end
     
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
@@ -45,4 +46,4 @@ end
 
 Dir['tasks/**/*.rake'].each { |rake| load rake }
 
-task :default => [:spec, :cucumber]
+task :default => [:jar, :compile, :spec, :cucumber]

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :minor: 0
-:patch: 3
+:patch: 4
 :major: 0

data/bin/gherkin

@@ -6,5 +6,5 @@ require 'gherkin/tools/pretty_printer'
 
 listener = Gherkin::Tools::PrettyPrinter.new(STDOUT)
 parser = Gherkin::Parser.new(listener, true) # We could skip the parser here, if we don't want to verify well-formedness
-lexer = Gherkin::Lexer['en'].new(parser)
-lexer.scan(IO.read(ARGV[0]))
+lexer = Gherkin::I18nLexer.new(parser)
+lexer.scan(IO.read(ARGV[0]))

data/features/pretty_printer.feature

@@ -6,6 +6,9 @@ Feature: Pretty printer
     Given I have Cucumber's home dir defined in CUCUMBER_HOME
     When I find all of the .feature files
     And I parse the prettified representation of each of them
-    # Of course, we don't really want all those errors, but these are the current ones.
+    # Of course, we don't really want any errors. The last
+    # two files that were not identical are written to p1.feature
+    # and p2.feature. Do a diff -u p1.feature p2.feature
+    #
     Then the following files should have errors:
-      | Path                                                       | Error                                        |
+      | Path | Error |

data/features/step_definitions/pretty_printer_steps.rb

@@ -1,4 +1,5 @@
 require 'stringio'
+require 'gherkin'
 require 'gherkin/tools/pretty_printer'
 
 module PrettyPlease
@@ -27,12 +28,16 @@ end
 When /^I parse the prettified representation of each of them$/ do
   @errors = [['Path', 'Error']]
   @features.each do |feature|
+    pretty1 = nil
+    pretty2 = nil
     begin
+#      announce "========== #{feature}:"
       pretty1 = pretty(IO.read(feature))
       pretty2 = pretty(pretty1)
-
       pretty2.should == pretty1
     rescue Spec::Expectations::ExpectationNotMetError => e
+      File.open("p1.feature", "w") {|io| io.write(pretty1)}
+      File.open("p2.feature", "w") {|io| io.write(pretty2)}
       announce "========== #{feature}:"
       if(e.message =~ /(@@.*)/m)
         announce $1

data/gherkin.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gherkin}
-  s.version = "0.0.3"
+  s.version = "0.0.4"
   s.platform = %q{universal-java-1.5}
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Mike Sassak", "Gregory Hnatiuk", "Aslak Helles\303\270y"]
-  s.date = %q{2009-11-25}
+  s.date = %q{2009-11-28}
   s.default_executable = %q{gherkin}
   s.description = %q{A fast Gherkin lexer in Ragel}
   s.email = %q{cukes@googlegroups.com}
@@ -27,8 +27,6 @@ Gem::Specification.new do |s|
      "VERSION.yml",
      "bin/gherkin",
      "cucumber.yml",
-     "ext/gherkin_lexer/.gitignore",
-     "ext/gherkin_lexer/extconf.rb",
      "features/feature_parser.feature",
      "features/native_lexer.feature",
      "features/parser_with_native_lexer.feature",
@@ -53,6 +51,7 @@ Gem::Specification.new do |s|
      "lib/gherkin.jar",
      "lib/gherkin.rb",
      "lib/gherkin/c_lexer.rb",
+     "lib/gherkin/core_ext/array.rb",
      "lib/gherkin/i18n.yml",
      "lib/gherkin/i18n_lexer.rb",
      "lib/gherkin/java_lexer.rb",
@@ -64,54 +63,16 @@ Gem::Specification.new do |s|
      "lib/gherkin/rb_lexer.rb",
      "lib/gherkin/rb_lexer/.gitignore",
      "lib/gherkin/rb_lexer/README.rdoc",
-     "lib/gherkin/rb_lexer/ar.rb",
-     "lib/gherkin/rb_lexer/bg.rb",
-     "lib/gherkin/rb_lexer/cat.rb",
-     "lib/gherkin/rb_lexer/cs.rb",
-     "lib/gherkin/rb_lexer/cy.rb",
-     "lib/gherkin/rb_lexer/da.rb",
-     "lib/gherkin/rb_lexer/de.rb",
-     "lib/gherkin/rb_lexer/en-au.rb",
-     "lib/gherkin/rb_lexer/en-lol.rb",
-     "lib/gherkin/rb_lexer/en-tx.rb",
-     "lib/gherkin/rb_lexer/en.rb",
-     "lib/gherkin/rb_lexer/es.rb",
-     "lib/gherkin/rb_lexer/et.rb",
-     "lib/gherkin/rb_lexer/fi.rb",
-     "lib/gherkin/rb_lexer/fr.rb",
-     "lib/gherkin/rb_lexer/he.rb",
-     "lib/gherkin/rb_lexer/hr.rb",
-     "lib/gherkin/rb_lexer/hu.rb",
-     "lib/gherkin/rb_lexer/id.rb",
-     "lib/gherkin/rb_lexer/it.rb",
-     "lib/gherkin/rb_lexer/ja.rb",
-     "lib/gherkin/rb_lexer/ko.rb",
-     "lib/gherkin/rb_lexer/lt.rb",
-     "lib/gherkin/rb_lexer/lv.rb",
-     "lib/gherkin/rb_lexer/nl.rb",
-     "lib/gherkin/rb_lexer/no.rb",
-     "lib/gherkin/rb_lexer/pl.rb",
-     "lib/gherkin/rb_lexer/pt.rb",
-     "lib/gherkin/rb_lexer/ro.rb",
-     "lib/gherkin/rb_lexer/ro2.rb",
-     "lib/gherkin/rb_lexer/ru.rb",
-     "lib/gherkin/rb_lexer/se.rb",
-     "lib/gherkin/rb_lexer/sk.rb",
-     "lib/gherkin/rb_lexer/sr-Latn.rb",
-     "lib/gherkin/rb_lexer/sr.rb",
-     "lib/gherkin/rb_lexer/tr.rb",
-     "lib/gherkin/rb_lexer/uz.rb",
-     "lib/gherkin/rb_lexer/vi.rb",
-     "lib/gherkin/rb_lexer/zh-CN.rb",
-     "lib/gherkin/rb_lexer/zh-TW.rb",
      "lib/gherkin/rb_parser.rb",
      "lib/gherkin/tools/pretty_printer.rb",
+     "nativegems.sh",
      "ragel/i18n/.gitignore",
      "ragel/lexer.c.rl.erb",
      "ragel/lexer.java.rl.erb",
      "ragel/lexer.rb.rl.erb",
      "ragel/lexer_common.rl.erb",
      "spec/gherkin/c_lexer_spec.rb",
+     "spec/gherkin/fixtures/1.feature",
      "spec/gherkin/fixtures/complex.feature",
      "spec/gherkin/fixtures/i18n_fr.feature",
      "spec/gherkin/fixtures/i18n_no.feature",
@@ -133,9 +94,9 @@ Gem::Specification.new do |s|
      "tasks/bench/feature_builder.rb",
      "tasks/bench/generated/.gitignore",
      "tasks/bench/null_listener.rb",
+     "tasks/compile.rake",
      "tasks/cucumber.rake",
-     "tasks/ext.rake",
-     "tasks/ragel.rake",
+     "tasks/ragel_task.rb",
      "tasks/rdoc.rake",
      "tasks/rspec.rake"
   ]
@@ -145,8 +106,7 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{Fast Gherkin lexer}
   s.test_files = [
-    "spec/spec_helper.rb",
-     "spec/gherkin/c_lexer_spec.rb",
+    "spec/gherkin/c_lexer_spec.rb",
      "spec/gherkin/i18n_spec.rb",
      "spec/gherkin/java_lexer_spec.rb",
      "spec/gherkin/parser_spec.rb",
@@ -155,7 +115,8 @@ Gem::Specification.new do |s|
      "spec/gherkin/shared/lexer_spec.rb",
      "spec/gherkin/shared/py_string_spec.rb",
      "spec/gherkin/shared/table_spec.rb",
-     "spec/gherkin/shared/tags_spec.rb"
+     "spec/gherkin/shared/tags_spec.rb",
+     "spec/spec_helper.rb"
   ]
 
   if s.respond_to? :specification_version then
@@ -164,14 +125,17 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, ["= 1.2.9"])
-      s.add_development_dependency(%q<cucumber>, [">= 0"])
+      s.add_development_dependency(%q<cucumber>, ["= 0.4.4"])
+      s.add_development_dependency(%q<rake-compiler>, ["= 0.6.0"])
     else
       s.add_dependency(%q<rspec>, ["= 1.2.9"])
-      s.add_dependency(%q<cucumber>, [">= 0"])
+      s.add_dependency(%q<cucumber>, ["= 0.4.4"])
+      s.add_dependency(%q<rake-compiler>, ["= 0.6.0"])
     end
   else
     s.add_dependency(%q<rspec>, ["= 1.2.9"])
-    s.add_dependency(%q<cucumber>, [">= 0"])
+    s.add_dependency(%q<cucumber>, ["= 0.4.4"])
+    s.add_dependency(%q<rake-compiler>, ["= 0.6.0"])
   end
 end
 

data/lib/.gitignore

@@ -1,2 +1,4 @@
-gherkin_lexer.bundle
-gherkin.jar
+*.bundle
+*.dll
+*.so
+*.jar

data/lib/gherkin/c_lexer.rb

@@ -1,9 +1,9 @@
-require 'gherkin_lexer'
-
 module Gherkin
   module CLexer
     def self.[](i18n_language)
-      i18n_lexer_class_name = i18n_language.gsub(/[\s-]/, '').capitalize
+      name = i18n_language.gsub(/[\s-]/, '')
+      require "gherkin_lexer_#{name}"
+      i18n_lexer_class_name = name.capitalize
       const_get(i18n_lexer_class_name)
     end
   end

data/lib/gherkin/core_ext/array.rb

@@ -0,0 +1,5 @@
+class Array
+  def utf8_pack(fmt)
+    (RUBY_VERSION =~ /^1\.9/) ? pack(fmt).force_encoding("UTF-8") : pack(fmt)
+  end
+end

data/lib/gherkin/lexer.rb

@@ -6,20 +6,21 @@ module Gherkin
     class << self
       def [](i18n_lang)
         begin
+          # Uncomment the line below (during development) to force use of Ruby lexer
+          # return rb[i18n_lang]
+
           if defined?(JRUBY_VERSION)
             java[i18n_lang]
           else
             begin
               c[i18n_lang]
-            rescue NameError => e
+            rescue NameError, LoadError => e
               warn("WARNING: #{e.message}. Reverting to Ruby lexer")
               rb[i18n_lang]
-            rescue LoadError
-              rb[i18n_lang]
             end
           end
-        rescue LoadError
-          raise I18nLexerNotFound, "No lexer was found for #{i18n_lang}. Supported languages are listed in gherkin/i18n.yml."
+        rescue LoadError => e
+          raise I18nLexerNotFound, "No lexer was found for #{i18n_lang} (#{e.message}). Supported languages are listed in gherkin/i18n.yml."
         end
       end
 

data/lib/gherkin/rb_lexer.rb

@@ -1,8 +1,9 @@
 module Gherkin
   module RbLexer
     def self.[](i18n_language)
-      require "gherkin/rb_lexer/#{i18n_language}"
-      i18n_lexer_class_name = i18n_language.gsub(/[\s-]/, '').capitalize
+      name = i18n_language.gsub(/[\s-]/, '')
+      require "gherkin/rb_lexer/#{name}"
+      i18n_lexer_class_name = name.capitalize
       const_get(i18n_lexer_class_name)
     end
   end

data/lib/gherkin/tools/pretty_printer.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 module Gherkin
   module Tools
     class PrettyPrinter
@@ -53,9 +54,7 @@ module Gherkin
       end
 
       def py_string(string, line)
-        @io.puts '      """'
-        @io.puts string.gsub(/^/, '      ')
-        @io.puts '      """'
+        @io.puts "      \"\"\"\n" + string.gsub(START, '      ') + "\n      \"\"\""
       end
 
       def syntax_error(state, event, legal_events, line)
@@ -63,10 +62,17 @@ module Gherkin
       end
 
     private
+      if(RUBY_VERSION =~ /^1\.9/)
+        START = /#{"^".encode('UTF-8')}/
+        NL    = Regexp.new("\n".encode('UTF-8'))
+      else
+        START = /^/
+        NL    = /\n/n
+      end
 
       def indent(string, indentation)
         indent = ""
-        string.split(/\n/n).map do |l|
+        string.split(NL).map do |l|
           s = "#{indent}#{l}"
           indent = indentation
           s

data/nativegems.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+# Builds gems for all supported platforms
+rake gemspec build PLATFORM=universal-java-1.5
+rake cross compile gemspec build PLATFORM=i386-mswin32 RUBY_CC_VERSION=1.8.6
+rake gemspec build PLATFORM=i386-mingw32 RUBY_CC_VERSION=1.8.6

data/ragel/lexer.c.rl.erb

@@ -5,6 +5,19 @@
 #include <stddef.h>
 #endif
 
+#ifdef HAVE_RUBY_RE_H
+#include <ruby/re.h>
+#endif
+
+#ifdef HAVE_RUBY_ENCODING_H
+#include <ruby/encoding.h>
+#define ENCODED_STR_NEW(ptr, len) \
+    rb_enc_str_new(ptr, len, rb_utf8_encoding());
+#else
+#define ENCODED_STR_NEW(ptr, len) \
+    rb_str_new(ptr, len); 
+#endif
+
 #ifndef RSTRING_PTR
 #define RSTRING_PTR(s) (RSTRING(s)->ptr)
 #endif
@@ -115,10 +128,12 @@ static VALUE rb_eGherkinLexerError;
   
   action store_comment_content {
     STORE_ATTR(comment)
+    lexer->mark = 0;
   }
   
   action store_tag_content {
     STORE_ATTR(tag)
+    lexer->mark = 0;
   }
   
   action inc_line_number {
@@ -160,7 +175,7 @@ static VALUE rb_eGherkinLexerError;
 
   action store_cell_content {
     VALUE con = Qnil;
-    con = rb_str_new(PTR_TO(content_start), LEN(content_start, p));
+    con = ENCODED_STR_NEW(PTR_TO(content_start), LEN(content_start, p));
     rb_funcall(con, rb_intern("strip!"), 0);
 
     rb_ary_push(current_row, con);
@@ -217,7 +232,7 @@ static VALUE rb_eGherkinLexerError;
     }
   }
 
-  include lexer_common "lexer_common.<%= i18n_language %>.rl";   
+  include lexer_common "lexer_common.<%= @i18n %>.rl";   
 
 }%%
 
@@ -251,8 +266,8 @@ store_kw_con(VALUE listener, const char * event_name,
              int current_line)
 {
   VALUE con = Qnil, kw = Qnil;
-  kw = rb_str_new(keyword_at, keyword_length);
-  con = rb_str_new(at, length);
+  kw = ENCODED_STR_NEW(keyword_at, keyword_length);
+  con = ENCODED_STR_NEW(at, length);
   con = multiline_strip(con);
   rb_funcall(con, rb_intern("strip!"), 0);
   rb_funcall(kw, rb_intern("strip!"), 0);
@@ -264,7 +279,7 @@ store_attr(VALUE listener, const char * attr_type,
            const char * at, size_t length, 
            int line)
 {
-  VALUE val = rb_str_new(at, length);
+  VALUE val = ENCODED_STR_NEW(at, length);
   rb_funcall(listener, rb_intern(attr_type), 2, val, INT2FIX(line));
 }
 
@@ -274,8 +289,8 @@ store_pystring_content(VALUE listener,
           const char *at, size_t length, 
           int current_line)
 {
-  VALUE con = rb_str_new(at, length);
-  // Gherkin will crash gracefully if the string representation of start_col pushes the pattern past 64 characters
+  VALUE con = ENCODED_STR_NEW(at, length);
+  // Gherkin will crash gracefully if the string representation of start_col pushes the pattern past 32 characters
   char pat[32]; 
   snprintf(pat, 32, "^ {0,%d}", start_col); 
   VALUE re = rb_reg_regcomp(rb_str_new2(pat));
@@ -331,9 +346,10 @@ static VALUE CLexer_scan(VALUE self, VALUE input)
   lexer_state *lexer = NULL;
   DATA_GET(self, lexer_state, lexer);
 
-  rb_str_append(input, rb_str_new2("\n%_FEATURE_END_%"));
-  char *data = RSTRING_PTR(input);
-  long len = RSTRING_LEN(input);
+  VALUE input_copy = rb_str_dup(input);
+  rb_str_append(input_copy, rb_str_new2("\n%_FEATURE_END_%"));
+  char *data = RSTRING_PTR(input_copy);
+  long len = RSTRING_LEN(input_copy);
  
   if (len == 0) { 
     rb_raise(rb_eGherkinLexerError, "No content to lex.");
@@ -371,14 +387,14 @@ static VALUE CLexer_scan(VALUE self, VALUE input)
   }
 }
 
-void Init_gherkin_lexer()
+void Init_gherkin_lexer_<%= @i18n %>()
 {
   mGherkin = rb_define_module("Gherkin");
   mLexer = rb_const_get(mGherkin, rb_intern("Lexer"));
   rb_eGherkinLexerError = rb_const_get(mLexer, rb_intern("LexingError"));
 
   mCLexer = rb_define_module_under(mGherkin, "CLexer");
-  cI18nLexer = rb_define_class_under(mCLexer, "En", rb_cObject);
+  cI18nLexer = rb_define_class_under(mCLexer, "<%= @i18n.capitalize %>", rb_cObject);
   rb_define_alloc_func(cI18nLexer, CLexer_alloc);
   rb_define_method(cI18nLexer, "initialize", CLexer_init, 1);
   rb_define_method(cI18nLexer, "scan", CLexer_scan, 1);