gherkin 1.0.30-universal-dotnet

data/ragel/lexer.java.rl.erb

@@ -0,0 +1,216 @@
+package gherkin.lexer;
+
+import java.io.UnsupportedEncodingException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+import gherkin.Lexer;
+import gherkin.Listener;
+import gherkin.LexingError;
+
+public class <%= @i18n.underscored_iso_code.upcase %> implements Lexer {
+  %%{
+    machine lexer;
+    alphtype byte;
+
+    action begin_content {
+      contentStart = p;
+      currentLine = lineNumber;
+    }
+
+    action start_pystring {
+      currentLine = lineNumber;
+      startCol = p - lastNewline;
+    }
+    
+    action begin_pystring_content {
+      contentStart = p;
+    }
+
+    action store_pystring_content {
+      String con = unindent(startCol, substring(data, contentStart, nextKeywordStart-1).replaceFirst("(\\r?\\n)?([\\t ])*\\Z", "").replaceAll("\\\\\"\\\\\"\\\\\"", "\"\"\""));
+      listener.pyString(con, currentLine);
+    }
+
+    action store_feature_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart).trim());
+      listener.feature(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_background_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.background(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_scenario_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.scenario(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_scenario_outline_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.scenarioOutline(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_examples_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.examples(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_step_content {
+      listener.step(keyword, substring(data, contentStart, p).trim(), currentLine);
+    }
+
+    action store_comment_content {
+      listener.comment(substring(data, contentStart, p).trim(), lineNumber);
+      keywordStart = -1;
+    }
+
+    action store_tag_content {
+      listener.tag(substring(data, contentStart, p).trim(), currentLine);
+      keywordStart = -1;
+    }
+
+    action inc_line_number {
+      lineNumber++;
+    }
+
+    action last_newline {
+      lastNewline = p + 1;
+    }
+
+    action start_keyword {
+      if(keywordStart == -1) keywordStart = p;
+    }
+
+    action end_keyword {
+      keyword = substring(data, keywordStart, p).replaceFirst(":$","");
+      keywordStart = -1;
+    }
+
+    action next_keyword_start {
+      nextKeywordStart = p;
+    }
+
+    action start_row {
+      p = p - 1;
+      currentRow = new ArrayList<String>();
+      currentLine = lineNumber;
+    }
+
+    action begin_cell_content {
+      contentStart = p;
+    }
+
+    action store_cell_content {
+      String con = substring(data, contentStart, p).trim();
+      currentRow.add(con.replaceAll("\\\\\\|", "|").replaceAll("\\\\\\\\", "\\\\"));
+    }
+
+    action store_row {
+      listener.row(currentRow, currentLine);
+    }
+
+    action end_feature {
+      if(cs < lexer_first_final) {
+        String content = currentLineContent(data, lastNewline);
+        throw new LexingError("Lexing error on line " + lineNumber + ": '" + content + "'");
+      } else {
+        listener.eof();
+      }
+    }
+
+    include lexer_common "lexer_common.<%= @i18n.underscored_iso_code %>.rl"; 
+  }%%
+
+  private final Listener listener;
+
+  public <%= @i18n.underscored_iso_code.upcase %>(Listener listener) {
+    this.listener = listener;
+  }
+
+  %% write data noerror;
+
+  public void scan(String inputSequence)  {
+    String input = inputSequence.toString() + "\n%_FEATURE_END_%";
+    byte[] data = null;
+    try {
+      data = input.getBytes("UTF-8");
+    } catch(UnsupportedEncodingException e) {
+      throw new RuntimeException(e);
+    }
+    int cs, p = 0, pe = data.length;
+    int eof = pe;
+
+    int lineNumber = 1;
+    int lastNewline = 0;
+
+    int contentStart = -1;
+    int currentLine = -1;
+    int startCol = -1;
+    int nextKeywordStart = -1;
+    int keywordStart = -1;
+    String keyword = null;
+    List<String> currentRow = null;
+
+    %% write init;
+    %% write exec;
+  }
+
+  private String keywordContent(byte[] data, int p, int eof, int nextKeywordStart, int contentStart) {
+    int endPoint = (nextKeywordStart == -1 || (p == eof)) ? p : nextKeywordStart;
+    return substring(data, contentStart, endPoint);
+  }
+
+  private static final Pattern CRLF_RE = Pattern.compile("\r\n");
+  private static final Pattern LF_RE = Pattern.compile("[^\r]\n");
+  private static final String CRLF = "\r\n";
+  private static final String LF = "\n";
+
+  private String multilineStrip(String text) {
+    int crlfCount = matchCount(CRLF_RE.matcher(text));
+    int lfCount = matchCount(LF_RE.matcher(text));
+    String eol = crlfCount > lfCount ? CRLF : LF;
+
+    StringBuffer result = new StringBuffer();
+    for(String s : text.split("\r?\n")) {
+      result.append(s.trim()).append(eol);
+    }
+    return result.toString().trim();
+  }
+
+  private int matchCount(Matcher m) {
+    int count = 0;
+    while(m.find()) {
+      count++;
+    }
+    return count;
+  }
+
+  private String unindent(int startCol, String text) {
+    return Pattern.compile("^[\t ]{0," + startCol + "}", Pattern.MULTILINE).matcher(text).replaceAll("");
+  }
+
+  private String currentLineContent(byte[] data, int lastNewline) {
+    return substring(data, lastNewline, data.length).trim();
+  }
+
+  private String substring(byte[] data, int start, int end) {
+    try {
+      return new String(data, start, end-start, "utf-8");
+    } catch(java.io.UnsupportedEncodingException e) {
+      throw new RuntimeException("Internal error", e);
+    }
+  }
+}

data/ragel/lexer.rb.rl.erb

@@ -0,0 +1,173 @@
+module Gherkin
+  module RbLexer
+    class <%= @i18n.underscored_iso_code.capitalize %> #:nodoc:
+      %%{
+        machine lexer;
+ 
+        action begin_content {
+          @content_start = p
+          @current_line = @line_number
+        }
+
+        action start_pystring {
+          @current_line = @line_number
+          @start_col = p - @last_newline
+        }
+        
+        action begin_pystring_content {
+          @content_start = p
+        }
+
+        action store_pystring_content {
+          con = unindent(@start_col, utf8_pack(data[@content_start...@next_keyword_start-1]).sub(/(\r?\n)?([\t ])*\Z/, '').gsub(/\\"\\"\\"/, '"""'))
+          @listener.py_string(con, @current_line) 
+        }
+ 
+        action store_feature_content {
+          store_keyword_content(:feature, data, p, eof) { |con| multiline_strip(con) }
+          p = @next_keyword_start - 1 if @next_keyword_start          
+          @next_keyword_start = nil
+        }
+ 
+        action store_background_content {
+          store_keyword_content(:background, data, p, eof) { |con| multiline_strip(con) }
+          p = @next_keyword_start - 1 if @next_keyword_start                    
+          @next_keyword_start = nil
+        }
+
+        action store_scenario_content {
+          store_keyword_content(:scenario, data, p, eof) { |con| multiline_strip(con) }
+          p = @next_keyword_start - 1 if @next_keyword_start                    
+          @next_keyword_start = nil
+        }
+
+        action store_scenario_outline_content {
+          store_keyword_content(:scenario_outline, data, p, eof) { |con| multiline_strip(con) }
+          p = @next_keyword_start - 1 if @next_keyword_start                    
+          @next_keyword_start = nil
+        }
+
+        action store_examples_content {
+          store_keyword_content(:examples, data, p, eof) { |con| multiline_strip(con) }
+          p = @next_keyword_start - 1 if @next_keyword_start                    
+          @next_keyword_start = nil
+        }
+
+        action store_step_content {
+          con = utf8_pack(data[@content_start...p]).strip
+          @listener.step(@keyword, con, @current_line)
+        }
+ 
+        action store_comment_content {
+          con = utf8_pack(data[@content_start...p]).strip
+          @listener.comment(con, @line_number)
+          @keyword_start = nil
+        }
+ 
+        action store_tag_content {
+          con = utf8_pack(data[@content_start...p]).strip
+          @listener.tag(con, @current_line)
+          @keyword_start = nil
+        }
+ 
+        action inc_line_number {
+          @line_number += 1
+        }
+ 
+        action last_newline {
+          @last_newline = p + 1
+        }
+
+        action start_keyword {
+          @keyword_start ||= p
+        }
+ 
+        action end_keyword {
+          @keyword = utf8_pack(data[@keyword_start...p]).sub(/:$/,'')
+          @keyword_start = nil
+        }
+ 
+        action next_keyword_start {
+          @next_keyword_start = p
+        }
+
+        action start_row {
+          p = p - 1
+          current_row = []
+          @current_line = @line_number
+        }
+
+        action begin_cell_content {
+          @content_start = p
+        }
+
+        action store_cell_content {
+          con = utf8_pack(data[@content_start...p]).strip
+          current_row << con.gsub(/\\\|/, "|").gsub(/\\\\/, "\\")
+        }
+
+        action store_row {
+          @listener.row(current_row, @current_line)
+        }
+
+        action end_feature {
+          if cs < lexer_first_final
+            content = current_line_content(data, p)
+            raise LexingError.new("Lexing error on line %d: '%s'." % [@line_number, content])
+          else
+            @listener.eof
+          end
+        }
+
+        include lexer_common "lexer_common.<%= @i18n.underscored_iso_code %>.rl"; 
+      }%%
+ 
+      def initialize(listener)
+        @listener = listener
+        %% write data;
+      end
+ 
+      def scan(data)
+        data = (data + "\n%_FEATURE_END_%").unpack("c*") # Explicit EOF simplifies things considerably
+        eof = pe = data.length
+ 
+        @line_number = 1
+        @last_newline = 0
+ 
+        %% write init;
+        %% write exec;
+      end
+
+      CRLF_RE = /\r\n/
+      LF_RE = /[^\r]\n/
+      CRLF = "\r\n"
+      LF = "\n"
+
+      def multiline_strip(text)
+        crlf_count = text.scan(CRLF_RE).size
+        lf_count = text.scan(LF_RE).size
+        eol = crlf_count > lf_count ? CRLF : LF
+        text.split(/\r?\n/).map{|s| s.strip}.join(eol).strip
+      end
+
+      def unindent(startcol, text)
+        text.gsub(/^[\t ]{0,#{startcol}}/, "")
+      end
+ 
+      def store_keyword_content(event, data, p, eof)
+        end_point = (!@next_keyword_start or (p == eof)) ? p : @next_keyword_start
+        con = yield utf8_pack(data[@content_start...end_point])
+        @listener.send(event, @keyword, con, @current_line)
+      end
+      
+      def current_line_content(data, p)
+        rest = data[@last_newline..-1]
+        utf8_pack(rest[0..rest.index(10)||-1]).strip
+      end
+
+      def utf8_pack(array)
+        (RUBY_VERSION =~ /^1\.9/) ? array.pack("c*").force_encoding("UTF-8") : array.pack("c*")
+      end
+    end
+  end
+end

data/ragel/lexer_common.rl.erb

@@ -0,0 +1,50 @@
+%%{
+  machine lexer_common;
+
+  # Language specific
+  I18N_Feature = (<%= ragel_list(@i18n.keywords('feature')) %> ':') >start_keyword %end_keyword; 
+  I18N_Background = (<%= ragel_list(@i18n.keywords('background')) %> ':') >start_keyword %end_keyword;
+  I18N_ScenarioOutline = (<%= ragel_list(@i18n.keywords('scenario_outline')) %> ':') >start_keyword %end_keyword;
+  I18N_Scenario = (<%= ragel_list(@i18n.keywords('scenario')) %> ':') >start_keyword %end_keyword;
+  I18N_Step = <%= ragel_list(@i18n.step_keywords) %> >start_keyword %end_keyword;
+  I18N_Examples = (<%= ragel_list(@i18n.keywords('examples')) %> ':') >start_keyword %end_keyword;
+
+  EOF = '%_FEATURE_END_%'; # Explicit EOF added before scanning begins 
+  EOL = ('\n' | '\r\n') @inc_line_number @last_newline;
+  BOM = 0xEF 0xBB 0xBF; # http://en.wikipedia.org/wiki/Byte_order_mark
+
+  PIPE = '|';
+  ESCAPED_PIPE = '\\|';
+
+  FeatureHeadingEnd = EOL+ space* (I18N_Background | I18N_Scenario | I18N_ScenarioOutline | '@' | '#' | EOF) >next_keyword_start;
+  ScenarioHeadingEnd = EOL+ space* ( I18N_Scenario | I18N_ScenarioOutline | I18N_Step | '@' | '#' | EOF ) >next_keyword_start;
+  BackgroundHeadingEnd = EOL+ space* ( I18N_Scenario | I18N_ScenarioOutline | I18N_Step | '@' | '#'| EOF ) >next_keyword_start;
+  ScenarioOutlineHeadingEnd = EOL+ space* ( I18N_Scenario | I18N_Step | '@' | '#' | EOF ) >next_keyword_start;
+  ExamplesHeadingEnd = EOL+ space* '|' >next_keyword_start;
+ 
+  FeatureHeading = space* I18N_Feature %begin_content ^FeatureHeadingEnd* :>> FeatureHeadingEnd @store_feature_content;
+  BackgroundHeading = space* I18N_Background %begin_content ^BackgroundHeadingEnd* :>> BackgroundHeadingEnd @store_background_content;
+  ScenarioHeading = space* I18N_Scenario %begin_content ^ScenarioHeadingEnd* :>> ScenarioHeadingEnd @store_scenario_content;
+  ScenarioOutlineHeading = space* I18N_ScenarioOutline %begin_content ^ScenarioOutlineHeadingEnd* :>> ScenarioOutlineHeadingEnd @store_scenario_outline_content;
+  ExamplesHeading = space* I18N_Examples %begin_content ^ExamplesHeadingEnd* :>> ExamplesHeadingEnd @store_examples_content;
+
+  Step = space* I18N_Step %begin_content ^EOL+ %store_step_content :> EOL+;
+  Comment = space* '#' >begin_content ^EOL* %store_comment_content :> EOL+;
+  
+  Tag = ( ('@' [^@\r\n\t ]+) >begin_content ) %store_tag_content;
+  Tags = space* (Tag space*)+ EOL+;
+  
+  StartRow = space* PIPE >start_row;
+  EndRow = EOL space* ^PIPE >next_keyword_start;    
+  Cell = PIPE (ESCAPED_PIPE | (any - PIPE)+ )* >begin_cell_content %store_cell_content;
+  RowBody = space* Cell** PIPE :>> (space* EOL+ space*) %store_row;
+  Row = StartRow :>> RowBody <: EndRow?;
+  
+  StartPyString = '"""' >start_pystring space* :>> EOL; 
+  EndPyString = (space* '"""') >next_keyword_start; 
+  PyString = space* StartPyString %begin_pystring_content (^EOL | EOL)* :>> EndPyString %store_pystring_content space* EOL+;
+  
+  Tokens = BOM? (space | EOL)* (Tags | Comment | FeatureHeading | BackgroundHeading | ScenarioHeading | ScenarioOutlineHeading | ExamplesHeading | Step | Row | PyString)* (space | EOL)* EOF;
+
+  main := Tokens %end_feature @!end_feature;
+}%%

data/spec/gherkin/c_lexer_spec.rb

@@ -0,0 +1,21 @@
+#encoding: utf-8
+unless defined?(JRUBY_VERSION)
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'gherkin_lexer_en'
+
+module Gherkin
+  module Lexer
+    describe "C Lexer" do
+      before do
+        @listener = Gherkin::SexpRecorder.new
+        @lexer = Gherkin::CLexer::En.new(@listener)
+      end
+
+      it_should_behave_like "a Gherkin lexer"
+      it_should_behave_like "a Gherkin lexer lexing tags"
+      it_should_behave_like "a Gherkin lexer lexing py_strings"
+      it_should_behave_like "a Gherkin lexer lexing rows"
+    end
+  end
+end
+end