gherkin 1.0.2 → 1.0.3

data/ragel/lexer.csharp.rl.erb

@@ -0,0 +1,240 @@
+namespace Gherkin.Lexer
+{
+	using System.Text;
+	using System.Text.RegularExpressions;
+	using System.IO;
+	using System.Collections.Generic;
+	using System.Linq;
+	
+	[Language("<%= @i18n.sanitized_key.downcase %>")]
+	public class <%= @i18n.sanitized_key.capitalize %> : ILexer {
+		%%{
+			machine lexer;
+			alphtype char;
+
+			action begin_content {
+				contentStart = p;
+				contentCol = p - lastNewline + 1;
+	            contentNewline = lastNewline;
+				currentLine = lineNumber;
+			}
+
+			action start_pystring {
+				currentLine = lineNumber;
+				pystringStartCol = p - lastNewline;
+				pystringStartNewline = lastNewline;
+			}
+		    
+			action begin_pystring_content {
+				contentStart = p;
+			}
+
+			action store_pystring_content {
+				string con = Unindent(pystringStartCol, new Regex("(\\r?\\n)?( )*\\Z").Replace(Substring(data, contentStart, nextKeywordStart-1), "", 1));
+				listener.PythonString(new Token(con, GetCharacterPosition(data, pystringStartNewline, currentLine, pystringStartCol+1)));
+			}
+
+			action store_feature_content {
+				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart).Trim());
+				listener.Feature(new Token(keyword, GetCharacterPosition(data, keywordNewline, currentLine, keywordCol)), new Token(con, GetCharacterPosition(data, contentNewline, currentLine, contentCol)));
+				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+				nextKeywordStart = -1;
+			}
+
+			action store_background_content {
+				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart));
+				listener.Background(new Token(keyword, GetCharacterPosition(data, keywordNewline, currentLine, keywordCol)), new Token(con, GetCharacterPosition(data, contentNewline, currentLine, contentCol)));
+				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+				nextKeywordStart = -1;
+			}
+
+			action store_scenario_content {
+				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart));
+				listener.Scenario(new Token(keyword, GetCharacterPosition(data, keywordNewline, currentLine, keywordCol)), new Token(con, GetCharacterPosition(data, contentNewline, currentLine, contentCol)));
+				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+				nextKeywordStart = -1;
+			}
+
+			action store_scenario_outline_content {
+				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart));
+				listener.ScenarioOutline(new Token(keyword, GetCharacterPosition(data, keywordNewline, currentLine, keywordCol)), new Token(con, GetCharacterPosition(data, contentNewline, currentLine, contentCol)));
+				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+				nextKeywordStart = -1;
+			}
+
+			action store_examples_content {
+				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart));
+				listener.Examples(new Token(keyword, GetCharacterPosition(data, keywordNewline, currentLine, keywordCol)), new Token(con, GetCharacterPosition(data, contentNewline, currentLine, contentCol)));
+				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+				nextKeywordStart = -1;
+			}
+
+			action store_step_content {
+				listener.Step(
+					new Token(keyword, GetCharacterPosition(data, keywordNewline, currentLine, keywordCol)), 
+					new Token(Substring(data, contentStart, p).Trim(), GetCharacterPosition(data, contentNewline, currentLine, contentCol)),
+					LookupStepKind(keyword));
+			}
+
+			action store_comment_content {
+				listener.Comment(new Token(Substring(data, contentStart, p).Trim(), GetCharacterPosition(data, contentNewline, currentLine, contentCol)));
+				keywordStart = -1;
+			}
+
+			action store_tag_content {
+				listener.Tag(new Token(Substring(data, contentStart, p).Trim(), GetCharacterPosition(data, contentNewline, currentLine, contentCol-1)));
+				keywordStart = -1;
+			}
+
+			action inc_line_number {
+				lineNumber++;
+			}
+
+			action last_newline {
+				lastNewline = p + 1;
+			}
+
+			action start_keyword {
+				if(keywordStart == -1) keywordStart = p;
+				if(nextKeywordStart == -1)
+				{
+					keywordCol = p - lastNewline + 1;
+					keywordNewline = lastNewline;
+				}
+			}
+
+			action end_keyword {
+				keyword = new Regex(":$").Replace(Substring(data, keywordStart, p), "", 1);
+				keywordStart = -1;
+			}
+
+			action next_keyword_start {
+				nextKeywordStart = p;
+			}
+
+			action start_row {
+				p = p - 1;
+				contentCol = p - lastNewline + 1;
+				currentRow = new List<Token>();
+				currentLine = lineNumber;
+			}
+
+			action begin_cell_content {
+				contentStart = p;
+			}
+
+			action store_cell_content {
+				currentRow.Add(new Token(Substring(data, contentStart, p).Trim(), GetCharacterPosition(data, lastNewline, lineNumber, contentStart - lastNewline + 1)));
+			}
+
+			action store_row {
+				listener.Row(currentRow, GetCharacterPosition(data, contentNewline, currentLine, contentCol));
+			}
+
+			action end_feature {
+				if(cs < lexer_first_final) {
+					string content = CurrentLineContent(data, lastNewline);
+					throw new LexingException("Lexing error on line " + lineNumber + ": '" + content + "'");
+				} else {
+				  listener.Eof();
+				}
+			}
+
+			include lexer_common "lexer_common.<%= @i18n.sanitized_key %>.rl"; 
+		}%%
+
+		private readonly IListener listener;
+
+		private static readonly IDictionary<string, StepKind> stepKeywords = new Dictionary<string, StepKind>();
+
+		static <%= @i18n.sanitized_key.capitalize %>()
+		{
+			<% @i18n.keywords('given', true).reject{|kw| kw == '* '}.each do |keyword| %>
+			stepKeywords["<%=keyword%>"] = StepKind.Given;
+			<% end %>
+			<% @i18n.keywords('when', true).reject{|kw| kw == '* '}.each do |keyword| %>
+			stepKeywords["<%=keyword%>"] = StepKind.When;
+			<% end %>
+			<% @i18n.keywords('then', true).reject{|kw| kw == '* '}.each do |keyword| %>
+			stepKeywords["<%=keyword%>"] = StepKind.Then;
+			<% end %>
+			<% @i18n.keywords('and', true).reject{|kw| kw == '* '}.each do |keyword| %>
+			stepKeywords["<%=keyword%>"] = StepKind.And;
+			<% end %>
+			<% @i18n.keywords('but', true).reject{|kw| kw == '* '}.each do |keyword| %>
+			stepKeywords["<%=keyword%>"] = StepKind.But;
+			<% end %>
+			stepKeywords["* "] = StepKind.Any;
+		}
+
+		private static StepKind LookupStepKind(string keyword)
+		{
+			if (!stepKeywords.Keys.Contains(keyword))
+			   return StepKind.Unknown;
+			return stepKeywords[keyword];
+		}
+
+		public <%= @i18n.sanitized_key.capitalize %>(IListener listener) {
+			this.listener = listener;
+
+		}
+
+		%% write data noerror;
+
+		public void Scan(TextReader inputSequence) {
+			string input = inputSequence.ReadToEnd() + "\n%_FEATURE_END_%";
+			char[] data = Encoding.GetEncoding("iso-8859-1").GetChars(Encoding.UTF8.GetBytes(input));
+			int cs, p = 0, pe = data.Length;
+			int eof = pe;
+
+			int lineNumber = 1;
+			int lastNewline = 0;
+
+			int keywordCol = -1;
+		    int keywordNewline = 0;
+			int contentCol = -1;
+		    int contentNewline = 0;
+			int contentStart = -1;
+			int currentLine = -1;
+			int pystringStartNewline = 0;
+			int pystringStartCol = -1;
+			int nextKeywordStart = -1;
+			int keywordStart = -1;
+			string keyword = null;
+			IList<Token> currentRow = null;
+
+			%% write init;
+			%% write exec;
+		}
+
+		private string KeywordContent(char[] data, int p, int eof, int nextKeywordStart, int contentStart) {
+			int endPoint = (nextKeywordStart == -1 || (p == eof)) ? p : nextKeywordStart;
+			return Substring(data, contentStart, endPoint);
+		}
+
+		private string MultilineStrip(string text) {
+			var result = new StringBuilder();
+			foreach (var s in text.Split(new [] {'\n'})) {
+				result.AppendLine(s.Trim());
+			}
+			return result.ToString().Trim();
+		}
+
+		private string Unindent(int startCol, string text) {
+			return new Regex("^ {0," + startCol + "}", RegexOptions.Multiline).Replace(text, "");
+		}
+
+		private string CurrentLineContent(char[] data, int lastNewline) {
+			return Substring(data, lastNewline, data.Length).Trim();
+		}
+
+		private string Substring(char[] data, int start, int end) {
+			return Encoding.UTF8.GetString(Encoding.GetEncoding("iso-8859-1").GetBytes(data, start, end-start));
+		}
+		
+        private Position GetCharacterPosition(char[] data, int lineStart, int line, int bytecol) {
+            return new Position(line,
+                Encoding.UTF8.GetCharCount(Encoding.GetEncoding("iso-8859-1").GetBytes(data, lineStart, bytecol)));
+        }
+	}
+}

data/ragel/lexer.java.rl.erb

@@ -3,6 +3,7 @@ package gherkin.lexer;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.regex.Pattern;
+import java.util.regex.Matcher;
 import gherkin.Lexer;
 import gherkin.Listener;
 import gherkin.LexingError;
@@ -93,7 +94,7 @@ public class <%= @i18n.sanitized_key.capitalize %> implements Lexer {
     }
 
     action end_keyword {
-      keyword = substring(data, keywordStart, p).replaceFirst(":$","").trim();
+      keyword = substring(data, keywordStart, p).replaceFirst(":$","");
       keywordStart = -1;
     }
 
@@ -101,14 +102,10 @@ public class <%= @i18n.sanitized_key.capitalize %> implements Lexer {
       nextKeywordStart = p;
     }
 
-    action start_table {
-      p = p - 1;
-      rows = new ArrayList<List<String>>();
-      currentLine = lineNumber;
-    }
-
     action start_row {
+      p = p - 1;
       currentRow = new ArrayList<String>();
+      currentLine = lineNumber;
     }
 
     action begin_cell_content {
@@ -120,19 +117,15 @@ public class <%= @i18n.sanitized_key.capitalize %> implements Lexer {
     }
 
     action store_row {
-      rows.add(currentRow);
-    }
-
-    action store_table {
-      if(!rows.isEmpty()) {
-        listener.table(rows, currentLine);
-      }
+      listener.row(currentRow, currentLine);
     }
 
     action end_feature {
       if(cs < lexer_first_final) {
         String content = currentLineContent(data, lastNewline);
-        throw new LexingError("Lexing error on line " + lineNumber);
+        throw new LexingError("Lexing error on line " + lineNumber + ": '" + content + "'");
+      } else {
+        listener.eof();
       }
     }
 
@@ -162,7 +155,6 @@ public class <%= @i18n.sanitized_key.capitalize %> implements Lexer {
     int nextKeywordStart = -1;
     int keywordStart = -1;
     String keyword = null;
-    List<List<String>> rows = null;
     List<String> currentRow = null;
 
     %% write init;
@@ -174,14 +166,31 @@ public class <%= @i18n.sanitized_key.capitalize %> implements Lexer {
     return substring(data, contentStart, endPoint);
   }
 
+  private static final Pattern CRLF_RE = Pattern.compile("\r\n");
+  private static final Pattern LF_RE = Pattern.compile("[^\r]\n");
+  private static final String CRLF = "\r\n";
+  private static final String LF = "\n";
+
   private String multilineStrip(String text) {
+    int crlfCount = matchCount(CRLF_RE.matcher(text));
+    int lfCount = matchCount(LF_RE.matcher(text));
+    String eol = crlfCount > lfCount ? CRLF : LF;
+
     StringBuffer result = new StringBuffer();
-    for(String s : text.split("\n")) {
-      result.append(s.trim()).append("\n");
+    for(String s : text.split("\r?\n")) {
+      result.append(s.trim()).append(eol);
     }
     return result.toString().trim();
   }
 
+  private int matchCount(Matcher m) {
+    int count = 0;
+    while(m.find()) {
+      count++;
+    }
+    return count;
+  }
+
   private String unindent(int startCol, String text) {
     return Pattern.compile("^ {0," + startCol + "}", Pattern.MULTILINE).matcher(text).replaceAll("");
   }

data/ragel/lexer.rb.rl.erb

@@ -85,7 +85,7 @@ module Gherkin
         }
  
         action end_keyword {
-          @keyword = data[@keyword_start...p].utf8_pack("c*").sub(/:$/,'').strip
+          @keyword = data[@keyword_start...p].utf8_pack("c*").sub(/:$/,'')
           @keyword_start = nil
         }
  
@@ -93,16 +93,12 @@ module Gherkin
           @next_keyword_start = p
         }
 
-        action start_table {
+        action start_row {
           p = p - 1
-          @rows = []
+          current_row = []
           @current_line = @line_number
         }
 
-        action start_row {
-          current_row = []
-        }
- 
         action begin_cell_content {
           @content_start = p
         }
@@ -113,19 +109,15 @@ module Gherkin
         }
 
         action store_row {
-          @rows << current_row
+          @listener.row(current_row, @current_line)
         }
 
-        action store_table {
-          if @rows.size != 0
-            @listener.table(@rows, @current_line)
-          end 
-        }
- 
         action end_feature {
           if cs < lexer_first_final
             content = current_line_content(data, p)
-            raise Lexer::LexingError.new("Lexing error on line %d: '%s'." % [@line_number, content])
+            raise LexingError.new("Lexing error on line %d: '%s'." % [@line_number, content])
+          else
+            @listener.eof
           end
         }
 
@@ -147,9 +139,17 @@ module Gherkin
         %% write init;
         %% write exec;
       end
- 
+
+      CRLF_RE = /\r\n/
+      LF_RE = /[^\r]\n/
+      CRLF = "\r\n"
+      LF = "\n"
+
       def multiline_strip(text)
-        text.split("\n").map{|s| s.strip}.join("\n").strip
+        crlf_count = text.scan(CRLF_RE).size
+        lf_count = text.scan(LF_RE).size
+        eol = crlf_count > lf_count ? CRLF : LF
+        text.split(/\r?\n/).map{|s| s.strip}.join(eol).strip
       end
 
       def unindent(startcol, text)

data/ragel/lexer_common.rl.erb

@@ -10,7 +10,7 @@
   I18N_Examples = (<%= ragel_list(@i18n.examples_keywords) %> ':') >start_keyword %end_keyword;
 
   EOF = '%_FEATURE_END_%'; # Explicit EOF added before scanning begins 
-  EOL = ('\r'? '\n') @inc_line_number @last_newline;
+  EOL = ('\n' | '\r\n') @inc_line_number @last_newline;
 
   FeatureHeadingEnd = EOL+ space* (I18N_Background | I18N_Scenario | I18N_ScenarioOutline | '@' | '#' | EOF) >next_keyword_start;
   ScenarioHeadingEnd = EOL+ space* ( I18N_Scenario | I18N_ScenarioOutline | I18N_Step | '@' | '#' | EOF ) >next_keyword_start;
@@ -24,23 +24,23 @@
   ScenarioOutlineHeading = space* I18N_ScenarioOutline %begin_content ^ScenarioOutlineHeadingEnd* :>> ScenarioOutlineHeadingEnd @store_scenario_outline_content;
   ExamplesHeading = space* I18N_Examples %begin_content ^ExamplesHeadingEnd* :>> ExamplesHeadingEnd @store_examples_content;
 
-  Step = space* I18N_Step %begin_content ^EOL+ %store_step_content EOL+;
-  Comment = space* '#' >begin_content ^EOL* %store_comment_content EOL+;
+  Step = space* I18N_Step %begin_content ^EOL+ %store_step_content :> EOL+;
+  Comment = space* '#' >begin_content ^EOL* %store_comment_content :> EOL+;
   
   Tag = ( '@' [^@\r\n\t ]+ >begin_content ) %store_tag_content;
   Tags = space* (Tag space*)+ EOL+;
   
-  StartTable = space* '|' >start_table;
-  EndTable = EOL space* ^('|') >next_keyword_start;    
+  StartRow = space* '|' >start_row;
+  EndRow = EOL space* ^('|') >next_keyword_start;    
   Cell = '|' (any - '|')* >begin_cell_content %store_cell_content;
-  Row = space* Cell* >start_row '|' :>> (space* EOL+ space*) %store_row;
-  Table = StartTable :>> Row+ %store_table <: EndTable?;
+  RowBody = space* Cell* '|' :>> (space* EOL+ space*) %store_row;
+  Row = StartRow :>> RowBody <: EndRow?;
   
   StartPyString = '"""' >start_pystring space* :>> EOL; 
   EndPyString = (space* '"""') >next_keyword_start; 
   PyString = space* StartPyString %begin_pystring_content (^EOL | EOL)* :>> EndPyString %store_pystring_content space* EOL+;
   
-  Tokens = (space | EOL)* (Tags | Comment | FeatureHeading | BackgroundHeading | ScenarioHeading | ScenarioOutlineHeading | ExamplesHeading | Step | Table | PyString)* (space | EOL)* EOF;
+  Tokens = (space | EOL)* (Tags | Comment | FeatureHeading | BackgroundHeading | ScenarioHeading | ScenarioOutlineHeading | ExamplesHeading | Step | Row | PyString)* (space | EOL)* EOF;
 
   main := Tokens %end_feature @!end_feature;
 }%%

data/spec/gherkin/c_lexer_spec.rb

@@ -1,21 +1,21 @@
 #encoding: utf-8
 unless defined?(JRUBY_VERSION)
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
-require 'gherkin/c_lexer'
+require 'gherkin_lexer_en'
 
 module Gherkin
   module Lexer
     describe "C Lexer" do
       before do
         @listener = Gherkin::SexpRecorder.new
-        @lexer = Gherkin::CLexer['en'].new(@listener)
+        @lexer = Gherkin::CLexer::En.new(@listener)
       end
 
       it_should_behave_like "a Gherkin lexer"
       it_should_behave_like "a Gherkin lexer lexing tags"
       it_should_behave_like "a Gherkin lexer lexing py_strings"
-      it_should_behave_like "a Gherkin lexer lexing tables"
+      it_should_behave_like "a Gherkin lexer lexing rows"
     end
   end
 end
-end
+end