bryanl-gherkin 2.11.1.1

data/ragel/lexer.java.rl.erb

@@ -0,0 +1,219 @@
+package gherkin.lexer.i18n;
+
+import java.io.UnsupportedEncodingException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.regex.Pattern;
+import gherkin.lexer.Lexer;
+import gherkin.lexer.Listener;
+import gherkin.lexer.LexingError;
+
+public class <%= @i18n.underscored_iso_code.upcase %> implements Lexer {
+  %%{
+    machine lexer;
+    alphtype byte;
+
+    action begin_content {
+      contentStart = p;
+      currentLine = lineNumber;
+      if(keyword != null) {
+        startCol = p - lastNewline - (keyword.length() + 1);
+      }
+    }
+
+    action start_docstring {
+      currentLine = lineNumber;
+      startCol = p - lastNewline;
+    }
+    
+    action begin_docstring_content {
+      contentStart = p;
+    }
+
+    action start_docstring_content_type {
+      docstringContentTypeStart = p;
+    }
+
+    action end_docstring_content_type {
+      docstringContentTypeEnd = p;
+    }
+
+    action store_docstring_content {
+      String con = unindent(startCol, substring(data, contentStart, nextKeywordStart-1).replaceFirst("(\\r?\\n)?([\\t ])*\\Z", "").replaceAll("\\\\\"\\\\\"\\\\\"", "\"\"\""));
+      String conType = substring(data, docstringContentTypeStart, docstringContentTypeEnd).trim();
+      listener.docString(conType, con, currentLine);
+    }
+
+    action store_feature_content {
+      String[] nameDescription = nameAndUnindentedDescription(startCol, keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.feature(keyword, nameDescription[0], nameDescription[1], currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_background_content {
+      String[] nameDescription = nameAndUnindentedDescription(startCol, keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.background(keyword, nameDescription[0], nameDescription[1], currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_scenario_content {
+      String[] nameDescription = nameAndUnindentedDescription(startCol, keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.scenario(keyword, nameDescription[0], nameDescription[1], currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_scenario_outline_content {
+      String[] nameDescription = nameAndUnindentedDescription(startCol, keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.scenarioOutline(keyword, nameDescription[0], nameDescription[1], currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_examples_content {
+      String[] nameDescription = nameAndUnindentedDescription(startCol, keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.examples(keyword, nameDescription[0], nameDescription[1], currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_step_content {
+      listener.step(keyword, substring(data, contentStart, p).trim(), currentLine);
+    }
+
+    action store_comment_content {
+      listener.comment(substring(data, contentStart, p).trim(), lineNumber);
+      keywordStart = -1;
+    }
+
+    action store_tag_content {
+      listener.tag(substring(data, contentStart, p).trim(), currentLine);
+      keywordStart = -1;
+    }
+
+    action inc_line_number {
+      lineNumber++;
+    }
+
+    action last_newline {
+      lastNewline = p + 1;
+    }
+
+    action start_keyword {
+      if(keywordStart == -1) keywordStart = p;
+    }
+
+    action end_keyword {
+      keyword = substring(data, keywordStart, p).replaceFirst(":$","");
+      keywordStart = -1;
+    }
+
+    action next_keyword_start {
+      nextKeywordStart = p;
+    }
+
+    action start_row {
+      p = p - 1;
+      currentRow = new ArrayList<String>();
+      currentLine = lineNumber;
+    }
+
+    action begin_cell_content {
+      contentStart = p;
+    }
+
+    action store_cell_content {
+      String con = substring(data, contentStart, p).trim();
+      currentRow.add(con
+        .replaceAll("\\\\\\|", "|")
+        .replaceAll("\\\\n", "\n")
+        .replaceAll("\\\\\\\\", "\\\\")
+      );
+    }
+
+    action store_row {
+      listener.row(currentRow, currentLine);
+    }
+
+    action end_feature {
+      if(cs < lexer_first_final) {
+        String content = currentLineContent(data, lastNewline);
+        throw new LexingError("Lexing error on line " + lineNumber + ": '" + content + "'. See http://wiki.github.com/cucumber/gherkin/lexingerror for more information.");
+      } else {
+        listener.eof();
+      }
+    }
+
+    include lexer_common "lexer_common.<%= @i18n.underscored_iso_code %>.rl"; 
+  }%%
+
+  private final Listener listener;
+
+  public <%= @i18n.underscored_iso_code.upcase %>(Listener listener) {
+    this.listener = listener;
+  }
+
+  %% write data noerror;
+
+  public void scan(String source)  {
+    String input = source + "\n%_FEATURE_END_%";
+    byte[] data = null;
+    try {
+      data = input.getBytes("UTF-8");
+    } catch(UnsupportedEncodingException e) {
+      throw new RuntimeException(e);
+    }
+    int cs, p = 0, pe = data.length;
+    int eof = pe;
+
+    int lineNumber = 1;
+    int lastNewline = 0;
+
+    int contentStart = -1;
+    int currentLine = -1;
+    int docstringContentTypeStart = -1;
+    int docstringContentTypeEnd = -1;
+    int startCol = -1;
+    int nextKeywordStart = -1;
+    int keywordStart = -1;
+    String keyword = null;
+    List<String> currentRow = null;
+
+    %% write init;
+    %% write exec;
+  }
+
+  private String keywordContent(byte[] data, int p, int eof, int nextKeywordStart, int contentStart) {
+    int endPoint = (nextKeywordStart == -1 || (p == eof)) ? p : nextKeywordStart;
+    return substring(data, contentStart, endPoint);
+  }
+
+  private String[] nameAndUnindentedDescription(int startCol, String text) {
+    String[] lines = text.split("\n");
+    String name = lines.length > 0 ? lines[0].trim() : "";
+    StringBuffer description = new StringBuffer();
+    for(int i = 1; i < lines.length; i++) {
+      description.append(lines[i]);
+      description.append("\n");
+    }
+    return new String[]{name, unindent(startCol+2, description.toString()).replaceAll("\\s+$", "")};
+  }
+
+  private String unindent(int startCol, String text) {
+    return Pattern.compile("^[\t ]{0," + startCol + "}", Pattern.MULTILINE).matcher(text).replaceAll("");
+  }
+
+  private String currentLineContent(byte[] data, int lastNewline) {
+    return substring(data, lastNewline, data.length).trim();
+  }
+
+  private String substring(byte[] data, int start, int end) {
+    try {
+      return new String(data, start, end-start, "utf-8");
+    } catch(java.io.UnsupportedEncodingException e) {
+      throw new RuntimeException("Internal error", e);
+    }
+  }
+}

data/ragel/lexer.js.rl.erb

@@ -0,0 +1,227 @@
+;(function() {
+
+%%{
+  machine lexer;
+
+  action begin_content {
+    this.content_start = p;
+    this.current_line = this.line_number;
+    this.start_col = p - this.last_newline - (this.keyword+':').length;
+  }
+
+  action start_docstring {
+    this.current_line = this.line_number;
+    this.start_col = p - this.last_newline;
+  }
+  
+  action begin_docstring_content {
+    this.content_start = p;
+  }
+
+  action start_docstring_content_type {
+    this.docstring_content_type_start = p;
+  }
+
+  action end_docstring_content_type {
+    this.docstring_content_type_end = p;
+  }
+
+  action store_docstring_content {
+    var con = this.unindent(
+      this.start_col, 
+      this.bytesToString(data.slice(this.content_start, this.next_keyword_start-1)).replace(/(\r?\n)?([\t ])*$/, '').replace(/ESCAPED_TRIPLE_QUOTE/mg, '"""')
+    );
+    var con_type = this.bytesToString(data.slice(this.docstring_content_type_start, this.docstring_content_type_end)).trim();
+    this.listener.doc_string(con_type, con, this.current_line); 
+  }
+
+  action store_feature_content {
+    p = this.store_keyword_content('feature', data, p, eof);
+  }
+
+  action store_background_content {
+    p = this.store_keyword_content('background', data, p, eof);
+  }
+
+  action store_scenario_content {
+    p = this.store_keyword_content('scenario', data, p, eof);
+  }
+
+  action store_scenario_outline_content {
+    p = this.store_keyword_content('scenario_outline', data, p, eof);
+  }
+
+  action store_examples_content {
+    p = this.store_keyword_content('examples', data, p, eof);
+  }
+
+  action store_step_content {
+    var con = this.bytesToString(data.slice(this.content_start, p)).trim();
+    this.listener.step(this.keyword, con, this.current_line);
+  }
+
+  action store_comment_content {
+    var con = this.bytesToString(data.slice(this.content_start, p)).trim();
+    this.listener.comment(con, this.line_number);
+    this.keyword_start = null;
+  }
+
+  action store_tag_content {
+    var con = this.bytesToString(data.slice(this.content_start, p)).trim();
+    this.listener.tag(con, this.line_number);
+    this.keyword_start = null;
+  }
+
+  action inc_line_number {
+    this.line_number++;
+  }
+
+  action last_newline {
+    this.last_newline = p + 1;
+  }
+
+  action start_keyword {
+    this.keyword_start = this.keyword_start || p;
+  }
+
+  action end_keyword {
+    this.keyword = this.bytesToString(data.slice(this.keyword_start, p)).replace(/:$/, '');
+    this.keyword_start = null;
+  }
+
+  action next_keyword_start {
+    this.next_keyword_start = p;
+  }
+
+  action start_row {
+    p = p - 1;
+    current_row = [];
+    this.current_line = this.line_number;
+  }
+
+  action begin_cell_content {
+    this.content_start = p;
+  }
+
+  action store_cell_content {
+    var con = this.bytesToString(data.slice(this.content_start, p)).trim();
+    current_row.push(con.replace(/\\\|/, "|").replace(/\\n/, "\n").replace(/\\\\/, "\\"));
+  }
+
+  action store_row {
+    this.listener.row(current_row, this.current_line);
+  }
+
+  action end_feature {
+    if(this.cs < lexer_first_final) {
+      var content = this.current_line_content(data, p);
+      throw "Lexing error on line " + this.line_number + ": '" + content + "'. See http://wiki.github.com/cucumber/gherkin/lexingerror for more information.";
+    } else {
+      this.listener.eof();
+    }
+    
+  }
+
+  include lexer_common "lexer_common.<%= @i18n.underscored_iso_code %>.rl"; 
+}%%
+
+%% write data;
+%% access this.;
+%% variable data data;
+
+var Lexer = function(listener) {
+  // Check that listener has the required functions
+  var events = ['comment', 'tag', 'feature', 'background', 'scenario', 'scenario_outline', 'examples', 'step', 'doc_string', 'row', 'eof'];
+  for(e in events) {
+    var event = events[e];
+    if(typeof listener[event] != 'function') {
+      "Error. No " + event + " function exists on " + JSON.stringify(listener);
+    }
+  }
+  this.listener = listener;  
+};
+
+Lexer.prototype.scan = function(data) {
+  var ending = "\n%_FEATURE_END_%";
+  if(typeof data == 'string') {
+    data = this.stringToBytes(data + ending);
+  } else if(typeof Buffer != 'undefined' && Buffer.isBuffer(data)) {
+    // Node.js
+    var buf = new Buffer(data.length + ending.length);
+    data.copy(buf, 0, 0);
+    new Buffer(ending).copy(buf, data.length, 0);
+    data = buf;
+  }
+  var eof = pe = data.length;
+  var p = 0;
+
+  this.line_number = 1;
+  this.last_newline = 0;
+
+  %% write init;
+  %% write exec;
+};
+
+Lexer.prototype.bytesToString = function(bytes) {
+  if(typeof bytes.write == 'function') {
+    // Node.js
+    return bytes.toString('utf-8');
+  } else {
+    var result = "";
+    for(var b in bytes) {
+      result += String.fromCharCode(bytes[b]);
+    }
+    return result;
+  }
+};
+
+Lexer.prototype.stringToBytes = function(string) {
+  var bytes = [];
+  for(var i = 0; i < string.length; i++) {
+    bytes[i] = string.charCodeAt(i);
+  }
+  return bytes;
+};
+
+Lexer.prototype.unindent = function(startcol, text) {
+  startcol = startcol || 0;
+  return text.replace(new RegExp('^[\t ]{0,' + startcol + '}', 'gm'), ''); 
+};
+
+Lexer.prototype.store_keyword_content = function(event, data, p, eof) {
+  var end_point = (!this.next_keyword_start || (p == eof)) ? p : this.next_keyword_start;
+  var content = this.unindent(this.start_col + 2, this.bytesToString(data.slice(this.content_start, end_point))).replace(/\s+$/,"");
+  var content_lines = content.split("\n")
+  var name = content_lines.shift() || "";
+  name = name.trim();
+  var description = content_lines.join("\n");
+  this.listener[event](this.keyword, name, description, this.current_line);
+  var nks = this.next_keyword_start;
+  this.next_keyword_start = null;
+  return nks ? nks - 1 : p;
+};
+
+Lexer.prototype.current_line_content = function(data, p) {
+  var rest = data.slice(this.last_newline, -1);
+  var end = rest.indexOf(10) || -1;
+  return this.bytesToString(rest.slice(0, end)).trim();
+};
+
+// Node.js export
+if(typeof exports !== 'undefined') {
+  exports.Lexer = Lexer;
+}
+// Require.js export
+if (typeof define !== 'undefined') {
+  if(define.amd) {
+    define('gherkin/lexer/<%= @i18n.underscored_iso_code %>', [], function() {
+      return Lexer
+    });
+  } else {
+    define('gherkin/lexer/<%= @i18n.underscored_iso_code %>', function(require, exports, module) {
+      exports.Lexer = Lexer;
+    });
+  }
+}
+
+})();

data/ragel/lexer.rb.rl.erb

@@ -0,0 +1,174 @@
+require 'gherkin/lexer/i18n_lexer'
+
+module Gherkin
+  module RbLexer
+    class <%= @i18n.underscored_iso_code.capitalize %> #:nodoc:
+      %%{
+        machine lexer;
+        
+        action begin_content {
+          @content_start = p
+          @current_line = @line_number
+          @start_col = p - @last_newline - "#{@keyword}:".length
+        }
+
+        action start_docstring {
+          @current_line = @line_number
+          @start_col = p - @last_newline
+        }
+        
+        action begin_docstring_content {
+          @content_start = p
+        }
+
+        action start_docstring_content_type {
+          @docstring_content_type_start = p
+        }
+        action end_docstring_content_type {
+          @docstring_content_type_end = p
+        }
+
+        action store_docstring_content {
+          con = unindent(@start_col, utf8_pack(data[@content_start...@next_keyword_start-1]).sub(/(\r?\n)?([\t ])*\Z/, '').gsub(/\\"\\"\\"/, '"""'))
+          con_type = utf8_pack(data[@docstring_content_type_start...@docstring_content_type_end]).strip
+          @listener.doc_string(con_type, con, @current_line) 
+        }
+        action store_feature_content {
+          p = store_keyword_content(:feature, data, p, eof)
+        }
+ 
+        action store_background_content {
+          p = store_keyword_content(:background, data, p, eof)
+        }
+
+        action store_scenario_content {
+          p = store_keyword_content(:scenario, data, p, eof)
+        }
+
+        action store_scenario_outline_content {
+          p = store_keyword_content(:scenario_outline, data, p, eof)
+        }
+
+        action store_examples_content {
+          p = store_keyword_content(:examples, data, p, eof)
+        }
+
+        action store_step_content {
+          con = utf8_pack(data[@content_start...p]).strip
+          @listener.step(@keyword, con, @current_line)
+        }
+ 
+        action store_comment_content {
+          con = utf8_pack(data[@content_start...p]).strip
+          @listener.comment(con, @line_number)
+          @keyword_start = nil
+        }
+ 
+        action store_tag_content {
+          con = utf8_pack(data[@content_start...p]).strip
+          @listener.tag(con, @current_line)
+          @keyword_start = nil
+        }
+ 
+        action inc_line_number {
+          @line_number += 1
+        }
+ 
+        action last_newline {
+          @last_newline = p + 1
+        }
+
+        action start_keyword {
+          @keyword_start ||= p
+        }
+ 
+        action end_keyword {
+          @keyword = utf8_pack(data[@keyword_start...p]).sub(/:$/,'')
+          @keyword_start = nil
+        }
+ 
+        action next_keyword_start {
+          @next_keyword_start = p
+        }
+
+        action start_row {
+          p = p - 1
+          current_row = []
+          @current_line = @line_number
+        }
+
+        action begin_cell_content {
+          @content_start = p
+        }
+
+        action store_cell_content {
+          con = utf8_pack(data[@content_start...p]).strip
+          current_row << con.gsub(/\\\|/, "|").gsub(/\\n/, "\n").gsub(/\\\\/, "\\")
+        }
+
+        action store_row {
+          @listener.row(current_row, @current_line)
+        }
+
+        action end_feature {
+          if cs < lexer_first_final
+            content = current_line_content(data, p)
+            raise Gherkin::Lexer::LexingError.new("Lexing error on line %d: '%s'. See http://wiki.github.com/cucumber/gherkin/lexingerror for more information." % [@line_number, content])
+          else
+            @listener.eof
+          end
+        }
+
+        include lexer_common "lexer_common.<%= @i18n.underscored_iso_code %>.rl"; 
+      }%%
+ 
+      def initialize(listener)
+        @listener = listener
+        %% write data;
+      end
+ 
+      def scan(data)
+        data = (data + "\n%_FEATURE_END_%").unpack("c*") # Explicit EOF simplifies things considerably
+        eof = pe = data.length
+ 
+        @line_number = 1
+        @last_newline = 0
+ 
+        %% write init;
+        %% write exec;
+      end
+
+      def unindent(startcol, text)
+        text.gsub(/^[\t ]{0,#{startcol}}/, "")
+      end
+
+      def store_keyword_content(event, data, p, eof)
+        end_point = (!@next_keyword_start or (p == eof)) ? p : @next_keyword_start
+        content = unindent(@start_col + 2, utf8_pack(data[@content_start...end_point])).rstrip
+        content_lines = content.split("\n")
+        name = content_lines.shift || ""
+        name.strip!
+        description = content_lines.join("\n")
+        @listener.__send__(event, @keyword, name, description, @current_line)
+        @next_keyword_start ? @next_keyword_start - 1 : p
+      ensure
+        @next_keyword_start = nil
+      end
+      
+      def current_line_content(data, p)
+        rest = data[@last_newline..-1]
+        utf8_pack(rest[0..rest.index(10)||-1]).strip # 10 is \n
+      end
+
+      if (RUBY_VERSION =~ /^1\.9/)
+        def utf8_pack(array)
+          array.pack("c*").force_encoding("UTF-8")
+        end
+      else
+        def utf8_pack(array)
+          array.pack("c*")
+        end
+      end
+    end
+  end
+end