gherkin 1.0.2-java

data/nativegems.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+# Builds gems for all supported platforms
+rake gemspec build PLATFORM=java
+rake cross compile gemspec build PLATFORM=i386-mswin32 RUBY_CC_VERSION=1.8.6
+rake gemspec build PLATFORM=i386-mingw32 RUBY_CC_VERSION=1.8.6

data/ragel/i18n/.gitignore

@@ -0,0 +1 @@
+*.rl

data/ragel/lexer.c.rl.erb

@@ -0,0 +1,403 @@
+#include <assert.h>
+#include <ruby.h>
+
+#if defined(_WIN32)
+#include <stddef.h>
+#endif
+
+#ifdef HAVE_RUBY_RE_H
+#include <ruby/re.h>
+#else
+#include <re.h>
+#endif
+
+#ifdef HAVE_RUBY_ENCODING_H
+#include <ruby/encoding.h>
+#define ENCODED_STR_NEW(ptr, len) \
+    rb_enc_str_new(ptr, len, rb_utf8_encoding());
+#else
+#define ENCODED_STR_NEW(ptr, len) \
+    rb_str_new(ptr, len); 
+#endif
+
+#ifndef RSTRING_PTR
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#endif
+
+#ifndef RSTRING_LEN
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+#endif
+
+#define DATA_GET(FROM, TYPE, NAME) \
+  Data_Get_Struct(FROM, TYPE, NAME); \
+  if (NAME == NULL) { \
+    rb_raise(rb_eArgError, "NULL found for " # NAME " when it shouldn't be."); \
+  }
+ 
+typedef struct lexer_state {
+  int content_len;
+  int line_number;
+  int current_line;
+  int start_col;
+  size_t mark;
+  size_t keyword_start;
+  size_t keyword_end;
+  size_t next_keyword_start;
+  size_t content_start;
+  size_t content_end;
+  size_t field_len;
+  size_t query_start;
+  size_t last_newline;
+  size_t final_newline;
+} lexer_state;
+
+static VALUE mGherkin;
+static VALUE mLexer;
+static VALUE mCLexer;
+static VALUE cI18nLexer;
+static VALUE rb_eGherkinLexerError;
+
+#define LEN(AT, P) (P - data - lexer->AT)
+#define MARK(M, P) (lexer->M = (P) - data)
+#define PTR_TO(P) (data + lexer->P)
+
+#define STORE_KW_END_CON(EVENT) \
+  store_kw_con(listener, # EVENT, \
+    PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end - 1)), \
+    PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), \
+    lexer->current_line); \
+    if (lexer->content_end != 0) { \
+      p = PTR_TO(content_end - 1); \
+    } \
+    lexer->content_end = 0;
+
+#define STORE_ATTR(ATTR) \
+    store_attr(listener, # ATTR, \
+      PTR_TO(content_start), LEN(content_start, p), \
+      lexer->line_number);
+
+%%{
+  machine lexer;
+ 
+  action begin_content {
+		MARK(content_start, p);
+    lexer->current_line = lexer->line_number;
+  }
+  
+  action begin_pystring_content {
+    MARK(content_start, p);
+  }
+
+  action start_pystring {
+    lexer->current_line = lexer->line_number;
+    lexer->start_col = p - data - lexer->last_newline;
+  }
+ 
+  action store_pystring_content {
+    int len = LEN(content_start, PTR_TO(final_newline));
+
+    if (len < 0) len = 0;
+
+    store_pystring_content(listener, lexer->start_col, PTR_TO(content_start), len, lexer->current_line);
+  } 
+ 
+  action store_feature_content {
+    STORE_KW_END_CON(feature)
+  }
+  
+  action store_background_content {
+    STORE_KW_END_CON(background)
+  }
+  
+  action store_scenario_content {
+    STORE_KW_END_CON(scenario)
+  }
+  
+  action store_scenario_outline_content {
+    STORE_KW_END_CON(scenario_outline)
+  }
+  
+  action store_examples_content {
+    STORE_KW_END_CON(examples)
+  }
+  
+  action store_step_content {
+    store_kw_con(listener, "step",
+      PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)),
+      PTR_TO(content_start), LEN(content_start, p), 
+      lexer->current_line);
+  }
+  
+  action store_comment_content {
+    STORE_ATTR(comment)
+    lexer->mark = 0;
+  }
+  
+  action store_tag_content {
+    STORE_ATTR(tag)
+    lexer->mark = 0;
+  }
+  
+  action inc_line_number {
+    lexer->line_number += 1;
+    MARK(final_newline, p);
+  }
+    
+  action last_newline {
+    MARK(last_newline, p + 1);
+  }
+ 
+  action start_keyword {
+    if (lexer->mark == 0) {
+      MARK(mark, p);
+    }
+  }
+  
+  action end_keyword {
+    MARK(keyword_end, p);
+    MARK(keyword_start, PTR_TO(mark));
+    MARK(content_start, p + 1);
+    lexer->mark = 0;
+  }
+  
+  action next_keyword_start {
+    MARK(content_end, p);
+  }
+
+  action start_table {
+    p = p - 1;
+    lexer->current_line = lexer->line_number;
+    rb_ary_clear(rows);
+    rb_ary_clear(current_row);
+  }
+  
+  action begin_cell_content {
+		MARK(content_start, p);
+  }
+
+  action store_cell_content {
+    VALUE con = Qnil;
+    con = ENCODED_STR_NEW(PTR_TO(content_start), LEN(content_start, p));
+    rb_funcall(con, rb_intern("strip!"), 0);
+
+    rb_ary_push(current_row, con);
+  }
+
+  action start_row {
+    current_row = rb_ary_new();
+  }  
+
+  action store_row {
+    rb_ary_push(rows, current_row);
+  }
+
+  action store_table {
+    rb_funcall(listener, rb_intern("table"), 2, rows, INT2FIX(lexer->current_line));
+  }
+
+  action end_feature {
+    if (cs < lexer_first_final) {
+      if (raise_lexer_error != NULL) {
+        int count = 0;
+        int newstr_count = 0;
+        size_t len;
+        const char *buff;
+        if (lexer->last_newline != 0) {
+          len = LEN(last_newline, eof);
+          buff = PTR_TO(last_newline);
+        } else {
+          len = strlen(data);
+          buff = data;
+        }
+
+        char newstr[len]; 
+
+        for (count = 0; count < len; count++) {
+          if(buff[count] == 10) {
+            newstr[newstr_count] = '\0'; // terminate new string at first newline found
+            break;
+          } else {
+            if (buff[count] == '%') {
+              newstr[newstr_count++] = buff[count];
+              newstr[newstr_count] = buff[count];
+            } else {
+              newstr[newstr_count] = buff[count];
+            }
+          }
+          newstr_count++;
+        }
+
+        int line = lexer->line_number;
+        lexer_init(lexer); // Re-initialize so we can scan again with the same lexer
+        raise_lexer_error(listener, newstr, line);
+      }
+    }
+  }
+
+  include lexer_common "lexer_common.<%= @i18n.sanitized_key %>.rl";   
+
+}%%
+
+/** Data **/
+%% write data;
+
+static VALUE 
+strip_i(VALUE str, VALUE ary)
+{
+  rb_funcall(str, rb_intern("strip!"), 0);
+  rb_ary_push(ary, str);
+  
+  return Qnil;
+}
+
+static VALUE 
+multiline_strip(VALUE text)
+{
+  VALUE map = rb_ary_new();
+  VALUE split = rb_str_split(text, "\n");
+  
+  rb_iterate(rb_each, split, strip_i, map);
+  
+  return rb_ary_join(split, rb_str_new2("\n"));
+}
+
+static void 
+store_kw_con(VALUE listener, const char * event_name, 
+             const char * keyword_at, size_t keyword_length, 
+             const char * at,         size_t length, 
+             int current_line)
+{
+  VALUE con = Qnil, kw = Qnil;
+  kw = ENCODED_STR_NEW(keyword_at, keyword_length);
+  con = ENCODED_STR_NEW(at, length);
+  con = multiline_strip(con);
+  rb_funcall(con, rb_intern("strip!"), 0);
+  rb_funcall(kw, rb_intern("strip!"), 0);
+  rb_funcall(listener, rb_intern(event_name), 3, kw, con, INT2FIX(current_line)); 
+}
+
+static void 
+store_attr(VALUE listener, const char * attr_type,
+           const char * at, size_t length, 
+           int line)
+{
+  VALUE val = ENCODED_STR_NEW(at, length);
+  rb_funcall(listener, rb_intern(attr_type), 2, val, INT2FIX(line));
+}
+
+static void 
+store_pystring_content(VALUE listener, 
+          int start_col, 
+          const char *at, size_t length, 
+          int current_line)
+{
+  VALUE con = ENCODED_STR_NEW(at, length);
+  // Gherkin will crash gracefully if the string representation of start_col pushes the pattern past 32 characters
+  char pat[32]; 
+  snprintf(pat, 32, "^ {0,%d}", start_col); 
+  VALUE re = rb_reg_regcomp(rb_str_new2(pat));
+  rb_funcall(con, rb_intern("gsub!"), 2, re, rb_str_new2(""));
+  rb_funcall(listener, rb_intern("py_string"), 2, con, INT2FIX(current_line));
+}
+
+static void 
+raise_lexer_error(VALUE listener, const char * at, int line)
+{ 
+  rb_raise(rb_eGherkinLexerError, "Lexing error on line %d: '%s'.", line, at);
+}
+
+static void lexer_init(lexer_state *lexer) {
+  lexer->content_start = 0;
+  lexer->content_end = 0;
+  lexer->content_len = 0;
+  lexer->mark = 0;
+  lexer->field_len = 0;
+  lexer->keyword_start = 0;
+  lexer->keyword_end = 0;
+  lexer->next_keyword_start = 0;
+  lexer->line_number = 1;
+  lexer->last_newline = 0;
+  lexer->final_newline = 0;
+  lexer->start_col = 0;
+}
+
+static VALUE CLexer_alloc(VALUE klass)
+{
+  VALUE obj;
+  lexer_state *lxr = ALLOC(lexer_state);
+  lexer_init(lxr);
+
+  obj = Data_Wrap_Struct(klass, NULL, -1, lxr);
+
+  return obj;
+}
+
+static VALUE CLexer_init(VALUE self, VALUE listener)
+{
+  rb_iv_set(self, "@listener", listener);
+  
+  lexer_state *lxr = NULL;
+  DATA_GET(self, lexer_state, lxr);
+  lexer_init(lxr);
+  
+  return self;
+}
+
+static VALUE CLexer_scan(VALUE self, VALUE input)
+{
+  lexer_state *lexer = NULL;
+  DATA_GET(self, lexer_state, lexer);
+
+  VALUE input_copy = rb_str_dup(input);
+  rb_str_append(input_copy, rb_str_new2("\n%_FEATURE_END_%"));
+  char *data = RSTRING_PTR(input_copy);
+  long len = RSTRING_LEN(input_copy);
+  
+  if (len == 0) { 
+    rb_raise(rb_eGherkinLexerError, "No content to lex.");
+  } else {
+    const char *p, *pe, *eof;
+    int cs = 0;
+    
+    VALUE listener = rb_iv_get(self, "@listener");
+    VALUE rows = rb_ary_new();
+    VALUE current_row = rb_ary_new();
+    
+    p = data;
+    pe = data + len;
+    eof = pe;
+    
+    assert(*pe == '\0' && "pointer does not end on NULL");
+    assert(pe - p == len && "pointers aren't same distance");
+    
+    %% write init;
+    %% write exec;
+    
+    assert(p <= pe && "data overflow after parsing execute");
+    assert(lexer->content_start <= len && "content starts after data end");
+    assert(lexer->mark < len && "mark is after data end");
+    assert(lexer->field_len <= len && "field has length longer than the whole data");
+    
+    // Reset lexer by re-initializing the whole thing
+    lexer_init(lexer);
+
+    if (cs == lexer_error) {
+      rb_raise(rb_eGherkinLexerError, "Invalid format, lexing fails.");
+    } else {
+      return Qtrue;
+    }
+  }
+}
+
+void Init_gherkin_lexer_<%= @i18n.sanitized_key %>()
+{
+  mGherkin = rb_define_module("Gherkin");
+  mLexer = rb_const_get(mGherkin, rb_intern("Lexer"));
+  rb_eGherkinLexerError = rb_const_get(mLexer, rb_intern("LexingError"));
+
+  mCLexer = rb_define_module_under(mGherkin, "CLexer");
+  cI18nLexer = rb_define_class_under(mCLexer, "<%= @i18n.sanitized_key.capitalize %>", rb_cObject);
+  rb_define_alloc_func(cI18nLexer, CLexer_alloc);
+  rb_define_method(cI18nLexer, "initialize", CLexer_init, 1);
+  rb_define_method(cI18nLexer, "scan", CLexer_scan, 1);
+}

data/ragel/lexer.java.rl.erb

@@ -0,0 +1,200 @@
+package gherkin.lexer;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.regex.Pattern;
+import gherkin.Lexer;
+import gherkin.Listener;
+import gherkin.LexingError;
+
+public class <%= @i18n.sanitized_key.capitalize %> implements Lexer {
+  %%{
+    machine lexer;
+    alphtype byte;
+
+    action begin_content {
+      contentStart = p;
+      currentLine = lineNumber;
+    }
+
+    action start_pystring {
+      currentLine = lineNumber;
+      startCol = p - lastNewline;
+    }
+    
+    action begin_pystring_content {
+      contentStart = p;
+    }
+
+    action store_pystring_content {
+      String con = unindent(startCol, substring(data, contentStart, nextKeywordStart-1).replaceFirst("(\\r?\\n)?( )*\\Z", ""));
+      listener.py_string(con, currentLine);
+    }
+
+    action store_feature_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart).trim());
+      listener.feature(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_background_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.background(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_scenario_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.scenario(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_scenario_outline_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.scenario_outline(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_examples_content {
+      String con = multilineStrip(keywordContent(data, p, eof, nextKeywordStart, contentStart));
+      listener.examples(keyword, con, currentLine);
+      if(nextKeywordStart != -1) p = nextKeywordStart - 1;
+      nextKeywordStart = -1;
+    }
+
+    action store_step_content {
+      listener.step(keyword, substring(data, contentStart, p).trim(), currentLine);
+    }
+
+    action store_comment_content {
+      listener.comment(substring(data, contentStart, p).trim(), lineNumber);
+      keywordStart = -1;
+    }
+
+    action store_tag_content {
+      listener.tag(substring(data, contentStart, p).trim(), currentLine);
+      keywordStart = -1;
+    }
+
+    action inc_line_number {
+      lineNumber++;
+    }
+
+    action last_newline {
+      lastNewline = p + 1;
+    }
+
+    action start_keyword {
+      if(keywordStart == -1) keywordStart = p;
+    }
+
+    action end_keyword {
+      keyword = substring(data, keywordStart, p).replaceFirst(":$","").trim();
+      keywordStart = -1;
+    }
+
+    action next_keyword_start {
+      nextKeywordStart = p;
+    }
+
+    action start_table {
+      p = p - 1;
+      rows = new ArrayList<List<String>>();
+      currentLine = lineNumber;
+    }
+
+    action start_row {
+      currentRow = new ArrayList<String>();
+    }
+
+    action begin_cell_content {
+      contentStart = p;
+    }
+
+    action store_cell_content {
+      currentRow.add(substring(data, contentStart, p).trim());
+    }
+
+    action store_row {
+      rows.add(currentRow);
+    }
+
+    action store_table {
+      if(!rows.isEmpty()) {
+        listener.table(rows, currentLine);
+      }
+    }
+
+    action end_feature {
+      if(cs < lexer_first_final) {
+        String content = currentLineContent(data, lastNewline);
+        throw new LexingError("Lexing error on line " + lineNumber);
+      }
+    }
+
+    include lexer_common "lexer_common.<%= @i18n.sanitized_key %>.rl"; 
+  }%%
+
+  private final Listener listener;
+
+  public <%= @i18n.sanitized_key.capitalize %>(Listener listener) {
+    this.listener = listener;
+  }
+
+  %% write data noerror;
+
+  public void scan(CharSequence inputSequence) {
+    String input = inputSequence.toString() + "\n%_FEATURE_END_%";
+    byte[] data = input.getBytes();
+    int cs, p = 0, pe = data.length;
+    int eof = pe;
+
+    int lineNumber = 1;
+    int lastNewline = 0;
+
+    int contentStart = -1;
+    int currentLine = -1;
+    int startCol = -1;
+    int nextKeywordStart = -1;
+    int keywordStart = -1;
+    String keyword = null;
+    List<List<String>> rows = null;
+    List<String> currentRow = null;
+
+    %% write init;
+    %% write exec;
+  }
+
+  private String keywordContent(byte[] data, int p, int eof, int nextKeywordStart, int contentStart) {
+    int endPoint = (nextKeywordStart == -1 || (p == eof)) ? p : nextKeywordStart;
+    return substring(data, contentStart, endPoint);
+  }
+
+  private String multilineStrip(String text) {
+    StringBuffer result = new StringBuffer();
+    for(String s : text.split("\n")) {
+      result.append(s.trim()).append("\n");
+    }
+    return result.toString().trim();
+  }
+
+  private String unindent(int startCol, String text) {
+    return Pattern.compile("^ {0," + startCol + "}", Pattern.MULTILINE).matcher(text).replaceAll("");
+  }
+
+  private String currentLineContent(byte[] data, int lastNewline) {
+    return substring(data, lastNewline, data.length).trim();
+  }
+
+  private String substring(byte[] data, int start, int end) {
+    try {
+      return new String(data, start, end-start, "utf-8");
+    } catch(java.io.UnsupportedEncodingException e) {
+      throw new RuntimeException("Internal error", e);
+    }
+  }
+}