gherkin3-pre-alpha 3.0.0.alpha.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4b4620019eafdd1a5b88e0cb91fbddaec8dc3a7d
+  data.tar.gz: f9d2e1a7044759658aefa6a0d02ea3691c419bda
+SHA512:
+  metadata.gz: 1e5d41da5cd9f1bf76bb829427cf823afb2c7e602471b9d63e8bf0d963e58071be0d966312b5d3e1c6ae8814e0a5d26ffbfc74cf1bf0734ff15a11d78192b3a2
+  data.tar.gz: 6dac43c4da02ce6503523cd1c5ad8b9790afd47dab83adb37112dd419940f287cf178f6ae74225e7ec465adc4a087d3ddc70b8d7a4a44af808cf5cb4fc35bb55

data/.travis.yml

@@ -0,0 +1 @@
+language: ruby

data/CONTRIBUTING.md

@@ -0,0 +1,16 @@
+Please read [CONTRIBUTING](https://github.com/cucumber/gherkin3/blob/master/CONTRIBUTING.md) first.
+You should clone the [cucumber/gherkin3](https://github.com/cucumber/gherkin3) repo if you want
+to contribute.
+
+## Using make
+
+Just run `make` from this directory.
+
+Even if you prefer `make` - run `rake` occasionally, as it reports better warnings.
+
+## Using rake
+
+Just run `rake` from this directory.
+
+Keep in mind that this will only run unit tests. The acceptance tests are only
+run when you build with `make`.

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014-2015 Cucumber Ltd, Gaspar Nagy, TechTalk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Makefile

@@ -0,0 +1,59 @@
+GOOD_FEATURE_FILES = $(shell find ../testdata/good -name "*.feature")
+BAD_FEATURE_FILES  = $(shell find ../testdata/bad -name "*.feature")
+
+TOKENS   = $(patsubst ../testdata/%.feature,acceptance/testdata/%.feature.tokens,$(GOOD_FEATURE_FILES))
+ASTS     = $(patsubst ../testdata/%.feature,acceptance/testdata/%.feature.ast.json,$(GOOD_FEATURE_FILES))
+ERRORS   = $(patsubst ../testdata/%.feature,acceptance/testdata/%.feature.errors,$(BAD_FEATURE_FILES))
+
+RUBY_FILES = $(shell find . -name "*.rb")
+
+all: .compared
+.PHONY: all
+
+.compared: .built $(TOKENS) $(ASTS) $(ERRORS)
+	touch $@
+
+.built: show-version-info lib/gherkin3/parser.rb lib/gherkin3/gherkin-languages.json $(RUBY_FILES) Gemfile.lock LICENSE
+	bundle exec rspec --color
+	touch $@
+
+show-version-info:
+	ruby --version
+.PHONY: show-version-info
+
+acceptance/testdata/%.feature.tokens: ../testdata/%.feature ../testdata/%.feature.tokens .built
+	mkdir -p `dirname $@`
+	bin/gherkin-generate-tokens $< > $@
+	diff --unified $<.tokens $@
+.DELETE_ON_ERROR: acceptance/testdata/%.feature.tokens
+
+acceptance/testdata/%.feature.ast.json: ../testdata/%.feature ../testdata/%.feature.ast.json .built
+	mkdir -p `dirname $@`
+	bin/gherkin-generate-ast $< | jq --sort-keys "." > $@
+	diff --unified $<.ast.json $@
+.DELETE_ON_ERROR: acceptance/testdata/%.feature.ast.json
+
+acceptance/testdata/%.feature.errors: ../testdata/%.feature ../testdata/%.feature.errors .built
+	mkdir -p `dirname $@`
+	! bin/gherkin-generate-ast $< 2> $@
+	diff --unified $<.errors $@
+.DELETE_ON_ERROR: acceptance/testdata/%.feature.errors
+
+lib/gherkin3/gherkin-languages.json: ../gherkin-languages.json
+	cp $^ $@
+
+clean:
+	rm -rf .compared .built acceptance lib/gherkin3/parser.rb lib/gherkin3/gherkin-languages.json coverage
+.PHONY: clean
+
+lib/gherkin3/parser.rb: ../gherkin.berp gherkin-ruby.razor ../bin/berp.exe
+	mono ../bin/berp.exe -g ../gherkin.berp -t gherkin-ruby.razor -o $@
+	# Remove BOM
+	tail -c +4 $@ > $@.nobom
+	mv $@.nobom $@
+
+Gemfile.lock: Gemfile
+	bundle install
+
+LICENSE: ../LICENSE
+	cp $< $@

data/README.md

@@ -0,0 +1,3 @@
+[![Build Status](https://secure.travis-ci.org/cucumber/gherkin-ruby.png)](http://travis-ci.org/cucumber/gherkin-ruby)
+
+Gherkin parser/compiler for Ruby. Please see [Gherkin3](https://github.com/cucumber/gherkin3) for details.

data/Rakefile

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require 'rubygems'
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+$:.unshift File.expand_path("../lib", __FILE__)
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.ruby_opts  = %w[-r./spec/coverage -w]
+  t.rspec_opts = %w[--color]
+end
+
+require_relative 'spec/capture_warnings'
+include CaptureWarnings
+namespace :spec do
+  task :warnings do
+    report_warnings do
+      Rake::Task['spec'].invoke
+    end
+  end
+end
+
+task default: ['spec:warnings']

data/bin/gherkin-generate-ast

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__),"../lib"))
+require 'gherkin3/parser'
+require 'gherkin3/token_scanner'
+require 'gherkin3/token_matcher'
+require 'gherkin3/ast_builder'
+require 'json'
+
+parser = Gherkin3::Parser.new
+parser.stop_at_first_error = false
+files = ARGV + (STDIN.tty? ? [] : [STDIN])
+files.each do |file|
+  scanner = Gherkin3::TokenScanner.new(file)
+  builder = Gherkin3::AstBuilder.new
+  begin
+    puts JSON.generate(parser.parse(scanner, builder, Gherkin3::TokenMatcher.new))
+  rescue Gherkin3::ParserError => e
+    STDERR.puts e.message
+    exit 1
+  end
+end

data/bin/gherkin-generate-tokens

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__),"../lib"))
+require 'gherkin3/parser'
+require 'gherkin3/token_scanner'
+require 'gherkin3/token_formatter_builder'
+require 'gherkin3/token_matcher'
+
+parser = Gherkin3::Parser.new
+files = ARGV + (STDIN.tty? ? [] : [STDIN])
+files.each do |file|
+  scanner = Gherkin3::TokenScanner.new(file)
+  builder = Gherkin3::TokenFormatterBuilder.new
+  print parser.parse(scanner, builder, Gherkin3::TokenMatcher.new)
+end

data/gherkin-ruby.razor

@@ -0,0 +1,210 @@
+@using Berp;
+@helper CallProduction(ProductionRule production)
+{
+  switch(production.Type)
+  {
+    case ProductionRuleType.Start:
+      @:  start_rule(context, :@production.RuleName);
+      break;
+    case ProductionRuleType.End:
+      @:  end_rule(context, :@production.RuleName);
+      break;
+    case ProductionRuleType.Process:
+      @:  build(context, token);
+      break;
+  }
+}
+@helper HandleParserError(IEnumerable<string> expectedTokens, State state)
+{<text>
+      state_comment = "State: @state.Id - @Raw(state.Comment)"
+      token.detach
+      expected_tokens = ["@Raw(string.Join("\", \"", expectedTokens))"]
+      error = token.eof? ? UnexpectedEOFException.new(token, expected_tokens, state_comment) : UnexpectedTokenException.new(token, expected_tokens, state_comment)
+      raise error if (stop_at_first_error)
+      add_error(context, error)
+      return @state.Id</text>}
+@helper MatchToken(TokenType tokenType)
+{<text>match_@(tokenType)(context, token)</text>}
+# This file is generated. Do not edit! Edit gherkin-ruby.razor instead.
+require_relative 'ast_builder'
+require_relative 'errors'
+
+module Gherkin3
+
+  RULE_TYPE = [
+    :None,
+    @foreach(var rule in Model.RuleSet.Where(r => !r.TempRule))
+    {<text>    :@rule.Name.Replace("#", "_"), # @rule.ToString(true)
+</text>}
+  ]
+
+  class ParserContext
+    attr_reader :token_scanner, :ast_builder, :token_matcher, :token_queue, :errors
+
+    def initialize(token_scanner, ast_builder, token_matcher, token_queue, errors)
+      @@token_scanner = token_scanner
+      @@ast_builder = ast_builder
+      @@token_matcher = token_matcher
+      @@token_queue = token_queue
+      @@errors = errors
+    end
+  end
+
+  class @Model.ParserClassName
+    attr_accessor :stop_at_first_error
+
+    def parse(token_scanner, ast_builder, token_matcher)
+      context = ParserContext.new(
+        token_scanner,
+        ast_builder,
+        token_matcher,
+        [],
+        []
+      )
+
+      start_rule(context, :@Model.RuleSet.StartRule.Name);
+      state = 0
+      token = nil
+      begin
+        token = read_token(context)
+        state = match_token(state, token, context)
+      end until(token.eof?)
+
+      end_rule(context, :@Model.RuleSet.StartRule.Name)
+
+      raise CompositeParserException.new(context.errors) if context.errors.any?
+
+      get_result(context)
+    end
+
+    def build(context, token)
+      handle_ast_error(context) do
+        context.ast_builder.build(token)
+      end
+    end
+
+    def add_error(context, error)
+      context.errors.push(error)
+      raise CompositeParserException, context.errors if context.errors.length > 10
+    end
+
+    def start_rule(context, rule_type)
+      handle_ast_error(context) do
+        context.ast_builder.start_rule(rule_type)
+      end
+    end
+
+    def end_rule(context, rule_type)
+      handle_ast_error(context) do
+        context.ast_builder.end_rule(rule_type)
+      end
+    end
+
+    def get_result(context)
+      context.ast_builder.get_result
+    end
+
+    def read_token(context)
+      context.token_queue.any? ? context.token_queue.shift : context.token_scanner.read
+    end
+
+@foreach(var rule in Model.RuleSet.TokenRules)
+{<text>
+    def match_@(rule.Name.Replace("#", ""))( context, token)
+      @if (rule.Name != "#EOF")
+      {
+      @:return false if token.eof?
+      }
+      return handle_external_error(context, false) do
+        context.token_matcher.match_@(rule.Name.Replace("#", ""))(token)
+      end
+    end</text>
+}
+
+    def match_token(state, token, context)
+      case state
+      @foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
+      {
+      @:when @state.Id
+        @:match_token_at_@(state.Id)(token, context)
+      }
+      else
+        raise InvalidOperationException, "Unknown state: #{state}"
+      end
+    end
+
+@foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
+{<text>
+    # @Raw(state.Comment)
+    def match_token_at_@(state.Id)(token, context)
+      @foreach(var transition in state.Transitions)
+      {
+      @:if @MatchToken(transition.TokenType)
+        if (transition.LookAheadHint != null)
+        {
+        @:if lookahead_@(transition.LookAheadHint.Id)(context, token)
+        }
+        foreach(var production in transition.Productions)
+        {
+          @CallProduction(production)
+        }
+        @:return @transition.TargetState
+        if (transition.LookAheadHint != null)
+        {
+        @:end
+        }
+      @:end
+      }
+      @HandleParserError(state.Transitions.Select(t => "#" + t.TokenType.ToString()).Distinct(), state)
+    end</text>
+}
+
+@foreach(var lookAheadHint in Model.RuleSet.LookAheadHints)
+{
+    <text>
+    def lookahead_@(lookAheadHint.Id)(context, currentToken)
+      currentToken.detach
+      token = nil
+      queue = []
+      match = false
+      loop do
+        token = read_token(context)
+        token.detach
+        queue.unshift(token)
+
+        if (false @foreach(var tokenType in lookAheadHint.ExpectedTokens) {<text>|| @MatchToken(tokenType)</text>})
+          match = true
+          break
+        end
+
+        break unless (false @foreach(var tokenType in lookAheadHint.Skip) {<text>|| @MatchToken(tokenType)</text>})
+      end
+
+      context.token_queue.concat(queue)
+
+      return match
+    end
+    </text>
+}
+
+    private
+
+    def handle_ast_error(context, &action)
+      handle_external_error(context, true, &action)
+    end
+
+    def handle_external_error(context, default_value, &action)
+      return action.call if stop_at_first_error
+
+      begin
+        return action.call
+      rescue CompositeParserException => e
+        e.errors.each { |error| add_error(context, error) }
+      rescue ParserException => e
+        add_error(context, e)
+      end
+      default_value
+    end
+
+  end
+end

data/gherkin3.gemspec

@@ -0,0 +1,26 @@
+# encoding: utf-8
+Gem::Specification.new do |s|
+  s.name        = 'gherkin3-pre-alpha'
+  s.version     = '3.0.0.alpha.1'
+  s.authors     = ["Gáspár Nagy", "Aslak Hellesøy", "Steve Tooke"]
+  s.description = 'Gherkin parser'
+  s.summary     = "gherkin-#{s.version}"
+  s.email       = 'cukes@googlegroups.com'
+  s.homepage    = "https://github.com/cucumber/gherkin3"
+  s.platform    = Gem::Platform::RUBY
+  s.license     = "MIT"
+  s.required_ruby_version = ">= 1.9.3"
+
+  s.add_development_dependency 'bundler',   '>= 1.7.6'
+  s.add_development_dependency 'rake',      '>= 10.4.2'
+  s.add_development_dependency 'rspec',     '>= 3.2.0'
+
+  # For coverage reports
+  s.add_development_dependency 'coveralls', '>= 0.8.1'
+
+  s.rubygems_version = ">= 1.6.1"
+  s.files            = `git ls-files`.split("\n").reject {|path| path =~ /\.gitignore$/ }
+  s.test_files       = `git ls-files -- spec/*`.split("\n")
+  s.rdoc_options     = ["--charset=UTF-8"]
+  s.require_path     = "lib"
+end

data/lib/gherkin3/ast_builder.rb

@@ -0,0 +1,239 @@
+require_relative 'ast_node'
+
+module Gherkin3
+  class AstBuilder
+    def initialize
+      @stack = [AstNode.new(:None)]
+      @comments = []
+    end
+
+    def start_rule(rule_type)
+      @stack.push AstNode.new(rule_type)
+    end
+
+    def end_rule(rule_type)
+      node = @stack.pop
+      current_node.add(node.rule_type, transform_node(node))
+    end
+
+    def build(token)
+      if token.matched_type == :Comment
+        @comments.push({
+          type: 'Comment',
+          location: get_location(token),
+          text: token.matched_text
+        })
+      else
+        current_node.add(token.matched_type, token)
+      end
+    end
+
+    def get_result
+      current_node.get_single(:Feature)
+    end
+
+    def current_node
+      @stack.last
+    end
+
+    def get_location(token, column=nil)
+      # TODO: translated from JS... is it right?
+      (column.nil? || column.zero?) ? token.location : {line: token.location[:line], column: column}
+    end
+
+    def get_tags(node)
+      tags = []
+      tags_node = node.get_single(:Tags)
+      return tags unless tags_node
+
+      tags_node.get_tokens(:TagLine).each do |token|
+        token.matched_items.each do |tag_item|
+          tags.push({
+            type: 'Tag',
+            location: get_location(token, tag_item.column),
+            name: tag_item.text
+          })
+        end
+      end
+
+      tags
+    end
+
+    def get_table_rows(node)
+      rows = node.get_tokens(:TableRow).map do |token|
+        {
+          type: 'TableRow',
+          location: get_location(token),
+          cells: get_cells(token)
+        }
+      end
+      ensure_cell_count(rows);
+      rows
+    end
+
+    def ensure_cell_count(rows)
+      return if rows.empty?
+      cell_count = rows[0][:cells].length
+      rows.each do |row|
+          if (row[:cells].length != cell_count)
+            raise AstBuilderException.new("inconsistent cell count within the table", row[:location]);
+          end
+      end
+    end
+
+    def get_cells(table_row_token)
+      table_row_token.matched_items.map do |cell_item|
+        {
+          type: 'TableCell',
+          location: get_location(table_row_token, cell_item.column),
+          value: cell_item.text
+        }
+      end
+    end
+
+    def get_description(node)
+      node.get_single(:Description)
+    end
+
+    def get_steps(node)
+      node.get_items(:Step)
+    end
+
+    def transform_node(node)
+      case node.rule_type
+      when :Step
+        step_line = node.get_token(:StepLine)
+        step_argument = node.get_single(:DataTable) || node.get_single(:DocString) || nil
+
+        reject_nils(
+          type: node.rule_type,
+          location: get_location(step_line),
+          keyword: step_line.matched_keyword,
+          text: step_line.matched_text,
+          argument: step_argument
+        )
+      when :DocString
+        separator_token = node.get_tokens(:DocStringSeparator)[0]
+        content_type = separator_token.matched_text
+        line_tokens = node.get_tokens(:Other)
+        content = line_tokens.map { |t| t.matched_text }.join("\n")
+
+        reject_nils(
+          type: node.rule_type,
+          location: get_location(separator_token),
+          contentType: content_type,
+          content: content
+        )
+      when :DataTable
+        rows = get_table_rows(node)
+        reject_nils(
+          type: node.rule_type,
+          location: rows[0][:location],
+          rows: rows,
+        )
+      when :Background
+        background_line = node.get_token(:BackgroundLine)
+        description = get_description(node)
+        steps = get_steps(node)
+
+        reject_nils(
+          type: node.rule_type,
+          location: get_location(background_line),
+          keyword: background_line.matched_keyword,
+          name: background_line.matched_text,
+          description: description,
+          steps: steps
+        )
+      when :Scenario_Definition
+        tags = get_tags(node)
+        scenario_node = node.get_single(:Scenario)
+        if(scenario_node)
+          scenario_line = scenario_node.get_token(:ScenarioLine)
+          description = get_description(scenario_node)
+          steps = get_steps(scenario_node)
+
+          reject_nils(
+            type: scenario_node.rule_type,
+            tags: tags,
+            location: get_location(scenario_line),
+            keyword: scenario_line.matched_keyword,
+            name: scenario_line.matched_text,
+            description: description,
+            steps: steps
+          )
+        else
+          scenario_outline_node = node.get_single(:ScenarioOutline)
+          raise 'Internal grammar error' unless scenario_outline_node
+
+          scenario_outline_line = scenario_outline_node.get_token(:ScenarioOutlineLine)
+          description = get_description(scenario_outline_node)
+          steps = get_steps(scenario_outline_node)
+          examples = scenario_outline_node.get_items(:Examples_Definition)
+
+          reject_nils(
+            type: scenario_outline_node.rule_type,
+            tags: tags,
+            location: get_location(scenario_outline_line),
+            keyword: scenario_outline_line.matched_keyword,
+            name: scenario_outline_line.matched_text,
+            description: description,
+            steps: steps,
+            examples: examples
+          )
+        end
+      when :Examples_Definition
+        tags = get_tags(node)
+        examples_node = node.get_single(:Examples)
+        examples_line = examples_node.get_token(:ExamplesLine)
+        description = get_description(examples_node)
+        rows = get_table_rows(examples_node)
+
+        reject_nils(
+          type: examples_node.rule_type,
+          tags: tags,
+          location: get_location(examples_line),
+          keyword: examples_line.matched_keyword,
+          name: examples_line.matched_text,
+          description: description,
+          tableHeader: rows.first,
+          tableBody: rows[1..-1]
+        )
+      when :Description
+        line_tokens = node.get_tokens(:Other)
+        # Trim trailing empty lines
+        last_non_empty = line_tokens.rindex { |token| !token.line.trimmed_line_text.empty? }
+        description = line_tokens[0..last_non_empty].map { |token| token.matched_text }.join("\n")
+        return description
+      when :Feature
+        header = node.get_single(:Feature_Header)
+        return unless header
+        tags = get_tags(header)
+        feature_line = header.get_token(:FeatureLine)
+        return unless feature_line
+        background = node.get_single(:Background)
+        scenario_definitions = node.get_items(:Scenario_Definition)
+        description = get_description(header)
+        language = feature_line.matched_gherkin_dialect
+
+        reject_nils(
+          type: node.rule_type,
+          tags: tags,
+          location: get_location(feature_line),
+          language: language,
+          keyword: feature_line.matched_keyword,
+          name: feature_line.matched_text,
+          description: description,
+          background: background,
+          scenarioDefinitions: scenario_definitions,
+          comments: @comments
+        )
+      else
+        return node
+      end
+    end
+
+    def reject_nils(values)
+      values.reject { |k,v| v.nil? }
+    end
+  end
+end