gherkin 2.12.2 → 3.2.0

data/Rakefile

@@ -1,26 +1,24 @@
 # encoding: utf-8
-GHERKIN_VERSION = Gem::Specification.load(File.dirname(__FILE__) + '/gherkin.gemspec').version.version
 require 'rubygems'
-unless ENV['RUBY_CC_VERSION']
-  require 'bundler'
-  Bundler.setup
-  Bundler::GemHelper.install_tasks
-end
-ENV['RUBYOPT'] = nil # Necessary to prevent Bundler from *&^%$#ing up rake-compiler.
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
-require 'rake/clean'
+$:.unshift File.expand_path("../lib", __FILE__)
 
-begin
-  # Support Rake >= 0.9.0
-  require 'rake/dsl_definition'
-  include Rake::DSL
-rescue LoadError
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.ruby_opts  = %w[-r./spec/coverage -w]
+  t.rspec_opts = %w[--color]
 end
 
-$:.unshift(File.dirname(__FILE__) + '/lib')
-
-Dir['tasks/**/*.rake'].each { |rake| load File.expand_path(rake) }
+require_relative 'spec/capture_warnings'
+include CaptureWarnings
+namespace :spec do
+  task :warnings do
+    report_warnings do
+      Rake::Task['spec'].invoke
+    end
+  end
+end
 
-task :default  => [:spec, :cucumber]
-task :spec     => defined?(JRUBY_VERSION) ? :jar : :compile
-task :cucumber => defined?(JRUBY_VERSION) ? :jar : :compile
+task default: ['spec:warnings']

data/bin/gherkin-generate-ast

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__),"../lib"))
+require 'gherkin/parser'
+require 'json'
+
+parser = Gherkin::Parser.new
+parser.stop_at_first_error = false
+files = ARGV.any? ? ARGV : (STDIN.tty? ? [] : [STDIN])
+start_time = Time.now
+files.each do |file|
+  begin
+    File.open(file) do |io|
+      puts JSON.generate(parser.parse(io))
+    end
+  rescue Gherkin::ParserError => e
+    STDERR.puts e.message
+    exit 1
+  end
+end
+end_time = Time.now
+STDERR.puts (end_time - start_time)*1000 if ENV['GHERKIN_PERF']

data/bin/gherkin-generate-pickles

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__),"../lib"))
+require 'gherkin/pickles/compiler'
+require 'gherkin/parser'
+require 'json'
+
+compiler = Gherkin::Pickles::Compiler.new
+parser = Gherkin::Parser.new
+parser.stop_at_first_error = false
+files = ARGV.any? ? ARGV : (STDIN.tty? ? [] : [STDIN])
+start_time = Time.now
+files.each do |file|
+  begin
+    File.open(file) do |io|
+      feature = parser.parse(io)
+      pickles = compiler.compile(feature, file);
+      puts JSON.generate(pickles)
+    end
+  rescue Gherkin::ParserError => e
+    STDERR.puts e.message
+    exit 1
+  end
+end
+end_time = Time.now
+STDERR.puts (end_time - start_time)*1000 if ENV['GHERKIN_PERF']

data/bin/gherkin-generate-tokens

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__),"../lib"))
+require 'gherkin/parser'
+require 'gherkin/token_formatter_builder'
+
+parser = Gherkin::Parser.new(Gherkin::TokenFormatterBuilder.new)
+files = ARGV + (STDIN.tty? ? [] : [STDIN])
+files.each do |file|
+  File.open(file) do |io|
+    print parser.parse(io)
+  end
+end

data/gherkin-ruby.razor

@@ -0,0 +1,218 @@
+@using Berp;
+@helper CallProduction(ProductionRule production)
+{
+  switch(production.Type)
+  {
+    case ProductionRuleType.Start:
+      @:  start_rule(context, :@production.RuleName);
+      break;
+    case ProductionRuleType.End:
+      @:  end_rule(context, :@production.RuleName);
+      break;
+    case ProductionRuleType.Process:
+      @:  build(context, token);
+      break;
+  }
+}
+@helper HandleParserError(IEnumerable<string> expectedTokens, State state)
+{<text>
+      state_comment = "State: @state.Id - @Raw(state.Comment)"
+      token.detach
+      expected_tokens = ["@Raw(string.Join("\", \"", expectedTokens))"]
+      error = token.eof? ? UnexpectedEOFException.new(token, expected_tokens, state_comment) : UnexpectedTokenException.new(token, expected_tokens, state_comment)
+      raise error if (stop_at_first_error)
+      add_error(context, error)
+      return @state.Id</text>}
+@helper MatchToken(TokenType tokenType)
+{<text>match_@(tokenType)(context, token)</text>}
+# This file is generated. Do not edit! Edit gherkin-ruby.razor instead.
+require_relative 'ast_builder'
+require_relative 'token_matcher'
+require_relative 'token_scanner'
+require_relative 'errors'
+
+module Gherkin
+
+  RULE_TYPE = [
+    :None,
+    @foreach(var rule in Model.RuleSet.Where(r => !r.TempRule))
+    {<text>    :@rule.Name.Replace("#", "_"), # @rule.ToString(true)
+</text>}
+  ]
+
+  class ParserContext
+    attr_reader :token_scanner, :token_matcher, :token_queue, :errors
+
+    def initialize(token_scanner, token_matcher, token_queue, errors)
+      @@token_scanner = token_scanner
+      @@token_matcher = token_matcher
+      @@token_queue = token_queue
+      @@errors = errors
+    end
+  end
+
+  class @Model.ParserClassName
+    attr_accessor :stop_at_first_error
+
+    def initialize(ast_builder=AstBuilder.new)
+      @@ast_builder = ast_builder
+    end
+
+    def parse(token_scanner, token_matcher=TokenMatcher.new)
+      token_scanner = token_scanner.is_a?(TokenScanner) ? token_scanner : TokenScanner.new(token_scanner)
+
+      @@ast_builder.reset
+      token_matcher.reset
+      context = ParserContext.new(
+        token_scanner,
+        token_matcher,
+        [],
+        []
+      )
+
+      start_rule(context, :@Model.RuleSet.StartRule.Name);
+      state = 0
+      token = nil
+      begin
+        token = read_token(context)
+        state = match_token(state, token, context)
+      end until(token.eof?)
+
+      end_rule(context, :@Model.RuleSet.StartRule.Name)
+
+      raise CompositeParserException.new(context.errors) if context.errors.any?
+
+      get_result()
+    end
+
+    def build(context, token)
+      handle_ast_error(context) do
+        @@ast_builder.build(token)
+      end
+    end
+
+    def add_error(context, error)
+      context.errors.push(error)
+      raise CompositeParserException, context.errors if context.errors.length > 10
+    end
+
+    def start_rule(context, rule_type)
+      handle_ast_error(context) do
+        @@ast_builder.start_rule(rule_type)
+      end
+    end
+
+    def end_rule(context, rule_type)
+      handle_ast_error(context) do
+        @@ast_builder.end_rule(rule_type)
+      end
+    end
+
+    def get_result()
+      @@ast_builder.get_result
+    end
+
+    def read_token(context)
+      context.token_queue.any? ? context.token_queue.shift : context.token_scanner.read
+    end
+
+@foreach(var rule in Model.RuleSet.TokenRules)
+{<text>
+    def match_@(rule.Name.Replace("#", ""))( context, token)
+      @if (rule.Name != "#EOF")
+      {
+      @:return false if token.eof?
+      }
+      return handle_external_error(context, false) do
+        context.token_matcher.match_@(rule.Name.Replace("#", ""))(token)
+      end
+    end</text>
+}
+
+    def match_token(state, token, context)
+      case state
+      @foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
+      {
+      @:when @state.Id
+        @:match_token_at_@(state.Id)(token, context)
+      }
+      else
+        raise InvalidOperationException, "Unknown state: #{state}"
+      end
+    end
+
+@foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
+{<text>
+    # @Raw(state.Comment)
+    def match_token_at_@(state.Id)(token, context)
+      @foreach(var transition in state.Transitions)
+      {
+      @:if @MatchToken(transition.TokenType)
+        if (transition.LookAheadHint != null)
+        {
+        @:if lookahead_@(transition.LookAheadHint.Id)(context, token)
+        }
+        foreach(var production in transition.Productions)
+        {
+          @CallProduction(production)
+        }
+        @:return @transition.TargetState
+        if (transition.LookAheadHint != null)
+        {
+        @:end
+        }
+      @:end
+      }
+      @HandleParserError(state.Transitions.Select(t => "#" + t.TokenType.ToString()).Distinct(), state)
+    end</text>
+}
+
+@foreach(var lookAheadHint in Model.RuleSet.LookAheadHints)
+{
+    <text>
+    def lookahead_@(lookAheadHint.Id)(context, currentToken)
+      currentToken.detach
+      token = nil
+      queue = []
+      match = false
+      loop do
+        token = read_token(context)
+        token.detach
+        queue.push(token)
+
+        if (false @foreach(var tokenType in lookAheadHint.ExpectedTokens) {<text>|| @MatchToken(tokenType)</text>})
+          match = true
+          break
+        end
+
+        break unless (false @foreach(var tokenType in lookAheadHint.Skip) {<text>|| @MatchToken(tokenType)</text>})
+      end
+
+      context.token_queue.concat(queue)
+
+      return match
+    end
+    </text>
+}
+
+    private
+
+    def handle_ast_error(context, &action)
+      handle_external_error(context, true, &action)
+    end
+
+    def handle_external_error(context, default_value, &action)
+      return action.call if stop_at_first_error
+
+      begin
+        return action.call
+      rescue CompositeParserException => e
+        e.errors.each { |error| add_error(context, error) }
+      rescue ParserException => e
+        add_error(context, e)
+      end
+      default_value
+    end
+
+  end
+end

data/gherkin.gemspec

@@ -1,80 +1,26 @@
-# -*- encoding: utf-8 -*-
-$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
-require 'gherkin/platform'
-
+# encoding: utf-8
 Gem::Specification.new do |s|
-  s.name        = "gherkin"
-  # If the MINOR version is bumped (MAJOR.MINOR.PATCH), verify that the change is ok:
-  #
-  # * Comment out the cucumber dependency below
-  # * Uncomment the cucumber location in Gemfile
-  # * Bump cucumber's gherkin dependency to the new version
-  # * bundle update
-  # * bundle exec rake
-  #
-  # Repeat these step for cucumber (swap `cucumber` with `gherkin`).
-  #
-  # When both are building OK, do a `bundle exec rake install` in both cucumber and gherkin projects, revert the changes in the first 2 steps
-  # and release both projects. Do this for both ruby 1.9.3, ruby 2.0.0 and jruby.
-  #
-  s.version     = Gherkin::VERSION
-  s.authors     = ["Mike Sassak", "Gregory Hnatiuk", "Aslak Hellesøy"]
-  s.description = "A fast Gherkin lexer/parser based on the Ragel State Machine Compiler."
-  s.summary     = "#{s.name}-#{s.version}"
-  s.email       = "cukes@googlegroups.com"
-  s.homepage    = "http://github.com/cucumber/gherkin"
-  s.license     = 'MIT'
-
-  s.files            = `git ls-files`.split("\n")
-  s.test_files       = `git ls-files -- {spec,features}/*`.split("\n")
-  s.executables      = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.name        = 'gherkin'
+  s.version     = '3.2.0'
+  s.authors     = ["Gáspár Nagy", "Aslak Hellesøy", "Steve Tooke"]
+  s.description = 'Gherkin parser'
+  s.summary     = "gherkin-#{s.version}"
+  s.email       = 'cukes@googlegroups.com'
+  s.homepage    = "https://github.com/cucumber/gherkin"
+  s.platform    = Gem::Platform::RUBY
+  s.license     = "MIT"
+  s.required_ruby_version = ">= 1.9.3"
+
+  s.add_development_dependency 'bundler',   '~> 1.7'
+  s.add_development_dependency 'rake',      '~> 10.4'
+  s.add_development_dependency 'rspec',     '~> 3.3'
+
+  # For coverage reports
+  s.add_development_dependency 'coveralls', '~> 0.8', '< 0.8.8'
+
+  s.rubygems_version = ">= 1.6.1"
+  s.files            = `git ls-files`.split("\n").reject {|path| path =~ /\.gitignore$/ }
+  s.test_files       = `git ls-files -- spec/*`.split("\n")
   s.rdoc_options     = ["--charset=UTF-8"]
   s.require_path     = "lib"
-
-  s.files -= Dir['ikvm/**/*']
-  s.files -= Dir['java/**/*']
-  s.files -= Dir['js/**/*']
-  s.files -= Dir['ext/**/*']
-  s.files -= Dir['lib/gherkin.jar']
-  s.files -= Dir['lib/**/*.dll']
-  s.files -= Dir['lib/**/*.bundle']
-  s.files -= Dir['lib/**/*.so']
-
-  if ENV['GEM_PLATFORM']
-    puts "GEM_PLATFORM:#{ENV['GEM_PLATFORM']}"
-  end
-  s.platform = ENV['GEM_PLATFORM'] if ENV['GEM_PLATFORM']
-  case s.platform.to_s
-  when /java/
-    s.files += Dir['lib/*.jar']
-  when /mswin|mingw32/
-    s.files += Dir['lib/*/*.so']
-  when /dotnet/
-    s.files += Dir['lib/*.dll']
-  else # MRI or Rubinius
-    s.files += Dir['lib/gherkin/lexer/*.rb']
-    s.files += Dir['ext/**/*.c']
-    s.extensions = Dir['ext/**/extconf.rb']
-    s.add_development_dependency('rake-compiler', '= 0.8.2')
-  end
-  s.files -= Dir['**/.gitignore']
-
-  s.add_runtime_dependency('multi_json', '~> 1.3')
-
-  s.add_development_dependency('cucumber', '>= 1.3.8')
-  s.add_development_dependency('rake', '>= 10.1.0')
-  s.add_development_dependency('bundler', '>= 1.3.5') # Make sure it's in sync with /.travis.yml
-  s.add_development_dependency('rspec', '~> 2.14.1')
-  s.add_development_dependency('rubyzip', '>= 1.0.0')
-  s.add_development_dependency('ruby-beautify', '= 0.92.2')
-
-  unless ENV['RUBY_CC_VERSION'] || Gherkin::JRUBY
-    s.add_development_dependency('therubyracer', '>= 0.12.0') if ENV['GHERKIN_JS']
-    # For Documentation:
-    s.add_development_dependency('yard', '>= 0.8.7.2')
-    s.add_development_dependency('rdiscount', '>= 2.1.6')
-  end
-
-  # Only needed by Cucumber. Remove when Cucumber no longer needs those.
-  s.add_development_dependency('builder', '>= 3.2.2')
 end

data/lib/gherkin/ast_builder.rb

@@ -0,0 +1,243 @@
+require_relative 'ast_node'
+
+module Gherkin
+  class AstBuilder
+    def initialize
+      reset
+    end
+
+    def reset
+      @stack = [AstNode.new(:None)]
+      @comments = []
+    end
+
+    def start_rule(rule_type)
+      @stack.push AstNode.new(rule_type)
+    end
+
+    def end_rule(rule_type)
+      node = @stack.pop
+      current_node.add(node.rule_type, transform_node(node))
+    end
+
+    def build(token)
+      if token.matched_type == :Comment
+        @comments.push({
+          type: 'Comment',
+          location: get_location(token),
+          text: token.matched_text
+        })
+      else
+        current_node.add(token.matched_type, token)
+      end
+    end
+
+    def get_result
+      current_node.get_single(:Feature)
+    end
+
+    def current_node
+      @stack.last
+    end
+
+    def get_location(token, column=nil)
+      # TODO: translated from JS... is it right?
+      (column.nil? || column.zero?) ? token.location : {line: token.location[:line], column: column}
+    end
+
+    def get_tags(node)
+      tags = []
+      tags_node = node.get_single(:Tags)
+      return tags unless tags_node
+
+      tags_node.get_tokens(:TagLine).each do |token|
+        token.matched_items.each do |tag_item|
+          tags.push({
+            type: 'Tag',
+            location: get_location(token, tag_item.column),
+            name: tag_item.text
+          })
+        end
+      end
+
+      tags
+    end
+
+    def get_table_rows(node)
+      rows = node.get_tokens(:TableRow).map do |token|
+        {
+          type: 'TableRow',
+          location: get_location(token),
+          cells: get_cells(token)
+        }
+      end
+      ensure_cell_count(rows);
+      rows
+    end
+
+    def ensure_cell_count(rows)
+      return if rows.empty?
+      cell_count = rows[0][:cells].length
+      rows.each do |row|
+          if (row[:cells].length != cell_count)
+            raise AstBuilderException.new("inconsistent cell count within the table", row[:location]);
+          end
+      end
+    end
+
+    def get_cells(table_row_token)
+      table_row_token.matched_items.map do |cell_item|
+        {
+          type: 'TableCell',
+          location: get_location(table_row_token, cell_item.column),
+          value: cell_item.text
+        }
+      end
+    end
+
+    def get_description(node)
+      node.get_single(:Description)
+    end
+
+    def get_steps(node)
+      node.get_items(:Step)
+    end
+
+    def transform_node(node)
+      case node.rule_type
+      when :Step
+        step_line = node.get_token(:StepLine)
+        step_argument = node.get_single(:DataTable) || node.get_single(:DocString) || nil
+
+        reject_nils(
+          type: node.rule_type,
+          location: get_location(step_line),
+          keyword: step_line.matched_keyword,
+          text: step_line.matched_text,
+          argument: step_argument
+        )
+      when :DocString
+        separator_token = node.get_tokens(:DocStringSeparator)[0]
+        content_type = separator_token.matched_text == '' ? nil : separator_token.matched_text
+        line_tokens = node.get_tokens(:Other)
+        content = line_tokens.map { |t| t.matched_text }.join("\n")
+
+        reject_nils(
+          type: node.rule_type,
+          location: get_location(separator_token),
+          contentType: content_type,
+          content: content
+        )
+      when :DataTable
+        rows = get_table_rows(node)
+        reject_nils(
+          type: node.rule_type,
+          location: rows[0][:location],
+          rows: rows,
+        )
+      when :Background
+        background_line = node.get_token(:BackgroundLine)
+        description = get_description(node)
+        steps = get_steps(node)
+
+        reject_nils(
+          type: node.rule_type,
+          location: get_location(background_line),
+          keyword: background_line.matched_keyword,
+          name: background_line.matched_text,
+          description: description,
+          steps: steps
+        )
+      when :Scenario_Definition
+        tags = get_tags(node)
+        scenario_node = node.get_single(:Scenario)
+        if(scenario_node)
+          scenario_line = scenario_node.get_token(:ScenarioLine)
+          description = get_description(scenario_node)
+          steps = get_steps(scenario_node)
+
+          reject_nils(
+            type: scenario_node.rule_type,
+            tags: tags,
+            location: get_location(scenario_line),
+            keyword: scenario_line.matched_keyword,
+            name: scenario_line.matched_text,
+            description: description,
+            steps: steps
+          )
+        else
+          scenario_outline_node = node.get_single(:ScenarioOutline)
+          raise 'Internal grammar error' unless scenario_outline_node
+
+          scenario_outline_line = scenario_outline_node.get_token(:ScenarioOutlineLine)
+          description = get_description(scenario_outline_node)
+          steps = get_steps(scenario_outline_node)
+          examples = scenario_outline_node.get_items(:Examples_Definition)
+
+          reject_nils(
+            type: scenario_outline_node.rule_type,
+            tags: tags,
+            location: get_location(scenario_outline_line),
+            keyword: scenario_outline_line.matched_keyword,
+            name: scenario_outline_line.matched_text,
+            description: description,
+            steps: steps,
+            examples: examples
+          )
+        end
+      when :Examples_Definition
+        tags = get_tags(node)
+        examples_node = node.get_single(:Examples)
+        examples_line = examples_node.get_token(:ExamplesLine)
+        description = get_description(examples_node)
+        rows = get_table_rows(examples_node)
+
+        reject_nils(
+          type: examples_node.rule_type,
+          tags: tags,
+          location: get_location(examples_line),
+          keyword: examples_line.matched_keyword,
+          name: examples_line.matched_text,
+          description: description,
+          tableHeader: rows.first,
+          tableBody: rows[1..-1]
+        )
+      when :Description
+        line_tokens = node.get_tokens(:Other)
+        # Trim trailing empty lines
+        last_non_empty = line_tokens.rindex { |token| !token.line.trimmed_line_text.empty? }
+        description = line_tokens[0..last_non_empty].map { |token| token.matched_text }.join("\n")
+        return description
+      when :Feature
+        header = node.get_single(:Feature_Header)
+        return unless header
+        tags = get_tags(header)
+        feature_line = header.get_token(:FeatureLine)
+        return unless feature_line
+        background = node.get_single(:Background)
+        scenario_definitions = node.get_items(:Scenario_Definition)
+        description = get_description(header)
+        language = feature_line.matched_gherkin_dialect
+
+        reject_nils(
+          type: node.rule_type,
+          tags: tags,
+          location: get_location(feature_line),
+          language: language,
+          keyword: feature_line.matched_keyword,
+          name: feature_line.matched_text,
+          description: description,
+          background: background,
+          scenarioDefinitions: scenario_definitions,
+          comments: @comments
+        )
+      else
+        return node
+      end
+    end
+
+    def reject_nils(values)
+      values.reject { |k,v| v.nil? }
+    end
+  end
+end