gherkin 7.0.4 → 8.0.0

data/lib/gherkin/pickles/compiler.rb

@@ -0,0 +1,233 @@
+require 'cucumber/messages'
+require 'digest'
+
+module Gherkin
+  module Pickles
+    class Compiler
+      def compile(gherkin_document, source)
+        pickles = []
+
+        return pickles unless gherkin_document.feature
+        feature = gherkin_document.feature
+        language = feature.language
+        tags = feature.tags
+
+        compile_feature(pickles, language, tags, feature, source)
+        pickles
+      end
+
+      private
+
+      def compile_feature(pickles, language, tags, feature, source)
+        background_steps = []
+        feature.children.each do |child|
+          if child.background
+            background_steps.concat(pickle_steps(child.background.steps))
+          elsif child.rule
+            compile_rule(pickles, language, tags, background_steps, child.rule, source)
+          else
+            scenario = child.scenario
+            if scenario.examples.empty?
+              compile_scenario(tags, background_steps, scenario, language, pickles, source)
+            else
+              compile_scenario_outline(tags, background_steps, scenario, language, pickles, source)
+            end
+          end
+        end
+      end
+
+      def compile_rule(pickles, language, tags, feature_background_steps, rule, source)
+        background_steps = feature_background_steps.dup
+        rule.children.each do |child|
+          if child.background
+            background_steps.concat(pickle_steps(child.background.steps))
+          else
+            scenario = child.scenario
+            if scenario.examples.empty?
+              compile_scenario(tags, background_steps, scenario, language, pickles, source)
+            else
+              compile_scenario_outline(tags, background_steps, scenario, language, pickles, source)
+            end
+          end
+        end
+      end
+
+      def compile_scenario(feature_tags, background_steps, scenario, language, pickles, source)
+        steps = scenario.steps.empty? ? [] : [].concat(background_steps)
+
+        tags = [].concat(feature_tags).concat(scenario.tags)
+
+        scenario.steps.each do |step|
+          steps.push(pickle_step(step))
+        end
+
+        pickle = Cucumber::Messages::Pickle.new(
+          uri: source.uri,
+          id: calculate_id(source.data, [scenario.location]),
+          tags: pickle_tags(tags),
+          name: scenario.name,
+          language: language,
+          locations: [scenario.location],
+          steps: steps
+        )
+        pickles.push(pickle)
+      end
+
+      def compile_scenario_outline(feature_tags, background_steps, scenario, language, pickles, source)
+        scenario.examples.reject { |examples| examples.table_header.nil? }.each do |examples|
+          variable_cells = examples.table_header.cells
+          examples.table_body.each do |values|
+            value_cells = values.cells
+            steps = scenario.steps.empty? ? [] : [].concat(background_steps)
+            tags = [].concat(feature_tags).concat(scenario.tags).concat(examples.tags)
+
+            scenario.steps.each do |scenario_outline_step|
+              step_props = pickle_step_props(scenario_outline_step, variable_cells, value_cells)
+              step_props[:locations].push(values.location)
+              steps.push(Cucumber::Messages::Pickle::PickleStep.new(step_props))
+            end
+
+            pickle = Cucumber::Messages::Pickle.new(
+              uri: source.uri,
+              id: calculate_id(source.data, [scenario.location, values.location]),
+              name: interpolate(scenario.name, variable_cells, value_cells),
+              language: language,
+              steps: steps,
+              tags: pickle_tags(tags),
+              locations: [
+                scenario.location,
+                values.location
+              ]
+            )
+            pickles.push(pickle);
+
+          end
+        end
+      end
+
+      def create_pickle_arguments(argument, variable_cells, value_cells)
+        result = []
+        return result if argument.nil?
+        if (argument[:type] == :DataTable)
+          table = {
+            rows: argument[:rows].map do |row|
+              {
+                cells: row[:cells].map do |cell|
+                  {
+                    location: cell.location,
+                    value: interpolate(cell.value, variable_cells, value_cells)
+                  }
+                end
+              }
+            end
+          }
+          result.push(table)
+        elsif (argument[:type] == :DocString)
+          doc_string = {
+            location: pickle_location(argument.location),
+            content: interpolate(argument[:content], variable_cells, value_cells)
+          }
+          if argument.key?(:contentType)
+            doc_string[:contentType] = interpolate(argument[:contentType], variable_cells, value_cells)
+          end
+          result.push(doc_string)
+        else
+          raise 'Internal error'
+        end
+        result
+      end
+
+      def interpolate(name, variable_cells, value_cells)
+        variable_cells.each_with_index do |variable_cell, n|
+          value_cell = value_cells[n]
+          name = name.gsub('<' + variable_cell.value + '>', value_cell.value)
+        end
+        name
+      end
+
+      def calculate_id(source_data, locations)
+        sha1 = Digest::SHA1.new
+        sha1 << source_data
+        locations.each do |location|
+          sha1 << [location.line, location.column].pack('V*')
+        end
+        sha1.hexdigest
+      end
+
+      def pickle_steps(steps)
+        steps.map do |step|
+          pickle_step(step)
+        end
+      end
+
+      def pickle_step(step)
+        Cucumber::Messages::Pickle::PickleStep.new(pickle_step_props(step, [], []))
+      end
+
+      def pickle_step_props(step, variable_cells, value_cells)
+        props = {
+          text: interpolate(step.text, variable_cells, value_cells),
+          locations: [pickle_step_location(step)]
+        }
+
+        if step.data_table
+          data_table = Cucumber::Messages::PickleStepArgument.new(
+            data_table: pickle_data_table(step.data_table, variable_cells, value_cells)
+          )
+          props[:argument] = data_table
+        end
+        if step.doc_string
+          doc_string = Cucumber::Messages::PickleStepArgument.new(
+            doc_string: pickle_doc_string(step.doc_string, variable_cells, value_cells)
+          )
+          props[:argument] = doc_string
+        end
+        props
+      end
+
+      def pickle_data_table(data_table, variable_cells, value_cells)
+        Cucumber::Messages::PickleStepArgument::PickleTable.new(
+          rows: data_table.rows.map do |row|
+            Cucumber::Messages::PickleStepArgument::PickleTable::PickleTableRow.new(
+              cells: row.cells.map do |cell|
+                Cucumber::Messages::PickleStepArgument::PickleTable::PickleTableRow::PickleTableCell.new(
+                  location: cell.location,
+                  value: interpolate(cell.value, variable_cells, value_cells)
+                )
+              end
+            )
+          end
+        )
+      end
+
+      def pickle_doc_string(doc_string, variable_cells, value_cells)
+        props = {
+          location: doc_string.location,
+          content: interpolate(doc_string.content, variable_cells, value_cells)
+        }
+        if doc_string.content_type
+          props[:contentType] = interpolate(doc_string.content_type, variable_cells, value_cells)
+        end
+        Cucumber::Messages::PickleStepArgument::PickleDocString.new(props)
+      end
+
+      def pickle_step_location(step)
+        Cucumber::Messages::Location.new(
+          line: step.location.line,
+          column: step.location.column + (step.keyword ? step.keyword.length : 0)
+        )
+      end
+
+      def pickle_tags(tags)
+        tags.map {|tag| pickle_tag(tag)}
+      end
+
+      def pickle_tag(tag)
+        Cucumber::Messages::Pickle::PickleTag.new(
+          name: tag.name,
+          location: tag.location
+        )
+      end
+    end
+  end
+end

data/lib/gherkin/stream/parser_message_stream.rb

@@ -0,0 +1,93 @@
+require 'cucumber/messages'
+require 'gherkin/parser'
+require 'gherkin/token_matcher'
+require 'gherkin/pickles/compiler'
+
+module Gherkin
+  module Stream
+    class ParserMessageStream
+      def initialize(paths, sources, options)
+        @paths = paths
+        @sources = sources
+        @options = options
+        @parser = Parser.new
+        @compiler = Pickles::Compiler.new
+      end
+
+      def messages
+        Enumerator.new do |y|
+          sources.each do |source|
+            y.yield(Cucumber::Messages::Envelope.new(source: source)) if @options[:include_source]
+            begin
+              gherkin_document = nil
+
+              if @options[:include_gherkin_document]
+                gherkin_document = build_gherkin_document(source)
+                y.yield(Cucumber::Messages::Envelope.new(gherkinDocument: gherkin_document))
+              end
+              if @options[:include_pickles]
+                gherkin_document ||= build_gherkin_document(source)
+                pickles = @compiler.compile(gherkin_document, source)
+                pickles.each do |pickle|
+                  y.yield(Cucumber::Messages::Envelope.new(pickle: pickle))
+                end
+              end
+            rescue CompositeParserException => err
+              yield_error_attachments(y, err.errors, source.uri)
+            rescue ParserException => err
+              yield_error_attachments(y, [err], source.uri)
+            end
+          end
+        end
+      end
+
+      private
+
+      def yield_error_attachments(y, errors, uri)
+        errors.each do |err|
+          attachment = Cucumber::Messages::Attachment.new(
+            source: Cucumber::Messages::SourceReference.new(
+              uri: uri,
+              location: Cucumber::Messages::Location.new(
+                line: err.location[:line],
+                column: err.location[:column]
+              )
+            ),
+            data: err.message
+          )
+          y.yield(Cucumber::Messages::Envelope.new(attachment: attachment))
+        end
+      end
+
+      def sources
+        Enumerator.new do |y|
+          @paths.each do |path|
+            source = Cucumber::Messages::Source.new(
+              uri: path,
+              data: File.open(path, 'r:UTF-8', &:read),
+              media: Cucumber::Messages::Media.new(
+                encoding: :UTF8,
+                content_type: 'text/x.cucumber.gherkin+plain'
+              )
+            )
+            y.yield(source)
+          end
+          @sources.each do |source|
+            y.yield(source)
+          end
+        end
+      end
+
+      def build_gherkin_document(source)
+        if @options[:default_dialect]
+          token_matcher = TokenMatcher.new(@options[:default_dialect])
+          gd = @parser.parse(source.data, token_matcher)
+        else
+          gd = @parser.parse(source.data)
+        end
+        gd[:uri] = source.uri
+        Cucumber::Messages::GherkinDocument.new(gd)
+      end
+    end
+  end
+end

data/lib/gherkin/stream/protobuf_message_stream.rb

@@ -0,0 +1,21 @@
+require 'cucumber/messages'
+
+module Gherkin
+  module Stream
+    class ProtobufMessageStream
+      def initialize(io)
+        @io = io
+      end
+
+      def messages
+        Enumerator.new do |y|
+          until @io.eof?
+            STDERR.puts "HELLO"
+            envelope = Cucumber::Messages::Envelope.parse_delimited_from(@io)
+            y.yield envelope
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gherkin/stream/subprocess_message_stream.rb

@@ -0,0 +1,26 @@
+require 'open3'
+require 'gherkin/stream/protobuf_message_stream'
+
+module Gherkin
+  module Stream
+    class SubprocessMessageStream
+      def initialize(gherkin_executable, paths, print_source, print_ast, print_pickles)
+        @gherkin_executable, @paths, @print_source, @print_ast, @print_pickles = gherkin_executable, paths, print_source, print_ast, print_pickles
+      end
+
+      def messages
+        args = [@gherkin_executable]
+        args.push('--no-source') unless @print_source
+        args.push('--no-ast') unless @print_ast
+        args.push('--no-pickles') unless @print_pickles
+        args = args.concat(@paths)
+        stdin, stdout, stderr, wait_thr = Open3.popen3(*args)
+        if(stdout.eof?)
+          error = stderr.read
+          raise error
+        end
+        ProtobufMessageStream.new(stdout).messages
+      end
+    end
+  end
+end

data/lib/gherkin/token.rb

@@ -0,0 +1,18 @@
+module Gherkin
+  class Token < Struct.new(:line, :location)
+    attr_accessor :matched_type, :matched_text, :matched_keyword, :matched_indent,
+                  :matched_items, :matched_gherkin_dialect
+
+    def eof?
+      line.nil?
+    end
+
+    def detach
+      # TODO: detach line - is this needed?
+    end
+
+    def token_value
+      eof? ? "EOF" : line.get_line_text(-1)
+    end
+  end
+end

data/lib/gherkin/token_formatter_builder.rb

@@ -0,0 +1,39 @@
+module Gherkin
+  class TokenFormatterBuilder
+    def initialize
+      reset
+    end
+
+    def reset
+      @tokens_text = ""
+    end
+
+    def build(token)
+      @tokens_text << "#{format_token(token)}\n"
+    end
+
+    def start_rule(rule_type)
+    end
+
+    def end_rule(rule_type)
+    end
+
+    def get_result
+      @tokens_text
+    end
+
+    private
+    def format_token(token)
+      return "EOF" if token.eof?
+
+      sprintf "(%s:%s)%s:%s/%s/%s",
+        token.location[:line],
+        token.location[:column],
+        token.matched_type,
+        token.matched_keyword,
+        token.matched_text,
+        Array(token.matched_items).map { |i| "#{i.column}:#{i.text}"}.join(',')
+    end
+
+  end
+end

data/lib/gherkin/token_matcher.rb

@@ -0,0 +1,169 @@
+require 'gherkin/dialect'
+require 'gherkin/errors'
+
+module Gherkin
+  class TokenMatcher
+    LANGUAGE_PATTERN = /^\s*#\s*language\s*:\s*([a-zA-Z\-_]+)\s*$/
+
+    def initialize(dialect_name = 'en')
+      @default_dialect_name = dialect_name
+      change_dialect(dialect_name, nil)
+      reset
+    end
+
+    def reset
+      change_dialect(@default_dialect_name, nil) unless @dialect_name == @default_dialect_name
+      @active_doc_string_separator = nil
+      @indent_to_remove = 0
+    end
+
+    def match_TagLine(token)
+      return false unless token.line.start_with?('@')
+
+      set_token_matched(token, :TagLine, nil, nil, nil, token.line.tags)
+      true
+    end
+
+    def match_FeatureLine(token)
+      match_title_line(token, :FeatureLine, @dialect.feature_keywords)
+    end
+
+    def match_RuleLine(token)
+      match_title_line(token, :RuleLine, @dialect.rule_keywords)
+    end
+
+    def match_ScenarioLine(token)
+      match_title_line(token, :ScenarioLine, @dialect.scenario_keywords) ||
+          match_title_line(token, :ScenarioLine, @dialect.scenario_outline_keywords)
+    end
+
+    def match_BackgroundLine(token)
+      match_title_line(token, :BackgroundLine, @dialect.background_keywords)
+    end
+
+    def match_ExamplesLine(token)
+      match_title_line(token, :ExamplesLine, @dialect.examples_keywords)
+    end
+
+    def match_TableRow(token)
+      return false unless token.line.start_with?('|')
+      # TODO: indent
+      set_token_matched(token, :TableRow, nil, nil, nil, token.line.table_cells)
+      true
+    end
+
+    def match_Empty(token)
+      return false unless token.line.empty?
+      set_token_matched(token, :Empty, nil, nil, 0)
+      true
+    end
+
+    def match_Comment(token)
+      return false unless token.line.start_with?('#')
+      text = token.line.get_line_text(0) #take the entire line, including leading space
+      set_token_matched(token, :Comment, text, nil, 0)
+      true
+    end
+
+    def match_Language(token)
+      return false unless token.line.trimmed_line_text =~ LANGUAGE_PATTERN
+
+      dialect_name = $1
+      set_token_matched(token, :Language, dialect_name)
+
+      change_dialect(dialect_name, token.location)
+
+      true
+    end
+
+    def match_DocStringSeparator(token)
+      if @active_doc_string_separator.nil?
+        # open
+        _match_DocStringSeparator(token, '"""', true) ||
+        _match_DocStringSeparator(token, '```', true)
+      else
+        # close
+        _match_DocStringSeparator(token, @active_doc_string_separator, false)
+      end
+    end
+
+    def _match_DocStringSeparator(token, separator, is_open)
+      return false unless token.line.start_with?(separator)
+
+      content_type = nil
+      if is_open
+        content_type = token.line.get_rest_trimmed(separator.length)
+        @active_doc_string_separator = separator
+        @indent_to_remove = token.line.indent
+      else
+        @active_doc_string_separator = nil
+        @indent_to_remove = 0
+      end
+
+      set_token_matched(token, :DocStringSeparator, content_type, separator)
+      true
+    end
+
+    def match_EOF(token)
+      return false unless token.eof?
+      set_token_matched(token, :EOF)
+      true
+    end
+
+    def match_Other(token)
+      text = token.line.get_line_text(@indent_to_remove) # take the entire line, except removing DocString indents
+      set_token_matched(token, :Other, unescape_docstring(text), nil, 0)
+      true
+    end
+
+    def match_StepLine(token)
+      keywords = @dialect.given_keywords +
+                 @dialect.when_keywords +
+                 @dialect.then_keywords +
+                 @dialect.and_keywords +
+                 @dialect.but_keywords
+
+      keyword = keywords.detect { |k| token.line.start_with?(k) }
+
+      return false unless keyword
+
+      title = token.line.get_rest_trimmed(keyword.length)
+      set_token_matched(token, :StepLine, title, keyword)
+      return true
+    end
+
+    private
+
+    def change_dialect(dialect_name, location)
+      dialect = Dialect.for(dialect_name)
+      raise NoSuchLanguageException.new(dialect_name, location) if dialect.nil?
+
+      @dialect_name = dialect_name
+      @dialect = dialect
+    end
+
+    def match_title_line(token, token_type, keywords)
+      keyword = keywords.detect { |k| token.line.start_with_title_keyword?(k) }
+
+      return false unless keyword
+
+      title = token.line.get_rest_trimmed(keyword.length + ':'.length)
+      set_token_matched(token, token_type, title, keyword)
+      true
+    end
+
+    def set_token_matched(token, matched_type, text=nil, keyword=nil, indent=nil, items=[])
+      token.matched_type = matched_type
+      token.matched_text = text && text.chomp
+      token.matched_keyword = keyword
+      token.matched_indent = indent || (token.line && token.line.indent) || 0
+      token.matched_items = items
+      token.location[:column] = token.matched_indent + 1
+      token.matched_gherkin_dialect = @dialect_name
+    end
+
+    def unescape_docstring(text)
+      @active_doc_string_separator ? text.gsub("\\\"\\\"\\\"", "\"\"\"") : text
+    end
+  end
+end