cucumber-cucumber-expressions 10.3.0 → 11.0.0

data/lib/cucumber/cucumber_expressions/cucumber_expression_parser.rb

@@ -0,0 +1,219 @@
+require 'cucumber/cucumber_expressions/ast'
+require 'cucumber/cucumber_expressions/errors'
+require 'cucumber/cucumber_expressions/cucumber_expression_tokenizer'
+
+module Cucumber
+  module CucumberExpressions
+    class CucumberExpressionParser
+      def parse(expression)
+        # text := whitespace | ')' | '}' | .
+        parse_text = lambda do |_, tokens, current|
+          token = tokens[current]
+          case token.type
+          when TokenType::WHITE_SPACE, TokenType::TEXT, TokenType::END_PARAMETER, TokenType::END_OPTIONAL
+            return 1, [Node.new(NodeType::TEXT, nil, token.text, token.start, token.end)]
+          when TokenType::ALTERNATION
+            raise AlternationNotAllowedInOptional.new(expression, token)
+          when TokenType::BEGIN_PARAMETER, TokenType::START_OF_LINE, TokenType::END_OF_LINE, TokenType::BEGIN_OPTIONAL
+          else
+            # If configured correctly this will never happen
+            return 0, nil
+          end
+          # If configured correctly this will never happen
+          return 0, nil
+        end
+
+        # name := whitespace | .
+        parse_name = lambda do |_, tokens, current|
+          token = tokens[current]
+          case token.type
+          when TokenType::WHITE_SPACE, TokenType::TEXT
+            return 1, [Node.new(NodeType::TEXT, nil, token.text, token.start, token.end)]
+          when TokenType::BEGIN_PARAMETER, TokenType::END_PARAMETER, TokenType::BEGIN_OPTIONAL, TokenType::END_OPTIONAL, TokenType::ALTERNATION
+            raise InvalidParameterTypeNameInNode.new(expression, token)
+          when TokenType::START_OF_LINE, TokenType::END_OF_LINE
+            # If configured correctly this will never happen
+            return 0, nil
+          else
+            # If configured correctly this will never happen
+            return 0, nil
+          end
+        end
+
+        # parameter := '{' + name* + '}'
+        parse_parameter = parse_between(
+            NodeType::PARAMETER,
+            TokenType::BEGIN_PARAMETER,
+            TokenType::END_PARAMETER,
+            [parse_name]
+        )
+
+        # optional := '(' + option* + ')'
+        # option := optional | parameter | text
+        optional_sub_parsers = []
+        parse_optional = parse_between(
+            NodeType::OPTIONAL,
+            TokenType::BEGIN_OPTIONAL,
+            TokenType::END_OPTIONAL,
+            optional_sub_parsers
+        )
+        optional_sub_parsers << parse_optional << parse_parameter << parse_text
+
+        # alternation := alternative* + ( '/' + alternative* )+
+        parse_alternative_separator = lambda do |_, tokens, current|
+          unless looking_at(tokens, current, TokenType::ALTERNATION)
+            return 0, nil
+          end
+          token = tokens[current]
+          return 1, [Node.new(NodeType::ALTERNATIVE, nil, token.text, token.start, token.end)]
+        end
+
+        alternative_parsers = [
+            parse_alternative_separator,
+            parse_optional,
+            parse_parameter,
+            parse_text,
+        ]
+
+        # alternation := (?<=left-boundary) + alternative* + ( '/' + alternative* )+ + (?=right-boundary)
+        # left-boundary := whitespace | } | ^
+        # right-boundary := whitespace | { | $
+        # alternative: = optional | parameter | text
+        parse_alternation = lambda do |expr, tokens, current|
+          previous = current - 1
+          unless looking_at_any(tokens, previous, [TokenType::START_OF_LINE, TokenType::WHITE_SPACE, TokenType::END_PARAMETER])
+            return 0, nil
+          end
+
+          consumed, ast = parse_tokens_until(expr, alternative_parsers, tokens, current, [TokenType::WHITE_SPACE, TokenType::END_OF_LINE, TokenType::BEGIN_PARAMETER])
+          sub_current = current + consumed
+          unless ast.map { |astNode| astNode.type }.include? NodeType::ALTERNATIVE
+            return 0, nil
+          end
+
+          start = tokens[current].start
+          _end = tokens[sub_current].start
+          # Does not consume right hand boundary token
+          return consumed, [Node.new(NodeType::ALTERNATION, split_alternatives(start, _end, ast), nil, start, _end)]
+        end
+
+        #
+        # cucumber-expression :=  ( alternation | optional | parameter | text )*
+        #
+        parse_cucumber_expression = parse_between(
+            NodeType::EXPRESSION,
+            TokenType::START_OF_LINE,
+            TokenType::END_OF_LINE,
+            [parse_alternation, parse_optional, parse_parameter, parse_text]
+        )
+
+        tokenizer = CucumberExpressionTokenizer.new
+        tokens = tokenizer.tokenize(expression)
+        _, ast = parse_cucumber_expression.call(expression, tokens, 0)
+        ast[0]
+      end
+
+      private
+
+      def parse_between(type, begin_token, end_token, parsers)
+        lambda do |expression, tokens, current|
+          unless looking_at(tokens, current, begin_token)
+            return 0, nil
+          end
+          sub_current = current + 1
+          consumed, ast = parse_tokens_until(expression, parsers, tokens, sub_current, [end_token, TokenType::END_OF_LINE])
+          sub_current += consumed
+
+          # endToken not found
+          unless looking_at(tokens, sub_current, end_token)
+            raise MissingEndToken.new(expression, begin_token, end_token, tokens[current])
+          end
+          # consumes endToken
+          start = tokens[current].start
+          _end = tokens[sub_current].end
+          consumed = sub_current + 1 - current
+          ast = [Node.new(type, ast, nil, start, _end)]
+          return consumed, ast
+        end
+      end
+
+      def parse_token(expression, parsers, tokens, start_at)
+        parsers.each do |parser|
+          consumed, ast = parser.call(expression, tokens, start_at)
+          return consumed, ast unless consumed == 0
+        end
+        # If configured correctly this will never happen
+        raise 'No eligible parsers for ' + tokens
+      end
+
+      def parse_tokens_until(expression, parsers, tokens, start_at, end_tokens)
+        current = start_at
+        size = tokens.length
+        ast = []
+        while current < size do
+          if looking_at_any(tokens, current, end_tokens)
+            break
+          end
+          consumed, sub_ast = parse_token(expression, parsers, tokens, current)
+          if consumed == 0
+            # If configured correctly this will never happen
+            # Keep to avoid infinite loops
+            raise 'No eligible parsers for ' + tokens
+          end
+          current += consumed
+          ast += sub_ast
+        end
+        [current - start_at, ast]
+      end
+
+      def looking_at_any(tokens, at, token_types)
+        token_types.detect { |token_type| looking_at(tokens, at, token_type) }
+      end
+
+      def looking_at(tokens, at, token)
+        if at < 0
+          # If configured correctly this will never happen
+          # Keep for completeness
+          return token == TokenType::START_OF_LINE
+        end
+        if at >= tokens.length
+          return token == TokenType::END_OF_LINE
+        end
+        tokens[at].type == token
+      end
+
+      def split_alternatives(start, _end, alternation)
+        separators = []
+        alternatives = []
+        alternative = []
+        alternation.each { |n|
+          if NodeType::ALTERNATIVE == n.type
+            separators.push(n)
+            alternatives.push(alternative)
+            alternative = []
+          else
+            alternative.push(n)
+          end
+        }
+        alternatives.push(alternative)
+        create_alternative_nodes(start, _end, separators, alternatives)
+      end
+
+      def create_alternative_nodes(start, _end, separators, alternatives)
+        alternatives.each_with_index.map do |n, i|
+          if i == 0
+            right_separator = separators[i]
+            Node.new(NodeType::ALTERNATIVE, n, nil, start, right_separator.start)
+          elsif i == alternatives.length - 1
+            left_separator = separators[i - 1]
+            Node.new(NodeType::ALTERNATIVE, n, nil, left_separator.end, _end)
+          else
+            left_separator = separators[i - 1]
+            right_separator = separators[i]
+            Node.new(NodeType::ALTERNATIVE, n, nil, left_separator.end, right_separator.start)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cucumber/cucumber_expressions/cucumber_expression_tokenizer.rb

@@ -0,0 +1,95 @@
+require 'cucumber/cucumber_expressions/ast'
+require 'cucumber/cucumber_expressions/errors'
+
+module Cucumber
+  module CucumberExpressions
+    class CucumberExpressionTokenizer
+      def tokenize(expression)
+        @expression = expression
+        tokens = []
+        @buffer = []
+        previous_token_type = TokenType::START_OF_LINE
+        treat_as_text = false
+        @escaped = 0
+        @buffer_start_index = 0
+
+        codepoints = expression.codepoints
+
+        if codepoints.empty?
+          tokens.push(Token.new(TokenType::START_OF_LINE, '', 0, 0))
+        end
+
+        codepoints.each do |codepoint|
+          if !treat_as_text && Token.is_escape_character(codepoint)
+            @escaped += 1
+            treat_as_text = true
+            next
+          end
+          current_token_type = token_type_of(codepoint, treat_as_text)
+          treat_as_text = false
+
+          if should_create_new_token?(previous_token_type, current_token_type)
+            token = convert_buffer_to_token(previous_token_type)
+            previous_token_type = current_token_type
+            @buffer.push(codepoint)
+            tokens.push(token)
+          else
+            previous_token_type = current_token_type
+            @buffer.push(codepoint)
+          end
+        end
+
+        if @buffer.length > 0
+          token = convert_buffer_to_token(previous_token_type)
+          tokens.push(token)
+        end
+
+        raise TheEndOfLineCannotBeEscaped.new(expression) if treat_as_text
+
+        tokens.push(Token.new(TokenType::END_OF_LINE, '', codepoints.length, codepoints.length))
+        tokens
+      end
+
+      private
+
+      # TODO: Make these lambdas
+
+      def convert_buffer_to_token(token_type)
+        escape_tokens = 0
+        if token_type == TokenType::TEXT
+          escape_tokens = @escaped
+          @escaped = 0
+        end
+
+        consumed_index = @buffer_start_index + @buffer.length + escape_tokens
+        t = Token.new(
+            token_type,
+            @buffer.map { |codepoint| codepoint.chr(Encoding::UTF_8) }.join(''),
+            @buffer_start_index,
+            consumed_index
+        )
+        @buffer = []
+        @buffer_start_index = consumed_index
+        t
+      end
+
+      def token_type_of(codepoint, treat_as_text)
+        unless treat_as_text
+          return Token.type_of(codepoint)
+        end
+        if Token.can_escape(codepoint)
+          return TokenType::TEXT
+        end
+        raise CantEscape.new(
+            @expression,
+            @buffer_start_index + @buffer.length + @escaped
+        )
+      end
+
+      def should_create_new_token?(previous_token_type, current_token_type)
+        current_token_type != previous_token_type ||
+            (current_token_type != TokenType::WHITE_SPACE && current_token_type != TokenType::TEXT)
+      end
+    end
+  end
+end

data/lib/cucumber/cucumber_expressions/errors.rb

@@ -1,11 +1,182 @@
+require 'cucumber/cucumber_expressions/ast'
+
 module Cucumber
   module CucumberExpressions
     class CucumberExpressionError < StandardError
+
+      def build_message(
+          index,
+          expression,
+          pointer,
+          problem,
+          solution
+      )
+        m = <<-EOF
+This Cucumber Expression has a problem at column #{index + 1}:
+
+#{expression}
+#{pointer}
+#{problem}.
+#{solution}
+        EOF
+        m.strip
+      end
+
+      def point_at(index)
+        ' ' * index + '^'
+      end
+
+      def point_at_located(node)
+        pointer = [point_at(node.start)]
+        if node.start + 1 < node.end
+          for _ in node.start + 1...node.end - 1
+            pointer.push('-')
+          end
+          pointer.push('^')
+        end
+        pointer.join('')
+      end
     end
 
-    class UndefinedParameterTypeError < CucumberExpressionError
+    class AlternativeMayNotExclusivelyContainOptionals < CucumberExpressionError
+      def initialize(node, expression)
+        super(build_message(
+                  node.start,
+                  expression,
+                  point_at_located(node),
+                  'An alternative may not exclusively contain optionals',
+                  "If you did not mean to use an optional you can use '\\(' to escape the the '('"
+              ))
+      end
+    end
+
+    class AlternativeMayNotBeEmpty < CucumberExpressionError
+      def initialize(node, expression)
+        super(build_message(
+                  node.start,
+                  expression,
+                  point_at_located(node),
+                  'Alternative may not be empty',
+                  "If you did not mean to use an alternative you can use '\\/' to escape the the '/'"
+              ))
+      end
+    end
+
+    class CantEscape < CucumberExpressionError
+      def initialize(expression, index)
+        super(build_message(
+                  index,
+                  expression,
+                  point_at(index),
+                  "Only the characters '{', '}', '(', ')', '\\', '/' and whitespace can be escaped",
+                  "If you did mean to use an '\\' you can use '\\\\' to escape it"
+              ))
+      end
+    end
+
+    class OptionalMayNotBeEmpty < CucumberExpressionError
+      def initialize(node, expression)
+        super(build_message(
+                  node.start,
+                  expression,
+                  point_at_located(node),
+                  'An optional must contain some text',
+                  "If you did not mean to use an optional you can use '\\(' to escape the the '('"
+              ))
+      end
+    end
+
+    class ParameterIsNotAllowedInOptional < CucumberExpressionError
+      def initialize(node, expression)
+        super(build_message(
+                  node.start,
+                  expression,
+                  point_at_located(node),
+                  'An optional may not contain a parameter type',
+                  "If you did not mean to use an parameter type you can use '\\{' to escape the the '{'"
+              ))
+      end
+    end
+
+    class OptionalIsNotAllowedInOptional < CucumberExpressionError
+      def initialize(node, expression)
+        super(build_message(
+                  node.start,
+                  expression,
+                  point_at_located(node),
+                  'An optional may not contain an other optional',
+                  "If you did not mean to use an optional type you can use '\\(' to escape the the '('. For more complicated expressions consider using a regular expression instead."
+              ))
+      end
+    end
+
+    class TheEndOfLineCannotBeEscaped < CucumberExpressionError
+      def initialize(expression)
+        index = expression.codepoints.length - 1
+        super(build_message(
+                  index,
+                  expression,
+                  point_at(index),
+                  'The end of line can not be escaped',
+                  "You can use '\\\\' to escape the the '\\'"
+              ))
+      end
+    end
+
+    class MissingEndToken < CucumberExpressionError
+      def initialize(expression, begin_token, end_token, current)
+        begin_symbol = Token::symbol_of(begin_token)
+        end_symbol = Token::symbol_of(end_token)
+        purpose = Token::purpose_of(begin_token)
+        super(build_message(
+                  current.start,
+                  expression,
+                  point_at_located(current),
+                  "The '#{begin_symbol}' does not have a matching '#{end_symbol}'",
+                  "If you did not intend to use #{purpose} you can use '\\#{begin_symbol}' to escape the #{purpose}"
+              ))
+      end
+    end
+
+    class AlternationNotAllowedInOptional < CucumberExpressionError
+      def initialize(expression, current)
+        super(build_message(
+                  current.start,
+                  expression,
+                  point_at_located(current),
+                  "An alternation can not be used inside an optional",
+                  "You can use '\\/' to escape the the '/'"
+              ))
+      end
+    end
+
+    class InvalidParameterTypeName < CucumberExpressionError
       def initialize(type_name)
-        super("Undefined parameter type {#{type_name}}")
+        super("Illegal character in parameter name {#{type_name}}. " +
+                  "Parameter names may not contain '{', '}', '(', ')', '\\' or '/'")
+      end
+    end
+
+
+    class InvalidParameterTypeNameInNode < CucumberExpressionError
+      def initialize(expression, token)
+        super(build_message(
+                  token.start,
+                  expression,
+                  point_at_located(token),
+                  "Parameter names may not contain '{', '}', '(', ')', '\\' or '/'",
+                  "Did you mean to use a regular expression?"
+              ))
+      end
+    end
+
+    class UndefinedParameterTypeError < CucumberExpressionError
+      def initialize(node, expression, parameter_type_name)
+        super(build_message(node.start,
+                            expression,
+                            point_at_located(node),
+                            "Undefined parameter type '#{parameter_type_name}'",
+                            "Please register a ParameterType for '#{parameter_type_name}'"))
       end
     end
 
@@ -29,11 +200,11 @@ I couldn't decide which one to use. You have two options:
       private
 
       def parameter_type_names(parameter_types)
-        parameter_types.map{|p| "{#{p.name}}"}.join("\n   ")
+        parameter_types.map { |p| "{#{p.name}}" }.join("\n   ")
       end
 
       def expressions(generated_expressions)
-        generated_expressions.map{|ge| ge.source}.join("\n   ")
+        generated_expressions.map { |ge| ge.source }.join("\n   ")
       end
     end
   end