mustermann 0.3.1 → 0.4.0

data/lib/mustermann/ast/param_scanner.rb

@@ -0,0 +1,20 @@
+require 'mustermann/ast/translator'
+
+module Mustermann
+  module AST
+    # Scans an AST for param converters.
+    # @!visibility private
+    # @see Mustermann::AST::Pattern#to_templates
+    class ParamScanner < Translator
+      # @!visibility private
+      def self.scan_params(ast)
+        new.translate(ast)
+      end
+
+      translate(:node)    { t(payload) }
+      translate(Array)    { map { |e| t(e) }.inject(:merge) }
+      translate(Object)   { {} }
+      translate(:capture) { convert ? { name => convert } : {} }
+    end
+  end
+end

data/lib/mustermann/ast/parser.rb

@@ -11,8 +11,8 @@ module Mustermann
       # @param [String] string to be parsed
       # @return [Mustermann::AST::Node] parse tree for string
       # @!visibility private
-      def self.parse(string)
-        new.parse(string)
+      def self.parse(string, **options)
+        new(**options).parse(string)
       end
 
       # Defines another grammar rule for first character.
@@ -31,24 +31,29 @@ module Mustermann
       #
       # @see Mustermann::Sinatra
       # @!visibility private
-      def self.suffix(pattern = /./, &block)
+      def self.suffix(pattern = /./, after: :node, &block)
         @suffix ||= []
-        @suffix << [pattern, block] if block
+        @suffix << [pattern, after, block] if block
         @suffix
       end
 
       # @!visibility private
-      attr_reader :buffer, :string
+      attr_reader :buffer, :string, :pattern
 
       extend Forwardable
-      def_delegators :buffer, :eos?, :getch
+      def_delegators :buffer, :eos?, :getch, :pos
+
+      # @!visibility private
+      def initialize(pattern: nil, **options)
+        @pattern = pattern
+      end
 
       # @param [String] string to be parsed
       # @return [Mustermann::AST::Node] parse tree for string
       # @!visibility private
       def parse(string)
         @string = string
-        @buffer = StringScanner.new(string)
+        @buffer = ::StringScanner.new(string)
         node(:root, string) { read unless eos? }
       end
 
@@ -59,8 +64,10 @@ module Mustermann
       # @return [Mustermann::AST::Node]
       # @!visibility private
       def node(type, *args, &block)
-        type = Node[type] unless type.respond_to? :new
-        block ? type.parse(*args, &block) : type.new(*args)
+        type  = Node[type] unless type.respond_to? :new
+        start = pos
+        node  = block ? type.parse(*args, &block) : type.new(*args)
+        min_size(start, pos, node)
       end
 
       # Create a node for a character we don't have an explicit rule for.
@@ -76,20 +83,35 @@ module Mustermann
       # @return [Mustermann::AST::Node] next element
       # @!visibility private
       def read
-        char    = getch
-        method  = "read %p" % char
-        element = respond_to?(method) ? send(method, char) : default_node(char)
+        start  = pos
+        char   = getch
+        method = "read %p" % char
+        element= respond_to?(method) ? send(method, char) : default_node(char)
+        min_size(start, pos, element)
         read_suffix(element)
       end
 
+      # sets start on node to start if it's not set to a lower value.
+      # sets stop on node to stop if it's not set to a higher value.
+      # @return [Mustermann::AST::Node] the node passed as third argument
+      # @!visibility private
+      def min_size(start, stop, node)
+        stop  ||= start
+        start ||= stop
+        node.start = start unless node.start and node.start < start
+        node.stop  = stop  unless node.stop  and node.stop  > stop
+        node
+      end
+
       # Checks for a potential suffix on the buffer.
       # @param [Mustermann::AST::Node] element node without suffix
       # @return [Mustermann::AST::Node] node with suffix
       # @!visibility private
       def read_suffix(element)
-        self.class.suffix.inject(element) do |ele, (regexp, callback)|
-          next ele unless payload = scan(regexp)
-          instance_exec(payload, ele, &callback)
+        self.class.suffix.inject(element) do |ele, (regexp, after, callback)|
+          next ele unless ele.is_a?(after) and payload = scan(regexp)
+          content = instance_exec(payload, ele, &callback)
+          min_size(element.start, pos, content)
         end
       end
 
@@ -102,8 +124,25 @@ module Mustermann
       # @return [String, MatchData, nil]
       # @!visibility private
       def scan(regexp)
+        match_buffer(:scan, regexp)
+      end
+
+      # Wrapper around {StringScanner#check} that turns strings into escaped
+      # regular expressions and returns a MatchData if the regexp has any
+      # named captures.
+      #
+      # @param [Regexp, String] regexp
+      # @see StringScanner#check
+      # @return [String, MatchData, nil]
+      # @!visibility private
+      def check(regexp)
+        match_buffer(:check, regexp)
+      end
+
+      # @!visibility private
+      def match_buffer(method, regexp)
         regexp = Regexp.new(Regexp.escape(regexp)) unless regexp.is_a? Regexp
-        string = buffer.scan(regexp)
+        string = buffer.public_send(method, regexp)
         regexp.names.any? ? regexp.match(string) : string
       end
 
@@ -114,8 +153,85 @@ module Mustermann
       # @return [String, MatchData] the match
       # @raise [Mustermann::ParseError] if expectation wasn't met
       # @!visibility private
-      def expect(regexp, **options)
-        scan(regexp)|| unexpected(**options)
+      def expect(regexp, char: nil, **options)
+        scan(regexp) || unexpected(char, **options)
+      end
+
+      # Allows to read a string inside brackets. It does not expect the string
+      # to start with an opening bracket.
+      #
+      # @example
+      #   buffer.string = "fo<o>>ba<r>"
+      #   read_brackets(?<, ?>) # => "fo<o>"
+      #   buffer.rest # => "ba<r>"
+      #
+      # @!visibility private
+      def read_brackets(open, close, char: nil, escape: ?\\, quote: false, **options)
+        result = ""
+        escape = false if escape.nil?
+        while current = getch
+          case current
+          when close  then return result
+          when open   then result << open   << read_brackets(open, close) << close
+          when escape then result << escape << getch
+          else result << current
+          end
+        end
+        unexpected(char, **options)
+      end
+
+
+      # Reads an argument string of the format arg1,args2,key:value
+      #
+      # @!visibility private
+      def read_args(key_separator, close, separator: ?,, symbol_keys: true, **options)
+        list, map = [], {}
+        while buffer.peek(1) != close
+          scan(separator)
+          entries = read_list(close, separator, separator: key_separator, **options)
+          case entries.size
+          when 1 then list += entries
+          when 2 then map[symbol_keys ? entries.first.to_sym : entries.first] = entries.last
+          else        unexpected(key_separator)
+          end
+          buffer.pos -= 1
+        end
+        expect(close)
+        [list, map]
+      end
+
+      # Reads a separated list with the ability to quote, escape and add spaces.
+      #
+      # @!visibility private
+      def read_list(*close, separator: ?,, escape: ?\\, quotes: [?", ?'], ignore: " ", **options)
+        result = []
+        while current = getch
+          element = result.empty? ? result : result.last
+          case current
+          when *close    then return result
+          when ignore    then nil # do nothing
+          when separator then result  << ""
+          when escape    then element << getch
+          when *quotes   then element << read_escaped(current, escape: escape)
+          else element << current
+          end
+        end
+        unexpected(current, **options)
+      end
+
+      # Read a string until a terminating character, ignoring escaped versions of said character.
+      #
+      # @!visibility private
+      def read_escaped(close, escape: ?\\, **options)
+        result = ""
+        while current = getch
+          case current
+          when close  then return result
+          when escape then result << getch
+          else result << current
+          end
+        end
+        unexpected(current, **options)
       end
 
       # Helper for raising an exception for an unexpected character.
@@ -124,10 +240,13 @@ module Mustermann
       # @param [String, nil] char the unexpected character
       # @raise [Mustermann::ParseError, Exception]
       # @!visibility private
-      def unexpected(char = getch, exception: ParseError)
+      def unexpected(char = nil, exception: ParseError)
+        char ||= getch
         char = "space" if char == " "
         raise exception, "unexpected #{char || "end of string"} while parsing #{string.inspect}"
       end
+
+      private :match_buffer
     end
 
     private_constant :Parser

data/lib/mustermann/ast/pattern.rb

@@ -1,7 +1,10 @@
 require 'mustermann/ast/parser'
+require 'mustermann/ast/boundaries'
 require 'mustermann/ast/compiler'
 require 'mustermann/ast/transformer'
 require 'mustermann/ast/validation'
+require 'mustermann/ast/template_generator'
+require 'mustermann/ast/param_scanner'
 require 'mustermann/regexp_based'
 require 'mustermann/expander'
 require 'tool/equality_map'
@@ -16,8 +19,9 @@ module Mustermann
 
       extend Forwardable, SingleForwardable
       single_delegate on: :parser, suffix: :parser
-      instance_delegate %i[parser compiler transformer validation] => 'self.class'
-      instance_delegate parse: :parser, transform: :transformer, validate: :validation
+      instance_delegate %i[parser compiler transformer validation template_generator param_scanner boundaries] => 'self.class'
+      instance_delegate parse: :parser, transform: :transformer, validate: :validation,
+        generate_templates: :template_generator, scan_params: :param_scanner, set_boundaries: :boundaries
 
       # @api private
       # @return [#parse] parser object for pattern
@@ -36,7 +40,14 @@ module Mustermann
       end
 
       # @api private
-      # @return [#transform] compiler object for pattern
+      # @return [#set_boundaries] translator making sure start and stop is set on all nodes
+      # @!visibility private
+      def self.boundaries
+        Boundaries
+      end
+
+      # @api private
+      # @return [#transform] transformer object for pattern
       # @!visibility private
       def self.transformer
         Transformer
@@ -49,6 +60,20 @@ module Mustermann
         Validation
       end
 
+      # @api private
+      # @return [#generate_templates] generates URI templates for pattern
+      # @!visibility private
+      def self.template_generator
+        TemplateGenerator
+      end
+
+      # @api private
+      # @return [#scan_params] param scanner for pattern
+      # @!visibility private
+      def self.param_scanner
+        ParamScanner
+      end
+
       # @!visibility private
       def compile(**options)
         options[:except] &&= parse options[:except]
@@ -62,7 +87,12 @@ module Mustermann
       # @!visibility private
       def to_ast
         @ast_cache ||= Tool::EqualityMap.new
-        @ast_cache.fetch(@string) { validate(transform(parse(@string))) }
+        @ast_cache.fetch(@string) do
+          ast   = parse(@string, pattern: self)
+          ast &&= transform(ast)
+          ast &&= set_boundaries(ast, string: @string)
+          validate(ast)
+        end
       end
 
       # All AST-based pattern implementations support expanding.
@@ -73,12 +103,34 @@ module Mustermann
       # @raise (see Mustermann::Pattern#expand)
       # @see Mustermann::Pattern#expand
       # @see Mustermann::Expander
-      def expand(**values)
+      def expand(behavior = nil, values = {})
         @expander ||= Mustermann::Expander.new(self)
-        @expander.expand(**values)
+        @expander.expand(behavior, values)
+      end
+
+      # All AST-based pattern implementations support generating templates.
+      #
+      # @example (see Mustermann::Pattern#to_templates)
+      # @param (see Mustermann::Pattern#to_templates)
+      # @return (see Mustermann::Pattern#to_templates)
+      # @see Mustermann::Pattern#to_templates
+      def to_templates
+        @to_templates ||= generate_templates(to_ast)
+      end
+
+      # @!visibility private
+      # @see Mustermann::Pattern#map_param
+      def map_param(key, value)
+        return super unless param_converters.include? key
+        param_converters[key][super]
+      end
+
+      # @!visibility private
+      def param_converters
+        @param_converters ||= scan_params(to_ast)
       end
 
-      private :compile
+      private :compile, :parse, :transform, :validate, :generate_templates, :param_converters, :scan_params, :set_boundaries
     end
   end
 end

data/lib/mustermann/ast/template_generator.rb

@@ -0,0 +1,28 @@
+require 'mustermann/ast/translator'
+
+module Mustermann
+  module AST
+    # Turns an AST into an Array of URI templates representing the AST.
+    # @!visibility private
+    # @see Mustermann::AST::Pattern#to_templates
+    class TemplateGenerator < Translator
+      # @!visibility private
+      def self.generate_templates(ast)
+        new.translate(ast).uniq
+      end
+
+      # translate(:expression) is not needed, since template patterns simply call to_s
+      translate(:root, :group)        { t(payload) || [""]            }
+      translate(:separator, :char)    { t.escape(payload)             }
+      translate(:capture)             { "{#{name}}"                   }
+      translate(:optional)            { [t(payload), ""]              }
+      translate(:named_splat, :splat) { "{+#{name}}"                  }
+      translate(:with_look_ahead)     { t([head, payload])            }
+      translate(:union)               { payload.flat_map { |e| t(e) } }
+
+      translate(Array) do
+        map { |e| Array(t(e)) }.inject { |first, second| first.product(second).map(&:join) }
+      end
+    end
+  end
+end

data/lib/mustermann/ast/transformer.rb

@@ -44,7 +44,7 @@ module Mustermann
           self.operator = OPERATORS.fetch(operator) { raise CompileError, "#{operator} operator not supported" }
           separator     = Node[:separator].new(operator.separator)
           prefix        = Node[:separator].new(operator.prefix)
-          self.payload  = Array(payload.inject { |list, element| Array(list) << t(separator) << t(element) })
+          self.payload  = Array(payload.inject { |list, element| Array(list) << t(separator.dup) << t(element) })
           payload.unshift(prefix) if operator.prefix
           self
         end
@@ -91,7 +91,7 @@ module Mustermann
         # @!visibility private
         def create_lookahead(elements, *args)
           return elements unless elements.size > 1
-          [Node[:with_look_ahead].new(elements, *args)]
+          [Node[:with_look_ahead].new(elements, *args, start: elements.first.start, stop: elements.last.stop)]
         end
 
         # can the given element be used in a look-ahead?

data/lib/mustermann/ast/translator.rb

@@ -74,6 +74,26 @@ module Mustermann
         end
       end
 
+      # Enables quick creation of a translator object.
+      #
+      # @example
+      #   require 'mustermann'
+      #   require 'mustermann/ast/translator'
+      #   
+      #   translator = Mustermann::AST::Translator.create do
+      #     translate(:node)  { [type, *t(payload)].flatten.compact }
+      #     translate(Array)  { map { |e| t(e) } }
+      #     translate(Object) { }
+      #   end
+      #   
+      #   ast = Mustermann.new('/:name').to_ast
+      #   translator.translate(ast) # => [:root, :separator, :capture]
+      #
+      # @!visibility private
+      def self.create(&block)
+        Class.new(self, &block).new
+      end
+
       raises Mustermann::Error
 
       # @param [Mustermann::AST::Node, Object] node to translate

data/lib/mustermann/ast/validation.rb

@@ -21,14 +21,15 @@ module Mustermann
       translate(Object, :splat) {}
       translate(:node) { t(payload) }
       translate(Array) { each { |p| t(p)} }
-      translate(:capture, :variable, :named_splat) { t.check_name(name) }
+      translate(:capture) { t.check_name(name, forbidden: ['captures', 'splat'])}
+      translate(:variable, :named_splat) { t.check_name(name, forbidden: 'captures')}
 
       # @raise [Mustermann::CompileError] if name is not acceptable
       # @!visibility private
-      def check_name(name)
+      def check_name(name, forbidden: [])
         raise CompileError, "capture name can't be empty" if name.nil? or name.empty?
         raise CompileError, "capture name must start with underscore or lower case letter" unless name =~ /^[a-z_]/
-        raise CompileError, "capture name can't be #{name}" if name == "splat" or name == "captures"
+        raise CompileError, "capture name can't be #{name}" if Array(forbidden).include? name
         raise CompileError, "can't use the same capture name twice" if names.include? name
         names << name
       end