actionpack 5.2.1 → 7.0.2.4

data/lib/action_dispatch/journey/gtg/builder.rb

@@ -6,52 +6,51 @@ module ActionDispatch
   module Journey # :nodoc:
     module GTG # :nodoc:
       class Builder # :nodoc:
-        DUMMY = Nodes::Dummy.new
+        DUMMY_END_NODE = Nodes::Dummy.new
 
         attr_reader :root, :ast, :endpoints
 
         def initialize(root)
           @root      = root
-          @ast       = Nodes::Cat.new root, DUMMY
-          @followpos = nil
+          @ast       = Nodes::Cat.new root, DUMMY_END_NODE
+          @followpos = build_followpos
         end
 
         def transition_table
           dtrans   = TransitionTable.new
-          marked   = {}
-          state_id = Hash.new { |h, k| h[k] = h.length }
+          marked   = {}.compare_by_identity
+          state_id = Hash.new { |h, k| h[k] = h.length }.compare_by_identity
+          dstates  = [firstpos(root)]
 
-          start   = firstpos(root)
-          dstates = [start]
           until dstates.empty?
             s = dstates.shift
             next if marked[s]
             marked[s] = true # mark s
 
             s.group_by { |state| symbol(state) }.each do |sym, ps|
-              u = ps.flat_map { |l| followpos(l) }
+              u = ps.flat_map { |l| @followpos[l] }.uniq
               next if u.empty?
 
-              if u.uniq == [DUMMY]
-                from = state_id[s]
-                to   = state_id[Object.new]
-                dtrans[from, to] = sym
+              from = state_id[s]
 
+              if u.all? { |pos| pos == DUMMY_END_NODE }
+                to = state_id[Object.new]
+                dtrans[from, to] = sym
                 dtrans.add_accepting(to)
+
                 ps.each { |state| dtrans.add_memo(to, state.memo) }
               else
-                dtrans[state_id[s], state_id[u]] = sym
-
-                if u.include?(DUMMY)
-                  to = state_id[u]
-
-                  accepting = ps.find_all { |l| followpos(l).include?(DUMMY) }
+                to = state_id[u]
+                dtrans[from, to] = sym
 
-                  accepting.each { |accepting_state|
-                    dtrans.add_memo(to, accepting_state.memo)
-                  }
+                if u.include?(DUMMY_END_NODE)
+                  ps.each do |state|
+                    if @followpos[state].include?(DUMMY_END_NODE)
+                      dtrans.add_memo(to, state.memo)
+                    end
+                  end
 
-                  dtrans.add_accepting(state_id[u])
+                  dtrans.add_accepting(to)
                 end
               end
 
@@ -67,7 +66,10 @@ module ActionDispatch
           when Nodes::Group
             true
           when Nodes::Star
-            true
+            # the default star regex is /(.+)/ which is NOT nullable
+            # but since different constraints can be provided we must
+            # actually check if this is the case or not.
+            node.regexp.match?("")
           when Nodes::Or
             node.children.any? { |c| nullable?(c) }
           when Nodes::Cat
@@ -92,7 +94,7 @@ module ActionDispatch
               firstpos(node.left)
             end
           when Nodes::Or
-            node.children.flat_map { |c| firstpos(c) }.uniq
+            node.children.flat_map { |c| firstpos(c) }.tap(&:uniq!)
           when Nodes::Unary
             firstpos(node.left)
           when Nodes::Terminal
@@ -105,9 +107,9 @@ module ActionDispatch
         def lastpos(node)
           case node
           when Nodes::Star
-            firstpos(node.left)
+            lastpos(node.left)
           when Nodes::Or
-            node.children.flat_map { |c| lastpos(c) }.uniq
+            node.children.flat_map { |c| lastpos(c) }.tap(&:uniq!)
           when Nodes::Cat
             if nullable?(node.right)
               lastpos(node.left) | lastpos(node.right)
@@ -123,40 +125,22 @@ module ActionDispatch
           end
         end
 
-        def followpos(node)
-          followpos_table[node]
-        end
-
         private
-
-          def followpos_table
-            @followpos ||= build_followpos
-          end
-
           def build_followpos
-            table = Hash.new { |h, k| h[k] = [] }
+            table = Hash.new { |h, k| h[k] = [] }.compare_by_identity
             @ast.each do |n|
               case n
               when Nodes::Cat
                 lastpos(n.left).each do |i|
                   table[i] += firstpos(n.right)
                 end
-              when Nodes::Star
-                lastpos(n).each do |i|
-                  table[i] += firstpos(n)
-                end
               end
             end
             table
           end
 
           def symbol(edge)
-            case edge
-            when Journey::Nodes::Symbol
-              edge.regexp
-            else
-              edge.left
-            end
+            edge.symbol? ? edge.regexp : edge.left
           end
       end
     end

data/lib/action_dispatch/journey/gtg/simulator.rb

@@ -14,6 +14,8 @@ module ActionDispatch
       end
 
       class Simulator # :nodoc:
+        INITIAL_STATE = [ [0, nil] ].freeze
+
         attr_reader :tt
 
         def initialize(transition_table)
@@ -22,18 +24,23 @@ module ActionDispatch
 
         def memos(string)
           input = StringScanner.new(string)
-          state = [0]
+          state = INITIAL_STATE
+          start_index = 0
+
           while sym = input.scan(%r([/.?]|[^/.?]+))
-            state = tt.move(state, sym)
-          end
+            end_index = start_index + sym.length
 
-          acceptance_states = state.find_all { |s|
-            tt.accepting? s
-          }
+            state = tt.move(state, string, start_index, end_index)
 
-          return yield if acceptance_states.empty?
+            start_index = end_index
+          end
+
+          acceptance_states = state.each_with_object([]) do |s_d, memos|
+            s, idx = s_d
+            memos.concat(tt.memo(s)) if idx.nil? && tt.accepting?(s)
+          end
 
-          acceptance_states.flat_map { |x| tt.memo(x) }.compact
+          acceptance_states.empty? ? yield : acceptance_states
         end
       end
     end

data/lib/action_dispatch/journey/gtg/transition_table.rb

@@ -10,11 +10,15 @@ module ActionDispatch
 
         attr_reader :memos
 
+        DEFAULT_EXP = /[^.\/?]+/
+        DEFAULT_EXP_ANCHORED = /\A#{DEFAULT_EXP}\Z/
+
         def initialize
-          @regexp_states = {}
-          @string_states = {}
-          @accepting     = {}
-          @memos         = Hash.new { |h, k| h[k] = [] }
+          @stdparam_states = {}
+          @regexp_states   = {}
+          @string_states   = {}
+          @accepting       = {}
+          @memos           = Hash.new { |h, k| h[k] = [] }
         end
 
         def add_accepting(state)
@@ -41,20 +45,54 @@ module ActionDispatch
           Array(t)
         end
 
-        def move(t, a)
+        def move(t, full_string, start_index, end_index)
           return [] if t.empty?
 
-          regexps = []
+          next_states = []
 
-          t.map { |s|
-            if states = @regexp_states[s]
-              regexps.concat states.map { |re, v| re === a ? v : nil }
+          tok = full_string.slice(start_index, end_index - start_index)
+          token_matches_default_component = DEFAULT_EXP_ANCHORED.match?(tok)
+
+          t.each { |s, previous_start|
+            if previous_start.nil?
+              # In the simple case of a "default" param regex do this fast-path
+              # and add all next states.
+              if token_matches_default_component && states = @stdparam_states[s]
+                states.each { |re, v| next_states << [v, nil].freeze if !v.nil? }
+              end
+
+              # When we have a literal string, we can just pull the next state
+              if states = @string_states[s]
+                next_states << [states[tok], nil].freeze unless states[tok].nil?
+              end
             end
 
-            if states = @string_states[s]
-              states[a]
+            # For regexes that aren't the "default" style, they may potentially
+            # not be terminated by the first "token" [./?], so we need to continue
+            # to attempt to match this regexp as well as any successful paths that
+            # continue out of it. both paths could be valid.
+            if states = @regexp_states[s]
+              slice_start = if previous_start.nil?
+                start_index
+              else
+                previous_start
+              end
+
+              slice_length = end_index - slice_start
+              curr_slice = full_string.slice(slice_start, slice_length)
+
+              states.each { |re, v|
+                # if we match, we can try moving past this
+                next_states << [v, nil].freeze if !v.nil? && re.match?(curr_slice)
+              }
+
+              # and regardless, we must continue accepting tokens and retrying this regexp.
+              # we need to remember where we started as well so we can take bigger slices.
+              next_states << [s, slice_start].freeze
             end
-          }.compact.concat regexps
+          }
+
+          next_states
         end
 
         def as_json(options = nil)
@@ -67,9 +105,10 @@ module ActionDispatch
           end
 
           {
-            regexp_states: simple_regexp,
-            string_states: @string_states,
-            accepting:     @accepting
+            regexp_states:   simple_regexp,
+            string_states:   @string_states,
+            stdparam_states: @stdparam_states,
+            accepting:       @accepting
           }
         end
 
@@ -91,7 +130,7 @@ module ActionDispatch
           states    = "function tt() { return #{to_json}; }"
 
           fun_routes = paths.sample(3).map do |ast|
-            ast.map { |n|
+            ast.filter_map { |n|
               case n
               when Nodes::Symbol
                 case n.left
@@ -104,7 +143,7 @@ module ActionDispatch
               else
                 nil
               end
-            }.compact.join
+            }.join
           end
 
           stylesheets = [fsm_css]
@@ -123,31 +162,49 @@ module ActionDispatch
 
         def []=(from, to, sym)
           to_mappings = states_hash_for(sym)[from] ||= {}
+          case sym
+          when Regexp
+            # we must match the whole string to a token boundary
+            if sym == DEFAULT_EXP
+              sym = DEFAULT_EXP_ANCHORED
+            else
+              sym = /\A#{sym}\Z/
+            end
+          when Symbol
+            # account for symbols in the constraints the same as strings
+            sym = sym.to_s
+          end
           to_mappings[sym] = to
         end
 
         def states
           ss = @string_states.keys + @string_states.values.flat_map(&:values)
+          ps = @stdparam_states.keys + @stdparam_states.values.flat_map(&:values)
           rs = @regexp_states.keys + @regexp_states.values.flat_map(&:values)
-          (ss + rs).uniq
+          (ss + ps + rs).uniq
         end
 
         def transitions
           @string_states.flat_map { |from, hash|
             hash.map { |s, to| [from, s, to] }
+          } + @stdparam_states.flat_map { |from, hash|
+            hash.map { |s, to| [from, s, to] }
           } + @regexp_states.flat_map { |from, hash|
             hash.map { |s, to| [from, s, to] }
           }
         end
 
         private
-
           def states_hash_for(sym)
             case sym
-            when String
+            when String, Symbol
               @string_states
             when Regexp
-              @regexp_states
+              if sym == DEFAULT_EXP
+                @stdparam_states
+              else
+                @regexp_states
+              end
             else
               raise ArgumentError, "unknown symbol: %s" % sym.class
             end

data/lib/action_dispatch/journey/nfa/dot.rb

@@ -9,17 +9,6 @@ module ActionDispatch
             "  #{from} -> #{to} [label=\"#{sym || 'ε'}\"];"
           }
 
-          # memo_nodes = memos.values.flatten.map { |n|
-          #   label = n
-          #   if Journey::Route === n
-          #     label = "#{n.verb.source} #{n.path.spec}"
-          #   end
-          #   "  #{n.object_id} [label=\"#{label}\", shape=box];"
-          # }
-          # memo_edges = memos.flat_map { |k, memos|
-          #   (memos || []).map { |v| "  #{k} -> #{v.object_id};" }
-          # }.uniq
-
           <<-eodot
 digraph nfa {
   rankdir=LR;

data/lib/action_dispatch/journey/nodes/node.rb

@@ -4,6 +4,66 @@ require "action_dispatch/journey/visitors"
 
 module ActionDispatch
   module Journey # :nodoc:
+    class Ast # :nodoc:
+      attr_reader :names, :path_params, :tree, :wildcard_options, :terminals
+      alias :root :tree
+
+      def initialize(tree, formatted)
+        @tree = tree
+        @path_params = []
+        @names = []
+        @symbols = []
+        @stars = []
+        @terminals = []
+        @wildcard_options = {}
+
+        visit_tree(formatted)
+      end
+
+      def requirements=(requirements)
+        # inject any regexp requirements for `star` nodes so they can be
+        # determined nullable, which requires knowing if the regex accepts an
+        # empty string.
+        (symbols + stars).each do |node|
+          re = requirements[node.to_sym]
+          node.regexp = re if re
+        end
+      end
+
+      def route=(route)
+        terminals.each { |n| n.memo = route }
+      end
+
+      def glob?
+        stars.any?
+      end
+
+      private
+        attr_reader :symbols, :stars
+
+        def visit_tree(formatted)
+          tree.each do |node|
+            if node.symbol?
+              path_params << node.to_sym
+              names << node.name
+              symbols << node
+            elsif node.star?
+              stars << node
+
+              if formatted != false
+                # Add a constraint for wildcard route to make it non-greedy and
+                # match the optional format part of the route by default.
+                wildcard_options[node.name.to_sym] ||= /.+?/m
+              end
+            end
+
+            if node.terminal?
+              terminals << node
+            end
+          end
+        end
+    end
+
     module Nodes # :nodoc:
       class Node # :nodoc:
         include Enumerable
@@ -32,7 +92,7 @@ module ActionDispatch
         end
 
         def name
-          left.tr "*:".freeze, "".freeze
+          -left.tr("*:", "")
         end
 
         def type
@@ -65,12 +125,12 @@ module ActionDispatch
         def literal?; false; end
       end
 
-      %w{ Symbol Slash Dot }.each do |t|
-        class_eval <<-eoruby, __FILE__, __LINE__ + 1
-          class #{t} < Terminal;
-            def type; :#{t.upcase}; end
-          end
-        eoruby
+      class Slash < Terminal # :nodoc:
+        def type; :SLASH; end
+      end
+
+      class Dot < Terminal # :nodoc:
+        def type; :DOT; end
       end
 
       class Symbol < Terminal # :nodoc:
@@ -78,17 +138,15 @@ module ActionDispatch
         alias :symbol :regexp
         attr_reader :name
 
-        DEFAULT_EXP = /[^\.\/\?]+/
-        def initialize(left)
-          super
-          @regexp = DEFAULT_EXP
-          @name = left.tr "*:".freeze, "".freeze
-        end
-
-        def default_regexp?
-          regexp == DEFAULT_EXP
+        DEFAULT_EXP = /[^.\/?]+/
+        GREEDY_EXP = /(.+)/
+        def initialize(left, regexp = DEFAULT_EXP)
+          super(left)
+          @regexp = regexp
+          @name = -left.tr("*:", "")
         end
 
+        def type; :SYMBOL; end
         def symbol?; true; end
       end
 
@@ -102,6 +160,15 @@ module ActionDispatch
       end
 
       class Star < Unary # :nodoc:
+        attr_accessor :regexp
+
+        def initialize(left)
+          super(left)
+
+          # By default wildcard routes are non-greedy and must match something.
+          @regexp = /.+?/m
+        end
+
         def star?; true; end
         def type; :STAR; end
 

data/lib/action_dispatch/journey/parser.rb

@@ -1,6 +1,6 @@
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by Racc 1.4.14
+# This file is automatically generated by Racc 1.4.16
 # from Racc grammar file "".
 #
 
@@ -135,11 +135,11 @@ Racc_debug_parser = false
 # reduce 0 omitted
 
 def _reduce_1(val, _values)
- Cat.new(val.first, val.last) 
+ Cat.new(val.first, val.last)
 end
 
 def _reduce_2(val, _values)
- val.first 
+ val.first
 end
 
 # reduce 3 omitted
@@ -151,19 +151,19 @@ end
 # reduce 6 omitted
 
 def _reduce_7(val, _values)
- Group.new(val[1]) 
+ Group.new(val[1])
 end
 
 def _reduce_8(val, _values)
- Or.new([val.first, val.last]) 
+ Or.new([val.first, val.last])
 end
 
 def _reduce_9(val, _values)
- Or.new([val.first, val.last]) 
+ Or.new([val.first, val.last])
 end
 
 def _reduce_10(val, _values)
- Star.new(Symbol.new(val.last)) 
+ Star.new(Symbol.new(val.last, Symbol::GREEDY_EXP))
 end
 
 # reduce 11 omitted
@@ -175,19 +175,19 @@ end
 # reduce 14 omitted
 
 def _reduce_15(val, _values)
- Slash.new(val.first) 
+ Slash.new(val.first)
 end
 
 def _reduce_16(val, _values)
- Symbol.new(val.first) 
+ Symbol.new(val.first)
 end
 
 def _reduce_17(val, _values)
- Literal.new(val.first) 
+ Literal.new(val.first)
 end
 
 def _reduce_18(val, _values)
- Dot.new(val.first) 
+ Dot.new(val.first)
 end
 
 def _reduce_none(val, _values)
@@ -195,5 +195,5 @@ def _reduce_none(val, _values)
 end
 
     end   # class Parser
-    end   # module Journey
-  end   # module ActionDispatch
+  end   # module Journey
+end   # module ActionDispatch

data/lib/action_dispatch/journey/parser.y

@@ -21,7 +21,7 @@ rule
     | expression OR or { Or.new([val.first, val.last]) }
     ;
   star
-    : STAR       { Star.new(Symbol.new(val.last)) }
+    : STAR       { Star.new(Symbol.new(val.last, Symbol::GREEDY_EXP)) }
     ;
   terminal
     : symbol