actionview_precompiler 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f3734d67f86056784f7949e66fc5027be5cc29eb554fdc69a0b00f4692c0254
-  data.tar.gz: 010ffab3de043d3d71a6d0d58c3e6a2789f164f8edd5b18c54ce5b7429a266ca
+  metadata.gz: 48e7feef8ebfac2d8a2d18188dcfe98ce8e156e84567a8c632ebd713facb9a6e
+  data.tar.gz: 4b5efa2f1ce1a221bbf1f1729ec4548a0dc6250a4086a5e5dc1e1ef485cd59aa
 SHA512:
-  metadata.gz: ca66b59f8768cdfe813b17dd9277211e269da5d978013ae9dda7b3e07d588a2b0a8c412849f8f5c866d44b23b3eadf6327ce92366f7b29888c131c48f6585c3b
-  data.tar.gz: dfb74620b2c5aaa56e008ce39172654cec82c98d951e3af57f63cea0d55f027d06489b48f412dfcee7efd46ba75ce6280d2456c82493dd9ad7094476c89ccc58
+  metadata.gz: 84eccaa58fb87a5a9dd06183e9f2fda66e258eaeaf57e0abbe27a4e8eed691e7cf1981a3d0ab4e55dbe6e9df841ea7f75ab924938ab8458ac8c7a2068ffa8c8f
+  data.tar.gz: bc95591c0489a70a3c1de6975c3ab0deb7890ee7d0ce4be018b4283098d3dabcb65eddaca299e50220673dbdb425d795ce20fabec178a5f20a6a6021038766fe

data/README.md

@@ -1,6 +1,6 @@
 # ActionviewPrecompiler
 
-Provides eager loading of ActionView templates.
+Provides eager loading of Action View templates.
 
 This optimization aims to improve cold render times and to allow more memory to be shared via CoW on forking web servers.
 
@@ -31,7 +31,7 @@ We determine the locals passed to each template by parsing all templates looking
 Right now this assumes every template with the same `virtual_path` takes the same locals (there may be smarter options, we just aren't doing them).
 A curse/blessing/actually still a curse of this approach is that mis-predicting render calls doesn't cause any issues, it just wastes RAM.
 
-Templates are half-compiled using standard ActionView handlers, so this should work for erb/builder/haml/whatever.
+Templates are half-compiled using standard Action View handlers, so this should work for erb/builder/haml/whatever.
 Parsing is done using either Ruby 2.6's `RubyVM::AbstractSyntaxTree` or JRuby's parser.
 
 ## Installation

data/lib/actionview_precompiler/ast_parser/jruby.rb

@@ -1,5 +1,5 @@
 module ActionviewPrecompiler
-  module ASTParser
+  module JRubyASTParser
     class Node
       def self.wrap(node)
         if org::jruby::ast::Node === node
@@ -25,14 +25,37 @@ module ActionviewPrecompiler
       def string?; org::jruby::ast::StrNode    === @node; end
       def symbol?; org::jruby::ast::SymbolNode === @node; end
 
+      def variable_reference?
+        org::jruby::ast::InstVarNode === @node ||
+          org::jruby::ast::GlobalVarNode === @node ||
+          org::jruby::ast::ClassVarNode === @node
+      end
+
+      def vcall?
+        org::jruby::ast::VCallNode === @node;
+      end
+
+      def call?
+        org::jruby::ast::CallNode === @node;
+      end
+
+      def variable_name
+        self[0][0]
+      end
+
+      def call_method_name
+        self.last.first
+      end
+
       def argument_nodes
-        @node.args_node.to_a[0...@node.args_node.size].map do |arg|
+        @node.args_node.children.to_a[0...@node.args_node.size].map do |arg|
           self.class.wrap(arg)
         end
       end
 
       def to_hash
         @node.pairs.each_with_object({}) do |pair, object|
+          return nil if pair.key == nil
           object[self.class.wrap(pair.key)] = self.class.wrap(pair.value)
         end
       end
@@ -41,6 +64,14 @@ module ActionviewPrecompiler
         @node.value
       end
 
+      def variable_name
+        @node.name.to_s
+      end
+
+      def call_method_name
+        @node.name.to_s
+      end
+
       def to_symbol
         @node.name
       end
@@ -53,8 +84,17 @@ module ActionviewPrecompiler
       end
     end
 
-    def parse(code)
-      Node.wrap(JRuby.parse(code))
+    extend self
+
+    METHODS_TO_PARSE = %i(render render_to_string layout)
+
+    def parse_render_nodes(code)
+      node = Node.wrap(JRuby.parse(code))
+
+      renders = extract_render_nodes(node)
+      renders.group_by(&:first).collect do |method, nodes|
+        [ method, nodes.collect { |v| v[1] } ]
+      end.to_h
     end
 
     def node?(node)
@@ -64,5 +104,19 @@ module ActionviewPrecompiler
     def fcall?(node, name)
       node.fcall_named?(name)
     end
+
+    def extract_render_nodes(node)
+      return [] unless node?(node)
+      renders = node.children.flat_map { |c| extract_render_nodes(c) }
+
+      method_name = render_call?(node)
+      renders << [method_name, node] if method_name
+
+      renders
+    end
+
+    def render_call?(node)
+      METHODS_TO_PARSE.detect { |m| fcall?(node, m) }
+    end
   end
 end

data/lib/actionview_precompiler/ast_parser/ripper.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "ripper"
+
+module ActionviewPrecompiler
+  module RipperASTParser
+    class Node < ::Array
+      attr_reader :type
+
+      def initialize(type, arr, opts = {})
+        @type = type
+        super(arr)
+      end
+
+      def children
+        to_a
+      end
+
+      def inspect
+        typeinfo = type && type != :list ? ':' + type.to_s + ', ' : ''
+        's(' + typeinfo + map(&:inspect).join(", ") + ')'
+      end
+
+      def fcall?
+        type == :command || type == :fcall
+      end
+
+      def fcall_named?(name)
+        fcall? &&
+          self[0].type == :@ident &&
+          self[0][0] == name
+      end
+
+      def argument_nodes
+        raise unless fcall?
+        return [] if self[1].nil?
+        if self[1].last == false || self[1].last.type == :vcall
+          self[1][0...-1]
+        else
+          self[1][0..-1]
+        end
+      end
+
+      def string?
+        type == :string_literal
+      end
+
+      def variable_reference?
+        type == :var_ref
+      end
+
+      def vcall?
+        type == :vcall
+      end
+
+      def call?
+        type == :call
+      end
+
+      def variable_name
+        self[0][0]
+      end
+
+      def call_method_name
+        self.last.first
+      end
+
+      def to_string
+        raise unless string?
+        raise "fixme" unless self[0].type == :string_content
+        raise "fixme" unless self[0][0].type == :@tstring_content
+        self[0][0][0]
+      end
+
+      def hash?
+        type == :bare_assoc_hash || type == :hash
+      end
+
+      def to_hash
+        if type == :bare_assoc_hash
+          hash_from_body(self[0])
+        elsif type == :hash && self[0] == nil
+          {}
+        elsif type == :hash && self[0].type == :assoclist_from_args
+          hash_from_body(self[0][0])
+        else
+          raise "not a hash? #{inspect}"
+        end
+      end
+
+      def hash_from_body(body)
+        body.map do |hash_node|
+          return nil if hash_node.type != :assoc_new
+
+          [hash_node[0], hash_node[1]]
+        end.to_h
+      end
+
+      def symbol?
+        type == :@label || type == :symbol_literal
+      end
+
+      def to_symbol
+        if type == :@label && self[0] =~ /\A(.+):\z/
+          $1.to_sym
+        elsif type == :symbol_literal && self[0].type == :symbol && self[0][0].type == :@ident
+          self[0][0][0].to_sym
+        else
+          raise "not a symbol?: #{self.inspect}"
+        end
+      end
+    end
+
+    class NodeParser < ::Ripper
+      PARSER_EVENTS.each do |event|
+        arity = PARSER_EVENT_TABLE[event]
+
+        if /_new\z/ =~ event.to_s && arity == 0
+          module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            def on_#{event}(*args)
+              Node.new(:list, args, lineno: lineno(), column: column())
+            end
+          eof
+        elsif /_add(_.+)?\z/ =~ event.to_s
+          module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            begin; undef on_#{event}; rescue NameError; end
+            def on_#{event}(list, item)
+              list.push(item)
+              list
+            end
+          eof
+        else
+          module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            begin; undef on_#{event}; rescue NameError; end
+            def on_#{event}(*args)
+              Node.new(:#{event}, args, lineno: lineno(), column: column())
+            end
+          eof
+        end
+      end
+
+      SCANNER_EVENTS.each do |event|
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(tok)
+            Node.new(:@#{event}, [tok], lineno: lineno(), column: column())
+          end
+        End
+      end
+    end
+
+    class RenderCallParser < NodeParser
+      attr_reader :render_calls
+
+      METHODS_TO_PARSE = %w(render render_to_string layout)
+
+      def initialize(*args)
+        super
+
+        @render_calls = []
+      end
+
+      private
+
+      def on_fcall(name, *args)
+        on_render_call(super)
+      end
+
+      def on_command(name, *args)
+        on_render_call(super)
+      end
+
+      def on_render_call(node)
+        METHODS_TO_PARSE.each do |method|
+          if node.fcall_named?(method)
+            @render_calls << [method, node]
+            return node
+          end
+        end
+        node
+      end
+
+      def on_arg_paren(content)
+        content
+      end
+
+      def on_paren(content)
+        content
+      end
+    end
+
+    extend self
+
+    def parse_render_nodes(code)
+      parser = RenderCallParser.new(code)
+      parser.parse
+
+      parser.render_calls.group_by(&:first).collect do |method, nodes|
+        [ method.to_sym, nodes.collect { |v| v[1] } ]
+      end.to_h
+    end
+  end
+end

data/lib/actionview_precompiler/ast_parser/ruby26.rb

@@ -1,5 +1,5 @@
 module ActionviewPrecompiler
-  module ASTParser
+  module Ruby26ASTParser
     class Node
       def self.wrap(node)
         if RubyVM::AbstractSyntaxTree::Node === node
@@ -24,7 +24,13 @@ module ActionviewPrecompiler
       end
 
       def argument_nodes
-        children[1].children[0...-1]
+        if children[1].array?
+          children[1].children[0...-1]
+        elsif children[1].block_pass?
+          children[1].children[0].children[0...-1]
+        else
+          raise "can't call argument_nodes on #{inspect}"
+        end
       end
 
       def array?
@@ -39,10 +45,34 @@ module ActionviewPrecompiler
         type == :HASH
       end
 
+      def block_pass?
+        type == :BLOCK_PASS
+      end
+
       def string?
         type == :STR && String === children[0]
       end
 
+      def variable_reference?
+        type == :IVAR || type == :GVAR || type == :CVAR
+      end
+
+      def variable_name
+        children[0].to_s
+      end
+
+      def vcall?
+        type == :VCALL
+      end
+
+      def call?
+        type == :CALL
+      end
+
+      def call_method_name
+        children[1].to_s
+      end
+
       def symbol?
         type == :LIT && Symbol === children[0]
       end
@@ -52,7 +82,9 @@ module ActionviewPrecompiler
         if list.nil?
           {}
         else
-          list.children[0..-2].each_slice(2).to_h
+          hash = list.children[0..-2].each_slice(2).to_h
+          return nil if hash.key?(nil)
+          hash
         end
       end
 
@@ -68,7 +100,7 @@ module ActionviewPrecompiler
         fcall? &&
           children[0] == name &&
           children[1] &&
-          children[1].array?
+          (children[1].array? || (children[1].block_pass? && children[1].children[0].array?))
       end
 
       private
@@ -78,6 +110,18 @@ module ActionviewPrecompiler
       end
     end
 
+    extend self
+
+    METHODS_TO_PARSE = %i(render render_to_string layout)
+
+    def parse_render_nodes(code)
+      renders = extract_render_nodes(parse(code))
+
+      renders.group_by(&:first).collect do |method, nodes|
+        [ method, nodes.collect { |v| v[1] } ]
+      end.to_h
+    end
+
     def parse(code)
       Node.wrap(RubyVM::AbstractSyntaxTree.parse(code))
     end
@@ -89,5 +133,25 @@ module ActionviewPrecompiler
     def fcall?(node, name)
       node.fcall_named?(name)
     end
+
+    def extract_render_nodes(root)
+      renders = []
+      queue = [root]
+
+      while node = queue.shift
+        node.children.each do |child|
+          queue << child if node?(child)
+        end
+
+        method_name = render_call?(node)
+        renders << [method_name, node] if method_name
+      end
+
+      renders
+    end
+
+    def render_call?(node)
+      METHODS_TO_PARSE.detect { |m| fcall?(node, m) }
+    end
   end
 end

data/lib/actionview_precompiler/ast_parser/syntax_tree.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "syntax_tree"
+
+module ActionviewPrecompiler
+  module SyntaxTreeASTParser
+    # This error is raised whenever an assumption we made wasn't met by the AST.
+    class CompilationError < StandardError
+    end
+
+    # Each call object is responsible for holding a list of arguments and should
+    # respond to a single #arguments_node method that returns an array of
+    # arguments.
+    class RenderCall
+      attr_reader :argument_nodes
+
+      def initialize(argument_nodes)
+        @argument_nodes = argument_nodes
+      end
+    end
+
+    # This class represents a node in the tree that is returned by the parser
+    # that corresponds to an argument to a render call, or a child of one of
+    # those nodes.
+    class RenderNode
+      attr_reader :node
+
+      def initialize(node)
+        @node = node
+      end
+
+      def call?
+        node.is_a?(SyntaxTree::Call)
+      end
+
+      def hash?
+        node.is_a?(SyntaxTree::HashLiteral) || node.is_a?(SyntaxTree::BareAssocHash)
+      end
+
+      def string?
+        node.is_a?(SyntaxTree::StringLiteral)
+      end
+
+      def symbol?
+        node.is_a?(SyntaxTree::Label) || node.is_a?(SyntaxTree::SymbolLiteral)
+      end
+
+      def variable_reference?
+        node.is_a?(SyntaxTree::VarRef)
+      end
+
+      def vcall?
+        node.is_a?(SyntaxTree::VCall)
+      end
+
+      # Converts the node into a hash where the keys and values are nodes. This
+      # will raise an error if the hash doesn't match the format we expect or
+      # if the hash contains any splats.
+      def to_hash
+        case node
+        in SyntaxTree::HashLiteral[assocs:]
+        in SyntaxTree::BareAssocHash[assocs:]
+        else
+          raise CompilationError, "Unexpected node type: #{node.class.name}"
+        end
+
+        assocs.to_h do |assoc|
+          case assoc
+          in SyntaxTree::Assoc[key:, value:]
+            [RenderNode.new(key), RenderNode.new(value)]
+          else
+            raise CompilationError, "Unexpected node type: #{node.class.name}"
+          end
+        end
+      end
+
+      # Converts the node into a string value. Only handles plain string
+      # content, and will raise an error if the node contains interpolation.
+      def to_string
+        case node
+        in SyntaxTree::StringLiteral[parts: [SyntaxTree::TStringContent[value:]]]
+          value
+        in SyntaxTree::StringLiteral[parts:]
+          raise CompilationError, "Unexpected string parts type: #{parts.inspect}"
+        else
+          raise CompilationError, "Unexpected node type: #{node.class.name}"
+        end
+      end
+
+      # Converts the node into a symbol value. Only handles labels and plain
+      # symbols, and will raise an error if the node contains interpolation.
+      def to_symbol
+        case node
+        in SyntaxTree::Label[value:]
+          value.chomp(":").to_sym
+        in SyntaxTree::SymbolLiteral[value: SyntaxTree::Ident[value:]]
+          value.to_sym
+        in SyntaxTree::SymbolLiteral[value:]
+          raise CompilationError, "Unexpected symbol value type: #{value.inspect}"
+        else
+          raise CompilationError, "Unexpected node type: #{node.class.name}"
+        end
+      end
+    end
+
+    # This visitor is responsible for visiting the parsed tree and extracting
+    # out the render calls. After visiting the tree, the #render_calls method
+    # will return the hash expected by the #parse_render_nodes method.
+    class RenderVisitor < SyntaxTree::Visitor
+      MESSAGE = /\A(render|render_to_string|layout)\z/
+
+      attr_reader :render_calls
+
+      def initialize
+        @render_calls = Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      visit_method def visit_command(node)
+        case node
+        in SyntaxTree::Command[
+             message: SyntaxTree::Ident[value: MESSAGE],
+             arguments: SyntaxTree::Args[parts:]
+           ]
+          argument_nodes = parts.map { |part| RenderNode.new(part) }
+          render_calls[$1.to_sym] << RenderCall.new(argument_nodes)
+        else
+        end
+
+        super
+      end
+
+      visit_method def visit_fcall(node)
+        case node
+        in SyntaxTree::FCall[
+             value: SyntaxTree::Ident[value: MESSAGE],
+             arguments: SyntaxTree::ArgParen[arguments: SyntaxTree::Args[parts:]]
+           ]
+          argument_nodes = parts.map { |part| RenderNode.new(part) }
+          render_calls[$1.to_sym] << RenderCall.new(argument_nodes)
+        else
+        end
+
+        super
+      end
+    end
+
+    # Main entrypoint into this AST parser variant. It's responsible for
+    # returning a hash of render calls. The keys are the method names, and the
+    # values are arrays of call objects.
+    def self.parse_render_nodes(code)
+      visitor = RenderVisitor.new
+      SyntaxTree.parse(code).accept(visitor)
+      visitor.render_calls
+    rescue SyntaxTree::Parser::ParseError
+      raise CompilationError, "Unable to parse the template"
+    end
+  end
+end

data/lib/actionview_precompiler/ast_parser.rb

@@ -1,5 +1,20 @@
-if RUBY_ENGINE == 'jruby'
-  require "actionview_precompiler/ast_parser/jruby"
-else
-  require "actionview_precompiler/ast_parser/ruby26"
+module ActionviewPrecompiler
+  parser = ENV["PRECOMPILER_PARSER"]
+  parser ||= "jruby" if RUBY_ENGINE == 'jruby'
+  parser ||= "rubyvm_ast" if RUBY_ENGINE == 'ruby'
+
+  case parser
+  when "rubyvm_ast"
+    require "actionview_precompiler/ast_parser/ruby26"
+    ASTParser = Ruby26ASTParser
+  when "jruby"
+    require "actionview_precompiler/ast_parser/jruby"
+    ASTParser = JRubyASTParser
+  when "syntax_tree"
+    require "actionview_precompiler/ast_parser/syntax_tree"
+    ASTParser = SyntaxTreeASTParser
+  else
+    require "actionview_precompiler/ast_parser/ripper"
+    ASTParser = RipperASTParser
+  end
 end

data/lib/actionview_precompiler/controller_parser.rb

@@ -0,0 +1,13 @@
+module ActionviewPrecompiler
+  class ControllerParser
+    def initialize(filename)
+      @filename = filename
+    end
+
+    def render_calls
+      src = File.read(@filename)
+      return [] unless src.include?("render")
+      RenderParser.new(src, from_controller: true).render_calls
+    end
+  end
+end

data/lib/actionview_precompiler/controller_scanner.rb

@@ -0,0 +1,48 @@
+module ActionviewPrecompiler
+  class ControllerScanner
+    attr_reader :controller_dir
+
+    def initialize(controller_dir)
+      @controller_dir = controller_dir
+    end
+
+    def template_renders
+      template_renders = []
+
+      each_controller do |path, fullpath|
+        parser = ControllerParser.new(fullpath)
+
+        if path =~ /\A(.*)_controller\.rb\z/
+          controller_prefix = $1
+        end
+
+        parser.render_calls.each do |render_call|
+          virtual_path = render_call.virtual_path
+
+          unless virtual_path.include?("/")
+            next unless controller_prefix
+
+            virtual_path = "#{controller_prefix}/#{virtual_path}"
+          end
+
+          locals = render_call.locals_keys.map(&:to_s).sort
+
+          template_renders << [virtual_path, locals]
+        end
+      end
+
+      template_renders.uniq
+    end
+
+    private
+
+    def each_controller
+      Dir["**/*.rb", base: controller_dir].sort.map do |file|
+        fullpath = File.expand_path(file, controller_dir)
+        next if File.directory?(fullpath)
+
+        yield file, fullpath
+      end
+    end
+  end
+end