herb 0.10.3 → 0.11.0

data/lib/herb/engine/errors.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+# typed: false
+
+require_relative "../diagnostic"
+
+module Herb
+  class Engine
+    class CompilationError < SyntaxError
+      attr_reader :details, :diagnostics, :visitors, :parser_options
+
+      def initialize(message, details: nil, diagnostics: [], visitors: [], parser_options: {})
+        @details = details
+        @diagnostics = diagnostics
+        @visitors = visitors
+        @parser_options = parser_options
+
+        super(message)
+      end
+
+      def detailed_message(highlight: false, **)
+        report = formatted_errors(highlight: highlight)
+
+        return super unless report
+
+        "#{super}\n\n#{report}"
+      end
+
+      def formatted_errors(highlight: false)
+        @details&.format_all(highlight: highlight)
+      end
+    end
+
+    module LocatedError
+      attr_reader :line, :column, :filename, :suggestion
+
+      def initialize(message, line: nil, column: nil, filename: nil, suggestion: nil)
+        @line = line
+        @column = column
+        @filename = filename
+        @suggestion = suggestion
+
+        super(build_error_message(message))
+      end
+
+      private
+
+      def build_error_message(message)
+        parts = [] #: Array[String]
+
+        if @filename || (@line && @column)
+          location_parts = [] #: Array[String]
+
+          location_parts << @filename if @filename
+          location_parts << "#{@line}:#{@column}" if @line && @column
+
+          parts << location_parts.join(":")
+        end
+
+        parts << message
+
+        parts << "Suggestion: #{@suggestion}" if @suggestion
+
+        parts.join(" - ")
+      end
+    end
+
+    class GeneratorTemplateError < CompilationError
+      include LocatedError
+    end
+
+    class InvalidRubyError < CompilationError
+      attr_reader :compiled_source
+
+      def initialize(message, compiled_source: nil)
+        @compiled_source = compiled_source
+
+        super(message)
+      end
+    end
+
+    class SecurityError < StandardError
+      include LocatedError
+    end
+
+    class ParseError < CompilationError
+      CONTEXT_LINES = 3 #: Integer
+
+      attr_reader :source #: String
+      attr_reader :filename #: String?
+
+      #: (String, diagnostics: Array[Herb::Diagnostic], source: String, ?filename: String?, ?visitors: Array[String], ?parser_options: Hash[Symbol, untyped], ?details: untyped) -> void
+      def initialize(message, diagnostics:, source:, filename: nil, visitors: [], parser_options: {}, details: nil)
+        @source = source
+        @filename = filename
+
+        super(message, details: details, diagnostics: diagnostics, visitors: visitors, parser_options: parser_options)
+      end
+
+      #: () -> Integer?
+      def line_number
+        location = diagnostics.first&.location
+
+        location&.start&.line
+      end
+
+      #: () -> Array[String]
+      def annotated_source_code
+        line = line_number
+
+        return [] unless line
+
+        lines = source.lines
+        first = [line - CONTEXT_LINES, 1].max
+        last = [line + CONTEXT_LINES, lines.length].min
+
+        (first..last).map { |number|
+          format("%<number>5d  %<line>s", number: number, line: lines[number - 1].to_s.chomp)
+        }
+      end
+    end
+  end
+end

data/lib/herb/engine/helpers.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+# typed: true
+
+module Herb
+  class Engine
+    module Helpers
+      #: (String) -> bool
+      def self.comment?(code)
+        stripped = code.rstrip
+
+        return false unless stripped.include?("#")
+        return true unless prism_available?
+
+        Prism.parse(stripped).comments.any? { |comment| comment.location.end_offset >= stripped.bytesize }
+      rescue StandardError
+        true
+      end
+
+      #: (String) -> String
+      def self.without_trailing_spaces(code)
+        code.sub(/[ \t]+\z/, "")
+      end
+
+      #: (String) -> bool
+      def self.ends_on_heredoc_terminator?(code)
+        return false unless code.include?("<<")
+
+        tokens = heredoc_tokens(code)
+
+        return true unless tokens
+
+        tokens.any? { |token| token.type == :HEREDOC_END && token.location.start_line == code.lines.size }
+      end
+
+      #: (String) -> bool
+      def self.heredoc?(code)
+        return false unless code.include?("<<")
+
+        tokens = heredoc_tokens(code)
+
+        return true unless tokens
+
+        tokens.any? { |token| token.type == :HEREDOC_START }
+      end
+
+      #: (String) -> Array[untyped]?
+      def self.heredoc_tokens(code)
+        return nil unless prism_available?
+
+        Prism.lex(code).value.map(&:first)
+      rescue StandardError
+        nil
+      end
+
+      #: (String) -> String
+      def self.strip_trailing_comment(code)
+        return code unless code.include?("#")
+        return code unless prism_available?
+
+        comment = Prism.parse(code).comments.find { |found| found.location.end_offset >= code.bytesize }
+
+        return code unless comment
+
+        without_trailing_spaces(code.byteslice(0, comment.location.start_offset).to_s)
+      rescue StandardError
+        code
+      end
+
+      #: () -> bool
+      def self.prism_available?
+        return @prism_available unless @prism_available.nil?
+
+        @prism_available = begin
+          require "prism"
+          true
+        rescue LoadError
+          false
+        end
+      end
+    end
+  end
+end

data/lib/herb/engine/inline_render/inliner.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative "../../analysis/partial_resolver"
+
+module Herb
+  class Engine
+    module InlineRender
+      class Inliner
+        VIRTUAL_PATH_DEPENDENT = /\bI18n\.[tl]\b|\b(?:t|l|translate|localize)\(|\bcache\b/
+        STRICT_LOCALS = /<%#[-\s]*locals:/
+
+        attr_reader :filename
+
+        def initialize(options = {})
+          @project_path = options[:project_path] || Pathname.new(Dir.pwd)
+          @filename = options[:filename]
+          @resolver = options[:resolver] || Analysis::PartialResolver.new(@project_path)
+          @format = find_format
+        end
+
+        def can_inline?(node, shadowed: [])
+          return false unless node.static_partial?
+          return false if node.body&.any?
+
+          return can_inline_collection?(node, shadowed: shadowed) if node.keywords&.collection
+
+          resolved = resolve_path(node)
+
+          return false unless resolved
+
+          safe_to_inline?(resolved, shadowed: shadowed)
+        end
+
+        def can_inline_collection?(node, shadowed: [])
+          return false unless node.static_partial?
+          return false if node.keywords&.spacer_template
+
+          resolved = resolve_path(node)
+
+          return false unless resolved
+
+          safe_to_inline?(resolved, item: collection_item_name(node), shadowed: shadowed)
+        end
+
+        def collection?(node)
+          !!node.keywords&.collection
+        end
+
+        def collection_expression(node)
+          node.keywords&.collection&.value
+        end
+
+        def collection_item_name(node)
+          as_name = node.keywords&.as_name&.value
+          return as_name if as_name
+
+          partial = node.partial_path
+          return nil unless partial
+
+          File.basename(partial)
+        end
+
+        def resolve_path(node)
+          @resolver.resolve(node.partial_path.to_s, from: @filename, format: @format)&.path
+        end
+
+        def own_locals(source)
+          require "prism"
+
+          ::Prism.parse(::Herb.extract_ruby(source)).value.locals.map(&:to_s)
+        rescue StandardError
+          []
+        end
+
+        def local_assignments(node)
+          locals = {} #: Hash[String, String]
+
+          node.keywords&.locals&.each do |local|
+            name = local.name&.value
+            value = local.value&.content
+
+            next unless name && value
+
+            value = name if value == "#{name}:" # Shorthand hash syntax
+
+            locals[name] = value
+          end
+
+          locals
+        end
+
+        private
+
+        def safe_to_inline?(file_path, item: nil, shadowed: [])
+          source = File.read(file_path)
+
+          # TODO: we might want to revisit these three conditions later
+          return false if source.include?("content_for")
+          return false if source.match?(/\byield\b/)
+          return false if source.include?("local_assigns")
+          return false if source.match?(VIRTUAL_PATH_DEPENDENT)
+          return false if source.match?(STRICT_LOCALS)
+          return false if item && source.include?("#{item}_iteration")
+
+          at_risk = shadowed - own_locals(source)
+
+          return false if at_risk.any? { |name| source.match?(/\b#{Regexp.escape(name)}\b/) }
+
+          true
+        end
+
+        def find_format
+          return nil unless @filename
+
+          parts = File.basename(@filename).split(".")
+
+          parts.length >= 3 ? parts[-2] : nil
+        end
+      end
+    end
+  end
+end

data/lib/herb/engine/inline_render/visitor.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+# typed: false
+
+require_relative "../../visitor"
+require_relative "../../visitor/context_aware"
+require_relative "../../visitor/experimental"
+
+module Herb
+  class Engine
+    module InlineRender
+      # Replaces a `render` of a static partial with the partial itself, so the rendered page costs
+      # no partial lookup at run time.
+      #
+      #     require "herb/engine/inline_render/visitor"
+      #
+      #     Herb::Engine.new(source, visitors: [Herb::Engine::InlineRender::Visitor.new])
+      #
+      # It has to run first, and the engine refuses a stack that puts it anywhere else. Everything
+      # after it sees the partial's markup as part of the template it landed in, which is what holds
+      # a partial to whatever the template around it is held to: a validator that refuses markup in a
+      # template refuses it in a partial, and a transform that rewrites a tag rewrites it wherever it
+      # was written. Were it to run last, moving markup into a partial would be a way of turning
+      # those off.
+      #
+      # What it moves is recorded on `Herb::Visitor::Context::Origin`, because a tree alone cannot say that a
+      # node was written somewhere else or that nobody wrote it at all. That is what lets a position
+      # reported from inside an inlined partial name the partial rather than the template it landed
+      # in, and leaves the wrapper written here attributed to nobody.
+      #
+      # A partial is inlined only when the file it names is knowable and the copy would mean what the
+      # original did. Some of what a partial means is answered by where it is rather than by what it
+      # says, and none of that survives being copied, so `Inliner` reads the partial and leaves it
+      # alone when it finds any of it. Anything left alone stays the `render` call it was written as.
+      #
+      class Visitor < Herb::Visitor
+        extend Herb::Visitor::Experimental
+        include Herb::Visitor::ContextAware
+
+        ARRAY_PROPERTIES = [:children, :body, :statements].freeze #: Array[Symbol]
+        NODE_PROPERTIES = [:subsequent, :else_clause, :rescue_clause, :ensure_clause].freeze #: Array[Symbol]
+
+        ASSIGNMENT_NODES = [
+          :LocalVariableWriteNode,
+          :LocalVariableOperatorWriteNode,
+          :LocalVariableOrWriteNode,
+          :LocalVariableAndWriteNode
+        ].freeze #: Array[Symbol]
+
+        required_parser_option track_locations: true
+        recommended_parser_option render_nodes: true
+        experimental "Inlining renders is experimental. What a partial is inlined into and when it is left alone may change."
+
+        #: () -> bool
+        def self.inlines_renders?
+          true
+        end
+
+        #: (Herb::AST::DocumentNode) -> void
+        def visit_document_node(node)
+          @inlining = [] #: Array[String]
+          @inliner = nil
+          @assigned = assigned_locals(node)
+
+          super
+
+          inline(node)
+        end
+
+        #: () -> String
+        def inspect
+          "#<#{self.class.name}>"
+        end
+
+        private
+
+        #: (Herb::AST::Node) -> void
+        def inline(node)
+          ARRAY_PROPERTIES.each do |property|
+            next unless node.respond_to?(property) && node.send(property).is_a?(Array)
+
+            expand(node.send(property))
+          end
+
+          NODE_PROPERTIES.each do |property|
+            next unless node.respond_to?(property) && node.send(property)
+
+            inline(node.send(property))
+          end
+
+          nil
+        end
+
+        #: (Array[Herb::AST::Node]) -> void
+        def expand(nodes)
+          nodes.replace(nodes.flat_map { |child|
+            inline(child)
+
+            inlinable?(child) ? spliced(child) : [child]
+          })
+
+          nil
+        end
+
+        #: (Herb::AST::Node) -> bool
+        def inlinable?(node)
+          node.is_a?(Herb::AST::ERBRenderNode) && outputs?(node) && inliner.can_inline?(node, shadowed: shadowed(node))
+        end
+
+        #: (Herb::AST::Node) -> Array[String]
+        def shadowed(node)
+          @assigned - inliner.local_assignments(node).keys
+        end
+
+        #: (Herb::AST::Node, ?Array[String]) -> Array[String]
+        def assigned_locals(node, names = [])
+          assigned = assigned_name(node)
+
+          names << assigned if assigned
+
+          ARRAY_PROPERTIES.each do |property|
+            next unless node.respond_to?(property) && node.send(property).is_a?(Array)
+
+            node.send(property).each { |child| assigned_locals(child, names) if child.is_a?(Herb::AST::Node) }
+          end
+
+          NODE_PROPERTIES.each do |property|
+            next unless node.respond_to?(property) && node.send(property).is_a?(Herb::AST::Node)
+
+            assigned_locals(node.send(property), names)
+          end
+
+          names
+        end
+
+        #: (Herb::AST::Node) -> String?
+        def assigned_name(node)
+          return nil unless node.respond_to?(:parsed_prism_node)
+
+          prism = node.send(:parsed_prism_node)
+
+          return nil unless prism
+          return nil unless ASSIGNMENT_NODES.include?(prism.class.name.to_s.split("::").last&.to_sym)
+
+          prism.name.to_s
+        end
+
+        #: (Herb::AST::Node) -> bool
+        def outputs?(node)
+          node.send(:tag_opening)&.value.to_s.include?("=")
+        end
+
+        #: (Herb::AST::Node) -> Array[Herb::AST::Node]
+        def spliced(node)
+          path = inliner.resolve_path(node)
+
+          return [node] unless path
+          return [node] if @inlining.include?(path.to_s)
+
+          source = File.read(path)
+          partial = parse(source)
+
+          return [node] unless partial
+
+          @inlining.push(path.to_s)
+
+          body = partial.children
+
+          inline_nested(body)
+
+          @inlining.pop
+
+          [block(node, body, path, inliner.own_locals(source))]
+        end
+
+        #: (Pathname) -> String
+        def relative(path)
+          Herb::Visitor::Context.derive_relative_file_path(
+            Pathname.new(File.expand_path(path.to_s)),
+            Pathname.new(File.expand_path(context.project_path))
+          )
+        end
+
+        #: (Array[Herb::AST::Node]) -> void
+        def inline_nested(body)
+          body.each { |child| inline(child) }
+
+          expand(body)
+
+          nil
+        end
+
+        #: (String) -> Herb::AST::DocumentNode?
+        def parse(source)
+          configured = context.options[:parser_options] || {} #: Hash[Symbol, untyped]
+          options = Herb::Visitor.parser_options_for([self], configured)
+
+          result = ::Herb.parse(source, **options, track_whitespace: true)
+
+          return nil if result.errors.any?
+
+          result.value
+        end
+
+        #: (Herb::AST::Node, Array[String]) -> String
+        def opening_for(node, own)
+          locals = inliner.local_assignments(node)
+
+          return "->(#{locals.keys.join(", ")}#{block_locals(own, locals.keys)}) {" unless inliner.collection?(node)
+
+          item = inliner.collection_item_name(node)
+          taken = [item, "#{item}_counter"]
+          assigned = locals.map { |name, value| "#{name} = (#{value});" }.join(" ")
+
+          "((#{inliner.collection_expression(node)}) || []).each_with_index " \
+            "do |#{item}, #{item}_counter#{block_locals(own + locals.keys, taken)}| #{assigned}"
+        end
+
+        #: (Array[String], Array[String]) -> String
+        def block_locals(names, taken)
+          scoped = names.uniq - taken
+
+          return "" if scoped.empty?
+
+          "; #{scoped.join(", ")}"
+        end
+
+        #: (Herb::AST::Node) -> String
+        def closing_for_code(node)
+          return "end" if inliner.collection?(node)
+
+          "}.call(#{inliner.local_assignments(node).values.map { |value| "(#{value})" }.join(", ")})"
+        end
+
+        #: (Herb::AST::Node, Array[Herb::AST::Node], Pathname, Array[String]) -> Herb::AST::ERBBlockNode
+        def block(node, body, path, own)
+          Herb::AST::ERBBlockNode.build(
+            tag_opening: Herb::Token.from("TOKEN_ERB_START", "<%"),
+            content: Herb::Token.from("TOKEN_ERB_CONTENT", " #{opening_for(node, own)} "),
+            tag_closing: Herb::Token.from("TOKEN_ERB_END", "%>"),
+            body: body,
+            end_node: closing_for(node),
+            location: node.location
+          ).tap { |block| origin.authored(block, relative(path), from: node) }
+        end
+
+        #: (Herb::AST::Node) -> Herb::AST::ERBEndNode
+        def closing_for(node)
+          Herb::AST::ERBEndNode.build(
+            tag_opening: Herb::Token.from("TOKEN_ERB_START", "<%"),
+            content: Herb::Token.from("TOKEN_ERB_CONTENT", " #{closing_for_code(node)} "),
+            tag_closing: Herb::Token.from("TOKEN_ERB_END", "%>"),
+            location: node.location
+          )
+        end
+
+        #: () -> untyped
+        def inliner
+          @inliner ||= begin
+            require_relative "inliner"
+
+            Inliner.new(project_path: context.project_path, filename: context.relative_file_path, resolver: context.resolver)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/herb/engine/runtime/entry.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+# typed: true
+
+module Herb
+  class Engine
+    module Runtime
+      # What was observed at one position in one template while it rendered.
+      #
+      # A collector puts things here by name, so several can share a position without knowing about
+      # each other: a query count and a render time sit side by side under different keys.
+      #
+      class Entry
+        attr_reader :template #: String?
+        attr_reader :line #: Integer
+        attr_reader :column #: Integer
+        attr_reader :end_line #: Integer?
+        attr_reader :end_column #: Integer?
+        attr_reader :observations #: Hash[Symbol, Array[untyped]]
+
+        #: (String?, Integer, Integer, ?Integer?, ?Integer?) -> void
+        def initialize(template, line, column, end_line = nil, end_column = nil)
+          @template = template
+          @line = line
+          @column = column
+          @end_line = end_line
+          @end_column = end_column
+          @observations = {} #: Hash[Symbol, Array[untyped]]
+        end
+
+        #: (Symbol, untyped) -> void
+        def observe(key, value)
+          (observations[key] ||= []) << value
+
+          nil
+        end
+
+        #: (Symbol) -> Array[untyped]
+        def [](key)
+          observations.fetch(key, [])
+        end
+
+        #: () -> bool
+        def empty?
+          observations.empty?
+        end
+
+        #: () -> Herb::Location
+        def location
+          Herb::Location.from(line, column, end_line || line, end_column || column)
+        end
+
+        #: () -> String
+        def to_s
+          summary = observations.map { |key, values| "#{values.size} #{key}" }.join(", ")
+          position = location.start.to_one_based
+
+          "#{template || "(unknown)"}:#{position[:line]}:#{position[:column]} (#{summary})"
+        end
+      end
+    end
+  end
+end