herb 0.10.1 → 0.11.1

data/lib/herb/project.rb

@@ -8,11 +8,16 @@ require "pathname"
 require "English"
 require "stringio"
 
+require_relative "colors"
+require_relative "configuration"
+
 module Herb
   class Project
     include Colors
 
-    attr_accessor :project_path, :output_file, :no_log_file, :no_timing, :silent, :verbose, :isolate, :validate_ruby, :file_paths, :arena_stats, :leak_check
+    attr_accessor :project_path, :output_file, :no_log_file, :no_timing, :silent, :verbose, :isolate, :validate_ruby, :file_paths, :arena_stats, :leak_check, :file_timeout
+
+    DEFAULT_FILE_TIMEOUT = 1 # seconds per file for parse + compile
 
     # Known error types that indicate issues in the user's template, not bugs in the parser.
     TEMPLATE_ERRORS = [
@@ -26,6 +31,7 @@ module Herb
       "MissingERBEndTagError",
       "ERBMultipleBlocksInTagError",
       "ERBCaseWithConditionsError",
+      "ERBCaseInlinePatternMatchError",
       "ConditionalElementMultipleTagsError",
       "ConditionalElementConditionMismatchError",
       "InvalidCommentClosingTagError",
@@ -35,8 +41,17 @@ module Herb
       "UnclosedQuoteError",
       "MissingAttributeValueError",
       "UnclosedERBTagError",
+      "MalformedERBClosingTagError",
       "StrayERBClosingTagError",
-      "NestedERBTagError"
+      "NestedERBTagError",
+      "MissingWhitespaceBetweenAttributesError",
+      "UnexpectedCharacterInAttributeNameError",
+      "UnexpectedCharacterInUnquotedAttributeValueError",
+      "UnexpectedEqualsSignBeforeAttributeNameError",
+      "UnexpectedSolidusInTagError",
+      "EndTagWithTrailingSolidusError",
+      "UnclosedCommentError",
+      "NestedCommentError"
     ].freeze
 
     ISSUE_TYPES = [
@@ -126,6 +141,7 @@ module Herb
 
       date = Time.now.strftime("%Y-%m-%d_%H-%M-%S")
       @output_file = output_file || "#{date}_erb_parsing_result_#{@project_path.basename}.log"
+      @file_timeout = DEFAULT_FILE_TIMEOUT
     end
 
     def configuration
@@ -151,16 +167,7 @@ module Herb
     private
 
     def find_files
-      included = include_patterns.flat_map do |pattern|
-        Dir[File.join(@project_path, pattern)]
-      end.uniq
-
-      return included if exclude_patterns.empty?
-
-      included.reject do |file|
-        relative_path = file.sub("#{@project_path}/", "")
-        exclude_patterns.any? { |pattern| File.fnmatch?(pattern, relative_path, File::FNM_PATHNAME) }
-      end
+      configuration.find_files(@project_path)
     end
 
     public
@@ -325,13 +332,13 @@ module Herb
       return unless issue_type[:key] == :invalid_ruby && file_content
 
       begin
-        engine = Herb::Engine.new(file_content, filename: file_path, escape: true, validation_mode: :none)
+        engine = Herb::Engine.new(file_content, filename: file_path, escape: true, visitors: [])
         puts ""
         puts "**Compiled Ruby:**"
         puts "```ruby"
         puts engine.src
         puts "```"
-      rescue StandardError
+      rescue Herb::Engine::CompilationError, StandardError
         # Skip if compilation fails entirely
       end
     end
@@ -354,7 +361,7 @@ module Herb
         result[:leak_check] = capture_leak_check(file_content)
       end
 
-      Timeout.timeout(1) do
+      Timeout.timeout(file_timeout) do
         parse_result = Herb.parse(file_content)
 
         if parse_result.failed?
@@ -369,7 +376,7 @@ module Herb
       result
     rescue Timeout::Error
       result.merge(status: :timeout, file_content: file_content,
-                   log: "⏱️ Parsing #{file_path} timed out after 1 second")
+                   log: "⏱️ Parsing #{file_path} timed out after #{file_timeout} #{pluralize(file_timeout, "second")}")
     rescue StandardError => e
       file_content ||= begin
         File.read(file_path)
@@ -388,7 +395,7 @@ module Herb
       stdout_file = Tempfile.new("stdout")
       stderr_file = Tempfile.new("stderr")
 
-      Timeout.timeout(1) do
+      Timeout.timeout(file_timeout) do
         pid = Process.fork do
           $stdout.reopen(stdout_file.path, "w")
           $stderr.reopen(stderr_file.path, "w")
@@ -431,7 +438,7 @@ module Herb
         nil
       end
 
-      { file_path: file_path, status: :timeout, file_content: file_content, log: "⏱️ Parsing #{file_path} timed out after 1 second" }
+      { file_path: file_path, status: :timeout, file_content: file_content, log: "⏱️ Parsing #{file_path} timed out after #{file_timeout} #{pluralize(file_timeout, "second")}" }
     rescue StandardError => e
       file_content ||= begin
         File.read(file_path)
@@ -482,7 +489,16 @@ module Herb
     end
 
     def compile_file(file_path, file_content)
-      Herb::Engine.new(file_content, filename: file_path, escape: true, validate_ruby: validate_ruby)
+      require_relative "engine"
+      require_relative "engine/validators"
+
+      Herb::Engine.new(
+        file_content,
+        filename: file_path,
+        escape: true,
+        validate_ruby: validate_ruby,
+        visitors: Herb::Engine::Validators.all
+      )
 
       { status: :successful, log: "✅ Compiled #{file_path} successfully" }
     rescue Herb::Engine::GeneratorTemplateError => e
@@ -498,13 +514,13 @@ module Herb
 
       # Retry without validators
       begin
-        Herb::Engine.new(file_content, filename: file_path, escape: true, validation_mode: :none, validate_ruby: validate_ruby)
+        Herb::Engine.new(file_content, filename: file_path, escape: true, visitors: [], validate_ruby: validate_ruby)
         error_name = e.is_a?(Herb::Engine::SecurityError) ? "SecurityError" : "ValidationError"
         return { status: :validation_error, file_content: file_content,
                  compilation_error: compilation_error,
                  diagnostics: [{ name: error_name, message: e.message }],
                  log: "⚠️ Compilation failed for #{file_path} (validation error)" }
-      rescue StandardError
+      rescue Herb::Engine::CompilationError, StandardError
         # Not a validator-caused error, continue with other checks
       end
 
@@ -515,7 +531,7 @@ module Herb
                  compilation_error: compilation_error,
                  diagnostics: [{ name: "CompilationError", message: "#{e.message} (strict mode)" }],
                  log: "🔒 Compilation failed for #{file_path} (strict mode error)" }
-      rescue StandardError
+      rescue Herb::Engine::CompilationError, StandardError
         # Fall through
       end
 
@@ -903,7 +919,7 @@ module Herb
     def ensure_parallel!
       return if defined?(Parallel)
 
-      Herb.ensure_installed { gem "parallel" } # steep:ignore
+      Herb.ensure_installed("parallel")
     end
 
     def separator

data/lib/herb/range.rb

@@ -23,6 +23,11 @@ module Herb
       new(from, to)
     end
 
+    #: () -> Range
+    def self.zero
+      new(0, 0)
+    end
+
     #: () -> serialized_range
     def to_a
       [from, to] #: Herb::serialized_range

data/lib/herb/result.rb

@@ -16,12 +16,12 @@ module Herb
 
     #: () -> bool
     def success?
-      false
+      errors.empty?
     end
 
     #: () -> bool
     def failed?
-      true
+      errors.any?
     end
   end
 end

data/lib/herb/ruby_program.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+# typed: true
+
+require "prism"
+
+module Herb
+  # Answers what Ruby a span of a template holds, out of the one Prism program the parser built
+  # for the whole document. `extract_ruby` blanks the HTML in place, so a Prism byte offset and a
+  # template byte offset are the same number, which is what lets a `Herb::Location` address Ruby.
+  #
+  #     program = Herb::RubyProgram.for(document)
+  #     program.resolve(node.content.location)&.nodes #=> [#<Prism::CallNode>]
+  #
+  # The document has to have been parsed with the `prism_program` parser option, and `for` answers
+  # `nil` when it was not. `Herb::Locate` is the same question asked of the HTML tree, and
+  # `Herb::Analysis::RubyLocalsIndex::OffsetTable` walks the other way, from a Prism offset back
+  # to a `Herb::Location`.
+  #
+  class RubyProgram
+    Resolved = Data.define(
+      :nodes,  #: Array[untyped]
+      :offset, #: Integer
+      :source  #: String
+    )
+
+    #: (untyped) -> RubyProgram?
+    def self.for(document)
+      return nil unless document.respond_to?(:source) && document.respond_to?(:deserialized_prism_node)
+
+      source = document.source
+      program = document.deserialized_prism_node
+
+      return nil unless source.is_a?(String) && program
+
+      new(source, program)
+    end
+
+    #: (String, untyped) -> void
+    def initialize(source, program)
+      @source = source
+      @lines = source.lines
+      @starts = [0] #: Array[Integer]
+
+      @lines.each { |line| @starts << (@starts.fetch(-1) + line.bytesize) }
+
+      @nodes = flatten(program)
+    end
+
+    #: (Herb::Location?) -> Resolved?
+    def resolve(location)
+      return nil unless location
+
+      from = offset_of(location.start)
+      to = offset_of(location.end)
+
+      return nil unless from && to && from < to
+
+      Resolved.new(nodes: enclosed(from, to), offset: from, source: @source.byteslice(from, to - from).to_s)
+    end
+
+    private
+
+    #: (untyped) -> Array[untyped]
+    def flatten(program)
+      found = [] #: Array[untyped]
+      queue = [program] #: Array[untyped]
+
+      until queue.empty?
+        node = queue.shift
+
+        found << node unless node.is_a?(Prism::StatementsNode) || node.is_a?(Prism::ProgramNode)
+        queue.concat(node.compact_child_nodes)
+      end
+
+      found.sort_by { |node| [node.location.start_offset, -node.location.end_offset] }
+    end
+
+    #: (Integer, Integer) -> Array[untyped]
+    def enclosed(from, to)
+      found = [] #: Array[untyped]
+      index = @nodes.bsearch_index { |node| node.location.start_offset >= from } || @nodes.length
+      reach = from
+
+      while (node = @nodes[index])
+        location = node.location
+
+        break if location.start_offset >= to
+
+        if location.start_offset >= reach && location.end_offset <= to
+          found << node
+          reach = location.end_offset
+        end
+
+        index += 1
+      end
+
+      found
+    end
+
+    #: (untyped) -> Integer?
+    def offset_of(position)
+      start = @starts[position.line - 1]
+      line = @lines[position.line - 1]
+
+      return nil unless start && line
+
+      start + line[0, position.column].to_s.bytesize
+    end
+  end
+end

data/lib/herb/source_path.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+# typed: true
+
+require "pathname"
+
+module Herb
+  # A path, and optionally where in it something is.
+  #
+  #     Herb::SourcePath.new("app/views/posts/_card.html.erb", line: 8, column: 2).to_s
+  #     #=> "app/views/posts/_card.html.erb:8:3"
+  #
+  #     Herb::SourcePath.parse("app/views/posts/_card.html.erb:8:3").position
+  #     #=> #<Herb::Position (8:2)>
+  #
+  # `path:line:column` is what an editor linkifies, what a terminal makes clickable, and what
+  # `data-herb-source` holds. Herb wrote it by hand in five places and they did not agree, because
+  # the column the parser reports is 0-based and the column an editor shows is 1-based. Everything
+  # that reads or writes one of these strings goes through here so that conversion happens once.
+  #
+  # A part that is not known is left out, so a path on its own stays a path:
+  #
+  #     Herb::SourcePath.new("index.html.erb").to_s        #=> "index.html.erb"
+  #     Herb::SourcePath.new("index.html.erb", line: 8).to_s #=> "index.html.erb:8"
+  #
+  # A scheme is optional too, and turns the same reference into something an editor opens:
+  #
+  #     Herb::SourcePath.new(path, line: 8, column: 2, scheme: "vscode").to_s
+  #     #=> "vscode://file/app/views/posts/_card.html.erb:8:3"
+  #
+  # A project turns the same reference into either form. A stamp and a diagnostic show the relative
+  # path, and an editor scheme needs the absolute one:
+  #
+  #     reference = Herb::SourcePath.at(path, position, project_path: Rails.root)
+  #
+  #     reference.relative.to_s                    #=> "app/views/posts/_card.html.erb:8:3"
+  #     reference.absolute.with_scheme("vscode").to_s
+  #     #=> "vscode://file/Users/marco/blog/app/views/posts/_card.html.erb:8:3"
+  #
+  # Neither is guessed. `to_s` writes the path it was handed, so a caller says which one it wants.
+  # An editor scheme names a file from the root, so call `absolute` before putting one on.
+  #
+  # Lines count from one and columns count from zero, which is what the parser reports and what
+  # `Herb::Position` carries. Only `to_s` and `parse` speak the 1-based column an editor shows.
+  #
+  class SourcePath
+    PATTERN = %r{
+      \A
+      (?:(?<scheme>[a-z][a-z0-9+.-]*)://file(?=/))?
+      (?<path>.*?)
+      (?::(?<line>\d+)(?::(?<column>\d+))?)?
+      \z
+    }ix #: Regexp
+
+    FILE_SCHEME_SEPARATOR = "://file" #: String
+
+    attr_reader :path #: Pathname
+    attr_reader :project_path #: Pathname?
+    attr_reader :line #: Integer?
+    attr_reader :column #: Integer?
+    attr_reader :scheme #: String?
+
+    #: ((String | Pathname), ?project_path: (String | Pathname)?, ?line: Integer?, ?column: Integer?, ?scheme: String?) -> void
+    def initialize(path, project_path: nil, line: nil, column: nil, scheme: nil)
+      @path = self.class.coerce(path) #: Pathname
+      @project_path = project_path.nil? ? nil : self.class.coerce(project_path)
+      @line = line
+      @column = line.nil? ? nil : column
+      @scheme = scheme
+
+      freeze
+    end
+
+    #: ((String | Pathname), Herb::Position?, ?project_path: (String | Pathname)?, ?scheme: String?) -> SourcePath
+    def self.at(path, position, project_path: nil, scheme: nil)
+      return new(path, project_path: project_path, scheme: scheme) unless position
+
+      new(path, project_path: project_path, line: position.line, column: position.column, scheme: scheme)
+    end
+
+    #: ((String | Pathname)) -> Pathname
+    def self.coerce(path)
+      path.is_a?(Pathname) ? path : Pathname.new(path)
+    end
+
+    #: (String, ?project_path: (String | Pathname)?) -> SourcePath?
+    def self.parse(string, project_path: nil)
+      match = PATTERN.match(string.to_s)
+
+      return nil unless match
+
+      path = match[:path]
+
+      return nil if path.nil? || path.empty?
+
+      new(
+        path,
+        project_path: project_path,
+        line: match[:line]&.to_i,
+        column: reported_column(match[:column]),
+        scheme: match[:scheme]
+      )
+    end
+
+    #: (String?) -> Integer?
+    def self.reported_column(column)
+      return nil unless column
+
+      [column.to_i - 1, 0].max
+    end
+
+    #: () -> Herb::Position?
+    def position
+      line = @line
+
+      return nil unless line
+
+      Position.new(line, column || 0)
+    end
+
+    #: () -> bool
+    def position?
+      !@line.nil?
+    end
+
+    #: ((String | Pathname)) -> SourcePath
+    def with_path(path)
+      self.class.new(path, project_path: project_path, line: line, column: column, scheme: scheme)
+    end
+
+    #: ((String | Pathname)?) -> SourcePath
+    def with_project_path(project_path)
+      self.class.new(path, project_path: project_path, line: line, column: column, scheme: scheme)
+    end
+
+    #: () -> bool
+    def absolute?
+      path.absolute?
+    end
+
+    #: () -> Pathname
+    def absolute_path
+      return path if path.absolute?
+
+      project = project_path
+
+      return path unless project
+
+      project + path
+    end
+
+    #: () -> Pathname
+    def relative_path
+      project = project_path
+
+      return path unless project
+
+      absolute_path.relative_path_from(project)
+    rescue ArgumentError
+      path
+    end
+
+    #: () -> SourcePath
+    def absolute
+      with_path(absolute_path)
+    end
+
+    #: () -> SourcePath
+    def relative
+      with_path(relative_path)
+    end
+
+    #: (Herb::Position?) -> SourcePath
+    def with_position(position)
+      self.class.at(path, position, project_path: project_path, scheme: scheme)
+    end
+
+    #: (String?) -> SourcePath
+    def with_scheme(scheme)
+      self.class.new(path, project_path: project_path, line: line, column: column, scheme: scheme)
+    end
+
+    #: () -> String
+    def to_s
+      "#{prefix}#{written_path}#{suffix}"
+    end
+
+    #: () -> Pathname
+    def to_pathname
+      path
+    end
+
+    #: (untyped) -> bool
+    def ==(other)
+      other.is_a?(SourcePath) && to_s == other.to_s
+    end
+
+    alias eql? ==
+
+    #: () -> Integer
+    def hash
+      to_s.hash
+    end
+
+    #: () -> String
+    def inspect
+      %(#<Herb::SourcePath #{self}>)
+    end
+
+    private
+
+    #: () -> String
+    def prefix
+      scheme ? "#{scheme}#{FILE_SCHEME_SEPARATOR}" : ""
+    end
+
+    #: () -> String
+    def written_path
+      written = path.to_s
+
+      return written unless scheme
+
+      written.start_with?("/") ? written : "/#{written}"
+    end
+
+    #: () -> String
+    def suffix
+      return "" unless line
+
+      column ? ":#{line}:#{column + 1}" : ":#{line}"
+    end
+  end
+end

data/lib/herb/token.rb

@@ -16,6 +16,11 @@ module Herb
     attr_reader :location #: Location
     attr_reader :type #: String
 
+    #: (String | Symbol, String, ?location: Location, ?range: Range) -> Token
+    def self.from(type, value, location: Location.zero, range: Range.zero)
+      new(value.dup, range, location, type.to_s)
+    end
+
     #: (String, Range, Location, String) -> void
     def initialize(value, range, location, type)
       @value = value
@@ -41,7 +46,9 @@ module Herb
 
     #: () -> String
     def tree_inspect
-      "#{green("\"#{value.force_encoding("utf-8")}\"")} #{dimmed("(location: #{location.tree_inspect})")}"
+      location_inspect = location ? location.tree_inspect : "∅"
+
+      "#{green("\"#{value.force_encoding("utf-8")}\"")} #{dimmed("(location: #{location_inspect})")}"
     end
 
     #: () -> String
@@ -55,17 +62,21 @@ module Herb
 
     #: () -> String
     def colorize_range
+      return "∅" unless range
+
       white("[") + cyan(range.from.to_s) + white(", ") + cyan(range.to.to_s) + white("]")
     end
 
-    #: (Position) -> String
+    #: (Position?) -> String
     def colorize_position(position)
+      return "∅" unless position
+
       white("(") + cyan(position.line.to_s) + white(":") + cyan(position.column.to_s) + white(")")
     end
 
     #: () -> String
     def inspect
-      "#{white("#<")}#{bold(yellow("Herb::Token"))} #{white("type=")}#{bright_magenta("\"#{type}\"")} #{white("value=")}#{green(value_inspect)} #{white("range=")}#{colorize_range} #{white("start=")}#{colorize_position(location.start)} #{white("end=")}#{colorize_position(location.end)}#{white(">")}"
+      "#{white("#<")}#{bold(yellow("Herb::Token"))} #{white("type=")}#{bright_magenta("\"#{type}\"")} #{white("value=")}#{green(value_inspect)} #{white("range=")}#{colorize_range} #{white("start=")}#{colorize_position(location&.start)} #{white("end=")}#{colorize_position(location&.end)}#{white(">")}"
     end
   end
 end

data/lib/herb/version.rb

@@ -2,5 +2,5 @@
 # typed: true
 
 module Herb
-  VERSION = "0.10.1"
+  VERSION = "0.11.1"
 end

data/lib/herb/visitor/context/origin.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+# typed: false
+
+module Herb
+  class Visitor
+    class Context
+      # Which nodes hold content that was written somewhere other than the file being compiled.
+      #
+      # A visitor that splices one template into another leaves a tree that no longer answers "where
+      # was this written" by itself, and the node it wraps the splice in is one nobody wrote at all.
+      # Both matter to anything reporting a position back to a developer, and neither is visible from
+      # the node alone.
+      #
+      #     origin.authored(wrapper, "app/views/posts/_card.html.erb", from: render_node)
+      #
+      #     origin.of(wrapper).file #=> "app/views/posts/_card.html.erb"
+      #     origin.of(wrapper).from #=> the `render` tag it replaced
+      #
+      # What is recorded is the node holding the content rather than each node of it, because what
+      # was moved is the partial and not each node of the partial. Recording the nodes themselves
+      # would leave nothing saying which of them belong together, and a partial spliced into the
+      # middle of another would split the one around it in two.
+      #
+      # Only nodes that hold content from elsewhere are recorded, so an untouched tree costs nothing.
+      # This lives on the compile rather than on a visitor, because the visitor that needs the answer
+      # is never the one that knows it.
+      #
+      class Origin
+        Entry = Data.define(:file, :from)
+
+        #: () -> void
+        def initialize
+          entries = {} #: Hash[Herb::AST::Node, Entry]
+
+          @entries = entries.compare_by_identity
+        end
+
+        #: (Herb::AST::Node, String, ?from: Herb::AST::Node?) -> void
+        def authored(node, file, from: nil)
+          @entries[node] = Entry.new(file, from) unless @entries.key?(node)
+
+          nil
+        end
+
+        #: (Herb::AST::Node) -> Entry?
+        def of(node)
+          @entries[node]
+        end
+      end
+    end
+  end
+end