klenod-build 0.0.1

data/lib/klenod/build/ruby_import_rewriter.rb

@@ -0,0 +1,436 @@
+# frozen_string_literal: true
+
+require "ripper"
+require "syntax_tree"
+
+require_relative "dependency"
+require_relative "errors"
+require_relative "watched_pattern"
+
+module Klenod
+  module Build
+    class RubyImportRewriter
+      ImportCall =
+        Data.define(:specifier, :location, :dynamic, :method_name, :eager_override) do
+          def eager
+            eager_override.nil? ? RubyImportRewriter::IMPORT_METHODS.fetch(method_name).fetch(:eager) : eager_override
+          end
+
+          def replacement
+            RubyImportRewriter::IMPORT_METHODS.fetch(method_name).fetch(:replacement)
+          end
+        end
+      Result = Data.define(:code, :dependencies, :watched_patterns)
+
+      IMPORT_METHODS = {
+        "import" => {
+          replacement: "__klenod_import__",
+          eager: true
+        },
+        "lazy_import" => {
+          replacement: "__klenod_lazy_import__",
+          eager: false
+        },
+        "import_glob" => {
+          replacement: nil,
+          eager: true
+        }
+      }.freeze
+      IMPORT_SOURCE_PATTERN = /(?<![A-Za-z0-9_])(?:import|lazy_import|import_glob)\s*(?:\(|["'])/
+      FAST_LITERAL_IMPORT_PATTERN = /(?<![A-Za-z0-9_])(import|lazy_import)\s*\(\s*("(?:\\.|[^"\\#])*")\s*\)/
+
+      def initialize(module_id:, kind:, source_dir: nil, profiler: nil, dependency_id_offset: 0, source_line_offset: 0, source_column_offset: 0)
+        @module_id = module_id
+        @kind = kind
+        @source_dir = source_dir && Pathname.new(source_dir).expand_path
+        @profiler = profiler
+        @dependency_id_offset = dependency_id_offset
+        @source_line_offset = source_line_offset
+        @source_column_offset = source_column_offset
+      end
+
+      def rewrite(code)
+        return Result.new(code, [], []) unless import_source?(code)
+        fast_result = rewrite_literal_import_calls(code)
+        return fast_result if fast_result
+
+        ast = measure(:ruby_import_parse) { SyntaxTree.parse(code) }
+        calls = measure(:ruby_import_scan) { import_calls(ast) }
+        expanded = expand_import_calls(calls)
+
+        rewritten = measure(:ruby_import_rewrite_source) { rewrite_import_calls(code, expanded) }
+        Result.new(
+          rewritten,
+          expanded.flat_map(&:dependencies),
+          expanded.flat_map(&:watched_patterns)
+        )
+      end
+
+      private
+
+      ExpandedImport = Data.define(:call, :dependencies, :replacement_source, :watched_patterns)
+
+      def import_source?(code)
+        code.match?(IMPORT_SOURCE_PATTERN)
+      end
+
+      def rewrite_literal_import_calls(code)
+        matches = []
+        code.to_enum(:scan, IMPORT_SOURCE_PATTERN).each do
+          start = Regexp.last_match.begin(0)
+          match = FAST_LITERAL_IMPORT_PATTERN.match(code, start)
+          return nil unless match && match.begin(0) == start
+
+          matches << match
+        end
+        return nil if matches.empty?
+        return nil unless bare_import_tokens?(code, matches)
+
+        dependencies =
+          matches.each_with_index.map do |match, index|
+            specifier =
+              begin
+                match[2].undump
+              rescue RuntimeError
+                return nil
+              end
+
+            Dependency
+              .create(
+                specifier: specifier,
+                importer_id: @module_id,
+                kind: @kind,
+                loc: source_location_for_offset(code, match.begin(0))
+              )
+              .with(eager: IMPORT_METHODS.fetch(match[1]).fetch(:eager))
+              .with(id: "#{@module_id}:dependency:#{@dependency_id_offset + index}")
+          end
+
+        rewritten =
+          matches
+            .zip(dependencies)
+            .reverse_each
+            .each_with_object(code.dup) do |(match, dependency), source|
+              replacement = IMPORT_METHODS.fetch(match[1]).fetch(:replacement)
+              source[match.begin(0)...match.end(0)] = "#{replacement}(#{dependency.id.inspect})"
+            end
+
+        Result.new(rewritten, dependencies, [])
+      end
+
+      def bare_import_tokens?(code, matches)
+        match_starts = matches.to_h { |match| [match.begin(1), match[1]] }
+        return true if match_starts.empty?
+
+        line_offsets = line_offsets_for(code)
+        previous_significant_token = nil
+        matched = {}
+
+        Ripper.lex(code).each do |(line, column), type, token, _state|
+          offset = line_offsets.fetch(line - 1) + column
+          expected = match_starts[offset]
+          if expected
+            return false unless type == :on_ident && token == expected
+            return false if receiver_token?(previous_significant_token)
+
+            matched[offset] = true
+          end
+
+          previous_significant_token = [type, token] unless insignificant_token?(type)
+        end
+
+        matched.length == match_starts.length
+      end
+
+      def receiver_token?(token)
+        token == [:on_period, "."] || token == [:on_op, "::"] || token == [:on_op, "&."]
+      end
+
+      def line_offsets_for(code)
+        offsets = [0]
+        code.each_line(chomp: false) { |line| offsets << offsets.last + line.length }
+        offsets
+      end
+
+      def insignificant_token?(type)
+        type == :on_sp || type == :on_ignored_nl || type == :on_nl || type == :on_comment
+      end
+
+      def measure(name, &block)
+        return yield unless @profiler
+
+        @profiler.measure(name, module_id: @module_id.to_s, kind: @kind, &block)
+      end
+
+      def import_calls(node)
+        calls = []
+        walk(node) do |child|
+          if child.instance_of?(::SyntaxTree::CallNode)
+            next unless child.instance_variable_get(:@receiver).nil?
+            next unless IMPORT_METHODS.key?(child.instance_variable_get(:@message)&.value)
+
+            calls << build_import_call(child)
+          elsif child.instance_of?(::SyntaxTree::Command)
+            next unless IMPORT_METHODS.key?(child.instance_variable_get(:@message)&.value)
+
+            calls << build_import_command(child)
+          end
+        end
+        calls
+      end
+
+      def build_import_call(node)
+        arguments = node.instance_variable_get(:@arguments)
+        parts = arguments&.instance_variable_get(:@arguments)&.instance_variable_get(:@parts) || []
+        build_import_from_parts(node, parts)
+      end
+
+      def build_import_command(node)
+        arguments = node.instance_variable_get(:@arguments)
+        parts = arguments&.instance_variable_get(:@parts) || []
+        build_import_from_parts(node, parts)
+      end
+
+      def build_import_from_parts(node, parts)
+        return build_import_glob_from_parts(node, parts) if node.instance_variable_get(:@message).value == "import_glob"
+
+        first = unwrap_paren(parts.first)
+        string_parts = first&.instance_variable_get(:@parts)
+        literal =
+          (parts.length == 1) &&
+          first&.class&.name == "SyntaxTree::StringLiteral" &&
+          string_parts&.length == 1 &&
+          string_parts.first.instance_of?(::SyntaxTree::TStringContent)
+
+        ImportCall.new(
+          literal ? string_parts.first.value : nil,
+          node.instance_variable_get(:@location),
+          !literal,
+          node.instance_variable_get(:@message).value,
+          nil
+        )
+      end
+
+      def build_import_glob_from_parts(node, parts)
+        first = unwrap_paren(parts.first)
+        string_parts = first&.instance_variable_get(:@parts)
+        literal =
+          first&.class&.name == "SyntaxTree::StringLiteral" &&
+          string_parts&.length == 1 &&
+          string_parts.first.instance_of?(::SyntaxTree::TStringContent)
+
+        eager = true
+        valid_options = true
+        if parts.length == 2
+          eager = eager_option_value(parts.fetch(1))
+          valid_options = !eager.nil?
+        elsif parts.length != 1
+          valid_options = false
+        end
+
+        ImportCall.new(
+          (literal && valid_options) ? string_parts.first.value : nil,
+          node.instance_variable_get(:@location),
+          !literal || !valid_options,
+          node.instance_variable_get(:@message).value,
+          eager
+        )
+      end
+
+      def eager_option_value(node)
+        return unless node.instance_of?(::SyntaxTree::BareAssocHash)
+
+        assocs = node.instance_variable_get(:@assocs)
+        return unless assocs.length == 1
+
+        assoc = assocs.fetch(0)
+        key = assoc.instance_variable_get(:@key)
+        return unless key.instance_of?(::SyntaxTree::Label) && key.value == "eager:"
+
+        bool_value(assoc.instance_variable_get(:@value))
+      end
+
+      def bool_value(node)
+        return unless node.instance_of?(::SyntaxTree::VarRef)
+
+        value = node.instance_variable_get(:@value)
+        return true if value.respond_to?(:value) && value.value == "true"
+        return false if value.respond_to?(:value) && value.value == "false"
+
+        nil
+      end
+
+      def unwrap_paren(node)
+        return node unless node.instance_of?(::SyntaxTree::Paren)
+
+        statements = node.instance_variable_get(:@contents)
+        body = statements&.instance_variable_get(:@body) || []
+        (body.length == 1) ? body.first : node
+      end
+
+      def expand_import_calls(calls)
+        dependency_index = @dependency_id_offset
+
+        calls.map do |call|
+          raise_dynamic_import!(call) if call.dynamic
+
+          if call.method_name == "import_glob"
+            dependencies, watched_patterns = expand_glob_import(call, dependency_index)
+            dependency_index += dependencies.length
+            ExpandedImport.new(call, dependencies, glob_replacement_source(dependencies, eager: call.eager), watched_patterns)
+          else
+            dependency = build_dependency(call, dependency_index)
+            dependency_index += 1
+            ExpandedImport.new(call, [dependency], "#{call.replacement}(#{dependency.id.inspect})", [])
+          end
+        end
+      end
+
+      def raise_dynamic_import!(call)
+        expected = (call.method_name == "import_glob") ? "import_glob(\"...\")" : "import(\"...\")"
+        raise DynamicImportError, "Only literal #{expected} calls are supported in #{@module_id}"
+      end
+
+      def build_dependency(call, dependency_index)
+        Dependency
+          .create(
+            specifier: call.specifier,
+            importer_id: @module_id,
+            kind: @kind,
+            loc: source_location(call.location)
+          )
+          .with(eager: call.eager)
+          .with(id: "#{@module_id}:dependency:#{dependency_index}")
+      end
+
+      def expand_glob_import(call, dependency_index)
+        source_dir = @source_dir || raise(DynamicImportError, "Cannot expand import_glob in #{@module_id} without a source directory")
+        path_pattern, query = call.specifier.split("?", 2)
+        absolute_pattern = absolute_pattern_for(path_pattern)
+        assert_pattern_inside_source_dir!(absolute_pattern)
+
+        matches =
+          Dir
+            .glob(absolute_pattern.to_s)
+            .select { |path| File.file?(path) }
+            .sort
+
+        dependencies =
+          matches.each_with_index.map do |absolute_path, index|
+            specifier = specifier_for_glob_match(path_pattern, absolute_path)
+            specifier = "#{specifier}?#{query}" if query
+
+            Dependency
+              .create(
+                specifier: specifier,
+                importer_id: @module_id,
+                kind: @kind,
+                loc: source_location(call.location)
+              )
+              .with(eager: call.eager)
+              .with(metadata: {glob_key: specifier.split("?", 2).first})
+              .with(id: "#{@module_id}:dependency:#{dependency_index + index}")
+          end
+
+        watched_patterns = [
+          WatchedPattern.new(
+            @module_id,
+            absolute_pattern.relative_path_from(source_dir).to_s,
+            :import_glob,
+            {specifier: call.specifier}
+          )
+        ]
+
+        [dependencies, watched_patterns]
+      end
+
+      def source_location(location)
+        SourceLocation.new(
+          @module_id.to_s,
+          location.start_line + @source_line_offset,
+          location.start_column + 1 + @source_column_offset
+        )
+      end
+
+      def source_location_for_offset(source, offset)
+        prefix = source[0...offset]
+        line = prefix.count("\n") + 1
+        column = prefix.length - (prefix.rindex("\n") || -1)
+        SourceLocation.new(@module_id.to_s, line + @source_line_offset, column + @source_column_offset)
+      end
+
+      def absolute_pattern_for(path_pattern)
+        source_dir = @source_dir
+
+        if path_pattern.start_with?("/")
+          source_dir.join(path_pattern.delete_prefix("/"))
+        elsif path_pattern.start_with?(".")
+          source_dir.join(@module_id.dirname, path_pattern)
+        else
+          source_dir.join(path_pattern)
+        end.expand_path
+      end
+
+      def assert_pattern_inside_source_dir!(absolute_pattern)
+        pattern = absolute_pattern.to_s
+        source = @source_dir.to_s
+        return if pattern == source || pattern.start_with?("#{source}/")
+
+        raise DynamicImportError, "Glob import escapes source_dir: #{pattern}"
+      end
+
+      def specifier_for_glob_match(path_pattern, absolute_path)
+        relative_source_path = Pathname.new(absolute_path).relative_path_from(@source_dir).to_s
+        return "/#{relative_source_path}" if path_pattern.start_with?("/")
+        return relative_source_path unless path_pattern.start_with?(".")
+
+        importer_dir = @source_dir.join(@module_id.dirname)
+        relative = Pathname.new(absolute_path).relative_path_from(importer_dir).to_s
+        relative.start_with?(".") ? relative : "./#{relative}"
+      end
+
+      def glob_replacement_source(dependencies, eager:)
+        import_method = eager ? "__klenod_import__" : "__klenod_lazy_import__"
+        entries =
+          dependencies.map do |dependency|
+            key = dependency.metadata.fetch(:glob_key)
+            "#{key.inspect} => #{import_method}(#{dependency.id.inspect})"
+          end
+
+        "{#{entries.join(", ")}}"
+      end
+
+      def rewrite_import_calls(code, expanded)
+        expanded
+          .reverse_each
+          .each_with_object(code.dup) do |import, rewritten|
+            call = import.call
+            next rewritten if call.dynamic
+
+            location = call.location
+            original = code[location.start_char...location.end_char]
+            unless original.match?(/\A#{Regexp.escape(call.method_name)}\s*\(/)
+              raise DynamicImportError, "Could not safely rewrite import at #{location.start_line}:#{location.start_column}"
+            end
+
+            rewritten[location.start_char...location.end_char] = import.replacement_source
+          end
+      end
+
+      def walk(value, &block)
+        yield value
+
+        case value
+        when Array
+          value.each { |item| walk(item, &block) }
+        else
+          return unless value.respond_to?(:instance_variables)
+
+          value.instance_variables.each do |ivar|
+            walk(value.instance_variable_get(ivar), &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/klenod/build/source_map/editor.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require_relative "map"
+
+module Klenod
+  module Build
+    module SourceMap
+      Edit = Data.define(:start_offset, :end_offset, :replacement) do
+        def self.replace(start_offset, end_offset, replacement)
+          new(start_offset, end_offset, replacement)
+        end
+
+        def self.delete(start_offset, end_offset)
+          new(start_offset, end_offset, "")
+        end
+      end
+
+      EditResult = Data.define(:code, :source_map)
+
+      class Editor
+        def initialize(code, source_map)
+          @code = code
+          @source_map = source_map
+          @line_starts = line_starts(code)
+        end
+
+        def apply(edits)
+          edits = edits.sort_by(&:start_offset)
+          validate_edits!(edits)
+          code = apply_code_edits(edits)
+          final_line_starts = line_starts(code)
+
+          segments =
+            @source_map.segments.filter_map do |segment|
+              offset = offset_for(segment.generated_line, segment.generated_column)
+              translated_offset = translate_offset(offset, edits)
+              next unless translated_offset
+
+              line, column = line_column_for(translated_offset, final_line_starts)
+              segment.with_generated(line, column)
+            end
+
+          EditResult.new(code, @source_map.with(segments: segments))
+        end
+
+        private
+
+        def validate_edits!(edits)
+          edits.each_cons(2) do |current, successor|
+            next if current.end_offset <= successor.start_offset
+
+            raise ArgumentError, "Source map edits must not overlap"
+          end
+        end
+
+        def apply_code_edits(edits)
+          result = +""
+          offset = 0
+
+          edits.each do |edit|
+            result << @code[offset...edit.start_offset]
+            result << edit.replacement
+            offset = edit.end_offset
+          end
+
+          result << @code[offset..]
+          result
+        end
+
+        def translate_offset(offset, edits)
+          delta = 0
+
+          edits.each do |edit|
+            return offset + delta if offset < edit.start_offset
+
+            if offset < edit.end_offset
+              return nil if edit.replacement.empty?
+              return edit.start_offset + delta if offset == edit.start_offset
+
+              return nil
+            end
+
+            delta += edit.replacement.length - (edit.end_offset - edit.start_offset)
+          end
+
+          offset + delta
+        end
+
+        def offset_for(line, column)
+          @line_starts.fetch(line) + column
+        end
+
+        def line_starts(code)
+          starts = [0]
+          code.each_char.with_index do |char, index|
+            starts << index + 1 if char == "\n"
+          end
+          starts
+        end
+
+        def line_column_for(offset, line_starts)
+          index = line_starts.bsearch_index { |line_start| line_start > offset }
+          line = index ? index - 1 : line_starts.length - 1
+
+          [line, offset - line_starts.fetch(line)]
+        end
+      end
+    end
+  end
+end

data/lib/klenod/build/source_map/map.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "vlq"
+
+module Klenod
+  module Build
+    module SourceMap
+      Segment = Data.define(:generated_line, :generated_column, :source_index, :original_line, :original_column, :name_index) do
+        def with_generated(line, column)
+          self.class.new(line, column, source_index, original_line, original_column, name_index)
+        end
+
+        def with_original(line:, column:)
+          self.class.new(generated_line, generated_column, source_index, line, column, name_index)
+        end
+
+        def mapped?
+          source_index && original_line && original_column
+        end
+      end
+
+      class Map
+        attr_reader :version, :source_root, :sources, :sources_content, :names, :segments
+
+        def self.parse(json)
+          hash = JSON.parse(json)
+          new(
+            version: hash.fetch("version"),
+            source_root: hash["sourceRoot"],
+            sources: hash.fetch("sources"),
+            sources_content: hash["sourcesContent"],
+            names: hash.fetch("names", []),
+            segments: decode_mappings(hash.fetch("mappings"))
+          )
+        end
+
+        def self.decode_mappings(mappings)
+          segments = []
+          previous_source_index = 0
+          previous_original_line = 0
+          previous_original_column = 0
+          previous_name_index = 0
+
+          mappings.split(";", -1).each_with_index do |line, generated_line|
+            previous_generated_column = 0
+            next if line.empty?
+
+            line.split(",").each do |encoded_segment|
+              values = VLQ.decode(encoded_segment)
+              previous_generated_column += values.fetch(0)
+
+              if values.length == 1
+                segments << Segment.new(generated_line, previous_generated_column, nil, nil, nil, nil)
+                next
+              end
+
+              previous_source_index += values.fetch(1)
+              previous_original_line += values.fetch(2)
+              previous_original_column += values.fetch(3)
+              name_index =
+                if values.length > 4
+                  previous_name_index += values.fetch(4)
+                end
+
+              segments << Segment.new(
+                generated_line,
+                previous_generated_column,
+                previous_source_index,
+                previous_original_line,
+                previous_original_column,
+                name_index
+              )
+            end
+          end
+
+          segments
+        end
+
+        def initialize(version:, source_root:, sources:, sources_content:, names:, segments:)
+          @version = version
+          @source_root = source_root
+          @sources = sources
+          @sources_content = sources_content
+          @names = names
+          @segments = segments
+        end
+
+        def to_json(*)
+          JSON.generate(to_h)
+        end
+
+        def to_h
+          {
+            "version" => version,
+            "sourceRoot" => source_root,
+            "mappings" => encode_mappings,
+            "sources" => sources,
+            "sourcesContent" => sources_content,
+            "names" => names
+          }.compact
+        end
+
+        def with(sources: @sources, sources_content: @sources_content, segments: @segments)
+          self.class.new(
+            version: version,
+            source_root: source_root,
+            sources: sources,
+            sources_content: sources_content,
+            names: names,
+            segments: segments
+          )
+        end
+
+        def map_original_lines(source_index:)
+          with(
+            segments: segments.map do |segment|
+              if segment.source_index == source_index
+                segment.with_original(line: yield(segment.original_line), column: segment.original_column)
+              else
+                segment
+              end
+            end
+          )
+        end
+
+        def encode_mappings
+          return "" if segments.empty?
+
+          grouped = segments.group_by(&:generated_line)
+          max_line = segments.map(&:generated_line).max
+          previous_source_index = 0
+          previous_original_line = 0
+          previous_original_column = 0
+          previous_name_index = 0
+
+          (0..max_line).map do |generated_line|
+            previous_generated_column = 0
+            (grouped[generated_line] || [])
+              .sort_by(&:generated_column)
+              .map do |segment|
+                values = [segment.generated_column - previous_generated_column]
+                previous_generated_column = segment.generated_column
+
+                if segment.mapped?
+                  values << segment.source_index - previous_source_index
+                  values << segment.original_line - previous_original_line
+                  values << segment.original_column - previous_original_column
+                  previous_source_index = segment.source_index
+                  previous_original_line = segment.original_line
+                  previous_original_column = segment.original_column
+
+                  if segment.name_index
+                    values << segment.name_index - previous_name_index
+                    previous_name_index = segment.name_index
+                  end
+                end
+
+                VLQ.encode(values)
+              end
+              .join(",")
+          end.join(";")
+        end
+      end
+    end
+  end
+end