dialecto-rails 0.1.0

data/lib/dialecto_rails/usages/scanner.rb

@@ -0,0 +1,241 @@
+require "prism"
+require_relative "../version"
+require_relative "../scan/catalogs"
+require_relative "catalog_keys"
+require_relative "ruby_visitor"
+require_relative "wrappers"
+require_relative "known_wrappers"
+require_relative "templates"
+
+module DialectoRails
+  module Usages
+    # The usage scan (spec-0023 §6): every place the code at one commit reads a translation key, in the shape
+    # spec-0022 §6 fixes (the `railsUsages` object). Reads committed blobs through `Scan::Git`, never the
+    # working tree.
+    class Scanner
+      VERSION = "1.0.0".freeze
+      MAX_BYTES = 1_000_000
+      MAX_NAMED_KEYS = 200
+      DEFAULT_EXCLUDES = %w[node_modules vendor tmp log coverage public/assets app/assets/builds].freeze
+      EXCLUDED_PREFIXES = %w[public/packs].freeze
+      # What a generator copies into a new app is text for another app: it holds ERB inside `.rb` and is never loaded.
+      GENERATOR_TEMPLATES = %r{(?:\A|/)lib/generators/(?:.+/)?templates/}
+      RUBY_EXTENSIONS = %w[.rb .rake .jbuilder .builder .arb].freeze
+      SCRIPT_EXTENSIONS = %w[.js .jsx .ts .tsx .mjs .cjs .coffee].freeze
+      I18N_JS_CONFIGS = %w[config/i18n.yml config/i18n-js.yml config/i18n.js].freeze
+      JS_STRING = /"((?:[^"\\\n]|\\.){1,200})"|'((?:[^'\\\n]|\\.){1,200})'|`([^`$\\\n]{1,200})`/
+
+      Source = Struct.new(:path, :lang, :bytes, keyword_init: true)
+
+      # The catalog keys read at the scanned commit, available after `call`.
+      attr_reader :catalog
+
+      def initialize(git:, sha:, config:)
+        @git = git
+        @sha = sha
+        @config = config
+        @unscanned = []
+        @catalog = CatalogKeys.new
+      end
+
+      def call
+        entries = @git.entries(@sha).reject { |entry| excluded?(entry.path) }
+        contents = {}
+        read = ->(entry) { contents[entry.path] ||= @git.blob(entry.blob) }
+        catalog_entries = entries.select { |entry| catalogs.match?(entry.path) }
+        catalog_entries.each { |entry| @catalog.add(read.call(entry)) }
+
+        sources = []
+        entries.each do |entry|
+          next if catalogs.match?(entry.path)
+
+          classify(entry, sources, read)
+        end
+        contents.clear
+
+        locked_gems = gems(entries, read)
+        declared = KnownWrappers.declared(@config.wrappers, locked_gems.map { |gem| gem["name"] })
+        wrappers = Wrappers.new(i18n_functions: @config.i18n_functions, declared: declared)
+        programs = {}
+        sources.each do |source|
+          program = parse(source)
+          next unless program
+
+          programs[source.path] = [source, program]
+          guarded(source) { wrappers.scout(source.path, program) unless RubyVisitor::TEST_FILE.match?(source.path) }
+        end
+        wrappers.resolve!
+
+        usages = []
+        conventions = []
+        programs.each do |path, (source, program)|
+          guarded(source) do
+            result = RubyVisitor.new(file: path, lang: source.lang.to_s, bytes: source.bytes, wrappers: wrappers,
+              catalog: @catalog, i18n_functions: @config.i18n_functions).call(program)
+            usages.concat(result.records)
+            conventions.concat(result.conventions)
+          end
+        end
+
+        {
+          "version" => VERSION,
+          "parsers" => parsers,
+          "gems" => locked_gems,
+          "usages" => order(resolve_overlaps(usages)),
+          "unscanned" => @unscanned.sort_by { |item| [item["file"], item["reason"]] },
+          "conventions" => conventions.sort_by { |item| [item["file"], item["line"], item["key"].to_s] }
+        }
+      end
+
+      private
+
+      def catalogs = @catalogs ||= Scan::Catalogs.new(@config.catalog_paths)
+
+      def custom_excludes = @custom_excludes ||= Scan::Catalogs.new(@config.usages_exclude)
+
+      def excluded?(path)
+        return true if DEFAULT_EXCLUDES.any? { |name| path == name || path.start_with?("#{name}/") || path.include?("/#{name}/") }
+        return true if EXCLUDED_PREFIXES.any? { |prefix| path.start_with?(prefix) }
+        return true if GENERATOR_TEMPLATES.match?(path)
+
+        !@config.usages_exclude.empty? && custom_excludes.match?(path)
+      end
+
+      def classify(entry, sources, read)
+        path = entry.path
+        extension = File.extname(path).downcase
+        if path.end_with?(".slim")
+          return unscanned(path, "Slim isn't scanned")
+        elsif SCRIPT_EXTENSIONS.include?(extension)
+          return script(entry, read)
+        end
+
+        lang = template_language(path, extension)
+        return unless lang
+
+        bytes = entry.size > MAX_BYTES ? nil : read.call(entry)
+        return unscanned(path, "could not be parsed: file is larger than #{MAX_BYTES} bytes") unless bytes
+
+        bytes = bytes.dup.force_encoding(Encoding::UTF_8)
+        return unscanned(path, "could not be parsed: the file is not valid UTF-8") unless bytes.valid_encoding?
+
+        sources << Source.new(path: path, lang: lang, bytes: bytes)
+      end
+
+      def template_language(path, extension)
+        return :ruby if RUBY_EXTENSIONS.include?(extension) || %w[Rakefile Gemfile].include?(File.basename(path))
+        return :erb if extension == ".erb"
+
+        :haml if extension == ".haml"
+      end
+
+      def script(entry, read)
+        return if entry.size > MAX_BYTES
+
+        if i18n_js_configured?
+          return unscanned(entry.path, "JavaScript that reads the locale files isn't scanned")
+        end
+        return if @catalog.empty?
+
+        text = read.call(entry).dup.force_encoding(Encoding::UTF_8).scrub("")
+        named = text.scan(JS_STRING).map { |groups| groups.compact.first.to_s }.select { |name| @catalog.include?(name) }.uniq.sort
+        return if named.empty?
+
+        # The keys the file names as strings stay locked server-side; the file itself is a place to check.
+        # A list past the cap would silently unlock the rest, so it is sent as "all": the file locks every key.
+        keys = named.size > MAX_NAMED_KEYS ? "all" : named
+        unscanned(entry.path, "JavaScript that names a catalog key isn't scanned", keys: keys)
+      end
+
+      def i18n_js_configured?
+        return @i18n_js unless @i18n_js.nil?
+
+        paths = @git.entries(@sha).map(&:path)
+        @i18n_js = I18N_JS_CONFIGS.any? { |name| paths.include?(name) }
+      end
+
+      def unscanned(path, reason, keys: nil)
+        item = {"file" => path, "reason" => reason}
+        item["keys"] = keys if keys
+        @unscanned << item
+      end
+
+      def parse(source)
+        ruby = case source.lang
+        when :ruby then source.bytes
+        when :erb then Templates.erb(source.bytes)
+        when :haml then Templates.haml(source.bytes)
+        end
+        return haml_unavailable(source) if ruby.nil?
+
+        result = Prism.parse(ruby, version: "latest")
+        errors = source.lang == :ruby ? result.errors : result.errors.reject { |error| TEMPLATE_TOLERATED.include?(error.message) }
+        unless errors.empty?
+          unscanned(source.path, parse_reason(errors.first.message))
+          return nil
+        end
+        result.value
+      rescue Templates::Unreadable => e
+        unscanned(source.path, parse_reason(e.message)) # our own message: a line number, never source text
+        nil
+      rescue => e
+        # A parser's exception message may quote the source, so only its class leaves the machine.
+        unscanned(source.path, parse_reason("#{e.class.name} while reading the template"))
+        nil
+      end
+
+      REASON_LIMIT = 200
+
+      # `could not be parsed: <message>`, the message cut to 200 bytes on a character boundary.
+      def parse_reason(message)
+        text = message.to_s.lines.first.to_s.strip.byteslice(0, REASON_LIMIT).to_s.dup.force_encoding(Encoding::UTF_8)
+        text = text.byteslice(0, text.bytesize - 1).force_encoding(Encoding::UTF_8) until text.empty? || text.valid_encoding?
+        "could not be parsed: #{text}"
+      end
+
+      # A template's Ruby isn't inside a method: a layout's `yield` is fine there.
+      TEMPLATE_TOLERATED = ["Invalid yield", "Invalid return in class/module body"].freeze
+
+      # A bug in the scan on one file must not take the others down; the file is reported, not hidden.
+      def guarded(source)
+        yield
+      rescue => e
+        unscanned(source.path, parse_reason("internal error (#{e.class.name})"))
+      end
+
+      def haml_unavailable(source)
+        unscanned(source.path, "HAML isn't scanned")
+        nil
+      end
+
+      def gems(entries, read)
+        lock = entries.find { |entry| entry.path == "Gemfile.lock" }
+        return [] unless lock
+
+        specs = read.call(lock).dup.force_encoding(Encoding::UTF_8).scrub("").scan(/^ {4}(\S+) \(([^)\s]+)\)\s*$/)
+        specs.map { |name, version| {"name" => name, "version" => version} }.uniq.sort_by { |gem| [gem["name"], gem["version"]] }
+      end
+
+      def parsers
+        {"dialecto-rails" => VERSION_STRING, "prism" => Prism::VERSION, "herb" => Templates.herb_version,
+         "haml" => Templates.haml_version}.compact
+      end
+
+      VERSION_STRING = DialectoRails::VERSION
+
+      # A `literal` that lies inside a call record's anchor is already part of that use.
+      def resolve_overlaps(usages)
+        usages.group_by { |usage| usage["file"] }.flat_map do |_file, records|
+          claims = records.reject { |record| record["kind"] == "literal" }.map { |record| claim(record) }
+          records.reject do |record|
+            record["kind"] == "literal" && claims.any? { |from, to| record["offset"] >= from && record["offset"] < to }
+          end
+        end
+      end
+
+      def claim(record) = [record["offset"], record["offset"] + record["content"].bytesize]
+
+      def order(usages) = usages.sort_by { |usage| [usage["file"].b, usage["offset"]] }
+    end
+  end
+end

data/lib/dialecto_rails/usages/summary.rb

@@ -0,0 +1,191 @@
+require "json"
+
+module DialectoRails
+  module Usages
+    # The developer's view of a usage scan (spec-0023 §6.7): uses by kind, every catalog key sorted by how a
+    # rename could treat it, and what wasn't read. It follows the server's classification (dlocal
+    # `Dlocal.Localize.KeyRename`, spec-0022 §5.6), which decides; a parity test holds the two together.
+    #
+    #   alone      read only by calls that name it
+    #   family     a shared template or a group lookup reaches it: renamed only with its group
+    #   locked     Rails or a gem reads it by name, a JavaScript file names it as a string, or a call whose
+    #              key is partly known could reach it. A file the scan can't read and a lookup of a wholly
+    #              unknown key lock nothing: they are places to check, listed at the end.
+    #   unreached  nothing reads it
+    class Summary
+      # What Rails and the gems read by name; the server owns this file and the gem ships a copy.
+      READ_BY_NAME = JSON.parse(File.read(File.join(__dir__, "read_by_name.json"))).freeze
+
+      Row = Struct.new(:key, :klass, :because, :prefix, keyword_init: true)
+
+      def initialize(envelope, catalog_keys)
+        @envelope = envelope
+        @keys = catalog_keys.sort
+      end
+
+      def kinds = @envelope["usages"].group_by { |usage| usage["kind"] }.transform_values(&:length).sort.to_h
+
+      def rows
+        @rows ||= begin
+          index
+          @keys.map { |key| classify(key) }
+        end
+      end
+
+      def counts = rows.group_by(&:klass).transform_values(&:length)
+
+      def render
+        lines = ["Uses: #{kinds.map { |kind, count| "#{count} #{kind}" }.join(", ").then { |text| text.empty? ? "none" : text }}"]
+        counts.then { |by| lines << "Catalog keys: " + %w[alone family locked unreached].map { |name| "#{by[name.to_sym] || 0} #{name}" }.join(", ") }
+        rows.select { |row| row.klass == :locked }.each { |row| lines << "  #{row.key} can't be renamed: #{row.because}" }
+        unreached = rows.select { |row| row.klass == :unreached }
+        lines << "Nothing reads #{unreached.length} keys (first: #{unreached.first(3).map(&:key).join(", ")})." unless unreached.empty?
+        @envelope["unscanned"].each { |item| lines << "Not scanned: #{item["file"]} (#{item["reason"]})" }
+        unknown = @envelope["usages"].select { |usage| usage["kind"] == "pattern" && wholly_unknown?(usage) }
+        unknown.each { |usage| lines << "Unknown key: #{usage["file"]}:#{usage["start_line"]} (#{reason_text(usage["reason"])})" }
+        places = @envelope["unscanned"].length + unknown.length
+        lines << "#{places} places Dialecto can't read; you check them before a rename." if places.positive?
+        lines.join("\n")
+      end
+
+      private
+
+      def index
+        @index ||= begin
+          usages = @envelope["usages"]
+          static = {}
+          set_by = Hash.new { |hash, key| hash[key] = [] }
+          templates = {}
+          patterns = []
+          implicit = Hash.new { |hash, key| hash[key] = [] }
+          usages.each do |usage|
+            case usage["kind"]
+            when "static" then static[usage["key"]] = true
+            when "template" then templates[usage["id"]] = usage
+            when "set" then usage["keys"].each { |key| set_by[key] << usage }
+            when "pattern" then patterns << [usage, tokens(usage["parts"])] unless wholly_unknown?(usage)
+            end
+            Array(usage["defaults"]).each do |default|
+              if default.key?("span") then static[default["key"]] = true
+              else implicit[default["key"]] << usage
+              end
+            end
+          end
+          { static: static, set_by: set_by, templates: templates, patterns: patterns, implicit: implicit,
+            rules: read_by_name_rules }
+        end
+      end
+
+      # A lookup of a key nothing is known about (`t(key)`): a place to check, not a lock on every key.
+      def wholly_unknown?(usage) = usage["parts"].none? { |part| part.key?("lit") }
+
+      # Rails' namespaces, the prefixes of the scan's gems and the scan's own conventions.
+      def read_by_name_rules
+        namespaces = READ_BY_NAME["namespaces"].map { |entry| { prefix: entry["prefix"], reader: entry["reader"] } }
+        gems = Array(@envelope["gems"]).flat_map do |gem|
+          Array(READ_BY_NAME["gems"][gem["name"]]).map { |prefix| { prefix: prefix, reader: "the #{gem["name"]} gem reads this key by its name" } }
+        end
+        conventions = Array(@envelope["conventions"]).map do |entry|
+          { prefix: entry["prefix"], key: entry["key"], reader: "the app reads this key by its name" }
+        end
+        namespaces + gems.uniq + conventions
+      end
+
+      def classify(key)
+        because = lock_reason(key)
+        return Row.new(key: key, klass: :locked, because: because) if because
+
+        groups = group_starts(key)
+        if index[:set_by].key?(key) || !groups.empty?
+          family(key, groups)
+        elsif index[:static][key] then Row.new(key: key, klass: :alone)
+        else Row.new(key: key, klass: :unreached)
+        end
+      end
+
+      def lock_reason(key)
+        # A file the scan can't read is a place to check, not a lock; a key the file names as a string stays locked.
+        unscanned = @envelope["unscanned"].find { |item| item["keys"] == "all" || Array(item["keys"]).include?(key) }
+        return "#{unscanned["file"]} names it as a string (#{unscanned["reason"]})" if unscanned
+
+        rule = index[:rules].find { |entry| entry[:prefix] ? key.start_with?(entry[:prefix]) : entry[:key] == key }
+        return rule[:reader] if rule
+
+        implicit = index[:implicit][key].first
+        return "#{implicit["file"]}:#{implicit["start_line"]} reads it as a fallback Rails adds" if implicit
+
+        hit = index[:patterns].find { |_usage, tokens| matches?(tokens, key) }
+        return "#{hit.first["file"]}:#{hit.first["start_line"]} #{reason_text(hit.first["reason"])}" if hit
+      end
+
+      def family(key, groups)
+        sets = index[:set_by][key]
+        starts = sets.map { |set| start_of(set) } + groups
+        prefix = starts.any?(&:nil?) ? "" : shared_prefix(starts)
+        if prefix.empty?
+          because = sets.map { |set| "#{set["file"]}:#{set["start_line"]} builds it from a template with no fixed dotted start" }.first
+          Row.new(key: key, klass: :locked, because: because)
+        else
+          Row.new(key: key, klass: :family, because: prefix, prefix: prefix)
+        end
+      end
+
+      def start_of(set)
+        origin = set["via"] && index[:templates][set["via"].first]
+        parts = set["via"] ? origin && origin["parts"] : set["parts"]
+        first = parts&.first
+        first && first["lit"]
+      end
+
+      # The longest dotted prefix every start shares ("" when they share no dot).
+      def shared_prefix(starts)
+        common = starts.reduce do |left, right|
+          left[0, (0...[left.length, right.length].min).take_while { |at| left[at] == right[at] }.length]
+        end
+        segments = common.split(".", -1)
+        segments.length < 2 ? "" : segments[0...-1].join(".") + "."
+      end
+
+      # `users.` for `users.index.title` when a call names the group `users`, which is no catalog key.
+      def group_starts(key)
+        @catalog ||= @keys.to_h { |catalog_key| [catalog_key, true] }
+        positions = (0...key.length).select { |at| key[at] == "." && at.positive? }
+        positions.filter_map do |at|
+          group = key[0, at]
+          "#{group}." if index[:static][group] && !@catalog[group]
+        end
+      end
+
+      # A pattern is a glob: literals and `*`. A wildcard matches any text, empty included.
+      def tokens(parts)
+        parts.each_with_object([]) do |part, acc|
+          if part["wild"] then acc << :wild unless acc.last == :wild
+          elsif !part["lit"].to_s.empty?
+            acc.last.is_a?(String) ? acc.last << part["lit"] : acc << part["lit"].dup
+          end
+        end
+      end
+
+      def matches?(tokens, key)
+        regexp = Regexp.new("\\A" + tokens.map { |token| token == :wild ? ".*" : Regexp.escape(token) }.join + "\\z", Regexp::MULTILINE)
+        regexp.match?(key)
+      end
+
+      def reason_text(reason)
+        {
+          "dynamic_scope" => "reads a key under a scope that isn't a literal",
+          "lazy_action_unknown" => "reads a lazy key outside a known action",
+          "lazy_outside_view" => "reads a lazy key outside a view",
+          "custom_separator" => "uses a custom separator",
+          "splat" => "passes options the scan can't read",
+          "argument" => "passes the key through a method the scan can't follow",
+          "concatenated_literal" => "builds a key from adjacent strings",
+          "interpolation" => "builds a key by interpolation",
+          "variable" => "reads a key from a variable",
+          "constant" => "reads a key from a constant",
+          "method_call" => "reads a key from a method call"
+        }.fetch(reason, "reads a key the scan can't resolve")
+      end
+    end
+  end
+end

data/lib/dialecto_rails/usages/templates.rb

@@ -0,0 +1,45 @@
+module DialectoRails
+  module Usages
+    # Template languages reduced to the Ruby inside them, with every byte kept where it was, so an offset in
+    # the reduced text is an offset in the template.
+    module Templates
+      # A template construct whose Ruby can't be put back in place; the message holds a line number, never source.
+      Unreadable = Class.new(StandardError)
+
+      module_function
+
+      def erb(bytes)
+        require "herb"
+        Herb.extract_ruby(bytes)
+      end
+
+      # HAML is read when the app's bundle has the `haml` gem; nil otherwise.
+      def haml(bytes)
+        return nil unless haml_loadable?
+
+        require_relative "haml_extractor"
+        HamlExtractor.extract(bytes)
+      end
+
+      def haml_loadable?
+        return @haml_loadable unless @haml_loadable.nil?
+
+        @haml_loadable = begin
+          require "haml"
+          true
+        rescue LoadError
+          false
+        end
+      end
+
+      def herb_version
+        require "herb"
+        Herb::VERSION
+      rescue LoadError
+        nil
+      end
+
+      def haml_version = haml_loadable? ? Haml::VERSION : nil
+    end
+  end
+end