i18n-tasks 0.9.37 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9cad835aebcfc5621cb07d3a5ef4134a01cfed6286411776a9a5e04cfefe5862
-  data.tar.gz: 3b9f5a35077463778d1b940bd01738e49225fe7775d02858da5bdd9a9ded8308
+  metadata.gz: 4d600c7425116c27f5cadffdc23d6e29de6137f90cea588e1a046ca1ec5970f5
+  data.tar.gz: 91ea505ba06486d962e7be61eb52ca6af18bc8d481fdeb6a193f715b3c2f55a2
 SHA512:
-  metadata.gz: 8ed7363e9cfd2e8cf179aa0f28c0f00b926b4a53ba95f19ca61d8a08901322c561e84ebab51616f592400dac985f32c82b0a08911ec5f22223fe22e7858dd6af
-  data.tar.gz: 04b44a9180dc9b1e9d739eb5e5881877f68321bbd7efa1028a2f2d89fd3acf1cf742d587aab566a8cbd43813619693132ec803221d6c7ad31cd703ba516622f3
+  metadata.gz: 0e74a51969f4a82f8fa479dc31cb9236cc9de5d6255fe99d6de9eb392fb85e8771f4b21cbaed35f8c114258ada6c88086fb6fae0f1e0d97d8eb1f2d317c4f834
+  data.tar.gz: 4a23c041ddef63f1b798f1455af637f77f369dcfb64e93f72c9592caf2105d2e9b15531679b30d7404746674596ee70078bba3525cbc2eacdce00d9d0ebc7054

data/README.md

@@ -1,5 +1,7 @@
 # i18n-tasks [![Build Status][badge-ci]][ci] [![Coverage Status][badge-coverage]][coverage] [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/glebm/i18n-tasks?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
+[![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://stand-with-ukraine.pp.ua/)
+
 i18n-tasks helps you find and manage missing and unused translations.
 
 <img width="539" height="331" src="https://i.imgur.com/XZBd8l7.png">
@@ -22,7 +24,7 @@ i18n-tasks can be used with any project using the ruby [i18n gem][i18n-gem] (def
 Add i18n-tasks to the Gemfile:
 
 ```ruby
-gem 'i18n-tasks', '~> 0.9.37'
+gem 'i18n-tasks', '~> 1.0.5'
 ```
 
 Copy the default [configuration file](#configuration):
@@ -221,7 +223,7 @@ See the full list of tasks with `i18n-tasks --help`.
 
 ### Features and limitations
 
-`i18n-tasks` uses an AST scanner for `.rb` files, and a regexp-based scanner for other files, such as `.haml`.
+`i18n-tasks` uses an AST scanner for `.rb` and `.html.erb` files, and a regexp-based scanner for other files, such as `.haml`.
 
 #### Relative keys
 

data/i18n-tasks.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |s| # rubocop:disable Metrics/BlockLength
 
   s.add_dependency 'activesupport', '>= 4.0.2'
   s.add_dependency 'ast', '>= 2.1.0'
+  s.add_dependency 'better_html', '~> 1.0'
   s.add_dependency 'erubi'
   s.add_dependency 'highline', '>= 2.0.0'
   s.add_dependency 'i18n'

data/lib/i18n/tasks/command/commander.rb

@@ -16,6 +16,7 @@ module I18n::Tasks
       end
 
       def run(name, opts = {})
+        log_stderr "#{Rainbow('#StandWith').bg(:blue)}#{Rainbow('Ukraine').bg(:yellow)}"
         name = name.to_sym
         public_name = name.to_s.tr '_', '-'
         log_verbose "task: #{public_name}(#{opts.map { |k, v| "#{k}: #{v.inspect}" } * ', '})"

data/lib/i18n/tasks/data/adapter/yaml_adapter.rb

@@ -5,11 +5,13 @@ module I18n::Tasks
   module Data
     module Adapter
       module YamlAdapter
+        EMOJI_REGEX = /\\u[\da-f]{8}/i
+
         class << self
           # @return [Hash] locale tree
           def parse(str, options)
             if YAML.method(:load).arity.abs == 2
-              YAML.load(str, **(options || {}))
+              YAML.safe_load(str, **(options || {}), permitted_classes: [Symbol], aliases: true)
             else
               # older jruby and rbx 2.2.7 do not accept options
               YAML.load(str)
@@ -18,7 +20,12 @@ module I18n::Tasks
 
           # @return [String]
           def dump(tree, options)
-            tree.to_yaml(options || {})
+            restore_emojis(tree.to_yaml(options || {}))
+          end
+
+          # @return [String]
+          def restore_emojis(yaml)
+            yaml.gsub(EMOJI_REGEX) { |m| [m[-8..].to_i(16)].pack("U") }
           end
         end
       end

data/lib/i18n/tasks/interpolations.rb

@@ -2,22 +2,26 @@
 
 module I18n::Tasks
   module Interpolations
-    VARIABLE_REGEX = /%{[^}]+}/.freeze
+    class << self
+      attr_accessor :variable_regex
+    end
+    @variable_regex = /%{[^}]+}/.freeze
 
     def inconsistent_interpolations(locales: nil, base_locale: nil) # rubocop:disable Metrics/AbcSize
       locales ||= self.locales
       base_locale ||= self.base_locale
       result = empty_forest
+      variable_regex = I18n::Tasks::Interpolations.variable_regex
 
       data[base_locale].key_values.each do |key, value|
         next if !value.is_a?(String) || ignore_key?(key, :inconsistent_interpolations)
 
-        base_vars = Set.new(value.scan(VARIABLE_REGEX))
+        base_vars = Set.new(value.scan(variable_regex))
         (locales - [base_locale]).each do |current_locale|
           node = data[current_locale].first.children[key]
           next unless node&.value.is_a?(String)
 
-          if base_vars != Set.new(node.value.scan(VARIABLE_REGEX))
+          if base_vars != Set.new(node.value.scan(variable_regex))
             result.merge!(node.walk_to_root.reduce(nil) { |c, p| [p.derive(children: c)] })
           end
         end

data/lib/i18n/tasks/scanners/erb_ast_processor.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'ast'
+require 'set'
+require 'i18n/tasks/scanners/local_ruby_parser'
+
+module I18n::Tasks::Scanners
+  class ErbAstProcessor
+    include AST::Processor::Mixin
+    def initialize
+      super()
+      @ruby_parser = LocalRubyParser.new(ignore_blocks: true)
+      @comments = []
+    end
+
+    def process_and_extract_comments(ast)
+      result = process(ast)
+      [result, @comments]
+    end
+
+    def on_code(node)
+      parsed, comments = @ruby_parser.parse(
+        node.children[0],
+        location: node.location
+      )
+      @comments.concat(comments)
+
+      unless parsed.nil?
+        parsed = parsed.updated(
+          nil,
+          parsed.children.map { |child| node?(child) ? process(child) : child }
+        )
+        node = node.updated(:send, parsed)
+      end
+      node
+    end
+
+    # @param node [::Parser::AST::Node]
+    # @return [::Parser::AST::Node]
+    def handler_missing(node)
+      node = transform_misparsed_comment(node)
+      node.updated(
+        nil,
+        node.children.map { |child| node?(child) ? process(child) : child }
+      )
+    end
+
+    private
+
+    # Works around incorrect handling of comments of the form:
+    # <%# ... #>
+    # (no space between % and #)
+    #
+    # With a space the AST is:
+    #
+    #     s(:erb, nil, nil,
+    #       s(:code, " # this should not fail: ' "), nil)
+    #
+    # Without a space the AST is:
+    #
+    #     s(:erb,
+    #       s(:indicator, "#"), nil,
+    #       s(:code, " this should not fail: ' "), nil)
+    # @param node [::Parser::AST::Node]
+    # @return [::Parser::AST::Node]
+    def transform_misparsed_comment(node)
+      return node unless node.type == :erb && node.children.size == 4 &&
+        node.children[0]&.type == :indicator && node.children[0].children[0] == "#" &&
+        node.children[1].nil? &&
+        node.children[2]&.type == :code &&
+        node.children[3].nil?
+      code_node = node.children[2]
+
+      # Prepend # to each line to make it a valid Ruby comment.
+      code = code_node.children[0].split("\n").map do |line|
+        next line if line =~ /^\s*#/
+        "##{line}"
+      end.join("\n")
+
+      node.updated(
+        nil,
+        [nil, nil, code_node.updated(nil, [code]), nil]
+      )
+    end
+
+    def node?(node)
+      node.is_a?(::Parser::AST::Node)
+    end
+  end
+end

data/lib/i18n/tasks/scanners/erb_ast_scanner.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/scanners/ruby_ast_scanner'
+require 'i18n/tasks/scanners/erb_ast_processor'
+require 'better_html/errors'
+require 'better_html/parser'
+
+module I18n::Tasks::Scanners
+  # Scan for I18n.translate calls in ERB-file better-html and ASTs
+  class ErbAstScanner < RubyAstScanner
+    def initialize(**args)
+      super(**args)
+      @erb_ast_processor = ErbAstProcessor.new
+    end
+
+    private
+
+    # Parse file on path and returns AST and comments.
+    #
+    # @param path Path to file to parse
+    # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
+    def path_to_ast_and_comments(path)
+      parser = BetterHtml::Parser.new(make_buffer(path))
+      ast = convert_better_html(parser.ast)
+      @erb_ast_processor.process_and_extract_comments(ast)
+    end
+
+    # Convert BetterHtml nodes to Parser::AST::Node
+    #
+    # @param node BetterHtml::Parser::AST::Node
+    # @return Parser::AST::Node
+    def convert_better_html(node)
+      definition = Parser::Source::Map::Definition.new(
+        node.location.begin,
+        node.location.begin,
+        node.location.begin,
+        node.location.end
+      )
+      Parser::AST::Node.new(
+        node.type,
+        node.children.map { |child| child.is_a?(BetterHtml::AST::Node) ? convert_better_html(child) : child },
+        {
+          location: definition
+        }
+      )
+    end
+  end
+end

data/lib/i18n/tasks/scanners/local_ruby_parser.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'parser/current'
+
+module I18n::Tasks::Scanners
+  class LocalRubyParser
+    # ignore_blocks feature inspired by shopify/better-html
+    # https://github.com/Shopify/better-html/blob/087943ffd2a5877fa977d71532010b0c91239519/lib/better_html/test_helper/ruby_node.rb#L24
+    BLOCK_EXPR = /\s*((\s+|\))do|\{)(\s*\|[^|]*\|)?\s*\Z/.freeze
+
+    def initialize(ignore_blocks: false)
+      @parser = ::Parser::CurrentRuby.new
+      @ignore_blocks = ignore_blocks
+    end
+
+    # Parse string and normalize location
+    def parse(source, location: nil)
+      buffer = ::Parser::Source::Buffer.new('(string)')
+      buffer.source = if @ignore_blocks
+                        source.sub(BLOCK_EXPR, '')
+                      else
+                        source
+                      end
+
+      @parser.reset
+      ast, comments = @parser.parse_with_comments(buffer)
+      ast = normalize_location(ast, location)
+      comments = comments.map { |comment| normalize_comment_location(comment, location) }
+      [ast, comments]
+    end
+
+    # Normalize location for all parsed nodes
+
+    # @param node {Parser::AST::Node} Node in parsed code
+    # @param location {Parser::Source::Map} Global location for the parsed string
+    # @return {Parser::AST::Node}
+    def normalize_location(node, location)
+      return node.map { |child| normalize_location(child, location) } if node.is_a?(Array)
+
+      return node unless node.is_a?(::Parser::AST::Node)
+
+      node.updated(
+        nil,
+        node.children.map { |child| normalize_location(child, location) },
+        { location: updated_location(location, node.location) }
+      )
+    end
+
+    # Calculate location relative to a global location
+    #
+    # @param global_location {Parser::Source::Map} Global location where the code was parsed
+    # @param local_location {Parser::Source::Map} Local location in the parsed string
+    # @return {Parser::Source::Map}
+    def updated_location(global_location, local_location)
+      return global_location if local_location.expression.nil?
+
+      range = ::Parser::Source::Range.new(
+        global_location.expression.source_buffer,
+        global_location.expression.to_range.begin + local_location.expression.to_range.begin,
+        global_location.expression.to_range.begin + local_location.expression.to_range.end
+      )
+
+      ::Parser::Source::Map::Definition.new(
+        range.begin,
+        range.begin,
+        range.begin,
+        range.end
+      )
+    end
+
+    # Normalize location for comment
+    #
+    # @param comment {Parser::Source::Comment} A comment with local location
+    # @param location {Parser::Source::Map} Global location for the parsed string
+    # @return {Parser::Source::Comment}
+    def normalize_comment_location(comment, location)
+      range = ::Parser::Source::Range.new(
+        location.expression.source_buffer,
+        location.expression.to_range.begin + comment.location.expression.to_range.begin,
+        location.expression.to_range.begin + comment.location.expression.to_range.end
+      )
+      ::Parser::Source::Comment.new(range)
+    end
+  end
+end

data/lib/i18n/tasks/scanners/results/occurrence.rb

@@ -48,7 +48,7 @@ module I18n::Tasks
         # rubocop:enable Metrics/ParameterLists
 
         def inspect
-          "Occurrence(#{@path}:#{@line_num}:#{@line_pos}:#{@pos}:#{@raw_key}:#{@default_arg})"
+          "Occurrence(#{@path}:#{@line_num}, line_pos: #{@line_pos}, pos: #{@pos}, raw_key: #{@raw_key}, default_arg: #{@default_arg}, line: #{@line})"
         end
 
         def ==(other)

data/lib/i18n/tasks/scanners/ruby_ast_call_finder.rb

@@ -2,6 +2,7 @@
 
 require 'ast'
 require 'set'
+
 module I18n::Tasks::Scanners
   class RubyAstCallFinder
     include AST::Processor::Mixin

data/lib/i18n/tasks/scanners/ruby_ast_scanner.rb

@@ -32,32 +32,22 @@ module I18n::Tasks::Scanners
     #
     # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
     def scan_file(path)
-      @parser.reset
-      ast, comments = @parser.parse_with_comments(make_buffer(path))
-
-      results = @call_finder.collect_calls ast do |send_node, method_name|
-        send_node_to_key_occurrence(send_node, method_name)
-      end
+      ast, comments = path_to_ast_and_comments(path)
 
-      magic_comments  = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
-      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
-        # transform_values is only available in ActiveSupport 4.2+
-        h.each { |k, v| h[k] = v.first }
-      end.invert
-      results + (magic_comments.flat_map do |comment|
-        @parser.reset
-        associated_node = comment_to_node[comment]
-        @call_finder.collect_calls(
-          @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
-        ) do |send_node, _method_name|
-          # method_name is not available at this stage
-          send_node_to_key_occurrence(send_node, nil, location: associated_node || comment.location)
-        end
-      end)
+      ast_to_occurences(ast) + comments_to_occurences(path, ast, comments)
     rescue Exception => e # rubocop:disable Lint/RescueException
       raise ::I18n::Tasks::CommandError.new(e, "Error scanning #{path}: #{e.message}")
     end
 
+    # Parse file on path and returns AST and comments.
+    #
+    # @param path Path to file to parse
+    # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
+    def path_to_ast_and_comments(path)
+      @parser.reset
+      @parser.parse_with_comments(make_buffer(path))
+    end
+
     # @param send_node [Parser::AST::Node]
     # @param method_name [Symbol, nil]
     # @param location [Parser::Source::Map]
@@ -204,6 +194,41 @@ module I18n::Tasks::Scanners
         buffer.raw_source = contents
       end
     end
+
+    # Convert an array of {Parser::Source::Comment} to occurrences.
+    #
+    # @param path Path to file
+    # @param ast Parser::AST::Node
+    # @param comments [Parser::Source::Comment]
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def comments_to_occurences(path, ast, comments)
+      magic_comments = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
+      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
+        # transform_values is only available in ActiveSupport 4.2+
+        h.each { |k, v| h[k] = v.first }
+      end.invert
+
+      magic_comments.flat_map do |comment|
+        @parser.reset
+        associated_node = comment_to_node[comment]
+        @call_finder.collect_calls(
+          @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
+        ) do |send_node, _method_name|
+          # method_name is not available at this stage
+          send_node_to_key_occurrence(send_node, nil, location: associated_node || comment.location)
+        end
+      end
+    end
+
+    # Convert {Parser::AST::Node} to occurrences.
+    #
+    # @param ast {Parser::Source::Comment}
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def ast_to_occurences(ast)
+      @call_finder.collect_calls(ast) do |send_node, method_name|
+        send_node_to_key_occurrence(send_node, method_name)
+      end
+    end
   end
 end
 # rubocop:enable Metrics/AbcSize,Metrics/BlockNesting,Metrics/PerceivedComplexity

data/lib/i18n/tasks/used_keys.rb

@@ -3,6 +3,7 @@
 require 'find'
 require 'i18n/tasks/scanners/pattern_with_scope_scanner'
 require 'i18n/tasks/scanners/ruby_ast_scanner'
+require 'i18n/tasks/scanners/erb_ast_scanner'
 require 'i18n/tasks/scanners/scanner_multiplexer'
 require 'i18n/tasks/scanners/files/caching_file_finder_provider'
 require 'i18n/tasks/scanners/files/caching_file_reader'
@@ -20,7 +21,8 @@ module I18n::Tasks
       relative_roots: %w[app/controllers app/helpers app/mailers app/presenters app/views].freeze,
       scanners: [
         ['::I18n::Tasks::Scanners::RubyAstScanner', { only: %w[*.rb] }],
-        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.rb] }]
+        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.erb] }],
+        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.erb *.rb] }]
       ],
       strict: true
     }.freeze

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '0.9.37'
+    VERSION = '1.0.5'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-tasks
 version: !ruby/object:Gem::Version
-  version: 0.9.37
+  version: 1.0.5
 platform: ruby
 authors:
 - glebm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-26 00:00:00.000000000 Z
+date: 2022-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: better_html
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: erubi
   requirement: !ruby/object:Gem::Requirement
@@ -357,12 +371,15 @@ files:
 - lib/i18n/tasks/references.rb
 - lib/i18n/tasks/reports/base.rb
 - lib/i18n/tasks/reports/terminal.rb
+- lib/i18n/tasks/scanners/erb_ast_processor.rb
+- lib/i18n/tasks/scanners/erb_ast_scanner.rb
 - lib/i18n/tasks/scanners/file_scanner.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder_provider.rb
 - lib/i18n/tasks/scanners/files/caching_file_reader.rb
 - lib/i18n/tasks/scanners/files/file_finder.rb
 - lib/i18n/tasks/scanners/files/file_reader.rb
+- lib/i18n/tasks/scanners/local_ruby_parser.rb
 - lib/i18n/tasks/scanners/occurrence_from_position.rb
 - lib/i18n/tasks/scanners/pattern_mapper.rb
 - lib/i18n/tasks/scanners/pattern_scanner.rb